def main():
data = fits.getdata(os.path.join('benchmark_data', '2d.fits'))
for outfile in '2d1.fits 2d2.fits 2d3.fits'.split():
d = Dendrogram.compute(data, verbose=True, **BENCHMARKS[outfile])
d.save_to(os.path.join('benchmark_data', outfile))
data = fits.getdata(os.path.join('benchmark_data', '3d.fits'))
for outfile in '3d1.fits 3d2.fits 3d3.fits'.split():
d = Dendrogram.compute(data, verbose=True, **BENCHMARKS[outfile])
d.save_to(os.path.join('benchmark_data', outfile))
def main():
data = fits.getdata(os.path.join('benchmark_data', '2d.fits'))
for outfile in '2d1.fits 2d2.fits 2d3.fits'.split():
d = Dendrogram.compute(data, verbose=True, **BENCHMARKS[outfile])
d.save_to(os.path.join('benchmark_data', outfile))
data = fits.getdata(os.path.join('benchmark_data', '3d.fits'))
for outfile in '3d1.fits 3d2.fits 3d3.fits'.split():
d = Dendrogram.compute(data, verbose=True, **BENCHMARKS[outfile])
d.save_to(os.path.join('benchmark_data', outfile))
def calc_dendrogram(data,sigma,snr,delta,ppb, save=False,fout='temp'):
# Survey designs
# sigma - K, noise level
# ppb - pixels/beam
# delta - flux derivation between sources
def custom_independent(structure, index=None, value=None):
peak_index, peak_value = structure.get_peak()
return peak_value > 1.5*sigma
d = Dendrogram.compute(data, min_value=snr*sigma, \
min_delta=delta*sigma, min_npix=ppb*ppb, \
is_independent=custom_independent, \
verbose = 1)
#v=d.viewer()
#v.show()
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Generate the catalog
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
#print("Generate a catalog of dendrogram structures")
#metadata = {}
#metadata['data_unit'] = u.Jy #This should be Kelvin (not yet implemented)!
#cat = ppv_catalog(d, metadata)
#print(cat)
if save:
d.save_to(fout+'.hdf5')
return d
def grad_dendro(self, min_value, min_npix, min_delta):
"""
Calculate the dendrogram
Parameters
------
min_value: minimum value in density to consider. See astrodendro.
min_npix: minimum number of voxels in a structure for it to be
considered. See astrodendro.
min_delta: minimum difference in density for a structure to be
considered, i.e. from the saddle point where it joins with
another structure to the peak. See astrodendro.
"""
dendro = Dendrogram.compute(self.N,
min_value=min_value,
min_npix=min_npix,
min_delta=min_delta)
self.dendro = dendro
df_massflow = defaultdict(list)
for i in range(len(dendro)):
mask_i = dendro[i].get_mask()
Sigma_i, dV_i, dR_i = quick2D(self.N, self.v, mask_i, ev=self.ev)
df_massflow['size'].append(2. * dR_i)
df_massflow['massflow'].append(Sigma_i * dV_i * (2. * dR_i)**2.)
df_massflow = pd.DataFrame(df_massflow)
self.df_massflow_dendro = df_massflow
def compute_dendro(self, verbose=False):
'''
Compute the dendrogram and prune to the minimum deltas.
** min_deltas must be in ascending order! **
Parameters
----------
verbose : optional, bool
'''
d = Dendrogram.compute(self.cube, verbose=verbose,
min_delta=self.min_deltas[0],
min_value=self.dendro_params["min_value"],
min_npix=self.dendro_params["min_npix"])
self.numfeatures[0] = len(d)
self.values.append(
np.asarray([struct.vmax for struct in d.all_structures]))
for i, delta in enumerate(self.min_deltas[1:]):
if verbose:
print "On %s of %s" % (i + 1, len(self.min_deltas[1:]))
d.prune(min_delta=delta)
self.numfeatures[i + 1] = len(d)
self.values.append([struct.vmax for struct in d.all_structures])
return self
def test_identify_edge_structures_4():
# Case 4: something more complex.
data = np.zeros((4,5,6))
# a non-edge structure
data[2,2,1] = 1
data[2,2,2] = 1
# an edge structure on zeros
data[0,0,3] = 1
data[0,1,3] = 1
data[1,1,3] = 1
# an edge structure on the max side
data[1,1,5] = 1
data[2,1,5] = 1
d = Dendrogram.compute(data, min_value=0)
on_edge = identify_edge_structures(d)
expected_onedge_len = 2
expected_offedge_len = 1
assert_equal(len(d), 3)
assert_equal(np.sum(on_edge), expected_onedge_len)
assert_equal(len(on_edge) - np.sum(on_edge), expected_offedge_len)
def make_dend(cube, noise, view=True, write=True,
min_npix=100,
min_nsig_value=3,
min_nsig_delta=2,
outfn="DendroMask_H2CO303202.hdf5"):
"""
Given a cube and a 2D noise map, extract dendrograms.
"""
# Use a little sigma-rejection to get a decently robust noise estimate
noise_std = noise[noise==noise].std()
noise_mean = noise[noise==noise].mean()
err_estimate = noise[(noise > (noise_mean-noise_std)) &
(noise < (noise_mean+noise_std))].mean()
bad_noise = np.isnan(noise)
log.info("{1} Estimated error: {0}".format(err_estimate, outfn))
dend = Dendrogram.compute(cube.filled_data[:].value,
min_value=min_nsig_value*err_estimate,
min_delta=min_nsig_delta*err_estimate,
min_npix=min_npix,
verbose=True, wcs=cube.wcs)
if view:
dend.viewer()
if write:
dend.save_to(hpath(outfn))
return dend
def make_dend(cube,
noise,
view=True,
write=True,
min_npix=100,
min_nsig_value=3,
min_nsig_delta=2,
outfn="DendroMask_H2CO303202.hdf5"):
"""
Given a cube and a 2D noise map, extract dendrograms.
"""
# Use a little sigma-rejection to get a decently robust noise estimate
noise_std = noise[noise == noise].std()
noise_mean = noise[noise == noise].mean()
err_estimate = noise[(noise > (noise_mean - noise_std))
& (noise < (noise_mean + noise_std))].mean()
bad_noise = np.isnan(noise)
log.info("{1} Estimated error: {0}".format(err_estimate, outfn))
dend = Dendrogram.compute(cube.filled_data[:].value,
min_value=min_nsig_value * err_estimate,
min_delta=min_nsig_delta * err_estimate,
min_npix=min_npix,
verbose=True,
wcs=cube.wcs)
if view:
dend.viewer()
if write:
dend.save_to(hpath(outfn))
return dend
def doDendro(self, data, minV=3, minDelta=3, minP=1000):
print "Calculating dendrogram...."
d = Dendrogram.compute(data,
min_value=minV * self.sigma,
min_delta=minDelta * self.sigma,
min_npix=minP)
print "Done"
return d
def compute_regions(min_val, min_del, npix, filename, reg_file, pdf=False):
contfile = fits.open(filename)
mywcs = wcs.WCS(contfile[0].header).celestial
array = contfile[0].data.squeeze()
beam = radio_beam.Beam.from_fits_header(contfile[0].header)
d = Dendrogram.compute(array,
min_value=min_val,
min_delta=min_del,
min_npix=npix,
wcs=mywcs,
verbose=False)
metadata = {}
metadata['data_unit'] = u.Jy / u.beam
pixel_scale = np.abs(
mywcs.pixel_scale_matrix.diagonal().prod())**0.5 * u.deg
metadata['spatial_scale'] = pixel_scale
metadata['beam_major'] = beam.major
metadata['beam_minor'] = beam.minor
#metadata['wavelength'] = contfile[0].header['CRVAL3']*u.GHz
metadata['wcs'] = mywcs
cat = pp_catalog(d, metadata)
with open(reg_file, 'w') as fh:
fh.write("fk5\n")
for row in cat:
fh.write(
"ellipse({x_cen}, {y_cen}, {major_sigma}, "
"{minor_sigma}, {position_angle}) # text={{{_idx}}}\n".format(
**dict(zip(row.colnames, row))))
if pdf == True:
ax = pl.gca()
ax.cla()
pl.imshow(array,
cmap='gray_r',
interpolation='none',
origin='lower',
vmax=0.01,
vmin=-0.001)
pltr = d.plotter()
for struct in d.leaves:
cntr = pltr.plot_contour(ax,
structure=struct,
colors=['r'],
linewidths=[0.9],
zorder=5)
if struct.parent:
while struct.parent:
struct = struct.parent
cntr_g = pltr.plot_contour(ax,
structure=struct,
colors=[(0, 1, 0, 1)],
linewidths=[0.2])
pl.savefig(reg_file + '.pdf')
return d, cat
def get_dendrogram(self, max_pos):
# if the dendrogram file doesn't exist
if not os.path.isfile('dendrogram.fits'):
dendro = Dendrogram.compute(self.vol_data, min_value=1.0, min_delta=1, min_npix=10, verbose=True)
dendro.save_to('dendrogram.fits')
dendro = Dendrogram.load_from('dendrogram.fits')
substructure = dendro.structure_at(max_pos) # max_pos should be (z, y, x)
dendro_mask = substructure.get_mask(shape=self.vol_data.shape)
return dendro_mask
def test_identify_edge_structures_1():
# Case 1: no edge structures
data = np.zeros((3,3,3))
data[1,1,1] = 1
d = Dendrogram.compute(data, min_value=0)
on_edge = identify_edge_structures(d)
expected = np.array([0])
assert_equal(on_edge, expected)
def test_identify_edge_structures_3():
# Case 3: an edge structure towards the "max" side
data = np.zeros((3,3,3))
data[1,1,1] = 1
data[1,1,2] = 1
d = Dendrogram.compute(data, min_value=0)
on_edge = identify_edge_structures(d)
expected = np.array([1])
assert_equal(on_edge, expected)
def test_reload_dendrogram_catalog_output():
# 1. construct a silly dendrogram, catalog, header, and metadata
data = np.zeros((3, 3, 3))
data[1, 1, 1] = 1
d = Dendrogram.compute(data, min_value=0)
catalog = Table()
catalog['test_column'] = np.zeros(10)
header = Header.fromstring(
"SIMPLE = T / conforms to FITS standard BITPIX = 64 / array data type NAXIS = 3 / number of array dimensions NAXIS1 = 6 NAXIS2 = 5 NAXIS3 = 4 CTYPE1 = 'VELO-LSR' CTYPE2 = 'GLON-CAR' CTYPE3 = 'GLAT-CAR' CRVAL1 = '' CRVAL2 = '' CRVAL3 = '' CDELT1 = '' CDELT2 = '' CDELT3 = '' CRPIX1 = '' CRPIX2 = '' CRPIX3 = '' END "
)
metadata = {}
metadata['data_unit'] = u.K
# 2. save it all "manually"
filename_base = filepath + "not_real_data_reloadtest.fits_1.000_2.000_3"
d.save_to(filename_base + "_d.hdf5")
catalog.write(filename_base + "_catalog.fits", overwrite=True)
header.tofile(filename_base + "_header.fits", clobber=True)
pickle.dump(metadata, open(filename_base + "_metadata.p", 'wb'))
# 3. reconstitute it using the magic function
kwargs = {
'data_filename': 'not_real_data_reloadtest.fits',
'min_value': 1,
'min_delta': 2,
'min_npix': 3,
'savepath': filepath
}
d2, catalog2, header2, metadata2 = reload_dendrogram_catalog_output(
**kwargs)
#4 and assert they equal the mock things.
assert_equal(d2.index_map, d.index_map)
assert_array_equal(catalog2, catalog)
assert_equal(header2, header)
assert_equal(metadata2, metadata)
# 4.5. stress test.
for i in range(5):
reload_dendrogram_catalog_output(**kwargs)
#5 then delete the saved objects.
os.remove(filename_base + "_d.hdf5")
os.remove(filename_base + "_catalog.fits")
os.remove(filename_base + "_header.fits")
os.remove(filename_base + "_metadata.p")
def dend_spec(xx=12, yy=17, min_nsig=3, min_delta=1, min_npix=3):
b303 = cube303[brick_slice]
n303 = noise[brick_slice[1:]]
dend = Dendrogram.compute(b303[:,yy,xx].value, min_value=n303[yy,xx]*min_nsig,
min_delta=n303[yy,xx]*min_delta, min_npix=min_npix,
verbose=True,)
dend_guesses = [b303.spectral_axis[leaf.indices()].mean() for leaf in dend]
import pylab as pl
pl.plot(b303[:,yy,xx].value, drawstyle='steps-mid', color='k', linewidth=0.5)
for ii in range(len(dend)):
pl.plot(dend[ii].indices()[0],
b303[:,yy,xx].value[dend[ii].indices()], '.')
return dend_guesses,dend
def get_dendrogram(self, max_pos):
# if the dendrogram file doesn't exist
if not os.path.isfile('dendrogram.fits'):
dendro = Dendrogram.compute(self.vol_data,
min_value=1.0,
min_delta=1,
min_npix=10,
verbose=True)
dendro.save_to('dendrogram.fits')
dendro = Dendrogram.load_from('dendrogram.fits')
substructure = dendro.structure_at(
max_pos) # max_pos should be (z, y, x)
dendro_mask = substructure.get_mask(shape=self.vol_data.shape)
return dendro_mask
def compute_dendro(cube, header, delta, save_name, verbose=False, save=True,\
save_path=None): ## Pass **kwargs into Dendrogram class??
'''
Computes a dendrogram and (optional) save it in HDF5 format.
'''
dendrogram = Dendrogram.compute(cube, min_delta=delta, verbose=verbose, min_value=0.25, min_npix=10)
if save:
dendrogram.save_to(save_name+"_"+str(delta)+"_dendrogram.hdf5")
return os.path.join(save_path, save_name+"_"+str(delta)+"_dendrogram.hdf5")
return dendrogram
def to_dendrogram(self,
min_value=None,
min_delta=None,
min_npix=None,
save=True):
"""
Calculates a dendrogram for the image.
Parameters
----------
min_value : float, optional
Minimum detection level to be considered in the dendrogram.
min_delta : float, optional
How significant a dendrogram structure has to be in order to be
considered a separate entity.
min_npix : float, optional
Minimum number of pixel needed for a dendrogram structure to be
considered a separate entity.
save : bool, optional
If enabled, the resulting dendrogram will be saved as an instance
attribute. Default is True.
Returns
----------
`~astrodendro.dendrogram.Dendrogram` object
A dendrogram object calculated from the radio image.
"""
if not min_value:
min_value = self.min_value
if not min_delta:
min_delta = self.min_delta
if not min_npix:
min_npix = self.min_npix
dend = Dendrogram.compute(self.data,
min_value=min_value,
min_delta=min_delta,
min_npix=min_npix,
wcs=self.wcs,
verbose=True)
if save:
self.dendrogram = dend
return dend
def test_save_dendrogram_catalog_output():
# 1. construct a silly dendrogram, catalog, header, and metadata
data = np.zeros((3, 3, 3))
data[1, 1, 1] = 1
d = Dendrogram.compute(data, min_value=0)
catalog = Table()
catalog['test_column'] = np.zeros(10)
header = Header.fromstring(
"SIMPLE = T / conforms to FITS standard BITPIX = 64 / array data type NAXIS = 3 / number of array dimensions NAXIS1 = 6 NAXIS2 = 5 NAXIS3 = 4 CTYPE1 = 'VELO-LSR' CTYPE2 = 'GLON-CAR' CTYPE3 = 'GLAT-CAR' CRVAL1 = '' CRVAL2 = '' CRVAL3 = '' CDELT1 = '' CDELT2 = '' CDELT3 = '' CRPIX1 = '' CRPIX2 = '' CRPIX3 = '' END "
)
metadata = {}
metadata['data_unit'] = u.K
# 2. run save_dendrogram_catalog_output on them - to some safe location
kwargs = {
'data_filename': 'not_real_data_savetest.fits',
'min_value': 1,
'min_delta': 2,
'min_npix': 3,
'savepath': filepath
}
save_dendrogram_catalog_output(d, catalog, header, metadata, **kwargs)
# 3. reload those things
filename_base = filepath + "not_real_data_savetest.fits_1.000_2.000_3"
d2 = Dendrogram.load_from(filename_base + "_d.hdf5")
catalog2 = Table.read(filename_base + "_catalog.fits")
header2 = getheader(filename_base + "_header.fits")
metadata2 = pickle.load(open(filename_base + "_metadata.p", 'rb'))
#4 and assert they equal the mock things.
assert_equal(d2.index_map, d.index_map)
assert_array_equal(catalog2, catalog)
assert_equal(header2, header)
assert_equal(metadata2, metadata)
#5 then delete the saved objects.
os.remove(filename_base + "_d.hdf5")
os.remove(filename_base + "_catalog.fits")
os.remove(filename_base + "_header.fits")
os.remove(filename_base + "_metadata.p")
def dend_guess_npeaks(cube, noise_2d, min_nsig=3, min_delta=1, min_npix=3):
valid_indices = np.where(np.isfinite(noise_2d))
guesses = {}
for yy,xx in ProgressBar(zip(*valid_indices)):
dend = Dendrogram.compute(cube[:, yy, xx].value,
min_value=min_nsig*noise_2d[yy,xx],
min_delta=min_delta*noise_2d[yy,xx],
min_npix=min_npix,)
guesses[(yy,xx)] = [x
for leaf in dend
for x in
(cube.spectral_axis[leaf.indices()].mean().value,
cube.spectral_axis[leaf.indices()].std().value)
]
return guesses
def compute_dendro(self,
verbose=False,
save_dendro=False,
dendro_name=None,
dendro_obj=None):
'''
Compute the dendrogram and prune to the minimum deltas.
** min_deltas must be in ascending order! **
Parameters
----------
verbose : optional, bool
Enables the progress bar in astrodendro.
save_dendro : optional, bool
Saves the dendrogram in HDF5 format. **Requires pyHDF5**
dendro_name : str, optional
Save name when save_dendro is enabled. ".hdf5" appended
automatically.
dendro_obj : Dendrogram, optional
Input a pre-computed dendrogram object. It is assumed that
the dendrogram has already been computed!
'''
if dendro_obj is None:
d = \
Dendrogram.compute(self.data, verbose=verbose,
min_delta=self.min_deltas[0],
min_value=self.dendro_params["min_value"],
min_npix=self.dendro_params["min_npix"])
else:
d = dendro_obj
self.numfeatures[0] = len(d)
self.values.append(
np.asarray([struct.vmax for struct in d.all_structures]))
if len(self.min_deltas) > 1:
for i, delta in enumerate(self.min_deltas[1:]):
if verbose:
print "On %s of %s" % (i + 1, len(self.min_deltas[1:]))
d.prune(min_delta=delta)
self.numfeatures[i + 1] = len(d)
self.values.append(
[struct.vmax for struct in d.all_structures])
return self
def make_sn_dend(sncube, view=True, write=True,
outfn="DendroMask_H2CO303202_signal_to_noise.hdf5"):
"""
Obsolete: not used
"""
raise DeprecationWarning("Obsolete")
dend = Dendrogram.compute(sncube, min_value=3, min_delta=2, min_npix=50,
verbose=True)
if view:
dend.viewer
if write:
dend.save_to(hpath(outfn))
return dend
def to_dendrogram(self,
min_value=None,
min_delta=None,
min_npix=None,
save=True):
"""
Calculates a dendrogram for the image.
Documentation needed
Parameters
----------
min_value : float, optional
min_delta : float, optional
min_npix : float, optional
save : bool, optional
If enabled, the resulting dendrogram will be saved as an instance
attribute. Default is True.
Returns
----------
~astrodendro.dendrogram.Dendrogram object
A dendrogram object calculated from the radio image.
"""
if not min_value:
min_value = self.min_value
if not min_delta:
min_delta = self.min_delta
if not min_npix:
min_npix = self.min_npix
dend = Dendrogram.compute(self.data,
min_value=min_value,
min_delta=min_delta,
min_npix=min_npix,
wcs=self.wcs,
verbose=True)
if save:
self.dendrogram = dend
return dend
def dend_guess_npeaks(cube, noise_2d, min_nsig=3, min_delta=1, min_npix=3):
valid_indices = np.where(np.isfinite(noise_2d))
guesses = {}
for yy, xx in ProgressBar(zip(*valid_indices)):
dend = Dendrogram.compute(
cube[:, yy, xx].value,
min_value=min_nsig * noise_2d[yy, xx],
min_delta=min_delta * noise_2d[yy, xx],
min_npix=min_npix,
)
guesses[(yy, xx)] = [
x for leaf in dend
for x in (cube.spectral_axis[leaf.indices()].mean().value,
cube.spectral_axis[leaf.indices()].std().value)
]
return guesses
def get_dendrogram(cogal, min_snr=5., fac_delta=1., fac_npix=1.):
import time, datetime
from astrodendro import Dendrogram
co = cogal[0]
gal = cogal[1]
# get data and info
file = data[co][gal]['noise matched']['file']
cube = fits.open(file)[0]
noise = data[co][gal]['noise matched']['rms'].to(u.K)
bmin = angle_to_parsec(cube.header['bmin'] * u.degree, source=gal)
bmaj = angle_to_parsec(cube.header['bmaj'] * u.degree, source=gal)
beam_area = 1.13309 * bmin * bmaj
pix1 = u.Quantity(
str(np.abs(cube.header['cdelt1'])) + cube.header['cunit1'])
pix2 = u.Quantity(
str(np.abs(cube.header['cdelt2'])) + cube.header['cunit2'])
pix3 = u.Quantity(
str(np.abs(cube.header['cdelt3'])) + cube.header['cunit3'])
pix_beam = (beam_area / pix1 / pix2)
# time execution
print("\nComputing dendrogram: " + co + " " + gal + "\n")
start = time.time()
# calculate dendrogram
d = Dendrogram.compute(
cube.data,
#wcs = WCS(cube.header), # causes a buffer overflow b/o wcslib errors: https://github.com/astropy/astropy/issues/7412
min_value=(min_snr * noise).value,
min_delta=(fac_delta * noise).value,
min_npix=(fac_npix * pix_beam).value,
verbose=True)
savepath = join(compdir, gal + '.' + co + '.dendrogram.fits')
mkdir(os.path.dirname(escape_filename(savepath)))
d.save_to(savepath)
# time execution
stop = time.time()
exec_time = np.round(stop - start, 1)
print("\nFinished dendrogram: " + co + " " + gal + "\nExecution took " +
str(datetime.timedelta(seconds=exec_time)) + "hours.\n")
def make_dendogram(self, field, axis, res):
"""Make a dendrogram of a field on a specified axis
and dump the dendrogram object to disk.
"""
self.get_fits_name(field, axis)
self.get_dendrogram_name(field, axis)
if glob.glob(self.dendrogram_name) != []:
print "Existing file at the following directory. I won't overwrite."
print self.dendrogram_name
return -1
print 'making dendrogram'
data = self.get_fits_array(field, axis)
d = Dendrogram.compute(data,
min_value=2.0,
min_delta=1.0,
min_npix=5,
verbose=True)
d.save_to(self.dendrogram_name)
return d
def get_dendrogram(self, axis, prefix='dendrogram_1', fpickle_args=None):
"""Unsure how to save these to disk"""
self.get_fits_name('density', axis)
dendro_filename = self.fits_dir + "/%s.fits" % prefix
if len(glob.glob(dendro_filename)) > 0:
pyfits.open(dendro_filename)[0].data
#return fPickle.load(dendro_filename)
else:
density = self.get_fits_array('density', axis)
if fpickle_args is None:
fpickle_args = {
'min_value': 2.0,
'min_delta': 1.0,
'min_npix': 10.0,
'verbose': True
}
d = dg.compute(density, **fpickle_args)
#fPickle.dump(d,dendro_filename)
return d
def compute_dendro(cube, header, delta, save_name, verbose=False, save=True,\
save_path=None): ## Pass **kwargs into Dendrogram class??
'''
Computes a dendrogram and (optional) save it in HDF5 format.
'''
dendrogram = Dendrogram.compute(cube,
min_delta=delta,
verbose=verbose,
min_value=0.25,
min_npix=10)
if save:
dendrogram.save_to(save_name + "_" + str(delta) + "_dendrogram.hdf5")
return os.path.join(save_path,
save_name + "_" + str(delta) + "_dendrogram.hdf5")
return dendrogram
def make_sn_dend(sncube,
view=True,
write=True,
outfn="DendroMask_H2CO303202_signal_to_noise.hdf5"):
"""
Obsolete: not used
"""
raise DeprecationWarning("Obsolete")
dend = Dendrogram.compute(sncube,
min_value=3,
min_delta=2,
min_npix=50,
verbose=True)
if view:
dend.viewer
if write:
dend.save_to(hpath(outfn))
return dend
def dend_spec(xx=12, yy=17, min_nsig=3, min_delta=1, min_npix=3):
b303 = cube303[brick_slice]
n303 = noise[brick_slice[1:]]
dend = Dendrogram.compute(
b303[:, yy, xx].value,
min_value=n303[yy, xx] * min_nsig,
min_delta=n303[yy, xx] * min_delta,
min_npix=min_npix,
verbose=True,
)
dend_guesses = [b303.spectral_axis[leaf.indices()].mean() for leaf in dend]
import pylab as pl
pl.plot(b303[:, yy, xx].value,
drawstyle='steps-mid',
color='k',
linewidth=0.5)
for ii in range(len(dend)):
pl.plot(dend[ii].indices()[0], b303[:, yy,
xx].value[dend[ii].indices()], '.')
return dend_guesses, dend
def dendrogram(self, periodic=True, **kwargs):
"""
Calculate the dendrogram
Parameters
------
perodic: whether the arrays have a periodic boundary condition. The
default is True.
min_value: minimum value in density to consider. See astrodendro.
min_npix: minimum number of voxels in a structure for it to be
considered. See astrodendro.
min_delta: minimum difference in density for a structure to be
considered, i.e. from the saddle point where it joins with
another structure to the peak. See astrodendro.
"""
# test for non-optional parameters for astrodendro
for k in ['min_value', 'min_npix', 'min_delta']:
if k not in kwargs.keys():
raise AttributeError('See astrodendro documentation.')
# calculate dendrogram; indicate whether the box is periodic
self.periodic = periodic
neighbours = periodic_neighbours([0, 1, 2]) if self.periodic else None
dendro = Dendrogram.compute(self.density,
neighbours=neighbours,
**kwargs)
# output
self.dendro = dendro
print('Number of structures:', len(dendro))
print('Number of leaves:', len(dendro.leaves))
save_name=osjoin(fig_path, "delvar_design4_break.png"))
# Look at difference w/ non-periodic boundary handling
# This needs to be revisited with the astropy convolution updates
# delvar.run(verbose=True, xunit=u.pix, xlow=4 * u.pix, xhigh=30 * u.pix,
# boundary='fill',
# save_name=osjoin(fig_path, "delvar_design4_boundaryfill.png"))
# Dendrograms
if run_dendro:
from turbustat.statistics import Dendrogram_Stats
from astrodendro import Dendrogram
cube = fits.open(osjoin(data_path, "Design4_flatrho_0021_00_radmc.fits"))[0]
d = Dendrogram.compute(cube.data, min_value=0.005, min_delta=0.1, min_npix=50,
verbose=True)
ax = plt.subplot(111)
d.plotter().plot_tree(ax)
plt.ylabel("Intensity (K)")
plt.savefig(osjoin(fig_path, "design4_dendrogram.png"))
plt.close()
dend_stat = Dendrogram_Stats(cube, min_deltas=np.logspace(-2, 0, 50),
dendro_params={"min_value": 0.005,
"min_npix": 50})
dend_stat.run(verbose=True,
save_name=osjoin(fig_path, "design4_dendrogram_stats.png"))
# Periodic boundaries
dend_stat.run(verbose=True, periodic_bounds=True,
def compute_dendro(self, show_progress=False, save_dendro=False,
dendro_name=None, dendro_obj=None,
periodic_bounds=False):
'''
Compute the dendrogram and prune to the minimum deltas.
** min_deltas must be in ascending order! **
Parameters
----------
show_progress : optional, bool
Enables the progress bar in astrodendro.
save_dendro : optional, bool
Saves the dendrogram in HDF5 format. **Requires pyHDF5**
dendro_name : str, optional
Save name when save_dendro is enabled. ".hdf5" appended
automatically.
dendro_obj : Dendrogram, optional
Input a pre-computed dendrogram object. It is assumed that
the dendrogram has already been computed!
periodic_bounds : bool, optional
Enable when the data is periodic in the spatial dimensions.
'''
self._numfeatures = np.empty(self.min_deltas.shape, dtype=int)
self._values = []
if dendro_obj is None:
if periodic_bounds:
# Find the spatial dimensions
num_axes = self.data.ndim
spat_axes = []
for i, axis_type in enumerate(self._wcs.get_axis_types()):
if axis_type["coordinate_type"] == u"celestial":
spat_axes.append(num_axes - i - 1)
neighbours = periodic_neighbours(spat_axes)
else:
neighbours = None
d = Dendrogram.compute(self.data, verbose=show_progress,
min_delta=self.min_deltas[0],
min_value=self.dendro_params["min_value"],
min_npix=self.dendro_params["min_npix"],
neighbours=neighbours)
else:
d = dendro_obj
self._numfeatures[0] = len(d)
self._values.append(np.array([struct.vmax for struct in
d.all_structures]))
if len(self.min_deltas) > 1:
# Another progress bar for pruning steps
if show_progress:
print("Pruning steps.")
bar = ProgressBar(len(self.min_deltas[1:]))
for i, delta in enumerate(self.min_deltas[1:]):
d.prune(min_delta=delta)
self._numfeatures[i + 1] = len(d)
self._values.append(np.array([struct.vmax for struct in
d.all_structures]))
if show_progress:
bar.update(i + 1)
def compute_dendro(self,
show_progress=False,
save_dendro=False,
dendro_name=None,
dendro_obj=None,
periodic_bounds=False):
'''
Compute the dendrogram and prune to the minimum deltas.
** min_deltas must be in ascending order! **
Parameters
----------
show_progress : optional, bool
Enables the progress bar in astrodendro.
save_dendro : optional, bool
Saves the dendrogram in HDF5 format. **Requires pyHDF5**
dendro_name : str, optional
Save name when save_dendro is enabled. ".hdf5" appended
automatically.
dendro_obj : Dendrogram, optional
Input a pre-computed dendrogram object. It is assumed that
the dendrogram has already been computed!
periodic_bounds : bool, optional
Enable when the data is periodic in the spatial dimensions.
'''
self._numfeatures = np.empty(self.min_deltas.shape, dtype=int)
self._values = []
if dendro_obj is None:
if periodic_bounds:
# Find the spatial dimensions
num_axes = self.data.ndim
spat_axes = []
for i, axis_type in enumerate(self._wcs.get_axis_types()):
if axis_type["coordinate_type"] == u"celestial":
spat_axes.append(num_axes - i - 1)
neighbours = periodic_neighbours(spat_axes)
else:
neighbours = None
d = Dendrogram.compute(self.data,
verbose=show_progress,
min_delta=self.min_deltas[0],
min_value=self.dendro_params["min_value"],
min_npix=self.dendro_params["min_npix"],
neighbours=neighbours)
else:
d = dendro_obj
self._numfeatures[0] = len(d)
self._values.append(
np.array([struct.vmax for struct in d.all_structures]))
if len(self.min_deltas) > 1:
# Another progress bar for pruning steps
if show_progress:
print("Pruning steps.")
bar = ProgressBar(len(self.min_deltas[1:]))
for i, delta in enumerate(self.min_deltas[1:]):
d.prune(min_delta=delta)
self._numfeatures[i + 1] = len(d)
self._values.append(
np.array([struct.vmax for struct in d.all_structures]))
if show_progress:
bar.update(i + 1)
do_topview = True
# Make the dendrogram of the full cube/image
if do_make:
print 'Make dendrogram from the full cube'
data = fits.getdata(data_file)
if size(shape(data))==4:
data = data[0,:,:,:]
rms = 0.4
pix_beam = 14
d = Dendrogram.compute(data, min_value=2*rms, min_delta=2*rms, min_npix=10, verbose = 1)
d.save_to(dendro_file+'.fits')
d.viewer()
# Load a premade dendrogram
if do_load:
data = fits.getdata(data_file)
if size(shape(data))==4:
data = data[0,:,:,:]
print 'Load dendrogram file: '+load_file
d = Dendrogram.load_from(load_file)
def dendrogram():
'''
'''
############################# Read in the data ################################
count, data, filenames = 0, [], []
# Loop through all fits files
for file in glob.glob('*.fits'):
count += 1
# Read in using astropy and then append the data to a list for use later
contents = fits.open(str(file))[1]
'''
# Append the name of the file to the position (count+len(data)) followed by
# the file contents
data.insert(count+len(data), file)
data.append(contents)
filenames.append(file)
'''
# Define a dendrogram instance
d = Dendrogram.compute(contents.data, verbose=True)
# Let the dendrogram become an interactive plot
p = d.plotter()
# Add the data to a figure
fig = figure()
# Define subplots to plot with
ax1 = subplot2grid((6,6), (0,0), colspan=4,rowspan=4)
ax2 = subplot2grid((6,6), (5,0), colspan=4,rowspan=1)
#ax13 = subplot2grid((3, 3), (1, 0), colspan=2, rowspan=2)
#ax14 = subplot2grid((3, 3), (1, 2), rowspan=2)
leaves = []
# Find the structure within the data
for i in range(0,len(d)):
struct = d[i]
# Get the mask of the region highlighted
mask = struct.get_mask()
# Create FITS HDU objects
mask_hdu = fits.PrimaryHDU(mask.astype('short'), contents.header)
ax1.imshow(contents.data, origin='lower', interpolation='nearest')
# Show contour for ``min_value``
p.plot_contour(ax1, color='black')
#p.colorbar()
# Determine the type of structure
if struct.is_leaf:
leaves.append(struct)
# Extract the indices of the core and its value
indices = struct.indices(subtree=True)
vals = struct.values(subtree=True)
#print indices
#print vals
# Highlight two branches
p.plot_contour(ax1, structure=struct, lw=3, colors="#%06x" % random.randint(0, 0xFFFFFF))
# Plot the entire tree in black
p.plot_tree(ax2, color='black')
# Add labels to the plot
if 'T_chi' in file:
ax1.set_xlabel('X')
ax1.set_ylabel('Y')
ax1.set_title('A Map Showing the Structure\n of $T$ for the $\chi^{2}$ Recovered Values')
ax2.set_xlabel('Structure')
ax2.set_ylabel('$T\/(K)$')
ax2.set_title('A Dendrogram Showing the Structure\n of $T$ for the $\chi^{2}$ Recovered Values')
elif 'N_chi' in file:
ax1.set_xlabel('X')
ax1.set_ylabel('Y')
ax1.set_title('A Map Showing the Structure\n of $N$ for the $\chi^{2}$ Recovered Values')
ax2.set_xlabel('Structure')
ax2.set_ylabel('$N\/(g\/cm^{-3})$')
ax2.set_title('A Dendrogram Showing the Structure\n of $N$ for the $\chi^{2}$ Recovered Values')
elif 'T_data' in file:
ax1.set_xlabel('X')
ax1.set_ylabel('Y')
ax1.set_title('A Map Showing the Structure\n of $T$ for the Data Input Values')
ax2.set_xlabel('Structure')
ax2.set_ylabel('$T\/(K)$')
ax2.set_title('A Dendrogram Showing the Structure\n of $T$ for the Data Input Values')
elif 'N_data' in file:
ax1.set_xlabel('X')
ax1.set_ylabel('Y')
ax1.set_title('A Map Showing the Structure\n of $N$ for the Data Input Values')
ax2.set_xlabel('Structure')
ax2.set_ylabel('$N\/(g\/cm^{-3})$')
ax2.set_title('A Dendrogram Showing the Structure\n of $N$ for the Data Input Values')
#ax1.imshow(contents, origin='lower', interpolation='nearest', cmap=cm.Blues, vmax1=4.0)
# Plot black contours on to the data
# Show contour for ``min_value``
#p.plot_contour(ax1, color='black')
# Save the image and then close to save memory and allow the next image to take its place
fig.savefig(str(file)+str('.jpg'), dpi=300)
close()
else: id = pars[0] == args.incl + args.unit
#**************************************************
#COMPUTING DENDROGRAM
#**************************************************
hdu.data *= 1e-3
mean_val = hdu.data.mean()
min_val = hdu.data.mean() / 10
if pars.ndim == 1:
min_delta = float(pars[1]) * mean_val
min_npix = int(pars[2])
else:
min_delta = float(pars[1][id]) * mean_val
min_npix = int(pars[2][id])
d = Dendrogram.compute(hdu.data.squeeze(),
min_value=min_val,
min_delta=min_delta,
min_npix=min_npix)
d.save_to('img_moment0_dendrogram_%s_%s.fits' % (args.unit, args.incl))
w = wcs.WCS(hdu.header)
#**************************************************
#CREATING MAIN MASK FOR LEAVES
#**************************************************
mask = np.zeros(hdu.data.shape, dtype=bool)
for leaf in d.leaves:
mask = mask | leaf.get_mask()
print('Number of leaves:', len(d.leaves))
# Now we create a FITS HDU object to contain this, with the correct header
mask_hdu = fits.PrimaryHDU(mask.astype('short'), hdu.header)
def run_dendro(
label='mycloud',
cubefile=None,
flatfile=None,
redo='n',
nsigma=3.,
min_delta=2.5,
min_bms=2.,
position_dependent_noise=False, # will use rms map in dendro
criteria=['volume'], # for SCIMES
doplots=True,
dendro_in=None, # use this instead of re-loading
**kwargs):
global cubedata, rmsmap, threshold_sigma
#%&%&%&%&%&%&%&%&%&%&%&%
# Make dendrogram
#%&%&%&%&%&%&%&%&%&%&%&%
hdu3 = fits.open(cubefile)[0]
hd3 = hdu3.header
cubedata = hdu3.data
# Deal with oddities in 30 Dor cube
if hd3['NAXIS'] == 3:
for key in [
'CTYPE4', 'CRVAL4', 'CDELT4', 'CRPIX4', 'CUNIT4', 'NAXIS4'
]:
if key in hd3.keys():
hd3.remove(key)
# Get cube parameters
sigma = stats.mad_std(hdu3.data[~np.isnan(hdu3.data)])
print('Robustly estimated RMS: ', sigma)
ppb = 1.133 * hd3['bmaj'] * hd3['bmin'] / (abs(
hd3['cdelt1'] * hd3['cdelt2']))
print('Pixels per beam: ', ppb)
# Make the dendrogram if not present or redo=y
if position_dependent_noise:
dendrofile = 'dendro_dendrogram_rmsmap.hdf5'
else:
dendrofile = 'dendro_dendrogram.hdf5'
if dendro_in != None:
d = dendro_in
elif redo == 'n' and os.path.isfile(dendrofile):
print('Loading pre-existing dendrogram')
d = Dendrogram.load_from(dendrofile)
else:
print('Make dendrogram from the full cube')
if position_dependent_noise:
mask3d = cubedata < 3 * sigma
rmsmap = np.nanstd(cubedata * mask3d,
axis=0) # assumes spectral 1st
threshold_sigma = min_delta # for custom_independent function
d = Dendrogram.compute(hdu3.data,
min_value=nsigma * sigma,
min_delta=min_delta * sigma,
min_npix=min_bms * ppb,
verbose=1,
is_independent=custom_independent)
else:
d = Dendrogram.compute(hdu3.data,
min_value=nsigma * sigma,
min_delta=min_delta * sigma,
min_npix=min_bms * ppb,
verbose=1)
d.save_to(dendrofile)
if doplots:
# checks/creates directory to place plots
if os.path.isdir('dendro_plots') == 0:
os.makedirs('dendro_plots')
# Plot the tree
fig = plt.figure(figsize=(14, 8))
ax = fig.add_subplot(111)
#ax.set_yscale('log')
ax.set_xlabel('Structure')
ax.set_ylabel('Intensity [' + hd3['BUNIT'] + ']')
p = d.plotter()
branch = [
s for s in d.all_structures
if s not in d.leaves and s not in d.trunk
]
tronly = [s for s in d.trunk if s not in d.leaves]
for st in tronly:
p.plot_tree(ax, structure=[st], color='brown', subtree=False)
for st in branch:
p.plot_tree(ax, structure=[st], color='black', subtree=False)
for st in d.leaves:
p.plot_tree(ax, structure=[st], color='green')
#p.plot_tree(ax, color='black')
plt.savefig('dendro_plots/' + label + '_dendrogram.pdf',
bbox_inches='tight')
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Generate the catalog
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Generate a catalog of dendrogram structures")
metadata = {}
if hd3['BUNIT'].upper() == 'JY/BEAM':
metadata['data_unit'] = u.Jy / u.beam
elif hd3['BUNIT'].upper() == 'K':
metadata['data_unit'] = u.K
else:
print("Warning: Unrecognized brightness unit")
metadata['vaxis'] = 0
if 'RESTFREQ' in hd3.keys():
freq = hd3['RESTFREQ'] * u.Hz
elif 'RESTFRQ' in hd3.keys():
freq = hd3['RESTFRQ'] * u.Hz
metadata['wavelength'] = freq.to(u.m, equivalencies=u.spectral())
metadata['spatial_scale'] = hd3['cdelt2'] * 3600. * u.arcsec
if hd3['ctype3'][0:3] == 'VEL' or hd3['ctype3'][0:4] == 'VRAD':
dv = hd3['cdelt3'] / 1000. * u.km / u.s
else:
assert hd3['ctype3'][0:4] == 'FREQ'
dv = 2.99792458e5 * np.absolute(
hd3['cdelt3']) / freq.value * u.km / u.s
metadata['velocity_scale'] = dv
bmaj = hd3['bmaj'] * 3600. * u.arcsec # FWHM
bmin = hd3['bmin'] * 3600. * u.arcsec # FWHM
metadata['beam_major'] = bmaj
metadata['beam_minor'] = bmin
if not (redo == "n" and os.path.exists(label + '_full_catalog.txt')):
print("generating catalog")
cat = ppv_catalog(d, metadata)
print(cat.info())
# Add additional properties: Average Peak Tb and Maximum Tb
srclist = cat['_idx'].tolist()
tmax = np.zeros(len(srclist), dtype=np.float64)
tpkav = np.zeros(len(srclist), dtype=np.float64)
for i, c in enumerate(srclist):
peakim = np.nanmax(hdu3.data * d[c].get_mask(), axis=0)
peakim[peakim == 0] = np.nan
tmax[i] = np.nanmax(peakim)
tpkav[i] = np.nanmean(peakim)
if hd3['BUNIT'].upper() == 'JY/BEAM':
omega_B = np.pi / (4 * np.log(2)) * bmaj * bmin
convfac = (u.Jy).to(u.K,
equivalencies=u.brightness_temperature(
omega_B, freq))
tmax *= convfac
tpkav *= convfac
newcol = Column(tmax, name='tmax')
newcol.unit = 'K'
cat.add_column(newcol)
newcol = Column(tpkav, name='tpkav')
newcol.unit = 'K'
cat.add_column(newcol)
cat.write(label + '_full_catalog.txt',
format='ascii.ecsv',
overwrite=True)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Generate the catalog with clipping
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
if not (redo == "n"
and os.path.exists(label + '_full_catalog_clipped.txt')):
print("generating clipped catalog")
ccat = ppv_catalog(d, metadata, clipping=True)
print(ccat.info())
# Add additional properties: Average Peak Tb and Maximum Tb
srclist = ccat['_idx'].tolist()
tmax = np.zeros(len(srclist), dtype=np.float64)
tpkav = np.zeros(len(srclist), dtype=np.float64)
for i, c in enumerate(srclist):
peakim = np.nanmax(hdu3.data * d[c].get_mask(), axis=0)
peakim[peakim == 0] = np.nan
clmin = np.nanmin(hdu3.data * d[c].get_mask())
tmax[i] = np.nanmax(peakim) - clmin
tpkav[i] = np.nanmean(peakim) - clmin
if hd3['BUNIT'].upper() == 'JY/BEAM':
omega_B = np.pi / (4 * np.log(2)) * bmaj * bmin
convfac = (u.Jy).to(u.K,
equivalencies=u.brightness_temperature(
omega_B, freq))
tmax *= convfac
tpkav *= convfac
newcol = Column(tmax, name='tmax-tmin')
newcol.unit = 'K'
ccat.add_column(newcol)
newcol = Column(tpkav, name='tpkav-tmin')
newcol.unit = 'K'
ccat.add_column(newcol)
ccat.write(label + '_full_catalog_clipped.txt',
format='ascii.ecsv',
overwrite=True)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Running SCIMES
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# print("Running SCIMES")
# dclust = SpectralCloudstering(d, cat, criteria = criteria, keepall=True)
# print(dclust.clusters)
#
# print("Visualize the clustered dendrogram")
# dclust.showdendro()
# plt.savefig('dendro_plots/'+label+'_clusters_tree.pdf')
#
# print("Produce the assignment cube")
# dclust.asgncube(hd3)
# try:
# os.remove(label+'_asgncube.fits.gz')
# except OSError:
# pass
# dclust.asgn.writeto(label+'_asgncube.fits.gz')
#sys.exit("Stopping here")
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the trunks
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
if doplots:
print("Image the trunks")
hdu2 = fits.open(flatfile)[0]
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
vmax = np.nanmax(hdu2.data) / 2.
im = ax.matshow(hdu2.data,
origin='lower',
cmap=plt.cm.Blues,
vmax=vmax)
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
if 'xlims' in kwargs:
ax.set_xlim(kwargs['xlims'])
if 'ylims' in kwargs:
ax.set_ylim(kwargs['ylims'])
# Make a trunk list
tronly = [s for s in d.trunk if s not in d.leaves]
f = open(label + '_trunks.txt', 'w')
for c in tronly:
f.write('{:<4d} | '.format(c.idx))
# Plot the actual structure boundaries
mask = d[c.idx].get_mask()
mask_coll = np.amax(mask, axis=0)
plt.contour(mask_coll, colors='red', linewidths=1, levels=[0])
# Plot the ellipse fits
s = analysis.PPVStatistic(d[c.idx])
ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none')
ax.add_patch(ellipse)
# Make sub-lists of descendants
print('Finding descendants of trunk ', c.idx)
desclist = []
if len(d[c.idx].descendants) > 0:
for s in d[c.idx].descendants:
desclist.append(s.idx)
desclist.sort()
liststr = ','.join(map(str, desclist))
f.write(liststr)
f.write("\n")
f.close()
fig.colorbar(im, ax=ax)
plt.savefig('dendro_plots/' + label + '_trunks_map.pdf',
bbox_inches='tight')
plt.close()
# Make a branch list
branch = [
s for s in d.all_structures
if s not in d.leaves and s not in d.trunk
]
slist = []
for c in branch:
slist.append(c.idx)
slist.sort()
with open(label + '_branches.txt', 'w') as output:
writer = csv.writer(output)
for val in slist:
writer.writerow([val])
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the leaves
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Image the leaves")
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
vmax = np.nanmax(hdu2.data) / 2.
im = ax.matshow(hdu2.data,
origin='lower',
cmap=plt.cm.Blues,
vmax=vmax)
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
if 'xlims' in kwargs:
ax.set_xlim(kwargs['xlims'])
if 'ylims' in kwargs:
ax.set_ylim(kwargs['ylims'])
# Make a leaf list
slist = []
for c in d.leaves:
slist.append(c.idx)
# Plot the actual structure boundaries
mask = d[c.idx].get_mask()
mask_coll = np.amax(mask, axis=0)
plt.contour(mask_coll, colors='green', linewidths=1, levels=[0])
# Plot the ellipse fits
s = analysis.PPVStatistic(d[c.idx])
ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none')
ax.add_patch(ellipse)
slist.sort()
with open(label + '_leaves.txt', "w") as output:
writer = csv.writer(output)
for val in slist:
writer.writerow([val])
fig.colorbar(im, ax=ax)
plt.savefig('dendro_plots/' + label + '_leaves_map.pdf',
bbox_inches='tight')
plt.close()
f = np.power(10, f)
NH2 = f
# plt.show()
outpath1 = r'./W43_main_N_erosion.fits'
if os.path.exists(outpath1):
os.remove(outpath1)
if os.path.exists(outpath1):
os.remove(outpath1)
print 'Writing', outpath1
fits.writeto(outpath1, np.log10(f), header=hdulist1[0].header)
params = {'mathtext.default': 'regular'}
plt.rcParams.update(params)
d = Dendrogram.compute(f, min_value=7.0e21, min_delta=8e20 / 2.35 * 5., min_npix=7.0)
metadata = {}
metadata['data_unit'] = u.Jy
metadata['spatial_scale'] = 1.5 * u.arcsec
metadata['beam_major'] = 10.0 * u.arcsec
metadata['beam_minor'] = 10.0 * u.arcsec
threshold = 10 * 8e20 / 2.35 * 5.
d.save_to('W43_main_my_dendrogram_erosion.fits')
NH2_mean = []
for i, leaf in enumerate(d.leaves):
NH2_mean = np.append(NH2_mean, np.nanmean(NH2[leaf.indices()[0], leaf.indices()[1]]))
fig = plt.figure(figsize=(17, 7))
p3 = d.plotter()
ax1 = fig.add_subplot(1, 3, 1)
ax2 = fig.add_subplot(1, 3, 2)
# In [55]: (1*u.K).to(u.Jy,jytok((46*u.arcsec)**2*pi,110*u.GHz))
# Out[55]: <Quantity 58.0861309512 Jy>
metadata = dict(data_unit=u.Jy / 58.06,
spatial_scale=22.5 * u.arcsec,
velocity_scale=grsh['CDELT3'] * u.m / u.s,
vaxis=0,
wcs=wcs.WCS(grsh))
# surface density ~ 4.92x10^20 T_mb dV = 4.92x10^20 * 58.06 * dv
# then divide by 2e33, multiply by 1.67e-24, and multiply by 3.08e18**2
if 'grsD' not in locals():
grsD = Dendrogram.compute(data=grs,
min_value=0.25,
min_npix=10,
min_delta=0.5,
verbose=True)
def get_velo_stats(leaf):
stats = []
L = leaf
while hasattr(L, 'parent'):
ps = PPVStatistic(L, metadata=metadata)
stats.append(ps)
L = L.parent
return stats
def get_trunk(x):
# 306 = 60 km/s
cropn = 50
#grs = fits.getdata('grs-28-cube.fits')[:306,:150,:150]
grs = fits.getdata('grs-28-cube.fits')[:,:100,:100]
grsh = fits.getheader('grs-28-cube.fits')
# In [55]: (1*u.K).to(u.Jy,jytok((46*u.arcsec)**2*pi,110*u.GHz))
# Out[55]: <Quantity 58.0861309512 Jy>
metadata = dict(data_unit=u.Jy/58.06, spatial_scale=22.5*u.arcsec, velocity_scale=grsh['CDELT3']*u.m/u.s, vaxis=0, wcs=wcs.WCS(grsh))
# surface density ~ 4.92x10^20 T_mb dV = 4.92x10^20 * 58.06 * dv
# then divide by 2e33, multiply by 1.67e-24, and multiply by 3.08e18**2
if 'grsD' not in locals():
grsD = Dendrogram.compute(data=grs, min_value=0.25, min_npix=10, min_delta=0.5, verbose=True)
def get_velo_stats(leaf):
stats = []
L = leaf
while hasattr(L,'parent'):
ps = PPVStatistic(L, metadata=metadata)
stats.append(ps)
L = L.parent
return stats
def get_trunk(x):
if x.parent is None:
return x
return get_trunk(x.parent)
for file in glob.glob('*.fits'):
count += 1
# Read in using astropy and then append the data to a list for use later
contents = fits.open(str(file))[1]
'''
# Append the name of the file to the position (count+len(data)) followed by
# the file contents
data.insert(count+len(data), file)
data.append(contents)
filenames.append(file)
'''
# Define a dendrogram instance
d = Dendrogram.compute(contents.data, verbose=True)
# Let the dendrogram become an interactive plot
p = d.plotter()
# Add the data to a figure
fig = figure()
# Define subplots to plot with
ax1 = subplot2grid((6,6), (0,0), colspan=4,rowspan=4)
ax2 = subplot2grid((6,6), (5,0), colspan=4,rowspan=1)
#ax13 = subplot2grid((3, 3), (1, 0), colspan=2, rowspan=2)
#ax14 = subplot2grid((3, 3), (1, 2), rowspan=2)
leaves = []
def main(args):
#%&%&%&%&%&%&%&%&%&%&%&%
# Make dendrogram
#%&%&%&%&%&%&%&%&%&%&%&%
print('Make dendrogram from the full cube')
hdu = fits.open(args.fits_file)[0]
data = hdu.data[801:1000, :, :]
hd = hdu.header
# Survey designs
sigma = args.sigma #K, noise level
ppb = args.ppb #pixels/beam
def custom_independent(structure, index=None, value=None):
peak_index, peak_value = structure.get_peak()
return peak_value > 3. * sigma
d = Dendrogram.compute(data, min_value=sigma, \
min_delta=args.delta*sigma, min_npix=1.*ppb, \
is_independent=custom_independent, \
verbose = 1)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Generate the catalog
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Generate a catalog of dendrogram structures")
metadata = {}
metadata['data_unit'] = u.Jy #This should be Kelvin (not yet implemented)!
cat = ppv_catalog(d, metadata)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Running SCIMES
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Running SCIMES")
dclust = scimes.SpectralCloudstering(d, cat, hd, rms=sigma, \
user_iter=args.iter, \
save_all_leaves = True, \
)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the result
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Visualize the clustered dendrogram")
dclust.showdendro()
print("Visualize collapsed maps of the assignment cubes")
cubes = [dclust.clusters_asgn,\
dclust.leaves_asgn,\
dclust.trunks_asgn]
titles = ['Clusters', 'Leaves', 'Trunks']
for cube, title in zip(cubes, titles):
plt.figure()
plt.imshow(np.nanmax(cube.data,axis=0),origin='lower',\
interpolation='nearest',cmap='jet')
plt.title(title + ' assignment map')
plt.colorbar(label='Structure label')
plt.xlabel('X [pixel]')
plt.ylabel('Y [pixel]')
# plt.show()
print("Image the results with APLpy")
clusts = dclust.clusters
colors = dclust.colors
# Create Orion integrated intensity map
mhdu = mom0map(hdu)
fig = aplpy.FITSFigure(mhdu, figsize=(8, 6), convention='wells')
fig.show_colorscale(cmap='gray') #, vmax=36, stretch = 'sqrt')
count = 0
for c in clusts:
mask = d[c].get_mask()
mask_hdu = fits.PrimaryHDU(mask.astype('short'), hdu.header)
mask_coll = np.amax(mask_hdu.data, axis=0)
mask_coll_hdu = fits.PrimaryHDU(mask_coll.astype('short'),
hd2d(hdu.header))
fig.show_contour(mask_coll_hdu,
colors=colors[count],
linewidths=2,
convention='wells',
levels=[0])
count = count + 1
print(count)
# fig.tick_labels.set_xformat('dd')
# fig.tick_labels.set_yformat('dd')
fig.add_colorbar()
fig.colorbar.set_axis_label_text(r'[(K km/s)$^{1/2}$]')
fig.save('b.png')
plt.show()
return 0
# Define the number of pixels in one leaf to be regarded a detection
condition = 4.
# Loop through all fits files
for file in glob.glob('*.fits'):
print 'File is...', file
count += 1
# Read in using astropy and then append the data to a list for use later
contents = fits.open(str(file))[1]
if 'N_chi_inp' in file:
# Define a dendrogram instance
d = Dendrogram.compute(contents.data, min_value=N_j/100, min_npix=4, verbose=True)
N_chi_indices, N_chi_vals = [], []
N_chi_data = contents.data
# Save a file for storage of determined values
N_chi_mass = open('N_chi_mass.txt', 'w')
cs = csv.writer(N_chi_mass, delimiter=' ')
# Save a line to allow better understanding of the columns
cs_towrite = ['First x Pixel', 'First y Pixel', 'U', 'GPE', 'Mass/ms', 'Sigma M/ms', 'Ratio']
cs.writerow(cs_towrite)
# Open the temperature file and extract the numerical data
T_data = fits.open(str('T_chi_inp.fits'))[1].data
def run_scimes(criteria=['volume'],
label='scimes',
cubefile=None,
mom0file=None,
bmajas=None,
bminas=None,
rfreq=None,
redo='n',
verbose=True,
**kwargs):
#%&%&%&%&%&%&%&%&%&%&%&%
# Make dendrogram
#%&%&%&%&%&%&%&%&%&%&%&%
hdu3 = fits.open(cubefile)[0]
hd3 = hdu3.header
# Deal with oddities in 30 Dor cube
if hd3['NAXIS'] == 3:
for key in [
'CTYPE4', 'CRVAL4', 'CDELT4', 'CRPIX4', 'CUNIT4', 'NAXIS4'
]:
if key in hd3.keys():
hd3.remove(key)
# Provide beam info if missing
if bmajas is not None:
hd3['bmaj'] = bmajas / 3600.
if bminas is not None:
hd3['bmin'] = bminas / 3600.
# Get cube parameters
sigma = stats.mad_std(hdu3.data[~np.isnan(hdu3.data)])
print('Robustly estimated RMS: {:.3f}'.format(sigma))
ppb = 1.133 * hd3['bmaj'] * hd3['bmin'] / (abs(
hd3['cdelt1'] * hd3['cdelt2']))
print('Pixels per beam: {:.2f}'.format(ppb))
# Make the dendrogram if not present or redo=y
if redo == 'n' and os.path.isfile(label + '_dendrogram.hdf5'):
print('Loading pre-existing dendrogram')
d = Dendrogram.load_from(label + '_dendrogram.hdf5')
else:
print('Make dendrogram from the full cube')
d = Dendrogram.compute(hdu3.data,
min_value=3 * sigma,
min_delta=2.5 * sigma,
min_npix=2 * ppb,
verbose=verbose)
d.save_to(label + '_dendrogram.hdf5')
# checks/creates directory to place plots
if os.path.isdir('plots') == 0:
os.makedirs('plots')
# Plot the tree
fig = plt.figure(figsize=(14, 8))
ax = fig.add_subplot(111)
ax.set_yscale('log')
ax.set_xlabel('Structure')
ax.set_ylabel('Intensity [' + hd3['BUNIT'] + ']')
p = d.plotter()
branch = [
s for s in d.all_structures if s not in d.leaves and s not in d.trunk
]
tronly = [s for s in d.trunk if s not in d.leaves]
for st in tronly:
p.plot_tree(ax, structure=[st], color='brown', subtree=False)
for st in branch:
p.plot_tree(ax, structure=[st], color='black', subtree=False)
for st in d.leaves:
p.plot_tree(ax, structure=[st], color='green')
plt.savefig('plots/' + label + '_dendrogram.pdf', bbox_inches='tight')
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Generate the catalog
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print('Generate a catalog of dendrogram structures')
metadata = {}
if hd3['BUNIT'].upper() == 'JY/BEAM':
metadata['data_unit'] = u.Jy / u.beam
elif hd3['BUNIT'].upper() == 'K':
metadata['data_unit'] = u.K
else:
print('Warning: Unrecognized brightness unit')
metadata['vaxis'] = 0
if rfreq is None:
if 'RESTFREQ' in hd3.keys():
freq = hd3['RESTFREQ'] * u.Hz
elif 'RESTFRQ' in hd3.keys():
freq = hd3['RESTFRQ'] * u.Hz
else:
freq = rfreq * u.GHz
metadata['wavelength'] = freq.to(u.m, equivalencies=u.spectral())
metadata['spatial_scale'] = hd3['cdelt2'] * 3600. * u.arcsec
metadata['velocity_scale'] = abs(hd3['cdelt3']) * u.meter / u.second
bmaj = hd3['bmaj'] * 3600. * u.arcsec # FWHM
bmin = hd3['bmin'] * 3600. * u.arcsec # FWHM
metadata['beam_major'] = bmaj
metadata['beam_minor'] = bmin
cat = ppv_catalog(d, metadata, verbose=verbose)
print(cat.info())
# Add additional properties: Average Peak Tb and Maximum Tb
srclist = cat['_idx'].tolist()
tmax = np.zeros(len(srclist), dtype=np.float64)
tpkav = np.zeros(len(srclist), dtype=np.float64)
for i, c in enumerate(srclist):
peakim = np.nanmax(hdu3.data * d[c].get_mask(), axis=0)
peakim[peakim == 0] = np.nan
tmax[i] = np.nanmax(peakim)
tpkav[i] = np.nanmean(peakim)
if hd3['BUNIT'].upper() == 'JY/BEAM':
omega_B = np.pi / (4 * np.log(2)) * bmaj * bmin
convfac = (u.Jy).to(u.K,
equivalencies=u.brightness_temperature(
omega_B, freq))
tmax *= convfac
tpkav *= convfac
newcol = Column(tmax, name='tmax')
newcol.unit = 'K'
cat.add_column(newcol)
newcol = Column(tpkav, name='tpkav')
newcol.unit = 'K'
cat.add_column(newcol)
cat.write(label + '_full_catalog.txt', format='ascii.ecsv', overwrite=True)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Running SCIMES
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Running SCIMES")
dclust = SpectralCloudstering(d, cat, criteria=criteria, keepall=True)
print(dclust.clusters)
print("Visualize the clustered dendrogram")
dclust.showdendro()
plt.savefig('plots/' + label + '_clusters_tree.pdf')
print("Produce the assignment cube")
dclust.asgncube(hd3)
try:
os.remove(label + '_asgncube.fits.gz')
except OSError:
pass
dclust.asgn.writeto(label + '_asgncube.fits.gz')
#sys.exit("Stopping here")
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the trunks
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Image the trunks")
hdu2 = fits.open(mom0file)[0]
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
vmax = np.nanmax(hdu2.data) / 2.
im = ax.matshow(hdu2.data, origin='lower', cmap=plt.cm.Blues, vmax=vmax)
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
if 'xlims' in kwargs:
ax.set_xlim(kwargs['xlims'])
if 'ylims' in kwargs:
ax.set_ylim(kwargs['ylims'])
# Make a trunk list
tronly = [s for s in d.trunk if s not in d.leaves]
f = open(label + '_trunks.txt', 'w')
for c in tronly:
f.write('{:<4d} | '.format(c.idx))
# Plot the actual structure boundaries
mask = d[c.idx].get_mask()
mask_coll = np.amax(mask, axis=0)
plt.contour(mask_coll, colors='red', linewidths=1, levels=[0])
# Plot the ellipse fits
s = analysis.PPVStatistic(d[c.idx])
ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none')
ax.add_patch(ellipse)
# Make sub-lists of descendants
print('Finding descendants of trunk {}'.format(c.idx))
desclist = []
if len(d[c.idx].descendants) > 0:
for s in d[c.idx].descendants:
desclist.append(s.idx)
desclist.sort()
liststr = ','.join(map(str, desclist))
f.write(liststr)
f.write("\n")
f.close()
fig.colorbar(im, ax=ax)
plt.savefig('plots/' + label + '_trunks_map.pdf', bbox_inches='tight')
plt.close()
# Make a branch list
branch = [
s for s in d.all_structures if s not in d.leaves and s not in d.trunk
]
slist = []
for c in branch:
slist.append(c.idx)
slist.sort()
with open(label + '_branches.txt', 'w') as output:
writer = csv.writer(output)
for val in slist:
writer.writerow([val])
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the leaves
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Image the leaves")
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
vmax = np.nanmax(hdu2.data) / 2.
im = ax.matshow(hdu2.data, origin='lower', cmap=plt.cm.Blues, vmax=vmax)
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
if 'xlims' in kwargs:
ax.set_xlim(kwargs['xlims'])
if 'ylims' in kwargs:
ax.set_ylim(kwargs['ylims'])
# Make a leaf list
slist = []
for c in d.leaves:
slist.append(c.idx)
# Plot the actual structure boundaries
mask = d[c.idx].get_mask()
mask_coll = np.amax(mask, axis=0)
plt.contour(mask_coll, colors='green', linewidths=1, levels=[0])
# Plot the ellipse fits
s = analysis.PPVStatistic(d[c.idx])
ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none')
ax.add_patch(ellipse)
slist.sort()
with open(label + '_leaves.txt', "w") as output:
writer = csv.writer(output)
for val in slist:
writer.writerow([val])
fig.colorbar(im, ax=ax)
plt.savefig('plots/' + label + '_leaves_map.pdf', bbox_inches='tight')
plt.close()
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the clusters
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Image the resulting clusters")
clusts = np.array(dclust.clusters)
colors = np.array(dclust.colors)
inds = np.argsort(clusts)
clusts = clusts[inds]
colors = colors[inds]
print(clusts)
print(colors)
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
vmax = np.nanmax(hdu2.data) / 2.
im = ax.matshow(hdu2.data, origin='lower', cmap=plt.cm.Blues, vmax=vmax)
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
if 'xlims' in kwargs:
ax.set_xlim(kwargs['xlims'])
if 'ylims' in kwargs:
ax.set_ylim(kwargs['ylims'])
# Make a cluster list
f = open(label + '_clusters.txt', 'w')
for i, c in enumerate(clusts):
f.write('{:<4d} {:7} | '.format(c, colors[i]))
# Plot the actual structure boundaries
mask = d[c].get_mask()
mask_coll = np.amax(mask, axis=0)
plt.contour(mask_coll, colors=colors[i], linewidths=1, levels=[0])
# Plot the ellipse fits
s = analysis.PPVStatistic(d[c])
ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none')
ax.add_patch(ellipse)
# Make sub-lists of descendants
print('Finding descendants of cluster {}'.format(c))
desclist = []
if len(d[c].descendants) > 0:
for s in d[c].descendants:
desclist.append(s.idx)
desclist.sort()
liststr = ','.join(map(str, desclist))
f.write(liststr)
f.write("\n")
f.close()
fig.colorbar(im, ax=ax)
plt.savefig('plots/' + label + '_clusters_map.pdf', bbox_inches='tight')
plt.close()
return
def detect(infile,
region,
band,
min_value=0.000325,
min_delta=0.0005525,
min_npix=7.5,
plot=False,
verbose=True):
outfile = 'region{}_band{}_val{:.5g}_delt{:.5g}_pix{}'.format(
region, band, min_value, min_delta, min_npix)
contfile = fits.open(infile) # load in fits image
da = contfile[0].data.squeeze() # get rid of extra axes
print(da)
mywcs = wcs.WCS(
contfile[0].header
).celestial # set up world coordinate system, ditch extra dimensions
beam = radio_beam.Beam.from_fits_header(contfile[0].header)
d = Dendrogram.compute(da,
min_value=min_value,
min_delta=min_delta,
min_npix=min_npix,
wcs=mywcs,
verbose=verbose)
pixel_scale = np.abs(
mywcs.pixel_scale_matrix.diagonal().prod())**0.5 * u.deg
metadata = {
'data_unit': u.Jy / u.beam,
'spatial_scale': pixel_scale,
'beam_major': beam.major,
'beam_minor': beam.minor,
'wavelength': 9.298234612192E+10 * u.Hz,
'velocity_scale': u.km / u.s,
'wcs': mywcs,
}
cat = pp_catalog(d.leaves, metadata) # set up position-position catalog
cat['_idx'] = range(len(cat))
# Use FWHM for ellipse dimensions instead of sigma
cat['major_sigma'] = cat['major_sigma'] * np.sqrt(8 * np.log(2))
cat['minor_sigma'] = cat['minor_sigma'] * np.sqrt(8 * np.log(2))
cat.rename_column('major_sigma', 'major_fwhm')
cat.rename_column('minor_sigma', 'minor_fwhm')
cat.rename_column('flux', 'dend_flux_band{}'.format(band))
# Rename _idx to include the band number in the hundreds digit
for i in range(len(cat)):
cat['_idx'][i] = int('{}{:02d}'.format(band, cat['_idx'][i]))
# Output the catalog and region files
cat.write('./cat/cat_' + outfile + '.dat', format='ascii')
savereg(cat, './reg/reg_' + outfile + '.reg')
if plot: # create PDF plots of contour regions, if enabled
ax = plt.gca()
ax.cla()
plt.imshow(da,
cmap='gray_r',
interpolation='none',
origin='lower',
vmax=0.01,
vmin=-0.001)
pltr = d.plotter()
if verbose:
print("Plotting contours to PDF...")
pb = ProgressBar(len(d.leaves))
for struct in d.leaves: # iterate over each of the leaf structures
pltr.plot_contour(ax,
structure=struct,
colors=['r'],
linewidths=[0.9],
zorder=5)
if struct.parent:
while struct.parent:
struct = struct.parent
pltr.plot_contour(ax,
structure=struct,
colors=[(0, 1, 0, 1)],
linewidths=[0.5])
if verbose:
pb.update()
cntr = plt.gca().collections
plt.setp([x for x in cntr if x.get_color()[0, 0] == 1], linewidth=0.25)
plt.setp([x for x in cntr if x.get_color()[0, 1] == 1], linewidth=0.25)
plt.savefig('./contour/contour_' + outfile + '.pdf')
plt.axis((1125.4006254228616, 1670.3650637799306, 1291.6829155596627,
1871.8063499397681))
plt.setp([x for x in cntr if x.get_color()[0, 0] == 1],
linewidth=0.75) # Red
plt.setp([x for x in cntr if x.get_color()[0, 1] == 1],
linewidth=0.5) # Green
plt.savefig('./contour/contour_' + outfile + 'zoom.pdf')
'''
Test some pruning
'''
from astrodendro import Dendrogram
from astropy.io.fits import getdata
img = getdata("hd22.13co.intintensity.fits")
d1 = Dendrogram.compute(img, verbose=True, min_delta=1.0, min_npix=2)
d2 = Dendrogram.compute(img, verbose=True, min_delta=0.5, min_npix=2)
d2.prune(min_delta=1.0)
from astrodendro.tests import test_pruning
test_pruning.compare_dendrograms(d1, d2)
filename = 'orion_12CO'
#%&%&%&%&%&%&%&%&%&%&%&%
# Make dendrogram
#%&%&%&%&%&%&%&%&%&%&%&%
print 'Make dendrogram from the full cube'
hdu = fits.open(filename+'.fits')[0]
data = hdu.data
hd = hdu.header
# Survey designs
sigma = 0.3 #K, noise level
ppb = 1.3 #pixels/beam
d = Dendrogram.compute(data, min_value=sigma, \
min_delta=2*sigma, min_npix=3*ppb, verbose = 1)
# Plot the tree
fig = plt.figure(figsize=(14, 8))
ax = fig.add_subplot(111)
ax.set_yscale('log')
ax.set_xlabel('Structure')
ax.set_ylabel('Flux')
p = d.plotter()
p.plot_tree(ax, color='black')
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Generate the catalog
