def doScimes(self,dendroFITS,hd ): self.metadata["wcs"]= WCS(hd) d = Dendrogram.load_from( dendroFITS ) cat = ppv_catalog(d, self.metadata) res = SpectralCloudstering(d, cat, hd, criteria = ['volume' ], blind = True, rms = self.rms, s2nlim = 3, save_all = True, user_iter=1) res.clusters_asgn.writeto (self.regionName+'Cluster_asgn.fits', overwrite=True)
def deriv_decimate_leaves(d,
cube,
meta,
fscale=2,
sigdiscont=0.5,
nredun=2,
**kwargs):
leaves = get_leaves(d)
goodleaf = np.ones(len(leaves), dtype=np.bool)
for i, leaf in enumerate(leaves):
if not goodleaf[i] or (leaf.parent is None):
continue
parentobj = ppv_catalog([leaf.parent], meta, verbose=False)
children = ppv_catalog(leaf.parent.children, meta, verbose=False)
thisleaf = np.array(children['_idx']) == leaf.idx
dmajor = np.array(
(parentobj['major_sigma'] - children['major_sigma']) /
children['major_sigma']) > sigdiscont
dminor = np.array(
(parentobj['minor_sigma'] - children['minor_sigma']) /
children['minor_sigma']) > sigdiscont
dsigv = np.array((parentobj['v_rms'] - children['v_rms']) /
children['v_rms']) > sigdiscont
dflux = np.array((parentobj['v_rms'] - children['v_rms']) /
children['v_rms']) > (sigdiscont * fscale)
disc = np.any(
np.sum(np.c_[dmajor, dminor, dsigv, dflux], axis=1) >= nredun)
goodleaf[i] = disc
leaflist = []
for i, leaf in enumerate(leaves):
if goodleaf[i]:
leaflist += [leaf]
peaks = [leaf.get_peak()[0] for leaf in leaflist]
indices = tuples_to_arrays(peaks)
return (indices)
def get_catalog(cogal):
from astrodendro import Dendrogram
from astrodendro import ppv_catalog
co = cogal[0]
gal = cogal[1]
# get data and info
file = data[co][gal]['noise matched']['file']
cube = fits.open(file)[0]
noise = data[co][gal]['noise matched']['rms'].to(u.K)
bmin = cube.header['bmin'] * u.degree
bmaj = cube.header['bmaj'] * u.degree
beam_area = 1.13309 * bmin * bmaj
pix1 = u.Quantity(
str(np.abs(cube.header['cdelt1'])) + cube.header['cunit1'])
pix2 = u.Quantity(
str(np.abs(cube.header['cdelt2'])) + cube.header['cunit2'])
pix3 = u.Quantity(
str(np.abs(cube.header['cdelt3'])) + cube.header['cunit3'])
pix_beam = (beam_area / pix1 / pix2)
d = Dendrogram.load_from(join(compdir,
gal + '.' + co + '.dendrogram.fits'))
# time execution
print("\nComputing catalog: " + co + " " + gal + "\n")
start = time.time()
# calculate catalog
metadata = {
'data_unit': u.K,
'spatial_scale': parsec_to_angle(pix1, source=gal),
'velocity_scale': pix3,
'beam_major': bmaj,
'beam_minor': bmin,
'vaxis': 0,
'wavelength': lines[co]['restfreq'],
'wcs': WCS(cube.header)
}
catalog = ppv_catalog(d, metadata)
fnpickle(catalog, join(compdir, gal + '.' + co + '.catalog.pickle'))
# time execution
stop = time.time()
exec_time = np.round(stop - start, 1)
print("\nFinished catalog: " + co + " " + gal + "\nExecution took " +
str(datetime.timedelta(seconds=exec_time)) + "hours for " +
str(len(catalog)) + " structures.\n")
def doCluster(self, dendroCase, saveName, head):
savePath = self.dataPath + saveName + "/"
trunkN = len(dendroCase.trunk)
print "trunkN: {} ".format(trunkN)
os.system("mkdir " + savePath)
saveCloudFITS = savePath + saveName + "CloudAsgn.fits"
saveTrunkFITS = savePath + saveName + "TrunkAsgn.fits"
saveCloudCat = savePath + saveName + "CloudCat.fit"
saveDenroCat = savePath + saveName + "DendroCat.fit"
saveDenroFITS = savePath + saveName + "DendroFITS.fits"
dendroCase.save_to(saveDenroFITS) #save dendro
self.metadata['wcs'] = WCS(head)
cat = ppv_catalog(dendroCase, self.metadata)
dclust = SpectralCloudstering(dendroCase, cat, head)
cloudHDU = dclust.clusters_asgn
cloudHDU.writeto(saveCloudFITS, overwrite=True)
trunkHDU = dclust.trunks_asgn
trunkHDU.writeto(saveTrunkFITS, overwrite=True)
cCat = cat[dclust.clusters]
cCat.write(saveCloudCat, overwrite=True)
cat.write(saveDenroCat, overwrite=True)
cloudN = len(set(cloudHDU.data.reshape(-1))) - 1
#save dendro cat
print "trunkN: {}, cloudN: {}".format(trunkN, cloudN)
return trunkN, cloudN
def WriteCatalog(self):
"""
"""
print "Writing catalg"
if self.dendroData == None:
self.readDendro()
#define metadata
metadata = {}
metadata['data_unit'] = u.Kelvin #u.Jy / u.beam
metadata['spatial_scale'] = 30 * u.arcsec
metadata['beam_major'] = 50 * u.arcsec # FWHM
metadata['beam_minor'] = 50 * u.arcsec # FWHM
# the velocity_scale should be mentioned, other wise the v_rms would be in pixels
if self.regionName == "Mopra":
metadata['beam_major'] = 33 * u.arcsec # FWHM
metadata['beam_minor'] = 33 * u.arcsec # FWHM
metadata['wcs'] = WCS(self.CO12Head) # 22.9 * u.arcsec # FWHM
c = 299792458.
f = 115271202000.0
wavelength = c / f * u.meter
metadata['wavelength'] = wavelength # 22.9 * u.arcsec # FWHM
cat = ppv_catalog(self.dendroData, metadata)
#write catalog
try:
os.remove(self.dendroCat)
except:
pass
cat.write(self.dendroCat, format='fits')
def dendrogram_downaddmom_cube(*args, **kwargs):
datacube_dt, datacube_dt_header = downsample_addnoise_and_momentmask(*args)
datacube_dt_wcs = wcs.wcs.WCS(datacube_dt_header)
beam_size = 1/8 * u.deg
frequency = 115 * u.GHz
d = astrodendro.Dendrogram.compute(datacube_dt, wcs=datacube_dt_wcs, **kwargs)
v_scale = datacube_dt_header['cdelt3']
v_unit = u.km / u.s
l_scale = datacube_dt_header['cdelt1']
b_scale = datacube_dt_header['cdelt2']
metadata = {}
metadata['data_unit'] = u.K
metadata['spatial_scale'] = b_scale * u.deg
metadata['velocity_scale'] = v_scale * v_unit
metadata['wavelength'] = frequency
metadata['beam_major'] = beam_size
metadata['beam_minor'] = beam_size
metadata['vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v)
metadata['wcs'] = datacube_dt_wcs
catalog = astrodendro.ppv_catalog(d, metadata, verbose=True)
flux = u.Quantity(catalog['flux'])
area_exact = catalog['area_exact'].unit*catalog['area_exact'].data
# average brightness temperature integrated over area_exact
flux_kelvin = flux.to('K', equivalencies=u.brightness_temperature(area_exact, frequency))
# flux integrated over area and velocity
flux_kelvin_kms_deg2 = flux_kelvin * metadata['velocity_scale'] * area_exact
catalog.add_column(astropy.table.Column(data=flux_kelvin_kms_deg2, name='flux_kelvin_kms_deg2'))
return d, catalog, datacube_dt_header, metadata
ax = fig.add_subplot(111)
ax.set_yscale('log')
ax.set_xlabel('Structure')
ax.set_ylabel('Flux')
p = d.plotter()
p.plot_tree(ax, color='black')
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Generate the catalog
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print "Generate a catalog of dendrogram structures"
metadata = {}
metadata['data_unit'] = u.Jy #This should be Kelvin (not yet implemented)!
cat = ppv_catalog(d, metadata)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Running SCIMES
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print "Running SCIMES"
dclust = SpectralCloudstering(d, cat, criteria = ['volume'])
print "Visualize the clustered dendrogram"
dclust.showdendro()
print "Produce the assignment cube"
dclust.asgncube(hd)
def compute_catalog(d, header):
"""
Computes a catalog on a dendrogram.
Parameters
----------
d : astrodendro.dendrogram.Dendrogram
Dendrogram on which to compute a catalog
header : astropy.io.fits.header.Header
Header corresponding exactly to `d.data`.
Used for unit and scaling information to calculate flux and
world coordinates.
Returns
-------
catalog : astropy.table.table.Table
Catalog describing the structures in dendrogram `d`
using units provided in `header`.
metadata : dict
Explicitly lists unit properties used in `catalog`
"""
# rough average of North and South telescopes:
# see Dame et al. 2001, p. 794.
# "The Northern telescope... has a beamwidth of 8'.4 +/- 0'.2....
# The Southern telescope has nearly the same beamwidth to within
# the uncertainties: 8'.8 +/- 0'.2"
beam_size = 8.5 * u.arcmin
frequency = 115.27 * u.GHz # http://www.cv.nrao.edu/php/splat/
# Remember: by this point, the data ought to be in (v, b, l), with FITS header order opposite that
if 'vel' not in header['ctype3'].lower():
raise ValueError("CTYPE3 must be velocity - check that the data were permuted correctly")
v_scale = header['cdelt3']
v_unit = u.km / u.s
l_scale = header['cdelt1']
b_scale = header['cdelt2']
metadata = {}
metadata['data_unit'] = u.K
metadata['spatial_scale'] = b_scale * u.deg
metadata['velocity_scale'] = v_scale * v_unit
metadata['wavelength'] = frequency # formerly: (c.c / frequency).to('mm') but now compute_flux can handle frequency in spectral equivalency
metadata['beam_major'] = beam_size
metadata['beam_minor'] = beam_size
metadata['vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v)
metadata['wcs'] = d.wcs
catalog = astrodendro.ppv_catalog(d, metadata, verbose=True)
if catalog['flux'].unit.is_equivalent('Jy'):
# Workaround because flux is computed wrong (see https://github.com/dendrograms/astrodendro/issues/107)
flux = u.Quantity(catalog['flux'])
area_exact = u.Quantity(catalog['area_exact']) #.unit*catalog['area_exact'].data
# average brightness temperature integrated over area_exact
flux_kelvin = flux.to('K', equivalencies=u.brightness_temperature(area_exact, frequency))
# flux integrated over area and velocity
flux_kelvin_kms_sr = flux_kelvin * metadata['velocity_scale'] * area_exact.to(u.steradian)
catalog['flux_true'] = flux_kelvin_kms_sr
background_flux_array = np.zeros_like(catalog['flux_true'])
# do a clipping --
# this uses the approximation that any given structure's background is well-represented by its lowest-valued pixel, which can get messy sometimes.
for i, row in enumerate(catalog):
background_kelvins = d[i].vmin * u.K
background_flux = background_kelvins * d[i].get_npix() * metadata['velocity_scale'] * metadata['spatial_scale']**2
background_flux_array[i] = background_flux.to(u.K * u.km/u.s * u.steradian).value
catalog['flux_clipped'] = catalog['flux_true'] - background_flux_array
return catalog, metadata
def run_dendro(
label='mycloud',
cubefile=None,
flatfile=None,
redo='n',
nsigma=3.,
min_delta=2.5,
min_bms=2.,
position_dependent_noise=False, # will use rms map in dendro
criteria=['volume'], # for SCIMES
doplots=True,
dendro_in=None, # use this instead of re-loading
**kwargs):
global cubedata, rmsmap, threshold_sigma
#%&%&%&%&%&%&%&%&%&%&%&%
# Make dendrogram
#%&%&%&%&%&%&%&%&%&%&%&%
hdu3 = fits.open(cubefile)[0]
hd3 = hdu3.header
cubedata = hdu3.data
# Deal with oddities in 30 Dor cube
if hd3['NAXIS'] == 3:
for key in [
'CTYPE4', 'CRVAL4', 'CDELT4', 'CRPIX4', 'CUNIT4', 'NAXIS4'
]:
if key in hd3.keys():
hd3.remove(key)
# Get cube parameters
sigma = stats.mad_std(hdu3.data[~np.isnan(hdu3.data)])
print('Robustly estimated RMS: ', sigma)
ppb = 1.133 * hd3['bmaj'] * hd3['bmin'] / (abs(
hd3['cdelt1'] * hd3['cdelt2']))
print('Pixels per beam: ', ppb)
# Make the dendrogram if not present or redo=y
if position_dependent_noise:
dendrofile = 'dendro_dendrogram_rmsmap.hdf5'
else:
dendrofile = 'dendro_dendrogram.hdf5'
if dendro_in != None:
d = dendro_in
elif redo == 'n' and os.path.isfile(dendrofile):
print('Loading pre-existing dendrogram')
d = Dendrogram.load_from(dendrofile)
else:
print('Make dendrogram from the full cube')
if position_dependent_noise:
mask3d = cubedata < 3 * sigma
rmsmap = np.nanstd(cubedata * mask3d,
axis=0) # assumes spectral 1st
threshold_sigma = min_delta # for custom_independent function
d = Dendrogram.compute(hdu3.data,
min_value=nsigma * sigma,
min_delta=min_delta * sigma,
min_npix=min_bms * ppb,
verbose=1,
is_independent=custom_independent)
else:
d = Dendrogram.compute(hdu3.data,
min_value=nsigma * sigma,
min_delta=min_delta * sigma,
min_npix=min_bms * ppb,
verbose=1)
d.save_to(dendrofile)
if doplots:
# checks/creates directory to place plots
if os.path.isdir('dendro_plots') == 0:
os.makedirs('dendro_plots')
# Plot the tree
fig = plt.figure(figsize=(14, 8))
ax = fig.add_subplot(111)
#ax.set_yscale('log')
ax.set_xlabel('Structure')
ax.set_ylabel('Intensity [' + hd3['BUNIT'] + ']')
p = d.plotter()
branch = [
s for s in d.all_structures
if s not in d.leaves and s not in d.trunk
]
tronly = [s for s in d.trunk if s not in d.leaves]
for st in tronly:
p.plot_tree(ax, structure=[st], color='brown', subtree=False)
for st in branch:
p.plot_tree(ax, structure=[st], color='black', subtree=False)
for st in d.leaves:
p.plot_tree(ax, structure=[st], color='green')
#p.plot_tree(ax, color='black')
plt.savefig('dendro_plots/' + label + '_dendrogram.pdf',
bbox_inches='tight')
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Generate the catalog
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Generate a catalog of dendrogram structures")
metadata = {}
if hd3['BUNIT'].upper() == 'JY/BEAM':
metadata['data_unit'] = u.Jy / u.beam
elif hd3['BUNIT'].upper() == 'K':
metadata['data_unit'] = u.K
else:
print("Warning: Unrecognized brightness unit")
metadata['vaxis'] = 0
if 'RESTFREQ' in hd3.keys():
freq = hd3['RESTFREQ'] * u.Hz
elif 'RESTFRQ' in hd3.keys():
freq = hd3['RESTFRQ'] * u.Hz
metadata['wavelength'] = freq.to(u.m, equivalencies=u.spectral())
metadata['spatial_scale'] = hd3['cdelt2'] * 3600. * u.arcsec
if hd3['ctype3'][0:3] == 'VEL' or hd3['ctype3'][0:4] == 'VRAD':
dv = hd3['cdelt3'] / 1000. * u.km / u.s
else:
assert hd3['ctype3'][0:4] == 'FREQ'
dv = 2.99792458e5 * np.absolute(
hd3['cdelt3']) / freq.value * u.km / u.s
metadata['velocity_scale'] = dv
bmaj = hd3['bmaj'] * 3600. * u.arcsec # FWHM
bmin = hd3['bmin'] * 3600. * u.arcsec # FWHM
metadata['beam_major'] = bmaj
metadata['beam_minor'] = bmin
if not (redo == "n" and os.path.exists(label + '_full_catalog.txt')):
print("generating catalog")
cat = ppv_catalog(d, metadata)
print(cat.info())
# Add additional properties: Average Peak Tb and Maximum Tb
srclist = cat['_idx'].tolist()
tmax = np.zeros(len(srclist), dtype=np.float64)
tpkav = np.zeros(len(srclist), dtype=np.float64)
for i, c in enumerate(srclist):
peakim = np.nanmax(hdu3.data * d[c].get_mask(), axis=0)
peakim[peakim == 0] = np.nan
tmax[i] = np.nanmax(peakim)
tpkav[i] = np.nanmean(peakim)
if hd3['BUNIT'].upper() == 'JY/BEAM':
omega_B = np.pi / (4 * np.log(2)) * bmaj * bmin
convfac = (u.Jy).to(u.K,
equivalencies=u.brightness_temperature(
omega_B, freq))
tmax *= convfac
tpkav *= convfac
newcol = Column(tmax, name='tmax')
newcol.unit = 'K'
cat.add_column(newcol)
newcol = Column(tpkav, name='tpkav')
newcol.unit = 'K'
cat.add_column(newcol)
cat.write(label + '_full_catalog.txt',
format='ascii.ecsv',
overwrite=True)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Generate the catalog with clipping
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
if not (redo == "n"
and os.path.exists(label + '_full_catalog_clipped.txt')):
print("generating clipped catalog")
ccat = ppv_catalog(d, metadata, clipping=True)
print(ccat.info())
# Add additional properties: Average Peak Tb and Maximum Tb
srclist = ccat['_idx'].tolist()
tmax = np.zeros(len(srclist), dtype=np.float64)
tpkav = np.zeros(len(srclist), dtype=np.float64)
for i, c in enumerate(srclist):
peakim = np.nanmax(hdu3.data * d[c].get_mask(), axis=0)
peakim[peakim == 0] = np.nan
clmin = np.nanmin(hdu3.data * d[c].get_mask())
tmax[i] = np.nanmax(peakim) - clmin
tpkav[i] = np.nanmean(peakim) - clmin
if hd3['BUNIT'].upper() == 'JY/BEAM':
omega_B = np.pi / (4 * np.log(2)) * bmaj * bmin
convfac = (u.Jy).to(u.K,
equivalencies=u.brightness_temperature(
omega_B, freq))
tmax *= convfac
tpkav *= convfac
newcol = Column(tmax, name='tmax-tmin')
newcol.unit = 'K'
ccat.add_column(newcol)
newcol = Column(tpkav, name='tpkav-tmin')
newcol.unit = 'K'
ccat.add_column(newcol)
ccat.write(label + '_full_catalog_clipped.txt',
format='ascii.ecsv',
overwrite=True)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Running SCIMES
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# print("Running SCIMES")
# dclust = SpectralCloudstering(d, cat, criteria = criteria, keepall=True)
# print(dclust.clusters)
#
# print("Visualize the clustered dendrogram")
# dclust.showdendro()
# plt.savefig('dendro_plots/'+label+'_clusters_tree.pdf')
#
# print("Produce the assignment cube")
# dclust.asgncube(hd3)
# try:
# os.remove(label+'_asgncube.fits.gz')
# except OSError:
# pass
# dclust.asgn.writeto(label+'_asgncube.fits.gz')
#sys.exit("Stopping here")
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the trunks
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
if doplots:
print("Image the trunks")
hdu2 = fits.open(flatfile)[0]
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
vmax = np.nanmax(hdu2.data) / 2.
im = ax.matshow(hdu2.data,
origin='lower',
cmap=plt.cm.Blues,
vmax=vmax)
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
if 'xlims' in kwargs:
ax.set_xlim(kwargs['xlims'])
if 'ylims' in kwargs:
ax.set_ylim(kwargs['ylims'])
# Make a trunk list
tronly = [s for s in d.trunk if s not in d.leaves]
f = open(label + '_trunks.txt', 'w')
for c in tronly:
f.write('{:<4d} | '.format(c.idx))
# Plot the actual structure boundaries
mask = d[c.idx].get_mask()
mask_coll = np.amax(mask, axis=0)
plt.contour(mask_coll, colors='red', linewidths=1, levels=[0])
# Plot the ellipse fits
s = analysis.PPVStatistic(d[c.idx])
ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none')
ax.add_patch(ellipse)
# Make sub-lists of descendants
print('Finding descendants of trunk ', c.idx)
desclist = []
if len(d[c.idx].descendants) > 0:
for s in d[c.idx].descendants:
desclist.append(s.idx)
desclist.sort()
liststr = ','.join(map(str, desclist))
f.write(liststr)
f.write("\n")
f.close()
fig.colorbar(im, ax=ax)
plt.savefig('dendro_plots/' + label + '_trunks_map.pdf',
bbox_inches='tight')
plt.close()
# Make a branch list
branch = [
s for s in d.all_structures
if s not in d.leaves and s not in d.trunk
]
slist = []
for c in branch:
slist.append(c.idx)
slist.sort()
with open(label + '_branches.txt', 'w') as output:
writer = csv.writer(output)
for val in slist:
writer.writerow([val])
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the leaves
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Image the leaves")
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
vmax = np.nanmax(hdu2.data) / 2.
im = ax.matshow(hdu2.data,
origin='lower',
cmap=plt.cm.Blues,
vmax=vmax)
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
if 'xlims' in kwargs:
ax.set_xlim(kwargs['xlims'])
if 'ylims' in kwargs:
ax.set_ylim(kwargs['ylims'])
# Make a leaf list
slist = []
for c in d.leaves:
slist.append(c.idx)
# Plot the actual structure boundaries
mask = d[c.idx].get_mask()
mask_coll = np.amax(mask, axis=0)
plt.contour(mask_coll, colors='green', linewidths=1, levels=[0])
# Plot the ellipse fits
s = analysis.PPVStatistic(d[c.idx])
ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none')
ax.add_patch(ellipse)
slist.sort()
with open(label + '_leaves.txt', "w") as output:
writer = csv.writer(output)
for val in slist:
writer.writerow([val])
fig.colorbar(im, ax=ax)
plt.savefig('dendro_plots/' + label + '_leaves_map.pdf',
bbox_inches='tight')
plt.close()
def downsampled_demo(data_file,
downsample_factor=4,
transpose_tuple=(2, 0, 1),
min_value=0.01,
min_delta=0.005,
min_npix=2000,
neighbours=None,
resample=False,
compute_catalog=True,
recenter=True,
data_path=data_path,
memmap=True):
df = downsample_factor
tt = transpose_tuple
print(
"\n ** Downsampled Demo Parameters:"
"\n downsample_factor={0}"
"\n transpose_tuple={1}"
"\n min_value={2}"
"\n min_delta={3}"
"\n min_npix={4}\n".format(downsample_factor, transpose_tuple,
min_value, min_delta, min_npix))
beginning = datetime.datetime.now()
beginning_string = datetime.datetime.strftime(beginning,
"%Y-%m-%d %H:%M:%S")
print("\n ** Beginning at: {0}".format(beginning_string))
print "\n** loading data from {0}: ...\n".format(data_path + data_file)
datacube, datacube_header = getdata(data_path + data_file,
memmap=memmap,
header=True)
print "\n** transposing {0} and downsampling ({1}) data: ...\n".format(
tt, df)
datacube_dt, datacube_dt_header = \
downsample_and_transpose_data_and_header(datacube, datacube_header, df, tt, resample=resample, recenter=recenter)
datacube_dt_wcs = wcs.wcs.WCS(datacube_dt_header)
datacube_dt_wcs.wcs.bounds_check(pix2world=False)
beam_size = 1 / 8 * u.deg
frequency = 115 * u.GHz
print "\n** computing dendrogram on data with dimensions {} and n_pixels {:,}: ...\n".format(
np.shape(datacube_dt), np.size(datacube_dt))
d = astrodendro.Dendrogram.compute(
datacube_dt,
min_value=min_value,
min_delta=min_delta, #these are arbitrary
min_npix=min_npix,
verbose=True,
neighbours=neighbours,
wcs=datacube_dt_wcs)
v_scale = datacube_dt_header['cdelt3']
v_unit = u.km / u.s
l_scale = datacube_dt_header['cdelt1']
b_scale = datacube_dt_header['cdelt2']
metadata = {}
metadata['data_unit'] = u.K
metadata['spatial_scale'] = b_scale * u.deg
metadata['velocity_scale'] = v_scale * v_unit
metadata[
'wavelength'] = frequency # formerly: (c.c / frequency).to('mm') but now compute_flux can handle frequency in spectral equivalency
metadata['beam_major'] = beam_size
metadata['beam_minor'] = beam_size
metadata[
'vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v)
metadata['wcs'] = datacube_dt_wcs
print "\n** computing catalog for {:,} structures: ...\n".format(len(d))
if compute_catalog:
catalog = astrodendro.ppv_catalog(d, metadata, verbose=True)
else:
catalog = None
return d, catalog, datacube_dt_header, metadata
if catalog['flux'].unit.is_equivalent('Jy'):
# Workaround because flux is computed wrong
flux = quantify_column(catalog['flux'])
area_exact = catalog['area_exact'].unit * catalog['area_exact'].data
# average brightness temperature integrated over area_exact
flux_kelvin = flux.to('K',
equivalencies=u.brightness_temperature(
area_exact, frequency))
# flux integrated over area and velocity
flux_kelvin_kms_deg2 = flux_kelvin * metadata[
'velocity_scale'] * area_exact
catalog.add_column(
astropy.table.Column(data=flux_kelvin_kms_deg2,
name='flux_kelvin_kms_deg2'))
end = datetime.datetime.now()
end_string = datetime.datetime.strftime(end, "%Y-%m-%d %H:%M:%S")
print("\n ** Ending at: {0}".format(end_string))
time_elapsed = (end - beginning)
print "\n ** Time elapsed: {0}".format(time_elapsed)
return d, catalog, datacube_dt_header, metadata
def main(args):
#%&%&%&%&%&%&%&%&%&%&%&%
# Make dendrogram
#%&%&%&%&%&%&%&%&%&%&%&%
print('Make dendrogram from the full cube')
hdu = fits.open(args.fits_file)[0]
data = hdu.data[801:1000, :, :]
hd = hdu.header
# Survey designs
sigma = args.sigma #K, noise level
ppb = args.ppb #pixels/beam
def custom_independent(structure, index=None, value=None):
peak_index, peak_value = structure.get_peak()
return peak_value > 3. * sigma
d = Dendrogram.compute(data, min_value=sigma, \
min_delta=args.delta*sigma, min_npix=1.*ppb, \
is_independent=custom_independent, \
verbose = 1)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Generate the catalog
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Generate a catalog of dendrogram structures")
metadata = {}
metadata['data_unit'] = u.Jy #This should be Kelvin (not yet implemented)!
cat = ppv_catalog(d, metadata)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Running SCIMES
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Running SCIMES")
dclust = scimes.SpectralCloudstering(d, cat, hd, rms=sigma, \
user_iter=args.iter, \
save_all_leaves = True, \
)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the result
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Visualize the clustered dendrogram")
dclust.showdendro()
print("Visualize collapsed maps of the assignment cubes")
cubes = [dclust.clusters_asgn,\
dclust.leaves_asgn,\
dclust.trunks_asgn]
titles = ['Clusters', 'Leaves', 'Trunks']
for cube, title in zip(cubes, titles):
plt.figure()
plt.imshow(np.nanmax(cube.data,axis=0),origin='lower',\
interpolation='nearest',cmap='jet')
plt.title(title + ' assignment map')
plt.colorbar(label='Structure label')
plt.xlabel('X [pixel]')
plt.ylabel('Y [pixel]')
# plt.show()
print("Image the results with APLpy")
clusts = dclust.clusters
colors = dclust.colors
# Create Orion integrated intensity map
mhdu = mom0map(hdu)
fig = aplpy.FITSFigure(mhdu, figsize=(8, 6), convention='wells')
fig.show_colorscale(cmap='gray') #, vmax=36, stretch = 'sqrt')
count = 0
for c in clusts:
mask = d[c].get_mask()
mask_hdu = fits.PrimaryHDU(mask.astype('short'), hdu.header)
mask_coll = np.amax(mask_hdu.data, axis=0)
mask_coll_hdu = fits.PrimaryHDU(mask_coll.astype('short'),
hd2d(hdu.header))
fig.show_contour(mask_coll_hdu,
colors=colors[count],
linewidths=2,
convention='wells',
levels=[0])
count = count + 1
print(count)
# fig.tick_labels.set_xformat('dd')
# fig.tick_labels.set_yformat('dd')
fig.add_colorbar()
fig.colorbar.set_axis_label_text(r'[(K km/s)$^{1/2}$]')
fig.save('b.png')
plt.show()
return 0
def measure_dendrogram_properties(dend=None,
cube303=cube303,
cube321=cube321,
cube13co=cube13co,
cube18co=cube18co,
noise_cube=noise_cube,
sncube=sncube,
suffix="",
last_index=None,
plot_some=True,
line='303',
write=True):
assert (cube321.shape == cube303.shape == noise_cube.shape ==
cube13co.shape == cube18co.shape == sncube.shape)
assert sncube.wcs is cube303.wcs is sncube.mask._wcs
metadata = {}
metadata['data_unit'] = u.K
metadata['spatial_scale'] = 7.2 * u.arcsec
metadata['beam_major'] = 30 * u.arcsec
metadata['beam_minor'] = 30 * u.arcsec
metadata['wavelength'] = 218.22219 * u.GHz
metadata['velocity_scale'] = u.km / u.s
metadata['wcs'] = cube303.wcs
keys = [
'density_chi2',
'expected_density',
'dmin1sig_chi2',
'dmax1sig_chi2',
'column_chi2',
'expected_column',
'cmin1sig_chi2',
'cmax1sig_chi2',
'temperature_chi2',
'expected_temperature',
'tmin1sig_chi2',
'tmax1sig_chi2',
'eratio321303',
'ratio321303',
'logh2column',
'elogh2column',
'logabundance',
'elogabundance',
]
obs_keys = [
'Stot303',
'Smin303',
'Smax303',
'Stot321',
'Smean303',
'Smean321',
'npix',
'e303',
'e321',
'r321303',
'er321303',
'13cosum',
'c18osum',
'13comean',
'c18omean',
's_ntotal',
'index',
'is_leaf',
'parent',
'root',
'lon',
'lat',
'vcen',
'higaldusttem',
'reff',
'dustmass',
'dustmindens',
'bad',
#'tkin_turb',
]
columns = {k: [] for k in (keys + obs_keys)}
log.debug("Initializing dendrogram temperature fitting loop")
# FORCE wcs to match
# (technically should reproject here)
cube13co._wcs = cube18co._wcs = cube303.wcs
cube13co.mask._wcs = cube18co.mask._wcs = cube303.wcs
if line == '303':
maincube = cube303
elif line == '321':
maincube = cube321
else:
raise ValueError("Unrecognized line: {0}".format(line))
# Prepare an array to hold the fitted temperatures
tcubedata = np.empty(maincube.shape, dtype='float32')
tcubedata[:] = np.nan
tcubeleafdata = np.empty(maincube.shape, dtype='float32')
tcubeleafdata[:] = np.nan
nbad = 0
catalog = ppv_catalog(dend, metadata)
pb = ProgressBar(len(catalog))
for ii, row in enumerate(catalog):
structure = dend[row['_idx']]
assert structure.idx == row['_idx'] == ii
dend_obj_mask = BooleanArrayMask(structure.get_mask(), wcs=cube303.wcs)
dend_inds = structure.indices()
view = (
slice(dend_inds[0].min(), dend_inds[0].max() + 1),
slice(dend_inds[1].min(), dend_inds[1].max() + 1),
slice(dend_inds[2].min(), dend_inds[2].max() + 1),
)
#view2 = cube303.subcube_slices_from_mask(dend_obj_mask)
submask = dend_obj_mask[view]
#assert np.count_nonzero(submask.include()) == np.count_nonzero(dend_obj_mask.include())
sn = sncube[view].with_mask(submask)
sntot = sn.sum().value
#np.testing.assert_almost_equal(sntot, structure.values().sum(), decimal=0)
c303 = cube303[view].with_mask(submask)
c321 = cube321[view].with_mask(submask)
co13sum = cube13co[view].with_mask(submask).sum().value
co18sum = cube18co[view].with_mask(submask).sum().value
if hasattr(co13sum, '__len__'):
raise TypeError(
".sum() applied to an array has yielded a non scalar.")
npix = submask.include().sum()
assert npix == structure.get_npix()
Stot303 = c303.sum().value
if np.isnan(Stot303):
raise ValueError("NaN in cube. This can't happen: the data from "
"which the dendrogram was derived can't have "
"NaN pixels.")
Smax303 = c303.max().value
Smin303 = c303.min().value
Stot321 = c321.sum().value
if npix == 0:
raise ValueError("npix=0. This is impossible.")
Smean303 = Stot303 / npix
if Stot303 <= 0 and line == '303':
raise ValueError(
"The 303 flux is <=0. This isn't possible because "
"the dendrogram was derived from the 303 data with a "
"non-zero threshold.")
elif Stot303 <= 0 and line == '321':
Stot303 = 0
Smean303 = 0
elif Stot321 <= 0 and line == '321':
raise ValueError(
"The 321 flux is <=0. This isn't possible because "
"the dendrogram was derived from the 321 data with a "
"non-zero threshold.")
if np.isnan(Stot321):
raise ValueError("NaN in 321 line")
Smean321 = Stot321 / npix
#error = (noise_cube[view][submask.include()]).sum() / submask.include().sum()**0.5
var = ((noise_cube[dend_obj_mask.include()]**2).sum() / npix**2)
error = var**0.5
if np.isnan(error):
raise ValueError("error is nan: this is impossible by definition.")
if line == '321' and Stot303 == 0:
r321303 = np.nan
er321303 = np.nan
elif Stot321 < 0:
r321303 = error / Smean303
er321303 = (r321303**2 * (var / Smean303**2 + 1))**0.5
else:
r321303 = Stot321 / Stot303
er321303 = (r321303**2 *
(var / Smean303**2 + var / Smean321**2))**0.5
for c in columns:
assert len(columns[c]) == ii
columns['index'].append(row['_idx'])
columns['s_ntotal'].append(sntot)
columns['Stot303'].append(Stot303)
columns['Smax303'].append(Smax303)
columns['Smin303'].append(Smin303)
columns['Stot321'].append(Stot321)
columns['Smean303'].append(Smean303)
columns['Smean321'].append(Smean321)
columns['npix'].append(npix)
columns['e303'].append(error)
columns['e321'].append(error)
columns['r321303'].append(r321303)
columns['er321303'].append(er321303)
columns['13cosum'].append(co13sum)
columns['c18osum'].append(co18sum)
columns['13comean'].append(co13sum / npix)
columns['c18omean'].append(co18sum / npix)
columns['is_leaf'].append(structure.is_leaf)
columns['parent'].append(
structure.parent.idx if structure.parent else -1)
columns['root'].append(get_root(structure).idx)
s_main = maincube._data[dend_inds]
x, y, z = maincube.world[dend_inds]
lon = ((z.value - (360 *
(z.value > 180))) * s_main).sum() / s_main.sum()
lat = (y * s_main).sum() / s_main.sum()
vel = (x * s_main).sum() / s_main.sum()
columns['lon'].append(lon)
columns['lat'].append(lat.value)
columns['vcen'].append(vel.value)
mask2d = dend_obj_mask.include().max(axis=0)[view[1:]]
logh2column = np.log10(
np.nanmean(column_regridded.data[view[1:]][mask2d]) * 1e22)
if np.isnan(logh2column):
log.info("Source #{0} has NaNs".format(ii))
logh2column = 24
elogh2column = elogabundance
columns['higaldusttem'].append(
np.nanmean(dusttem_regridded.data[view[1:]][mask2d]))
r_arcsec = row['radius'] * u.arcsec
reff = (r_arcsec * (8.5 * u.kpc)).to(u.pc, u.dimensionless_angles())
mass = ((10**logh2column * u.cm**-2) * np.pi * reff**2 * 2.8 *
constants.m_p).to(u.M_sun)
density = (mass / (4 / 3. * np.pi * reff**3) / constants.m_p / 2.8).to(
u.cm**-3)
columns['reff'].append(reff.value)
columns['dustmass'].append(mass.value)
columns['dustmindens'].append(density.value)
mindens = np.log10(density.value)
if mindens < 3:
mindens = 3
if (r321303 < 0 or np.isnan(r321303)) and line != '321':
raise ValueError("Ratio <0: This can't happen any more because "
"if either num/denom is <0, an exception is "
"raised earlier")
#for k in columns:
# if k not in obs_keys:
# columns[k].append(np.nan)
elif (r321303 < 0 or np.isnan(r321303)) and line == '321':
for k in keys:
columns[k].append(np.nan)
else:
# Replace negatives for fitting
if Smean321 <= 0:
Smean321 = error
mf.set_constraints(
ratio321303=r321303,
eratio321303=er321303,
#ratio321322=ratio2, eratio321322=eratio2,
logh2column=logh2column,
elogh2column=elogh2column,
logabundance=logabundance,
elogabundance=elogabundance,
taline303=Smean303,
etaline303=error,
taline321=Smean321,
etaline321=error,
mindens=mindens,
linewidth=10)
row_data = mf.get_parconstraints()
row_data['ratio321303'] = r321303
row_data['eratio321303'] = er321303
for k in row_data:
columns[k].append(row_data[k])
# Exclude bad velocities from cubes
if row['v_cen'] < -80e3 or row['v_cen'] > 180e3:
# Skip: there is no real structure down here
nbad += 1
is_bad = True
else:
is_bad = False
tcubedata[
dend_obj_mask.include()] = row_data['expected_temperature']
if structure.is_leaf:
tcubeleafdata[dend_obj_mask.include(
)] = row_data['expected_temperature']
columns['bad'].append(is_bad)
width = row['v_rms'] * u.km / u.s
lengthscale = reff
#REMOVED in favor of despotic version done in dendrograms.py
# we use the analytic version here; the despotic version is
# computed elsewhere (with appropriate gcor factors)
#columns['tkin_turb'].append(heating.tkin_all(10**row_data['density_chi2']*u.cm**-3,
# width,
# lengthscale,
# width/lengthscale,
# columns['higaldusttem'][-1]*u.K,
# crir=0./u.s))
if len(set(len(c) for k, c in columns.items())) != 1:
print("Columns are different lengths. This is not allowed.")
import ipdb
ipdb.set_trace()
for c in columns:
assert len(columns[c]) == ii + 1
if plot_some and not is_bad and (ii - nbad % 100 == 0
or ii - nbad < 50):
try:
log.info(
"T: [{tmin1sig_chi2:7.2f},{expected_temperature:7.2f},{tmax1sig_chi2:7.2f}]"
" R={ratio321303:8.4f}+/-{eratio321303:8.4f}"
" Smean303={Smean303:8.4f} +/- {e303:8.4f}"
" Stot303={Stot303:8.2e} npix={npix:6d}".format(
Smean303=Smean303,
Stot303=Stot303,
npix=npix,
e303=error,
**row_data))
pl.figure(1)
pl.clf()
mf.denstemplot()
pl.savefig(
fpath("dendrotem/diagnostics/{0}_{1}.png".format(
suffix, ii)))
pl.figure(2).clf()
mf.parplot1d_all(levels=[0.68268949213708585])
pl.savefig(
fpath("dendrotem/diagnostics/1dplot{0}_{1}.png".format(
suffix, ii)))
pl.draw()
pl.show()
except Exception as ex:
print ex
pass
else:
pb.update(ii + 1)
if last_index is not None and ii >= last_index:
break
if last_index is not None:
catalog = catalog[:last_index + 1]
for k in columns:
if k not in catalog.keys():
catalog.add_column(table.Column(name=k, data=columns[k]))
for mid, lo, hi, letter in (('expected_temperature', 'tmin1sig_chi2',
'tmax1sig_chi2', 't'),
('expected_density', 'dmin1sig_chi2',
'dmax1sig_chi2', 'd'),
('expected_column', 'cmin1sig_chi2',
'cmax1sig_chi2', 'c')):
catalog.add_column(
table.Column(name='elo_' + letter,
data=catalog[mid] - catalog[lo]))
catalog.add_column(
table.Column(name='ehi_' + letter,
data=catalog[hi] - catalog[mid]))
if write:
catalog.write(tpath('PPV_H2CO_Temperature{0}.ipac'.format(suffix)),
format='ascii.ipac')
# Note that there are overlaps in the catalog, which means that ORDER MATTERS
# in the above loop. I haven't yet checked whether large scale overwrites
# small or vice-versa; it may be that both views of the data are interesting.
tcube = SpectralCube(
data=tcubedata,
wcs=cube303.wcs,
mask=cube303.mask,
meta={'unit': 'K'},
header=cube303.header,
)
tcubeleaf = SpectralCube(
data=tcubeleafdata,
wcs=cube303.wcs,
mask=cube303.mask,
meta={'unit': 'K'},
header=cube303.header,
)
if write:
log.info("Writing TemperatureCube")
outpath = 'TemperatureCube_DendrogramObjects{0}.fits'
tcube.write(hpath(outpath.format(suffix)), overwrite=True)
outpath_leaf = 'TemperatureCube_DendrogramObjects{0}_leaves.fits'
tcubeleaf.write(hpath(outpath_leaf.format(suffix)), overwrite=True)
return catalog, tcube
def run_scimes(criteria=['volume'],
label='scimes',
cubefile=None,
mom0file=None,
bmajas=None,
bminas=None,
rfreq=None,
redo='n',
verbose=True,
**kwargs):
#%&%&%&%&%&%&%&%&%&%&%&%
# Make dendrogram
#%&%&%&%&%&%&%&%&%&%&%&%
hdu3 = fits.open(cubefile)[0]
hd3 = hdu3.header
# Deal with oddities in 30 Dor cube
if hd3['NAXIS'] == 3:
for key in [
'CTYPE4', 'CRVAL4', 'CDELT4', 'CRPIX4', 'CUNIT4', 'NAXIS4'
]:
if key in hd3.keys():
hd3.remove(key)
# Provide beam info if missing
if bmajas is not None:
hd3['bmaj'] = bmajas / 3600.
if bminas is not None:
hd3['bmin'] = bminas / 3600.
# Get cube parameters
sigma = stats.mad_std(hdu3.data[~np.isnan(hdu3.data)])
print('Robustly estimated RMS: {:.3f}'.format(sigma))
ppb = 1.133 * hd3['bmaj'] * hd3['bmin'] / (abs(
hd3['cdelt1'] * hd3['cdelt2']))
print('Pixels per beam: {:.2f}'.format(ppb))
# Make the dendrogram if not present or redo=y
if redo == 'n' and os.path.isfile(label + '_dendrogram.hdf5'):
print('Loading pre-existing dendrogram')
d = Dendrogram.load_from(label + '_dendrogram.hdf5')
else:
print('Make dendrogram from the full cube')
d = Dendrogram.compute(hdu3.data,
min_value=3 * sigma,
min_delta=2.5 * sigma,
min_npix=2 * ppb,
verbose=verbose)
d.save_to(label + '_dendrogram.hdf5')
# checks/creates directory to place plots
if os.path.isdir('plots') == 0:
os.makedirs('plots')
# Plot the tree
fig = plt.figure(figsize=(14, 8))
ax = fig.add_subplot(111)
ax.set_yscale('log')
ax.set_xlabel('Structure')
ax.set_ylabel('Intensity [' + hd3['BUNIT'] + ']')
p = d.plotter()
branch = [
s for s in d.all_structures if s not in d.leaves and s not in d.trunk
]
tronly = [s for s in d.trunk if s not in d.leaves]
for st in tronly:
p.plot_tree(ax, structure=[st], color='brown', subtree=False)
for st in branch:
p.plot_tree(ax, structure=[st], color='black', subtree=False)
for st in d.leaves:
p.plot_tree(ax, structure=[st], color='green')
plt.savefig('plots/' + label + '_dendrogram.pdf', bbox_inches='tight')
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Generate the catalog
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print('Generate a catalog of dendrogram structures')
metadata = {}
if hd3['BUNIT'].upper() == 'JY/BEAM':
metadata['data_unit'] = u.Jy / u.beam
elif hd3['BUNIT'].upper() == 'K':
metadata['data_unit'] = u.K
else:
print('Warning: Unrecognized brightness unit')
metadata['vaxis'] = 0
if rfreq is None:
if 'RESTFREQ' in hd3.keys():
freq = hd3['RESTFREQ'] * u.Hz
elif 'RESTFRQ' in hd3.keys():
freq = hd3['RESTFRQ'] * u.Hz
else:
freq = rfreq * u.GHz
metadata['wavelength'] = freq.to(u.m, equivalencies=u.spectral())
metadata['spatial_scale'] = hd3['cdelt2'] * 3600. * u.arcsec
metadata['velocity_scale'] = abs(hd3['cdelt3']) * u.meter / u.second
bmaj = hd3['bmaj'] * 3600. * u.arcsec # FWHM
bmin = hd3['bmin'] * 3600. * u.arcsec # FWHM
metadata['beam_major'] = bmaj
metadata['beam_minor'] = bmin
cat = ppv_catalog(d, metadata, verbose=verbose)
print(cat.info())
# Add additional properties: Average Peak Tb and Maximum Tb
srclist = cat['_idx'].tolist()
tmax = np.zeros(len(srclist), dtype=np.float64)
tpkav = np.zeros(len(srclist), dtype=np.float64)
for i, c in enumerate(srclist):
peakim = np.nanmax(hdu3.data * d[c].get_mask(), axis=0)
peakim[peakim == 0] = np.nan
tmax[i] = np.nanmax(peakim)
tpkav[i] = np.nanmean(peakim)
if hd3['BUNIT'].upper() == 'JY/BEAM':
omega_B = np.pi / (4 * np.log(2)) * bmaj * bmin
convfac = (u.Jy).to(u.K,
equivalencies=u.brightness_temperature(
omega_B, freq))
tmax *= convfac
tpkav *= convfac
newcol = Column(tmax, name='tmax')
newcol.unit = 'K'
cat.add_column(newcol)
newcol = Column(tpkav, name='tpkav')
newcol.unit = 'K'
cat.add_column(newcol)
cat.write(label + '_full_catalog.txt', format='ascii.ecsv', overwrite=True)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Running SCIMES
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Running SCIMES")
dclust = SpectralCloudstering(d, cat, criteria=criteria, keepall=True)
print(dclust.clusters)
print("Visualize the clustered dendrogram")
dclust.showdendro()
plt.savefig('plots/' + label + '_clusters_tree.pdf')
print("Produce the assignment cube")
dclust.asgncube(hd3)
try:
os.remove(label + '_asgncube.fits.gz')
except OSError:
pass
dclust.asgn.writeto(label + '_asgncube.fits.gz')
#sys.exit("Stopping here")
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the trunks
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Image the trunks")
hdu2 = fits.open(mom0file)[0]
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
vmax = np.nanmax(hdu2.data) / 2.
im = ax.matshow(hdu2.data, origin='lower', cmap=plt.cm.Blues, vmax=vmax)
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
if 'xlims' in kwargs:
ax.set_xlim(kwargs['xlims'])
if 'ylims' in kwargs:
ax.set_ylim(kwargs['ylims'])
# Make a trunk list
tronly = [s for s in d.trunk if s not in d.leaves]
f = open(label + '_trunks.txt', 'w')
for c in tronly:
f.write('{:<4d} | '.format(c.idx))
# Plot the actual structure boundaries
mask = d[c.idx].get_mask()
mask_coll = np.amax(mask, axis=0)
plt.contour(mask_coll, colors='red', linewidths=1, levels=[0])
# Plot the ellipse fits
s = analysis.PPVStatistic(d[c.idx])
ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none')
ax.add_patch(ellipse)
# Make sub-lists of descendants
print('Finding descendants of trunk {}'.format(c.idx))
desclist = []
if len(d[c.idx].descendants) > 0:
for s in d[c.idx].descendants:
desclist.append(s.idx)
desclist.sort()
liststr = ','.join(map(str, desclist))
f.write(liststr)
f.write("\n")
f.close()
fig.colorbar(im, ax=ax)
plt.savefig('plots/' + label + '_trunks_map.pdf', bbox_inches='tight')
plt.close()
# Make a branch list
branch = [
s for s in d.all_structures if s not in d.leaves and s not in d.trunk
]
slist = []
for c in branch:
slist.append(c.idx)
slist.sort()
with open(label + '_branches.txt', 'w') as output:
writer = csv.writer(output)
for val in slist:
writer.writerow([val])
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the leaves
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Image the leaves")
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
vmax = np.nanmax(hdu2.data) / 2.
im = ax.matshow(hdu2.data, origin='lower', cmap=plt.cm.Blues, vmax=vmax)
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
if 'xlims' in kwargs:
ax.set_xlim(kwargs['xlims'])
if 'ylims' in kwargs:
ax.set_ylim(kwargs['ylims'])
# Make a leaf list
slist = []
for c in d.leaves:
slist.append(c.idx)
# Plot the actual structure boundaries
mask = d[c.idx].get_mask()
mask_coll = np.amax(mask, axis=0)
plt.contour(mask_coll, colors='green', linewidths=1, levels=[0])
# Plot the ellipse fits
s = analysis.PPVStatistic(d[c.idx])
ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none')
ax.add_patch(ellipse)
slist.sort()
with open(label + '_leaves.txt', "w") as output:
writer = csv.writer(output)
for val in slist:
writer.writerow([val])
fig.colorbar(im, ax=ax)
plt.savefig('plots/' + label + '_leaves_map.pdf', bbox_inches='tight')
plt.close()
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the clusters
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Image the resulting clusters")
clusts = np.array(dclust.clusters)
colors = np.array(dclust.colors)
inds = np.argsort(clusts)
clusts = clusts[inds]
colors = colors[inds]
print(clusts)
print(colors)
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
vmax = np.nanmax(hdu2.data) / 2.
im = ax.matshow(hdu2.data, origin='lower', cmap=plt.cm.Blues, vmax=vmax)
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
if 'xlims' in kwargs:
ax.set_xlim(kwargs['xlims'])
if 'ylims' in kwargs:
ax.set_ylim(kwargs['ylims'])
# Make a cluster list
f = open(label + '_clusters.txt', 'w')
for i, c in enumerate(clusts):
f.write('{:<4d} {:7} | '.format(c, colors[i]))
# Plot the actual structure boundaries
mask = d[c].get_mask()
mask_coll = np.amax(mask, axis=0)
plt.contour(mask_coll, colors=colors[i], linewidths=1, levels=[0])
# Plot the ellipse fits
s = analysis.PPVStatistic(d[c])
ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none')
ax.add_patch(ellipse)
# Make sub-lists of descendants
print('Finding descendants of cluster {}'.format(c))
desclist = []
if len(d[c].descendants) > 0:
for s in d[c].descendants:
desclist.append(s.idx)
desclist.sort()
liststr = ','.join(map(str, desclist))
f.write(liststr)
f.write("\n")
f.close()
fig.colorbar(im, ax=ax)
plt.savefig('plots/' + label + '_clusters_map.pdf', bbox_inches='tight')
plt.close()
return
def doDendroAndScimes(self,fitsFile,rms=0.5, minPix=500 ,reDo=False,subRegion="" ,saveAll=True,vScale=10,useVolume=True,useLuminosity=True,useVelociy=True ,calDendro=True, inputDendro=None,iinputDenroCatFile=None ):
"""
:param fitsFile:
:param rms: #not important anymore
:param minPix:
:param reDo:
:param subRegion: #only used to save
:param saveAll:
:param vScale: only useful when useVeloicty is true
:param useVolume:
:param useLuminosity:
:param useVelociy:
:return:
"""
if not calDendro and (inputDendro ==None or iinputDenroCatFile==None):
print "If you do not want to calculate dendrogram, you need to provide the inputDendro and the catalog"
return
criteriaUsed=[]
scales=[]
saveMark=''
if useLuminosity:
criteriaUsed.append('luminosity')
scales.append(None)
saveMark=saveMark+'Lu'
if useVolume:
criteriaUsed.append('volume')
#criteriaUsed.append('trueVolume')
saveMark=saveMark+'Vo'
scales.append(None)
if useVelociy:
#criteriaUsed.append('trueVelocity')
criteriaUsed.append('v_rms')
scales.append(vScale)
saveMark=saveMark+'Ve'
saveMark=saveMark+"{}_{}".format(minPix,vScale)
subRegion=subRegion+saveMark
hdu=fits.open(fitsFile)[0]
data= hdu.data
hd=hdu.header
saveDendro=self.regionName+subRegion+"Dendro.fits"
cdelt1 = abs(hd.get('CDELT1'))
cdelt2 = abs(hd.get('CDELT2'))
ppbeam = abs((bmaj*bmin)/(cdelt1*cdelt2)*2*np.pi/(8*np.log(2))) # Pixels per beam
if not calDendro and os.path.isfile( self.saveSCIMESPath+saveDendro ):
print self.regionName," has been done, skipping..."
return
print("Making dendrogram...")
#from pruning import all_true, min_vchan, min_delta, min_area
#is_independent = all_true((min_delta(3*rms), min_area(288), min_vchan(2)))
#is_independent = all_true( ( min_area(288) ) )
#d = Dendrogram.compute(data, min_value=0, is_independent=is_independent, verbose=1, min_npix=10000 )
#d = Dendrogram.compute(data, min_value=0, verbose=1, min_npix=minPix ,min_delta=1.5 , is_independent=min_area(288) )
catName=self.saveSCIMESPath+self.regionName+subRegion +"dendroCat.fit"
treeFile=self.saveSCIMESPath+subRegion+"DendroTree.txt"
if calDendro: #just for test
d = Dendrogram.compute(data, min_value=0, verbose=1, min_npix=minPix, min_delta=1.5 )
d.save_to( saveDendro )
self.metadata["wcs"]= WCS(hd)
cat = ppv_catalog(d, self.metadata)
print len(cat),"<-- total number of structures?"
#print dir(cat)
cat.write(catName,overwrite=True)
self.writeTreeStructure(d,treeFile)
# add levels.
#if saveDendro:
#cat.write(self.saveSCIMESPath+saveDenroMark+".fit" ,overwrite=True)
#return d,self.saveSCIMESPath+saveDenroMark+".fit"
else: #just for test
d =inputDendro # do not read dendro here
cat = Table.read(iinputDenroCatFile)
print len(cat),"<-- total number of structures?"
#print dir(cat)
self.writeTreeStructure(d,treeFile)
# add levels.
#newVo,newVe=self.getTrueVolumeAndVrms(treeFile,iinputDenroCatFile,subRegion=subRegion)
#cat["trueVolume"]=newVo
#cat['trueVelocity']=newVe
###Test, what if we multiply the volumen by the dispersion of velocity dispersion in km/s
#cat.sort("v_rms")
#print cat
#print cat.colnames
res = SpectralCloudstering(d, cat, hd, criteria = criteriaUsed , user_scalpars=scales, blind = True , rms = self.rms, s2nlim = 3, save_all = saveAll, user_iter=1)
res.clusters_asgn.writeto (self.saveSCIMESPath+self.regionName+subRegion+'Cluster_asgn.fits', overwrite=True)
clusterCat=cat[res.clusters]
clusterCat.write( self.saveSCIMESPath +self.regionName+subRegion +"ClusterCat.fit",overwrite=True)
def compute_catalog(d, header):
"""
Computes a catalog on a dendrogram.
Parameters
----------
d : astrodendro.dendrogram.Dendrogram
Dendrogram on which to compute a catalog
header : astropy.io.fits.header.Header
Header corresponding exactly to `d.data`.
Used for unit and scaling information to calculate flux and
world coordinates.
Returns
-------
catalog : astropy.table.table.Table
Catalog describing the structures in dendrogram `d`
using units provided in `header`.
metadata : dict
Explicitly lists unit properties used in `catalog`
"""
# rough average of North and South telescopes:
# see Dame et al. 2001, p. 794.
# "The Northern telescope... has a beamwidth of 8'.4 +/- 0'.2....
# The Southern telescope has nearly the same beamwidth to within
# the uncertainties: 8'.8 +/- 0'.2"
beam_size = 8.5 * u.arcmin
frequency = 115.27 * u.GHz # http://www.cv.nrao.edu/php/splat/
# Remember: by this point, the data ought to be in (v, b, l), with FITS header order opposite that
if 'vel' not in header['ctype3'].lower():
raise ValueError(
"CTYPE3 must be velocity - check that the data were permuted correctly"
)
v_scale = header['cdelt3']
v_unit = u.km / u.s
l_scale = header['cdelt1']
b_scale = header['cdelt2']
metadata = {}
metadata['data_unit'] = u.K
metadata['spatial_scale'] = b_scale * u.deg
metadata['velocity_scale'] = v_scale * v_unit
metadata[
'wavelength'] = frequency # formerly: (c.c / frequency).to('mm') but now compute_flux can handle frequency in spectral equivalency
metadata['beam_major'] = beam_size
metadata['beam_minor'] = beam_size
metadata[
'vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v)
metadata['wcs'] = d.wcs
catalog = astrodendro.ppv_catalog(d, metadata, verbose=True)
if catalog['flux'].unit.is_equivalent('Jy'):
# Workaround because flux is computed wrong (see https://github.com/dendrograms/astrodendro/issues/107)
flux = u.Quantity(catalog['flux'])
area_exact = u.Quantity(
catalog['area_exact']) #.unit*catalog['area_exact'].data
# average brightness temperature integrated over area_exact
flux_kelvin = flux.to('K',
equivalencies=u.brightness_temperature(
area_exact, frequency))
# flux integrated over area and velocity
flux_kelvin_kms_sr = flux_kelvin * metadata[
'velocity_scale'] * area_exact.to(u.steradian)
catalog['flux_true'] = flux_kelvin_kms_sr
background_flux_array = np.zeros_like(catalog['flux_true'])
# do a clipping --
# this uses the approximation that any given structure's background is well-represented by its lowest-valued pixel, which can get messy sometimes.
for i, row in enumerate(catalog):
background_kelvins = d[i].vmin * u.K
background_flux = background_kelvins * d[i].get_npix(
) * metadata['velocity_scale'] * metadata['spatial_scale']**2
background_flux_array[i] = background_flux.to(u.K * u.km / u.s *
u.steradian).value
catalog['flux_clipped'] = catalog['flux_true'] - background_flux_array
return catalog, metadata
def compare_jansky_to_kelvin(data_file, downsample_factor=4, transpose_tuple=(2,0,1),
min_value=0.01, min_delta=0.005, min_npix=1000):
df = downsample_factor
tt = transpose_tuple
print "loading data: ..."
datacube, datacube_header = getdata(data_path+data_file, memmap=True,
header=True)
print "transposing, downsampling, and unit-converting data: ..."
datacube_dt, datacube_dt_header = \
downsample_and_transpose_data_and_header(datacube, datacube_header, df, tt)
datacube_dt_wcs = wcs.wcs.WCS(datacube_dt_header)
beam_size = 1/8 * u.deg
# Convert the data from kelvin to jansky-per-beam
omega_beam = np.pi * (0.5 * beam_size)**2 # Beam width is 1/8 degree
frequency = 115 * u.GHz
K_to_Jy = u.K.to(u.Jy, equivalencies=
u.brightness_temperature(omega_beam, frequency))
datacube_dt_jansky_perbeam = datacube_dt * K_to_Jy
print "computing dendrogram: ..."
d_jansky = astrodendro.Dendrogram.compute(
datacube_dt_jansky_perbeam,
min_value=min_value*K_to_Jy, min_delta=min_delta*K_to_Jy, #these are arbitrary
min_npix=min_npix//df**3, verbose=True)
d_kelvin = astrodendro.Dendrogram.compute(
datacube_dt,
min_value=min_value, min_delta=min_delta, #these are arbitrary
min_npix=min_npix//df**3, verbose=True)
print len(list(d_jansky.all_structures))
print len(list(d_kelvin.all_structures))
v_scale = datacube_dt_header['cdelt3']
v_unit = u.km / u.s
l_scale = datacube_dt_header['cdelt1']
b_scale = datacube_dt_header['cdelt2']
metadata_jansky = {}
metadata_jansky['data_unit'] = u.Jy / u.beam # According to A. Ginsburg
metadata_jansky['spatial_scale'] = b_scale * u.deg
metadata_jansky['velocity_scale'] = v_scale * v_unit
metadata_jansky['wavelength'] = (c.c / frequency).to('mm')
metadata_jansky['beam_major'] = beam_size
metadata_jansky['beam_minor'] = beam_size
metadata_jansky['vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v)
metadata_jansky['wcs'] = datacube_dt_wcs
metadata_kelvin = metadata_jansky.copy()
metadata_kelvin['data_unit'] = u.K # overwrite Jy/beam
catalog_jansky = astrodendro.ppv_catalog(d_jansky, metadata_jansky)
catalog_kelvin = astrodendro.ppv_catalog(d_kelvin, metadata_kelvin)
beams_per_pixel = metadata_jansky['spatial_scale'] ** 2 / (metadata_jansky['beam_minor'] * metadata_jansky['beam_major'] * 1.1331) * u.beam
print beams_per_pixel
# if catalog['flux'].unit.is_equivalent('Jy'):
# # Workaround because flux is computed wrong
# flux = quantify_column(catalog['flux'])
# area_exact = catalog['area_exact'].unit*catalog['area_exact'].data
# flux_kelvin = flux.to('K', equivalencies=u.brightness_temperature(area_exact, frequency))
# flux_kelvin_kms_deg2 = flux_kelvin * metadata['velocity_scale'] * area_exact
# catalog.add_column(astropy.table.Column(data=flux_kelvin_kms_deg2, name='flux_kelvin_kms_deg2'))
return_dict = {}
return_dict['jansky'] = (d_jansky, catalog_jansky)
return_dict['kelvin'] = (d_kelvin, catalog_kelvin)
return return_dict
def compare_jansky_to_kelvin(data_file,
downsample_factor=4,
transpose_tuple=(2, 0, 1),
min_value=0.01,
min_delta=0.005,
min_npix=1000):
df = downsample_factor
tt = transpose_tuple
print "loading data: ..."
datacube, datacube_header = getdata(data_path + data_file,
memmap=True,
header=True)
print "transposing, downsampling, and unit-converting data: ..."
datacube_dt, datacube_dt_header = \
downsample_and_transpose_data_and_header(datacube, datacube_header, df, tt)
datacube_dt_wcs = wcs.wcs.WCS(datacube_dt_header)
beam_size = 1 / 8 * u.deg
# Convert the data from kelvin to jansky-per-beam
omega_beam = np.pi * (0.5 * beam_size)**2 # Beam width is 1/8 degree
frequency = 115 * u.GHz
K_to_Jy = u.K.to(u.Jy,
equivalencies=u.brightness_temperature(
omega_beam, frequency))
datacube_dt_jansky_perbeam = datacube_dt * K_to_Jy
print "computing dendrogram: ..."
d_jansky = astrodendro.Dendrogram.compute(
datacube_dt_jansky_perbeam,
min_value=min_value * K_to_Jy,
min_delta=min_delta * K_to_Jy, #these are arbitrary
min_npix=min_npix // df**3,
verbose=True)
d_kelvin = astrodendro.Dendrogram.compute(
datacube_dt,
min_value=min_value,
min_delta=min_delta, #these are arbitrary
min_npix=min_npix // df**3,
verbose=True)
print len(list(d_jansky.all_structures))
print len(list(d_kelvin.all_structures))
v_scale = datacube_dt_header['cdelt3']
v_unit = u.km / u.s
l_scale = datacube_dt_header['cdelt1']
b_scale = datacube_dt_header['cdelt2']
metadata_jansky = {}
metadata_jansky['data_unit'] = u.Jy / u.beam # According to A. Ginsburg
metadata_jansky['spatial_scale'] = b_scale * u.deg
metadata_jansky['velocity_scale'] = v_scale * v_unit
metadata_jansky['wavelength'] = (c.c / frequency).to('mm')
metadata_jansky['beam_major'] = beam_size
metadata_jansky['beam_minor'] = beam_size
metadata_jansky[
'vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v)
metadata_jansky['wcs'] = datacube_dt_wcs
metadata_kelvin = metadata_jansky.copy()
metadata_kelvin['data_unit'] = u.K # overwrite Jy/beam
catalog_jansky = astrodendro.ppv_catalog(d_jansky, metadata_jansky)
catalog_kelvin = astrodendro.ppv_catalog(d_kelvin, metadata_kelvin)
beams_per_pixel = metadata_jansky['spatial_scale']**2 / (
metadata_jansky['beam_minor'] * metadata_jansky['beam_major'] *
1.1331) * u.beam
print beams_per_pixel
# if catalog['flux'].unit.is_equivalent('Jy'):
# # Workaround because flux is computed wrong
# flux = quantify_column(catalog['flux'])
# area_exact = catalog['area_exact'].unit*catalog['area_exact'].data
# flux_kelvin = flux.to('K', equivalencies=u.brightness_temperature(area_exact, frequency))
# flux_kelvin_kms_deg2 = flux_kelvin * metadata['velocity_scale'] * area_exact
# catalog.add_column(astropy.table.Column(data=flux_kelvin_kms_deg2, name='flux_kelvin_kms_deg2'))
return_dict = {}
return_dict['jansky'] = (d_jansky, catalog_jansky)
return_dict['kelvin'] = (d_kelvin, catalog_kelvin)
return return_dict
def measure_dendrogram_properties(dend=None, cube303=cube303,
cube321=cube321, cube13co=cube13co,
cube18co=cube18co, noise_cube=noise_cube,
sncube=sncube,
suffix="",
last_index=None,
plot_some=True,
line='303',
write=True):
assert (cube321.shape == cube303.shape == noise_cube.shape ==
cube13co.shape == cube18co.shape == sncube.shape)
assert sncube.wcs is cube303.wcs is sncube.mask._wcs
metadata = {}
metadata['data_unit'] = u.K
metadata['spatial_scale'] = 7.2 * u.arcsec
metadata['beam_major'] = 30 * u.arcsec
metadata['beam_minor'] = 30 * u.arcsec
metadata['wavelength'] = 218.22219*u.GHz
metadata['velocity_scale'] = u.km/u.s
metadata['wcs'] = cube303.wcs
keys = [
'density_chi2',
'expected_density',
'dmin1sig_chi2',
'dmax1sig_chi2',
'column_chi2',
'expected_column',
'cmin1sig_chi2',
'cmax1sig_chi2',
'temperature_chi2',
'expected_temperature',
'tmin1sig_chi2',
'tmax1sig_chi2',
'eratio321303',
'ratio321303',
'logh2column',
'elogh2column',
'logabundance',
'elogabundance',
]
obs_keys = [
'Stot303',
'Smin303',
'Smax303',
'Stot321',
'Smean303',
'Smean321',
'npix',
'e303',
'e321',
'r321303',
'er321303',
'13cosum',
'c18osum',
'13comean',
'c18omean',
's_ntotal',
'index',
'is_leaf',
'parent',
'root',
'lon',
'lat',
'vcen',
'higaldusttem',
'reff',
'dustmass',
'dustmindens',
'bad',
#'tkin_turb',
]
columns = {k:[] for k in (keys+obs_keys)}
log.debug("Initializing dendrogram temperature fitting loop")
# FORCE wcs to match
# (technically should reproject here)
cube13co._wcs = cube18co._wcs = cube303.wcs
cube13co.mask._wcs = cube18co.mask._wcs = cube303.wcs
if line == '303':
maincube = cube303
elif line == '321':
maincube = cube321
else:
raise ValueError("Unrecognized line: {0}".format(line))
# Prepare an array to hold the fitted temperatures
tcubedata = np.empty(maincube.shape, dtype='float32')
tcubedata[:] = np.nan
tcubeleafdata = np.empty(maincube.shape, dtype='float32')
tcubeleafdata[:] = np.nan
nbad = 0
catalog = ppv_catalog(dend, metadata)
pb = ProgressBar(len(catalog))
for ii,row in enumerate(catalog):
structure = dend[row['_idx']]
assert structure.idx == row['_idx'] == ii
dend_obj_mask = BooleanArrayMask(structure.get_mask(), wcs=cube303.wcs)
dend_inds = structure.indices()
view = (slice(dend_inds[0].min(), dend_inds[0].max()+1),
slice(dend_inds[1].min(), dend_inds[1].max()+1),
slice(dend_inds[2].min(), dend_inds[2].max()+1),)
#view2 = cube303.subcube_slices_from_mask(dend_obj_mask)
submask = dend_obj_mask[view]
#assert np.count_nonzero(submask.include()) == np.count_nonzero(dend_obj_mask.include())
sn = sncube[view].with_mask(submask)
sntot = sn.sum().value
#np.testing.assert_almost_equal(sntot, structure.values().sum(), decimal=0)
c303 = cube303[view].with_mask(submask)
c321 = cube321[view].with_mask(submask)
co13sum = cube13co[view].with_mask(submask).sum().value
co18sum = cube18co[view].with_mask(submask).sum().value
if hasattr(co13sum,'__len__'):
raise TypeError(".sum() applied to an array has yielded a non scalar.")
npix = submask.include().sum()
assert npix == structure.get_npix()
Stot303 = c303.sum().value
if np.isnan(Stot303):
raise ValueError("NaN in cube. This can't happen: the data from "
"which the dendrogram was derived can't have "
"NaN pixels.")
Smax303 = c303.max().value
Smin303 = c303.min().value
Stot321 = c321.sum().value
if npix == 0:
raise ValueError("npix=0. This is impossible.")
Smean303 = Stot303/npix
if Stot303 <= 0 and line=='303':
raise ValueError("The 303 flux is <=0. This isn't possible because "
"the dendrogram was derived from the 303 data with a "
"non-zero threshold.")
elif Stot303 <= 0 and line=='321':
Stot303 = 0
Smean303 = 0
elif Stot321 <= 0 and line=='321':
raise ValueError("The 321 flux is <=0. This isn't possible because "
"the dendrogram was derived from the 321 data with a "
"non-zero threshold.")
if np.isnan(Stot321):
raise ValueError("NaN in 321 line")
Smean321 = Stot321/npix
#error = (noise_cube[view][submask.include()]).sum() / submask.include().sum()**0.5
var = ((noise_cube[dend_obj_mask.include()]**2).sum() / npix**2)
error = var**0.5
if np.isnan(error):
raise ValueError("error is nan: this is impossible by definition.")
if line == '321' and Stot303 == 0:
r321303 = np.nan
er321303 = np.nan
elif Stot321 < 0:
r321303 = error / Smean303
er321303 = (r321303**2 * (var/Smean303**2 + 1))**0.5
else:
r321303 = Stot321 / Stot303
er321303 = (r321303**2 * (var/Smean303**2 + var/Smean321**2))**0.5
for c in columns:
assert len(columns[c]) == ii
columns['index'].append(row['_idx'])
columns['s_ntotal'].append(sntot)
columns['Stot303'].append(Stot303)
columns['Smax303'].append(Smax303)
columns['Smin303'].append(Smin303)
columns['Stot321'].append(Stot321)
columns['Smean303'].append(Smean303)
columns['Smean321'].append(Smean321)
columns['npix'].append(npix)
columns['e303'].append(error)
columns['e321'].append(error)
columns['r321303'].append(r321303)
columns['er321303'].append(er321303)
columns['13cosum'].append(co13sum)
columns['c18osum'].append(co18sum)
columns['13comean'].append(co13sum/npix)
columns['c18omean'].append(co18sum/npix)
columns['is_leaf'].append(structure.is_leaf)
columns['parent'].append(structure.parent.idx if structure.parent else -1)
columns['root'].append(get_root(structure).idx)
s_main = maincube._data[dend_inds]
x,y,z = maincube.world[dend_inds]
lon = ((z.value-(360*(z.value>180)))*s_main).sum()/s_main.sum()
lat = (y*s_main).sum()/s_main.sum()
vel = (x*s_main).sum()/s_main.sum()
columns['lon'].append(lon)
columns['lat'].append(lat.value)
columns['vcen'].append(vel.value)
mask2d = dend_obj_mask.include().max(axis=0)[view[1:]]
logh2column = np.log10(np.nanmean(column_regridded.data[view[1:]][mask2d]) * 1e22)
if np.isnan(logh2column):
log.info("Source #{0} has NaNs".format(ii))
logh2column = 24
elogh2column = elogabundance
columns['higaldusttem'].append(np.nanmean(dusttem_regridded.data[view[1:]][mask2d]))
r_arcsec = row['radius']*u.arcsec
reff = (r_arcsec*(8.5*u.kpc)).to(u.pc, u.dimensionless_angles())
mass = ((10**logh2column*u.cm**-2)*np.pi*reff**2*2.8*constants.m_p).to(u.M_sun)
density = (mass/(4/3.*np.pi*reff**3)/constants.m_p/2.8).to(u.cm**-3)
columns['reff'].append(reff.value)
columns['dustmass'].append(mass.value)
columns['dustmindens'].append(density.value)
mindens = np.log10(density.value)
if mindens < 3:
mindens = 3
if (r321303 < 0 or np.isnan(r321303)) and line != '321':
raise ValueError("Ratio <0: This can't happen any more because "
"if either num/denom is <0, an exception is "
"raised earlier")
#for k in columns:
# if k not in obs_keys:
# columns[k].append(np.nan)
elif (r321303 < 0 or np.isnan(r321303)) and line == '321':
for k in keys:
columns[k].append(np.nan)
else:
# Replace negatives for fitting
if Smean321 <= 0:
Smean321 = error
mf.set_constraints(ratio321303=r321303, eratio321303=er321303,
#ratio321322=ratio2, eratio321322=eratio2,
logh2column=logh2column, elogh2column=elogh2column,
logabundance=logabundance, elogabundance=elogabundance,
taline303=Smean303, etaline303=error,
taline321=Smean321, etaline321=error,
mindens=mindens,
linewidth=10)
row_data = mf.get_parconstraints()
row_data['ratio321303'] = r321303
row_data['eratio321303'] = er321303
for k in row_data:
columns[k].append(row_data[k])
# Exclude bad velocities from cubes
if row['v_cen'] < -80e3 or row['v_cen'] > 180e3:
# Skip: there is no real structure down here
nbad += 1
is_bad = True
else:
is_bad = False
tcubedata[dend_obj_mask.include()] = row_data['expected_temperature']
if structure.is_leaf:
tcubeleafdata[dend_obj_mask.include()] = row_data['expected_temperature']
columns['bad'].append(is_bad)
width = row['v_rms']*u.km/u.s
lengthscale = reff
#REMOVED in favor of despotic version done in dendrograms.py
# we use the analytic version here; the despotic version is
# computed elsewhere (with appropriate gcor factors)
#columns['tkin_turb'].append(heating.tkin_all(10**row_data['density_chi2']*u.cm**-3,
# width,
# lengthscale,
# width/lengthscale,
# columns['higaldusttem'][-1]*u.K,
# crir=0./u.s))
if len(set(len(c) for k,c in columns.items())) != 1:
print("Columns are different lengths. This is not allowed.")
import ipdb; ipdb.set_trace()
for c in columns:
assert len(columns[c]) == ii+1
if plot_some and not is_bad and (ii-nbad % 100 == 0 or ii-nbad < 50):
try:
log.info("T: [{tmin1sig_chi2:7.2f},{expected_temperature:7.2f},{tmax1sig_chi2:7.2f}]"
" R={ratio321303:8.4f}+/-{eratio321303:8.4f}"
" Smean303={Smean303:8.4f} +/- {e303:8.4f}"
" Stot303={Stot303:8.2e} npix={npix:6d}"
.format(Smean303=Smean303, Stot303=Stot303,
npix=npix, e303=error, **row_data))
pl.figure(1)
pl.clf()
mf.denstemplot()
pl.savefig(fpath("dendrotem/diagnostics/{0}_{1}.png".format(suffix,ii)))
pl.figure(2).clf()
mf.parplot1d_all(levels=[0.68268949213708585])
pl.savefig(fpath("dendrotem/diagnostics/1dplot{0}_{1}.png".format(suffix,ii)))
pl.draw()
pl.show()
except Exception as ex:
print ex
pass
else:
pb.update(ii+1)
if last_index is not None and ii >= last_index:
break
if last_index is not None:
catalog = catalog[:last_index+1]
for k in columns:
if k not in catalog.keys():
catalog.add_column(table.Column(name=k, data=columns[k]))
for mid,lo,hi,letter in (('expected_temperature','tmin1sig_chi2','tmax1sig_chi2','t'),
('expected_density','dmin1sig_chi2','dmax1sig_chi2','d'),
('expected_column','cmin1sig_chi2','cmax1sig_chi2','c')):
catalog.add_column(table.Column(name='elo_'+letter,
data=catalog[mid]-catalog[lo]))
catalog.add_column(table.Column(name='ehi_'+letter,
data=catalog[hi]-catalog[mid]))
if write:
catalog.write(tpath('PPV_H2CO_Temperature{0}.ipac'.format(suffix)), format='ascii.ipac')
# Note that there are overlaps in the catalog, which means that ORDER MATTERS
# in the above loop. I haven't yet checked whether large scale overwrites
# small or vice-versa; it may be that both views of the data are interesting.
tcube = SpectralCube(data=tcubedata, wcs=cube303.wcs,
mask=cube303.mask, meta={'unit':'K'},
header=cube303.header,
)
tcubeleaf = SpectralCube(data=tcubeleafdata, wcs=cube303.wcs,
mask=cube303.mask, meta={'unit':'K'},
header=cube303.header,
)
if write:
log.info("Writing TemperatureCube")
outpath = 'TemperatureCube_DendrogramObjects{0}.fits'
tcube.write(hpath(outpath.format(suffix)),
overwrite=True)
outpath_leaf = 'TemperatureCube_DendrogramObjects{0}_leaves.fits'
tcubeleaf.write(hpath(outpath_leaf.format(suffix)),
overwrite=True)
return catalog, tcube
def doDendro(self,
COFITS,
minV=3,
minPix=8,
RMS=0.5,
minDelta=3,
saveName=None,
doSCIMES=False):
"""
COFITS may not be masedk
:param COFITS:
:param minV:
:param minPix:
:param RMS:
:param minDelta:
:return:
"""
if saveName == None:
saveName = ""
saveMark = saveName + "minV{}minP{}minD{}".format(
minV, minPix, minDelta)
treeFile = saveMark + "_Tree.txt"
catFile = saveMark + "_dendroCat.fit"
dendroFile = saveMark + "_dendro.fits"
trunkCatFile = saveMark + "_dendroCatTrunk.fit"
trunkFITS = saveMark + "_TrunkAsign.fits"
hdu = fits.open(COFITS)[0]
dataCO = hdu.data
headCO = hdu.header
#mask the data by minValue
print "Cmputing dendrogram with min value {}, min pixel number {}".format(
minV * RMS, minPix)
#data[data<minV*RMS]=0
d = Dendrogram.compute(dataCO,
min_value=minV * RMS,
verbose=1,
min_npix=minPix,
min_delta=minDelta * RMS)
self.produceAssignFITS(d, COFITS, trunkFITS)
d.save_to(dendroFile)
#calculate the catalog
cat = ppv_catalog(d, self.metadata)
cat.write(catFile, overwrite=True)
self.writeTreeStructure(d, treeFile)
doTree = dendroTree(treeFile)
trunks = doTree.getAllTrunk()
trunkCat = cat[trunks]
print "{} trunks found!".format(len(trunkCat))
trunkCat.write(trunkCatFile, overwrite=True)
if doSCIMES:
self.doSCIMES(COFITS,
dendroFile,
catFile,
saveMark + "Ve20",
inputD=d,
criteriaUsed=[self.myVrms],
scales=[20])
self.doSCIMES(COFITS,
dendroFile,
catFile,
saveMark + "Ve10",
inputD=d,
criteriaUsed=[self.myVrms],
scales=[10])
self.doSCIMES(COFITS,
dendroFile,
catFile,
saveMark + "Ve15",
inputD=d,
criteriaUsed=[self.myVrms],
scales=[15])
hd = hdu.header
# Survey designs
sigma = 0.3 #K, noise level
ppb = 1.3 #pixels/beam
d = Dendrogram.compute(data, min_value=sigma, \
min_delta=2*sigma, min_npix=3*ppb, verbose = 1)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Generate the catalog
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Generate a catalog of dendrogram structures")
metadata = {}
metadata['data_unit'] = u.Jy #This should be Kelvin (not yet implemented)!
cat = ppv_catalog(d, metadata)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Running SCIMES
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Running SCIMES")
dclust = scimes.SpectralCloudstering(d, cat, hd, rms=sigma)
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Image the result
#%&%&%&%&%&%&%&%&%&%&%&%&%&%
print("Visualize the clustered dendrogram")
dclust.showdendro()
print("Visualize collapsed maps of the assignment cubes")
cubes = [dclust.clusters_asgn,\
def downsampled_demo(data_file, downsample_factor=4, transpose_tuple=(2,0,1),
min_value=0.01, min_delta=0.005, min_npix=2000,
neighbours=None, resample=False, compute_catalog=True, recenter=True,
data_path=data_path, memmap=True):
df = downsample_factor
tt = transpose_tuple
print("\n ** Downsampled Demo Parameters:"
"\n downsample_factor={0}"
"\n transpose_tuple={1}"
"\n min_value={2}"
"\n min_delta={3}"
"\n min_npix={4}\n".format(downsample_factor, transpose_tuple, min_value, min_delta, min_npix))
beginning = datetime.datetime.now()
beginning_string = datetime.datetime.strftime(beginning,"%Y-%m-%d %H:%M:%S")
print("\n ** Beginning at: {0}".format(beginning_string))
print "\n** loading data from {0}: ...\n".format(data_path+data_file)
datacube, datacube_header = getdata(data_path+data_file, memmap=memmap,
header=True)
print "\n** transposing {0} and downsampling ({1}) data: ...\n".format(tt, df)
datacube_dt, datacube_dt_header = \
downsample_and_transpose_data_and_header(datacube, datacube_header, df, tt, resample=resample, recenter=recenter)
datacube_dt_wcs = wcs.wcs.WCS(datacube_dt_header)
datacube_dt_wcs.wcs.bounds_check(pix2world=False)
beam_size = 1/8 * u.deg
frequency = 115 * u.GHz
print "\n** computing dendrogram on data with dimensions {} and n_pixels {:,}: ...\n".format(np.shape(datacube_dt), np.size(datacube_dt))
d = astrodendro.Dendrogram.compute(
datacube_dt,
min_value=min_value, min_delta=min_delta, #these are arbitrary
min_npix=min_npix, verbose=True, neighbours=neighbours, wcs=datacube_dt_wcs)
v_scale = datacube_dt_header['cdelt3']
v_unit = u.km / u.s
l_scale = datacube_dt_header['cdelt1']
b_scale = datacube_dt_header['cdelt2']
metadata = {}
metadata['data_unit'] = u.K
metadata['spatial_scale'] = b_scale * u.deg
metadata['velocity_scale'] = v_scale * v_unit
metadata['wavelength'] = frequency # formerly: (c.c / frequency).to('mm') but now compute_flux can handle frequency in spectral equivalency
metadata['beam_major'] = beam_size
metadata['beam_minor'] = beam_size
metadata['vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v)
metadata['wcs'] = datacube_dt_wcs
print "\n** computing catalog for {:,} structures: ...\n".format(len(d))
if compute_catalog:
catalog = astrodendro.ppv_catalog(d, metadata, verbose=True)
else:
catalog = None
return d, catalog, datacube_dt_header, metadata
if catalog['flux'].unit.is_equivalent('Jy'):
# Workaround because flux is computed wrong
flux = quantify_column(catalog['flux'])
area_exact = catalog['area_exact'].unit*catalog['area_exact'].data
# average brightness temperature integrated over area_exact
flux_kelvin = flux.to('K', equivalencies=u.brightness_temperature(area_exact, frequency))
# flux integrated over area and velocity
flux_kelvin_kms_deg2 = flux_kelvin * metadata['velocity_scale'] * area_exact
catalog.add_column(astropy.table.Column(data=flux_kelvin_kms_deg2, name='flux_kelvin_kms_deg2'))
end = datetime.datetime.now()
end_string = datetime.datetime.strftime(end,"%Y-%m-%d %H:%M:%S")
print("\n ** Ending at: {0}".format(end_string))
time_elapsed = (end - beginning)
print "\n ** Time elapsed: {0}".format(time_elapsed)
return d, catalog, datacube_dt_header, metadata
