def doScimes(self,dendroFITS,hd ): self.metadata["wcs"]= WCS(hd) d = Dendrogram.load_from( dendroFITS ) cat = ppv_catalog(d, self.metadata) res = SpectralCloudstering(d, cat, hd, criteria = ['volume' ], blind = True, rms = self.rms, s2nlim = 3, save_all = True, user_iter=1) res.clusters_asgn.writeto (self.regionName+'Cluster_asgn.fits', overwrite=True)
def deriv_decimate_leaves(d, cube, meta, fscale=2, sigdiscont=0.5, nredun=2, **kwargs): leaves = get_leaves(d) goodleaf = np.ones(len(leaves), dtype=np.bool) for i, leaf in enumerate(leaves): if not goodleaf[i] or (leaf.parent is None): continue parentobj = ppv_catalog([leaf.parent], meta, verbose=False) children = ppv_catalog(leaf.parent.children, meta, verbose=False) thisleaf = np.array(children['_idx']) == leaf.idx dmajor = np.array( (parentobj['major_sigma'] - children['major_sigma']) / children['major_sigma']) > sigdiscont dminor = np.array( (parentobj['minor_sigma'] - children['minor_sigma']) / children['minor_sigma']) > sigdiscont dsigv = np.array((parentobj['v_rms'] - children['v_rms']) / children['v_rms']) > sigdiscont dflux = np.array((parentobj['v_rms'] - children['v_rms']) / children['v_rms']) > (sigdiscont * fscale) disc = np.any( np.sum(np.c_[dmajor, dminor, dsigv, dflux], axis=1) >= nredun) goodleaf[i] = disc leaflist = [] for i, leaf in enumerate(leaves): if goodleaf[i]: leaflist += [leaf] peaks = [leaf.get_peak()[0] for leaf in leaflist] indices = tuples_to_arrays(peaks) return (indices)
def get_catalog(cogal): from astrodendro import Dendrogram from astrodendro import ppv_catalog co = cogal[0] gal = cogal[1] # get data and info file = data[co][gal]['noise matched']['file'] cube = fits.open(file)[0] noise = data[co][gal]['noise matched']['rms'].to(u.K) bmin = cube.header['bmin'] * u.degree bmaj = cube.header['bmaj'] * u.degree beam_area = 1.13309 * bmin * bmaj pix1 = u.Quantity( str(np.abs(cube.header['cdelt1'])) + cube.header['cunit1']) pix2 = u.Quantity( str(np.abs(cube.header['cdelt2'])) + cube.header['cunit2']) pix3 = u.Quantity( str(np.abs(cube.header['cdelt3'])) + cube.header['cunit3']) pix_beam = (beam_area / pix1 / pix2) d = Dendrogram.load_from(join(compdir, gal + '.' + co + '.dendrogram.fits')) # time execution print("\nComputing catalog: " + co + " " + gal + "\n") start = time.time() # calculate catalog metadata = { 'data_unit': u.K, 'spatial_scale': parsec_to_angle(pix1, source=gal), 'velocity_scale': pix3, 'beam_major': bmaj, 'beam_minor': bmin, 'vaxis': 0, 'wavelength': lines[co]['restfreq'], 'wcs': WCS(cube.header) } catalog = ppv_catalog(d, metadata) fnpickle(catalog, join(compdir, gal + '.' + co + '.catalog.pickle')) # time execution stop = time.time() exec_time = np.round(stop - start, 1) print("\nFinished catalog: " + co + " " + gal + "\nExecution took " + str(datetime.timedelta(seconds=exec_time)) + "hours for " + str(len(catalog)) + " structures.\n")
def doCluster(self, dendroCase, saveName, head): savePath = self.dataPath + saveName + "/" trunkN = len(dendroCase.trunk) print "trunkN: {} ".format(trunkN) os.system("mkdir " + savePath) saveCloudFITS = savePath + saveName + "CloudAsgn.fits" saveTrunkFITS = savePath + saveName + "TrunkAsgn.fits" saveCloudCat = savePath + saveName + "CloudCat.fit" saveDenroCat = savePath + saveName + "DendroCat.fit" saveDenroFITS = savePath + saveName + "DendroFITS.fits" dendroCase.save_to(saveDenroFITS) #save dendro self.metadata['wcs'] = WCS(head) cat = ppv_catalog(dendroCase, self.metadata) dclust = SpectralCloudstering(dendroCase, cat, head) cloudHDU = dclust.clusters_asgn cloudHDU.writeto(saveCloudFITS, overwrite=True) trunkHDU = dclust.trunks_asgn trunkHDU.writeto(saveTrunkFITS, overwrite=True) cCat = cat[dclust.clusters] cCat.write(saveCloudCat, overwrite=True) cat.write(saveDenroCat, overwrite=True) cloudN = len(set(cloudHDU.data.reshape(-1))) - 1 #save dendro cat print "trunkN: {}, cloudN: {}".format(trunkN, cloudN) return trunkN, cloudN
def WriteCatalog(self): """ """ print "Writing catalg" if self.dendroData == None: self.readDendro() #define metadata metadata = {} metadata['data_unit'] = u.Kelvin #u.Jy / u.beam metadata['spatial_scale'] = 30 * u.arcsec metadata['beam_major'] = 50 * u.arcsec # FWHM metadata['beam_minor'] = 50 * u.arcsec # FWHM # the velocity_scale should be mentioned, other wise the v_rms would be in pixels if self.regionName == "Mopra": metadata['beam_major'] = 33 * u.arcsec # FWHM metadata['beam_minor'] = 33 * u.arcsec # FWHM metadata['wcs'] = WCS(self.CO12Head) # 22.9 * u.arcsec # FWHM c = 299792458. f = 115271202000.0 wavelength = c / f * u.meter metadata['wavelength'] = wavelength # 22.9 * u.arcsec # FWHM cat = ppv_catalog(self.dendroData, metadata) #write catalog try: os.remove(self.dendroCat) except: pass cat.write(self.dendroCat, format='fits')
def dendrogram_downaddmom_cube(*args, **kwargs): datacube_dt, datacube_dt_header = downsample_addnoise_and_momentmask(*args) datacube_dt_wcs = wcs.wcs.WCS(datacube_dt_header) beam_size = 1/8 * u.deg frequency = 115 * u.GHz d = astrodendro.Dendrogram.compute(datacube_dt, wcs=datacube_dt_wcs, **kwargs) v_scale = datacube_dt_header['cdelt3'] v_unit = u.km / u.s l_scale = datacube_dt_header['cdelt1'] b_scale = datacube_dt_header['cdelt2'] metadata = {} metadata['data_unit'] = u.K metadata['spatial_scale'] = b_scale * u.deg metadata['velocity_scale'] = v_scale * v_unit metadata['wavelength'] = frequency metadata['beam_major'] = beam_size metadata['beam_minor'] = beam_size metadata['vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v) metadata['wcs'] = datacube_dt_wcs catalog = astrodendro.ppv_catalog(d, metadata, verbose=True) flux = u.Quantity(catalog['flux']) area_exact = catalog['area_exact'].unit*catalog['area_exact'].data # average brightness temperature integrated over area_exact flux_kelvin = flux.to('K', equivalencies=u.brightness_temperature(area_exact, frequency)) # flux integrated over area and velocity flux_kelvin_kms_deg2 = flux_kelvin * metadata['velocity_scale'] * area_exact catalog.add_column(astropy.table.Column(data=flux_kelvin_kms_deg2, name='flux_kelvin_kms_deg2')) return d, catalog, datacube_dt_header, metadata
ax = fig.add_subplot(111) ax.set_yscale('log') ax.set_xlabel('Structure') ax.set_ylabel('Flux') p = d.plotter() p.plot_tree(ax, color='black') #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Generate the catalog #%&%&%&%&%&%&%&%&%&%&%&%&%&% print "Generate a catalog of dendrogram structures" metadata = {} metadata['data_unit'] = u.Jy #This should be Kelvin (not yet implemented)! cat = ppv_catalog(d, metadata) #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Running SCIMES #%&%&%&%&%&%&%&%&%&%&%&%&%&% print "Running SCIMES" dclust = SpectralCloudstering(d, cat, criteria = ['volume']) print "Visualize the clustered dendrogram" dclust.showdendro() print "Produce the assignment cube" dclust.asgncube(hd)
def compute_catalog(d, header): """ Computes a catalog on a dendrogram. Parameters ---------- d : astrodendro.dendrogram.Dendrogram Dendrogram on which to compute a catalog header : astropy.io.fits.header.Header Header corresponding exactly to `d.data`. Used for unit and scaling information to calculate flux and world coordinates. Returns ------- catalog : astropy.table.table.Table Catalog describing the structures in dendrogram `d` using units provided in `header`. metadata : dict Explicitly lists unit properties used in `catalog` """ # rough average of North and South telescopes: # see Dame et al. 2001, p. 794. # "The Northern telescope... has a beamwidth of 8'.4 +/- 0'.2.... # The Southern telescope has nearly the same beamwidth to within # the uncertainties: 8'.8 +/- 0'.2" beam_size = 8.5 * u.arcmin frequency = 115.27 * u.GHz # http://www.cv.nrao.edu/php/splat/ # Remember: by this point, the data ought to be in (v, b, l), with FITS header order opposite that if 'vel' not in header['ctype3'].lower(): raise ValueError("CTYPE3 must be velocity - check that the data were permuted correctly") v_scale = header['cdelt3'] v_unit = u.km / u.s l_scale = header['cdelt1'] b_scale = header['cdelt2'] metadata = {} metadata['data_unit'] = u.K metadata['spatial_scale'] = b_scale * u.deg metadata['velocity_scale'] = v_scale * v_unit metadata['wavelength'] = frequency # formerly: (c.c / frequency).to('mm') but now compute_flux can handle frequency in spectral equivalency metadata['beam_major'] = beam_size metadata['beam_minor'] = beam_size metadata['vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v) metadata['wcs'] = d.wcs catalog = astrodendro.ppv_catalog(d, metadata, verbose=True) if catalog['flux'].unit.is_equivalent('Jy'): # Workaround because flux is computed wrong (see https://github.com/dendrograms/astrodendro/issues/107) flux = u.Quantity(catalog['flux']) area_exact = u.Quantity(catalog['area_exact']) #.unit*catalog['area_exact'].data # average brightness temperature integrated over area_exact flux_kelvin = flux.to('K', equivalencies=u.brightness_temperature(area_exact, frequency)) # flux integrated over area and velocity flux_kelvin_kms_sr = flux_kelvin * metadata['velocity_scale'] * area_exact.to(u.steradian) catalog['flux_true'] = flux_kelvin_kms_sr background_flux_array = np.zeros_like(catalog['flux_true']) # do a clipping -- # this uses the approximation that any given structure's background is well-represented by its lowest-valued pixel, which can get messy sometimes. for i, row in enumerate(catalog): background_kelvins = d[i].vmin * u.K background_flux = background_kelvins * d[i].get_npix() * metadata['velocity_scale'] * metadata['spatial_scale']**2 background_flux_array[i] = background_flux.to(u.K * u.km/u.s * u.steradian).value catalog['flux_clipped'] = catalog['flux_true'] - background_flux_array return catalog, metadata
def run_dendro( label='mycloud', cubefile=None, flatfile=None, redo='n', nsigma=3., min_delta=2.5, min_bms=2., position_dependent_noise=False, # will use rms map in dendro criteria=['volume'], # for SCIMES doplots=True, dendro_in=None, # use this instead of re-loading **kwargs): global cubedata, rmsmap, threshold_sigma #%&%&%&%&%&%&%&%&%&%&%&% # Make dendrogram #%&%&%&%&%&%&%&%&%&%&%&% hdu3 = fits.open(cubefile)[0] hd3 = hdu3.header cubedata = hdu3.data # Deal with oddities in 30 Dor cube if hd3['NAXIS'] == 3: for key in [ 'CTYPE4', 'CRVAL4', 'CDELT4', 'CRPIX4', 'CUNIT4', 'NAXIS4' ]: if key in hd3.keys(): hd3.remove(key) # Get cube parameters sigma = stats.mad_std(hdu3.data[~np.isnan(hdu3.data)]) print('Robustly estimated RMS: ', sigma) ppb = 1.133 * hd3['bmaj'] * hd3['bmin'] / (abs( hd3['cdelt1'] * hd3['cdelt2'])) print('Pixels per beam: ', ppb) # Make the dendrogram if not present or redo=y if position_dependent_noise: dendrofile = 'dendro_dendrogram_rmsmap.hdf5' else: dendrofile = 'dendro_dendrogram.hdf5' if dendro_in != None: d = dendro_in elif redo == 'n' and os.path.isfile(dendrofile): print('Loading pre-existing dendrogram') d = Dendrogram.load_from(dendrofile) else: print('Make dendrogram from the full cube') if position_dependent_noise: mask3d = cubedata < 3 * sigma rmsmap = np.nanstd(cubedata * mask3d, axis=0) # assumes spectral 1st threshold_sigma = min_delta # for custom_independent function d = Dendrogram.compute(hdu3.data, min_value=nsigma * sigma, min_delta=min_delta * sigma, min_npix=min_bms * ppb, verbose=1, is_independent=custom_independent) else: d = Dendrogram.compute(hdu3.data, min_value=nsigma * sigma, min_delta=min_delta * sigma, min_npix=min_bms * ppb, verbose=1) d.save_to(dendrofile) if doplots: # checks/creates directory to place plots if os.path.isdir('dendro_plots') == 0: os.makedirs('dendro_plots') # Plot the tree fig = plt.figure(figsize=(14, 8)) ax = fig.add_subplot(111) #ax.set_yscale('log') ax.set_xlabel('Structure') ax.set_ylabel('Intensity [' + hd3['BUNIT'] + ']') p = d.plotter() branch = [ s for s in d.all_structures if s not in d.leaves and s not in d.trunk ] tronly = [s for s in d.trunk if s not in d.leaves] for st in tronly: p.plot_tree(ax, structure=[st], color='brown', subtree=False) for st in branch: p.plot_tree(ax, structure=[st], color='black', subtree=False) for st in d.leaves: p.plot_tree(ax, structure=[st], color='green') #p.plot_tree(ax, color='black') plt.savefig('dendro_plots/' + label + '_dendrogram.pdf', bbox_inches='tight') #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Generate the catalog #%&%&%&%&%&%&%&%&%&%&%&%&%&% print("Generate a catalog of dendrogram structures") metadata = {} if hd3['BUNIT'].upper() == 'JY/BEAM': metadata['data_unit'] = u.Jy / u.beam elif hd3['BUNIT'].upper() == 'K': metadata['data_unit'] = u.K else: print("Warning: Unrecognized brightness unit") metadata['vaxis'] = 0 if 'RESTFREQ' in hd3.keys(): freq = hd3['RESTFREQ'] * u.Hz elif 'RESTFRQ' in hd3.keys(): freq = hd3['RESTFRQ'] * u.Hz metadata['wavelength'] = freq.to(u.m, equivalencies=u.spectral()) metadata['spatial_scale'] = hd3['cdelt2'] * 3600. * u.arcsec if hd3['ctype3'][0:3] == 'VEL' or hd3['ctype3'][0:4] == 'VRAD': dv = hd3['cdelt3'] / 1000. * u.km / u.s else: assert hd3['ctype3'][0:4] == 'FREQ' dv = 2.99792458e5 * np.absolute( hd3['cdelt3']) / freq.value * u.km / u.s metadata['velocity_scale'] = dv bmaj = hd3['bmaj'] * 3600. * u.arcsec # FWHM bmin = hd3['bmin'] * 3600. * u.arcsec # FWHM metadata['beam_major'] = bmaj metadata['beam_minor'] = bmin if not (redo == "n" and os.path.exists(label + '_full_catalog.txt')): print("generating catalog") cat = ppv_catalog(d, metadata) print(cat.info()) # Add additional properties: Average Peak Tb and Maximum Tb srclist = cat['_idx'].tolist() tmax = np.zeros(len(srclist), dtype=np.float64) tpkav = np.zeros(len(srclist), dtype=np.float64) for i, c in enumerate(srclist): peakim = np.nanmax(hdu3.data * d[c].get_mask(), axis=0) peakim[peakim == 0] = np.nan tmax[i] = np.nanmax(peakim) tpkav[i] = np.nanmean(peakim) if hd3['BUNIT'].upper() == 'JY/BEAM': omega_B = np.pi / (4 * np.log(2)) * bmaj * bmin convfac = (u.Jy).to(u.K, equivalencies=u.brightness_temperature( omega_B, freq)) tmax *= convfac tpkav *= convfac newcol = Column(tmax, name='tmax') newcol.unit = 'K' cat.add_column(newcol) newcol = Column(tpkav, name='tpkav') newcol.unit = 'K' cat.add_column(newcol) cat.write(label + '_full_catalog.txt', format='ascii.ecsv', overwrite=True) #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Generate the catalog with clipping #%&%&%&%&%&%&%&%&%&%&%&%&%&% if not (redo == "n" and os.path.exists(label + '_full_catalog_clipped.txt')): print("generating clipped catalog") ccat = ppv_catalog(d, metadata, clipping=True) print(ccat.info()) # Add additional properties: Average Peak Tb and Maximum Tb srclist = ccat['_idx'].tolist() tmax = np.zeros(len(srclist), dtype=np.float64) tpkav = np.zeros(len(srclist), dtype=np.float64) for i, c in enumerate(srclist): peakim = np.nanmax(hdu3.data * d[c].get_mask(), axis=0) peakim[peakim == 0] = np.nan clmin = np.nanmin(hdu3.data * d[c].get_mask()) tmax[i] = np.nanmax(peakim) - clmin tpkav[i] = np.nanmean(peakim) - clmin if hd3['BUNIT'].upper() == 'JY/BEAM': omega_B = np.pi / (4 * np.log(2)) * bmaj * bmin convfac = (u.Jy).to(u.K, equivalencies=u.brightness_temperature( omega_B, freq)) tmax *= convfac tpkav *= convfac newcol = Column(tmax, name='tmax-tmin') newcol.unit = 'K' ccat.add_column(newcol) newcol = Column(tpkav, name='tpkav-tmin') newcol.unit = 'K' ccat.add_column(newcol) ccat.write(label + '_full_catalog_clipped.txt', format='ascii.ecsv', overwrite=True) #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Running SCIMES #%&%&%&%&%&%&%&%&%&%&%&%&%&% # print("Running SCIMES") # dclust = SpectralCloudstering(d, cat, criteria = criteria, keepall=True) # print(dclust.clusters) # # print("Visualize the clustered dendrogram") # dclust.showdendro() # plt.savefig('dendro_plots/'+label+'_clusters_tree.pdf') # # print("Produce the assignment cube") # dclust.asgncube(hd3) # try: # os.remove(label+'_asgncube.fits.gz') # except OSError: # pass # dclust.asgn.writeto(label+'_asgncube.fits.gz') #sys.exit("Stopping here") #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Image the trunks #%&%&%&%&%&%&%&%&%&%&%&%&%&% if doplots: print("Image the trunks") hdu2 = fits.open(flatfile)[0] fig = plt.figure() ax = fig.add_subplot(1, 1, 1) vmax = np.nanmax(hdu2.data) / 2. im = ax.matshow(hdu2.data, origin='lower', cmap=plt.cm.Blues, vmax=vmax) ax.axes.get_xaxis().set_ticks([]) ax.axes.get_yaxis().set_ticks([]) if 'xlims' in kwargs: ax.set_xlim(kwargs['xlims']) if 'ylims' in kwargs: ax.set_ylim(kwargs['ylims']) # Make a trunk list tronly = [s for s in d.trunk if s not in d.leaves] f = open(label + '_trunks.txt', 'w') for c in tronly: f.write('{:<4d} | '.format(c.idx)) # Plot the actual structure boundaries mask = d[c.idx].get_mask() mask_coll = np.amax(mask, axis=0) plt.contour(mask_coll, colors='red', linewidths=1, levels=[0]) # Plot the ellipse fits s = analysis.PPVStatistic(d[c.idx]) ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none') ax.add_patch(ellipse) # Make sub-lists of descendants print('Finding descendants of trunk ', c.idx) desclist = [] if len(d[c.idx].descendants) > 0: for s in d[c.idx].descendants: desclist.append(s.idx) desclist.sort() liststr = ','.join(map(str, desclist)) f.write(liststr) f.write("\n") f.close() fig.colorbar(im, ax=ax) plt.savefig('dendro_plots/' + label + '_trunks_map.pdf', bbox_inches='tight') plt.close() # Make a branch list branch = [ s for s in d.all_structures if s not in d.leaves and s not in d.trunk ] slist = [] for c in branch: slist.append(c.idx) slist.sort() with open(label + '_branches.txt', 'w') as output: writer = csv.writer(output) for val in slist: writer.writerow([val]) #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Image the leaves #%&%&%&%&%&%&%&%&%&%&%&%&%&% print("Image the leaves") fig = plt.figure() ax = fig.add_subplot(1, 1, 1) vmax = np.nanmax(hdu2.data) / 2. im = ax.matshow(hdu2.data, origin='lower', cmap=plt.cm.Blues, vmax=vmax) ax.axes.get_xaxis().set_ticks([]) ax.axes.get_yaxis().set_ticks([]) if 'xlims' in kwargs: ax.set_xlim(kwargs['xlims']) if 'ylims' in kwargs: ax.set_ylim(kwargs['ylims']) # Make a leaf list slist = [] for c in d.leaves: slist.append(c.idx) # Plot the actual structure boundaries mask = d[c.idx].get_mask() mask_coll = np.amax(mask, axis=0) plt.contour(mask_coll, colors='green', linewidths=1, levels=[0]) # Plot the ellipse fits s = analysis.PPVStatistic(d[c.idx]) ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none') ax.add_patch(ellipse) slist.sort() with open(label + '_leaves.txt', "w") as output: writer = csv.writer(output) for val in slist: writer.writerow([val]) fig.colorbar(im, ax=ax) plt.savefig('dendro_plots/' + label + '_leaves_map.pdf', bbox_inches='tight') plt.close()
def downsampled_demo(data_file, downsample_factor=4, transpose_tuple=(2, 0, 1), min_value=0.01, min_delta=0.005, min_npix=2000, neighbours=None, resample=False, compute_catalog=True, recenter=True, data_path=data_path, memmap=True): df = downsample_factor tt = transpose_tuple print( "\n ** Downsampled Demo Parameters:" "\n downsample_factor={0}" "\n transpose_tuple={1}" "\n min_value={2}" "\n min_delta={3}" "\n min_npix={4}\n".format(downsample_factor, transpose_tuple, min_value, min_delta, min_npix)) beginning = datetime.datetime.now() beginning_string = datetime.datetime.strftime(beginning, "%Y-%m-%d %H:%M:%S") print("\n ** Beginning at: {0}".format(beginning_string)) print "\n** loading data from {0}: ...\n".format(data_path + data_file) datacube, datacube_header = getdata(data_path + data_file, memmap=memmap, header=True) print "\n** transposing {0} and downsampling ({1}) data: ...\n".format( tt, df) datacube_dt, datacube_dt_header = \ downsample_and_transpose_data_and_header(datacube, datacube_header, df, tt, resample=resample, recenter=recenter) datacube_dt_wcs = wcs.wcs.WCS(datacube_dt_header) datacube_dt_wcs.wcs.bounds_check(pix2world=False) beam_size = 1 / 8 * u.deg frequency = 115 * u.GHz print "\n** computing dendrogram on data with dimensions {} and n_pixels {:,}: ...\n".format( np.shape(datacube_dt), np.size(datacube_dt)) d = astrodendro.Dendrogram.compute( datacube_dt, min_value=min_value, min_delta=min_delta, #these are arbitrary min_npix=min_npix, verbose=True, neighbours=neighbours, wcs=datacube_dt_wcs) v_scale = datacube_dt_header['cdelt3'] v_unit = u.km / u.s l_scale = datacube_dt_header['cdelt1'] b_scale = datacube_dt_header['cdelt2'] metadata = {} metadata['data_unit'] = u.K metadata['spatial_scale'] = b_scale * u.deg metadata['velocity_scale'] = v_scale * v_unit metadata[ 'wavelength'] = frequency # formerly: (c.c / frequency).to('mm') but now compute_flux can handle frequency in spectral equivalency metadata['beam_major'] = beam_size metadata['beam_minor'] = beam_size metadata[ 'vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v) metadata['wcs'] = datacube_dt_wcs print "\n** computing catalog for {:,} structures: ...\n".format(len(d)) if compute_catalog: catalog = astrodendro.ppv_catalog(d, metadata, verbose=True) else: catalog = None return d, catalog, datacube_dt_header, metadata if catalog['flux'].unit.is_equivalent('Jy'): # Workaround because flux is computed wrong flux = quantify_column(catalog['flux']) area_exact = catalog['area_exact'].unit * catalog['area_exact'].data # average brightness temperature integrated over area_exact flux_kelvin = flux.to('K', equivalencies=u.brightness_temperature( area_exact, frequency)) # flux integrated over area and velocity flux_kelvin_kms_deg2 = flux_kelvin * metadata[ 'velocity_scale'] * area_exact catalog.add_column( astropy.table.Column(data=flux_kelvin_kms_deg2, name='flux_kelvin_kms_deg2')) end = datetime.datetime.now() end_string = datetime.datetime.strftime(end, "%Y-%m-%d %H:%M:%S") print("\n ** Ending at: {0}".format(end_string)) time_elapsed = (end - beginning) print "\n ** Time elapsed: {0}".format(time_elapsed) return d, catalog, datacube_dt_header, metadata
def main(args): #%&%&%&%&%&%&%&%&%&%&%&% # Make dendrogram #%&%&%&%&%&%&%&%&%&%&%&% print('Make dendrogram from the full cube') hdu = fits.open(args.fits_file)[0] data = hdu.data[801:1000, :, :] hd = hdu.header # Survey designs sigma = args.sigma #K, noise level ppb = args.ppb #pixels/beam def custom_independent(structure, index=None, value=None): peak_index, peak_value = structure.get_peak() return peak_value > 3. * sigma d = Dendrogram.compute(data, min_value=sigma, \ min_delta=args.delta*sigma, min_npix=1.*ppb, \ is_independent=custom_independent, \ verbose = 1) #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Generate the catalog #%&%&%&%&%&%&%&%&%&%&%&%&%&% print("Generate a catalog of dendrogram structures") metadata = {} metadata['data_unit'] = u.Jy #This should be Kelvin (not yet implemented)! cat = ppv_catalog(d, metadata) #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Running SCIMES #%&%&%&%&%&%&%&%&%&%&%&%&%&% print("Running SCIMES") dclust = scimes.SpectralCloudstering(d, cat, hd, rms=sigma, \ user_iter=args.iter, \ save_all_leaves = True, \ ) #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Image the result #%&%&%&%&%&%&%&%&%&%&%&%&%&% print("Visualize the clustered dendrogram") dclust.showdendro() print("Visualize collapsed maps of the assignment cubes") cubes = [dclust.clusters_asgn,\ dclust.leaves_asgn,\ dclust.trunks_asgn] titles = ['Clusters', 'Leaves', 'Trunks'] for cube, title in zip(cubes, titles): plt.figure() plt.imshow(np.nanmax(cube.data,axis=0),origin='lower',\ interpolation='nearest',cmap='jet') plt.title(title + ' assignment map') plt.colorbar(label='Structure label') plt.xlabel('X [pixel]') plt.ylabel('Y [pixel]') # plt.show() print("Image the results with APLpy") clusts = dclust.clusters colors = dclust.colors # Create Orion integrated intensity map mhdu = mom0map(hdu) fig = aplpy.FITSFigure(mhdu, figsize=(8, 6), convention='wells') fig.show_colorscale(cmap='gray') #, vmax=36, stretch = 'sqrt') count = 0 for c in clusts: mask = d[c].get_mask() mask_hdu = fits.PrimaryHDU(mask.astype('short'), hdu.header) mask_coll = np.amax(mask_hdu.data, axis=0) mask_coll_hdu = fits.PrimaryHDU(mask_coll.astype('short'), hd2d(hdu.header)) fig.show_contour(mask_coll_hdu, colors=colors[count], linewidths=2, convention='wells', levels=[0]) count = count + 1 print(count) # fig.tick_labels.set_xformat('dd') # fig.tick_labels.set_yformat('dd') fig.add_colorbar() fig.colorbar.set_axis_label_text(r'[(K km/s)$^{1/2}$]') fig.save('b.png') plt.show() return 0
def measure_dendrogram_properties(dend=None, cube303=cube303, cube321=cube321, cube13co=cube13co, cube18co=cube18co, noise_cube=noise_cube, sncube=sncube, suffix="", last_index=None, plot_some=True, line='303', write=True): assert (cube321.shape == cube303.shape == noise_cube.shape == cube13co.shape == cube18co.shape == sncube.shape) assert sncube.wcs is cube303.wcs is sncube.mask._wcs metadata = {} metadata['data_unit'] = u.K metadata['spatial_scale'] = 7.2 * u.arcsec metadata['beam_major'] = 30 * u.arcsec metadata['beam_minor'] = 30 * u.arcsec metadata['wavelength'] = 218.22219 * u.GHz metadata['velocity_scale'] = u.km / u.s metadata['wcs'] = cube303.wcs keys = [ 'density_chi2', 'expected_density', 'dmin1sig_chi2', 'dmax1sig_chi2', 'column_chi2', 'expected_column', 'cmin1sig_chi2', 'cmax1sig_chi2', 'temperature_chi2', 'expected_temperature', 'tmin1sig_chi2', 'tmax1sig_chi2', 'eratio321303', 'ratio321303', 'logh2column', 'elogh2column', 'logabundance', 'elogabundance', ] obs_keys = [ 'Stot303', 'Smin303', 'Smax303', 'Stot321', 'Smean303', 'Smean321', 'npix', 'e303', 'e321', 'r321303', 'er321303', '13cosum', 'c18osum', '13comean', 'c18omean', 's_ntotal', 'index', 'is_leaf', 'parent', 'root', 'lon', 'lat', 'vcen', 'higaldusttem', 'reff', 'dustmass', 'dustmindens', 'bad', #'tkin_turb', ] columns = {k: [] for k in (keys + obs_keys)} log.debug("Initializing dendrogram temperature fitting loop") # FORCE wcs to match # (technically should reproject here) cube13co._wcs = cube18co._wcs = cube303.wcs cube13co.mask._wcs = cube18co.mask._wcs = cube303.wcs if line == '303': maincube = cube303 elif line == '321': maincube = cube321 else: raise ValueError("Unrecognized line: {0}".format(line)) # Prepare an array to hold the fitted temperatures tcubedata = np.empty(maincube.shape, dtype='float32') tcubedata[:] = np.nan tcubeleafdata = np.empty(maincube.shape, dtype='float32') tcubeleafdata[:] = np.nan nbad = 0 catalog = ppv_catalog(dend, metadata) pb = ProgressBar(len(catalog)) for ii, row in enumerate(catalog): structure = dend[row['_idx']] assert structure.idx == row['_idx'] == ii dend_obj_mask = BooleanArrayMask(structure.get_mask(), wcs=cube303.wcs) dend_inds = structure.indices() view = ( slice(dend_inds[0].min(), dend_inds[0].max() + 1), slice(dend_inds[1].min(), dend_inds[1].max() + 1), slice(dend_inds[2].min(), dend_inds[2].max() + 1), ) #view2 = cube303.subcube_slices_from_mask(dend_obj_mask) submask = dend_obj_mask[view] #assert np.count_nonzero(submask.include()) == np.count_nonzero(dend_obj_mask.include()) sn = sncube[view].with_mask(submask) sntot = sn.sum().value #np.testing.assert_almost_equal(sntot, structure.values().sum(), decimal=0) c303 = cube303[view].with_mask(submask) c321 = cube321[view].with_mask(submask) co13sum = cube13co[view].with_mask(submask).sum().value co18sum = cube18co[view].with_mask(submask).sum().value if hasattr(co13sum, '__len__'): raise TypeError( ".sum() applied to an array has yielded a non scalar.") npix = submask.include().sum() assert npix == structure.get_npix() Stot303 = c303.sum().value if np.isnan(Stot303): raise ValueError("NaN in cube. This can't happen: the data from " "which the dendrogram was derived can't have " "NaN pixels.") Smax303 = c303.max().value Smin303 = c303.min().value Stot321 = c321.sum().value if npix == 0: raise ValueError("npix=0. This is impossible.") Smean303 = Stot303 / npix if Stot303 <= 0 and line == '303': raise ValueError( "The 303 flux is <=0. This isn't possible because " "the dendrogram was derived from the 303 data with a " "non-zero threshold.") elif Stot303 <= 0 and line == '321': Stot303 = 0 Smean303 = 0 elif Stot321 <= 0 and line == '321': raise ValueError( "The 321 flux is <=0. This isn't possible because " "the dendrogram was derived from the 321 data with a " "non-zero threshold.") if np.isnan(Stot321): raise ValueError("NaN in 321 line") Smean321 = Stot321 / npix #error = (noise_cube[view][submask.include()]).sum() / submask.include().sum()**0.5 var = ((noise_cube[dend_obj_mask.include()]**2).sum() / npix**2) error = var**0.5 if np.isnan(error): raise ValueError("error is nan: this is impossible by definition.") if line == '321' and Stot303 == 0: r321303 = np.nan er321303 = np.nan elif Stot321 < 0: r321303 = error / Smean303 er321303 = (r321303**2 * (var / Smean303**2 + 1))**0.5 else: r321303 = Stot321 / Stot303 er321303 = (r321303**2 * (var / Smean303**2 + var / Smean321**2))**0.5 for c in columns: assert len(columns[c]) == ii columns['index'].append(row['_idx']) columns['s_ntotal'].append(sntot) columns['Stot303'].append(Stot303) columns['Smax303'].append(Smax303) columns['Smin303'].append(Smin303) columns['Stot321'].append(Stot321) columns['Smean303'].append(Smean303) columns['Smean321'].append(Smean321) columns['npix'].append(npix) columns['e303'].append(error) columns['e321'].append(error) columns['r321303'].append(r321303) columns['er321303'].append(er321303) columns['13cosum'].append(co13sum) columns['c18osum'].append(co18sum) columns['13comean'].append(co13sum / npix) columns['c18omean'].append(co18sum / npix) columns['is_leaf'].append(structure.is_leaf) columns['parent'].append( structure.parent.idx if structure.parent else -1) columns['root'].append(get_root(structure).idx) s_main = maincube._data[dend_inds] x, y, z = maincube.world[dend_inds] lon = ((z.value - (360 * (z.value > 180))) * s_main).sum() / s_main.sum() lat = (y * s_main).sum() / s_main.sum() vel = (x * s_main).sum() / s_main.sum() columns['lon'].append(lon) columns['lat'].append(lat.value) columns['vcen'].append(vel.value) mask2d = dend_obj_mask.include().max(axis=0)[view[1:]] logh2column = np.log10( np.nanmean(column_regridded.data[view[1:]][mask2d]) * 1e22) if np.isnan(logh2column): log.info("Source #{0} has NaNs".format(ii)) logh2column = 24 elogh2column = elogabundance columns['higaldusttem'].append( np.nanmean(dusttem_regridded.data[view[1:]][mask2d])) r_arcsec = row['radius'] * u.arcsec reff = (r_arcsec * (8.5 * u.kpc)).to(u.pc, u.dimensionless_angles()) mass = ((10**logh2column * u.cm**-2) * np.pi * reff**2 * 2.8 * constants.m_p).to(u.M_sun) density = (mass / (4 / 3. * np.pi * reff**3) / constants.m_p / 2.8).to( u.cm**-3) columns['reff'].append(reff.value) columns['dustmass'].append(mass.value) columns['dustmindens'].append(density.value) mindens = np.log10(density.value) if mindens < 3: mindens = 3 if (r321303 < 0 or np.isnan(r321303)) and line != '321': raise ValueError("Ratio <0: This can't happen any more because " "if either num/denom is <0, an exception is " "raised earlier") #for k in columns: # if k not in obs_keys: # columns[k].append(np.nan) elif (r321303 < 0 or np.isnan(r321303)) and line == '321': for k in keys: columns[k].append(np.nan) else: # Replace negatives for fitting if Smean321 <= 0: Smean321 = error mf.set_constraints( ratio321303=r321303, eratio321303=er321303, #ratio321322=ratio2, eratio321322=eratio2, logh2column=logh2column, elogh2column=elogh2column, logabundance=logabundance, elogabundance=elogabundance, taline303=Smean303, etaline303=error, taline321=Smean321, etaline321=error, mindens=mindens, linewidth=10) row_data = mf.get_parconstraints() row_data['ratio321303'] = r321303 row_data['eratio321303'] = er321303 for k in row_data: columns[k].append(row_data[k]) # Exclude bad velocities from cubes if row['v_cen'] < -80e3 or row['v_cen'] > 180e3: # Skip: there is no real structure down here nbad += 1 is_bad = True else: is_bad = False tcubedata[ dend_obj_mask.include()] = row_data['expected_temperature'] if structure.is_leaf: tcubeleafdata[dend_obj_mask.include( )] = row_data['expected_temperature'] columns['bad'].append(is_bad) width = row['v_rms'] * u.km / u.s lengthscale = reff #REMOVED in favor of despotic version done in dendrograms.py # we use the analytic version here; the despotic version is # computed elsewhere (with appropriate gcor factors) #columns['tkin_turb'].append(heating.tkin_all(10**row_data['density_chi2']*u.cm**-3, # width, # lengthscale, # width/lengthscale, # columns['higaldusttem'][-1]*u.K, # crir=0./u.s)) if len(set(len(c) for k, c in columns.items())) != 1: print("Columns are different lengths. This is not allowed.") import ipdb ipdb.set_trace() for c in columns: assert len(columns[c]) == ii + 1 if plot_some and not is_bad and (ii - nbad % 100 == 0 or ii - nbad < 50): try: log.info( "T: [{tmin1sig_chi2:7.2f},{expected_temperature:7.2f},{tmax1sig_chi2:7.2f}]" " R={ratio321303:8.4f}+/-{eratio321303:8.4f}" " Smean303={Smean303:8.4f} +/- {e303:8.4f}" " Stot303={Stot303:8.2e} npix={npix:6d}".format( Smean303=Smean303, Stot303=Stot303, npix=npix, e303=error, **row_data)) pl.figure(1) pl.clf() mf.denstemplot() pl.savefig( fpath("dendrotem/diagnostics/{0}_{1}.png".format( suffix, ii))) pl.figure(2).clf() mf.parplot1d_all(levels=[0.68268949213708585]) pl.savefig( fpath("dendrotem/diagnostics/1dplot{0}_{1}.png".format( suffix, ii))) pl.draw() pl.show() except Exception as ex: print ex pass else: pb.update(ii + 1) if last_index is not None and ii >= last_index: break if last_index is not None: catalog = catalog[:last_index + 1] for k in columns: if k not in catalog.keys(): catalog.add_column(table.Column(name=k, data=columns[k])) for mid, lo, hi, letter in (('expected_temperature', 'tmin1sig_chi2', 'tmax1sig_chi2', 't'), ('expected_density', 'dmin1sig_chi2', 'dmax1sig_chi2', 'd'), ('expected_column', 'cmin1sig_chi2', 'cmax1sig_chi2', 'c')): catalog.add_column( table.Column(name='elo_' + letter, data=catalog[mid] - catalog[lo])) catalog.add_column( table.Column(name='ehi_' + letter, data=catalog[hi] - catalog[mid])) if write: catalog.write(tpath('PPV_H2CO_Temperature{0}.ipac'.format(suffix)), format='ascii.ipac') # Note that there are overlaps in the catalog, which means that ORDER MATTERS # in the above loop. I haven't yet checked whether large scale overwrites # small or vice-versa; it may be that both views of the data are interesting. tcube = SpectralCube( data=tcubedata, wcs=cube303.wcs, mask=cube303.mask, meta={'unit': 'K'}, header=cube303.header, ) tcubeleaf = SpectralCube( data=tcubeleafdata, wcs=cube303.wcs, mask=cube303.mask, meta={'unit': 'K'}, header=cube303.header, ) if write: log.info("Writing TemperatureCube") outpath = 'TemperatureCube_DendrogramObjects{0}.fits' tcube.write(hpath(outpath.format(suffix)), overwrite=True) outpath_leaf = 'TemperatureCube_DendrogramObjects{0}_leaves.fits' tcubeleaf.write(hpath(outpath_leaf.format(suffix)), overwrite=True) return catalog, tcube
def run_scimes(criteria=['volume'], label='scimes', cubefile=None, mom0file=None, bmajas=None, bminas=None, rfreq=None, redo='n', verbose=True, **kwargs): #%&%&%&%&%&%&%&%&%&%&%&% # Make dendrogram #%&%&%&%&%&%&%&%&%&%&%&% hdu3 = fits.open(cubefile)[0] hd3 = hdu3.header # Deal with oddities in 30 Dor cube if hd3['NAXIS'] == 3: for key in [ 'CTYPE4', 'CRVAL4', 'CDELT4', 'CRPIX4', 'CUNIT4', 'NAXIS4' ]: if key in hd3.keys(): hd3.remove(key) # Provide beam info if missing if bmajas is not None: hd3['bmaj'] = bmajas / 3600. if bminas is not None: hd3['bmin'] = bminas / 3600. # Get cube parameters sigma = stats.mad_std(hdu3.data[~np.isnan(hdu3.data)]) print('Robustly estimated RMS: {:.3f}'.format(sigma)) ppb = 1.133 * hd3['bmaj'] * hd3['bmin'] / (abs( hd3['cdelt1'] * hd3['cdelt2'])) print('Pixels per beam: {:.2f}'.format(ppb)) # Make the dendrogram if not present or redo=y if redo == 'n' and os.path.isfile(label + '_dendrogram.hdf5'): print('Loading pre-existing dendrogram') d = Dendrogram.load_from(label + '_dendrogram.hdf5') else: print('Make dendrogram from the full cube') d = Dendrogram.compute(hdu3.data, min_value=3 * sigma, min_delta=2.5 * sigma, min_npix=2 * ppb, verbose=verbose) d.save_to(label + '_dendrogram.hdf5') # checks/creates directory to place plots if os.path.isdir('plots') == 0: os.makedirs('plots') # Plot the tree fig = plt.figure(figsize=(14, 8)) ax = fig.add_subplot(111) ax.set_yscale('log') ax.set_xlabel('Structure') ax.set_ylabel('Intensity [' + hd3['BUNIT'] + ']') p = d.plotter() branch = [ s for s in d.all_structures if s not in d.leaves and s not in d.trunk ] tronly = [s for s in d.trunk if s not in d.leaves] for st in tronly: p.plot_tree(ax, structure=[st], color='brown', subtree=False) for st in branch: p.plot_tree(ax, structure=[st], color='black', subtree=False) for st in d.leaves: p.plot_tree(ax, structure=[st], color='green') plt.savefig('plots/' + label + '_dendrogram.pdf', bbox_inches='tight') #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Generate the catalog #%&%&%&%&%&%&%&%&%&%&%&%&%&% print('Generate a catalog of dendrogram structures') metadata = {} if hd3['BUNIT'].upper() == 'JY/BEAM': metadata['data_unit'] = u.Jy / u.beam elif hd3['BUNIT'].upper() == 'K': metadata['data_unit'] = u.K else: print('Warning: Unrecognized brightness unit') metadata['vaxis'] = 0 if rfreq is None: if 'RESTFREQ' in hd3.keys(): freq = hd3['RESTFREQ'] * u.Hz elif 'RESTFRQ' in hd3.keys(): freq = hd3['RESTFRQ'] * u.Hz else: freq = rfreq * u.GHz metadata['wavelength'] = freq.to(u.m, equivalencies=u.spectral()) metadata['spatial_scale'] = hd3['cdelt2'] * 3600. * u.arcsec metadata['velocity_scale'] = abs(hd3['cdelt3']) * u.meter / u.second bmaj = hd3['bmaj'] * 3600. * u.arcsec # FWHM bmin = hd3['bmin'] * 3600. * u.arcsec # FWHM metadata['beam_major'] = bmaj metadata['beam_minor'] = bmin cat = ppv_catalog(d, metadata, verbose=verbose) print(cat.info()) # Add additional properties: Average Peak Tb and Maximum Tb srclist = cat['_idx'].tolist() tmax = np.zeros(len(srclist), dtype=np.float64) tpkav = np.zeros(len(srclist), dtype=np.float64) for i, c in enumerate(srclist): peakim = np.nanmax(hdu3.data * d[c].get_mask(), axis=0) peakim[peakim == 0] = np.nan tmax[i] = np.nanmax(peakim) tpkav[i] = np.nanmean(peakim) if hd3['BUNIT'].upper() == 'JY/BEAM': omega_B = np.pi / (4 * np.log(2)) * bmaj * bmin convfac = (u.Jy).to(u.K, equivalencies=u.brightness_temperature( omega_B, freq)) tmax *= convfac tpkav *= convfac newcol = Column(tmax, name='tmax') newcol.unit = 'K' cat.add_column(newcol) newcol = Column(tpkav, name='tpkav') newcol.unit = 'K' cat.add_column(newcol) cat.write(label + '_full_catalog.txt', format='ascii.ecsv', overwrite=True) #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Running SCIMES #%&%&%&%&%&%&%&%&%&%&%&%&%&% print("Running SCIMES") dclust = SpectralCloudstering(d, cat, criteria=criteria, keepall=True) print(dclust.clusters) print("Visualize the clustered dendrogram") dclust.showdendro() plt.savefig('plots/' + label + '_clusters_tree.pdf') print("Produce the assignment cube") dclust.asgncube(hd3) try: os.remove(label + '_asgncube.fits.gz') except OSError: pass dclust.asgn.writeto(label + '_asgncube.fits.gz') #sys.exit("Stopping here") #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Image the trunks #%&%&%&%&%&%&%&%&%&%&%&%&%&% print("Image the trunks") hdu2 = fits.open(mom0file)[0] fig = plt.figure() ax = fig.add_subplot(1, 1, 1) vmax = np.nanmax(hdu2.data) / 2. im = ax.matshow(hdu2.data, origin='lower', cmap=plt.cm.Blues, vmax=vmax) ax.axes.get_xaxis().set_ticks([]) ax.axes.get_yaxis().set_ticks([]) if 'xlims' in kwargs: ax.set_xlim(kwargs['xlims']) if 'ylims' in kwargs: ax.set_ylim(kwargs['ylims']) # Make a trunk list tronly = [s for s in d.trunk if s not in d.leaves] f = open(label + '_trunks.txt', 'w') for c in tronly: f.write('{:<4d} | '.format(c.idx)) # Plot the actual structure boundaries mask = d[c.idx].get_mask() mask_coll = np.amax(mask, axis=0) plt.contour(mask_coll, colors='red', linewidths=1, levels=[0]) # Plot the ellipse fits s = analysis.PPVStatistic(d[c.idx]) ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none') ax.add_patch(ellipse) # Make sub-lists of descendants print('Finding descendants of trunk {}'.format(c.idx)) desclist = [] if len(d[c.idx].descendants) > 0: for s in d[c.idx].descendants: desclist.append(s.idx) desclist.sort() liststr = ','.join(map(str, desclist)) f.write(liststr) f.write("\n") f.close() fig.colorbar(im, ax=ax) plt.savefig('plots/' + label + '_trunks_map.pdf', bbox_inches='tight') plt.close() # Make a branch list branch = [ s for s in d.all_structures if s not in d.leaves and s not in d.trunk ] slist = [] for c in branch: slist.append(c.idx) slist.sort() with open(label + '_branches.txt', 'w') as output: writer = csv.writer(output) for val in slist: writer.writerow([val]) #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Image the leaves #%&%&%&%&%&%&%&%&%&%&%&%&%&% print("Image the leaves") fig = plt.figure() ax = fig.add_subplot(1, 1, 1) vmax = np.nanmax(hdu2.data) / 2. im = ax.matshow(hdu2.data, origin='lower', cmap=plt.cm.Blues, vmax=vmax) ax.axes.get_xaxis().set_ticks([]) ax.axes.get_yaxis().set_ticks([]) if 'xlims' in kwargs: ax.set_xlim(kwargs['xlims']) if 'ylims' in kwargs: ax.set_ylim(kwargs['ylims']) # Make a leaf list slist = [] for c in d.leaves: slist.append(c.idx) # Plot the actual structure boundaries mask = d[c.idx].get_mask() mask_coll = np.amax(mask, axis=0) plt.contour(mask_coll, colors='green', linewidths=1, levels=[0]) # Plot the ellipse fits s = analysis.PPVStatistic(d[c.idx]) ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none') ax.add_patch(ellipse) slist.sort() with open(label + '_leaves.txt', "w") as output: writer = csv.writer(output) for val in slist: writer.writerow([val]) fig.colorbar(im, ax=ax) plt.savefig('plots/' + label + '_leaves_map.pdf', bbox_inches='tight') plt.close() #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Image the clusters #%&%&%&%&%&%&%&%&%&%&%&%&%&% print("Image the resulting clusters") clusts = np.array(dclust.clusters) colors = np.array(dclust.colors) inds = np.argsort(clusts) clusts = clusts[inds] colors = colors[inds] print(clusts) print(colors) fig = plt.figure() ax = fig.add_subplot(1, 1, 1) vmax = np.nanmax(hdu2.data) / 2. im = ax.matshow(hdu2.data, origin='lower', cmap=plt.cm.Blues, vmax=vmax) ax.axes.get_xaxis().set_ticks([]) ax.axes.get_yaxis().set_ticks([]) if 'xlims' in kwargs: ax.set_xlim(kwargs['xlims']) if 'ylims' in kwargs: ax.set_ylim(kwargs['ylims']) # Make a cluster list f = open(label + '_clusters.txt', 'w') for i, c in enumerate(clusts): f.write('{:<4d} {:7} | '.format(c, colors[i])) # Plot the actual structure boundaries mask = d[c].get_mask() mask_coll = np.amax(mask, axis=0) plt.contour(mask_coll, colors=colors[i], linewidths=1, levels=[0]) # Plot the ellipse fits s = analysis.PPVStatistic(d[c]) ellipse = s.to_mpl_ellipse(edgecolor='black', facecolor='none') ax.add_patch(ellipse) # Make sub-lists of descendants print('Finding descendants of cluster {}'.format(c)) desclist = [] if len(d[c].descendants) > 0: for s in d[c].descendants: desclist.append(s.idx) desclist.sort() liststr = ','.join(map(str, desclist)) f.write(liststr) f.write("\n") f.close() fig.colorbar(im, ax=ax) plt.savefig('plots/' + label + '_clusters_map.pdf', bbox_inches='tight') plt.close() return
def doDendroAndScimes(self,fitsFile,rms=0.5, minPix=500 ,reDo=False,subRegion="" ,saveAll=True,vScale=10,useVolume=True,useLuminosity=True,useVelociy=True ,calDendro=True, inputDendro=None,iinputDenroCatFile=None ): """ :param fitsFile: :param rms: #not important anymore :param minPix: :param reDo: :param subRegion: #only used to save :param saveAll: :param vScale: only useful when useVeloicty is true :param useVolume: :param useLuminosity: :param useVelociy: :return: """ if not calDendro and (inputDendro ==None or iinputDenroCatFile==None): print "If you do not want to calculate dendrogram, you need to provide the inputDendro and the catalog" return criteriaUsed=[] scales=[] saveMark='' if useLuminosity: criteriaUsed.append('luminosity') scales.append(None) saveMark=saveMark+'Lu' if useVolume: criteriaUsed.append('volume') #criteriaUsed.append('trueVolume') saveMark=saveMark+'Vo' scales.append(None) if useVelociy: #criteriaUsed.append('trueVelocity') criteriaUsed.append('v_rms') scales.append(vScale) saveMark=saveMark+'Ve' saveMark=saveMark+"{}_{}".format(minPix,vScale) subRegion=subRegion+saveMark hdu=fits.open(fitsFile)[0] data= hdu.data hd=hdu.header saveDendro=self.regionName+subRegion+"Dendro.fits" cdelt1 = abs(hd.get('CDELT1')) cdelt2 = abs(hd.get('CDELT2')) ppbeam = abs((bmaj*bmin)/(cdelt1*cdelt2)*2*np.pi/(8*np.log(2))) # Pixels per beam if not calDendro and os.path.isfile( self.saveSCIMESPath+saveDendro ): print self.regionName," has been done, skipping..." return print("Making dendrogram...") #from pruning import all_true, min_vchan, min_delta, min_area #is_independent = all_true((min_delta(3*rms), min_area(288), min_vchan(2))) #is_independent = all_true( ( min_area(288) ) ) #d = Dendrogram.compute(data, min_value=0, is_independent=is_independent, verbose=1, min_npix=10000 ) #d = Dendrogram.compute(data, min_value=0, verbose=1, min_npix=minPix ,min_delta=1.5 , is_independent=min_area(288) ) catName=self.saveSCIMESPath+self.regionName+subRegion +"dendroCat.fit" treeFile=self.saveSCIMESPath+subRegion+"DendroTree.txt" if calDendro: #just for test d = Dendrogram.compute(data, min_value=0, verbose=1, min_npix=minPix, min_delta=1.5 ) d.save_to( saveDendro ) self.metadata["wcs"]= WCS(hd) cat = ppv_catalog(d, self.metadata) print len(cat),"<-- total number of structures?" #print dir(cat) cat.write(catName,overwrite=True) self.writeTreeStructure(d,treeFile) # add levels. #if saveDendro: #cat.write(self.saveSCIMESPath+saveDenroMark+".fit" ,overwrite=True) #return d,self.saveSCIMESPath+saveDenroMark+".fit" else: #just for test d =inputDendro # do not read dendro here cat = Table.read(iinputDenroCatFile) print len(cat),"<-- total number of structures?" #print dir(cat) self.writeTreeStructure(d,treeFile) # add levels. #newVo,newVe=self.getTrueVolumeAndVrms(treeFile,iinputDenroCatFile,subRegion=subRegion) #cat["trueVolume"]=newVo #cat['trueVelocity']=newVe ###Test, what if we multiply the volumen by the dispersion of velocity dispersion in km/s #cat.sort("v_rms") #print cat #print cat.colnames res = SpectralCloudstering(d, cat, hd, criteria = criteriaUsed , user_scalpars=scales, blind = True , rms = self.rms, s2nlim = 3, save_all = saveAll, user_iter=1) res.clusters_asgn.writeto (self.saveSCIMESPath+self.regionName+subRegion+'Cluster_asgn.fits', overwrite=True) clusterCat=cat[res.clusters] clusterCat.write( self.saveSCIMESPath +self.regionName+subRegion +"ClusterCat.fit",overwrite=True)
def compute_catalog(d, header): """ Computes a catalog on a dendrogram. Parameters ---------- d : astrodendro.dendrogram.Dendrogram Dendrogram on which to compute a catalog header : astropy.io.fits.header.Header Header corresponding exactly to `d.data`. Used for unit and scaling information to calculate flux and world coordinates. Returns ------- catalog : astropy.table.table.Table Catalog describing the structures in dendrogram `d` using units provided in `header`. metadata : dict Explicitly lists unit properties used in `catalog` """ # rough average of North and South telescopes: # see Dame et al. 2001, p. 794. # "The Northern telescope... has a beamwidth of 8'.4 +/- 0'.2.... # The Southern telescope has nearly the same beamwidth to within # the uncertainties: 8'.8 +/- 0'.2" beam_size = 8.5 * u.arcmin frequency = 115.27 * u.GHz # http://www.cv.nrao.edu/php/splat/ # Remember: by this point, the data ought to be in (v, b, l), with FITS header order opposite that if 'vel' not in header['ctype3'].lower(): raise ValueError( "CTYPE3 must be velocity - check that the data were permuted correctly" ) v_scale = header['cdelt3'] v_unit = u.km / u.s l_scale = header['cdelt1'] b_scale = header['cdelt2'] metadata = {} metadata['data_unit'] = u.K metadata['spatial_scale'] = b_scale * u.deg metadata['velocity_scale'] = v_scale * v_unit metadata[ 'wavelength'] = frequency # formerly: (c.c / frequency).to('mm') but now compute_flux can handle frequency in spectral equivalency metadata['beam_major'] = beam_size metadata['beam_minor'] = beam_size metadata[ 'vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v) metadata['wcs'] = d.wcs catalog = astrodendro.ppv_catalog(d, metadata, verbose=True) if catalog['flux'].unit.is_equivalent('Jy'): # Workaround because flux is computed wrong (see https://github.com/dendrograms/astrodendro/issues/107) flux = u.Quantity(catalog['flux']) area_exact = u.Quantity( catalog['area_exact']) #.unit*catalog['area_exact'].data # average brightness temperature integrated over area_exact flux_kelvin = flux.to('K', equivalencies=u.brightness_temperature( area_exact, frequency)) # flux integrated over area and velocity flux_kelvin_kms_sr = flux_kelvin * metadata[ 'velocity_scale'] * area_exact.to(u.steradian) catalog['flux_true'] = flux_kelvin_kms_sr background_flux_array = np.zeros_like(catalog['flux_true']) # do a clipping -- # this uses the approximation that any given structure's background is well-represented by its lowest-valued pixel, which can get messy sometimes. for i, row in enumerate(catalog): background_kelvins = d[i].vmin * u.K background_flux = background_kelvins * d[i].get_npix( ) * metadata['velocity_scale'] * metadata['spatial_scale']**2 background_flux_array[i] = background_flux.to(u.K * u.km / u.s * u.steradian).value catalog['flux_clipped'] = catalog['flux_true'] - background_flux_array return catalog, metadata
def compare_jansky_to_kelvin(data_file, downsample_factor=4, transpose_tuple=(2,0,1), min_value=0.01, min_delta=0.005, min_npix=1000): df = downsample_factor tt = transpose_tuple print "loading data: ..." datacube, datacube_header = getdata(data_path+data_file, memmap=True, header=True) print "transposing, downsampling, and unit-converting data: ..." datacube_dt, datacube_dt_header = \ downsample_and_transpose_data_and_header(datacube, datacube_header, df, tt) datacube_dt_wcs = wcs.wcs.WCS(datacube_dt_header) beam_size = 1/8 * u.deg # Convert the data from kelvin to jansky-per-beam omega_beam = np.pi * (0.5 * beam_size)**2 # Beam width is 1/8 degree frequency = 115 * u.GHz K_to_Jy = u.K.to(u.Jy, equivalencies= u.brightness_temperature(omega_beam, frequency)) datacube_dt_jansky_perbeam = datacube_dt * K_to_Jy print "computing dendrogram: ..." d_jansky = astrodendro.Dendrogram.compute( datacube_dt_jansky_perbeam, min_value=min_value*K_to_Jy, min_delta=min_delta*K_to_Jy, #these are arbitrary min_npix=min_npix//df**3, verbose=True) d_kelvin = astrodendro.Dendrogram.compute( datacube_dt, min_value=min_value, min_delta=min_delta, #these are arbitrary min_npix=min_npix//df**3, verbose=True) print len(list(d_jansky.all_structures)) print len(list(d_kelvin.all_structures)) v_scale = datacube_dt_header['cdelt3'] v_unit = u.km / u.s l_scale = datacube_dt_header['cdelt1'] b_scale = datacube_dt_header['cdelt2'] metadata_jansky = {} metadata_jansky['data_unit'] = u.Jy / u.beam # According to A. Ginsburg metadata_jansky['spatial_scale'] = b_scale * u.deg metadata_jansky['velocity_scale'] = v_scale * v_unit metadata_jansky['wavelength'] = (c.c / frequency).to('mm') metadata_jansky['beam_major'] = beam_size metadata_jansky['beam_minor'] = beam_size metadata_jansky['vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v) metadata_jansky['wcs'] = datacube_dt_wcs metadata_kelvin = metadata_jansky.copy() metadata_kelvin['data_unit'] = u.K # overwrite Jy/beam catalog_jansky = astrodendro.ppv_catalog(d_jansky, metadata_jansky) catalog_kelvin = astrodendro.ppv_catalog(d_kelvin, metadata_kelvin) beams_per_pixel = metadata_jansky['spatial_scale'] ** 2 / (metadata_jansky['beam_minor'] * metadata_jansky['beam_major'] * 1.1331) * u.beam print beams_per_pixel # if catalog['flux'].unit.is_equivalent('Jy'): # # Workaround because flux is computed wrong # flux = quantify_column(catalog['flux']) # area_exact = catalog['area_exact'].unit*catalog['area_exact'].data # flux_kelvin = flux.to('K', equivalencies=u.brightness_temperature(area_exact, frequency)) # flux_kelvin_kms_deg2 = flux_kelvin * metadata['velocity_scale'] * area_exact # catalog.add_column(astropy.table.Column(data=flux_kelvin_kms_deg2, name='flux_kelvin_kms_deg2')) return_dict = {} return_dict['jansky'] = (d_jansky, catalog_jansky) return_dict['kelvin'] = (d_kelvin, catalog_kelvin) return return_dict
def compare_jansky_to_kelvin(data_file, downsample_factor=4, transpose_tuple=(2, 0, 1), min_value=0.01, min_delta=0.005, min_npix=1000): df = downsample_factor tt = transpose_tuple print "loading data: ..." datacube, datacube_header = getdata(data_path + data_file, memmap=True, header=True) print "transposing, downsampling, and unit-converting data: ..." datacube_dt, datacube_dt_header = \ downsample_and_transpose_data_and_header(datacube, datacube_header, df, tt) datacube_dt_wcs = wcs.wcs.WCS(datacube_dt_header) beam_size = 1 / 8 * u.deg # Convert the data from kelvin to jansky-per-beam omega_beam = np.pi * (0.5 * beam_size)**2 # Beam width is 1/8 degree frequency = 115 * u.GHz K_to_Jy = u.K.to(u.Jy, equivalencies=u.brightness_temperature( omega_beam, frequency)) datacube_dt_jansky_perbeam = datacube_dt * K_to_Jy print "computing dendrogram: ..." d_jansky = astrodendro.Dendrogram.compute( datacube_dt_jansky_perbeam, min_value=min_value * K_to_Jy, min_delta=min_delta * K_to_Jy, #these are arbitrary min_npix=min_npix // df**3, verbose=True) d_kelvin = astrodendro.Dendrogram.compute( datacube_dt, min_value=min_value, min_delta=min_delta, #these are arbitrary min_npix=min_npix // df**3, verbose=True) print len(list(d_jansky.all_structures)) print len(list(d_kelvin.all_structures)) v_scale = datacube_dt_header['cdelt3'] v_unit = u.km / u.s l_scale = datacube_dt_header['cdelt1'] b_scale = datacube_dt_header['cdelt2'] metadata_jansky = {} metadata_jansky['data_unit'] = u.Jy / u.beam # According to A. Ginsburg metadata_jansky['spatial_scale'] = b_scale * u.deg metadata_jansky['velocity_scale'] = v_scale * v_unit metadata_jansky['wavelength'] = (c.c / frequency).to('mm') metadata_jansky['beam_major'] = beam_size metadata_jansky['beam_minor'] = beam_size metadata_jansky[ 'vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v) metadata_jansky['wcs'] = datacube_dt_wcs metadata_kelvin = metadata_jansky.copy() metadata_kelvin['data_unit'] = u.K # overwrite Jy/beam catalog_jansky = astrodendro.ppv_catalog(d_jansky, metadata_jansky) catalog_kelvin = astrodendro.ppv_catalog(d_kelvin, metadata_kelvin) beams_per_pixel = metadata_jansky['spatial_scale']**2 / ( metadata_jansky['beam_minor'] * metadata_jansky['beam_major'] * 1.1331) * u.beam print beams_per_pixel # if catalog['flux'].unit.is_equivalent('Jy'): # # Workaround because flux is computed wrong # flux = quantify_column(catalog['flux']) # area_exact = catalog['area_exact'].unit*catalog['area_exact'].data # flux_kelvin = flux.to('K', equivalencies=u.brightness_temperature(area_exact, frequency)) # flux_kelvin_kms_deg2 = flux_kelvin * metadata['velocity_scale'] * area_exact # catalog.add_column(astropy.table.Column(data=flux_kelvin_kms_deg2, name='flux_kelvin_kms_deg2')) return_dict = {} return_dict['jansky'] = (d_jansky, catalog_jansky) return_dict['kelvin'] = (d_kelvin, catalog_kelvin) return return_dict
def measure_dendrogram_properties(dend=None, cube303=cube303, cube321=cube321, cube13co=cube13co, cube18co=cube18co, noise_cube=noise_cube, sncube=sncube, suffix="", last_index=None, plot_some=True, line='303', write=True): assert (cube321.shape == cube303.shape == noise_cube.shape == cube13co.shape == cube18co.shape == sncube.shape) assert sncube.wcs is cube303.wcs is sncube.mask._wcs metadata = {} metadata['data_unit'] = u.K metadata['spatial_scale'] = 7.2 * u.arcsec metadata['beam_major'] = 30 * u.arcsec metadata['beam_minor'] = 30 * u.arcsec metadata['wavelength'] = 218.22219*u.GHz metadata['velocity_scale'] = u.km/u.s metadata['wcs'] = cube303.wcs keys = [ 'density_chi2', 'expected_density', 'dmin1sig_chi2', 'dmax1sig_chi2', 'column_chi2', 'expected_column', 'cmin1sig_chi2', 'cmax1sig_chi2', 'temperature_chi2', 'expected_temperature', 'tmin1sig_chi2', 'tmax1sig_chi2', 'eratio321303', 'ratio321303', 'logh2column', 'elogh2column', 'logabundance', 'elogabundance', ] obs_keys = [ 'Stot303', 'Smin303', 'Smax303', 'Stot321', 'Smean303', 'Smean321', 'npix', 'e303', 'e321', 'r321303', 'er321303', '13cosum', 'c18osum', '13comean', 'c18omean', 's_ntotal', 'index', 'is_leaf', 'parent', 'root', 'lon', 'lat', 'vcen', 'higaldusttem', 'reff', 'dustmass', 'dustmindens', 'bad', #'tkin_turb', ] columns = {k:[] for k in (keys+obs_keys)} log.debug("Initializing dendrogram temperature fitting loop") # FORCE wcs to match # (technically should reproject here) cube13co._wcs = cube18co._wcs = cube303.wcs cube13co.mask._wcs = cube18co.mask._wcs = cube303.wcs if line == '303': maincube = cube303 elif line == '321': maincube = cube321 else: raise ValueError("Unrecognized line: {0}".format(line)) # Prepare an array to hold the fitted temperatures tcubedata = np.empty(maincube.shape, dtype='float32') tcubedata[:] = np.nan tcubeleafdata = np.empty(maincube.shape, dtype='float32') tcubeleafdata[:] = np.nan nbad = 0 catalog = ppv_catalog(dend, metadata) pb = ProgressBar(len(catalog)) for ii,row in enumerate(catalog): structure = dend[row['_idx']] assert structure.idx == row['_idx'] == ii dend_obj_mask = BooleanArrayMask(structure.get_mask(), wcs=cube303.wcs) dend_inds = structure.indices() view = (slice(dend_inds[0].min(), dend_inds[0].max()+1), slice(dend_inds[1].min(), dend_inds[1].max()+1), slice(dend_inds[2].min(), dend_inds[2].max()+1),) #view2 = cube303.subcube_slices_from_mask(dend_obj_mask) submask = dend_obj_mask[view] #assert np.count_nonzero(submask.include()) == np.count_nonzero(dend_obj_mask.include()) sn = sncube[view].with_mask(submask) sntot = sn.sum().value #np.testing.assert_almost_equal(sntot, structure.values().sum(), decimal=0) c303 = cube303[view].with_mask(submask) c321 = cube321[view].with_mask(submask) co13sum = cube13co[view].with_mask(submask).sum().value co18sum = cube18co[view].with_mask(submask).sum().value if hasattr(co13sum,'__len__'): raise TypeError(".sum() applied to an array has yielded a non scalar.") npix = submask.include().sum() assert npix == structure.get_npix() Stot303 = c303.sum().value if np.isnan(Stot303): raise ValueError("NaN in cube. This can't happen: the data from " "which the dendrogram was derived can't have " "NaN pixels.") Smax303 = c303.max().value Smin303 = c303.min().value Stot321 = c321.sum().value if npix == 0: raise ValueError("npix=0. This is impossible.") Smean303 = Stot303/npix if Stot303 <= 0 and line=='303': raise ValueError("The 303 flux is <=0. This isn't possible because " "the dendrogram was derived from the 303 data with a " "non-zero threshold.") elif Stot303 <= 0 and line=='321': Stot303 = 0 Smean303 = 0 elif Stot321 <= 0 and line=='321': raise ValueError("The 321 flux is <=0. This isn't possible because " "the dendrogram was derived from the 321 data with a " "non-zero threshold.") if np.isnan(Stot321): raise ValueError("NaN in 321 line") Smean321 = Stot321/npix #error = (noise_cube[view][submask.include()]).sum() / submask.include().sum()**0.5 var = ((noise_cube[dend_obj_mask.include()]**2).sum() / npix**2) error = var**0.5 if np.isnan(error): raise ValueError("error is nan: this is impossible by definition.") if line == '321' and Stot303 == 0: r321303 = np.nan er321303 = np.nan elif Stot321 < 0: r321303 = error / Smean303 er321303 = (r321303**2 * (var/Smean303**2 + 1))**0.5 else: r321303 = Stot321 / Stot303 er321303 = (r321303**2 * (var/Smean303**2 + var/Smean321**2))**0.5 for c in columns: assert len(columns[c]) == ii columns['index'].append(row['_idx']) columns['s_ntotal'].append(sntot) columns['Stot303'].append(Stot303) columns['Smax303'].append(Smax303) columns['Smin303'].append(Smin303) columns['Stot321'].append(Stot321) columns['Smean303'].append(Smean303) columns['Smean321'].append(Smean321) columns['npix'].append(npix) columns['e303'].append(error) columns['e321'].append(error) columns['r321303'].append(r321303) columns['er321303'].append(er321303) columns['13cosum'].append(co13sum) columns['c18osum'].append(co18sum) columns['13comean'].append(co13sum/npix) columns['c18omean'].append(co18sum/npix) columns['is_leaf'].append(structure.is_leaf) columns['parent'].append(structure.parent.idx if structure.parent else -1) columns['root'].append(get_root(structure).idx) s_main = maincube._data[dend_inds] x,y,z = maincube.world[dend_inds] lon = ((z.value-(360*(z.value>180)))*s_main).sum()/s_main.sum() lat = (y*s_main).sum()/s_main.sum() vel = (x*s_main).sum()/s_main.sum() columns['lon'].append(lon) columns['lat'].append(lat.value) columns['vcen'].append(vel.value) mask2d = dend_obj_mask.include().max(axis=0)[view[1:]] logh2column = np.log10(np.nanmean(column_regridded.data[view[1:]][mask2d]) * 1e22) if np.isnan(logh2column): log.info("Source #{0} has NaNs".format(ii)) logh2column = 24 elogh2column = elogabundance columns['higaldusttem'].append(np.nanmean(dusttem_regridded.data[view[1:]][mask2d])) r_arcsec = row['radius']*u.arcsec reff = (r_arcsec*(8.5*u.kpc)).to(u.pc, u.dimensionless_angles()) mass = ((10**logh2column*u.cm**-2)*np.pi*reff**2*2.8*constants.m_p).to(u.M_sun) density = (mass/(4/3.*np.pi*reff**3)/constants.m_p/2.8).to(u.cm**-3) columns['reff'].append(reff.value) columns['dustmass'].append(mass.value) columns['dustmindens'].append(density.value) mindens = np.log10(density.value) if mindens < 3: mindens = 3 if (r321303 < 0 or np.isnan(r321303)) and line != '321': raise ValueError("Ratio <0: This can't happen any more because " "if either num/denom is <0, an exception is " "raised earlier") #for k in columns: # if k not in obs_keys: # columns[k].append(np.nan) elif (r321303 < 0 or np.isnan(r321303)) and line == '321': for k in keys: columns[k].append(np.nan) else: # Replace negatives for fitting if Smean321 <= 0: Smean321 = error mf.set_constraints(ratio321303=r321303, eratio321303=er321303, #ratio321322=ratio2, eratio321322=eratio2, logh2column=logh2column, elogh2column=elogh2column, logabundance=logabundance, elogabundance=elogabundance, taline303=Smean303, etaline303=error, taline321=Smean321, etaline321=error, mindens=mindens, linewidth=10) row_data = mf.get_parconstraints() row_data['ratio321303'] = r321303 row_data['eratio321303'] = er321303 for k in row_data: columns[k].append(row_data[k]) # Exclude bad velocities from cubes if row['v_cen'] < -80e3 or row['v_cen'] > 180e3: # Skip: there is no real structure down here nbad += 1 is_bad = True else: is_bad = False tcubedata[dend_obj_mask.include()] = row_data['expected_temperature'] if structure.is_leaf: tcubeleafdata[dend_obj_mask.include()] = row_data['expected_temperature'] columns['bad'].append(is_bad) width = row['v_rms']*u.km/u.s lengthscale = reff #REMOVED in favor of despotic version done in dendrograms.py # we use the analytic version here; the despotic version is # computed elsewhere (with appropriate gcor factors) #columns['tkin_turb'].append(heating.tkin_all(10**row_data['density_chi2']*u.cm**-3, # width, # lengthscale, # width/lengthscale, # columns['higaldusttem'][-1]*u.K, # crir=0./u.s)) if len(set(len(c) for k,c in columns.items())) != 1: print("Columns are different lengths. This is not allowed.") import ipdb; ipdb.set_trace() for c in columns: assert len(columns[c]) == ii+1 if plot_some and not is_bad and (ii-nbad % 100 == 0 or ii-nbad < 50): try: log.info("T: [{tmin1sig_chi2:7.2f},{expected_temperature:7.2f},{tmax1sig_chi2:7.2f}]" " R={ratio321303:8.4f}+/-{eratio321303:8.4f}" " Smean303={Smean303:8.4f} +/- {e303:8.4f}" " Stot303={Stot303:8.2e} npix={npix:6d}" .format(Smean303=Smean303, Stot303=Stot303, npix=npix, e303=error, **row_data)) pl.figure(1) pl.clf() mf.denstemplot() pl.savefig(fpath("dendrotem/diagnostics/{0}_{1}.png".format(suffix,ii))) pl.figure(2).clf() mf.parplot1d_all(levels=[0.68268949213708585]) pl.savefig(fpath("dendrotem/diagnostics/1dplot{0}_{1}.png".format(suffix,ii))) pl.draw() pl.show() except Exception as ex: print ex pass else: pb.update(ii+1) if last_index is not None and ii >= last_index: break if last_index is not None: catalog = catalog[:last_index+1] for k in columns: if k not in catalog.keys(): catalog.add_column(table.Column(name=k, data=columns[k])) for mid,lo,hi,letter in (('expected_temperature','tmin1sig_chi2','tmax1sig_chi2','t'), ('expected_density','dmin1sig_chi2','dmax1sig_chi2','d'), ('expected_column','cmin1sig_chi2','cmax1sig_chi2','c')): catalog.add_column(table.Column(name='elo_'+letter, data=catalog[mid]-catalog[lo])) catalog.add_column(table.Column(name='ehi_'+letter, data=catalog[hi]-catalog[mid])) if write: catalog.write(tpath('PPV_H2CO_Temperature{0}.ipac'.format(suffix)), format='ascii.ipac') # Note that there are overlaps in the catalog, which means that ORDER MATTERS # in the above loop. I haven't yet checked whether large scale overwrites # small or vice-versa; it may be that both views of the data are interesting. tcube = SpectralCube(data=tcubedata, wcs=cube303.wcs, mask=cube303.mask, meta={'unit':'K'}, header=cube303.header, ) tcubeleaf = SpectralCube(data=tcubeleafdata, wcs=cube303.wcs, mask=cube303.mask, meta={'unit':'K'}, header=cube303.header, ) if write: log.info("Writing TemperatureCube") outpath = 'TemperatureCube_DendrogramObjects{0}.fits' tcube.write(hpath(outpath.format(suffix)), overwrite=True) outpath_leaf = 'TemperatureCube_DendrogramObjects{0}_leaves.fits' tcubeleaf.write(hpath(outpath_leaf.format(suffix)), overwrite=True) return catalog, tcube
def doDendro(self, COFITS, minV=3, minPix=8, RMS=0.5, minDelta=3, saveName=None, doSCIMES=False): """ COFITS may not be masedk :param COFITS: :param minV: :param minPix: :param RMS: :param minDelta: :return: """ if saveName == None: saveName = "" saveMark = saveName + "minV{}minP{}minD{}".format( minV, minPix, minDelta) treeFile = saveMark + "_Tree.txt" catFile = saveMark + "_dendroCat.fit" dendroFile = saveMark + "_dendro.fits" trunkCatFile = saveMark + "_dendroCatTrunk.fit" trunkFITS = saveMark + "_TrunkAsign.fits" hdu = fits.open(COFITS)[0] dataCO = hdu.data headCO = hdu.header #mask the data by minValue print "Cmputing dendrogram with min value {}, min pixel number {}".format( minV * RMS, minPix) #data[data<minV*RMS]=0 d = Dendrogram.compute(dataCO, min_value=minV * RMS, verbose=1, min_npix=minPix, min_delta=minDelta * RMS) self.produceAssignFITS(d, COFITS, trunkFITS) d.save_to(dendroFile) #calculate the catalog cat = ppv_catalog(d, self.metadata) cat.write(catFile, overwrite=True) self.writeTreeStructure(d, treeFile) doTree = dendroTree(treeFile) trunks = doTree.getAllTrunk() trunkCat = cat[trunks] print "{} trunks found!".format(len(trunkCat)) trunkCat.write(trunkCatFile, overwrite=True) if doSCIMES: self.doSCIMES(COFITS, dendroFile, catFile, saveMark + "Ve20", inputD=d, criteriaUsed=[self.myVrms], scales=[20]) self.doSCIMES(COFITS, dendroFile, catFile, saveMark + "Ve10", inputD=d, criteriaUsed=[self.myVrms], scales=[10]) self.doSCIMES(COFITS, dendroFile, catFile, saveMark + "Ve15", inputD=d, criteriaUsed=[self.myVrms], scales=[15])
hd = hdu.header # Survey designs sigma = 0.3 #K, noise level ppb = 1.3 #pixels/beam d = Dendrogram.compute(data, min_value=sigma, \ min_delta=2*sigma, min_npix=3*ppb, verbose = 1) #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Generate the catalog #%&%&%&%&%&%&%&%&%&%&%&%&%&% print("Generate a catalog of dendrogram structures") metadata = {} metadata['data_unit'] = u.Jy #This should be Kelvin (not yet implemented)! cat = ppv_catalog(d, metadata) #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Running SCIMES #%&%&%&%&%&%&%&%&%&%&%&%&%&% print("Running SCIMES") dclust = scimes.SpectralCloudstering(d, cat, hd, rms=sigma) #%&%&%&%&%&%&%&%&%&%&%&%&%&% # Image the result #%&%&%&%&%&%&%&%&%&%&%&%&%&% print("Visualize the clustered dendrogram") dclust.showdendro() print("Visualize collapsed maps of the assignment cubes") cubes = [dclust.clusters_asgn,\
def downsampled_demo(data_file, downsample_factor=4, transpose_tuple=(2,0,1), min_value=0.01, min_delta=0.005, min_npix=2000, neighbours=None, resample=False, compute_catalog=True, recenter=True, data_path=data_path, memmap=True): df = downsample_factor tt = transpose_tuple print("\n ** Downsampled Demo Parameters:" "\n downsample_factor={0}" "\n transpose_tuple={1}" "\n min_value={2}" "\n min_delta={3}" "\n min_npix={4}\n".format(downsample_factor, transpose_tuple, min_value, min_delta, min_npix)) beginning = datetime.datetime.now() beginning_string = datetime.datetime.strftime(beginning,"%Y-%m-%d %H:%M:%S") print("\n ** Beginning at: {0}".format(beginning_string)) print "\n** loading data from {0}: ...\n".format(data_path+data_file) datacube, datacube_header = getdata(data_path+data_file, memmap=memmap, header=True) print "\n** transposing {0} and downsampling ({1}) data: ...\n".format(tt, df) datacube_dt, datacube_dt_header = \ downsample_and_transpose_data_and_header(datacube, datacube_header, df, tt, resample=resample, recenter=recenter) datacube_dt_wcs = wcs.wcs.WCS(datacube_dt_header) datacube_dt_wcs.wcs.bounds_check(pix2world=False) beam_size = 1/8 * u.deg frequency = 115 * u.GHz print "\n** computing dendrogram on data with dimensions {} and n_pixels {:,}: ...\n".format(np.shape(datacube_dt), np.size(datacube_dt)) d = astrodendro.Dendrogram.compute( datacube_dt, min_value=min_value, min_delta=min_delta, #these are arbitrary min_npix=min_npix, verbose=True, neighbours=neighbours, wcs=datacube_dt_wcs) v_scale = datacube_dt_header['cdelt3'] v_unit = u.km / u.s l_scale = datacube_dt_header['cdelt1'] b_scale = datacube_dt_header['cdelt2'] metadata = {} metadata['data_unit'] = u.K metadata['spatial_scale'] = b_scale * u.deg metadata['velocity_scale'] = v_scale * v_unit metadata['wavelength'] = frequency # formerly: (c.c / frequency).to('mm') but now compute_flux can handle frequency in spectral equivalency metadata['beam_major'] = beam_size metadata['beam_minor'] = beam_size metadata['vaxis'] = 0 # keep it this way if you think the (post-downsample/transposed) input data is (l, b, v) metadata['wcs'] = datacube_dt_wcs print "\n** computing catalog for {:,} structures: ...\n".format(len(d)) if compute_catalog: catalog = astrodendro.ppv_catalog(d, metadata, verbose=True) else: catalog = None return d, catalog, datacube_dt_header, metadata if catalog['flux'].unit.is_equivalent('Jy'): # Workaround because flux is computed wrong flux = quantify_column(catalog['flux']) area_exact = catalog['area_exact'].unit*catalog['area_exact'].data # average brightness temperature integrated over area_exact flux_kelvin = flux.to('K', equivalencies=u.brightness_temperature(area_exact, frequency)) # flux integrated over area and velocity flux_kelvin_kms_deg2 = flux_kelvin * metadata['velocity_scale'] * area_exact catalog.add_column(astropy.table.Column(data=flux_kelvin_kms_deg2, name='flux_kelvin_kms_deg2')) end = datetime.datetime.now() end_string = datetime.datetime.strftime(end,"%Y-%m-%d %H:%M:%S") print("\n ** Ending at: {0}".format(end_string)) time_elapsed = (end - beginning) print "\n ** Time elapsed: {0}".format(time_elapsed) return d, catalog, datacube_dt_header, metadata