def combine_karachantsev_tables(write_to_disk=False): dname = pth.join(SUP_DATA_DIR, "karachantsev_sfr_catalogue/") sfr = ascii.read(dname+"table3.dat", readme=dname+"ReadMe.txt") dname = pth.join(SUP_DATA_DIR, "karachantsev_list_of_galaxies/") phot = ascii.read(dname+"table2.dat", readme=dname+"readme.txt") dist = ascii.read(dname+"table6.dat", readme=dname+"readme.txt") wanted_col_names = ["Name", "RAh", "RAm", "RAs", "DE-", "DEd", "DEm", "DEs", "TT", "SFRa"] dtypes = ["<U17", "<i4", "<i4", "<f8", "<U1", "<i4", "<i4", "<i4", "<i4", "<f8"] tbl = Table(names=wanted_col_names + ["KLum"] + ["DM"], dtype=dtypes+["<f8", "<f8"]) for row in dist: if row["Name"] in sfr["Name"] and row["Name"] in phot["Name"]: sfr_ii = np.where(sfr["Name"] == row["Name"])[0][0] phot_ii = np.where(phot["Name"] == row["Name"])[0][0] vals = [sfr[sfr_ii][col] for col in wanted_col_names] + [phot[phot_ii]["KLum"]] + [row["DM"]] tbl.add_row(vals) dist_col = Column(name="Dist", data=np.round(10**(0.2*tbl["DM"]+1) * 1e-6, 3)) sfr_col = Column(name="SFR", data=np.round(10**(tbl["SFRa"]+tbl["KLum"]), 3)) tbl.add_columns([dist_col, sfr_col]) if write_to_disk: tbl.write(pth.join(SUP_DATA_DIR, "combined_karachentsev.dat"), format="ascii.fixed_width", overwrite=True) return tbl
def create_basic_table(ra_values, dec_values, magnitudes, location_units, minimum_index=1): """Create astropy table containing the basic catalog info NOTE THAT THIS IS OUTSIDE CLASSES """ basic_table = Table() # Add index, filename, RA, Dec or x, y columns index_col = Column(np.arange(minimum_index, minimum_index + len(ra_values)), name='index') ra_col = Column(ra_values, name='x_or_RA') dec_col = Column(dec_values, name='y_or_Dec') basic_table.add_columns([index_col, ra_col, dec_col]) # Add magnitude columns for key in magnitudes: mag_values = magnitudes[key][1] mag_sys = magnitudes[key][0] mag_column = Column(mag_values, name=key) basic_table.add_column(mag_column) # Add magnitude system and position units as metadata basic_table.meta['comments'] = [location_units, mag_sys] return basic_table
def table(self): #@TODO: make this work for map data ? '''Construct the table of input Measurements, and if the fit has been run, the density, radiation field, and :math:`\chi^2` values :rtype: :class:`astropy.table.Table` ''' v = self._measurements.values() # This only works for astropy version >= 4.1 # and for some reason requirements.txt did not install it # for a test user. #t = Table(self._measurements, # units=[m.unit for m in v] # ) t = Table() cols = [Column(data=d,unit=d.unit) for d in v] t.add_columns(cols=cols, names=[m.id for m in v]) if self._observedratios is not None: v = self._observedratios.values() cols = [Column(data=d,unit=d.unit) for d in v] t.add_columns(cols=cols, names=[m.id for m in v]) if self.radiation_field is not None: t.add_column(col=Column(self.radiation_field, unit=self.radiation_field_unit), name=self.radiation_field.id) if self.density is not None: t.add_column(col=Column(self.density, unit=self.density_unit), name=self.density.id) if self._chisq_min is not None: t.add_column(col=Column(self._chisq_min, unit=None), name="Chi-square") for j in t.columns: t[j].format = '3.2E' return t
def get_r_gtan(self, write=False): """ Populates self.r with radial distance of galaxies from (xc,yc) and self.gtan/gcross with tangential and cross ellipticities Failed shape measurements with g1/g2 = -999999 get filtered out """ xc = self.annular_info['nfw_center'][0] yc = self.annular_info['nfw_center'][1] #wg=(self.mu>1) wg = (self.g1 > -2) print("%d galaxies were actually lensed" % len(wg.nonzero()[0])) g1 = self.g1[wg] g2 = self.g2[wg] x = self.x[wg] y = self.y[wg] self.r = np.sqrt(((x - xc)**2.0) + ((y - yc)**2)) phi = np.arctan2((y - yc), (x - xc)) g = np.sqrt(g1**2 + g2**2) print("## Mean g: %f " % np.mean(g)) self.gtan = -1.0 * (g1 * np.cos(2.0 * phi) + g2 * np.sin(2.0 * phi)) self.gcross = g1 * np.sin(2.0 * phi) + g2 * np.cos(2.0 * phi) if write == True: newtab = Table() newtab.add_columns([x, y, self.r, self.gtan, self.gcross], names=['x', 'y', 'r', 'gcross', 'gtan']) newtab.write('shear_inputs.csv', format='csv', overwrite=True) return
def make_annular_cat(self, hdu_ext=2, filter_g=True, clobber=True): """ make a catalog of positions and shapes ready for annular.c and lensing in general Filter_g : whether to clean failed fits or not """ # actually load the master coadd catalog try: coadd_cat = Table.read(self.coadd_catalog,hdu=hdu_ext) except: print("wrong coadd extension, try again?") pdb.set_trace() # do matching fitvd_matcher=htm.Matcher(16,ra=self.master_fitvd['ra'],dec=self.master_fitvd['dec']) sex_ind,fitvd_ind,dist=fitvd_matcher.match(ra=coadd_cat['ALPHAWIN_J2000'],dec=coadd_cat['DELTAWIN_J2000'],maxmatch=1,radius=5.5E-4) for_annular=Table() for_annular.add_columns([coadd_cat['ALPHAWIN_J2000'][sex_ind],coadd_cat[sex_ind]['DELTAWIN_J2000'],coadd_cat[sex_ind]['X_IMAGE'], coadd_cat[sex_ind]['Y_IMAGE']]) for_annular.add_columns([self.master_fitvd[fitvd_ind]['exp_g'][:,0],self.master_fitvd[fitvd_ind]['exp_g'][:,1]],names=['g1','g2']) self.annular_cat = for_annular # write to file if filter_g==True: for_annular_clean=for_annular[for_annular['g1']>-10.] for_annular_clean.write('fitvd-out.csv',format='ascii.csv',overwrite=clobber) else: for_annular.write('fitvd-out.csv',format='ascii.csv',overwrite=clobber)
def fits_write(self, filename, header=None, overwrite=False): # Generate a binary FITS table from astropy.table import Table, Column """ Writes out a Spectrum, as binary FITS table - wavelength, flux, error, continuum. `overwrite` can be True or False. """ # Overwrite? if os.path.lexists(filename) and not overwrite: c = raw_input('File %s exists - overwrite? (y) or n: ' % filename) if c != '': if c.strip().lower()[0] == 'n': print('returning without writing anything...') return # Generate FITS table and write cwa = Column(data=self.wa,name=str('wa')) cfl = Column(data=self.fl,name=str('fl')) cer = Column(data=self.er,name=str('er')) cco = Column(data=self.co,name=str('co')) sp = Table() #pdb.set_trace() sp.add_columns([cwa,cfl,cer,cco]) sp.write(filename, format='fits',overwrite=True) # Save filename if self.filename is None: self.filename = filename
def create_salp_background(): #Full sample of objects should be same length as background-corrected observed: 11719 nobjs = 1000 e=[] mlisttab = Table.read('teff_spt_table.txt',format='ascii') fullm = np.asarray(mlisttab['Msun'][6:97]) for i in range(len(fullm)): e.append(salp_imf(fullm[i],[-2.35])) e = [a/max(e) for a in e] backg_obj_masses = choices(fullm,weights=e,k=nobjs) mass_col = Column(backg_obj_masses,name='Mass(msol)') alist = np.arange(0.25,30.25,0.25) av_list = np.random.choice(alist,nobjs) #av_list = np.random.normal(10.0,2.0,nobjs) av_col = Column(av_list,name='RAv') dlist = np.arange(10.,400.,0.5) d = np.random.choice(dlist,nobjs) d_col = Column(d,name='Dist(pc)') t = Table() t.add_columns([mass_col,av_col,d_col]) t = add_spt(t,'Msun','Mass(msol)','b') return t
def make_grid(): tab = Table.read('l17_files_qplot.txt',format='ascii') stds = tab['Typ'] #av_dist = Table.read('serpens_extinction_probabilities/ss_av_converted_prob_density_higher_conversion.txt',format='ascii') #av_min = min(av_dist['Av Value']) #av_max = max(av_dist['Av Value']) av_min = 0.0 av_max = 30.25 av_list = np.arange(av_min,av_max+0.25,0.25) model = Table() typ,av,q_cols,H_mags = [],[],[],[] for i in range(len(stds)): print(i) for j in range(len(av_list)): kw,kj,kh,jw,hw,Hap = find_wband_cols(-av_list[j],stds[i],'realfile') q_cols.append(round(jw + (1.85*hw),3)) H_mags.append(round(Hap,3)) typ.append(stds[i]) av.append(av_list[j]) typ_col = Column(typ,name='SpT') av_col = Column(av,name='RAv') H_apcol = Column(H_mags,name='H_app') q_col_col = Column(q_cols,name='Q(cols)') model.add_columns([typ_col,av_col,H_apcol,q_col_col]) model.write('spt_av_grid_0-30_converted_av_higher.txt',format='ascii') return model
def _map_ids(tbl: Table) -> dict[int, int]: gaia_ids = np.zeros(len(tbl), dtype=np.int64) cel_ids = np.zeros(len(tbl), dtype=np.uint32) id_idx = {} for i, ids in enumerate(tbl['ids']): for ident in ids.split('|'): if ident.startswith('HIP'): hip = int(ident[3:].strip()) assert hip not in id_idx id_idx[hip] = i cel_ids[i] = hip elif ident.startswith('TYC'): tycs = [int(t) for t in ident[3:].strip().split('-')] tyc = tycs[0] + tycs[1] * 10000 + tycs[2] * 1000000000 assert tyc not in id_idx id_idx[tyc] = i if cel_ids[i] == 0: cel_ids[i] = tyc elif ident.startswith('Gaia DR2'): gaia_ids[i] = int(ident[8:].strip()) tbl.add_columns([ MaskedColumn(data=gaia_ids, name='gaia'), MaskedColumn(data=cel_ids, name='hip'), ]) return id_idx
def readStack(stack,line): cols=[] nrow=len(stack[line+'_stack_sum']) rows=[] for key in stack.keys(): data=stack[key] if key=='spectral_axis': col=Column(name=key,dtype=np.float, shape=data.shape) elif key == 'bin_type' or key=='bin_unit': col=Column(name=key,dtype='U10') elif key == line+'_stack_profile': col=Column(name=key, dtype=np.float, shape=data.shape[1]) else: col=Column(name=key, dtype=np.float) cols.append(col) tab=Table() tab.add_columns(cols) for i in range(nrow): row=[] for key in stack.keys(): data=stack[key] if key=='spectral_axis' or key=='bin_type' or key=='bin_unit': r=data elif key==line+'_stack_profile': r=data[i,:] else: r=data[i] row.append(r) tab.add_row(row) return tab
def add_spec(data): tab = Table(data) names = ['TEFF_SPEC', 'LOGG_SPEC'] newcols = Table(np.zeros([len(tab), len(names)]) - 9999., names=names) tab.add_columns(newcols.columns.values()) tab['TEFF_SPEC'] = tab['FPARAM'][:, 0] tab['LOGG_SPEC'] = tab['FPARAM'][:, 1] return (tab)
def write_index_file(info_table): """ Write a summary index file in tabular format from ``info_table`` dict, e.g. updated_file baseline_file date dy_acis_i dz_acis_i dy_acis_s dz_acis_s ------------------- ------------------- ----------------- --------- --------- --------- --------- CHARACTERIS_12OCT15 CHARACTERIS_12MAR15 2015:285:01:21:25 -5.00 10.00 15.00 -20.00 """ index_file = os.path.join(opt.data_root, 'characteristics', 'index') if os.path.exists(index_file): index = Table.read(index_file, format='ascii.fixed_width_two_line', guess=False) matching = index['updated_file'] == info_table['updated_file'] index = index[~matching] if np.any(matching): logger.info('WARNING: replacing existing entry for updated_file={}' .format(info_table['updated_file'])) shutil.copy(index_file, index_file + '.bak') else: index = Table(names=['updated_file', 'baseline_file', 'date', 'dy_acis_i', 'dz_acis_i', 'dy_acis_s', 'dz_acis_s'], dtype=['S19', 'S19', 'S17', 'f', 'f', 'f', 'f']) index.add_row(info_table) for colname in ('dy_acis_i', 'dz_acis_i', 'dy_acis_s', 'dz_acis_s'): index[colname].format = '.2f' logger.info('Writing index file {}'.format(index_file)) index.write(index_file, format='ascii.fixed_width_two_line') # Write an HTML index file as a table def self_link(vals): """Turn vals into a list of self-referenced links""" return ['<a href="{0}">{0}</a>'.format(val) for val in vals] # Turn table entries into HTML links to same diffs = self_link(x + '_diff.html' for x in index['updated_file']) jsons = self_link(x + '.json' for x in index['updated_file']) updated_files = self_link(index['updated_file']) baseline_files = self_link(index['baseline_file']) index.remove_columns(['updated_file', 'baseline_file']) index.add_columns([Column(updated_files, name='updated_file'), Column(baseline_files, name='baseline_file'), Column(jsons, name='JSON_info'), Column(diffs, name='diff')], [0, 0, 0, 0]) index_file = index_file + '.html' logger.info('Writing index.html file {}'.format(index_file)) if os.path.exists(index_file): shutil.copy(index_file, index_file + '.bak') index.write(index_file, format='ascii.html') # Hack: undo the HTML escaping that table write does. # TODO: just do the whole thing with jinja template. lines = (re.sub(r'>', '>', line) for line in open(index_file, 'r')) lines = [re.sub(r'<', '<', line) for line in lines] with open(index_file, 'w') as fh: fh.writelines(lines)
def add_gaussian_sources(self, within=(0, 1), cat_gen=pos_uniform, **kwargs): """Add gaussian sources into the map. Parameters ---------- within : tuple of 2 int force the sources within this relative range in the map cat_gen : function (`pos_uniform`|`pos_gridded`|`pos_list`|...) the function used to generate the pixel positions and flux of the sources (see Notes below) **kwargs any keyword arguments to be passed to the `cat_gen` function Notes ----- the `cat_gen` function is used to generate the list of x, y pixel positions and fluxes and must at least support the `shape=None, within=(0, 1), mask=None` arguments. """ shape = self.shape x_mean, y_mean, peak_flux = cat_gen(shape=shape, within=within, mask=self.mask, **kwargs) nsources = x_mean.shape[0] sources = Table(masked=True) sources["amplitude"] = peak_flux.to(self.unit * u.beam) sources["x_mean"] = x_mean sources["y_mean"] = y_mean sources["x_stddev"] = np.ones(nsources) * self.beam.sigma_pix.value sources["y_stddev"] = np.ones(nsources) * self.beam.sigma_pix.value sources["theta"] = np.zeros(nsources) # Crude check to be within the finite part of the map if self.mask is not None: within_coverage = ~self.mask[sources["y_mean"].astype(int), sources["x_mean"].astype(int)] sources = sources[within_coverage] # Gaussian sources... self._data += make_gaussian_sources_image(shape, sources) # Add an ID column sources.add_column(Column(np.arange(len(sources)), name="fake_id"), 0) # Transform pixel to world coordinates a, d = self.wcs.pixel_to_world_values(sources["x_mean"], sources["y_mean"]) sources.add_columns([Column(a * u.deg, name="ra"), Column(d * u.deg, name="dec")]) sources["_ra"] = sources["ra"] sources["_dec"] = sources["dec"] # Remove unnecessary columns sources.remove_columns(["x_mean", "y_mean", "x_stddev", "y_stddev", "theta"]) self.fake_sources = sources
def build_data_array(dirname): colnames = ['PA', 'ELLIP', 'STOP'] buffsize = 100 d = Table() d.add_column(Column(name='SMA', data=arange(0, buffsize*2, 2))) for i, fn in enumerate(glob(dirname+'/*.dat')): t = read_ellipse_output(fn) if i is 0: d.add_columns([Column(name=name, data=zeros([buffsize, 100])-99) for name in colnames]) assert array_equiv(d['SMA'][:len(t)].data, t['SMA'].data), "SMA array do not match" for col in colnames: d[col][:len(t),i] = t[col] return d
def test_quick_add_keys_records_history(self, keyword_arg, file_arg, file_column): ic = ImageFileCollection(self.test_dir.strpath, keywords=['imagetyp']) ic.summary.keep_columns('file') file_list = os.path.join(ic.location, 'files.txt') keyword_list = os.path.join(ic.location, 'keys.txt') full_paths = [ os.path.join(self.test_dir.strpath, fil) for fil in ic.summary['file'] ] print('fill paths: %s' % ' '.join(full_paths)) ic.summary['file'][:] = full_paths ic.summary.remove_column('file') ic.summary.add_column(Column(data=full_paths, name=file_column)) ic.summary.write(file_list, format='ascii') if file_column != 'file': ic.summary.rename_column(file_column, 'file') dumb_keyword = 'munkeez'.upper() dumb_value = 'bananaz' keywords = Column(data=[dumb_keyword], name='Keyword') vals = Column(data=[dumb_value], name='value') keyword_table = Table() keyword_table.add_columns([keywords, vals]) keyword_table.write(keyword_list, format='ascii') args_for = {} args_for['--key-file'] = [keyword_list] args_for['--key-value'] = [dumb_keyword, dumb_value] args_for['--file-list'] = [file_list] args_for[''] = full_paths argslist = [keyword_arg] argslist.extend(args_for[keyword_arg]) if file_arg: argslist.append(file_arg) argslist.extend(args_for[file_arg]) if file_column.lower() != 'file' and file_arg: with pytest.raises(ValueError): quick_add_keys_to_file.main(argslist) return else: quick_add_keys_to_file.main(argslist) # add_keys(file_list=file_list, key_file=keyword_list) for header in ic.headers(): assert (header[dumb_keyword] == dumb_value) history_string = ' '.join(header['history']) assert (dumb_keyword in history_string) assert (dumb_value in history_string)
def output_j19_comparison(sbid, gaskap_table, j19_table, idx_j19, d2d_j19, j19_match, j19_unmatched, title, filename, match_cat=None): print (title, filename) gaskap_targets = gaskap_table[j19_match] j19_targets = j19_table[idx_j19] j19_targets = j19_targets[j19_match] sort_order = gaskap_targets.argsort(['comp_name']) #comp_names = sorted(targets['comp_name']) gaskap_tgt_ordered = gaskap_targets[sort_order] j19_tgt_ordered = j19_targets[sort_order] with open(filename, 'w') as f: output_header(f, title) for rating in 'ABCDEF': mask = gaskap_tgt_ordered['rating']==rating subset = gaskap_tgt_ordered[mask] j19_subset = j19_tgt_ordered[mask] print('Rating {} has {} spectra'.format(rating, len(subset))) output_block_title(f, rating, rating=='A', len(subset)) for idx, gaskap_src in enumerate(subset): gaskap_name = gaskap_src['comp_name'] j19_name = j19_subset[idx]['Source'] output_j19_img(f, gaskap_name, j19_name, rating) # Add a section for missed spectra output_block_title(f, None, False, len(j19_unmatched)) for row in j19_unmatched: gaskap_name = row['closest_gaskap'] j19_name = row['Source'] output_j19_img(f, gaskap_name, j19_name, rating, sep=row['gaskap_sep']) output_footer(f) if match_cat: augmented_table = Table(gaskap_tgt_ordered) close_j19_comp_names = j19_tgt_ordered['Source'] col_closest = Column(name='closest_j19', data=close_j19_comp_names) col_gaskap_ra = Column(name='j19_ra', data=j19_tgt_ordered['ra']*u.deg) col_gaskap_dec = Column(name='j19_dec', data=j19_tgt_ordered['dec']*u.deg) sep_vals = d2d_j19[j19_match] sep_vals_sorted = sep_vals[sort_order] col_sep = Column(name='j19_sep', data=sep_vals_sorted.to(u.arcsec)) augmented_table.add_columns([col_closest, col_gaskap_ra, col_gaskap_dec, col_sep]) #print (augmented_table) j19_match_vo_table = from_table(augmented_table) writeto(j19_match_vo_table, match_cat)
def test_quick_add_keys_records_history(self, keyword_arg, file_arg, file_column): ic = ImageFileCollection(self.test_dir.strpath, keywords=['imagetyp']) ic.summary_info.keep_columns('file') file_list = os.path.join(ic.location, 'files.txt') keyword_list = os.path.join(ic.location, 'keys.txt') full_paths = [os.path.join(self.test_dir.strpath, fil) for fil in ic.summary_info['file']] print('fill paths: %s' % ' '.join(full_paths)) ic.summary_info['file'][:] = full_paths ic.summary_info.remove_column('file') ic.summary_info.add_column(Column(data=full_paths, name=file_column)) ic.summary_info.write(file_list, format='ascii') if file_column != 'file': ic.summary_info.rename_column(file_column, 'file') dumb_keyword = 'munkeez'.upper() dumb_value = 'bananaz' keywords = Column(data=[dumb_keyword], name='Keyword') vals = Column(data=[dumb_value], name='value') keyword_table = Table() keyword_table.add_columns([keywords, vals]) keyword_table.write(keyword_list, format='ascii') args_for = {} args_for['--key-file'] = [keyword_list] args_for['--key-value'] = [dumb_keyword, dumb_value] args_for['--file-list'] = [file_list] args_for[''] = full_paths argslist = [keyword_arg] argslist.extend(args_for[keyword_arg]) if file_arg: argslist.append(file_arg) argslist.extend(args_for[file_arg]) if file_column.lower() != 'file' and file_arg: with pytest.raises(ValueError): quick_add_keys_to_file.main(argslist) return else: quick_add_keys_to_file.main(argslist) # add_keys(file_list=file_list, key_file=keyword_list) for header in ic.headers(): assert (header[dumb_keyword] == dumb_value) history_string = ' '.join(header['history']) assert (dumb_keyword in history_string) assert (dumb_value in history_string)
def make_output_table(target_list, wds_id_list): """Construct an astropy Table from the target ID list and the filtered WDS component list """ output_table = Table() target_column = Column( data=target_list, description='Original user-specified target identifier', name='Star', format='{}') wds_column = Column(data=wds_id_list, description='WDS identifiers for selected components', name='WDS', format='{}') output_table.add_columns(cols=[target_column, wds_column]) return output_table
def make_flux_table(source_tbl, src, wave_min, wave_max, area): fluxes = np.zeros(len(src.spectra)) ref_set = list(set(source_tbl["ref"])) flux_set = src.photons_in_range(wave_min, wave_max, area, ref_set) fluxes[ref_set] = flux_set ref = source_tbl["ref"] weight = source_tbl["weight"] flux_col = Column(name="flux", data=fluxes[ref] * weight) x_col = source_tbl["x"] y_col = source_tbl["y"] tbl = Table() tbl.add_columns([x_col, y_col, flux_col]) return tbl
class Parser(object): """Read in a MOP formatted file""" def __init__(self, expnum, ccd, extension, type='p', prefix=None): """does nothing""" self.expnum = expnum self.ccd = ccd self.extension = extension self.type = type self.prefix = prefix self.fobj = None self.keywords = [] self.values = [] self.formats = {} self.header = {} self.cdata ={} def parse(self): """read in a file and return a MOPFile object.""" self.filename = storage.get_file(self.expnum, self.ccd, ext=self.extension, version=self.type, prefix=self.prefix) self.fobj = open(self.filename,'r') lines = self.fobj.read().split('\n') self.header = HeaderParser(self.extension).parser(lines) if 'matt' in self.extension: usecols=[0,1,2,3,4] data = numpy.genfromtxt(self.filename, usecols=usecols) else: data = numpy.genfromtxt(self.filename) self.data = Table(data, names=self.header.column_names[0:data.shape[1]]) ast_header = storage.get_astheader(self.expnum, self.ccd) self.wcs = wcs.WCS(ast_header) flip_these_extensions = range(1,19) flip_these_extensions.append(37) flip_these_extensions.append(38) if self.ccd + 1 in flip_these_extensions: self.data['X'] = float(self.header.keywords['NAX1'][0])-self.data['X'] + 1 self.data['Y'] = float(self.header.keywords['NAX2'][0])-self.data['Y'] + 1 ra, dec = self.wcs.xy2sky(self.data['X'], self.data['Y'], usepv=True) self.data.add_columns([Column(ra, name='RA_J2000'), Column(dec, name='DE_J2000')]) return self
def _write_ref_ISM_table(): """ Write a reference table enabling faster I/O for ISM-related lists For developer use only. Note that after running this, you need to manually copy the table produced to linetools/data/lines/ISM_table.fits inside the github repository, and then check it in. """ from linetools.lists.linelist import LineList ism = LineList('ISM', use_ISM_table=False) strong = LineList('Strong', use_ISM_table=False) euv = LineList('EUV', use_ISM_table=False) hi = LineList('HI', use_ISM_table=False) # need a Table, not QTable to write tab = Table(ism._data.copy()) tab.sort(('wrest')) # Using np.in1d doesn't work for some reason. Do it the long way cond = [] for table in (strong, euv, hi): igood = [] for row in Table(table._data): ind = tab['wrest'].searchsorted(row['wrest']) dw = abs(tab['wrest'][ind] - row['wrest']) if abs(tab['wrest'][ind+1] - row['wrest']) < dw: ind = ind + 1 rism = tab[ind] # check this is the right row. if all(row[k] == rism[k] for k in hi._data.colnames if not hasattr(row[k], 'mask') or not row[k].mask): igood.append(ind) else: raise RuntimeError('No match found!') cond.append(np.zeros(len(tab), dtype=bool)) cond[-1][np.array(igood)] = True col1 = Column(cond[0], name='is_Strong') col2 = Column(cond[1], name='is_EUV') col3 = Column(cond[2], name='is_HI') tab.add_columns([col1, col2, col3]) tab.write('ISM_table.fits', overwrite=True)
def _write_ref_ISM_table(): """ Write a reference table enabling faster I/O for ISM-related lists For developer use only. Note that after running this, you need to manually copy the table produced to linetools/data/lines/ISM_table.fits inside the github repository, and then check it in. """ from linetools.lists.linelist import LineList ism = LineList('ISM', use_ISM_table=False) strong = LineList('Strong', use_ISM_table=False) euv = LineList('EUV', use_ISM_table=False) hi = LineList('HI', use_ISM_table=False) # need a Table, not QTable to write tab = Table(ism._data.copy()) tab.sort(('wrest')) # Using np.in1d doesn't work for some reason. Do it the long way cond = [] for table in (strong, euv, hi): igood = [] for row in Table(table._data): ind = tab['wrest'].searchsorted(row['wrest']) dw = abs(tab['wrest'][ind] - row['wrest']) if abs(tab['wrest'][ind + 1] - row['wrest']) < dw: ind = ind + 1 rism = tab[ind] # check this is the right row. if all(row[k] == rism[k] for k in hi._data.colnames if not hasattr(row[k], 'mask') or not row[k].mask): igood.append(ind) else: raise RuntimeError('No match found!') cond.append(np.zeros(len(tab), dtype=bool)) cond[-1][np.array(igood)] = True col1 = Column(cond[0], name='is_Strong') col2 = Column(cond[1], name='is_EUV') col3 = Column(cond[2], name='is_HI') tab.add_columns([col1, col2, col3]) tab.write('ISM_table.fits', overwrite=True)
def list_filters(): """Print the list of filters in the pcigale database. """ with Database() as base: filters = {name: base.get_filter(name) for name in base.get_filter_names()} name = Column(data=[filters[f].name for f in filters], name='Name') description = Column(data=[filters[f].description for f in filters], name='Description') wl = Column(data=[filters[f].effective_wavelength for f in filters], name='Effective Wavelength', unit=u.nm, format='%d') filter_type = Column(data=[filters[f].trans_type for f in filters], name='Type') samples = Column(data=[filters[f].trans_table[0].size for f in filters], name="Points") t = Table() t.add_columns([name, description, wl, filter_type, samples]) t.sort(['Effective Wavelength']) t.pprint(max_lines=-1, max_width=-1)
def tableFill(dam, ra, dec, appender, nme): t = Table(None) Am = Column(name='Arcminute') North = Column(name='North') East = Column(name='East') South = Column(name='South') West = Column(name='West') t.add_columns([Am, North, East, South, West]) tA_v = [] curVal = [None] * 4 #n = 0, e = 1, s = 2, w = 3 coord = [None] * 4 #n = 0, e = 1, s = 2, w = 3 #get values for each arcminute for j in range(0, dam + 1): fourCoord(j, ra, dec, coord) t.add_row() t[j][0] = j for i in range(0, 4): C = coordinates.SkyCoord(coord[i]) table = IrsaDust.get_extinction_table(C.fk5, show_progress=False) curVal[i] = (table['A_SandF'][2]) t[j][i + 1] = curVal[i] curVal = curVal[:] tA_v.append(curVal) t.add_row() for i in range( 0, 5): #this adds a blank line to the table to separate queries t[j + 1][i] = None n = [nme] namesTable = Table([n], names=('n')) final_name = namesTable.to_pandas() final_vals = t.to_pandas() from pandas import ExcelWriter with open('A_v Values.csv', appender) as f: final_name.to_csv(f, header=False, index=False) appender = 'a' with open('A_v Values.csv', appender) as f: final_vals.to_csv(f, header=True, index=False, sep=',') return (tA_v) #gets the data from IRSA database and stores A_v in array
def get_ellipticities(incat,all_ind,truth_cat,truth_ind): e1= (incat['X2_IMAGE']-incat['Y2_IMAGE']) / (incat['X2_IMAGE']+incat['Y2_IMAGE']) e2=-2.0*incat['XY_IMAGE']/(incat['X2_IMAGE']+incat['Y2_IMAGE']) ellip=np.sqrt(e1**2+e2**2) e1=e1[all_ind] e1.colname='$e_1$' e2=e2[all_ind] e2.colname='$e_2$' ellip=np.sqrt(e1**2+e2**2) ellip.colname='$e$' ra = incat['ALPHAWIN_J2000'][all_ind]; dec = incat['DELTAWIN_J2000'][all_ind] x = incat['X_IMAGE'][all_ind]; y=incat['Y_IMAGE'][all_ind] fwhm = incat['FWHM_IMAGE'][all_ind] mag = incat['MAG_AUTO'][all_ind] flux = incat['FLUX_AUTO'][all_ind] flux_rad = incat['FLUX_RADIUS'][all_ind] # copy over relevant quantities from truth catalog as well! truth_fwhm = truth_cat[truth_ind]['mom_size']*2.355 truth_flux = truth_cat[truth_ind]['flux'] truth_g1_meas = truth_cat[truth_ind]['g1_meas'] truth_g2_meas = truth_cat[truth_ind]['g2_meas'] z =truth_cat[truth_ind]['redshift'] # also add fwhm and mag, useful quantities for stars, ellip_tab=Table() ellip_tab.add_columns([ra, dec, x, y, e1,e2,ellip,truth_g1_meas,truth_g2_meas, fwhm,flux_rad, mag,flux,truth_fwhm, truth_flux,z], names=['ra','dec','x', 'y', 'e1','e2','e','g1_meas','g2_meas', 'fwhm','flux_rad','mag','flux','truth_fwhm','truth_flux','redshift']) return ellip_tab
def save(self, save_path: Path = '.'): save_path = Path(save_path) npar = len(self.ps) if self.de: de = xa.DataArray(self.de.population, dims='pvector parameter'.split(), coords={'parameter': self.ps.names}) else: de = None if self.sampler is not None: mc = xa.DataArray(self.sampler.chain, dims='pvector step parameter'.split(), coords={'parameter': self.ps.names}, attrs={'ndim': npar, 'npop': self.sampler.nwalkers}) else: mc = None ds = xa.Dataset(data_vars={'de_population': de, 'mcmc_samples': mc}, attrs={'created': strftime('%Y-%m-%d %H:%M:%S'), 'name': self.name}) ds.to_netcdf(save_path.joinpath(f'{self.name}.nc')) try: if self.sampler is not None: fname = save_path / f'{self.name}.fits' chains = self.sampler.chain nchains = chains.shape[0] nsteps = chains.shape[1] idch = repeat(arange(nchains), nsteps) idst = tile(arange(nsteps), nchains) flc = chains.reshape([-1, chains.shape[2]]) tb1 = Table([idch, idst], names=['chain', 'step']) tb1.add_columns(flc.T, names=self.ps.names) tb2 = Table([idch, idst], names=['chain', 'step']) tb2.add_column(self.sampler.lnprobability.ravel(), name='lnp') tbhdu1 = pf.BinTableHDU(tb1, name='posterior') tbhdu2 = pf.BinTableHDU(tb2, name='sample_stats') hdul = pf.HDUList([pf.PrimaryHDU(), tbhdu1, tbhdu2]) hdul.writeto(fname, overwrite=True) except ValueError: print('Could not save the samples in fits format.')
def match_planted(fk_candidate_observations, match_filename, bright_limit=BRIGHT_LIMIT, object_planted=OBJECT_PLANTED, minimum_bright_detections=MINIMUM_BRIGHT_DETECTIONS, bright_fraction=MINIMUM_BRIGHT_FRACTION): """ Using the fk_candidate_observations as input get the Object.planted file from VOSpace and match planted sources with found sources. The Object.planted list is pulled from VOSpace based on the standard file-layout and name of the first exposure as read from the .astrom file. :param fk_candidate_observations: name of the fk*reals.astrom file to check against Object.planted :param match_filename: a file that will contain a list of all planted sources and the matched found source @param minimum_bright_detections: if there are too few bright detections we raise an error. """ found_pos = [] detections = fk_candidate_observations.get_sources() for detection in detections: reading = detection.get_reading(0) # create a list of positions, to be used later by match_lists found_pos.append([reading.x, reading.y]) # Now get the Object.planted file, either from the local FS or from VOSpace. objects_planted_uri = object_planted if not os.access(objects_planted_uri, os.F_OK): objects_planted_uri = fk_candidate_observations.observations[ 0].get_object_planted_uri() try: lines = storage.open_vos_or_local(objects_planted_uri) lines = lines.read().decode('utf-8') except Exception as ex: logging.critical(f'{ex}') print(lines) raise ex # we are changing the format of the Object.planted header to be compatible with astropy.io.ascii but # there are some old Object.planted files out there so we do these string/replace calls to reset those. new_lines = lines.replace("pix rate", "pix_rate") new_lines = new_lines.replace("""''/h rate""", "sky_rate") planted_objects_table = ascii.read(new_lines, header_start=-1, data_start=0) planted_objects_table.meta = None # The match_list method expects a list that contains a position, not an x and a y vector, so we transpose. planted_pos = numpy.transpose( [planted_objects_table['x'].data, planted_objects_table['y'].data]) # match_idx is an order list. The list is in the order of the first list of positions and each entry # is the index of the matching position from the second list. (match_idx, match_fnd) = util.match_lists(numpy.array(planted_pos), numpy.array(found_pos)) assert isinstance(match_idx, numpy.ma.MaskedArray) assert isinstance(match_fnd, numpy.ma.MaskedArray) false_positives_table = Table() # Once we've matched the two lists we'll need some new columns to store the information in. # these are masked columns so that object.planted entries that have no detected match are left 'blank'. new_columns = [ MaskedColumn(name="measure_x", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_y", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_rate", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_angle", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_mag1", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_merr1", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_mag2", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_merr2", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_mag3", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_merr3", length=len(planted_objects_table), mask=True) ] planted_objects_table.add_columns(new_columns) tlength = 0 new_columns = [ MaskedColumn(name="measure_x", length=tlength, mask=True), MaskedColumn(name="measure_y", length=tlength, mask=True), MaskedColumn(name="measure_rate", length=0, mask=True), MaskedColumn(name="measure_angle", length=0, mask=True), MaskedColumn(name="measure_mag1", length=0, mask=True), MaskedColumn(name="measure_merr1", length=0, mask=True), MaskedColumn(name="measure_mag2", length=0, mask=True), MaskedColumn(name="measure_merr2", length=0, mask=True), MaskedColumn(name="measure_mag3", length=tlength, mask=True), MaskedColumn(name="measure_merr3", length=tlength, mask=True) ] false_positives_table.add_columns(new_columns) # We do some 'checks' on the Object.planted match to diagnose pipeline issues. Those checks are made using just # those planted sources we should have detected. bright = planted_objects_table['mag'] < bright_limit n_bright_planted = numpy.count_nonzero( planted_objects_table['mag'][bright]) measures = [] idxs = [] for idx in range(len(match_idx)): # The match_idx value is False if nothing was found. if not match_idx.mask[idx]: # Each 'source' has multiple 'readings' measures.append(detections[match_idx[idx]].get_readings()) idxs.append(idx) observations = measure_mags(measures) for oidx in range(len(measures)): idx = idxs[oidx] readings = measures[oidx] start_jd = Time(readings[0].obs.header['MJD_OBS_CENTER'], format='mpc', scale='utc').jd end_jd = Time(readings[-1].obs.header['MJD_OBS_CENTER'], format='mpc', scale='utc').jd rate = math.sqrt( (readings[-1].x - readings[0].x)**2 + (readings[-1].y - readings[0].y)**2) / (24 * (end_jd - start_jd)) rate = int(rate * 100) / 100.0 angle = math.degrees( math.atan2(readings[-1].y - readings[0].y, readings[-1].x - readings[0].x)) angle = int(angle * 100) / 100.0 planted_objects_table[idx]['measure_rate'] = rate planted_objects_table[idx]['measure_angle'] = angle planted_objects_table[idx]['measure_x'] = observations[ readings[0].obs]['mags']["XCENTER"][oidx] planted_objects_table[idx]['measure_y'] = observations[ readings[0].obs]['mags']["YCENTER"][oidx] for ridx in range(len(readings)): reading = readings[ridx] mags = observations[reading.obs]['mags'] planted_objects_table[idx]['measure_mag{}'.format( ridx + 1)] = mags["MAG"][oidx] planted_objects_table[idx]['measure_merr{}'.format( ridx + 1)] = mags["MERR"][oidx] # for idx in range(len(match_fnd)): # if match_fnd.mask[idx]: # measures = detections[idx].get_readings() # false_positives_table.add_row() # false_positives_table[-1] = measure_mags(measures, false_positives_table[-1]) # Count an object as detected if it has a measured magnitude in the first frame of the triplet. n_bright_found = numpy.count_nonzero( planted_objects_table['measure_mag1'][bright]) # Also compute the offset and standard deviation of the measured magnitude from that planted ones. offset = numpy.mean(planted_objects_table['mag'][bright] - planted_objects_table['measure_mag1'][bright]) try: offset = "{:5.2f}".format(offset) except: offset = "indef" std = numpy.std(planted_objects_table['mag'][bright] - planted_objects_table['measure_mag1'][bright]) try: std = "{:5.2f}".format(std) except: std = "indef" if os.access(match_filename, os.R_OK): fout = open(match_filename, 'a') else: fout = open(match_filename, 'w') fout.write("#K {:10s} {:10s}\n".format("EXPNUM", "FWHM")) for measure in detections[0].get_readings(): fout.write('#V {:10s} {:10s}\n'.format(measure.obs.header['EXPNUM'], measure.obs.header['FWHM'])) fout.write("#K ") for keyword in ["RMIN", "RMAX", "ANGLE", "AWIDTH"]: fout.write("{:10s} ".format(keyword)) fout.write("\n") fout.write("#V ") for keyword in ["RMIN", "RMAX", "ANGLE", "AWIDTH"]: fout.write("{:10s} ".format( fk_candidate_observations.sys_header[keyword])) fout.write("\n") fout.write("#K ") for keyword in ["NBRIGHT", "NFOUND", "OFFSET", "STDEV"]: fout.write("{:10s} ".format(keyword)) fout.write("\n") fout.write("#V {:<10} {:<10} {:<10} {:<10}\n".format( n_bright_planted, n_bright_found, offset, std)) try: writer = ascii.FixedWidth # add a hash to the start of line that will have header columns: for JMP fout.write("# ") fout.flush() ascii.write(planted_objects_table, output=fout, Writer=writer, delimiter=None) if len(false_positives_table) > 0: with open(match_filename + ".fp", 'a') as fpout: fpout.write("#") ascii.write(false_positives_table, output=fpout, Writer=writer, delimiter=None) except Exception as e: logging.error(str(e)) raise e finally: fout.close() # Some simple checks to report a failure how we're doing. if n_bright_planted < minimum_bright_detections: raise RuntimeError(1, "Too few bright objects planted.") if n_bright_found / float(n_bright_planted) < bright_fraction: raise RuntimeError(2, "Too few bright objects found.") return "{} {} {} {}".format(n_bright_planted, n_bright_found, offset, std)
def main(): epilog='''The SCUSS pipeline is designed for images taken by the Bok Telescope. It make photometry for both SDSS detected objects and Bok-only sources. The photometric methods include automatic aperture, circular aperture, PSF and model photometry. The model photometry can use SDSS shape parameters for force photometry or use self-guessed parameters to refit. Two models are Dev (1/4) and Exp laws. Gain and read-out noise is read from the FITS header. if stacked image, only weight image should provided. if single image, only flag should be provided. if weight, flag, psfex, sdss is empty, the program will automatically find or calculate them''' parser = argparse.ArgumentParser(description='SCUSS Photometric Pipeline.', epilog=epilog,fromfile_prefix_chars='@') parser.add_argument('fits', metavar='FITS', type=str, nargs='*', help='Bok calibrated images to do photometry. @filelist to read from file') parser.add_argument('--stacked', action='store_true', help='indicate FITS are stacked image') parser.add_argument('-f','--flag',type=str,nargs='*',default=[], help='flag image (bad pixels). @filelist to read from file') parser.add_argument('-w','--weight',type=str,nargs='*',default=[], help='weight image (exposure times for mosaics). @filelist to read from file') parser.add_argument('--psfex',type=str,nargs='*',metavar='psfex',default=[], help='PSF file derived by PSFEx') parser.add_argument('--phot-sdss', action='store_true', help='photometry with SDSS detections with automatic parameters') parser.add_argument('--force-sdss', action='store_true', help='force SDSS shape paramters on Bok image.') parser.add_argument('--force-filter',type=str,default=None, help='Force Photometry with SDSS filter shape parameters, default None (use the same filter as the image') parser.add_argument('--save-sky', action='store_true', help='save the 2d sky background and rms map') parser.add_argument('--save-segment', action='store_true', help='save the 2d sky background and rms map') parser.add_argument('--save-model', action='store_true', help='save the fitted model and residual image') parser.add_argument('--plot', action='store_true', help='plot all kinds of corrections in the catalog') parser.add_argument('--nthread',type=int,default=1, help='number of threads to parallel processing, default 1') parser.add_argument('--version', action='version',version='%(prog)s 1.0') # parse arguments args = parser.parse_args() ###CCD parameters saturate=30000.0 gain=1.375 rdnoise=8.0 ## detecting and analysis detect_nsigma=1.0 aly_nsigma=1.5 conv_sigma=1.2 # not using seeing FWHM/2.35 if seeing too large many faint objects cannot be found! we use 1.2 for all bok image remove_nthresh=64 sky_grid=216 # photometry apertures=[3,4,5,6,8,10,13,16,20,25,30,40] sky_annuli=[40,20] sdssmag_limit=23.5 edge_nfwhm=3.0 # n*fwhm of seeing to regarded as edge pixels min_dist=3.3 # min distance between scuss and sdss pixels ab_limit=[1.0,7.0] re_limit=[1e-10,5.0] nmax_fit=7 nmax_force_fit=10 nradius=10 # n*effective radius to create models and residual map remove_big_width=500 # remove big objects # get list of fits, psfs, flags or weights fitslist=args.fits flaglist=args.flag weightlist=args.weight psflist=args.psfex nfits=len(fitslist) nflag=len(flaglist) nweight=len(weightlist) npsf=len(psflist) # print help if no parameters provided if len(args.fits) == 0: parser.print_help() # verify whether file numbers are equal if npsf !=0 and npsf != nfits: print('The number of PSF files should be same as the FITS image') sys.exit(-1) stacked=args.stacked if stacked: if nweight !=0 and nfits != nweight: print('The number of weight images should be same as the Stacked images') sys.exit(-1) else: if nflag !=0 and nflag != nfits: print('The number of flag images should be same as the single-epoch images') sys.exit(-1) print(datetime.datetime.now()) for i,ifits in enumerate(fitslist): start=time.time() print('phot '+ifits+' ...') if not os.path.isfile(ifits): continue # get FITS image fitsname,_=os.path.splitext(ifits) fitspath,name=os.path.split(ifits) data=fits.getdata(ifits) head=fits.getheader(ifits) if 'CCD_NO:' in head: head.rename_keyword('CCD_NO:','CCD_NO') filt=head['filter'] if filt == 'bokr': filt='r' if not stacked: datestr=head['date-obs'] timestr=head['time-obs'] tt=Time(datestr+' '+timestr,format='iso',scale='utc') mjd=tt.mjd seeing = None if 'seeing' in head: seeing=head['seeing']/0.455 # seeing in pixels edge_pixels=np.ceil(edge_nfwhm*seeing).astype('int') #get weight, flag, mask mask_sat=data > saturate if stacked: if nweight ==0: weight=fits.getdata(os.path.join(fitspath,'b_'+name)) else: weight=fits.getdata(weightlist[i]) if mask_sat.any(): weight[mask_sat]+=1 flag=(weight == 0).astype('int8') mask=flag.copy() tmpphot=pyphot.phot.PyPhot(weight,mask=mask) tmpphot.get_sky_single(method="median",nsample=10000,sigma=3.0,iters=0,show=False) wmed=tmpphot.sky gain=gain*wmed rdnoise=rdnoise/np.sqrt(wmed) print('weight file: '+weightlist[i]) else: if nflag ==0: ccd=str(head['CCD_NO']) date=head['date-obs'] flagfile=get_flagfile(date,ccd) flag=fits.getdata(flagfile) else: flag=fits.getdata(flaglist[i]) flag=flag != 0 mask=flag.copy() weight=None print('flag file: '+flagfile) flag[mask_sat]=np.bitwise_or(flag[mask_sat],2) if edge_pixels > 0: flag[:edge_pixels,:]=np.bitwise_or(flag[:edge_pixels,:],4) flag[-edge_pixels:,:]=np.bitwise_or(flag[-edge_pixels:,:],4) flag[:,:edge_pixels]=np.bitwise_or(flag[:,:edge_pixels],4) flag[:,-edge_pixels:]=np.bitwise_or(flag[:,-edge_pixels:],4) psffile=fitsname+'.psf' # get psf file if npsf == 0: if not os.path.isfile(psffile): get_psfex(ifits,stacked=stacked) else: if os.path.isfile(psflist[i]): psffile=psflist[i] else: get_psfex(ifits,stacked=stacked) # get sdss catalog sdsscat=fitsname+'-sdss.fits' #if args.add_sdsscat or args.force_sdss or (not args.add_sdsscat and not args.force_sdss and args.phot_sdss) : ra,dec=fits_center(head,stacked=stacked) ra=ra[0] ; dec=dec[0] print('fits center: {}, {}'.format(ra,dec)) if not os.path.isfile(sdsscat): d=DownSDSSCat() d.get_allmag_cat(ra=ra,dec=dec,width=0.6,psfmag_limit=sdssmag_limit,filename=sdsscat,fmt='fits') if os.path.isfile(sdsscat): print('sdss catalog: '+sdsscat) else: print('sdss catalog isnot found!') # processing sdss catalogs sdss_params=None if os.path.isfile(sdsscat) and os.path.getsize(sdsscat)!=0: sdss=fits.getdata(sdsscat,1) sdss_params,sdss=params_from_sdss(sdss,head,force_filter=args.force_filter) # get psfex PSF, initialize photometry psf=pyphot.psfex.PSFEx() psf.from_fits(psffile) phot=pyphot.phot.PyPhot(data,uncertainty=None,mask=mask,flags=flag,weight=weight,saturate=saturate,gain=gain,rdnoise=rdnoise,seeing=seeing,nthread=args.nthread) if os.path.isfile(fitsname+'-sky.fits') and os.path.isfile(fitsname+'-sky_rms.fits'): print('get sky and rms map from saved fits') phot.sky=fits.getdata(fitsname+'-sky.fits') phot.sky_rms=fits.getdata(fitsname+'-sky_rms.fits') else: phot.get_sky_map(nsample=30000,grid_size=sky_grid) if stacked: #phot.get_sky_single(nsample=100000) phot.weighted_gain_rdnoise() #else: # phot.get_sky_map(nsample=30000,grid_size=216) # get flux error map if seeing is None: phot.get_seeing() phot.update_data_uncertainty(poisson=True) if args.save_sky: print('save sky and rms maps') fits.writeto(fitsname+'-sky.fits',phot.sky,header=head,clobber=True,output_verify='fix+warn') fits.writeto(fitsname+'-sky_rms.fits',phot.sky_rms,header=head,clobber=True,output_verify='fix+warn') # detecting source in bok image tmp_prop=phot.detect_objects(detect_nsigma=detect_nsigma,conv_sigma=conv_sigma,analysis_nsigma=aly_nsigma,remove_nthresh=remove_nthresh,sig_flag=False,remove_after_segment=True,remove_big_width=remove_big_width) coord_scuss=np.array((tmp_prop['x_window'],tmp_prop['y_window'])).transpose() nscuss=len(coord_scuss) # get SDSS coordinates if also fitting them coord=coord_scuss if sdss_params is not None: coord_sdss=np.array((sdss_params['x'],sdss_params['y'])).transpose() index_scuss,index_sdss=closest_match(coord_scuss,coord_sdss,min_dist=min_dist) index_scuss=np.array(index_scuss) indsdss_match=np.array([idx[0] for idx in index_sdss]) indsdss_nomatch=np.setdiff1d(np.arange(len(coord_sdss)),indsdss_match) nuniq_sdss=len(indsdss_nomatch) if args.phot_sdss and nuniq_sdss > 0: coord=np.vstack((coord_scuss,coord_sdss[indsdss_nomatch])) is_sdss=Table.Column(np.zeros(len(coord)).astype('int'),'is_sdss') is_sdss[index_scuss]=1 if nuniq_sdss > 0: is_sdss[nscuss:]=1 else: is_sdss=Table.Column(np.zeros(len(coord)).astype('int'),'is_sdss') # calculate MJD for single-epoch images if not stacked: mjd_col=Table.Column(np.ones(len(coord))*mjd,'bok_mjd') # Self-photometry with either Bok objects or toghether with SDSS objects print('Photometry with shape parameters of the image itself ...') if sdss_params is not None and args.phot_sdss: det_sc=phot.detect_objects(detect_nsigma=detect_nsigma,conv_sigma=conv_sigma,analysis_nsigma=aly_nsigma,remove_nthresh=remove_nthresh,sig_flag=False,remove_after_segment=True,coordinates=coord,remove_big_width=remove_big_width) else: det_sc=tmp_prop if det_sc == False: continue if args.save_segment: fits.writeto(fitsname+'-segment.fits',phot.seglabels,header=head,clobber=True,output_verify='fix+warn') # self model photometry param_det=phot.parameter_guess(det_sc,xname='x_window',yname='y_window',remove_seeing=False) mod_sc=phot.model_photometry(psf,param_guess=param_det,pos_limit=True,ab_limit=ab_limit,re_limit=re_limit,nmax_fit=nmax_fit,remove_seeing=False) if args.save_model: model,residual=phot.create_model_residual(mod_sc,psf,nradius=nradius,add_seeing=False) fits.writeto(fitsname+'-model.fits',model,header=head,clobber=True,output_verify='fix+warn') fits.writeto(fitsname+'-residual.fits',residual,header=head,clobber=True,output_verify='fix+warn') #mod_sc_bak=mod_sc.copy() det_sc=Table(det_sc) mod_sc=Table(mod_sc) rename_table(det_sc,prefix='bok_',postfix='_'+filt) rename_table(mod_sc,prefix='bok_',postfix='_'+filt) del_cols=['x_min','x_max','y_min','y_max','cxx','cxx_error','cyy','cyy_error','cxy','cxy_error'] for col in del_cols: det_sc.remove_column('bok_'+col+'_'+filt) del_cols=['id','x_in','y_in','x_psf','x_psf_error','y_psf','y_psf_error','x_dev','x_dev_error','y_dev','y_dev_error','x_exp','x_exp_error','y_exp','y_exp_error'] for col in del_cols: mod_sc.remove_column('bok_'+col+'_'+filt) fluxadu=det_sc fluxadu.add_columns(mod_sc.columns.values()) index_det=det_sc['bok_id_'+filt] if not stacked: fluxadu.add_column(mjd_col[index_det]) fluxadu.add_column(is_sdss[index_det]) # add SDSS information bok_id='bok_id_'+filt if sdss_params is not None: sdss=Table(sdss) rename_table(sdss,prefix='sdss_') sdss.add_columns([table.Column(sdss_params['x'],name='sdss_bokx_'+filt), table.Column(sdss_params['y'],name='sdss_boky_'+filt)],indexes=[0,0]) sdss[bok_id]=-1 index1=np.array([ind for ind in np.arange(len(indsdss_match)) if index_scuss[ind] in index_det]) if len(index1) > 0: sdss[bok_id][indsdss_match[index1]]=index_scuss[index1] if nuniq_sdss > 0: if args.phot_sdss: index2=np.array([ind for ind in np.arange(nuniq_sdss) if ind+nscuss in index_det]) if len(index2)>0: sdss[bok_id][indsdss_nomatch[index2]]=index2+nscuss else: for ii in range(nuniq_sdss): fluxadu.add_row() nscuss_det=len(index_det) fluxadu[bok_id][nscuss_det:]=nscuss+np.arange(nuniq_sdss) fluxadu['is_sdss'][nscuss_det:]=1 fluxadu=table.join(fluxadu,sdss,join_type='left') fluxadu=fluxadu.filled(fill_value=0) # force photometry if sdss_params is not None and args.force_sdss: print('Force Photometry with SDSS shape parameters ...') det_sd=phot.detect_objects(detect_nsigma=detect_nsigma,conv_sigma=conv_sigma,analysis_nsigma=aly_nsigma,remove_nthresh=remove_nthresh,sig_flag=False,remove_after_segment=True,coordinates=coord_sdss,only_position=True,remove_big_width=remove_big_width) if det_sd != False: sdss_params=sdss_params[det_sd['id']] sdss_params['x']=det_sd['x_window'] sdss_params['y']=det_sd['y_window'] mod_sd=phot.model_photometry(psf,param_guess=sdss_params,pos_limit=True,ab_limit=True,re_limit=True,theta_limit=True,nmax_fit=nmax_force_fit,remove_seeing=True,models=['psf','dev','exp']) if args.save_model: fmodel,fresidual=phot.create_model_residual(mod_sd,psf,nradius=nradius,add_seeing=True) fits.writeto(fitsname+'-fmodel.fits',fmodel,clobber=True,output_verify='fix+warn') fits.writeto(fitsname+'-fresidual.fits',fresidual,clobber=True,output_verify='fix+warn') mod_sd=Table(mod_sd) rename_table(mod_sd,prefix='bokf_',postfix='_'+filt) del_cols=['x_psf','x_psf_error','y_psf','y_psf_error','x_dev','x_dev_error','y_dev','y_dev_error','re_dev','re_dev_error','ab_dev','ab_dev_error','theta_dev','theta_dev_error','x_exp','x_exp_error','y_exp','y_exp_error','re_exp','re_exp_error','ab_exp','ab_exp_error','theta_exp','theta_exp_error'] for col in del_cols: mod_sd.remove_column('bokf_'+col+'_'+filt) mod_sd.rename_column('bokf_x_in_'+filt,'bokf_x_window_'+filt) mod_sd.rename_column('bokf_y_in_'+filt,'bokf_y_window_'+filt) mod_sd.rename_column('bokf_id_'+filt,bok_id) mod_sd[bok_id]=sdss[bok_id][det_sd['id']] fluxadu=table.join(fluxadu,mod_sd,join_type='left') fluxadu=fluxadu.filled(fill_value=0) # get coordinates for aperture photometry aper_coord=np.array([fluxadu['bok_x_window_'+filt],fluxadu['bok_y_window_'+filt]]).transpose() if sdss_params is not None and not args.phot_sdss and nuniq_sdss>0: aper_coord[nscuss_det:,0]=fluxadu['sdss_bokx_'+filt][nscuss_det:] aper_coord[nscuss_det:,1]=fluxadu['sdss_boky_'+filt][nscuss_det:] if args.force_sdss: index=np.array([ind for ind in sdss[bok_id][det_sd['id']] if ind >= nscuss]) if len(index) > 0: aper_coord[index,0]=fluxadu['bokf_x_window_'+filt][index] aper_coord[index,1]=fluxadu['bokf_y_window_'+filt][index] print('Aperture Photometry ....') aper_prop=phot.aperture_photometry(aper_coord,apertures=apertures,sky_annulus=sky_annuli,aper_type='circle',sky_type='global') aper_prop=Table(aper_prop) rename_table(aper_prop,prefix='bok_',postfix='_'+filt) del_cols=['id'] for col in del_cols: aper_prop.remove_column('bok_'+col+'_'+filt) aper_prop.rename_column('bok_x_in_'+filt,'bok_x_aper_'+filt) aper_prop.rename_column('bok_y_in_'+filt,'bok_y_aper_'+filt) fluxadu.add_columns(aper_prop.columns.values()) #re-order the objects fluxadu[bok_id]=np.arange(len(fluxadu)) fluxfile=fitsname+'-fluxadu.fits' fits.writeto(fluxfile,fluxadu.as_array(),clobber=True,output_verify='fix+warn') ind_head=head.index('RA-OBS') f=fits.open(fluxfile,mode='update') f[1].header["apersize"]=('3,4,5,6,8,10,13,16,20,25,30,40','aperture radii in pixels') f[1].header.extend(head.cards[ind_head:],bottom=True) f.flush(output_verify='fix+warn') f.close() # calibrate flux and aperture corrections fluxcalib=calibrate_fluxadu(fluxadu,head,stacked=stacked) if fluxcalib == False: continue plotname=None if args.plot: plotname=fitsname fluxcor=magnitude_correction(fluxcalib,head,plot_name=plotname) corfile=fitsname+'-fluxcor.fits' fits.writeto(corfile,fluxcor.as_array(),clobber=True,output_verify='fix+warn') f=fits.open(corfile,mode='update') f[1].header.extend(head.cards[ind_head:],bottom=True) f.flush(output_verify='fix+warn') f.close() print('Total processing time for '+fitsname,time.time()-start) print(datetime.datetime.now())
'panstarrs_gMAG': 'F334', 'panstarrs_rMAG': 'F335', 'panstarrs_iMAG': 'F336', 'panstarrs_zMAG': 'F337', 'panstarrs_yMAG': 'F338'} filt_type = ['narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', 'narrowband', \ 'sdss', 'sdss', 'sdss', 'sdss', 'sdss', \ 'allwise', 'allwise', \ 'panstarrs', 'panstarrs', 'panstarrs', 'panstarrs', 'panstarrs', \ 'clash','clash','clash','clash','clash','clash','clash','clash'] filt_lam = [4920, 5000, 5100, 5200, 5320, 5400, 5500, 5600, 5680, 5800, 5890, 6000, 6100, 6200, 6300, 6400, 6500, 6600, \ 3543, 4770, 6231, 7625, 9134, \ 33682, 46179, \ 4849.11, 6201.19, 7534.96, 8674.18, 9627.77, \ 3817.2, 4448, 5470.2, 6505.4, 7960.5, 7671.2, 9028.2, 21574] filt_table = Table() filt_table.add_columns([filt_name, filt_lam, filt_type], names=['name', 'lam', 'type']) # # Find where magnitudes end # imagerr_end = np.where(np.array(cat.colnames) == 'filter_count')[0][0] # fnames_original = cat.colnames[2:imagerr_end:2] # # Translate original filter names from gal_table to eazy filter names using dictionary # fnames = [filt_Dict[fnames_original[x]] for x in range(len(fnames_original))] # hfilt = np.in1d(filt_table['name'], fnames) # filt_table = filt_table[hfilt] fnames = cat.colnames[4:-4:2] # print(fnames) hfilt = np.in1d(filt_table['name'], fnames) filt_table = filt_table[hfilt] # print(filt_table)
def read_spectrum(filepath, filesource='auto'): """read SDSS/LAMOST spectrum Parameters ---------- filepath: string input file path filesource: string {'sdss_dr12' / 'lamost_dr2' / 'lamost_dr3'} Returns ------- specdata: astropy.table.Table spectra as a table """ # auto-identify the spectrum origination if filesource == 'auto': telescope = fits.open(filepath)[0].header['TELESCOP'] if telescope == 'SDSS 2.5-M': return read_spectrum(filepath, filesource='sdss_dr12') if telescope == 'LAMOST': return read_spectrum(filepath, filesource='lamost_dr3') # SDSS DR7 spectrum if filesource == 'sdss_dr7': hdulist = fits.open(filepath) # 1. flux, flux_err, mask data = hdulist[0].data # 5 rows flux = data[0][:] flux_err = data[2][:] mask = data[3][:] # 2. wave # http://iraf.net/irafdocs/specwcs.php # wi = CRVALi + CDi_i * (li - CRPIXi) CRVAL1 = hdulist[0].header['CRVAL1'] CD1_1 = hdulist[0].header['CD1_1'] CRPIX1 = hdulist[0].header['CRPIX1'] Npix = len(flux) wavelog = CRVAL1 + (np.arange(Npix) + 1 - CRPIX1) * CD1_1 wave = np.power(10, wavelog) spec = Table([wave, flux, flux_err, mask], names=['wave', 'flux', 'flux_err', 'mask']) return spec # SDSS DR10/DR12 spectrum if filesource == 'sdss_dr10' or filesource == 'sdss_dr12': data = fits.open(filepath) specdata = Table(data[1].data) wave = Column(name='wave', data=np.power(10., specdata['loglam'])) flux_err = Column(name='flux_err', data=(specdata['ivar']) ** -0.5) specdata.add_columns([wave, flux_err]) return specdata # LAMOST DR2/DR3 spectrum if filesource == 'lamost_dr3' or filesource == 'lamost_dr2' or filesource == 'lamost_dr1': data = fits.open(filepath) specdata = Table(data[0].data.T) flux = Column(name='flux', data=specdata['col0']) ivar = Column(name='ivar', data=specdata['col1']) flux_err = Column(name='flux_err', data=(specdata['col1']) ** -0.5) wave = Column(name='wave', data=specdata['col2']) and_mask = Column(name='and_mask', data=specdata['col3']) or_mask = Column(name='or_mask', data=specdata['col4']) # for flux_err, convert inf to nan flux_err[np.isinf(flux_err.data)] = np.nan return Table([wave, flux, flux_err, ivar, and_mask, or_mask]) return None
def join_cats(self, no_conv, check_phot): """ """ if no_conv: images = self.par_info['image'] else: images = self.par_info['convim'] # read in F110W catalog (all UVIS fields have F110W) f110cat = os.path.join(self.outdir, 'F110W_cat.fits') f = fits.open(f110cat) uvisdata = f[1].data f.close() # get all other catalogs cats = glob(os.path.join(self.outdir, 'F*cat.fits')) cats = [x for x in cats if x != f110cat] # read in original catalog c = os.path.join(self.datadir,self.par,'DATA/DIRECT_GRISM/fin_F110.cat') refdata = np.genfromtxt(c) idx,separc = match_cats(uvisdata['x_world'], uvisdata['y_world'], refdata[:,7], refdata[:,8]) match = (separc.value*3600. <= 0.2) # d['x_world'][match] # dref[:,7][idx[match]] t = Table(uvisdata[match]) print for phot in ['ISO', 'AUTO', 'APER']: # 'PETRO' # fix errors w110 = np.where(self.par_info['filt'] == 'F110W') print 'fix errors for F110W, %s' % phot eflux,emag = calc_errors(t, images[w110][0], self.par_info['rms'][w110][0], os.path.join(self.outdir,'F110W_seg.fits'), self.par_info['exptime'][w110][0], phot=phot) #eflux=eflux.reshape(t['FLUXERR_%s'%phot].shape) t['FLUXERR_%s'%phot] = eflux t['MAGERR_%s'%phot] = emag # rename F110 photometry columns t.rename_column('FLUX_%s'%phot, 'FLUX_%s_F110W'%phot) t.rename_column('FLUXERR_%s'%phot, 'FLUXERR_%s_F110W'%phot) t.rename_column('MAG_%s'%phot, 'MAG_%s_F110W'%phot) t.rename_column('MAGERR_%s'%phot, 'MAGERR_%s_F110W'%phot) # add column of indices from fin_F110.cat t.add_column(Column(data=refdata[:,1][idx[match]], name='WISP_NUMBER'), index=0) # add in photometry from other catalogs for i, cat in enumerate(cats): i += 1 print cat f = fits.getdata(cat) d = f[match] filtstr = os.path.basename(cat).split('_')[0] for j,phot in enumerate(['ISO', 'AUTO', 'APER']): index = (i*12 + 2) + (j * 4) # fix errors - UVIS errors are fine if filtstr == 'F160W': w160 = np.where(self.par_info['filt'] == 'F160W') print 'fix errors for %s, %s' % (filtstr, phot) eflux,emag = calc_errors(d, images[w160][0], self.par_info['rms'][w160][0], os.path.join(self.outdir,'F110W_seg.fits'), self.par_info['exptime'][w160][0], phot=phot) d['FLUXERR_%s'%phot] = eflux d['MAGERR_%s'%phot] = emag t.add_columns([Column(data=d['FLUX_%s'%phot], name='FLUX_%s_%s'%(phot,filtstr)), Column(data=d['FLUXERR_%s'%phot], name='FLUXERR_%s_%s'%(phot,filtstr)), Column(data=d['MAG_%s'%phot], name='MAG_%s_%s'%(phot,filtstr)), Column(data=d['MAGERR_%s'%phot], name='MAGERR_%s_%s'%(phot,filtstr))], indexes=[index, index, index, index]) # sort by WISP number t.sort(['WISP_NUMBER']) output = os.path.join(self.outdir, '%s_cat.fits'%self.par) t.write(output, format='fits') """ hdu0 = fits.PrimaryHDU() hdu1 = fits.BinTableHDU(np.array(t)) hdulist = fits.HDUList([hdu0, hdu1]) output = os.path.join(self.outdir, '%s_cat.fits'%self.par) hdulist.writeto(output, clobber=True) """ # make region file region_wcs(os.path.splitext(output)[0]+'.reg', t['X_WORLD'], t['Y_WORLD'], t['A_WORLD'], t['B_WORLD'], t['THETA_WORLD'], t['NUMBER']) # make region file of original F110W catalog region_wcs(os.path.join(self.outdir, 'F110W_orig.reg'), refdata[:,7], refdata[:,8], refdata[:,9], refdata[:,10], refdata[:,11], refdata[:,1], color='red') if check_phot is True: print c print f110cat check_conv_phot(c, output, 'F110W') c = os.path.join(self.datadir,self.par,'DATA/DIRECT_GRISM/fin_F160.cat') print c check_conv_phot(c, output, 'F160W')
def main(): parser = argparse.ArgumentParser(description="Returns photometric tables of catalog stars") parser.add_argument('catalog',type=str,help='JSON catalog of sources.') parser.add_argument('-WISE',type=str,required=True,help='Directory of WISE tables') parser.add_argument('-2MASS',type=str,dest='MASS',required=True,help='Directory of 2MASS tables') args = parser.parse_args() # Get starlist print 'Loading JSON data from: %s' % args.catalog theList = Star.load(args.catalog) print '\tLoaded %i sources.' % len(theList) print print 'Getting photometry from catalogs...' t = star_photometry(theList) print 'Locating sources in %s' % args.MASS t = add_2MASS(t,theList,args.MASS) print 'Locating sources in %s' % args.WISE t = add_WISE(t,theList,args.WISE) print #t.write('photometry.tsv',format='ascii.tab') #exit() outfile = 'photometry_ZOMG' colnames = [x for x in t.colnames if 'lam' in x] for col in colnames: t[col] = [99.99 if ((x is None) or (x is 'None')) else x for x in t[col]] #for Roberta rTable = Table() rTable.add_columns([t[x] for x in ['ID','Gal']]) for col in ['U','B','V','R','I','J','H','K', '3.6','4.5','5.8','8.0', 'W1','W2','W3','W4', 'F_U_Jy','F_B_Jy','F_V_Jy','F_R_Jy','F_I_Jy', 'F_J_Jy','F_H_Jy','F_K_Jy', 'F_3.6_Jy','F_4.5_Jy','F_5.8_Jy','F_8.0_Jy', 'F_W1_Jy','F_W2_Jy','F_W3_Jy','F_W4_Jy', 'F_0.36_um','lam_F_0.36_um','F_0.44_um','lam_F_0.44_um','F_0.55_um','lam_F_0.55_um','F_0.71_um','lam_F_0.71_um','F_0.97_um','lam_F_0.97_um', 'F_1.24_um','lam_F_1.24_um','F_1.66_um','lam_F_1.66_um','F_2.16_um','lam_F_2.16_um', 'F_3.55_um','lam_F_3.55_um','F_4.44_um','lam_F_4.44_um','F_5.73_um','lam_F_5.73_um','F_7.87_um','lam_F_7.87_um', 'F_3.35_um','lam_F_3.35_um','F_4.60_um','lam_F_4.60_um','F_11.56_um','lam_F_11.56_um','F_22.09_um','lam_F_22.09_um']: c = Column([np.float(x) if x else 99.99 for x in t[col]],name=col,dtype=np.float) rTable.add_column(c) rTable = photo_corr(rTable) rTable.write(outfile+'.tsv',format='ascii.tab') rTable.write(outfile+'.fits') exit() newCols = [] for col in colnames: c = Column([np.float(x) for x in t[col]],name=col,dtype=np.float) newCols.append(c) #print colnames eTable = Table() eTable.add_column(t['ID']) #eTable.add_column(t['lam_F_0.55_um']) #c = Column([str(x) for x in t['ID']],name='ID',dtype=str) #eTable.add_column(c) #eTable.add_columns([t[x] for x in colnames]) eTable.add_columns(newCols) #print eTable['ID'].dtype #exit() #print eTable.colnames #for col in eTable.colnames: # print eTable[col] #print eTable outfile = 'photometry.fits' print 'Writing table to %s' % outfile eTable.write(outfile)
""" Generate input file for single sersic model """ import numpy as np from astropy.table import Table, Column from astropy.io import fits source_table = '../../testser/RAWFIT00000.00268.fits' outname = 'input_ser.fits' source = Table.read(source_table) t_input = Table() t_input.add_column(source['NAME']) # put best-fit params of test Sersic fit as initial values val = Column( name='SER_VAL', data=source['FIT_SER'].data) fix = Column( name='SER_FIX', data=np.zeros((len(source), 8)) ) fix[:, 4] = 1. # do not fit shape of the isophotes t_input.add_columns([val, fix]) t_input.write(outname)
def make_sensitivity_curves(tint=1200, spec_res=100, sq_aper_diam=0.30, seeing_limited=False): mag = np.arange(10, 22) snr_z = np.zeros(len(mag), dtype=float) snr_y = np.zeros(len(mag), dtype=float) snr_j = np.zeros(len(mag), dtype=float) snr_h = np.zeros(len(mag), dtype=float) snr_sum_z = np.zeros(len(mag), dtype=float) snr_sum_y = np.zeros(len(mag), dtype=float) snr_sum_j = np.zeros(len(mag), dtype=float) snr_sum_h = np.zeros(len(mag), dtype=float) bkg_z = np.zeros(len(mag), dtype=float) bkg_y = np.zeros(len(mag), dtype=float) bkg_j = np.zeros(len(mag), dtype=float) bkg_h = np.zeros(len(mag), dtype=float) star_z = np.zeros(len(mag), dtype=float) star_y = np.zeros(len(mag), dtype=float) star_j = np.zeros(len(mag), dtype=float) star_h = np.zeros(len(mag), dtype=float) spec_z_tab = None spec_y_tab = None spec_j_tab = None spec_h_tab = None # Calculate the number of supernovae. N_SNe = 4500.0 * 0.6 * 10**(mag - 18.9) out_file = 'roboAO_sensitivity_t{0:d}_R{1:d}_ap{2:0.3f}'.format(tint, spec_res, sq_aper_diam) # Save the output to a table. _out = open(out_file + '.txt', 'w') meta1 = '# tint = {0:5d}, R = {1:5d}, sq_ap_diam = {2:5.3f}"\n' _out.write(meta1.format(tint, spec_res, sq_aper_diam)) _out.write('# Sensitivity integrated over broad band.') hdr = '{0:5s} {1:6s} {2:5s} {3:5s} {4:5s} {5:5s} {6:5s} {7:5s}\n' fmt = '{0:5.1f} {1:6.1f} {2:5.1f} {3:5.1f} {4:5.1f} {5:5.1f} {6:5.1f} {7:5.1f}\n' _out.write(hdr.format('# Mag', 'N_SNe', 'J_SNR', 'H_SNR', 'J_ms', 'H_ms', 'J_mb', 'H_mb')) for mm in range(len(mag)): print 'Mag: ', mag[mm] blah_z = etc_uh_roboAO(mag[mm], 'Z', tint, spec_res=spec_res, sq_aper_diam=sq_aper_diam, seeing_limited=seeing_limited) blah_y = etc_uh_roboAO(mag[mm], 'Y', tint, spec_res=spec_res, sq_aper_diam=sq_aper_diam, seeing_limited=seeing_limited) blah_j = etc_uh_roboAO(mag[mm], 'J', tint, spec_res=spec_res, sq_aper_diam=sq_aper_diam, seeing_limited=seeing_limited) blah_h = etc_uh_roboAO(mag[mm], 'H', tint, spec_res=spec_res, sq_aper_diam=sq_aper_diam, seeing_limited=seeing_limited) col_z_suffix = '_Z_{0:d}'.format(mag[mm]) col_y_suffix = '_Y_{0:d}'.format(mag[mm]) col_j_suffix = '_J_{0:d}'.format(mag[mm]) col_h_suffix = '_H_{0:d}'.format(mag[mm]) spec_signal_z = Column(name='sig'+col_z_suffix, data=blah_z[4]) spec_signal_y = Column(name='sig'+col_y_suffix, data=blah_y[4]) spec_signal_j = Column(name='sig'+col_j_suffix, data=blah_j[4]) spec_signal_h = Column(name='sig'+col_h_suffix, data=blah_h[4]) spec_bkg_z = Column(name='bkg'+col_z_suffix, data=blah_z[5]) spec_bkg_y = Column(name='bkg'+col_y_suffix, data=blah_y[5]) spec_bkg_j = Column(name='bkg'+col_j_suffix, data=blah_j[5]) spec_bkg_h = Column(name='bkg'+col_h_suffix, data=blah_h[5]) spec_snr_z = Column(name='snr'+col_z_suffix, data=blah_z[6]) spec_snr_y = Column(name='snr'+col_y_suffix, data=blah_y[6]) spec_snr_j = Column(name='snr'+col_j_suffix, data=blah_j[6]) spec_snr_h = Column(name='snr'+col_h_suffix, data=blah_h[6]) if spec_z_tab == None: spec_z_tab = Table([blah_z[3]], names=['wave_Z']) if spec_y_tab == None: spec_y_tab = Table([blah_y[3]], names=['wave_Y']) if spec_j_tab == None: spec_j_tab = Table([blah_j[3]], names=['wave_J']) if spec_h_tab == None: spec_h_tab = Table([blah_h[3]], names=['wave_H']) spec_z_tab.add_columns([spec_signal_z, spec_bkg_z, spec_snr_z]) spec_y_tab.add_columns([spec_signal_y, spec_bkg_y, spec_snr_y]) spec_j_tab.add_columns([spec_signal_j, spec_bkg_j, spec_snr_j]) spec_h_tab.add_columns([spec_signal_h, spec_bkg_h, spec_snr_h]) snr_z[mm] = blah_z[0] snr_y[mm] = blah_y[0] snr_j[mm] = blah_j[0] snr_h[mm] = blah_h[0] snr_sum_z[mm] = math.sqrt((spec_snr_z**2).sum()) snr_sum_y[mm] = math.sqrt((spec_snr_y**2).sum()) snr_sum_j[mm] = math.sqrt((spec_snr_j**2).sum()) snr_sum_h[mm] = math.sqrt((spec_snr_h**2).sum()) star_z[mm] = blah_z[1] star_y[mm] = blah_y[1] star_j[mm] = blah_j[1] star_h[mm] = blah_h[1] bkg_z[mm] = blah_z[2] bkg_y[mm] = blah_y[2] bkg_j[mm] = blah_j[2] bkg_h[mm] = blah_h[2] avg_tab = Table([mag, snr_z, snr_y, snr_j, snr_h, snr_sum_z, snr_sum_y, snr_sum_j, snr_sum_h, star_z, star_y, star_j, star_h, bkg_z, bkg_y, bkg_j, bkg_h], names=['mag', 'snr_z', 'snr_y', 'snr_j', 'snr_h', 'snr_sum_z', 'snr_sum_y', 'snr_sum_j', 'snr_sum_h', 'star_z', 'star_y', 'star_j', 'star_h', 'bkg_z', 'bkg_y', 'bkg_j', 'bkg_h']) out_file = 'roboAO_sensitivity_t{0:d}_R{1:d}_ap{2:0.3f}'.format(tint, spec_res, sq_aper_diam) if seeing_limited: out_file += '_seeing' # Save the tables spec_z_tab.write(out_file + '_spec_z_tab.fits', overwrite=True) spec_y_tab.write(out_file + '_spec_y_tab.fits', overwrite=True) spec_j_tab.write(out_file + '_spec_j_tab.fits', overwrite=True) spec_h_tab.write(out_file + '_spec_h_tab.fits', overwrite=True) avg_tab.write(out_file + '_avg_tab.fits', overwrite=True) return
def make_tint_curves(mag=20, spec_res=100, sq_aper_diam=0.30, seeing_limited=False): tint = np.arange(300, 3600+1, 300) snr_y = np.zeros(len(tint), dtype=float) snr_j = np.zeros(len(tint), dtype=float) snr_h = np.zeros(len(tint), dtype=float) snr_sum_y = np.zeros(len(tint), dtype=float) snr_sum_j = np.zeros(len(tint), dtype=float) snr_sum_h = np.zeros(len(tint), dtype=float) spec_y_tab = None spec_j_tab = None spec_h_tab = None for tt in range(len(tint)): print 'Tint: ', tint[tt] blah_y = etc_uh_roboAO(mag, 'Y', tint[tt], spec_res=spec_res, sq_aper_diam=sq_aper_diam, seeing_limited=seeing_limited) blah_j = etc_uh_roboAO(mag, 'J', tint[tt], spec_res=spec_res, sq_aper_diam=sq_aper_diam, seeing_limited=seeing_limited) blah_h = etc_uh_roboAO(mag, 'H', tint[tt], spec_res=spec_res, sq_aper_diam=sq_aper_diam, seeing_limited=seeing_limited) col_y_suffix = '_Y_{0:d}'.format(tint[tt]) col_j_suffix = '_J_{0:d}'.format(tint[tt]) col_h_suffix = '_H_{0:d}'.format(tint[tt]) spec_signal_y = Column(name='sig'+col_y_suffix, data=blah_y[4]) spec_signal_j = Column(name='sig'+col_j_suffix, data=blah_j[4]) spec_signal_h = Column(name='sig'+col_h_suffix, data=blah_h[4]) spec_bkg_y = Column(name='bkg'+col_y_suffix, data=blah_y[5]) spec_bkg_j = Column(name='bkg'+col_j_suffix, data=blah_j[5]) spec_bkg_h = Column(name='bkg'+col_h_suffix, data=blah_h[5]) spec_snr_y = Column(name='snr'+col_y_suffix, data=blah_y[6]) spec_snr_j = Column(name='snr'+col_j_suffix, data=blah_j[6]) spec_snr_h = Column(name='snr'+col_h_suffix, data=blah_h[6]) if spec_y_tab == None: spec_y_tab = Table([blah_y[3]], names=['wave_Y']) if spec_j_tab == None: spec_j_tab = Table([blah_j[3]], names=['wave_J']) if spec_h_tab == None: spec_h_tab = Table([blah_h[3]], names=['wave_H']) spec_y_tab.add_columns([spec_signal_y, spec_bkg_y, spec_snr_y]) spec_j_tab.add_columns([spec_signal_j, spec_bkg_j, spec_snr_j]) spec_h_tab.add_columns([spec_signal_h, spec_bkg_h, spec_snr_h]) snr_y[tt] = blah_y[0] snr_j[tt] = blah_j[0] snr_h[tt] = blah_h[0] snr_sum_y[tt] = math.sqrt((spec_snr_y**2).sum()) snr_sum_j[tt] = math.sqrt((spec_snr_j**2).sum()) snr_sum_h[tt] = math.sqrt((spec_snr_h**2).sum()) avg_tab = Table([tint, snr_y, snr_sum_y, snr_j, snr_sum_j, snr_h, snr_sum_h], names=['tint', 'snr_y', 'snr_sum_y', 'snr_j', 'snr_sum_j', 'snr_h', 'snr_sum_h']) out_file = 'roboAO_tint_m{0:d}_R{1:d}_ap{2:0.3f}'.format(mag, spec_res, sq_aper_diam) if seeing_limited: out_file += '_seeing' # Save the tables spec_y_tab.write(out_file + '_spec_y_tab.fits', overwrite=True) spec_j_tab.write(out_file + '_spec_j_tab.fits', overwrite=True) spec_h_tab.write(out_file + '_spec_h_tab.fits', overwrite=True) avg_tab.write(out_file + '_avg_tab.fits', overwrite=True) return
def main(): parser = argparse.ArgumentParser(description='Perform centroiding and profiling of input galaxies.') parser.add_argument('data',type=str,help='Image file') parser.add_argument('region',type=str,help='Region file') # Parse dat ass args = parser.parse_args() # Parse coordinates regions = pyregion.open(args.region) names = [r.comment for r in regions] coords = [(int(r.coord_list[0]),int(r.coord_list[1])) for r in regions] # Grab those juicy data meats data,header = fits.getdata(args.data,header=True) # For each coord, initialize and centroid galaxy galaxies = [Galaxy(name,co,data) for co,name in zip(coords,names)] # Show each swatch pp = PdfPages('galaxy_im.pdf') for gal in galaxies: pp.savefig(gal.show()[0]) #plt.show() pp.close() # Perform aperture photometry rows = [] for gal in galaxies: ap = utils.CircularAperture((gal.xc,gal.yc), gal.swatch, 15,19) rows.append(ap.run()) t = Table(rows=rows) t.add_column(Column([gal.name for gal in galaxies],name='name'),index=0) # Convert to counts/arcsec^2 pltscale = header['pltscale'] # "/mm pltscale = pltscale /1000 # "/um pixscaleX = header['xpixelsz'] # um/pix pixscaleX = pixscaleX * pltscale # "/pix pixscaleY = header['ypixelsz'] # um/pix pixscaleY = pixscaleY * pltscale # "/pix pixscale = pixscaleX * pixscaleY # arcsec^2/pix^2 cpa2 = t['counts/pix^2'] / pixscale epa2 = t['err_cp2'] * pixscale cpa2 = Column(cpa2,name='counts/arcsec^2') epa2 = Column(epa2,name='err_ca2') t.add_columns([cpa2,epa2]) t.pprint() ''' name #pix counts/pix^2 err_cp2 counts/arcsec^2 err_ca2 ------- ------------- ------------- ------------- --------------- ------------- NGC4875 706.858347058 7721.59941623 117.896411641 7584.60432695 120.025887186 NGC4869 706.858347058 11151.8722709 173.611128029 10954.0179592 176.746937224 GMP4277 706.858347058 9756.15839964 139.752361268 9583.06656737 142.276604641 GMP4350 706.858347058 8855.62011973 130.378485589 8698.50545946 132.73341559 NGC4860 706.858347058 11748.7754577 157.304664726 11540.3310078 160.145942354 NGC4881 706.858347058 11499.1497714 161.710512127 11295.1341311 164.631369312 NGC4921 706.858347058 12822.257865 215.253917975 12594.7679026 219.141890035 ''' # prep PDF pp = PdfPages('galaxy_prof.pdf') pp2= PdfPages('galaxy_contour.pdf') contours = [] for gal in galaxies: # perform contour isophotal fitting for profile r,med,err,levels = utils.contour_profile(gal.swatch,get_levels=True) contours.append(levels) fig = plt.figure() # plot profile with errorbars plt.errorbar(np.array(r)*pixscaleX,med,yerr=err,fmt='o',ms=8) # fit sersic function try: # hard-coded bullshit!! if gal.name == 'NGC4860': xi,yi = r[:-2],med[:-2] elif gal.name == 'GMP4350': xi,yi = r[1:],med[1:] elif gal.name == 'NGC4869': xi,yi = r[:-4],med[:-4] else: xi,yi = r,med xf,yf,p = utils.sersic_fit(xi,yi,rebin=100) except: plt.legend([gal.name]) continue # plot fit plt.plot(xf*pixscaleX,yf,'r-') plt.ylabel('Counts') plt.xlabel('Radius [arcsec]') #plt.xlim([0,12]) #plt.legend([gal.name,'n = %.2f' % p[0]]) # plot Re as just half max Ie = yf.max()/2 ReIdx,Ie = utils.find_nearest_element(yf,Ie,index=True) Re = xf[ReIdx]*pixscaleX # calculate Re from exp prof? #Re = utils.exp_Re(xf*pixscaleX,yf) plt.legend([gal.name,'n = %.2f\nRe = %.2f' % (p[0],Re)]) plt.plot([np.min(xf)*pixscaleX,Re],[Ie,Ie],'r-.') pp.savefig(fig) # plot image and contours for gal in galaxies: fig = plt.figure() plt.imshow(gal.swatch,origin='lower',cmap=plt.cm.gist_heat_r) plt.contour(gal.swatch,levels=levels,alpha=0.7) plt.title(gal.name) pp2.savefig(fig) #plt.show() pp.close() pp2.close() exit() '''
cleaned = np.vectorize( lambda s: s.strip(':?') )(data) empty = np.vectorize( lambda s: s=='' )(data) table = Table(cleaned, names=fields) #convert RA/DEC columns This is a massive HACK!! ra, dec = table['ra'], table['dec'] decmap = map(lambda dec: dec.rsplit(' ', 1)[0], dec) coords = list(map( ' '.join, zip(ra, decmap) )) #coords = Column( name='coords', data=SkyCoord( coords, unit=('h', 'deg') ) ) coords = SkyCoord( coords, unit=('h', 'deg') ) #TODO: Try use this as column?? ra_col = Column( coords.ra, 'ra', float ) dec_col = Column( coords.dec, 'dec', float ) i = table.colnames.index('ra') table.remove_columns( ('ra', 'dec') ) table.add_columns( [ra_col, dec_col], [i,i] ) #Type filter types = [c for c in table.colnames if 'type' in c] mtypes = ('AM', 'AS', 'LA', 'IP', 'LI') #magnetic systems 'IP', 'LI' ltype = np.array( [any(t in mtypes for t in tt) for tt in table[types]] ) #Hour angle filter #lra = (9 > coords.ra.hour) & (coords.ra.hour < 19) tq = table[ltype] #&lra #tq.sort('ra') raise SystemExit #Magnitude filter
#============================================================================== # Make Table #============================================================================== make_table = False if make_table: data = Table() fluxes = np.squeeze(synphot.fluxes).value for i, n in enumerate(Names): flux = (fluxes[i] * Ms[:, :]).ravel() error = 0.01 * flux noise = [np.random.normal(0, err) for err in error] print np.median(noise / flux) * 100 data.add_columns([ Column(flux + noise, 'F%s' % (i + 1)), Column(error, 'E%s' % (i + 1)) ]) id = Column(name='id', data=np.arange(1, len(data) + 1)) zspec = Column(name='zspec', data=-1 * np.ones(len(data))) data.add_column(id, 0) data.add_column(zspec) df = data.to_pandas() np.savetxt('Lilly/Lilly_SFH.cat', data, header=' '.join(data.colnames), fmt=['%d'] + ['%.5e' for i in range(20)] + ['%.2f'])
t['caLHacorr'].description = 'Log of Halpha luminosity, extinction corrected, from <log_L_Ha_cor> column 147 in get_proc_elines_CALIFA.csv' t['caMstars'].unit = 'dex(solMass)' t['caMstars'].description = 'Log of stellar mass, from column 73 log_Mass in get_proc_elines_CALIFA.csv' t['caDistMpc'].unit = 'Mpc' t['caDistMpc'].description = 'Luminosity distance in Mpc computed from caZgas assuming Ho=70, Om=0.27, Ol=0.73' if 'califa' in rewrite: outcols = ['Name'] for cname in t.colnames: if cname.startswith("ca") or cname.startswith("S"): outcols.append(cname) tsel = t[outcols] newt = join(tsel, dl_pipe3d, keys='Name', join_type='left') joint = join(tsel, ca_qc, keys='Name', join_type='left') newt.add_columns([ joint['FLAG_CAL_WL_V500'], joint['FLAG_CAL_REGISTRATION_V500'], joint['FLAG_CAL_IMGQUAL_V500'] ], names=['caFlgWav5', 'caFlgReg5', 'caFlgImg5']) newt.add_columns([ joint['FLAG_CAL_WL_V1200'], joint['FLAG_CAL_REGISTRATION_V1200'], joint['FLAG_CAL_IMGQUAL_V1200'] ], names=['caFlgWav12', 'caFlgReg12', 'caFlgImg12']) newt['caDistP3d'].unit = 'Mpc' newt[ 'caDistP3d'].description = 'Luminosity distance in Mpc from get_proc_elines_CALIFA.csv' newt[ 'caFlgWav5'].description = 'Flag (-1/0/1/2=NA/good/minor/bad) for wavelength calibration V500' newt[ 'caFlgWav12'].description = 'Flag (-1/0/1/2=NA/good/minor/bad) for wavelength calibration V1200' newt[
def make_table(data,details,units,prefix,ra_lims,dec_lims): ##Find all of the data in the columns as specified by the ##user inputs shape = data.shape entries = shape[0] names = data[details[0]] RA = get_units(data,details[1],units[0],'angle',entries) RA_error = get_units(data,details[2],units[1],'angle',entries) Dec = get_units(data,details[3],units[2],'angle',entries) Dec_error = get_units(data,details[4],units[3],'angle',entries) freq = details[5] flux = get_units_blanks(data,details[6],units[4],'flux',entries) flux_error = get_units_blanks(data,details[7],units[5],'flux',entries) PA = get_units_blanks(data,details[8],units[6],'angle',entries) major = get_units_blanks(data,details[9],units[7],'angle',entries) minor = get_units_blanks(data,details[10],units[8],'angle',entries) if details[11]=='-': flags = np.empty(entries); flags.fill(-100000.0) else: flags = data[details[11]] if details[12]=='-': ID = np.empty(entries); ID.fill(-100000.0) else: ID = data[details[12]] freqs = [] fluxs = [] ferrs = [] ##This handles the case of more than one frequency in a single catalogue if len(details)>13: length = len(details) - 13 num_of_freqs = length / 3 for i in xrange(num_of_freqs): freqs.append(details[13+(3*i)]) fluxs.append(get_units_blanks(data,details[13+1+(3*i)],units[9+(2*i)],'flux',entries)) ferrs.append(get_units_blanks(data,details[13+2+(3*i)],units[10+(2*i)],'flux',entries)) def get_lune(ra1,ra2,dec1,dec2): '''Calculates the steradian coverage of a lune defined by two RA,Dec coords''' return abs((ra2*dr-ra1*dr)*(np.sin(dec2*dr)-np.sin(dec1*dr))) ##Count the number of sources within the requested user lune to calculate the scaled ##source density of the catalogues ##If the ra lims do not cross over the RA = 0 line if ra_lims[0]<ra_lims[1]: sources_in_bounds = [i for i in xrange(len(RA)) if (RA[i]>=ra_lims[0] and RA[i]<=ra_lims[1]) and (Dec[i]>=dec_lims[0] and Dec[i]<=dec_lims[1])] area = get_lune(ra_lims[0],ra_lims[1],dec_lims[0],dec_lims[1]) ##If they do, searching the coords slightly differently and do some rearranging to get the area else: sources_in_bounds = [i for i in xrange(len(RA)) if (RA[i]>=ra_lims[0] or RA[i]<=ra_lims[1]) and (Dec[i]>=dec_lims[0] and Dec[i]<=dec_lims[1])] extra = 360.0 - ra_lims[0] area = get_lune(0,ra_lims[1]+extra,dec_lims[0],dec_lims[1]) scaled_source_density = (4*np.pi*len(sources_in_bounds))/area ##Create a new table, and populate with the data in correct units t=Table(masked=True,meta={'src_dens':scaled_source_density}) t_names = Column(name='%s_name' %prefix,data=names,description='Name from catalogue',dtype=str) t_ras = Column(name='%s_RAJ2000' %prefix,data=RA,description='Right Ascension of source (J2000)',unit='deg',dtype=float) t_rerrs = Column(name='%s_e_RAJ2000' %prefix,data=RA_error,description='Error on Right Ascension',unit='deg',dtype=float) t_decs = Column(name='%s_DEJ2000' %prefix,data=Dec,description='Declination of source (J2000)',unit='deg',dtype=float) t_derrs = Column(name='%s_e_DEJ2000' %prefix,data=Dec_error,description='Error on Declination',unit='deg',dtype=float) t_fluxes = Column(name='%s_S%d' %(prefix,float(freq)),data=flux,description='Source flux at %.1fMHz' %float(freq),unit='Jy',dtype=float) t_ferrs = Column(name='%s_e_S%d' %(prefix,float(freq)),data=flux_error,description='Error on flux at %.1fMHz' %float(freq),unit='Jy',dtype=float) t_majors = Column(name='%s_MajAxis' %prefix,data=major,description='Fitted major axis',unit='deg',dtype=float) t_minors = Column(name='%s_MinAxis' %prefix,data=minor,description='Fitted minor axis',unit='deg',dtype=float) t_PAs = Column(name='%s_PA' %prefix,data=PA,description='Fitted Position Angle',unit='deg',dtype=float) mask = [] for i in flags: if type(i)==np.ma.core.MaskedConstant: mask.append(True) else: mask.append(False) t_flags = MaskedColumn(name='%s_flag' %prefix,data=flags,description='Any meta flag for inclusion',mask=mask,fill_value='--',dtype=str) mask = [] for i in ID: if type(i)==np.ma.core.MaskedConstant: mask.append(True) else: mask.append(False) t_fields = MaskedColumn(name='%s_FieldID' %prefix,data=ID,description='If avaiable, image field ID',mask=mask,fill_value='--',dtype=str) t.add_columns([t_names,t_ras,t_rerrs,t_decs,t_derrs,t_fluxes,t_ferrs,t_majors,t_minors,t_PAs,t_flags,t_fields]) #Again, handles multiple frequencies in one catalogues if len(freqs)>0: for i in xrange(len(freqs)): t_fextra = Column(name='%s_Flux_%.1f' %(prefix,float(freqs[i])),data=fluxs[i],description='Source flux at %.1fMHz' %float(freqs[i]),unit='Jy',dtype=float) t_ferrextra = Column(name='%s_Flux_%.1f_err' %(prefix,float(freqs[i])),data=ferrs[i],description='Error on flux at %.1fMHz' %float(freqs[i]),unit='Jy',dtype=float) t.add_columns([t_fextra,t_ferrextra]) ##Add the source density to the table #t.add_keyword('%s_nu' %prefix,str(scaled_source_density)) t.write('simple_%s.fits' %prefix,overwrite=True,format='fits') return scaled_source_density
def main(): epilog='''The BASS DR1 pipeline is designed for images taken by the Bok Telescope. It make photometry for both SDSS detected objects and Bok-only sources. The photometric methods include automatic aperture, circular aperture, PSF and model photometry. The model photometry can use SDSS shape parameters for force photometry or use self-guessed parameters to refit. Two models are Dev (1/4) and Exp laws. Gain and read-out noise is read from the FITS header. if single image, only flag should be provided. If weight, flag, psfex, sdss is empty, the program will automatically find or calculate them''' parser = argparse.ArgumentParser(description='BASS DR1 Photometric Pipeline.', epilog=epilog,fromfile_prefix_chars='@') parser.add_argument('fits', metavar='FITS', type=str, nargs='*', help='Bok calibrated images to do photometry. @filelist to read from file') parser.add_argument('-f','--flag',type=str,nargs='*',default=[], help='flag image (bad pixels). @filelist to read from file') parser.add_argument('--psfex',type=str,nargs='*',metavar='psfex',default=[], help='PSF file derived by PSFEx') parser.add_argument('--force-filter',type=str,default=None, help='Force Photometry with SDSS filter shape parameters, default None (use the same filter as the image') parser.add_argument('--save-sky', action='store_true', help='save the 2d sky background and rms map') parser.add_argument('--save-segment', action='store_true', help='save the 2d sky background and rms map') parser.add_argument('--save-model', action='store_true', help='save the fitted model and residual image') parser.add_argument('--plot', action='store_true', help='plot all kinds of corrections in the catalog') parser.add_argument('--nthread',type=int,default=0, help='number of threads to parallel processing, default 0') parser.add_argument('--version', action='version',version='%(prog)s 1.0') # parse arguments args = parser.parse_args() ###CCD parameters saturate=30000.0 gain=1.4 rdnoise=8.0 ## detecting and analysis detect_nsigma=1.2 aly_nsigma=1.5 conv_sigma=1.2 # not using seeing FWHM/2.35 if seeing too large many faint objects cannot be found! we use 1.2 for all bok image sky_grid=216 remove_nthresh=0 sig_flag=True sig_level=3.0 med_window=21 remove_big_width=None # remove big objects nradius=10 # n*effective radius to create models and residual map # photometry psfmag_limit=24.0 # SDSS r-band mag limit to get sources apertures=[3,4,5,6,8,10,13,16,20,25,30,40] sky_annuli=[40,20] edge_nfwhm=3.0 # n*fwhm of seeing to regarded as edge pixels min_dist=3.3 # min distance between scuss and sdss pixels ab_limit=[0.1,10.0] re_limit=[1e-10,5.0] theta_limit=[None,None] nmax_fit=1 nmax_force_fit=1 # get list of fits, psfs, flags or weights fitslist=args.fits flaglist=args.flag psflist=args.psfex nfits=len(fitslist) nflag=len(flaglist) npsf=len(psflist) # print help if no parameters provided if len(args.fits) == 0: parser.print_help() # verify whether file numbers are equal if npsf !=0 and npsf != nfits: print('The number of PSF files should be same as the FITS image') sys.exit(-1) if nflag !=0 and nflag != nfits: print('The number of flag images should be same as the single-epoch images') sys.exit(-1) for i,ifits in enumerate(fitslist): start=time.time() print('phot '+ifits+' ...') # check file exist if not os.path.isfile(ifits): continue # get FITS image fitsname,_=os.path.splitext(ifits) fitspath,name=os.path.split(ifits) data=fits.getdata(ifits) head=fits.getheader(ifits) head['ctype1']='RA---TPV' head['ctype2']='DEC--TPV' filt=head['filter'] if filt == 'bokr': filt='r' # calculate the julian day datestr=head['date-obs'] timestr=head['time-obs'] tt=Time(datestr+' '+timestr,format='iso',scale='utc') mjd=tt.mjd # get seeing value seeing=None if 'seeing' in head: seeing=head['seeing']/0.455 # seeing in pixels #get flag, mask mask_sat=data > saturate if nflag ==0: ccd=str(head['CCD_NO']) date=head['date-obs'] flagfile=get_flagfile(date,ccd) flag=fits.getdata(flagfile) else: flag=fits.getdata(flaglist[i]) flag=(flag != 0).astype('int') mask=(flag != 0) | (data > 70000.0) | (data< 1.0) print('flag file: '+flagfile) # flag the saturated pixels flag[mask_sat]=np.bitwise_or(flag[mask_sat],2) psffile=fitsname+'.psf' # get psf file if npsf == 0: if not os.path.isfile(psffile): get_psfex(ifits) else: if os.path.isfile(psflist[i]): psffile=psflist[i] else: get_psfex(ifits) # download sdss catalog sdsscat=fitsname+'-sdss.fits' ra,dec=fits_center(head) print(ra,dec) print('fits center: {}, {}'.format(ra,dec)) if not os.path.isfile(sdsscat): d=DownSDSSCat() d.get_allmag_cat(ra=ra,dec=dec,width=0.6,psfmag_limit=psfmag_limit,filename=sdsscat,fmt='fits') if os.path.isfile(sdsscat): print('sdss catalog: '+sdsscat) else: print('sdss catalog is not found!') # processing sdss catalogs: transform the astrophot input parameters and get sdss catalogs within the image sdss_params=None if os.path.isfile(sdsscat) and os.path.getsize(sdsscat)!=0: sdss=fits.getdata(sdsscat,1) sdss_params,sdss=params_from_sdss(sdss,head,force_filter=args.force_filter,faint_limit=23.0) # get psfex PSF, initialize photometry psf=pyphot.psfex.PSFEx() try: psf.from_fits(psffile) except: continue phot=pyphot.phot.PyPhot(data,uncertainty=None,mask=mask,flags=flag,weight=None,saturate=saturate,gain=gain,rdnoise=rdnoise,seeing=seeing,nthread=args.nthread) if os.path.isfile(fitsname+'-sky.fits') and os.path.isfile(fitsname+'-sky_rms.fits'): print('get sky and rms map from saved fits') phot.sky=fits.getdata(fitsname+'-sky.fits') phot.sky_rms=fits.getdata(fitsname+'-sky_rms.fits') else: phot.get_sky_map(nsample=30000,grid_size=sky_grid) # get flux error map if seeing is None: phot.get_seeing() seeing=phot.seeing # define the edge pixels and flags edge_pixels=np.ceil(edge_nfwhm*seeing).astype('int') if edge_pixels > 0: flag[:edge_pixels,:]=np.bitwise_or(flag[:edge_pixels,:],4) flag[-edge_pixels:,:]=np.bitwise_or(flag[-edge_pixels:,:],4) flag[:,:edge_pixels]=np.bitwise_or(flag[:,:edge_pixels],4) flag[:,-edge_pixels:]=np.bitwise_or(flag[:,-edge_pixels:],4) phot.flag=flag phot.update_data_uncertainty(poisson=True) if args.save_sky: print('save sky and rms maps') fits.writeto(fitsname+'-sky.fits',phot.sky.astype('float32'),header=head,clobber=True,output_verify='fix+warn') fits.writeto(fitsname+'-sky_rms.fits',phot.sky_rms.astype('float32'),header=head,clobber=True,output_verify='fix+warn') # detecting source in bok image tmp_prop=phot.detect_objects(detect_nsigma=detect_nsigma,conv_sigma=conv_sigma,analysis_nsigma=aly_nsigma,remove_nthresh=remove_nthresh,sig_flag=sig_flag,sig_level=sig_level,remove_after_segment=False,remove_big_width=remove_big_width) coord_scuss=np.array((tmp_prop['x_window'],tmp_prop['y_window'])).transpose() nscuss=len(coord_scuss) # get SDSS coordinates and combine them into Bok detected sources coord=coord_scuss if sdss_params is not None: coord_sdss=np.array((sdss_params['x'],sdss_params['y'])).transpose() index_scuss,index_sdss=closest_match(coord_scuss,coord_sdss,min_dist=min_dist) index_scuss=np.array(index_scuss) indsdss_match=np.array([idx[0] for idx in index_sdss]) indsdss_nomatch=np.setdiff1d(np.arange(len(coord_sdss)),indsdss_match) nuniq_sdss=len(indsdss_nomatch) coord=np.vstack((coord_scuss,coord_sdss[indsdss_nomatch])) is_sdss=Table.Column(np.zeros(len(coord)).astype('bool'),'is_sdss') if len(index_scuss) !=0 : is_sdss[index_scuss]=True if nuniq_sdss > 0: is_sdss[nscuss:]=True else: is_sdss=Table.Column(np.zeros(len(coord)).astype('bool'),'is_sdss') is_bok=Table.Column(np.zeros(len(coord)).astype('bool'),'is_bok') is_bok[:nscuss]=True # calculate MJD for single-epoch images mjd_col=Table.Column(np.ones(len(coord))*mjd,'bok_mjd') # get detection properties if sdss_params is not None: det_sc=phot.detect_objects(detect_nsigma=detect_nsigma,conv_sigma=conv_sigma,analysis_nsigma=aly_nsigma,remove_nthresh=remove_nthresh,sig_flag=sig_flag,sig_level=sig_level,remove_after_segment=False,coordinates=coord,remove_big_width=remove_big_width) else: det_sc=tmp_prop # save segmentations if args.save_segment: fits.writeto(fitsname+'-segment.fits',phot.seglabels,header=head,clobber=True,output_verify='fix+warn') param_det=phot.parameter_guess(det_sc,xname='x_window',yname='y_window') # clean resulting data det_sc=Table(det_sc) rename_table(det_sc,prefix='bok_',postfix='_'+filt) del_cols=['x_min','x_max','y_min','y_max','cxx','cxx_error','cyy','cyy_error','cxy','cxy_error','flux_kron','flux_kron_error','kron_radius'] for col in del_cols: det_sc.remove_column('bok_'+col+'_'+filt) fluxadu=det_sc index_det=det_sc['bok_id_'+filt] is_sdss=is_sdss[index_det] is_bok=is_bok[index_det] mjd_col=mjd_col[index_det] if sdss_params is not None: tmpindex=np.arange(len(coord)) if len(index_scuss) !=0 : tmpindex[index_scuss]=indsdss_match tmpindex[nscuss:]=indsdss_nomatch tmpindex=tmpindex[index_det] tmpindex=tmpindex[is_sdss] sdss=sdss[tmpindex] sdss_params=sdss_params[tmpindex] fluxadu.add_column(mjd_col) fluxadu.add_column(is_sdss) fluxadu.add_column(is_bok) # parameter guess: if SDSS exist use SDSS, else use guess to force photometry bok_id='bok_id_'+filt if sdss_params is not None: param_det['flux_psf'][is_sdss]=sdss_params['flux_psf'] param_det['re_dev'][is_sdss]=sdss_params['re_dev'] param_det['re_exp'][is_sdss]=sdss_params['re_exp'] param_det['ab_dev'][is_sdss]=sdss_params['ab_dev'] param_det['ab_exp'][is_sdss]=sdss_params['ab_exp'] param_det['theta_dev'][is_sdss]=sdss_params['theta_dev'] param_det['theta_exp'][is_sdss]=sdss_params['theta_exp'] param_det['flux_dev'][is_sdss]=sdss_params['flux_dev'] param_det['flux_exp'][is_sdss]=sdss_params['flux_exp'] param_det['frac_dev'][is_sdss]=sdss_params['frac_dev'] mod_sd1=phot.model_photometry(psf,param_guess=param_det[is_sdss],pos_limit=True,ab_limit=True,re_limit=True,theta_limit=True,nmax_fit=nmax_force_fit,models=['psf','dev','exp']) mod_sd1=Table(mod_sd1) groupmax=np.max(mod_sd1['group']) rename_table(mod_sd1,prefix='bok_',postfix='_'+filt) del_cols=['x_in','y_in']#,'x_psf','x_psf_error','y_psf','y_psf_error','x_dev','x_dev_error','y_dev','y_dev_error','re_dev','re_dev_error','ab_dev','ab_dev_error','theta_dev','theta_dev_error','x_exp','x_exp_error','y_exp','y_exp_error','re_exp','re_exp_error','ab_exp','ab_exp_error','theta_exp','theta_exp_error'] for col in del_cols: mod_sd1.remove_column('bok_'+col+'_'+filt) mod_sd1[bok_id]=fluxadu[bok_id][is_sdss] fluxadu=table.join(fluxadu,mod_sd1,join_type='left') fluxadu=fluxadu.filled(fill_value=0) #fluxadu=fluxadu.filled() tmpmask=np.logical_and(is_bok,np.logical_not(is_sdss)) if tmpmask.any(): mod_sd2=phot.model_photometry(psf,param_guess=param_det[tmpmask],pos_limit=True,ab_limit=True,theta_limit=True,re_limit=True,nmax_fit=nmax_force_fit,models=['psf','dev','exp']) if sdss_params is not None: mod_sd2['group']+=groupmax+1 mod_sd2=Table(mod_sd2) rename_table(mod_sd2,prefix='bok_',postfix='_'+filt) del_cols=['x_in','y_in']#,'x_psf','x_psf_error','y_psf','y_psf_error','x_dev','x_dev_error','y_dev','y_dev_error','re_dev','re_dev_error','ab_dev','ab_dev_error','theta_dev','theta_dev_error','x_exp','x_exp_error','y_exp','y_exp_error','re_exp','re_exp_error','ab_exp','ab_exp_error','theta_exp','theta_exp_error'] for col in del_cols: mod_sd2.remove_column('bok_'+col+'_'+filt) if sdss_params is not None: #if sdss exist merge allcols=mod_sd2.colnames for col in allcols: fluxadu[col][tmpmask]=mod_sd2[col] else: # if no sdss add new columns mod_sd2[bok_id]=fluxadu[bok_id][tmpmask] fluxadu=table.join(fluxadu,mod_sd2,join_type='left') fluxadu.filled(fill_value=0) # merge sdss information if sdss_params is not None: sdss=Table(sdss) rename_table(sdss,prefix='sdss_') sdss.add_columns([table.Column(sdss_params['x'],name='sdss_bokx_'+filt), table.Column(sdss_params['y'],name='sdss_boky_'+filt)],indexes=[0,0]) sdss[bok_id]=-1 sdss[bok_id]=fluxadu[bok_id][is_sdss] fluxadu=table.join(fluxadu,sdss,join_type='left') fluxadu=fluxadu.filled(fill_value=0) # get coordinates for aperture photometry aper_coord=np.array([fluxadu['bok_x_window_'+filt],fluxadu['bok_y_window_'+filt]]).transpose() print('Aperture Photometry ....') aper_prop=phot.aperture_photometry(aper_coord,apertures=apertures,sky_annulus=sky_annuli,aper_type='circle',sky_type='global') aper_prop=Table(aper_prop) rename_table(aper_prop,prefix='bok_',postfix='_'+filt) del_cols=['id','x_in','y_in'] for col in del_cols: aper_prop.remove_column('bok_'+col+'_'+filt) fluxadu.add_columns(aper_prop.columns.values()) #re-order the objects fluxadu[bok_id]=np.arange(len(fluxadu)) if args.save_model: tmpdata=fluxadu.copy() cols=['id','x_psf','x_psf_error','y_psf','y_psf_error','x_dev','x_dev_error','y_dev','y_dev_error','re_dev','re_dev_error','ab_dev','ab_dev_error','theta_dev','theta_dev_error','x_exp','x_exp_error','y_exp','y_exp_error','re_exp','re_exp_error','ab_exp','ab_exp_error','theta_exp','theta_exp_error','chi2_psf','chi2_dev','chi2_exp','psf_fwhm','flux_psf','flux_dev','flux_exp'] for col in cols: tmpdata.rename_column('bok_'+col+'_'+filt,col) model,residual=phot.create_model_residual(tmpdata,psf,nradius=nradius) fits.writeto(fitsname+'-model.fits',model.astype('float32'),header=head,clobber=True,output_verify='fix+warn') fits.writeto(fitsname+'-residual.fits',residual.astype('float32'),header=head,clobber=True,output_verify='fix+warn') #del_cols=['x_psf','x_psf_error','y_psf','y_psf_error','x_dev','x_dev_error','y_dev','y_dev_error','re_dev','re_dev_error','ab_dev','ab_dev_error','theta_dev','theta_dev_error','x_exp','x_exp_error','y_exp','y_exp_error','re_exp','re_exp_error','ab_exp','ab_exp_error','theta_exp','theta_exp_error'] #for col in del_cols: # fluxadu.remove_column('bok_'+col+'_'+filt) fluxfile=fitsname+'-fluxadu.fits' if os.path.isfile(fluxfile): os.remove(fluxfile) fluxadu.write(fluxfile,format='fits') #fits.writeto(fluxfile,fluxadu.as_array(),clobber=True,output_verify='fix+warn') ind_head=head.index('RA-OBS') f=fits.open(fluxfile,mode='update') f[1].header["apersize"]=('3,4,5,6,8,10,13,16,20,25,30,40','aperture radii in pixels') f[1].header.extend(head.cards[ind_head:],bottom=True) f.flush(output_verify='fix+warn') f.close() # calibrate flux and aperture corrections fluxcalib=calibrate_fluxadu(fluxadu,head) if fluxcalib == False: continue plotname=None if args.plot: plotname=fitsname fluxcor=magnitude_correction(fluxcalib,head,elp_lim=1.0,sigma=2.5,plot_name=plotname) corfile=fitsname+'-fluxcor.fits' if os.path.isfile(corfile): os.remove(corfile) fluxcor.write(corfile,format='fits') #fits.writeto(corfile,fluxcor.as_array(),clobber=True,output_verify='fix+warn') f=fits.open(corfile,mode='update') f[1].header["apersize"]=('3,4,5,6,8,10,13,16,20,25,30,40','aperture radii in pixels') f[1].header.extend(head.cards[ind_head:],bottom=True) f.flush(output_verify='fix+warn') f.close() print('Total processing time for '+fitsname,time.time()-start)
# journal_counts = {} # for journal in os.listdir(main_dir): # journal_counts[journal] = sum([len(files) for r, d, files in os.walk(main_dir + "/" + journal)]) # print journal_counts final_table = Table() final_columns = [] stat_keys = [] with open('final_analysis_new_java.txt', 'r') as final_analysis: for line in final_analysis.readlines(): for jour in journal_counts.keys(): if jour in line: dict_journ = ast.literal_eval( line[len(jour) + 3:][:-1].translate(None, ')')) for key, value in dict_journ.iteritems(): dict_journ[key] = value / journal_counts[jour] # print (jour, dict_journ) stat_keys = dict_journ.keys() distri_column = Column(name=jour, data=list(dict_journ.values())) final_columns.append(distri_column) print("--- %s seconds ---" % (time.time() - start_time)) stat_column = Column(name='stat. Methods', data=stat_keys) final_table.add_column(stat_column) final_table.add_columns(final_columns) # ascii.write(final_table, 'final_distribution_new_second.dat', format='fixed_width') ascii.write(final_table, 'final_distribution_new_java.dat', format='fixed_width') # ======================================= END: Generate percentage distribution for all journals ======================================
def load_plot_etc_target_type(metadata_path, spectra_path, test_inds, target_type, no_plot=False, save_out=False, restrict_delta=False, use_spca=False, use_pca=False): obs_metadata = trim_observation_metadata(load_observation_metadata(metadata_path)) if use_filter_split: c_sources, c_mixing, c_exposures, c_wavelengths, c_filter_split_arr = load_spectra_data(spectra_path, target_type=target_type, filter_str='nonem', use_spca=use_spca, use_pca=use_pca) c_sources_e, c_mixing_e, c_exposures_e, c_wavelengths_e, c_filter_split_arr_e = load_spectra_data(spectra_path, target_type=target_type, filter_str='em', use_spca=use_spca, use_pca=use_pca) else: c_sources, c_mixing, c_exposures, c_wavelengths, c_filter_split_arr = load_spectra_data(spectra_path, target_type=target_type, filter_str='both', use_spca=use_spca, use_pca=use_pca) reduced_obs_metadata = obs_metadata[np.in1d(obs_metadata['EXP_ID'], c_exposures)] reduced_obs_metadata.sort('EXP_ID') sorted_inds = np.argsort(c_exposures) if use_filter_split: sorted_e_inds = np.argsort(c_exposures_e) if not linear_only: if reg_type == 'etr': rfr = ensemble.ExtraTreesRegressor(n_estimators=n_estimators, min_samples_split=min_samples_split, random_state=rfr_random_state, n_jobs=-1, verbose=False, bootstrap=bootstrap) if use_filter_split: rfr_e = ensemble.ExtraTreesRegressor(n_estimators=n_estimators, min_samples_split=min_samples_split, random_state=rfr_random_state, n_jobs=-1, verbose=False, bootstrap=bootstrap) else: rfr = ensemble.RandomForestRegressor(n_estimators=n_estimators, min_samples_split=min_samples_split, random_state=rfr_random_state, n_jobs=-1, verbose=False, bootstrap=bootstrap) if use_filter_split: rfr_e = ensemble.RandomForestRegressor(n_estimators=n_estimators, min_samples_split=min_samples_split, random_state=rfr_random_state, n_jobs=-1, verbose=False, bootstrap=bootstrap) if include_knn: knn = neighbors.KNeighborsRegressor(weights='distance', n_neighbors=10, p=64) if use_filter_split: knn_e = neighbors.KNeighborsRegressor(weights='distance', n_neighbors=10, p=64) if include_linear: linear = Linear(fit_intercept=True, copy_X=True, n_jobs=-1) poly_2_linear = Pipeline([('poly', PolynomialFeatures(degree=2)), ('linear', Linear(fit_intercept=True, copy_X=True, n_jobs=-1))]) poly_3_linear = Pipeline([('poly', PolynomialFeatures(degree=3)), ('linear', Linear(fit_intercept=True, copy_X=True, n_jobs=-1))]) poly_4_linear = Pipeline([('poly', PolynomialFeatures(degree=4)), ('linear', Linear(fit_intercept=True, copy_X=True, n_jobs=-1))]) if use_filter_split: linear_e = Linear(fit_intercept=True, copy_X=True, n_jobs=-1) poly_2_linear_e = Pipeline([('poly', PolynomialFeatures(degree=2)), ('linear', Linear(fit_intercept=True, copy_X=True, n_jobs=-1))]) poly_3_linear_e = Pipeline([('poly', PolynomialFeatures(degree=3)), ('linear', Linear(fit_intercept=True, copy_X=True, n_jobs=-1))]) poly_4_linear_e = Pipeline([('poly', PolynomialFeatures(degree=4)), ('linear', Linear(fit_intercept=True, copy_X=True, n_jobs=-1))]) reduced_obs_metadata.remove_column('EXP_ID') md_len = len(reduced_obs_metadata) var_count = len(reduced_obs_metadata.columns) X_arr = np.array(reduced_obs_metadata).view('f8').reshape((md_len,-1)) ica = None if not use_spca and not use_pca: if use_filter_split: ica = ICAize.unpickle_FastICA(path=spectra_path, target_type=target_type, filter_str='nonem') ica_e = ICAize.unpickle_FastICA(path=spectra_path, target_type=target_type, filter_str='em') else: ica = ICAize.unpickle_FastICA(path=spectra_path, target_type=target_type, filter_str='both') elif use_spca: ica = ICAize.unpickle_SPCA(path=spectra_path, target_type=target_type) else: if use_filter_split: ica = ICAize.unpickle_PCA(path=spectra_path, target_type=target_type, filter_str='nonem') ica_e = ICAize.unpickle_PCA(path=spectra_path, target_type=target_type, filter_str='em') else: ica = ICAize.unpickle_PCA(path=spectra_path, target_type=target_type, filter_str='both') spectra_dir_list = os.listdir(spectra_path) ################################################################ results = None for test_ind in test_inds: test_X = X_arr[test_ind] train_X = np.vstack( [X_arr[:test_ind], X_arr[test_ind+1:]] ) test_y = (c_sources[sorted_inds])[test_ind] train_y = np.vstack( [(c_sources[sorted_inds])[:test_ind], (c_sources[sorted_inds])[test_ind+1:]] ) if use_filter_split: test_y_e = (c_sources_e[sorted_e_inds])[test_ind] train_y_e = np.vstack( [(c_sources_e[sorted_e_inds])[:test_ind], (c_sources_e[sorted_e_inds])[test_ind+1:]] ) if scale: scaler = StandardScaler(with_std=scale_std) train_X = scaler.fit_transform(train_X) test_X = scaler.transform(test_X) title_str = "exp{}, {}".format(c_exposures[sorted_inds[test_ind]], target_type) if not linear_only: rfr.fit(X=train_X, y=train_y) if use_filter_split: rfr_e.fit(X=train_X, y=train_y_e) if include_knn: knn.fit(X=train_X, y=train_y) if user_filter_split: knn_e.fit(X=train_X, y=train_y_e) if include_linear: linear.fit(train_X, train_y) poly_2_linear.fit(train_X, train_y) if order_3: poly_3_linear.fit(train_X, train_y) if order_4: poly_4_linear.fit(train_X, train_y) if use_filter_split and include_linear: linear_e.fit(train_X, train_y_e) poly_2_linear_e.fit(train_X, train_y_e) if order_3: poly_3_linear_e.fit(train_X, train_y_e) if order_4: poly_4_linear_e.fit(train_X, train_y_e) print test_ind, c_exposures[sorted_inds[test_ind]], data = None actual = None mask = None delta_mask = None ivar = None for file in spectra_dir_list: if fnmatch.fnmatch(file, "stacked_sky_*exp{}.csv".format(c_exposures[sorted_inds[test_ind]])): data = Table.read(os.path.join(spectra_path, file), format="ascii.csv") ivar = data['ivar'] mask = (data['ivar'] == 0) delta_mask = mask.copy() if restrict_delta: if restrict_color == 'blue': delta_mask[2700:] = True else: delta_mask[:2700] = True actual = data['flux'] break if actual is None: continue if not linear_only: rfr_prediction = rfr.predict(test_X) if not use_spca and not use_pca: rfr_predicted = ica.inverse_transform(rfr_prediction, copy=True) else: rfr_predicted = np.zeros( (1, ica.components_.shape[1]) ) rfr_predicted[0,:] = np.sum(rfr_prediction.T * ica.components_, 0) if use_filter_split: rfr_e_prediction = rfr_e.predict(test_X) if not use_spca and not use_pca: rfr_e_predicted = ica_e.inverse_transform(rfr_e_prediction, copy=True) else: rfr_e_predicted = np.zeros( (1, ica_e.components_.shape[1]) ) rfr_e_predicted[0,:] = np.sum(rfr_e_prediction.T * ica_e.components_, 0) rfr_predicted = rfr_predicted + rfr_e_predicted rfr_delta = rfr_predicted[0] - actual if not no_plot: plt.plot(c_wavelengths[~mask], rfr_predicted[0][~mask]) plt.plot(c_wavelengths[~mask], actual[~mask]) plt.plot(c_wavelengths[~mask], rfr_delta[~mask]) if not no_plot: plt.plot(c_wavelengths, [0]*len(c_wavelengths)) err_term = np.sum(np.power(rfr_delta[~delta_mask], 2))/len(c_wavelengths[~delta_mask]) err_sum = np.sum(rfr_delta[~delta_mask])/len(rfr_delta[~delta_mask]) red_chi = np.sum(np.power(rfr_delta[~delta_mask], 2)*ivar[~delta_mask])/(len(c_wavelengths[~delta_mask])-var_count-1) if not no_plot: plt.legend(['Predicted', 'Actual', 'Delta {:0.5f}'.format(err_term)]) plt.tight_layout() plt.title("Random Forest Regressor: {}".format(title_str)) plt.show() plt.close() print err_term, red_chi, err_sum, if include_knn: knn_prediction = knn.predict(test_X) if not use_spca and not use_pca: knn_predicted = ica.inverse_transform(knn_prediction, copy=True) else: knn_predicted = np.zeros( (1, ica.components_.shape[1]) ) knn_predicted[0,:] = np.sum(knn_prediction.T * ica.components_, 0) if use_filter_split: knn_e_prediction = knn_e.predict(test_X) if not use_spca and not use_pca: knn_e_predicted = ica_e.inverse_transform(knn_e_prediction, copy=True) else: knn_e_predicted = np.zeros( (1, ica_e.components_.shape[1]) ) knn_e_predicted[0,:] = np.sum(knn_e_prediction.T * ica_e.components_, 0) knn_predicted = knn_predicted + knn_e_predicted if not no_plot: plt.plot(c_wavelengths[~mask], knn_predicted[0][~mask]) plt.plot(c_wavelengths[~mask], actual[~mask]) knn_delta = knn_predicted[0] - actual err_term = np.sum(np.power(knn_delta[~delta_mask], 2))/len(c_wavelengths[~delta_mask]) err_sum = np.sum(knn_delta[~delta_mask])/len(knn_delta[~delta_mask]) red_chi = np.sum(np.power(knn_delta[~delta_mask], 2)*ivar[~delta_mask])/(len(c_wavelengths[~delta_mask])-var_count-1) if not no_plot: plt.plot(c_wavelengths[~mask], knn_delta[~mask]) plt.plot(c_wavelengths, [0]*len(c_wavelengths)) plt.legend(['Predicted', 'Actual', 'Delta {:0.5f}'.format(err_term)]) plt.tight_layout() plt.title("Good 'ol K-NN: {}".format(title_str)) plt.show() plt.close() print err_term, red_chi, err_sum, if include_linear: poly_1_prediction = linear.predict(test_X) if not use_spca and not use_pca: poly_1_predicted = ica.inverse_transform(poly_1_prediction, copy=True) else: poly_1_predicted = np.zeros( (1, ica.components_.shape[1]) ) poly_1_predicted[0,:] = np.sum(poly_1_prediction.T * ica.components_, 0) if use_filter_split: poly_1_e_prediction = linear.predict(test_X) if not use_spca and not use_pca: poly_1_e_predicted = ica_e.inverse_transform(poly_1_e_prediction, copy=True) else: poly_1_e_predicted = np.zeros( (1, ica_e.components_.shape[1]) ) poly_1_e_predicted[0,:] = np.sum(poly_1_e_prediction.T * ica_e.components_, 0) poly_1_predicted = poly_1_predicted + poly_1_e_predicted poly_1_delta = poly_1_predicted[0] - actual if not no_plot: plt.plot(c_wavelengths[~mask], poly_1_predicted[0][~mask]) plt.plot(c_wavelengths[~mask], actual[~mask]) err_term = np.sum(np.power(poly_1_delta[~delta_mask], 2))/len(c_wavelengths[~delta_mask]) err_sum = np.sum(poly_1_delta[~delta_mask])/len(poly_1_delta[~delta_mask]) red_chi = np.sum(np.power(poly_1_delta[~delta_mask], 2)*ivar[~delta_mask])/(len(c_wavelengths[~delta_mask])-var_count-1) if not no_plot: plt.plot(c_wavelengths[~mask], poly_1_delta[~mask]) plt.plot(c_wavelengths, [0]*len(c_wavelengths)) plt.legend(['Predicted', 'Actual', 'Delta {:0.5f}'.format(err_term)]) plt.tight_layout() plt.title("Poly 1: {}".format(title_str)) plt.show() plt.close() print err_term, red_chi, err_sum, poly_2_prediction = poly_2_linear.predict(test_X) if not use_spca and not use_pca: poly_2_predicted = ica.inverse_transform(poly_2_prediction, copy=True) else: poly_2_predicted = np.zeros( (1, ica.components_.shape[1]) ) poly_2_predicted[0,:] = np.sum(poly_2_prediction.T * ica.components_, 0) poly_2_delta = poly_2_predicted[0] - actual if not no_plot: plt.plot(c_wavelengths[~mask], poly_2_predicted[0][~mask]) plt.plot(c_wavelengths[~mask], actual[~mask]) err_term = np.sum(np.power(poly_2_delta[~delta_mask], 2))/len(c_wavelengths[~delta_mask]) err_sum = np.sum(poly_2_delta[~delta_mask])/len(poly_2_delta[~delta_mask]) red_chi = np.sum(np.power(poly_2_delta[~delta_mask], 2)*ivar[~delta_mask])/(len(c_wavelengths[~delta_mask])-var_count-1) if not no_plot: plt.plot(c_wavelengths[~mask], poly_2_delta[~mask]) plt.plot(c_wavelengths, [0]*len(c_wavelengths)) plt.legend(['Predicted', 'Actual', 'Delta {:0.5f}'.format(err_term)]) plt.tight_layout() plt.title("Poly 2: {}".format(title_str)) plt.show() plt.close() print err_term, red_chi, err_sum, err_ind =+ 1 if order_3: poly_3_prediction = poly_3_linear.predict(test_X) if not use_spca and not use_pca: poly_3_predicted = ica.inverse_transform(poly_3_prediction, copy=True) else: poly_3_predicted = np.zeros( (1, ica.components_.shape[1]) ) poly_3_predicted[0,:] = np.sum(poly_3_prediction.T * ica.components_, 0) poly_3_delta = poly_3_predicted[0] - actual if not no_plot: plt.plot(c_wavelengths[~mask], poly_3_predicted[0][~mask]) plt.plot(c_wavelengths[~mask], actual[~mask]) err_term = np.sum(np.power(poly_3_delta[~delta_mask], 2))/len(c_wavelengths[~delta_mask]) err_sum = np.sum(poly_3_delta[~delta_mask])/len(poly_3_delta[~delta_mask]) red_chi = np.sum(np.power(poly_3_delta[~delta_mask], 2)*ivar[~delta_mask])/(len(c_wavelengths[~delta_mask])-var_count-1) if not no_plot: plt.plot(c_wavelengths[~mask], poly_3_delta[~mask]) plt.plot(c_wavelengths, [0]*len(c_wavelengths)) plt.legend(['Predicted', 'Actual', 'Delta {:0.5f}'.format(err_term)]) plt.tight_layout() plt.title("Poly 3: {}".format(title_str)) plt.show() plt.close() print err_term, red_chi, err_sum, err_ind =+ 1 if order_4: poly_4_prediction = poly_4_linear.predict(test_X) if not use_spca and not use_pca: poly_4_predicted = ica.inverse_transform(poly_4_prediction, copy=True) else: poly_4_predicted = np.zeros( (1, ica.components_.shape[1]) ) poly_4_predicted[0,:] = np.sum(poly_4_prediction.T * ica.components_, 0) poly_4_delta = poly_4_predicted[0] - actual if not no_plot: plt.plot(c_wavelengths[~mask], poly_4_predicted[0][~mask]) plt.plot(c_wavelengths[~mask], actual[~mask]) err_term = np.sum(np.power(poly_4_delta[~delta_mask], 2))/len(c_wavelengths[~delta_mask]) err_sum = np.sum(poly_4_delta[~delta_mask])/len(poly_4_delta[~delta_mask]) red_chi = np.sum(np.power(poly_4_delta[~delta_mask], 2)*ivar[~delta_mask])/(len(c_wavelengths[~delta_mask])-var_count-1) if not no_plot: plt.plot(c_wavelengths[~mask], poly_4_delta[~mask]) plt.plot(c_wavelengths, [0]*len(c_wavelengths)) plt.legend(['Predicted', 'Actual', 'Delta {:0.5f}'.format(err_term)]) plt.tight_layout() plt.title("Poly 4: {}".format(title_str)) plt.show() plt.close() print err_term, red_chi, err_sum, err_ind =+ 1 print if save_out: out_table = Table() wavelength_col = Column(c_wavelengths, name="wavelength", dtype=float) out_table.add_columns([wavelength_col]) if not linear_only: rf_col = Column(rfr_predicted[0], name="rf_flux", dtype=float) out_table.add_columns([rf_col]) if include_knn: knn_col = Column(knn_predicted[0], name="knn_flux", dtype=float) avg_col = Column(avg_predicted[0], name="avg_flux", dtype=float) out_table.add_columns([knn_col, avg_col]) if include_linear: poly_1_col = Column(poly_1_predicted[0], name="poly_1_flux", dtype=float) poly_2_col = Column(poly_2_predicted[0], name="poly_2_flux", dtype=float) out_table.add_columns([poly_1_col, poly_2_col]) if order_3: poly_3_col = Column(poly_3_predicted[0], name="poly_3_flux", dtype=float) out_table.add_columns([poly_3_col]) if order_4: poly_4_col = Column(poly_4_predicted[0], name="poly_4_flux", dtype=float) out_table.add_columns([poly_4_col]) mask_col = Column(~mask, name="mask_col", dtype=bool) out_table.add_columns([mask_col]) out_table.write("predicted_sky_exp{}.csv".format(c_exposures[sorted_inds[test_ind]]), format="ascii.csv")
def bal_cluster(line, k, g): """ unsupervised clustering using KMeans. param: line: which line to use for the clustering features k: number of clusters g0: normalized EW, Vmin, Vmax g1: normalized EW, Vmin, dV g2: normalized EW, Vmax, dV g3: normalized EW, Vmin, EW/dv g4: normalized EW, Vmax, EW/dv g5: normalized EW, Vmin, Vmax, dV g6: normalized EW, Vmin, Vmax, EW/dv g7: normalized Vmin, Vmax, EW/dV g8: normalized EW, dV, EW/dV g9: normalized EW, dV g11: normalized EW, Vmax, BI0/dV g12: normalized EW, Vmin, Vmax, BI0/dV g13: normalized BI0, Vmax, BI0/dV g14: normalized BI0, Vmin, Vmax, BI0/dV g15: normalized BI0, dV, Vmax g16: normalized BI0, EW, Vmin, Vmax, dV, BI0/dV g17: normalized BI0, EW, Vmin, Vmax, BI0/dV g18: normalized EW, Vmin, Vmax, dV/EW """ clstr_name= line+str(k) data= Table.read('myBALs.fits') #data= Table.read('myBALCat_xtra.csv', format= 'ascii.csv') #selec the sample: line has an absorption trough: BI0 >0 , S/N >3, and a redshift cutoff to restrict the bandwidth if line == "MgII": z1= 1.1 z2= 2.2 lum= "logF2500" else: z1= 1.79 z2= 3.7 lum= "logF1400" s= data[(data[line+'-BIO'] >0) & (data['SN1700'] >3) & (data['Z_HW'] >z1) & (data['Z_HW'] <z2)]# & (data[lum] !=-999)] print "sample has", len(s), "objects" #features """ redshift= s['Z_HW'] bi= s[line+'-BIO'] ew= s[line+'-EW'] vmin= s[line+'-vmin'] vmax= s[line+'-vmax'] fdeep= s[line+'-fdeep'] """ s['Z_HW'].fill_value= -999 redshift= s['Z_HW'].filled() #redshift s[line+'-BI'].fill_value= -999 bi= s[line+'-BI'].filled() # balnicity: integration 3000-25000 s[line+'-BIO'].fill_value= -999 bi0= s[line+'-BIO'].filled() # modified balnicity: integration 0-25000 s[line+'-EW'].fill_value= -999 ew= s[line+'-EW'].filled() # restframe absorption EW s[line+'-vmin'].fill_value= -999 vmin= s[line+'-vmin'].filled() # minimum velocity s[line+'-vmax'].fill_value= -999 vmax= s[line+'-vmax'].filled() # maximum velocity s[line+'-fdeep'].fill_value= -999 fdeep= s[line+'-fdeep'].filled() s[lum].fill_value= -999 cl= s[lum].filled() # Log of 1400 or 2500 monochromatic luminosity s['SDSSName'].fill_value= -999 names= s['SDSSName'].filled() # SDSS name dv= vmax- vmin # delta v dd= ew/dv # some sort of an estimate of the trough depth dp= bi0/dv # another dimensionless parameter to estimate trough depth dc4= dv/ew # another param to estimate the depth of CIV #standardize (normalized) parameters before using them in clustering ew_n= (ew - mean(ew))/std(ew) vmin_n= (vmin - mean(vmin))/std(vmin) vmax_n= (vmax - mean(vmax))/std(vmax) dv_n = (dv - mean(dv))/std(dv) dd_n= (dd- mean(dd))/std(dd) bi_n= (bi0 - mean(bi0))/std(bi0) dp_n= (dp - mean(dp))/std(dp) dc4_n= (dc4- mean(dc4))/std(dc4) #cl_n= (cl - mean(cl))/std(cl) if g== 'g0': f = [ew_n, vmin_n, vmax_n] colnames= ('EW', 'Vmin', 'Vmax') datatype= ('float64', 'float64', 'float64') elif g== 'g1': f = [ew_n, vmin_n, dv_n] colnames= ('EW', 'Vmin', 'dV') datatype= ('float64', 'float64', 'float64') elif g== 'g2': f = [ew_n, vmax_n, dv_n] colnames= ('EW', 'Vmax', 'dV') datatype= ('float64', 'float64', 'float64') elif g== 'g3': f= [ew_n, vmin_n, dd_n] colnames= ('EW', 'Vmin', 'EW_dV') datatype= ('float64', 'float64', 'float64') elif g== 'g4': f= [ew_n, vmax_n, dd_n] colnames= ('EW', 'Vmax', 'EW_dV') datatype= ('float64', 'float64', 'float64') elif g== 'g5': f = [ew_n, vmin_n, vmax_n, dv_n] colnames= ('EW', 'Vmin', 'Vmax', 'dV') datatype= ('float64', 'float64', 'float64', 'float64') elif g== 'g6': f= [ew_n, vmin_n, vmax_n, dd_n] colnames= ('EW', 'Vmin', 'Vmax', 'EW_dV') datatype= ('float64', 'float64', 'float64', 'float64') elif g== 'g7': f= [vmin_n, vmax_n, dd_n] colnames= ('Vmin', 'Vmax', 'EW_dV') datatype= ('float64', 'float64', 'float64') elif g== 'g8': f= [ew_n, dv_n, dd_n] colnames= ('EW', 'dV', 'EW_dV') datatype= ('float64', 'float64', 'float64') elif g== 'g9': f= [ew_n, dv_n] colnames= ('EW', 'dV') datatype= ('float64', 'float64') elif g== 'g11': f= [ew_n, vmax_n, dp_n] colnames= ('EW', 'Vmax', 'BI0_dV') datatype= ('float64', 'float64', 'float64') elif g== 'g12': f= [ew_n, vmin_n, vmax_n, dp_n] colnames= ('EW', 'Vmin', 'Vmax', 'BI0_dV') datatype= ('float64', 'float64', 'float64', 'float64') elif g== 'g13': f= [bi_n, vmax_n, dp_n] colnames= ('BI0', 'Vmax', 'BI0_dV') datatype= ('float64', 'float64', 'float64') elif g== 'g14': f= [bi_n, vmin_n, vmax_n, dp_n] colnames= ('BI0', 'Vmin', 'Vmax', 'BI0_dV') datatype= ('float64', 'float64', 'float64', 'float64') elif g== 'g15': f= [bi_n, vmax_n, dv_n] colnames= ('BI0', 'Vmax', 'dV') datatype= ('float64', 'float64', 'float64') elif g== 'g16': f= [bi_n, ew_n, vmin_n, vmax_n, dv_n, dp_n] colnames= ('BI0', 'EW', 'Vmin', 'Vmax', 'dV', 'BI0_dV') datatype= ('float64', 'float64', 'float64', 'float64', 'float64', 'float64') elif g== 'g18': f= [ew_n, vmin_n, vmax_n, dc4_n] colnames= ['EW', 'Vmin', 'Vmax', 'dCIV'] datatype= ('float64', 'float64', 'float64', 'float64') qs= np.column_stack(param for param in f) # 2D array to do clustering on #do the clustering kmeans= KMeans(init= 'k-means++', n_clusters= k, n_init= 10) kmeans.fit(qs) labels= kmeans.predict(qs) sc= metrics.silhouette_score(qs, labels) cntrs= kmeans.cluster_centers_ ## file to save centroids and silhouette scores for every run param_f= open("./clusters/"+g+"/"+clstr_name+"param.txt", 'wr') param_f.write(g+", \t features: "+str(colnames)+ "\n") param_f.write("Silhouette score= "+ str(sc)+ "\n") param_f.write("centroids: "+"\n"+str(cntrs)+ "\n") param_f.close() print "Silhouette score= ", sc print "centroids: "+"\n", cntrs # save the results in a FITS table # table with clustering results clstr_tbl= Table([qs[:,b] for b in range(len(f))], names= colnames, dtype= datatype) print "table has", len(clstr_tbl), "objects" lab= Column(name= 'label', data= labels) clstr_tbl.add_columns([lab, names, redshift]) clstr_tbl.write("./clusters/"+g+"/"+clstr_name+"clstrs.fits", format= 'fits') return
from glob import glob from os import path as pth from astropy.io import fits, ascii from astropy.table import Table files = glob("wfc3_ir_f*_mjd_*_syn.fits") output_format = "../TER_filter_{}.dat" for fname in files: data = fits.getdata(fname) tbl = Table(data) tbl.columns[0].name = "wavelength" tbl.columns[1].name = "transmission" new_table = Table() new_table.add_columns([tbl["wavelength"], tbl["transmission"]]) cmts = ["author: Kieran Leschinski", "source: https://stsynphot.readthedocs.io/en/latest/stsynphot/data_hst.html", "description: WFC3 IR Filter Curve", "date_created: 2019-11-19", "date_modified: 2019-11-19", "status: measured", "orig_filename: {}".format(fname), "wavelength_unit: Angstrom"] new_table.meta["comments"] = cmts new_table.write(output_format.format(fname[8:13].upper()), format="ascii.fixed_width", overwrite=True, delimiter=None)
def create_young_pop(imf,a,difflen): #Remove last element of parameter of array as this is not an IMF parameter # a = a[0:len(a)-1] #Possible imf keywords: Chabrier-log ,Kroupa nsamp = 250 #Full sample of objects should be same length as background-corrected observed: 11719 nobjs = difflen t,e = [],[] full_m = np.linspace(0.07,0.4,nsamp) start = time.time() if 'chab' in imf: for i in range(nsamp): #Generate FULL distribution of masses and probabilities, to a fine resolution: e = number of each mass, need to normalise? e.append(chab_log_imf(full_m[i],a)) elif 'kroup' in imf: for i in range(nsamp): e.append(kroupa_imf(full_m[i],a)) elif 'salp': for i in range(nsamp): e.append(salp_imf(full_m[i],a)) #Masses of objects e = [a/max(e) for a in e] young_obj_masses = choices(full_m,weights=e,k=nobjs) end = time.time() #print('Took',str(end-start)) mass_col = Column(young_obj_masses,name='Mass(msol)') #Extinctions of objects: random choices from a distribution with weights #av_dist = Table.read('serpens_extinction_probabilities/ss_av_converted_prob_density_higher_conversion.txt',format='ascii') #av_list = choices(av_dist['Av Value'],weights=av_dist['Frequency'],k=nobjs) #Extinctions: random choices without weights #alist = np.arange(10.0,30.25,0.25) alist = np.arange(0.0,5.0,0.25) av_list = np.random.choice(alist,nobjs) #av_list = [0 for i in range(nobjs)] av_col = Column(av_list,name='RAv') t = Table() #Option to have distance variation DIST = False if DIST: #Distances of objects - just use gaussian distribution for now dist = [438,11] dist_list = [] for i in range(nsamp*100): dist_list.append(np.random.normal(dist[0],dist[1])) dhist,dvals = np.histogram(dist_list,bins=10000) dvals = dvals[0:len(dvals)-1] d_list = choices(dvals,weights=dhist,k=nobjs) d_col = Column(d_list,name='Dist(pc)') t.add_columns([mass_col,av_col,d_col]) t.add_columns([mass_col,av_col]) start = time.time() t = add_spt(t,'Msun','Mass(msol)','y') end = time.time() #print('Took',str(end-start)) return t
def add_gaia(data): """ Add GAIA data to input structure, with 2MASS match and coordinate match to (cross-matched) GAIA reference file """ # get the GAIA data from both matches gaia_twomass, gaia_posn = getdata(data) # add new columns tab = Table(data) in_names = ('source_id', 'parallax', 'parallax_error', 'pmra', 'pmra_error', 'pmdec', 'pmdec_error', 'phot_g_mean_mag', 'phot_bp_mean_mag', 'phot_rp_mean_mag', 'a_g_val', 'e_bp_min_rp_val', 'radial_velocity', 'radial_velocity_error', 'r_est', 'r_lo', 'r_hi') dtypes = ('i8', 'f8', 'f8', 'f8', 'f8', 'f8', 'f8', 'f4', 'f4', 'f4', 'f4', 'f4', 'f8', 'f8', 'f8', 'f8', 'f8') out_names = [] for name in in_names: out_names.append(('gaia_' + name).upper()) # initialize newcols = Table(np.zeros([len(tab), len(out_names)]) - 9999., names=out_names, dtype=dtypes) # for source_id, default to 0, not -9999. newcols['GAIA_SOURCE_ID'] = 0 # rename targetting proper motions to avoid confusion! try: tab.rename_column('PMRA', 'TARG_PMRA') except: pass try: tab.rename_column('PMDEC', 'TARG_PMDEC') except: pass try: tab.rename_column('PM_SRC', 'TARG_PM_SRC') except: pass # add unpopulated columns tab.add_columns(newcols.columns.values()) # remove dups in GAIA twomass in favor of brightest print('number in GAIA-2MASS xmatch catalog: ', len(gaia_twomass), len(set(gaia_twomass['original_ext_source_id']))) ind = [] for tid in set(gaia_twomass['original_ext_source_id']): j = np.where(gaia_twomass['original_ext_source_id'] == tid)[0] if len(j) > 1: ii = np.argsort(gaia_twomass['phot_rp_mean_mag'][j]) ind.append(j[ii[0]]) print('duplicate 2MASS: ', gaia_twomass['phot_rp_mean_mag'][j[ii]]) else: ind.append(j) # read gaia 2MASS matched file, match by 2MASS ID, and populate while True: # loop for matches since we may have repeats and want them all matched j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0] print('Number missing gaia_source_id: ', len(j)) if len(j) == 0: break m1, m2 = match.match( np.core.defchararray.replace(tab['APOGEE_ID'][j], b'2M', b''), gaia_twomass['original_ext_source_id']) print('Number matched by 2MASS: ', len(m1)) if len(m1) == 0: break for inname, outname in zip(in_names, out_names): tab[outname][j[m1]] = gaia_twomass[inname][m2] j = np.where(tab['GAIA_SOURCE_ID'] > 0)[0] print('number of unique APOGEE_ID matches: ', len(set(tab['APOGEE_ID'][j]))) j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0] print('missing sources after 2MASS matches: ', len(j)) h = htm.HTM() # now do a positional match, take the brightest object within 3 arcsec (which is the max from the GAIA crossmatch) maxrad = 3. / 3600. m1, m2, rad = h.match(tab['RA'][j], tab['DEC'][j], gaia_posn['ra'], gaia_posn['dec'], maxrad, maxmatch=10) for m in set(m1): jj = np.where(m1 == m)[0] ii = np.argsort(gaia_posn['phot_rp_mean_mag'][m2[jj]]) for inname, outname in zip(in_names, out_names): tab[outname][j[m]] = gaia_posn[inname][m2[jj[ii[0]]]] j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0] print('missing sources after second match: ', len(j)) # replace NaNs for name in out_names: bd = np.where(np.isnan(tab[name]))[0] tab[name][bd] = -9999. return tab
def unify_cluster_catalog_GMBCG(catalog, cosmo): """For a given catalog of GMBCG clusters, it unifies it to the required properties (RA, DEC, redshift, richness, Mass, etc) as used in our analysis. It returns a new catalog of only relevant information.""" clusters = Table() #Unify name name = Column(name='NAME_ORIG',data=catalog['NAME']) clusters.add_column(name) #Unify ID, add column with original indices for hao objid = np.array([int(i+1) for i in range(len(catalog))]) objid = Column(name = 'objid_gmbcg', data=objid) clusters.add_column(objid) #Unify RA/DEC ra = Column(name='ra_d' ,data=catalog['RA']) dec = Column(name='dec_d',data=catalog['DEC']) clusters.add_columns([ra,dec]) #unify best redshifts (spec-z when available, otherwise photo-z) z = np.where(catalog['SPZ']!=0,catalog['SPZ'],catalog['PHOTOZ']) z = Column(name = 'redshift', data=z) clusters.add_column(z) z_err = np.where(catalog['SPZ']!=0,0.0002,catalog['PHOTOZ_ERR']) z_err = Column(name='redshift_err',data=z_err) clusters.add_column(z_err) #add photometric and spectroscopic redshifts zsp = Column(name='zsp',data=np.where(catalog['SPZ']>0,catalog['SPZ'],-1)) zph = Column(name='zph',data=catalog['PHOTOZ']) zph_err = Column(name='zph_err',data=catalog['PHOTOZ_ERR']) clusters.add_columns([zsp,zph,zph_err]) #add flag_spec flag_zsp = (catalog['SPZ'] > 0)*1. flag_zsp = Column(name='flag_zsp',data=flag_zsp) clusters.add_column(flag_zsp) #create unified richness for hao richness = np.where(catalog['WEIGHTOK'],catalog['GM_NGALS_WEIGHTED'],catalog['GM_SCALED_NGALS']) richness = Column(name = 'richness', data=richness) clusters.add_column(richness) #add mass estimate #add column with cluster mass using calibration given by http://gmbcg.blogspot.com/ #cluster mass = 8.8e13*((0.66*Ngals+9.567)/20.)^1.28 (h^-1 M_sun) mass = 8.8e13 * np.power((0.66 * clusters['richness'] + 9.567)/20,1.28)/(cosmo.H0/100) # in M_sun/h mass = Column(name = 'mass', data=mass) clusters.add_column(mass) #add column with r200 in Mpc rho_200 = 200 * cosmo.critical_density(clusters['redshift']).to('Msun/Mpc3') #in Msun/Mpc3 mass_200 = clusters['mass']*u.Msun # in Msun r_200 = np.power(mass_200/rho_200/(4*np.pi/3.), 1/3.).value #in Mpc clusters['r200_mpc'] = r_200 # give unified name name = [give_name(ra,dec) for ra,dec in zip(clusters['ra_d'],clusters['dec_d'])] clusters['name'] = name return clusters
def fitsextract(input, header=None, stride=[1,1,1], keepref=True, keepnan=True, zselect=None, col_lbl='imdat', ra_gc=None, dec_gc=None, pa=0, inc=0, ortlabel='default', bunit=None, first=False, use_hexgrid=False, sidelen=2, starting_angle=0, precision=2, header_hex=None, hexgrid_output=None): """ Sample data from an image into an AstroPy table indexed by coordinates. Simple approach taking every nth pixel along each image axis. Pseudocubes are handled as separate images and are detected by a blank value for CTYPE3 in the header. Parameters ---------- input : str or `~numpy.ndarray` The input data to reproject. This can be: * The name of a FITS file * A numpy array (in which case header must be separately given) header : `~astropy.io.fits.Header` object Header corresponding to the input array. Must be provided if the input is a numpy array. stride : tuple of ints, optional step size to select pixels along each axis. Axes are ordered using the FITS convention, not numpy convention (i.e. velaxis last). Default is [1,1,1] to keep all pixels. Note: stride in z is ignored for pseudocubes. keepref : bool, optional If dropping pixels, try to ensure that the reference pixel is kept. Default is True (keep the reference pixel). keepnan : bool, optional If False, the output table drops rows which are all-NaN. Default is True (keep the NaNs). zselect : list of ints, optional Indices of planes in cube/pseudocube to be selected. Default is to keep all planes. col_lbl : string or list of strings, optional Column label for the data values, can be list corresponding to each plane to be selected, for CALIFA pseudocubes. Default is "imdat"+possible integer. ra_gc : float, optional Right ascension of galaxy center. Used to determine polar coordinates of each sample in the plane of the galaxy. Default is reference RA value of the image. dec_gc : float, optional Declination of galaxy center. Used to determine polar coordinates of each sample in the plane of the galaxy. Default is reference DEC value of the image. pa : float, optional Position angle of the galaxy disk in degrees E of N. Used to determine polar coordinates of each sample in the plane of the galaxy. Default is 0 (due north). inc : float, optional Inclination of the galaxy disk in degrees from face-on. Used to determine polar coordinates of each sample in the plane of the galaxy. Default is 0 (face-on). ortlabel : string, optional String label describing the origin of the orientation parameters. bunit : string or list of strings, optional Astropy units for data values, can be list corresponding to each plane to be selected, for CALIFA pseudocubes. Default is obtained from BUNIT in the header. first : bool, optional True to write the coordinate columns, which only need to be done once per table. When combining multiple FITS images into a table, fitsextract should be called initially with first=True and then subsequently with first=False. Default is False. Returns ------- tab : `~astropy.Table` The selected pixels as a 1-D table. """ # Read the inputs if isinstance(input, np.ndarray): if header is None: raise TypeError("Header must be given if input is not FITS file") else: hdr = header data = input else: hdu = fits.open(input)[0] data = hdu.data hdr = hdu.header if header is None else header bunit = hdr['BUNIT'] if bunit is None else bunit w = WCS(hdr) print('RA ref is',w.wcs.crval[0]) print('DEC ref is',w.wcs.crval[1]) ndim = len(data.shape) iscube = (ndim > 2 and data.shape[ndim-3] > 1) if 'CTYPE3' in hdr.keys(): pseudo = (hdr['CTYPE3'] == '') else: pseudo = False # Create the coordinate columns. First case is for PPV cubes. if iscube and not pseudo: print('This is a data cube of shape', data.shape) data = np.squeeze(data) if len(data.shape) > 3: raise ('Data cannot be squeezed to three dimensions') naxis = 3 nz,ny,nx = data.shape ix,iy,iz = np.meshgrid(np.arange(nx), np.arange(ny), np.arange(nz), indexing='ij') tab = Table([np.ravel(ix), np.ravel(iy), np.ravel(iz)], names=('ix','iy','iz'),dtype=('i4','i4','i4')) tab['ix'].description = '0-based pixel index in x direction' tab['iy'].description = '0-based pixel index in y direction' tab['iz'].description = '0-based pixel index in z direction' # Get the pixel coordinates as tuples wcsin = (np.array([tab['ix'],tab['iy'],tab['iz']])).T # Next case is for 2D images or pseudo-cubes else: print('This is an image of shape', data.shape) data = np.squeeze(data) naxis = 2 if pseudo: nz,ny,nx = data.shape else: ny,nx = data.shape ix,iy = np.meshgrid(np.arange(nx), np.arange(ny), indexing='ij') tab = Table([np.ravel(ix), np.ravel(iy)], names=('ix','iy'),dtype=('i4','i4')) tab['ix'].description = '0-based pixel index in x direction' tab['iy'].description = '0-based pixel index in y direction' # Get the pixel coordinates as tuples wcsin = (np.array([tab['ix'],tab['iy']])).T # Reduce WCS to 2-D for pseudocubes wfix = w.sub(naxis) if first: wcsout = wfix.wcs_pix2world(wcsin,0) col_ra = Column(wcsout.T[0], name='ra_abs', dtype='f4', unit='deg', format='.6f', description='sample ra coord') col_dc = Column(wcsout.T[1], name='dec_abs', dtype='f4', unit='deg', format='.6f', description='sample dec coord') col_raoff = Column(wcsout.T[0]-w.wcs.crval[0], name='ra_off', dtype='f4', unit='deg', format='.6f', description='ra offset from ref pixel') col_dcoff = Column(wcsout.T[1]-w.wcs.crval[1], name='dec_off', dtype='f4', unit='deg', format='.6f', description='dec offset from ref pixel') if ra_gc is None: ra_gc = w.wcs.crval[0] if dec_gc is None: dec_gc = w.wcs.crval[1] r, theta = gc_polr(wcsout.T[0], wcsout.T[1], ra_gc, dec_gc, pa, inc) col_r = Column(r, name='rad_arc', dtype='f4', unit='arcsec', format='.3f', description='radius based on {}'.format(ortlabel)) col_th = Column(theta, name='azi_ang', dtype='f4', unit='deg', format='.3f', description='azang based on {}'.format(ortlabel)) tab.add_columns([col_ra,col_dc,col_raoff,col_dcoff,col_r,col_th]) if iscube and not pseudo: col_vel = Column(wcsout.T[2]/1000., name='vel', dtype='f4', unit='km/s', description='velocity in LSRK frame using radio def') tab.add_column(col_vel) # Flatten the cube into a 1-D table # Use order = 'F' because the velocity axis is first # For pseudocubes, each plane becomes a separate column if pseudo: zsel = range(nz) if zselect is None else zselect if not isinstance(bunit, list): bunit = [bunit]*len(zsel) if not isinstance(col_lbl, list): col_lbl = [col_lbl+str(i) for i in range(len(zsel))] for iz, sel in enumerate(zsel): try: desc = hdr['DESC_'+str(sel)].strip() except: desc = '' col_data = Column(np.ravel(data[sel],order='F'), name=col_lbl[iz], dtype='f4', description=desc, unit=bunit[iz]) tab.add_column(col_data) else: if isinstance(bunit, list): bunit = bunit[0] if isinstance(col_lbl, list): col_lbl = col_lbl[0] col_data = Column(np.ravel(data,order='F'), name=col_lbl, dtype='f4', unit=bunit) tab.add_column(col_data) idx = ['ix', 'iy', 'iz'] rem = [0, 0, 0] select = [[],[],[]] if not use_hexgrid: # Use stride to select the desired rows from the full table if keepref: for i in range(naxis): crpix = wfix.wcs.crpix[i] if crpix < 1 or crpix > hdr['naxis'+str(i+1)] or not crpix.is_integer(): print('Cannot use keepref on axis {}: crpix={}'.format(i+1,crpix)) continue else: print('Axis {}: crpix={}'.format(i+1,crpix)) rem[i] = int(crpix-1) % stride[i] print('Remainder: ',rem) for i in range(naxis): select[i]=np.where(tab[idx[i]] % stride[i] == rem[i])[0] xy = np.intersect1d(select[0], select[1]) if iscube and not pseudo: xyz = np.intersect1d(xy, select[2]) if len(xyz) < len(tab): newtab = tab[xyz] tab = newtab else: if len(xy) < len(tab): newtab = tab[xy] tab = newtab else: # print(f'reference pix is {wfix.wcs.crpix[:2]}') if iscube and not pseudo: iz_data = [] tab_length = 0 zlist = np.where(np.unique(tab['iz']) % stride[2] == rem[2])[0] for iz in zlist: if len(tab[tab['iz'] == iz]) == 0: continue sample = hex_sampler(tab[tab['iz'] == iz], sidelen, keepref, wfix.wcs.crpix[:2] - 1., w.wcs.crval[0], w.wcs.crval[1], ra_gc, dec_gc, pa, inc, starting_angle, precision, hexgrid_output) iz_data.append(sample) sample['iz'] = iz tab_length += len(sample) tab = tab[:tab_length] init = 0 for tabs in iz_data: tab[init:(init+len(tabs))] = tabs init += len(tabs) else: sample = hex_sampler(tab, sidelen, keepref, wfix.wcs.crpix[:2] - 1., w.wcs.crval[0], w.wcs.crval[1], ra_gc, dec_gc, pa, inc, starting_angle, precision, hexgrid_output) tab = sample # Remove NaN rows if desired if not keepnan: if not pseudo: # newtab = tab[~np.isnan(tab[cname])] newtab = tab[~np.isnan(tab)] tab = newtab else: df = tab.to_pandas() df.dropna(how='all', subset=col_lbl) tab = Table.from_pandas(df) return tab
filters = FilterSet() filters.addEAZYFilter(eazy_library, range(len(eazy_library.filternames))) Names = ['FUV', 'NUV', 'U', 'B', 'V', 'R', 'I', 'J', 'H', 'K'] synphot1 = Observe(SP1, filters, redshift=0.001) synphot2 = Observe(SP2, filters, redshift=0.001) fluxes1 = np.squeeze(synphot1.fluxes).value fluxes2 = np.squeeze(synphot2.fluxes).value data = Table() for i, n in enumerate(Names): flux = np.array([]) for k, w in enumerate(weight): flux = np.append(flux, (w * fluxes1[i] * Ms[sp1,:] + \ (1-w) * fluxes2[i] * Ms[sp2,:]).ravel()) err = 0.01 * flux data.add_columns( [Column(flux, 'F%s' % (i + 1)), Column(err, 'E%s' % (i + 1))]) id = Column(name='id', data=np.arange(1, len(data) + 1)) zspec = Column(name='zspec', data=-1 * np.ones(len(data))) data.add_column(id, 0) data.add_column(zspec) df = data.to_pandas() np.savetxt('New/Composite/composite_AC.cat', data, header=' '.join(data.colnames), fmt=['%d'] + ['%.5e' for k in range(20)] + ['%.2f'])
def match_planted(fk_candidate_observations, match_filename, bright_limit=BRIGHT_LIMIT, object_planted=OBJECT_PLANTED, minimum_bright_detections=MINIMUM_BRIGHT_DETECTIONS, bright_fraction=MINIMUM_BRIGHT_FRACTION): """ Using the fk_candidate_observations as input get the Object.planted file from VOSpace and match planted sources with found sources. The Object.planted list is pulled from VOSpace based on the standard file-layout and name of the first exposure as read from the .astrom file. :param fk_candidate_observations: name of the fk*reals.astrom file to check against Object.planted :param match_filename: a file that will contain a list of all planted sources and the matched found source @param minimum_bright_detections: if there are too few bright detections we raise an error. """ found_pos = [] detections = fk_candidate_observations.get_sources() for detection in detections: reading = detection.get_reading(0) # create a list of positions, to be used later by match_lists found_pos.append([reading.x, reading.y]) # Now get the Object.planted file, either from the local FS or from VOSpace. objects_planted_uri = object_planted if not os.access(objects_planted_uri, os.F_OK): objects_planted_uri = fk_candidate_observations.observations[0].get_object_planted_uri() lines = open(objects_planted_uri).read() # we are changing the format of the Object.planted header to be compatible with astropy.io.ascii but # there are some old Object.planted files out there so we do these string/replace calls to reset those. new_lines = lines.replace("pix rate", "pix_rate") new_lines = new_lines.replace("""''/h rate""", "sky_rate") planted_objects_table = ascii.read(new_lines, header_start=-1, data_start=0) planted_objects_table.meta = None # The match_list method expects a list that contains a position, not an x and a y vector, so we transpose. planted_pos = numpy.transpose([planted_objects_table['x'].data, planted_objects_table['y'].data]) # match_idx is an order list. The list is in the order of the first list of positions and each entry # is the index of the matching position from the second list. (match_idx, match_fnd) = util.match_lists(numpy.array(planted_pos), numpy.array(found_pos)) assert isinstance(match_idx, numpy.ma.MaskedArray) assert isinstance(match_fnd, numpy.ma.MaskedArray) false_positives_table = Table() # Once we've matched the two lists we'll need some new columns to store the information in. # these are masked columns so that object.planted entries that have no detected match are left 'blank'. new_columns = [MaskedColumn(name="measure_x", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_y", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_rate", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_angle", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_mag1", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_merr1", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_mag2", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_merr2", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_mag3", length=len(planted_objects_table), mask=True), MaskedColumn(name="measure_merr3", length=len(planted_objects_table), mask=True)] planted_objects_table.add_columns(new_columns) tlength = 0 new_columns = [MaskedColumn(name="measure_x", length=tlength, mask=True), MaskedColumn(name="measure_y", length=tlength, mask=True), MaskedColumn(name="measure_rate", length=0, mask=True), MaskedColumn(name="measure_angle", length=0, mask=True), MaskedColumn(name="measure_mag1", length=0, mask=True), MaskedColumn(name="measure_merr1", length=0, mask=True), MaskedColumn(name="measure_mag2", length=0, mask=True), MaskedColumn(name="measure_merr2", length=0, mask=True), MaskedColumn(name="measure_mag3", length=tlength, mask=True), MaskedColumn(name="measure_merr3", length=tlength, mask=True)] false_positives_table.add_columns(new_columns) # We do some 'checks' on the Object.planted match to diagnose pipeline issues. Those checks are made using just # those planted sources we should have detected. bright = planted_objects_table['mag'] < bright_limit n_bright_planted = numpy.count_nonzero(planted_objects_table['mag'][bright]) measures = [] idxs = [] for idx in range(len(match_idx)): # The match_idx value is False if nothing was found. if not match_idx.mask[idx]: # Each 'source' has multiple 'readings' measures.append(detections[match_idx[idx]].get_readings()) idxs.append(idx) observations = measure_mags(measures) for oidx in range(len(measures)): idx = idxs[oidx] readings = measures[oidx] start_jd = util.Time(readings[0].obs.header['MJD_OBS_CENTER'], format='mpc', scale='utc').jd end_jd = util.Time(readings[-1].obs.header['MJD_OBS_CENTER'], format='mpc', scale='utc').jd rate = math.sqrt((readings[-1].x - readings[0].x) ** 2 + (readings[-1].y - readings[0].y) ** 2) / ( 24 * (end_jd - start_jd)) rate = int(rate * 100) / 100.0 angle = math.degrees(math.atan2(readings[-1].y - readings[0].y, readings[-1].x - readings[0].x)) angle = int(angle * 100) / 100.0 planted_objects_table[idx]['measure_rate'] = rate planted_objects_table[idx]['measure_angle'] = angle planted_objects_table[idx]['measure_x'] = observations[readings[0].obs]['mags']["XCENTER"][oidx] planted_objects_table[idx]['measure_y'] = observations[readings[0].obs]['mags']["YCENTER"][oidx] for ridx in range(len(readings)): reading = readings[ridx] mags = observations[reading.obs]['mags'] planted_objects_table[idx]['measure_mag{}'.format(ridx+1)] = mags["MAG"][oidx] planted_objects_table[idx]['measure_merr{}'.format(ridx+1)] = mags["MERR"][oidx] # for idx in range(len(match_fnd)): # if match_fnd.mask[idx]: # measures = detections[idx].get_readings() # false_positives_table.add_row() # false_positives_table[-1] = measure_mags(measures, false_positives_table[-1]) # Count an object as detected if it has a measured magnitude in the first frame of the triplet. n_bright_found = numpy.count_nonzero(planted_objects_table['measure_mag1'][bright]) # Also compute the offset and standard deviation of the measured magnitude from that planted ones. offset = numpy.mean(planted_objects_table['mag'][bright] - planted_objects_table['measure_mag1'][bright]) try: offset = "{:5.2f}".format(offset) except: offset = "indef" std = numpy.std(planted_objects_table['mag'][bright] - planted_objects_table['measure_mag1'][bright]) try: std = "{:5.2f}".format(std) except: std = "indef" if os.access(match_filename, os.R_OK): fout = open(match_filename, 'a') else: fout = open(match_filename, 'w') fout.write("#K {:10s} {:10s}\n".format("EXPNUM", "FWHM")) for measure in detections[0].get_readings(): fout.write('#V {:10s} {:10s}\n'.format(measure.obs.header['EXPNUM'], measure.obs.header['FWHM'])) fout.write("#K ") for keyword in ["RMIN", "RMAX", "ANGLE", "AWIDTH"]: fout.write("{:10s} ".format(keyword)) fout.write("\n") fout.write("#V ") for keyword in ["RMIN", "RMAX", "ANGLE", "AWIDTH"]: fout.write("{:10s} ".format(fk_candidate_observations.sys_header[keyword])) fout.write("\n") fout.write("#K ") for keyword in ["NBRIGHT", "NFOUND", "OFFSET", "STDEV"]: fout.write("{:10s} ".format(keyword)) fout.write("\n") fout.write("#V {:<10} {:<10} {:<10} {:<10}\n".format(n_bright_planted, n_bright_found, offset, std)) try: writer = ascii.FixedWidth # add a hash to the start of line that will have header columns: for JMP fout.write("#") fout.flush() ascii.write(planted_objects_table, output=fout, Writer=writer, delimiter=None) if len(false_positives_table) > 0: with open(match_filename+".fp", 'a') as fpout: fpout.write("#") ascii.write(false_positives_table, output=fpout, Writer=writer, delimiter=None) except Exception as e: logging.error(str(e)) raise e finally: fout.close() # Some simple checks to report a failure how we're doing. if n_bright_planted < minimum_bright_detections: raise RuntimeError(1, "Too few bright objects planted.") if n_bright_found / float(n_bright_planted) < bright_fraction: raise RuntimeError(2, "Too few bright objects found.") return "{} {} {} {}".format(n_bright_planted, n_bright_found, offset, std)
def make_sensitivity_curves(tint=1200, spec_res=100, aper_radius=0.15, seeing_limited=False): mag = np.arange(10, 22) snr_y = np.zeros(len(mag), dtype=float) snr_j = np.zeros(len(mag), dtype=float) snr_h = np.zeros(len(mag), dtype=float) snr_sum_y = np.zeros(len(mag), dtype=float) snr_sum_j = np.zeros(len(mag), dtype=float) snr_sum_h = np.zeros(len(mag), dtype=float) bkg_y = np.zeros(len(mag), dtype=float) bkg_j = np.zeros(len(mag), dtype=float) bkg_h = np.zeros(len(mag), dtype=float) star_y = np.zeros(len(mag), dtype=float) star_j = np.zeros(len(mag), dtype=float) star_h = np.zeros(len(mag), dtype=float) spec_y_tab = None spec_j_tab = None spec_h_tab = None # Calculate the number of supernovae. N_SNe = 4500.0 * 0.6 * 10 ** (mag - 18.9) out_file = "roboAO_sensitivity_t{0:d}_R{1:d}_ap{2:0.3f}".format(tint, spec_res, aper_radius) # Save the output to a table. _out = open(out_file + ".txt", "w") meta1 = '# tint = {0:5d}, R = {1:5d}, apRad = {2:5.3f}"\n' _out.write(meta1.format(tint, spec_res, aper_radius)) _out.write("# Sensitivity integrated over broad band.") hdr = "{0:5s} {1:6s} {2:5s} {3:5s} {4:5s} {5:5s} {6:5s} {7:5s}\n" fmt = "{0:5.1f} {1:6.1f} {2:5.1f} {3:5.1f} {4:5.1f} {5:5.1f} {6:5.1f} {7:5.1f}\n" _out.write(hdr.format("# Mag", "N_SNe", "J_SNR", "H_SNR", "J_ms", "H_ms", "J_mb", "H_mb")) for mm in range(len(mag)): print "Mag: ", mag[mm] blah_y = etc_uh_roboAO( mag[mm], "Y", tint, spec_res=spec_res, aper_radius=aper_radius, seeing_limited=seeing_limited ) blah_j = etc_uh_roboAO( mag[mm], "J", tint, spec_res=spec_res, aper_radius=aper_radius, seeing_limited=seeing_limited ) blah_h = etc_uh_roboAO( mag[mm], "H", tint, spec_res=spec_res, aper_radius=aper_radius, seeing_limited=seeing_limited ) col_y_suffix = "_Y_{0:d}".format(mag[mm]) col_j_suffix = "_J_{0:d}".format(mag[mm]) col_h_suffix = "_H_{0:d}".format(mag[mm]) spec_signal_y = Column(name="sig" + col_y_suffix, data=blah_y[4]) spec_signal_j = Column(name="sig" + col_j_suffix, data=blah_j[4]) spec_signal_h = Column(name="sig" + col_h_suffix, data=blah_h[4]) spec_bkg_y = Column(name="bkg" + col_y_suffix, data=blah_y[5]) spec_bkg_j = Column(name="bkg" + col_j_suffix, data=blah_j[5]) spec_bkg_h = Column(name="bkg" + col_h_suffix, data=blah_h[5]) spec_snr_y = Column(name="snr" + col_y_suffix, data=blah_y[6]) spec_snr_j = Column(name="snr" + col_j_suffix, data=blah_j[6]) spec_snr_h = Column(name="snr" + col_h_suffix, data=blah_h[6]) if spec_y_tab == None: spec_y_tab = Table([blah_y[3]], names=["wave_Y"]) if spec_j_tab == None: spec_j_tab = Table([blah_j[3]], names=["wave_J"]) if spec_h_tab == None: spec_h_tab = Table([blah_h[3]], names=["wave_H"]) spec_y_tab.add_columns([spec_signal_y, spec_bkg_y, spec_snr_y]) spec_j_tab.add_columns([spec_signal_j, spec_bkg_j, spec_snr_j]) spec_h_tab.add_columns([spec_signal_h, spec_bkg_h, spec_snr_h]) snr_y[mm] = blah_y[0] snr_j[mm] = blah_j[0] snr_h[mm] = blah_h[0] snr_sum_y[mm] = math.sqrt((spec_snr_y ** 2).sum()) snr_sum_j[mm] = math.sqrt((spec_snr_j ** 2).sum()) snr_sum_h[mm] = math.sqrt((spec_snr_h ** 2).sum()) star_y[mm] = blah_y[1] star_j[mm] = blah_j[1] star_h[mm] = blah_h[1] bkg_y[mm] = blah_y[2] bkg_j[mm] = blah_j[2] bkg_h[mm] = blah_h[2] avg_tab = Table( [mag, snr_y, snr_j, snr_h, snr_sum_y, snr_sum_j, snr_sum_h, star_y, star_j, star_h, bkg_y, bkg_j, bkg_h], names=[ "mag", "snr_y", "snr_j", "snr_h", "snr_sum_y", "snr_sum_j", "snr_sum_h", "star_y", "star_j", "star_h", "bkg_y", "bkg_j", "bkg_h", ], ) out_file = "roboAO_sensitivity_t{0:d}_R{1:d}_ap{2:0.3f}".format(tint, spec_res, aper_radius) if seeing_limited: out_file += "_seeing" # Save the tables spec_y_tab.write(out_file + "_spec_y_tab.fits", overwrite=True) spec_j_tab.write(out_file + "_spec_j_tab.fits", overwrite=True) spec_h_tab.write(out_file + "_spec_h_tab.fits", overwrite=True) avg_tab.write(out_file + "_avg_tab.fits", overwrite=True) return
def make_tint_curves(mag=20, spec_res=100, aper_radius=0.15, seeing_limited=False): tint = np.arange(300, 3600 + 1, 300) snr_y = np.zeros(len(tint), dtype=float) snr_j = np.zeros(len(tint), dtype=float) snr_h = np.zeros(len(tint), dtype=float) snr_sum_y = np.zeros(len(tint), dtype=float) snr_sum_j = np.zeros(len(tint), dtype=float) snr_sum_h = np.zeros(len(tint), dtype=float) spec_y_tab = None spec_j_tab = None spec_h_tab = None for tt in range(len(tint)): print "Tint: ", tint[tt] blah_y = etc_uh_roboAO( mag, "Y", tint[tt], spec_res=spec_res, aper_radius=aper_radius, seeing_limited=seeing_limited ) blah_j = etc_uh_roboAO( mag, "J", tint[tt], spec_res=spec_res, aper_radius=aper_radius, seeing_limited=seeing_limited ) blah_h = etc_uh_roboAO( mag, "H", tint[tt], spec_res=spec_res, aper_radius=aper_radius, seeing_limited=seeing_limited ) col_y_suffix = "_Y_{0:d}".format(tint[tt]) col_j_suffix = "_J_{0:d}".format(tint[tt]) col_h_suffix = "_H_{0:d}".format(tint[tt]) spec_signal_y = Column(name="sig" + col_y_suffix, data=blah_y[4]) spec_signal_j = Column(name="sig" + col_j_suffix, data=blah_j[4]) spec_signal_h = Column(name="sig" + col_h_suffix, data=blah_h[4]) spec_bkg_y = Column(name="bkg" + col_y_suffix, data=blah_y[5]) spec_bkg_j = Column(name="bkg" + col_j_suffix, data=blah_j[5]) spec_bkg_h = Column(name="bkg" + col_h_suffix, data=blah_h[5]) spec_snr_y = Column(name="snr" + col_y_suffix, data=blah_y[6]) spec_snr_j = Column(name="snr" + col_j_suffix, data=blah_j[6]) spec_snr_h = Column(name="snr" + col_h_suffix, data=blah_h[6]) if spec_y_tab == None: spec_y_tab = Table([blah_y[3]], names=["wave_Y"]) if spec_j_tab == None: spec_j_tab = Table([blah_j[3]], names=["wave_J"]) if spec_h_tab == None: spec_h_tab = Table([blah_h[3]], names=["wave_H"]) spec_y_tab.add_columns([spec_signal_y, spec_bkg_y, spec_snr_y]) spec_j_tab.add_columns([spec_signal_j, spec_bkg_j, spec_snr_j]) spec_h_tab.add_columns([spec_signal_h, spec_bkg_h, spec_snr_h]) snr_y[tt] = blah_y[0] snr_j[tt] = blah_j[0] snr_h[tt] = blah_h[0] snr_sum_y[tt] = math.sqrt((spec_snr_y ** 2).sum()) snr_sum_j[tt] = math.sqrt((spec_snr_j ** 2).sum()) snr_sum_h[tt] = math.sqrt((spec_snr_h ** 2).sum()) avg_tab = Table( [tint, snr_y, snr_sum_y, snr_j, snr_sum_j, snr_h, snr_sum_h], names=["tint", "snr_y", "snr_sum_y", "snr_j", "snr_sum_j", "snr_h", "snr_sum_h"], ) out_file = "roboAO_tint_m{0:d}_R{1:d}_ap{2:0.3f}".format(mag, spec_res, aper_radius) if seeing_limited: out_file += "_seeing" # Save the tables spec_y_tab.write(out_file + "_spec_y_tab.fits", overwrite=True) spec_j_tab.write(out_file + "_spec_j_tab.fits", overwrite=True) spec_h_tab.write(out_file + "_spec_h_tab.fits", overwrite=True) avg_tab.write(out_file + "_avg_tab.fits", overwrite=True) return
unit='deg', dtype=float) t_orrerr = Column(name='e_original_RAJ2000', data=np.array(original_rerrs), description='Error on original right ascension of source', unit='deg', dtype=float) t_orderr = Column(name='e_original_DECJ2000', data=np.array(original_derrs), description='Error on original declination of source', unit='deg', dtype=float) ##Add the columns t.add_columns([ t_name, t_base_name, t_upra, t_updec, t_uprerr, t_upderr, t_orra, t_ordec, t_orrerr, t_orderr ]) ##Check to see if we have repeated frequencies between different catalogues ##If we do, we need to make unique names for the flux columns all_freqs = [] for freqs in cat_freqs: for freq in freqs: all_freqs.append(freq) if len(all_freqs) == len(np.unique(all_freqs)): duplicates = False else: duplicates = False ##For every catalogue in the match
##Create many columns of data t_name = Column(name='Name',data=names,description='Name based on position of combined source',dtype=str) prim_names = [source.names[0] for source in sources] t_base_name = Column(name='%s_name' %matched_cats[0],data=prim_names,description='Name of %s component' %matched_cats[0],dtype=str) t_upra = Column(name='updated_RAJ2000',data=np.array(updated_ras),description='Updated right ascension of source',unit='deg',dtype=float) t_updec = Column(name='updated_DECJ2000',data=np.array(updated_decs),description='Updated declination of source',unit='deg',dtype=float) t_uprerr = Column(name='e_updated_RAJ2000',data=np.array(updated_rerrs),description='Error on updated right ascension of source',unit='deg',dtype=float) t_upderr = Column(name='e_updated_DECJ2000',data=np.array(updated_derrs),description='Error on updated declination of source',unit='deg',dtype=float) t_orra = Column(name='original_RAJ2000',data=np.array(original_ras),description='Original right ascension of source',unit='deg',dtype=float) t_ordec = Column(name='original_DECJ2000',data=np.array(original_decs),description='Original feclination of source',unit='deg',dtype=float) t_orrerr = Column(name='e_original_RAJ2000',data=np.array(original_rerrs),description='Error on original right ascension of source',unit='deg',dtype=float) t_orderr = Column(name='e_original_DECJ2000',data=np.array(original_derrs),description='Error on original declination of source',unit='deg',dtype=float) ##Add the columns t.add_columns([t_name,t_base_name,t_upra,t_updec,t_uprerr,t_upderr,t_orra,t_ordec,t_orrerr,t_orderr]) ##For every catalogue in the match for cat in xrange(len(num_freqs)): ##See how many frequencies that source has num_freq = num_freqs[cat] ##For every frequency, make a column of fluxes and flux errors, masking every value with -100000.0 for freq in xrange(num_freq): fluxs = np.array([src.fluxs[cat][freq] for src in sources]) ferrs = np.array([src.ferrs[cat][freq] for src in sources]) t_flux = MaskedColumn(name='S_%d' %int(cat_freqs[cat][freq]),data=fluxs,description='Flux at %.1fMHz' %float(cat_freqs[cat][freq]),mask=fluxs==-100000.0, fill_value=None,unit='Jy',dtype=float) t_ferr = MaskedColumn(name='e_S_%d' %int(cat_freqs[cat][freq]),data=ferrs,description='Flux error at %.1fMHz' %float(cat_freqs[cat][freq]),mask=ferrs==-100000.0, fill_value=None,unit='Jy',dtype=float) t.add_columns([t_flux,t_ferr]) ##Extrapolate the base catalogue frequency flux density, using the fitted values for late comparison extrap_freq = cat_freqs[0][0]