def prepare_fit_range(indata, bounds=None): """ `indata` must be binned on common velocity grid If no range is passed, full data range is fitted. """ mcols = {} if not bounds: bounds = indata['wave'].min(), indata['wave'].max() bounds = np.array(bounds) idx = np.where((indata['Velocity'] > bounds.min()) & (indata['Velocity'] < bounds.max())) fitdata = indata[idx] mcols["Velocity"] = fitdata["Velocity"] for col in fitdata.colnames: if not col.endswith('flux'): # & col.startswith("Si"): continue if not col[:-5] in fikdict: continue species = " ".join(col.split(" ")[:-1]) mflcol = Table.MaskedColumn(fitdata[col], name="col", mask=fitdata[species + " mask"].data) mercol = Table.MaskedColumn(fitdata[species + " errs"], mask=fitdata[species + " mask"].data) mcols[col] = mflcol mcols[species + " errs"] = mercol fittable = Table(mcols) return fittable
def mask_nan_values(table): """ Mask all nan values contained in columns of dtype=float. Useful when reading a fits table. :param table: :return: """ for col_name in table.colnames: if isinstance(table[col_name], Column): if np.issubdtype( table[col_name].dtype, np.floating) and np.sum(np.isnan(table[col_name])) > 0: table[col_name] = Table.MaskedColumn(table[col_name].data, mask=np.isnan( table[col_name].data), unit=table[col_name].unit)
# File can be downloaded (in CSV format) from https://docs.google.com/spreadsheets/d/18w3dbA3lga8So5sCUml7xUFyOPjfTEZ-vXSzobYprhY/edit?usp=sharing # You *may* need to remove commas (as astropy.table may not like them... if os.getcwd().split('/')[-1] == 'python': relDir = '../' else: relDir = './' fileIn = os.path.join(relDir, "data/data.csv") tabIn = Table.read(fileIn) jsonOut = [] for row in tabIn: entry = {} for col in tabIn.colnames: if type(tabIn[col]) == type(Table.MaskedColumn()): if not tabIn[col].mask[row.index]: entry[col] = row[col] else: entry[col] = row[col] jsonOut.append(entry) json.dump(jsonOut, open(os.path.join(relDir, 'data/data.json'), 'w'), indent=2) fIn = open(os.path.join(relDir, 'data/data.json'), 'r') fOut = open(os.path.join(relDir, 'data/data.jsonp'), 'w') lines = fIn.readlines() lines[0] = 'data(' + lines[0] lines[-1] = lines[-1] + ');' for l in lines: fOut.write(l) fOut.close() fIn.close()
def main(mtlz_path, mtlz_name, correlation_cut=0.2, summary_column_subset=True): maskname = mtlz_name.split('_')[2] print(maskname) tab = Table.read(os.path.join(mtlz_path, mtlz_name), format='ascii.csv') ## Make appropriate cuts if 'SDSS_only' in tab.colnames: if type(tab['SDSS_only'][0]) in [bool, np.bool, np.bool_]: boolcut = [(not row) for row in tab['SDSS_only']] else: boolcut = [row.lower() == 'false' for row in tab['SDSS_only']] tab = tab[boolcut] if 'cor' in tab.colnames: tab = tab[tab['cor'] >= correlation_cut] if 'ID' in tab.colnames: tab = tab[['GAL' in id for id in tab['ID']]] outnames = [] for name in tab['ID']: outnames.append(name.replace('GAL', '{}-'.format(maskname))) tab.remove_column('ID') tab.add_column(Table.Column(name='TARGETID', data=outnames)) ## Make sure the names look good # ra_colname_bool = np.any([col.strip(' \t\r').upper() == 'RA' for col in tab.colnames]) # if not ra_colname_bool: if 'RA' not in tab.colnames: tab.rename_column('RA_targeted', 'RA') tab.rename_column('DEC_targeted', 'DEC') # sdss_colname_bool = np.any([col.strip(' \t\r').upper() == 'SDSS_SDSS12' for col in tab.colnames]) # if sdss_colname_bool: if 'sdss_SDSS12' in tab.colnames: newcol = [] for ii in range(len(tab)): newcol.append('SDSS' + str(tab['sdss_SDSS12'][ii])) tab.add_column(Table.Column(data=newcol, name='SDSS12_OBJID')) tab.remove_column('sdss_SDSS12') else: tab.add_column( tab.MaskedColumn( data=[''] * len(tab), name='SDSS12_OBJID')) #,mask=np.ones(len(tab)).astype(bool))) if 'sdss_zsp' in tab.colnames: tab.rename_column('sdss_zsp', 'SDSS_zsp') else: tab.add_column( Table.MaskedColumn( data=np.zeros(len(tab)), name='SDSS_zsp')) #,mask=np.ones(len(tab)).astype(bool))) if 'z_est_bary' in tab.colnames: tab.add_column(Table.Column(data=tab['z_est_bary'].copy(), name='z')) if 'Proj_R_asec' in tab.colnames: tab.rename_column('Proj_R_asec', 'R [asec]') if 'velocity' in tab.colnames: tab.rename_column('velocity', 'v [km/s]') if 'FIBNAME' in tab.colnames: tab.rename_column('FIBNAME', 'FIBERNUM') ## Load up in the right order if summary_column_subset: tab = tab[[ 'TARGETID', 'FIBERNUM', 'RA', 'DEC', 'SDSS12_OBJID', 'z', 'R [asec]', 'v [km/s]', 'SDSS_zsp' ]] if 'description' in dict(tab.meta).keys(): desc = tab.meta.pop('description') tab.meta['DESCRP'] = desc return tab.filled()
if os.path.exists(args[2]): os.remove(args[2]) FREQ='118-129' OFFSET = 0.347 CAT='tgss' SNR_THRESH = 0.0 MATCH_THRESH = 2.0*OFFSET t = Table.read(args[0]) t['p'] = where(t['sigma_ips_%s' % CAT] > SNR_THRESH, ones(len(t)), zeros(len(t))) t2 = Table.read(args[1]) # add new columns to t2 t2["p_match1"] = ones(len(t2)) t2["name_match2"] = Table.MaskedColumn([""]*len(t2), mask=[True]*len(t2), dtype='S24') t2["p_match2"] = zeros(len(t2)) n_suspicious = 0 n_undetected = 0 hi_lr = 0 lo_snr = 0 n_100 =0 n_99 = 0 n_95 = 0 n_nomatch = 0 for g in range(t['GroupID'].min(), t['GroupID'].max()): #group_gt_thresh = (t['GroupID'] == g) & (t['sigma_ips_%s' % CAT] > SNR_THRESH) group_gt_thresh = (t['GroupID'] == g) seps = t[group_gt_thresh]['SepArcM_%s' % CAT] source_names = t[group_gt_thresh]['Source_name_%s' % CAT] if len(seps) < 2:
def slFiles2dataTables(slNames): """Reads in data from sourcelists, returns some or all of the following data columns in a pair of properly formatted astropy.table objects: * X Position * Y Position * Right Ascension * Declination * Flux (Inner Aperture) * Flux (Outer Aperture) * Magnitude (Inner Aperture) * Magnitude (Outer Aperture) * Flag Value .. note:: 'X position' and 'Y position' data columns will always be returned. However, not every sourcelist or catalog file is guaranteed to have any of the seven remaining data columns in the above list. Parameters ---------- slNames : list A list containing the reference sourcelist filename and the comparison sourcelist filename, in that order. Returns ------- refData : astropy.table object data from the reference sourcelist compData : astropy.table object data from the comparison sourcelist """ if slNames[0].endswith(".ecsv"): refData_in = Table.read(slNames[0], format='ascii.ecsv') else: try: refData_in = Table.read(slNames[0], format='ascii.daophot') except Exception: refData_in = Table.read(slNames[0], format='ascii') if slNames[1].endswith(".ecsv"): compData_in = Table.read(slNames[1], format='ascii.ecsv') else: try: compData_in = Table.read(slNames[1], format='ascii.daophot') except Exception: compData_in = Table.read(slNames[1], format='ascii') titleSwapDict_dao1 = {"X": "X-Center", "Y": "Y-Center", "RA": "RA", "DEC": "DEC", "FLUX1": "n/a", "FLUX2": "Flux(0.15)", "MAGNITUDE1": "MagAp(0.05)", "MAGNITUDE2": "MagAp(0.15)", "MERR1": "MagErr(0.05)", "MERR2": "MagErr(0.15)", "MSKY": "MSky(0.15)", "STDEV": "Stdev(0.15)", "FLAGS": "Flags", "ID": "ID", "CI": "CI"} titleSwapDict_dao2 = {"X": "X-Center", "Y": "Y-Center", "RA": "RA", "DEC": "DEC", "FLUX1": "n/a", "FLUX2": "Flux(0.45)", "MAGNITUDE1": "MagAp(0.15)", "MAGNITUDE2": "MagAp(0.45)", "MERR1": "MagErr(0.15)", "MERR2": "MagErr(0.45)", "MSKY": "MSky(0.45)", "STDEV": "Stdev(0.45)", "FLAGS": "Flags", "ID": "ID", "CI": "CI"} titleSwapDict_dao3 = {"X": "X-Center", "Y": "Y-Center", "RA": "RA", "DEC": "DEC", "FLUX1": "n/a", "FLUX2": "Flux(0.125)", "MAGNITUDE1": "MagAp(0.03)", "MAGNITUDE2": "MagAp(0.125)", "MERR1": "MagErr(0.03)", "MERR2": "MagErr(0.125)", "MSKY": "MSky(0.125)", "STDEV": "Stdev(0.125)", "FLAGS": "Flags", "ID": "ID", "CI": "CI"} titleSwapDict_point = {"X": "X-Center", "Y": "Y-Center", "RA": "RA", "DEC": "DEC", "FLUX1": "n/a", "FLUX2": "FluxAp2", "MAGNITUDE1": "MagAp1", "MAGNITUDE2": "MagAp2", "MERR1": "MagErrAp1", "MERR2": "MagErrAp2", "MSKY": "MSkyAp2", "STDEV": "StdevAp2", "FLAGS": "Flags", "ID": "ID", "CI": "CI"} titleSwapDict_segment = {"X": "X-Centroid", "Y": "Y-Centroid", "RA": "RA", "DEC": "DEC", "FLUX1": "n/a", "FLUX2": "FluxAp2", "MAGNITUDE1": "MagAp1", "MAGNITUDE2": "MagAp2", "MERR1": "MagErrAp1", "MERR2": "MagErrAp2", "MSKY": "n/a", "STDEV": "n/a", "FLAGS": "Flags", "ID": "ID", "CI": "CI"} titleSwapDict_daoTemp = {"X": "XCENTER", "Y": "YCENTER", "RA": "n/a", "DEC": "n/a", "FLUX1": "FLUX1", "FLUX2": "FLUX2", "MAGNITUDE1": "MAG1", "MAGNITUDE2": "MAG2", "FLAGS": "n/a", "ID": "ID", "MERR1": "MERR1", "MERR2": "MERR2", "MSKY": "MSKY", "STDEV": "STDEV"} titleSwapDict_sourceX = {"X": "X_IMAGE", "Y": "Y_IMAGE", "RA": "RA", "DEC": "DEC", "FLUX1": "FLUX_APER1", "FLUX2": "FLUX_APER2", "MAGNITUDE1": "MAG_APER1", "MAGNITUDE2": "MAG_APER2", "FLAGS": "FLAGS", "ID": "NUMBER"} titleSwapDict_cooNew = {"X": "col1", "Y": "col2", "RA": "n/a", "DEC": "n/a", "FLUX1": "n/a", "FLUX2": "n/a", "MAGNITUDE1": "n/a", "MAGNITUDE2": "n/a", "FLAGS": "n/a", "ID": "col7"} # titleSwapDict_cooOld = {"X": "XCENTER", "Y": "YCENTER", "RA": "n/a", "DEC": "n/a", "FLUX1": "n/a", # "FLUX2": "n/a", "MAGNITUDE1": "n/a", "MAGNITUDE2": "n/a", "FLAGS": "n/a", # "ID": "ID"} titleSwapDict_cooOld2 = {"X": "XCENTER", "Y": "YCENTER", "RA": "RA", "DEC": "DEC", "FLUX1": "FLUX_0.05", "FLUX2": "FLUX_0.15", "MAGNITUDE1": "MAG_0.05", "MAGNITUDE2": "MAG_0.15", "FLAGS": "n/a", "ID": "ID", "MERR1": "MERR_0.05", "MERR2": "MERR_0.15", "MSKY": "MSKY", "STDEV": "STDEV"} titleSwapDict_daorep = {"X": "X", "Y": "Y", "RA": "n/a", "DEC": "n/a", "FLUX1": "flux_0", "FLUX2": "flux_1", "MAGNITUDE1": "mag_0", "MAGNITUDE2": "mag_1", "FLAGS": "n/a", "ID": "n/a"} ctr = 1 for dataTable in [refData_in, compData_in]: if "X-Center" in list(dataTable.keys()): if (("MagAp(0.05)" in list(dataTable.keys())) and ( "MagAp(0.15)" in list(dataTable.keys()))): # ACS/WFC, WFC3/UVIS log.info("titleSwapDict_dao1") titleSwapDict = titleSwapDict_dao1 elif (("MagAp(0.15)" in list(dataTable.keys())) and ("MagAp(0.45)" in list(dataTable.keys()))): # WFC3/IR log.info("titleSwapDict_dao2") titleSwapDict = titleSwapDict_dao2 elif (("MagAp(0.03)" in list(dataTable.keys())) and ("MagAp(0.125)" in list(dataTable.keys()))): # ACS/HRC log.info("titleSwapDict_dao3") titleSwapDict = titleSwapDict_dao3 elif "MagAp1" in list(dataTable.keys()): log.info("titleSwapDict_point") titleSwapDict = titleSwapDict_point else: sys.exit("ERROR: Unrecognized format. Exiting...") elif ("XCENTER" in list(dataTable.keys()) and "FLUX1" in list(dataTable.keys())): log.info("titleSwapDict_daoTemp") titleSwapDict = titleSwapDict_daoTemp elif "X_IMAGE" in list(dataTable.keys()): log.info("titleSwapDict_sourceX") titleSwapDict = titleSwapDict_sourceX elif "col1" in list(dataTable.keys()): log.info("titleSwapDict_cooNew") titleSwapDict = titleSwapDict_cooNew elif "XCENTER" in list(dataTable.keys()): log.info("titleSwapDict_cooOld2") titleSwapDict = titleSwapDict_cooOld2 elif "X" in list(dataTable.keys()): log.info("titleSwapDict_daorep") titleSwapDict = titleSwapDict_daorep elif "X-Centroid" in list(dataTable.keys()) and "MagAp1" in list(dataTable.keys()): log.info("titleSwapDict_segment") titleSwapDict = titleSwapDict_segment else: sys.exit("ERROR: Unrecognized format. Exiting...") outTable = Table() for swapKey in list(titleSwapDict.keys()): if titleSwapDict[swapKey] != "n/a": try: col2add = Table.Column(name=swapKey, data=dataTable[titleSwapDict[swapKey]]) except TypeError: col2add = Table.MaskedColumn(name=swapKey, data=dataTable[titleSwapDict[swapKey]]) outTable.add_column(col2add) if ctr == 1: refData = outTable if ctr == 2: compData = outTable ctr += 1 return (refData, compData)
def write_table(origtable, tablename=None, tabletype=None, joinsymb='|', overwrite=True, verbose=False, comma_replacement=';', write_empty=False, use_specprod=True): """ Workflow function to write exposure, processing, and unprocessed tables. It allows for multi-valued table cells, which are reduced to strings using the joinsymb. It writes to a temp file before moving the fully written file to the name given by tablename (or the default for table of type tabletype). Args: origtable, Table. Either exposure table or processing table. tablename, str. Full pathname of where the table should be saved, including the extension. Originally save to *.temp.{ext} and then moved to *.{ext}. If None, it looks up the default for typetable. tabletype, str. Used if tablename is None to get the default name for the type of table. joinsymb, str. The symbol used to join values in a list/array when saving. Should not be a comma. overwrite, bool. Whether to overwrite the file on disk if it already exists. Default is currently True. verbose, bool. Whether to give verbose amounts of information (True) or succinct/no outputs (False). Default is False. write_empty, bool. Whether to write an empty table to disk. The default is False. Warning: code is less robust to column datatypes on read/write if the table is empty. May cause issues if this is set to True. comma_replacement, str. Replace instances of this symbol with commas when loading scalar columns in a table, as e.g. BADAMPS is used in the pipeline and symbols like ';' are problematic on the command line. use_specprod, bool. If True and tablename not specified and tabletype is exposure table, this looks for the table in the SPECPROD rather than the exptab repository. Default is True. Returns: Nothing. """ log = get_logger() if tablename is None and tabletype is None: log.error("Pathname or type of table is required to save the table") return if tabletype is not None: tabletype = standardize_tabletype(tabletype) if tablename is None: tablename = translate_type_to_pathname(tabletype, use_specprod=use_specprod) if not write_empty and len(origtable) == 0: log.warning(f'NOT writing zero length table to {tablename}') return if verbose: log.info("In write table", tablename,'\n', tabletype) log.info(origtable[0:2]) basename, ext = os.path.splitext(tablename) temp_name = f'{basename}.temp{ext}' if verbose: log.info(ext ,temp_name) table = origtable.copy() if ext in ['.csv', '.ecsv']: if verbose: log.info("Given table: ", table.info) # replace_cols = {} for nam in table.colnames: ndim = table[nam].ndim if ndim > 1 or type(table[nam][0]) in [list, np.ndarray, np.array] or table[nam].dtype is object: if verbose: log.info(f'{nam} is {ndim} dimensions, changing to string') col = [ensure_scalar(row, joinsymb=joinsymb) for row in table[nam]] # replace_cols[nam] = Table.Column(name=nam,data=col) if type(table[nam]) is Table.MaskedColumn: col = Table.MaskedColumn(name=nam, data=col) else: col = Table.Column(name=nam, data=col) table.replace_column(nam, col) elif type(table[nam][0]) in [str, np.str, np.str_]: col = [row.replace(',', comma_replacement) for row in table[nam]] if type(table[nam]) is Table.MaskedColumn: col = Table.MaskedColumn(name=nam, data=col) else: col = Table.Column(name=nam, data=col) table.replace_column(nam, col) # for nam, col in replace_cols.items(): # t.replace_column(nam,col) if np.any([c.ndim > 1 or type(table[nam][0]) in [list, np.ndarray, np.array] for c in table.itercols()]) and verbose: log.warning("A column was still more than one dimensional") log.info(table.info()) table.write(temp_name, format=f'ascii{ext}', overwrite=overwrite) else: table.write(temp_name, overwrite=True) os.rename(temp_name, tablename) if verbose: log.info("Written table: ", table.info)
def combine_vvv_2mass(vvvpsf_file, twomass_file, out_dir=dirconfig.cross_vvv_2mass, max_error=1.00): """ This function add 2MASS sources to the VVV-PSF catalog :param twomass_file: string :param vvvpsf_file: string :param out_dir: string :param max_error: number :return: """ # Check if files exist if files_exist(twomass_file, vvvpsf_file): print('Combining: ', vvvpsf_file, twomass_file) # Read catalogs twomass_table = read_fits_table(twomass_file) vvvpsf_table = read_fits_table(vvvpsf_file) # Check if tile match if not twomass_table.meta['TILE'] == vvvpsf_table.meta['TILE']: raise ValueError(f'Files do not correspond to the same tile') # Cross-match c2mass = SkyCoord(twomass_table['RAJ2000'], twomass_table['DEJ2000'], unit='deg') cvvv = SkyCoord(vvvpsf_table['ra'], vvvpsf_table['dec'], unit='deg') idx, d2d, d3d = c2mass.match_to_catalog_sky(cvvv) match = d2d > max_error * u.arcsec # In this case repeated sources are not removed (otherwise they will be included in the output catalog) unpaired_2mass_sources = twomass_table[match] # Create a new table to store combined data unp_table = Table() # Add unpaired 2MASS sources to new_catalog unp_table['ra'] = unpaired_2mass_sources['RAJ2000'] unp_table['dec'] = unpaired_2mass_sources['DEJ2000'] unp_table['l'] = unpaired_2mass_sources['l'] unp_table['b'] = unpaired_2mass_sources['b'] unp_table['mag_J'] = unpaired_2mass_sources['J_vista'] unp_table['eJ'] = unpaired_2mass_sources['e_Jmag'] unp_table['mag_H'] = unpaired_2mass_sources['H_vista'] unp_table['eH'] = unpaired_2mass_sources['e_Hmag'] unp_table['mag_Ks'] = unpaired_2mass_sources['Ks_vista'] unp_table['eKs'] = unpaired_2mass_sources['e_Kmag'] unp_table['H-Ks'] = unpaired_2mass_sources[ 'H_vista'] - unpaired_2mass_sources['Ks_vista'] unp_table['J-Ks'] = unpaired_2mass_sources[ 'J_vista'] - unpaired_2mass_sources['Ks_vista'] unp_table['J-H'] = unpaired_2mass_sources[ 'J_vista'] - unpaired_2mass_sources['H_vista'] unp_table['catalog'] = [ '2MASS' for _ in range(len(unpaired_2mass_sources)) ] unp_table['id'] = unpaired_2mass_sources['id'] # Aux catalog for VVV-PSF sources aux_table = Table() # Add VVV-PSF sources to new_catalog aux_table['ra'] = vvvpsf_table['ra'] aux_table['dec'] = vvvpsf_table['dec'] aux_table['l'] = vvvpsf_table['l'] aux_table['b'] = vvvpsf_table['b'] aux_table['mag_Z'] = Table.MaskedColumn(vvvpsf_table['mag_Z'].data, mask=np.isnan( vvvpsf_table['mag_Z'].data)) aux_table['er_Z'] = Table.MaskedColumn(vvvpsf_table['er_Z'].data, mask=np.isnan( vvvpsf_table['er_Z'].data)) aux_table['mag_Y'] = Table.MaskedColumn(vvvpsf_table['mag_Y'].data, mask=np.isnan( vvvpsf_table['mag_Y'].data)) aux_table['er_Y'] = Table.MaskedColumn(vvvpsf_table['er_Y'].data, mask=np.isnan( vvvpsf_table['er_Y'].data)) aux_table['mag_J'] = vvvpsf_table['mag_J'] aux_table['eJ'] = vvvpsf_table['er_J'] aux_table['mag_H'] = vvvpsf_table['mag_H'] aux_table['eH'] = vvvpsf_table['er_H'] aux_table['mag_Ks'] = vvvpsf_table['mag_Ks'] aux_table['eKs'] = vvvpsf_table['er_Ks'] aux_table['H-Ks'] = vvvpsf_table['H-Ks'] aux_table['J-Ks'] = vvvpsf_table['J-Ks'] aux_table['J-H'] = vvvpsf_table['J-H'] aux_table['catalog'] = ['PSF-VVV' for _ in range(len(vvvpsf_table))] aux_table['id'] = vvvpsf_table['id'] output_table = vstack([unp_table, aux_table]) # Add metadata to the new file date_time = datetime.utcnow() tile = vvvpsf_table.meta['TILE'] catype = vvvpsf_table.meta['CATYPE'] + '-' + twomass_table.meta['CATYPE'] output_table.meta = { 'TILE': tile, 'F2MASS': twomass_file, 'N2MASS': len(unp_table), 'FVVV': vvvpsf_file, 'NVVV': len(vvvpsf_table), 'STAGE': 'combine_vvv_2mass', 'CATYPE': catype, 'CDATE': date_time.strftime('%Y-%m-%d'), 'CTIME': date_time.strftime('%H:%M:%S'), 'AUTHOR': 'Jorge Anais' } # Write output table fname = f't{tile:03d}_{catype}.fits' output_file = path.join(out_dir, fname) write_fits_table(output_table, output_file)