def main(): assert database.connected file_ = os.environ['CATALOGDB_DIR'] + '/cataclysmicvariables.fits' data = astropy.table.Table.read(file_) copy_data(data, database, 'cataclysmic_variables', schema='catalogdb')
def main(): assert database.connected file_ = os.environ['CATALOGDB_DIR'] + '/Kounkel_Clustering.fits' data = astropy.table.Table.read(file_) copy_data(data, database, 'yso_clustering', schema='catalogdb')
def main(): assert database.connected file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/lmxb_good.fits' data = astropy.table.Table.read(file_) source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) copy_data(data, database, 'lmxb', schema='catalogdb')
def main(): assert database.connected database.become_admin() file_ = os.environ['CATALOGDB_DIR'] + '/MIPSGAL/MIPSGAL_catalog.fits' data = astropy.table.Table.read(file_) data.meta = {} copy_data(data, database, 'mipsgal', schema='catalogdb') file_ = os.environ['CATALOGDB_DIR'] + '/MIPSGAL/MIPSGAL_archive.fits' data = astropy.table.Table.read(file_) data.meta = {} copy_data(data, database, 'mipsgal', schema='catalogdb')
def main(): assert database.connected file_ = ( '/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/' 'target/catalogs/csc/csc_for_catalogdb_v0/CSC2_stub1_realonly_v0.1.0.fits' ) data = astropy.table.Table.read(file_) data.add_column(astropy.table.Column(numpy.arange(len(data)) + 1, 'pk'), 0) # Add id pk. copy_data(data, database, 'bhm_csc', schema='catalogdb')
def main(): assert database.connected replacements = { 'RAdeg': 'ra', 'DEdeg': 'dec', 'e_RAdeg': 'e_ra', 'e_DEdeg': 'e_dec', 'pmDE': 'pmdec', 'e_pmDE': 'e_pmdec', 'E(BR/RP)': 'e_br_rp', '(S/N)': 'snr', 'FG': 'fg_gaia', 'e_FG': 'e_fg_gaia', 'Gmag': 'g_gaia_mag' } path_ = ( '/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/wd/dr2/' ) for file_, table in (('J_MNRAS_482_4570_gaia2wd.dat.gz.fits.gz', 'gaia_dr2_wd'), ('J_MNRAS_482_4570_gaiasdss.dat.gz.fits.gz', 'gaia_dr2_wd_sdss')): data = astropy.table.Table.read(path_ + file_) for column_name in data.colnames: if column_name in replacements: new_column_name = replacements[column_name] else: new_column_name = column_name.lower() data.rename_column(column_name, new_column_name) data['wd'][:] = numpy.core.defchararray.strip(data['wd'].data) if 'gaia2wd' in file_: name_split = numpy.core.defchararray.split(data['dr2name']) source_id = numpy.array(list(zip(*name_split))[2], numpy.int64) data.add_column(astropy.table.Column(source_id, 'source_id'), data.index_column('source')) else: pk = numpy.arange(len(data)) + 1 data.add_column(astropy.table.Column(pk, 'pk'), 0) copy_data(data, database, table, schema='catalogdb')
def main(): assert database.connected path_ = ('/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/' 'target/catalogs/eRosita/eFEDS_for_catalogdb_v0/') for file_, table in (('BHM_SPIDERS_EFEDS_SUPERSET_CLUS_v0.1.2.fits', 'bhm_spiders_clusters_superset'), ('BHM_SPIDERS_EFEDS_SUPERSET_AGN_v0.1.1.fits', 'bhm_spiders_agn_superset')): data = astropy.table.Table.read(path_ + file_) data.add_column(astropy.table.Column( numpy.arange(len(data)) + 1, 'pk'), 0) # Add id pk. copy_data(data, database, table, schema='catalogdb')
def build_query(self, version_id, query_region=None): self.log.debug(f'Processing file {self._file_path}.') # We need to copy the data to a temporary table so that we can # join on it. We could use a Peewee ValueList but for large tables # that will hit the limit of 1GB in PSQL. # Create model for temporary table from FITS table columns. # This works fine because we know there are no arrays. temp_table = self.name.lower() + '_temp' temp = create_model_from_table(temp_table, self._table) temp._meta.database = self.database temp.create_table(temporary=True) # Copy data. copy_data(self._table, self.database, temp_table) self.database.execute_sql(f'CREATE INDEX ON "{temp_table}" ("Gaia_DR2_Source_ID")') self.database.execute_sql(f'CREATE INDEX ON "{temp_table}" ("LegacySurvey_DR8_ID")') self.database.execute_sql(f'CREATE INDEX ON "{temp_table}" ("PanSTARRS_DR2_ID")') self.database.execute_sql(f'CREATE INDEX ON "{temp_table}" ("TwoMASS_ID")') vacuum_table(self.database, temp_table, vacuum=False, analyze=True) inertial_case = peewee.Case( None, ((temp.inertial.cast('boolean').is_null(), False),), temp.inertial.cast('boolean')) query_common = (Catalog .select(Catalog.catalogid, temp.Gaia_DR2_Source_ID.alias('gaia_source_id'), temp.LegacySurvey_DR8_ID.alias('ls_id'), temp.PanSTARRS_DR2_ID.alias('catid_objid'), temp.TwoMASS_ID.alias('designation'), Catalog.ra, Catalog.dec, temp.delta_ra.cast('double precision'), temp.delta_dec.cast('double precision'), inertial_case.alias('inertial'), temp.cadence, temp.priority, temp.instrument, peewee.Value(0).alias('value')) .distinct(Catalog.catalogid)) query_gaia_dr2 = \ (query_common .join(CatalogToTIC_v8) .join(TIC_v8, on=(CatalogToTIC_v8.target_id == TIC_v8.id)) .join(Gaia_DR2, on=(TIC_v8.gaia_int == Gaia_DR2.source_id)) .join(temp, on=(temp.Gaia_DR2_Source_ID == Gaia_DR2.source_id)) .switch(Catalog) .where(CatalogToTIC_v8.version_id == version_id, (CatalogToTIC_v8.best >> True) | CatalogToTIC_v8.best.is_null(), Catalog.version_id == version_id)) query_legacysurvey_dr8 = \ (query_common .join(CatalogToLegacy_Survey_DR8) .join(Legacy_Survey_DR8) .join(temp, on=(temp.LegacySurvey_DR8_ID == Legacy_Survey_DR8.ls_id)) .switch(Catalog) .where(CatalogToLegacy_Survey_DR8.version_id == version_id, (CatalogToLegacy_Survey_DR8.best >> True) | CatalogToLegacy_Survey_DR8.best.is_null(), Catalog.version_id == version_id)) query_panstarrs_dr2 = \ (query_common .join(CatalogToPanstarrs1) .join(Panstarrs1) .join(temp, on=(temp.PanSTARRS_DR2_ID == Panstarrs1.catid_objid)) .switch(Catalog) .where(CatalogToPanstarrs1.version_id == version_id, (CatalogToPanstarrs1.best >> True) | CatalogToPanstarrs1.best.is_null(), Catalog.version_id == version_id)) query_twomass_psc = \ (query_common .join(CatalogToTIC_v8, on=(Catalog.catalogid == CatalogToTIC_v8.catalogid)) .join(TIC_v8, on=(CatalogToTIC_v8.target_id == TIC_v8.id)) .join(TwoMassPSC, on=(TIC_v8.twomass_psc == TwoMassPSC.designation)) .join(temp, on=(temp.TwoMASS_ID == TwoMassPSC.designation)) .switch(Catalog) .where(CatalogToTIC_v8.version_id == version_id, (CatalogToTIC_v8.best >> True) | CatalogToTIC_v8.best.is_null(), Catalog.version_id == version_id)) len_table = len(self._table) len_gaia_dr2 =\ len(self._table[self._table['Gaia_DR2_Source_ID'] > 0]) len_legacysurvey_dr8 =\ len(self._table[self._table['LegacySurvey_DR8_ID'] > 0]) len_panstarrs_dr2 =\ len(self._table[self._table['PanSTARRS_DR2_ID'] > 0]) # TwoMass_ID corresponds to the designation column of # the table catalogdb.twomass_psc. # Since the designation column is a text column, below # we are comparing it to the string 'NA' and not the integer 0. # len_twomass_psc =\ len(self._table[self._table['TwoMASS_ID'] != 'NA']) # There must be exactly one non-zero id per row else raise an exception. if ((len_gaia_dr2 + len_legacysurvey_dr8 + len_panstarrs_dr2 + len_twomass_psc) != len_table): raise TargetSelectionError('error in get_file_carton(): ' + '(len_gaia_dr2 + len_legacysurvey_dr8 + ' + 'len_panstarrs_dr2 + len_twomass_psc) != ' + 'len_table') if (len_gaia_dr2 > 0): is_gaia_dr2 = True else: is_gaia_dr2 = False if (len_legacysurvey_dr8 > 0): is_legacysurvey_dr8 = True else: is_legacysurvey_dr8 = False if (len_panstarrs_dr2 > 0): is_panstarrs_dr2 = True else: is_panstarrs_dr2 = False if (len_twomass_psc > 0): is_twomass_psc = True else: is_twomass_psc = False query = None if(is_gaia_dr2 is True): if(query is None): query = query_gaia_dr2 else: query = query | query_gaia_dr2 if(is_legacysurvey_dr8 is True): if(query is None): query = query_legacysurvey_dr8 else: query = query | query_legacysurvey_dr8 if(is_panstarrs_dr2 is True): if(query is None): query = query_panstarrs_dr2 else: query = query | query_panstarrs_dr2 if(is_twomass_psc is True): if(query is None): query = query_twomass_psc else: query = query | query_twomass_psc if(query is None): # At least one of the four boolean variables above # must be True, so we should not get here. raise TargetSelectionError('error in get_file_carton(): ' + '(is_gaia_dr2 is False) and ' + '(is_legacysurvey_dr8 is False) and ' + '(is_panstarrs_dr2 is False) and ' + '(is_twomass_psc is False)') if 'lambda_eff' in self._table.colnames: query = query.select_extend(temp.lambda_eff.alias('lambda_eff')) return query