def main():
assert database.connected
file_ = os.environ['CATALOGDB_DIR'] + '/cataclysmicvariables.fits'
data = astropy.table.Table.read(file_)
copy_data(data, database, 'cataclysmic_variables', schema='catalogdb')
def main():
assert database.connected
file_ = os.environ['CATALOGDB_DIR'] + '/Kounkel_Clustering.fits'
data = astropy.table.Table.read(file_)
copy_data(data, database, 'yso_clustering', schema='catalogdb')
def main():
assert database.connected
file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/lmxb_good.fits'
data = astropy.table.Table.read(file_)
source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME']))
data.add_column(astropy.table.Column(source_id, 'gaia_source_id'))
copy_data(data, database, 'lmxb', schema='catalogdb')
def main():
assert database.connected
database.become_admin()
file_ = os.environ['CATALOGDB_DIR'] + '/MIPSGAL/MIPSGAL_catalog.fits'
data = astropy.table.Table.read(file_)
data.meta = {}
copy_data(data, database, 'mipsgal', schema='catalogdb')
file_ = os.environ['CATALOGDB_DIR'] + '/MIPSGAL/MIPSGAL_archive.fits'
data = astropy.table.Table.read(file_)
data.meta = {}
copy_data(data, database, 'mipsgal', schema='catalogdb')
def main():
assert database.connected
file_ = (
'/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/'
'target/catalogs/csc/csc_for_catalogdb_v0/CSC2_stub1_realonly_v0.1.0.fits'
)
data = astropy.table.Table.read(file_)
data.add_column(astropy.table.Column(numpy.arange(len(data)) + 1, 'pk'),
0) # Add id pk.
copy_data(data, database, 'bhm_csc', schema='catalogdb')
def main():
assert database.connected
replacements = {
'RAdeg': 'ra',
'DEdeg': 'dec',
'e_RAdeg': 'e_ra',
'e_DEdeg': 'e_dec',
'pmDE': 'pmdec',
'e_pmDE': 'e_pmdec',
'E(BR/RP)': 'e_br_rp',
'(S/N)': 'snr',
'FG': 'fg_gaia',
'e_FG': 'e_fg_gaia',
'Gmag': 'g_gaia_mag'
}
path_ = (
'/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/wd/dr2/'
)
for file_, table in (('J_MNRAS_482_4570_gaia2wd.dat.gz.fits.gz',
'gaia_dr2_wd'),
('J_MNRAS_482_4570_gaiasdss.dat.gz.fits.gz',
'gaia_dr2_wd_sdss')):
data = astropy.table.Table.read(path_ + file_)
for column_name in data.colnames:
if column_name in replacements:
new_column_name = replacements[column_name]
else:
new_column_name = column_name.lower()
data.rename_column(column_name, new_column_name)
data['wd'][:] = numpy.core.defchararray.strip(data['wd'].data)
if 'gaia2wd' in file_:
name_split = numpy.core.defchararray.split(data['dr2name'])
source_id = numpy.array(list(zip(*name_split))[2], numpy.int64)
data.add_column(astropy.table.Column(source_id, 'source_id'),
data.index_column('source'))
else:
pk = numpy.arange(len(data)) + 1
data.add_column(astropy.table.Column(pk, 'pk'), 0)
copy_data(data, database, table, schema='catalogdb')
def main():
assert database.connected
path_ = ('/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/'
'target/catalogs/eRosita/eFEDS_for_catalogdb_v0/')
for file_, table in (('BHM_SPIDERS_EFEDS_SUPERSET_CLUS_v0.1.2.fits',
'bhm_spiders_clusters_superset'),
('BHM_SPIDERS_EFEDS_SUPERSET_AGN_v0.1.1.fits',
'bhm_spiders_agn_superset')):
data = astropy.table.Table.read(path_ + file_)
data.add_column(astropy.table.Column(
numpy.arange(len(data)) + 1, 'pk'), 0) # Add id pk.
copy_data(data, database, table, schema='catalogdb')
def build_query(self, version_id, query_region=None):
self.log.debug(f'Processing file {self._file_path}.')
# We need to copy the data to a temporary table so that we can
# join on it. We could use a Peewee ValueList but for large tables
# that will hit the limit of 1GB in PSQL.
# Create model for temporary table from FITS table columns.
# This works fine because we know there are no arrays.
temp_table = self.name.lower() + '_temp'
temp = create_model_from_table(temp_table, self._table)
temp._meta.database = self.database
temp.create_table(temporary=True)
# Copy data.
copy_data(self._table, self.database, temp_table)
self.database.execute_sql(f'CREATE INDEX ON "{temp_table}" ("Gaia_DR2_Source_ID")')
self.database.execute_sql(f'CREATE INDEX ON "{temp_table}" ("LegacySurvey_DR8_ID")')
self.database.execute_sql(f'CREATE INDEX ON "{temp_table}" ("PanSTARRS_DR2_ID")')
self.database.execute_sql(f'CREATE INDEX ON "{temp_table}" ("TwoMASS_ID")')
vacuum_table(self.database, temp_table, vacuum=False, analyze=True)
inertial_case = peewee.Case(
None,
((temp.inertial.cast('boolean').is_null(), False),),
temp.inertial.cast('boolean'))
query_common = (Catalog
.select(Catalog.catalogid,
temp.Gaia_DR2_Source_ID.alias('gaia_source_id'),
temp.LegacySurvey_DR8_ID.alias('ls_id'),
temp.PanSTARRS_DR2_ID.alias('catid_objid'),
temp.TwoMASS_ID.alias('designation'),
Catalog.ra,
Catalog.dec,
temp.delta_ra.cast('double precision'),
temp.delta_dec.cast('double precision'),
inertial_case.alias('inertial'),
temp.cadence,
temp.priority,
temp.instrument,
peewee.Value(0).alias('value'))
.distinct(Catalog.catalogid))
query_gaia_dr2 = \
(query_common
.join(CatalogToTIC_v8)
.join(TIC_v8, on=(CatalogToTIC_v8.target_id == TIC_v8.id))
.join(Gaia_DR2, on=(TIC_v8.gaia_int == Gaia_DR2.source_id))
.join(temp,
on=(temp.Gaia_DR2_Source_ID == Gaia_DR2.source_id))
.switch(Catalog)
.where(CatalogToTIC_v8.version_id == version_id,
(CatalogToTIC_v8.best >> True) |
CatalogToTIC_v8.best.is_null(),
Catalog.version_id == version_id))
query_legacysurvey_dr8 = \
(query_common
.join(CatalogToLegacy_Survey_DR8)
.join(Legacy_Survey_DR8)
.join(temp,
on=(temp.LegacySurvey_DR8_ID == Legacy_Survey_DR8.ls_id))
.switch(Catalog)
.where(CatalogToLegacy_Survey_DR8.version_id == version_id,
(CatalogToLegacy_Survey_DR8.best >> True) |
CatalogToLegacy_Survey_DR8.best.is_null(),
Catalog.version_id == version_id))
query_panstarrs_dr2 = \
(query_common
.join(CatalogToPanstarrs1)
.join(Panstarrs1)
.join(temp,
on=(temp.PanSTARRS_DR2_ID == Panstarrs1.catid_objid))
.switch(Catalog)
.where(CatalogToPanstarrs1.version_id == version_id,
(CatalogToPanstarrs1.best >> True) |
CatalogToPanstarrs1.best.is_null(),
Catalog.version_id == version_id))
query_twomass_psc = \
(query_common
.join(CatalogToTIC_v8,
on=(Catalog.catalogid == CatalogToTIC_v8.catalogid))
.join(TIC_v8,
on=(CatalogToTIC_v8.target_id == TIC_v8.id))
.join(TwoMassPSC,
on=(TIC_v8.twomass_psc == TwoMassPSC.designation))
.join(temp,
on=(temp.TwoMASS_ID == TwoMassPSC.designation))
.switch(Catalog)
.where(CatalogToTIC_v8.version_id == version_id,
(CatalogToTIC_v8.best >> True) |
CatalogToTIC_v8.best.is_null(),
Catalog.version_id == version_id))
len_table = len(self._table)
len_gaia_dr2 =\
len(self._table[self._table['Gaia_DR2_Source_ID'] > 0])
len_legacysurvey_dr8 =\
len(self._table[self._table['LegacySurvey_DR8_ID'] > 0])
len_panstarrs_dr2 =\
len(self._table[self._table['PanSTARRS_DR2_ID'] > 0])
# TwoMass_ID corresponds to the designation column of
# the table catalogdb.twomass_psc.
# Since the designation column is a text column, below
# we are comparing it to the string 'NA' and not the integer 0.
#
len_twomass_psc =\
len(self._table[self._table['TwoMASS_ID'] != 'NA'])
# There must be exactly one non-zero id per row else raise an exception.
if ((len_gaia_dr2 + len_legacysurvey_dr8 +
len_panstarrs_dr2 + len_twomass_psc) != len_table):
raise TargetSelectionError('error in get_file_carton(): ' +
'(len_gaia_dr2 + len_legacysurvey_dr8 + ' +
'len_panstarrs_dr2 + len_twomass_psc) != ' +
'len_table')
if (len_gaia_dr2 > 0):
is_gaia_dr2 = True
else:
is_gaia_dr2 = False
if (len_legacysurvey_dr8 > 0):
is_legacysurvey_dr8 = True
else:
is_legacysurvey_dr8 = False
if (len_panstarrs_dr2 > 0):
is_panstarrs_dr2 = True
else:
is_panstarrs_dr2 = False
if (len_twomass_psc > 0):
is_twomass_psc = True
else:
is_twomass_psc = False
query = None
if(is_gaia_dr2 is True):
if(query is None):
query = query_gaia_dr2
else:
query = query | query_gaia_dr2
if(is_legacysurvey_dr8 is True):
if(query is None):
query = query_legacysurvey_dr8
else:
query = query | query_legacysurvey_dr8
if(is_panstarrs_dr2 is True):
if(query is None):
query = query_panstarrs_dr2
else:
query = query | query_panstarrs_dr2
if(is_twomass_psc is True):
if(query is None):
query = query_twomass_psc
else:
query = query | query_twomass_psc
if(query is None):
# At least one of the four boolean variables above
# must be True, so we should not get here.
raise TargetSelectionError('error in get_file_carton(): ' +
'(is_gaia_dr2 is False) and ' +
'(is_legacysurvey_dr8 is False) and ' +
'(is_panstarrs_dr2 is False) and ' +
'(is_twomass_psc is False)')
if 'lambda_eff' in self._table.colnames:
query = query.select_extend(temp.lambda_eff.alias('lambda_eff'))
return query