def load_photometry(fieldname, instrument, band1, band2, server="marvin"):
"""Get photometry from starplex."""
connect_to_server(server, echo=True)
session = Session()
# potentially a factor 10 speed gain with this
# session.execute('SET ENABLE_NESTLOOP TO FALSE')
mag1obs = aliased(Observation)
mag2obs = aliased(Observation)
bp1 = aliased(Bandpass)
bp2 = aliased(Bandpass)
q = session.query(CatalogStar.cfrac, mag1obs.mag, mag2obs.mag_err,
mag2obs.mag, mag2obs.mag_err)\
.join(mag1obs, CatalogStar.observations)\
.join(mag2obs, CatalogStar.observations)\
.join(Catalog)\
.filter(Catalog.name == fieldname)\
.filter(Catalog.instrument == instrument)\
.join(bp1, mag1obs.bandpass)\
.filter(bp1.name == band1)\
.join(bp2, mag2obs.bandpass)\
.filter(bp2.name == band2)
dt = [('cfrac', np.float), ('m1', np.float), ('e1', np.float),
('m2', np.float), ('e2', np.float)]
data = np.array(q.all(), dtype=np.dtype(dt))
log.info("Field {0} {2} has {1:d} stars".
format(fieldname, data.shape[0], instrument))
# session.execute('SET ENABLE_NESTLOOP TO TRUE')
session.close()
if len(data) == 0:
raise NoDataError
return data
def main():
connect_to_server("marvin", echo=True)
session = Session()
catalogs = session.query(Catalog)
for catalog in catalogs:
catalog.metajson = catalog.meta
session.commit()
def main():
parser = argparse.ArgumentParser(
description="Ingest 2MASS PSC into Starplex")
parser.add_argument('data_dir', action='store',
help="Directory with psc_*.gz files")
parser.add_argument('--name', action='store', default='starplex',
help="Database name")
parser.add_argument('--user', action='store', default='starplex',
help="Database user")
parser.add_argument('--pw', action='store', default=None,
help="Database password")
parser.add_argument('--url', action='store', default='localhost',
help="Database URL")
parser.add_argument('--port', action='store', default=5432, type=int,
help="Database port")
parser.add_argument('--ra', action='store', nargs=2,
default=[0., 360.], type=float,
help="Min and max RA range")
parser.add_argument('--dec', action='store', nargs=2,
default=[-90., 90.], type=float,
help="Min and max Dec range")
args = parser.parse_args()
log.setLevel('INFO')
connect(user=args.user, name=args.name)
session = Session()
create_all()
tm_ingester = TwoMassPSCIngest(session, args.data_dir)
tm_ingester.ingest_region('2MASS_PSC', [7.5, 17], [36, 47])
class TestObservationsORM(object):
def setup_class(self):
connect(user='******', name='starplex_test')
self.session = Session()
create_all()
def teardown_class(self):
self.session.rollback()
self.session.close()
def test_bandpass_uniqueness(self):
bp0 = Bandpass.as_unique(self.session, "V", "Vega")
bp1 = Bandpass.as_unique(self.session, "B", "Vega")
bp0b = Bandpass.as_unique(self.session, "V", "Vega")
assert bp0 is bp0b
assert bp1 is not bp0
class TestObservationsORM(object):
def setup_class(self):
connect(user='******', name='starplex_test')
self.session = Session()
create_all()
def teardown_class(self):
self.session.rollback()
self.session.close()
def test_bandpass_uniqueness(self):
bp0 = Bandpass.as_unique(self.session, "V", "Vega")
bp1 = Bandpass.as_unique(self.session, "B", "Vega")
bp0b = Bandpass.as_unique(self.session, "V", "Vega")
assert bp0 is bp0b
assert bp1 is not bp0
def setup_class(self):
connect(user='******', name='starplex_test')
self.session = Session()
drop_all()
create_all()
init_catalog(self.session,
self.mock.catalog_name,
self.mock.instrument_name,
self.mock.bands,
self.mock.band_sys,
self.mock.footprints,
meta=self.mock.meta)
add_observations(self.session, self.mock.catalog_name,
self.mock.instrument_name, self.mock.bands,
self.mock.band_sys, self.mock.x, self.mock.y,
self.mock.ra, self.mock.ra_err, self.mock.dec,
self.mock.dec_err, self.mock.mag, self.mock.mag_err,
self.mock.cfrac)
def setup_class(self):
print "engine init", engine
connect(user='******', name='starplex_test')
print "engine after connect", engine
self.session = Session()
drop_all()
create_all()
catalog = Catalog(self.mock_dataset.catalog_name,
self.mock_dataset.instrument_name,
None,
fits_path=self.mock_dataset.fits_path)
for i in xrange(self.mock_dataset.n):
cstar = CatalogStar(0., 0., self.mock_dataset.ra[i],
self.mock_dataset.dec[i], 0., 0., 1.)
for j, bandname in enumerate(self.mock_dataset.bands):
bp = Bandpass.as_unique(self.session, bandname,
self.mock_dataset.band_sys)
obs = Observation(self.mock_dataset.mags[j][i], 0.)
obs.bandpass = bp
cstar.observations.append(obs)
catalog.catalog_stars.append(cstar)
self.session.add(catalog)
class TestObservationsORM(object):
mock_dataset = MockCatalog("test1", "myinstr", ['B', 'V'],
[10.0, 10.5], [40.0, 41.])
def setup_class(self):
print "engine init", engine
connect(user='******', name='starplex_test')
print "engine after connect", engine
self.session = Session()
drop_all()
create_all()
catalog = Catalog(self.mock_dataset.catalog_name,
self.mock_dataset.instrument_name,
None,
fits_path=self.mock_dataset.fits_path)
for i in xrange(self.mock_dataset.n):
cstar = CatalogStar(0., 0., self.mock_dataset.ra[i],
self.mock_dataset.dec[i], 0., 0., 1.)
for j, bandname in enumerate(self.mock_dataset.bands):
bp = Bandpass.as_unique(self.session, bandname,
self.mock_dataset.band_sys)
obs = Observation(self.mock_dataset.mags[j][i], 0.)
obs.bandpass = bp
cstar.observations.append(obs)
catalog.catalog_stars.append(cstar)
self.session.add(catalog)
def teardown_class(self):
self.session.rollback()
self.session.close()
def test_band_insert(self):
bp = self.session.query(Bandpass).\
filter(Bandpass.name == self.mock_dataset.bands[0]).one()
print bp
print bp.name, bp.system
assert bp.name == self.mock_dataset.bands[0]
assert bp.system == self.mock_dataset.band_sys
def test_hstore_metadata_read(self):
c = self.session.query(Catalog).\
filter(Catalog.name == self.mock_dataset.catalog_name).one()
assert c.meta['fits_path'] == self.mock_dataset.fits_path
class TestObservationsORM(object):
mock_dataset = MockCatalog("test1", "myinstr", ['B', 'V'], [10.0, 10.5],
[40.0, 41.])
def setup_class(self):
print "engine init", engine
connect(user='******', name='starplex_test')
print "engine after connect", engine
self.session = Session()
drop_all()
create_all()
catalog = Catalog(self.mock_dataset.catalog_name,
self.mock_dataset.instrument_name,
None,
fits_path=self.mock_dataset.fits_path)
for i in xrange(self.mock_dataset.n):
cstar = CatalogStar(0., 0., self.mock_dataset.ra[i],
self.mock_dataset.dec[i], 0., 0., 1.)
for j, bandname in enumerate(self.mock_dataset.bands):
bp = Bandpass.as_unique(self.session, bandname,
self.mock_dataset.band_sys)
obs = Observation(self.mock_dataset.mags[j][i], 0.)
obs.bandpass = bp
cstar.observations.append(obs)
catalog.catalog_stars.append(cstar)
self.session.add(catalog)
def teardown_class(self):
self.session.rollback()
self.session.close()
def test_band_insert(self):
bp = self.session.query(Bandpass).\
filter(Bandpass.name == self.mock_dataset.bands[0]).one()
print bp
print bp.name, bp.system
assert bp.name == self.mock_dataset.bands[0]
assert bp.system == self.mock_dataset.band_sys
def test_hstore_metadata_read(self):
c = self.session.query(Catalog).\
filter(Catalog.name == self.mock_dataset.catalog_name).one()
assert c.meta['fits_path'] == self.mock_dataset.fits_path
def load_photometry(fieldname, tile, radec_poly,
use_vega, disable_intercal,
server='marvin'):
"""Load WIRCam photometry from Starplex, converted to AB mag."""
instrument = "wircam"
connect_to_server(server)
session = Session()
mag1obs = aliased(Observation)
mag2obs = aliased(Observation)
bp1 = aliased(Bandpass)
bp2 = aliased(Bandpass)
catalog = session.query(Catalog).\
filter(Catalog.name == fieldname).\
filter(Catalog.instrument == instrument).\
one()
q = session.query(CatalogStar.cfrac, mag1obs.mag, mag2obs.mag_err,
mag2obs.mag, mag2obs.mag_err).\
join(mag1obs, CatalogStar.observations).\
join(mag2obs, CatalogStar.observations).\
join(Catalog).\
filter(Catalog.name == fieldname).\
filter(Catalog.instrument == instrument).\
join(bp1, mag1obs.bandpass).\
filter(bp1.name == "J").\
join(bp2, mag2obs.bandpass).\
filter(bp2.name == "Ks").\
filter(func.q3c_poly_query(CatalogStar.ra,
CatalogStar.dec,
np.array(radec_poly).flatten().tolist()))
dt = [('cfrac', np.float), ('J', np.float), ('J_err', np.float),
('Ks', np.float), ('Ks_err', np.float)]
data = np.array(q.all(), dtype=np.dtype(dt))
# Apply the intercal ZP correction
if not disable_intercal:
if 'intercal' in catalog.meta:
for band in ['J', 'Ks']:
if band in catalog.meta['intercal']:
data[band] += catalog.meta['intercal'][band]['zp']
# Convert to AB
if not use_vega:
data['J'] = wircam_vega_to_ab(data['J'], "J")
data['Ks'] = wircam_vega_to_ab(data['Ks'], "Ks")
log.info("Field {0} {2} {3:d} has {1:d} stars".
format(fieldname, data.shape[0], instrument, tile))
session.close()
if len(data) == 0:
raise NoDataError
return data
class TestObservationsCoreIngest(object):
mock = MockCatalog("test1", "myinstr", ['B', 'V'], [10.0, 10.5],
[40.0, 41.])
def setup_class(self):
connect(user='******', name='starplex_test')
self.session = Session()
drop_all()
create_all()
init_catalog(self.session,
self.mock.catalog_name,
self.mock.instrument_name,
self.mock.bands,
self.mock.band_sys,
self.mock.footprints,
meta=self.mock.meta)
add_observations(self.session, self.mock.catalog_name,
self.mock.instrument_name, self.mock.bands,
self.mock.band_sys, self.mock.x, self.mock.y,
self.mock.ra, self.mock.ra_err, self.mock.dec,
self.mock.dec_err, self.mock.mag, self.mock.mag_err,
self.mock.cfrac)
def teardown_class(self):
# drop_all()
# create_all()
# self.session.commit()
self.session.close()
def test_band_insert(self):
bp = self.session.query(Bandpass).\
filter(Bandpass.name == self.mock.bands[0]).one()
assert bp.name == self.mock.bands[0]
assert bp.system == self.mock.band_sys
def test_hstore_metadata_read(self):
c = self.session.query(Catalog).\
filter(Catalog.name == self.mock.catalog_name).one()
assert c.meta['fits_path'] == self.mock.meta['fits_path']
def test_count_catalog_stars(self):
c = self.session.query(Catalog).\
filter(Catalog.name == self.mock.catalog_name).one()
assert self.mock.n == self.session.query(CatalogStar).\
filter(CatalogStar.catalog == c).\
count()
def setup_class(self):
print "engine init", engine
connect(user='******', name='starplex_test')
print "engine after connect", engine
self.session = Session()
drop_all()
create_all()
catalog = Catalog(self.mock_dataset.catalog_name,
self.mock_dataset.instrument_name,
None,
fits_path=self.mock_dataset.fits_path)
for i in xrange(self.mock_dataset.n):
cstar = CatalogStar(0., 0., self.mock_dataset.ra[i],
self.mock_dataset.dec[i], 0., 0., 1.)
for j, bandname in enumerate(self.mock_dataset.bands):
bp = Bandpass.as_unique(self.session, bandname,
self.mock_dataset.band_sys)
obs = Observation(self.mock_dataset.mags[j][i], 0.)
obs.bandpass = bp
cstar.observations.append(obs)
catalog.catalog_stars.append(cstar)
self.session.add(catalog)
def setup_class(self):
connect(user='******', name='starplex_test')
self.session = Session()
create_all()
def load_photometry(fieldname,
use_vega=False,
apply_intercal=False,
server='marvin'):
"""Load WIRCam photometry from Starplex, converted to AB mag.
Filter out MW stars with a rudimentary J-Ks > 0.9 (Vega) color cut.
"""
instrument = "wircam"
connect_to_server(server)
session = Session()
mag1obs = aliased(Observation)
mag2obs = aliased(Observation)
bp1 = aliased(Bandpass)
bp2 = aliased(Bandpass)
catalog = session.query(Catalog).\
filter(Catalog.name == fieldname).\
filter(Catalog.instrument == instrument).\
one()
q = session.query(CatalogStar.cfrac, CatalogStar.ra, CatalogStar.dec,
mag1obs.mag, mag1obs.mag_err,
mag2obs.mag, mag2obs.mag_err).\
join(mag1obs, CatalogStar.observations).\
join(mag2obs, CatalogStar.observations).\
join(Catalog).\
filter(Catalog.name == fieldname).\
filter(Catalog.instrument == instrument).\
join(bp1, mag1obs.bandpass).\
filter(bp1.name == "J").\
join(bp2, mag2obs.bandpass).\
filter(bp2.name == "Ks")
dt = [('cfrac', np.float), ('ra', np.float), ('dec', np.float),
('J', np.float), ('J_err', np.float), ('Ks', np.float),
('Ks_err', np.float)]
data = np.array(q.all(), dtype=np.dtype(dt))
# Filter out MW stars
# FIXME rudimentary
# Using Vega Mag here!
sel = np.where((data['J'] - data['Ks']) > 0.9)[0]
data = data[sel]
# Apply the intercal ZP correction
if apply_intercal:
if 'intercal' in catalog.meta:
for band in ['J', 'Ks']:
if band in catalog.meta['intercal']:
data[band] += catalog.meta['intercal'][band]['zp']
# Convert to AB
if not use_vega:
data['J'] = wircam_vega_to_ab(data['J'], "J")
data['Ks'] = wircam_vega_to_ab(data['Ks'], "Ks")
log.info("Field {0} {2} has {1:d} stars".format(fieldname, data.shape[0],
instrument))
session.close()
if len(data) == 0:
raise NoDataError
return data
def load_photometry(fieldname,
use_vega=False, apply_intercal=False,
server='marvin'):
"""Load WIRCam photometry from Starplex, converted to AB mag.
Filter out MW stars with a rudimentary J-Ks > 0.9 (Vega) color cut.
"""
instrument = "wircam"
connect_to_server(server)
session = Session()
mag1obs = aliased(Observation)
mag2obs = aliased(Observation)
bp1 = aliased(Bandpass)
bp2 = aliased(Bandpass)
catalog = session.query(Catalog).\
filter(Catalog.name == fieldname).\
filter(Catalog.instrument == instrument).\
one()
q = session.query(CatalogStar.cfrac, CatalogStar.ra, CatalogStar.dec,
mag1obs.mag, mag1obs.mag_err,
mag2obs.mag, mag2obs.mag_err).\
join(mag1obs, CatalogStar.observations).\
join(mag2obs, CatalogStar.observations).\
join(Catalog).\
filter(Catalog.name == fieldname).\
filter(Catalog.instrument == instrument).\
join(bp1, mag1obs.bandpass).\
filter(bp1.name == "J").\
join(bp2, mag2obs.bandpass).\
filter(bp2.name == "Ks")
dt = [('cfrac', np.float), ('ra', np.float), ('dec', np.float),
('J', np.float), ('J_err', np.float),
('Ks', np.float), ('Ks_err', np.float)]
data = np.array(q.all(), dtype=np.dtype(dt))
# Filter out MW stars
# FIXME rudimentary
# Using Vega Mag here!
sel = np.where((data['J'] - data['Ks']) > 0.9)[0]
data = data[sel]
# Apply the intercal ZP correction
if apply_intercal:
if 'intercal' in catalog.meta:
for band in ['J', 'Ks']:
if band in catalog.meta['intercal']:
data[band] += catalog.meta['intercal'][band]['zp']
# Convert to AB
if not use_vega:
data['J'] = wircam_vega_to_ab(data['J'], "J")
data['Ks'] = wircam_vega_to_ab(data['Ks'], "Ks")
log.info("Field {0} {2} has {1:d} stars".
format(fieldname, data.shape[0], instrument))
session.close()
if len(data) == 0:
raise NoDataError
return data
def setup_class(self):
connect(user='******', name='starplex_test')
self.session = Session()
create_all()