def __init__(self, repo, db_info=None):
"""
Keep track of the Level 2 repository and create a db connection.
"""
self.repo = repo
if db_info is None:
db_info = dict(db='jc_desc', read_default_file='~/.my.cnf')
self.conn = DbConnection(**db_info)
def __init__(self, repo=None, db_info=None, tract='0', patch='0,0',
project=None):
"""
Class constructor.
"""
self.repo = repo
if db_info is None:
db_info = dict(database='DESC_Twinkles_Level_2',
host='scidb1.nersc.gov')
self.conn = DbConnection(**db_info)
self.tract = str(tract)
self.patch = patch
self.project = project
import time
import numpy as np
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
from desc.pserv import DbConnection
from light_curve_service import LightCurveFactory
plt.ion()
def normed_hist(x, bins=20):
xnorm = x/max(x)
my_hist = np.histogram(xnorm, bins=bins)
return my_hist[0]
db_info = dict(db='jc_desc', read_default_file='~/.my.cnf')
jc_desc = DbConnection(**db_info)
lc_factory = LightCurveFactory(**db_info)
band = 'u'
#band = 'r'
# find the number of visits for the selected band
num_visits = jc_desc.apply("select count(1) from CcdVisit where filterName='%(band)s'" % locals(), lambda curs: [x[0] for x in curs][0])
dof = num_visits - 1
# Find all of the light curves from deblended sources with chisq > 1e4
query = '''select cs.objectId from Chisq cs
join Object obj on cs.objectId=obj.objectId
join ObjectNumChildren numCh on obj.objectId=numCh.objectId
where cs.filterName='%(band)s' and cs.dof=%(dof)i
and cs.chisq > 1e4
class Level2DataService(object):
"""Provide access to the Twinkles Level 2 data from a MySQL/pserv
database.
Parameters
----------
repo : str or None, optional
The output repository from the Level 2 analysis.
db_info : dict or None, optional
Connection information for the Level 2 database. It should
have the form
{'database': <database name>,
'host': <database server host address>}
If None, then the default info for using the Twinkles Level 2
database at NERSC is used:
{'database': 'DESC_Twinkles_Level_2',
'host': 'scidb1.nersc.gov'}
tract : int or str, optional
The "tract", as assigned by the LSST Stack.
patch : str, optional
The "patch", as assigned by the LSST Stack, e.g., "0,0".
project : str or None, optional
The name of the DESC project. This is part of the primary key
in the pserv Level 2 database tables, so it is generally
needed to differentiate results from different projects using
the same database instance.
Attributes
----------
repo : str or None
The output repository from the Level 2 analysis.
conn : desc.pserv.DbConnection
The connection object that executes the queries to the Level 2 db.
project : str or None
The name of the DESC project.
tract : str
The "tract", as assigned by the LSST Stack.
patch : str
The "patch", as assigned by the LSST Stack, e.g., "0,0".
Notes
-----
Database credentials and connection info (e.g., the port to use)
are accessed via lsst.daf.persistence.DbAuth from the user's
~/.lsst/db-auth.paf file.
"""
def __init__(self, repo=None, db_info=None, tract='0', patch='0,0',
project=None):
"""
Class constructor.
"""
self.repo = repo
if db_info is None:
db_info = dict(database='DESC_Twinkles_Level_2',
host='scidb1.nersc.gov')
self.conn = DbConnection(**db_info)
self.tract = str(tract)
self.patch = patch
self.project = project
def get_pixel_data(self, objectId, band, size=10, pickle_file=None):
"""Get the pixel data for a cutout region for all visits.
Use PostageStampMaker to extract the pixel data cutout for the
specified objectId and band for each visit in the repository.
Parameters
----------
objectId : int
The id of the object in the Level2 Object db table.
band : str
The LSST band (u, g, r, i, z, or y).
size : int, optional
The size of the cutout region in arcsec.
Cutouts of size x size centered on the RA, Dec of the
object will be extracted.
pickle_file : str, optional
The name of the pickle file to contain the extracted pixel
data. If it is None, then a default name of the form
'pixel_data_%(objectId)i_%(band)s.pkl' will be used. If
that file exists, the pixel data will be loaded from it.
If it does not exist, the pixel data will be extracted from
the deepCoadd tempExp frames.
Returns
-------
pixel_data : OrderedDict
A dictionary of 2D numpy arrays containing the pixel values.
The keys are the visitId numbers from the CcdVisit table.
pickle_file : str
The name of the pickle file.
"""
if pickle_file is None:
pickle_file = 'pixel_data_%(objectId)i_%(band)s.pkl' % locals()
if os.path.isfile(pickle_file):
with open(pickle_file, 'r') as input_:
pixel_data = pickle.load(input_)
return pixel_data, pickle_file
pixel_data = OrderedDict()
visits = self.get_visits(band)
ra, dec = self.get_coords(objectId)
for visit in visits:
print("working on visit", visit)
sys.stdout.flush()
exp_file = self._get_warped_visit_frame(band, visit)
maker = PostageStampMaker(exp_file)
stamp = maker.create(ra, dec, size)
pixel_data[visit] = \
copy.deepcopy(stamp.getMaskedImage().getImage().getArray())
with open(pickle_file, 'w') as output:
pickle.dump(pixel_data, output)
return pixel_data, pickle_file
def get_deepCoadd_frame(self, band):
"""Get the deepCoadd frame for the specified band
Parameters
----------
band : str
The LSST band (u, g, r, i, z, or y).
Returns
-------
str
File path to the FITS file containing the coadd the
specified band.
Raises
------
RuntimeError
If the repo attribute has not been set.
Notes
-----
This should be re-implemented to use the Data Butler.
"""
if self.repo is None:
raise RuntimeError('Level 2 respository not set.')
return os.path.join(self.repo, 'deepCoadd', band,
self.tract, self.patch + '.fits')
def _get_warped_visit_frame(self, band, visit):
"""Get the deepCoadd tempExp frame for the specified band and visit.
Parameters
----------
band : str
The LSST band (u, g, r, i, z, or y).
visit : int
The visitId of the visit from the CcdVisit table.
Returns
-------
str
File path to the FITS file containing the warped exposure for
the specified band and visit.
Raises
------
RuntimeError
If the repo attribute has not been set.
Notes
-----
This should be re-implemented to use the Data Butler.
"""
if self.repo is None:
raise RuntimeError('Level 2 respository not set.')
return os.path.join(self.repo, 'deepCoadd', band,
self.tract, self.patch + 'tempExp',
'v%(visit)i-f%(band)s.fits' % locals())
def get_light_curve(self, objectId, band):
""" Get the light curve for the requested objectId and band.
Parameters
----------
objectId : int
The id of the object as specified in the Level 2 Object table.
band : str
The LSST band (u, g, r, i, z, or y).
Returns
-------
dict
A dictionary of numpy arrays containing the light curve data.
The format is
{'mjd': <Visit obs times in MJD>,
'flux': <Fluxes in nmgy>,
'fluxerr': <Flux errors in nmgy>}
Notes
-----
This method queries the CcdVisit and ForcedSource tables for
the CcdVisit.obsStart, ForcedSource.psFlux, and
ForcedSource.psFlux_Sigma values.
"""
query = """select cv.obsStart, fs.psFlux, fs.psFlux_Sigma from
CcdVisit cv join ForcedSource fs on cv.ccdVisitId=fs.ccdVisitId
and cv.project=fs.project
where cv.filterName='%(band)s' and fs.objectId=%(objectId)i""" \
% locals()
if self.project is not None:
query += " and fs.project='%s'" % self.project
query += " order by cv.obsStart asc"
rows = self.conn.apply(query, lambda curs: np.array([x for x in curs]))
obsStart, flux, fluxerr = (np.array(col) for col in rows.transpose())
mjd = astropy.time.Time(obsStart).mjd
return dict(mjd=mjd, flux=flux, fluxerr=fluxerr)
def get_visits(self, band):
"""Get the visitIds corresponding to the specified band.
Parameters
----------
band : str
The LSST band (u, g, r, i, z, or y).
Returns
-------
list
A list of visitIds from the CcdVisit table.
"""
query = """select visitId from CcdVisit where
filterName='%(band)s'""" % locals()
if self.project is not None:
query += " and project='%s'" % self.project
query += " order by visitId"
return np.unique(self.conn.apply(query,
lambda curs: [x[0] for x in curs]))
def get_objects(self, ra, dec, box_size):
"""Get coadd objects within a box of size box_size x box_size arcsec
centered on ra, dec.
Parameters
----------
ra : float
Right Ascension (J2000) in degrees
dec : float
Declination (J2000) in degrees
box_size : float
Size of search box in arcsec.
Returns
-------
dict
A dictionary of (RA, Dec) tuples keyed by objectId.
"""
size = box_size/3600. # convert from arcsec to degrees
cos_dec = np.abs(np.cos(dec*np.pi/180.))
ra_min, ra_max = ra - size/cos_dec, ra + size/cos_dec
dec_min, dec_max = dec - size, dec + size
query = '''select objectId, psRa, psDecl from Object obj where
%(ra_min)12.8f < psRa and psRa < %(ra_max)12.8f and
%(dec_min)12.8f < psDecl and psDecl < %(dec_max)12.8f''' \
% locals()
if self.project is not None:
query += " and project='%s'" % self.project
return self.conn.apply(query, lambda curs: dict([(x[0], x[1:3])
for x in curs]))
def get_coords(self, objectId):
"""Get the RA, Dec of the requested object from the Object db table.
Parameters
----------
objectId : int
The id of the object as specified in the Level 2 Object table.
Returns
-------
tuple
The objects RA, Dec in degrees.
"""
return self.conn.apply('''select psRA, psDecl from Object where
objectId=%(objectId)i''' % locals(),
lambda curs: tuple([x for x in curs][0]))
import numpy as np
import matplotlib.pyplot as plt
import astropy.io.fits as fits
import astropy.wcs
from desc.pserv import DbConnection
from light_curve_service import LightCurveFactory
from desc.twinkles import PostageStampMaker, render_fits_image
plt.ion()
try:
objectId = int(sys.argv[1])
except:
objectId = 57500
db_info = dict(db='jc_desc', read_default_file='~/.my.cnf')
jc_desc = DbConnection(**db_info)
lc_factory = LightCurveFactory(**db_info)
lc = lc_factory.create(objectId)
lc.plot(figtext='objectId %i' % objectId)
query = 'select psRa, psDecl from Object where objectId=%(objectId)i' % locals()
ra, dec = jc_desc.apply(query, lambda curs : tuple([x for x in curs][0]))
print ra, dec
chisq_min = 0
dof = 0
size_arcsec = 10.
size = size_arcsec/3600. # convert to degrees
class Level2DataService(object):
"""
Access to the Twinkles Level 2 data and pserv database.
"""
def __init__(self, repo, db_info=None):
"""
Keep track of the Level 2 repository and create a db connection.
"""
self.repo = repo
if db_info is None:
db_info = dict(db='jc_desc', read_default_file='~/.my.cnf')
self.conn = DbConnection(**db_info)
def get_pixel_data(self, objectId, band, size=10, pickle_file=None):
"""
Use PostageStampMaker to extract the pixel data cutout for the
specified objectId andn band for each visit.
"""
if pickle_file is None:
pickle_file = 'pixel_data_%(objectId)i_%(band)s.pkl' % locals()
if os.path.isfile(pickle_file):
with open(pickle_file, 'r') as input_:
pixel_data = pickle.load(input_)
return pixel_data, pickle_file
pixel_data = OrderedDict()
visits = self.get_visits(band)
ra, dec = self.get_coords(objectId)
for visit in visits:
print("working on visit", visit)
sys.stdout.flush()
exp_file = os.path.join(repo, 'deepCoadd', band, '0/0,0tempExp',
'v%(visit)i-f%(band)s.fits' % locals())
maker = PostageStampMaker(exp_file)
stamp = maker.create(ra, dec, size)
pixel_data[visit] = \
copy.deepcopy(stamp.getMaskedImage().getImage().getArray())
with open(pickle_file, 'w') as output:
pickle.dump(pixel_data, output)
return pixel_data, pickle_file
def get_light_curve(self, objectId, band):
"""
Get the light curve for the requested objectId and band.
"""
query = """select cv.obsStart, fs.psFlux, fs.psFlux_Sigma from
CcdVisit cv join ForcedSource fs on cv.ccdVisitId=fs.ccdVisitId
join Object obj on fs.objectId=obj.objectId where
cv.filterName='%(band)s' and fs.objectId=%(objectId)i
order by cv.obsStart asc""" % locals()
rows = self.conn.apply(query, lambda curs: np.array([x for x in curs]))
obsStart, flux, fluxerr = (np.array(col) for col in rows.transpose())
mjd = astropy.time.Time(obsStart).mjd
return dict(mjd=mjd, flux=flux, fluxerr=fluxerr)
def get_visits(self, band):
"""
Get the visitIds corresponding to the specified band.
"""
return self.conn.apply('''select visitId from CcdVisit where
filterName='%(band)s' order by visitId'''
% locals(),
lambda curs: [x[0] for x in curs])
def get_coords(self, objectId):
"""
Get the RA, Dec of the requested object from the Object table.
"""
return self.conn.apply('''select psRA, psDecl from Object where
objectId=%(objectId)i''' % locals(),
lambda curs: tuple([x for x in curs][0]))