def __init__(self, repo, db_info=None):
     """
     Keep track of the Level 2 repository and create a db connection.
     """
     self.repo = repo
     if db_info is None:
         db_info = dict(db='jc_desc', read_default_file='~/.my.cnf')
     self.conn = DbConnection(**db_info)
 def __init__(self, repo=None, db_info=None, tract='0', patch='0,0',
              project=None):
     """
     Class constructor.
     """
     self.repo = repo
     if db_info is None:
         db_info = dict(database='DESC_Twinkles_Level_2',
                        host='scidb1.nersc.gov')
     self.conn = DbConnection(**db_info)
     self.tract = str(tract)
     self.patch = patch
     self.project = project
Example #3
0
import time
import numpy as np
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
from desc.pserv import DbConnection
from light_curve_service import LightCurveFactory
plt.ion()

def normed_hist(x, bins=20):
    xnorm = x/max(x)
    my_hist = np.histogram(xnorm, bins=bins)
    return my_hist[0]

db_info = dict(db='jc_desc', read_default_file='~/.my.cnf')

jc_desc = DbConnection(**db_info)
lc_factory = LightCurveFactory(**db_info)

band = 'u'
#band = 'r'

# find the number of visits for the selected band
num_visits = jc_desc.apply("select count(1) from CcdVisit where filterName='%(band)s'" % locals(), lambda curs: [x[0] for x in curs][0])
dof = num_visits - 1

# Find all of the light curves from deblended sources with chisq > 1e4
query = '''select cs.objectId from Chisq cs
           join Object obj on cs.objectId=obj.objectId
           join ObjectNumChildren numCh on obj.objectId=numCh.objectId
           where cs.filterName='%(band)s' and cs.dof=%(dof)i
           and cs.chisq > 1e4
class Level2DataService(object):
    """Provide access to the Twinkles Level 2 data from a MySQL/pserv
    database.

    Parameters
    ----------
    repo : str or None, optional
        The output repository from the Level 2 analysis.
    db_info : dict or None, optional
        Connection information for the Level 2 database.  It should
        have the form

        {'database': <database name>,
         'host': <database server host address>}

        If None, then the default info for using the Twinkles Level 2
        database at NERSC is used:

        {'database': 'DESC_Twinkles_Level_2',
         'host': 'scidb1.nersc.gov'}
    tract : int or str, optional
        The "tract", as assigned by the LSST Stack.
    patch : str, optional
        The "patch", as assigned by the LSST Stack, e.g., "0,0".
    project : str or None, optional
        The name of the DESC project.  This is part of the primary key
        in the pserv Level 2 database tables, so it is generally
        needed to differentiate results from different projects using
        the same database instance.

    Attributes
    ----------
    repo : str or None
        The output repository from the Level 2 analysis.
    conn : desc.pserv.DbConnection
        The connection object that executes the queries to the Level 2 db.
    project : str or None
        The name of the DESC project.
    tract : str
        The "tract", as assigned by the LSST Stack.
    patch : str
        The "patch", as assigned by the LSST Stack, e.g., "0,0".

    Notes
    -----
    Database credentials and connection info (e.g., the port to use)
    are accessed via lsst.daf.persistence.DbAuth from the user's
    ~/.lsst/db-auth.paf file.

    """
    def __init__(self, repo=None, db_info=None, tract='0', patch='0,0',
                 project=None):
        """
        Class constructor.
        """
        self.repo = repo
        if db_info is None:
            db_info = dict(database='DESC_Twinkles_Level_2',
                           host='scidb1.nersc.gov')
        self.conn = DbConnection(**db_info)
        self.tract = str(tract)
        self.patch = patch
        self.project = project

    def get_pixel_data(self, objectId, band, size=10, pickle_file=None):
        """Get the pixel data for a cutout region for all visits.

        Use PostageStampMaker to extract the pixel data cutout for the
        specified objectId and band for each visit in the repository.

        Parameters
        ----------
        objectId : int
           The id of the object in the Level2 Object db table.
        band : str
           The LSST band (u, g, r, i, z, or y).
        size : int, optional
           The size of the cutout region in arcsec.
           Cutouts of size x size centered on the RA, Dec of the
           object will be extracted.
        pickle_file : str, optional
           The name of the pickle file to contain the extracted pixel
           data.  If it is None, then a default name of the form
           'pixel_data_%(objectId)i_%(band)s.pkl' will be used.  If
           that file exists, the pixel data will be loaded from it.
           If it does not exist, the pixel data will be extracted from
           the deepCoadd tempExp frames.

        Returns
        -------
        pixel_data : OrderedDict
            A dictionary of 2D numpy arrays containing the pixel values.
            The keys are the visitId numbers from the CcdVisit table.
        pickle_file : str
            The name of the pickle file.
        """
        if pickle_file is None:
            pickle_file = 'pixel_data_%(objectId)i_%(band)s.pkl' % locals()
        if os.path.isfile(pickle_file):
            with open(pickle_file, 'r') as input_:
                pixel_data = pickle.load(input_)
            return pixel_data, pickle_file
        pixel_data = OrderedDict()
        visits = self.get_visits(band)
        ra, dec = self.get_coords(objectId)
        for visit in visits:
            print("working on visit", visit)
            sys.stdout.flush()
            exp_file = self._get_warped_visit_frame(band, visit)
            maker = PostageStampMaker(exp_file)
            stamp = maker.create(ra, dec, size)
            pixel_data[visit] = \
                copy.deepcopy(stamp.getMaskedImage().getImage().getArray())
        with open(pickle_file, 'w') as output:
            pickle.dump(pixel_data, output)
        return pixel_data, pickle_file

    def get_deepCoadd_frame(self, band):
        """Get the deepCoadd frame for the specified band

        Parameters
        ----------
        band : str
            The LSST band (u, g, r, i, z, or y).

        Returns
        -------
        str
            File path to the FITS file containing the coadd the
            specified band.

        Raises
        ------
        RuntimeError
            If the repo attribute has not been set.

        Notes
        -----
        This should be re-implemented to use the Data Butler.
        """
        if self.repo is None:
            raise RuntimeError('Level 2 respository not set.')
        return os.path.join(self.repo, 'deepCoadd', band,
                            self.tract, self.patch + '.fits')

    def _get_warped_visit_frame(self, band, visit):
        """Get the deepCoadd tempExp frame for the specified band and visit.

        Parameters
        ----------
        band : str
            The LSST band (u, g, r, i, z, or y).
        visit : int
            The visitId of the visit from the CcdVisit table.

        Returns
        -------
        str
            File path to the FITS file containing the warped exposure for
            the specified band and visit.

        Raises
        ------
        RuntimeError
            If the repo attribute has not been set.

        Notes
        -----
        This should be re-implemented to use the Data Butler.
        """
        if self.repo is None:
            raise RuntimeError('Level 2 respository not set.')
        return os.path.join(self.repo, 'deepCoadd', band,
                            self.tract, self.patch + 'tempExp',
                            'v%(visit)i-f%(band)s.fits' % locals())

    def get_light_curve(self, objectId, band):
        """ Get the light curve for the requested objectId and band.

        Parameters
        ----------
        objectId : int
            The id of the object as specified in the Level 2 Object table.
        band : str
            The LSST band (u, g, r, i, z, or y).

        Returns
        -------
        dict
            A dictionary of numpy arrays containing the light curve data.
            The format is
            {'mjd': <Visit obs times in MJD>,
             'flux': <Fluxes in nmgy>,
             'fluxerr': <Flux errors in nmgy>}

        Notes
        -----
        This method queries the CcdVisit and ForcedSource tables for
        the CcdVisit.obsStart, ForcedSource.psFlux, and
        ForcedSource.psFlux_Sigma values.
        """
        query = """select cv.obsStart, fs.psFlux, fs.psFlux_Sigma from
               CcdVisit cv join ForcedSource fs on cv.ccdVisitId=fs.ccdVisitId
               and cv.project=fs.project
               where cv.filterName='%(band)s' and fs.objectId=%(objectId)i""" \
            % locals()
        if self.project is not None:
            query += " and fs.project='%s'" % self.project
        query += " order by cv.obsStart asc"
        rows = self.conn.apply(query, lambda curs: np.array([x for x in curs]))
        obsStart, flux, fluxerr = (np.array(col) for col in rows.transpose())
        mjd = astropy.time.Time(obsStart).mjd
        return dict(mjd=mjd, flux=flux, fluxerr=fluxerr)

    def get_visits(self, band):
        """Get the visitIds corresponding to the specified band.

        Parameters
        ----------
        band : str
            The LSST band (u, g, r, i, z, or y).

        Returns
        -------
        list
            A list of visitIds from the CcdVisit table.
        """
        query = """select visitId from CcdVisit where
                   filterName='%(band)s'""" % locals()
        if self.project is not None:
            query += " and project='%s'" % self.project
        query += " order by visitId"
        return np.unique(self.conn.apply(query,
                                         lambda curs: [x[0] for x in curs]))

    def get_objects(self, ra, dec, box_size):
        """Get coadd objects within a box of size box_size x box_size arcsec
        centered on ra, dec.

        Parameters
        ----------
        ra : float
            Right Ascension (J2000) in degrees
        dec : float
            Declination (J2000) in degrees
        box_size : float
            Size of search box in arcsec.

        Returns
        -------
        dict
            A dictionary of (RA, Dec) tuples keyed by objectId.
        """
        size = box_size/3600.  # convert from arcsec to degrees
        cos_dec = np.abs(np.cos(dec*np.pi/180.))
        ra_min, ra_max = ra - size/cos_dec, ra + size/cos_dec
        dec_min, dec_max = dec - size, dec + size
        query = '''select objectId, psRa, psDecl from Object obj where
                   %(ra_min)12.8f < psRa and psRa < %(ra_max)12.8f and
                   %(dec_min)12.8f < psDecl and psDecl < %(dec_max)12.8f''' \
            % locals()
        if self.project is not None:
            query += " and project='%s'" % self.project
        return self.conn.apply(query, lambda curs: dict([(x[0], x[1:3])
                                                         for x in curs]))

    def get_coords(self, objectId):
        """Get the RA, Dec of the requested object from the Object db table.

        Parameters
        ----------
        objectId : int
            The id of the object as specified in the Level 2 Object table.

        Returns
        -------
        tuple
            The objects RA, Dec in degrees.
        """
        return self.conn.apply('''select psRA, psDecl from Object where
                                  objectId=%(objectId)i''' % locals(),
                               lambda curs: tuple([x for x in curs][0]))
Example #5
0
import numpy as np
import matplotlib.pyplot as plt
import astropy.io.fits as fits
import astropy.wcs
from desc.pserv import DbConnection
from light_curve_service import LightCurveFactory
from desc.twinkles import PostageStampMaker, render_fits_image
plt.ion()

try:
    objectId = int(sys.argv[1])
except:
    objectId = 57500

db_info = dict(db='jc_desc', read_default_file='~/.my.cnf')
jc_desc = DbConnection(**db_info)
lc_factory = LightCurveFactory(**db_info)

lc = lc_factory.create(objectId)
lc.plot(figtext='objectId %i' % objectId)

query = 'select psRa, psDecl from Object where objectId=%(objectId)i' % locals()
ra, dec = jc_desc.apply(query, lambda curs : tuple([x for x in curs][0]))
print ra, dec

chisq_min = 0
dof = 0

size_arcsec = 10.

size = size_arcsec/3600. # convert to degrees
class Level2DataService(object):
    """
    Access to the Twinkles Level 2 data and pserv database.
    """
    def __init__(self, repo, db_info=None):
        """
        Keep track of the Level 2 repository and create a db connection.
        """
        self.repo = repo
        if db_info is None:
            db_info = dict(db='jc_desc', read_default_file='~/.my.cnf')
        self.conn = DbConnection(**db_info)

    def get_pixel_data(self, objectId, band, size=10, pickle_file=None):
        """
        Use PostageStampMaker to extract the pixel data cutout for the
        specified objectId andn band for each visit.
        """
        if pickle_file is None:
            pickle_file = 'pixel_data_%(objectId)i_%(band)s.pkl' % locals()
        if os.path.isfile(pickle_file):
            with open(pickle_file, 'r') as input_:
                pixel_data = pickle.load(input_)
            return pixel_data, pickle_file
        pixel_data = OrderedDict()
        visits = self.get_visits(band)
        ra, dec = self.get_coords(objectId)
        for visit in visits:
            print("working on visit", visit)
            sys.stdout.flush()
            exp_file = os.path.join(repo, 'deepCoadd', band, '0/0,0tempExp',
                                    'v%(visit)i-f%(band)s.fits' % locals())
            maker = PostageStampMaker(exp_file)
            stamp = maker.create(ra, dec, size)
            pixel_data[visit] = \
                copy.deepcopy(stamp.getMaskedImage().getImage().getArray())
        with open(pickle_file, 'w') as output:
            pickle.dump(pixel_data, output)
        return pixel_data, pickle_file

    def get_light_curve(self, objectId, band):
        """
        Get the light curve for the requested objectId and band.
        """
        query = """select cv.obsStart, fs.psFlux, fs.psFlux_Sigma from
               CcdVisit cv join ForcedSource fs on cv.ccdVisitId=fs.ccdVisitId
               join Object obj on fs.objectId=obj.objectId where
               cv.filterName='%(band)s' and fs.objectId=%(objectId)i
               order by cv.obsStart asc""" % locals()
        rows = self.conn.apply(query, lambda curs: np.array([x for x in curs]))
        obsStart, flux, fluxerr = (np.array(col) for col in rows.transpose())
        mjd = astropy.time.Time(obsStart).mjd
        return dict(mjd=mjd, flux=flux, fluxerr=fluxerr)

    def get_visits(self, band):
        """
        Get the visitIds corresponding to the specified band.
        """
        return self.conn.apply('''select visitId from CcdVisit where
                                  filterName='%(band)s' order by visitId'''
                               % locals(),
                               lambda curs: [x[0] for x in curs])

    def get_coords(self, objectId):
        """
        Get the RA, Dec of the requested object from the Object table.
        """
        return self.conn.apply('''select psRA, psDecl from Object where
                                  objectId=%(objectId)i''' % locals(),
                               lambda curs: tuple([x for x in curs][0]))