def get_data(self, date=None, timedelta='30m'):
self.set_date(date)
tmax = self.date
tmin = self.date - pd.Timedelta(timedelta)
if self.df is None:
# Don't want to create the DB each time?
try:
db = Database()
db.connect()
query ="""
select date, qc_fwhm as fwhm, airmass, filter from exposure
where date > '%s' and date < '%s'
--and filter != 'VR' and qc_fwhm is not NULL
and qc_fwhm is not NULL and qc_fwhm > 0
"""%(tmin, tmax)
logging.debug(query)
raw = db.query2rec(query)
except Exception as e:
logging.warn("Couldn't connect to database:\n%s"%str(e))
dtype=[('date', '<M8[ns]'), ('fwhm', '<f8'),
('airmass', '<f8'), ('filter', 'S4')]
raw = np.recarray(0,dtype=dtype)
else:
sel = (self.df.index > tmin) & (self.df.index < tmax)
raw = self.df[sel].to_records()
return raw
def survey_coverage(nside=1024,
bands=['g', 'r', 'i', 'z'],
propid=None,
exptime=30,
teff=0.01):
from obztak.utils.database import Database
import healpy as hp
bands = np.atleast_1d(bands)
db = Database()
db.connect()
params = dict(propid='')
if propid:
params['propid'] = "(propid = '%s')" % propid
else:
params['propid'] = "(propid not like '%-9999')"
params['exptime'] = "(exptime > %s)" % exptime
params['teff'] = "not (qc_teff < %s)" % teff
params['filter'] = "filter in (%s)" % (','.join(
["'%s'" % b for b in bands]))
query = """select id as expnum, telra as ra, teldec as dec, filter, propid,
exptime from exposure where
discard = False and delivered = True and flavor = 'object'
and %(propid)s and %(exptime)s and %(teff)s and %(filter)s;
""" % params
logging.debug(query)
data = db.query2rec(query)
out = dict()
for b in bands:
hpxmap = np.zeros(hp.nside2npix(nside))
d = data[data['filter'] == b]
theta = np.radians(90. - d['dec'])
phi = np.radians(d['ra'])
vec = hp.ang2vec(theta, phi)
for i, v in enumerate(vec):
if i % 10000 == 0:
logging.info('%i/%i' % (i, len(vec)))
pix = hp.query_disc(nside, v, radius=np.radians(constants.DECAM))
hpxmap[pix] += d[i]['exptime']
hpxmap[np.where(hpxmap == 0)] = hp.UNSEEN
out[b] = hpxmap
return out
def get_data(self, date=None, timedelta='30m'):
self.set_date(date)
tmax = self.date
tmin = self.date - pd.Timedelta(timedelta)
if self.df is None:
# Don't want to create the DB each time?
db = Database(self.db)
db.connect()
query ="""
select date, dimm2see as fwhm from exposure
where date > '%s' and date < '%s'
and dimm2see is not NULL
"""%(tmin, tmax)
logging.debug(query)
raw = db.query2rec(query)
else:
sel = (self.df.index > tmin) & (self.df.index < tmax)
raw = self.df[sel].to_records()
return raw
def post_qcinv(token=None,
channel=None,
propid=None,
timedelta=None,
debug=False):
"""Post inventory results to Slack.
Parameters:
-----------
token : slack bot token
channel: slack channel
propid : proposal id
timedelta: time to collect exposure info
debug : execute but don't post
Returns:
--------
df,pkg : data frame and text package
"""
db = Database()
db.connect()
df = db.qcInv(propid=propid, timedelta=timedelta)
if not len(df):
logging.debug("No exposures found")
return df, None
kwargs = dict(index=False, float_format='{:.2f}'.format, justify='right')
package = """Observing update @ {time} CST:
```
{content}
```""".format(content=df.fillna('').to_string(**kwargs),
time=datetime.now().strftime("%H:%M"))
logging.debug(package)
if debug:
logging.debug("Exiting without posting.")
return df, package
ret = post_message(package, token=token, channel=channel)
return df, package
def plot_bliss_nightsum(fields,nitestr):
plot_bliss_coverage(fields)
plt.savefig('nightsum_coverage_%s.png'%nitestr)
new = (np.array(map(utc2nite,fields['DATE'])) == nitestr)
new_fields = fields[new]
old_fields = fields[~new]
db = Database()
db.connect()
query = """select id, qc_fwhm as psf, qc_teff as teff from exposure
where exptime = 90 and delivered = True and propid = '%s'
and qc_teff is not NULL and qc_fwhm is not NULL
and to_timestamp(utc_beg) %s '%s'
"""
new = db.query2recarray(query%(fields.PROPID,'>',datestr(date)))
try:
old = db.query2recarray(query%(fields.PROPID,'<',date))
except ValueError as e:
print(e)
old = np.recarray(0,dtype=new.dtype)
nbins = 35
kwargs = dict(normed=True)
step_kwargs = dict(kwargs,histtype='step',lw=3.5)
fill_kwargs = dict(kwargs,histtype='stepfilled',lw=1.0,alpha=0.7)
plt.figure()
step_kwargs['bins'] = np.linspace(0.5,2.5,nbins)
fill_kwargs['bins'] = np.linspace(0.5,2.5,nbins)
plt.hist(new['psf'],color='green',zorder=10, label='Observed tonight', **fill_kwargs)
plt.hist(new['psf'],color='green',zorder=10, **step_kwargs)
plt.hist(old['psf'],color='0.5', label='Observed previously', **fill_kwargs)
plt.hist(old['psf'],color='0.5', **step_kwargs)
plt.axvline(1.20,ls='--',lw=2,color='gray')
plt.legend()
plt.title('Seeing (%s)'%nitestr)
plt.xlabel('FWHM (arcsec)')
plt.ylabel('Normalized Number of Exposures')
plt.savefig('nightsum_psf_%s.png'%nitestr,bbox_inches='tight')
plt.figure()
step_kwargs['bins'] = np.linspace(0,1.5,nbins)
fill_kwargs['bins'] = np.linspace(0,1.5,nbins)
plt.hist(new['teff'],color='green',zorder=10,label='Observed tonight', **fill_kwargs)
plt.hist(new['teff'],color='green',zorder=10, **step_kwargs)
plt.hist(old['teff'],color='0.5',label='Observed previously', **fill_kwargs)
plt.hist(old['teff'],color='0.5', **step_kwargs)
plt.axvline(0.25,ls='--',lw=2,color='gray')
plt.legend()
plt.title('Effective Depth (%s)'%nitestr)
plt.xlabel('Teff')
plt.ylabel('Normalized Number of Exposures')
plt.savefig('nightsum_teff_%s.png'%nitestr,bbox_inches='tight')
def load_database(cls, database=None):
"""
Load fields from the telemetry database.
Parameters:
-----------
database : String or Database object to connect to.
Returns:
--------
fields : A FieldArray filled from the database
"""
try:
from obztak.utils.database import Database
except ImportError as e:
logging.warn(e)
return cls()
try:
database = Database(database)
except IOError as e:
logging.warn(e)
return cls()
database.connect()
query = cls.query()
logging.debug(query)
data = database.execute(query)
names = map(str.upper, database.get_columns())
objidx = names.index('OBJECT')
if not len(data):
logging.warn("No fields found in database.")
return cls()
fields = cls()
for d in data:
f = cls(1)
for i, key in enumerate(names):
if key in f.dtype.names:
f[key] = d[i]
f.from_object(d[objidx])
fields = fields + f
return fields
#!/usr/bin/env python
"""
Generic python script.
"""
import sys
from collections import OrderedDict as odict
import healpy as hp
import numpy as np
import pylab as plt
from obztak.utils.database import Database
from obztak.utils.ortho import DECamBasemap, DECamMcBride
db = Database()
db.connect()
query ="""
SELECT id as expnum, telra as ra, teldec as dec, expTime, filter,
COALESCE(qc_teff,'NaN') as teff
FROM exposure where exptime >= 30 and discard = False and delivered = True
and flavor = 'object' and telra between 0 and 360 and teldec between -90 and 90
and filter in ('u','g','r','i','z','Y') and propid NOT LIKE '%-9999'
ORDER BY id;
"""
data = db.query2recarray(query)
exposures = odict([
('u',data[data['filter'] =='u']),
('g',data[data['filter'] =='g']),
import pandas as pd
import ephem
from obztak.utils import fileio
from obztak.utils.fileio import get_datafile, to_csv, read_csv
from obztak.utils.database import Database
import obztak.delve
import datetime
import skymap
import skymap.survey
from status import *
filename = EXPOSURES
if not os.path.exists(filename):
db = Database('db-fnal')
db.connect()
data = db.query2recarray(QUERY)
print("Writing %s..." % filename)
fileio.rec2csv(filename, data)
else:
print("Reading %s..." % filename)
data = fileio.csv2rec(filename)
good = data[select_good_exposures(data)]
guid = np.char.rpartition(good['object'].astype(str), ' ')[:, -1]
# Unique fields
guid, idx = np.unique(guid, return_index=True)
good = good[idx]
fields = obztak.delve.DelveFieldArray().load(get_datafile(TARGETS))
def plot_nightsum(fields,nitestr):
""" Plot the night summary for MagLiteS.
Parameters:
-----------
fields: the fields observed tonight
nitestr: the nite in strig format
Returns:
--------
None
"""
import pylab as plt
from obztak.utils.database import Database
from obztak.utils.ortho import makePlot
#fields = FieldArray.load_database()
#new = np.char.startswith(fields['DATE'],date)
date = nite2utc(nitestr)
new = (np.array(map(utc2nite,fields['DATE'])) == nitestr)
new_fields = fields[new]
old_fields = fields[~new]
kwargs = dict(edgecolor='none', s=50, vmin=0, vmax=4)
fig,basemap = makePlot(date=nitestr,name='nightsum',moon=False,airmass=False,center=(0,-90),bliss=False)
plt.title('Coverage (%s)'%nitestr)
kwargs['cmap'] = 'gray_r'
proj = basemap.proj(old_fields['RA'], old_fields['DEC'])
basemap.scatter(*proj, c=old_fields['TILING'],**kwargs)
kwargs['cmap'] = 'summer_r'
proj = basemap.proj(new_fields['RA'], new_fields['DEC'])
basemap.scatter(*proj, c=new_fields['TILING'], **kwargs)
colorbar = plt.colorbar()
colorbar.set_label('Tiling')
plt.plot(np.nan, np.nan,'o',color='green',mec='green',label='Observed tonight')
plt.plot(np.nan, np.nan,'o',color='0.7',mec='0.7',label='Observed previously')
plt.legend(fontsize=10,loc='lower left',scatterpoints=1)
plt.savefig('nightsum_coverage_%s.png'%nitestr,bbox_inches='tight')
db = Database()
db.connect()
query = """
select id, qc_fwhm as psf, qc_teff as teff from exposure
where exptime = 90 and delivered = True
and propid = '%s'
and qc_teff is not NULL and qc_fwhm is not NULL
and to_timestamp(utc_beg) %s '%s'
"""
new = db.query2recarray(query%(PROPID,'>',date))
old = db.query2recarray(query%(PROPID,'<',date))
nbins = 35
kwargs = dict(normed=True)
step_kwargs = dict(kwargs,histtype='step',lw=3.5)
fill_kwargs = dict(kwargs,histtype='stepfilled',lw=1.0,alpha=0.7)
plt.figure()
step_kwargs['bins'] = np.linspace(0.5,2.5,nbins)
fill_kwargs['bins'] = np.linspace(0.5,2.5,nbins)
plt.hist(new['psf'],color='green',zorder=10, label='Observed tonight', **fill_kwargs)
plt.hist(new['psf'],color='green',zorder=10, **step_kwargs)
plt.hist(old['psf'],color='0.5', label='Observed previously', **fill_kwargs)
plt.hist(old['psf'],color='0.5', **step_kwargs)
plt.axvline(1.20,ls='--',lw=2,color='gray')
plt.legend()
plt.title('Seeing (%s)'%nitestr)
plt.xlabel('FWHM (arcsec)')
plt.ylabel('Normalized Number of Exposures')
plt.savefig('nightsum_psf_%s.png'%nitestr,bbox_inches='tight')
plt.figure()
step_kwargs['bins'] = np.linspace(0,1.5,nbins)
fill_kwargs['bins'] = np.linspace(0,1.5,nbins)
plt.hist(new['teff'],color='green',zorder=10,label='Observed tonight', **fill_kwargs)
plt.hist(new['teff'],color='green',zorder=10, **step_kwargs)
plt.hist(old['teff'],color='0.5',label='Observed previously', **fill_kwargs)
plt.hist(old['teff'],color='0.5', **step_kwargs)
plt.axvline(0.25,ls='--',lw=2,color='gray')
plt.legend()
plt.title('Effective Depth (%s)'%nitestr)
plt.xlabel('Teff')
plt.ylabel('Normalized Number of Exposures')
plt.savefig('nightsum_teff_%s.png'%nitestr,bbox_inches='tight')
def plot_nightsum(fields,nitestr,date):
""" Plot the bliss night summary.
Parameters:
-----------
fields: the fields observed tonight
nitestr: the nite in strig format
Returns:
--------
None
"""
from obztak.utils.database import Database
plt.ioff()
# Select the fields from the database
db = Database()
db.connect()
query = """
select id, qc_fwhm as psf, qc_teff as teff, filter from exposure
where delivered = True and propid = '%s'
and flavor = 'object'
and qc_teff is not NULL
and qc_fwhm is not NULL
and to_timestamp(utc_beg) %s
"""
#new = db.query2recarray(query%(fields.PROPID,'>',datestr(date)))
d = datestr(date)
q = query%(fields.PROPID,"between (timestamp '%s') AND (timestamp '%s' + interval '12 hours')"%(d,d))
logging.debug(q)
new = db.query2recarray(q)
try:
q = query%(fields.PROPID,"< (timestamp '%s')"%d)
logging.debug(q)
old = db.query2recarray(q)
except ValueError as e:
print(e)
old = np.recarray(0,dtype=new.dtype)
for b in ['u','g','r','i','z','Y']:
f = new[new['filter'] == b]
print ' %s-band:'%b, len(f)
if not len(new):
logging.warn("No new exposures...")
return
##########################
plot_coverage(fields,nitestr)
##########################
fig,axes = plt.subplots(1,2,figsize=(12,5))
for i,d in enumerate(['2017/02/08 07:00:00','2017/02/08 19:00:00']):
plt.sca(axes[i])
bmap = DECamOrtho(date=d)
for b in np.unique(fields['FILTER']):
f = fields[fields['FILTER']==b]
bmap.draw_focal_planes(f['RA'],f['DEC'],color=COLORS[b],alpha=0.3)
bmap.draw_bliss()
bmap.draw_galaxy()
bmap.draw_des()
plt.suptitle('Coverage (%s)'%nitestr,fontsize=16)
plt.savefig('nightsum_summary_%s.png'%nitestr)
new_sel = (np.array(map(utc2nite,fields['DATE'])) == nitestr)
new_fields = fields[new_sel]
old_fields = fields[~new_sel]
##########################
fig,axes = plt.subplots(1,2,figsize=(12,5))
plt.sca(axes[0])
plt.plot(np.nan,np.nan,'-w',label='all')
plot_psf(new,old)
plt.title('Seeing (%s)'%nitestr)
plt.sca(axes[1])
plt.plot(np.nan,np.nan,'-w',label='all')
plot_teff(new,old)
plt.title('Effective Depth (%s)'%nitestr)
plt.savefig('nightsum_psf_teff_%s.png'%nitestr,bbox_inches='tight')
fig,axes = plt.subplots(2,2,figsize=(14,10))
axes = axes.flatten()
for i,b in enumerate(['g','r','i','z']):
plt.sca(axes[i])
plt.plot(np.nan,np.nan,'-w',label='%s-band'%b)
plot_psf(new[new['filter'] == b],old[old['filter'] == b])
plt.savefig('nightsum_psf_%s.png'%nitestr,bbox_inches='tight')
fig,axes = plt.subplots(2,2,figsize=(14,10))
axes = axes.flatten()
for i,b in enumerate(['g','r','i','z']):
plt.sca(axes[i])
plt.plot(np.nan,np.nan,'-w',label='%s-band'%b)
plot_teff(new[new['filter'] == b],old[old['filter'] == b])
plt.savefig('nightsum_teff_%s.png'%nitestr,bbox_inches='tight')