def timed_out(self, dt):
now = datenow()
dt = (now - self.lastNewFile).total_seconds()
print('No new images seen for', dt, 'seconds.')
markmjds = []
if dt > self.longtime:
edate = (self.lastNewFile +
datetime.timedelta(0, self.longtime))
markmjds.append((datetomjd(edate),'r'))
self.plot_recent(markmjds=markmjds)
def queue_initial_exposures(self):
# Queue two exposures to start
e0 = self.queue_exposure()
e1 = self.queue_exposure()
# When should we queue the next exposure?
dt = e0['expTime'] + e1['expTime'] + 2 * self.nom.overhead
# margin
dt -= self.queueMargin
self.queuetime = (datenow() +
datetime.timedelta(0, dt))
self.queuetime = self.queuetime.replace(microsecond = 0)
def heartbeat(self):
if self.queuetime is None:
return
## Is it time to queue a new exposure?
now = datenow().replace(microsecond=0)
print('Heartbeat. Now is', now.isoformat(),
'queue time is', self.queuetime.isoformat(),
'dt %i' % (int((self.queuetime - now).total_seconds())))
if now < self.queuetime:
return
print('Time to queue an exposure!')
e = self.queue_exposure()
# schedule next one
dt = e['expTime'] + self.nom.overhead - self.queueMargin
self.queuetime = now + datetime.timedelta(0, dt)
self.queuetime = self.queuetime.replace(microsecond = 0)
self.write_plans()
def main(cmdlineargs=None, get_copilot=False):
global gSFD
import optparse
parser = optparse.OptionParser(usage='%prog')
# Mosaic or Decam?
from camera import (nominal_cal, ephem_observer, default_extension,
tile_path)
nom = nominal_cal
obs = ephem_observer()
plotfn_default = 'recent.png'
parser.add_option('--ext', default=default_extension,
help='Extension to read for computing observing conditions: default %default')
parser.add_option('--extnum', type=int, help='Integer extension to read')
parser.add_option('--rawdata', help='Directory to monitor for new images: default $MOS3_DATA if set, else "rawdata"', default=None)
parser.add_option('--n-fwhm', default=None, type=int, help='Number of stars on which to measure FWHM')
parser.add_option('--no-db', dest='db', default=True, action='store_false',
help='Do not append results to database')
parser.add_option('--no-focus', dest='focus', default=True,
action='store_false', help='Do not analyze focus frames')
parser.add_option('--fits', help='Write database to given FITS table')
parser.add_option('--plot', action='store_true',
help='Plot recent data and quit')
parser.add_option('--plot-filename', default=None,
help='Save plot to given file, default %s' % plotfn_default)
parser.add_option('--nightplot', '--night', action='store_true',
help="Plot tonight's data and quit")
parser.add_option('--qa-plots', dest='doplots', default=False,
action='store_true', help='Create QA plots')
parser.add_option('--keep-plots', action='store_true',
help='Do not remove PNG-format plots (normally merged into PDF)')
parser.add_option('--mjdstart', type=float, default=None,
help='MJD (UTC) at which to start plot')
now = mjdnow()
parser.add_option('--mjdend', type=float, default=None,
help='MJD (UTC) at which to end plot (default: now, which is %.3f)' % now)
parser.add_option('--skip', action='store_true',
help='Skip images that already exist in the database')
parser.add_option('--threads', type=int, default=None,
help='Run multi-threaded when processing list of files on command-line')
parser.add_option('--fix-db', action='store_true')
parser.add_option('--tiles', default=tile_path,
help='Tiles table, default %default')
parser.add_option('--no-show', dest='show', default=True, action='store_false',
help='Do not show plot window, just save it.')
if cmdlineargs is None:
opt,args = parser.parse_args()
else:
opt,args = parser.parse_args(cmdlineargs)
if not opt.show:
import matplotlib
matplotlib.use('Agg')
imagedir = opt.rawdata
if imagedir is None:
imagedir = os.environ.get('MOS3_DATA', 'rawdata')
rawext = opt.ext
if opt.extnum is not None:
rawext = opt.extnum
assert(rawext is not None)
from astrometry.util.fits import fits_table
tiles = fits_table(opt.tiles)
from django.conf import settings
import obsdb
import pylab as plt
plt.figure(figsize=(8,10))
markmjds = []
if opt.nightplot:
opt.plot = True
if opt.plot_filename is None:
opt.plot_filename = 'night.png'
# Are we at Tololo or Kitt Peak? Look for latest image.
o = obsdb.MeasuredCCD.objects.all().order_by('-mjd_obs')
cam = o[0].camera
print('Camera:', cam)
if opt.mjdstart is not None:
sdate = ephem.Date(mjdtodate(opt.mjdend))
else:
sdate = ephem.Date(datenow())
(sunset, eve12, eve18, morn18, morn12, sunrise) = get_twilight(
cam, sdate)
if opt.mjdstart is None:
opt.mjdstart = ephemdate_to_mjd(sunset)
print('Set mjd start to sunset:', sunset, opt.mjdstart)
if opt.mjdend is None:
opt.mjdend = ephemdate_to_mjd(sunrise)
print('Set mjd end to sunrise', sunrise, opt.mjdend)
markmjds.append((ephemdate_to_mjd(eve18),'b'))
print('Evening twi18:', eve18, markmjds[-1])
markmjds.append((ephemdate_to_mjd(morn18),'b'))
print('Morning twi18:', morn18, markmjds[-1])
markmjds.append((ephemdate_to_mjd(eve12),'g'))
print('Evening twi12:', eve12, markmjds[-1])
markmjds.append((ephemdate_to_mjd(morn12),'g'))
print('Morning twi12:', morn12, markmjds[-1])
if opt.plot_filename is None:
opt.plot_filename = plotfn_default
if opt.fits:
ccds = obsdb.MeasuredCCD.objects.all()
print(ccds.count(), 'measured CCDs')
T = db_to_fits(ccds)
T.writeto(opt.fits)
print('Wrote', opt.fits)
return 0
if opt.fix_db:
from astrometry.util.fits import fits_table
tiles = fits_table('obstatus/mosaic-tiles_obstatus.fits')
now = mjdnow()
#ccds = obsdb.MeasuredCCD.objects.all()
#ccds = obsdb.MeasuredCCD.objects.all().filter(mjd_obs__gt=now - 0.25)
ccds = obsdb.MeasuredCCD.objects.all().filter(mjd_obs__gt=57434)
print(ccds.count(), 'measured CCDs')
for ccd in ccds:
try:
hdr = fitsio.read_header(ccd.filename, ext=0)
# band = hdr['FILTER']
# band = band.split()[0]
# ccd.band = band
set_tile_fields(ccd, hdr, tiles)
ccd.save()
print('Fixed', ccd.filename)
except:
import traceback
traceback.print_exc()
return 0
if opt.plot:
plot_recent(opt, nom, tiles=tiles, markmjds=markmjds, show_plot=False)
return 0
print('Loading SFD maps...')
sfd = SFDMap()
if len(args) > 0:
mp = None
if opt.threads > 1:
gSFD = sfd
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
if opt.skip:
fns = skip_existing_files(args, rawext)
else:
fns = args
if mp is None:
for fn in fns:
process_image(fn, rawext, nom, sfd, opt, obs, tiles)
else:
sfd = None
mp.map(bounce_process_image,
[(fn, rawext, nom, sfd, opt, obs, tiles) for fn in fns])
plot_recent(opt, nom, tiles=tiles, markmjds=markmjds, show_plot=False)
return 0
copilot = Copilot(imagedir, rawext, opt, nom, sfd, obs, tiles)
# for testability
if get_copilot:
return copilot
copilot.run()
return 0
def process_image(fn, ext, nom, sfd, opt, obs, tiles):
db = opt.db
print('Reading', fn)
if sfd is None:
sfd = gSFD
# Read primary FITS header
phdr = fitsio.read_header(fn)
obstype = phdr.get('OBSTYPE','').strip()
print('obstype:', obstype)
exptime = phdr.get('EXPTIME', 0)
expnum = phdr.get('EXPNUM', 0)
filt = phdr.get('FILTER', None)
if filt is not None:
filt = filt.strip()
filt = filt.split()[0]
if filt is None:
filt = ''
airmass = phdr.get('AIRMASS', 0.)
ra = hmsstring2ra (phdr.get('RA', '0'))
dec = dmsstring2dec(phdr.get('DEC', '0'))
# Write QA plots to files named by the exposure number
print('Exposure number:', expnum)
skip = False
if obstype in ['zero', 'focus', 'dome flat', '']:
print('Skipping obstype =', obstype)
skip = True
if exptime == 0:
print('Exposure time EXPTIME in header =', exptime)
skip = True
if expnum == '':
print('No expnum in header')
skip = True
if filt == 'solid':
print('Solid (block) filter.')
skip = True
if skip and not db:
return None
if db:
import obsdb
if ext is None:
ext = get_default_extension(fn)
m,created = obsdb.MeasuredCCD.objects.get_or_create(
filename=fn, extension=ext)
m.obstype = obstype
m.camera = camera_name(phdr)
m.expnum = expnum
m.exptime = exptime
m.mjd_obs = phdr.get('MJD-OBS', 0.)
m.airmass = airmass
m.rabore = ra
m.decbore = dec
m.band = filt
m.bad_pixcnt = ('PIXCNT1' in phdr)
m.readtime = phdr.get('READTIME', 0.)
if opt.focus and obstype == 'focus' and m.camera == 'mosaic3':
from mosaic_focus import Mosaic3FocusMeas
show_plot = opt.show
if show_plot:
import pylab as plt
plt.figure(2, figsize=(8,10))
if ext is None:
ext = get_default_extension(fn)
meas = Mosaic3FocusMeas(fn, ext, nom)
focusfn = 'focus.png'
meas.run(ps=None, plotfn=focusfn)
print('Wrote', focusfn)
if show_plot:
plt.draw()
plt.show(block=False)
plt.pause(0.001)
plt.figure(1)
if skip:
m.save()
return None
if opt.doplots:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('qa-%i' % expnum)
ps.printfn = False
else:
ps = None
# Measure the new image
kwa = {}
if ext is not None:
kwa.update(ext=ext)
if opt.n_fwhm is not None:
kwa.update(n_fwhm=opt.n_fwhm)
M = measure_raw(fn, ps=ps, **kwa)
if opt.doplots:
from glob import glob
# Gather all the QAplots into a single pdf and clean them up.
qafile = 'qa-%i.pdf' % expnum
pnglist = sorted(glob('qa-%i-??.png' % expnum))
cmd = 'convert {} {}'.format(' '.join(pnglist), qafile)
print('Writing out {}'.format(qafile))
#print(cmd)
os.system(cmd)
if not opt.keep_plots:
[os.remove(png) for png in pnglist]
# (example results for testig)
#M = {'seeing': 1.4890481099577366, 'airmass': 1.34,
#'skybright': 18.383479116033314, 'transparency': 0.94488537276869045,
#'band': 'z', 'zp': 26.442847814941093}
#print('Measurements:', M)
trans = M.get('transparency', 0)
band = M['band']
# Look up E(B-V) in SFD map
ebv = sfd.ebv(ra, dec)[0]
print('E(B-V): %.3f' % ebv)
if trans > 0:
fid = nom.fiducial_exptime(band)
expfactor = exposure_factor(fid, nom,
airmass, ebv, M['seeing'], M['skybright'],
trans)
print('Exposure factor: %6.3f' % expfactor)
t_exptime = expfactor * fid.exptime
print('Target exposure time: %6.1f' % t_exptime)
t_exptime = np.clip(t_exptime, fid.exptime_min, fid.exptime_max)
print('Clipped exposure time: %6.1f' % t_exptime)
if band == 'z':
t_sat = nom.saturation_time(band, M['skybright'])
if t_exptime > t_sat:
t_exptime = t_sat
print('Reduced exposure time to avoid z-band saturation: %.1f' % t_exptime)
print
print('Actual exposure time taken: %6.1f' % exptime)
print('Depth (exposure time) factor: %6.3f' % (exptime / t_exptime))
# If you were going to re-plan, you would run with these args:
plandict = dict(seeing=M['seeing'], transparency=trans)
# Assume the sky is as much brighter than canonical in each band... unlikely
dsky = M['skybright'] - nom.sky(M['band'])
for b in 'grz':
plandict['sb'+b] = nom.sky(b) + dsky
# Note that nightlystrategy.py takes UTC dates.
start = datenow()
# Start the strategy 5 minutes from now.
start += datetime.timedelta(0, 5*60)
d = start.date()
plandict['startdate'] = '%04i-%02i-%02i' % (d.year, d.month, d.day)
t = start.time()
plandict['starttime'] = t.strftime('%H:%M:%S')
# Make an hour-long plan
end = start + datetime.timedelta(0, 3600)
d = end.date()
plandict['enddate'] = '%04i-%02i-%02i' % (d.year, d.month, d.day)
t = end.time()
plandict['endtime'] = t.strftime('%H:%M:%S')
# Set "--date" to be the UTC date at previous sunset.
# (nightlystrategy will ask for the next setting of the sun below
# -18-degrees from that date to define the sn_18). We could
# probably also get away with subtracting, like, 12 hours from
# now()...
sun = ephem.Sun()
obs.date = datenow()
# not the proper horizon, but this doesn't matter -- just need it to
# be before -18-degree twilight.
obs.horizon = 0.
sunset = obs.previous_setting(sun)
# pyephem's Date.tuple() splits a date into y,m,d,h,m,s
d = sunset.tuple()
#print('Date at sunset, UTC:', d)
year,month,day = d[:3]
plandict['date'] = '%04i-%02i-%02i' % (year, month, day)
# Decide the pass.
goodseeing = plandict['seeing'] < 1.3
photometric = plandict['transparency'] > 0.9
if goodseeing and photometric:
passnum = 1
elif goodseeing or photometric:
passnum = 2
else:
passnum = 3
plandict['pass'] = passnum
## ??
plandict['portion'] = 1.0
print('Replan command:')
print()
print('python2.7 nightlystrategy.py --seeg %(seeing).3f --seer %(seeing).3f --seez %(seeing).3f --sbg %(sbg).3f --sbr %(sbr).3f --sbz %(sbz).3f --transparency %(transparency).3f --start-date %(startdate)s --start-time %(starttime)s --end-date %(enddate)s --end-time %(endtime)s --date %(date)s --portion %(portion)f --pass %(pass)i' % plandict)
print()
else:
plandict = None
expfactor = 0.
rtn = (M, plandict, expnum)
if not db:
return rtn
m.racenter = M['ra_ccd']
m.deccenter = M['dec_ccd']
m.ebv = ebv
zp = M.get('zp', 0.)
if zp is None:
zp = 0.
m.zeropoint = zp
m.transparency = trans
m.seeing = M.get('seeing', 0.)
m.sky = M['skybright']
m.expfactor = expfactor
m.dx = M.get('dx', 0)
m.dy = M.get('dy', 0)
m.nmatched = M.get('nmatched',0)
img = fitsio.read(fn, ext=1)
cheaphash = np.sum(img)
# cheaphash becomes an int64.
m.md5sum = cheaphash
set_tile_fields(m, phdr, tiles)
m.save()
return rtn