def run(self, indir):
'''TODO: document'''
log = desiutil.log.get_logger()
results = list()
infiles = glob.glob(os.path.join(indir, 'qframe-*.fits'))
if len(infiles) == 0 :
log.error("no qframe in {}".format(indir))
return None
for filename in infiles:
qframe = read_qframe(filename)
night = int(qframe.meta['NIGHT'])
expid = int(qframe.meta['EXPID'])
cam = qframe.meta['CAMERA'][0].upper()
spectro = int(qframe.meta['CAMERA'][1])
log.debug("computing scores for {} {} {} {}".format(night,expid,cam,spectro))
scores,comments = compute_frame_scores(qframe,suffix="RAW",flux_per_angstrom=True)
has_calib_frame = False
cfilename=filename.replace("qframe","qcframe")
if os.path.isfile(cfilename) : # add scores of calibrated sky-subtracted frame
qcframe = read_qframe(cfilename)
cscores,comments = compute_frame_scores(qcframe,suffix="CALIB",flux_per_angstrom=True)
for k in cscores.keys() :
scores[k] = cscores[k]
has_calib_frame = True
nfibers=scores['INTEG_RAW_FLUX_'+cam].size
for f in range(nfibers) :
fiber = int(qframe.fibermap["FIBER"][f])
if has_calib_frame :
results.append(collections.OrderedDict(
NIGHT=night, EXPID=expid, SPECTRO=spectro, CAM=cam, FIBER=fiber,
INTEG_RAW_FLUX=scores['INTEG_RAW_FLUX_'+cam][f],
MEDIAN_RAW_FLUX=scores['MEDIAN_RAW_FLUX_'+cam][f],
MEDIAN_RAW_SNR=scores['MEDIAN_RAW_SNR_'+cam][f],
INTEG_CALIB_FLUX=scores['INTEG_CALIB_FLUX_'+cam][f],
MEDIAN_CALIB_FLUX=scores['MEDIAN_CALIB_FLUX_'+cam][f],
MEDIAN_CALIB_SNR=scores['MEDIAN_CALIB_SNR_'+cam][f]))
else :
results.append(collections.OrderedDict(
NIGHT=night, EXPID=expid, SPECTRO=spectro, CAM=cam, FIBER=fiber,
INTEG_RAW_FLUX=scores['INTEG_RAW_FLUX_'+cam][f],
MEDIAN_RAW_FLUX=scores['MEDIAN_RAW_FLUX_'+cam][f],
MEDIAN_RAW_SNR=scores['MEDIAN_RAW_SNR_'+cam][f]))
if len(results)==0 :
return None
return Table(results, names=results[0].keys())
def run(self, indir):
'''TODO: document'''
log = desiutil.log.get_logger()
results = list()
infiles = glob.glob(os.path.join(indir, 'qframe-*.fits'))
if len(infiles) == 0:
log.error("no qframe in {}".format(indir))
return None
for filename in infiles:
qframe = read_qframe(filename)
night = int(qframe.meta['NIGHT'])
expid = int(qframe.meta['EXPID'])
cam = qframe.meta['CAMERA'][0].upper()
spectro = int(qframe.meta['CAMERA'][1])
try:
cfinder = CalibFinder([qframe.meta])
except:
log.error(
"failed to find calib for qframe {}".format(filename))
continue
if not cfinder.haskey("FIBERFLAT"):
log.warning(
"no known fiberflat for qframe {}".format(filename))
continue
fflat = read_fiberflat(cfinder.findfile("FIBERFLAT"))
tmp = np.median(fflat.fiberflat, axis=1)
reference_fflat = tmp / np.median(tmp)
tmp = np.median(qframe.flux, axis=1)
this_fflat = tmp / np.median(tmp)
for f, fiber in enumerate(qframe.fibermap["FIBER"]):
results.append(
collections.OrderedDict(NIGHT=night,
EXPID=expid,
SPECTRO=spectro,
CAM=cam,
FIBER=fiber,
FIBERFLAT=this_fflat[f],
REF_FIBERFLAT=reference_fflat[f]))
if len(results) == 0:
return None
return Table(results, names=results[0].keys())
def run(self, indir):
'''TODO: document'''
log = desiutil.log.get_logger()
infiles = glob.glob(os.path.join(indir, 'psf-*.fits'))
if len(infiles) == 0:
log.error('No {}/psf*.fits files found'.format(indir))
return None
results = list()
for filename in infiles:
log.debug(filename)
hdr = fitsio.read_header(filename)
night = hdr['NIGHT']
expid = hdr['EXPID']
cam = hdr['CAMERA'][0].upper()
spectro = int(hdr['CAMERA'][1])
dico={"NIGHT":night,"EXPID":expid,"SPECTRO":spectro,"CAM":cam}
for k in ["MEANDX","MINDX","MAXDX","MEANDY","MINDY","MAXDY"] :
dico[k]=hdr[k]
log.debug("{} {} {} {}".format(night,expid,cam,spectro))
results.append(collections.OrderedDict(**dico))
return Table(results, names=results[0].keys())
def write_tables(indir, outdir, expnights=None):
'''
Parses directory for available nights, exposures to generate
nights and exposures tables
Args:
indir : directory of nights
outdir : directory where to write nights table
Options:
expnights (list) : only update exposures tables for these nights
'''
import re
from astropy.table import Table
from nightwatch.webpages import tables as web_tables
from pkg_resources import resource_filename
from shutil import copyfile
from collections import Counter
log = desiutil.log.get_logger()
log.info(f'Tabulating exposures in {indir}')
#- Count night/expid directories to get num exp per night
expdirs = sorted(glob.glob(f"{indir}/20*/[0-9]*"))
nights = list()
re_expid = re.compile('^\d{8}$')
re_night = re.compile('^20\d{6}$')
for expdir in expdirs:
expid = os.path.basename(expdir)
night = os.path.basename(os.path.dirname(expdir))
if re_expid.match(expid) and re_night.match(night):
nights.append(night)
num_exp_per_night = Counter(nights)
#- Build the exposures table for the requested nights
rows = list()
for expdir in expdirs:
expid = os.path.basename(expdir)
night = os.path.basename(os.path.dirname(expdir))
if re_expid.match(expid) and re_night.match(night) and \
(expnights is None or int(night) in expnights):
night = int(night)
expid = int(expid)
qafile = os.path.join(expdir, 'qa-{:08d}.fits'.format(expid))
#- gets the list of failed qprocs for each expid
expfiles = os.listdir(expdir)
preproc_cams = [
i.split("-")[1] for i in expfiles
if re.match(r'preproc-.*-.*.fits', i)
]
log_cams = [
i.split("-")[1] for i in expfiles if re.match(r'.*\.log', i)
]
qfails = [i for i in log_cams if i not in preproc_cams]
if os.path.exists(qafile):
try:
with fitsio.FITS(qafile) as fits:
qproc_status = fits['QPROC_STATUS'].read()
exitcode = np.count_nonzero(qproc_status['QPROC_EXIT'])
except IOError:
exitcode = 0
rows.append(
dict(NIGHT=night,
EXPID=expid,
FAIL=0,
QPROC=qfails,
QPROC_EXIT=exitcode))
else:
log.error('Missing {}'.format(qafile))
rows.append(
dict(NIGHT=night,
EXPID=expid,
FAIL=1,
QPROC=None,
QPROC_EXIT=None))
if len(rows) == 0:
msg = "No exp dirs found in {}/NIGHT/EXPID".format(indir)
raise RuntimeError(msg)
exposures = Table(rows)
caldir = os.path.join(outdir, 'static')
if not os.path.isdir(caldir):
os.makedirs(caldir)
files = [
'bootstrap.js', 'bootstrap.css', 'bootstrap-year-calendar.css',
'bootstrap-year-calendar.js', 'jquery_min.js', 'popper_min.js',
'live.js'
]
for f in files:
outfile = os.path.join(outdir, 'static', f)
if not os.path.exists(outfile):
infile = resource_filename('nightwatch', os.path.join('static', f))
copyfile(infile, outfile)
nightsfile = os.path.join(outdir, 'nights.html')
web_tables.write_calendar(nightsfile, num_exp_per_night)
web_tables.write_exposures_tables(indir,
outdir,
exposures,
nights=expnights)
def run_qproc(rawfile, outdir, ncpu=None, cameras=None):
'''
Determine the obstype of the rawfile, and run qproc with appropriate options
Args:
rawfile: input desi-EXPID.fits.fz raw data file
outdir: directory to write qproc-CAM-EXPID.fits files
Options:
ncpu: number of CPU cores to use for parallelism; serial if ncpu<=1
cameras: list of cameras to process; default all found in rawfile
Returns header of HDU 0 of the input raw data file, plus dictionary of return codes for each qproc process run.
'''
log = desiutil.log.get_logger()
if not os.path.isdir(outdir):
log.info('Creating {}'.format(outdir))
os.makedirs(outdir, exist_ok=True)
hdr = fitsio.read_header(rawfile, 0)
if ('OBSTYPE' not in hdr) and ('FLAVOR' not in hdr):
log.warning(
"no obstype nor flavor keyword in first hdu header, moving to the next one"
)
try:
hdr = fitsio.read_header(rawfile, 1)
except OSError as err:
log.error("fitsio error reading HDU 1, trying 2 then giving up")
hdr = fitsio.read_header(rawfile, 2)
try:
if 'OBSTYPE' in hdr:
obstype = hdr['OBSTYPE'].rstrip().upper()
else:
log.warning('Use FLAVOR instead of missing OBSTYPE')
obstype = hdr['FLAVOR'].rstrip().upper()
night, expid = get_night_expid_header(hdr)
except KeyError as e:
log.error(str(e))
raise (e)
#- copy coordfile to new folder for pos accuracy
indir = os.path.abspath(os.path.dirname(rawfile))
coord_infile = '{}/coordinates-{:08d}.fits'.format(indir, expid)
coord_outfile = '{}/coordinates-{:08d}.fits'.format(outdir, expid)
print(coord_infile)
if os.path.isfile(coord_infile):
print('copying coordfile')
copyfile(coord_infile, coord_outfile)
else:
log.warning('No coordinate file for positioner accuracy')
#- HACK: Workaround for data on 20190626/27 that have blank NIGHT keywords
#- Note: get_night_expid_header(hdr) should take care of this now, but
#- this is left in for robustness just in case
if night == ' ' or night is None:
log.error(
'Correcting blank NIGHT keyword based upon directory structure')
#- /path/to/NIGHT/EXPID/rawfile.fits
night = os.path.basename(
os.path.dirname(os.path.dirname(os.path.abspath(rawfile))))
if re.match('20\d{6}', night):
log.info('Setting NIGHT to {}'.format(night))
else:
raise RuntimeError('Unable to derive NIGHT for {}'.format(rawfile))
cmdlist = list()
loglist = list()
msglist = list()
rawcameras = which_cameras(rawfile)
if cameras is None:
cameras = rawcameras
elif len(set(cameras) - set(rawcameras)) > 0:
missing_cameras = set(cameras) - set(rawcameras)
for cam in sorted(missing_cameras):
log.error('{} missing camera {}'.format(os.path.basename(rawfile),
cam))
cameras = sorted(set(cameras) & set(rawcameras))
for camera in cameras:
outfiles = dict(
rawfile=rawfile,
fibermap='{}/fibermap-{:08d}.fits'.format(outdir, expid),
logfile='{}/qproc-{}-{:08d}.log'.format(outdir, camera, expid),
outdir=outdir,
camera=camera)
cmd = "desi_qproc -i {rawfile} --fibermap {fibermap} --auto --auto-output-dir {outdir} --cam {camera}".format(
**outfiles)
cmdlist.append(cmd)
loglist.append(outfiles['logfile'])
msglist.append('qproc {}/{} {}'.format(night, expid, camera))
ncpu = min(len(cmdlist), get_ncpu(ncpu))
if ncpu > 1 and len(cameras) > 1:
log.info('Running qproc in parallel on {} cores for {} cameras'.format(
ncpu, len(cameras)))
pool = mp.Pool(ncpu)
errs = pool.starmap(runcmd, zip(cmdlist, loglist, msglist))
pool.close()
pool.join()
else:
errs = []
log.info('Running qproc serially for {} cameras'.format(len(cameras)))
for cmd, logfile, msg in zip(cmdlist, loglist, msglist):
err = runcmd(cmd, logfile, msg)
errs.append(err)
errorcodes = dict()
for err in errs:
for key in err.keys():
errorcodes[key] = err[key]
jsonfile = '{}/errorcodes-{:08d}.txt'.format(outdir, expid)
with open(jsonfile, 'w') as outfile:
json.dump(errorcodes, outfile)
print('Wrote {}'.format(jsonfile))
return hdr
def main(args):
"""Command-line driver for updating the survey plan.
"""
# Check for a valid fa-delay value.
if args.fa_delay[-1] not in ('d', 'm', 'q'):
raise ValueError('fa-delay must have the form Nd, Nm or Nq.')
fa_delay_type = args.fa_delay[-1]
try:
fa_delay = int(args.fa_delay[:-1])
except ValueError:
raise ValueError('invalid number in fa-delay.')
if fa_delay < 0:
raise ValueError('fa-delay value must be >= 0.')
# Set up the logger
if args.debug:
log = desiutil.log.get_logger(desiutil.log.DEBUG)
args.verbose = True
elif args.verbose:
log = desiutil.log.get_logger(desiutil.log.INFO)
else:
log = desiutil.log.get_logger(desiutil.log.WARNING)
# Freeze IERS table for consistent results.
desisurvey.utils.freeze_iers()
# Set the output path if requested.
config = desisurvey.config.Configuration(file_name=args.config_file)
if args.output_path is not None:
config.set_output_path(args.output_path)
# Load ephemerides.
ephem = desisurvey.ephem.get_ephem()
# Initialize scheduler.
if not os.path.exists(config.get_path('scheduler.fits')):
# Tabulate data used by the scheduler if necessary.
desisurvey.old.schedule.initialize(ephem)
scheduler = desisurvey.old.schedule.Scheduler()
# Read priority rules.
rules = desisurvey.rules.Rules(args.rules)
if args.create:
# Load initial design hour angles for each tile.
design = astropy.table.Table.read(config.get_path('surveyinit.fits'))
# Create an empty progress record.
progress = desisurvey.progress.Progress()
# Initialize the observing priorities.
priorities = rules.apply(progress)
# Create the initial plan.
plan = desisurvey.plan.create(design['HA'], priorities)
# Start the survey from scratch.
start = config.first_day()
else:
# Load an existing plan and progress record.
if not os.path.exists(config.get_path('plan.fits')):
log.error('No plan.fits found in output path.')
return -1
if not os.path.exists(config.get_path('progress.fits')):
log.error('No progress.fits found in output path.')
return -1
plan = astropy.table.Table.read(config.get_path('plan.fits'))
progress = desisurvey.progress.Progress('progress.fits')
# Start the new plan from the last observing date.
with open(config.get_path('last_date.txt'), 'r') as f:
start = desisurvey.utils.get_date(f.read().rstrip())
num_complete, num_total, pct = progress.completed(as_tuple=True)
# Already observed all tiles?
if num_complete == num_total:
log.info('All tiles observed!')
# Return a shell exit code so scripts can detect this condition.
sys.exit(9)
# Reached end of the survey?
if start >= config.last_day():
log.info('Reached survey end date!')
# Return a shell exit code so scripts can detect this condition.
sys.exit(9)
day_number = desisurvey.utils.day_number(start)
log.info(
'Planning night[{0}] {1} with {2:.1f} / {3} ({4:.1f}%) completed.'.
format(day_number, start, num_complete, num_total, pct))
bookmarked = False
if not args.create:
# Update the priorities for the progress so far.
new_priority = rules.apply(progress)
changed_priority = (new_priority != plan['priority'])
if np.any(changed_priority):
changed_passes = np.unique(plan['pass'][changed_priority])
log.info('Priorities changed in pass(es) {0}.'.format(', '.join(
[str(p) for p in changed_passes])))
plan['priority'] = new_priority
bookmarked = True
# Identify any new tiles that are available for fiber assignment.
plan = desisurvey.plan.update_available(plan, progress, start, ephem,
fa_delay, fa_delay_type)
# Will update design HA assignments here...
pass
# Update the progress table for the new plan.
ptable = progress._table
new_cover = (ptable['covered'] < 0) & (plan['covered'] <= day_number)
ptable['covered'][new_cover] = day_number
new_avail = (ptable['available'] < 0) & plan['available']
ptable['available'][new_avail] = day_number
new_plan = (ptable['planned'] < 0) & (plan['priority'] > 0)
ptable['planned'][new_plan] = day_number
# Save updated progress.
progress.save('progress.fits')
# Save the plan.
plan.write(config.get_path('plan.fits'), overwrite=True)
if bookmarked:
# Save a backup of the plan and progress at this point.
plan.write(config.get_path('plan_{0}.fits'.format(start)))
progress.save('progress_{0}.fits'.format(start))
def run(self, indir, outfile=None, jsonfile=None):
'''TODO: document'''
log = desiutil.log.get_logger()
log.debug('Running QA in {}'.format(indir))
print('here qa.runner', indir)
preprocfiles = sorted(glob.glob('{}/preproc-*.fits'.format(indir)))
if len(preprocfiles) == 0:
log.error('No preproc files found in {}'.format(indir))
return None
# We can have different obstypes (signal+dark) with calibration data
# obtained with a calibration slit hooked to a single spectrograph.
# So we have to loop over all frames to check if there are science,
# arc, or flat obstypes as guessed by qproc.
qframefiles = sorted(glob.glob('{}/qframe-*.fits'.format(indir)))
if len(qframefiles) == 0: # no qframe so it's either zero or dark
hdr = fitsio.read_header(preprocfiles[0], 0)
if 'OBSTYPE' in hdr:
obstype = hdr['FLAVOR'].strip()
else:
log.warning("Using FLAVOR instead of missing OBSTYPE")
obstype = hdr['FLAVOR'].strip()
else:
obstype = None
log.debug("Reading qframe headers to guess flavor ...")
for qframefile in qframefiles: # look at all of them and prefer arc or flat over dark or zero
hdr = fitsio.read_header(qframefile, 0)
if 'OBSTYPE' in hdr:
this_obstype = hdr['OBSTYPE'].strip().upper()
else:
log.warning("Using FLAVOR instead of missing OBSTYPE")
obstype = hdr['FLAVOR'].strip()
if this_obstype == "ARC" or this_obstype == "FLAT" \
or this_obstype == "TESTARC" or this_obstype == "TESTFLAT" :
obstype = this_obstype
# we use this so we exit the loop
break
elif obstype == None:
obstype = this_obstype
# we stay in the loop in case another frame has another obstype
log.debug('Found OBSTYPE={} files'.format(obstype))
results = dict()
for qa in self.qalist:
if qa.valid_obstype(obstype):
log.info('{} Running {} {}'.format(timestamp(), qa,
qa.output_type))
qa_results = None
try:
qa_results = qa.run(indir)
except Exception as err:
log.warning('{} failed on {} because {}; skipping'.format(
qa, indir, str(err)))
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
del exc_info
#raise(err)
#- TODO: print traceback somewhere useful
if qa_results is not None:
if qa.output_type not in results:
results[qa.output_type] = list()
results[qa.output_type].append(qa_results)
else:
log.debug('Skip {} {} for {}'.format(qa, qa.output_type,
obstype))
#- Combine results for different types of QA
join_keys = dict(
PER_AMP=['NIGHT', 'EXPID', 'SPECTRO', 'CAM', 'AMP'],
PER_CAMERA=['NIGHT', 'EXPID', 'SPECTRO', 'CAM'],
PER_FIBER=['NIGHT', 'EXPID', 'SPECTRO', 'FIBER'],
PER_CAMFIBER=['NIGHT', 'EXPID', 'SPECTRO', 'CAM', 'FIBER'],
PER_SPECTRO=['NIGHT', 'EXPID', 'SPECTRO'],
PER_EXP=['NIGHT', 'EXPID'],
QPROC_STATUS=['NIGHT', 'EXPID', 'SPECTRO', 'CAM'])
if jsonfile is not None:
if os.path.exists(jsonfile):
with open(jsonfile, 'r') as myfile:
json_data = myfile.read()
json_data = json.loads(json_data)
else:
json_data = dict()
rewrite_necessary = False
for key1 in results:
if key1 != "PER_CAMFIBER" and key1.startswith("PER_"):
colnames_lst = results[key1][0].colnames
if key1 in join_keys:
for i in join_keys[key1]:
colnames_lst.remove(i)
if key1 not in json_data:
json_data[key1] = colnames_lst
rewrite_necessary = True
else:
for aspect in colnames_lst:
if aspect not in json_data[key1]:
json_data[key1] += [aspect]
rewrite_necessary = True
if rewrite_necessary:
if os.path.isdir(os.path.dirname(jsonfile)):
with open(jsonfile, 'w') as out:
json.dump(json_data, out)
print('Wrote {}'.format(jsonfile))
for qatype in list(results.keys()):
if len(results[qatype]) == 1:
results[qatype] = results[qatype][0]
else:
tx = results[qatype][0]
for i in range(1, len(results[qatype])):
tx = join(tx,
results[qatype][i],
keys=join_keys[qatype],
join_type='outer')
results[qatype] = tx
#- convert python string to bytes for FITS format compatibility
if 'AMP' in results[qatype].colnames:
results[qatype]['AMP'] = results[qatype]['AMP'].astype('S1')
if 'CAM' in results[qatype].colnames:
results[qatype]['CAM'] = results[qatype]['CAM'].astype('S1')
#- TODO: NIGHT/EXPID/SPECTRO/FIBER int64 -> int32 or int16
#- TODO: metrics from float64 -> float32
if outfile is not None:
for tx in results.values():
night = tx['NIGHT'][0]
expid = tx['EXPID'][0]
break
#- To do: consider propagating header from indir/desi*.fits.fz
log.info('{} Writing {}'.format(timestamp(), outfile))
tmpfile = outfile + '.tmp'
with fitsio.FITS(tmpfile, 'rw', clobber=True) as fx:
fx.write(np.zeros(3, dtype=float),
extname='PRIMARY',
header=hdr)
for qatype, qatable in results.items():
fx.write_table(qatable.as_array(),
extname=qatype,
header=hdr)
os.rename(tmpfile, outfile)
log.info('{} Finished writing {}'.format(timestamp(), outfile))
return results
def run(self, indir):
'''TODO: document'''
log = desiutil.log.get_logger()
results = list()
infiles = glob.glob(os.path.join(indir, 'qcframe-*.fits'))
if len(infiles) == 0:
log.error("no qcframe in {}".format(indir))
return None
# find number of spectros
spectros = []
for filename in infiles:
hdr = fitsio.read_header(filename)
s = int(hdr['CAMERA'][1])
spectros.append(s)
spectros = np.unique(spectros)
for spectro in spectros:
infiles = glob.glob(
os.path.join(indir, 'qcframe-?{}-*.fits'.format(spectro)))
qframes = {}
fmap = None
for infile in infiles:
qframe = read_qframe(infile)
cam = qframe.meta["CAMERA"][0].upper()
qframes[cam] = qframe
if fmap is None:
fmap = qframe.fibermap
night = int(qframe.meta['NIGHT'])
expid = int(qframe.meta['EXPID'])
# need to decode fibermap
columns, masks, survey = targets.main_cmx_or_sv(fmap)
desi_target = fmap[columns[0]] # (SV1_)DESI_TARGET
desi_mask = masks[0] # (sv1) desi_mask
stars = (
desi_target
& desi_mask.mask('STD_WD|STD_FAINT|STD_BRIGHT')) != 0
qsos = (desi_target & desi_mask.QSO) != 0
# sw = np.zeros(self.rwave.size)
# swf = np.zeros(self.rwave.size)
#- we need a ridiculously large array to cover the r filter, but precache
#- what wavelength ranges matter for each band
iiband = dict()
for c in ["B", "R", "Z"]:
if c in qframes:
qframe = qframes[c]
iiband[c] = (qframe.wave[0][0] < self.rwave)
iiband[c] &= (self.rwave < qframe.wave[0][-1])
#- for each fiber, generate list of arguments to pass to get_dico
#- get_fiber_data extracts data for only *one* fiber from qframes, fmap, which have data for all fibers
#- this reduces the parallel processing overhead
argslist = [(self, iiband,
get_fiber_data(qframes, fmap, f, fiber, night, expid,
spectro, stars, qsos))
for f, fiber in enumerate(fmap["FIBER"])]
ncpu = get_ncpu(None)
if ncpu > 1:
pool = mp.Pool(ncpu)
results = pool.starmap(get_dico, argslist)
else:
for args in argslist:
results.append(get_dico(**args))
if len(results) == 0:
return None
return Table(results, names=results[0].keys())