def corr(img, d0=4, d1=4, nrand=50000):
"""
Computes the correlation function of an image.
Args:
img : 2D numpy array
d0 : size of output correlation along axis 0
d1 : size of output correlation along axis 1
return correlation function as a 2D array of shape (d0,d1)
"""
log = desiutil.log.get_logger()
mean, rms = calc_overscan(img, nsigma=5, niter=3)
tmp = (img - mean) / rms
log.debug("mean={:3.2f} rms={:3.2f}".format(mean, rms))
n0 = tmp.shape[0]
n1 = tmp.shape[1]
if nrand >= n0 * n1:
ii0 = np.arange(n0 - d0)
ii1 = np.arange(n1 - d1)
else: # random subsampling to run faster
ii0 = np.random.choice(n0 - d0, size=nrand)
ii1 = np.random.choice(n1 - d1, size=nrand)
corrimg = np.zeros((d0, d1))
for i0 in range(d0):
for i1 in range(d1):
corrimg[i0,
i1] = np.median(tmp[ii0, ii1] * tmp[ii0 + i0, ii1 + i1])
#log.debug("corr[{},{}] = {:4.3f}".format(i0,i1,corrimg[i0,i1]))
corrimg /= corrimg[0, 0]
return corrimg
def run(self, indir):
'''TODO: document'''
log = desiutil.log.get_logger()
infiles = glob.glob(os.path.join(indir, 'psf-*.fits'))
results = list()
for filename in infiles:
log.debug(filename)
hdr = fitsio.read_header(filename)
night = hdr['NIGHT']
expid = hdr['EXPID']
cam = hdr['CAMERA'][0].upper()
spectro = int(hdr['CAMERA'][1])
dico={"NIGHT":night,"EXPID":expid,"SPECTRO":spectro,"CAM":cam}
xsig = fitsio.read(filename,"XSIG")
ysig = fitsio.read(filename,"XSIG")
dico["MEANXSIG"]=np.mean(xsig[:,0])
dico["MINXSIG"]=np.min(xsig[:,0])
dico["MAXXSIG"]=np.max(xsig[:,0])
dico["MEANYSIG"]=np.mean(ysig[:,0])
dico["MINYSIG"]=np.min(ysig[:,0])
dico["MAXYSIG"]=np.max(ysig[:,0])
log.debug("{} {} {} {} mean xsig={:3.2f} ysig={:3.2f}".format(night,expid,cam,spectro,dico["MEANXSIG"],dico["MEANYSIG"]))
results.append(collections.OrderedDict(**dico))
return Table(results, names=results[0].keys())
def get_simulator(config='desi',
num_fibers=1,
camera_output=True,
params=None):
'''
returns new or cached specsim.simulator.Simulator object
Also adds placeholder for BGS fiberloss if that isn't already in the config
'''
if isinstance(config, Configuration):
w = config.wavelength
wavehash = (np.min(w), np.max(w), len(w))
key = (config.name, wavehash, num_fibers, camera_output)
else:
key = (config, num_fibers, camera_output)
parammsg = 'with telescope {0}'.format(
params['telescope']) if params else ''
msg = '{0} Simulator for {1} {2}'.format(key, config, parammsg)
if key in _simulators:
log.debug('Returning cached {0}'.format(msg))
qsim = _simulators[key]
defaults = _simdefaults[key]
qsim.source.focal_xy = defaults['focal_xy']
qsim.atmosphere.airmass = defaults['airmass']
qsim.observation.exposure_time = defaults['exposure_time']
qsim.atmosphere.moon.moon_phase = defaults['moon_phase']
qsim.atmosphere.moon.separation_angle = defaults['moon_angle']
qsim.atmosphere.moon.moon_zenith = defaults['moon_zenith']
else:
log.debug('Creating new {0}'.format(msg))
#- New config; create Simulator object
import specsim.simulator
qsim = specsim.simulator.Simulator(config,
num_fibers,
camera_output=camera_output,
params=params)
#- Cache defaults to reset back to original state later
defaults = dict()
defaults['focal_xy'] = qsim.source.focal_xy
defaults['airmass'] = qsim.atmosphere.airmass
defaults['exposure_time'] = qsim.observation.exposure_time
defaults['moon_phase'] = qsim.atmosphere.moon.moon_phase
defaults['moon_angle'] = qsim.atmosphere.moon.separation_angle
defaults['moon_zenith'] = qsim.atmosphere.moon.moon_zenith
_simulators[key] = qsim
_simdefaults[key] = defaults
# update the parameters
if params:
update_params(qsim, params)
return qsim
def get_ephem(use_cache=True, write_cache=True):
"""Return tabulated ephemerides for (START_DATE,STOP_DATE).
The pyephem module must be installed to calculate ephemerides,
but is not necessary when a FITS file of precalcuated data is
available.
Parameters
----------
use_cache : bool
Use cached ephemerides from memory or disk if possible
when True. Otherwise, always calculate from scratch.
write_cache : bool
When True, write a generated table so it is available for
future invocations. Writing only takes place when a
cached object is not available or ``use_cache`` is False.
Returns
-------
Ephemerides
Object with tabulated ephemerides for (START_DATE,STOP_DATE).
"""
global _ephem
# Freeze IERS table for consistent results.
desisurvey.utils.freeze_iers()
# Use standardized string representation of dates.
start_iso = START_DATE.isoformat()
stop_iso = STOP_DATE.isoformat()
range_iso = '({},{})'.format(start_iso, stop_iso)
log = desiutil.log.get_logger()
# First check for a cached object in memory.
if use_cache and _ephem is not None:
if _ephem.start_date != START_DATE or _ephem.stop_date != STOP_DATE:
raise RuntimeError('START_DATE, STOP_DATE have changed.')
log.debug('Returning cached ephemerides for {}.'.format(range_iso))
return _ephem
# Next check for a FITS file on disk.
config = desisurvey.config.Configuration()
filename = config.get_path('ephem_{}_{}.fits'.format(start_iso, stop_iso))
if use_cache and os.path.exists(filename):
# Save restored object in memory.
_ephem = Ephemerides(START_DATE, STOP_DATE, restore=filename)
log.info('Restored ephemerides for {} from {}.'
.format(range_iso, filename))
return _ephem
# Finally, create new ephemerides and save in the memory cache.
log.info('Building ephemerides for {}...'.format(range_iso))
_ephem = Ephemerides(START_DATE, STOP_DATE)
if write_cache:
# Save the tabulated ephemerides to disk.
_ephem._table.write(filename, overwrite=True)
log.info('Saved ephemerides for {} to {}'.format(range_iso, filename))
return _ephem
def get_ephem(use_cache=True, write_cache=True):
"""Return tabulated ephemerides for (START_DATE,STOP_DATE).
The pyephem module must be installed to calculate ephemerides,
but is not necessary when a FITS file of precalcuated data is
available.
Parameters
----------
use_cache : bool
Use cached ephemerides from memory or disk if possible
when True. Otherwise, always calculate from scratch.
write_cache : bool
When True, write a generated table so it is available for
future invocations. Writing only takes place when a
cached object is not available or ``use_cache`` is False.
Returns
-------
Ephemerides
Object with tabulated ephemerides for (START_DATE,STOP_DATE).
"""
global _ephem
# Freeze IERS table for consistent results.
desisurvey.utils.freeze_iers()
# Use standardized string representation of dates.
start_iso = START_DATE.isoformat()
stop_iso = STOP_DATE.isoformat()
range_iso = '({},{})'.format(start_iso, stop_iso)
log = desiutil.log.get_logger()
# First check for a cached object in memory.
if use_cache and _ephem is not None:
if _ephem.start_date != START_DATE or _ephem.stop_date != STOP_DATE:
raise RuntimeError('START_DATE, STOP_DATE have changed.')
log.debug('Returning cached ephemerides for {}.'.format(range_iso))
return _ephem
# Next check for a FITS file on disk.
config = desisurvey.config.Configuration()
filename = config.get_path('ephem_{}_{}.fits'.format(start_iso, stop_iso))
if use_cache and os.path.exists(filename):
# Save restored object in memory.
_ephem = Ephemerides(START_DATE, STOP_DATE, restore=filename)
log.info('Restored ephemerides for {} from {}.'
.format(range_iso, filename))
return _ephem
# Finally, create new ephemerides and save in the memory cache.
log.info('Building ephemerides for {}...'.format(range_iso))
_ephem = Ephemerides(START_DATE, STOP_DATE)
if write_cache:
# Save the tabulated ephemerides to disk.
_ephem._table.write(filename, overwrite=True)
log.info('Saved ephemerides for {} to {}'.format(range_iso, filename))
return _ephem
def get_simulator(config='desi', num_fibers=1):
'''
Returns new or cached specsim.simulator.Simulator object;
Also adds placeholder for BGS fiberloss if that isn't already in the config.
'''
key = (config, num_fibers)
if key in _simulators:
log.debug('Returning cached {} simulator'.format(config))
qsim = _simulators[key]
defaults = _simdefaults[key]
qsim.source.focal_xy = defaults['focal_xy']
qsim.atmosphere.airmass = defaults['airmass']
qsim.atmosphere.moon.moon_phase = defaults['moon_phase']
qsim.atmosphere.moon.separation_angle = defaults['moon_angle']
qsim.atmosphere.moon.moon_zenith = defaults['moon_zenith']
qsim.observation.exposure_time = defaults['exposure_time']
else:
log.debug('Creating new {} simulator'.format(config))
## New config; create Simulator object
import specsim.simulator
qsim = specsim.simulator.Simulator(config, num_fibers)
## TODO FIXME HACK: desimodel/specsim doesn't have BGS fiberloss yet,
## so scale from LRG
if 'bgs' not in qsim.instrument.fiber_acceptance_dict.keys():
log.warning('Treating BGS fiberloss = 0.5 * LRG fiberloss')
qsim.instrument.fiber_acceptance_dict[
'bgs'] = 0.5 * qsim.instrument.fiber_acceptance_dict['lrg']
## Cache defaults to reset back to original state later.
defaults = dict()
defaults['focal_xy'] = qsim.source.focal_xy
defaults['airmass'] = qsim.atmosphere.airmass
defaults['exposure_time'] = qsim.observation.exposure_time
defaults['moon_phase'] = qsim.atmosphere.moon.moon_phase
defaults['moon_angle'] = qsim.atmosphere.moon.separation_angle
defaults['moon_zenith'] = qsim.atmosphere.moon.moon_zenith
_simulators[key] = qsim
_simdefaults[key] = defaults
return qsim
def run(self, indir):
'''TODO: document'''
log = desiutil.log.get_logger()
results = list()
infiles = glob.glob(os.path.join(indir, 'qframe-*.fits'))
if len(infiles) == 0 :
log.error("no qframe in {}".format(indir))
return None
for filename in infiles:
qframe = read_qframe(filename)
night = int(qframe.meta['NIGHT'])
expid = int(qframe.meta['EXPID'])
cam = qframe.meta['CAMERA'][0].upper()
spectro = int(qframe.meta['CAMERA'][1])
log.debug("computing scores for {} {} {} {}".format(night,expid,cam,spectro))
scores,comments = compute_frame_scores(qframe,suffix="RAW",flux_per_angstrom=True)
has_calib_frame = False
cfilename=filename.replace("qframe","qcframe")
if os.path.isfile(cfilename) : # add scores of calibrated sky-subtracted frame
qcframe = read_qframe(cfilename)
cscores,comments = compute_frame_scores(qcframe,suffix="CALIB",flux_per_angstrom=True)
for k in cscores.keys() :
scores[k] = cscores[k]
has_calib_frame = True
nfibers=scores['INTEG_RAW_FLUX_'+cam].size
for f in range(nfibers) :
fiber = int(qframe.fibermap["FIBER"][f])
if has_calib_frame :
results.append(collections.OrderedDict(
NIGHT=night, EXPID=expid, SPECTRO=spectro, CAM=cam, FIBER=fiber,
INTEG_RAW_FLUX=scores['INTEG_RAW_FLUX_'+cam][f],
MEDIAN_RAW_FLUX=scores['MEDIAN_RAW_FLUX_'+cam][f],
MEDIAN_RAW_SNR=scores['MEDIAN_RAW_SNR_'+cam][f],
INTEG_CALIB_FLUX=scores['INTEG_CALIB_FLUX_'+cam][f],
MEDIAN_CALIB_FLUX=scores['MEDIAN_CALIB_FLUX_'+cam][f],
MEDIAN_CALIB_SNR=scores['MEDIAN_CALIB_SNR_'+cam][f]))
else :
results.append(collections.OrderedDict(
NIGHT=night, EXPID=expid, SPECTRO=spectro, CAM=cam, FIBER=fiber,
INTEG_RAW_FLUX=scores['INTEG_RAW_FLUX_'+cam][f],
MEDIAN_RAW_FLUX=scores['MEDIAN_RAW_FLUX_'+cam][f],
MEDIAN_RAW_SNR=scores['MEDIAN_RAW_SNR_'+cam][f]))
if len(results)==0 :
return None
return Table(results, names=results[0].keys())
def get_simulator(config='desi', num_fibers=1, camera_output=True):
'''
returns new or cached specsim.simulator.Simulator object
Also adds placeholder for BGS fiberloss if that isn't already in the config
'''
if isinstance(config, Configuration):
w = config.wavelength
wavehash = (np.min(w), np.max(w), len(w))
key = (config.name, wavehash, num_fibers, camera_output)
else:
key = (config, num_fibers, camera_output)
if key in _simulators:
log.debug('Returning cached {} Simulator'.format(key))
qsim = _simulators[key]
defaults = _simdefaults[key]
qsim.source.focal_xy = defaults['focal_xy']
qsim.atmosphere.airmass = defaults['airmass']
qsim.observation.exposure_time = defaults['exposure_time']
qsim.atmosphere.moon.moon_phase = defaults['moon_phase']
qsim.atmosphere.moon.separation_angle = defaults['moon_angle']
qsim.atmosphere.moon.moon_zenith = defaults['moon_zenith']
else:
log.debug('Creating new {} Simulator'.format(key))
#- New config; create Simulator object
import specsim.simulator
qsim = specsim.simulator.Simulator(config, num_fibers,
camera_output=camera_output)
#- Cache defaults to reset back to original state later
defaults = dict()
defaults['focal_xy'] = qsim.source.focal_xy
defaults['airmass'] = qsim.atmosphere.airmass
defaults['exposure_time'] = qsim.observation.exposure_time
defaults['moon_phase'] = qsim.atmosphere.moon.moon_phase
defaults['moon_angle'] = qsim.atmosphere.moon.separation_angle
defaults['moon_zenith'] = qsim.atmosphere.moon.moon_zenith
_simulators[key] = qsim
_simdefaults[key] = defaults
return qsim
def get_tiles(tiles_file=None, use_cache=True, write_cache=True):
"""Return a Tiles object with optional caching.
You should normally always use the default arguments to ensure
that tiles are defined consistently and efficiently between
different classes.
Parameters
----------
tiles_file : str or None
Use the specified name to override config.tiles_file.
use_cache : bool
Use tiles previously cached in memory when True.
Otherwise, (re)load tiles from disk.
write_cache : bool
If tiles need to be loaded from disk with this call,
save them in a memory cache for future calls.
"""
global _cached_tiles
log = desiutil.log.get_logger()
config = desisurvey.config.Configuration()
tiles_file = tiles_file or config.tiles_file()
if use_cache and tiles_file in _cached_tiles:
tiles = _cached_tiles[tiles_file]
log.debug('Using cached tiles for "{}".'.format(tiles_file))
else:
tiles = Tiles(tiles_file)
log.info('Initialized tiles from "{}".'.format(tiles_file))
for pname in Tiles.PROGRAMS:
pinfo = []
for passnum in tiles.program_passes[pname]:
pinfo.append('{}({})'.format(passnum,
tiles.pass_ntiles[passnum]))
log.info('{:6s} passes(tiles): {}.'.format(pname,
', '.join(pinfo)))
if write_cache:
_cached_tiles[tiles_file] = tiles
else:
log.info('Tiles not cached for "{}".'.format(tiles_file))
return tiles
def get_tiles(tiles_file=None, use_cache=True, write_cache=True):
"""Return a Tiles object with optional caching.
You should normally always use the default arguments to ensure
that tiles are defined consistently and efficiently between
different classes.
Parameters
----------
tiles_file : str or None
Use the specified name to override config.tiles_file.
use_cache : bool
Use tiles previously cached in memory when True.
Otherwise, (re)load tiles from disk.
write_cache : bool
If tiles need to be loaded from disk with this call,
save them in a memory cache for future calls.
"""
global _cached_tiles
log = desiutil.log.get_logger()
config = desisurvey.config.Configuration()
tiles_file = tiles_file or config.tiles_file()
if use_cache and tiles_file in _cached_tiles:
tiles = _cached_tiles[tiles_file]
log.debug('Using cached tiles for "{}".'.format(tiles_file))
else:
tiles = Tiles(tiles_file)
log.info('Initialized tiles from "{}".'.format(tiles_file))
for pname in Tiles.PROGRAMS:
pinfo = []
for passnum in tiles.program_passes[pname]:
pinfo.append('{}({})'.format(passnum, tiles.pass_ntiles[passnum]))
log.info('{:6s} passes(tiles): {}.'.format(pname, ', '.join(pinfo)))
if write_cache:
_cached_tiles[tiles_file] = tiles
else:
log.info('Tiles not cached for "{}".'.format(tiles_file))
return tiles
def _fix_amp_names(hdr):
'''In-place fix of header `hdr` amp names 1-4 to A-D if needed.'''
#- Assume that if any are right, all are right
if 'DATASECA' in hdr:
return
log = desiutil.log.get_logger()
log.debug('Correcting AMP 1-4 to A-D for night {} expid {}'.format(
hdr['NIGHT'], hdr['EXPID']))
for prefix in [
'GAIN', 'RDNOISE', 'PRESEC', 'PRRSEC', 'DATASEC', 'TRIMSEC',
'BIASSEC', 'ORSEC', 'CCDSEC', 'DETSEC', 'AMPSEC', 'OBSRDN',
'OVERSCN'
]:
for ampnum, ampname in [('1', 'A'), ('2', 'B'), ('3', 'C'),
('4', 'D')]:
if prefix + ampnum in hdr:
hdr[prefix + ampname] = hdr[prefix + ampnum]
hdr.delete(prefix + ampnum)
def run(self, indir):
'''TODO: document'''
log = desiutil.log.get_logger()
infiles = glob.glob(os.path.join(indir, 'psf-*.fits'))
if len(infiles) == 0:
log.error('No {}/psf*.fits files found'.format(indir))
return None
results = list()
for filename in infiles:
log.debug(filename)
hdr = fitsio.read_header(filename)
night = hdr['NIGHT']
expid = hdr['EXPID']
cam = hdr['CAMERA'][0].upper()
spectro = int(hdr['CAMERA'][1])
dico={"NIGHT":night,"EXPID":expid,"SPECTRO":spectro,"CAM":cam}
for k in ["MEANDX","MINDX","MAXDX","MEANDY","MINDY","MAXDY"] :
dico[k]=hdr[k]
log.debug("{} {} {} {}".format(night,expid,cam,spectro))
results.append(collections.OrderedDict(**dico))
return Table(results, names=results[0].keys())
def get_simulator(config='desi', num_fibers=1):
'''
returns new or cached specsim.simulator.Simulator object
Also adds placeholder for BGS fiberloss if that isn't already in the config
'''
key = (config, num_fibers)
if key in _simulators:
log.debug('Returning cached {} Simulator'.format(config))
qsim = _simulators[key]
defaults = _simdefaults[key]
qsim.source.focal_xy = defaults['focal_xy']
qsim.atmosphere.airmass = defaults['airmass']
qsim.observation.exposure_time = defaults['exposure_time']
qsim.atmosphere.moon.moon_phase = defaults['moon_phase']
qsim.atmosphere.moon.separation_angle = defaults['moon_angle']
qsim.atmosphere.moon.moon_zenith = defaults['moon_zenith']
else:
log.debug('Creating new {} Simulator'.format(config))
#- New config; create Simulator object
import specsim.simulator
qsim = specsim.simulator.Simulator(config, num_fibers)
#- Cache defaults to reset back to original state later
defaults = dict()
defaults['focal_xy'] = qsim.source.focal_xy
defaults['airmass'] = qsim.atmosphere.airmass
defaults['exposure_time'] = qsim.observation.exposure_time
defaults['moon_phase'] = qsim.atmosphere.moon.moon_phase
defaults['moon_angle'] = qsim.atmosphere.moon.separation_angle
defaults['moon_zenith'] = qsim.atmosphere.moon.moon_zenith
_simulators[key] = qsim
_simdefaults[key] = defaults
return qsim
def __init__(self):
self.output_type = "PER_FIBER"
log = desiutil.log.get_logger()
# load things we will use several times
desiparams = load_desiparams()
geometric_area_cm2 = 1e4 * desiparams["area"]["geometric_area"] # m2
self.thru_conversion_wavelength = 6000 # A
energy_per_photon = (
constants.h * constants.c /
(self.thru_conversion_wavelength * units.Angstrom)).to(
units.erg).value #ergs
self.thru_conversion_factor_ergs_per_cm2 = 1e17 * energy_per_photon / geometric_area_cm2 # ergs/cm2
# preload filters
self.filters = dict()
for photsys in ["N", "S"]:
for band in ["G", "R", "Z", "W1", "W2"]:
self.filters[band + photsys] = load_legacy_survey_filter(
band=band, photsys=photsys)
# define wavelength array
min_filter_wave = 100000
max_filter_wave = 0
for k in self.filters.keys():
log.debug(self.filters[k].name)
min_filter_wave = min(min_filter_wave,
np.min(self.filters[k].wavelength)) - 0.1
max_filter_wave = max(max_filter_wave,
np.max(self.filters[k].wavelength)) + 0.1
log.debug("min max wavelength for filters= {:d} , {:d}".format(
int(min_filter_wave), int(max_filter_wave)))
self.rwave = np.linspace(min_filter_wave, max_filter_wave,
int(max_filter_wave - min_filter_wave))
def stdouterr_redirected(to=None, comm=None):
"""
Redirect stdout and stderr to a file.
The general technique is based on:
http://stackoverflow.com/questions/5081657
http://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/
One difference here is that each process in the communicator
redirects to a different temporary file, and the upon exit
from the context the rank zero process concatenates these
in order to the file result.
Args:
to (str): The output file name.
comm (mpi4py.MPI.Comm): The optional MPI communicator.
"""
nproc = 1
rank = 0
if comm is not None:
nproc = comm.size
rank = comm.rank
# The currently active POSIX file descriptors
fd_out = sys.stdout.fileno()
fd_err = sys.stderr.fileno()
# The DESI loggers.
desi_loggers = desiutil.log._desiutil_log_root
def _redirect(out_to, err_to):
# Flush the C-level buffers
if c_stdout is not None:
libc.fflush(c_stdout)
if c_stderr is not None:
libc.fflush(c_stderr)
# This closes the python file handles, and marks the POSIX
# file descriptors for garbage collection- UNLESS those
# are the special file descriptors for stderr/stdout.
sys.stdout.close()
sys.stderr.close()
# Close fd_out/fd_err if they are open, and copy the
# input file descriptors to these.
os.dup2(out_to, fd_out)
os.dup2(err_to, fd_err)
# Create a new sys.stdout / sys.stderr that points to the
# redirected POSIX file descriptors. In Python 3, these
# are actually higher level IO objects.
if sys.version_info[0] < 3:
sys.stdout = os.fdopen(fd_out, "wb")
sys.stderr = os.fdopen(fd_err, "wb")
else:
# Python 3 case
sys.stdout = io.TextIOWrapper(os.fdopen(fd_out, 'wb'))
sys.stderr = io.TextIOWrapper(os.fdopen(fd_err, 'wb'))
# update DESI logging to use new stdout
for name, logger in desi_loggers.items():
hformat = None
while len(logger.handlers) > 0:
h = logger.handlers[0]
if hformat is None:
hformat = h.formatter._fmt
logger.removeHandler(h)
# Add the current stdout.
ch = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(hformat, datefmt='%Y-%m-%dT%H:%M:%S')
ch.setFormatter(formatter)
logger.addHandler(ch)
# redirect both stdout and stderr to the same file
if to is None:
to = "/dev/null"
if rank == 0:
log = get_logger()
log.debug("Begin log redirection to {} at {}".format(
to, time.asctime()))
# Save the original file descriptors so we can restore them later
saved_fd_out = os.dup(fd_out)
saved_fd_err = os.dup(fd_err)
try:
pto = to
if to != "/dev/null":
pto = "{}_{}".format(to, rank)
# open python file, which creates low-level POSIX file
# descriptor.
file = open(pto, "w")
# redirect stdout/stderr to this new file descriptor.
_redirect(out_to=file.fileno(), err_to=file.fileno())
yield # allow code to be run with the redirected output
# close python file handle, which will mark POSIX file
# descriptor for garbage collection. That is fine since
# we are about to overwrite those in the finally clause.
file.close()
finally:
# restore old stdout and stderr
_redirect(out_to=saved_fd_out, err_to=saved_fd_err)
if comm is not None:
comm.barrier()
# concatenate per-process files
if rank == 0:
with open(to, "w") as outfile:
for p in range(nproc):
outfile.write(
"================= Process {} =================\n".
format(p))
fname = "{}_{}".format(to, p)
with open(fname) as infile:
outfile.write(infile.read())
os.remove(fname)
if comm is not None:
comm.barrier()
if rank == 0:
log = get_logger()
log.debug("End log redirection to {} at {}".format(
to, time.asctime()))
# flush python handles for good measure
sys.stdout.flush()
sys.stderr.flush()
return
def run(self, indir, outfile=None, jsonfile=None):
'''TODO: document'''
log = desiutil.log.get_logger()
log.debug('Running QA in {}'.format(indir))
print('here qa.runner', indir)
preprocfiles = sorted(glob.glob('{}/preproc-*.fits'.format(indir)))
if len(preprocfiles) == 0:
log.error('No preproc files found in {}'.format(indir))
return None
# We can have different obstypes (signal+dark) with calibration data
# obtained with a calibration slit hooked to a single spectrograph.
# So we have to loop over all frames to check if there are science,
# arc, or flat obstypes as guessed by qproc.
qframefiles = sorted(glob.glob('{}/qframe-*.fits'.format(indir)))
if len(qframefiles) == 0: # no qframe so it's either zero or dark
hdr = fitsio.read_header(preprocfiles[0], 0)
if 'OBSTYPE' in hdr:
obstype = hdr['FLAVOR'].strip()
else:
log.warning("Using FLAVOR instead of missing OBSTYPE")
obstype = hdr['FLAVOR'].strip()
else:
obstype = None
log.debug("Reading qframe headers to guess flavor ...")
for qframefile in qframefiles: # look at all of them and prefer arc or flat over dark or zero
hdr = fitsio.read_header(qframefile, 0)
if 'OBSTYPE' in hdr:
this_obstype = hdr['OBSTYPE'].strip().upper()
else:
log.warning("Using FLAVOR instead of missing OBSTYPE")
obstype = hdr['FLAVOR'].strip()
if this_obstype == "ARC" or this_obstype == "FLAT" \
or this_obstype == "TESTARC" or this_obstype == "TESTFLAT" :
obstype = this_obstype
# we use this so we exit the loop
break
elif obstype == None:
obstype = this_obstype
# we stay in the loop in case another frame has another obstype
log.debug('Found OBSTYPE={} files'.format(obstype))
results = dict()
for qa in self.qalist:
if qa.valid_obstype(obstype):
log.info('{} Running {} {}'.format(timestamp(), qa,
qa.output_type))
qa_results = None
try:
qa_results = qa.run(indir)
except Exception as err:
log.warning('{} failed on {} because {}; skipping'.format(
qa, indir, str(err)))
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
del exc_info
#raise(err)
#- TODO: print traceback somewhere useful
if qa_results is not None:
if qa.output_type not in results:
results[qa.output_type] = list()
results[qa.output_type].append(qa_results)
else:
log.debug('Skip {} {} for {}'.format(qa, qa.output_type,
obstype))
#- Combine results for different types of QA
join_keys = dict(
PER_AMP=['NIGHT', 'EXPID', 'SPECTRO', 'CAM', 'AMP'],
PER_CAMERA=['NIGHT', 'EXPID', 'SPECTRO', 'CAM'],
PER_FIBER=['NIGHT', 'EXPID', 'SPECTRO', 'FIBER'],
PER_CAMFIBER=['NIGHT', 'EXPID', 'SPECTRO', 'CAM', 'FIBER'],
PER_SPECTRO=['NIGHT', 'EXPID', 'SPECTRO'],
PER_EXP=['NIGHT', 'EXPID'],
QPROC_STATUS=['NIGHT', 'EXPID', 'SPECTRO', 'CAM'])
if jsonfile is not None:
if os.path.exists(jsonfile):
with open(jsonfile, 'r') as myfile:
json_data = myfile.read()
json_data = json.loads(json_data)
else:
json_data = dict()
rewrite_necessary = False
for key1 in results:
if key1 != "PER_CAMFIBER" and key1.startswith("PER_"):
colnames_lst = results[key1][0].colnames
if key1 in join_keys:
for i in join_keys[key1]:
colnames_lst.remove(i)
if key1 not in json_data:
json_data[key1] = colnames_lst
rewrite_necessary = True
else:
for aspect in colnames_lst:
if aspect not in json_data[key1]:
json_data[key1] += [aspect]
rewrite_necessary = True
if rewrite_necessary:
if os.path.isdir(os.path.dirname(jsonfile)):
with open(jsonfile, 'w') as out:
json.dump(json_data, out)
print('Wrote {}'.format(jsonfile))
for qatype in list(results.keys()):
if len(results[qatype]) == 1:
results[qatype] = results[qatype][0]
else:
tx = results[qatype][0]
for i in range(1, len(results[qatype])):
tx = join(tx,
results[qatype][i],
keys=join_keys[qatype],
join_type='outer')
results[qatype] = tx
#- convert python string to bytes for FITS format compatibility
if 'AMP' in results[qatype].colnames:
results[qatype]['AMP'] = results[qatype]['AMP'].astype('S1')
if 'CAM' in results[qatype].colnames:
results[qatype]['CAM'] = results[qatype]['CAM'].astype('S1')
#- TODO: NIGHT/EXPID/SPECTRO/FIBER int64 -> int32 or int16
#- TODO: metrics from float64 -> float32
if outfile is not None:
for tx in results.values():
night = tx['NIGHT'][0]
expid = tx['EXPID'][0]
break
#- To do: consider propagating header from indir/desi*.fits.fz
log.info('{} Writing {}'.format(timestamp(), outfile))
tmpfile = outfile + '.tmp'
with fitsio.FITS(tmpfile, 'rw', clobber=True) as fx:
fx.write(np.zeros(3, dtype=float),
extname='PRIMARY',
header=hdr)
for qatype, qatable in results.items():
fx.write_table(qatable.as_array(),
extname=qatype,
header=hdr)
os.rename(tmpfile, outfile)
log.info('{} Finished writing {}'.format(timestamp(), outfile))
return results
def simulate_night(night, scheduler, stats, explist, weather,
use_twilight=False, update_interval=10.,
plot=False, verbose=False):
"""Simulate one night of observing.
Uses the online tile scheduler and exposure time calculator.
Parameters
----------
night : datetime.date
Date that the simulated night starts.
scheduler : :class:`desisurvey.scheduler.Scheduler`
Next tile scheduler to use.
stats : :class:`surveysim.stats.SurveyStatistics`
Object for accumulating simulated survey statistics.
explist : :class:`surveysim.exposures.ExposureList`
Object for recording simulated exposures.
weather : :class:`surveysim.weather.Weather`
Simulated weather conditions to use.
use_twlight : bool
Observe during twilight when True.
update_interval : float
Interval in seconds for simulated ETC updates.
plot : bool
Generate a plot summarizing the simulated night when True.
verbose : bool
Produce verbose output on simulation progress when True.
"""
log = desiutil.log.get_logger()
update_interval_days = update_interval / 86400.
night = desisurvey.utils.get_date(night)
nightstats = stats.get_night(night)
label = str(night)
# Lookup this night's sunset and sunrise MJD values.
night_ephem = scheduler.ephem.get_night(night)
if use_twilight:
begin = night_ephem['brightdusk']
end = night_ephem['brightdawn']
else:
begin = night_ephem['dusk']
end = night_ephem['dawn']
nightstats['tsched'] = end - begin
log.debug('Simulating observing on {} from MJD {:.5f} - {:.5f}.'
.format(night, begin, end))
# Find weather time steps that cover this night.
weather_mjd = weather._table['mjd'].data
ilo = np.searchsorted(weather_mjd, begin, side='left') - 1
ihi = np.searchsorted(weather_mjd, end, side='right') + 2
assert weather_mjd[ilo] < begin and weather_mjd[ihi] > end
weather_mjd = weather_mjd[ilo:ihi]
seeing = weather._table['seeing'].data[ilo:ihi]
transp = weather._table['transparency'].data[ilo:ihi]
# Fix this in the weather generator instead?
transp = np.maximum(0.1, transp)
dome = weather._table['open'].data[ilo:ihi]
if not np.any(dome):
log.debug('Dome closed all night.')
return
scheduler.init_night(night, use_twilight=use_twilight)
ETC = desisurvey.etc.ExposureTimeCalculator(save_history=plot)
nexp_last = explist.nexp
# Build linear interpolators for observing conditions.
# This implementation is faster than scipy.interpolate.interp1d()
# when mjd values are gradually increasing.
weather_idx = 0
dmjd_weather = weather_mjd[1] - weather_mjd[0]
def get_weather(mjd):
nonlocal weather_idx
while mjd >= weather_mjd[weather_idx + 1]:
weather_idx += 1
s = (mjd - weather_mjd[weather_idx]) / dmjd_weather
return (
seeing[weather_idx] * (1 - s) + seeing[weather_idx + 1] * s,
transp[weather_idx] * (1 - s) + transp[weather_idx + 1] * s)
# Define time intervals to use in units of days (move to config?)
NO_TILE_AVAIL_DELAY = 30. / 86400.
# Step through the night.
dome_is_open = False
mjd_now = weather_mjd[0]
completed_last = scheduler.completed_by_pass.copy()
while mjd_now < end:
if not dome_is_open:
# Advance to the next dome opening, if any.
idx_now = np.searchsorted(weather_mjd, mjd_now, side='left')
if not np.any(dome[idx_now:]):
# Dome is closed for the rest of the night.
mjd_now = end
break
idx_open = idx_now + np.argmax(dome[idx_now:])
assert dome[idx_open] == True and (idx_open == 0 or dome[idx_open - 1] == False)
mjd_now = weather_mjd[idx_open]
if mjd_now >= end:
# The next dome opening is after the end of the night.
# This can happen if we are not using twilight.
break
# Find the next closing.
if np.all(dome[idx_open:]):
next_dome_closing = end
else:
idx_close = idx_open + np.argmin(dome[idx_open:])
assert dome[idx_close] == False and dome[idx_close - 1] == True
next_dome_closing = min(end, weather_mjd[idx_close])
dome_is_open = True
weather_idx = idx_open
# == NEXT TILE ===========================================================
# Dome is open from mjd_now to next_dome_closing.
mjd_last = mjd_now
tdead = 0.
# Get the current observing conditions.
seeing_now, transp_now = get_weather(mjd_now)
# Get the next tile to observe from the scheduler.
tileid, passnum, snr2frac_start, exposure_factor, airmass, sched_program, mjd_program_end = \
scheduler.next_tile(mjd_now, ETC, seeing_now, transp_now)
if tileid is None:
# Deadtime while we delay and try again.
mjd_now += NO_TILE_AVAIL_DELAY
if mjd_now >= next_dome_closing:
# Dome closed during deadtime.
mjd_now = next_dome_closing
dome_is_open = False
tdead += mjd_now - mjd_last
else:
# Setup for a new field.
mjd_now += ETC.NEW_FIELD_SETUP
if mjd_now >= next_dome_closing:
# Setup interrupted by dome closing.
mjd_now = next_dome_closing
dome_is_open = False
# Record an aborted setup.
nightstats['nsetup_abort'][passnum] += 1
else:
# Record a completed setup.
nightstats['nsetup'][passnum] += 1
# Charge this as setup time whether or not it was aborted.
nightstats['tsetup'][passnum] += mjd_now - mjd_last
if dome_is_open:
# Lookup the program of the next tile, which might be
# different from the scheduled program in ``sched_program``.
tile_program = scheduler.tiles.pass_program[passnum]
# Loop over repeated exposures of the same tile.
continue_this_tile = True
while continue_this_tile:
# -- NEXT EXPOSURE ---------------------------------------------------
# Get the current observing conditions.
seeing_now, transp_now = get_weather(mjd_now)
sky_now = 1.
# Use the ETC to control the shutter.
mjd_open_shutter = mjd_now
ETC.start(mjd_now, tileid, tile_program, snr2frac_start, exposure_factor,
seeing_now, transp_now, sky_now)
integrating = True
while integrating:
mjd_now += update_interval_days
if mjd_now >= next_dome_closing:
# Current exposure is interrupted by dome closing.
mjd_now = next_dome_closing
dome_is_open = False
integrating = False
continue_this_tile = False
elif mjd_now >= mjd_program_end:
# Current exposure is interrupted by a program change.
mjd_now = mjd_program_end
integrating = False
continue_this_tile = False
# Get the current observing conditions.
seeing_now, transp_now = get_weather(mjd_now)
sky_now = 1.
# Update the SNR.
if not ETC.update(mjd_now, seeing_now, transp_now, sky_now):
# Current exposure reached its target SNR according to the ETC.
integrating= False
# stop() will return False if this is a cosmic split and
# more integration is still required.
if ETC.stop(mjd_now):
continue_this_tile = False
# Record this exposure
assert np.allclose(ETC.exptime, mjd_now - mjd_open_shutter)
nightstats['tscience'][passnum] += ETC.exptime
nightstats['nexp'][passnum] += 1
explist.add(
mjd_now - ETC.exptime, 86400 * ETC.exptime, tileid, ETC.snr2frac,
airmass, seeing_now, transp_now, sky_now)
scheduler.update_snr(tileid, ETC.snr2frac)
if continue_this_tile:
# Prepare for the next exposure of the same tile.
snr2frac_start = ETC.snr2frac
mjd_split_start = mjd_now
mjd_now += ETC.SAME_FIELD_SETUP
if mjd_now >= next_dome_closing:
# Setup for next exposure of same tile interrupted by dome closing.
mjd_now = next_dome_closing
dome_is_open = False
continue_this_tile = False
# Record an aborted split.
nightstats['nsplit_abort'][passnum] += 1
else:
# Record a completed split.
nightstats['nsplit'][passnum] += 1
# Charge this as split time, whether or not is was aborted.
nightstats['tsplit'][passnum] += mjd_now - mjd_split_start
# --------------------------------------------------------------------
# Update statistics for the scheduled program (which might be different from
# the program of the tile we just observed).
pidx = scheduler.tiles.PROGRAM_INDEX[sched_program]
nightstats['tdead'][pidx] += tdead
nightstats['topen'][pidx] += mjd_now - mjd_last
# All done if we have observed all tiles.
if scheduler.survey_completed():
break
# ========================================================================
# Save the number of tiles completed per pass in the nightly statistics.
nightstats['completed'][:] = scheduler.completed_by_pass - completed_last
if plot:
import matplotlib.pyplot as plt
fig, axes = plt.subplots(2, 1, figsize=(15, 10), sharex=True)
ax = axes[0]
ax.plot(weather_mjd, seeing, 'r-', label='Seeing')
ax.plot([], [], 'b-', label='Transparency')
ax.legend(ncol=2, loc='lower center')
ax.set_ylabel('Seeing FWHM [arcsec]')
rhs = ax.twinx()
rhs.plot(weather_mjd, transp, 'b-')
rhs.set_ylabel('Transparency')
ax = axes[1]
changes = np.where(np.abs(np.diff(dome)) == 1)[0]
for idx in changes:
ax.axvline(weather_mjd[idx + 1], ls='-', c='r')
mjd_history = np.array(ETC.history['mjd'])
snr2frac_history = np.array(ETC.history['snr2frac'])
for expinfo in explist._exposures[nexp_last: explist.nexp]:
passnum = scheduler.tiles.passnum[scheduler.tiles.index(expinfo['TILEID'])]
program = scheduler.tiles.pass_program[passnum]
color = desisurvey.plots.program_color[program]
t1 = expinfo['MJD']
t2 = t1 + expinfo['EXPTIME'] / 86400
sel = (mjd_history >= t1) & (mjd_history <= t2)
ax.fill_between(mjd_history[sel], snr2frac_history[sel], color=color, alpha=0.5, lw=0)
for t in scheduler.night_changes:
ax.axvline(t, c='b', ls=':')
ax.set_xlim(weather_mjd[0], weather_mjd[-1])
ax.set_xlabel('MJD During {}'.format(label))
ax.set_ylim(0, 1)
ax.set_ylabel('Integrated SNR2 Fraction')
def find_latest_expdir(basedir, processed, startdate=None):
'''
finds the earliest unprocessed basedir/YEARMMDD/EXPID from the latest
YEARMMDD without traversing the whole tree
Args:
basedir : a directory of nights with exposures
processed : set of exposure directories already processed
Options:
startdate : the earliest night to consider processing YYYYMMDD
Returns directory, or None if no matching directories are found
Note: if you want the first unprocessed directory, use
`find_unprocessed_expdir` instead
'''
if startdate:
startdate = str(startdate)
else:
startdate = ''
log = desiutil.log.get_logger(level='DEBUG')
### log.debug('Looking for unprocessed exposures at {}'.format(time.asctime()))
#- Search for most recent basedir/YEARMMDD
for dirname in sorted(os.listdir(basedir), reverse=True):
nightdir = os.path.join(basedir, dirname)
if re.match('20\d{6}', dirname) and dirname >= startdate and \
os.path.isdir(nightdir):
break
#- if for loop completes without finding nightdir to break, run this else
else:
log.debug('No YEARMMDD dirs found in {}'.format(basedir))
return None
night = dirname
log.debug('{} Looking for exposures in {}'.format(timestamp(), nightdir))
spectrofiles = sorted(glob.glob(nightdir + '/*/desi*.fits.fz'))
if len(spectrofiles) > 0:
log.debug('{} found {} desi spectro files though {}'.format(
timestamp(), len(spectrofiles),
os.path.basename(spectrofiles[-1])))
else:
log.debug('{} no new spectro files yet'.format(timestamp()))
return None
for filename in spectrofiles:
dirname = os.path.dirname(filename)
if dirname not in processed:
log.debug('{} selected {}'.format(timestamp(), filename))
return dirname
else:
log.debug('{} no new spectro files found'.format(timestamp()))
return None
def make_plots(infile,
basedir,
preprocdir=None,
logdir=None,
rawdir=None,
cameras=None):
'''Make plots for a single exposure
Args:
infile: input QA fits file with HDUs like PER_AMP, PER_FIBER, ...
basedir: write output HTML to basedir/NIGHT/EXPID/
Options:
preprocdir: directory to where the "preproc-*-*.fits" are located. If
not provided, function will NOT generate any image files from any
preproc fits file.
logdir: directory to where the "qproc-*-*.log" are located. If
not provided, function will NOT display any logfiles.
rawdir: directory to where the raw data files are located, including
"guide-rois-*.fits" and "centroid-*.json" files, are located. If
not provided, the function will not plot the guide plots.
cameras: list of cameras (strings) to generate image files of. If not
provided, will generate a cameras list from parcing through the
preproc fits files in the preprocdir
'''
from nightwatch.webpages import amp as web_amp
from nightwatch.webpages import camfiber as web_camfiber
from nightwatch.webpages import camera as web_camera
from nightwatch.webpages import summary as web_summary
from nightwatch.webpages import lastexp as web_lastexp
from nightwatch.webpages import guide as web_guide
from nightwatch.webpages import guideimage as web_guideimage
from nightwatch.webpages import placeholder as web_placeholder
from . import io
log = desiutil.log.get_logger()
qadata = io.read_qa(infile)
header = qadata['HEADER']
night = header['NIGHT']
expid = header['EXPID']
#- Early data have wrong NIGHT in header; check by hand
#- YEARMMDD/EXPID/infile
dirnight = os.path.basename(os.path.dirname(os.path.dirname(infile)))
if re.match('20\d{6}', dirnight) and dirnight != str(night):
log.warning('Correcting {} header night {} to {}'.format(
infile, night, dirnight))
night = int(dirnight)
header['NIGHT'] = night
#- Create output exposures plot directory if needed
expdir = os.path.join(basedir, str(night), '{:08d}'.format(expid))
if not os.path.isdir(expdir):
log.info('Creating {}'.format(expdir))
os.makedirs(expdir, exist_ok=True)
if 'PER_AMP' in qadata:
htmlfile = '{}/qa-amp-{:08d}.html'.format(expdir, expid)
pc = web_amp.write_amp_html(htmlfile, qadata['PER_AMP'], header)
print('Wrote {}'.format(htmlfile))
else:
htmlfile = '{}/qa-amp-{:08d}.html'.format(expdir, expid)
pc = web_placeholder.write_placeholder_html(htmlfile, header,
"PER_AMP")
htmlfile = '{}/qa-camfiber-{:08d}.html'.format(expdir, expid)
if 'PER_CAMFIBER' in qadata:
try:
pc = web_camfiber.write_camfiber_html(htmlfile,
qadata['PER_CAMFIBER'],
header)
print('Wrote {}'.format(htmlfile))
except Exception as err:
web_placeholder.handle_failed_plot(htmlfile, header,
"PER_CAMFIBER")
else:
pc = web_placeholder.write_placeholder_html(htmlfile, header,
"PER_CAMFIBER")
htmlfile = '{}/qa-camera-{:08d}.html'.format(expdir, expid)
if 'PER_CAMERA' in qadata:
try:
pc = web_camera.write_camera_html(htmlfile, qadata['PER_CAMERA'],
header)
print('Wrote {}'.format(htmlfile))
except Exception as err:
web_placeholder.handle_failed_plot(htmlfile, header, "PER_CAMERA")
else:
pc = web_placeholder.write_placeholder_html(htmlfile, header,
"PER_CAMERA")
htmlfile = '{}/qa-summary-{:08d}.html'.format(expdir, expid)
web_summary.write_summary_html(htmlfile, qadata, preprocdir)
print('Wrote {}'.format(htmlfile))
#- Note: last exposure goes in basedir, not expdir=basedir/NIGHT/EXPID
htmlfile = '{}/qa-lastexp.html'.format(basedir)
web_lastexp.write_lastexp_html(htmlfile, qadata, preprocdir)
print('Wrote {}'.format(htmlfile))
if rawdir:
#- plot guide metric plots
try:
guidedata = io.get_guide_data(night, expid, rawdir)
htmlfile = '{}/qa-guide-{:08d}.html'.format(expdir, expid)
web_guide.write_guide_html(htmlfile, header, guidedata)
print('Wrote {}'.format(htmlfile))
except (FileNotFoundError, OSError, IOError):
print('Unable to find guide data, not plotting guide plots')
htmlfile = '{}/qa-guide-{:08d}.html'.format(expdir, expid)
pc = web_placeholder.write_placeholder_html(
htmlfile, header, "GUIDING")
#- plot guide image movies
try:
htmlfile = '{expdir}/guide-image-{expid:08d}.html'.format(
expdir=expdir, expid=expid)
image_data = io.get_guide_images(night, expid, rawdir)
web_guideimage.write_guide_image_html(image_data, htmlfile, night,
expid)
print('Wrote {}'.format(htmlfile))
except (FileNotFoundError, OSError, IOError):
print('Unable to find guide data, not plotting guide image plots')
htmlfile = '{expdir}/guide-image-{expid:08d}.html'.format(
expdir=expdir, expid=expid)
pc = web_placeholder.write_placeholder_html(
htmlfile, header, "GUIDE_IMAGES")
#- regardless of if logdir or preprocdir, identifying failed qprocs by comparing
#- generated preproc files to generated logfiles
qproc_fails = []
if cameras is None:
cameras = []
import glob
for preprocfile in glob.glob(
os.path.join(preprocdir, 'preproc-*-*.fits')):
cameras += [os.path.basename(preprocfile).split('-')[1]]
log_cams = []
log_outputs = [
i for i in os.listdir(logdir) if re.match(r'qproc.*\.log', i)
]
for log_output in log_outputs:
l_cam = log_output.split("-")[1]
log_cams += [l_cam]
if l_cam not in cameras:
qproc_fails.append(l_cam)
from nightwatch.webpages import plotimage as web_plotimage
if (preprocdir is not None):
#- plot preprocessed images
downsample = 4
ncpu = get_ncpu(None)
pinput = os.path.join(preprocdir, "preproc-{}-{:08d}.fits")
output = os.path.join(expdir, "preproc-{}-{:08d}-4x.html")
argslist = [(pinput.format(cam, expid), output.format(cam, expid),
downsample, night) for cam in cameras]
if ncpu > 1:
pool = mp.Pool(ncpu)
pool.starmap(web_plotimage.write_image_html, argslist)
pool.close()
pool.join()
else:
for args in argslist:
web_plotimage.write_image_html(*args)
#- plot preproc nav table
navtable_output = '{}/qa-amp-{:08d}-preproc_table.html'.format(
expdir, expid)
web_plotimage.write_preproc_table_html(preprocdir, night, expid,
downsample, navtable_output)
if (logdir is not None):
#- plot logfiles
log.debug('Log directory: {}'.format(logdir))
error_colors = dict()
for log_cam in log_cams:
qinput = os.path.join(logdir,
"qproc-{}-{:08d}.log".format(log_cam, expid))
output = os.path.join(
expdir, "qproc-{}-{:08d}-logfile.html".format(log_cam, expid))
log.debug('qproc log: {}'.format(qinput))
e = web_summary.write_logfile_html(qinput, output, night)
error_colors[log_cam] = e
#- plot logfile nav table
htmlfile = '{}/qa-summary-{:08d}-logfiles_table.html'.format(
expdir, expid)
web_summary.write_logtable_html(htmlfile,
logdir,
night,
expid,
available=log_cams,
error_colors=error_colors)
def freeze_iers(name='iers_frozen.ecsv', ignore_warnings=True):
"""Use a frozen IERS table saved with this package.
This should be called at the beginning of a script that calls
astropy time and coordinates functions which refer to the UT1-UTC
and polar motions tabulated by IERS. The purpose is to ensure
identical results across systems and astropy releases, to avoid a
potential network download, and to eliminate some astropy warnings.
After this call, the loaded table will be returned by
:func:`astropy.utils.iers.IERS_Auto.open()` and treated like a
a normal IERS table by all astropy code. Specifically, this method
registers an instance of a custom IERS_Frozen class that inherits from
IERS_B and overrides
:meth:`astropy.utils.iers.IERS._check_interpolate_indices` to prevent
any IERSRangeError being raised.
See http://docs.astropy.org/en/stable/utils/iers.html for details.
This function returns immediately after the first time it is called,
so it it safe to insert anywhere that consistent IERS models are
required, and subsequent calls with different args will have no
effect.
The :func:`desisurvey.utils.plot_iers` function is useful for inspecting
IERS tables and how they are extrapolated to DESI survey dates.
Parameters
----------
name : str
Name of the file to load the frozen IERS table from. Should normally
be relative and then refers to this package's data/ directory.
Must end with the .ecsv extension.
ignore_warnings : bool
Ignore ERFA and IERS warnings about future dates generated by
astropy time and coordinates functions. Specifically, ERFA warnings
containing the string "dubious year" are filtered out, as well
as AstropyWarnings related to IERS table extrapolation.
"""
log = desiutil.log.get_logger()
if desisurvey.utils._iers_is_frozen:
log.debug('IERS table already frozen.')
return
log.info('Freezing IERS table used by astropy time, coordinates.')
# Validate the save_name extension.
_, ext = os.path.splitext(name)
if ext != '.ecsv':
raise ValueError('Expected .ecsv extension for {0}.'.format(name))
# Locate the file in our package data/ directory.
if not os.path.isabs(name):
name = astropy.utils.data._find_pkg_data_path(
os.path.join('data', name))
if not os.path.exists(name):
raise ValueError('No such IERS file: {0}.'.format(name))
# Clear any current IERS table.
astropy.utils.iers.IERS.close()
# Initialize the global IERS table. We load the table by
# hand since the IERS open() method hardcodes format='cds'.
try:
table = astropy.table.Table.read(name, format='ascii.ecsv').filled()
except IOError:
raise RuntimeError('Unable to load IERS table from {0}.'.format(name))
# Define a subclass of IERS_B that overrides _check_interpolate_indices
# to prevent any IERSRangeError being raised.
class IERS_Frozen(astropy.utils.iers.IERS_B):
def _check_interpolate_indices(self, indices_orig, indices_clipped,
max_input_mjd): pass
# Create and register an instance of this class from the table.
iers = IERS_Frozen(table)
astropy.utils.iers.IERS.iers_table = iers
# Prevent any attempts to automatically download updated IERS-A tables.
astropy.utils.iers.conf.auto_download = False
astropy.utils.iers.conf.auto_max_age = None
astropy.utils.iers.conf.iers_auto_url = 'frozen'
# Sanity check.
if not (astropy.utils.iers.IERS_Auto.open() is iers):
raise RuntimeError('Frozen IERS is not installed as the default.')
if ignore_warnings:
warnings.filterwarnings(
'ignore', category=astropy._erfa.core.ErfaWarning, message=
r'ERFA function \"[a-z0-9_]+\" yielded [0-9]+ of \"dubious year')
warnings.filterwarnings(
'ignore', category=astropy.utils.exceptions.AstropyWarning,
message=r'Tried to get polar motions for times after IERS data')
warnings.filterwarnings(
'ignore', category=astropy.utils.exceptions.AstropyWarning,
message=r'\(some\) times are outside of range covered by IERS')
# Shortcircuit any subsequent calls to this function.
desisurvey.utils._iers_is_frozen = True
def simulate_night(night,
scheduler,
stats,
explist,
weather,
use_twilight=False,
use_simplesky=False,
update_interval=10.,
plot=False,
verbose=False):
"""Simulate one night of observing.
Uses the online tile scheduler and exposure time calculator.
Parameters
----------
night : datetime.date
Date that the simulated night starts.
scheduler : :class:`desisurvey.scheduler.Scheduler`
Next tile scheduler to use.
stats : :class:`surveysim.stats.SurveyStatistics`
Object for accumulating simulated survey statistics.
explist : :class:`surveysim.exposures.ExposureList`
Object for recording simulated exposures.
weather : :class:`surveysim.weather.Weather`
Simulated weather conditions to use.
use_twlight : bool
Observe during twilight when True.
update_interval : float
Interval in seconds for simulated ETC updates.
plot : bool
Generate a plot summarizing the simulated night when True.
verbose : bool
Produce verbose output on simulation progress when True.
"""
log = desiutil.log.get_logger()
update_interval_days = update_interval / 86400.
night = desisurvey.utils.get_date(night)
nightstats = stats.get_night(night)
label = str(night)
# Lookup this night's sunset and sunrise MJD values.
night_ephem = scheduler.ephem.get_night(night)
night_programs, night_changes = scheduler.ephem.get_night_program(night)
if use_twilight:
begin = night_ephem['brightdusk']
end = night_ephem['brightdawn']
else:
begin = night_ephem['dusk']
end = night_ephem['dawn']
nightstats['tsched'] = end - begin
log.debug('Simulating observing on {} from MJD {:.5f} - {:.5f}.'.format(
night, begin, end))
config = desisurvey.config.Configuration()
# Find weather time steps that cover this night.
weather_mjd = weather._table['mjd'].data
ilo = np.searchsorted(weather_mjd, begin, side='left') - 1
ihi = np.searchsorted(weather_mjd, end, side='right') + 2
assert weather_mjd[ilo] < begin and weather_mjd[ihi] > end
weather_mjd = weather_mjd[ilo:ihi]
seeing = weather._table['seeing'].data[ilo:ihi]
transp = weather._table['transparency'].data[ilo:ihi]
# Fix this in the weather generator instead?
transp = np.maximum(0.1, transp)
dome = weather._table['open'].data[ilo:ihi]
if not np.any(dome):
log.debug('Dome closed all night.')
return
scheduler.init_night(night, use_twilight=use_twilight)
ETC = desisurvey.etc.ExposureTimeCalculator(save_history=plot)
nexp_last = explist.nexp
# Build linear interpolators for observing conditions.
# This implementation is faster than scipy.interpolate.interp1d()
# when mjd values are gradually increasing.
weather_idx = 0
dmjd_weather = weather_mjd[1] - weather_mjd[0]
# moon illumination for the night
moon_ill = night_ephem['moon_illum_frac']
# for moon ephem calculation
moon_DECRA = desisurvey.ephem.get_object_interpolator(night_ephem,
'moon',
altaz=False)
moon_ALTAZ = desisurvey.ephem.get_object_interpolator(night_ephem,
'moon',
altaz=True)
# for sun ephem calculation
sun_DECRA = desisurvey.ephem.get_object_interpolator(night_ephem,
'sun',
altaz=False)
sun_ALTAZ = desisurvey.ephem.get_object_interpolator(night_ephem,
'sun',
altaz=True)
skylevel_cache, skylevel_cache_time = None, None
def get_weather(mjd, ra=None, dec=None):
nonlocal weather_idx, night_changes, night_programs, config, skylevel_cache, skylevel_cache_time
while mjd >= weather_mjd[weather_idx + 1]:
weather_idx += 1
s = (mjd - weather_mjd[weather_idx]) / dmjd_weather
if not use_simplesky:
# use on-the-fly sky level calculations
if ra is None:
sky = desisurvey.etc.sky_level(mjd,
ra,
dec,
moon_ill=moon_ill,
moon_DECRA=moon_DECRA,
moon_ALTAZ=moon_ALTAZ,
sun_DECRA=sun_DECRA,
sun_ALTAZ=sun_ALTAZ)
else:
# update sky level every 10 mins
if (skylevel_cache_time is None) or (mjd - skylevel_cache_time
> 0.006944444445252884):
sky = desisurvey.etc.sky_level(mjd,
ra,
dec,
moon_ill=moon_ill,
moon_DECRA=moon_DECRA,
moon_ALTAZ=moon_ALTAZ,
sun_DECRA=sun_DECRA,
sun_ALTAZ=sun_ALTAZ)
skylevel_cache_time = mjd
skylevel_cache = sky
else:
sky = skylevel_cache
else:
# use simple sky level calculation based on moon_up factor
cond_ind = np.interp(mjd, night_changes,
np.arange(len(night_changes)))
if (cond_ind < 0) or (cond_ind >= len(night_programs)):
cond = 'BRIGHT'
else:
cond = night_programs[int(np.floor(cond_ind))]
sky = getattr(config.conditions, cond).moon_up_factor()
return (seeing[weather_idx] * (1 - s) + seeing[weather_idx + 1] * s,
transp[weather_idx] * (1 - s) + transp[weather_idx + 1] * s,
sky)
# Define time intervals to use in units of days (move to config?)
NO_TILE_AVAIL_DELAY = 30. / 86400.
# Step through the night.
dome_is_open = False
mjd_now = weather_mjd[0]
if mjd_now < begin:
mjd_now = begin
completed_last = scheduler.plan.obsend_by_program()
current_ra = None
current_dec = None
while mjd_now < end:
if not dome_is_open:
current_ra = current_dec = None
# Advance to the next dome opening, if any.
idx_now = np.searchsorted(weather_mjd, mjd_now, side='left')
if not np.any(dome[idx_now:]):
# Dome is closed for the rest of the night.
mjd_now = end
break
idx_open = idx_now + np.argmax(dome[idx_now:])
assert dome[idx_open] == True and (idx_open == 0
or dome[idx_open - 1] == False
or mjd_now == begin)
mjd_now = weather_mjd[idx_open]
if mjd_now >= end:
# The next dome opening is after the end of the night.
# This can happen if we are not using twilight.
break
# Find the next closing.
if np.all(dome[idx_open:]):
next_dome_closing = end
else:
idx_close = idx_open + np.argmin(dome[idx_open:])
assert dome[idx_close] == False and dome[idx_close - 1] == True
next_dome_closing = min(end, weather_mjd[idx_close])
dome_is_open = True
weather_idx = idx_open
# == NEXT TILE ===========================================================
# Dome is open from mjd_now to next_dome_closing.
mjd_last = mjd_now
tdead = 0.
# Get the current observing conditions.
seeing_tile, transp_tile, sky_tile = get_weather(mjd_now,
ra=None,
dec=None)
# Get the next tile to observe from the scheduler.
tileid, passnum, snr2frac_start, exposure_factor, airmass, sched_program, mjd_program_end = \
scheduler.next_tile(
mjd_now, ETC, seeing_tile, transp_tile,
sky_tile, current_ra=current_ra, current_dec=current_dec)
if tileid is None:
# Deadtime while we delay and try again.
mjd_now += NO_TILE_AVAIL_DELAY
if mjd_now >= next_dome_closing:
# Dome closed during deadtime.
mjd_now = next_dome_closing
dome_is_open = False
tdead += mjd_now - mjd_last
current_ra = current_dec = None
else:
idx = scheduler.tiles.index(tileid)
tileprogram = scheduler.tiles.tileprogram[idx]
programnum = scheduler.tiles.program_index[tileprogram]
current_ra = scheduler.tiles.tileRA[idx]
current_dec = scheduler.tiles.tileDEC[idx]
# Setup for a new field.
mjd_now += ETC.NEW_FIELD_SETUP
# slew time goes here
if mjd_now >= next_dome_closing:
# Setup interrupted by dome closing.
mjd_now = next_dome_closing
dome_is_open = False
# Record an aborted setup.
nightstats['nsetup_abort'][programnum] += 1
else:
# Record a completed setup.
nightstats['nsetup'][programnum] += 1
# Charge this as setup time whether or not it was aborted.
nightstats['tsetup'][programnum] += mjd_now - mjd_last
if dome_is_open:
# Lookup the program of the next tile, which might be
# different from the scheduled program in ``sched_program``.
# Loop over repeated exposures of the same tile.
continue_this_tile = True
while continue_this_tile:
# -- NEXT EXPOSURE ---------------------------------------------------
# Use the ETC to control the shutter.
mjd_open_shutter = mjd_now
ETC.start(mjd_now, tileid, tileprogram, snr2frac_start,
exposure_factor, seeing_tile, transp_tile,
sky_tile)
integrating = True
while integrating:
mjd_now += update_interval_days
if mjd_now >= next_dome_closing:
# Current exposure is interrupted by dome closing.
mjd_now = next_dome_closing
dome_is_open = False
integrating = False
continue_this_tile = False
elif mjd_now >= mjd_program_end:
# Current exposure is interrupted by a program change.
mjd_now = mjd_program_end
integrating = False
continue_this_tile = False
# Get the current observing conditions.
seeing_now, transp_now, sky_now = get_weather(
mjd_now,
ra=scheduler.tiles.tileRA[idx],
dec=scheduler.tiles.tileDEC[idx])
# Update the SNR.
if not ETC.update(mjd_now, seeing_now, transp_now,
sky_now):
# Current exposure reached its target SNR according to the ETC.
integrating = False
# stop() will return False if this is a cosmic split and
# more integration is still required.
if ETC.stop(mjd_now):
continue_this_tile = False
# Record this exposure
assert np.allclose(ETC.exptime, mjd_now - mjd_open_shutter)
nightstats['tscience'][programnum] += ETC.exptime
nightstats['nexp'][programnum] += 1
explist.add(mjd_now - ETC.exptime, 86400 * ETC.exptime,
tileid, ETC.snr2frac,
ETC.snr2frac - snr2frac_start, airmass,
seeing_now, transp_now, sky_now)
scheduler.update_snr(tileid, ETC.snr2frac)
# All done if we have observed all tiles.
if scheduler.plan.survey_completed():
break
if continue_this_tile:
# Prepare for the next exposure of the same tile.
snr2frac_start = ETC.snr2frac
mjd_split_start = mjd_now
mjd_now += ETC.SAME_FIELD_SETUP
if mjd_now >= next_dome_closing:
# Setup for next exposure of same tile interrupted by dome closing.
mjd_now = next_dome_closing
dome_is_open = False
continue_this_tile = False
# Record an aborted split.
nightstats['nsplit_abort'][programnum] += 1
else:
# Record a completed split.
nightstats['nsplit'][programnum] += 1
# Charge this as split time, whether or not is was aborted.
nightstats['tsplit'][
programnum] += mjd_now - mjd_split_start
# --------------------------------------------------------------------
# Update statistics for the scheduled program (which might be different from
# the program of the tile we just observed).
if scheduler.tiles.nogray:
sched_program = ('DARK'
if sched_program == 'GRAY' else sched_program)
pidx = scheduler.tiles.program_index[sched_program]
nightstats['tdead'][pidx] += tdead
nightstats['topen'][pidx] += mjd_now - mjd_last
# ========================================================================
# Save the number of tiles completed per program in the nightly statistics.
nightstats['completed'][:] = (scheduler.plan.obsend_by_program() -
completed_last)
if plot:
import matplotlib.pyplot as plt
fig, axes = plt.subplots(2, 1, figsize=(15, 10), sharex=True)
ax = axes[0]
ax.plot(weather_mjd, seeing, 'r-', label='Seeing')
ax.plot([], [], 'b-', label='Transparency')
ax.legend(ncol=2, loc='lower center')
ax.set_ylabel('Seeing FWHM [arcsec]')
rhs = ax.twinx()
rhs.plot(weather_mjd, transp, 'b-')
rhs.set_ylabel('Transparency')
ax = axes[1]
changes = np.where(np.abs(np.diff(dome)) == 1)[0]
for idx in changes:
ax.axvline(weather_mjd[idx + 1], ls='-', c='r')
mjd_history = np.array(ETC.history['mjd'])
snr2frac_history = np.array(ETC.history['snr2frac'])
for expinfo in explist._exposures[nexp_last:explist.nexp]:
program = scheduler.tiles.tileprogram[scheduler.tiles.index(
expinfo['TILEID'])]
color = desisurvey.plots.program_color[program]
t1 = expinfo['MJD']
t2 = t1 + expinfo['EXPTIME'] / 86400
sel = (mjd_history >= t1) & (mjd_history <= t2)
ax.fill_between(mjd_history[sel],
snr2frac_history[sel],
color=color,
alpha=0.5,
lw=0)
for t in scheduler.night_changes:
ax.axvline(t, c='b', ls=':')
ax.set_xlim(weather_mjd[0], weather_mjd[-1])
ax.set_xlabel('MJD During {}'.format(label))
ax.set_ylim(0, 1)
ax.set_ylabel('Integrated SNR2 Fraction')
def write_exposures_tables(indir, outdir, exposures, nights=None):
"""
Writes exposures table for each night available
Args:
outfile: output HTML files to outdir/YEARMMDD/exposures.html
exposures: table with columns NIGHT, EXPID
Options:
nights: optional list of nights to process
"""
log = desiutil.log.get_logger()
env = jinja2.Environment(
loader=jinja2.PackageLoader('nightwatch.webpages', 'templates'),
autoescape=select_autoescape(disabled_extensions=('txt', ),
default_for_string=True,
default=True))
template = env.get_template('exposures.html')
if nights is None:
nights = np.unique(exposures['NIGHT'])
for night in nights:
log.debug('{} Generating exposures table for {}'.format(
time.asctime(), night))
ii = (exposures['NIGHT'] == int(night))
explist = list()
night_exps = exposures[ii]
night_exps.sort('EXPID')
for row in night_exps:
expid = row['EXPID']
#- adds failed expid to table
if row['FAIL'] == 1:
link = '{expid:08d}/qa-summary-{expid:08}-logfiles_table.html'.format(
night=night, expid=expid)
expinfo = dict(night=night, expid=expid, link=link, fail=1)
explist.append(expinfo)
continue
qafile = io.findfile('qa', night, expid, basedir=indir)
qadata = io.read_qa(qafile)
status = get_status(qadata, night)
if 'OBSTYPE' in qadata['HEADER']:
obstype = qadata['HEADER']['OBSTYPE'].rstrip().upper()
else:
log.warning('Use FLAVOR instead of missing OBSTYPE')
obstype = qadata['HEADER']['FLAVOR'].rstrip().upper()
exptime = qadata['HEADER']['EXPTIME']
from ..plots.core import parse_numlist
peramp = status.get('PER_AMP')
if peramp:
list_specs = list(set(peramp['SPECTRO']))
spectros = parse_numlist(list_specs)
else:
spectros = '???'
link = '{expid:08d}/qa-summary-{expid:08d}.html'.format(
night=night, expid=expid)
expinfo = dict(night=night,
expid=expid,
obstype=obstype,
link=link,
exptime=exptime,
spectros=spectros,
fail=0)
hdr = qadata['HEADER']
expinfo['PROGRAM'] = hdr['PROGRAM'] if 'PROGRAM' in hdr else '?'
#- TILEID with link to fiberassign QA
if 'TILEID' in hdr:
tileid = hdr['TILEID']
expinfo['TILEID'] = tileid
tilegroup = '{:03d}'.format(tileid // 1000)
expinfo[
'TILEID_LINK'] = f'https://data.desi.lbl.gov/desi/target/fiberassign/tiles/trunk/{tilegroup}/fiberassign-{tileid:06d}.png'
else:
expinfo['TILEID'] = -1
expinfo['TILEID_LINK'] = 'na'
#- KPNO local time (MST=Mountain Standard Time)
if 'MJD-OBS' in hdr:
expinfo['MST'] = Time(hdr['MJD-OBS'] - 7 / 24,
format='mjd').strftime('%H:%M')
else:
expinfo['MST'] = '?'
#- Adds qproc to the expid status
#- TODO: add some catches to this for robustness, e.g. the '-' if QPROC is missing
if len(row['QPROC']) == 0 and row['QPROC_EXIT'] == 0:
expinfo['QPROC'] = 'ok'
else:
expinfo['QPROC'] = 'error'
expinfo[
'QPROC_link'] = '{expid:08d}/qa-summary-{expid:08d}-logfiles_table.html'.format(
expid=expid)
#- TODO: have actual thresholds
for i, qatype in enumerate([
'PER_AMP', 'PER_CAMERA', 'PER_FIBER', 'PER_CAMFIBER',
'PER_SPECTRO', 'PER_EXP'
]):
if qatype not in status:
expinfo[qatype] = '-'
expinfo[qatype + "_link"] = "na"
else:
qastatus = Status(np.max(status[qatype]['QASTATUS']))
short_name = qatype.split("_")[1].lower()
expinfo[qatype] = qastatus.name
if qatype != 'QPROC':
expinfo[
qatype +
"_link"] = '{expid:08d}/qa-{name}-{expid:08d}.html'.format(
expid=expid, name=short_name)
explist.append(expinfo)
html = template.render(night=night,
exposures=explist,
autoreload=True,
staticdir='../static')
outfile = os.path.join(outdir, str(night), 'exposures.html')
tmpfile = outfile + '.tmp' + str(os.getpid())
with open(tmpfile, 'w') as fx:
fx.write(html)
os.rename(tmpfile, outfile)
#- Update expid and night links only once at the end
_write_expid_links(outdir, exposures, nights)
_write_night_links(outdir)
