def main(args):
"""Command-line driver for initializing the survey plan.
"""
# Set up the logger
if args.debug:
log = desiutil.log.get_logger(desiutil.log.DEBUG)
args.verbose = True
elif args.verbose:
log = desiutil.log.get_logger(desiutil.log.INFO)
else:
log = desiutil.log.get_logger(desiutil.log.WARNING)
# Set the output path if requested.
config = desisurvey.config.Configuration(file_name=args.config_file)
if args.output_path is not None:
config.set_output_path(args.output_path)
if args.tiles_file is not None:
config.tiles_file.set_value(args.tiles_file)
# Tabulate emphemerides if necessary.
ephem = desisurvey.ephem.get_ephem(use_cache=not args.recalc)
# Calculate design hour angles if necessary.
fullname = config.get_path(args.save)
if args.recalc or not os.path.exists(fullname):
calculate_initial_plan(args)
else:
log.info('Initial plan has already been created.')
def run_assemble_fibermap(rawfile, outdir):
'''Run assemble_fibermap using NIGHT, EXPID, and TILE from input raw data file
Args:
rawfile: input desi-EXPID.fits.fz raw data file
outdir: directory to write fibermap-EXPID.fits files
Returns:
path to written fibermap
'''
hdr = fitsio.read_header(rawfile, 1)
night, expid = get_night_expid_header(hdr)
log = desiutil.log.get_logger()
if 'TILEID' in hdr:
if not os.path.isdir(outdir):
log.info('Creating {}'.format(outdir))
os.makedirs(outdir, exist_ok=True)
fibermap = os.path.join(outdir, 'fibermap-{:08d}.fits'.format(expid))
cmd = f'assemble_fibermap -n {night} -e {expid} -o {fibermap} --overwrite'
logfile = '{}/assemble_fibermap-{:08d}.log'.format(outdir, expid)
msg = 'assemble_fibermap {}/{}'.format(night, expid)
err = runcmd(cmd, logfile, msg)
return fibermap
return None
def main(args):
"""Command-line driver for initializing the survey plan.
"""
# Set up the logger
if args.debug:
log = desiutil.log.get_logger(desiutil.log.DEBUG)
args.verbose = True
elif args.verbose:
log = desiutil.log.get_logger(desiutil.log.INFO)
else:
log = desiutil.log.get_logger(desiutil.log.WARNING)
# Set the output path if requested.
config = desisurvey.config.Configuration(file_name=args.config_file)
if args.output_path is not None:
config.set_output_path(args.output_path)
if args.tiles_file is not None:
config.tiles_file.set_value(args.tiles_file)
# Tabulate emphemerides if necessary.
ephem = desisurvey.ephem.get_ephem(use_cache=not args.recalc)
# Calculate design hour angles if necessary.
fullname = config.get_path(args.save)
if args.recalc or not os.path.exists(fullname):
calculate_initial_plan(args)
else:
log.info('Initial plan has already been created.')
def save(self, name='stats.fits', comment='', overwrite=True):
"""Save a snapshot of these statistics as a binary FITS table.
The saved file size is ~800 Kb.
Parameters
----------
name : str
File name to write. Will be located in the configuration
output path unless it is an absolute path. Pass the same
name to the constructor's ``restore`` argument to restore
this snapshot.
comment : str
Comment to include in the saved header, for documentation
purposes.
overwrite : bool
Silently overwrite any existing file when True.
"""
hdus = astropy.io.fits.HDUList()
header = astropy.io.fits.Header()
header['TILES'] = self.tiles.tiles_file
header['START'] = self.start_date.isoformat()
header['STOP'] = self.stop_date.isoformat()
header['COMMENT'] = comment
header['EXTNAME'] = 'STATS'
hdus.append(astropy.io.fits.PrimaryHDU())
hdus.append(astropy.io.fits.BinTableHDU(self._data, header=header, name='STATS'))
config = desisurvey.config.Configuration()
fullname = config.get_path(name)
hdus.writeto(fullname, overwrite=overwrite)
log = desiutil.log.get_logger()
log.info('Saved stats to {}'.format(fullname))
if comment:
log.info('Saved with comment "{}".'.format(header['COMMENT']))
def __init__(self,
start_date=None,
stop_date=None,
restore=None,
tiles_file=None,
bgs_footprint=None):
if tiles_file is None:
self.tiles = desisurvey.tiles.Tiles(bgs_footprint=bgs_footprint)
else:
self.tiles = desisurvey.tiles.Tiles(tiles_file=tiles_file,
bgs_footprint=bgs_footprint)
config = desisurvey.config.Configuration()
if start_date is None:
self.start_date = config.first_day()
else:
self.start_date = desisurvey.utils.get_date(start_date)
if stop_date is None:
self.stop_date = config.last_day()
else:
self.stop_date = desisurvey.utils.get_date(stop_date)
self.num_nights = (self.stop_date - self.start_date).days
if self.num_nights <= 0:
raise ValueError('Expected start_date < stop_date.')
# Build our internal array.
dtype = []
for name in 'MJD', 'tsched', :
dtype.append((name, np.float))
nprograms = len(self.tiles.PROGRAMS)
for name in 'topen', 'tdead', :
dtype.append((name, np.float, (nprograms, )))
for name in 'tscience', 'tsetup', 'tsplit', :
dtype.append((name, np.float, (self.tiles.npasses, )))
for name in 'completed', 'nexp', 'nsetup', 'nsplit', 'nsetup_abort', 'nsplit_abort', :
dtype.append((name, np.int32, (self.tiles.npasses, )))
self._data = np.zeros(self.num_nights, dtype)
if restore is not None:
# Restore array contents from a FITS file.
fullname = config.get_path(restore)
with astropy.io.fits.open(fullname, memmap=None) as hdus:
header = hdus[1].header
comment = header['COMMENT']
if header['TILES'] != self.tiles.tiles_file:
raise ValueError('Header mismatch for TILES.')
if header['START'] != self.start_date.isoformat():
raise ValueError('Header mismatch for START.')
if header['STOP'] != self.stop_date.isoformat():
raise ValueError('Header mismatch for STOP.')
self._data[:] = hdus['STATS'].data
log = desiutil.log.get_logger()
log.info('Restored stats from {}'.format(fullname))
if comment:
log.info(' Comment: "{}".'.format(comment))
else:
# Initialize local-noon MJD timestamp for each night.
first_noon = desisurvey.utils.local_noon_on_date(
self.start_date).mjd
self._data['MJD'] = first_noon + np.arange(self.num_nights)
def update(iframe):
if (iframe + 1) % args.log_interval == 0:
log.info('Drawing frame {0}/{1}.'.format(iframe + 1, nframes))
if args.nightly:
while not animator.draw_exposure(iexp[0], nightly=True):
iexp[0] += 1
else:
animator.draw_exposure(iexp=iframe, nightly=False)
return animator.artists
def update(iframe):
if (iframe + 1) % args.log_interval == 0:
log.info('Drawing frame {0}/{1}.'
.format(iframe + 1, nframes))
if args.nightly:
while not animator.draw_exposure(iexp[0], nightly=True):
iexp[0] += 1
else:
animator.draw_exposure(iexp=iframe, nightly=False)
return animator.artists
def load(name, extra_nexp=0):
config = desisurvey.config.Configuration()
name = config.get_path(name)
with astropy.io.fits.open(name) as hdus:
header = hdus[0].header
comment = header['COMMENT']
nexp = header['NEXP']
max_nexp = nexp + extra_nexp
explist = ExposureList(max_nexp=max_nexp)
explist._exposures[:nexp] = hdus['EXPOSURES'].data
explist._tiledata[:] = hdus['TILEDATA'].data
log = desiutil.log.get_logger()
log.info('Loaded {} exposures from {}'.format(nexp, name))
if comment:
log.info('Loaded with comment "{}".'.format(comment))
return explist
def __init__(self, start_date=None, stop_date=None, restore=None):
self.tiles = desisurvey.tiles.Tiles()
config = desisurvey.config.Configuration()
if start_date is None:
self.start_date = config.first_day()
else:
self.start_date = desisurvey.utils.get_date(start_date)
if stop_date is None:
self.stop_date = config.last_day()
else:
self.stop_date = desisurvey.utils.get_date(stop_date)
self.num_nights = (self.stop_date - self.start_date).days
if self.num_nights <= 0:
raise ValueError('Expected start_date < stop_date.')
# Build our internal array.
dtype = []
for name in 'MJD', 'tsched',:
dtype.append((name, np.float))
nprograms = len(self.tiles.PROGRAMS)
for name in 'topen', 'tdead',:
dtype.append((name, np.float, (nprograms,)))
for name in 'tscience', 'tsetup', 'tsplit',:
dtype.append((name, np.float, (self.tiles.npasses,)))
for name in 'completed', 'nexp', 'nsetup', 'nsplit', 'nsetup_abort', 'nsplit_abort',:
dtype.append((name, np.int32, (self.tiles.npasses,)))
self._data = np.zeros(self.num_nights, dtype)
if restore is not None:
# Restore array contents from a FITS file.
fullname = config.get_path(restore)
with astropy.io.fits.open(fullname, memmap=None) as hdus:
header = hdus[1].header
comment = header['COMMENT']
if header['TILES'] != self.tiles.tiles_file:
raise ValueError('Header mismatch for TILES.')
if header['START'] != self.start_date.isoformat():
raise ValueError('Header mismatch for START.')
if header['STOP'] != self.stop_date.isoformat():
raise ValueError('Header mismatch for STOP.')
self._data[:] = hdus['STATS'].data
log = desiutil.log.get_logger()
log.info('Restored stats from {}'.format(fullname))
if comment:
log.info(' Comment: "{}".'.format(comment))
else:
# Initialize local-noon MJD timestamp for each night.
first_noon = desisurvey.utils.local_noon_on_date(self.start_date).mjd
self._data['MJD'] = first_noon + np.arange(self.num_nights)
def main(args=None):
'''
TODO: document
Note: this bypasses specsim since we don't have an arclamp model in
surface brightness units; we only have electrons on the CCD
'''
import desiutil.log
log = desiutil.log.get_logger()
from desiutil.iers import freeze_iers
freeze_iers()
if isinstance(args, (list, tuple, type(None))):
args = parse(args)
log.info('reading arc data from {}'.format(args.arcfile))
arcdata = astropy.table.Table.read(args.arcfile)
wave, phot, fibermap = \
desisim.simexp.simarc(arcdata, nspec=args.nspec, nonuniform=args.nonuniform)
log.info('Writing {}'.format(args.fibermap))
fibermap.meta['NIGHT'] = args.night
fibermap.meta['EXPID'] = args.expid
fibermap.meta['EXTNAME'] = 'FIBERMAP'
fibermap.write(args.fibermap, overwrite=args.clobber)
#- TODO: explain bypassing desisim.io.write_simspec
header = fits.Header()
desiutil.depend.add_dependencies(header)
header['EXPID'] = args.expid
header['NIGHT'] = args.night
header['FLAVOR'] = 'arc'
header['DOSVER'] = 'SIM'
header['EXPTIME'] = 5 #- TODO: add exptime support
#- TODO: DATE-OBS on night instead of now
tx = astropy.time.Time(datetime.datetime(*time.gmtime()[0:6]))
header['DATE-OBS'] = tx.utc.isot
desisim.io.write_simspec_arc(args.simspec,
wave,
phot,
header,
fibermap,
overwrite=args.clobber)
def main(args=None):
'''
Generates a new flat exposure; see newflat --help for usage options
'''
import desiutil.log
log = desiutil.log.get_logger()
from desiutil.iers import freeze_iers
freeze_iers()
if isinstance(args, (list, tuple, type(None))):
args = parse(args)
sim, fibermap = \
desisim.simexp.simflat(args.flatfile, nspec=args.nspec, nonuniform=args.nonuniform)
log.info('Writing {}'.format(args.fibermap))
fibermap.meta['NIGHT'] = args.night
fibermap.meta['EXPID'] = args.expid
fibermap.meta['EXTNAME'] = 'FIBERMAP'
fibermap.write(args.fibermap, overwrite=args.clobber)
header = fits.Header()
desiutil.depend.add_dependencies(header)
header['EXPID'] = args.expid
header['NIGHT'] = args.night
header['FLAVOR'] = 'flat'
header['DOSVER'] = 'SIM'
#- Set calibrations as happening at 15:00 AZ local time = 22:00 UTC
year = int(args.night[0:4])
month = int(args.night[4:6])
day = int(args.night[6:8])
tx = astropy.time.Time(datetime.datetime(year, month, day, 22, 0, 0))
header['DATE-OBS'] = tx.utc.isot
#- metadata truth and obs dictionary are None
desisim.io.write_simspec(sim,
None,
fibermap,
None,
args.expid,
args.night,
filename=args.simspec,
header=header,
overwrite=args.clobber)
def __init__(self, restore=None, max_nexp=60000,
existing_exposures=None):
self.tiles = desisurvey.tiles.get_tiles()
self._exposures = np.empty(max_nexp, dtype=[
('EXPID', np.int32),
('MJD', np.float64),
('EXPTIME', np.float32),
('TILEID', np.int32),
('SNR2FRAC', np.float32),
('DSNR2FRAC', np.float32),
('AIRMASS', np.float32),
('SEEING', np.float32),
('TRANSP', np.float32),
('SKY', np.float32),
])
self._tiledata = np.empty(self.tiles.ntiles, dtype=[
('AVAIL', np.int32),
('PLANNED', np.int32),
('EXPTIME', np.float32),
('SNR2FRAC', np.float32),
('NEXP', np.int32)
])
if restore is not None:
config = desisurvey.config.Configuration()
fullname = config.get_path(restore)
with astropy.io.fits.open(fullname, memmap=False) as hdus:
header = hdus[0].header
comment = header['COMMENT']
self.nexp = header['NEXP']
self._exposures[:self.nexp] = hdus['EXPOSURES'].data
self._tiledata[:] = hdus['TILEDATA'].data
self.initial_night = desisurvey.utils.get_date(header['INITIAL']) if header['INITIAL'] else None
log = desiutil.log.get_logger()
log.info('Restored stats from {}'.format(fullname))
if comment:
log.info(' Comment: "{}".'.format(comment))
else:
self.nexp = 0
self._tiledata['AVAIL'] = -1
self._tiledata['PLANNED'] = -1
self._tiledata['EXPTIME'] = 0.
self._tiledata['SNR2FRAC'] = 0.
self._tiledata['NEXP'] = 0
self.initial_night = None
if existing_exposures is not None:
self.add_existing_exposures(existing_exposures)
def save(self, name='exposures.fits', comment='', overwrite=True):
"""Save exposures to a FITS file with two binary tables.
The saved file size scales linearly with the number of exposures
added so far, and is independent of the memory size of this
object.
Parameters
----------
name : str
File name to write. Will be located in the configuration
output path unless it is an absolute path. Pass the same
name to the constructor's ``restore`` argument to restore
this snapshot.
comment : str
Comment to include in the saved header, for documentation
purposes.
overwrite : bool
Silently overwrite any existing file when True.
"""
hdus = astropy.io.fits.HDUList()
header = astropy.io.fits.Header()
header['TILES'] = self.tiles.tiles_file
header['NEXP'] = self.nexp
header['COMMENT'] = comment
header['INITIAL'] = self.initial_night.isoformat(
) if self.initial_night else ''
header['EXTNAME'] = 'META'
hdus.append(astropy.io.fits.PrimaryHDU(header=header))
hdus.append(
astropy.io.fits.BinTableHDU(self._exposures[:self.nexp],
name='EXPOSURES'))
hdus.append(
astropy.io.fits.BinTableHDU(self._tiledata, name='TILEDATA'))
config = desisurvey.config.Configuration()
name = config.get_path(name)
hdus.writeto(name, overwrite=overwrite)
log = desiutil.log.get_logger()
log.info('Saved {} exposures to {}'.format(self.nexp, name))
if comment:
log.info('Saved with comment "{}".'.format(header['COMMENT']))
def main(args=None):
'''
TODO: document
Note: this bypasses specsim since we don't have an arclamp model in
surface brightness units; we only have electrons on the CCD
'''
import desiutil.log
log = desiutil.log.get_logger()
if isinstance(args, (list, tuple, type(None))):
args = parse(args)
log.info('reading arc data from {}'.format(args.arcfile))
arcdata = astropy.table.Table.read(args.arcfile)
wave, phot, fibermap = \
desisim.simexp.simarc(arcdata, nspec=args.nspec, nonuniform=args.nonuniform)
log.info('Writing {}'.format(args.fibermap))
fibermap.meta['NIGHT'] = args.night
fibermap.meta['EXPID'] = args.expid
fibermap.meta['EXTNAME'] = 'FIBERMAP'
fibermap.write(args.fibermap, overwrite=args.clobber)
#- TODO: explain bypassing desisim.io.write_simspec
header = fits.Header()
desiutil.depend.add_dependencies(header)
header['EXPID'] = args.expid
header['NIGHT'] = args.night
header['FLAVOR'] = 'arc'
header['DOSVER'] = 'SIM'
header['EXPTIME'] = 5 #- TODO: add exptime support
#- TODO: DATE-OBS on night instead of now
tx = astropy.time.Time(datetime.datetime(*time.gmtime()[0:6]))
header['DATE-OBS'] = tx.utc.isot
desisim.io.write_simspec_arc(args.simspec, wave, phot, header, fibermap, overwrite=args.clobber)
def save(self, name='stats.fits', comment='', overwrite=True):
"""Save a snapshot of these statistics as a binary FITS table.
The saved file size is ~800 Kb.
Parameters
----------
name : str
File name to write. Will be located in the configuration
output path unless it is an absolute path. Pass the same
name to the constructor's ``restore`` argument to restore
this snapshot.
comment : str
Comment to include in the saved header, for documentation
purposes.
overwrite : bool
Silently overwrite any existing file when True.
"""
hdus = astropy.io.fits.HDUList()
header = astropy.io.fits.Header()
header['TILES'] = self.tiles.tiles_file
header['START'] = self.start_date.isoformat()
header['STOP'] = self.stop_date.isoformat()
header['COMMENT'] = comment
header['EXTNAME'] = 'STATS'
hdus.append(astropy.io.fits.PrimaryHDU())
hdus.append(
astropy.io.fits.BinTableHDU(self._data,
header=header,
name='STATS'))
config = desisurvey.config.Configuration()
fullname = config.get_path(name)
hdus.writeto(fullname, overwrite=overwrite)
log = desiutil.log.get_logger()
log.info('Saved stats to {}'.format(fullname))
if comment:
log.info('Saved with comment "{}".'.format(header['COMMENT']))
def main(args=None):
'''
Generates a new flat exposure; see newflat --help for usage options
'''
import desiutil.log
log = desiutil.log.get_logger()
if isinstance(args, (list, tuple, type(None))):
args = parse(args)
sim, fibermap = \
desisim.simexp.simflat(args.flatfile, nspec=args.nspec, nonuniform=args.nonuniform)
log.info('Writing {}'.format(args.fibermap))
fibermap.meta['NIGHT'] = args.night
fibermap.meta['EXPID'] = args.expid
fibermap.meta['EXTNAME'] = 'FIBERMAP'
fibermap.write(args.fibermap, overwrite=args.clobber)
header = fits.Header()
desiutil.depend.add_dependencies(header)
header['EXPID'] = args.expid
header['NIGHT'] = args.night
header['FLAVOR'] = 'flat'
header['DOSVER'] = 'SIM'
#- Set calibrations as happening at 15:00 AZ local time = 22:00 UTC
year = int(args.night[0:4])
month = int(args.night[4:6])
day = int(args.night[6:8])
tx = astropy.time.Time(datetime.datetime(year, month, day, 22, 0, 0))
header['DATE-OBS'] = tx.utc.isot
#- metadata truth and obs dictionary are None
desisim.io.write_simspec(sim, None, fibermap, None, args.expid, args.night,
filename=args.simspec, header=header, overwrite=args.clobber)
def get_ephem(use_cache=True, write_cache=True):
"""Return tabulated ephemerides for (START_DATE,STOP_DATE).
The pyephem module must be installed to calculate ephemerides,
but is not necessary when a FITS file of precalcuated data is
available.
Parameters
----------
use_cache : bool
Use cached ephemerides from memory or disk if possible
when True. Otherwise, always calculate from scratch.
write_cache : bool
When True, write a generated table so it is available for
future invocations. Writing only takes place when a
cached object is not available or ``use_cache`` is False.
Returns
-------
Ephemerides
Object with tabulated ephemerides for (START_DATE,STOP_DATE).
"""
global _ephem
# Freeze IERS table for consistent results.
desisurvey.utils.freeze_iers()
# Use standardized string representation of dates.
start_iso = START_DATE.isoformat()
stop_iso = STOP_DATE.isoformat()
range_iso = '({},{})'.format(start_iso, stop_iso)
log = desiutil.log.get_logger()
# First check for a cached object in memory.
if use_cache and _ephem is not None:
if _ephem.start_date != START_DATE or _ephem.stop_date != STOP_DATE:
raise RuntimeError('START_DATE, STOP_DATE have changed.')
log.debug('Returning cached ephemerides for {}.'.format(range_iso))
return _ephem
# Next check for a FITS file on disk.
config = desisurvey.config.Configuration()
filename = config.get_path('ephem_{}_{}.fits'.format(start_iso, stop_iso))
if use_cache and os.path.exists(filename):
# Save restored object in memory.
_ephem = Ephemerides(START_DATE, STOP_DATE, restore=filename)
log.info('Restored ephemerides for {} from {}.'
.format(range_iso, filename))
return _ephem
# Finally, create new ephemerides and save in the memory cache.
log.info('Building ephemerides for {}...'.format(range_iso))
_ephem = Ephemerides(START_DATE, STOP_DATE)
if write_cache:
# Save the tabulated ephemerides to disk.
_ephem._table.write(filename, overwrite=True)
log.info('Saved ephemerides for {} to {}'.format(range_iso, filename))
return _ephem
def get_ephem(use_cache=True, write_cache=True):
"""Return tabulated ephemerides for (START_DATE,STOP_DATE).
The pyephem module must be installed to calculate ephemerides,
but is not necessary when a FITS file of precalcuated data is
available.
Parameters
----------
use_cache : bool
Use cached ephemerides from memory or disk if possible
when True. Otherwise, always calculate from scratch.
write_cache : bool
When True, write a generated table so it is available for
future invocations. Writing only takes place when a
cached object is not available or ``use_cache`` is False.
Returns
-------
Ephemerides
Object with tabulated ephemerides for (START_DATE,STOP_DATE).
"""
global _ephem
# Freeze IERS table for consistent results.
desisurvey.utils.freeze_iers()
# Use standardized string representation of dates.
start_iso = START_DATE.isoformat()
stop_iso = STOP_DATE.isoformat()
range_iso = '({},{})'.format(start_iso, stop_iso)
log = desiutil.log.get_logger()
# First check for a cached object in memory.
if use_cache and _ephem is not None:
if _ephem.start_date != START_DATE or _ephem.stop_date != STOP_DATE:
raise RuntimeError('START_DATE, STOP_DATE have changed.')
log.debug('Returning cached ephemerides for {}.'.format(range_iso))
return _ephem
# Next check for a FITS file on disk.
config = desisurvey.config.Configuration()
filename = config.get_path('ephem_{}_{}.fits'.format(start_iso, stop_iso))
if use_cache and os.path.exists(filename):
# Save restored object in memory.
_ephem = Ephemerides(START_DATE, STOP_DATE, restore=filename)
log.info('Restored ephemerides for {} from {}.'
.format(range_iso, filename))
return _ephem
# Finally, create new ephemerides and save in the memory cache.
log.info('Building ephemerides for {}...'.format(range_iso))
_ephem = Ephemerides(START_DATE, STOP_DATE)
if write_cache:
# Save the tabulated ephemerides to disk.
_ephem._table.write(filename, overwrite=True)
log.info('Saved ephemerides for {} to {}'.format(range_iso, filename))
return _ephem
def update_iers(save_name='iers_frozen.ecsv', num_avg=1000):
"""Update the IERS table used by astropy time, coordinates.
Downloads the current IERS-A table, replaces the last entry (which is
repeated for future times) with the average of the last ``num_avg``
entries, and saves the table in ECSV format.
This should only be called every few months, e.g., with major releases.
The saved file should then be copied to this package's data/ directory
and committed to the git repository.
Requires a network connection in order to download the current IERS-A table.
Prints information about the update process.
The :func:`desisurvey.utils.plot_iers` function is useful for inspecting
IERS tables and how they are extrapolated to DESI survey dates.
Parameters
----------
save_name : str
Name where frozen IERS table should be saved. Must end with the
.ecsv extension.
num_avg : int
Number of rows from the end of the current table to average and
use for calculating UT1-UTC offsets and polar motion at times
beyond the table.
"""
log = desiutil.log.get_logger()
# Validate the save_name extension.
_, ext = os.path.splitext(save_name)
if ext != '.ecsv':
raise ValueError('Expected .ecsv extension for {0}.'.format(save_name))
# Download the latest IERS_A table
iers = astropy.utils.iers.IERS_A.open(astropy.utils.iers.IERS_A_URL)
last = astropy.time.Time(iers['MJD'][-1], format='mjd').datetime
log.info('Updating to current IERS-A table with coverage up to {0}.'
.format(last.date()))
# Loop over the columns used by the astropy IERS routines.
for name in 'UT1_UTC', 'PM_x', 'PM_y':
# Replace the last entry with the mean of recent samples.
mean_value = np.mean(iers[name][-num_avg:].value)
unit = iers[name].unit
iers[name][-1] = mean_value * unit
log.info('Future {0:7s} = {1:.3}'.format(name, mean_value * unit))
# Strip the original table metadata since ECSV cannot handle it.
# We only need a single keyword that is checked by IERS_Auto.open().
iers.meta = dict(data_url='frozen')
# Save the table. The IERS-B table provided with astropy uses the
# ascii.cds format but astropy cannot write this format.
iers.write(save_name, format='ascii.ecsv', overwrite=True)
log.info('Wrote updated table to {0}.'.format(save_name))
def run_preproc(rawfile, outdir, fibermap=None, ncpu=None, cameras=None):
'''Runs preproc on the input raw data file, outputting to outdir
Args:
rawfile: input desi-EXPID.fits.fz raw data file
outdir: directory to write preproc-CAM-EXPID.fits files
Options:
fibermap : path to fibermap-EXPID.fits file
ncpu: number of CPU cores to use for parallelism; serial if ncpu<=1
cameras: list of cameras to process; default all found in rawfile
Returns header of HDU 0 of the input raw data file
'''
if not os.path.exists(rawfile):
raise ValueError("{} doesn't exist".format(rawfile))
log = desiutil.log.get_logger()
if not os.path.isdir(outdir):
log.info('Creating {}'.format(outdir))
os.makedirs(outdir, exist_ok=True)
if cameras is None:
cameras = which_cameras(rawfile)
header = fitsio.read_header(rawfile, 0)
arglist = list()
for camera in cameras:
args = [
'--infile', rawfile, '--outdir', outdir, '--fibermap', fibermap,
'--cameras', camera
]
arglist.append(args)
ncpu = min(len(arglist), get_ncpu(ncpu))
if ncpu > 1:
log.info(
'Running preproc in parallel on {} cores for {} cameras'.format(
ncpu, len(cameras)))
pool = mp.Pool(ncpu)
pool.map(desispec.scripts.preproc.main, arglist)
pool.close()
pool.join()
else:
log.info('Running preproc serially for {} cameras'.format(
len(cameras)))
for args in arglist:
desispec.scripts.preproc.main(args)
return header
def get_tiles(tiles_file=None, use_cache=True, write_cache=True):
"""Return a Tiles object with optional caching.
You should normally always use the default arguments to ensure
that tiles are defined consistently and efficiently between
different classes.
Parameters
----------
tiles_file : str or None
Use the specified name to override config.tiles_file.
use_cache : bool
Use tiles previously cached in memory when True.
Otherwise, (re)load tiles from disk.
write_cache : bool
If tiles need to be loaded from disk with this call,
save them in a memory cache for future calls.
"""
global _cached_tiles
log = desiutil.log.get_logger()
config = desisurvey.config.Configuration()
tiles_file = tiles_file or config.tiles_file()
if use_cache and tiles_file in _cached_tiles:
tiles = _cached_tiles[tiles_file]
log.debug('Using cached tiles for "{}".'.format(tiles_file))
else:
tiles = Tiles(tiles_file)
log.info('Initialized tiles from "{}".'.format(tiles_file))
for pname in Tiles.PROGRAMS:
pinfo = []
for passnum in tiles.program_passes[pname]:
pinfo.append('{}({})'.format(passnum,
tiles.pass_ntiles[passnum]))
log.info('{:6s} passes(tiles): {}.'.format(pname,
', '.join(pinfo)))
if write_cache:
_cached_tiles[tiles_file] = tiles
else:
log.info('Tiles not cached for "{}".'.format(tiles_file))
return tiles
def get_tiles(tiles_file=None, use_cache=True, write_cache=True):
"""Return a Tiles object with optional caching.
You should normally always use the default arguments to ensure
that tiles are defined consistently and efficiently between
different classes.
Parameters
----------
tiles_file : str or None
Use the specified name to override config.tiles_file.
use_cache : bool
Use tiles previously cached in memory when True.
Otherwise, (re)load tiles from disk.
write_cache : bool
If tiles need to be loaded from disk with this call,
save them in a memory cache for future calls.
"""
global _cached_tiles
log = desiutil.log.get_logger()
config = desisurvey.config.Configuration()
tiles_file = tiles_file or config.tiles_file()
if use_cache and tiles_file in _cached_tiles:
tiles = _cached_tiles[tiles_file]
log.debug('Using cached tiles for "{}".'.format(tiles_file))
else:
tiles = Tiles(tiles_file)
log.info('Initialized tiles from "{}".'.format(tiles_file))
for pname in Tiles.PROGRAMS:
pinfo = []
for passnum in tiles.program_passes[pname]:
pinfo.append('{}({})'.format(passnum, tiles.pass_ntiles[passnum]))
log.info('{:6s} passes(tiles): {}.'.format(pname, ', '.join(pinfo)))
if write_cache:
_cached_tiles[tiles_file] = tiles
else:
log.info('Tiles not cached for "{}".'.format(tiles_file))
return tiles
def run_qproc(rawfile, outdir, ncpu=None, cameras=None):
'''
Determine the obstype of the rawfile, and run qproc with appropriate options
Args:
rawfile: input desi-EXPID.fits.fz raw data file
outdir: directory to write qproc-CAM-EXPID.fits files
Options:
ncpu: number of CPU cores to use for parallelism; serial if ncpu<=1
cameras: list of cameras to process; default all found in rawfile
Returns header of HDU 0 of the input raw data file, plus dictionary of return codes for each qproc process run.
'''
log = desiutil.log.get_logger()
if not os.path.isdir(outdir):
log.info('Creating {}'.format(outdir))
os.makedirs(outdir, exist_ok=True)
hdr = fitsio.read_header(rawfile, 0)
if ('OBSTYPE' not in hdr) and ('FLAVOR' not in hdr):
log.warning(
"no obstype nor flavor keyword in first hdu header, moving to the next one"
)
try:
hdr = fitsio.read_header(rawfile, 1)
except OSError as err:
log.error("fitsio error reading HDU 1, trying 2 then giving up")
hdr = fitsio.read_header(rawfile, 2)
try:
if 'OBSTYPE' in hdr:
obstype = hdr['OBSTYPE'].rstrip().upper()
else:
log.warning('Use FLAVOR instead of missing OBSTYPE')
obstype = hdr['FLAVOR'].rstrip().upper()
night, expid = get_night_expid_header(hdr)
except KeyError as e:
log.error(str(e))
raise (e)
#- copy coordfile to new folder for pos accuracy
indir = os.path.abspath(os.path.dirname(rawfile))
coord_infile = '{}/coordinates-{:08d}.fits'.format(indir, expid)
coord_outfile = '{}/coordinates-{:08d}.fits'.format(outdir, expid)
print(coord_infile)
if os.path.isfile(coord_infile):
print('copying coordfile')
copyfile(coord_infile, coord_outfile)
else:
log.warning('No coordinate file for positioner accuracy')
#- HACK: Workaround for data on 20190626/27 that have blank NIGHT keywords
#- Note: get_night_expid_header(hdr) should take care of this now, but
#- this is left in for robustness just in case
if night == ' ' or night is None:
log.error(
'Correcting blank NIGHT keyword based upon directory structure')
#- /path/to/NIGHT/EXPID/rawfile.fits
night = os.path.basename(
os.path.dirname(os.path.dirname(os.path.abspath(rawfile))))
if re.match('20\d{6}', night):
log.info('Setting NIGHT to {}'.format(night))
else:
raise RuntimeError('Unable to derive NIGHT for {}'.format(rawfile))
cmdlist = list()
loglist = list()
msglist = list()
rawcameras = which_cameras(rawfile)
if cameras is None:
cameras = rawcameras
elif len(set(cameras) - set(rawcameras)) > 0:
missing_cameras = set(cameras) - set(rawcameras)
for cam in sorted(missing_cameras):
log.error('{} missing camera {}'.format(os.path.basename(rawfile),
cam))
cameras = sorted(set(cameras) & set(rawcameras))
for camera in cameras:
outfiles = dict(
rawfile=rawfile,
fibermap='{}/fibermap-{:08d}.fits'.format(outdir, expid),
logfile='{}/qproc-{}-{:08d}.log'.format(outdir, camera, expid),
outdir=outdir,
camera=camera)
cmd = "desi_qproc -i {rawfile} --fibermap {fibermap} --auto --auto-output-dir {outdir} --cam {camera}".format(
**outfiles)
cmdlist.append(cmd)
loglist.append(outfiles['logfile'])
msglist.append('qproc {}/{} {}'.format(night, expid, camera))
ncpu = min(len(cmdlist), get_ncpu(ncpu))
if ncpu > 1 and len(cameras) > 1:
log.info('Running qproc in parallel on {} cores for {} cameras'.format(
ncpu, len(cameras)))
pool = mp.Pool(ncpu)
errs = pool.starmap(runcmd, zip(cmdlist, loglist, msglist))
pool.close()
pool.join()
else:
errs = []
log.info('Running qproc serially for {} cameras'.format(len(cameras)))
for cmd, logfile, msg in zip(cmdlist, loglist, msglist):
err = runcmd(cmd, logfile, msg)
errs.append(err)
errorcodes = dict()
for err in errs:
for key in err.keys():
errorcodes[key] = err[key]
jsonfile = '{}/errorcodes-{:08d}.txt'.format(outdir, expid)
with open(jsonfile, 'w') as outfile:
json.dump(errorcodes, outfile)
print('Wrote {}'.format(jsonfile))
return hdr
def make_plots(infile,
basedir,
preprocdir=None,
logdir=None,
rawdir=None,
cameras=None):
'''Make plots for a single exposure
Args:
infile: input QA fits file with HDUs like PER_AMP, PER_FIBER, ...
basedir: write output HTML to basedir/NIGHT/EXPID/
Options:
preprocdir: directory to where the "preproc-*-*.fits" are located. If
not provided, function will NOT generate any image files from any
preproc fits file.
logdir: directory to where the "qproc-*-*.log" are located. If
not provided, function will NOT display any logfiles.
rawdir: directory to where the raw data files are located, including
"guide-rois-*.fits" and "centroid-*.json" files, are located. If
not provided, the function will not plot the guide plots.
cameras: list of cameras (strings) to generate image files of. If not
provided, will generate a cameras list from parcing through the
preproc fits files in the preprocdir
'''
from nightwatch.webpages import amp as web_amp
from nightwatch.webpages import camfiber as web_camfiber
from nightwatch.webpages import camera as web_camera
from nightwatch.webpages import summary as web_summary
from nightwatch.webpages import lastexp as web_lastexp
from nightwatch.webpages import guide as web_guide
from nightwatch.webpages import guideimage as web_guideimage
from nightwatch.webpages import placeholder as web_placeholder
from . import io
log = desiutil.log.get_logger()
qadata = io.read_qa(infile)
header = qadata['HEADER']
night = header['NIGHT']
expid = header['EXPID']
#- Early data have wrong NIGHT in header; check by hand
#- YEARMMDD/EXPID/infile
dirnight = os.path.basename(os.path.dirname(os.path.dirname(infile)))
if re.match('20\d{6}', dirnight) and dirnight != str(night):
log.warning('Correcting {} header night {} to {}'.format(
infile, night, dirnight))
night = int(dirnight)
header['NIGHT'] = night
#- Create output exposures plot directory if needed
expdir = os.path.join(basedir, str(night), '{:08d}'.format(expid))
if not os.path.isdir(expdir):
log.info('Creating {}'.format(expdir))
os.makedirs(expdir, exist_ok=True)
if 'PER_AMP' in qadata:
htmlfile = '{}/qa-amp-{:08d}.html'.format(expdir, expid)
pc = web_amp.write_amp_html(htmlfile, qadata['PER_AMP'], header)
print('Wrote {}'.format(htmlfile))
else:
htmlfile = '{}/qa-amp-{:08d}.html'.format(expdir, expid)
pc = web_placeholder.write_placeholder_html(htmlfile, header,
"PER_AMP")
htmlfile = '{}/qa-camfiber-{:08d}.html'.format(expdir, expid)
if 'PER_CAMFIBER' in qadata:
try:
pc = web_camfiber.write_camfiber_html(htmlfile,
qadata['PER_CAMFIBER'],
header)
print('Wrote {}'.format(htmlfile))
except Exception as err:
web_placeholder.handle_failed_plot(htmlfile, header,
"PER_CAMFIBER")
else:
pc = web_placeholder.write_placeholder_html(htmlfile, header,
"PER_CAMFIBER")
htmlfile = '{}/qa-camera-{:08d}.html'.format(expdir, expid)
if 'PER_CAMERA' in qadata:
try:
pc = web_camera.write_camera_html(htmlfile, qadata['PER_CAMERA'],
header)
print('Wrote {}'.format(htmlfile))
except Exception as err:
web_placeholder.handle_failed_plot(htmlfile, header, "PER_CAMERA")
else:
pc = web_placeholder.write_placeholder_html(htmlfile, header,
"PER_CAMERA")
htmlfile = '{}/qa-summary-{:08d}.html'.format(expdir, expid)
web_summary.write_summary_html(htmlfile, qadata, preprocdir)
print('Wrote {}'.format(htmlfile))
#- Note: last exposure goes in basedir, not expdir=basedir/NIGHT/EXPID
htmlfile = '{}/qa-lastexp.html'.format(basedir)
web_lastexp.write_lastexp_html(htmlfile, qadata, preprocdir)
print('Wrote {}'.format(htmlfile))
if rawdir:
#- plot guide metric plots
try:
guidedata = io.get_guide_data(night, expid, rawdir)
htmlfile = '{}/qa-guide-{:08d}.html'.format(expdir, expid)
web_guide.write_guide_html(htmlfile, header, guidedata)
print('Wrote {}'.format(htmlfile))
except (FileNotFoundError, OSError, IOError):
print('Unable to find guide data, not plotting guide plots')
htmlfile = '{}/qa-guide-{:08d}.html'.format(expdir, expid)
pc = web_placeholder.write_placeholder_html(
htmlfile, header, "GUIDING")
#- plot guide image movies
try:
htmlfile = '{expdir}/guide-image-{expid:08d}.html'.format(
expdir=expdir, expid=expid)
image_data = io.get_guide_images(night, expid, rawdir)
web_guideimage.write_guide_image_html(image_data, htmlfile, night,
expid)
print('Wrote {}'.format(htmlfile))
except (FileNotFoundError, OSError, IOError):
print('Unable to find guide data, not plotting guide image plots')
htmlfile = '{expdir}/guide-image-{expid:08d}.html'.format(
expdir=expdir, expid=expid)
pc = web_placeholder.write_placeholder_html(
htmlfile, header, "GUIDE_IMAGES")
#- regardless of if logdir or preprocdir, identifying failed qprocs by comparing
#- generated preproc files to generated logfiles
qproc_fails = []
if cameras is None:
cameras = []
import glob
for preprocfile in glob.glob(
os.path.join(preprocdir, 'preproc-*-*.fits')):
cameras += [os.path.basename(preprocfile).split('-')[1]]
log_cams = []
log_outputs = [
i for i in os.listdir(logdir) if re.match(r'qproc.*\.log', i)
]
for log_output in log_outputs:
l_cam = log_output.split("-")[1]
log_cams += [l_cam]
if l_cam not in cameras:
qproc_fails.append(l_cam)
from nightwatch.webpages import plotimage as web_plotimage
if (preprocdir is not None):
#- plot preprocessed images
downsample = 4
ncpu = get_ncpu(None)
pinput = os.path.join(preprocdir, "preproc-{}-{:08d}.fits")
output = os.path.join(expdir, "preproc-{}-{:08d}-4x.html")
argslist = [(pinput.format(cam, expid), output.format(cam, expid),
downsample, night) for cam in cameras]
if ncpu > 1:
pool = mp.Pool(ncpu)
pool.starmap(web_plotimage.write_image_html, argslist)
pool.close()
pool.join()
else:
for args in argslist:
web_plotimage.write_image_html(*args)
#- plot preproc nav table
navtable_output = '{}/qa-amp-{:08d}-preproc_table.html'.format(
expdir, expid)
web_plotimage.write_preproc_table_html(preprocdir, night, expid,
downsample, navtable_output)
if (logdir is not None):
#- plot logfiles
log.debug('Log directory: {}'.format(logdir))
error_colors = dict()
for log_cam in log_cams:
qinput = os.path.join(logdir,
"qproc-{}-{:08d}.log".format(log_cam, expid))
output = os.path.join(
expdir, "qproc-{}-{:08d}-logfile.html".format(log_cam, expid))
log.debug('qproc log: {}'.format(qinput))
e = web_summary.write_logfile_html(qinput, output, night)
error_colors[log_cam] = e
#- plot logfile nav table
htmlfile = '{}/qa-summary-{:08d}-logfiles_table.html'.format(
expdir, expid)
web_summary.write_logtable_html(htmlfile,
logdir,
night,
expid,
available=log_cams,
error_colors=error_colors)
def initialize(ephem, start_date=None, stop_date=None, step_size=5.0,
healpix_nside=16, output_name='scheduler.fits'):
"""Calculate exposure-time factors over a grid of times and pointings.
Takes about 9 minutes to run and writes a 1.3Gb output file with the
default parameters.
Requires that healpy is installed.
Parameters
----------
ephem : desisurvey.ephem.Ephemerides
Tabulated ephemerides data to use for planning.
start_date : date or None
Survey planning starts on the evening of this date. Must be convertible
to a date using :func:`desisurvey.utils.get_date`. Use the first night
of the ephemerides when None.
stop_date : date or None
Survey planning stops on the morning of this date. Must be convertible
to a date using :func:`desisurvey.utils.get_date`. Use the first night
of the ephemerides when None.
step_size : :class:`astropy.units.Quantity`
Exposure-time factors are tabulated at this interval during each night.
healpix_nside : int
Healpix NSIDE parameter to use for binning the sky. Must be a power of
two. Values larger than 16 will lead to holes in the footprint with
the current implementation.
output_name : str
Name of the FITS output file where results are saved. A relative path
refers to the :meth:`configuration output path
<desisurvey.config.Configuration.get_path>`.
"""
import healpy
if not isinstance(step_size, u.Quantity):
step_size = step_size * u.min
log = desiutil.log.get_logger()
# Freeze IERS table for consistent results.
desisurvey.utils.freeze_iers()
config = desisurvey.config.Configuration()
output_name = config.get_path(output_name)
start_date = desisurvey.utils.get_date(start_date or config.first_day())
stop_date = desisurvey.utils.get_date(stop_date or config.last_day())
if start_date >= stop_date:
raise ValueError('Expected start_date < stop_date.')
mjd = ephem._table['noon']
sel = ((mjd >= desisurvey.utils.local_noon_on_date(start_date).mjd) &
(mjd < desisurvey.utils.local_noon_on_date(stop_date).mjd))
t = ephem._table[sel]
num_nights = len(t)
# Build a grid of elapsed time relative to local midnight during each night.
midnight = t['noon'] + 0.5
t_edges = desisurvey.ephem.get_grid(step_size)
t_centers = 0.5 * (t_edges[1:] + t_edges[:-1])
num_points = len(t_centers)
# Create an empty HDU0 with header info.
header = astropy.io.fits.Header()
header['START'] = str(start_date)
header['STOP'] = str(stop_date)
header['NSIDE'] = healpix_nside
header['NPOINTS'] = num_points
header['STEP'] = step_size.to(u.min).value
hdus = astropy.io.fits.HDUList()
hdus.append(astropy.io.fits.ImageHDU(header=header))
# Save time grid.
hdus.append(astropy.io.fits.ImageHDU(name='GRID', data=t_edges))
# Load the list of tiles to observe.
tiles = astropy.table.Table(
desimodel.io.load_tiles(onlydesi=True, extra=False,
tilesfile=config.tiles_file()))
# Build the footprint as a healpix map of the requested size.
# The footprint includes any pixel containing at least one tile center.
npix = healpy.nside2npix(healpix_nside)
footprint = np.zeros(npix, bool)
pixels = healpy.ang2pix(
healpix_nside, np.radians(90 - tiles['DEC'].data),
np.radians(tiles['RA'].data))
footprint[np.unique(pixels)] = True
footprint_pixels = np.where(footprint)[0]
num_footprint = len(footprint_pixels)
log.info('Footprint contains {0} pixels.'.format(num_footprint))
# Sort pixels in order of increasing phi + 60deg so that the north and south
# galactic caps are contiguous in the arrays we create below.
pix_theta, pix_phi = healpy.pix2ang(healpix_nside, footprint_pixels)
pix_dphi = np.fmod(pix_phi + np.pi / 3, 2 * np.pi)
sort_order = np.argsort(pix_dphi)
footprint_pixels = footprint_pixels[sort_order]
# Calculate sorted pixel (ra,dec).
pix_theta, pix_phi = healpy.pix2ang(healpix_nside, footprint_pixels)
pix_ra, pix_dec = np.degrees(pix_phi), 90 - np.degrees(pix_theta)
# Record per-tile info needed for planning.
table = astropy.table.Table()
table['tileid'] = tiles['TILEID'].astype(np.int32)
table['ra'] = tiles['RA'].astype(np.float32)
table['dec'] = tiles['DEC'].astype(np.float32)
table['EBV'] = tiles['EBV_MED'].astype(np.float32)
table['pass'] = tiles['PASS'].astype(np.int16)
# Map each tile ID to the corresponding index in our spatial arrays.
mapper = np.zeros(npix, int)
mapper[footprint_pixels] = np.arange(len(footprint_pixels))
table['map'] = mapper[pixels].astype(np.int16)
# Use a small int to identify the program, ordered by sky brightness:
# 1=DARK, 2=GRAY, 3=BRIGHT.
table['program'] = np.full(len(tiles), 4, np.int16)
for i, program in enumerate(('DARK', 'GRAY', 'BRIGHT')):
table['program'][tiles['PROGRAM'] == program] = i + 1
assert np.all(table['program'] > 0)
hdu = astropy.io.fits.table_to_hdu(table)
hdu.name = 'TILES'
hdus.append(hdu)
# Average E(B-V) for all tiles falling into a pixel.
tiles_per_pixel = np.bincount(pixels, minlength=npix)
EBV = np.bincount(pixels, weights=tiles['EBV_MED'], minlength=npix)
EBV[footprint] /= tiles_per_pixel[footprint]
# Calculate dust extinction exposure-time factor.
f_EBV = 1. / desisurvey.etc.dust_exposure_factor(EBV)
# Save HDU with the footprint and static dust exposure map.
table = astropy.table.Table()
table['pixel'] = footprint_pixels
table['dust'] = f_EBV[footprint_pixels]
table['ra'] = pix_ra
table['dec'] = pix_dec
hdu = astropy.io.fits.table_to_hdu(table)
hdu.name = 'STATIC'
hdus.append(hdu)
# Prepare a table of calendar data.
calendar = astropy.table.Table()
calendar['midnight'] = midnight
calendar['monsoon'] = np.zeros(num_nights, bool)
calendar['fullmoon'] = np.zeros(num_nights, bool)
calendar['weather'] = np.zeros(num_nights, np.float32)
# Hardcode annualized average weight.
weather_weights = np.full(num_nights, 0.723)
# Prepare a table of ephemeris data.
etable = astropy.table.Table()
# Program codes ordered by increasing sky brightness:
# 1=DARK, 2=GRAY, 3=BRIGHT, 4=DAYTIME.
etable['program'] = np.full(num_nights * num_points, 4, dtype=np.int16)
etable['moon_frac'] = np.zeros(num_nights * num_points, dtype=np.float32)
etable['moon_ra'] = np.zeros(num_nights * num_points, dtype=np.float32)
etable['moon_dec'] = np.zeros(num_nights * num_points, dtype=np.float32)
etable['moon_alt'] = np.zeros(num_nights * num_points, dtype=np.float32)
etable['zenith_ra'] = np.zeros(num_nights * num_points, dtype=np.float32)
etable['zenith_dec'] = np.zeros(num_nights * num_points, dtype=np.float32)
# Tabulate MJD and apparent LST values for each time step. We don't save
# MJD values since they are cheap to reconstruct from the index, but
# do use them below.
mjd0 = desisurvey.utils.local_noon_on_date(start_date).mjd + 0.5
mjd = mjd0 + np.arange(num_nights)[:, np.newaxis] + t_centers
times = astropy.time.Time(
mjd, format='mjd', location=desisurvey.utils.get_location())
etable['lst'] = times.sidereal_time('apparent').flatten().to(u.deg).value
# Build sky coordinates for each pixel in the footprint.
pix_theta, pix_phi = healpy.pix2ang(healpix_nside, footprint_pixels)
pix_ra, pix_dec = np.degrees(pix_phi), 90 - np.degrees(pix_theta)
pix_sky = astropy.coordinates.ICRS(pix_ra * u.deg, pix_dec * u.deg)
# Initialize exposure factor calculations.
alt, az = np.full(num_points, 90.) * u.deg, np.zeros(num_points) * u.deg
fexp = np.zeros((num_nights * num_points, num_footprint), dtype=np.float32)
vband_extinction = 0.15154
one = np.ones((num_points, num_footprint))
# Loop over nights.
for i in range(num_nights):
night = ephem.get_night(midnight[i])
date = desisurvey.utils.get_date(midnight[i])
if date.day == 1:
log.info('Starting {0} (completed {1}/{2} nights)'
.format(date.strftime('%b %Y'), i, num_nights))
# Initialize the slice of the fexp[] time index for this night.
sl = slice(i * num_points, (i + 1) * num_points)
# Do we expect to observe on this night?
calendar[i]['monsoon'] = desisurvey.utils.is_monsoon(midnight[i])
calendar[i]['fullmoon'] = ephem.is_full_moon(midnight[i])
# Look up expected dome-open fraction due to weather.
calendar[i]['weather'] = weather_weights[i]
# Calculate the program during this night (default is 4=DAYTIME).
mjd = midnight[i] + t_centers
dark, gray, bright = ephem.tabulate_program(mjd)
etable['program'][sl][dark] = 1
etable['program'][sl][gray] = 2
etable['program'][sl][bright] = 3
# Zero the exposure factor whenever we are not oberving.
##fexp[sl] = (dark | gray | bright)[:, np.newaxis]
fexp[sl] = 1.
# Transform the local zenith to (ra,dec).
zenith = desisurvey.utils.get_observer(
times[i], alt=alt, az=az).transform_to(astropy.coordinates.ICRS)
etable['zenith_ra'][sl] = zenith.ra.to(u.deg).value
etable['zenith_dec'][sl] = zenith.dec.to(u.deg).value
# Calculate zenith angles to each pixel in the footprint.
pix_sep = pix_sky.separation(zenith[:, np.newaxis])
# Zero the exposure factor for pixels below the horizon.
visible = pix_sep < 90 * u.deg
fexp[sl][~visible] = 0.
# Calculate the airmass exposure-time penalty.
X = desisurvey.utils.cos_zenith_to_airmass(np.cos(pix_sep[visible]))
fexp[sl][visible] /= desisurvey.etc.airmass_exposure_factor(X)
# Loop over objects we need to avoid.
for name in config.avoid_bodies.keys:
f_obj = desisurvey.ephem.get_object_interpolator(night, name)
# Calculate this object's (dec,ra) path during the night.
obj_dec, obj_ra = f_obj(mjd)
sky_obj = astropy.coordinates.ICRS(
ra=obj_ra[:, np.newaxis] * u.deg,
dec=obj_dec[:, np.newaxis] * u.deg)
# Calculate the separation angles to each pixel in the footprint.
obj_sep = pix_sky.separation(sky_obj)
if name == 'moon':
etable['moon_ra'][sl] = obj_ra
etable['moon_dec'][sl] = obj_dec
# Calculate moon altitude during the night.
moon_alt, _ = desisurvey.ephem.get_object_interpolator(
night, 'moon', altaz=True)(mjd)
etable['moon_alt'][sl] = moon_alt
moon_zenith = (90 - moon_alt[:,np.newaxis]) * u.deg
moon_up = moon_alt > 0
assert np.all(moon_alt[gray] > 0)
# Calculate the moon illuminated fraction during the night.
moon_frac = ephem.get_moon_illuminated_fraction(mjd)
etable['moon_frac'][sl] = moon_frac
# Convert to temporal moon phase.
moon_phase = np.arccos(2 * moon_frac[:,np.newaxis] - 1) / np.pi
# Calculate scattered moon V-band brightness at each pixel.
V = specsim.atmosphere.krisciunas_schaefer(
pix_sep, moon_zenith, obj_sep,
moon_phase, desisurvey.etc._vband_extinction).value
# Estimate the exposure time factor from V.
X = np.dstack((one, np.exp(-V), 1/V, 1/V**2, 1/V**3))
T = X.dot(desisurvey.etc._moonCoefficients)
# No penalty when the moon is below the horizon.
T[moon_alt < 0, :] = 1.
fexp[sl] *= 1. / T
# Veto pointings within avoidance size when the moon is
# above the horizon. Apply Gaussian smoothing to the veto edge.
veto = np.ones_like(T)
dsep = (obj_sep - config.avoid_bodies.moon()).to(u.deg).value
veto[dsep <= 0] = 0.
veto[dsep > 0] = 1 - np.exp(-0.5 * (dsep[dsep > 0] / 3) ** 2)
veto[moon_alt < 0] = 1.
fexp[sl] *= veto
else:
# Lookup the avoidance size for this object.
size = getattr(config.avoid_bodies, name)()
# Penalize the exposure-time with a factor
# 1 - exp(-0.5*(obj_sep/size)**2)
penalty = 1. - np.exp(-0.5 * (obj_sep / size).to(1).value ** 2)
fexp[sl] *= penalty
# Save calendar table.
hdu = astropy.io.fits.table_to_hdu(calendar)
hdu.name = 'CALENDAR'
hdus.append(hdu)
# Save ephemerides table.
hdu = astropy.io.fits.table_to_hdu(etable)
hdu.name = 'EPHEM'
hdus.append(hdu)
# Save dynamic exposure-time factors.
hdus.append(astropy.io.fits.ImageHDU(name='DYNAMIC', data=fexp))
# Finalize the output file.
try:
hdus.writeto(output_name, overwrite=True)
except TypeError:
# astropy < 1.3 uses the now deprecated clobber.
hdus.writeto(output_name, clobber=True)
log.info('Plan initialization saved to {0}'.format(output_name))
def run(self, indir, outfile=None, jsonfile=None):
'''TODO: document'''
log = desiutil.log.get_logger()
log.debug('Running QA in {}'.format(indir))
print('here qa.runner', indir)
preprocfiles = sorted(glob.glob('{}/preproc-*.fits'.format(indir)))
if len(preprocfiles) == 0:
log.error('No preproc files found in {}'.format(indir))
return None
# We can have different obstypes (signal+dark) with calibration data
# obtained with a calibration slit hooked to a single spectrograph.
# So we have to loop over all frames to check if there are science,
# arc, or flat obstypes as guessed by qproc.
qframefiles = sorted(glob.glob('{}/qframe-*.fits'.format(indir)))
if len(qframefiles) == 0: # no qframe so it's either zero or dark
hdr = fitsio.read_header(preprocfiles[0], 0)
if 'OBSTYPE' in hdr:
obstype = hdr['FLAVOR'].strip()
else:
log.warning("Using FLAVOR instead of missing OBSTYPE")
obstype = hdr['FLAVOR'].strip()
else:
obstype = None
log.debug("Reading qframe headers to guess flavor ...")
for qframefile in qframefiles: # look at all of them and prefer arc or flat over dark or zero
hdr = fitsio.read_header(qframefile, 0)
if 'OBSTYPE' in hdr:
this_obstype = hdr['OBSTYPE'].strip().upper()
else:
log.warning("Using FLAVOR instead of missing OBSTYPE")
obstype = hdr['FLAVOR'].strip()
if this_obstype == "ARC" or this_obstype == "FLAT" \
or this_obstype == "TESTARC" or this_obstype == "TESTFLAT" :
obstype = this_obstype
# we use this so we exit the loop
break
elif obstype == None:
obstype = this_obstype
# we stay in the loop in case another frame has another obstype
log.debug('Found OBSTYPE={} files'.format(obstype))
results = dict()
for qa in self.qalist:
if qa.valid_obstype(obstype):
log.info('{} Running {} {}'.format(timestamp(), qa,
qa.output_type))
qa_results = None
try:
qa_results = qa.run(indir)
except Exception as err:
log.warning('{} failed on {} because {}; skipping'.format(
qa, indir, str(err)))
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
del exc_info
#raise(err)
#- TODO: print traceback somewhere useful
if qa_results is not None:
if qa.output_type not in results:
results[qa.output_type] = list()
results[qa.output_type].append(qa_results)
else:
log.debug('Skip {} {} for {}'.format(qa, qa.output_type,
obstype))
#- Combine results for different types of QA
join_keys = dict(
PER_AMP=['NIGHT', 'EXPID', 'SPECTRO', 'CAM', 'AMP'],
PER_CAMERA=['NIGHT', 'EXPID', 'SPECTRO', 'CAM'],
PER_FIBER=['NIGHT', 'EXPID', 'SPECTRO', 'FIBER'],
PER_CAMFIBER=['NIGHT', 'EXPID', 'SPECTRO', 'CAM', 'FIBER'],
PER_SPECTRO=['NIGHT', 'EXPID', 'SPECTRO'],
PER_EXP=['NIGHT', 'EXPID'],
QPROC_STATUS=['NIGHT', 'EXPID', 'SPECTRO', 'CAM'])
if jsonfile is not None:
if os.path.exists(jsonfile):
with open(jsonfile, 'r') as myfile:
json_data = myfile.read()
json_data = json.loads(json_data)
else:
json_data = dict()
rewrite_necessary = False
for key1 in results:
if key1 != "PER_CAMFIBER" and key1.startswith("PER_"):
colnames_lst = results[key1][0].colnames
if key1 in join_keys:
for i in join_keys[key1]:
colnames_lst.remove(i)
if key1 not in json_data:
json_data[key1] = colnames_lst
rewrite_necessary = True
else:
for aspect in colnames_lst:
if aspect not in json_data[key1]:
json_data[key1] += [aspect]
rewrite_necessary = True
if rewrite_necessary:
if os.path.isdir(os.path.dirname(jsonfile)):
with open(jsonfile, 'w') as out:
json.dump(json_data, out)
print('Wrote {}'.format(jsonfile))
for qatype in list(results.keys()):
if len(results[qatype]) == 1:
results[qatype] = results[qatype][0]
else:
tx = results[qatype][0]
for i in range(1, len(results[qatype])):
tx = join(tx,
results[qatype][i],
keys=join_keys[qatype],
join_type='outer')
results[qatype] = tx
#- convert python string to bytes for FITS format compatibility
if 'AMP' in results[qatype].colnames:
results[qatype]['AMP'] = results[qatype]['AMP'].astype('S1')
if 'CAM' in results[qatype].colnames:
results[qatype]['CAM'] = results[qatype]['CAM'].astype('S1')
#- TODO: NIGHT/EXPID/SPECTRO/FIBER int64 -> int32 or int16
#- TODO: metrics from float64 -> float32
if outfile is not None:
for tx in results.values():
night = tx['NIGHT'][0]
expid = tx['EXPID'][0]
break
#- To do: consider propagating header from indir/desi*.fits.fz
log.info('{} Writing {}'.format(timestamp(), outfile))
tmpfile = outfile + '.tmp'
with fitsio.FITS(tmpfile, 'rw', clobber=True) as fx:
fx.write(np.zeros(3, dtype=float),
extname='PRIMARY',
header=hdr)
for qatype, qatable in results.items():
fx.write_table(qatable.as_array(),
extname=qatype,
header=hdr)
os.rename(tmpfile, outfile)
log.info('{} Finished writing {}'.format(timestamp(), outfile))
return results
def main(args):
"""Command-line driver to visualize survey scheduling and progress.
"""
# Set up the logger
if args.debug:
log = desiutil.log.get_logger(desiutil.log.DEBUG)
args.verbose = True
elif args.verbose:
log = desiutil.log.get_logger(desiutil.log.INFO)
else:
log = desiutil.log.get_logger(desiutil.log.WARNING)
# Freeze IERS table for consistent results.
desisurvey.utils.freeze_iers()
# Set the output path if requested.
config = desisurvey.config.Configuration()
if args.output_path is not None:
config.set_output_path(args.output_path)
if args.tiles_file is not None:
config.tiles_file.set_value(args.tiles_file)
# Look for the exposures file in the output path by default.
args.exposures = config.get_path(args.exposures)
# Initialize.
animator = Animator(args.exposures, args.start, args.stop, args.label,
args.scores)
log.info('Found {0} exposures from {1} to {2} ({3} nights).'.format(
animator.num_exp, args.start, args.stop, animator.num_nights))
animator.init_figure(args.nightly)
if args.expid is not None:
expid = animator.exposures['EXPID']
assert np.all(expid == expid[0] + np.arange(len(expid)))
if (args.expid < expid[0]) or (args.expid > expid[-1]):
raise RuntimeError(
'Requested exposure ID {0} not available.'.format(args.expid))
animator.draw_exposure(args.expid - expid[0], args.nightly)
save_name = args.save + '.png'
plt.savefig(save_name)
log.info('Saved {0}.'.format(save_name))
else:
nframes = animator.num_nights if args.nightly else animator.num_exp
iexp = [0]
def init():
return animator.artists
def update(iframe):
if (iframe + 1) % args.log_interval == 0:
log.info('Drawing frame {0}/{1}.'.format(iframe + 1, nframes))
if args.nightly:
while not animator.draw_exposure(iexp[0], nightly=True):
iexp[0] += 1
else:
animator.draw_exposure(iexp=iframe, nightly=False)
return animator.artists
log.info('Movie will be {:.1f} mins long at {:.1f} frames/sec.'.format(
nframes / (60 * args.fps), args.fps))
animation = matplotlib.animation.FuncAnimation(animator.figure,
update,
init_func=init,
blit=True,
frames=nframes)
writer = matplotlib.animation.writers['ffmpeg'](
bitrate=2400, fps=args.fps, metadata=dict(artist='surveymovie'))
save_name = args.save + '.mp4'
animation.save(save_name, writer=writer, dpi=animator.dpi)
log.info('Saved {0}.'.format(save_name))
def freeze_iers(name='iers_frozen.ecsv', ignore_warnings=True):
"""Use a frozen IERS table saved with this package.
This should be called at the beginning of a script that calls
astropy time and coordinates functions which refer to the UT1-UTC
and polar motions tabulated by IERS. The purpose is to ensure
identical results across systems and astropy releases, to avoid a
potential network download, and to eliminate some astropy warnings.
After this call, the loaded table will be returned by
:func:`astropy.utils.iers.IERS_Auto.open()` and treated like a
a normal IERS table by all astropy code. Specifically, this method
registers an instance of a custom IERS_Frozen class that inherits from
IERS_B and overrides
:meth:`astropy.utils.iers.IERS._check_interpolate_indices` to prevent
any IERSRangeError being raised.
See http://docs.astropy.org/en/stable/utils/iers.html for details.
This function returns immediately after the first time it is called,
so it it safe to insert anywhere that consistent IERS models are
required, and subsequent calls with different args will have no
effect.
The :func:`desisurvey.utils.plot_iers` function is useful for inspecting
IERS tables and how they are extrapolated to DESI survey dates.
Parameters
----------
name : str
Name of the file to load the frozen IERS table from. Should normally
be relative and then refers to this package's data/ directory.
Must end with the .ecsv extension.
ignore_warnings : bool
Ignore ERFA and IERS warnings about future dates generated by
astropy time and coordinates functions. Specifically, ERFA warnings
containing the string "dubious year" are filtered out, as well
as AstropyWarnings related to IERS table extrapolation.
"""
log = desiutil.log.get_logger()
if desisurvey.utils._iers_is_frozen:
log.debug('IERS table already frozen.')
return
log.info('Freezing IERS table used by astropy time, coordinates.')
# Validate the save_name extension.
_, ext = os.path.splitext(name)
if ext != '.ecsv':
raise ValueError('Expected .ecsv extension for {0}.'.format(name))
# Locate the file in our package data/ directory.
if not os.path.isabs(name):
name = astropy.utils.data._find_pkg_data_path(
os.path.join('data', name))
if not os.path.exists(name):
raise ValueError('No such IERS file: {0}.'.format(name))
# Clear any current IERS table.
astropy.utils.iers.IERS.close()
# Initialize the global IERS table. We load the table by
# hand since the IERS open() method hardcodes format='cds'.
try:
table = astropy.table.Table.read(name, format='ascii.ecsv').filled()
except IOError:
raise RuntimeError('Unable to load IERS table from {0}.'.format(name))
# Define a subclass of IERS_B that overrides _check_interpolate_indices
# to prevent any IERSRangeError being raised.
class IERS_Frozen(astropy.utils.iers.IERS_B):
def _check_interpolate_indices(self, indices_orig, indices_clipped,
max_input_mjd): pass
# Create and register an instance of this class from the table.
iers = IERS_Frozen(table)
astropy.utils.iers.IERS.iers_table = iers
# Prevent any attempts to automatically download updated IERS-A tables.
astropy.utils.iers.conf.auto_download = False
astropy.utils.iers.conf.auto_max_age = None
astropy.utils.iers.conf.iers_auto_url = 'frozen'
# Sanity check.
if not (astropy.utils.iers.IERS_Auto.open() is iers):
raise RuntimeError('Frozen IERS is not installed as the default.')
if ignore_warnings:
warnings.filterwarnings(
'ignore', category=astropy._erfa.core.ErfaWarning, message=
r'ERFA function \"[a-z0-9_]+\" yielded [0-9]+ of \"dubious year')
warnings.filterwarnings(
'ignore', category=astropy.utils.exceptions.AstropyWarning,
message=r'Tried to get polar motions for times after IERS data')
warnings.filterwarnings(
'ignore', category=astropy.utils.exceptions.AstropyWarning,
message=r'\(some\) times are outside of range covered by IERS')
# Shortcircuit any subsequent calls to this function.
desisurvey.utils._iers_is_frozen = True
def main(args):
"""Command-line driver to visualize survey scheduling and progress.
"""
# Set up the logger
if args.debug:
log = desiutil.log.get_logger(desiutil.log.DEBUG)
args.verbose = True
elif args.verbose:
log = desiutil.log.get_logger(desiutil.log.INFO)
else:
log = desiutil.log.get_logger(desiutil.log.WARNING)
# Freeze IERS table for consistent results.
desisurvey.utils.freeze_iers()
# Set the output path if requested.
config = desisurvey.config.Configuration()
if args.output_path is not None:
config.set_output_path(args.output_path)
if args.tiles_file is not None:
config.tiles_file.set_value(args.tiles_file)
# Look for the exposures file in the output path by default.
args.exposures = config.get_path(args.exposures)
# Initialize.
animator = Animator(
args.exposures, args.start, args.stop, args.label, args.scores)
log.info('Found {0} exposures from {1} to {2} ({3} nights).'
.format(animator.num_exp, args.start, args.stop,
animator.num_nights))
animator.init_figure(args.nightly)
if args.expid is not None:
expid = animator.exposures['EXPID']
assert np.all(expid == expid[0] + np.arange(len(expid)))
if (args.expid < expid[0]) or (args.expid > expid[-1]):
raise RuntimeError('Requested exposure ID {0} not available.'
.format(args.expid))
animator.draw_exposure(args.expid - expid[0], args.nightly)
save_name = args.save + '.png'
plt.savefig(save_name)
log.info('Saved {0}.'.format(save_name))
else:
nframes = animator.num_nights if args.nightly else animator.num_exp
iexp = [0]
def init():
return animator.artists
def update(iframe):
if (iframe + 1) % args.log_interval == 0:
log.info('Drawing frame {0}/{1}.'
.format(iframe + 1, nframes))
if args.nightly:
while not animator.draw_exposure(iexp[0], nightly=True):
iexp[0] += 1
else:
animator.draw_exposure(iexp=iframe, nightly=False)
return animator.artists
log.info('Movie will be {:.1f} mins long at {:.1f} frames/sec.'
.format(nframes / (60 * args.fps), args.fps))
animation = matplotlib.animation.FuncAnimation(
animator.figure, update, init_func=init, blit=True, frames=nframes)
writer = matplotlib.animation.writers['ffmpeg'](
bitrate=2400, fps=args.fps, metadata=dict(artist='surveymovie'))
save_name = args.save + '.mp4'
animation.save(save_name, writer=writer, dpi=animator.dpi)
log.info('Saved {0}.'.format(save_name))
def main(args):
"""Command-line driver for updating the survey plan.
"""
# Check for a valid fa-delay value.
if args.fa_delay[-1] not in ('d', 'm', 'q'):
raise ValueError('fa-delay must have the form Nd, Nm or Nq.')
fa_delay_type = args.fa_delay[-1]
try:
fa_delay = int(args.fa_delay[:-1])
except ValueError:
raise ValueError('invalid number in fa-delay.')
if fa_delay < 0:
raise ValueError('fa-delay value must be >= 0.')
# Set up the logger
if args.debug:
log = desiutil.log.get_logger(desiutil.log.DEBUG)
args.verbose = True
elif args.verbose:
log = desiutil.log.get_logger(desiutil.log.INFO)
else:
log = desiutil.log.get_logger(desiutil.log.WARNING)
# Freeze IERS table for consistent results.
desisurvey.utils.freeze_iers()
# Set the output path if requested.
config = desisurvey.config.Configuration(file_name=args.config_file)
if args.output_path is not None:
config.set_output_path(args.output_path)
# Load ephemerides.
ephem = desisurvey.ephem.get_ephem()
# Initialize scheduler.
if not os.path.exists(config.get_path('scheduler.fits')):
# Tabulate data used by the scheduler if necessary.
desisurvey.old.schedule.initialize(ephem)
scheduler = desisurvey.old.schedule.Scheduler()
# Read priority rules.
rules = desisurvey.rules.Rules(args.rules)
if args.create:
# Load initial design hour angles for each tile.
design = astropy.table.Table.read(config.get_path('surveyinit.fits'))
# Create an empty progress record.
progress = desisurvey.progress.Progress()
# Initialize the observing priorities.
priorities = rules.apply(progress)
# Create the initial plan.
plan = desisurvey.plan.create(design['HA'], priorities)
# Start the survey from scratch.
start = config.first_day()
else:
# Load an existing plan and progress record.
if not os.path.exists(config.get_path('plan.fits')):
log.error('No plan.fits found in output path.')
return -1
if not os.path.exists(config.get_path('progress.fits')):
log.error('No progress.fits found in output path.')
return -1
plan = astropy.table.Table.read(config.get_path('plan.fits'))
progress = desisurvey.progress.Progress('progress.fits')
# Start the new plan from the last observing date.
with open(config.get_path('last_date.txt'), 'r') as f:
start = desisurvey.utils.get_date(f.read().rstrip())
num_complete, num_total, pct = progress.completed(as_tuple=True)
# Already observed all tiles?
if num_complete == num_total:
log.info('All tiles observed!')
# Return a shell exit code so scripts can detect this condition.
sys.exit(9)
# Reached end of the survey?
if start >= config.last_day():
log.info('Reached survey end date!')
# Return a shell exit code so scripts can detect this condition.
sys.exit(9)
day_number = desisurvey.utils.day_number(start)
log.info(
'Planning night[{0}] {1} with {2:.1f} / {3} ({4:.1f}%) completed.'.
format(day_number, start, num_complete, num_total, pct))
bookmarked = False
if not args.create:
# Update the priorities for the progress so far.
new_priority = rules.apply(progress)
changed_priority = (new_priority != plan['priority'])
if np.any(changed_priority):
changed_passes = np.unique(plan['pass'][changed_priority])
log.info('Priorities changed in pass(es) {0}.'.format(', '.join(
[str(p) for p in changed_passes])))
plan['priority'] = new_priority
bookmarked = True
# Identify any new tiles that are available for fiber assignment.
plan = desisurvey.plan.update_available(plan, progress, start, ephem,
fa_delay, fa_delay_type)
# Will update design HA assignments here...
pass
# Update the progress table for the new plan.
ptable = progress._table
new_cover = (ptable['covered'] < 0) & (plan['covered'] <= day_number)
ptable['covered'][new_cover] = day_number
new_avail = (ptable['available'] < 0) & plan['available']
ptable['available'][new_avail] = day_number
new_plan = (ptable['planned'] < 0) & (plan['priority'] > 0)
ptable['planned'][new_plan] = day_number
# Save updated progress.
progress.save('progress.fits')
# Save the plan.
plan.write(config.get_path('plan.fits'), overwrite=True)
if bookmarked:
# Save a backup of the plan and progress at this point.
plan.write(config.get_path('plan_{0}.fits'.format(start)))
progress.save('progress_{0}.fits'.format(start))
def main(args):
"""Command-line driver for running survey simulations.
"""
# Set up the logger
if args.debug:
os.environ['DESI_LOGLEVEL'] = 'DEBUG'
args.verbose = True
elif args.verbose:
os.environ['DESI_LOGLEVEL'] = 'INFO'
else:
os.environ['DESI_LOGLEVEL'] = 'WARNING'
log = desiutil.log.get_logger()
# Set the output path if requested.
config = desisurvey.config.Configuration()
if args.output_path is not None:
config.set_output_path(args.output_path)
if args.tiles_file is not None:
config.tiles_file.set_value(args.tiles_file)
# Initialize simulation progress tracking.
stats = surveysim.stats.SurveyStatistics(args.start, args.stop)
explist = surveysim.exposures.ExposureList()
# Initialize the survey strategy rules.
rules = desisurvey.rules.Rules(args.rules)
# Initialize afternoon planning.
planner = desisurvey.plan.Planner(rules)
# Initialize next tile selection.
scheduler = desisurvey.scheduler.Scheduler()
# Generate random weather conditions.
weather = surveysim.weather.Weather(seed=args.seed, replay=args.replay)
# Loop over nights.
num_simulated = 0
num_nights = (args.stop - args.start).days
for num_simulated in range(num_nights):
night = args.start + datetime.timedelta(num_simulated)
if args.save_restore and num_simulated > 0:
# Restore the planner and scheduler saved after the previous night.
planner = desisurvey.plan.Planner(rules, restore='planner_{}.fits'.format(last_night))
scheduler = desisurvey.scheduler.Scheduler(restore='scheduler_{}.fits'.format(last_night))
scheduler.update_tiles(planner.tile_available, planner.tile_priority)
# Perform afternoon planning.
explist.update_tiles(night, *scheduler.update_tiles(*planner.afternoon_plan(night, scheduler.completed)))
if not desisurvey.utils.is_monsoon(night) and not scheduler.ephem.is_full_moon(night):
# Simulate one night of observing.
surveysim.nightops.simulate_night(
night, scheduler, stats, explist, weather=weather, use_twilight=args.twilight)
if scheduler.survey_completed():
log.info('Survey complete on {}.'.format(night))
break
if args.save_restore:
last_night = night.isoformat()
planner.save('planner_{}.fits'.format(last_night))
scheduler.save('scheduler_{}.fits'.format(last_night))
if num_simulated % args.log_interval == args.log_interval - 1:
log.info('Completed {} / {} tiles after {} / {} nights.'.format(
scheduler.completed_by_pass.sum(), scheduler.tiles.ntiles,
num_simulated + 1, num_nights))
explist.save('exposures_{}.fits'.format(args.name), comment=args.comment)
stats.save('stats_{}.fits'.format(args.name), comment=args.comment)
if args.verbose:
stats.summarize()
def main(args):
"""Command-line driver for running survey simulations.
"""
# Set up the logger
if args.debug:
os.environ['DESI_LOGLEVEL'] = 'DEBUG'
args.verbose = True
elif args.verbose:
os.environ['DESI_LOGLEVEL'] = 'INFO'
else:
os.environ['DESI_LOGLEVEL'] = 'WARNING'
log = desiutil.log.get_logger()
# Set the output path if requested.
config = desisurvey.config.Configuration()
if args.output_path is not None:
config.set_output_path(args.output_path)
if args.tiles_file is not None:
config.tiles_file.set_value(args.tiles_file)
# Initialize simulation progress tracking.
stats = surveysim.stats.SurveyStatistics(args.start, args.stop)
explist = surveysim.exposures.ExposureList()
# Initialize the survey strategy rules.
rules = desisurvey.rules.Rules(args.rules)
# Initialize afternoon planning.
planner = desisurvey.plan.Planner(rules)
# Initialize next tile selection.
scheduler = desisurvey.scheduler.Scheduler()
# Generate random weather conditions.
weather = surveysim.weather.Weather(seed=args.seed, replay=args.replay)
# Loop over nights.
num_simulated = 0
num_nights = (args.stop - args.start).days
for num_simulated in range(num_nights):
night = args.start + datetime.timedelta(num_simulated)
if args.save_restore and num_simulated > 0:
# Restore the planner and scheduler saved after the previous night.
planner = desisurvey.plan.Planner(
rules, restore='planner_{}.fits'.format(last_night))
scheduler = desisurvey.scheduler.Scheduler(
restore='scheduler_{}.fits'.format(last_night))
scheduler.update_tiles(planner.tile_available,
planner.tile_priority)
# Perform afternoon planning.
explist.update_tiles(
night,
*scheduler.update_tiles(
*planner.afternoon_plan(night, scheduler.completed)))
if not desisurvey.utils.is_monsoon(
night) and not scheduler.ephem.is_full_moon(night):
# Simulate one night of observing.
surveysim.nightops.simulate_night(night,
scheduler,
stats,
explist,
weather=weather,
use_twilight=args.twilight)
if scheduler.survey_completed():
log.info('Survey complete on {}.'.format(night))
break
if args.save_restore:
last_night = night.isoformat()
planner.save('planner_{}.fits'.format(last_night))
scheduler.save('scheduler_{}.fits'.format(last_night))
if num_simulated % args.log_interval == args.log_interval - 1:
log.info('Completed {} / {} tiles after {} / {} nights.'.format(
scheduler.completed_by_pass.sum(), scheduler.tiles.ntiles,
num_simulated + 1, num_nights))
explist.save('exposures_{}.fits'.format(args.name), comment=args.comment)
stats.save('stats_{}.fits'.format(args.name), comment=args.comment)
if args.verbose:
stats.summarize()
def calculate_initial_plan(args):
"""Calculate the initial survey plan.
Use :func:`desisurvey.plan.load_weather` and
:func:`desisurvey.plan.load_design_hourangles` to retrieve
these data from the saved plan.
Parameters
----------
args : object
Object with attributes for parsed command-line arguments.
"""
log = desiutil.log.get_logger()
config = desisurvey.config.Configuration()
tiles = desisurvey.tiles.get_tiles()
ephem = desisurvey.ephem.get_ephem()
# Initialize the output file to write.
hdus = fits.HDUList()
hdr = fits.Header()
# Calculate average weather factors for each day covered by
# the ephemerides.
first = desisurvey.ephem.START_DATE
last = desisurvey.ephem.STOP_DATE
years = np.arange(2007, 2018)
fractions = []
for year in years:
fractions.append(
desimodel.weather.dome_closed_fractions(first,
last,
replay='Y{}'.format(year)))
weather = 1 - np.mean(fractions, axis=0)
# Save the weather fractions as the primary HDU.
hdr['FIRST'] = first.isoformat()
hdr['YEARS'] = ','.join(['{}'.format(yr) for yr in years])
start = config.first_day()
stop = config.last_day()
assert start >= first and stop <= last
hdr['START'] = start.isoformat()
hdr['STOP'] = stop.isoformat()
hdr['TWILIGHT'] = args.include_twilight
hdus.append(fits.ImageHDU(weather, header=hdr, name='WEATHER'))
# Calculate the distribution of available LST in each program
# during the nominal survey [start, stop).
ilo, ihi = (start - first).days, (stop - first).days
lst_hist, lst_bins = ephem.get_available_lst(
nbins=args.nbins,
weather=weather[ilo:ihi],
include_twilight=args.include_twilight)
# Initialize the output results table.
design = astropy.table.Table()
design['INIT'] = np.zeros(tiles.ntiles)
design['HA'] = np.zeros(tiles.ntiles)
design['TEXP'] = np.zeros(tiles.ntiles)
# Optimize each program separately.
stretches = dict(DARK=args.dark_stretch,
GRAY=args.gray_stretch,
BRIGHT=args.bright_stretch)
for pindex, program in enumerate(tiles.PROGRAMS):
sel = tiles.program_mask[program]
if not np.any(sel):
log.info('Skipping {} program with no tiles.'.format(program))
continue
# Initialize an LST summary table.
table = astropy.table.Table(meta={'ORIGIN': lst_bins[0]})
table['AVAIL'] = lst_hist[pindex]
# Initailize an optimizer for this program.
opt = desisurvey.optimize.Optimizer(program,
lst_bins,
lst_hist[pindex],
init=args.init,
center=None,
stretch=stretches[program])
table['INIT'] = opt.plan_hist.copy()
design['INIT'][sel] = opt.ha_initial
# Initialize annealing cycles.
ncycles = 0
binsize = 360. / args.nbins
frac = args.adjust / binsize
smoothing = args.smooth
# Loop over annealing cycles.
while ncycles < args.max_cycles:
start_score = opt.eval_score(opt.plan_hist)
for i in range(opt.ntiles):
opt.improve(frac)
if smoothing > 0:
opt.smooth(alpha=smoothing)
stop_score = opt.eval_score(opt.plan_hist)
delta = (stop_score - start_score) / start_score
RMSE = opt.RMSE_history[-1]
loss = opt.loss_history[-1]
log.info('[{:03d}] dHA={:5.3f}deg '.format(ncycles + 1, frac *
binsize) +
'RMSE={:6.2f}% LOSS={:5.2f}% delta(score)={:+5.1f}%'.
format(1e2 * RMSE, 1e2 * loss, 1e2 * delta))
# Both conditions must be satisfied to terminate.
if RMSE < args.max_rmse and delta > -args.epsilon:
break
# Anneal parameters for next cycle.
frac *= args.anneal
smoothing *= args.anneal
ncycles += 1
plan_sum = opt.plan_hist.sum()
avail_sum = opt.lst_hist_sum
margin = (avail_sum - plan_sum) / plan_sum
log.info(
'{} plan uses {:.1f}h with {:.1f}h avail ({:.1f}% margin).'.format(
program, plan_sum, avail_sum, 1e2 * margin))
# Save planned LST usage.
table['PLAN'] = opt.plan_hist
hdus.append(fits.BinTableHDU(table, name=program))
# Calculate exposure times in (solar) seconds.
texp, _ = opt.get_exptime(opt.ha)
texp *= 24. * 3600. / 360. * 0.99726956583
# Save results for this program.
design['HA'][sel] = opt.ha
design['TEXP'][sel] = texp
hdus.append(fits.BinTableHDU(design, name='DESIGN'))
fullname = config.get_path(args.save)
hdus.writeto(fullname, overwrite=True)
log.info('Saved initial plan to "{}".'.format(fullname))
def main(args):
"""Command-line driver for running survey simulations.
"""
# Set up the logger
if args.debug:
os.environ['DESI_LOGLEVEL'] = 'DEBUG'
args.verbose = True
elif args.verbose:
os.environ['DESI_LOGLEVEL'] = 'INFO'
else:
os.environ['DESI_LOGLEVEL'] = 'WARNING'
log = desiutil.log.get_logger()
# Set the output path if requested.
config = desisurvey.config.Configuration()
if args.output_path is not None:
config.set_output_path(args.output_path)
if args.tiles_file is not None:
config.tiles_file.set_value(args.tiles_file)
if args.existing_exposures is not None:
exps = Table.read(args.existing_exposures)
tiles = desisurvey.tiles.get_tiles()
idx, mask = tiles.index(exps['TILEID'], return_mask=True)
firstnight = max(exps['NIGHT'][mask])
if args.start != config.first_day():
raise ValueError('Cannot set both start and existing-exposures!')
args.start = '-'.join(
[firstnight[:4], firstnight[4:6], firstnight[6:]])
args.start = desisurvey.utils.get_date(args.start)
exps = exps[mask]
else:
exps = None
# Initialize simulation progress tracking.
stats = surveysim.stats.SurveyStatistics(args.start, args.stop)
explist = surveysim.exposures.ExposureList(existing_exposures=exps)
# Initialize the survey strategy rules.
if args.rules is None:
rulesfile = config.rules_file()
else:
rulesfile = args.rules
rules = desisurvey.rules.Rules(rulesfile)
log.info('Rules loaded from {}.'.format(rulesfile))
# Initialize afternoon planning.
planner = desisurvey.plan.Planner(rules, simulate=True)
# Initialize next tile selection.
scheduler = desisurvey.scheduler.Scheduler(planner)
# Generate random weather conditions.
weather = surveysim.weather.Weather(
seed=args.seed, replay=args.replay,
extra_downtime=args.extra_downtime)
# Loop over nights.
num_simulated = 0
num_nights = (args.stop - args.start).days
for num_simulated in range(num_nights):
night = args.start + datetime.timedelta(num_simulated)
if args.save_restore and num_simulated > 0:
# Restore the planner and scheduler saved after the previous night.
planner = desisurvey.plan.Planner(rules, restore='desi-status-{}.ecsv'.format(last_night),
simulate=True)
scheduler = desisurvey.scheduler.Scheduler(planner)
# Perform afternoon planning.
explist.update_tiles(night, *planner.afternoon_plan(night))
if not desisurvey.utils.is_monsoon(night) and not scheduler.ephem.is_full_moon(night):
# Simulate one night of observing.
surveysim.nightops.simulate_night(
night, scheduler, stats, explist, weather=weather,
use_twilight=args.twilight, use_simplesky=args.simplesky)
if scheduler.plan.survey_completed():
log.info('Survey complete on {}.'.format(night))
break
if args.save_restore:
last_night = desisurvey.utils.night_to_str(night)
planner.save('desi-status-{}.ecsv'.format(last_night))
if num_simulated % args.log_interval == args.log_interval - 1:
log.info('Completed {} / {} tiles after {} / {} nights.'.format(
scheduler.plan.obsend().sum(),
scheduler.tiles.ntiles,
num_simulated + 1, num_nights))
explist.save('exposures_{}.fits'.format(args.name), comment=args.comment)
stats.save('stats_{}.fits'.format(args.name), comment=args.comment)
planner.save('desi-status-end-{}.ecsv'.format(args.name))
if args.verbose:
stats.summarize()
def write_tables(indir, outdir, expnights=None):
'''
Parses directory for available nights, exposures to generate
nights and exposures tables
Args:
indir : directory of nights
outdir : directory where to write nights table
Options:
expnights (list) : only update exposures tables for these nights
'''
import re
from astropy.table import Table
from nightwatch.webpages import tables as web_tables
from pkg_resources import resource_filename
from shutil import copyfile
from collections import Counter
log = desiutil.log.get_logger()
log.info(f'Tabulating exposures in {indir}')
#- Count night/expid directories to get num exp per night
expdirs = sorted(glob.glob(f"{indir}/20*/[0-9]*"))
nights = list()
re_expid = re.compile('^\d{8}$')
re_night = re.compile('^20\d{6}$')
for expdir in expdirs:
expid = os.path.basename(expdir)
night = os.path.basename(os.path.dirname(expdir))
if re_expid.match(expid) and re_night.match(night):
nights.append(night)
num_exp_per_night = Counter(nights)
#- Build the exposures table for the requested nights
rows = list()
for expdir in expdirs:
expid = os.path.basename(expdir)
night = os.path.basename(os.path.dirname(expdir))
if re_expid.match(expid) and re_night.match(night) and \
(expnights is None or int(night) in expnights):
night = int(night)
expid = int(expid)
qafile = os.path.join(expdir, 'qa-{:08d}.fits'.format(expid))
#- gets the list of failed qprocs for each expid
expfiles = os.listdir(expdir)
preproc_cams = [
i.split("-")[1] for i in expfiles
if re.match(r'preproc-.*-.*.fits', i)
]
log_cams = [
i.split("-")[1] for i in expfiles if re.match(r'.*\.log', i)
]
qfails = [i for i in log_cams if i not in preproc_cams]
if os.path.exists(qafile):
try:
with fitsio.FITS(qafile) as fits:
qproc_status = fits['QPROC_STATUS'].read()
exitcode = np.count_nonzero(qproc_status['QPROC_EXIT'])
except IOError:
exitcode = 0
rows.append(
dict(NIGHT=night,
EXPID=expid,
FAIL=0,
QPROC=qfails,
QPROC_EXIT=exitcode))
else:
log.error('Missing {}'.format(qafile))
rows.append(
dict(NIGHT=night,
EXPID=expid,
FAIL=1,
QPROC=None,
QPROC_EXIT=None))
if len(rows) == 0:
msg = "No exp dirs found in {}/NIGHT/EXPID".format(indir)
raise RuntimeError(msg)
exposures = Table(rows)
caldir = os.path.join(outdir, 'static')
if not os.path.isdir(caldir):
os.makedirs(caldir)
files = [
'bootstrap.js', 'bootstrap.css', 'bootstrap-year-calendar.css',
'bootstrap-year-calendar.js', 'jquery_min.js', 'popper_min.js',
'live.js'
]
for f in files:
outfile = os.path.join(outdir, 'static', f)
if not os.path.exists(outfile):
infile = resource_filename('nightwatch', os.path.join('static', f))
copyfile(infile, outfile)
nightsfile = os.path.join(outdir, 'nights.html')
web_tables.write_calendar(nightsfile, num_exp_per_night)
web_tables.write_exposures_tables(indir,
outdir,
exposures,
nights=expnights)
def initialize(ephem,
start_date=None,
stop_date=None,
step_size=5.0,
healpix_nside=16,
output_name='scheduler.fits'):
"""Calculate exposure-time factors over a grid of times and pointings.
Takes about 9 minutes to run and writes a 1.3Gb output file with the
default parameters.
Requires that healpy is installed.
Parameters
----------
ephem : desisurvey.ephem.Ephemerides
Tabulated ephemerides data to use for planning.
start_date : date or None
Survey planning starts on the evening of this date. Must be convertible
to a date using :func:`desisurvey.utils.get_date`. Use the first night
of the ephemerides when None.
stop_date : date or None
Survey planning stops on the morning of this date. Must be convertible
to a date using :func:`desisurvey.utils.get_date`. Use the first night
of the ephemerides when None.
step_size : :class:`astropy.units.Quantity`
Exposure-time factors are tabulated at this interval during each night.
healpix_nside : int
Healpix NSIDE parameter to use for binning the sky. Must be a power of
two. Values larger than 16 will lead to holes in the footprint with
the current implementation.
output_name : str
Name of the FITS output file where results are saved. A relative path
refers to the :meth:`configuration output path
<desisurvey.config.Configuration.get_path>`.
"""
import healpy
if not isinstance(step_size, u.Quantity):
step_size = step_size * u.min
log = desiutil.log.get_logger()
# Freeze IERS table for consistent results.
desisurvey.utils.freeze_iers()
config = desisurvey.config.Configuration()
output_name = config.get_path(output_name)
start_date = desisurvey.utils.get_date(start_date or config.first_day())
stop_date = desisurvey.utils.get_date(stop_date or config.last_day())
if start_date >= stop_date:
raise ValueError('Expected start_date < stop_date.')
mjd = ephem._table['noon']
sel = ((mjd >= desisurvey.utils.local_noon_on_date(start_date).mjd) &
(mjd < desisurvey.utils.local_noon_on_date(stop_date).mjd))
t = ephem._table[sel]
num_nights = len(t)
# Build a grid of elapsed time relative to local midnight during each night.
midnight = t['noon'] + 0.5
t_edges = desisurvey.ephem.get_grid(step_size)
t_centers = 0.5 * (t_edges[1:] + t_edges[:-1])
num_points = len(t_centers)
# Create an empty HDU0 with header info.
header = astropy.io.fits.Header()
header['START'] = str(start_date)
header['STOP'] = str(stop_date)
header['NSIDE'] = healpix_nside
header['NPOINTS'] = num_points
header['STEP'] = step_size.to(u.min).value
hdus = astropy.io.fits.HDUList()
hdus.append(astropy.io.fits.ImageHDU(header=header))
# Save time grid.
hdus.append(astropy.io.fits.ImageHDU(name='GRID', data=t_edges))
# Load the list of tiles to observe.
tiles = astropy.table.Table(
desimodel.io.load_tiles(onlydesi=True,
extra=False,
tilesfile=config.tiles_file()))
# Build the footprint as a healpix map of the requested size.
# The footprint includes any pixel containing at least one tile center.
npix = healpy.nside2npix(healpix_nside)
footprint = np.zeros(npix, bool)
pixels = healpy.ang2pix(healpix_nside, np.radians(90 - tiles['DEC'].data),
np.radians(tiles['RA'].data))
footprint[np.unique(pixels)] = True
footprint_pixels = np.where(footprint)[0]
num_footprint = len(footprint_pixels)
log.info('Footprint contains {0} pixels.'.format(num_footprint))
# Sort pixels in order of increasing phi + 60deg so that the north and south
# galactic caps are contiguous in the arrays we create below.
pix_theta, pix_phi = healpy.pix2ang(healpix_nside, footprint_pixels)
pix_dphi = np.fmod(pix_phi + np.pi / 3, 2 * np.pi)
sort_order = np.argsort(pix_dphi)
footprint_pixels = footprint_pixels[sort_order]
# Calculate sorted pixel (ra,dec).
pix_theta, pix_phi = healpy.pix2ang(healpix_nside, footprint_pixels)
pix_ra, pix_dec = np.degrees(pix_phi), 90 - np.degrees(pix_theta)
# Record per-tile info needed for planning.
table = astropy.table.Table()
table['tileid'] = tiles['TILEID'].astype(np.int32)
table['ra'] = tiles['RA'].astype(np.float32)
table['dec'] = tiles['DEC'].astype(np.float32)
table['EBV'] = tiles['EBV_MED'].astype(np.float32)
table['pass'] = tiles['PASS'].astype(np.int16)
# Map each tile ID to the corresponding index in our spatial arrays.
mapper = np.zeros(npix, int)
mapper[footprint_pixels] = np.arange(len(footprint_pixels))
table['map'] = mapper[pixels].astype(np.int16)
# Use a small int to identify the program, ordered by sky brightness:
# 1=DARK, 2=GRAY, 3=BRIGHT.
table['program'] = np.full(len(tiles), 4, np.int16)
for i, program in enumerate(('DARK', 'GRAY', 'BRIGHT')):
table['program'][tiles['PROGRAM'] == program] = i + 1
assert np.all(table['program'] > 0)
hdu = astropy.io.fits.table_to_hdu(table)
hdu.name = 'TILES'
hdus.append(hdu)
# Average E(B-V) for all tiles falling into a pixel.
tiles_per_pixel = np.bincount(pixels, minlength=npix)
EBV = np.bincount(pixels, weights=tiles['EBV_MED'], minlength=npix)
EBV[footprint] /= tiles_per_pixel[footprint]
# Calculate dust extinction exposure-time factor.
f_EBV = 1. / desisurvey.etc.dust_exposure_factor(EBV)
# Save HDU with the footprint and static dust exposure map.
table = astropy.table.Table()
table['pixel'] = footprint_pixels
table['dust'] = f_EBV[footprint_pixels]
table['ra'] = pix_ra
table['dec'] = pix_dec
hdu = astropy.io.fits.table_to_hdu(table)
hdu.name = 'STATIC'
hdus.append(hdu)
# Prepare a table of calendar data.
calendar = astropy.table.Table()
calendar['midnight'] = midnight
calendar['monsoon'] = np.zeros(num_nights, bool)
calendar['fullmoon'] = np.zeros(num_nights, bool)
calendar['weather'] = np.zeros(num_nights, np.float32)
# Hardcode annualized average weight.
weather_weights = np.full(num_nights, 0.723)
# Prepare a table of ephemeris data.
etable = astropy.table.Table()
# Program codes ordered by increasing sky brightness:
# 1=DARK, 2=GRAY, 3=BRIGHT, 4=DAYTIME.
etable['program'] = np.full(num_nights * num_points, 4, dtype=np.int16)
etable['moon_frac'] = np.zeros(num_nights * num_points, dtype=np.float32)
etable['moon_ra'] = np.zeros(num_nights * num_points, dtype=np.float32)
etable['moon_dec'] = np.zeros(num_nights * num_points, dtype=np.float32)
etable['moon_alt'] = np.zeros(num_nights * num_points, dtype=np.float32)
etable['zenith_ra'] = np.zeros(num_nights * num_points, dtype=np.float32)
etable['zenith_dec'] = np.zeros(num_nights * num_points, dtype=np.float32)
# Tabulate MJD and apparent LST values for each time step. We don't save
# MJD values since they are cheap to reconstruct from the index, but
# do use them below.
mjd0 = desisurvey.utils.local_noon_on_date(start_date).mjd + 0.5
mjd = mjd0 + np.arange(num_nights)[:, np.newaxis] + t_centers
times = astropy.time.Time(mjd,
format='mjd',
location=desisurvey.utils.get_location())
etable['lst'] = times.sidereal_time('apparent').flatten().to(u.deg).value
# Build sky coordinates for each pixel in the footprint.
pix_theta, pix_phi = healpy.pix2ang(healpix_nside, footprint_pixels)
pix_ra, pix_dec = np.degrees(pix_phi), 90 - np.degrees(pix_theta)
pix_sky = astropy.coordinates.ICRS(pix_ra * u.deg, pix_dec * u.deg)
# Initialize exposure factor calculations.
alt, az = np.full(num_points, 90.) * u.deg, np.zeros(num_points) * u.deg
fexp = np.zeros((num_nights * num_points, num_footprint), dtype=np.float32)
vband_extinction = 0.15154
one = np.ones((num_points, num_footprint))
# Loop over nights.
for i in range(num_nights):
night = ephem.get_night(midnight[i])
date = desisurvey.utils.get_date(midnight[i])
if date.day == 1:
log.info('Starting {0} (completed {1}/{2} nights)'.format(
date.strftime('%b %Y'), i, num_nights))
# Initialize the slice of the fexp[] time index for this night.
sl = slice(i * num_points, (i + 1) * num_points)
# Do we expect to observe on this night?
calendar[i]['monsoon'] = desisurvey.utils.is_monsoon(midnight[i])
calendar[i]['fullmoon'] = ephem.is_full_moon(midnight[i])
# Look up expected dome-open fraction due to weather.
calendar[i]['weather'] = weather_weights[i]
# Calculate the program during this night (default is 4=DAYTIME).
mjd = midnight[i] + t_centers
dark, gray, bright = ephem.tabulate_program(mjd)
etable['program'][sl][dark] = 1
etable['program'][sl][gray] = 2
etable['program'][sl][bright] = 3
# Zero the exposure factor whenever we are not oberving.
##fexp[sl] = (dark | gray | bright)[:, np.newaxis]
fexp[sl] = 1.
# Transform the local zenith to (ra,dec).
zenith = desisurvey.utils.get_observer(
times[i], alt=alt, az=az).transform_to(astropy.coordinates.ICRS)
etable['zenith_ra'][sl] = zenith.ra.to(u.deg).value
etable['zenith_dec'][sl] = zenith.dec.to(u.deg).value
# Calculate zenith angles to each pixel in the footprint.
pix_sep = pix_sky.separation(zenith[:, np.newaxis])
# Zero the exposure factor for pixels below the horizon.
visible = pix_sep < 90 * u.deg
fexp[sl][~visible] = 0.
# Calculate the airmass exposure-time penalty.
X = desisurvey.utils.cos_zenith_to_airmass(np.cos(pix_sep[visible]))
fexp[sl][visible] /= desisurvey.etc.airmass_exposure_factor(X)
# Loop over objects we need to avoid.
for name in config.avoid_bodies.keys:
f_obj = desisurvey.ephem.get_object_interpolator(night, name)
# Calculate this object's (dec,ra) path during the night.
obj_dec, obj_ra = f_obj(mjd)
sky_obj = astropy.coordinates.ICRS(
ra=obj_ra[:, np.newaxis] * u.deg,
dec=obj_dec[:, np.newaxis] * u.deg)
# Calculate the separation angles to each pixel in the footprint.
obj_sep = pix_sky.separation(sky_obj)
if name == 'moon':
etable['moon_ra'][sl] = obj_ra
etable['moon_dec'][sl] = obj_dec
# Calculate moon altitude during the night.
moon_alt, _ = desisurvey.ephem.get_object_interpolator(
night, 'moon', altaz=True)(mjd)
etable['moon_alt'][sl] = moon_alt
moon_zenith = (90 - moon_alt[:, np.newaxis]) * u.deg
moon_up = moon_alt > 0
assert np.all(moon_alt[gray] > 0)
# Calculate the moon illuminated fraction during the night.
moon_frac = ephem.get_moon_illuminated_fraction(mjd)
etable['moon_frac'][sl] = moon_frac
# Convert to temporal moon phase.
moon_phase = np.arccos(2 * moon_frac[:, np.newaxis] -
1) / np.pi
# Calculate scattered moon V-band brightness at each pixel.
V = specsim.atmosphere.krisciunas_schaefer(
pix_sep, moon_zenith, obj_sep, moon_phase,
desisurvey.etc._vband_extinction).value
# Estimate the exposure time factor from V.
X = np.dstack((one, np.exp(-V), 1 / V, 1 / V**2, 1 / V**3))
T = X.dot(desisurvey.etc._moonCoefficients)
# No penalty when the moon is below the horizon.
T[moon_alt < 0, :] = 1.
fexp[sl] *= 1. / T
# Veto pointings within avoidance size when the moon is
# above the horizon. Apply Gaussian smoothing to the veto edge.
veto = np.ones_like(T)
dsep = (obj_sep - config.avoid_bodies.moon()).to(u.deg).value
veto[dsep <= 0] = 0.
veto[dsep > 0] = 1 - np.exp(-0.5 * (dsep[dsep > 0] / 3)**2)
veto[moon_alt < 0] = 1.
fexp[sl] *= veto
else:
# Lookup the avoidance size for this object.
size = getattr(config.avoid_bodies, name)()
# Penalize the exposure-time with a factor
# 1 - exp(-0.5*(obj_sep/size)**2)
penalty = 1. - np.exp(-0.5 * (obj_sep / size).to(1).value**2)
fexp[sl] *= penalty
# Save calendar table.
hdu = astropy.io.fits.table_to_hdu(calendar)
hdu.name = 'CALENDAR'
hdus.append(hdu)
# Save ephemerides table.
hdu = astropy.io.fits.table_to_hdu(etable)
hdu.name = 'EPHEM'
hdus.append(hdu)
# Save dynamic exposure-time factors.
hdus.append(astropy.io.fits.ImageHDU(name='DYNAMIC', data=fexp))
# Finalize the output file.
try:
hdus.writeto(output_name, overwrite=True)
except TypeError:
# astropy < 1.3 uses the now deprecated clobber.
hdus.writeto(output_name, clobber=True)
log.info('Plan initialization saved to {0}'.format(output_name))
def stdouterr_redirected(to=None, comm=None):
"""
Redirect stdout and stderr to a file.
The general technique is based on:
http://stackoverflow.com/questions/5081657
http://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/
One difference here is that each process in the communicator
redirects to a different temporary file, and the upon exit
from the context the rank zero process concatenates these
in order to the file result.
Args:
to (str): The output file name.
comm (mpi4py.MPI.Comm): The optional MPI communicator.
"""
nproc = 1
rank = 0
if comm is not None:
nproc = comm.size
rank = comm.rank
# The currently active POSIX file descriptors
fd_out = sys.stdout.fileno()
fd_err = sys.stderr.fileno()
# The DESI loggers.
desi_loggers = desiutil.log._desiutil_log_root
def _redirect(out_to, err_to):
# Flush the C-level buffers
if c_stdout is not None:
libc.fflush(c_stdout)
if c_stderr is not None:
libc.fflush(c_stderr)
# This closes the python file handles, and marks the POSIX
# file descriptors for garbage collection- UNLESS those
# are the special file descriptors for stderr/stdout.
sys.stdout.close()
sys.stderr.close()
# Close fd_out/fd_err if they are open, and copy the
# input file descriptors to these.
os.dup2(out_to, fd_out)
os.dup2(err_to, fd_err)
# Create a new sys.stdout / sys.stderr that points to the
# redirected POSIX file descriptors. In Python 3, these
# are actually higher level IO objects.
if sys.version_info[0] < 3:
sys.stdout = os.fdopen(fd_out, "wb")
sys.stderr = os.fdopen(fd_err, "wb")
else:
# Python 3 case
sys.stdout = io.TextIOWrapper(os.fdopen(fd_out, 'wb'))
sys.stderr = io.TextIOWrapper(os.fdopen(fd_err, 'wb'))
# update DESI logging to use new stdout
for name, logger in desi_loggers.items():
hformat = None
while len(logger.handlers) > 0:
h = logger.handlers[0]
if hformat is None:
hformat = h.formatter._fmt
logger.removeHandler(h)
# Add the current stdout.
ch = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(hformat, datefmt='%Y-%m-%dT%H:%M:%S')
ch.setFormatter(formatter)
logger.addHandler(ch)
# redirect both stdout and stderr to the same file
if to is None:
to = "/dev/null"
if rank == 0:
log = get_logger()
log.info("Begin log redirection to {} at {}".format(
to, time.asctime()))
# Save the original file descriptors so we can restore them later
saved_fd_out = os.dup(fd_out)
saved_fd_err = os.dup(fd_err)
try:
pto = to
if to != "/dev/null":
pto = "{}_{}".format(to, rank)
# open python file, which creates low-level POSIX file
# descriptor.
file = open(pto, "w")
# redirect stdout/stderr to this new file descriptor.
_redirect(out_to=file.fileno(), err_to=file.fileno())
yield # allow code to be run with the redirected output
# close python file handle, which will mark POSIX file
# descriptor for garbage collection. That is fine since
# we are about to overwrite those in the finally clause.
file.close()
finally:
# flush python handles for good measure
sys.stdout.flush()
sys.stderr.flush()
# restore old stdout and stderr
_redirect(out_to=saved_fd_out, err_to=saved_fd_err)
if nproc > 1:
comm.barrier()
# concatenate per-process files
if rank == 0:
with open(to, "w") as outfile:
for p in range(nproc):
outfile.write(
"================= Process {} =================\n".
format(p))
fname = "{}_{}".format(to, p)
with open(fname) as infile:
outfile.write(infile.read())
os.remove(fname)
if nproc > 1:
comm.barrier()
if rank == 0:
log = get_logger()
log.info("End log redirection to {} at {}".format(
to, time.asctime()))
# flush python handles for good measure
sys.stdout.flush()
sys.stderr.flush()
return
def add_calibration_exposures(exposures, flats_per_night=3, arcs_per_night=3,
darks_per_night=0, zeroes_per_night=0,
exptime=None, readout=30.0):
"""Prepare a list of science exposures for desisim.wrap-newexp.
Insert calibration exposures at the start of each night, and add
the following columns for all exposures: EXPID, PROGRAM, NIGHT,
FLAVOR.
Parameters
----------
exposures : table like or :class:`surveysim.exposures.ExposureList`
A table of science exposures including, at a minimum,
MJD, EXPTIME and TILEID columns. The exposures must be sorted
by increasing MJD. Could be a numpy recarray, an astropy
table, or an ExposureList object. Columns other than
the required ones are copied to the output.
flats_per_night : :class:`int`, optional
Add this many arc exposures per night (default 3).
arcs_per_night : :class:`int`, optional
Add this many arc exposures per night (default 3).
darks_per_night : :class:`int`, optional
Add this many dark exposures per night (default 0).
zeroes_per_night : :class:`int`, optional
Add this many zero exposures per night (default 0).
exptime : :class:`dict`, optional
A dictionary setting calibration exposure times for each
calibration flavor.
readout : :class:`float`, optional
Set readout time for calibration exposures (default 30.0 s).
Returns
-------
:class:`astropy.table.Table`
The output table augmented with calibration exposures and
additional columns.
Raises
------
ValueError
If the input is not sorted by increasing MJD/timestamp.
"""
if isinstance(exposures, surveysim.exposures.ExposureList):
exposures = exposures._exposures[:exposures.nexp]
nexp = len(exposures)
MJD = exposures['MJD']
if not np.all(np.diff(MJD) > 0):
raise ValueError("Input is not sorted by increasing MJD!")
if exptime is None:
exptime = {'flat': 10.0, 'arc': 10.0, 'dark': 1000.0, 'zero': 0.0}
# Define the start of night calibration sequence.
calib_time = lambda x: exptime[x] + readout
calib_sequence = (['arc']*arcs_per_night + ['flat']*flats_per_night +
['dark']*darks_per_night + ['zero']*zeroes_per_night)
calib_times = np.cumsum(np.array([calib_time(c) for c in calib_sequence]))[::-1]
# Group exposures by night.
MJD0 = desisurvey.utils.local_noon_on_date(desisurvey.utils.get_date(MJD[0])).mjd
night_idx = np.floor(MJD - MJD0).astype(int)
nights = np.unique(night_idx)
ncalib = len(calib_sequence) * len(nights)
# Initialize the output table.
output = astropy.table.Table()
nout = nexp + ncalib
output['EXPID'] = np.arange(nout, dtype=np.int32)
template = astropy.table.Table(dtype=exposures.dtype)
for colname in template.colnames:
col = template[colname]
output[colname] = astropy.table.Column(dtype=col.dtype, length=nout)
output['PROGRAM'] = astropy.table.Column(dtype=(str, len('BRIGHT')), length=nout)
output['NIGHT'] = astropy.table.Column(dtype=(str, len('YYYYMMDD')), length=nout)
output['FLAVOR'] = astropy.table.Column(dtype=(str, len('science')), length=nout)
tiles = desisurvey.tiles.get_tiles()
# Moon parameters are hardcoded for now.
output['MOONFRAC'] = 0.5
output['MOONALT'] = -10.
output['MOONSEP'] = 90.
# Loop over nights.
out_idx = 0
for n in nights:
sel = (night_idx == n)
nsel = np.count_nonzero(sel)
first = np.where(sel)[0][0]
MJD_first = MJD[first]
NIGHT = desisurvey.utils.get_date(MJD_first).isoformat().replace('-', '')
# Append the calibration sequence.
for j, c in enumerate(calib_sequence):
output['MJD'][out_idx] = MJD_first - calib_times[j]/86400.0
output['EXPTIME'][out_idx] = exptime[c]
output['TILEID'][out_idx] = -1
output['PROGRAM'][out_idx] = 'CALIB'
output['NIGHT'][out_idx] = NIGHT
output['FLAVOR'][out_idx] = c
out_idx += 1
# Append the night's science exposures.
outslice = slice(out_idx, out_idx + nsel)
for colname in template.colnames:
output[colname][outslice] = exposures[colname][sel]
TILEIDs = exposures['TILEID'][sel]
output['PROGRAM'][outslice] = [
tiles.pass_program[p] for p in tiles.passnum[tiles.index(TILEIDs)]]
output['NIGHT'][outslice] = NIGHT
output['FLAVOR'][outslice] = 'science'
out_idx += nsel
assert out_idx == nout
log = desiutil.log.get_logger()
log.info('Added {} nightly calibration sequences of {} exposures each to {} science exposures.'
.format(len(nights), len(calib_sequence), nexp))
return output
def calculate_initial_plan(args):
"""Calculate the initial survey plan.
Use :func:`desisurvey.plan.load_weather` and
:func:`desisurvey.plan.load_design_hourangles` to retrieve
these data from the saved plan.
Parameters
----------
args : object
Object with attributes for parsed command-line arguments.
"""
log = desiutil.log.get_logger()
config = desisurvey.config.Configuration()
tiles = desisurvey.tiles.get_tiles()
ephem = desisurvey.ephem.get_ephem()
# Initialize the output file to write.
hdus = fits.HDUList()
hdr = fits.Header()
# Calculate average weather factors for each day covered by
# the ephemerides.
first = desisurvey.ephem.START_DATE
last = desisurvey.ephem.STOP_DATE
years = np.arange(2007, 2018)
fractions = []
for year in years:
fractions.append(
desimodel.weather.dome_closed_fractions(first, last, replay='Y{}'.format(year)))
weather = 1 - np.mean(fractions, axis=0)
# Save the weather fractions as the primary HDU.
hdr['FIRST'] = first.isoformat()
hdr['YEARS'] = ','.join(['{}'.format(yr) for yr in years])
start = config.first_day()
stop = config.last_day()
assert start >= first and stop <= last
hdr['START'] = start.isoformat()
hdr['STOP'] = stop.isoformat()
hdr['TWILIGHT'] = args.include_twilight
hdus.append(fits.ImageHDU(weather, header=hdr, name='WEATHER'))
# Calculate the distribution of available LST in each program
# during the nominal survey [start, stop).
ilo, ihi = (start - first).days, (stop - first).days
lst_hist, lst_bins = ephem.get_available_lst(
nbins=args.nbins, weather=weather[ilo:ihi], include_twilight=args.include_twilight)
# Initialize the output results table.
design = astropy.table.Table()
design['INIT'] = np.zeros(tiles.ntiles)
design['HA'] = np.zeros(tiles.ntiles)
design['TEXP'] = np.zeros(tiles.ntiles)
# Optimize each program separately.
stretches = dict(
DARK=args.dark_stretch,
GRAY=args.gray_stretch,
BRIGHT=args.bright_stretch)
for pindex, program in enumerate(tiles.PROGRAMS):
sel = tiles.program_mask[program]
if not np.any(sel):
log.info('Skipping {} program with no tiles.'.format(program))
continue
# Initialize an LST summary table.
table = astropy.table.Table(meta={'ORIGIN': lst_bins[0]})
table['AVAIL'] = lst_hist[pindex]
# Initailize an optimizer for this program.
opt = desisurvey.optimize.Optimizer(
program, lst_bins, lst_hist[pindex], init=args.init, center=None,
stretch=stretches[program])
table['INIT'] = opt.plan_hist.copy()
design['INIT'][sel] = opt.ha_initial
# Initialize annealing cycles.
ncycles = 0
binsize = 360. / args.nbins
frac = args.adjust / binsize
smoothing = args.smooth
# Loop over annealing cycles.
while ncycles < args.max_cycles:
start_score = opt.eval_score(opt.plan_hist)
for i in range(opt.ntiles):
opt.improve(frac)
if smoothing > 0:
opt.smooth(alpha=smoothing)
stop_score = opt.eval_score(opt.plan_hist)
delta = (stop_score - start_score) / start_score
RMSE = opt.RMSE_history[-1]
loss = opt.loss_history[-1]
log.info(
'[{:03d}] dHA={:5.3f}deg '.format(ncycles + 1, frac * binsize) +
'RMSE={:6.2f}% LOSS={:5.2f}% delta(score)={:+5.1f}%'
.format(1e2*RMSE, 1e2*loss, 1e2*delta))
# Both conditions must be satisfied to terminate.
if RMSE < args.max_rmse and delta > -args.epsilon:
break
# Anneal parameters for next cycle.
frac *= args.anneal
smoothing *= args.anneal
ncycles += 1
plan_sum = opt.plan_hist.sum()
avail_sum = opt.lst_hist_sum
margin = (avail_sum - plan_sum) / plan_sum
log.info('{} plan uses {:.1f}h with {:.1f}h avail ({:.1f}% margin).'
.format(program, plan_sum, avail_sum, 1e2 * margin))
# Save planned LST usage.
table['PLAN'] = opt.plan_hist
hdus.append(fits.BinTableHDU(table, name=program))
# Calculate exposure times in (solar) seconds.
texp, _ = opt.get_exptime(opt.ha)
texp *= 24. * 3600. / 360. * 0.99726956583
# Save results for this program.
design['HA'][sel] = opt.ha
design['TEXP'][sel] = texp
hdus.append(fits.BinTableHDU(design, name='DESIGN'))
fullname = config.get_path(args.save)
hdus.writeto(fullname, overwrite=True)
log.info('Saved initial plan to "{}".'.format(fullname))
