def main(args): """Command-line driver for initializing the survey plan. """ # Set up the logger if args.debug: log = desiutil.log.get_logger(desiutil.log.DEBUG) args.verbose = True elif args.verbose: log = desiutil.log.get_logger(desiutil.log.INFO) else: log = desiutil.log.get_logger(desiutil.log.WARNING) # Set the output path if requested. config = desisurvey.config.Configuration(file_name=args.config_file) if args.output_path is not None: config.set_output_path(args.output_path) if args.tiles_file is not None: config.tiles_file.set_value(args.tiles_file) # Tabulate emphemerides if necessary. ephem = desisurvey.ephem.get_ephem(use_cache=not args.recalc) # Calculate design hour angles if necessary. fullname = config.get_path(args.save) if args.recalc or not os.path.exists(fullname): calculate_initial_plan(args) else: log.info('Initial plan has already been created.')
def run_assemble_fibermap(rawfile, outdir): '''Run assemble_fibermap using NIGHT, EXPID, and TILE from input raw data file Args: rawfile: input desi-EXPID.fits.fz raw data file outdir: directory to write fibermap-EXPID.fits files Returns: path to written fibermap ''' hdr = fitsio.read_header(rawfile, 1) night, expid = get_night_expid_header(hdr) log = desiutil.log.get_logger() if 'TILEID' in hdr: if not os.path.isdir(outdir): log.info('Creating {}'.format(outdir)) os.makedirs(outdir, exist_ok=True) fibermap = os.path.join(outdir, 'fibermap-{:08d}.fits'.format(expid)) cmd = f'assemble_fibermap -n {night} -e {expid} -o {fibermap} --overwrite' logfile = '{}/assemble_fibermap-{:08d}.log'.format(outdir, expid) msg = 'assemble_fibermap {}/{}'.format(night, expid) err = runcmd(cmd, logfile, msg) return fibermap return None
def save(self, name='stats.fits', comment='', overwrite=True): """Save a snapshot of these statistics as a binary FITS table. The saved file size is ~800 Kb. Parameters ---------- name : str File name to write. Will be located in the configuration output path unless it is an absolute path. Pass the same name to the constructor's ``restore`` argument to restore this snapshot. comment : str Comment to include in the saved header, for documentation purposes. overwrite : bool Silently overwrite any existing file when True. """ hdus = astropy.io.fits.HDUList() header = astropy.io.fits.Header() header['TILES'] = self.tiles.tiles_file header['START'] = self.start_date.isoformat() header['STOP'] = self.stop_date.isoformat() header['COMMENT'] = comment header['EXTNAME'] = 'STATS' hdus.append(astropy.io.fits.PrimaryHDU()) hdus.append(astropy.io.fits.BinTableHDU(self._data, header=header, name='STATS')) config = desisurvey.config.Configuration() fullname = config.get_path(name) hdus.writeto(fullname, overwrite=overwrite) log = desiutil.log.get_logger() log.info('Saved stats to {}'.format(fullname)) if comment: log.info('Saved with comment "{}".'.format(header['COMMENT']))
def __init__(self, start_date=None, stop_date=None, restore=None, tiles_file=None, bgs_footprint=None): if tiles_file is None: self.tiles = desisurvey.tiles.Tiles(bgs_footprint=bgs_footprint) else: self.tiles = desisurvey.tiles.Tiles(tiles_file=tiles_file, bgs_footprint=bgs_footprint) config = desisurvey.config.Configuration() if start_date is None: self.start_date = config.first_day() else: self.start_date = desisurvey.utils.get_date(start_date) if stop_date is None: self.stop_date = config.last_day() else: self.stop_date = desisurvey.utils.get_date(stop_date) self.num_nights = (self.stop_date - self.start_date).days if self.num_nights <= 0: raise ValueError('Expected start_date < stop_date.') # Build our internal array. dtype = [] for name in 'MJD', 'tsched', : dtype.append((name, np.float)) nprograms = len(self.tiles.PROGRAMS) for name in 'topen', 'tdead', : dtype.append((name, np.float, (nprograms, ))) for name in 'tscience', 'tsetup', 'tsplit', : dtype.append((name, np.float, (self.tiles.npasses, ))) for name in 'completed', 'nexp', 'nsetup', 'nsplit', 'nsetup_abort', 'nsplit_abort', : dtype.append((name, np.int32, (self.tiles.npasses, ))) self._data = np.zeros(self.num_nights, dtype) if restore is not None: # Restore array contents from a FITS file. fullname = config.get_path(restore) with astropy.io.fits.open(fullname, memmap=None) as hdus: header = hdus[1].header comment = header['COMMENT'] if header['TILES'] != self.tiles.tiles_file: raise ValueError('Header mismatch for TILES.') if header['START'] != self.start_date.isoformat(): raise ValueError('Header mismatch for START.') if header['STOP'] != self.stop_date.isoformat(): raise ValueError('Header mismatch for STOP.') self._data[:] = hdus['STATS'].data log = desiutil.log.get_logger() log.info('Restored stats from {}'.format(fullname)) if comment: log.info(' Comment: "{}".'.format(comment)) else: # Initialize local-noon MJD timestamp for each night. first_noon = desisurvey.utils.local_noon_on_date( self.start_date).mjd self._data['MJD'] = first_noon + np.arange(self.num_nights)
def update(iframe): if (iframe + 1) % args.log_interval == 0: log.info('Drawing frame {0}/{1}.'.format(iframe + 1, nframes)) if args.nightly: while not animator.draw_exposure(iexp[0], nightly=True): iexp[0] += 1 else: animator.draw_exposure(iexp=iframe, nightly=False) return animator.artists
def update(iframe): if (iframe + 1) % args.log_interval == 0: log.info('Drawing frame {0}/{1}.' .format(iframe + 1, nframes)) if args.nightly: while not animator.draw_exposure(iexp[0], nightly=True): iexp[0] += 1 else: animator.draw_exposure(iexp=iframe, nightly=False) return animator.artists
def load(name, extra_nexp=0): config = desisurvey.config.Configuration() name = config.get_path(name) with astropy.io.fits.open(name) as hdus: header = hdus[0].header comment = header['COMMENT'] nexp = header['NEXP'] max_nexp = nexp + extra_nexp explist = ExposureList(max_nexp=max_nexp) explist._exposures[:nexp] = hdus['EXPOSURES'].data explist._tiledata[:] = hdus['TILEDATA'].data log = desiutil.log.get_logger() log.info('Loaded {} exposures from {}'.format(nexp, name)) if comment: log.info('Loaded with comment "{}".'.format(comment)) return explist
def __init__(self, start_date=None, stop_date=None, restore=None): self.tiles = desisurvey.tiles.Tiles() config = desisurvey.config.Configuration() if start_date is None: self.start_date = config.first_day() else: self.start_date = desisurvey.utils.get_date(start_date) if stop_date is None: self.stop_date = config.last_day() else: self.stop_date = desisurvey.utils.get_date(stop_date) self.num_nights = (self.stop_date - self.start_date).days if self.num_nights <= 0: raise ValueError('Expected start_date < stop_date.') # Build our internal array. dtype = [] for name in 'MJD', 'tsched',: dtype.append((name, np.float)) nprograms = len(self.tiles.PROGRAMS) for name in 'topen', 'tdead',: dtype.append((name, np.float, (nprograms,))) for name in 'tscience', 'tsetup', 'tsplit',: dtype.append((name, np.float, (self.tiles.npasses,))) for name in 'completed', 'nexp', 'nsetup', 'nsplit', 'nsetup_abort', 'nsplit_abort',: dtype.append((name, np.int32, (self.tiles.npasses,))) self._data = np.zeros(self.num_nights, dtype) if restore is not None: # Restore array contents from a FITS file. fullname = config.get_path(restore) with astropy.io.fits.open(fullname, memmap=None) as hdus: header = hdus[1].header comment = header['COMMENT'] if header['TILES'] != self.tiles.tiles_file: raise ValueError('Header mismatch for TILES.') if header['START'] != self.start_date.isoformat(): raise ValueError('Header mismatch for START.') if header['STOP'] != self.stop_date.isoformat(): raise ValueError('Header mismatch for STOP.') self._data[:] = hdus['STATS'].data log = desiutil.log.get_logger() log.info('Restored stats from {}'.format(fullname)) if comment: log.info(' Comment: "{}".'.format(comment)) else: # Initialize local-noon MJD timestamp for each night. first_noon = desisurvey.utils.local_noon_on_date(self.start_date).mjd self._data['MJD'] = first_noon + np.arange(self.num_nights)
def main(args=None): ''' TODO: document Note: this bypasses specsim since we don't have an arclamp model in surface brightness units; we only have electrons on the CCD ''' import desiutil.log log = desiutil.log.get_logger() from desiutil.iers import freeze_iers freeze_iers() if isinstance(args, (list, tuple, type(None))): args = parse(args) log.info('reading arc data from {}'.format(args.arcfile)) arcdata = astropy.table.Table.read(args.arcfile) wave, phot, fibermap = \ desisim.simexp.simarc(arcdata, nspec=args.nspec, nonuniform=args.nonuniform) log.info('Writing {}'.format(args.fibermap)) fibermap.meta['NIGHT'] = args.night fibermap.meta['EXPID'] = args.expid fibermap.meta['EXTNAME'] = 'FIBERMAP' fibermap.write(args.fibermap, overwrite=args.clobber) #- TODO: explain bypassing desisim.io.write_simspec header = fits.Header() desiutil.depend.add_dependencies(header) header['EXPID'] = args.expid header['NIGHT'] = args.night header['FLAVOR'] = 'arc' header['DOSVER'] = 'SIM' header['EXPTIME'] = 5 #- TODO: add exptime support #- TODO: DATE-OBS on night instead of now tx = astropy.time.Time(datetime.datetime(*time.gmtime()[0:6])) header['DATE-OBS'] = tx.utc.isot desisim.io.write_simspec_arc(args.simspec, wave, phot, header, fibermap, overwrite=args.clobber)
def main(args=None): ''' Generates a new flat exposure; see newflat --help for usage options ''' import desiutil.log log = desiutil.log.get_logger() from desiutil.iers import freeze_iers freeze_iers() if isinstance(args, (list, tuple, type(None))): args = parse(args) sim, fibermap = \ desisim.simexp.simflat(args.flatfile, nspec=args.nspec, nonuniform=args.nonuniform) log.info('Writing {}'.format(args.fibermap)) fibermap.meta['NIGHT'] = args.night fibermap.meta['EXPID'] = args.expid fibermap.meta['EXTNAME'] = 'FIBERMAP' fibermap.write(args.fibermap, overwrite=args.clobber) header = fits.Header() desiutil.depend.add_dependencies(header) header['EXPID'] = args.expid header['NIGHT'] = args.night header['FLAVOR'] = 'flat' header['DOSVER'] = 'SIM' #- Set calibrations as happening at 15:00 AZ local time = 22:00 UTC year = int(args.night[0:4]) month = int(args.night[4:6]) day = int(args.night[6:8]) tx = astropy.time.Time(datetime.datetime(year, month, day, 22, 0, 0)) header['DATE-OBS'] = tx.utc.isot #- metadata truth and obs dictionary are None desisim.io.write_simspec(sim, None, fibermap, None, args.expid, args.night, filename=args.simspec, header=header, overwrite=args.clobber)
def __init__(self, restore=None, max_nexp=60000, existing_exposures=None): self.tiles = desisurvey.tiles.get_tiles() self._exposures = np.empty(max_nexp, dtype=[ ('EXPID', np.int32), ('MJD', np.float64), ('EXPTIME', np.float32), ('TILEID', np.int32), ('SNR2FRAC', np.float32), ('DSNR2FRAC', np.float32), ('AIRMASS', np.float32), ('SEEING', np.float32), ('TRANSP', np.float32), ('SKY', np.float32), ]) self._tiledata = np.empty(self.tiles.ntiles, dtype=[ ('AVAIL', np.int32), ('PLANNED', np.int32), ('EXPTIME', np.float32), ('SNR2FRAC', np.float32), ('NEXP', np.int32) ]) if restore is not None: config = desisurvey.config.Configuration() fullname = config.get_path(restore) with astropy.io.fits.open(fullname, memmap=False) as hdus: header = hdus[0].header comment = header['COMMENT'] self.nexp = header['NEXP'] self._exposures[:self.nexp] = hdus['EXPOSURES'].data self._tiledata[:] = hdus['TILEDATA'].data self.initial_night = desisurvey.utils.get_date(header['INITIAL']) if header['INITIAL'] else None log = desiutil.log.get_logger() log.info('Restored stats from {}'.format(fullname)) if comment: log.info(' Comment: "{}".'.format(comment)) else: self.nexp = 0 self._tiledata['AVAIL'] = -1 self._tiledata['PLANNED'] = -1 self._tiledata['EXPTIME'] = 0. self._tiledata['SNR2FRAC'] = 0. self._tiledata['NEXP'] = 0 self.initial_night = None if existing_exposures is not None: self.add_existing_exposures(existing_exposures)
def save(self, name='exposures.fits', comment='', overwrite=True): """Save exposures to a FITS file with two binary tables. The saved file size scales linearly with the number of exposures added so far, and is independent of the memory size of this object. Parameters ---------- name : str File name to write. Will be located in the configuration output path unless it is an absolute path. Pass the same name to the constructor's ``restore`` argument to restore this snapshot. comment : str Comment to include in the saved header, for documentation purposes. overwrite : bool Silently overwrite any existing file when True. """ hdus = astropy.io.fits.HDUList() header = astropy.io.fits.Header() header['TILES'] = self.tiles.tiles_file header['NEXP'] = self.nexp header['COMMENT'] = comment header['INITIAL'] = self.initial_night.isoformat( ) if self.initial_night else '' header['EXTNAME'] = 'META' hdus.append(astropy.io.fits.PrimaryHDU(header=header)) hdus.append( astropy.io.fits.BinTableHDU(self._exposures[:self.nexp], name='EXPOSURES')) hdus.append( astropy.io.fits.BinTableHDU(self._tiledata, name='TILEDATA')) config = desisurvey.config.Configuration() name = config.get_path(name) hdus.writeto(name, overwrite=overwrite) log = desiutil.log.get_logger() log.info('Saved {} exposures to {}'.format(self.nexp, name)) if comment: log.info('Saved with comment "{}".'.format(header['COMMENT']))
def main(args=None): ''' TODO: document Note: this bypasses specsim since we don't have an arclamp model in surface brightness units; we only have electrons on the CCD ''' import desiutil.log log = desiutil.log.get_logger() if isinstance(args, (list, tuple, type(None))): args = parse(args) log.info('reading arc data from {}'.format(args.arcfile)) arcdata = astropy.table.Table.read(args.arcfile) wave, phot, fibermap = \ desisim.simexp.simarc(arcdata, nspec=args.nspec, nonuniform=args.nonuniform) log.info('Writing {}'.format(args.fibermap)) fibermap.meta['NIGHT'] = args.night fibermap.meta['EXPID'] = args.expid fibermap.meta['EXTNAME'] = 'FIBERMAP' fibermap.write(args.fibermap, overwrite=args.clobber) #- TODO: explain bypassing desisim.io.write_simspec header = fits.Header() desiutil.depend.add_dependencies(header) header['EXPID'] = args.expid header['NIGHT'] = args.night header['FLAVOR'] = 'arc' header['DOSVER'] = 'SIM' header['EXPTIME'] = 5 #- TODO: add exptime support #- TODO: DATE-OBS on night instead of now tx = astropy.time.Time(datetime.datetime(*time.gmtime()[0:6])) header['DATE-OBS'] = tx.utc.isot desisim.io.write_simspec_arc(args.simspec, wave, phot, header, fibermap, overwrite=args.clobber)
def save(self, name='stats.fits', comment='', overwrite=True): """Save a snapshot of these statistics as a binary FITS table. The saved file size is ~800 Kb. Parameters ---------- name : str File name to write. Will be located in the configuration output path unless it is an absolute path. Pass the same name to the constructor's ``restore`` argument to restore this snapshot. comment : str Comment to include in the saved header, for documentation purposes. overwrite : bool Silently overwrite any existing file when True. """ hdus = astropy.io.fits.HDUList() header = astropy.io.fits.Header() header['TILES'] = self.tiles.tiles_file header['START'] = self.start_date.isoformat() header['STOP'] = self.stop_date.isoformat() header['COMMENT'] = comment header['EXTNAME'] = 'STATS' hdus.append(astropy.io.fits.PrimaryHDU()) hdus.append( astropy.io.fits.BinTableHDU(self._data, header=header, name='STATS')) config = desisurvey.config.Configuration() fullname = config.get_path(name) hdus.writeto(fullname, overwrite=overwrite) log = desiutil.log.get_logger() log.info('Saved stats to {}'.format(fullname)) if comment: log.info('Saved with comment "{}".'.format(header['COMMENT']))
def main(args=None): ''' Generates a new flat exposure; see newflat --help for usage options ''' import desiutil.log log = desiutil.log.get_logger() if isinstance(args, (list, tuple, type(None))): args = parse(args) sim, fibermap = \ desisim.simexp.simflat(args.flatfile, nspec=args.nspec, nonuniform=args.nonuniform) log.info('Writing {}'.format(args.fibermap)) fibermap.meta['NIGHT'] = args.night fibermap.meta['EXPID'] = args.expid fibermap.meta['EXTNAME'] = 'FIBERMAP' fibermap.write(args.fibermap, overwrite=args.clobber) header = fits.Header() desiutil.depend.add_dependencies(header) header['EXPID'] = args.expid header['NIGHT'] = args.night header['FLAVOR'] = 'flat' header['DOSVER'] = 'SIM' #- Set calibrations as happening at 15:00 AZ local time = 22:00 UTC year = int(args.night[0:4]) month = int(args.night[4:6]) day = int(args.night[6:8]) tx = astropy.time.Time(datetime.datetime(year, month, day, 22, 0, 0)) header['DATE-OBS'] = tx.utc.isot #- metadata truth and obs dictionary are None desisim.io.write_simspec(sim, None, fibermap, None, args.expid, args.night, filename=args.simspec, header=header, overwrite=args.clobber)
def get_ephem(use_cache=True, write_cache=True): """Return tabulated ephemerides for (START_DATE,STOP_DATE). The pyephem module must be installed to calculate ephemerides, but is not necessary when a FITS file of precalcuated data is available. Parameters ---------- use_cache : bool Use cached ephemerides from memory or disk if possible when True. Otherwise, always calculate from scratch. write_cache : bool When True, write a generated table so it is available for future invocations. Writing only takes place when a cached object is not available or ``use_cache`` is False. Returns ------- Ephemerides Object with tabulated ephemerides for (START_DATE,STOP_DATE). """ global _ephem # Freeze IERS table for consistent results. desisurvey.utils.freeze_iers() # Use standardized string representation of dates. start_iso = START_DATE.isoformat() stop_iso = STOP_DATE.isoformat() range_iso = '({},{})'.format(start_iso, stop_iso) log = desiutil.log.get_logger() # First check for a cached object in memory. if use_cache and _ephem is not None: if _ephem.start_date != START_DATE or _ephem.stop_date != STOP_DATE: raise RuntimeError('START_DATE, STOP_DATE have changed.') log.debug('Returning cached ephemerides for {}.'.format(range_iso)) return _ephem # Next check for a FITS file on disk. config = desisurvey.config.Configuration() filename = config.get_path('ephem_{}_{}.fits'.format(start_iso, stop_iso)) if use_cache and os.path.exists(filename): # Save restored object in memory. _ephem = Ephemerides(START_DATE, STOP_DATE, restore=filename) log.info('Restored ephemerides for {} from {}.' .format(range_iso, filename)) return _ephem # Finally, create new ephemerides and save in the memory cache. log.info('Building ephemerides for {}...'.format(range_iso)) _ephem = Ephemerides(START_DATE, STOP_DATE) if write_cache: # Save the tabulated ephemerides to disk. _ephem._table.write(filename, overwrite=True) log.info('Saved ephemerides for {} to {}'.format(range_iso, filename)) return _ephem
def update_iers(save_name='iers_frozen.ecsv', num_avg=1000): """Update the IERS table used by astropy time, coordinates. Downloads the current IERS-A table, replaces the last entry (which is repeated for future times) with the average of the last ``num_avg`` entries, and saves the table in ECSV format. This should only be called every few months, e.g., with major releases. The saved file should then be copied to this package's data/ directory and committed to the git repository. Requires a network connection in order to download the current IERS-A table. Prints information about the update process. The :func:`desisurvey.utils.plot_iers` function is useful for inspecting IERS tables and how they are extrapolated to DESI survey dates. Parameters ---------- save_name : str Name where frozen IERS table should be saved. Must end with the .ecsv extension. num_avg : int Number of rows from the end of the current table to average and use for calculating UT1-UTC offsets and polar motion at times beyond the table. """ log = desiutil.log.get_logger() # Validate the save_name extension. _, ext = os.path.splitext(save_name) if ext != '.ecsv': raise ValueError('Expected .ecsv extension for {0}.'.format(save_name)) # Download the latest IERS_A table iers = astropy.utils.iers.IERS_A.open(astropy.utils.iers.IERS_A_URL) last = astropy.time.Time(iers['MJD'][-1], format='mjd').datetime log.info('Updating to current IERS-A table with coverage up to {0}.' .format(last.date())) # Loop over the columns used by the astropy IERS routines. for name in 'UT1_UTC', 'PM_x', 'PM_y': # Replace the last entry with the mean of recent samples. mean_value = np.mean(iers[name][-num_avg:].value) unit = iers[name].unit iers[name][-1] = mean_value * unit log.info('Future {0:7s} = {1:.3}'.format(name, mean_value * unit)) # Strip the original table metadata since ECSV cannot handle it. # We only need a single keyword that is checked by IERS_Auto.open(). iers.meta = dict(data_url='frozen') # Save the table. The IERS-B table provided with astropy uses the # ascii.cds format but astropy cannot write this format. iers.write(save_name, format='ascii.ecsv', overwrite=True) log.info('Wrote updated table to {0}.'.format(save_name))
def run_preproc(rawfile, outdir, fibermap=None, ncpu=None, cameras=None): '''Runs preproc on the input raw data file, outputting to outdir Args: rawfile: input desi-EXPID.fits.fz raw data file outdir: directory to write preproc-CAM-EXPID.fits files Options: fibermap : path to fibermap-EXPID.fits file ncpu: number of CPU cores to use for parallelism; serial if ncpu<=1 cameras: list of cameras to process; default all found in rawfile Returns header of HDU 0 of the input raw data file ''' if not os.path.exists(rawfile): raise ValueError("{} doesn't exist".format(rawfile)) log = desiutil.log.get_logger() if not os.path.isdir(outdir): log.info('Creating {}'.format(outdir)) os.makedirs(outdir, exist_ok=True) if cameras is None: cameras = which_cameras(rawfile) header = fitsio.read_header(rawfile, 0) arglist = list() for camera in cameras: args = [ '--infile', rawfile, '--outdir', outdir, '--fibermap', fibermap, '--cameras', camera ] arglist.append(args) ncpu = min(len(arglist), get_ncpu(ncpu)) if ncpu > 1: log.info( 'Running preproc in parallel on {} cores for {} cameras'.format( ncpu, len(cameras))) pool = mp.Pool(ncpu) pool.map(desispec.scripts.preproc.main, arglist) pool.close() pool.join() else: log.info('Running preproc serially for {} cameras'.format( len(cameras))) for args in arglist: desispec.scripts.preproc.main(args) return header
def get_tiles(tiles_file=None, use_cache=True, write_cache=True): """Return a Tiles object with optional caching. You should normally always use the default arguments to ensure that tiles are defined consistently and efficiently between different classes. Parameters ---------- tiles_file : str or None Use the specified name to override config.tiles_file. use_cache : bool Use tiles previously cached in memory when True. Otherwise, (re)load tiles from disk. write_cache : bool If tiles need to be loaded from disk with this call, save them in a memory cache for future calls. """ global _cached_tiles log = desiutil.log.get_logger() config = desisurvey.config.Configuration() tiles_file = tiles_file or config.tiles_file() if use_cache and tiles_file in _cached_tiles: tiles = _cached_tiles[tiles_file] log.debug('Using cached tiles for "{}".'.format(tiles_file)) else: tiles = Tiles(tiles_file) log.info('Initialized tiles from "{}".'.format(tiles_file)) for pname in Tiles.PROGRAMS: pinfo = [] for passnum in tiles.program_passes[pname]: pinfo.append('{}({})'.format(passnum, tiles.pass_ntiles[passnum])) log.info('{:6s} passes(tiles): {}.'.format(pname, ', '.join(pinfo))) if write_cache: _cached_tiles[tiles_file] = tiles else: log.info('Tiles not cached for "{}".'.format(tiles_file)) return tiles
def run_qproc(rawfile, outdir, ncpu=None, cameras=None): ''' Determine the obstype of the rawfile, and run qproc with appropriate options Args: rawfile: input desi-EXPID.fits.fz raw data file outdir: directory to write qproc-CAM-EXPID.fits files Options: ncpu: number of CPU cores to use for parallelism; serial if ncpu<=1 cameras: list of cameras to process; default all found in rawfile Returns header of HDU 0 of the input raw data file, plus dictionary of return codes for each qproc process run. ''' log = desiutil.log.get_logger() if not os.path.isdir(outdir): log.info('Creating {}'.format(outdir)) os.makedirs(outdir, exist_ok=True) hdr = fitsio.read_header(rawfile, 0) if ('OBSTYPE' not in hdr) and ('FLAVOR' not in hdr): log.warning( "no obstype nor flavor keyword in first hdu header, moving to the next one" ) try: hdr = fitsio.read_header(rawfile, 1) except OSError as err: log.error("fitsio error reading HDU 1, trying 2 then giving up") hdr = fitsio.read_header(rawfile, 2) try: if 'OBSTYPE' in hdr: obstype = hdr['OBSTYPE'].rstrip().upper() else: log.warning('Use FLAVOR instead of missing OBSTYPE') obstype = hdr['FLAVOR'].rstrip().upper() night, expid = get_night_expid_header(hdr) except KeyError as e: log.error(str(e)) raise (e) #- copy coordfile to new folder for pos accuracy indir = os.path.abspath(os.path.dirname(rawfile)) coord_infile = '{}/coordinates-{:08d}.fits'.format(indir, expid) coord_outfile = '{}/coordinates-{:08d}.fits'.format(outdir, expid) print(coord_infile) if os.path.isfile(coord_infile): print('copying coordfile') copyfile(coord_infile, coord_outfile) else: log.warning('No coordinate file for positioner accuracy') #- HACK: Workaround for data on 20190626/27 that have blank NIGHT keywords #- Note: get_night_expid_header(hdr) should take care of this now, but #- this is left in for robustness just in case if night == ' ' or night is None: log.error( 'Correcting blank NIGHT keyword based upon directory structure') #- /path/to/NIGHT/EXPID/rawfile.fits night = os.path.basename( os.path.dirname(os.path.dirname(os.path.abspath(rawfile)))) if re.match('20\d{6}', night): log.info('Setting NIGHT to {}'.format(night)) else: raise RuntimeError('Unable to derive NIGHT for {}'.format(rawfile)) cmdlist = list() loglist = list() msglist = list() rawcameras = which_cameras(rawfile) if cameras is None: cameras = rawcameras elif len(set(cameras) - set(rawcameras)) > 0: missing_cameras = set(cameras) - set(rawcameras) for cam in sorted(missing_cameras): log.error('{} missing camera {}'.format(os.path.basename(rawfile), cam)) cameras = sorted(set(cameras) & set(rawcameras)) for camera in cameras: outfiles = dict( rawfile=rawfile, fibermap='{}/fibermap-{:08d}.fits'.format(outdir, expid), logfile='{}/qproc-{}-{:08d}.log'.format(outdir, camera, expid), outdir=outdir, camera=camera) cmd = "desi_qproc -i {rawfile} --fibermap {fibermap} --auto --auto-output-dir {outdir} --cam {camera}".format( **outfiles) cmdlist.append(cmd) loglist.append(outfiles['logfile']) msglist.append('qproc {}/{} {}'.format(night, expid, camera)) ncpu = min(len(cmdlist), get_ncpu(ncpu)) if ncpu > 1 and len(cameras) > 1: log.info('Running qproc in parallel on {} cores for {} cameras'.format( ncpu, len(cameras))) pool = mp.Pool(ncpu) errs = pool.starmap(runcmd, zip(cmdlist, loglist, msglist)) pool.close() pool.join() else: errs = [] log.info('Running qproc serially for {} cameras'.format(len(cameras))) for cmd, logfile, msg in zip(cmdlist, loglist, msglist): err = runcmd(cmd, logfile, msg) errs.append(err) errorcodes = dict() for err in errs: for key in err.keys(): errorcodes[key] = err[key] jsonfile = '{}/errorcodes-{:08d}.txt'.format(outdir, expid) with open(jsonfile, 'w') as outfile: json.dump(errorcodes, outfile) print('Wrote {}'.format(jsonfile)) return hdr
def make_plots(infile, basedir, preprocdir=None, logdir=None, rawdir=None, cameras=None): '''Make plots for a single exposure Args: infile: input QA fits file with HDUs like PER_AMP, PER_FIBER, ... basedir: write output HTML to basedir/NIGHT/EXPID/ Options: preprocdir: directory to where the "preproc-*-*.fits" are located. If not provided, function will NOT generate any image files from any preproc fits file. logdir: directory to where the "qproc-*-*.log" are located. If not provided, function will NOT display any logfiles. rawdir: directory to where the raw data files are located, including "guide-rois-*.fits" and "centroid-*.json" files, are located. If not provided, the function will not plot the guide plots. cameras: list of cameras (strings) to generate image files of. If not provided, will generate a cameras list from parcing through the preproc fits files in the preprocdir ''' from nightwatch.webpages import amp as web_amp from nightwatch.webpages import camfiber as web_camfiber from nightwatch.webpages import camera as web_camera from nightwatch.webpages import summary as web_summary from nightwatch.webpages import lastexp as web_lastexp from nightwatch.webpages import guide as web_guide from nightwatch.webpages import guideimage as web_guideimage from nightwatch.webpages import placeholder as web_placeholder from . import io log = desiutil.log.get_logger() qadata = io.read_qa(infile) header = qadata['HEADER'] night = header['NIGHT'] expid = header['EXPID'] #- Early data have wrong NIGHT in header; check by hand #- YEARMMDD/EXPID/infile dirnight = os.path.basename(os.path.dirname(os.path.dirname(infile))) if re.match('20\d{6}', dirnight) and dirnight != str(night): log.warning('Correcting {} header night {} to {}'.format( infile, night, dirnight)) night = int(dirnight) header['NIGHT'] = night #- Create output exposures plot directory if needed expdir = os.path.join(basedir, str(night), '{:08d}'.format(expid)) if not os.path.isdir(expdir): log.info('Creating {}'.format(expdir)) os.makedirs(expdir, exist_ok=True) if 'PER_AMP' in qadata: htmlfile = '{}/qa-amp-{:08d}.html'.format(expdir, expid) pc = web_amp.write_amp_html(htmlfile, qadata['PER_AMP'], header) print('Wrote {}'.format(htmlfile)) else: htmlfile = '{}/qa-amp-{:08d}.html'.format(expdir, expid) pc = web_placeholder.write_placeholder_html(htmlfile, header, "PER_AMP") htmlfile = '{}/qa-camfiber-{:08d}.html'.format(expdir, expid) if 'PER_CAMFIBER' in qadata: try: pc = web_camfiber.write_camfiber_html(htmlfile, qadata['PER_CAMFIBER'], header) print('Wrote {}'.format(htmlfile)) except Exception as err: web_placeholder.handle_failed_plot(htmlfile, header, "PER_CAMFIBER") else: pc = web_placeholder.write_placeholder_html(htmlfile, header, "PER_CAMFIBER") htmlfile = '{}/qa-camera-{:08d}.html'.format(expdir, expid) if 'PER_CAMERA' in qadata: try: pc = web_camera.write_camera_html(htmlfile, qadata['PER_CAMERA'], header) print('Wrote {}'.format(htmlfile)) except Exception as err: web_placeholder.handle_failed_plot(htmlfile, header, "PER_CAMERA") else: pc = web_placeholder.write_placeholder_html(htmlfile, header, "PER_CAMERA") htmlfile = '{}/qa-summary-{:08d}.html'.format(expdir, expid) web_summary.write_summary_html(htmlfile, qadata, preprocdir) print('Wrote {}'.format(htmlfile)) #- Note: last exposure goes in basedir, not expdir=basedir/NIGHT/EXPID htmlfile = '{}/qa-lastexp.html'.format(basedir) web_lastexp.write_lastexp_html(htmlfile, qadata, preprocdir) print('Wrote {}'.format(htmlfile)) if rawdir: #- plot guide metric plots try: guidedata = io.get_guide_data(night, expid, rawdir) htmlfile = '{}/qa-guide-{:08d}.html'.format(expdir, expid) web_guide.write_guide_html(htmlfile, header, guidedata) print('Wrote {}'.format(htmlfile)) except (FileNotFoundError, OSError, IOError): print('Unable to find guide data, not plotting guide plots') htmlfile = '{}/qa-guide-{:08d}.html'.format(expdir, expid) pc = web_placeholder.write_placeholder_html( htmlfile, header, "GUIDING") #- plot guide image movies try: htmlfile = '{expdir}/guide-image-{expid:08d}.html'.format( expdir=expdir, expid=expid) image_data = io.get_guide_images(night, expid, rawdir) web_guideimage.write_guide_image_html(image_data, htmlfile, night, expid) print('Wrote {}'.format(htmlfile)) except (FileNotFoundError, OSError, IOError): print('Unable to find guide data, not plotting guide image plots') htmlfile = '{expdir}/guide-image-{expid:08d}.html'.format( expdir=expdir, expid=expid) pc = web_placeholder.write_placeholder_html( htmlfile, header, "GUIDE_IMAGES") #- regardless of if logdir or preprocdir, identifying failed qprocs by comparing #- generated preproc files to generated logfiles qproc_fails = [] if cameras is None: cameras = [] import glob for preprocfile in glob.glob( os.path.join(preprocdir, 'preproc-*-*.fits')): cameras += [os.path.basename(preprocfile).split('-')[1]] log_cams = [] log_outputs = [ i for i in os.listdir(logdir) if re.match(r'qproc.*\.log', i) ] for log_output in log_outputs: l_cam = log_output.split("-")[1] log_cams += [l_cam] if l_cam not in cameras: qproc_fails.append(l_cam) from nightwatch.webpages import plotimage as web_plotimage if (preprocdir is not None): #- plot preprocessed images downsample = 4 ncpu = get_ncpu(None) pinput = os.path.join(preprocdir, "preproc-{}-{:08d}.fits") output = os.path.join(expdir, "preproc-{}-{:08d}-4x.html") argslist = [(pinput.format(cam, expid), output.format(cam, expid), downsample, night) for cam in cameras] if ncpu > 1: pool = mp.Pool(ncpu) pool.starmap(web_plotimage.write_image_html, argslist) pool.close() pool.join() else: for args in argslist: web_plotimage.write_image_html(*args) #- plot preproc nav table navtable_output = '{}/qa-amp-{:08d}-preproc_table.html'.format( expdir, expid) web_plotimage.write_preproc_table_html(preprocdir, night, expid, downsample, navtable_output) if (logdir is not None): #- plot logfiles log.debug('Log directory: {}'.format(logdir)) error_colors = dict() for log_cam in log_cams: qinput = os.path.join(logdir, "qproc-{}-{:08d}.log".format(log_cam, expid)) output = os.path.join( expdir, "qproc-{}-{:08d}-logfile.html".format(log_cam, expid)) log.debug('qproc log: {}'.format(qinput)) e = web_summary.write_logfile_html(qinput, output, night) error_colors[log_cam] = e #- plot logfile nav table htmlfile = '{}/qa-summary-{:08d}-logfiles_table.html'.format( expdir, expid) web_summary.write_logtable_html(htmlfile, logdir, night, expid, available=log_cams, error_colors=error_colors)
def initialize(ephem, start_date=None, stop_date=None, step_size=5.0, healpix_nside=16, output_name='scheduler.fits'): """Calculate exposure-time factors over a grid of times and pointings. Takes about 9 minutes to run and writes a 1.3Gb output file with the default parameters. Requires that healpy is installed. Parameters ---------- ephem : desisurvey.ephem.Ephemerides Tabulated ephemerides data to use for planning. start_date : date or None Survey planning starts on the evening of this date. Must be convertible to a date using :func:`desisurvey.utils.get_date`. Use the first night of the ephemerides when None. stop_date : date or None Survey planning stops on the morning of this date. Must be convertible to a date using :func:`desisurvey.utils.get_date`. Use the first night of the ephemerides when None. step_size : :class:`astropy.units.Quantity` Exposure-time factors are tabulated at this interval during each night. healpix_nside : int Healpix NSIDE parameter to use for binning the sky. Must be a power of two. Values larger than 16 will lead to holes in the footprint with the current implementation. output_name : str Name of the FITS output file where results are saved. A relative path refers to the :meth:`configuration output path <desisurvey.config.Configuration.get_path>`. """ import healpy if not isinstance(step_size, u.Quantity): step_size = step_size * u.min log = desiutil.log.get_logger() # Freeze IERS table for consistent results. desisurvey.utils.freeze_iers() config = desisurvey.config.Configuration() output_name = config.get_path(output_name) start_date = desisurvey.utils.get_date(start_date or config.first_day()) stop_date = desisurvey.utils.get_date(stop_date or config.last_day()) if start_date >= stop_date: raise ValueError('Expected start_date < stop_date.') mjd = ephem._table['noon'] sel = ((mjd >= desisurvey.utils.local_noon_on_date(start_date).mjd) & (mjd < desisurvey.utils.local_noon_on_date(stop_date).mjd)) t = ephem._table[sel] num_nights = len(t) # Build a grid of elapsed time relative to local midnight during each night. midnight = t['noon'] + 0.5 t_edges = desisurvey.ephem.get_grid(step_size) t_centers = 0.5 * (t_edges[1:] + t_edges[:-1]) num_points = len(t_centers) # Create an empty HDU0 with header info. header = astropy.io.fits.Header() header['START'] = str(start_date) header['STOP'] = str(stop_date) header['NSIDE'] = healpix_nside header['NPOINTS'] = num_points header['STEP'] = step_size.to(u.min).value hdus = astropy.io.fits.HDUList() hdus.append(astropy.io.fits.ImageHDU(header=header)) # Save time grid. hdus.append(astropy.io.fits.ImageHDU(name='GRID', data=t_edges)) # Load the list of tiles to observe. tiles = astropy.table.Table( desimodel.io.load_tiles(onlydesi=True, extra=False, tilesfile=config.tiles_file())) # Build the footprint as a healpix map of the requested size. # The footprint includes any pixel containing at least one tile center. npix = healpy.nside2npix(healpix_nside) footprint = np.zeros(npix, bool) pixels = healpy.ang2pix( healpix_nside, np.radians(90 - tiles['DEC'].data), np.radians(tiles['RA'].data)) footprint[np.unique(pixels)] = True footprint_pixels = np.where(footprint)[0] num_footprint = len(footprint_pixels) log.info('Footprint contains {0} pixels.'.format(num_footprint)) # Sort pixels in order of increasing phi + 60deg so that the north and south # galactic caps are contiguous in the arrays we create below. pix_theta, pix_phi = healpy.pix2ang(healpix_nside, footprint_pixels) pix_dphi = np.fmod(pix_phi + np.pi / 3, 2 * np.pi) sort_order = np.argsort(pix_dphi) footprint_pixels = footprint_pixels[sort_order] # Calculate sorted pixel (ra,dec). pix_theta, pix_phi = healpy.pix2ang(healpix_nside, footprint_pixels) pix_ra, pix_dec = np.degrees(pix_phi), 90 - np.degrees(pix_theta) # Record per-tile info needed for planning. table = astropy.table.Table() table['tileid'] = tiles['TILEID'].astype(np.int32) table['ra'] = tiles['RA'].astype(np.float32) table['dec'] = tiles['DEC'].astype(np.float32) table['EBV'] = tiles['EBV_MED'].astype(np.float32) table['pass'] = tiles['PASS'].astype(np.int16) # Map each tile ID to the corresponding index in our spatial arrays. mapper = np.zeros(npix, int) mapper[footprint_pixels] = np.arange(len(footprint_pixels)) table['map'] = mapper[pixels].astype(np.int16) # Use a small int to identify the program, ordered by sky brightness: # 1=DARK, 2=GRAY, 3=BRIGHT. table['program'] = np.full(len(tiles), 4, np.int16) for i, program in enumerate(('DARK', 'GRAY', 'BRIGHT')): table['program'][tiles['PROGRAM'] == program] = i + 1 assert np.all(table['program'] > 0) hdu = astropy.io.fits.table_to_hdu(table) hdu.name = 'TILES' hdus.append(hdu) # Average E(B-V) for all tiles falling into a pixel. tiles_per_pixel = np.bincount(pixels, minlength=npix) EBV = np.bincount(pixels, weights=tiles['EBV_MED'], minlength=npix) EBV[footprint] /= tiles_per_pixel[footprint] # Calculate dust extinction exposure-time factor. f_EBV = 1. / desisurvey.etc.dust_exposure_factor(EBV) # Save HDU with the footprint and static dust exposure map. table = astropy.table.Table() table['pixel'] = footprint_pixels table['dust'] = f_EBV[footprint_pixels] table['ra'] = pix_ra table['dec'] = pix_dec hdu = astropy.io.fits.table_to_hdu(table) hdu.name = 'STATIC' hdus.append(hdu) # Prepare a table of calendar data. calendar = astropy.table.Table() calendar['midnight'] = midnight calendar['monsoon'] = np.zeros(num_nights, bool) calendar['fullmoon'] = np.zeros(num_nights, bool) calendar['weather'] = np.zeros(num_nights, np.float32) # Hardcode annualized average weight. weather_weights = np.full(num_nights, 0.723) # Prepare a table of ephemeris data. etable = astropy.table.Table() # Program codes ordered by increasing sky brightness: # 1=DARK, 2=GRAY, 3=BRIGHT, 4=DAYTIME. etable['program'] = np.full(num_nights * num_points, 4, dtype=np.int16) etable['moon_frac'] = np.zeros(num_nights * num_points, dtype=np.float32) etable['moon_ra'] = np.zeros(num_nights * num_points, dtype=np.float32) etable['moon_dec'] = np.zeros(num_nights * num_points, dtype=np.float32) etable['moon_alt'] = np.zeros(num_nights * num_points, dtype=np.float32) etable['zenith_ra'] = np.zeros(num_nights * num_points, dtype=np.float32) etable['zenith_dec'] = np.zeros(num_nights * num_points, dtype=np.float32) # Tabulate MJD and apparent LST values for each time step. We don't save # MJD values since they are cheap to reconstruct from the index, but # do use them below. mjd0 = desisurvey.utils.local_noon_on_date(start_date).mjd + 0.5 mjd = mjd0 + np.arange(num_nights)[:, np.newaxis] + t_centers times = astropy.time.Time( mjd, format='mjd', location=desisurvey.utils.get_location()) etable['lst'] = times.sidereal_time('apparent').flatten().to(u.deg).value # Build sky coordinates for each pixel in the footprint. pix_theta, pix_phi = healpy.pix2ang(healpix_nside, footprint_pixels) pix_ra, pix_dec = np.degrees(pix_phi), 90 - np.degrees(pix_theta) pix_sky = astropy.coordinates.ICRS(pix_ra * u.deg, pix_dec * u.deg) # Initialize exposure factor calculations. alt, az = np.full(num_points, 90.) * u.deg, np.zeros(num_points) * u.deg fexp = np.zeros((num_nights * num_points, num_footprint), dtype=np.float32) vband_extinction = 0.15154 one = np.ones((num_points, num_footprint)) # Loop over nights. for i in range(num_nights): night = ephem.get_night(midnight[i]) date = desisurvey.utils.get_date(midnight[i]) if date.day == 1: log.info('Starting {0} (completed {1}/{2} nights)' .format(date.strftime('%b %Y'), i, num_nights)) # Initialize the slice of the fexp[] time index for this night. sl = slice(i * num_points, (i + 1) * num_points) # Do we expect to observe on this night? calendar[i]['monsoon'] = desisurvey.utils.is_monsoon(midnight[i]) calendar[i]['fullmoon'] = ephem.is_full_moon(midnight[i]) # Look up expected dome-open fraction due to weather. calendar[i]['weather'] = weather_weights[i] # Calculate the program during this night (default is 4=DAYTIME). mjd = midnight[i] + t_centers dark, gray, bright = ephem.tabulate_program(mjd) etable['program'][sl][dark] = 1 etable['program'][sl][gray] = 2 etable['program'][sl][bright] = 3 # Zero the exposure factor whenever we are not oberving. ##fexp[sl] = (dark | gray | bright)[:, np.newaxis] fexp[sl] = 1. # Transform the local zenith to (ra,dec). zenith = desisurvey.utils.get_observer( times[i], alt=alt, az=az).transform_to(astropy.coordinates.ICRS) etable['zenith_ra'][sl] = zenith.ra.to(u.deg).value etable['zenith_dec'][sl] = zenith.dec.to(u.deg).value # Calculate zenith angles to each pixel in the footprint. pix_sep = pix_sky.separation(zenith[:, np.newaxis]) # Zero the exposure factor for pixels below the horizon. visible = pix_sep < 90 * u.deg fexp[sl][~visible] = 0. # Calculate the airmass exposure-time penalty. X = desisurvey.utils.cos_zenith_to_airmass(np.cos(pix_sep[visible])) fexp[sl][visible] /= desisurvey.etc.airmass_exposure_factor(X) # Loop over objects we need to avoid. for name in config.avoid_bodies.keys: f_obj = desisurvey.ephem.get_object_interpolator(night, name) # Calculate this object's (dec,ra) path during the night. obj_dec, obj_ra = f_obj(mjd) sky_obj = astropy.coordinates.ICRS( ra=obj_ra[:, np.newaxis] * u.deg, dec=obj_dec[:, np.newaxis] * u.deg) # Calculate the separation angles to each pixel in the footprint. obj_sep = pix_sky.separation(sky_obj) if name == 'moon': etable['moon_ra'][sl] = obj_ra etable['moon_dec'][sl] = obj_dec # Calculate moon altitude during the night. moon_alt, _ = desisurvey.ephem.get_object_interpolator( night, 'moon', altaz=True)(mjd) etable['moon_alt'][sl] = moon_alt moon_zenith = (90 - moon_alt[:,np.newaxis]) * u.deg moon_up = moon_alt > 0 assert np.all(moon_alt[gray] > 0) # Calculate the moon illuminated fraction during the night. moon_frac = ephem.get_moon_illuminated_fraction(mjd) etable['moon_frac'][sl] = moon_frac # Convert to temporal moon phase. moon_phase = np.arccos(2 * moon_frac[:,np.newaxis] - 1) / np.pi # Calculate scattered moon V-band brightness at each pixel. V = specsim.atmosphere.krisciunas_schaefer( pix_sep, moon_zenith, obj_sep, moon_phase, desisurvey.etc._vband_extinction).value # Estimate the exposure time factor from V. X = np.dstack((one, np.exp(-V), 1/V, 1/V**2, 1/V**3)) T = X.dot(desisurvey.etc._moonCoefficients) # No penalty when the moon is below the horizon. T[moon_alt < 0, :] = 1. fexp[sl] *= 1. / T # Veto pointings within avoidance size when the moon is # above the horizon. Apply Gaussian smoothing to the veto edge. veto = np.ones_like(T) dsep = (obj_sep - config.avoid_bodies.moon()).to(u.deg).value veto[dsep <= 0] = 0. veto[dsep > 0] = 1 - np.exp(-0.5 * (dsep[dsep > 0] / 3) ** 2) veto[moon_alt < 0] = 1. fexp[sl] *= veto else: # Lookup the avoidance size for this object. size = getattr(config.avoid_bodies, name)() # Penalize the exposure-time with a factor # 1 - exp(-0.5*(obj_sep/size)**2) penalty = 1. - np.exp(-0.5 * (obj_sep / size).to(1).value ** 2) fexp[sl] *= penalty # Save calendar table. hdu = astropy.io.fits.table_to_hdu(calendar) hdu.name = 'CALENDAR' hdus.append(hdu) # Save ephemerides table. hdu = astropy.io.fits.table_to_hdu(etable) hdu.name = 'EPHEM' hdus.append(hdu) # Save dynamic exposure-time factors. hdus.append(astropy.io.fits.ImageHDU(name='DYNAMIC', data=fexp)) # Finalize the output file. try: hdus.writeto(output_name, overwrite=True) except TypeError: # astropy < 1.3 uses the now deprecated clobber. hdus.writeto(output_name, clobber=True) log.info('Plan initialization saved to {0}'.format(output_name))
def run(self, indir, outfile=None, jsonfile=None): '''TODO: document''' log = desiutil.log.get_logger() log.debug('Running QA in {}'.format(indir)) print('here qa.runner', indir) preprocfiles = sorted(glob.glob('{}/preproc-*.fits'.format(indir))) if len(preprocfiles) == 0: log.error('No preproc files found in {}'.format(indir)) return None # We can have different obstypes (signal+dark) with calibration data # obtained with a calibration slit hooked to a single spectrograph. # So we have to loop over all frames to check if there are science, # arc, or flat obstypes as guessed by qproc. qframefiles = sorted(glob.glob('{}/qframe-*.fits'.format(indir))) if len(qframefiles) == 0: # no qframe so it's either zero or dark hdr = fitsio.read_header(preprocfiles[0], 0) if 'OBSTYPE' in hdr: obstype = hdr['FLAVOR'].strip() else: log.warning("Using FLAVOR instead of missing OBSTYPE") obstype = hdr['FLAVOR'].strip() else: obstype = None log.debug("Reading qframe headers to guess flavor ...") for qframefile in qframefiles: # look at all of them and prefer arc or flat over dark or zero hdr = fitsio.read_header(qframefile, 0) if 'OBSTYPE' in hdr: this_obstype = hdr['OBSTYPE'].strip().upper() else: log.warning("Using FLAVOR instead of missing OBSTYPE") obstype = hdr['FLAVOR'].strip() if this_obstype == "ARC" or this_obstype == "FLAT" \ or this_obstype == "TESTARC" or this_obstype == "TESTFLAT" : obstype = this_obstype # we use this so we exit the loop break elif obstype == None: obstype = this_obstype # we stay in the loop in case another frame has another obstype log.debug('Found OBSTYPE={} files'.format(obstype)) results = dict() for qa in self.qalist: if qa.valid_obstype(obstype): log.info('{} Running {} {}'.format(timestamp(), qa, qa.output_type)) qa_results = None try: qa_results = qa.run(indir) except Exception as err: log.warning('{} failed on {} because {}; skipping'.format( qa, indir, str(err))) exc_info = sys.exc_info() traceback.print_exception(*exc_info) del exc_info #raise(err) #- TODO: print traceback somewhere useful if qa_results is not None: if qa.output_type not in results: results[qa.output_type] = list() results[qa.output_type].append(qa_results) else: log.debug('Skip {} {} for {}'.format(qa, qa.output_type, obstype)) #- Combine results for different types of QA join_keys = dict( PER_AMP=['NIGHT', 'EXPID', 'SPECTRO', 'CAM', 'AMP'], PER_CAMERA=['NIGHT', 'EXPID', 'SPECTRO', 'CAM'], PER_FIBER=['NIGHT', 'EXPID', 'SPECTRO', 'FIBER'], PER_CAMFIBER=['NIGHT', 'EXPID', 'SPECTRO', 'CAM', 'FIBER'], PER_SPECTRO=['NIGHT', 'EXPID', 'SPECTRO'], PER_EXP=['NIGHT', 'EXPID'], QPROC_STATUS=['NIGHT', 'EXPID', 'SPECTRO', 'CAM']) if jsonfile is not None: if os.path.exists(jsonfile): with open(jsonfile, 'r') as myfile: json_data = myfile.read() json_data = json.loads(json_data) else: json_data = dict() rewrite_necessary = False for key1 in results: if key1 != "PER_CAMFIBER" and key1.startswith("PER_"): colnames_lst = results[key1][0].colnames if key1 in join_keys: for i in join_keys[key1]: colnames_lst.remove(i) if key1 not in json_data: json_data[key1] = colnames_lst rewrite_necessary = True else: for aspect in colnames_lst: if aspect not in json_data[key1]: json_data[key1] += [aspect] rewrite_necessary = True if rewrite_necessary: if os.path.isdir(os.path.dirname(jsonfile)): with open(jsonfile, 'w') as out: json.dump(json_data, out) print('Wrote {}'.format(jsonfile)) for qatype in list(results.keys()): if len(results[qatype]) == 1: results[qatype] = results[qatype][0] else: tx = results[qatype][0] for i in range(1, len(results[qatype])): tx = join(tx, results[qatype][i], keys=join_keys[qatype], join_type='outer') results[qatype] = tx #- convert python string to bytes for FITS format compatibility if 'AMP' in results[qatype].colnames: results[qatype]['AMP'] = results[qatype]['AMP'].astype('S1') if 'CAM' in results[qatype].colnames: results[qatype]['CAM'] = results[qatype]['CAM'].astype('S1') #- TODO: NIGHT/EXPID/SPECTRO/FIBER int64 -> int32 or int16 #- TODO: metrics from float64 -> float32 if outfile is not None: for tx in results.values(): night = tx['NIGHT'][0] expid = tx['EXPID'][0] break #- To do: consider propagating header from indir/desi*.fits.fz log.info('{} Writing {}'.format(timestamp(), outfile)) tmpfile = outfile + '.tmp' with fitsio.FITS(tmpfile, 'rw', clobber=True) as fx: fx.write(np.zeros(3, dtype=float), extname='PRIMARY', header=hdr) for qatype, qatable in results.items(): fx.write_table(qatable.as_array(), extname=qatype, header=hdr) os.rename(tmpfile, outfile) log.info('{} Finished writing {}'.format(timestamp(), outfile)) return results
def main(args): """Command-line driver to visualize survey scheduling and progress. """ # Set up the logger if args.debug: log = desiutil.log.get_logger(desiutil.log.DEBUG) args.verbose = True elif args.verbose: log = desiutil.log.get_logger(desiutil.log.INFO) else: log = desiutil.log.get_logger(desiutil.log.WARNING) # Freeze IERS table for consistent results. desisurvey.utils.freeze_iers() # Set the output path if requested. config = desisurvey.config.Configuration() if args.output_path is not None: config.set_output_path(args.output_path) if args.tiles_file is not None: config.tiles_file.set_value(args.tiles_file) # Look for the exposures file in the output path by default. args.exposures = config.get_path(args.exposures) # Initialize. animator = Animator(args.exposures, args.start, args.stop, args.label, args.scores) log.info('Found {0} exposures from {1} to {2} ({3} nights).'.format( animator.num_exp, args.start, args.stop, animator.num_nights)) animator.init_figure(args.nightly) if args.expid is not None: expid = animator.exposures['EXPID'] assert np.all(expid == expid[0] + np.arange(len(expid))) if (args.expid < expid[0]) or (args.expid > expid[-1]): raise RuntimeError( 'Requested exposure ID {0} not available.'.format(args.expid)) animator.draw_exposure(args.expid - expid[0], args.nightly) save_name = args.save + '.png' plt.savefig(save_name) log.info('Saved {0}.'.format(save_name)) else: nframes = animator.num_nights if args.nightly else animator.num_exp iexp = [0] def init(): return animator.artists def update(iframe): if (iframe + 1) % args.log_interval == 0: log.info('Drawing frame {0}/{1}.'.format(iframe + 1, nframes)) if args.nightly: while not animator.draw_exposure(iexp[0], nightly=True): iexp[0] += 1 else: animator.draw_exposure(iexp=iframe, nightly=False) return animator.artists log.info('Movie will be {:.1f} mins long at {:.1f} frames/sec.'.format( nframes / (60 * args.fps), args.fps)) animation = matplotlib.animation.FuncAnimation(animator.figure, update, init_func=init, blit=True, frames=nframes) writer = matplotlib.animation.writers['ffmpeg']( bitrate=2400, fps=args.fps, metadata=dict(artist='surveymovie')) save_name = args.save + '.mp4' animation.save(save_name, writer=writer, dpi=animator.dpi) log.info('Saved {0}.'.format(save_name))
def freeze_iers(name='iers_frozen.ecsv', ignore_warnings=True): """Use a frozen IERS table saved with this package. This should be called at the beginning of a script that calls astropy time and coordinates functions which refer to the UT1-UTC and polar motions tabulated by IERS. The purpose is to ensure identical results across systems and astropy releases, to avoid a potential network download, and to eliminate some astropy warnings. After this call, the loaded table will be returned by :func:`astropy.utils.iers.IERS_Auto.open()` and treated like a a normal IERS table by all astropy code. Specifically, this method registers an instance of a custom IERS_Frozen class that inherits from IERS_B and overrides :meth:`astropy.utils.iers.IERS._check_interpolate_indices` to prevent any IERSRangeError being raised. See http://docs.astropy.org/en/stable/utils/iers.html for details. This function returns immediately after the first time it is called, so it it safe to insert anywhere that consistent IERS models are required, and subsequent calls with different args will have no effect. The :func:`desisurvey.utils.plot_iers` function is useful for inspecting IERS tables and how they are extrapolated to DESI survey dates. Parameters ---------- name : str Name of the file to load the frozen IERS table from. Should normally be relative and then refers to this package's data/ directory. Must end with the .ecsv extension. ignore_warnings : bool Ignore ERFA and IERS warnings about future dates generated by astropy time and coordinates functions. Specifically, ERFA warnings containing the string "dubious year" are filtered out, as well as AstropyWarnings related to IERS table extrapolation. """ log = desiutil.log.get_logger() if desisurvey.utils._iers_is_frozen: log.debug('IERS table already frozen.') return log.info('Freezing IERS table used by astropy time, coordinates.') # Validate the save_name extension. _, ext = os.path.splitext(name) if ext != '.ecsv': raise ValueError('Expected .ecsv extension for {0}.'.format(name)) # Locate the file in our package data/ directory. if not os.path.isabs(name): name = astropy.utils.data._find_pkg_data_path( os.path.join('data', name)) if not os.path.exists(name): raise ValueError('No such IERS file: {0}.'.format(name)) # Clear any current IERS table. astropy.utils.iers.IERS.close() # Initialize the global IERS table. We load the table by # hand since the IERS open() method hardcodes format='cds'. try: table = astropy.table.Table.read(name, format='ascii.ecsv').filled() except IOError: raise RuntimeError('Unable to load IERS table from {0}.'.format(name)) # Define a subclass of IERS_B that overrides _check_interpolate_indices # to prevent any IERSRangeError being raised. class IERS_Frozen(astropy.utils.iers.IERS_B): def _check_interpolate_indices(self, indices_orig, indices_clipped, max_input_mjd): pass # Create and register an instance of this class from the table. iers = IERS_Frozen(table) astropy.utils.iers.IERS.iers_table = iers # Prevent any attempts to automatically download updated IERS-A tables. astropy.utils.iers.conf.auto_download = False astropy.utils.iers.conf.auto_max_age = None astropy.utils.iers.conf.iers_auto_url = 'frozen' # Sanity check. if not (astropy.utils.iers.IERS_Auto.open() is iers): raise RuntimeError('Frozen IERS is not installed as the default.') if ignore_warnings: warnings.filterwarnings( 'ignore', category=astropy._erfa.core.ErfaWarning, message= r'ERFA function \"[a-z0-9_]+\" yielded [0-9]+ of \"dubious year') warnings.filterwarnings( 'ignore', category=astropy.utils.exceptions.AstropyWarning, message=r'Tried to get polar motions for times after IERS data') warnings.filterwarnings( 'ignore', category=astropy.utils.exceptions.AstropyWarning, message=r'\(some\) times are outside of range covered by IERS') # Shortcircuit any subsequent calls to this function. desisurvey.utils._iers_is_frozen = True
def main(args): """Command-line driver to visualize survey scheduling and progress. """ # Set up the logger if args.debug: log = desiutil.log.get_logger(desiutil.log.DEBUG) args.verbose = True elif args.verbose: log = desiutil.log.get_logger(desiutil.log.INFO) else: log = desiutil.log.get_logger(desiutil.log.WARNING) # Freeze IERS table for consistent results. desisurvey.utils.freeze_iers() # Set the output path if requested. config = desisurvey.config.Configuration() if args.output_path is not None: config.set_output_path(args.output_path) if args.tiles_file is not None: config.tiles_file.set_value(args.tiles_file) # Look for the exposures file in the output path by default. args.exposures = config.get_path(args.exposures) # Initialize. animator = Animator( args.exposures, args.start, args.stop, args.label, args.scores) log.info('Found {0} exposures from {1} to {2} ({3} nights).' .format(animator.num_exp, args.start, args.stop, animator.num_nights)) animator.init_figure(args.nightly) if args.expid is not None: expid = animator.exposures['EXPID'] assert np.all(expid == expid[0] + np.arange(len(expid))) if (args.expid < expid[0]) or (args.expid > expid[-1]): raise RuntimeError('Requested exposure ID {0} not available.' .format(args.expid)) animator.draw_exposure(args.expid - expid[0], args.nightly) save_name = args.save + '.png' plt.savefig(save_name) log.info('Saved {0}.'.format(save_name)) else: nframes = animator.num_nights if args.nightly else animator.num_exp iexp = [0] def init(): return animator.artists def update(iframe): if (iframe + 1) % args.log_interval == 0: log.info('Drawing frame {0}/{1}.' .format(iframe + 1, nframes)) if args.nightly: while not animator.draw_exposure(iexp[0], nightly=True): iexp[0] += 1 else: animator.draw_exposure(iexp=iframe, nightly=False) return animator.artists log.info('Movie will be {:.1f} mins long at {:.1f} frames/sec.' .format(nframes / (60 * args.fps), args.fps)) animation = matplotlib.animation.FuncAnimation( animator.figure, update, init_func=init, blit=True, frames=nframes) writer = matplotlib.animation.writers['ffmpeg']( bitrate=2400, fps=args.fps, metadata=dict(artist='surveymovie')) save_name = args.save + '.mp4' animation.save(save_name, writer=writer, dpi=animator.dpi) log.info('Saved {0}.'.format(save_name))
def main(args): """Command-line driver for updating the survey plan. """ # Check for a valid fa-delay value. if args.fa_delay[-1] not in ('d', 'm', 'q'): raise ValueError('fa-delay must have the form Nd, Nm or Nq.') fa_delay_type = args.fa_delay[-1] try: fa_delay = int(args.fa_delay[:-1]) except ValueError: raise ValueError('invalid number in fa-delay.') if fa_delay < 0: raise ValueError('fa-delay value must be >= 0.') # Set up the logger if args.debug: log = desiutil.log.get_logger(desiutil.log.DEBUG) args.verbose = True elif args.verbose: log = desiutil.log.get_logger(desiutil.log.INFO) else: log = desiutil.log.get_logger(desiutil.log.WARNING) # Freeze IERS table for consistent results. desisurvey.utils.freeze_iers() # Set the output path if requested. config = desisurvey.config.Configuration(file_name=args.config_file) if args.output_path is not None: config.set_output_path(args.output_path) # Load ephemerides. ephem = desisurvey.ephem.get_ephem() # Initialize scheduler. if not os.path.exists(config.get_path('scheduler.fits')): # Tabulate data used by the scheduler if necessary. desisurvey.old.schedule.initialize(ephem) scheduler = desisurvey.old.schedule.Scheduler() # Read priority rules. rules = desisurvey.rules.Rules(args.rules) if args.create: # Load initial design hour angles for each tile. design = astropy.table.Table.read(config.get_path('surveyinit.fits')) # Create an empty progress record. progress = desisurvey.progress.Progress() # Initialize the observing priorities. priorities = rules.apply(progress) # Create the initial plan. plan = desisurvey.plan.create(design['HA'], priorities) # Start the survey from scratch. start = config.first_day() else: # Load an existing plan and progress record. if not os.path.exists(config.get_path('plan.fits')): log.error('No plan.fits found in output path.') return -1 if not os.path.exists(config.get_path('progress.fits')): log.error('No progress.fits found in output path.') return -1 plan = astropy.table.Table.read(config.get_path('plan.fits')) progress = desisurvey.progress.Progress('progress.fits') # Start the new plan from the last observing date. with open(config.get_path('last_date.txt'), 'r') as f: start = desisurvey.utils.get_date(f.read().rstrip()) num_complete, num_total, pct = progress.completed(as_tuple=True) # Already observed all tiles? if num_complete == num_total: log.info('All tiles observed!') # Return a shell exit code so scripts can detect this condition. sys.exit(9) # Reached end of the survey? if start >= config.last_day(): log.info('Reached survey end date!') # Return a shell exit code so scripts can detect this condition. sys.exit(9) day_number = desisurvey.utils.day_number(start) log.info( 'Planning night[{0}] {1} with {2:.1f} / {3} ({4:.1f}%) completed.'. format(day_number, start, num_complete, num_total, pct)) bookmarked = False if not args.create: # Update the priorities for the progress so far. new_priority = rules.apply(progress) changed_priority = (new_priority != plan['priority']) if np.any(changed_priority): changed_passes = np.unique(plan['pass'][changed_priority]) log.info('Priorities changed in pass(es) {0}.'.format(', '.join( [str(p) for p in changed_passes]))) plan['priority'] = new_priority bookmarked = True # Identify any new tiles that are available for fiber assignment. plan = desisurvey.plan.update_available(plan, progress, start, ephem, fa_delay, fa_delay_type) # Will update design HA assignments here... pass # Update the progress table for the new plan. ptable = progress._table new_cover = (ptable['covered'] < 0) & (plan['covered'] <= day_number) ptable['covered'][new_cover] = day_number new_avail = (ptable['available'] < 0) & plan['available'] ptable['available'][new_avail] = day_number new_plan = (ptable['planned'] < 0) & (plan['priority'] > 0) ptable['planned'][new_plan] = day_number # Save updated progress. progress.save('progress.fits') # Save the plan. plan.write(config.get_path('plan.fits'), overwrite=True) if bookmarked: # Save a backup of the plan and progress at this point. plan.write(config.get_path('plan_{0}.fits'.format(start))) progress.save('progress_{0}.fits'.format(start))
def main(args): """Command-line driver for running survey simulations. """ # Set up the logger if args.debug: os.environ['DESI_LOGLEVEL'] = 'DEBUG' args.verbose = True elif args.verbose: os.environ['DESI_LOGLEVEL'] = 'INFO' else: os.environ['DESI_LOGLEVEL'] = 'WARNING' log = desiutil.log.get_logger() # Set the output path if requested. config = desisurvey.config.Configuration() if args.output_path is not None: config.set_output_path(args.output_path) if args.tiles_file is not None: config.tiles_file.set_value(args.tiles_file) # Initialize simulation progress tracking. stats = surveysim.stats.SurveyStatistics(args.start, args.stop) explist = surveysim.exposures.ExposureList() # Initialize the survey strategy rules. rules = desisurvey.rules.Rules(args.rules) # Initialize afternoon planning. planner = desisurvey.plan.Planner(rules) # Initialize next tile selection. scheduler = desisurvey.scheduler.Scheduler() # Generate random weather conditions. weather = surveysim.weather.Weather(seed=args.seed, replay=args.replay) # Loop over nights. num_simulated = 0 num_nights = (args.stop - args.start).days for num_simulated in range(num_nights): night = args.start + datetime.timedelta(num_simulated) if args.save_restore and num_simulated > 0: # Restore the planner and scheduler saved after the previous night. planner = desisurvey.plan.Planner(rules, restore='planner_{}.fits'.format(last_night)) scheduler = desisurvey.scheduler.Scheduler(restore='scheduler_{}.fits'.format(last_night)) scheduler.update_tiles(planner.tile_available, planner.tile_priority) # Perform afternoon planning. explist.update_tiles(night, *scheduler.update_tiles(*planner.afternoon_plan(night, scheduler.completed))) if not desisurvey.utils.is_monsoon(night) and not scheduler.ephem.is_full_moon(night): # Simulate one night of observing. surveysim.nightops.simulate_night( night, scheduler, stats, explist, weather=weather, use_twilight=args.twilight) if scheduler.survey_completed(): log.info('Survey complete on {}.'.format(night)) break if args.save_restore: last_night = night.isoformat() planner.save('planner_{}.fits'.format(last_night)) scheduler.save('scheduler_{}.fits'.format(last_night)) if num_simulated % args.log_interval == args.log_interval - 1: log.info('Completed {} / {} tiles after {} / {} nights.'.format( scheduler.completed_by_pass.sum(), scheduler.tiles.ntiles, num_simulated + 1, num_nights)) explist.save('exposures_{}.fits'.format(args.name), comment=args.comment) stats.save('stats_{}.fits'.format(args.name), comment=args.comment) if args.verbose: stats.summarize()
def main(args): """Command-line driver for running survey simulations. """ # Set up the logger if args.debug: os.environ['DESI_LOGLEVEL'] = 'DEBUG' args.verbose = True elif args.verbose: os.environ['DESI_LOGLEVEL'] = 'INFO' else: os.environ['DESI_LOGLEVEL'] = 'WARNING' log = desiutil.log.get_logger() # Set the output path if requested. config = desisurvey.config.Configuration() if args.output_path is not None: config.set_output_path(args.output_path) if args.tiles_file is not None: config.tiles_file.set_value(args.tiles_file) # Initialize simulation progress tracking. stats = surveysim.stats.SurveyStatistics(args.start, args.stop) explist = surveysim.exposures.ExposureList() # Initialize the survey strategy rules. rules = desisurvey.rules.Rules(args.rules) # Initialize afternoon planning. planner = desisurvey.plan.Planner(rules) # Initialize next tile selection. scheduler = desisurvey.scheduler.Scheduler() # Generate random weather conditions. weather = surveysim.weather.Weather(seed=args.seed, replay=args.replay) # Loop over nights. num_simulated = 0 num_nights = (args.stop - args.start).days for num_simulated in range(num_nights): night = args.start + datetime.timedelta(num_simulated) if args.save_restore and num_simulated > 0: # Restore the planner and scheduler saved after the previous night. planner = desisurvey.plan.Planner( rules, restore='planner_{}.fits'.format(last_night)) scheduler = desisurvey.scheduler.Scheduler( restore='scheduler_{}.fits'.format(last_night)) scheduler.update_tiles(planner.tile_available, planner.tile_priority) # Perform afternoon planning. explist.update_tiles( night, *scheduler.update_tiles( *planner.afternoon_plan(night, scheduler.completed))) if not desisurvey.utils.is_monsoon( night) and not scheduler.ephem.is_full_moon(night): # Simulate one night of observing. surveysim.nightops.simulate_night(night, scheduler, stats, explist, weather=weather, use_twilight=args.twilight) if scheduler.survey_completed(): log.info('Survey complete on {}.'.format(night)) break if args.save_restore: last_night = night.isoformat() planner.save('planner_{}.fits'.format(last_night)) scheduler.save('scheduler_{}.fits'.format(last_night)) if num_simulated % args.log_interval == args.log_interval - 1: log.info('Completed {} / {} tiles after {} / {} nights.'.format( scheduler.completed_by_pass.sum(), scheduler.tiles.ntiles, num_simulated + 1, num_nights)) explist.save('exposures_{}.fits'.format(args.name), comment=args.comment) stats.save('stats_{}.fits'.format(args.name), comment=args.comment) if args.verbose: stats.summarize()
def calculate_initial_plan(args): """Calculate the initial survey plan. Use :func:`desisurvey.plan.load_weather` and :func:`desisurvey.plan.load_design_hourangles` to retrieve these data from the saved plan. Parameters ---------- args : object Object with attributes for parsed command-line arguments. """ log = desiutil.log.get_logger() config = desisurvey.config.Configuration() tiles = desisurvey.tiles.get_tiles() ephem = desisurvey.ephem.get_ephem() # Initialize the output file to write. hdus = fits.HDUList() hdr = fits.Header() # Calculate average weather factors for each day covered by # the ephemerides. first = desisurvey.ephem.START_DATE last = desisurvey.ephem.STOP_DATE years = np.arange(2007, 2018) fractions = [] for year in years: fractions.append( desimodel.weather.dome_closed_fractions(first, last, replay='Y{}'.format(year))) weather = 1 - np.mean(fractions, axis=0) # Save the weather fractions as the primary HDU. hdr['FIRST'] = first.isoformat() hdr['YEARS'] = ','.join(['{}'.format(yr) for yr in years]) start = config.first_day() stop = config.last_day() assert start >= first and stop <= last hdr['START'] = start.isoformat() hdr['STOP'] = stop.isoformat() hdr['TWILIGHT'] = args.include_twilight hdus.append(fits.ImageHDU(weather, header=hdr, name='WEATHER')) # Calculate the distribution of available LST in each program # during the nominal survey [start, stop). ilo, ihi = (start - first).days, (stop - first).days lst_hist, lst_bins = ephem.get_available_lst( nbins=args.nbins, weather=weather[ilo:ihi], include_twilight=args.include_twilight) # Initialize the output results table. design = astropy.table.Table() design['INIT'] = np.zeros(tiles.ntiles) design['HA'] = np.zeros(tiles.ntiles) design['TEXP'] = np.zeros(tiles.ntiles) # Optimize each program separately. stretches = dict(DARK=args.dark_stretch, GRAY=args.gray_stretch, BRIGHT=args.bright_stretch) for pindex, program in enumerate(tiles.PROGRAMS): sel = tiles.program_mask[program] if not np.any(sel): log.info('Skipping {} program with no tiles.'.format(program)) continue # Initialize an LST summary table. table = astropy.table.Table(meta={'ORIGIN': lst_bins[0]}) table['AVAIL'] = lst_hist[pindex] # Initailize an optimizer for this program. opt = desisurvey.optimize.Optimizer(program, lst_bins, lst_hist[pindex], init=args.init, center=None, stretch=stretches[program]) table['INIT'] = opt.plan_hist.copy() design['INIT'][sel] = opt.ha_initial # Initialize annealing cycles. ncycles = 0 binsize = 360. / args.nbins frac = args.adjust / binsize smoothing = args.smooth # Loop over annealing cycles. while ncycles < args.max_cycles: start_score = opt.eval_score(opt.plan_hist) for i in range(opt.ntiles): opt.improve(frac) if smoothing > 0: opt.smooth(alpha=smoothing) stop_score = opt.eval_score(opt.plan_hist) delta = (stop_score - start_score) / start_score RMSE = opt.RMSE_history[-1] loss = opt.loss_history[-1] log.info('[{:03d}] dHA={:5.3f}deg '.format(ncycles + 1, frac * binsize) + 'RMSE={:6.2f}% LOSS={:5.2f}% delta(score)={:+5.1f}%'. format(1e2 * RMSE, 1e2 * loss, 1e2 * delta)) # Both conditions must be satisfied to terminate. if RMSE < args.max_rmse and delta > -args.epsilon: break # Anneal parameters for next cycle. frac *= args.anneal smoothing *= args.anneal ncycles += 1 plan_sum = opt.plan_hist.sum() avail_sum = opt.lst_hist_sum margin = (avail_sum - plan_sum) / plan_sum log.info( '{} plan uses {:.1f}h with {:.1f}h avail ({:.1f}% margin).'.format( program, plan_sum, avail_sum, 1e2 * margin)) # Save planned LST usage. table['PLAN'] = opt.plan_hist hdus.append(fits.BinTableHDU(table, name=program)) # Calculate exposure times in (solar) seconds. texp, _ = opt.get_exptime(opt.ha) texp *= 24. * 3600. / 360. * 0.99726956583 # Save results for this program. design['HA'][sel] = opt.ha design['TEXP'][sel] = texp hdus.append(fits.BinTableHDU(design, name='DESIGN')) fullname = config.get_path(args.save) hdus.writeto(fullname, overwrite=True) log.info('Saved initial plan to "{}".'.format(fullname))
def main(args): """Command-line driver for running survey simulations. """ # Set up the logger if args.debug: os.environ['DESI_LOGLEVEL'] = 'DEBUG' args.verbose = True elif args.verbose: os.environ['DESI_LOGLEVEL'] = 'INFO' else: os.environ['DESI_LOGLEVEL'] = 'WARNING' log = desiutil.log.get_logger() # Set the output path if requested. config = desisurvey.config.Configuration() if args.output_path is not None: config.set_output_path(args.output_path) if args.tiles_file is not None: config.tiles_file.set_value(args.tiles_file) if args.existing_exposures is not None: exps = Table.read(args.existing_exposures) tiles = desisurvey.tiles.get_tiles() idx, mask = tiles.index(exps['TILEID'], return_mask=True) firstnight = max(exps['NIGHT'][mask]) if args.start != config.first_day(): raise ValueError('Cannot set both start and existing-exposures!') args.start = '-'.join( [firstnight[:4], firstnight[4:6], firstnight[6:]]) args.start = desisurvey.utils.get_date(args.start) exps = exps[mask] else: exps = None # Initialize simulation progress tracking. stats = surveysim.stats.SurveyStatistics(args.start, args.stop) explist = surveysim.exposures.ExposureList(existing_exposures=exps) # Initialize the survey strategy rules. if args.rules is None: rulesfile = config.rules_file() else: rulesfile = args.rules rules = desisurvey.rules.Rules(rulesfile) log.info('Rules loaded from {}.'.format(rulesfile)) # Initialize afternoon planning. planner = desisurvey.plan.Planner(rules, simulate=True) # Initialize next tile selection. scheduler = desisurvey.scheduler.Scheduler(planner) # Generate random weather conditions. weather = surveysim.weather.Weather( seed=args.seed, replay=args.replay, extra_downtime=args.extra_downtime) # Loop over nights. num_simulated = 0 num_nights = (args.stop - args.start).days for num_simulated in range(num_nights): night = args.start + datetime.timedelta(num_simulated) if args.save_restore and num_simulated > 0: # Restore the planner and scheduler saved after the previous night. planner = desisurvey.plan.Planner(rules, restore='desi-status-{}.ecsv'.format(last_night), simulate=True) scheduler = desisurvey.scheduler.Scheduler(planner) # Perform afternoon planning. explist.update_tiles(night, *planner.afternoon_plan(night)) if not desisurvey.utils.is_monsoon(night) and not scheduler.ephem.is_full_moon(night): # Simulate one night of observing. surveysim.nightops.simulate_night( night, scheduler, stats, explist, weather=weather, use_twilight=args.twilight, use_simplesky=args.simplesky) if scheduler.plan.survey_completed(): log.info('Survey complete on {}.'.format(night)) break if args.save_restore: last_night = desisurvey.utils.night_to_str(night) planner.save('desi-status-{}.ecsv'.format(last_night)) if num_simulated % args.log_interval == args.log_interval - 1: log.info('Completed {} / {} tiles after {} / {} nights.'.format( scheduler.plan.obsend().sum(), scheduler.tiles.ntiles, num_simulated + 1, num_nights)) explist.save('exposures_{}.fits'.format(args.name), comment=args.comment) stats.save('stats_{}.fits'.format(args.name), comment=args.comment) planner.save('desi-status-end-{}.ecsv'.format(args.name)) if args.verbose: stats.summarize()
def write_tables(indir, outdir, expnights=None): ''' Parses directory for available nights, exposures to generate nights and exposures tables Args: indir : directory of nights outdir : directory where to write nights table Options: expnights (list) : only update exposures tables for these nights ''' import re from astropy.table import Table from nightwatch.webpages import tables as web_tables from pkg_resources import resource_filename from shutil import copyfile from collections import Counter log = desiutil.log.get_logger() log.info(f'Tabulating exposures in {indir}') #- Count night/expid directories to get num exp per night expdirs = sorted(glob.glob(f"{indir}/20*/[0-9]*")) nights = list() re_expid = re.compile('^\d{8}$') re_night = re.compile('^20\d{6}$') for expdir in expdirs: expid = os.path.basename(expdir) night = os.path.basename(os.path.dirname(expdir)) if re_expid.match(expid) and re_night.match(night): nights.append(night) num_exp_per_night = Counter(nights) #- Build the exposures table for the requested nights rows = list() for expdir in expdirs: expid = os.path.basename(expdir) night = os.path.basename(os.path.dirname(expdir)) if re_expid.match(expid) and re_night.match(night) and \ (expnights is None or int(night) in expnights): night = int(night) expid = int(expid) qafile = os.path.join(expdir, 'qa-{:08d}.fits'.format(expid)) #- gets the list of failed qprocs for each expid expfiles = os.listdir(expdir) preproc_cams = [ i.split("-")[1] for i in expfiles if re.match(r'preproc-.*-.*.fits', i) ] log_cams = [ i.split("-")[1] for i in expfiles if re.match(r'.*\.log', i) ] qfails = [i for i in log_cams if i not in preproc_cams] if os.path.exists(qafile): try: with fitsio.FITS(qafile) as fits: qproc_status = fits['QPROC_STATUS'].read() exitcode = np.count_nonzero(qproc_status['QPROC_EXIT']) except IOError: exitcode = 0 rows.append( dict(NIGHT=night, EXPID=expid, FAIL=0, QPROC=qfails, QPROC_EXIT=exitcode)) else: log.error('Missing {}'.format(qafile)) rows.append( dict(NIGHT=night, EXPID=expid, FAIL=1, QPROC=None, QPROC_EXIT=None)) if len(rows) == 0: msg = "No exp dirs found in {}/NIGHT/EXPID".format(indir) raise RuntimeError(msg) exposures = Table(rows) caldir = os.path.join(outdir, 'static') if not os.path.isdir(caldir): os.makedirs(caldir) files = [ 'bootstrap.js', 'bootstrap.css', 'bootstrap-year-calendar.css', 'bootstrap-year-calendar.js', 'jquery_min.js', 'popper_min.js', 'live.js' ] for f in files: outfile = os.path.join(outdir, 'static', f) if not os.path.exists(outfile): infile = resource_filename('nightwatch', os.path.join('static', f)) copyfile(infile, outfile) nightsfile = os.path.join(outdir, 'nights.html') web_tables.write_calendar(nightsfile, num_exp_per_night) web_tables.write_exposures_tables(indir, outdir, exposures, nights=expnights)
def initialize(ephem, start_date=None, stop_date=None, step_size=5.0, healpix_nside=16, output_name='scheduler.fits'): """Calculate exposure-time factors over a grid of times and pointings. Takes about 9 minutes to run and writes a 1.3Gb output file with the default parameters. Requires that healpy is installed. Parameters ---------- ephem : desisurvey.ephem.Ephemerides Tabulated ephemerides data to use for planning. start_date : date or None Survey planning starts on the evening of this date. Must be convertible to a date using :func:`desisurvey.utils.get_date`. Use the first night of the ephemerides when None. stop_date : date or None Survey planning stops on the morning of this date. Must be convertible to a date using :func:`desisurvey.utils.get_date`. Use the first night of the ephemerides when None. step_size : :class:`astropy.units.Quantity` Exposure-time factors are tabulated at this interval during each night. healpix_nside : int Healpix NSIDE parameter to use for binning the sky. Must be a power of two. Values larger than 16 will lead to holes in the footprint with the current implementation. output_name : str Name of the FITS output file where results are saved. A relative path refers to the :meth:`configuration output path <desisurvey.config.Configuration.get_path>`. """ import healpy if not isinstance(step_size, u.Quantity): step_size = step_size * u.min log = desiutil.log.get_logger() # Freeze IERS table for consistent results. desisurvey.utils.freeze_iers() config = desisurvey.config.Configuration() output_name = config.get_path(output_name) start_date = desisurvey.utils.get_date(start_date or config.first_day()) stop_date = desisurvey.utils.get_date(stop_date or config.last_day()) if start_date >= stop_date: raise ValueError('Expected start_date < stop_date.') mjd = ephem._table['noon'] sel = ((mjd >= desisurvey.utils.local_noon_on_date(start_date).mjd) & (mjd < desisurvey.utils.local_noon_on_date(stop_date).mjd)) t = ephem._table[sel] num_nights = len(t) # Build a grid of elapsed time relative to local midnight during each night. midnight = t['noon'] + 0.5 t_edges = desisurvey.ephem.get_grid(step_size) t_centers = 0.5 * (t_edges[1:] + t_edges[:-1]) num_points = len(t_centers) # Create an empty HDU0 with header info. header = astropy.io.fits.Header() header['START'] = str(start_date) header['STOP'] = str(stop_date) header['NSIDE'] = healpix_nside header['NPOINTS'] = num_points header['STEP'] = step_size.to(u.min).value hdus = astropy.io.fits.HDUList() hdus.append(astropy.io.fits.ImageHDU(header=header)) # Save time grid. hdus.append(astropy.io.fits.ImageHDU(name='GRID', data=t_edges)) # Load the list of tiles to observe. tiles = astropy.table.Table( desimodel.io.load_tiles(onlydesi=True, extra=False, tilesfile=config.tiles_file())) # Build the footprint as a healpix map of the requested size. # The footprint includes any pixel containing at least one tile center. npix = healpy.nside2npix(healpix_nside) footprint = np.zeros(npix, bool) pixels = healpy.ang2pix(healpix_nside, np.radians(90 - tiles['DEC'].data), np.radians(tiles['RA'].data)) footprint[np.unique(pixels)] = True footprint_pixels = np.where(footprint)[0] num_footprint = len(footprint_pixels) log.info('Footprint contains {0} pixels.'.format(num_footprint)) # Sort pixels in order of increasing phi + 60deg so that the north and south # galactic caps are contiguous in the arrays we create below. pix_theta, pix_phi = healpy.pix2ang(healpix_nside, footprint_pixels) pix_dphi = np.fmod(pix_phi + np.pi / 3, 2 * np.pi) sort_order = np.argsort(pix_dphi) footprint_pixels = footprint_pixels[sort_order] # Calculate sorted pixel (ra,dec). pix_theta, pix_phi = healpy.pix2ang(healpix_nside, footprint_pixels) pix_ra, pix_dec = np.degrees(pix_phi), 90 - np.degrees(pix_theta) # Record per-tile info needed for planning. table = astropy.table.Table() table['tileid'] = tiles['TILEID'].astype(np.int32) table['ra'] = tiles['RA'].astype(np.float32) table['dec'] = tiles['DEC'].astype(np.float32) table['EBV'] = tiles['EBV_MED'].astype(np.float32) table['pass'] = tiles['PASS'].astype(np.int16) # Map each tile ID to the corresponding index in our spatial arrays. mapper = np.zeros(npix, int) mapper[footprint_pixels] = np.arange(len(footprint_pixels)) table['map'] = mapper[pixels].astype(np.int16) # Use a small int to identify the program, ordered by sky brightness: # 1=DARK, 2=GRAY, 3=BRIGHT. table['program'] = np.full(len(tiles), 4, np.int16) for i, program in enumerate(('DARK', 'GRAY', 'BRIGHT')): table['program'][tiles['PROGRAM'] == program] = i + 1 assert np.all(table['program'] > 0) hdu = astropy.io.fits.table_to_hdu(table) hdu.name = 'TILES' hdus.append(hdu) # Average E(B-V) for all tiles falling into a pixel. tiles_per_pixel = np.bincount(pixels, minlength=npix) EBV = np.bincount(pixels, weights=tiles['EBV_MED'], minlength=npix) EBV[footprint] /= tiles_per_pixel[footprint] # Calculate dust extinction exposure-time factor. f_EBV = 1. / desisurvey.etc.dust_exposure_factor(EBV) # Save HDU with the footprint and static dust exposure map. table = astropy.table.Table() table['pixel'] = footprint_pixels table['dust'] = f_EBV[footprint_pixels] table['ra'] = pix_ra table['dec'] = pix_dec hdu = astropy.io.fits.table_to_hdu(table) hdu.name = 'STATIC' hdus.append(hdu) # Prepare a table of calendar data. calendar = astropy.table.Table() calendar['midnight'] = midnight calendar['monsoon'] = np.zeros(num_nights, bool) calendar['fullmoon'] = np.zeros(num_nights, bool) calendar['weather'] = np.zeros(num_nights, np.float32) # Hardcode annualized average weight. weather_weights = np.full(num_nights, 0.723) # Prepare a table of ephemeris data. etable = astropy.table.Table() # Program codes ordered by increasing sky brightness: # 1=DARK, 2=GRAY, 3=BRIGHT, 4=DAYTIME. etable['program'] = np.full(num_nights * num_points, 4, dtype=np.int16) etable['moon_frac'] = np.zeros(num_nights * num_points, dtype=np.float32) etable['moon_ra'] = np.zeros(num_nights * num_points, dtype=np.float32) etable['moon_dec'] = np.zeros(num_nights * num_points, dtype=np.float32) etable['moon_alt'] = np.zeros(num_nights * num_points, dtype=np.float32) etable['zenith_ra'] = np.zeros(num_nights * num_points, dtype=np.float32) etable['zenith_dec'] = np.zeros(num_nights * num_points, dtype=np.float32) # Tabulate MJD and apparent LST values for each time step. We don't save # MJD values since they are cheap to reconstruct from the index, but # do use them below. mjd0 = desisurvey.utils.local_noon_on_date(start_date).mjd + 0.5 mjd = mjd0 + np.arange(num_nights)[:, np.newaxis] + t_centers times = astropy.time.Time(mjd, format='mjd', location=desisurvey.utils.get_location()) etable['lst'] = times.sidereal_time('apparent').flatten().to(u.deg).value # Build sky coordinates for each pixel in the footprint. pix_theta, pix_phi = healpy.pix2ang(healpix_nside, footprint_pixels) pix_ra, pix_dec = np.degrees(pix_phi), 90 - np.degrees(pix_theta) pix_sky = astropy.coordinates.ICRS(pix_ra * u.deg, pix_dec * u.deg) # Initialize exposure factor calculations. alt, az = np.full(num_points, 90.) * u.deg, np.zeros(num_points) * u.deg fexp = np.zeros((num_nights * num_points, num_footprint), dtype=np.float32) vband_extinction = 0.15154 one = np.ones((num_points, num_footprint)) # Loop over nights. for i in range(num_nights): night = ephem.get_night(midnight[i]) date = desisurvey.utils.get_date(midnight[i]) if date.day == 1: log.info('Starting {0} (completed {1}/{2} nights)'.format( date.strftime('%b %Y'), i, num_nights)) # Initialize the slice of the fexp[] time index for this night. sl = slice(i * num_points, (i + 1) * num_points) # Do we expect to observe on this night? calendar[i]['monsoon'] = desisurvey.utils.is_monsoon(midnight[i]) calendar[i]['fullmoon'] = ephem.is_full_moon(midnight[i]) # Look up expected dome-open fraction due to weather. calendar[i]['weather'] = weather_weights[i] # Calculate the program during this night (default is 4=DAYTIME). mjd = midnight[i] + t_centers dark, gray, bright = ephem.tabulate_program(mjd) etable['program'][sl][dark] = 1 etable['program'][sl][gray] = 2 etable['program'][sl][bright] = 3 # Zero the exposure factor whenever we are not oberving. ##fexp[sl] = (dark | gray | bright)[:, np.newaxis] fexp[sl] = 1. # Transform the local zenith to (ra,dec). zenith = desisurvey.utils.get_observer( times[i], alt=alt, az=az).transform_to(astropy.coordinates.ICRS) etable['zenith_ra'][sl] = zenith.ra.to(u.deg).value etable['zenith_dec'][sl] = zenith.dec.to(u.deg).value # Calculate zenith angles to each pixel in the footprint. pix_sep = pix_sky.separation(zenith[:, np.newaxis]) # Zero the exposure factor for pixels below the horizon. visible = pix_sep < 90 * u.deg fexp[sl][~visible] = 0. # Calculate the airmass exposure-time penalty. X = desisurvey.utils.cos_zenith_to_airmass(np.cos(pix_sep[visible])) fexp[sl][visible] /= desisurvey.etc.airmass_exposure_factor(X) # Loop over objects we need to avoid. for name in config.avoid_bodies.keys: f_obj = desisurvey.ephem.get_object_interpolator(night, name) # Calculate this object's (dec,ra) path during the night. obj_dec, obj_ra = f_obj(mjd) sky_obj = astropy.coordinates.ICRS( ra=obj_ra[:, np.newaxis] * u.deg, dec=obj_dec[:, np.newaxis] * u.deg) # Calculate the separation angles to each pixel in the footprint. obj_sep = pix_sky.separation(sky_obj) if name == 'moon': etable['moon_ra'][sl] = obj_ra etable['moon_dec'][sl] = obj_dec # Calculate moon altitude during the night. moon_alt, _ = desisurvey.ephem.get_object_interpolator( night, 'moon', altaz=True)(mjd) etable['moon_alt'][sl] = moon_alt moon_zenith = (90 - moon_alt[:, np.newaxis]) * u.deg moon_up = moon_alt > 0 assert np.all(moon_alt[gray] > 0) # Calculate the moon illuminated fraction during the night. moon_frac = ephem.get_moon_illuminated_fraction(mjd) etable['moon_frac'][sl] = moon_frac # Convert to temporal moon phase. moon_phase = np.arccos(2 * moon_frac[:, np.newaxis] - 1) / np.pi # Calculate scattered moon V-band brightness at each pixel. V = specsim.atmosphere.krisciunas_schaefer( pix_sep, moon_zenith, obj_sep, moon_phase, desisurvey.etc._vband_extinction).value # Estimate the exposure time factor from V. X = np.dstack((one, np.exp(-V), 1 / V, 1 / V**2, 1 / V**3)) T = X.dot(desisurvey.etc._moonCoefficients) # No penalty when the moon is below the horizon. T[moon_alt < 0, :] = 1. fexp[sl] *= 1. / T # Veto pointings within avoidance size when the moon is # above the horizon. Apply Gaussian smoothing to the veto edge. veto = np.ones_like(T) dsep = (obj_sep - config.avoid_bodies.moon()).to(u.deg).value veto[dsep <= 0] = 0. veto[dsep > 0] = 1 - np.exp(-0.5 * (dsep[dsep > 0] / 3)**2) veto[moon_alt < 0] = 1. fexp[sl] *= veto else: # Lookup the avoidance size for this object. size = getattr(config.avoid_bodies, name)() # Penalize the exposure-time with a factor # 1 - exp(-0.5*(obj_sep/size)**2) penalty = 1. - np.exp(-0.5 * (obj_sep / size).to(1).value**2) fexp[sl] *= penalty # Save calendar table. hdu = astropy.io.fits.table_to_hdu(calendar) hdu.name = 'CALENDAR' hdus.append(hdu) # Save ephemerides table. hdu = astropy.io.fits.table_to_hdu(etable) hdu.name = 'EPHEM' hdus.append(hdu) # Save dynamic exposure-time factors. hdus.append(astropy.io.fits.ImageHDU(name='DYNAMIC', data=fexp)) # Finalize the output file. try: hdus.writeto(output_name, overwrite=True) except TypeError: # astropy < 1.3 uses the now deprecated clobber. hdus.writeto(output_name, clobber=True) log.info('Plan initialization saved to {0}'.format(output_name))
def stdouterr_redirected(to=None, comm=None): """ Redirect stdout and stderr to a file. The general technique is based on: http://stackoverflow.com/questions/5081657 http://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/ One difference here is that each process in the communicator redirects to a different temporary file, and the upon exit from the context the rank zero process concatenates these in order to the file result. Args: to (str): The output file name. comm (mpi4py.MPI.Comm): The optional MPI communicator. """ nproc = 1 rank = 0 if comm is not None: nproc = comm.size rank = comm.rank # The currently active POSIX file descriptors fd_out = sys.stdout.fileno() fd_err = sys.stderr.fileno() # The DESI loggers. desi_loggers = desiutil.log._desiutil_log_root def _redirect(out_to, err_to): # Flush the C-level buffers if c_stdout is not None: libc.fflush(c_stdout) if c_stderr is not None: libc.fflush(c_stderr) # This closes the python file handles, and marks the POSIX # file descriptors for garbage collection- UNLESS those # are the special file descriptors for stderr/stdout. sys.stdout.close() sys.stderr.close() # Close fd_out/fd_err if they are open, and copy the # input file descriptors to these. os.dup2(out_to, fd_out) os.dup2(err_to, fd_err) # Create a new sys.stdout / sys.stderr that points to the # redirected POSIX file descriptors. In Python 3, these # are actually higher level IO objects. if sys.version_info[0] < 3: sys.stdout = os.fdopen(fd_out, "wb") sys.stderr = os.fdopen(fd_err, "wb") else: # Python 3 case sys.stdout = io.TextIOWrapper(os.fdopen(fd_out, 'wb')) sys.stderr = io.TextIOWrapper(os.fdopen(fd_err, 'wb')) # update DESI logging to use new stdout for name, logger in desi_loggers.items(): hformat = None while len(logger.handlers) > 0: h = logger.handlers[0] if hformat is None: hformat = h.formatter._fmt logger.removeHandler(h) # Add the current stdout. ch = logging.StreamHandler(sys.stdout) formatter = logging.Formatter(hformat, datefmt='%Y-%m-%dT%H:%M:%S') ch.setFormatter(formatter) logger.addHandler(ch) # redirect both stdout and stderr to the same file if to is None: to = "/dev/null" if rank == 0: log = get_logger() log.info("Begin log redirection to {} at {}".format( to, time.asctime())) # Save the original file descriptors so we can restore them later saved_fd_out = os.dup(fd_out) saved_fd_err = os.dup(fd_err) try: pto = to if to != "/dev/null": pto = "{}_{}".format(to, rank) # open python file, which creates low-level POSIX file # descriptor. file = open(pto, "w") # redirect stdout/stderr to this new file descriptor. _redirect(out_to=file.fileno(), err_to=file.fileno()) yield # allow code to be run with the redirected output # close python file handle, which will mark POSIX file # descriptor for garbage collection. That is fine since # we are about to overwrite those in the finally clause. file.close() finally: # flush python handles for good measure sys.stdout.flush() sys.stderr.flush() # restore old stdout and stderr _redirect(out_to=saved_fd_out, err_to=saved_fd_err) if nproc > 1: comm.barrier() # concatenate per-process files if rank == 0: with open(to, "w") as outfile: for p in range(nproc): outfile.write( "================= Process {} =================\n". format(p)) fname = "{}_{}".format(to, p) with open(fname) as infile: outfile.write(infile.read()) os.remove(fname) if nproc > 1: comm.barrier() if rank == 0: log = get_logger() log.info("End log redirection to {} at {}".format( to, time.asctime())) # flush python handles for good measure sys.stdout.flush() sys.stderr.flush() return
def add_calibration_exposures(exposures, flats_per_night=3, arcs_per_night=3, darks_per_night=0, zeroes_per_night=0, exptime=None, readout=30.0): """Prepare a list of science exposures for desisim.wrap-newexp. Insert calibration exposures at the start of each night, and add the following columns for all exposures: EXPID, PROGRAM, NIGHT, FLAVOR. Parameters ---------- exposures : table like or :class:`surveysim.exposures.ExposureList` A table of science exposures including, at a minimum, MJD, EXPTIME and TILEID columns. The exposures must be sorted by increasing MJD. Could be a numpy recarray, an astropy table, or an ExposureList object. Columns other than the required ones are copied to the output. flats_per_night : :class:`int`, optional Add this many arc exposures per night (default 3). arcs_per_night : :class:`int`, optional Add this many arc exposures per night (default 3). darks_per_night : :class:`int`, optional Add this many dark exposures per night (default 0). zeroes_per_night : :class:`int`, optional Add this many zero exposures per night (default 0). exptime : :class:`dict`, optional A dictionary setting calibration exposure times for each calibration flavor. readout : :class:`float`, optional Set readout time for calibration exposures (default 30.0 s). Returns ------- :class:`astropy.table.Table` The output table augmented with calibration exposures and additional columns. Raises ------ ValueError If the input is not sorted by increasing MJD/timestamp. """ if isinstance(exposures, surveysim.exposures.ExposureList): exposures = exposures._exposures[:exposures.nexp] nexp = len(exposures) MJD = exposures['MJD'] if not np.all(np.diff(MJD) > 0): raise ValueError("Input is not sorted by increasing MJD!") if exptime is None: exptime = {'flat': 10.0, 'arc': 10.0, 'dark': 1000.0, 'zero': 0.0} # Define the start of night calibration sequence. calib_time = lambda x: exptime[x] + readout calib_sequence = (['arc']*arcs_per_night + ['flat']*flats_per_night + ['dark']*darks_per_night + ['zero']*zeroes_per_night) calib_times = np.cumsum(np.array([calib_time(c) for c in calib_sequence]))[::-1] # Group exposures by night. MJD0 = desisurvey.utils.local_noon_on_date(desisurvey.utils.get_date(MJD[0])).mjd night_idx = np.floor(MJD - MJD0).astype(int) nights = np.unique(night_idx) ncalib = len(calib_sequence) * len(nights) # Initialize the output table. output = astropy.table.Table() nout = nexp + ncalib output['EXPID'] = np.arange(nout, dtype=np.int32) template = astropy.table.Table(dtype=exposures.dtype) for colname in template.colnames: col = template[colname] output[colname] = astropy.table.Column(dtype=col.dtype, length=nout) output['PROGRAM'] = astropy.table.Column(dtype=(str, len('BRIGHT')), length=nout) output['NIGHT'] = astropy.table.Column(dtype=(str, len('YYYYMMDD')), length=nout) output['FLAVOR'] = astropy.table.Column(dtype=(str, len('science')), length=nout) tiles = desisurvey.tiles.get_tiles() # Moon parameters are hardcoded for now. output['MOONFRAC'] = 0.5 output['MOONALT'] = -10. output['MOONSEP'] = 90. # Loop over nights. out_idx = 0 for n in nights: sel = (night_idx == n) nsel = np.count_nonzero(sel) first = np.where(sel)[0][0] MJD_first = MJD[first] NIGHT = desisurvey.utils.get_date(MJD_first).isoformat().replace('-', '') # Append the calibration sequence. for j, c in enumerate(calib_sequence): output['MJD'][out_idx] = MJD_first - calib_times[j]/86400.0 output['EXPTIME'][out_idx] = exptime[c] output['TILEID'][out_idx] = -1 output['PROGRAM'][out_idx] = 'CALIB' output['NIGHT'][out_idx] = NIGHT output['FLAVOR'][out_idx] = c out_idx += 1 # Append the night's science exposures. outslice = slice(out_idx, out_idx + nsel) for colname in template.colnames: output[colname][outslice] = exposures[colname][sel] TILEIDs = exposures['TILEID'][sel] output['PROGRAM'][outslice] = [ tiles.pass_program[p] for p in tiles.passnum[tiles.index(TILEIDs)]] output['NIGHT'][outslice] = NIGHT output['FLAVOR'][outslice] = 'science' out_idx += nsel assert out_idx == nout log = desiutil.log.get_logger() log.info('Added {} nightly calibration sequences of {} exposures each to {} science exposures.' .format(len(nights), len(calib_sequence), nexp)) return output
def calculate_initial_plan(args): """Calculate the initial survey plan. Use :func:`desisurvey.plan.load_weather` and :func:`desisurvey.plan.load_design_hourangles` to retrieve these data from the saved plan. Parameters ---------- args : object Object with attributes for parsed command-line arguments. """ log = desiutil.log.get_logger() config = desisurvey.config.Configuration() tiles = desisurvey.tiles.get_tiles() ephem = desisurvey.ephem.get_ephem() # Initialize the output file to write. hdus = fits.HDUList() hdr = fits.Header() # Calculate average weather factors for each day covered by # the ephemerides. first = desisurvey.ephem.START_DATE last = desisurvey.ephem.STOP_DATE years = np.arange(2007, 2018) fractions = [] for year in years: fractions.append( desimodel.weather.dome_closed_fractions(first, last, replay='Y{}'.format(year))) weather = 1 - np.mean(fractions, axis=0) # Save the weather fractions as the primary HDU. hdr['FIRST'] = first.isoformat() hdr['YEARS'] = ','.join(['{}'.format(yr) for yr in years]) start = config.first_day() stop = config.last_day() assert start >= first and stop <= last hdr['START'] = start.isoformat() hdr['STOP'] = stop.isoformat() hdr['TWILIGHT'] = args.include_twilight hdus.append(fits.ImageHDU(weather, header=hdr, name='WEATHER')) # Calculate the distribution of available LST in each program # during the nominal survey [start, stop). ilo, ihi = (start - first).days, (stop - first).days lst_hist, lst_bins = ephem.get_available_lst( nbins=args.nbins, weather=weather[ilo:ihi], include_twilight=args.include_twilight) # Initialize the output results table. design = astropy.table.Table() design['INIT'] = np.zeros(tiles.ntiles) design['HA'] = np.zeros(tiles.ntiles) design['TEXP'] = np.zeros(tiles.ntiles) # Optimize each program separately. stretches = dict( DARK=args.dark_stretch, GRAY=args.gray_stretch, BRIGHT=args.bright_stretch) for pindex, program in enumerate(tiles.PROGRAMS): sel = tiles.program_mask[program] if not np.any(sel): log.info('Skipping {} program with no tiles.'.format(program)) continue # Initialize an LST summary table. table = astropy.table.Table(meta={'ORIGIN': lst_bins[0]}) table['AVAIL'] = lst_hist[pindex] # Initailize an optimizer for this program. opt = desisurvey.optimize.Optimizer( program, lst_bins, lst_hist[pindex], init=args.init, center=None, stretch=stretches[program]) table['INIT'] = opt.plan_hist.copy() design['INIT'][sel] = opt.ha_initial # Initialize annealing cycles. ncycles = 0 binsize = 360. / args.nbins frac = args.adjust / binsize smoothing = args.smooth # Loop over annealing cycles. while ncycles < args.max_cycles: start_score = opt.eval_score(opt.plan_hist) for i in range(opt.ntiles): opt.improve(frac) if smoothing > 0: opt.smooth(alpha=smoothing) stop_score = opt.eval_score(opt.plan_hist) delta = (stop_score - start_score) / start_score RMSE = opt.RMSE_history[-1] loss = opt.loss_history[-1] log.info( '[{:03d}] dHA={:5.3f}deg '.format(ncycles + 1, frac * binsize) + 'RMSE={:6.2f}% LOSS={:5.2f}% delta(score)={:+5.1f}%' .format(1e2*RMSE, 1e2*loss, 1e2*delta)) # Both conditions must be satisfied to terminate. if RMSE < args.max_rmse and delta > -args.epsilon: break # Anneal parameters for next cycle. frac *= args.anneal smoothing *= args.anneal ncycles += 1 plan_sum = opt.plan_hist.sum() avail_sum = opt.lst_hist_sum margin = (avail_sum - plan_sum) / plan_sum log.info('{} plan uses {:.1f}h with {:.1f}h avail ({:.1f}% margin).' .format(program, plan_sum, avail_sum, 1e2 * margin)) # Save planned LST usage. table['PLAN'] = opt.plan_hist hdus.append(fits.BinTableHDU(table, name=program)) # Calculate exposure times in (solar) seconds. texp, _ = opt.get_exptime(opt.ha) texp *= 24. * 3600. / 360. * 0.99726956583 # Save results for this program. design['HA'][sel] = opt.ha design['TEXP'][sel] = texp hdus.append(fits.BinTableHDU(design, name='DESIGN')) fullname = config.get_path(args.save) hdus.writeto(fullname, overwrite=True) log.info('Saved initial plan to "{}".'.format(fullname))