def corr(img, d0=4, d1=4, nrand=50000): """ Computes the correlation function of an image. Args: img : 2D numpy array d0 : size of output correlation along axis 0 d1 : size of output correlation along axis 1 return correlation function as a 2D array of shape (d0,d1) """ log = desiutil.log.get_logger() mean, rms = calc_overscan(img, nsigma=5, niter=3) tmp = (img - mean) / rms log.debug("mean={:3.2f} rms={:3.2f}".format(mean, rms)) n0 = tmp.shape[0] n1 = tmp.shape[1] if nrand >= n0 * n1: ii0 = np.arange(n0 - d0) ii1 = np.arange(n1 - d1) else: # random subsampling to run faster ii0 = np.random.choice(n0 - d0, size=nrand) ii1 = np.random.choice(n1 - d1, size=nrand) corrimg = np.zeros((d0, d1)) for i0 in range(d0): for i1 in range(d1): corrimg[i0, i1] = np.median(tmp[ii0, ii1] * tmp[ii0 + i0, ii1 + i1]) #log.debug("corr[{},{}] = {:4.3f}".format(i0,i1,corrimg[i0,i1])) corrimg /= corrimg[0, 0] return corrimg
def run(self, indir): '''TODO: document''' log = desiutil.log.get_logger() infiles = glob.glob(os.path.join(indir, 'psf-*.fits')) results = list() for filename in infiles: log.debug(filename) hdr = fitsio.read_header(filename) night = hdr['NIGHT'] expid = hdr['EXPID'] cam = hdr['CAMERA'][0].upper() spectro = int(hdr['CAMERA'][1]) dico={"NIGHT":night,"EXPID":expid,"SPECTRO":spectro,"CAM":cam} xsig = fitsio.read(filename,"XSIG") ysig = fitsio.read(filename,"XSIG") dico["MEANXSIG"]=np.mean(xsig[:,0]) dico["MINXSIG"]=np.min(xsig[:,0]) dico["MAXXSIG"]=np.max(xsig[:,0]) dico["MEANYSIG"]=np.mean(ysig[:,0]) dico["MINYSIG"]=np.min(ysig[:,0]) dico["MAXYSIG"]=np.max(ysig[:,0]) log.debug("{} {} {} {} mean xsig={:3.2f} ysig={:3.2f}".format(night,expid,cam,spectro,dico["MEANXSIG"],dico["MEANYSIG"])) results.append(collections.OrderedDict(**dico)) return Table(results, names=results[0].keys())
def get_simulator(config='desi', num_fibers=1, camera_output=True, params=None): ''' returns new or cached specsim.simulator.Simulator object Also adds placeholder for BGS fiberloss if that isn't already in the config ''' if isinstance(config, Configuration): w = config.wavelength wavehash = (np.min(w), np.max(w), len(w)) key = (config.name, wavehash, num_fibers, camera_output) else: key = (config, num_fibers, camera_output) parammsg = 'with telescope {0}'.format( params['telescope']) if params else '' msg = '{0} Simulator for {1} {2}'.format(key, config, parammsg) if key in _simulators: log.debug('Returning cached {0}'.format(msg)) qsim = _simulators[key] defaults = _simdefaults[key] qsim.source.focal_xy = defaults['focal_xy'] qsim.atmosphere.airmass = defaults['airmass'] qsim.observation.exposure_time = defaults['exposure_time'] qsim.atmosphere.moon.moon_phase = defaults['moon_phase'] qsim.atmosphere.moon.separation_angle = defaults['moon_angle'] qsim.atmosphere.moon.moon_zenith = defaults['moon_zenith'] else: log.debug('Creating new {0}'.format(msg)) #- New config; create Simulator object import specsim.simulator qsim = specsim.simulator.Simulator(config, num_fibers, camera_output=camera_output, params=params) #- Cache defaults to reset back to original state later defaults = dict() defaults['focal_xy'] = qsim.source.focal_xy defaults['airmass'] = qsim.atmosphere.airmass defaults['exposure_time'] = qsim.observation.exposure_time defaults['moon_phase'] = qsim.atmosphere.moon.moon_phase defaults['moon_angle'] = qsim.atmosphere.moon.separation_angle defaults['moon_zenith'] = qsim.atmosphere.moon.moon_zenith _simulators[key] = qsim _simdefaults[key] = defaults # update the parameters if params: update_params(qsim, params) return qsim
def get_ephem(use_cache=True, write_cache=True): """Return tabulated ephemerides for (START_DATE,STOP_DATE). The pyephem module must be installed to calculate ephemerides, but is not necessary when a FITS file of precalcuated data is available. Parameters ---------- use_cache : bool Use cached ephemerides from memory or disk if possible when True. Otherwise, always calculate from scratch. write_cache : bool When True, write a generated table so it is available for future invocations. Writing only takes place when a cached object is not available or ``use_cache`` is False. Returns ------- Ephemerides Object with tabulated ephemerides for (START_DATE,STOP_DATE). """ global _ephem # Freeze IERS table for consistent results. desisurvey.utils.freeze_iers() # Use standardized string representation of dates. start_iso = START_DATE.isoformat() stop_iso = STOP_DATE.isoformat() range_iso = '({},{})'.format(start_iso, stop_iso) log = desiutil.log.get_logger() # First check for a cached object in memory. if use_cache and _ephem is not None: if _ephem.start_date != START_DATE or _ephem.stop_date != STOP_DATE: raise RuntimeError('START_DATE, STOP_DATE have changed.') log.debug('Returning cached ephemerides for {}.'.format(range_iso)) return _ephem # Next check for a FITS file on disk. config = desisurvey.config.Configuration() filename = config.get_path('ephem_{}_{}.fits'.format(start_iso, stop_iso)) if use_cache and os.path.exists(filename): # Save restored object in memory. _ephem = Ephemerides(START_DATE, STOP_DATE, restore=filename) log.info('Restored ephemerides for {} from {}.' .format(range_iso, filename)) return _ephem # Finally, create new ephemerides and save in the memory cache. log.info('Building ephemerides for {}...'.format(range_iso)) _ephem = Ephemerides(START_DATE, STOP_DATE) if write_cache: # Save the tabulated ephemerides to disk. _ephem._table.write(filename, overwrite=True) log.info('Saved ephemerides for {} to {}'.format(range_iso, filename)) return _ephem
def get_simulator(config='desi', num_fibers=1): ''' Returns new or cached specsim.simulator.Simulator object; Also adds placeholder for BGS fiberloss if that isn't already in the config. ''' key = (config, num_fibers) if key in _simulators: log.debug('Returning cached {} simulator'.format(config)) qsim = _simulators[key] defaults = _simdefaults[key] qsim.source.focal_xy = defaults['focal_xy'] qsim.atmosphere.airmass = defaults['airmass'] qsim.atmosphere.moon.moon_phase = defaults['moon_phase'] qsim.atmosphere.moon.separation_angle = defaults['moon_angle'] qsim.atmosphere.moon.moon_zenith = defaults['moon_zenith'] qsim.observation.exposure_time = defaults['exposure_time'] else: log.debug('Creating new {} simulator'.format(config)) ## New config; create Simulator object import specsim.simulator qsim = specsim.simulator.Simulator(config, num_fibers) ## TODO FIXME HACK: desimodel/specsim doesn't have BGS fiberloss yet, ## so scale from LRG if 'bgs' not in qsim.instrument.fiber_acceptance_dict.keys(): log.warning('Treating BGS fiberloss = 0.5 * LRG fiberloss') qsim.instrument.fiber_acceptance_dict[ 'bgs'] = 0.5 * qsim.instrument.fiber_acceptance_dict['lrg'] ## Cache defaults to reset back to original state later. defaults = dict() defaults['focal_xy'] = qsim.source.focal_xy defaults['airmass'] = qsim.atmosphere.airmass defaults['exposure_time'] = qsim.observation.exposure_time defaults['moon_phase'] = qsim.atmosphere.moon.moon_phase defaults['moon_angle'] = qsim.atmosphere.moon.separation_angle defaults['moon_zenith'] = qsim.atmosphere.moon.moon_zenith _simulators[key] = qsim _simdefaults[key] = defaults return qsim
def run(self, indir): '''TODO: document''' log = desiutil.log.get_logger() results = list() infiles = glob.glob(os.path.join(indir, 'qframe-*.fits')) if len(infiles) == 0 : log.error("no qframe in {}".format(indir)) return None for filename in infiles: qframe = read_qframe(filename) night = int(qframe.meta['NIGHT']) expid = int(qframe.meta['EXPID']) cam = qframe.meta['CAMERA'][0].upper() spectro = int(qframe.meta['CAMERA'][1]) log.debug("computing scores for {} {} {} {}".format(night,expid,cam,spectro)) scores,comments = compute_frame_scores(qframe,suffix="RAW",flux_per_angstrom=True) has_calib_frame = False cfilename=filename.replace("qframe","qcframe") if os.path.isfile(cfilename) : # add scores of calibrated sky-subtracted frame qcframe = read_qframe(cfilename) cscores,comments = compute_frame_scores(qcframe,suffix="CALIB",flux_per_angstrom=True) for k in cscores.keys() : scores[k] = cscores[k] has_calib_frame = True nfibers=scores['INTEG_RAW_FLUX_'+cam].size for f in range(nfibers) : fiber = int(qframe.fibermap["FIBER"][f]) if has_calib_frame : results.append(collections.OrderedDict( NIGHT=night, EXPID=expid, SPECTRO=spectro, CAM=cam, FIBER=fiber, INTEG_RAW_FLUX=scores['INTEG_RAW_FLUX_'+cam][f], MEDIAN_RAW_FLUX=scores['MEDIAN_RAW_FLUX_'+cam][f], MEDIAN_RAW_SNR=scores['MEDIAN_RAW_SNR_'+cam][f], INTEG_CALIB_FLUX=scores['INTEG_CALIB_FLUX_'+cam][f], MEDIAN_CALIB_FLUX=scores['MEDIAN_CALIB_FLUX_'+cam][f], MEDIAN_CALIB_SNR=scores['MEDIAN_CALIB_SNR_'+cam][f])) else : results.append(collections.OrderedDict( NIGHT=night, EXPID=expid, SPECTRO=spectro, CAM=cam, FIBER=fiber, INTEG_RAW_FLUX=scores['INTEG_RAW_FLUX_'+cam][f], MEDIAN_RAW_FLUX=scores['MEDIAN_RAW_FLUX_'+cam][f], MEDIAN_RAW_SNR=scores['MEDIAN_RAW_SNR_'+cam][f])) if len(results)==0 : return None return Table(results, names=results[0].keys())
def get_simulator(config='desi', num_fibers=1, camera_output=True): ''' returns new or cached specsim.simulator.Simulator object Also adds placeholder for BGS fiberloss if that isn't already in the config ''' if isinstance(config, Configuration): w = config.wavelength wavehash = (np.min(w), np.max(w), len(w)) key = (config.name, wavehash, num_fibers, camera_output) else: key = (config, num_fibers, camera_output) if key in _simulators: log.debug('Returning cached {} Simulator'.format(key)) qsim = _simulators[key] defaults = _simdefaults[key] qsim.source.focal_xy = defaults['focal_xy'] qsim.atmosphere.airmass = defaults['airmass'] qsim.observation.exposure_time = defaults['exposure_time'] qsim.atmosphere.moon.moon_phase = defaults['moon_phase'] qsim.atmosphere.moon.separation_angle = defaults['moon_angle'] qsim.atmosphere.moon.moon_zenith = defaults['moon_zenith'] else: log.debug('Creating new {} Simulator'.format(key)) #- New config; create Simulator object import specsim.simulator qsim = specsim.simulator.Simulator(config, num_fibers, camera_output=camera_output) #- Cache defaults to reset back to original state later defaults = dict() defaults['focal_xy'] = qsim.source.focal_xy defaults['airmass'] = qsim.atmosphere.airmass defaults['exposure_time'] = qsim.observation.exposure_time defaults['moon_phase'] = qsim.atmosphere.moon.moon_phase defaults['moon_angle'] = qsim.atmosphere.moon.separation_angle defaults['moon_zenith'] = qsim.atmosphere.moon.moon_zenith _simulators[key] = qsim _simdefaults[key] = defaults return qsim
def get_tiles(tiles_file=None, use_cache=True, write_cache=True): """Return a Tiles object with optional caching. You should normally always use the default arguments to ensure that tiles are defined consistently and efficiently between different classes. Parameters ---------- tiles_file : str or None Use the specified name to override config.tiles_file. use_cache : bool Use tiles previously cached in memory when True. Otherwise, (re)load tiles from disk. write_cache : bool If tiles need to be loaded from disk with this call, save them in a memory cache for future calls. """ global _cached_tiles log = desiutil.log.get_logger() config = desisurvey.config.Configuration() tiles_file = tiles_file or config.tiles_file() if use_cache and tiles_file in _cached_tiles: tiles = _cached_tiles[tiles_file] log.debug('Using cached tiles for "{}".'.format(tiles_file)) else: tiles = Tiles(tiles_file) log.info('Initialized tiles from "{}".'.format(tiles_file)) for pname in Tiles.PROGRAMS: pinfo = [] for passnum in tiles.program_passes[pname]: pinfo.append('{}({})'.format(passnum, tiles.pass_ntiles[passnum])) log.info('{:6s} passes(tiles): {}.'.format(pname, ', '.join(pinfo))) if write_cache: _cached_tiles[tiles_file] = tiles else: log.info('Tiles not cached for "{}".'.format(tiles_file)) return tiles
def _fix_amp_names(hdr): '''In-place fix of header `hdr` amp names 1-4 to A-D if needed.''' #- Assume that if any are right, all are right if 'DATASECA' in hdr: return log = desiutil.log.get_logger() log.debug('Correcting AMP 1-4 to A-D for night {} expid {}'.format( hdr['NIGHT'], hdr['EXPID'])) for prefix in [ 'GAIN', 'RDNOISE', 'PRESEC', 'PRRSEC', 'DATASEC', 'TRIMSEC', 'BIASSEC', 'ORSEC', 'CCDSEC', 'DETSEC', 'AMPSEC', 'OBSRDN', 'OVERSCN' ]: for ampnum, ampname in [('1', 'A'), ('2', 'B'), ('3', 'C'), ('4', 'D')]: if prefix + ampnum in hdr: hdr[prefix + ampname] = hdr[prefix + ampnum] hdr.delete(prefix + ampnum)
def run(self, indir): '''TODO: document''' log = desiutil.log.get_logger() infiles = glob.glob(os.path.join(indir, 'psf-*.fits')) if len(infiles) == 0: log.error('No {}/psf*.fits files found'.format(indir)) return None results = list() for filename in infiles: log.debug(filename) hdr = fitsio.read_header(filename) night = hdr['NIGHT'] expid = hdr['EXPID'] cam = hdr['CAMERA'][0].upper() spectro = int(hdr['CAMERA'][1]) dico={"NIGHT":night,"EXPID":expid,"SPECTRO":spectro,"CAM":cam} for k in ["MEANDX","MINDX","MAXDX","MEANDY","MINDY","MAXDY"] : dico[k]=hdr[k] log.debug("{} {} {} {}".format(night,expid,cam,spectro)) results.append(collections.OrderedDict(**dico)) return Table(results, names=results[0].keys())
def get_simulator(config='desi', num_fibers=1): ''' returns new or cached specsim.simulator.Simulator object Also adds placeholder for BGS fiberloss if that isn't already in the config ''' key = (config, num_fibers) if key in _simulators: log.debug('Returning cached {} Simulator'.format(config)) qsim = _simulators[key] defaults = _simdefaults[key] qsim.source.focal_xy = defaults['focal_xy'] qsim.atmosphere.airmass = defaults['airmass'] qsim.observation.exposure_time = defaults['exposure_time'] qsim.atmosphere.moon.moon_phase = defaults['moon_phase'] qsim.atmosphere.moon.separation_angle = defaults['moon_angle'] qsim.atmosphere.moon.moon_zenith = defaults['moon_zenith'] else: log.debug('Creating new {} Simulator'.format(config)) #- New config; create Simulator object import specsim.simulator qsim = specsim.simulator.Simulator(config, num_fibers) #- Cache defaults to reset back to original state later defaults = dict() defaults['focal_xy'] = qsim.source.focal_xy defaults['airmass'] = qsim.atmosphere.airmass defaults['exposure_time'] = qsim.observation.exposure_time defaults['moon_phase'] = qsim.atmosphere.moon.moon_phase defaults['moon_angle'] = qsim.atmosphere.moon.separation_angle defaults['moon_zenith'] = qsim.atmosphere.moon.moon_zenith _simulators[key] = qsim _simdefaults[key] = defaults return qsim
def __init__(self): self.output_type = "PER_FIBER" log = desiutil.log.get_logger() # load things we will use several times desiparams = load_desiparams() geometric_area_cm2 = 1e4 * desiparams["area"]["geometric_area"] # m2 self.thru_conversion_wavelength = 6000 # A energy_per_photon = ( constants.h * constants.c / (self.thru_conversion_wavelength * units.Angstrom)).to( units.erg).value #ergs self.thru_conversion_factor_ergs_per_cm2 = 1e17 * energy_per_photon / geometric_area_cm2 # ergs/cm2 # preload filters self.filters = dict() for photsys in ["N", "S"]: for band in ["G", "R", "Z", "W1", "W2"]: self.filters[band + photsys] = load_legacy_survey_filter( band=band, photsys=photsys) # define wavelength array min_filter_wave = 100000 max_filter_wave = 0 for k in self.filters.keys(): log.debug(self.filters[k].name) min_filter_wave = min(min_filter_wave, np.min(self.filters[k].wavelength)) - 0.1 max_filter_wave = max(max_filter_wave, np.max(self.filters[k].wavelength)) + 0.1 log.debug("min max wavelength for filters= {:d} , {:d}".format( int(min_filter_wave), int(max_filter_wave))) self.rwave = np.linspace(min_filter_wave, max_filter_wave, int(max_filter_wave - min_filter_wave))
def stdouterr_redirected(to=None, comm=None): """ Redirect stdout and stderr to a file. The general technique is based on: http://stackoverflow.com/questions/5081657 http://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/ One difference here is that each process in the communicator redirects to a different temporary file, and the upon exit from the context the rank zero process concatenates these in order to the file result. Args: to (str): The output file name. comm (mpi4py.MPI.Comm): The optional MPI communicator. """ nproc = 1 rank = 0 if comm is not None: nproc = comm.size rank = comm.rank # The currently active POSIX file descriptors fd_out = sys.stdout.fileno() fd_err = sys.stderr.fileno() # The DESI loggers. desi_loggers = desiutil.log._desiutil_log_root def _redirect(out_to, err_to): # Flush the C-level buffers if c_stdout is not None: libc.fflush(c_stdout) if c_stderr is not None: libc.fflush(c_stderr) # This closes the python file handles, and marks the POSIX # file descriptors for garbage collection- UNLESS those # are the special file descriptors for stderr/stdout. sys.stdout.close() sys.stderr.close() # Close fd_out/fd_err if they are open, and copy the # input file descriptors to these. os.dup2(out_to, fd_out) os.dup2(err_to, fd_err) # Create a new sys.stdout / sys.stderr that points to the # redirected POSIX file descriptors. In Python 3, these # are actually higher level IO objects. if sys.version_info[0] < 3: sys.stdout = os.fdopen(fd_out, "wb") sys.stderr = os.fdopen(fd_err, "wb") else: # Python 3 case sys.stdout = io.TextIOWrapper(os.fdopen(fd_out, 'wb')) sys.stderr = io.TextIOWrapper(os.fdopen(fd_err, 'wb')) # update DESI logging to use new stdout for name, logger in desi_loggers.items(): hformat = None while len(logger.handlers) > 0: h = logger.handlers[0] if hformat is None: hformat = h.formatter._fmt logger.removeHandler(h) # Add the current stdout. ch = logging.StreamHandler(sys.stdout) formatter = logging.Formatter(hformat, datefmt='%Y-%m-%dT%H:%M:%S') ch.setFormatter(formatter) logger.addHandler(ch) # redirect both stdout and stderr to the same file if to is None: to = "/dev/null" if rank == 0: log = get_logger() log.debug("Begin log redirection to {} at {}".format( to, time.asctime())) # Save the original file descriptors so we can restore them later saved_fd_out = os.dup(fd_out) saved_fd_err = os.dup(fd_err) try: pto = to if to != "/dev/null": pto = "{}_{}".format(to, rank) # open python file, which creates low-level POSIX file # descriptor. file = open(pto, "w") # redirect stdout/stderr to this new file descriptor. _redirect(out_to=file.fileno(), err_to=file.fileno()) yield # allow code to be run with the redirected output # close python file handle, which will mark POSIX file # descriptor for garbage collection. That is fine since # we are about to overwrite those in the finally clause. file.close() finally: # restore old stdout and stderr _redirect(out_to=saved_fd_out, err_to=saved_fd_err) if comm is not None: comm.barrier() # concatenate per-process files if rank == 0: with open(to, "w") as outfile: for p in range(nproc): outfile.write( "================= Process {} =================\n". format(p)) fname = "{}_{}".format(to, p) with open(fname) as infile: outfile.write(infile.read()) os.remove(fname) if comm is not None: comm.barrier() if rank == 0: log = get_logger() log.debug("End log redirection to {} at {}".format( to, time.asctime())) # flush python handles for good measure sys.stdout.flush() sys.stderr.flush() return
def run(self, indir, outfile=None, jsonfile=None): '''TODO: document''' log = desiutil.log.get_logger() log.debug('Running QA in {}'.format(indir)) print('here qa.runner', indir) preprocfiles = sorted(glob.glob('{}/preproc-*.fits'.format(indir))) if len(preprocfiles) == 0: log.error('No preproc files found in {}'.format(indir)) return None # We can have different obstypes (signal+dark) with calibration data # obtained with a calibration slit hooked to a single spectrograph. # So we have to loop over all frames to check if there are science, # arc, or flat obstypes as guessed by qproc. qframefiles = sorted(glob.glob('{}/qframe-*.fits'.format(indir))) if len(qframefiles) == 0: # no qframe so it's either zero or dark hdr = fitsio.read_header(preprocfiles[0], 0) if 'OBSTYPE' in hdr: obstype = hdr['FLAVOR'].strip() else: log.warning("Using FLAVOR instead of missing OBSTYPE") obstype = hdr['FLAVOR'].strip() else: obstype = None log.debug("Reading qframe headers to guess flavor ...") for qframefile in qframefiles: # look at all of them and prefer arc or flat over dark or zero hdr = fitsio.read_header(qframefile, 0) if 'OBSTYPE' in hdr: this_obstype = hdr['OBSTYPE'].strip().upper() else: log.warning("Using FLAVOR instead of missing OBSTYPE") obstype = hdr['FLAVOR'].strip() if this_obstype == "ARC" or this_obstype == "FLAT" \ or this_obstype == "TESTARC" or this_obstype == "TESTFLAT" : obstype = this_obstype # we use this so we exit the loop break elif obstype == None: obstype = this_obstype # we stay in the loop in case another frame has another obstype log.debug('Found OBSTYPE={} files'.format(obstype)) results = dict() for qa in self.qalist: if qa.valid_obstype(obstype): log.info('{} Running {} {}'.format(timestamp(), qa, qa.output_type)) qa_results = None try: qa_results = qa.run(indir) except Exception as err: log.warning('{} failed on {} because {}; skipping'.format( qa, indir, str(err))) exc_info = sys.exc_info() traceback.print_exception(*exc_info) del exc_info #raise(err) #- TODO: print traceback somewhere useful if qa_results is not None: if qa.output_type not in results: results[qa.output_type] = list() results[qa.output_type].append(qa_results) else: log.debug('Skip {} {} for {}'.format(qa, qa.output_type, obstype)) #- Combine results for different types of QA join_keys = dict( PER_AMP=['NIGHT', 'EXPID', 'SPECTRO', 'CAM', 'AMP'], PER_CAMERA=['NIGHT', 'EXPID', 'SPECTRO', 'CAM'], PER_FIBER=['NIGHT', 'EXPID', 'SPECTRO', 'FIBER'], PER_CAMFIBER=['NIGHT', 'EXPID', 'SPECTRO', 'CAM', 'FIBER'], PER_SPECTRO=['NIGHT', 'EXPID', 'SPECTRO'], PER_EXP=['NIGHT', 'EXPID'], QPROC_STATUS=['NIGHT', 'EXPID', 'SPECTRO', 'CAM']) if jsonfile is not None: if os.path.exists(jsonfile): with open(jsonfile, 'r') as myfile: json_data = myfile.read() json_data = json.loads(json_data) else: json_data = dict() rewrite_necessary = False for key1 in results: if key1 != "PER_CAMFIBER" and key1.startswith("PER_"): colnames_lst = results[key1][0].colnames if key1 in join_keys: for i in join_keys[key1]: colnames_lst.remove(i) if key1 not in json_data: json_data[key1] = colnames_lst rewrite_necessary = True else: for aspect in colnames_lst: if aspect not in json_data[key1]: json_data[key1] += [aspect] rewrite_necessary = True if rewrite_necessary: if os.path.isdir(os.path.dirname(jsonfile)): with open(jsonfile, 'w') as out: json.dump(json_data, out) print('Wrote {}'.format(jsonfile)) for qatype in list(results.keys()): if len(results[qatype]) == 1: results[qatype] = results[qatype][0] else: tx = results[qatype][0] for i in range(1, len(results[qatype])): tx = join(tx, results[qatype][i], keys=join_keys[qatype], join_type='outer') results[qatype] = tx #- convert python string to bytes for FITS format compatibility if 'AMP' in results[qatype].colnames: results[qatype]['AMP'] = results[qatype]['AMP'].astype('S1') if 'CAM' in results[qatype].colnames: results[qatype]['CAM'] = results[qatype]['CAM'].astype('S1') #- TODO: NIGHT/EXPID/SPECTRO/FIBER int64 -> int32 or int16 #- TODO: metrics from float64 -> float32 if outfile is not None: for tx in results.values(): night = tx['NIGHT'][0] expid = tx['EXPID'][0] break #- To do: consider propagating header from indir/desi*.fits.fz log.info('{} Writing {}'.format(timestamp(), outfile)) tmpfile = outfile + '.tmp' with fitsio.FITS(tmpfile, 'rw', clobber=True) as fx: fx.write(np.zeros(3, dtype=float), extname='PRIMARY', header=hdr) for qatype, qatable in results.items(): fx.write_table(qatable.as_array(), extname=qatype, header=hdr) os.rename(tmpfile, outfile) log.info('{} Finished writing {}'.format(timestamp(), outfile)) return results
def simulate_night(night, scheduler, stats, explist, weather, use_twilight=False, update_interval=10., plot=False, verbose=False): """Simulate one night of observing. Uses the online tile scheduler and exposure time calculator. Parameters ---------- night : datetime.date Date that the simulated night starts. scheduler : :class:`desisurvey.scheduler.Scheduler` Next tile scheduler to use. stats : :class:`surveysim.stats.SurveyStatistics` Object for accumulating simulated survey statistics. explist : :class:`surveysim.exposures.ExposureList` Object for recording simulated exposures. weather : :class:`surveysim.weather.Weather` Simulated weather conditions to use. use_twlight : bool Observe during twilight when True. update_interval : float Interval in seconds for simulated ETC updates. plot : bool Generate a plot summarizing the simulated night when True. verbose : bool Produce verbose output on simulation progress when True. """ log = desiutil.log.get_logger() update_interval_days = update_interval / 86400. night = desisurvey.utils.get_date(night) nightstats = stats.get_night(night) label = str(night) # Lookup this night's sunset and sunrise MJD values. night_ephem = scheduler.ephem.get_night(night) if use_twilight: begin = night_ephem['brightdusk'] end = night_ephem['brightdawn'] else: begin = night_ephem['dusk'] end = night_ephem['dawn'] nightstats['tsched'] = end - begin log.debug('Simulating observing on {} from MJD {:.5f} - {:.5f}.' .format(night, begin, end)) # Find weather time steps that cover this night. weather_mjd = weather._table['mjd'].data ilo = np.searchsorted(weather_mjd, begin, side='left') - 1 ihi = np.searchsorted(weather_mjd, end, side='right') + 2 assert weather_mjd[ilo] < begin and weather_mjd[ihi] > end weather_mjd = weather_mjd[ilo:ihi] seeing = weather._table['seeing'].data[ilo:ihi] transp = weather._table['transparency'].data[ilo:ihi] # Fix this in the weather generator instead? transp = np.maximum(0.1, transp) dome = weather._table['open'].data[ilo:ihi] if not np.any(dome): log.debug('Dome closed all night.') return scheduler.init_night(night, use_twilight=use_twilight) ETC = desisurvey.etc.ExposureTimeCalculator(save_history=plot) nexp_last = explist.nexp # Build linear interpolators for observing conditions. # This implementation is faster than scipy.interpolate.interp1d() # when mjd values are gradually increasing. weather_idx = 0 dmjd_weather = weather_mjd[1] - weather_mjd[0] def get_weather(mjd): nonlocal weather_idx while mjd >= weather_mjd[weather_idx + 1]: weather_idx += 1 s = (mjd - weather_mjd[weather_idx]) / dmjd_weather return ( seeing[weather_idx] * (1 - s) + seeing[weather_idx + 1] * s, transp[weather_idx] * (1 - s) + transp[weather_idx + 1] * s) # Define time intervals to use in units of days (move to config?) NO_TILE_AVAIL_DELAY = 30. / 86400. # Step through the night. dome_is_open = False mjd_now = weather_mjd[0] completed_last = scheduler.completed_by_pass.copy() while mjd_now < end: if not dome_is_open: # Advance to the next dome opening, if any. idx_now = np.searchsorted(weather_mjd, mjd_now, side='left') if not np.any(dome[idx_now:]): # Dome is closed for the rest of the night. mjd_now = end break idx_open = idx_now + np.argmax(dome[idx_now:]) assert dome[idx_open] == True and (idx_open == 0 or dome[idx_open - 1] == False) mjd_now = weather_mjd[idx_open] if mjd_now >= end: # The next dome opening is after the end of the night. # This can happen if we are not using twilight. break # Find the next closing. if np.all(dome[idx_open:]): next_dome_closing = end else: idx_close = idx_open + np.argmin(dome[idx_open:]) assert dome[idx_close] == False and dome[idx_close - 1] == True next_dome_closing = min(end, weather_mjd[idx_close]) dome_is_open = True weather_idx = idx_open # == NEXT TILE =========================================================== # Dome is open from mjd_now to next_dome_closing. mjd_last = mjd_now tdead = 0. # Get the current observing conditions. seeing_now, transp_now = get_weather(mjd_now) # Get the next tile to observe from the scheduler. tileid, passnum, snr2frac_start, exposure_factor, airmass, sched_program, mjd_program_end = \ scheduler.next_tile(mjd_now, ETC, seeing_now, transp_now) if tileid is None: # Deadtime while we delay and try again. mjd_now += NO_TILE_AVAIL_DELAY if mjd_now >= next_dome_closing: # Dome closed during deadtime. mjd_now = next_dome_closing dome_is_open = False tdead += mjd_now - mjd_last else: # Setup for a new field. mjd_now += ETC.NEW_FIELD_SETUP if mjd_now >= next_dome_closing: # Setup interrupted by dome closing. mjd_now = next_dome_closing dome_is_open = False # Record an aborted setup. nightstats['nsetup_abort'][passnum] += 1 else: # Record a completed setup. nightstats['nsetup'][passnum] += 1 # Charge this as setup time whether or not it was aborted. nightstats['tsetup'][passnum] += mjd_now - mjd_last if dome_is_open: # Lookup the program of the next tile, which might be # different from the scheduled program in ``sched_program``. tile_program = scheduler.tiles.pass_program[passnum] # Loop over repeated exposures of the same tile. continue_this_tile = True while continue_this_tile: # -- NEXT EXPOSURE --------------------------------------------------- # Get the current observing conditions. seeing_now, transp_now = get_weather(mjd_now) sky_now = 1. # Use the ETC to control the shutter. mjd_open_shutter = mjd_now ETC.start(mjd_now, tileid, tile_program, snr2frac_start, exposure_factor, seeing_now, transp_now, sky_now) integrating = True while integrating: mjd_now += update_interval_days if mjd_now >= next_dome_closing: # Current exposure is interrupted by dome closing. mjd_now = next_dome_closing dome_is_open = False integrating = False continue_this_tile = False elif mjd_now >= mjd_program_end: # Current exposure is interrupted by a program change. mjd_now = mjd_program_end integrating = False continue_this_tile = False # Get the current observing conditions. seeing_now, transp_now = get_weather(mjd_now) sky_now = 1. # Update the SNR. if not ETC.update(mjd_now, seeing_now, transp_now, sky_now): # Current exposure reached its target SNR according to the ETC. integrating= False # stop() will return False if this is a cosmic split and # more integration is still required. if ETC.stop(mjd_now): continue_this_tile = False # Record this exposure assert np.allclose(ETC.exptime, mjd_now - mjd_open_shutter) nightstats['tscience'][passnum] += ETC.exptime nightstats['nexp'][passnum] += 1 explist.add( mjd_now - ETC.exptime, 86400 * ETC.exptime, tileid, ETC.snr2frac, airmass, seeing_now, transp_now, sky_now) scheduler.update_snr(tileid, ETC.snr2frac) if continue_this_tile: # Prepare for the next exposure of the same tile. snr2frac_start = ETC.snr2frac mjd_split_start = mjd_now mjd_now += ETC.SAME_FIELD_SETUP if mjd_now >= next_dome_closing: # Setup for next exposure of same tile interrupted by dome closing. mjd_now = next_dome_closing dome_is_open = False continue_this_tile = False # Record an aborted split. nightstats['nsplit_abort'][passnum] += 1 else: # Record a completed split. nightstats['nsplit'][passnum] += 1 # Charge this as split time, whether or not is was aborted. nightstats['tsplit'][passnum] += mjd_now - mjd_split_start # -------------------------------------------------------------------- # Update statistics for the scheduled program (which might be different from # the program of the tile we just observed). pidx = scheduler.tiles.PROGRAM_INDEX[sched_program] nightstats['tdead'][pidx] += tdead nightstats['topen'][pidx] += mjd_now - mjd_last # All done if we have observed all tiles. if scheduler.survey_completed(): break # ======================================================================== # Save the number of tiles completed per pass in the nightly statistics. nightstats['completed'][:] = scheduler.completed_by_pass - completed_last if plot: import matplotlib.pyplot as plt fig, axes = plt.subplots(2, 1, figsize=(15, 10), sharex=True) ax = axes[0] ax.plot(weather_mjd, seeing, 'r-', label='Seeing') ax.plot([], [], 'b-', label='Transparency') ax.legend(ncol=2, loc='lower center') ax.set_ylabel('Seeing FWHM [arcsec]') rhs = ax.twinx() rhs.plot(weather_mjd, transp, 'b-') rhs.set_ylabel('Transparency') ax = axes[1] changes = np.where(np.abs(np.diff(dome)) == 1)[0] for idx in changes: ax.axvline(weather_mjd[idx + 1], ls='-', c='r') mjd_history = np.array(ETC.history['mjd']) snr2frac_history = np.array(ETC.history['snr2frac']) for expinfo in explist._exposures[nexp_last: explist.nexp]: passnum = scheduler.tiles.passnum[scheduler.tiles.index(expinfo['TILEID'])] program = scheduler.tiles.pass_program[passnum] color = desisurvey.plots.program_color[program] t1 = expinfo['MJD'] t2 = t1 + expinfo['EXPTIME'] / 86400 sel = (mjd_history >= t1) & (mjd_history <= t2) ax.fill_between(mjd_history[sel], snr2frac_history[sel], color=color, alpha=0.5, lw=0) for t in scheduler.night_changes: ax.axvline(t, c='b', ls=':') ax.set_xlim(weather_mjd[0], weather_mjd[-1]) ax.set_xlabel('MJD During {}'.format(label)) ax.set_ylim(0, 1) ax.set_ylabel('Integrated SNR2 Fraction')
def find_latest_expdir(basedir, processed, startdate=None): ''' finds the earliest unprocessed basedir/YEARMMDD/EXPID from the latest YEARMMDD without traversing the whole tree Args: basedir : a directory of nights with exposures processed : set of exposure directories already processed Options: startdate : the earliest night to consider processing YYYYMMDD Returns directory, or None if no matching directories are found Note: if you want the first unprocessed directory, use `find_unprocessed_expdir` instead ''' if startdate: startdate = str(startdate) else: startdate = '' log = desiutil.log.get_logger(level='DEBUG') ### log.debug('Looking for unprocessed exposures at {}'.format(time.asctime())) #- Search for most recent basedir/YEARMMDD for dirname in sorted(os.listdir(basedir), reverse=True): nightdir = os.path.join(basedir, dirname) if re.match('20\d{6}', dirname) and dirname >= startdate and \ os.path.isdir(nightdir): break #- if for loop completes without finding nightdir to break, run this else else: log.debug('No YEARMMDD dirs found in {}'.format(basedir)) return None night = dirname log.debug('{} Looking for exposures in {}'.format(timestamp(), nightdir)) spectrofiles = sorted(glob.glob(nightdir + '/*/desi*.fits.fz')) if len(spectrofiles) > 0: log.debug('{} found {} desi spectro files though {}'.format( timestamp(), len(spectrofiles), os.path.basename(spectrofiles[-1]))) else: log.debug('{} no new spectro files yet'.format(timestamp())) return None for filename in spectrofiles: dirname = os.path.dirname(filename) if dirname not in processed: log.debug('{} selected {}'.format(timestamp(), filename)) return dirname else: log.debug('{} no new spectro files found'.format(timestamp())) return None
def make_plots(infile, basedir, preprocdir=None, logdir=None, rawdir=None, cameras=None): '''Make plots for a single exposure Args: infile: input QA fits file with HDUs like PER_AMP, PER_FIBER, ... basedir: write output HTML to basedir/NIGHT/EXPID/ Options: preprocdir: directory to where the "preproc-*-*.fits" are located. If not provided, function will NOT generate any image files from any preproc fits file. logdir: directory to where the "qproc-*-*.log" are located. If not provided, function will NOT display any logfiles. rawdir: directory to where the raw data files are located, including "guide-rois-*.fits" and "centroid-*.json" files, are located. If not provided, the function will not plot the guide plots. cameras: list of cameras (strings) to generate image files of. If not provided, will generate a cameras list from parcing through the preproc fits files in the preprocdir ''' from nightwatch.webpages import amp as web_amp from nightwatch.webpages import camfiber as web_camfiber from nightwatch.webpages import camera as web_camera from nightwatch.webpages import summary as web_summary from nightwatch.webpages import lastexp as web_lastexp from nightwatch.webpages import guide as web_guide from nightwatch.webpages import guideimage as web_guideimage from nightwatch.webpages import placeholder as web_placeholder from . import io log = desiutil.log.get_logger() qadata = io.read_qa(infile) header = qadata['HEADER'] night = header['NIGHT'] expid = header['EXPID'] #- Early data have wrong NIGHT in header; check by hand #- YEARMMDD/EXPID/infile dirnight = os.path.basename(os.path.dirname(os.path.dirname(infile))) if re.match('20\d{6}', dirnight) and dirnight != str(night): log.warning('Correcting {} header night {} to {}'.format( infile, night, dirnight)) night = int(dirnight) header['NIGHT'] = night #- Create output exposures plot directory if needed expdir = os.path.join(basedir, str(night), '{:08d}'.format(expid)) if not os.path.isdir(expdir): log.info('Creating {}'.format(expdir)) os.makedirs(expdir, exist_ok=True) if 'PER_AMP' in qadata: htmlfile = '{}/qa-amp-{:08d}.html'.format(expdir, expid) pc = web_amp.write_amp_html(htmlfile, qadata['PER_AMP'], header) print('Wrote {}'.format(htmlfile)) else: htmlfile = '{}/qa-amp-{:08d}.html'.format(expdir, expid) pc = web_placeholder.write_placeholder_html(htmlfile, header, "PER_AMP") htmlfile = '{}/qa-camfiber-{:08d}.html'.format(expdir, expid) if 'PER_CAMFIBER' in qadata: try: pc = web_camfiber.write_camfiber_html(htmlfile, qadata['PER_CAMFIBER'], header) print('Wrote {}'.format(htmlfile)) except Exception as err: web_placeholder.handle_failed_plot(htmlfile, header, "PER_CAMFIBER") else: pc = web_placeholder.write_placeholder_html(htmlfile, header, "PER_CAMFIBER") htmlfile = '{}/qa-camera-{:08d}.html'.format(expdir, expid) if 'PER_CAMERA' in qadata: try: pc = web_camera.write_camera_html(htmlfile, qadata['PER_CAMERA'], header) print('Wrote {}'.format(htmlfile)) except Exception as err: web_placeholder.handle_failed_plot(htmlfile, header, "PER_CAMERA") else: pc = web_placeholder.write_placeholder_html(htmlfile, header, "PER_CAMERA") htmlfile = '{}/qa-summary-{:08d}.html'.format(expdir, expid) web_summary.write_summary_html(htmlfile, qadata, preprocdir) print('Wrote {}'.format(htmlfile)) #- Note: last exposure goes in basedir, not expdir=basedir/NIGHT/EXPID htmlfile = '{}/qa-lastexp.html'.format(basedir) web_lastexp.write_lastexp_html(htmlfile, qadata, preprocdir) print('Wrote {}'.format(htmlfile)) if rawdir: #- plot guide metric plots try: guidedata = io.get_guide_data(night, expid, rawdir) htmlfile = '{}/qa-guide-{:08d}.html'.format(expdir, expid) web_guide.write_guide_html(htmlfile, header, guidedata) print('Wrote {}'.format(htmlfile)) except (FileNotFoundError, OSError, IOError): print('Unable to find guide data, not plotting guide plots') htmlfile = '{}/qa-guide-{:08d}.html'.format(expdir, expid) pc = web_placeholder.write_placeholder_html( htmlfile, header, "GUIDING") #- plot guide image movies try: htmlfile = '{expdir}/guide-image-{expid:08d}.html'.format( expdir=expdir, expid=expid) image_data = io.get_guide_images(night, expid, rawdir) web_guideimage.write_guide_image_html(image_data, htmlfile, night, expid) print('Wrote {}'.format(htmlfile)) except (FileNotFoundError, OSError, IOError): print('Unable to find guide data, not plotting guide image plots') htmlfile = '{expdir}/guide-image-{expid:08d}.html'.format( expdir=expdir, expid=expid) pc = web_placeholder.write_placeholder_html( htmlfile, header, "GUIDE_IMAGES") #- regardless of if logdir or preprocdir, identifying failed qprocs by comparing #- generated preproc files to generated logfiles qproc_fails = [] if cameras is None: cameras = [] import glob for preprocfile in glob.glob( os.path.join(preprocdir, 'preproc-*-*.fits')): cameras += [os.path.basename(preprocfile).split('-')[1]] log_cams = [] log_outputs = [ i for i in os.listdir(logdir) if re.match(r'qproc.*\.log', i) ] for log_output in log_outputs: l_cam = log_output.split("-")[1] log_cams += [l_cam] if l_cam not in cameras: qproc_fails.append(l_cam) from nightwatch.webpages import plotimage as web_plotimage if (preprocdir is not None): #- plot preprocessed images downsample = 4 ncpu = get_ncpu(None) pinput = os.path.join(preprocdir, "preproc-{}-{:08d}.fits") output = os.path.join(expdir, "preproc-{}-{:08d}-4x.html") argslist = [(pinput.format(cam, expid), output.format(cam, expid), downsample, night) for cam in cameras] if ncpu > 1: pool = mp.Pool(ncpu) pool.starmap(web_plotimage.write_image_html, argslist) pool.close() pool.join() else: for args in argslist: web_plotimage.write_image_html(*args) #- plot preproc nav table navtable_output = '{}/qa-amp-{:08d}-preproc_table.html'.format( expdir, expid) web_plotimage.write_preproc_table_html(preprocdir, night, expid, downsample, navtable_output) if (logdir is not None): #- plot logfiles log.debug('Log directory: {}'.format(logdir)) error_colors = dict() for log_cam in log_cams: qinput = os.path.join(logdir, "qproc-{}-{:08d}.log".format(log_cam, expid)) output = os.path.join( expdir, "qproc-{}-{:08d}-logfile.html".format(log_cam, expid)) log.debug('qproc log: {}'.format(qinput)) e = web_summary.write_logfile_html(qinput, output, night) error_colors[log_cam] = e #- plot logfile nav table htmlfile = '{}/qa-summary-{:08d}-logfiles_table.html'.format( expdir, expid) web_summary.write_logtable_html(htmlfile, logdir, night, expid, available=log_cams, error_colors=error_colors)
def freeze_iers(name='iers_frozen.ecsv', ignore_warnings=True): """Use a frozen IERS table saved with this package. This should be called at the beginning of a script that calls astropy time and coordinates functions which refer to the UT1-UTC and polar motions tabulated by IERS. The purpose is to ensure identical results across systems and astropy releases, to avoid a potential network download, and to eliminate some astropy warnings. After this call, the loaded table will be returned by :func:`astropy.utils.iers.IERS_Auto.open()` and treated like a a normal IERS table by all astropy code. Specifically, this method registers an instance of a custom IERS_Frozen class that inherits from IERS_B and overrides :meth:`astropy.utils.iers.IERS._check_interpolate_indices` to prevent any IERSRangeError being raised. See http://docs.astropy.org/en/stable/utils/iers.html for details. This function returns immediately after the first time it is called, so it it safe to insert anywhere that consistent IERS models are required, and subsequent calls with different args will have no effect. The :func:`desisurvey.utils.plot_iers` function is useful for inspecting IERS tables and how they are extrapolated to DESI survey dates. Parameters ---------- name : str Name of the file to load the frozen IERS table from. Should normally be relative and then refers to this package's data/ directory. Must end with the .ecsv extension. ignore_warnings : bool Ignore ERFA and IERS warnings about future dates generated by astropy time and coordinates functions. Specifically, ERFA warnings containing the string "dubious year" are filtered out, as well as AstropyWarnings related to IERS table extrapolation. """ log = desiutil.log.get_logger() if desisurvey.utils._iers_is_frozen: log.debug('IERS table already frozen.') return log.info('Freezing IERS table used by astropy time, coordinates.') # Validate the save_name extension. _, ext = os.path.splitext(name) if ext != '.ecsv': raise ValueError('Expected .ecsv extension for {0}.'.format(name)) # Locate the file in our package data/ directory. if not os.path.isabs(name): name = astropy.utils.data._find_pkg_data_path( os.path.join('data', name)) if not os.path.exists(name): raise ValueError('No such IERS file: {0}.'.format(name)) # Clear any current IERS table. astropy.utils.iers.IERS.close() # Initialize the global IERS table. We load the table by # hand since the IERS open() method hardcodes format='cds'. try: table = astropy.table.Table.read(name, format='ascii.ecsv').filled() except IOError: raise RuntimeError('Unable to load IERS table from {0}.'.format(name)) # Define a subclass of IERS_B that overrides _check_interpolate_indices # to prevent any IERSRangeError being raised. class IERS_Frozen(astropy.utils.iers.IERS_B): def _check_interpolate_indices(self, indices_orig, indices_clipped, max_input_mjd): pass # Create and register an instance of this class from the table. iers = IERS_Frozen(table) astropy.utils.iers.IERS.iers_table = iers # Prevent any attempts to automatically download updated IERS-A tables. astropy.utils.iers.conf.auto_download = False astropy.utils.iers.conf.auto_max_age = None astropy.utils.iers.conf.iers_auto_url = 'frozen' # Sanity check. if not (astropy.utils.iers.IERS_Auto.open() is iers): raise RuntimeError('Frozen IERS is not installed as the default.') if ignore_warnings: warnings.filterwarnings( 'ignore', category=astropy._erfa.core.ErfaWarning, message= r'ERFA function \"[a-z0-9_]+\" yielded [0-9]+ of \"dubious year') warnings.filterwarnings( 'ignore', category=astropy.utils.exceptions.AstropyWarning, message=r'Tried to get polar motions for times after IERS data') warnings.filterwarnings( 'ignore', category=astropy.utils.exceptions.AstropyWarning, message=r'\(some\) times are outside of range covered by IERS') # Shortcircuit any subsequent calls to this function. desisurvey.utils._iers_is_frozen = True
def simulate_night(night, scheduler, stats, explist, weather, use_twilight=False, use_simplesky=False, update_interval=10., plot=False, verbose=False): """Simulate one night of observing. Uses the online tile scheduler and exposure time calculator. Parameters ---------- night : datetime.date Date that the simulated night starts. scheduler : :class:`desisurvey.scheduler.Scheduler` Next tile scheduler to use. stats : :class:`surveysim.stats.SurveyStatistics` Object for accumulating simulated survey statistics. explist : :class:`surveysim.exposures.ExposureList` Object for recording simulated exposures. weather : :class:`surveysim.weather.Weather` Simulated weather conditions to use. use_twlight : bool Observe during twilight when True. update_interval : float Interval in seconds for simulated ETC updates. plot : bool Generate a plot summarizing the simulated night when True. verbose : bool Produce verbose output on simulation progress when True. """ log = desiutil.log.get_logger() update_interval_days = update_interval / 86400. night = desisurvey.utils.get_date(night) nightstats = stats.get_night(night) label = str(night) # Lookup this night's sunset and sunrise MJD values. night_ephem = scheduler.ephem.get_night(night) night_programs, night_changes = scheduler.ephem.get_night_program(night) if use_twilight: begin = night_ephem['brightdusk'] end = night_ephem['brightdawn'] else: begin = night_ephem['dusk'] end = night_ephem['dawn'] nightstats['tsched'] = end - begin log.debug('Simulating observing on {} from MJD {:.5f} - {:.5f}.'.format( night, begin, end)) config = desisurvey.config.Configuration() # Find weather time steps that cover this night. weather_mjd = weather._table['mjd'].data ilo = np.searchsorted(weather_mjd, begin, side='left') - 1 ihi = np.searchsorted(weather_mjd, end, side='right') + 2 assert weather_mjd[ilo] < begin and weather_mjd[ihi] > end weather_mjd = weather_mjd[ilo:ihi] seeing = weather._table['seeing'].data[ilo:ihi] transp = weather._table['transparency'].data[ilo:ihi] # Fix this in the weather generator instead? transp = np.maximum(0.1, transp) dome = weather._table['open'].data[ilo:ihi] if not np.any(dome): log.debug('Dome closed all night.') return scheduler.init_night(night, use_twilight=use_twilight) ETC = desisurvey.etc.ExposureTimeCalculator(save_history=plot) nexp_last = explist.nexp # Build linear interpolators for observing conditions. # This implementation is faster than scipy.interpolate.interp1d() # when mjd values are gradually increasing. weather_idx = 0 dmjd_weather = weather_mjd[1] - weather_mjd[0] # moon illumination for the night moon_ill = night_ephem['moon_illum_frac'] # for moon ephem calculation moon_DECRA = desisurvey.ephem.get_object_interpolator(night_ephem, 'moon', altaz=False) moon_ALTAZ = desisurvey.ephem.get_object_interpolator(night_ephem, 'moon', altaz=True) # for sun ephem calculation sun_DECRA = desisurvey.ephem.get_object_interpolator(night_ephem, 'sun', altaz=False) sun_ALTAZ = desisurvey.ephem.get_object_interpolator(night_ephem, 'sun', altaz=True) skylevel_cache, skylevel_cache_time = None, None def get_weather(mjd, ra=None, dec=None): nonlocal weather_idx, night_changes, night_programs, config, skylevel_cache, skylevel_cache_time while mjd >= weather_mjd[weather_idx + 1]: weather_idx += 1 s = (mjd - weather_mjd[weather_idx]) / dmjd_weather if not use_simplesky: # use on-the-fly sky level calculations if ra is None: sky = desisurvey.etc.sky_level(mjd, ra, dec, moon_ill=moon_ill, moon_DECRA=moon_DECRA, moon_ALTAZ=moon_ALTAZ, sun_DECRA=sun_DECRA, sun_ALTAZ=sun_ALTAZ) else: # update sky level every 10 mins if (skylevel_cache_time is None) or (mjd - skylevel_cache_time > 0.006944444445252884): sky = desisurvey.etc.sky_level(mjd, ra, dec, moon_ill=moon_ill, moon_DECRA=moon_DECRA, moon_ALTAZ=moon_ALTAZ, sun_DECRA=sun_DECRA, sun_ALTAZ=sun_ALTAZ) skylevel_cache_time = mjd skylevel_cache = sky else: sky = skylevel_cache else: # use simple sky level calculation based on moon_up factor cond_ind = np.interp(mjd, night_changes, np.arange(len(night_changes))) if (cond_ind < 0) or (cond_ind >= len(night_programs)): cond = 'BRIGHT' else: cond = night_programs[int(np.floor(cond_ind))] sky = getattr(config.conditions, cond).moon_up_factor() return (seeing[weather_idx] * (1 - s) + seeing[weather_idx + 1] * s, transp[weather_idx] * (1 - s) + transp[weather_idx + 1] * s, sky) # Define time intervals to use in units of days (move to config?) NO_TILE_AVAIL_DELAY = 30. / 86400. # Step through the night. dome_is_open = False mjd_now = weather_mjd[0] if mjd_now < begin: mjd_now = begin completed_last = scheduler.plan.obsend_by_program() current_ra = None current_dec = None while mjd_now < end: if not dome_is_open: current_ra = current_dec = None # Advance to the next dome opening, if any. idx_now = np.searchsorted(weather_mjd, mjd_now, side='left') if not np.any(dome[idx_now:]): # Dome is closed for the rest of the night. mjd_now = end break idx_open = idx_now + np.argmax(dome[idx_now:]) assert dome[idx_open] == True and (idx_open == 0 or dome[idx_open - 1] == False or mjd_now == begin) mjd_now = weather_mjd[idx_open] if mjd_now >= end: # The next dome opening is after the end of the night. # This can happen if we are not using twilight. break # Find the next closing. if np.all(dome[idx_open:]): next_dome_closing = end else: idx_close = idx_open + np.argmin(dome[idx_open:]) assert dome[idx_close] == False and dome[idx_close - 1] == True next_dome_closing = min(end, weather_mjd[idx_close]) dome_is_open = True weather_idx = idx_open # == NEXT TILE =========================================================== # Dome is open from mjd_now to next_dome_closing. mjd_last = mjd_now tdead = 0. # Get the current observing conditions. seeing_tile, transp_tile, sky_tile = get_weather(mjd_now, ra=None, dec=None) # Get the next tile to observe from the scheduler. tileid, passnum, snr2frac_start, exposure_factor, airmass, sched_program, mjd_program_end = \ scheduler.next_tile( mjd_now, ETC, seeing_tile, transp_tile, sky_tile, current_ra=current_ra, current_dec=current_dec) if tileid is None: # Deadtime while we delay and try again. mjd_now += NO_TILE_AVAIL_DELAY if mjd_now >= next_dome_closing: # Dome closed during deadtime. mjd_now = next_dome_closing dome_is_open = False tdead += mjd_now - mjd_last current_ra = current_dec = None else: idx = scheduler.tiles.index(tileid) tileprogram = scheduler.tiles.tileprogram[idx] programnum = scheduler.tiles.program_index[tileprogram] current_ra = scheduler.tiles.tileRA[idx] current_dec = scheduler.tiles.tileDEC[idx] # Setup for a new field. mjd_now += ETC.NEW_FIELD_SETUP # slew time goes here if mjd_now >= next_dome_closing: # Setup interrupted by dome closing. mjd_now = next_dome_closing dome_is_open = False # Record an aborted setup. nightstats['nsetup_abort'][programnum] += 1 else: # Record a completed setup. nightstats['nsetup'][programnum] += 1 # Charge this as setup time whether or not it was aborted. nightstats['tsetup'][programnum] += mjd_now - mjd_last if dome_is_open: # Lookup the program of the next tile, which might be # different from the scheduled program in ``sched_program``. # Loop over repeated exposures of the same tile. continue_this_tile = True while continue_this_tile: # -- NEXT EXPOSURE --------------------------------------------------- # Use the ETC to control the shutter. mjd_open_shutter = mjd_now ETC.start(mjd_now, tileid, tileprogram, snr2frac_start, exposure_factor, seeing_tile, transp_tile, sky_tile) integrating = True while integrating: mjd_now += update_interval_days if mjd_now >= next_dome_closing: # Current exposure is interrupted by dome closing. mjd_now = next_dome_closing dome_is_open = False integrating = False continue_this_tile = False elif mjd_now >= mjd_program_end: # Current exposure is interrupted by a program change. mjd_now = mjd_program_end integrating = False continue_this_tile = False # Get the current observing conditions. seeing_now, transp_now, sky_now = get_weather( mjd_now, ra=scheduler.tiles.tileRA[idx], dec=scheduler.tiles.tileDEC[idx]) # Update the SNR. if not ETC.update(mjd_now, seeing_now, transp_now, sky_now): # Current exposure reached its target SNR according to the ETC. integrating = False # stop() will return False if this is a cosmic split and # more integration is still required. if ETC.stop(mjd_now): continue_this_tile = False # Record this exposure assert np.allclose(ETC.exptime, mjd_now - mjd_open_shutter) nightstats['tscience'][programnum] += ETC.exptime nightstats['nexp'][programnum] += 1 explist.add(mjd_now - ETC.exptime, 86400 * ETC.exptime, tileid, ETC.snr2frac, ETC.snr2frac - snr2frac_start, airmass, seeing_now, transp_now, sky_now) scheduler.update_snr(tileid, ETC.snr2frac) # All done if we have observed all tiles. if scheduler.plan.survey_completed(): break if continue_this_tile: # Prepare for the next exposure of the same tile. snr2frac_start = ETC.snr2frac mjd_split_start = mjd_now mjd_now += ETC.SAME_FIELD_SETUP if mjd_now >= next_dome_closing: # Setup for next exposure of same tile interrupted by dome closing. mjd_now = next_dome_closing dome_is_open = False continue_this_tile = False # Record an aborted split. nightstats['nsplit_abort'][programnum] += 1 else: # Record a completed split. nightstats['nsplit'][programnum] += 1 # Charge this as split time, whether or not is was aborted. nightstats['tsplit'][ programnum] += mjd_now - mjd_split_start # -------------------------------------------------------------------- # Update statistics for the scheduled program (which might be different from # the program of the tile we just observed). if scheduler.tiles.nogray: sched_program = ('DARK' if sched_program == 'GRAY' else sched_program) pidx = scheduler.tiles.program_index[sched_program] nightstats['tdead'][pidx] += tdead nightstats['topen'][pidx] += mjd_now - mjd_last # ======================================================================== # Save the number of tiles completed per program in the nightly statistics. nightstats['completed'][:] = (scheduler.plan.obsend_by_program() - completed_last) if plot: import matplotlib.pyplot as plt fig, axes = plt.subplots(2, 1, figsize=(15, 10), sharex=True) ax = axes[0] ax.plot(weather_mjd, seeing, 'r-', label='Seeing') ax.plot([], [], 'b-', label='Transparency') ax.legend(ncol=2, loc='lower center') ax.set_ylabel('Seeing FWHM [arcsec]') rhs = ax.twinx() rhs.plot(weather_mjd, transp, 'b-') rhs.set_ylabel('Transparency') ax = axes[1] changes = np.where(np.abs(np.diff(dome)) == 1)[0] for idx in changes: ax.axvline(weather_mjd[idx + 1], ls='-', c='r') mjd_history = np.array(ETC.history['mjd']) snr2frac_history = np.array(ETC.history['snr2frac']) for expinfo in explist._exposures[nexp_last:explist.nexp]: program = scheduler.tiles.tileprogram[scheduler.tiles.index( expinfo['TILEID'])] color = desisurvey.plots.program_color[program] t1 = expinfo['MJD'] t2 = t1 + expinfo['EXPTIME'] / 86400 sel = (mjd_history >= t1) & (mjd_history <= t2) ax.fill_between(mjd_history[sel], snr2frac_history[sel], color=color, alpha=0.5, lw=0) for t in scheduler.night_changes: ax.axvline(t, c='b', ls=':') ax.set_xlim(weather_mjd[0], weather_mjd[-1]) ax.set_xlabel('MJD During {}'.format(label)) ax.set_ylim(0, 1) ax.set_ylabel('Integrated SNR2 Fraction')
def write_exposures_tables(indir, outdir, exposures, nights=None): """ Writes exposures table for each night available Args: outfile: output HTML files to outdir/YEARMMDD/exposures.html exposures: table with columns NIGHT, EXPID Options: nights: optional list of nights to process """ log = desiutil.log.get_logger() env = jinja2.Environment( loader=jinja2.PackageLoader('nightwatch.webpages', 'templates'), autoescape=select_autoescape(disabled_extensions=('txt', ), default_for_string=True, default=True)) template = env.get_template('exposures.html') if nights is None: nights = np.unique(exposures['NIGHT']) for night in nights: log.debug('{} Generating exposures table for {}'.format( time.asctime(), night)) ii = (exposures['NIGHT'] == int(night)) explist = list() night_exps = exposures[ii] night_exps.sort('EXPID') for row in night_exps: expid = row['EXPID'] #- adds failed expid to table if row['FAIL'] == 1: link = '{expid:08d}/qa-summary-{expid:08}-logfiles_table.html'.format( night=night, expid=expid) expinfo = dict(night=night, expid=expid, link=link, fail=1) explist.append(expinfo) continue qafile = io.findfile('qa', night, expid, basedir=indir) qadata = io.read_qa(qafile) status = get_status(qadata, night) if 'OBSTYPE' in qadata['HEADER']: obstype = qadata['HEADER']['OBSTYPE'].rstrip().upper() else: log.warning('Use FLAVOR instead of missing OBSTYPE') obstype = qadata['HEADER']['FLAVOR'].rstrip().upper() exptime = qadata['HEADER']['EXPTIME'] from ..plots.core import parse_numlist peramp = status.get('PER_AMP') if peramp: list_specs = list(set(peramp['SPECTRO'])) spectros = parse_numlist(list_specs) else: spectros = '???' link = '{expid:08d}/qa-summary-{expid:08d}.html'.format( night=night, expid=expid) expinfo = dict(night=night, expid=expid, obstype=obstype, link=link, exptime=exptime, spectros=spectros, fail=0) hdr = qadata['HEADER'] expinfo['PROGRAM'] = hdr['PROGRAM'] if 'PROGRAM' in hdr else '?' #- TILEID with link to fiberassign QA if 'TILEID' in hdr: tileid = hdr['TILEID'] expinfo['TILEID'] = tileid tilegroup = '{:03d}'.format(tileid // 1000) expinfo[ 'TILEID_LINK'] = f'https://data.desi.lbl.gov/desi/target/fiberassign/tiles/trunk/{tilegroup}/fiberassign-{tileid:06d}.png' else: expinfo['TILEID'] = -1 expinfo['TILEID_LINK'] = 'na' #- KPNO local time (MST=Mountain Standard Time) if 'MJD-OBS' in hdr: expinfo['MST'] = Time(hdr['MJD-OBS'] - 7 / 24, format='mjd').strftime('%H:%M') else: expinfo['MST'] = '?' #- Adds qproc to the expid status #- TODO: add some catches to this for robustness, e.g. the '-' if QPROC is missing if len(row['QPROC']) == 0 and row['QPROC_EXIT'] == 0: expinfo['QPROC'] = 'ok' else: expinfo['QPROC'] = 'error' expinfo[ 'QPROC_link'] = '{expid:08d}/qa-summary-{expid:08d}-logfiles_table.html'.format( expid=expid) #- TODO: have actual thresholds for i, qatype in enumerate([ 'PER_AMP', 'PER_CAMERA', 'PER_FIBER', 'PER_CAMFIBER', 'PER_SPECTRO', 'PER_EXP' ]): if qatype not in status: expinfo[qatype] = '-' expinfo[qatype + "_link"] = "na" else: qastatus = Status(np.max(status[qatype]['QASTATUS'])) short_name = qatype.split("_")[1].lower() expinfo[qatype] = qastatus.name if qatype != 'QPROC': expinfo[ qatype + "_link"] = '{expid:08d}/qa-{name}-{expid:08d}.html'.format( expid=expid, name=short_name) explist.append(expinfo) html = template.render(night=night, exposures=explist, autoreload=True, staticdir='../static') outfile = os.path.join(outdir, str(night), 'exposures.html') tmpfile = outfile + '.tmp' + str(os.getpid()) with open(tmpfile, 'w') as fx: fx.write(html) os.rename(tmpfile, outfile) #- Update expid and night links only once at the end _write_expid_links(outdir, exposures, nights) _write_night_links(outdir)