def spline_fit(output_wave,input_wave,input_flux,required_resolution,input_ivar=None,order=3,max_resolution=None): """Performs spline fit of input_flux vs. input_wave and resamples at output_wave Args: output_wave : 1D array of output wavelength samples input_wave : 1D array of input wavelengths input_flux : 1D array of input flux density required_resolution (float) : resolution for spline knot placement (same unit as wavelength) Options: input_ivar : 1D array of weights for input_flux order (int) : spline order max_resolution (float) : if not None and first fit fails, try once this resolution Returns: output_flux : 1D array of flux sampled at output_wave """ if input_ivar is not None : selection=np.where(input_ivar>0)[0] if selection.size < 2 : log=get_logger() log.error("cannot do spline fit because only {0:d} values with ivar>0".format(selection.size)) raise ValueError w1=input_wave[selection[0]] w2=input_wave[selection[-1]] else : w1=input_wave[0] w2=input_wave[-1] res=required_resolution n=int((w2-w1)/res) res=(w2-w1)/(n+1) knots=w1+res*(0.5+np.arange(n)) ## check that nodes are close to pixels dknots = abs(knots[:,None]-input_wave) mins = np.amin(dknots,axis=1) w=mins<res knots = knots[w] try : toto=scipy.interpolate.splrep(input_wave,input_flux,w=input_ivar,k=order,task=-1,t=knots) output_flux = scipy.interpolate.splev(output_wave,toto) except ValueError as err : log=get_logger() if max_resolution is not None and required_resolution < max_resolution : log.warning("spline fit failed with resolution={}, retrying with {}".format(required_resolution,max_resolution)) return spline_fit(output_wave,input_wave,input_flux,max_resolution,input_ivar=input_ivar,order=3,max_resolution=None) else : log.error("spline fit failed") raise ValueError return output_flux
def load_qa_frame(filename, frame=None, flavor=None): """ Load an existing QA_Frame or generate one, as needed Args: filename: str frame: Frame object, optional flavor: str, optional Type of QA_Frame Returns: qa_frame: QA_Frame object """ from lvmspec.qa.qa_frame import QA_Frame log=get_logger() if os.path.isfile(filename): # Read from file, if it exists qaframe = read_qa_frame(filename) log.info("Loaded QA file {:s}".format(filename)) # Check against frame, if provided if frame is not None: for key in ['camera','expid','night','flavor']: assert getattr(qaframe, key) == frame.meta[key.upper()] else: # Init if frame is None: log.error("QA file {:s} does not exist. Expecting frame input".format(filename)) qaframe = QA_Frame(frame) # Set flavor? if flavor is not None: qaframe.flavor = flavor # Return return qaframe
def write_qa_prod(outroot, qaprod, indent=True): """Write QA for a given production Args: outroot : str filename without format extension qa_prod : QA_Prod object Returns: outfile: str output filename """ from lvmutil.io import combine_dicts log=get_logger() outfile = outroot+'.json' outfile = makepath(outfile, 'qa') # Loop on exposures odict = {} for qaexp in qaprod.qa_exps: # Get the exposure dict idict = write_qa_exposure('foo', qaexp, ret_dict=True) odict = combine_dicts(odict, idict) ydict = yamlify(odict) # This works well for JSON too # Simple json with open(outfile, 'wt') as fh: json.dump(ydict, fh, indent=indent) log.info('Wrote QA_Prod file: {:s}'.format(outfile)) return outfile
def main(args): log = get_logger() #- Generate obsconditions with args.program, then override as needed args.program = args.program.upper() if args.program in ['ARC', 'FLAT']: obsconditions = None else: obsconditions = lvmsim.simexp.reference_conditions[args.program] if args.airmass is not None: obsconditions['AIRMASS'] = args.airmass if args.seeing is not None: obsconditions['SEEING'] = args.seeing if args.exptime is not None: obsconditions['EXPTIME'] = args.exptime if args.moonfrac is not None: obsconditions['MOONFRAC'] = args.moonfrac if args.moonalt is not None: obsconditions['MOONALT'] = args.moonalt if args.moonsep is not None: obsconditions['MOONSEP'] = args.moonsep sim, fibermap, meta, obs = lvmsim.obs.new_exposure( args.program, nspec=args.nspec, night=args.night, expid=args.expid, tileid=args.tileid, nproc=args.nproc, seed=args.seed, obsconditions=obsconditions, outdir=args.outdir)
def sim(night, nspec=25, clobber=False): """ Simulate data as part of the integration test. Args: night (str): YEARMMDD nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() output_dir = os.path.join('$LVM_SPECTRO_REDUX', 'calib2d') # Create input fibermaps, spectra, and quickgen data for expid, program in zip([0, 1, 2], ['flat', 'arc', 'dark']): cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format( expid=expid, program=program, nspec=nspec, night=night) simspec = lvmsim.io.findfile('simspec', night, expid) fibermap = '{}/fibermap-{:08d}.fits'.format(os.path.dirname(simspec), expid) if runcmd(cmd, clobber=clobber) != 0: raise RuntimeError('newexp failed for {} exposure {}'.format( program, expid)) cmd = "quickgen --simspec {} --fibermap {}".format(simspec, fibermap) if runcmd(cmd, clobber=clobber) != 0: raise RuntimeError('quickgen failed for {} exposure {}'.format( program, expid)) return
def default_options(extra={}): """ Get the default options for all workers. Args: extra (dict): optional extra options to add to the default options for each worker class. Returns (dict): the default options dictionary, suitable for writing to the default options.yaml file. """ log = get_logger() allopts = {} for step in step_types: defwork = default_workers[step] allopts["{}_worker".format(step)] = defwork if defwork in extra: allopts["{}_worker_opts".format(step)] = extra[defwork] else: allopts["{}_worker_opts".format(step)] = {} worker = get_worker(step, None, {}) allopts[step] = worker.default_options() return allopts
def main(): """Entry point for :command:`desi_dts_delivery`. Returns ------- :class:`int` An integer suitable for passing to :func:`sys.exit`. """ from os import environ from os.path import dirname, join from subprocess import Popen from lvmutil.log import get_logger log = get_logger() options = parse_delivery() remote_command = ['desi_{0.nightStatus}_night {0.night}'.format(options)] if options.prefix is not None: remote_command = options.prefix + remote_command remote_command = ('(' + '; '.join([c + ' &> /dev/null' for c in remote_command]) + ' &)') command = ['ssh', '-n', '-q', options.nersc_host, remote_command] log.info("Received file {0.filename} with exposure number {0.exposure:d}.". format(options)) dst = join(environ['LVM_SPECTRO_DATA'], options.night) log.info("Using {0} as raw data directory.".format(dst)) move_file(options.filename, dst) exposure_arrived = check_exposure(dst, options.exposure) if options.nightStatus in ('start', 'end') or exposure_arrived: log.info("Calling: {0}.".format(' '.join(command))) proc = Popen(command) return 0
def make_frameqa(self, make_plots=False, clobber=True): """ Work through the Production and make QA for all frames Parameters: make_plots: bool, optional Remake the plots too? clobber: bool, optional Returns: """ # imports from lvmspec.io import meta from lvmspec.io.qa import load_qa_frame, write_qa_frame from lvmspec.io.fiberflat import read_fiberflat from lvmspec.io.sky import read_sky from lvmspec.io.fluxcalibration import read_flux_calibration from lvmspec.qa import qa_plots from lvmspec.qa.qa_frame import qaframe_from_frame from lvmspec.io.fluxcalibration import read_stdstar_models log = get_logger() # Loop on nights path_nights = glob.glob(self.specprod_dir+'/exposures/*') nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] for night in nights: for exposure in get_exposures(night, specprod_dir = self.specprod_dir): # Object only?? frames_dict = get_files(filetype = str('frame'), night = night, expid = exposure, specprod_dir = self.specprod_dir) for camera,frame_fil in frames_dict.items(): # Load frame qaframe_from_frame(frame_fil, make_plots=make_plots) '''
def run(self, grph, task, opts, comm=None): """ Run the PSF combining. This is a serial call to libspecex to combine the PSF files. Args: grph (dict): pruned graph with this task and dependencies. task (str): the name of this task. opts (dict): options to use for this task. comm (mpi4py.MPI.Comm): optional MPI communicator. """ if comm is not None: if comm.size > 1: raise RuntimeError( "PSFCombine worker should only be called with one process") log = get_logger() node = grph[task] outfile = graph_path(task) infiles = [] for input in node["in"]: infiles.append(graph_path(input)) specex.mean_psf(infiles, outfile) return
def init_fluxcalib(self, re_init=False): """ Initialize parameters for FLUXCALIB QA Args: re_init: bool, (optional) Re-initialize parameter dict Returns: """ log = get_logger() assert self.flavor == 'science' # Standard FLUXCALIB input parameters flux_dict = dict( ZP_WAVE=0., # Wavelength for ZP evaluation (camera dependent) MAX_ZP_OFF=0.2, # Max offset in ZP for individual star ) if self.camera[0] == 'b': flux_dict['ZP_WAVE'] = 4800. # Ang elif self.camera[0] == 'r': flux_dict['ZP_WAVE'] = 6500. # Ang elif self.camera[0] == 'z': flux_dict['ZP_WAVE'] = 8250. # Ang else: log.error("Not ready for camera {}!".format(self.camera)) # Init self.init_qatype('FLUXCALIB', flux_dict, re_init=re_init)
def search_for_framefile(frame_file): """ Search for an input frame_file in the lvmspec redux hierarchy Args: frame_file: str Returns: mfile: str, full path to frame_file if found else raise error """ log = get_logger() # Parse frame file path, ifile = os.path.split(frame_file) splits = ifile.split('-') root = splits[0] camera = splits[1] fexposure = int(splits[2].split('.')[0]) # Loop on nights nights = get_nights() for night in nights: for exposure in get_exposures(night): if exposure == fexposure: mfile = findfile(root, camera=camera, night=night, expid=exposure) if os.path.isfile(mfile): return mfile else: log.error("Expected file {:s} not found..".format(mfile))
def move_file(filename, dst): """Move delivered file from the DTS spool to the final raw data area. This function will ensure that the destination directory exists. Parameters ---------- filename : :class:`str` The name, including full path, of the file to move. dst : :class:`str` The destination *directory*. Returns ------- :class:`str` The value returned by :func:`shutil.move`. """ from os import mkdir from os.path import exists, isdir from shutil import move from lvmutil.log import get_logger log = get_logger() if not exists(dst): log.info("mkdir('{0}', 0o2770)".format(dst)) mkdir(dst, 0o2770) log.info("move('{0}', '{1}')".format(filename, dst)) return move(filename, dst)
def update_logger(options): """Reconfigure the default logging object. Parameters ---------- options : :class:`argparse.Namespace` The parsed command-line options. Returns ------- :class:`logging.Logger` The updated object. """ from os import environ from os.path import join from logging import FileHandler from lvmutil.log import get_logger log = get_logger() fmt = None try: filename = join(environ['LVM_SPECTRO_DATA'], options.night, 'desi_{0.stage}_night_{0.night}.log'.format(options)) except (AttributeError, KeyError): # This can happen during tests. return log if log.hasHandlers(): for h in log.handlers: if fmt is None: fmt = h.formatter log.removeHandler(h) h = FileHandler(filename) h.setFormatter(fmt) log.addHandler(h) return log
def write_targets(filename, data, indir=None, qso_selection=None, sandboxcuts=False, nside=None): """Write a target catalogue. Parameters ---------- filename : output target selection file data : numpy structured array of targets to save nside: :class:`int` If passed, add a column to the targets array popluated with HEALPix pixels at resolution nside """ # FIXME: assert data and tsbits schema #ADM set up the default logger from lvmutil.log import get_logger log = get_logger() #ADM use RELEASE to determine the release string for the input targets if len(data) == 0: #ADM if there are no targets, then we don't know the Data Release drstring = 'unknowndr' else: drint = np.max(data['RELEASE'] // 1000) drstring = 'dr' + str(drint) #- Create header to include versions, etc. hdr = fitsio.FITSHDR() depend.setdep(hdr, 'lvmtarget', lvmtarget_version) depend.setdep(hdr, 'lvmtarget-git', gitversion()) depend.setdep(hdr, 'sandboxcuts', sandboxcuts) depend.setdep(hdr, 'photcat', drstring) if indir is not None: depend.setdep(hdr, 'tractor-files', indir) if qso_selection is None: log.warning('qso_selection method not specified for output file') depend.setdep(hdr, 'qso-selection', 'unknown') else: depend.setdep(hdr, 'qso-selection', qso_selection) #ADM add HEALPix column, if requested by input if nside is not None: theta, phi = np.radians(90 - data["DEC"]), np.radians(data["RA"]) hppix = hp.ang2pix(nside, theta, phi, nest=True) data = rfn.append_fields(data, 'HPXPIXEL', hppix, usemask=False) hdr['HPXNSIDE'] = nside hdr['HPXNEST'] = True #ADM add PHOTSYS column, mapped from RELEASE photsys = release_to_photsys(data["RELEASE"]) data = rfn.append_fields(data, 'PHOTSYS', photsys, usemask=False) fitsio.write(filename, data, extname='TARGETS', header=hdr, clobber=True)
def encode_mtl_targetid(targets): """ Sets targetid used in MTL, which encode both the target class and arbitrary tracibility data propagated from individual input sources. Allows rows in final MTL (and hence fibre map) to be mapped to input sources. """ #ADM set up the default logger from lvmutil.log import get_logger log = get_logger() encoded_targetid = targets['TARGETID'].copy() # Validate incoming target ids if not np.all(encoded_targetid <= ENCODE_MTL_USER_MASK): log.error('Invalid range of user-specfied targetid: cannot exceed {}' .format(ENCODE_MTL_USER_MASK)) desi_target = targets['LVM_TARGET'] != 0 bgs_target = targets['BGS_TARGET'] != 0 mws_target = targets['MWS_TARGET'] != 0 # Assumes surveys are mutually exclusive. assert(np.max(np.sum([desi_target,bgs_target,mws_target],axis=0)) == 1) # Set the survey bits #encoded_targetid[desi_target] += TARGETID_SURVEY_INDEX['desi'] << SOURCE_END #encoded_targetid[bgs_target ] += TARGETID_SURVEY_INDEX['bgs'] << SOURCE_END #encoded_targetid[mws_target] += TARGETID_SURVEY_INDEX['mws'] << SOURCE_END encoded_targetid[desi_target] += encode_survey_source(TARGETID_SURVEY_INDEX['desi'],0,0) encoded_targetid[bgs_target ] += encode_survey_source(TARGETID_SURVEY_INDEX['bgs'],0,0) encoded_targetid[mws_target] += encode_survey_source(TARGETID_SURVEY_INDEX['mws'],0,0) # Set the source bits. Will be different for each survey. desi_sources = ['ELG','LRG','QSO'] bgs_sources = ['BGS_FAINT','BGS_BRIGHT'] mws_sources = ['MWS_MAIN','MWS_WD','MWS_NEARBY'] for name in desi_sources: ii = (targets['LVM_TARGET'] & desi_mask[name]) != 0 assert(desi_mask[name] <= SOURCE_MAX) encoded_targetid[ii] += encode_survey_source(0,desi_mask[name],0) for name in bgs_sources: ii = (targets['BGS_TARGET'] & bgs_mask[name]) != 0 assert(bgs_mask[name] <= SOURCE_MAX) encoded_targetid[ii] += encode_survey_source(0,bgs_mask[name],0) for name in mws_sources: ii = (targets['MWS_TARGET'] & mws_mask[name]) != 0 assert(mws_mask[name] <= SOURCE_MAX) encoded_targetid[ii] += encode_survey_source(0,mws_mask[name],0) # FIXME (APC): expensive... assert(len(np.unique(encoded_targetid)) == len(encoded_targetid)) return encoded_targetid
def run(self, grph, task, opts, comm=None): """ Run Redrock on a spectral group. Args: grph (dict): pruned graph with this task and dependencies. task (str): the name of this task. opts (dict): options to use for this task. comm (mpi4py.MPI.Comm): optional MPI communicator. """ nproc = 1 rank = 0 if comm is not None: nproc = comm.size rank = comm.rank log = get_logger() node = grph[task] spectra = "spectra-{}-{}".format(node["nside"], node["pixel"]) specfile = graph_path(spectra) outfile = graph_path(task) outdir = os.path.dirname(outfile) details = os.path.join(outdir, "rrdetails_{}.h5".format(task)) options = {} options["output"] = details options["zbest"] = outfile options.update(opts) optarray = option_list(options) optarray.append(specfile) # write out the equivalent commandline if rank == 0: com = ["RUN", "rrdesi_mpi"] com.extend(optarray) log.info(" ".join(com)) failcount = 0 try: rrdesi(options=optarray, comm=comm) except: failcount += 1 sys.stdout.flush() if comm is not None: failcount = comm.allreduce(failcount) if failcount > 0: # all processes throw raise RuntimeError("some redshifts failed for task " "{}".format(task)) return
def run(self, grph, task, opts, comm=None): """ Compute the sky model. Args: grph (dict): pruned graph with this task and dependencies. task (str): the name of this task. opts (dict): options to use for this task. comm (mpi4py.MPI.Comm): optional MPI communicator. """ if comm is not None: if comm.size > 1: raise RuntimeError( "Sky worker should only be called with one process") log = get_logger() node = grph[task] frm = [] flat = [] for input in node["in"]: inode = grph[input] if inode["type"] == "frame": frm.append(input) elif inode["type"] == "fiberflat": flat.append(input) if len(frm) != 1: raise RuntimeError("sky needs exactly one frame file") if len(flat) != 1: raise RuntimeError("sky needs exactly one fiberflat file") framefile = graph_path(frm[0]) flatfile = graph_path(flat[0]) outfile = graph_path(task) #qafile, qafig = qa_path(outfile) options = {} options["infile"] = framefile options["fiberflat"] = flatfile #options["qafile"] = qafile #options["qafig"] = qafig options["outfile"] = outfile options.update(opts) optarray = option_list(options) # at debug level, write out the equivalent commandline com = ["RUN", "desi_compute_sky"] com.extend(optarray) log.info(" ".join(com)) args = skypkg.parse(optarray) skypkg.main(args) return
def compatible(head1, head2): log = get_logger() for k in [ "PSFTYPE", "NPIX_X", "NPIX_Y", "HSIZEX", "HSIZEY", "FIBERMIN", "FIBERMAX", "NPARAMS", "LEGDEG", "GHDEGX", "GHDEGY" ]: if (head1[k] != head2[k]): log.warning("different {} : {}, {}".format(k, head1[k], head2[k])) return False return True
def get_source_types(fibermap): ''' Return a list of specsim source types based upon fibermap['LVM_TARGET'] Args: fibermap: fibermap Table including LVM_TARGET column Returns array of source_types 'sky', 'elg', 'lrg', 'qso', 'star' Unassigned fibers fibermap['TARGETID'] == -1 will be treated as 'sky' If fibermap.meta['FLAVOR'] = 'arc' or 'flat', returned source types will match that flavor, though specsim doesn't use those as source_types TODO: specsim/lvmmodel doesn't have a fiber input loss model for BGS yet, so BGS targets get source_type = 'lrg' (!) ''' from lvmutil.log import get_logger log = get_logger('DEBUG') if 'LVM_TARGET' not in fibermap.dtype.names: log.warning( "LVM_TARGET not in fibermap table; using source_type='star' for everything" ) return np.array([ 'star', ] * len(fibermap)) source_type = np.zeros(len(fibermap), dtype='U4') assert np.all(source_type == '') tm = lvmtarget.desi_mask if 'TARGETID' in fibermap.dtype.names: unassigned = fibermap['TARGETID'] == -1 source_type[unassigned] = 'sky' source_type[(fibermap['OBJTYPE'] == 'FLAT')] = 'FLAT' source_type[(fibermap['OBJTYPE'] == 'ARC')] = 'ARC' source_type[(fibermap['LVM_TARGET'] & tm.SKY) != 0] = 'sky' source_type[(fibermap['LVM_TARGET'] & tm.ELG) != 0] = 'elg' source_type[(fibermap['LVM_TARGET'] & tm.LRG) != 0] = 'lrg' source_type[(fibermap['LVM_TARGET'] & tm.QSO) != 0] = 'qso' source_type[(fibermap['LVM_TARGET'] & tm.BGS_ANY) != 0] = 'bgs' starmask = tm.STD_FSTAR | tm.STD_WD | tm.STD_BRIGHT | tm.MWS_ANY source_type[(fibermap['LVM_TARGET'] & starmask) != 0] = 'star' assert not np.any(source_type == '') for name in sorted(np.unique(source_type)): n = np.count_nonzero(source_type == name) log.debug('{} {} targets'.format(name, n)) return source_type
def list_tractorfiles(root): """Return a list of tractor files found under `root` directory. """ from lvmutil.log import get_logger log = get_logger(timestamp=True) #ADM check for duplicate files in case the listing was run #ADM at too low a level in the directory structure check = [os.path.basename(x) for x in iter_tractorfiles(root)] if len(check) != len(set(check)): log.error("Duplicate Tractor files in root directory!") return [x for x in iter_tractorfiles(root)]
def run(self, grph, task, opts, comm=None): nproc = 1 rank = 0 if comm is not None: nproc = comm.size rank = comm.rank log = get_logger() node = grph[task] p = graph_path(task) log.info("NOOP Worker creating {}".format(p)) with open(p, "w") as f: f.write("NOOP\n") return
def main(args) : from lvmspec.io import meta from lvmspec.qa.qa_frame import qaframe_from_frame log=get_logger() log.info("starting") if args.reduxdir is None: specprod_dir = meta.specprod_root() else: specprod_dir = args.reduxdir # Generate qaframe (and figures?) _ = qaframe_from_frame(args.frame_file, specprod_dir=specprod_dir, make_plots=args.make_plots, output_dir=args.output_dir)
def default_options(self): log = get_logger() opts = {} if self.starmodels is not None: opts["starmodels"] = self.starmodels else: if "LVM_ROOT" in os.environ: opts["starmodels"] = os.environ[ "LVM_ROOT"] + "/spectro/templates/star_templates/v2.1/star_templates_v2.1.fits" else: log.warning( "$LVM_ROOT not set; using NERSC default /project/projectdirs/desi" ) opts[ "starmodels"] = "/project/projectdirs/desi/spectro/templates/star_templates/v2.1/star_templates_v2.1.fits" return opts
def run(self, grph, task, opts, comm=None): """ Compute the fiberflat. Args: grph (dict): pruned graph with this task and dependencies. task (str): the name of this task. opts (dict): options to use for this task. comm (mpi4py.MPI.Comm): optional MPI communicator. """ if comm is not None: if comm.size > 1: raise RuntimeError( "Fiberflat worker should only be called with one process") log = get_logger() node = grph[task] night, obj = graph_night_split(task) (temp, band, spec, expid) = graph_name_split(obj) cam = "{}{}".format(band, spec) if len(node["in"]) != 1: raise RuntimeError("fiberflat should have only one input frame") framefile = graph_path(node["in"][0]) outfile = graph_path(task) #qafile, qafig = qa_path(outfile) options = {} options["infile"] = framefile #options["qafile"] = qafile #options["qafig"] = qafig options["outfile"] = outfile options.update(opts) optarray = option_list(options) # at debug level, write out the equivalent commandline com = ["RUN", "desi_compute_fiberflat"] com.extend(optarray) log.info(" ".join(com)) args = fiberflat.parse(optarray) fiberflat.main(args) return
def load_qa_brick(filename): """ Load an existing QA_Brick or generate one, as needed Args: filename: str Returns: qa_brick: QA_Brick object """ from lvmspec.qa.qa_brick import QA_Brick log=get_logger() if os.path.isfile(filename): # Read from file, if it exists qabrick = read_qa_brick(filename) log.info("Loaded QA file {:s}".format(filename)) else: # Init qabrick = QA_Brick() # Return return qabrick
def apply_fiberflat(frame, fiberflat): """Apply fiberflat to frame. Modifies frame.flux and frame.ivar Args: frame : `lvmspec.Frame` object fiberflat : `lvmspec.FiberFlat` object The frame is divided by the fiberflat, except where the fiberflat=0. frame.mask gets bit specmask.BADFIBERFLAT set where * fiberflat.fiberflat == 0 * fiberflat.ivar == 0 * fiberflat.mask != 0 """ log=get_logger() log.info("starting") # check same wavelength, die if not the case if not np.allclose(frame.wave, fiberflat.wave): message = "frame and fiberflat do not have the same wavelength arrays" log.critical(message) raise ValueError(message) """ F'=F/C Var(F') = Var(F)/C**2 + F**2*( d(1/C)/dC )**2*Var(C) = 1/(ivar(F)*C**2) + F**2*(1/C**2)**2*Var(C) = 1/(ivar(F)*C**2) + F**2*Var(C)/C**4 = 1/(ivar(F)*C**2) + F**2/(ivar(C)*C**4) """ #- shorthand ff = fiberflat sp = frame #- sp=spectra for this frame #- update sp.ivar first since it depends upon the original sp.flux sp.ivar=(sp.ivar>0)*(ff.ivar>0)*(ff.fiberflat>0)/( 1./((sp.ivar+(sp.ivar==0))*(ff.fiberflat**2+(ff.fiberflat==0))) + sp.flux**2/(ff.ivar*ff.fiberflat**4+(ff.ivar*ff.fiberflat==0)) ) #- Then update sp.flux, taking care not to divide by 0 ii = np.where(ff.fiberflat > 0) sp.flux[ii] = sp.flux[ii] / ff.fiberflat[ii] badff = (ff.fiberflat == 0.0) | (ff.ivar == 0) | (ff.mask != 0) sp.mask[badff] |= specmask.BADFIBERFLAT log.info("done")
def vet(self): """ Perform very basic checks on the frame Generally run before writing to disk (or when read) Args: index: Returns: diagnosis: int (bitwise flag) 0: Pass 2**0: Improper meta data 2**1: Improper data shapes """ # Shapes log = get_logger() bad_shape = False if (self.nspec, self.nwave) != self.flux.shape: log.error( 'Frame nspec {} nwave {} inconsistent with flux.shape {}'. format(self.nspec, self.nwave, self.flux.shape)) bad_shape = True # Meta data bad_meta = False if self.meta is None: log.error('Frame.meta missing') bad_meta = True else: from lvmspec.io.params import read_params # Check flavor if 'FLAVOR' not in self.meta.keys(): log.error('Frame.meta missing FLAVOR keyword') bad_meta = True else: desi_params = read_params() if self.meta['FLAVOR'] not in desi_params['frame_types']: log.error("Frame.meta['FLAVOR'] = '{}' not in {}".format( self.meta['FLAVOR'], desi_params['frame_types'])) bad_meta = True #if bad_meta: # import pdb; pdb.set_trace() # Generate the flag diagnosis = 0 + 2**0 * bad_shape + 2**1 * bad_meta return diagnosis
def check_env(): """ Check required environment variables; raise RuntimeException if missing """ log = logging.get_logger() #- template locations missing_env = False if 'LVM_BASIS_TEMPLATES' not in os.environ: log.warning( 'missing $LVM_BASIS_TEMPLATES needed for simulating spectra'. format(name)) missing_env = True if not os.path.isdir(os.getenv('LVM_BASIS_TEMPLATES')): log.warning('missing $LVM_BASIS_TEMPLATES directory') log.warning( 'e.g. see NERSC:/project/projectdirs/desi/spectro/templates/basis_templates/v2.2' ) missing_env = True for name in ('LVM_SPECTRO_SIM', 'LVM_SPECTRO_REDUX', 'PIXPROD', 'SPECPROD', 'LVMMODEL'): if name not in os.environ: log.warning("missing ${0}".format(name)) missing_env = True if missing_env: log.warning("Why are these needed?") log.warning(" Simulations written to $LVM_SPECTRO_SIM/$PIXPROD/") log.warning(" Raw data read from $LVM_SPECTRO_DATA/") log.warning( " Spectro pipeline output written to $LVM_SPECTRO_REDUX/$SPECPROD/" ) log.warning(" Templates are read from $LVM_BASIS_TEMPLATES") #- Wait until end to raise exception so that we report everything that #- is missing before actually failing if missing_env: log.critical("missing env vars; exiting without running pipeline") sys.exit(1) #- Override $LVM_SPECTRO_DATA to match $LVM_SPECTRO_SIM/$PIXPROD os.environ['LVM_SPECTRO_DATA'] = os.path.join(os.getenv('LVM_SPECTRO_SIM'), os.getenv('PIXPROD'))
def load_simulated_data(session, obs_pass=0): """Load simulated frame and brick data. Parameters ---------- session : :class:`sqlalchemy.orm.session.Session` Database connection. obs_pass : :class:`int`, optional If set, only simulate one pass. """ log = get_logger() tiles = get_all_tiles(session, obs_pass=obs_pass) status = 'succeeded' for t in tiles: for band in 'brz': for spectrograph in range(10): frame, bricks = t.simulate_frame(session, band, spectrograph) try: q = session.query(Night).filter_by(night=frame.night).one() except NoResultFound: session.add(Night(night=frame.night)) try: q = session.query(ExposureFlavor).filter_by( flavor=frame.flavor).one() except NoResultFound: session.add(ExposureFlavor(flavor=frame.flavor)) # try: # q = session.query(Status).filter_by(status=status).one() # except NoResultFound: # session.add(Status(status=status)) session.add(frame) session.add( FrameStatus(frame_id=frame.id, status=status, stamp=frame.dateobs)) for brick in bricks: session.add( BrickStatus(brick_id=brick.id, status=status, stamp=frame.dateobs)) frame.bricks = bricks session.commit() log.info("Completed insert of tileid = {0:d}.".format(t.id)) return
def q3c_index(table): """Create a q3c index on a table. Parameters ---------- table : :class:`str` Name of the table to index. """ from lvmutil.log import get_logger log = get_logger() q3c_sql = """CREATE INDEX ix_{table}_q3c_ang2ipix ON {schema}.{table} (q3c_ang2ipix(ra, dec)); CLUSTER {schema}.{table} USING ix_{table}_q3c_ang2ipix; ANALYZE {schema}.{table}; """.format(schema=schemaname, table=table) log.info("Creating q3c index on %s.%s.", schemaname, table) dbSession.execute(q3c_sql) log.info("Finished q3c index on %s.%s.", schemaname, table) dbSession.commit() return