def isStdStar(fibermap, bright=None): """ Determines if target(s) are standard stars Args: fibermap: table including DESI_TARGET or SV1_DESI_TARGET bit mask(s) Optional: bright: if True, only bright time standards; if False, only darktime, otherwise both Returns bool or array of bool TODO: move out of scripts/stdstars.py """ target_colnames, target_masks, survey = main_cmx_or_sv(fibermap) desi_target = fibermap[target_colnames[0]] # (SV1_)DESI_TARGET desi_mask = target_masks[0] # (sv1) desi_mask yes = (desi_target & desi_mask.STD_WD) != 0 if bright is None: yes |= (desi_target & desi_mask.mask('STD_WD|STD_FAINT|STD_BRIGHT')) != 0 elif bright: yes |= (desi_target & desi_mask.mask('STD_WD|STD_BRIGHT')) != 0 else: yes |= (desi_target & desi_mask.mask('STD_WD|STD_FAINT')) != 0 return yes
def default_target_masks(data): """Return the column name and default mask values for the data table. This identifies the type of target data and returns the defaults for the program type. Args: data (Table): A Table or recarray. Returns: (tuple): The survey, column name, science mask, standard mask, sky mask, suppsky mask, safe mask, and exclude mask for the data. """ col = None filecols, filemasks, filesurvey = main_cmx_or_sv(data) if filesurvey == "main": col = "DESI_TARGET" elif filesurvey == "cmx": col = filecols[0] elif filesurvey == "sv1": col = "SV1_DESI_TARGET" sciencemask, stdmask, skymask, suppskymask, safemask, excludemask = \ default_survey_target_masks(filesurvey) return (filesurvey, col, sciencemask, stdmask, skymask, suppskymask, safemask, excludemask)
def fibermeta2fibermap(fiberassign, meta): ''' Convert a fiberassign + targeting metadata table into a fibermap Table A future refactor will standardize the column names of fiber assignment, target catalogs, and fibermaps, but in the meantime this is needed. ''' #- Handle DESI_TARGET vs. SV1_DESI_TARGET etc. target_colnames, target_masks, survey = main_cmx_or_sv(fiberassign) targetcol = target_colnames[0] #- DESI_TARGET or SV1_DESI_TARGET desi_mask = target_masks[0] #- desi_mask or sv1_desi_mask #- Copy column names in common fibermap = desispec.io.empty_fibermap(len(fiberassign)) for c in ['FIBER', 'TARGETID', 'BRICKNAME']: fibermap[c] = fiberassign[c] for c in target_colnames: fibermap[c] = fiberassign[c] for band in ['G', 'R', 'Z', 'W1', 'W2']: key = 'FLUX_' + band fibermap[key] = meta[key] #- TODO: FLUX_IVAR_* #- set OBJTYPE #- TODO: what about MWS science targets that are also standard stars? #- Loop over STD options for backwards/forwards compatibility stdmask = 0 for name in [ 'STD', 'STD_FSTAR', 'STD_WD' 'STD_FAINT', 'STD_FAINT_BEST', 'STD_BRIGHT', 'STD_BRIGHT_BEST' ]: if name in desi_mask.names(): stdmask |= desi_mask[name] isSTD = (fiberassign[targetcol] & stdmask) != 0 isSKY = (fiberassign[targetcol] & desi_mask.SKY) != 0 isSCI = (~isSTD & ~isSKY) fibermap['OBJTYPE'][isSKY] = 'SKY' fibermap['OBJTYPE'][isSCI | isSTD] = 'TGT' fibermap['LAMBDAREF'] = 5400.0 fibermap['TARGET_RA'] = fiberassign['TARGET_RA'] fibermap['TARGET_DEC'] = fiberassign['TARGET_DEC'] fibermap['FIBER_RA'] = fiberassign['TARGET_RA'] fibermap['FIBER_DEC'] = fiberassign['TARGET_DEC'] fibermap['FIBERASSIGN_X'] = fiberassign['FIBERASSIGN_X'] fibermap['FIBERASSIGN_Y'] = fiberassign['FIBERASSIGN_Y'] fibermap['DELTA_X'] = 0.0 fibermap['DELTA_Y'] = 0.0 #- TODO: POSITIONER -> LOCATION #- TODO: TARGETCAT (how should we propagate this info into here?) #- TODO: NaNs in fibermap for unassigned positioners targets return fibermap
def make_ledger_in_hp(targets, outdirname, nside, pixlist, obscon="DARK", indirname=None, verbose=True): """ Make an initial MTL ledger file for targets in a set of HEALPixels. Parameters ---------- targets : :class:`~numpy.array` Targets made by, e.g. `desitarget.cuts.select_targets()`. outdirname : :class:`str` Output directory to which to write the MTLs (the file names are constructed on the fly). nside : :class:`int` (NESTED) HEALPixel nside that corresponds to `pixnum`. pixlist : :class:`list` or `int` HEALPixels at `nside` at which to write the MTLs. obscon : :class:`str`, optional, defaults to "DARK" A string matching ONE obscondition in the desitarget bitmask yaml file (i.e. in `desitarget.targetmask.obsconditions`), e.g. "GRAY" Governs how priorities are set when merging targets. Also governs the sub-directory to which the ledger is written. indirname : :class:`str` A directory associated with the targets. Written to the headers of the output MTL files. verbose : :class:`bool`, optional, defaults to ``True`` If ``True`` then log target and file information. Returns ------- Nothing, but writes the `targets` out to `outdirname` split across each HEALPixel in `pixlist`. """ t0 = time() # ADM in case an integer was passed. pixlist = np.atleast_1d(pixlist) # ADM execute MTL. mtl = make_mtl(targets, obscon, trimcols=True) # ADM the HEALPixel within which each target in the MTL lies. theta, phi = np.radians(90-mtl["DEC"]), np.radians(mtl["RA"]) mtlpix = hp.ang2pix(nside, theta, phi, nest=True) # ADM write the MTLs. _, _, survey = main_cmx_or_sv(mtl) for pix in pixlist: inpix = mtlpix == pix ecsv = get_mtl_ledger_format() == "ecsv" nt, fn = io.write_mtl( outdirname, mtl[inpix].as_array(), indir=indirname, ecsv=ecsv, survey=survey, obscon=obscon, nsidefile=nside, hpxlist=pix) if verbose: log.info('{} targets written to {}...t={:.1f}s'.format( nt, fn, time()-t0)) return
def test_cmx_priorities(self): """Test that priority calculation can handle commissioning files. """ t = self.targets.copy() z = self.zcat # ADM restructure the table to look like a commissioning table. t.rename_column('DESI_TARGET', 'CMX_TARGET') t.remove_column('BGS_TARGET') t.remove_column('MWS_TARGET') # - No targeting bits set is priority=0 self.assertTrue(np.all(calc_priority(t, z, "GRAY|DARK") == 0)) # ADM retrieve the cmx_mask. colnames, masks, _ = main_cmx_or_sv(t) cmx_mask = masks[0] # ADM test handling of unobserved SV0_BGS and SV0_MWS for name, obscon in [("SV0_BGS", "BRIGHT"), ("SV0_MWS", "POOR")]: t['CMX_TARGET'] = cmx_mask[name] self.assertTrue( np.all( calc_priority(t, z, obscon) == cmx_mask[name].priorities['UNOBS'])) # ADM done is Done, regardless of ZWARN. for name, obscon in [("SV0_BGS", "BRIGHT"), ("SV0_MWS", "POOR")]: t['CMX_TARGET'] = cmx_mask[name] t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) # APC: Use NUMOBS_INIT here to avoid hardcoding NOBS corresponding to "done". numobs_done = t['NUMOBS_INIT'][0] z['NUMOBS'] = [0, numobs_done, numobs_done] z['ZWARN'] = [1, 1, 0] p = make_mtl(t, obscon, zcat=z)["PRIORITY"] self.assertEqual(p[0], cmx_mask[name].priorities['UNOBS']) self.assertEqual(p[1], cmx_mask[name].priorities['DONE']) self.assertEqual(p[2], cmx_mask[name].priorities['DONE']) # BGS ZGOOD targets always have lower priority than MWS targets that # are not DONE. lowest_bgs_priority_zgood = cmx_mask['SV0_BGS'].priorities[ 'MORE_ZGOOD'] lowest_mws_priority_unobs = cmx_mask['SV0_MWS'].priorities['UNOBS'] lowest_mws_priority_zwarn = cmx_mask['SV0_MWS'].priorities[ 'MORE_ZWARN'] lowest_mws_priority_zgood = cmx_mask['SV0_MWS'].priorities[ 'MORE_ZGOOD'] lowest_mws_priority = min(lowest_mws_priority_unobs, lowest_mws_priority_zwarn, lowest_mws_priority_zgood) self.assertLess(lowest_bgs_priority_zgood, lowest_mws_priority)
def test_isStdStar(self): """test isStdStar works for cmx, main, and sv1 fibermaps""" from desispec.fluxcalibration import isStdStar from desitarget.targetmask import desi_mask, mws_mask from desitarget.sv1.sv1_targetmask import desi_mask as sv1_desi_mask from desitarget.sv1.sv1_targetmask import mws_mask as sv1_mws_mask from desitarget.cmx.cmx_targetmask import cmx_mask from desitarget.targets import main_cmx_or_sv from astropy.table import Table #- CMX fm = Table() fm['CMX_TARGET'] = np.zeros(10, dtype=int) fm['CMX_TARGET'][0:2] = cmx_mask.STD_FAINT fm['CMX_TARGET'][2:4] = cmx_mask.SV0_STD_FAINT self.assertEqual(main_cmx_or_sv(fm)[2], 'cmx') self.assertEqual(np.count_nonzero(isStdStar(fm)), 4) #- SV1 fm = Table() fm['SV1_DESI_TARGET'] = np.zeros(10, dtype=int) fm['SV1_MWS_TARGET'] = np.zeros(10, dtype=int) fm['SV1_DESI_TARGET'][0:2] = sv1_desi_mask.STD_FAINT fm['SV1_MWS_TARGET'][2:4] = sv1_mws_mask.GAIA_STD_FAINT self.assertEqual(main_cmx_or_sv(fm)[2], 'sv1') self.assertEqual(np.count_nonzero(isStdStar(fm)), 4) #- Main fm = Table() fm['DESI_TARGET'] = np.zeros(10, dtype=int) fm['MWS_TARGET'] = np.zeros(10, dtype=int) fm['DESI_TARGET'][0:2] = desi_mask.STD_FAINT fm['DESI_TARGET'][2:4] = desi_mask.STD_BRIGHT fm['DESI_TARGET'][4:6] |= desi_mask.MWS_ANY fm['MWS_TARGET'][4:6] = sv1_mws_mask.GAIA_STD_FAINT self.assertEqual(main_cmx_or_sv(fm)[2], 'main') self.assertEqual(np.count_nonzero(isStdStar(fm)), 6) self.assertEqual(np.count_nonzero(isStdStar(fm, bright=False)), 4) self.assertEqual(np.count_nonzero(isStdStar(fm, bright=True)), 2)
def select_stdstars(data, fibermap, snrcut=10, verbose=False): """Select spectra based on S/N and targeting bit. """ from astropy.table import Table from desitarget.targets import main_cmx_or_sv night, expid = data['NIGHT'][0], data['EXPID'][0] out_fibermap = fibermap.copy() # Pre-select standards istd = np.where(isStdStar(out_fibermap))[0] target_colnames, target_masks, survey = main_cmx_or_sv(out_fibermap) target_col, target_mask = target_colnames[0], target_masks[ 0] # CMX_TARGET, CMX_MASK for commissioning #istd = np.where((out_fibermap[target_col] & target_mask.mask('STD_BRIGHT') != 0))[0] # For each standard, apply a minimum S/N cut (in any camera) to # subselect the fibers that were reasonably centered on the fiber. reject = np.ones(len(istd), dtype=bool) #if len(istd) == 0: # pdb.set_trace() if len(istd) > 0: for ii, fiber in enumerate(out_fibermap['FIBER'][istd]): wspec = np.where( (data['EXPID'] == expid) * (data['FIBER'] == fiber))[0] if len(wspec) > 0: isnr = np.where(data['MEDIAN_CALIB_SNR'][wspec] > snrcut)[0] if len(isnr) > 0: reject[ii] = False if np.sum(~reject) > 0: istd_keep = istd[~reject] else: istd_keep = [] if np.sum(reject) > 0: istd_reject = istd[reject] else: istd_reject = [] # Hack! Set the targeting bit for the failed standards to zero! if len(istd_reject) > 0: out_fibermap[target_col][istd_reject] = 0 if verbose: print('EXPID={}, NSTD={}/{}'.format(expid, len(istd_keep), len(istd))) return out_fibermap
def test_priorities(self): """Test that priorities are set correctly for both the main survey and SV. """ # ADM loop through once for SV and once for the main survey. for prefix in ["", "SV1_"]: t = self.targets.copy() z = self.zcat.copy() main_names = ['DESI_TARGET', 'BGS_TARGET', 'MWS_TARGET'] for name in main_names: t.rename_column(name, prefix + name) # ADM retrieve the mask and column names for this survey flavor. colnames, masks, _ = main_cmx_or_sv(t) desi_target, bgs_target, mws_target = colnames desi_mask, bgs_mask, mws_mask = masks # - No targeting bits set is priority=0 self.assertTrue(np.all(calc_priority(t, z, "BRIGHT") == 0)) # ADM test QSO > LRG > ELG for main survey and SV. t[desi_target] = desi_mask.ELG self.assertTrue( np.all( calc_priority(t, z, "GRAY|DARK") == desi_mask.ELG.priorities['UNOBS'])) t[desi_target] |= desi_mask.LRG self.assertTrue( np.all( calc_priority(t, z, "GRAY|DARK") == desi_mask.LRG.priorities['UNOBS'])) t[desi_target] |= desi_mask.QSO self.assertTrue( np.all( calc_priority(t, z, "GRAY|DARK") == desi_mask.QSO.priorities['UNOBS'])) # - different states -> different priorities # - Done is Done, regardless of ZWARN. t[desi_target] = desi_mask.ELG t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 1, 1] z['ZWARN'] = [1, 1, 0] p = make_mtl(t, "GRAY|DARK", zcat=z)["PRIORITY"] self.assertEqual(p[0], desi_mask.ELG.priorities['UNOBS']) self.assertEqual(p[1], desi_mask.ELG.priorities['DONE']) self.assertEqual(p[2], desi_mask.ELG.priorities['DONE']) # ADM In BRIGHT conditions BGS FAINT targets are # ADM never DONE, only MORE_ZGOOD. t[desi_target] = desi_mask.BGS_ANY t[bgs_target] = bgs_mask.BGS_FAINT t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 1, 1] z['ZWARN'] = [1, 1, 0] p = make_mtl(t, "BRIGHT", zcat=z)["PRIORITY"] self.assertEqual(p[0], bgs_mask.BGS_FAINT.priorities['UNOBS']) self.assertEqual(p[1], bgs_mask.BGS_FAINT.priorities['MORE_ZWARN']) self.assertEqual(p[2], bgs_mask.BGS_FAINT.priorities['MORE_ZGOOD']) # BGS_FAINT: {UNOBS: 2000, MORE_ZWARN: 2000, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} # ADM but in DARK conditions, BGS_FAINT should behave as # ADM for other target classes. z = self.zcat.copy() z['NUMOBS'] = [0, 1, 1] z['ZWARN'] = [1, 1, 0] p = make_mtl(t, "DARK|GRAY", zcat=z)["PRIORITY"] self.assertEqual(p[0], bgs_mask.BGS_FAINT.priorities['UNOBS']) self.assertEqual(p[1], bgs_mask.BGS_FAINT.priorities['DONE']) self.assertEqual(p[2], bgs_mask.BGS_FAINT.priorities['DONE']) # ADM In BRIGHT conditions BGS BRIGHT targets are # ADM never DONE, only MORE_ZGOOD. t[desi_target] = desi_mask.BGS_ANY t[bgs_target] = bgs_mask.BGS_BRIGHT t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 1, 1] z['ZWARN'] = [1, 1, 0] p = make_mtl(t, "BRIGHT", zcat=z)["PRIORITY"] self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS']) self.assertEqual(p[1], bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN']) self.assertEqual(p[2], bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD']) # BGS_BRIGHT: {UNOBS: 2100, MORE_ZWARN: 2100, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} # ADM In BRIGHT conditions BGS targets are # ADM NEVER done even after 100 observations t[desi_target] = desi_mask.BGS_ANY t[bgs_target] = bgs_mask.BGS_BRIGHT t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 100, 100] z['ZWARN'] = [1, 1, 0] p = calc_priority(t, z, "BRIGHT") self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS']) self.assertEqual(p[1], bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN']) self.assertEqual(p[2], bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD']) # BGS ZGOOD targets always have lower priority than MWS targets that # are not DONE. Exempting the MWS "BACKUP" targets. # ADM first discard N/S informational bits from bitmask as these # ADM should never trump the other bits. bgs_names = [ name for name in bgs_mask.names() if 'NORTH' not in name and 'SOUTH' not in name ] mws_names = [ name for name in mws_mask.names() if 'NORTH' not in name and 'SOUTH' not in name and 'BACKUP' not in name ] lowest_mws_priority_unobs = [ mws_mask[n].priorities['UNOBS'] for n in mws_names ] lowest_bgs_priority_zgood = np.min( [bgs_mask[n].priorities['MORE_ZGOOD'] for n in bgs_names]) # ADM MORE_ZGOOD and MORE_ZWARN are only meaningful if a # ADM target class requests more than 1 observation (except # ADM for BGS, which has a numobs=infinity exception) lowest_mws_priority_zwarn = [ mws_mask[n].priorities['MORE_ZWARN'] for n in mws_names if mws_mask[n].numobs > 1 ] lowest_mws_priority_zgood = [ mws_mask[n].priorities['MORE_ZGOOD'] for n in mws_names if mws_mask[n].numobs > 1 ] lowest_mws_priority = np.min( np.concatenate([ lowest_mws_priority_unobs, lowest_mws_priority_zwarn, lowest_mws_priority_zgood ])) self.assertLess(lowest_bgs_priority_zgood, lowest_mws_priority)
def select_fibers_to_fit(fibermap, sns, zbest_path=None, minsn=None, objtypes=None, expid_range=None, fit_targetid=None, zbest_select=False): """ Identify fibers to fit Parameters ---------- fibermap: Table Fibermap table object sns: dict of numpy arrays Array of S/Ns minsn: float Threshold S/N objtypes: list of regular expressions of DESI_TARGETs ['MWS_ANY,'STD_*'] or None expid_range: list The range of EXPID to consider fit_targetid: list of ints Fit only specific TARGETIDs Returns ------- ret: bool numpy array Array with True for selected spectra """ zbest_maxvel = 1500 # maximum velocity to consider a star zbest_type = 'STAR' try: import desitarget.targets as DT except ImportError: DT = None subset = np.ones(len(fibermap), dtype=bool) # Always apply EXPID range if expid_range is not None: mine, maxe = expid_range if mine is None: mine = -1 if maxe is None: maxe = np.inf if "EXPID" in fibermap.columns.names: subset = subset & (fibermap["EXPID"] > mine) & (fibermap['EXPID'] <= maxe) # ONLY select good fiberstatus ones if 'FIBERSTATUS' in fibermap.columns.names: subset = subset & (fibermap['FIBERSTATUS'] == 0) elif 'COADD_FIBERSTATUS' in fibermap.columns.names: subset = subset & (fibermap['COADD_FIBERSTATUS'] == 0) # Exclude skys but include anything else subset = subset & (fibermap['OBJTYPE'] != 'SKY') # Always apply TARGETID selection if provided if fit_targetid is not None: subset = subset & np.in1d(fibermap['TARGETID'], fit_targetid) # Always apply SN cut if provided if minsn is not None: maxsn = np.max(np.array(list(sns.values())), axis=0) with warnings.catch_warnings(): warnings.simplefilter('ignore') subset = subset & (maxsn > minsn) # compute the subset based on TARGET types fibermapT = atpy.Table(fibermap) types_subset = np.ones(len(fibermap), dtype=bool) selecting_by_type = False if DT is not None and objtypes is not None: selecting_by_type = True re_types = [re.compile(_) for _ in objtypes] for i, currow in enumerate(fibermapT): col_list, mask_list, _ = DT.main_cmx_or_sv(currow, scnd=True) # collist will be column list like # DESI_TARGET, BGS_TARGET, MWS_TARGET # extract the DESI_TARGET part colname = col_list[0] mask = mask_list[0] # all the possible types here objtypnames = list(mask.names()) objs = [] # check which names match our regular expression for curo in objtypnames: for r in re_types: if r.match(curo) is not None: objs.append(curo) # obtain integer values for each object type that matched # our RE and bitwise OR them bitmask = functools.reduce(operator.or_, [mask.mask(_) for _ in objs]) # check if the given row has any hits in the bitmask types_subset[i] = (currow[colname] & bitmask) > 0 # select objects based on redrock velocity or type if zbest_select: if zbest_path is None: warnings.warn( 'zbest selection requested, but the zbest file not found') if selecting_by_type: zbest_subset = np.zeros(len(fibermap), dtype=bool) else: # I fit everything zbest_subset = np.ones(len(fibermap), dtype=bool) else: zb = atpy.Table().read(zbest_path, format='fits', hdu='REDSHIFTS') assert (len(zb) == len(subset)) zbest_subset = (((zb['SPECTYPE'] == zbest_type) | (np.abs(zb['Z']) < zbest_maxvel / 3e5))) else: zbest_subset = np.zeros(len(fibermap), dtype=bool) # We select either based on type or zbest subset = subset & (zbest_subset | types_subset) return subset
def main(args): log = get_logger() cmd = [ 'desi_compute_fluxcalibration', ] for key, value in args.__dict__.items(): if value is not None: cmd += ['--' + key, str(value)] cmd = ' '.join(cmd) log.info(cmd) log.info("read frame") # read frame frame = read_frame(args.infile) # Set fibermask flagged spectra to have 0 flux and variance frame = get_fiberbitmasked_frame(frame, bitmask='flux', ivar_framemask=True) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models( args.models) ok = np.ones(len(model_metadata), dtype=bool) if args.chi2cut > 0: log.info("apply cut CHI2DOF<{}".format(args.chi2cut)) good = (model_metadata["CHI2DOF"] < args.chi2cut) bad = ~good ok &= good if np.any(bad): log.info(" discard {} stars with CHI2DOF= {}".format( np.sum(bad), list(model_metadata["CHI2DOF"][bad]))) legacy_filters = ('G-R', 'R-Z') gaia_filters = ('GAIA-BP-RP', 'GAIA-G-RP') model_column_list = model_metadata.columns.names if args.color is None: if 'MODEL_G-R' in model_column_list: color = 'G-R' elif 'MODEL_GAIA-BP-RP' in model_column_list: log.info('Using Gaia filters') color = 'GAIA-BP-RP' else: log.error( "Can't find either G-R or BP-RP color in the model file.") sys.exit(15) else: if args.color not in legacy_filters and args.color not in gaia_filters: log.error( 'Color name {} is not allowed, must be one of {} {}'.format( args.color, legacy_filters, gaia_filters)) sys.exit(14) color = args.color if color not in model_column_list: # This should't happen log.error( 'The color {} was not computed in the models'.format(color)) sys.exit(16) if args.delta_color_cut > 0: log.info("apply cut |delta color|<{}".format(args.delta_color_cut)) good = (np.abs(model_metadata["MODEL_" + color] - model_metadata["DATA_" + color]) < args.delta_color_cut) bad = ok & (~good) ok &= good if np.any(bad): vals = model_metadata["MODEL_" + color][bad] - model_metadata["DATA_" + color][bad] log.info(" discard {} stars with dcolor= {}".format( np.sum(bad), list(vals))) if args.min_color is not None: log.info("apply cut DATA_{}>{}".format(color, args.min_color)) good = (model_metadata["DATA_{}".format(color)] > args.min_color) bad = ok & (~good) ok &= good if np.any(bad): vals = model_metadata["DATA_{}".format(color)][bad] log.info(" discard {} stars with {}= {}".format( np.sum(bad), color, list(vals))) if args.chi2cut_nsig > 0: # automatically reject stars that ar chi2 outliers mchi2 = np.median(model_metadata["CHI2DOF"]) rmschi2 = np.std(model_metadata["CHI2DOF"]) maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2 log.info("apply cut CHI2DOF<{} based on chi2cut_nsig={}".format( maxchi2, args.chi2cut_nsig)) good = (model_metadata["CHI2DOF"] <= maxchi2) bad = ok & (~good) ok &= good if np.any(bad): log.info(" discard {} stars with CHI2DOF={}".format( np.sum(bad), list(model_metadata["CHI2DOF"][bad]))) ok = np.where(ok)[0] if ok.size == 0: log.error("selection cuts discarded all stars") sys.exit(12) nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning("discarding %d star(s) out of %d because of cuts" % (nbad, nstars)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] # check that the model_fibers are actually standard stars fibermap = frame.fibermap ## check whether star fibers from args.models are consistent with fibers from fibermap ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit fibermap_std_indices = np.where(isStdStar(fibermap))[0] if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)): target_colnames, target_masks, survey = main_cmx_or_sv(fibermap) colname = target_colnames[0] for i in model_fibers % 500: log.error( "inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap" .format(i, fibermap["OBJTYPE"][i], colname, fibermap[colname][i])) sys.exit(12) # Make sure the fibers of interest aren't entirely masked. if np.sum( np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) == frame.nwave) == len(model_fibers): log.warning('All standard-star spectra are masked!') return fluxcalib = compute_flux_calibration( frame, model_wave, model_flux, model_fibers % 500, highest_throughput_nstars=args.highest_throughput, exposure_seeing_fwhm=args.seeing_fwhm) # QA if (args.qafile is not None): from desispec.io import write_qa_frame from desispec.io.qa import load_qa_frame from desispec.qa import qa_plots log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame_meta=frame.meta, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # record inputs frame.meta['IN_FRAME'] = shorten_filename(args.infile) frame.meta['IN_SKY'] = shorten_filename(args.sky) frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat) frame.meta['STDMODEL'] = shorten_filename(args.models) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s" % args.outfile)
def make_mtl(targets, obscon, zcat=None, trim=False, scnd=None): """Adds NUMOBS, PRIORITY, and OBSCONDITIONS columns to a targets table. Parameters ---------- targets : :class:`~numpy.array` or `~astropy.table.Table` A numpy rec array or astropy Table with at least the columns ``TARGETID``, ``DESI_TARGET``, ``NUMOBS_INIT``, ``PRIORITY_INIT``. or the corresponding columns for SV or commissioning. obscon : :class:`str` A combination of strings that are in the desitarget bitmask yaml file (specifically in `desitarget.targetmask.obsconditions`), e.g. "DARK|GRAY". Governs the behavior of how priorities are set based on "obsconditions" in the desitarget bitmask yaml file. zcat : :class:`~astropy.table.Table`, optional Redshift catalog table with columns ``TARGETID``, ``NUMOBS``, ``Z``, ``ZWARN``. trim : :class:`bool`, optional If ``True`` (default), don't include targets that don't need any more observations. If ``False``, include every input target. scnd : :class:`~numpy.array`, `~astropy.table.Table`, optional A set of secondary targets associated with the `targets`. As with the `target` must include at least ``TARGETID``, ``NUMOBS_INIT``, ``PRIORITY_INIT`` or the corresponding SV columns. The secondary targets will be padded to have the same columns as the targets, and concatenated with them. Returns ------- :class:`~astropy.table.Table` MTL Table with targets columns plus: * NUMOBS_MORE - number of additional observations requested * PRIORITY - target priority (larger number = higher priority) * OBSCONDITIONS - replaces old GRAYLAYER """ start = time() # ADM set up the default logger. from desiutil.log import get_logger log = get_logger() # ADM if secondaries were passed, concatenate them with the targets. if scnd is not None: nrows = len(scnd) log.info( 'Pad {} primary targets with {} secondaries...t={:.1f}s'.format( len(targets), nrows, time() - start)) padit = np.zeros(nrows, dtype=targets.dtype) sharedcols = set(targets.dtype.names).intersection( set(scnd.dtype.names)) for col in sharedcols: padit[col] = scnd[col] targets = np.concatenate([targets, padit]) # APC Propagate a flag on which targets came from scnd is_scnd = np.repeat(False, len(targets)) is_scnd[-nrows:] = True log.info('Done with padding...t={:.1f}s'.format(time() - start)) # ADM determine whether the input targets are main survey, cmx or SV. colnames, masks, survey = main_cmx_or_sv(targets) # ADM set the first column to be the "desitarget" column desi_target, desi_mask = colnames[0], masks[0] # Trim targets from zcat that aren't in original targets table if zcat is not None: ok = np.in1d(zcat['TARGETID'], targets['TARGETID']) num_extra = np.count_nonzero(~ok) if num_extra > 0: log.warning("Ignoring {} zcat entries that aren't " "in the input target list".format(num_extra)) zcat = zcat[ok] n = len(targets) # ADM if the input target columns were incorrectly called NUMOBS or PRIORITY # ADM rename them to NUMOBS_INIT or PRIORITY_INIT. # ADM Note that the syntax is slightly different for a Table. for name in ['NUMOBS', 'PRIORITY']: if isinstance(targets, Table): try: targets.rename_column(name, name + '_INIT') except KeyError: pass else: targets.dtype.names = [ name + '_INIT' if col == name else col for col in targets.dtype.names ] # ADM if a redshift catalog was passed, order it to match the input targets # ADM catalog on 'TARGETID'. if zcat is not None: # ADM there might be a quicker way to do this? # ADM set up a dictionary of the indexes of each target id. d = dict(tuple(zip(targets["TARGETID"], np.arange(n)))) # ADM loop through the zcat and look-up the index in the dictionary. zmatcher = np.array([d[tid] for tid in zcat["TARGETID"]]) ztargets = zcat if ztargets.masked: unobs = ztargets['NUMOBS'].mask ztargets['NUMOBS'][unobs] = 0 unobsz = ztargets['Z'].mask ztargets['Z'][unobsz] = -1 unobszw = ztargets['ZWARN'].mask ztargets['ZWARN'][unobszw] = -1 else: ztargets = Table() ztargets['TARGETID'] = targets['TARGETID'] ztargets['NUMOBS'] = np.zeros(n, dtype=np.int32) ztargets['Z'] = -1 * np.ones(n, dtype=np.float32) ztargets['ZWARN'] = -1 * np.ones(n, dtype=np.int32) # ADM if zcat wasn't passed, there is a one-to-one correspondence # ADM between the targets and the zcat. zmatcher = np.arange(n) # ADM extract just the targets that match the input zcat. targets_zmatcher = targets[zmatcher] # ADM use passed value of NUMOBS_INIT instead of calling the memory-heavy calc_numobs. # ztargets['NUMOBS_MORE'] = np.maximum(0, calc_numobs(ztargets) - ztargets['NUMOBS']) ztargets['NUMOBS_MORE'] = np.maximum( 0, targets_zmatcher['NUMOBS_INIT'] - ztargets['NUMOBS']) # ADM need a minor hack to ensure BGS targets are observed once # ADM (and only once) every time during the BRIGHT survey, regardless # ADM of how often they've previously been observed. I've turned this # ADM off for commissioning. Not sure if we'll keep it in general. if survey != 'cmx': # ADM only if we're considering bright survey conditions. if (obsconditions.mask(obscon) & obsconditions.mask("BRIGHT")) != 0: ii = targets_zmatcher[desi_target] & desi_mask.BGS_ANY > 0 ztargets['NUMOBS_MORE'][ii] = 1 if survey == 'main': # If the object is confirmed to be a tracer QSO, then don't request more observations if (obsconditions.mask(obscon) & obsconditions.mask("DARK")) != 0: if zcat is not None: ii = ztargets['SPECTYPE'] == 'QSO' ii &= (ztargets['ZWARN'] == 0) ii &= (ztargets['Z'] < 2.1) ii &= (ztargets['NUMOBS'] > 0) ztargets['NUMOBS_MORE'][ii] = 0 # ADM assign priorities, note that only things in the zcat can have changed priorities. # ADM anything else will be assigned PRIORITY_INIT, below. priority = calc_priority(targets_zmatcher, ztargets, obscon) # If priority went to 0==DONOTOBSERVE or 1==OBS or 2==DONE, then NUMOBS_MORE should also be 0. # ## mtl['NUMOBS_MORE'] = ztargets['NUMOBS_MORE'] ii = (priority <= 2) log.info( '{:d} of {:d} targets have priority zero, setting N_obs=0.'.format( np.sum(ii), n)) ztargets['NUMOBS_MORE'][ii] = 0 # - Set the OBSCONDITIONS mask for each target bit. obsconmask = set_obsconditions(targets) # APC obsconmask will now be incorrect for secondary-only targets. Fix this # APC using the mask on secondary targets. if scnd is not None: obsconmask[is_scnd] = set_obsconditions(targets[is_scnd], scnd=True) # ADM set up the output mtl table. mtl = Table(targets) mtl.meta['EXTNAME'] = 'MTL' # ADM any target that wasn't matched to the ZCAT should retain its # ADM original (INIT) value of PRIORITY and NUMOBS. mtl['NUMOBS_MORE'] = mtl['NUMOBS_INIT'] mtl['PRIORITY'] = mtl['PRIORITY_INIT'] # ADM now populate the new mtl columns with the updated information. mtl['OBSCONDITIONS'] = obsconmask mtl['PRIORITY'][zmatcher] = priority mtl['NUMOBS_MORE'][zmatcher] = ztargets['NUMOBS_MORE'] # Filter out any targets marked as done. if trim: notdone = mtl['NUMOBS_MORE'] > 0 log.info('{:d} of {:d} targets are done, trimming these'.format( len(mtl) - np.sum(notdone), len(mtl))) mtl = mtl[notdone] # Filtering can reset the fill_value, which is just wrong wrong wrong # See https://github.com/astropy/astropy/issues/4707 # and https://github.com/astropy/astropy/issues/4708 mtl['NUMOBS_MORE'].fill_value = -1 log.info('Done...t={:.1f}s'.format(time() - start)) return mtl
def parse_assign(optlist=None): """Parse assignment options. This parses either sys.argv or a list of strings passed in. If passing an option list, you can create that more easily using the :func:`option_list` function. Args: optlist (list, optional): Optional list of arguments to parse instead of using sys.argv. Returns: (namespace): an ArgumentParser namespace. """ log = Logger.get() parser = argparse.ArgumentParser() parser.add_argument("--targets", type=str, required=True, nargs="+", help="Input file with targets of any type. This " "argument can be specified multiple times (for " "example if standards / skies / science targets are " "in different files). By default, the " "'--mask_column' (default DESI_TARGET)" "column and bitfield values defined in desitarget " "are used to determine the type of each target. " "Each filename may be optionally followed by comma " "and then one of the strings 'science', 'standard', " "'sky' or 'safe' to force all targets in that file " "to be treated as a fixed target type.") parser.add_argument("--sky", type=str, required=False, nargs="+", help="Input file with sky or 'bad sky' targets. " "This option exists in order to treat main-survey" " sky target files as valid for other survey types." " If you are running a main survey assignment, you" " can just pass the sky file to the --targets list.") parser.add_argument("--gfafile", type=str, required=False, default=None, help="Optional GFA targets FITS file") parser.add_argument("--footprint", type=str, required=False, default=None, help="Optional FITS file defining the footprint. If" " not specified, the default footprint from desimodel" " is used.") parser.add_argument("--tiles", type=str, required=False, default=None, help="Optional text file containing a subset of the" " tile IDs to use in the footprint, one ID per line." " Default uses all tiles in the footprint.") parser.add_argument("--rundate", type=str, required=False, default=None, help="Optional date to simulate for this run of " "fiber assignment, used to load the correct " "focalplane properties and state from desimodel. " "Default uses the current date. Format is " "YYYY-MM-DDTHH:mm:ss+-zz:zz.") parser.add_argument("--obsdate", type=str, required=False, default="2022-07-01", help="Plan field rotations for this date (YEARMMDD, " "or ISO 8601 YEAR-MM-DD with or without time).") parser.add_argument("--ha", type=float, required=False, default=0., help="Design for the given Hour Angle in degrees.") parser.add_argument("--fieldrot", type=float, required=False, default=None, help="Override obsdate and use this field rotation " "for all tiles (degrees counter clockwise in CS5)") parser.add_argument("--dir", type=str, required=False, default=None, help="Output directory.") parser.add_argument("--prefix", type=str, required=False, default="fba-", help="Prefix of each file (before the <tile>.fits).") parser.add_argument("--split", required=False, default=False, action="store_true", help="Split output into tile prefix directories.") parser.add_argument("--standards_per_petal", type=int, required=False, default=10, help="Required number of standards per" " petal") parser.add_argument("--sky_per_petal", type=int, required=False, default=40, help="Required number of sky targets per" " petal") parser.add_argument("--sky_per_slitblock", type=int, required=False, default=1, help="Required number of sky targets per" " fiber slitblock") parser.add_argument("--margin-pos", type=float, required=False, default=0., help="Add margin (in mm) around positioner keep-out polygons") parser.add_argument("--margin-petal", type=float, required=False, default=0., help="Add margin (in mm) around petal-boundary keep-out polygons") parser.add_argument("--margin-gfa", type=float, required=False, default=0., help="Add margin (in mm) around GFA keep-out polygons") parser.add_argument("--write_all_targets", required=False, default=False, action="store_true", help="When writing target properties, write data " "for all available targets, not just those which are " "assigned. This is convenient, but increases the " "write time and the file size.") parser.add_argument("--overwrite", required=False, default=False, action="store_true", help="Overwrite any pre-existing output files") parser.add_argument("--mask_column", required=False, default=None, help="FITS column to use for applying target " "masks") parser.add_argument("--sciencemask", required=False, default=None, help="Default DESI_TARGET mask to use for science " "targets") parser.add_argument("--stdmask", required=False, default=None, help="Default DESI_TARGET mask to use for stdstar " "targets") parser.add_argument("--skymask", required=False, default=None, help="Default DESI_TARGET mask to use for sky targets") parser.add_argument("--safemask", required=False, default=None, help="Default DESI_TARGET mask to use for safe " "backup targets") parser.add_argument("--excludemask", required=False, default=None, help="Default DESI_TARGET mask to exclude from " "any assignments") parser.add_argument("--by_tile", required=False, default=False, action="store_true", help="Do assignment one tile at a time. This disables" " redistribution.") parser.add_argument("--no_redistribute", required=False, default=False, action="store_true", help="Disable redistribution of science targets.") parser.add_argument("--no_zero_obsremain", required=False, default=False, action="store_true", help="Disable oversubscription of science targets with leftover fibers.") args = None if optlist is None: args = parser.parse_args() else: args = parser.parse_args(optlist) if args.sky is None: args.sky = list() # If any of the masks are strings, determine the survey type from # the first target file to know which bitmask to use if isinstance(args.sciencemask, str) or \ isinstance(args.stdmask, str) or \ isinstance(args.skymask, str) or \ isinstance(args.safemask, str) or \ isinstance(args.excludemask, str): import fitsio from desitarget.targets import main_cmx_or_sv data = fitsio.read(args.targets[0], 1, rows=[0,1]) filecols, filemasks, filesurvey = main_cmx_or_sv(data) desi_mask = filemasks[0] # convert str bit names -> int bit mask if isinstance(args.sciencemask, str): try: args.sciencemask = int(args.sciencemask) except ValueError: args.sciencemask = desi_mask.mask(args.sciencemask.replace(",","|")) if isinstance(args.stdmask, str): try: args.stdmask = int(args.stdmask) except ValueError: args.stdmask = desi_mask.mask(args.stdmask.replace(",", "|")) if isinstance(args.skymask, str): try: args.skymask = int(args.skymask) except ValueError: args.skymask = desi_mask.mask(args.skymask.replace(",", "|")) if isinstance(args.safemask, str): try: args.safemask = int(args.safemask) except ValueError: args.safemask = desi_mask.mask(args.safemask.replace(",", "|")) if isinstance(args.excludemask, str): try: args.excludemask = int(args.excludemask) except ValueError: args.excludemask = desi_mask.mask(args.excludemask.replace(",","|")) # convert YEARMMDD to YEAR-MM-DD to be ISO 8601 compatible if re.match('\d{8}', args.obsdate): year = args.obsdate[0:4] mm = args.obsdate[4:6] dd = args.obsdate[6:8] #- Note: ISO8601 does not require time portion args.obsdate = '{}-{}-{}'.format(year, mm, dd) # Set output directory if args.dir is None: if args.rundate is None: raise RuntimeError( "You must specify the output directory or the rundate") args.dir = "out_fiberassign_{}".format(args.rundate) # Set up margins dict args.margins = {} if args.margin_pos != 0: args.margins['theta'] = args.margin_pos args.margins['phi'] = args.margin_pos if args.margin_petal != 0: ## The petal polygon is listed in *clockwise* order, so we have to give it a negative margin! args.margins['petal'] = -args.margin_petal if args.margin_gfa != 0: args.margins['gfa'] = args.margin_gfa return args
def get_mock_spectra(fiberassign, mockdir=None, nside=64, obscon=None): ''' Args: fiberassign: table loaded from fiberassign tile file Options: mockdir (str): base directory under which files are found nside (int): healpix nside for file directory grouping obscon (str): (observing conditions) None/dark/bright extra dir level Returns (flux, wave, meta) tuple ''' nspec = len(fiberassign) flux = None meta = None wave = None objmeta = None target_colnames, target_masks, survey = main_cmx_or_sv(fiberassign) targetcol = target_colnames[0] #- DESI_TARGET or SV1_DESI_TARGET desi_mask = target_masks[0] #- desi_mask or sv1_desi_mask issky = (fiberassign[targetcol] & desi_mask.SKY) != 0 skyids = fiberassign['TARGETID'][issky] #- check several ways in which an unassigned fiber might appear unassigned = np.isnan(fiberassign['TARGET_RA']) unassigned |= np.isnan(fiberassign['TARGET_DEC']) unassigned |= (fiberassign['TARGETID'] < 0) ## TODO: check desi_mask.NO_TARGET once that bit exists for truthfile, targetids in zip(*targets2truthfiles( fiberassign[~unassigned], basedir=mockdir, nside=nside, obscon=obscon)): #- Sky fibers aren't in the truth files ok = ~np.in1d(targetids, skyids) tmpflux, tmpwave, tmpmeta, tmpobjmeta = read_mock_spectra( truthfile, targetids[ok]) if flux is None: nwave = tmpflux.shape[1] flux = np.zeros((nspec, nwave)) meta = np.zeros(nspec, dtype=tmpmeta.dtype) meta['TARGETID'] = -1 wave = tmpwave.astype('f8') objmeta = dict() for key in tmpobjmeta.keys(): objmeta[key] = list() ii = np.in1d(fiberassign['TARGETID'], tmpmeta['TARGETID']) flux[ii] = tmpflux meta[ii] = tmpmeta assert np.all(wave == tmpwave) for key in tmpobjmeta.keys(): if key not in objmeta: objmeta[key] = list() objmeta[key].append(tmpobjmeta[key]) #- Stack the per-objtype meta tables for key in objmeta.keys(): objmeta[key] = astropy.table.Table(np.hstack(objmeta[key])) #- Set meta['TARGETID'] for sky fibers #- TODO: other things to set? meta['TARGETID'][issky] = skyids meta['TARGETID'][unassigned] = fiberassign['TARGETID'][unassigned] assert np.all(fiberassign['TARGETID'] == meta['TARGETID']) return flux, wave, astropy.table.Table(meta), objmeta
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None): """Match secondary targets to primary targets and update bits. Parameters ---------- primtargs : :class:`~numpy.ndarray` An array of primary targets. scndout : :class`~numpy.ndarray` Name of a sub-directory to which to write the information in `desitarget.secondary.outdatamodel` with `TARGETID` and (the highest) `PRIORITY_INIT` updated with matching primary info. scxdir : :class:`str`, optional, defaults to `None` Name of the directory that hosts secondary targets. sep : :class:`float`, defaults to 1 arcsecond The separation at which to match in ARCSECONDS. pix : :class:`list`, optional, defaults to `None` Limit secondary targets to (NESTED) HEALpixels that touch pix at the supplied `nside`, as a speed-up. nside : :class:`int`, optional, defaults to `None` The (NESTED) HEALPixel nside to be used with `pixlist`. Returns ------- :class:`~numpy.ndarray` The array of primary targets, with the `SCND_TARGET` bit populated for matches to secondary targets """ # ADM add a SCND_TARGET column to the primary targets. dt = primtargs.dtype.descr dt.append(('SCND_TARGET', '>i8')) targs = np.zeros(len(primtargs), dtype=dt) for col in primtargs.dtype.names: targs[col] = primtargs[col] # ADM check if this is an SV or main survey file. cols, mx, surv = main_cmx_or_sv(targs, scnd=True) log.info('running on the {} survey...'.format(surv)) if surv != 'main': scxdir = os.path.join(scxdir, surv) # ADM read in non-OVERRIDE secondary targets. scxtargs = read_files(scxdir, mx[3]) scxtargs = scxtargs[~scxtargs["OVERRIDE"]] # ADM match primary targets to non-OVERRIDE secondary targets. inhp = np.ones(len(scxtargs), dtype="?") # ADM as a speed-up, save memory by limiting the secondary targets # ADM to just HEALPixels that could touch the primary targets. if nside is not None and pix is not None: # ADM remember to grab adjacent pixels in case of edge effects. allpix = add_hp_neighbors(nside, pix) inhp = is_in_hp(scxtargs, nside, allpix) # ADM it's unlikely that the matching separation is comparable # ADM to the HEALPixel resolution, but guard against that anyway. halfpix = np.degrees(hp.max_pixrad(nside)) * 3600. if sep > halfpix: msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format( sep, halfpix) log.critical(msg) raise ValueError(msg) # ADM warn the user if the secondary and primary samples are "large". big = 500000 if np.sum(inhp) > big and len(primtargs) > big: log.warning('Large secondary (N={}) and primary (N={}) samples'.format( np.sum(inhp), len(primtargs))) log.warning('The code may run slowly') # ADM for each secondary target, determine if there is a match # ADM with a primary target. Note that sense is important, here # ADM (the primary targets must be passed first). log.info( 'Matching primary and secondary targets for {} at {}"...t={:.1f}s'. format(scndout, sep, time() - start)) mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep) # ADM recast the indices to the full set of secondary targets, # ADM instead of just those that were in the relevant HEALPixels. mscx = np.where(inhp)[0][mscx] # ADM loop through the matches and update the SCND_TARGET # ADM column in the primary target list. The np.unique is a # ADM speed-up to assign singular matches first. umtargs, inv, cnt = np.unique(mtargs, return_inverse=True, return_counts=True) # ADM number of times each primary target was matched, ordered # ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs. nmtargs = cnt[inv] # ADM assign anything with nmtargs = 1 directly. singular = nmtargs == 1 targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][ mscx[singular]] # ADM loop through things with nmtargs > 1 and combine the bits. for i in range(len((mtargs[~singular]))): targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][ mscx[~singular][i]] # ADM also assign the SCND_ANY bit to the primary targets. desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True) targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY # ADM rename the SCND_TARGET column, in case this is an SV file. targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]}) # ADM update the secondary targets with the primary information. scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs] # ADM the maximum priority will be used to break ties in the # ADM unlikely event that a secondary matches two primaries. hipri = np.maximum(targs["PRIORITY_INIT_DARK"], targs["PRIORITY_INIT_BRIGHT"]) scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs] # ADM write the secondary targets that have updated TARGETIDs. ii = scxtargs["TARGETID"] != -1 nmatches = np.sum(ii) log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format( nmatches, scndout, time() - start)) if nmatches > 0: hdr = fitsio.FITSHDR() hdr["SURVEY"] = surv fitsio.write(scndout, scxtargs[ii], extname='SCND_TARG', header=hdr, clobber=True) log.info('Done...t={:.1f}s'.format(time() - start)) return targs
def finalize_secondary(scxtargs, scnd_mask, survey='main', sep=1., darkbright=False): """Assign secondary targets a realistic TARGETID, finalize columns. Parameters ---------- scxtargs : :class:`~numpy.ndarray` An array of secondary targets, must contain the columns `RA`, `DEC` and `TARGETID`. `TARGETID` should be -1 for objects that lack a `TARGETID`. scnd_mask : :class:`desiutil.bitmask.BitMask` A mask corresponding to a set of secondary targets, e.g, could be ``from desitarget.targetmask import scnd_mask`` for the main survey mask. survey : :class:`str`, optional, defaults to "main" string indicating whether we are working in the context of the Main Survey (`main`) or SV (e.g. `sv1`, `sv2` etc.). Used to set the `RELEASE` number in the `TARGETID` (see Notes). sep : :class:`float`, defaults to 1 arcsecond The separation at which to match secondary targets to themselves in ARCSECONDS. darkbright : :class:`bool`, optional, defaults to ``False`` If sent, then split `NUMOBS_INIT` and `PRIORITY_INIT` into `NUMOBS_INIT_DARK`, `NUMOBS_INIT_BRIGHT`, `PRIORITY_INIT_DARK` and `PRIORITY_INIT_BRIGHT` and calculate values appropriate to "BRIGHT" and "DARK|GRAY" observing conditions. Returns ------- :class:`~numpy.ndarray` The array of secondary targets, with the `TARGETID` bit updated to be unique and reasonable and the `SCND_TARGET` column renamed based on the flavor of `scnd_mask`. Notes ----- - Secondaries without `OVERRIDE` are also matched to themselves Such matches are given the same `TARGETID` (that of the primary if they match a primary) and the bitwise or of `SCND_TARGET` and `OBSCONDITIONS` bits across matches. The highest `PRIORITY_INIT` is retained, and others are set to -1. Only secondaries with priorities that are not -1 are written to the main file. If multiple matching secondary targets have the same (highest) priority, the first one encountered retains its `PRIORITY_INIT` - The secondary `TARGETID` is designed to be reproducible. It combines `BRICKID` based on location, `OBJID` based on the order of the targets in the secondary file (`SCND_ORDER`) and `RELEASE` from the secondary bit number (`SCND_TARGET`) and the input `survey`. `RELEASE` is set to ((X-1)*100)+np.log2(scnd_bit) with X from the `survey` string survey=svX and scnd_bit from `SCND_TARGET`. For the main survey (survey="main") X-1 is 5. """ # ADM assign new TARGETIDs to targets without a primary match. nomatch = scxtargs["TARGETID"] == -1 # ADM get the BRICKIDs for each source. brxid = bricks.brickid(scxtargs["RA"], scxtargs["DEC"]) # ADM ensure unique secondary bits for different iterations of SV # ADM and the Main Survey. if survey == 'main': Xm1 = 5 elif survey[0:2] == 'sv': # ADM the re.search just extracts the numbers in the string. Xm1 = int(re.search(r'\d+', survey).group()) - 1 # ADM we've allowed a max of up to sv5 (!). Fail if surpassed. if Xm1 >= 5: msg = "Only coded for up to 'sv5', not {}!!!".format(survey) log.critical(msg) raise ValueError(msg) else: msg = "allowed surveys: 'main', 'svX', not {}!!!".format(survey) log.critical(msg) raise ValueError(msg) # ADM the RELEASE for each source is the `SCND_TARGET` bit NUMBER. release = (Xm1 * 100) + np.log2(scxtargs["SCND_TARGET_INIT"]).astype('int') # ADM build the OBJIDs based on the values of SCND_ORDER. t0 = time() log.info("Begin assigning OBJIDs to bricks...") # ADM So as not to overwhelm the bit-limits for OBJID # ADM rank by SCND_ORDER for each brick and bit combination. # ADM First, create a unique ID based on brxid and release. scnd_order = scxtargs["SCND_ORDER"] sorter = (1000 * brxid) + release # ADM sort the unique IDs and split based on where they change. argsort = np.argsort(sorter) w = np.where(np.diff(sorter[argsort]))[0] soperbrxbit = np.split(scnd_order[argsort], w + 1) # ADM loop through each (brxid, release) and sort on scnd_order. # ADM double argsort returns the ascending ranked order of the entry # ADM (whereas a single argsort returns the indexes for ordering). sortperbrxbit = [np.argsort(np.argsort(so)) for so in soperbrxbit] # ADM finally unroll the (brxid, release) combinations... sortedobjid = np.array(list(itertools.chain.from_iterable(sortperbrxbit))) # ADM ...and reorder based on the initial argsort. objid = np.zeros_like(sortedobjid) - 1 objid[argsort] = sortedobjid log.info("Assigned OBJIDs to bricks in {:.1f}s".format(time() - t0)) # ADM check that the objid array was entirely populated. assert np.all(objid != -1) # ADM assemble the TARGETID, SCND objects have RELEASE==0. targetid = encode_targetid(objid=objid, brickid=brxid, release=release) # ADM a check that the generated TARGETIDs are unique. if len(set(targetid)) != len(targetid): msg = "duplicate TARGETIDs generated for secondary targets!!!" log.critical(msg) raise ValueError(msg) # ADM assign the unique TARGETIDs to the secondary objects. scxtargs["TARGETID"][nomatch] = targetid[nomatch] log.debug("Assigned {} targetids to unmatched secondaries".format( len(targetid[nomatch]))) # ADM match secondaries to themselves, to ensure duplicates # ADM share a TARGETID. Don't match special (OVERRIDE) targets # ADM or sources that have already been matched to a primary. w = np.where(~scxtargs["OVERRIDE"] & nomatch)[0] if len(w) > 0: log.info("Matching secondary targets to themselves...t={:.1f}s".format( time() - t0)) # ADM use astropy for the matching. At NERSC, astropy matches # ADM ~20M objects to themselves in about 10 minutes. c = SkyCoord(scxtargs["RA"][w] * u.deg, scxtargs["DEC"][w] * u.deg) m1, m2, _, _ = c.search_around_sky(c, sep * u.arcsec) log.info("Done with matching...t={:.1f}s".format(time() - t0)) # ADM restrict only to unique matches (and exclude self-matches). uniq = m1 > m2 m1, m2 = m1[uniq], m2[uniq] # ADM set same TARGETID for any matches. m2 must come first, here. scxtargs["TARGETID"][w[m2]] = scxtargs["TARGETID"][w[m1]] # ADM Ensure secondary targets with matching TARGETIDs have all the # ADM relevant SCND_TARGET bits set. By definition, targets with # ADM OVERRIDE set never have matching TARGETIDs. wnoov = np.where(~scxtargs["OVERRIDE"])[0] if len(wnoov) > 0: for _, inds in duplicates(scxtargs["TARGETID"][wnoov]): scnd_targ = 0 for ind in inds: scnd_targ |= scxtargs["SCND_TARGET"][wnoov[ind]] scxtargs["SCND_TARGET"][wnoov[inds]] = scnd_targ log.info("Done checking SCND_TARGET...t={:.1f}s".format(time() - t0)) # ADM change the data model depending on whether the mask # ADM is an SVX (X = 1, 2, etc.) mask or not. Nothing will # ADM change if the mask has no preamble. prepend = scnd_mask._name[:-9].upper() scxtargs = rfn.rename_fields(scxtargs, {'SCND_TARGET': prepend + 'SCND_TARGET'}) # APC same thing for DESI_TARGET scxtargs = rfn.rename_fields(scxtargs, {'DESI_TARGET': prepend + 'DESI_TARGET'}) # APC Remove duplicate targetids from secondary-only targets alldups = [] for _, dups in duplicates(scxtargs['TARGETID']): # Retain the duplicate with highest priority, breaking ties # on lowest index in list of duplicates dups = np.delete(dups, np.argmax(scxtargs['PRIORITY_INIT'][dups])) alldups.append(dups) alldups = np.hstack(alldups) log.debug( "Flagging {} duplicate secondary targetids with PRIORITY_INIT=-1". format(len(alldups))) # ADM and remove the INIT fields in prep for a dark/bright split. scxtargs = rfn.drop_fields(scxtargs, ["PRIORITY_INIT", "NUMOBS_INIT"]) # ADM set initial priorities, numobs and obsconditions for both # ADM BRIGHT and DARK|GRAY conditions, if requested. nscx = len(scxtargs) nodata = np.zeros(nscx, dtype='int') - 1 if darkbright: ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"] else: ender, obscon = [""], ["DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"] cols, vals, forms = [], [], [] for edr, oc in zip(ender, obscon): cols += ["{}_INIT{}".format(pn, edr) for pn in ["PRIORITY", "NUMOBS"]] vals += [nodata, nodata] forms += ['>i8', '>i8'] # ADM write the output array. newdt = [dt for dt in zip(cols, forms)] done = np.array(np.zeros(nscx), dtype=scxtargs.dtype.descr + newdt) for col in scxtargs.dtype.names: done[col] = scxtargs[col] for col, val in zip(cols, vals): done[col] = val # ADM add the actual PRIORITY/NUMOBS values. for edr, oc in zip(ender, obscon): pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr done[pc], done[nc] = initial_priority_numobs(done, obscon=oc, scnd=True) # APC Flagged duplicates are removed in io.write_secondary done[pc][alldups] = -1 # APC add secondary flag in DESI_TARGET cols, mx, surv = main_cmx_or_sv(done, scnd=True) done[cols[0]] = mx[0]['SCND_ANY'] # ADM set the OBSCONDITIONS. done["OBSCONDITIONS"] = set_obsconditions(done, scnd=True) return done
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None): """Match secondary targets to primary targets and update bits. Parameters ---------- primtargs : :class:`~numpy.ndarray` An array of primary targets. scndout : :class`~numpy.ndarray` Name of a sub-directory to which to write the information in `desitarget.secondary.outdatamodel` with `TARGETID` and (the highest) `PRIORITY_INIT` updated with matching primary info. scxdir : :class:`str`, optional, defaults to `None` Name of the directory that hosts secondary targets. sep : :class:`float`, defaults to 1 arcsecond The separation at which to match in ARCSECONDS. pix : :class:`list`, optional, defaults to `None` Limit secondary targets to (NESTED) HEALpixels that touch pix at the supplied `nside`, as a speed-up. nside : :class:`int`, optional, defaults to `None` The (NESTED) HEALPixel nside to be used with `pixlist`. Returns ------- :class:`~numpy.ndarray` The array of primary targets, with the `SCND_TARGET` bit populated for matches to secondary targets """ # ADM add a SCND_TARGET column to the primary targets. dt = primtargs.dtype.descr dt.append(('SCND_TARGET', '>i8')) targs = np.zeros(len(primtargs), dtype=dt) for col in primtargs.dtype.names: targs[col] = primtargs[col] # ADM check if this is an SV or main survey file. cols, mx, surv = main_cmx_or_sv(targs, scnd=True) log.info('running on the {} survey...'.format(surv)) if surv != 'main': scxdir = os.path.join(scxdir, surv) # ADM read in non-OVERRIDE secondary targets. scxtargs = read_files(scxdir, mx[3]) scxtargs = scxtargs[~scxtargs["OVERRIDE"]] # ADM match primary targets to non-OVERRIDE secondary targets. inhp = np.ones(len(scxtargs), dtype="?") # ADM as a speed-up, save memory by limiting the secondary targets # ADM to just HEALPixels that could touch the primary targets. if nside is not None and pix is not None: # ADM remember to grab adjacent pixels in case of edge effects. allpix = add_hp_neighbors(nside, pix) inhp = is_in_hp(scxtargs, nside, allpix) # ADM it's unlikely that the matching separation is comparable # ADM to the HEALPixel resolution, but guard against that anyway. halfpix = np.degrees(hp.max_pixrad(nside)) * 3600. if sep > halfpix: msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format( sep, halfpix) log.critical(msg) raise ValueError(msg) # ADM warn the user if the secondary and primary samples are "large". big = 500000 if np.sum(inhp) > big and len(primtargs) > big: log.warning('Large secondary (N={}) and primary (N={}) samples'.format( np.sum(inhp), len(primtargs))) log.warning('The code may run slowly') # ADM for each secondary target, determine if there is a match # ADM with a primary target. Note that sense is important, here # ADM (the primary targets must be passed first). log.info( 'Matching primary and secondary targets for {} at {}"...t={:.1f}s'. format(scndout, sep, time() - start)) mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep) # ADM recast the indices to the full set of secondary targets, # ADM instead of just those that were in the relevant HEALPixels. mscx = np.where(inhp)[0][mscx] # ADM loop through the matches and update the SCND_TARGET # ADM column in the primary target list. The np.unique is a # ADM speed-up to assign singular matches first. umtargs, inv, cnt = np.unique(mtargs, return_inverse=True, return_counts=True) # ADM number of times each primary target was matched, ordered # ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs. nmtargs = cnt[inv] # ADM assign anything with nmtargs = 1 directly. singular = nmtargs == 1 targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][ mscx[singular]] # ADM loop through things with nmtargs > 1 and combine the bits. for i in range(len((mtargs[~singular]))): targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][ mscx[~singular][i]] # ADM also assign the SCND_ANY bit to the primary targets. desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True) desi_mask = desimasks[0] targs[desicols[0]][umtargs] |= desi_mask.SCND_ANY # ADM rename the SCND_TARGET column, in case this is an SV file. targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]}) # APC Secondary target bits only affect PRIORITY, NUMOBS and # APC obsconditions for specific DESI_TARGET bits # APC See https://github.com/desihub/desitarget/pull/530 # APC Only consider primary targets with secondary bits set scnd_update = (targs[desicols[0]] & desi_mask['SCND_ANY']) != 0 if np.any(scnd_update): # APC Allow changes to primaries if the DESI_TARGET bitmask has # APC only the following bits set, in any combination. log.info( 'Testing if secondary targets can update {} matched primaries'. format(scnd_update.sum())) update_from_scnd_bits = desi_mask['SCND_ANY'] | desi_mask[ 'MWS_ANY'] | desi_mask['STD_BRIGHT'] | desi_mask[ 'STD_FAINT'] | desi_mask['STD_WD'] scnd_update &= ((targs[desicols[0]] & ~update_from_scnd_bits) == 0) log.info( 'Setting new priority, numobs and obsconditions from secondary for {} matched primaries' .format(scnd_update.sum())) # APC Primary and secondary obsconditions are or'd scnd_obscon = set_obsconditions(targs[scnd_update], scnd=True) targs['OBSCONDITIONS'][scnd_update] &= scnd_obscon # APC bit of a hack here # APC Check for _BRIGHT, _DARK split in column names darkbright = 'NUMOBS_INIT_DARK' in targs.dtype.names if darkbright: ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"] else: ender, obscon = [""], [ "DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18" ] # APC secondaries can increase priority and numobs for edr, oc in zip(ender, obscon): pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr scnd_priority, scnd_numobs = initial_priority_numobs( targs[scnd_update], obscon=oc, scnd=True) targs[nc][scnd_update] = np.maximum(targs[nc][scnd_update], scnd_numobs) targs[pc][scnd_update] = np.maximum(targs[pc][scnd_update], scnd_priority) # ADM update the secondary targets with the primary information. scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs] # ADM the maximum priority will be used to break ties in the # ADM unlikely event that a secondary matches two primaries. hipri = np.maximum(targs["PRIORITY_INIT_DARK"], targs["PRIORITY_INIT_BRIGHT"]) scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs] # ADM write the secondary targets that have updated TARGETIDs. ii = scxtargs["TARGETID"] != -1 nmatches = np.sum(ii) log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format( nmatches, scndout, time() - start)) if nmatches > 0: hdr = fitsio.FITSHDR() hdr["SURVEY"] = surv fitsio.write(scndout, scxtargs[ii], extname='SCND_TARG', header=hdr, clobber=True) log.info('Done...t={:.1f}s'.format(time() - start)) return targs
def main(args): log = get_logger() cmd = [ 'desi_compute_fluxcalibration', ] for key, value in args.__dict__.items(): if value is not None: cmd += ['--' + key, str(value)] cmd = ' '.join(cmd) log.info(cmd) log.info("read frame") # read frame frame = read_frame(args.infile) # Set fibermask flagged spectra to have 0 flux and variance frame = get_fiberbitmasked_frame(frame, bitmask='flux', ivar_framemask=True) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models( args.models) ok = np.ones(len(model_metadata), dtype=bool) if args.chi2cut > 0: log.info("Apply cut CHI2DOF<{}".format(args.chi2cut)) ok &= (model_metadata["CHI2DOF"] < args.chi2cut) if args.delta_color_cut > 0: log.info("Apply cut |delta color|<{}".format(args.delta_color_cut)) ok &= (np.abs(model_metadata["MODEL_G-R"] - model_metadata["DATA_G-R"]) < args.delta_color_cut) if args.min_color is not None: log.info("Apply cut DATA_G-R>{}".format(args.min_color)) ok &= (model_metadata["DATA_G-R"] > args.min_color) if args.chi2cut_nsig > 0: # automatically reject stars that ar chi2 outliers mchi2 = np.median(model_metadata["CHI2DOF"]) rmschi2 = np.std(model_metadata["CHI2DOF"]) maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2 log.info("Apply cut CHI2DOF<{} based on chi2cut_nsig={}".format( maxchi2, args.chi2cut_nsig)) ok &= (model_metadata["CHI2DOF"] <= maxchi2) ok = np.where(ok)[0] if ok.size == 0: log.error("cuts discarded all stars") sys.exit(12) nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning("discarding %d star(s) out of %d because of cuts" % (nbad, nstars)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] # check that the model_fibers are actually standard stars fibermap = frame.fibermap ## check whether star fibers from args.models are consistent with fibers from fibermap ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit fibermap_std_indices = np.where(isStdStar(fibermap))[0] if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)): target_colnames, target_masks, survey = main_cmx_or_sv(fibermap) colname = target_colnames[0] for i in model_fibers % 500: log.error( "inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap" .format(i, fibermap["OBJTYPE"][i], colname, fibermap[colname][i])) sys.exit(12) # Make sure the fibers of interest aren't entirely masked. if np.sum( np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) == frame.nwave) == len(model_fibers): log.warning('All standard-star spectra are masked!') return fluxcalib = compute_flux_calibration( frame, model_wave, model_flux, model_fibers % 500, highest_throughput_nstars=args.highest_throughput) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame_meta=frame.meta, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s" % args.outfile)
def get_source_types(fibermap): ''' Return a list of specsim source types based upon fibermap['DESI_TARGET'] Args: fibermap: fibermap Table including DESI_TARGET column Returns array of source_types 'sky', 'elg', 'lrg', 'qso', 'star' Unassigned fibers fibermap['TARGETID'] == -1 will be treated as 'sky' If fibermap.meta['FLAVOR'] = 'arc' or 'flat', returned source types will match that flavor, though specsim doesn't use those as source_types TODO: specsim/desimodel doesn't have a fiber input loss model for BGS yet, so BGS targets get source_type = 'lrg' (!) ''' from desiutil.log import get_logger log = get_logger() if ('DESI_TARGET' not in fibermap.dtype.names) and \ ('SV1_DESI_TARGET' not in fibermap.dtype.names): log.warning( "(SV1_)DESI_TARGET not in fibermap table; using source_type='star' for everything" ) return np.array([ 'star', ] * len(fibermap)) target_colnames, target_masks, survey = main_cmx_or_sv(fibermap) targetcol = target_colnames[0] #- DESI_TARGET or SV1_DESI_TARGET tm = target_masks[0] #- desi_mask or sv1_desi_mask source_type = np.zeros(len(fibermap), dtype='U4') assert np.all(source_type == '') if 'TARGETID' in fibermap.dtype.names: unassigned = fibermap['TARGETID'] == -1 source_type[unassigned] = 'sky' source_type[(fibermap['OBJTYPE'] == 'FLT')] = 'FLAT' source_type[(fibermap['OBJTYPE'] == 'ARC')] = 'ARC' source_type[(fibermap[targetcol] & tm.SKY) != 0] = 'sky' source_type[(fibermap[targetcol] & tm.ELG) != 0] = 'elg' source_type[(fibermap[targetcol] & tm.LRG) != 0] = 'lrg' source_type[(fibermap[targetcol] & tm.QSO) != 0] = 'qso' source_type[(fibermap[targetcol] & tm.BGS_ANY) != 0] = 'bgs' starmask = 0 for name in [ 'STD', 'STD_FSTAR', 'STD_WD', 'MWS_ANY', 'STD_FAINT', 'STD_FAINT_BEST', 'STD_BRIGHT', 'STD_BRIGHT_BEST' ]: if name in desitarget.targetmask.desi_mask.names(): starmask |= desitarget.targetmask.desi_mask[name] source_type[(fibermap[targetcol] & starmask) != 0] = 'star' #- Simulate unassigned fibers as sky ## TODO: when fiberassign and desitarget are updated, use ## desitarget.targetmask.desi_mask.NO_TARGET to identify these source_type[fibermap['TARGETID'] < 0] = 'sky' assert not np.any(source_type == '') for name in sorted(np.unique(source_type)): n = np.count_nonzero(source_type == name) log.debug('{} {} targets'.format(name, n)) return source_type
def make_mtl(targets, obscon, zcat=None, scnd=None, trim=False, trimcols=False, trimtozcat=False): """Adds fiberassign and zcat columns to a targets table. Parameters ---------- targets : :class:`~numpy.array` or `~astropy.table.Table` A numpy rec array or astropy Table with at least the columns ``TARGETID``, ``DESI_TARGET``, ``NUMOBS_INIT``, ``PRIORITY_INIT``. or the corresponding columns for SV or commissioning. obscon : :class:`str` A combination of strings that are in the desitarget bitmask yaml file (specifically in `desitarget.targetmask.obsconditions`), e.g. "DARK|GRAY". Governs the behavior of how priorities are set based on "obsconditions" in the desitarget bitmask yaml file. zcat : :class:`~astropy.table.Table`, optional Redshift catalog table with columns ``TARGETID``, ``NUMOBS``, ``Z``, ``ZWARN``. scnd : :class:`~numpy.array`, `~astropy.table.Table`, optional A set of secondary targets associated with the `targets`. As with the `target` must include at least ``TARGETID``, ``NUMOBS_INIT``, ``PRIORITY_INIT`` or the corresponding SV columns. The secondary targets will be padded to have the same columns as the targets, and concatenated with them. trim : :class:`bool`, optional If ``True`` (default), don't include targets that don't need any more observations. If ``False``, include every input target. trimcols : :class:`bool`, optional, defaults to ``False`` Only pass through columns in `targets` that are actually needed for fiberassign (see `desitarget.mtl.mtldatamodel`). trimtozcat : :class:`bool`, optional, defaults to ``False`` Only return targets that have been UPDATED (i.e. the targets with a match in `zcat`). Returns all targets if `zcat` is ``None``. Returns ------- :class:`~astropy.table.Table` MTL Table with targets columns plus: * NUMOBS_MORE - number of additional observations requested * PRIORITY - target priority (larger number = higher priority) * TARGET_STATE - the observing state that corresponds to PRIORITY * OBSCONDITIONS - replaces old GRAYLAYER * TIMESTAMP - time that (this) make_mtl() function was run * VERSION - version of desitarget used to run make_mtl() """ start = time() # ADM set up the default logger. from desiutil.log import get_logger log = get_logger() # ADM if trimcols was passed, reduce input target columns to minimal. if trimcols: mtldm = switch_main_cmx_or_sv(mtldatamodel, targets) cullcols = list(set(targets.dtype.names) - set(mtldm.dtype.names)) if isinstance(targets, Table): targets.remove_columns(cullcols) else: targets = rfn.drop_fields(targets, cullcols) # ADM determine whether the input targets are main survey, cmx or SV. colnames, masks, survey = main_cmx_or_sv(targets, scnd=True) # ADM set the first column to be the "desitarget" column desi_target, desi_mask = colnames[0], masks[0] scnd_target = colnames[-1] # ADM if secondaries were passed, concatenate them with the targets. if scnd is not None: nrows = len(scnd) log.info('Pad {} primary targets with {} secondaries...t={:.1f}s'.format( len(targets), nrows, time()-start)) padit = np.zeros(nrows, dtype=targets.dtype) sharedcols = set(targets.dtype.names).intersection(set(scnd.dtype.names)) for col in sharedcols: padit[col] = scnd[col] targets = np.concatenate([targets, padit]) # APC Propagate a flag on which targets came from scnd is_scnd = np.repeat(False, len(targets)) is_scnd[-nrows:] = True log.info('Done with padding...t={:.1f}s'.format(time()-start)) # Trim targets from zcat that aren't in original targets table. if zcat is not None: ok = np.in1d(zcat['TARGETID'], targets['TARGETID']) num_extra = np.count_nonzero(~ok) if num_extra > 0: log.warning("Ignoring {} zcat entries that aren't " "in the input target list".format(num_extra)) zcat = zcat[ok] n = len(targets) # ADM if a redshift catalog was passed, order it to match the input targets # ADM catalog on 'TARGETID'. if zcat is not None: # ADM there might be a quicker way to do this? # ADM set up a dictionary of the indexes of each target id. d = dict(tuple(zip(targets["TARGETID"], np.arange(n)))) # ADM loop through the zcat and look-up the index in the dictionary. zmatcher = np.array([d[tid] for tid in zcat["TARGETID"]]) ztargets = zcat if ztargets.masked: unobs = ztargets['NUMOBS'].mask ztargets['NUMOBS'][unobs] = 0 unobsz = ztargets['Z'].mask ztargets['Z'][unobsz] = -1 unobszw = ztargets['ZWARN'].mask ztargets['ZWARN'][unobszw] = -1 else: ztargets = Table() ztargets['TARGETID'] = targets['TARGETID'] ztargets['NUMOBS'] = np.zeros(n, dtype=np.int32) ztargets['Z'] = -1 * np.ones(n, dtype=np.float32) ztargets['ZWARN'] = -1 * np.ones(n, dtype=np.int32) # ADM if zcat wasn't passed, there is a one-to-one correspondence # ADM between the targets and the zcat. zmatcher = np.arange(n) # ADM extract just the targets that match the input zcat. targets_zmatcher = targets[zmatcher] # ADM update the number of observations for the targets. ztargets['NUMOBS_MORE'] = calc_numobs_more(targets_zmatcher, ztargets, obscon) # ADM assign priorities. Only things in the zcat can have changed # ADM priorities. Anything else is assigned PRIORITY_INIT, below. priority, target_state = calc_priority( targets_zmatcher, ztargets, obscon, state=True) # If priority went to 0==DONOTOBSERVE or 1==OBS or 2==DONE, then # NUMOBS_MORE should also be 0. # ## mtl['NUMOBS_MORE'] = ztargets['NUMOBS_MORE'] ii = (priority <= 2) log.info('{:d} of {:d} targets have priority zero, setting N_obs=0.'.format( np.sum(ii), n)) ztargets['NUMOBS_MORE'][ii] = 0 # - Set the OBSCONDITIONS mask for each target bit. obsconmask = set_obsconditions(targets) # APC obsconmask will now be incorrect for secondary-only targets. Fix this # APC using the mask on secondary targets. if scnd is not None: obsconmask[is_scnd] = set_obsconditions(targets[is_scnd], scnd=True) # ADM set up the output mtl table. mtl = Table(targets) mtl.meta['EXTNAME'] = 'MTL' # ADM add a placeholder for the secondary bit-mask, if it isn't there. if scnd_target not in mtl.dtype.names: mtl[scnd_target] = np.zeros(len(mtl), dtype=mtldatamodel["SCND_TARGET"].dtype) # ADM initialize columns to avoid zero-length/missing/format errors. zcols = ["NUMOBS_MORE", "NUMOBS", "Z", "ZWARN"] for col in zcols + ["TARGET_STATE", "TIMESTAMP", "VERSION"]: mtl[col] = np.empty(len(mtl), dtype=mtldatamodel[col].dtype) # ADM any target that wasn't matched to the ZCAT should retain its # ADM original (INIT) value of PRIORITY and NUMOBS. mtl['NUMOBS_MORE'] = mtl['NUMOBS_INIT'] mtl['PRIORITY'] = mtl['PRIORITY_INIT'] mtl['TARGET_STATE'] = "UNOBS" # ADM add the time and version of the desitarget code that was run. utc = datetime.utcnow().isoformat(timespec='seconds') mtl["TIMESTAMP"] = utc mtl["VERSION"] = dt_version # ADM now populate the new mtl columns with the updated information. mtl['OBSCONDITIONS'] = obsconmask mtl['PRIORITY'][zmatcher] = priority mtl['TARGET_STATE'][zmatcher] = target_state for col in zcols: mtl[col][zmatcher] = ztargets[col] # Filter out any targets marked as done. if trim: notdone = mtl['NUMOBS_MORE'] > 0 log.info('{:d} of {:d} targets are done, trimming these'.format( len(mtl) - np.sum(notdone), len(mtl)) ) mtl = mtl[notdone] # Filtering can reset the fill_value, which is just wrong wrong wrong # See https://github.com/astropy/astropy/issues/4707 # and https://github.com/astropy/astropy/issues/4708 mtl['NUMOBS_MORE'].fill_value = -1 # ADM assert the data model is complete. # ADM turning this off for now, useful for testing. # mtltypes = [mtl[i].dtype.type for i in mtl.dtype.names] # mtldmtypes = [mtldm[i].dtype.type for i in mtl.dtype.names] # assert set(mtl.dtype.names) == set(mtldm.dtype.names) # assert mtltypes == mtldmtypes log.info('Done...t={:.1f}s'.format(time()-start)) if trimtozcat: return mtl[zmatcher] return mtl
def test_priorities(self): """Test that priorities are set correctly for both the main survey and SV. """ # ADM loop through once for SV and once for the main survey. for prefix in ["", "SV1_"]: t = self.targets.copy() z = self.zcat.copy() main_names = ['DESI_TARGET', 'BGS_TARGET', 'MWS_TARGET'] for name in main_names: t.rename_column(name, prefix + name) # ADM retrieve the mask and column names for this survey flavor. colnames, masks, _ = main_cmx_or_sv(t) desi_target, bgs_target, mws_target = colnames desi_mask, bgs_mask, mws_mask = masks # - No targeting bits set is priority=0 self.assertTrue(np.all(calc_priority(t, z) == 0)) # - test QSO > (LRG_1PASS | LRG_2PASS) > ELG t[desi_target] = desi_mask.ELG self.assertTrue( np.all( calc_priority(t, z) == desi_mask.ELG.priorities['UNOBS'])) t[desi_target] |= desi_mask.LRG_1PASS self.assertTrue( np.all( calc_priority(t, z) == desi_mask.LRG.priorities['UNOBS'])) t[desi_target] |= desi_mask.LRG_2PASS self.assertTrue( np.all( calc_priority(t, z) == desi_mask.LRG.priorities['UNOBS'])) t[desi_target] |= desi_mask.QSO self.assertTrue( np.all( calc_priority(t, z) == desi_mask.QSO.priorities['UNOBS'])) # - different states -> different priorities # - Done is Done, regardless of ZWARN. t[desi_target] = desi_mask.ELG t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 1, 1] z['ZWARN'] = [1, 1, 0] p = make_mtl(t, z)["PRIORITY"] self.assertEqual(p[0], desi_mask.ELG.priorities['UNOBS']) self.assertEqual(p[1], desi_mask.ELG.priorities['DONE']) self.assertEqual(p[2], desi_mask.ELG.priorities['DONE']) # - BGS FAINT targets are never DONE, only MORE_ZGOOD. t[desi_target] = desi_mask.BGS_ANY t[bgs_target] = bgs_mask.BGS_FAINT t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 1, 1] z['ZWARN'] = [1, 1, 0] p = make_mtl(t, z)["PRIORITY"] self.assertEqual(p[0], bgs_mask.BGS_FAINT.priorities['UNOBS']) self.assertEqual(p[1], bgs_mask.BGS_FAINT.priorities['MORE_ZWARN']) self.assertEqual(p[2], bgs_mask.BGS_FAINT.priorities['MORE_ZGOOD']) # BGS_FAINT: {UNOBS: 2000, MORE_ZWARN: 2000, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} # - BGS BRIGHT targets are never DONE, only MORE_ZGOOD. t[desi_target] = desi_mask.BGS_ANY t[bgs_target] = bgs_mask.BGS_BRIGHT t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 1, 1] z['ZWARN'] = [1, 1, 0] p = make_mtl(t, z)["PRIORITY"] self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS']) self.assertEqual(p[1], bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN']) self.assertEqual(p[2], bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD']) # BGS_BRIGHT: {UNOBS: 2100, MORE_ZWARN: 2100, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} # BGS targets are NEVER done even after 100 observations t[desi_target] = desi_mask.BGS_ANY t[bgs_target] = bgs_mask.BGS_BRIGHT t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 100, 100] z['ZWARN'] = [1, 1, 0] p = calc_priority(t, z) self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS']) self.assertEqual(p[1], bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN']) self.assertEqual(p[2], bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD']) # BGS ZGOOD targets always have lower priority than MWS targets that # are not DONE. # ADM first discard N/S informational bits from bitmask as these # ADM should never trump the other bits. bgs_names = [ name for name in bgs_mask.names() if 'NORTH' not in name and 'SOUTH' not in name ] mws_names = [ name for name in mws_mask.names() if 'NORTH' not in name and 'SOUTH' not in name ] lowest_bgs_priority_zgood = min( [bgs_mask[n].priorities['MORE_ZGOOD'] for n in bgs_names]) lowest_mws_priority_unobs = min( [mws_mask[n].priorities['UNOBS'] for n in mws_names]) lowest_mws_priority_zwarn = min( [mws_mask[n].priorities['MORE_ZWARN'] for n in mws_names]) lowest_mws_priority_zgood = min( [mws_mask[n].priorities['MORE_ZGOOD'] for n in mws_names]) lowest_mws_priority = min(lowest_mws_priority_unobs, lowest_mws_priority_zwarn, lowest_mws_priority_zgood) self.assertLess(lowest_bgs_priority_zgood, lowest_mws_priority)
def match_secondary(infile, scxtargs, sep=1., scxdir=None): """Match secondary targets to primary targets and update bits. Parameters ---------- infile : :class:`str` The full path to a file containing primary targets. scxtargs : :class:`~numpy.ndarray` An array of secondary targets. sep : :class:`float`, defaults to 1 arcsecond The separation at which to match in ARCSECONDS. scxdir : :class:`str`, optional, defaults to `None` Name of the directory that hosts secondary targets. If passed, this is written to the output primary file header as `SCNDDIR`. Returns ------- :class:`~numpy.ndarray` The array of secondary targets, with the `TARGETID` bit updated with any matching primary targets from `infile`. Notes ----- - The primary target `infiles` are written back to their original path with `.fits` changed to `-wscnd.fits` and the `SCND_TARGET` bit populated for matching targets. """ # ADM just the file name for logging. fn = os.path.basename(infile) # ADM read in the primary targets. log.info('Reading primary targets file {}...t={:.1f}s'.format( infile, time() - start)) intargs, hdr = fitsio.read(infile, extension="TARGETS", header=True) # ADM fail if file's already been matched to secondary targets. if "SCNDDIR" in hdr: msg = "{} already matched to secondary targets".format(fn) \ + " (did you mean to remove {}?)!!!".format(fn) log.critical(msg) raise ValueError(msg) # ADM add the SCNDDIR to the primary targets file header. hdr["SCNDDIR"] = scxdir # ADM add a SCND_TARGET column to the primary targets. dt = intargs.dtype.descr dt.append(('SCND_TARGET', '>i8')) targs = np.zeros(len(intargs), dtype=dt) for col in intargs.dtype.names: targs[col] = intargs[col] # ADM match to all secondary targets for non-custom primary files. inhp = np.ones(len(scxtargs), dtype="?") # ADM as a speed-up, save memory by limiting the secondary targets # ADM to just HEALPixels that could touch the primary targets. if 'FILEHPX' in hdr: nside, pix = hdr['FILENSID'], hdr['FILEHPX'] # ADM remember to grab adjacent pixels in case of edge effects. allpix = add_hp_neighbors(nside, pix) inhp = is_in_hp(scxtargs, nside, allpix) # ADM it's unlikely that the matching separation is comparable # ADM to the HEALPixel resolution, but guard against that anyway. halfpix = np.degrees(hp.max_pixrad(nside)) * 3600. if sep > halfpix: msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format( sep, halfpix) log.critical(msg) raise ValueError(msg) # ADM warn the user if the secondary and primary samples are "large". big = 500000 if np.sum(inhp) > big and len(intargs) > big: log.warning('Large secondary (N={}) and primary (N={}) samples'.format( np.sum(inhp), len(intargs))) log.warning('The code may run slowly') # ADM for each secondary target, determine if there is a match # ADM with a primary target. Note that sense is important, here # ADM (the primary targets must be passed first). log.info( 'Matching primary and secondary targets for {} at {}"...t={:.1f}s'. format(fn, sep, time() - start)) mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep) # ADM recast the indices to the full set of secondary targets, # ADM instead of just those that were in the relevant HEALPixels. mscx = np.where(inhp)[0][mscx] # ADM loop through the matches and update the SCND_TARGET # ADM column in the primary target list. The np.unique is a # ADM speed-up to assign singular matches first. umtargs, inv, cnt = np.unique(mtargs, return_inverse=True, return_counts=True) # ADM number of times each primary target was matched, ordered # ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs. nmtargs = cnt[inv] # ADM assign anything with nmtargs = 1 directly. singular = nmtargs == 1 targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][ mscx[singular]] # ADM loop through things with nmtargs > 1 and combine the bits. for i in range(len((mtargs[~singular]))): targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][ mscx[~singular][i]] # ADM also assign the SCND_ANY bit to the primary targets. desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True) targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY # ADM rename the SCND_TARGET column, in case this is an SV file. targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]}) # ADM update the secondary targets with the primary TARGETID. scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs] # ADM form the output primary file name and write the file. base, ext = os.path.splitext(infile) outfile = "{}{}{}".format(base, '-wscnd', ext) log.info('Writing updated primary targets to {}...t={:.1f}s'.format( outfile, time() - start)) fitsio.write(outfile, targs, extname='TARGETS', header=hdr, clobber=True) log.info('Done for {}...t={:.1f}s'.format(fn, time() - start)) return scxtargs
def make_mtl(targets, zcat=None, trim=False): """Adds NUMOBS, PRIORITY, and OBSCONDITIONS columns to a targets table. Parameters ---------- targets : :class:`~numpy.array` or `~astropy.table.Table` A numpy rec array or astropy Table with at least the columns ``TARGETID``, ``DESI_TARGET``, ``NUMOBS_INIT``, ``PRIORITY_INIT``. or the corresponding columns for SV or commissioning. zcat : :class:`~astropy.table.Table`, optional Redshift catalog table with columns ``TARGETID``, ``NUMOBS``, ``Z``, ``ZWARN``. trim : :class:`bool`, optional If ``True`` (default), don't include targets that don't need any more observations. If ``False``, include every input target. Returns ------- :class:`~astropy.table.Table` MTL Table with targets columns plus: * NUMOBS_MORE - number of additional observations requested * PRIORITY - target priority (larger number = higher priority) * OBSCONDITIONS - replaces old GRAYLAYER """ # ADM set up the default logger. from desiutil.log import get_logger log = get_logger() # ADM determine whether the input targets are main survey, cmx or SV. colnames, masks, survey = main_cmx_or_sv(targets) # ADM set the first column to be the "desitarget" column desi_target, desi_mask = colnames[0], masks[0] # Trim targets from zcat that aren't in original targets table if zcat is not None: ok = np.in1d(zcat['TARGETID'], targets['TARGETID']) num_extra = np.count_nonzero(~ok) if num_extra > 0: log.warning("Ignoring {} zcat entries that aren't " "in the input target list".format(num_extra)) zcat = zcat[ok] n = len(targets) # ADM if the input target columns were incorrectly called NUMOBS or PRIORITY # ADM rename them to NUMOBS_INIT or PRIORITY_INIT. # ADM Note that the syntax is slightly different for a Table. for name in ['NUMOBS', 'PRIORITY']: if isinstance(targets, Table): try: targets.rename_column(name, name + '_INIT') except KeyError: pass else: targets.dtype.names = [ name + '_INIT' if col == name else col for col in targets.dtype.names ] # ADM if a redshift catalog was passed, order it to match the input targets # ADM catalog on 'TARGETID'. if zcat is not None: # ADM there might be a quicker way to do this? # ADM set up a dictionary of the indexes of each target id. d = dict(tuple(zip(targets["TARGETID"], np.arange(n)))) # ADM loop through the zcat and look-up the index in the dictionary. zmatcher = np.array([d[tid] for tid in zcat["TARGETID"]]) ztargets = zcat if ztargets.masked: unobs = ztargets['NUMOBS'].mask ztargets['NUMOBS'][unobs] = 0 unobsz = ztargets['Z'].mask ztargets['Z'][unobsz] = -1 unobszw = ztargets['ZWARN'].mask ztargets['ZWARN'][unobszw] = -1 else: ztargets = Table() ztargets['TARGETID'] = targets['TARGETID'] ztargets['NUMOBS'] = np.zeros(n, dtype=np.int32) ztargets['Z'] = -1 * np.ones(n, dtype=np.float32) ztargets['ZWARN'] = -1 * np.ones(n, dtype=np.int32) # ADM if zcat wasn't passed, there is a one-to-one correspondence # ADM between the targets and the zcat. zmatcher = np.arange(n) # ADM extract just the targets that match the input zcat. targets_zmatcher = targets[zmatcher] # ADM use passed value of NUMOBS_INIT instead of calling the memory-heavy calc_numobs. # ztargets['NUMOBS_MORE'] = np.maximum(0, calc_numobs(ztargets) - ztargets['NUMOBS']) ztargets['NUMOBS_MORE'] = np.maximum( 0, targets_zmatcher['NUMOBS_INIT'] - ztargets['NUMOBS']) # ADM we need a minor hack to ensure that BGS targets are observed once (and only once) # ADM every time, regardless of how many times they've previously been observed. # ADM I've turned this off for commissioning. Not sure if we'll keep it in general. if survey != 'cmx': ii = targets_zmatcher[desi_target] & desi_mask.BGS_ANY > 0 ztargets['NUMOBS_MORE'][ii] = 1 # ADM assign priorities, note that only things in the zcat can have changed priorities. # ADM anything else will be assigned PRIORITY_INIT, below. priority = calc_priority(targets_zmatcher, ztargets) # If priority went to 0==DONOTOBSERVE or 1==OBS or 2==DONE, then NUMOBS_MORE should also be 0. # ## mtl['NUMOBS_MORE'] = ztargets['NUMOBS_MORE'] ii = (priority <= 2) log.info( '{:d} of {:d} targets have priority zero, setting N_obs=0.'.format( np.sum(ii), n)) ztargets['NUMOBS_MORE'][ii] = 0 # - Set the OBSCONDITIONS mask for each target bit. obscon = set_obsconditions(targets) # ADM set up the output mtl table. mtl = Table(targets) mtl.meta['EXTNAME'] = 'MTL' # ADM any target that wasn't matched to the ZCAT should retain its # ADM original (INIT) value of PRIORITY and NUMOBS. mtl['NUMOBS_MORE'] = mtl['NUMOBS_INIT'] mtl['PRIORITY'] = mtl['PRIORITY_INIT'] # ADM now populate the new mtl columns with the updated information. mtl['OBSCONDITIONS'] = obscon mtl['PRIORITY'][zmatcher] = priority mtl['NUMOBS_MORE'][zmatcher] = ztargets['NUMOBS_MORE'] # Filter out any targets marked as done. if trim: notdone = mtl['NUMOBS_MORE'] > 0 log.info('{:d} of {:d} targets are done, trimming these'.format( len(mtl) - np.sum(notdone), len(mtl))) mtl = mtl[notdone] # Filtering can reset the fill_value, which is just wrong wrong wrong # See https://github.com/astropy/astropy/issues/4707 # and https://github.com/astropy/astropy/issues/4708 mtl['NUMOBS_MORE'].fill_value = -1 return mtl
def run(self, indir): '''TODO: document''' log = desiutil.log.get_logger() results = list() infiles = glob.glob(os.path.join(indir, 'qcframe-*.fits')) if len(infiles) == 0: log.error("no qcframe in {}".format(indir)) return None # find number of spectros spectros = [] for filename in infiles: hdr = fitsio.read_header(filename) s = int(hdr['CAMERA'][1]) spectros.append(s) spectros = np.unique(spectros) for spectro in spectros: infiles = glob.glob( os.path.join(indir, 'qcframe-?{}-*.fits'.format(spectro))) qframes = {} fmap = None for infile in infiles: qframe = read_qframe(infile) cam = qframe.meta["CAMERA"][0].upper() qframes[cam] = qframe if fmap is None: fmap = qframe.fibermap night = int(qframe.meta['NIGHT']) expid = int(qframe.meta['EXPID']) # need to decode fibermap columns, masks, survey = targets.main_cmx_or_sv(fmap) desi_target = fmap[columns[0]] # (SV1_)DESI_TARGET desi_mask = masks[0] # (sv1) desi_mask stars = ( desi_target & desi_mask.mask('STD_WD|STD_FAINT|STD_BRIGHT')) != 0 qsos = (desi_target & desi_mask.QSO) != 0 # sw = np.zeros(self.rwave.size) # swf = np.zeros(self.rwave.size) #- we need a ridiculously large array to cover the r filter, but precache #- what wavelength ranges matter for each band iiband = dict() for c in ["B", "R", "Z"]: if c in qframes: qframe = qframes[c] iiband[c] = (qframe.wave[0][0] < self.rwave) iiband[c] &= (self.rwave < qframe.wave[0][-1]) #- for each fiber, generate list of arguments to pass to get_dico #- get_fiber_data extracts data for only *one* fiber from qframes, fmap, which have data for all fibers #- this reduces the parallel processing overhead argslist = [(self, iiband, get_fiber_data(qframes, fmap, f, fiber, night, expid, spectro, stars, qsos)) for f, fiber in enumerate(fmap["FIBER"])] ncpu = get_ncpu(None) if ncpu > 1: pool = mp.Pool(ncpu) results = pool.starmap(get_dico, argslist) else: for args in argslist: results.append(get_dico(**args)) if len(results) == 0: return None return Table(results, names=results[0].keys())