def read_desi_spectra(fin): h = fitsio.FITS(fin) nbins = int((llmax - llmin) / dll) tids = h[1]["TARGETID"][:] nspec = len(tids) fl = np.zeros((nspec, nbins)) iv = np.zeros((nspec, nbins)) if nspec == 0: return None for band in ["B", "R", "Z"]: wave = h["{}_WAVELENGTH".format(band)].read() w = (np.log10(wave) > llmin) & (np.log10(wave) < llmax) wave = wave[w] bins = np.floor((np.log10(wave) - llmin) / dll).astype(int) fl_aux = h["{}_FLUX".format(band)].read()[:, w] iv_aux = h["{}_IVAR".format(band)].read()[:, w] for i in range(nspec): c = np.bincount(bins, weights=fl_aux[i] * iv_aux[i]) fl[i, :len(c)] += c c = np.bincount(bins, weights=iv_aux[i]) iv[i, :len(c)] += c w = iv > 0 fl[w] /= iv[w] fl = np.hstack((fl, iv)) ## select QSO targets: wqso = h[1]['DESI_TARGET'][:] & desi_mask.mask('QSO') wqso = wqso > 0 print("INFO: founds {} qso targets".format(wqso.sum())) fl = fl[wqso, :] tids = tids[wqso] return tids, fl
def read_desi_spectra(fin, ignore_quasar_mask=False): try: from desitarget import desi_mask quasar_mask = desi_mask.mask('QSO') except: print("WARN: can't load desi_mask, ignoring mask!") quasar_mask = 1 h = fitsio.FITS(fin) nbins = int((llmax - llmin) / dll) wqso = h[1]['DESI_TARGET'][:] & quasar_mask if ignore_quasar_mask: wqso |= 1 wqso = wqso > 0 print("INFO: found {} quasar targets".format(wqso.sum())) tids = h[1]["TARGETID"][:][wqso] utids = np.unique(tids) nspec = len(utids) fl = np.zeros((nspec, nbins)) iv = np.zeros((nspec, nbins)) if nspec == 0: return None for band in ["B", "R", "Z"]: wave = h["{}_WAVELENGTH".format(band)].read() w = (np.log10(wave) > llmin) & (np.log10(wave) < llmax) wave = wave[w] bins = np.floor((np.log10(wave) - llmin) / dll).astype(int) fl_aux = h["{}_FLUX".format(band)].read()[:, w] iv_aux = h["{}_IVAR".format(band)].read()[:, w] fl_aux = fl_aux[wqso] iv_aux = iv_aux[wqso] ivfl_aux = fl_aux * iv_aux for i, t in enumerate(tids): j = np.argwhere(utids == t)[0] c = np.bincount(bins, weights=ivfl_aux[i]) fl[j, :len(c)] += c c = np.bincount(bins, weights=iv_aux[i]) iv[j, :len(c)] += c w = iv > 0 fl[w] /= iv[w] fl = np.hstack((fl, iv)) print("INFO: founds {} good spectra".format(wqso.sum())) return utids, fl
def test_numobs(self): t = self.targets #- No target bits set is an error with self.assertRaises(ValueError): calc_numobs(t) #- ELGs and QSOs get one observation t['DESI_TARGET'] = desi_mask.ELG self.assertTrue(np.all(calc_numobs(t) == 1)) t['DESI_TARGET'] = desi_mask.QSO self.assertTrue(np.all(calc_numobs(t) == 4)) #- LRG numobs depends upon zflux (DECAM_FLUX index 4) t['DESI_TARGET'] = desi_mask.LRG nobs = calc_numobs(t) self.assertTrue(np.all(nobs == [1, 1, 2, 3, 3])) #- test astropy Table t = Table(t) nobs = calc_numobs(t) self.assertTrue(np.all(nobs == [1, 1, 2, 3, 3])) #- LRG numobs also works with ZFLUX instead of DECAM* t['ZFLUX'] = t['DECAM_FLUX'][:, 4] / t['DECAM_MW_TRANSMISSION'][:, 4] t.remove_column('DECAM_FLUX') t.remove_column('DECAM_MW_TRANSMISSION') nobs = calc_numobs(t) self.assertTrue(np.all(nobs == [1, 1, 2, 3, 3])) #- this is true even if other targeting bits are set t['DESI_TARGET'] |= desi_mask.mask('ELG|BGS_ANY') nobs = calc_numobs(t) self.assertTrue(np.all(nobs == [1, 1, 2, 3, 3])) #- But if no *FLUX available, default to LRGs with 2 obs t.remove_column('ZFLUX') nobs = calc_numobs(t) self.assertTrue(np.all(nobs == 2))
def test_numobs(self): t = self.targets #- default DESI_TARGET=0 should be no observations self.assertTrue(np.all(calc_numobs(t) == 0)) #- ELGs and QSOs get one observation t['DESI_TARGET'] = desi_mask.ELG self.assertTrue(np.all(calc_numobs(t) == 1)) t['DESI_TARGET'] = desi_mask.QSO self.assertTrue(np.all(calc_numobs(t) == 4)) #- LRG numobs depends upon zflux (DECAM_FLUX index 4) t['DESI_TARGET'] = desi_mask.LRG nobs = calc_numobs(t) self.assertTrue(np.all(nobs == [1, 1, 2, 3, 3])) #- test astropy Table t = Table(t) nobs = calc_numobs(t) self.assertTrue(np.all(nobs == [1, 1, 2, 3, 3])) #- LRG numobs also works with ZFLUX instead of DECAM* t['ZFLUX'] = t['DECAM_FLUX'][:,4] / t['DECAM_MW_TRANSMISSION'][:,4] t.remove_column('DECAM_FLUX') t.remove_column('DECAM_MW_TRANSMISSION') nobs = calc_numobs(t) self.assertTrue(np.all(nobs == [1, 1, 2, 3, 3])) #- this is true even if other targeting bits are set t['DESI_TARGET'] |= desi_mask.mask('ELG|BGS_ANY') nobs = calc_numobs(t) self.assertTrue(np.all(nobs == [1, 1, 2, 3, 3])) #- But if no *FLUX available, default to LRGs with 2 obs t.remove_column('ZFLUX') nobs = calc_numobs(t) self.assertTrue(np.all(nobs == 2))
INVARR = [] INVARZ = [] WAVB = [] WAVR = [] WAVZ = [] ID = [] print("Opening Files") for f in Folder_all: folder_list = os.listdir(Platedir + slash + f) for l in folder_list: filefolder_list = os.listdir(Platedir + slash + f + slash + l) for n in filefolder_list: if 'spectra' in n: c = Platedir + slash + f + slash + l + slash + n plate_ = fits.open(c, memmap=True) wqso = plate_[1].data['DESI_TARGET'][:] & desi_mask.mask('QSO') wqso = wqso > 0 Bin_info_ = plate_[1].data[wqso] Flux_B = plate_[3].data[wqso] Flux_R = plate_[8].data[wqso] Flux_Z = plate_[13].data[wqso] AND_B = plate_[5].data[wqso] AND_R = plate_[10].data[wqso] AND_Z = plate_[15].data[wqso] INV_B = plate_[4].data[wqso] INV_R = plate_[9].data[wqso] INV_Z = plate_[14].data[wqso] WAV_B = plate_[2].data WAV_R = plate_[7].data WAV_Z = plate_[12].data FluxB.append(Flux_B)
def qa_qso(targets, truth, qadir='.'): """Detailed QA plots for QSOs.""" dens = dict() these = np.where(targets['DESI_TARGET'] & desi_mask.mask('QSO') != 0)[0] dens['QSO_TARGETS'] = target_density(targets[these]) these = np.where((targets['DESI_TARGET'] & desi_mask.mask('QSO') != 0) * (truth['TRUEZ'] < 2.1))[0] dens['QSO_TRACER'] = target_density(targets[these]) these = np.where((targets['DESI_TARGET'] & desi_mask.mask('QSO') != 0) * (truth['TRUEZ'] >= 2.1))[0] dens['QSO_LYA'] = target_density(targets[these]) these = np.where( (targets['DESI_TARGET'] & desi_mask.mask('QSO') != 0) * (truth['CONTAM_TARGET'] & contam_mask.mask('QSO_IS_GALAXY')) != 0)[0] dens['QSO_IS_GALAXY'] = target_density(targets[these]) these = np.where( (targets['DESI_TARGET'] & desi_mask.mask('QSO') != 0) * (truth['CONTAM_TARGET'] & contam_mask.mask('QSO_IS_STAR')) != 0)[0] dens['QSO_IS_STAR'] = target_density(targets[these]) bins = 50 lim = (0, 280) fig, ax = plt.subplots(1, 2, figsize=(12, 5)) ax[0].hist(dens['QSO_TARGETS'], bins=bins, range=lim, label='All Targets', color='k', lw=1, histtype='step') ax[0].hist(dens['QSO_TRACER'], bins=bins, range=lim, alpha=0.6, ls='--', lw=2, label='Tracer QSOs') #, histtype='step') ax[0].hist(dens['QSO_LYA'], bins=bins, range=lim, lw=2, label='Lya QSOs') ax[0].set_ylabel('Number of Healpixels') ax[0].set_xlabel('Targets / deg$^2$') ax[0].set_title('True QSOs') ax[0].legend(loc='upper right') lim = (0, 100) #ax[1].hist(dens['QSO_TARGETS'], bins=bins, range=lim, label='All Targets', # color='k', lw=1, histtype='step') ax[1].hist(dens['QSO_IS_STAR'], bins=bins, range=lim, alpha=0.3, label='QSO_IS_STAR') ax[1].hist(dens['QSO_IS_GALAXY'], bins=bins, range=lim, alpha=0.5, label='QSO_IS_GALAXY') ax[1].set_ylabel('Number of Healpixels') ax[1].set_xlabel('Targets / deg$^2$') ax[1].set_title('QSO Contaminants') ax[1].legend(loc='upper right') pngfile = os.path.join(qadir, '{}_detail_density.png'.format('qso')) fig.savefig(pngfile) return pngfile
def qa_targets_truth(output_dir, verbose=True): """Generate QA plots from the joined targets and truth catalogs. time select_mock_targets --output_dir debug --qa """ import fitsio from desiutil.log import get_logger, DEBUG from desitarget import desi_mask, bgs_mask, mws_mask, contam_mask if verbose: log = get_logger(DEBUG) else: log = get_logger() qadir = os.path.join(output_dir, 'qa') if os.path.exists(qadir): if os.listdir(qadir): log.warning('Output directory {} is not empty.'.format(qadir)) else: log.info('Creating directory {}'.format(qadir)) os.makedirs(qadir) log.info('Writing to output QA directory {}'.format(qadir)) # Read the catalogs. targfile = os.path.join(output_dir, 'targets.fits') truthfile = os.path.join(output_dir, 'truth.fits') skyfile = os.path.join(output_dir, 'sky.fits') stddarkfile = os.path.join(output_dir, 'standards-dark.fits') stdbrightfile = os.path.join(output_dir, 'standards-bright.fits') cat = list() for ff in (targfile, truthfile, skyfile, stddarkfile, stdbrightfile): if os.path.exists(ff): log.info('Reading {}'.format(ff)) cat.append(fitsio.read(ff, ext=1, upper=True)) else: log.warning('File {} not found.'.format(ff)) cat.append(None) targets, truth, sky, stddark, stdbright = [cc for cc in cat] # Do some sanity checking of the catalogs. nobj, nsky, ndark, nbright = [ len(cc) for cc in (targets, sky, stddark, stdbright) ] if nobj != len(truth): log.fatal( 'Mismatch in the number of objects in targets.fits (N={}) and truth.fits (N={})!' .format(nobj, len(truth))) raise ValueError # Assign healpixels to estimate the area covered by the catalog. nside = 256 npix = hp.nside2npix(nside) areaperpix = hp.nside2pixarea(nside, degrees=True) pix = radec2pix(nside, targets['RA'], targets['DEC']) counts = np.bincount(pix, weights=None, minlength=npix) area = np.sum(counts > 10) * areaperpix log.info('Approximate area spanned by catalog = {:.2f} deg2'.format(area)) binarea = 1.0 htmlfile = os.path.join(qadir, 'index.html') log.info('Building {}'.format(htmlfile)) html = open(htmlfile, 'w') html.write('<html><body>\n') html.write('<h1>QA directory: {}</h1>\n'.format(qadir)) html.write('<ul>\n') html.write('<li>Approximate total area = {:.1f} deg2</li>\n'.format(area)) html.write('<li>Science targets = {}</li>\n'.format(nobj)) html.write('<li>Sky targets = {}</li>\n'.format(nsky)) html.write('<li>Dark standards = {}</li>\n'.format(ndark)) html.write('<li>Bright standards = {}</li>\n'.format(nbright)) html.write('</ul>\n') # Desired target densities, including contaminants. html.write('<hr>\n') html.write('<hr>\n') html.write('<h2>Raw target densities (including contaminants)</h2>\n') targdens = {'ELG': 2400, 'LRG': 350, 'QSO': 260, 'SKY': 1400} if nobj > 0: html.write( '<h3>Science targets - ELG, LRG, QSO, BGS_ANY, MWS_ANY</h3>\n') for obj in ('ELG', 'LRG', 'QSO', 'BGS_ANY', 'MWS_ANY'): these = np.where( (targets['DESI_TARGET'] & desi_mask.mask(obj)) != 0)[0] if len(these) > 0: pngfile = qadensity(targets[these], obj, targdens, qadir=qadir, max_bin_area=binarea) _img2html(html, pngfile, log) html.write('<h3>Science targets - BGS_BRIGHT, BGS_FAINT</h3>\n') for obj in ('BGS_BRIGHT', 'BGS_FAINT'): these = np.where( (targets['BGS_TARGET'] & bgs_mask.mask(obj)) != 0)[0] if len(these) > 0: pngfile = qadensity(targets[these], obj, targdens, qadir=qadir, max_bin_area=binarea) _img2html(html, pngfile, log) html.write( '<h3>Science targets - MWS_MAIN, MWS_MAIN_VERY_FAINT, MWS_NEARBY, MWS_WD</h3>\n' ) for obj in ('MWS_MAIN', 'MWS_MAIN_VERY_FAINT', 'MWS_NEARBY', 'MWS_WD'): these = np.where( (targets['MWS_TARGET'] & mws_mask.mask(obj)) != 0)[0] if len(these) > 0: pngfile = qadensity(targets[these], obj, targdens, qadir=qadir, max_bin_area=binarea) _img2html(html, pngfile, log) html.write('<hr>\n') if nsky > 0: html.write('<h3>Sky targets</h3>\n') obj = 'SKY' these = np.where((sky['DESI_TARGET'] & desi_mask.mask(obj)) != 0)[0] if len(these) > 0: pngfile = qadensity(sky[these], obj, targdens, qadir=qadir, max_bin_area=binarea) _img2html(html, pngfile, log) html.write('<hr>\n') if ndark > 0 or nbright > 0: html.write('<h3>Standard Stars</h3>\n') for cat, nn, obj in zip((stddark, stdbright), (ndark, nbright), ('STD_FSTAR', 'STD_BRIGHT')): if nn > 0: these = np.where( (cat['DESI_TARGET'] & desi_mask.mask(obj)) != 0)[0] if len(these) > 0: pngfile = qadensity(cat[these], obj, targdens, qadir=qadir, max_bin_area=binarea) _img2html(html, pngfile, log) html.write('<hr>\n') html.write('<hr>\n') # Desired target densities, including contaminants. html.write('<h2>Detailed target densities</h2>\n') html.write('<h3>QSOs</h3>\n') pngfile = qa_qso(targets, truth, qadir=qadir) _img2html(html, pngfile, log) html.write('<hr>\n') html.write('<h3>ELGs</h3>\n') pngfile = qa_elg(targets, truth, qadir=qadir) _img2html(html, pngfile, log) html.write('<hr>\n') html.write('</html></body>\n') html.close()
def qa_elg(targets, truth, qadir='.'): """Detailed QA plots for ELGs.""" dens_all = 2400 dens_loz = dens_all * 0.05 dens_hiz = dens_all * 0.05 dens_star = dens_all * 0.1 dens_rightz = dens_all - dens_loz - dens_hiz - dens_star dens = dict() these = np.where(targets['DESI_TARGET'] & desi_mask.mask('ELG') != 0)[0] dens['ELG_TARGETS'] = target_density(targets[these]) these = np.where((targets['DESI_TARGET'] & desi_mask.mask('ELG') != 0) * (truth['TRUEZ'] >= 0.6) * (truth['TRUEZ'] <= 1.6))[0] dens['ELG_IS_RIGHTZ'] = target_density(targets[these]) these = np.where((targets['DESI_TARGET'] & desi_mask.mask('QSO') != 0) * (truth['TRUEZ'] < 0.6))[0] dens['ELG_IS_LOZ'] = target_density(targets[these]) these = np.where((targets['DESI_TARGET'] & desi_mask.mask('QSO') != 0) * (truth['TRUEZ'] > 1.6))[0] dens['ELG_IS_HIZ'] = target_density(targets[these]) these = np.where( (targets['DESI_TARGET'] & desi_mask.mask('ELG') != 0) * (truth['CONTAM_TARGET'] & contam_mask.mask('ELG_IS_STAR')) != 0)[0] dens['ELG_IS_STAR'] = target_density(targets[these]) bins = 50 lim = (0, 3000) fig, ax = plt.subplots(1, 2, figsize=(12, 5)) #line = ax[0].axvline(x=dens_all, ls='-') ax[0].hist( dens['ELG_TARGETS'], bins=bins, range=lim, #color=line.get_color(), color='k', lw=1, label='All Targets', histtype='step') #line = ax[0].axvline(x=dens_rightz, ls='--') ax[0].hist( dens['ELG_IS_RIGHTZ'], bins=bins, range=lim, alpha=0.6, # color=line.get_color(), ls='--', lw=2, label='ELG (0.6<z<1.6)') #, histtype='step') ax[0].set_ylabel('Number of Healpixels') ax[0].set_xlabel('Targets / deg$^2$') ax[0].set_title('True ELGs') ax[0].legend(loc='upper left') lim = (0, 300) #ax[1].hist(dens['ELG_TARGETS'], bins=bins, range=lim, label='All Targets', # color='k', lw=1, histtype='step') #line = ax[1].axvline(x=dens_star, ls='-') ax[1].hist( dens['ELG_IS_STAR'], bins=bins, range=lim, #color=line.get_color(), alpha=0.5, label='ELG_IS_STAR') #line = ax[1].axvline(x=dens_loz, ls='-') ax[1].hist( dens['ELG_IS_LOZ'], bins=bins, range=lim, #color=line.get_color(), alpha=0.5, label='ELG_IS_LOZ (z<0.6)') #line = ax[1].axvline(x=dens_hiz, ls='-') ax[1].hist( dens['ELG_IS_HIZ'], bins=bins, range=lim, #color=line.get_color(), alpha=0.5, label='ELG_IS_HIZ (z>1.6)') ax[1].set_ylabel('Number of Healpixels') ax[1].set_xlabel('Targets / deg$^2$') ax[1].set_title('ELG Contaminants') ax[1].legend(loc='upper right') pngfile = os.path.join(qadir, '{}_detail_density.png'.format('elg')) fig.savefig(pngfile) return pngfile
def setUpClass(cls): cls.ntiles = 4 tiles = desimodel.io.load_tiles() cls.tileids = tiles['TILEID'][0:cls.ntiles] cls.tilefiles = ['tile-{:05d}.fits'.format(i) for i in cls.tileids] cls.tilefiles_multiobs = [ 'multitile-{:05d}.fits'.format(i) for i in cls.tileids ] cls.nspec = n = 1000 targets = Table() targets['TARGETID'] = np.random.randint(0, 2**60, size=n) targets['DESI_TARGET'] = 2**np.random.randint(0, 3, size=n) targets['BGS_TARGET'] = np.zeros(n, dtype=int) targets['MWS_TARGET'] = np.zeros(n, dtype=int) isLRG = (targets['DESI_TARGET'] & desi_mask.LRG) != 0 isELG = (targets['DESI_TARGET'] & desi_mask.ELG) != 0 isQSO = (targets['DESI_TARGET'] & desi_mask.QSO) != 0 cls.targets = targets #- Make a few of them BGS and MWS iibright = np.random.choice(np.arange(n), size=6, replace=False) isBGS = iibright[0:3] isMWS = iibright[3:6] targets['DESI_TARGET'][isBGS] = desi_mask.BGS_ANY targets['BGS_TARGET'][isBGS] = bgs_mask.BGS_BRIGHT targets['DESI_TARGET'][isMWS] = desi_mask.MWS_ANY targets['MWS_TARGET'][isMWS] = mws_mask.MWS_MAIN #- Add some fake photometry; no attempt to get colors right flux = np.zeros((n, 6)) #- ugrizY; DESI has grz flux[isLRG, 1] = np.random.uniform(0, 1.0, np.count_nonzero(isLRG)) flux[isLRG, 2] = np.random.uniform(0, 5.0, np.count_nonzero(isLRG)) flux[isLRG, 4] = np.random.uniform(0, 5.0, np.count_nonzero(isLRG)) flux[isELG, 1] = np.random.uniform(0, 4.0, np.count_nonzero(isELG)) flux[isELG, 2] = np.random.uniform(0, 4.0, np.count_nonzero(isELG)) flux[isELG, 4] = np.random.uniform(0, 10.0, np.count_nonzero(isELG)) flux[isQSO, 1] = np.random.uniform(0, 4.0, np.count_nonzero(isQSO)) flux[isQSO, 2] = np.random.uniform(0, 4.0, np.count_nonzero(isQSO)) flux[isQSO, 4] = np.random.uniform(0, 6.0, np.count_nonzero(isQSO)) flux[isBGS, 1] = np.random.uniform(10, 600, np.count_nonzero(isBGS)) flux[isBGS, 2] = np.random.uniform(15, 1000, np.count_nonzero(isBGS)) flux[isBGS, 4] = np.random.uniform(10, 1400, np.count_nonzero(isBGS)) flux[isMWS, 1] = np.random.uniform(10, 150, np.count_nonzero(isMWS)) flux[isMWS, 2] = np.random.uniform(15, 350, np.count_nonzero(isMWS)) flux[isMWS, 4] = np.random.uniform(10, 1500, np.count_nonzero(isMWS)) targets['DECAM_FLUX'] = flux truth = Table() truth['TARGETID'] = targets['TARGETID'] truth['TRUEZ'] = np.random.uniform(0, 1.5, size=n) truth['TRUESPECTYPE'] = np.zeros(n, dtype=(str, 10)) truth['GMAG'] = np.random.uniform(18.0, 24.0, size=n) ii = (targets['DESI_TARGET'] & desi_mask.mask('LRG|ELG|BGS_ANY')) != 0 truth['TRUESPECTYPE'][ii] = 'GALAXY' ii = (targets['DESI_TARGET'] == desi_mask.QSO) truth['TRUESPECTYPE'][ii] = 'QSO' starmask = desi_mask.mask('MWS_ANY|STD_FSTAR|STD_WD|STD_BRIGHT') ii = (targets['DESI_TARGET'] & starmask) != 0 truth['TRUESPECTYPE'][ii] = 'STAR' #- Add some fake [OII] fluxes for the ELGs; include some that will fail isELG = (targets['DESI_TARGET'] & desi_mask.ELG) != 0 nELG = np.count_nonzero(isELG) truth['OIIFLUX'] = np.zeros(n, dtype=float) truth['OIIFLUX'][isELG] = np.random.normal(2e-17, 2e-17, size=nELG).clip(0) cls.truth = truth fiberassign = truth['TARGETID', ] fiberassign['RA'] = np.random.uniform(0, 5, size=n) fiberassign['DEC'] = np.random.uniform(0, 5, size=n) fiberassign.meta['EXTNAME'] = 'FIBER_ASSIGNMENTS' nx = cls.nspec // cls.ntiles cls.targets_in_tile = dict() for i, filename in enumerate(cls.tilefiles): subset = fiberassign[i * nx:(i + 1) * nx] subset.write(filename) cls.targets_in_tile[cls.tileids[i]] = subset['TARGETID'] hdulist = fits.open(filename, mode='update') hdr = hdulist[1].header hdr.set('TILEID', cls.tileids[i]) hdulist.close() #- Also create a test of tile files that have multiple observations nx = cls.nspec // cls.ntiles for i, filename in enumerate(cls.tilefiles_multiobs): subset = fiberassign[0:(i + 1) * nx] subset.write(filename) hdulist = fits.open(filename, mode='update') hdr = hdulist[1].header hdr.set('TILEID', cls.tileids[i]) hdulist.close()