def page_subset_expo(fdir, exposure, frametype, petals, html_dir,
titlepage_prefix, mask, log, nspecperfile, snr_cut):
'''
Running prospect from frames : loop over petals for a given exposure
'''
nspec_done = 0
for petal_num in petals:
frames = [
desispec.io.read_frame(
os.path.join(
fdir, frametype + "-" + band + petal_num + "-" + exposure +
".fits")) for band in ['b', 'r', 'z']
]
spectra = utils_specviewer.frames2spectra(frames, with_scores=True)
if 'FIBERSTATUS' in spectra.fibermap.keys():
spectra = myspecselect.myspecselect(spectra,
clean_fiberstatus=True)
if spectra is None: return 0
# Selection
if (mask != None) or (snr_cut != None):
spectra = utils_specviewer.specviewer_selection(
spectra,
log=log,
mask=mask,
mask_type='CMX_TARGET',
snr_cut=snr_cut,
with_dirty_mask_merge=True)
if spectra == 0: continue
# Handle several html pages per exposure - sort by fiberid
nspec_expo = spectra.num_spectra()
log.info("Petal number " + petal_num + " : " + str(nspec_expo) +
" spectra")
sort_indices = np.argsort(spectra.fibermap["FIBER"])
nbpages = int(np.ceil((nspec_expo / nspecperfile)))
for i_page in range(1, 1 + nbpages):
log.info(" * Page " + str(i_page) + " / " + str(nbpages))
the_indices = sort_indices[(i_page - 1) * nspecperfile:i_page *
nspecperfile]
thespec = myspecselect.myspecselect(spectra, indices=the_indices)
titlepage = titlepage_prefix + "_petal" + petal_num + "_" + str(
i_page)
plotframes.plotspectra(thespec,
with_noise=True,
with_coaddcam=True,
is_coadded=False,
title=titlepage,
html_dir=html_dir,
mask_type='CMX_TARGET',
with_thumb_only_page=True)
nspec_done += nspec_expo
return nspec_done
def page_subset_tile(fdir, tile_db_subset, frametype, html_dir,
titlepage_prefix, mask, log, nspecperfile, snr_cut):
'''
Running prospect from frames : tile-based, do not separate pages per exposure.
tile_db_subset : subset of tile_db, all with the same tile
'''
tile = tile_db_subset['tile']
nspec_done = 0
all_spectra = None
for the_subset in tile_db_subset['db_subset']:
assert (the_subset['tile'] == tile)
log.info("Tile " + tile + " : reading frames from exposure " +
the_subset['exposure'])
for spectrograph_num in the_subset['spectrographs']:
frames = [
desispec.io.read_frame(
os.path.join(
fdir, the_subset['night'],
frametype + "-" + band + spectrograph_num + "-" +
the_subset['exposure'] + ".fits"))
for band in ['b', 'r', 'z']
]
### TMP TRICK (?) : need to have exposures in fibermaps, otherwise spectra.update() crashes !
for fr in frames:
if not ('EXPID' in fr.fibermap.keys()):
fr.fibermap['EXPID'] = fr.fibermap['FIBER']
for i in range(len(fr.fibermap)):
fr.fibermap['EXPID'][i] = the_subset['exposure']
### END TMP TRICK
### OTHER TRICK : need resolution data in spectra to pass coadd fct (could be changed...)
spectra = utils_specviewer.frames2spectra(
frames, with_scores=True, with_resolution_data=True)
# Filtering
if (mask != None) or (snr_cut != None):
spectra = utils_specviewer.specviewer_selection(
spectra,
log=log,
mask=mask,
mask_type='CMX_TARGET',
snr_cut=snr_cut)
if spectra == 0: continue
# Merge
if all_spectra is None:
all_spectra = spectra
else:
all_spectra.update(
spectra
) # NB update() does not copy scores. Filtering was done before.
if all_spectra is None:
log.info("Tile " + tile + " : no spectra !")
return 0
# Exposure-coadd
all_spectra = utils_specviewer.coadd_targets(all_spectra)
# Handle several html pages per exposure - sort by targetid
nspec_tile = all_spectra.num_spectra()
log.info("Tile " + tile + " : " + str(nspec_tile) +
" exposure-coadded spectra")
sort_indices = np.argsort(all_spectra.fibermap["TARGETID"])
nbpages = int(np.ceil((nspec_tile / nspecperfile)))
for i_page in range(1, 1 + nbpages):
log.info(" * Page " + str(i_page) + " / " + str(nbpages))
the_indices = sort_indices[(i_page - 1) * nspecperfile:i_page *
nspecperfile]
thespec = myspecselect.myspecselect(all_spectra, indices=the_indices)
titlepage = titlepage_prefix + "_" + str(i_page)
plotframes.plotspectra(thespec,
with_noise=True,
with_coaddcam=True,
is_coadded=True,
title=titlepage,
html_dir=html_dir,
mask_type='CMX_TARGET',
with_thumb_only_page=True)
nspec_done += nspec_tile
return nspec_done
def main(args):
log = get_logger()
specprod_dir = args.specprod_dir
if specprod_dir is None: specprod_dir = desispec.io.specprod_root()
webdir = args.webdir
if webdir is None:
webdir = os.environ[
"DESI_WWW"] + "/users/armengau/svdc2019c" # TMP, for test
nights = desispec.io.get_nights(specprod_dir=specprod_dir)
# TODO - Select night (eg. only last night)
nights = nights[8:11] # TMP, for test
for thenight in nights:
# Get spectra from tiles dir - To consolidate
specfiles = glob.glob(
os.path.join(specprod_dir,
"tiles/*/tilespectra-*-" + thenight + ".fits"))
for f in specfiles:
log.info("Working on file " + f)
file_label = f[
f.find("tilespectra-") + 12:f.find(thenight) -
1] # From tile-based file description - To consolidate
spectra = desispec.io.read_spectra(f)
zbfile = f.replace("tilespectra", "zbest")
zbest = Table.read(zbfile, 'ZBEST')
# Handle several html pages per pixel : sort by TARGETID
# NOTE : this way, individual spectra from the same target are together
# Does it make sense ? (they have the same fit)
nbpages = int(np.ceil((spectra.num_spectra() / args.nspecperfile)))
sort_indices = np.argsort(
spectra.fibermap["TARGETID"],
kind='mergesort') # keep order of equal elts
for i_page in range(1, 1 + nbpages):
log.info(" * Page " + str(i_page) + " / " + str(nbpages))
the_indices = sort_indices[(i_page - 1) *
args.nspecperfile:i_page *
args.nspecperfile]
thespec = myspecselect.myspecselect(spectra,
indices=the_indices)
thezb, kk = utils_specviewer.match_zcat_to_spectra(
zbest, thespec)
model = plotframes.create_model(thespec, thezb)
### No VI results to display by default
# vifile = os.environ['HOME']+"/prospect/vilist_prototype.fits"
# vidata = utils_specviewer.match_vi_targets(vifile, thespec.fibermap["TARGETID"])
titlepage = "specviewer_night" + thenight + "_" + file_label + "_" + str(
i_page)
html_dir = webdir + "/nights/night" + thenight
if not os.path.exists(html_dir):
os.makedirs(html_dir)
os.mkdir(html_dir + "/vignettes")
plotframes.plotspectra(thespec,
zcatalog=thezb,
vidata=None,
model=model,
title=titlepage,
html_dir=html_dir,
is_coadded=False)
for i_spec in range(thespec.num_spectra()):
saveplot = html_dir + "/vignettes/night" + thenight + "_" + file_label + "_" + str(
i_page) + "_" + str(i_spec) + ".png"
utils_specviewer.miniplot_spectrum(
thespec,
i_spec,
model=model,
saveplot=saveplot,
smoothing=args.vignette_smoothing)
def page_subset_tile(fdir,
tile_db_subset,
frametype,
html_dir,
titlepage_prefix,
mask,
log,
nspecperfile,
snr_cut,
with_zcatalog=False):
'''
Running prospect from frames : tile-based, do not separate pages per exposure.
tile_db_subset : subset of tile_db, all with the same tile
'''
tile = tile_db_subset['tile']
nspec_done = 0
all_spectra = None
for the_subset in tile_db_subset['db_subset']:
assert (the_subset['tile'] == tile)
log.info("Tile " + tile + " : reading frames from exposure " +
the_subset['exposure'])
for petal_num in the_subset['petals']:
frames = [
desispec.io.read_frame(
os.path.join(
fdir, the_subset['night'], frametype + "-" + band +
petal_num + "-" + the_subset['exposure'] + ".fits"))
for band in ['b', 'r', 'z']
]
# TRICK : need resolution data in spectra to pass coadd fct (could be changed...)
spectra = utils_specviewer.frames2spectra(
frames, with_scores=True, with_resolution_data=True)
# Filtering
if (mask != None) or (snr_cut != None):
spectra = utils_specviewer.specviewer_selection(
spectra,
log=log,
mask=mask,
mask_type='CMX_TARGET',
snr_cut=snr_cut,
with_dirty_mask_merge=True)
if spectra == 0: continue
# Merge
if all_spectra is None:
all_spectra = spectra
else: # NB update() does not copy scores. Filtering was done before.
all_spectra = myspecupdate.myspecupdate(all_spectra, spectra)
if all_spectra is None:
log.info("Tile " + tile + " : no spectra !")
return 0
elif 'FIBERSTATUS' in all_spectra.fibermap.keys():
all_spectra = myspecselect.myspecselect(all_spectra,
clean_fiberstatus=True,
remove_scores=True)
if all_spectra is None: return 0
# Exposure-coadd
all_spectra = utils_specviewer.coadd_targets(all_spectra)
# zcatalog
if with_zcatalog:
zcat_files = glob.glob(fdir + "/" + the_subset['night'] +
"/zbest*.fits") # Probably TMP ... ?
ztables = []
for f in zcat_files:
ztables.append(Table.read(f, 'ZBEST'))
zcat = vstack(ztables)
else:
zcat = None
# Handle several html pages per exposure - sort by targetid
nspec_tile = all_spectra.num_spectra()
log.info("Tile " + tile + " : " + str(nspec_tile) +
" exposure-coadded spectra")
sort_indices = np.argsort(all_spectra.fibermap["TARGETID"])
nbpages = int(np.ceil((nspec_tile / nspecperfile)))
for i_page in range(1, 1 + nbpages):
log.info(" * Page " + str(i_page) + " / " + str(nbpages))
the_indices = sort_indices[(i_page - 1) * nspecperfile:i_page *
nspecperfile]
thespec = myspecselect.myspecselect(all_spectra,
indices=the_indices,
remove_scores=True)
titlepage = titlepage_prefix + "_" + str(i_page)
plotframes.plotspectra(thespec,
with_noise=True,
with_coaddcam=True,
is_coadded=True,
zcatalog=zcat,
title=titlepage,
html_dir=html_dir,
mask_type='CMX_TARGET',
with_thumb_only_page=True)
nspec_done += nspec_tile
return nspec_done
def main(args):
log = get_logger()
specprod_dir = args.specprod_dir
if specprod_dir is None: specprod_dir = desispec.io.specprod_root()
webdir = args.webdir
if webdir is None:
webdir = os.environ[
"DESI_WWW"] + "/users/armengau/svdc2019c" # TMP, for test
# TODO - Selection on pixels based on existing specviewer pages
if args.pixel_list is None:
pixels = glob.glob(os.path.join(specprod_dir, "spectra-64/*/*"))
pixels = [x[x.rfind("/") + 1:] for x in pixels]
else:
pixels = np.loadtxt(args.pixel_list, dtype=str)
if args.random_pixels:
random.shuffle(pixels)
# Loop on pixels
nspec_done = 0
for pixel in pixels:
log.info("Working on pixel " + pixel)
thefile = desispec.io.findfile('spectra',
groupname=int(pixel),
specprod_dir=specprod_dir)
individual_spectra = desispec.io.read_spectra(thefile)
spectra = plotframes.coadd_targets(individual_spectra)
zbfile = thefile.replace('spectra-64-', 'zbest-64-')
if os.path.isfile(zbfile):
zbest = Table.read(zbfile, 'ZBEST')
else:
log.info("No associated zbest file found : skipping pixel")
continue
spectra = utils_specviewer.specviewer_selection(
spectra,
log=log,
mask=args.mask,
mask_type=args.mask_type,
gmag_cut=args.gcut,
rmag_cut=args.rcut,
chi2cut=args.chi2cut,
zbest=zbest)
if spectra == 0: continue
# Handle several html pages per pixel : sort by TARGETID
# TODO - Find a more useful sort ?
nbpages = int(np.ceil((spectra.num_spectra() / args.nspecperfile)))
sort_indices = np.argsort(spectra.fibermap["TARGETID"])
for i_page in range(1, 1 + nbpages):
log.info(" * Page " + str(i_page) + " / " + str(nbpages))
the_indices = sort_indices[(i_page - 1) *
args.nspecperfile:i_page *
args.nspecperfile]
thespec = myspecselect.myspecselect(spectra, indices=the_indices)
thezb, kk = utils_specviewer.match_zcat_to_spectra(zbest, thespec)
### No VI results to display by default
# VI "catalog" - location to define later ..
# vifile = os.environ['HOME']+"/prospect/vilist_prototype.fits"
# vidata = utils_specviewer.match_vi_targets(vifile, thespec.fibermap["TARGETID"])
titlepage = "pix" + pixel + "_" + str(i_page)
if args.gcut is not None:
titlepage = "gcut-" + str(args.gcut[0]) + "-" + str(
args.gcut[1]) + "_" + titlepage
if args.rcut is not None:
titlepage = "rcut-" + str(args.rcut[0]) + "-" + str(
args.rcut[1]) + "_" + titlepage
if args.chi2cut is not None:
titlepage = "chi2cut-" + str(args.chi2cut[0]) + "-" + str(
args.chi2cut[1]) + "_" + titlepage
if args.mask is not None:
titlepage = args.mask + "_" + titlepage
model = plotframes.create_model(thespec, thezb)
html_dir = os.path.join(webdir, "pix" + pixel)
if not os.path.exists(html_dir):
os.makedirs(html_dir)
os.mkdir(html_dir + "/vignettes")
plotframes.plotspectra(thespec,
zcatalog=zbest,
model_from_zcat=True,
vidata=None,
model=None,
title=titlepage,
html_dir=html_dir,
is_coadded=True,
mask_type=args.mask_type)
# for i_spec in range(thespec.num_spectra()) :
# saveplot = html_dir+"/vignettes/pix"+pixel+"_"+str(i_page)+"_"+str(i_spec)+".png"
# utils_specviewer.miniplot_spectrum(thespec, i_spec, model=model, saveplot=saveplot, smoothing = args.vignette_smoothing)
nspec_done += thespec.num_spectra()
# Stop running if needed, only once a full pixel is completed
if args.nmax_spectra is not None:
if nspec_done >= args.nmax_spectra:
log.info(
str(nspec_done) +
" spectra done : no other pixel will be processed")
break
def page_subset_tile(fdir,
tile_db_subset,
html_dir,
titlepage_prefix,
mask,
log,
nspecperfile,
snr_cut,
with_zcatalog=False,
template_dir=None,
clean_bad_fibers_cmx=False):
'''
Running prospect from coadds.
'''
tile = tile_db_subset['tile']
night = tile_db_subset['night']
nspec_done = 0
all_spectra = None
log.info("Tile " + tile + " : reading coadds from night " + night)
for petal_num in tile_db_subset['petals']:
fname = os.path.join(
fdir, "coadd-" + petal_num + "-" + tile + '-' + night + ".fits")
spectra = desispec.io.read_spectra(fname)
# Filtering
if (mask != None) or (snr_cut != None):
spectra = utils_specviewer.specviewer_selection(
spectra,
log=log,
mask=mask,
mask_type='CMX_TARGET',
snr_cut=snr_cut,
with_dirty_mask_merge=True)
if spectra == 0: continue
# Merge
if all_spectra is None:
all_spectra = spectra
else:
# NB update() does not copy scores. Score-based filtering (SNR) was done before.
all_spectra = myspecupdate.myspecupdate(all_spectra, spectra)
if all_spectra is None:
log.info("Tile " + tile + " - night " + night + ": no spectra.")
return 0
else:
clean_fiberstatus = True if 'FIBERSTATUS' in all_spectra.fibermap.keys(
) else False
fibers = None
if clean_bad_fibers_cmx:
fibers = np.arange(5000)
for cut_fiber_range in _bad_fibers_cmx:
fibers = fibers[((fibers < cut_fiber_range[0]) |
(fibers > cut_fiber_range[1]))]
all_spectra = myspecselect.myspecselect(
all_spectra,
clean_fiberstatus=clean_fiberstatus,
fibers=fibers,
remove_scores=True)
if all_spectra is None: return 0
# zcatalog
if with_zcatalog:
ztables = []
for petal_num in tile_db_subset['petals']:
fname = os.path.join(
fdir,
"zbest-" + petal_num + "-" + tile + '-' + night + ".fits")
ztables.append(Table.read(fname, 'ZBEST'))
zcat = vstack(ztables)
else:
zcat = None
# Create several html pages : sort by targetid
nspec_tile = all_spectra.num_spectra()
log.info("Tile " + tile + " - night " + night + ": " + str(nspec_tile) +
" exposure-coadded spectra")
sort_indices = np.argsort(all_spectra.fibermap["TARGETID"])
nbpages = int(np.ceil((nspec_tile / nspecperfile)))
for i_page in range(1, 1 + nbpages):
log.info(" * Page " + str(i_page) + " / " + str(nbpages))
the_indices = sort_indices[(i_page - 1) * nspecperfile:i_page *
nspecperfile]
thespec = myspecselect.myspecselect(all_spectra,
indices=the_indices,
remove_scores=True)
the_zcat, kk = utils_specviewer.match_zcat_to_spectra(zcat, thespec)
titlepage = titlepage_prefix + "_" + str(i_page)
plotframes.plotspectra(thespec,
with_noise=True,
with_coaddcam=False,
is_coadded=True,
zcatalog=the_zcat,
title=titlepage,
html_dir=html_dir,
mask_type='CMX_TARGET',
with_thumb_only_page=True,
template_dir=template_dir)
nspec_done += nspec_tile
return nspec_done