def main(args=None):
log = get_logger()
if args is None:
args = parse()
if args.lin_step is not None and args.log10_step is not None:
print(
"cannot have both linear and logarthmic bins :-), choose either --lin-step or --log10-step"
)
return 12
if args.coadd_cameras and (args.lin_step is not None
or args.log10_step is not None):
print(
"cannot specify a new wavelength binning along with --coadd-cameras option"
)
return 12
log.info("reading spectra ...")
spectra = read_spectra(args.infile)
if args.coadd_cameras:
log.info("coadding cameras ...")
spectra = coadd_cameras(spectra, cosmics_nsig=args.nsig)
else:
log.info("coadding ...")
coadd(spectra, cosmics_nsig=args.nsig)
if args.lin_step is not None:
log.info("resampling ...")
spectra = resample_spectra_lin_or_log(spectra,
linear_step=args.lin_step,
wave_min=args.wave_min,
wave_max=args.wave_max,
fast=args.fast,
nproc=args.nproc)
if args.log10_step is not None:
log.info("resampling ...")
spectra = resample_spectra_lin_or_log(spectra,
log10_step=args.log10_step,
wave_min=args.wave_min,
wave_max=args.wave_max,
fast=args.fast,
nproc=args.nproc)
log.info("writing {} ...".format(args.outfile))
write_spectra(args.outfile, spectra)
log.info("done")
def sim_source_spectra(allinfo, allzbest, infofile='source-truth.fits', debug=False):
"""Build the residual (source) spectra. No redshift-fitting.
"""
from desispec.io import read_spectra, write_spectra
from desispec.spectra import Spectra
from desispec.interpolation import resample_flux
from desispec.resolution import Resolution
from redrock.external.desi import DistTargetsDESI
from redrock.templates import find_templates, Template
assert(np.all(allinfo['TARGETID'] == allzbest['TARGETID']))
nsim = len(allinfo)
# Select the subset of objects for which we got the correct lens (BGS)
# redshift.
these = np.where((allzbest['SPECTYPE'] == 'GALAXY') *
(np.abs(allzbest['Z'] - allinfo['LENS_Z']) < 0.003))[0]
print('Selecting {}/{} lenses with the correct redshift'.format(len(these), nsim))
if len(these) == 0:
raise ValueError('No spectra passed the cuts!')
allinfo = allinfo[these]
allzbest = allzbest[these]
print('Writing {}'.format(infofile))
allinfo.write(infofile, overwrite=True)
tempfile = find_templates()[0]
rrtemp = Template(tempfile)
# loop on each chunk of lens+source spectra
nchunk = len(set(allinfo['CHUNK']))
for ichunk in set(allinfo['CHUNK']):
I = np.where(allinfo['CHUNK'] == ichunk)[0]
info = allinfo[I]
zbest = allzbest[I]
specfile = 'lenssource-spectra-chunk{:03d}.fits'.format(ichunk)
sourcefile = 'source-spectra-chunk{:03d}.fits'.format(ichunk)
spectra = read_spectra(specfile).select(targets=info['TARGETID'])
for igal, zz in enumerate(zbest):
zwave = rrtemp.wave * (1 + zz['Z'])
zflux = rrtemp.flux.T.dot(zz['COEFF']).T #/ (1 + zz['Z'])
if debug:
fig, ax = plt.subplots()
for band in spectra.bands:
R = Resolution(spectra.resolution_data[band][igal])
# use fastspecfit here
modelflux = R.dot(resample_flux(spectra.wave[band], zwave, zflux))
if debug:
ax.plot(spectra.wave[band], spectra.flux[band][igal, :])
ax.plot(spectra.wave[band], modelflux)
ax.set_ylim(np.median(spectra.flux['r'][igal, :]) + np.std(spectra.flux['r'][igal, :]) * np.array([-1.5, 3]))
#ax.set_xlim(4500, 5500)
spectra.flux[band][igal, :] -= modelflux # subtract
if debug:
qafile = 'source-spectra-chunk{:03d}-{}.png'.format(ichunk, igal)
fig.savefig(qafile)
plt.close()
print('Writing {} spectra to {}'.format(len(zbest), sourcefile))
write_spectra(outfile=sourcefile, spec=spectra)
return allinfo
def main(args=None):
log = get_logger()
if args is None:
args = parse()
if args.lin_step is not None and args.log10_step is not None:
log.critical(
"cannot have both linear and logarthmic bins :-), choose either --lin-step or --log10-step"
)
return 12
if args.coadd_cameras and (args.lin_step is not None
or args.log10_step is not None):
log.critical(
"cannot specify a new wavelength binning along with --coadd-cameras option"
)
return 12
if len(args.infile) == 0:
log.critical("You must specify input files")
return 12
log.info("reading input ...")
# inspect headers
input_is_frames = False
input_is_spectra = False
for filename in args.infile:
ifile = fitsio.FITS(filename)
head = ifile[0].read_header()
identified = False
if "EXTNAME" in head and head["EXTNAME"] == "FLUX":
print(filename, "is a frame")
input_is_frames = True
identified = True
ifile.close()
continue
for hdu in ifile:
head = hdu.read_header()
if "EXTNAME" in head and head["EXTNAME"].find("_FLUX") >= 0:
print(filename, "is a spectra")
input_is_spectra = True
identified = True
break
ifile.close()
if not identified:
log.error(
"{} not identified as frame of spectra file".format(filename))
sys.exit(1)
if input_is_frames and input_is_spectra:
log.error("cannot combine input spectra and frames")
sys.exit(1)
if input_is_spectra:
spectra = read_spectra(args.infile[0])
for filename in args.infile[1:]:
log.info("append {}".format(filename))
spectra.update(read_spectra(filename))
else: # frames
frames = dict()
cameras = {}
for filename in args.infile:
frame = read_frame(filename)
night = frame.meta['NIGHT']
expid = frame.meta['EXPID']
camera = frame.meta['CAMERA']
frames[(night, expid, camera)] = frame
if args.coadd_cameras:
cam, spec = camera[0], camera[1]
# Keep a list of cameras (b,r,z) for each exposure + spec
if (night, expid) not in cameras.keys():
cameras[(night, expid)] = {spec: [cam]}
elif spec not in cameras[(night, expid)].keys():
cameras[(night, expid)][spec] = [cam]
else:
cameras[(night, expid)][spec].append(cam)
if args.coadd_cameras:
# If not all 3 cameras are available, remove the incomplete sets
for (night, expid), camdict in cameras.items():
for spec, camlist in camdict.items():
log.info("Found {} for SP{} on NIGHT {} EXP {}".format(
camlist, spec, night, expid))
if len(camlist) != 3 or np.any(
np.sort(camlist) != np.array(['b', 'r', 'z'])):
for cam in camlist:
frames.pop((night, expid, cam + spec))
log.warning(
"Removing {}{} from Night {} EXP {}".format(
cam, spec, night, expid))
#import pdb
#pdb.set_trace()
spectra = frames2spectra(frames)
#- hacks to make SpectraLite like a Spectra
spectra.fibermap = Table(spectra.fibermap)
del frames #- maybe free some memory
if args.coadd_cameras:
log.info("coadding cameras ...")
spectra = coadd_cameras(spectra, cosmics_nsig=args.nsig)
else:
log.info("coadding ...")
coadd(spectra, cosmics_nsig=args.nsig)
if args.lin_step is not None:
log.info("resampling ...")
spectra = resample_spectra_lin_or_log(spectra,
linear_step=args.lin_step,
wave_min=args.wave_min,
wave_max=args.wave_max,
fast=args.fast,
nproc=args.nproc)
if args.log10_step is not None:
log.info("resampling ...")
spectra = resample_spectra_lin_or_log(spectra,
log10_step=args.log10_step,
wave_min=args.wave_min,
wave_max=args.wave_max,
fast=args.fast,
nproc=args.nproc)
#- Add input files to header
if spectra.meta is None:
spectra.meta = dict()
for i, filename in enumerate(args.infile):
spectra.meta['INFIL{:03d}'.format(i)] = os.path.basename(filename)
log.info("writing {} ...".format(args.outfile))
write_spectra(args.outfile, spectra)
log.info("done")
allzbest[idx],
keys='TARGETID')
fmap = join(fmap, classification, keys='TARGETID')
# Pack data into Spectra and write to FITS.
cand_spectra = Spectra(
bands=['brz'],
wave={'brz': allwave},
flux={'brz': allflux[idx]},
ivar={'brz': allivar[idx]},
resolution_data={'brz': allres[idx]},
fibermap=fmap)
outfits = '{}/transient_candidate_spectra_{}_{}.fits'.format(
out_path, obsdate, tile_number)
write_spectra(outfits, cand_spectra)
print('Output file saved in {}'.format(outfits))
# Make a plot of up to 16 transients
selection = sorted(
np.random.choice(idx.flatten(),
size=np.minimum(len(idx), 16),
replace=False))
fig, axes = plt.subplots(4,
4,
figsize=(15, 10),
sharex=True,
sharey=True,
gridspec_kw={
rsflux = rescale_flux(reflux)
# Run the classification.
if args.tfmodel is not None:
pred = classifier.predict(rsflux)
# Create output: selected target spectra.
selected_spectra = Spectra(bands=['brz'],
wave={'brz' : allwave},
flux={'brz' : allflux},
ivar={'brz' : allivar},
mask={'brz' : allmask},
resolution_data={'brz' : allres},
fibermap=allfmap)
write_spectra('selected-{}-{}.fits'.format(args.tile, args.date), selected_spectra)
# Append preprocess spectra to output.
hx = fits.HDUList()
hdu_rewave = fits.PrimaryHDU(rewave)
hdu_rewave.header['EXTNAME'] = 'REWAVE'
hdu_rewave.header['BUNIT'] = 'Angstrom'
hdu_rewave.header['AIRORVAC'] = ('vac', 'Vacuum wavelengths')
hx.append(hdu_rewave)
hdu_reflux = fits.ImageHDU(reflux)
hdu_reflux.header['EXTNAME'] = 'REFLUX'
hx.append(hdu_reflux)
hdu_rsflux = fits.ImageHDU(rsflux)
def main(args=None):
log = get_logger()
if isinstance(args, (list, tuple, type(None))):
args = parse(args)
thedate = datetime.now().strftime('%Y-%m-%d')
# Generate transient model if one is specified.
trans_model = None
if args.transient is not None:
trans_model = transients.get_model(args.transient)
# Generate list of HEALPix pixels to randomly sample the mocks.
rng = np.random.RandomState(args.seed)
nside = args.nside
healpixels = _get_healpixels_in_footprint(nside=args.nside)
npix = np.minimum(10 * args.nsim, len(healpixels))
pixels = rng.choice(healpixels, size=npix, replace=False)
ipix = iter(pixels)
# Set up the template generator.
fluxratio_range = 10**(-np.sort(args.magrange)[::-1] / 2.5)
epoch_range = np.sort(args.epochrange)
if args.host == 'bgs':
maker = BGSMaker(seed=args.seed)
tmpl_maker = BGS
elif args.host == 'elg':
maker = ELGMaker(seed=args.seed)
tmpl_maker = ELG
elif args.host == 'lrg':
maker = LRGMaker(seed=args.seed)
tmpl_maker = LRG
else:
raise ValueError('Unusable host type {}'.format(args.host))
maker.template_maker = tmpl_maker(transient=trans_model,
tr_fluxratio=fluxratio_range,
tr_epoch=epoch_range)
for j in range(args.nsim):
# Loop until finding a non-empty healpixel with mock galaxies.
tdata = []
while len(tdata) == 0:
pixel = next(ipix)
tdata = maker.read(healpixels=pixel, nside=args.nside)
# Generate spectral templates and write them to truth files.
# Keep producing templates until we have enough to pass brightness cuts.
wave = None
flux, targ, truth, objtr = [], [], [], []
ntosim = np.min([args.nspec, len(tdata['RA'])])
ngood = 0
while ngood < args.nspec:
idx = rng.choice(len(tdata['RA']), ntosim)
tflux, twave, ttarg, ttruth, tobj = maker.make_spectra(tdata,
indx=idx)
g, r, z, w1, w2 = [
ttruth['FLUX_{}'.format(_)]
for _ in ['G', 'R', 'Z', 'W1', 'W2']
]
rfib = ttarg['FIBERFLUX_R']
# print(g, r, z, w1, w2, rfib)
# Apply color cuts.
is_bright = isBGS_colors(rfib, g, r, z, w1, w2, targtype='bright')
is_faint = isBGS_colors(rfib, g, r, z, w1, w2, targtype='faint')
is_wise = isBGS_colors(rfib, g, r, z, w1, w2, targtype='wise')
keep = np.logical_or.reduce([is_bright, is_faint, is_wise])
_ngood = np.count_nonzero(keep)
if _ngood > 0:
ngood += _ngood
flux.append(tflux[keep, :])
targ.append(ttarg[keep])
truth.append(ttruth[keep])
objtr.append(tobj[keep])
wave = maker.wave
flux = np.vstack(flux)[:args.nspec, :]
targ = vstack(targ)[:args.nspec]
truth = vstack(truth)[:args.nspec]
objtr = vstack(objtr)[:args.nspec]
# Set up and verify the TARGETID across all truth tables.
n = len(truth)
new_id = 10000 * pixel + 100 * j + np.arange(1, n + 1)
targ['OBJID'][:] = new_id
truth['TARGETID'][:] = new_id
objtr['TARGETID'][:] = new_id
assert (len(truth) == args.nspec)
assert (np.all(targ['OBJID'] == truth['TARGETID']))
assert (len(targ) == len(np.unique(targ['OBJID'])))
assert (len(truth) == len(np.unique(truth['TARGETID'])))
assert (len(objtr) == len(np.unique(objtr['TARGETID'])))
truthfile = os.path.join(
args.outdir,
'{}_{}_{:04d}s_{:03d}_truth.fits'.format(args.host, thedate,
int(args.exptime), j + 1))
write_templates(truthfile, flux, wave, targ, truth, objtr)
log.info('Wrote {}'.format(truthfile))
# Get observing conditions and generate spectra.
obs = dict(AIRMASS=args.airmass,
EXPTIME=args.exptime,
MOONALT=args.moonalt,
MOONFRAC=args.moonfrac,
MOONSEP=args.moonsep,
SEEING=args.seeing)
fcols = dict(BRICKID=targ['BRICKID'],
BRICK_OBJID=targ['OBJID'],
FLUX_G=targ['FLUX_G'],
FLUX_R=targ['FLUX_R'],
FLUX_Z=targ['FLUX_Z'],
FLUX_W1=targ['FLUX_W1'],
FLUX_W2=targ['FLUX_W2'],
FLUX_IVAR_G=targ['FLUX_IVAR_G'],
FLUX_IVAR_R=targ['FLUX_IVAR_R'],
FLUX_IVAR_Z=targ['FLUX_IVAR_Z'],
FLUX_IVAR_W1=targ['FLUX_IVAR_W1'],
FLUX_IVAR_W2=targ['FLUX_IVAR_W2'],
FIBERFLUX_G=targ['FIBERFLUX_G'],
FIBERFLUX_R=targ['FIBERFLUX_R'],
FIBERFLUX_Z=targ['FIBERFLUX_Z'],
FIBERTOTFLUX_G=targ['FIBERTOTFLUX_G'],
FIBERTOTFLUX_R=targ['FIBERTOTFLUX_R'],
FIBERTOTFLUX_Z=targ['FIBERTOTFLUX_Z'],
MW_TRANSMISSION_G=targ['MW_TRANSMISSION_G'],
MW_TRANSMISSION_R=targ['MW_TRANSMISSION_R'],
MW_TRANSMISSION_Z=targ['MW_TRANSMISSION_Z'],
EBV=targ['EBV'])
specfile = os.path.join(
args.outdir,
'{}_{}_{:04d}s_{:03d}_spect.fits'.format(args.host, thedate,
int(args.exptime), j + 1))
# redshifts = truth['TRUEZ'] if args.host=='bgs' else None
redshifts = None
sim_spectra(wave,
flux,
args.host,
specfile,
sourcetype=args.host,
obsconditions=obs,
meta=obs,
fibermap_columns=fcols,
targetid=truth['TARGETID'],
redshift=redshifts,
ra=targ['RA'],
dec=targ['DEC'],
seed=args.seed,
expid=j)
if args.coadd_cameras:
coaddfile = specfile.replace('spect', 'coadd')
spectra = read_spectra(specfile)
spectra = coadd_cameras(spectra)
write_spectra(coaddfile, spectra)
log.info('Wrote {}'.format(coaddfile))