def integration_test(night=None, nspec=5, clobber=False):
"""Run an integration test from raw data simulations through redshifts
Args:
night (str, optional): YEARMMDD, defaults to current night
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = get_logger()
#- YEARMMDD string, rolls over at noon not midnight
if night is None:
night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600))
#- check for required environment variables
check_env()
#- parameter dictionary that will later be used for formatting commands
params = dict(night=night, nspec=nspec)
#-----
#- Input fibermaps, spectra, and pixel-level raw data
for expid, flavor in zip([0,1,2], ['flat', 'arc', 'science']):
cmd = "pixsim-desi --newexp {flavor} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, flavor=flavor, **params)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid)
inputs = []
outputs = [fibermap, simspec]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('pixsim newexp failed for {} exposure {}'.format(flavor, expid))
cmd = "pixsim-desi --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, **params)
inputs = [fibermap, simspec]
outputs = list()
for camera in ['b0', 'r0', 'z0']:
pixfile = io.findfile('pix', night, expid, camera)
outputs.append(pixfile)
outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-')))
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid))
#-----
#- Extract
waverange = dict(
b = "3570,5940,1.0",
r = "5630,7740,1.0",
z = "7440,9830,1.0",
)
for expid in [0,1,2]:
for channel in ['b', 'r', 'z']:
camera = channel+'0'
pixfile = io.findfile('pix', night, expid, camera)
psffile = '{}/data/specpsf/psf-{}.fits'.format(os.getenv('DESIMODEL'), channel)
framefile = io.findfile('frame', night, expid, camera)
cmd = "exspec -i {pix} -p {psf} --specrange 0,{nspec} -w {wave} -o {frame}".format(
pix=pixfile, psf=psffile, wave=waverange[channel], frame=framefile, **params)
inputs = [pixfile, psffile]
outputs = [framefile,]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('extraction failed for {} expid {}'.format(camera, expid))
#-----
#- Fiber flat
expid = 0
for channel in ['b', 'r', 'z']:
camera = channel+"0"
framefile = io.findfile('frame', night, expid, camera)
fiberflat = io.findfile('fiberflat', night, expid, camera)
cmd = "desi_compute_fiberflat.py --infile {frame} --outfile {fiberflat}".format(
frame=framefile, fiberflat=fiberflat, **params)
inputs = [framefile,]
outputs = [fiberflat,]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('fiberflat failed for '+camera)
#-----
#- Sky model
flat_expid = 0
expid = 2
for channel in ['b', 'r', 'z']:
camera = channel+"0"
framefile = io.findfile('frame', night, expid, camera)
fibermap = io.findfile('fibermap', night, expid)
fiberflat = io.findfile('fiberflat', night, flat_expid, camera)
skyfile = io.findfile('sky', night, expid, camera)
cmd="desi_compute_sky.py --infile {frame} --fibermap {fibermap} --fiberflat {fiberflat} --outfile {sky}".format(
frame=framefile, fibermap=fibermap, fiberflat=fiberflat, sky=skyfile, **params)
inputs = [framefile, fibermap, fiberflat]
outputs = [skyfile, ]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('sky model failed for '+camera)
#-----
#- Fit standard stars
if 'STD_TEMPLATES' in os.environ:
std_templates = os.getenv('STD_TEMPLATES')
else:
std_templates = os.getenv('DESI_ROOT')+'/spectro/templates/stellar_templates/v1.0/stdstar_templates_v1.0.fits'
stdstarfile = io.findfile('stdstars', night, expid, spectrograph=0)
cmd = """desi_fit_stdstars.py --spectrograph 0 \
--fibermap {fibermap} \
--fiberflatexpid {flat_expid} \
--models {std_templates} --outfile {stdstars}""".format(
flat_expid=flat_expid, fibermap=fibermap, std_templates=std_templates,
stdstars=stdstarfile)
inputs = [fibermap, std_templates]
outputs = [stdstarfile,]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('fitting stdstars failed')
#-----
#- Flux calibration
for channel in ['b', 'r', 'z']:
camera = channel+"0"
framefile = io.findfile('frame', night, expid, camera)
fibermap = io.findfile('fibermap', night, expid)
fiberflat = io.findfile('fiberflat', night, flat_expid, camera)
skyfile = io.findfile('sky', night, expid, camera)
calibfile = io.findfile('calib', night, expid, camera)
#- Compute flux calibration vector
cmd = """desi_compute_fluxcalibration.py \
--infile {frame} --fibermap {fibermap} --fiberflat {fiberflat} --sky {sky} \
--models {stdstars} --outfile {calib}""".format(
frame=framefile, fibermap=fibermap, fiberflat=fiberflat, sky=skyfile,
stdstars=stdstarfile, calib=calibfile,
)
inputs = [framefile, fibermap, fiberflat, skyfile, stdstarfile]
outputs = [calibfile,]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('flux calibration failed for '+camera)
#- Apply the flux calibration to write a cframe file
cframefile = io.findfile('cframe', night, expid, camera)
cmd = """desi_process_exposure.py \
--infile {frame} --fiberflat {fiberflat} --sky {sky} --calib {calib} \
--outfile {cframe}""".format(frame=framefile, fibermap=fibermap,
fiberflat=fiberflat, sky=skyfile, calib=calibfile, cframe=cframefile)
inputs = [framefile, fiberflat, skyfile, calibfile]
outputs = [cframefile, ]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('combining calibration steps failed for '+camera)
#-----
#- Bricks
inputs = list()
for camera in ['b0', 'r0', 'z0']:
inputs.append( io.findfile('cframe', night, expid, camera) )
outputs = list()
fibermap = io.read_fibermap(io.findfile('fibermap', night, expid))
bricks = set(fibermap['BRICKNAME'])
for b in bricks:
for channel in ['b', 'r', 'z']:
outputs.append( io.findfile('brick', brickid=b, band=channel))
cmd = "desi_make_bricks.py --night "+night
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('brick generation failed')
#-----
#- Redshifts!
for b in bricks:
inputs = [io.findfile('brick', brickid=b, band=channel) for channel in ['b', 'r', 'z']]
zbestfile = io.findfile('zbest', brickid=b)
outputs = [zbestfile, ]
cmd = "desi_zfind.py --brick {} -o {}".format(b, zbestfile)
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('redshifts failed for brick '+b)
#-----
#- Did it work?
#- (this combination of fibermap, simspec, and zbest is a pain)
simdir = os.path.dirname(io.findfile('fibermap', night=night, expid=expid))
simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid)
siminfo = fits.getdata(simspec, 'METADATA')
print()
print("--------------------------------------------------")
print("Brick True z -> Class z zwarn")
# print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok")
for b in bricks:
zbest = io.read_zbest(io.findfile('zbest', brickid=b))
for i in range(len(zbest.z)):
if zbest.type[i] == 'ssp_em_galaxy':
objtype = 'GAL'
elif zbest.type[i] == 'spEigenStar':
objtype = 'STAR'
else:
objtype = zbest.type[i]
z, zwarn = zbest.z[i], zbest.zwarn[i]
j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0]
truetype = siminfo['OBJTYPE'][j]
truez = siminfo['REDSHIFT'][j]
dv = 3e5*(z-truez)/(1+truez)
if truetype == 'SKY' and zwarn > 0:
status = 'ok'
elif zwarn == 0:
if truetype == 'LRG' and objtype == 'GAL' and abs(dv) < 150:
status = 'ok'
elif truetype == 'ELG' and objtype == 'GAL' and abs(dv) < 150:
status = 'ok'
elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750:
status = 'ok'
elif truetype == 'STD' and objtype == 'STAR':
status = 'ok'
else:
status = 'oops'
else:
status = 'oops'
print('{0} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.format(
b, truetype, truez, objtype, z, zwarn, status))
print("--------------------------------------------------")
def main():
log = get_logger()
key = 'DESI_ROOT'
if key not in os.environ:
log.fatal('Required ${} environment variable not set'.format(key))
return 0
desidir = os.getenv(key)
simsdir = os.path.join(desidir, 'spectro', 'sim', 'bgs-sims1.0')
brickdir = os.path.join(simsdir, 'bricks')
parser = argparse.ArgumentParser()
parser.add_argument('--sim', type=int, default=None, help='Simulation number (see documentation)')
parser.add_argument('--nproc', type=int, default=1, help='Number of processors to use.')
parser.add_argument('--simsdir', default=simsdir, help='Top-level simulation directory')
parser.add_argument('--bricks', action='store_true', help='Generate the brick files.')
parser.add_argument('--zfind', action='store_true', help='Fit for the redshifts.')
parser.add_argument('--results', action='store_true', help='Merge all the relevant results.')
parser.add_argument('--qaplots', action='store_true', help='Generate QAplots.')
args = parser.parse_args()
if args.sim is None:
parser.print_help()
sys.exit(1)
# --------------------------------------------------
# Initialize the parameters of each simulation here.
if args.sim == 1:
seed = 678245
brickname = 'sim01'
nbrick = 50
nspec = 20
phase = 0.25
rmag = (19.5, 19.5)
redshift = (0.1, 0.3)
zenith = 30
exptime = 300
angle = (0, 150)
simoptions = [
'--exptime-range', '{}'.format(exptime), '{}'.format(exptime),
'--rmagrange-bgs', '{}'.format(rmag[0]), '{}'.format(rmag[1]),
'--zrange-bgs', '{}'.format(redshift[0]), '{}'.format(redshift[1]),
'--moon-zenith-range', '{}'.format(zenith), '{}'.format(zenith),
'--moon-phase-range', '{}'.format(phase), '{}'.format(phase),
'--moon-angle-range', '{}'.format(angle[0]), '{}'.format(angle[1])
]
elif args.sim == 2:
seed = 991274
brickname = 'sim02'
nbrick = 50
nspec = 20
phase = (0.0, 1.0)
rmag = (19.5, 19.5)
redshift = (0.1, 0.3)
zenith = 30
exptime = 300
angle = 60
simoptions = [
'--exptime-range', '{}'.format(exptime), '{}'.format(exptime),
'--rmagrange-bgs', '{}'.format(rmag[0]), '{}'.format(rmag[1]),
'--zrange-bgs', '{}'.format(redshift[0]), '{}'.format(redshift[1]),
'--moon-zenith-range', '{}'.format(zenith), '{}'.format(zenith),
'--moon-phase-range', '{}'.format(phase[0]), '{}'.format(phase[1]),
'--moon-angle-range', '{}'.format(angle), '{}'.format(angle)
]
elif args.sim == 3:
seed = 471934
brickname = 'sim03'
nbrick = 50
nspec = 20
phase = (0.0, 1.0)
rmag = (19.5, 19.5)
redshift = (0.1, 0.3)
zenith = 30
exptime = 300
angle = (0, 150)
simoptions = [
'--exptime-range', '{}'.format(exptime), '{}'.format(exptime),
'--rmagrange-bgs', '{}'.format(rmag[0]), '{}'.format(rmag[1]),
'--zrange-bgs', '{}'.format(redshift[0]), '{}'.format(redshift[1]),
'--moon-zenith-range', '{}'.format(zenith), '{}'.format(zenith),
'--moon-phase-range', '{}'.format(phase[0]), '{}'.format(phase[1]),
'--moon-angle-range', '{}'.format(angle[0]), '{}'.format(angle[1])
]
elif args.sim == 4:
seed = 971234
brickname = 'sim04'
nbrick = 100
nspec = 50
phase = (0.0, 1.0)
rmag = (17.5, 20.0)
redshift = (0.1, 0.3)
zenith = 30
exptime = 300
angle = (0, 150)
simoptions = [
'--exptime-range', '{}'.format(exptime), '{}'.format(exptime),
'--rmagrange-bgs', '{}'.format(rmag[0]), '{}'.format(rmag[1]),
'--zrange-bgs', '{}'.format(redshift[0]), '{}'.format(redshift[1]),
'--moon-zenith-range', '{}'.format(zenith), '{}'.format(zenith),
'--moon-phase-range', '{}'.format(phase[0]), '{}'.format(phase[1]),
'--moon-angle-range', '{}'.format(angle[0]), '{}'.format(angle[1])
]
nobj = nbrick*nspec
rand = np.random.RandomState(seed)
# --------------------------------------------------
# Generate the brick and truth files.
if args.bricks:
brightoptions = [
'--brickname', '{}'.format(brickname),
'--nbrick', '{}'.format(nbrick),
'--nspec', '{}'.format(nspec),
'--outdir', '{}'.format(simsdir),
'--brickdir', '{}'.format(brickdir),
'--seed', '{}'.format(seed),
'--objtype', 'BGS']
brightargs = brightsims.parse(np.hstack((brightoptions, simoptions)))
brightargs.verbose = True
brightsims.main(brightargs)
# --------------------------------------------------
# Fit the redshifts
if args.zfind:
inputfile = os.path.join(simsdir, brickname+'-input.fits')
log.info('Reading {}'.format(inputfile))
cat = fits.getdata(inputfile, 1)
log.info('Testing with just one brick!')
#for ib in range(10, 11):
for ib in range(nbrick):
thisbrick = cat['BRICKNAME'][ib]
brickfiles = [os.path.join(brickdir, 'brick-{}-{}.fits'.format(ch, thisbrick)) for ch in ['b', 'r', 'z']]
redoptions = [
'--brick', thisbrick,
'--nproc', '{}'.format(args.nproc),
'--specprod_dir', simsdir,
'--zrange-galaxy', '{}'.format(redshift[0]), '{}'.format(redshift[1]),
'--outfile', os.path.join(brickdir, thisbrick, 'zbest-{}.fits'.format(thisbrick)),
'--objtype', 'ELG,LRG']
redargs = zfind.parse(redoptions)
zfind.main(redargs)
# --------------------------------------------------
# Parse and write out the simulation inputs, brick spectra, and redshifts
if args.results:
inputfile = os.path.join(simsdir, brickname+'-input.fits')
log.info('Reading {}'.format(inputfile))
cat = fits.getdata(inputfile, 1)
# Build a results table.
resultfile = makepath(os.path.join(simsdir, '{}-results.fits'.format(brickname)))
resultcols = [
('EXPTIME', 'f4'),
('AIRMASS', 'f4'),
('MOONPHASE', 'f4'),
('MOONANGLE', 'f4'),
('MOONZENITH', 'f4'),
('SNR_B', 'f4'),
('SNR_R', 'f4'),
('SNR_Z', 'f4'),
('TARGETID', 'i8'),
('RMAG', 'f4'),
('D4000', 'f4'),
('EWHBETA', 'f4'),
('ZTRUE', 'f4'),
('Z', 'f4'),
('ZERR', 'f4'),
('ZWARNING', 'f4')]
result = Table(np.zeros(nobj, dtype=resultcols))
result['EXPTIME'].unit = 's'
result['MOONANGLE'].unit = 'deg'
result['MOONZENITH'].unit = 'deg'
for ib in range(nbrick):
# Copy over some data.
thisbrick = cat['BRICKNAME'][ib]
result['EXPTIME'][nspec*ib:nspec*(ib+1)] = cat['EXPTIME'][ib]
result['AIRMASS'][nspec*ib:nspec*(ib+1)] = cat['AIRMASS'][ib]
result['MOONPHASE'][nspec*ib:nspec*(ib+1)] = cat['MOONPHASE'][ib]
result['MOONANGLE'][nspec*ib:nspec*(ib+1)] = cat['MOONANGLE'][ib]
result['MOONZENITH'][nspec*ib:nspec*(ib+1)] = cat['MOONZENITH'][ib]
# Read the truth file of the first channel to get the metadata.
truthfile = os.path.join(brickdir, thisbrick, 'truth-brick-{}-{}.fits'.format('b', thisbrick))
log.info('Reading {}'.format(truthfile))
truth = io.Brick(truthfile).hdu_list[4].data
result['TARGETID'][nspec*ib:nspec*(ib+1)] = truth['TARGETID']
result['RMAG'][nspec*ib:nspec*(ib+1)] = 22.5-2.5*np.log10(truth['DECAM_FLUX'][:,2])
result['D4000'][nspec*ib:nspec*(ib+1)] = truth['D4000']
result['EWHBETA'][nspec*ib:nspec*(ib+1)] = truth['EWHBETA']
result['ZTRUE'][nspec*ib:nspec*(ib+1)] = truth['TRUEZ']
# Finally read the zbest file.
zbestfile = os.path.join(brickdir, thisbrick, 'zbest-{}.fits'.format(thisbrick))
if os.path.isfile(zbestfile):
log.info('Reading {}'.format(zbestfile))
zbest = read_zbest(zbestfile)
# There's gotta be a better way than looping here!
for ii in range(nspec):
this = np.where(zbest.targetid[ii] == result['TARGETID'])[0]
result['Z'][this] = zbest.z[ii]
result['ZERR'][this] = zbest.zerr[ii]
result['ZWARNING'][this] = zbest.zwarn[ii]
#pdb.set_trace()
# Finally, read the spectra and truth tables, one per channel.
for channel in ('b','r','z'):
brickfile = os.path.join(brickdir, thisbrick, 'brick-{}-{}.fits'.format(channel, thisbrick))
log.info('Reading {}'.format(brickfile))
brick = io.Brick(brickfile)
wave = brick.get_wavelength_grid()
for iobj in range(nspec):
flux = brick.hdu_list[0].data[iobj,:]
ivar = brick.hdu_list[1].data[iobj,:]
these = np.where((wave>np.mean(wave)-50)*(wave<np.mean(wave)+50)*(flux>0))[0]
result['SNR_'+channel.upper()][nspec*ib+iobj] = \
np.median(np.sqrt(flux[these]*ivar[these]))
log.info('Writing {}'.format(resultfile))
write_bintable(resultfile, result, extname='RESULTS', clobber=True)
# --------------------------------------------------
# Build QAplots
if args.qaplots:
phaserange = (-0.05, 1.05)
snrrange = (0, 3)
rmagrange = (17.5, 20)
anglerange = (-5, 155)
d4000range = (0.9, 2.2)
snrinterval = 1.0
cmap = mpl.colors.ListedColormap(sns.color_palette('muted'))
resultfile = os.path.join(simsdir, '{}-results.fits'.format(brickname))
log.info('Reading {}'.format(resultfile))
res = fits.getdata(resultfile, 1)
qafile = os.path.join(simsdir, 'qa-{}.pdf'.format(brickname))
log.info('Writing {}'.format(qafile))
# ------------------------------
# Simulation 1
if args.sim == 1:
fig, ax0 = plt.subplots(1, 1, figsize=(6, 4.5))
ax0.scatter(res['MOONANGLE']+rand.normal(0, 0.2, nobj), res['SNR_B'], label='b channel', c=col[1])
ax0.scatter(res['MOONANGLE']+rand.normal(0, 0.2, nobj), res['SNR_R'], label='r channel', c=col[2])
ax0.scatter(res['MOONANGLE']+rand.normal(0, 0.2, nobj), res['SNR_Z'], label='z channel', c=col[0])
ax0.set_xlabel('Object-Moon Angle (deg)')
ax0.set_ylabel(r'Signal-to-Noise Ratio (pixel$^{-1}$)')
ax0.set_xlim(anglerange)
ax0.set_ylim(snrrange)
plt.legend(loc='upper left', labelspacing=0.25)
plt.text(0.95, 0.7, 't = {:g} s\nr = {:g} mag\nRedshift = {:.2f}-{:.2f}'.format(exptime, rmag,
redshift[0], redshift[1])+\
'\nLunar Zenith Angle = {:g} deg\nLunar Phase = {:g}'.format(zenith, phase),
horizontalalignment='right', transform=ax0.transAxes,
fontsize=11)
plt.subplots_adjust(bottom=0.2, right=0.95, left=0.15)
plt.savefig(qafile)
plt.close()
#pdb.set_trace()
# ------------------------------
# Simulation 2
if args.sim == 2:
fig, ax0 = plt.subplots(1, 1, figsize=(6, 4.5))
ax0.scatter(res['MOONPHASE']+rand.normal(0, 0.02, nobj), res['SNR_B'], label='b channel', c=col[1])
ax0.scatter(res['MOONPHASE']+rand.normal(0, 0.02, nobj), res['SNR_R'], label='r channel', c=col[2])
ax0.scatter(res['MOONPHASE']+rand.normal(0, 0.02, nobj), res['SNR_Z'], label='z channel', c=col[0])
ax0.set_xlabel('Lunar Phase (0=Full, 1=New)')
ax0.set_ylabel(r'Signal-to-Noise Ratio (pixel$^{-1}$)')
ax0.set_xlim(phaserange)
ax0.set_ylim(snrrange)
plt.legend(loc='upper left', labelspacing=0.25)
plt.text(0.95, 0.7, 't = {:g} s\nr = {:g} mag\nRedshift = {:.2f}-{:.2f}'.format(exptime, rmag,
redshift[0], redshift[1])+\
'\nLunar Zenith Angle = {:g} deg\nObject-Moon Angle = {:g} deg'.format(zenith, angle),
horizontalalignment='right', transform=ax0.transAxes,
fontsize=11)
plt.subplots_adjust(bottom=0.2, right=0.95, left=0.15)
plt.savefig(qafile)
plt.close()
# ------------------------------
# Simulation 3
if args.sim == 3:
fig, ax0 = plt.subplots(1, 1, figsize=(6, 4))
im = ax0.scatter(res['MOONANGLE']+rand.normal(0, 0.3, nobj), res['SNR_R'], c=res['MOONPHASE'], vmin=phaserange,
cmap=cmap)
ax0.set_xlabel('Object-Moon Angle (deg)')
ax0.set_ylabel(r'r channel Signal-to-Noise Ratio (pixel$^{-1}$)')
ax0.set_xlim(anglerange)
ax0.set_ylim(snrrange)
ax0.yaxis.set_major_locator(mpl.ticker.MultipleLocator(snrinterval))
plt.text(0.95, 0.7, 't = {:g} s\nr = {:g} mag\nRedshift = {:.2f}-{:.2f}'.format(exptime, rmag,
redshift[0], redshift[1])+\
'\nLunar Zenith Angle = {:g} deg'.format(zenith),
horizontalalignment='right', transform=ax0.transAxes,
fontsize=11)
cbar = fig.colorbar(im)
cbar.set_ticks([0, 0.5, 1])
cbar.ax.set_yticklabels(['Full','Quarter','New'], rotation=90)
cbar.ax.set_ylabel('Lunar Phase')
plt.subplots_adjust(bottom=0.2, right=1.0)
plt.savefig(qafile)
plt.close()
# ------------------------------
# Simulation 4
if args.sim == 4:
bins = 30
zgood = (np.abs(res['Z']-res['ZTRUE'])<5E-5)*(res['ZWARNING']==0)*1
H, xedges, yedges = np.histogram2d(res['RMAG'], res['MOONPHASE'], bins=bins, weights=zgood)
H2, _, _ = np.histogram2d(res['RMAG'], res['MOONPHASE'], bins=bins)
extent = np.array((rmagrange, phaserange)).flatten()
#pdb.set_trace()
fig, ax0 = plt.subplots(1, 1, figsize=(6, 4))
im = ax0.imshow(H/H2, extent=extent, interpolation='nearest')#, cmap=cmap)
ax0.set_xlabel('r (AB mag)')
ax0.set_ylabel('Lunar Phase (0=Full, 1=New)')
ax0.set_xlim(rmagrange)
ax0.set_ylim(phaserange)
#plt.text(0.95, 0.7, 't = {:g} s\nr = {:g} mag\nRedshift = {:.2f}-{:.2f}'.format(exptime, rmag,
# redshift[0], redshift[1])+\
# '\nLunar Zenith Angle = {:g} deg'.format(zenith),
# horizontalalignment='right', transform=ax0.transAxes,
# fontsize=11)
cbar = fig.colorbar(im)
#cbar.set_ticks([0, 0.5, 1])
#cbar.ax.set_yticklabels(['Full','Quarter','New'], rotation=90)
#cbar.ax.set_ylabel('Lunar Phase')
plt.subplots_adjust(bottom=0.2, right=1.0)
plt.savefig(qafile)
plt.close()
# ------------------------------
# Simulation 10
if args.sim == 10:
zgood = (np.abs(res['Z']-res['ZTRUE'])<0.001)*(res['ZWARNING']==0)*1
#pdb.set_trace()
fig, (ax0, ax1, ax2) = plt.subplots(3, 1, figsize=(6,6))
# moon phase vs S/N(r)
im = ax0.scatter(res['RMAG'], res['SNR_R'], c=res['MOONPHASE'], vmin=phaserange,
cmap=cmap)
ax0.set_xlabel('r (AB mag)')
ax0.set_ylabel('S/N (r channel)')
ax0.set_xlim(rmagrange)
ax0.set_ylim(snrrange)
ax0.yaxis.set_major_locator(mpl.ticker.MultipleLocator(snrinterval))
# object-moon angle vs S/N(r)
im = ax1.scatter(res['MOONANGLE'], res['SNR_R'], c=res['MOONPHASE'], vmin=phaserange,
cmap=cmap)
ax1.set_xlabel('Object-Moon Angle (deg)')
ax1.set_ylabel('S/N (r channel)')
ax1.set_xlim(anglerange)
ax1.set_ylim(snrrange)
ax1.yaxis.set_major_locator(mpl.ticker.MultipleLocator(snrinterval))
# D(4000) vs S/N(r)
im = ax2.scatter(res['D4000'], res['SNR_R'], c=res['MOONPHASE'], vmin=phaserange,
cmap=cmap)
ax2.set_xlabel('$D_{n}(4000)$')
ax2.set_ylabel('S/N (r channel)')
ax2.set_xlim(d4000range)
ax2.set_ylim(snrrange)
ax2.yaxis.set_major_locator(mpl.ticker.MultipleLocator(snrinterval))
# Shared colorbar
cbarax = fig.add_axes([0.83, 0.15, 0.03, 0.8])
cbar = fig.colorbar(im, cax=cbarax)
ticks = ['Full','Quarter','New']
cbar.set_ticks([0, 0.5, 1])
cbar.ax.set_yticklabels(ticks, rotation=-45)
plt.tight_layout(pad=0.5)#, h_pad=0.2, w_pad=0.3)
plt.subplots_adjust(right=0.78)
plt.savefig(qafile)
plt.close()
def integration_test(night=None, nspec=5, clobber=False):
"""Run an integration test from raw data simulations through redshifts
Args:
night (str, optional): YEARMMDD, defaults to current night
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
log.setLevel(logging.DEBUG)
# YEARMMDD string, rolls over at noon not midnight
# TODO: fix usage of night to be something other than today
if night is None:
#night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600))
night = "20160726"
# check for required environment variables
check_env()
# simulate inputs
sim(night, nspec=nspec, clobber=clobber)
# create production
# FIXME: someday run PSF estimation too...
### com = "desi_pipe --spectrographs 0 --fakeboot --fakepsf"
com = "desi_pipe --spectrographs 0 --fakeboot --fakepsf"
sp.check_call(com, shell=True)
# raw and production locations
rawdir = os.path.abspath(io.rawdata_root())
proddir = os.path.abspath(io.specprod_root())
# Modify options file to restrict the spectral range
optpath = os.path.join(proddir, "run", "options.yaml")
opts = pipe.yaml_read(optpath)
opts['extract']['specmin'] = 0
opts['extract']['nspec'] = nspec
pipe.yaml_write(optpath, opts)
# run the generated shell scripts
# FIXME: someday run PSF estimation too...
# print("Running bootcalib script...")
# com = os.path.join(proddir, "run", "scripts", "bootcalib_all.sh")
# sp.check_call(["bash", com])
# print("Running specex script...")
# com = os.path.join(proddir, "run", "scripts", "specex_all.sh")
# sp.check_call(["bash", com])
# print("Running psfcombine script...")
# com = os.path.join(proddir, "run", "scripts", "psfcombine_all.sh")
# sp.check_call(["bash", com])
com = os.path.join(proddir, "run", "scripts", "run_shell.sh")
print("Running extraction through calibration: " + com)
sp.check_call(["bash", com])
com = os.path.join(proddir, "run", "scripts", "spectra.sh")
print("Running spectral regrouping: " + com)
sp.check_call(["bash", com])
com = os.path.join(proddir, "run", "scripts", "redshift.sh")
print("Running redshift script " + com)
sp.check_call(["bash", com])
# #-----
# #- Did it work?
# #- (this combination of fibermap, simspec, and zbest is a pain)
expid = 2
fmfile = io.findfile('fibermap', night=night, expid=expid)
fibermap = io.read_fibermap(fmfile)
simdir = os.path.dirname(fmfile)
simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid)
siminfo = fits.getdata(simspec, 'METADATA')
brickdirs = glob.glob(os.path.join(proddir, "bricks", "*"))
bricks = [os.path.basename(x) for x in brickdirs]
print()
print("--------------------------------------------------")
print("Brick True z -> Class z zwarn")
# print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok")
for b in bricks:
zbest = io.read_zbest(io.findfile('zbest', brickname=b))
for i in range(len(zbest.z)):
if zbest.spectype[i] == 'ssp_em_galaxy':
objtype = 'GAL'
elif zbest.spectype[i] == 'spEigenStar':
objtype = 'STAR'
else:
objtype = zbest.spectype[i]
z, zwarn = zbest.z[i], zbest.zwarn[i]
j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0]
truetype = siminfo['OBJTYPE'][j]
oiiflux = siminfo['OIIFLUX'][j]
truez = siminfo['REDSHIFT'][j]
dv = 3e5 * (z - truez) / (1 + truez)
if truetype == 'SKY' and zwarn > 0:
status = 'ok'
elif truetype == 'ELG' and zwarn > 0 and oiiflux < 8e-17:
status = 'ok ([OII] flux {:.2g})'.format(oiiflux)
elif zwarn == 0:
if truetype == 'LRG' and objtype == 'GAL' and abs(dv) < 150:
status = 'ok'
elif truetype == 'ELG' and objtype == 'GAL':
if abs(dv) < 150 or oiiflux < 8e-17:
status = 'ok ([OII] flux {:.2g})'.format(oiiflux)
else:
status = 'OOPS ([OII] flux {:.2g})'.format(oiiflux)
elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750:
status = 'ok'
elif truetype in ('STD', 'FSTD') and objtype == 'STAR':
status = 'ok'
else:
status = 'OOPS'
else:
status = 'OOPS'
print('{0} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.
format(b, truetype, truez, objtype, z, zwarn, status))
print("--------------------------------------------------")
def integration_test(night=None, nspec=5, clobber=False):
"""Run an integration test from raw data simulations through redshifts
Args:
night (str, optional): YEARMMDD, defaults to current night
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
log.setLevel(logging.DEBUG)
# YEARMMDD string, rolls over at noon not midnight
# TODO: fix usage of night to be something other than today
if night is None:
#night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600))
night = "20160726"
# check for required environment variables
check_env()
# simulate inputs
sim(night, nspec=nspec, clobber=clobber)
# create production
# FIXME: someday run PSF estimation too...
### com = "desi_pipe --env env.txt --spectrographs 0 --fakeboot --fakepsf"
rawdir = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD'))
com = "desi_pipe --spectrographs 0 --fakeboot --fakepsf --raw {}".format(rawdir)
sp.check_call(com, shell=True)
# raw and production locations
rawdir = os.path.abspath(io.rawdata_root())
proddir = os.path.abspath(io.specprod_root())
# Modify options file to restrict the spectral range
optpath = os.path.join(proddir, "run", "options.yaml")
opts = pipe.read_options(optpath)
opts['extract']['specmin'] = 0
opts['extract']['nspec'] = nspec
pipe.write_options(optpath, opts)
# run the generated shell scripts
# FIXME: someday run PSF estimation too...
# print("Running bootcalib script...")
# com = os.path.join(proddir, "run", "scripts", "bootcalib_all.sh")
# sp.check_call(["bash", com])
# print("Running specex script...")
# com = os.path.join(proddir, "run", "scripts", "specex_all.sh")
# sp.check_call(["bash", com])
# print("Running psfcombine script...")
# com = os.path.join(proddir, "run", "scripts", "psfcombine_all.sh")
# sp.check_call(["bash", com])
com = os.path.join(proddir, "run", "scripts", "extract_all.sh")
print("Running extraction script "+com)
sp.check_call(["bash", com])
com = os.path.join(proddir, "run", "scripts", "fiberflat-procexp_all.sh")
print("Running calibration script "+com)
sp.check_call(["bash", com])
com = os.path.join(proddir, "run", "scripts", "bricks.sh")
print("Running makebricks script "+com)
sp.check_call(["bash", com])
com = os.path.join(proddir, "run", "scripts", "zfind_all.sh")
print("Running zfind script "+com)
sp.check_call(["bash", com])
# #-----
# #- Did it work?
# #- (this combination of fibermap, simspec, and zbest is a pain)
expid = 2
fmfile = io.findfile('fibermap', night=night, expid=expid)
fibermap = io.read_fibermap(fmfile)
simdir = os.path.dirname(fmfile)
simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid)
siminfo = fits.getdata(simspec, 'METADATA')
brickdirs = glob.glob(os.path.join(proddir, "bricks", "*"))
bricks = [ os.path.basename(x) for x in brickdirs ]
print()
print("--------------------------------------------------")
print("Brick True z -> Class z zwarn")
# print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok")
for b in bricks:
zbest = io.read_zbest(io.findfile('zbest', brickname=b))
for i in range(len(zbest.z)):
if zbest.spectype[i] == 'ssp_em_galaxy':
objtype = 'GAL'
elif zbest.spectype[i] == 'spEigenStar':
objtype = 'STAR'
else:
objtype = zbest.spectype[i]
z, zwarn = zbest.z[i], zbest.zwarn[i]
j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0]
truetype = siminfo['OBJTYPE'][j]
truez = siminfo['REDSHIFT'][j]
dv = 3e5*(z-truez)/(1+truez)
if truetype == 'SKY' and zwarn > 0:
status = 'ok'
elif zwarn == 0:
if truetype == 'LRG' and objtype == 'GAL' and abs(dv) < 150:
status = 'ok'
elif truetype == 'ELG' and objtype == 'GAL' and abs(dv) < 150:
status = 'ok'
elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750:
status = 'ok'
elif truetype == 'STD' and objtype == 'STAR':
status = 'ok'
else:
status = 'OOPS'
else:
status = 'OOPS'
print('{0} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.format(
b, truetype, truez, objtype, z, zwarn, status))
print("--------------------------------------------------")
