def test_compute_sky(self):
"""
Tests desi_compute_sky --infile frame.fits --fiberflat fiberflat.fits --outfile skymodel.fits
"""
self._write_frame(flavor='science',
camera='b0') # MUST MATCH FLUXCALIB ABOVE
self._write_fiberflat()
self._write_fibermap()
cmd = "{} {}/desi_compute_sky --infile {} --fiberflat {} --outfile {} --qafile {} --qafig {}".format(
sys.executable, self.binDir, self.framefile, self.fiberflatfile,
self.skyfile, self.qa_data_file, self.qafig)
inputs = [self.framefile, self.fiberflatfile]
outputs = [
self.skyfile,
self.qa_data_file,
self.qafig,
]
err = runcmd(cmd, inputs=inputs, outputs=outputs, clobber=True)
self.assertEqual(err, 0)
#- Remove outputs and call again via function instead of system call
self._remove_files(outputs)
args = lvmspec.scripts.sky.parse(cmd.split()[2:])
err = runcmd(lvmspec.scripts.sky.main,
args=[
args,
],
inputs=inputs,
outputs=outputs,
clobber=True)
self.assertEqual(err, None)
def sim(night, nspec=25, clobber=False):
"""
Simulate data as part of the integration test.
Args:
night (str): YEARMMDD
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
output_dir = os.path.join('$LVM_SPECTRO_REDUX', 'calib2d')
# Create input fibermaps, spectra, and quickgen data
for expid, program in zip([0, 1, 2], ['flat', 'arc', 'dark']):
cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, program=program, nspec=nspec, night=night)
simspec = lvmsim.io.findfile('simspec', night, expid)
fibermap = '{}/fibermap-{:08d}.fits'.format(os.path.dirname(simspec),
expid)
if runcmd(cmd, clobber=clobber) != 0:
raise RuntimeError('newexp failed for {} exposure {}'.format(
program, expid))
cmd = "quickgen --simspec {} --fibermap {}".format(simspec, fibermap)
if runcmd(cmd, clobber=clobber) != 0:
raise RuntimeError('quickgen failed for {} exposure {}'.format(
program, expid))
return
def test_existing_outputs(self):
token = uuid4().hex
cmd = 'echo {} > {}'.format(token, self.testfile)
self.assertEqual(0, util.runcmd(cmd, outputs=[self.outfile]))
fx = open(self.testfile)
line = fx.readline().strip()
#- command should not have run, so tokens should not be equal
self.assertNotEqual(token, line)
def test_clobber(self):
token = uuid4().hex
cmd = 'echo {} > {}'.format(token, self.testfile)
self.assertEqual(
0, util.runcmd(cmd, outputs=[self.outfile], clobber=True))
fx = open(self.testfile)
line = fx.readline().strip()
#- command should have run, so tokens should be equal
self.assertEqual(token, line)
def run_pa(self, deg, flatimage, arcimage, outputfile, args):
from lvmspec.util import runcmd
cmd = "desi_bootcalib --arcfile {} --fiberflat {} --outfile {}".format(
arcimage, flatimage, outputfile)
if runcmd(cmd) != 0:
raise RuntimeError('desi_bootcalib failed, psfboot not written')
img = args[0]
return img
def sim(night, nspec=5, clobber=False):
"""
Simulate data as part of the integration test.
Args:
night (str): YEARMMDD
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
# Create input fibermaps, spectra, and pixel-level raw data
for expid, program in zip([0, 1, 2], ['flat', 'arc', 'dark']):
cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, program=program, nspec=nspec, night=night)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap),
expid)
inputs = []
outputs = [fibermap, simspec]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError(
'newexp-random failed for {} exposure {}'.format(
program, expid))
cmd = "pixsim --preproc --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, nspec=nspec, night=night)
inputs = [fibermap, simspec]
outputs = list()
outputs.append(fibermap.replace('fibermap-', 'simpix-'))
for camera in ['b0', 'r0', 'z0']:
pixfile = io.findfile('pix', night, expid, camera)
outputs.append(pixfile)
#outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-')))
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(
program, expid))
return
def test_compute_fiberflat(self):
"""
Tests desi_compute_fiberflat --infile frame.fits --outfile fiberflat.fits
"""
self._write_frame(flavor='flat')
self._write_fibermap()
# QA fig requires fibermapfile
cmd = '{} {}/desi_compute_fiberflat --infile {} --outfile {} --qafile {} --qafig {}'.format(
sys.executable, self.binDir, self.framefile, self.fiberflatfile,
self.qa_calib_file, self.qafig)
outputs = [self.fiberflatfile, self.qa_calib_file, self.qafig]
inputs = [
self.framefile,
]
err = runcmd(cmd, inputs=inputs, outputs=outputs, clobber=True)
self.assertEqual(err, 0)
#- Confirm that the output file can be read as a fiberflat
ff1 = io.read_fiberflat(self.fiberflatfile)
#- Remove outputs and call again via function instead of system call
self._remove_files(outputs)
args = lvmspec.scripts.fiberflat.parse(cmd.split()[2:])
err = runcmd(lvmspec.scripts.fiberflat.main,
args=[
args,
],
inputs=inputs,
outputs=outputs,
clobber=True)
#- Confirm that the output file can be read as a fiberflat
ff2 = io.read_fiberflat(self.fiberflatfile)
self.assertTrue(np.all(ff1.fiberflat == ff2.fiberflat))
self.assertTrue(np.all(ff1.ivar == ff2.ivar))
self.assertTrue(np.all(ff1.mask == ff2.mask))
self.assertTrue(np.all(ff1.meanspec == ff2.meanspec))
self.assertTrue(np.all(ff1.wave == ff2.wave))
self.assertTrue(np.all(ff1.fibers == ff2.fibers))
def test_zz(self):
"""
Even if clobber=False and outputs exist, run cmd if inputs are
newer than outputs. Run this test last since it alters timestamps.
"""
#- update input timestamp to be newer than output
fx = open(self.infile, 'w')
fx.write('This file is leftover from a test; you can remove it\n')
fx.close()
#- run a command
token = uuid4().hex
cmd = 'echo {} > {}'.format(token, self.testfile)
self.assertEqual(
0, util.runcmd(cmd, outputs=[self.outfile], clobber=False))
#- command should have run even though outputs exist,
#- so tokens should be equal
fx = open(self.testfile)
line = fx.readline().strip()
self.assertNotEqual(token, line)
def test_function(self):
def blat(*args):
return list(args)
self.assertEqual(util.runcmd(blat, args=[1, 2, 3]), [1, 2, 3])
self.assertEqual(util.runcmd(blat), [])
def test_missing_outputs(self):
cmd = 'echo hello > /dev/null'
self.assertNotEqual(0, util.runcmd(cmd, outputs=[uuid4().hex]))
def test_existing_inputs(self):
cmd = 'echo hello > /dev/null'
self.assertEqual(0, util.runcmd(cmd, inputs=[self.infile]))
def test_runcmd(self):
self.assertEqual(0, util.runcmd('echo hello > /dev/null'))
def test_mergeQA(self):
os.environ['QL_SPEC_REDUX'] = self.testDir
cmd = "{} {}/desi_quicklook -i {} -n {} -c {} -e {} --rawdata_dir {} --specprod_dir {} --mergeQA".format(sys.executable,self.binDir,self.configfile,self.night,self.camera,self.expid,self.testDir,self.testDir)
if runcmd(cmd) != 0:
raise RuntimeError('quicklook pipeline failed')
def integration_test(args=None):
"""
Run an integration test from raw data simulations through QuickLook pipeline
Args:
night (str, optional): YEARMMDD
nspec (int, optional): number of spectra to simulate
overwrite (bool, otional) : overwrite existing files
delete (bool, optional) : delete all inputs and outputs
Raises:
RuntimeError if QuickLook fails
"""
#- Parse arguments and check environment variables
args = parse(args)
night = args.night
nspec = args.nspec
raw_dir = os.environ['QL_SPEC_DATA']
output_dir = os.environ['QL_SPEC_REDUX']
#- Simulate inputs
sim(night=night, nspec=nspec)
#- Get the configuration file from lvmspec/data/quicklook
from pkg_resources import resource_filename
configfile = resource_filename('lvmspec',
'data/quicklook/qlconfig_dark.yaml')
for camera in ['r0', 'z0']:
#- Verify that quicklook pipeline runs
com = "desi_quicklook -i {} -n {} -c {} -e 2".format(
configfile, night, camera)
if runcmd(com) != 0:
raise RuntimeError(
'quicklook pipeline failed for camera {}'.format(camera))
#- Remove all output if desired
if args.delete:
sim_dir = os.path.join(os.environ['LVM_SPECTRO_SIM'],
os.environ['PIXPROD'], args.night)
if os.path.exists(sim_dir):
sim_files = os.listdir(sim_dir)
for file in range(len(sim_files)):
sim_file = os.path.join(sim_dir, sim_files[file])
os.remove(sim_file)
os.rmdir(sim_dir)
data_dir = os.path.join(raw_dir, night)
if os.path.exists(data_dir):
data_files = os.listdir(data_dir)
for file in range(len(data_files)):
data_file = os.path.join(data_dir, data_files[file])
os.remove(data_file)
os.rmdir(data_dir)
if os.path.exists(output_dir):
exp_dir = os.path.join(output_dir, 'exposures', night)
if os.path.exists(exp_dir):
id_dir = os.path.join(exp_dir, '00000002')
if os.path.exists(id_dir):
id_files = os.listdir(id_dir)
for file in range(len(id_files)):
id_file = os.path.join(id_dir, id_files[file])
os.remove(id_file)
os.rmdir(id_dir)
exp_files = os.listdir(exp_dir)
for file in range(len(exp_files)):
exp_file = os.path.join(exp_dir, exp_files[file])
os.remove(exp_file)
os.rmdir(exp_dir)
calib_dir = os.path.join(output_dir, 'calib2d', night)
if os.path.exists(calib_dir):
calib_files = os.listdir(calib_dir)
for file in range(len(calib_files)):
calib_file = os.path.join(calib_dir, calib_files[file])
os.remove(calib_file)
os.rmdir(calib_dir)
psf_dir = os.path.join(output_dir, 'calib2d', 'psf', night)
if os.path.exists(psf_dir):
psf_files = os.listdir(psf_dir)
for ii in range(len(psf_files)):
thisfile = os.path.join(psf_dir, psf_files[ii])
os.remove(thisfile)
os.rmdir(psf_dir)
def sim(night, nspec):
"""
Simulate data as part of the QuickLook integration test.
Args:
night (str): YEARMMDD
nspec (int): number of spectra to simulate
Raises:
RuntimeError if any script fails
"""
psf_b = os.path.join(os.environ['LVMMODEL'], 'data', 'specpsf',
'psf-b.fits')
psf_r = os.path.join(os.environ['LVMMODEL'], 'data', 'specpsf',
'psf-r.fits')
psf_z = os.path.join(os.environ['LVMMODEL'], 'data', 'specpsf',
'psf-z.fits')
#- Create files needed to run quicklook
sim_dir = os.path.join(os.environ['LVM_SPECTRO_SIM'],
os.environ['PIXPROD'], night)
data_dir = os.path.join(os.environ['QL_SPEC_DATA'], night)
output_dir = os.environ['QL_SPEC_REDUX']
exp_dir = os.path.join(output_dir, 'exposures', night)
calib_dir = os.path.join(output_dir, 'calib2d', night)
psf_dir = os.path.join(output_dir, 'calib2d', 'psf', night)
cmd = "newarc --nspec {} --night {} --expid 0".format(nspec, night)
if runcmd(cmd) != 0:
raise RuntimeError('newexp failed for arc exposure')
cmd = "newflat --nspec {} --night {} --expid 1".format(nspec, night)
if runcmd(cmd) != 0:
raise RuntimeError('newexp failed for flat exposure')
cmd = "newexp-random --program dark --nspec {} --night {} --expid 2".format(
nspec, night)
if runcmd(cmd) != 0:
raise RuntimeError('newexp failed for dark exposure')
cmd = "pixsim --night {} --expid 0 --nspec {} --rawfile {}/desi-00000000.fits.fz --preproc --preproc_dir {}".format(
night, nspec, data_dir, data_dir)
if runcmd(cmd) != 0:
raise RuntimeError('pixsim failed for arc exposure')
cmd = "pixsim --night {} --expid 1 --nspec {} --rawfile {}/desi-00000001.fits.fz --preproc --preproc_dir {}".format(
night, nspec, data_dir, data_dir)
if runcmd(cmd) != 0:
raise RuntimeError('pixsim failed for flat exposure')
cmd = "pixsim --night {} --expid 2 --nspec {} --rawfile {}/desi-00000002.fits.fz".format(
night, nspec, data_dir)
if runcmd(cmd) != 0:
raise RuntimeError('pixsim failed for dark exposure')
# cmd = "desi_extract_spectra -i {}/pix-b0-00000001.fits -o {}/frame-b0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 3550,5730,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_b,nspec)
# if runcmd(cmd) != 0:
# raise RuntimeError('desi_extract_spectra failed for camera b0')
cmd = "desi_extract_spectra -i {}/pix-r0-00000001.fits -o {}/frame-r0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 5630,7740,0.8 -n {}".format(
data_dir, exp_dir, sim_dir, psf_r, nspec)
if runcmd(cmd) != 0:
raise RuntimeError('desi_extract_spectra failed for camera r0')
cmd = "desi_extract_spectra -i {}/pix-z0-00000001.fits -o {}/frame-z0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 7650,9830,0.8 -n {}".format(
data_dir, exp_dir, sim_dir, psf_z, nspec)
if runcmd(cmd) != 0:
raise RuntimeError('desi_extract_spectra failed for camera z0')
copyfile(os.path.join(sim_dir, 'fibermap-00000002.fits'),
os.path.join(data_dir, 'fibermap-00000002.fits'))
os.remove(os.path.join(data_dir, 'simpix-00000002.fits'))
for camera in ['r0', 'z0']:
cmd = "desi_compute_fiberflat --infile {}/frame-{}-00000001.fits --outfile {}/fiberflat-{}-00000001.fits".format(
exp_dir, camera, calib_dir, camera)
if runcmd(cmd) != 0:
raise RuntimeError(
'desi_compute_fiberflat failed for camera {}'.format(camera))
cmd = "desi_bootcalib --fiberflat {}/pix-{}-00000001.fits --arcfile {}/pix-{}-00000000.fits --outfile {}/psfboot-{}.fits".format(
data_dir, camera, data_dir, camera, psf_dir, camera)
if runcmd(cmd) != 0:
raise RuntimeError(
'desi_bootcalib failed for camera {}'.format(camera))
return
def integration_test(night="20160726", nspec=25, clobber=False):
"""Run an integration test from raw data simulations through redshifts
Args:
night (str, optional): YEARMMDD, defaults to current night
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
log.setLevel(logging.DEBUG)
flat_expid = 0
expid = 2
# check for required environment variables and simulate inputs
check_env()
sim(night, nspec=nspec, clobber=clobber)
for camera in ['b0', 'r0', 'z0']:
# find all necessary input and output files
framefile = lvmspec.io.findfile('frame', night, expid, camera)
fiberflatfile = lvmspec.io.findfile('fiberflat', night, flat_expid,
camera)
skyfile = lvmspec.io.findfile('sky', night, expid, camera)
skytestfile = lvmspec.io.findfile('sky', night, expid, camera) + 'test'
calibfile = lvmspec.io.findfile('calib', night, expid, camera)
calibtestfile = lvmspec.io.findfile('calib', night, expid,
camera) + 'test'
stdstarsfile = lvmspec.io.findfile('stdstars',
night,
expid,
spectrograph=0)
cframetestfile = lvmspec.io.findfile('cframe', night, expid,
camera) + 'test'
# verify that quickgen output works for full pipeline
com = "desi_compute_sky --infile {} --fiberflat {} --outfile {}".format(
framefile, fiberflatfile, skytestfile)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError(
'desi_compute_sky failed for camera {}'.format(camera))
# only fit stdstars once
if camera == 'b0':
com = "desi_fit_stdstars --frames {} --skymodels {} --fiberflats {} --starmodels $DESI_BASIS_TEMPLATES/star_templates_v2.1.fits --outfile {}".format(
framefile, skyfile, fiberflatfile, stdstarsfile)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError(
'desi_fit_stdstars failed for camera {}'.format(camera))
com = "desi_compute_fluxcalibration --infile {} --fiberflat {} --sky {} --models {} --outfile {}".format(
framefile, fiberflatfile, skyfile, stdstarsfile, calibtestfile)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError(
'desi_compute_fluxcalibration failed for camera {}'.format(
camera))
com = "desi_process_exposure --infile {} --fiberflat {} --sky {} --calib {} --outfile {}".format(
framefile, fiberflatfile, skyfile, calibfile, cframetestfile)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError(
'desi_process_exposure failed for camera {}'.format(camera))
com = "desi_make_bricks --night {}".format(night)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError('desi_make_bricks failed')