def sim(night, nspec=5, clobber=False):
"""
Simulate data as part of the integration test.
Args:
night (str): YEARMMDD
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
# Create input fibermaps, spectra, and pixel-level raw data
for expid, program in zip([0,1,2], ['flat', 'arc', 'dark']):
cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, program=program, nspec=nspec, night=night)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid)
inputs = []
outputs = [fibermap, simspec]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('newexp-random failed for {} exposure {}'.format(program, expid))
cmd = "pixsim --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, nspec=nspec, night=night)
inputs = [fibermap, simspec]
outputs = list()
outputs.append(fibermap.replace('fibermap-', 'simpix-'))
outputs.append(io.findfile('raw', night, expid))
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(program, expid))
return
def test_compute_fiberflat(self):
"""
Tests desi_compute_fiberflat --infile frame.fits --outfile fiberflat.fits
"""
self._write_frame(flavor='flat')
self._write_fibermap()
# QA fig requires fibermapfile
cmd = '{} {}/desi_compute_fiberflat --infile {} --outfile {} --qafile {} --qafig {}'.format(
sys.executable, self.binDir, self.framefile,
self.fiberflatfile, self.qa_calib_file, self.qafig)
outputs = [self.fiberflatfile,self.qa_calib_file,self.qafig]
inputs = [self.framefile,]
err = runcmd(cmd, inputs=inputs, outputs=outputs, clobber=True)
self.assertEqual(err, 0, 'FAILED: {}'.format(cmd))
#- Confirm that the output file can be read as a fiberflat
ff1 = io.read_fiberflat(self.fiberflatfile)
#- Remove outputs and call again via function instead of system call
self._remove_files(outputs)
args = desispec.scripts.fiberflat.parse(cmd.split()[2:])
err = runcmd(desispec.scripts.fiberflat.main, args=[args,],
inputs=inputs, outputs=outputs, clobber=True)
#- Confirm that the output file can be read as a fiberflat
ff2 = io.read_fiberflat(self.fiberflatfile)
self.assertTrue(np.all(ff1.fiberflat == ff2.fiberflat))
self.assertTrue(np.all(ff1.ivar == ff2.ivar))
self.assertTrue(np.all(ff1.mask == ff2.mask))
self.assertTrue(np.all(ff1.meanspec == ff2.meanspec))
self.assertTrue(np.all(ff1.wave == ff2.wave))
self.assertTrue(np.all(ff1.fibers == ff2.fibers))
def sim(night, nspec=25, clobber=False):
"""
Simulate data as part of the integration test.
Args:
night (str): YEARMMDD
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
output_dir = os.path.join('$DESI_SPECTRO_REDUX','calib2d')
# Create input fibermaps, spectra, and quickgen data
for expid, program in zip([0,1,2], ['flat', 'arc', 'dark']):
cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, program=program, nspec=nspec, night=night)
simspec = desisim.io.findfile('simspec', night, expid)
fibermap = '{}/fibermap-{:08d}.fits'.format(os.path.dirname(simspec),expid)
if runcmd(cmd, clobber=clobber) != 0:
raise RuntimeError('newexp failed for {} exposure {}'.format(program, expid))
cmd = "quickgen --simspec {} --fibermap {}".format(simspec,fibermap)
if runcmd(cmd, clobber=clobber) != 0:
raise RuntimeError('quickgen failed for {} exposure {}'.format(program, expid))
return
def integration_test(night="20160726", nspec=25, clobber=False):
"""Run an integration test from raw data simulations through redshifts
Args:
night (str, optional): YEARMMDD, defaults to current night
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
log.setLevel(logging.DEBUG)
flat_expid = 0
expid = 2
# check for required environment variables and simulate inputs
check_env()
sim(night, nspec=nspec, clobber=clobber)
for camera in ['b0', 'r0', 'z0']:
# find all necessary input and output files
framefile = desispec.io.findfile('frame', night, expid, camera)
fiberflatfile = desispec.io.findfile('fiberflat', night, flat_expid, camera)
skyfile = desispec.io.findfile('sky', night, expid, camera)
skytestfile = desispec.io.findfile('sky', night, expid, camera) + 'test'
calibfile = desispec.io.findfile('calib', night, expid, camera)
calibtestfile = desispec.io.findfile('calib', night, expid, camera) + 'test'
stdstarsfile = desispec.io.findfile('stdstars', night, expid, spectrograph=0)
cframetestfile = desispec.io.findfile('cframe', night, expid, camera) + 'test'
# verify that quickgen output works for full pipeline
com = "desi_compute_sky --infile {} --fiberflat {} --outfile {}".format(framefile, fiberflatfile, skytestfile)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError('desi_compute_sky failed for camera {}'.format(camera))
# only fit stdstars once
if camera == 'b0':
com = "desi_fit_stdstars --frames {} --skymodels {} --fiberflats {} --starmodels $DESI_BASIS_TEMPLATES/star_templates_v2.1.fits --outfile {}".format(framefile, skyfile, fiberflatfile, stdstarsfile)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError('desi_fit_stdstars failed for camera {}'.format(camera))
com = "desi_compute_fluxcalibration --infile {} --fiberflat {} --sky {} --models {} --outfile {}".format(framefile, fiberflatfile, skyfile, stdstarsfile, calibtestfile)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError('desi_compute_fluxcalibration failed for camera {}'.format(camera))
com = "desi_process_exposure --infile {} --fiberflat {} --sky {} --calib {} --outfile {}".format(framefile, fiberflatfile, skyfile, calibfile, cframetestfile)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError('desi_process_exposure failed for camera {}'.format(camera))
com = "desi_make_bricks --night {}".format(night)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError('desi_make_bricks failed')
def test_clobber(self):
token = uuid4().hex
cmd = 'echo {} > {}'.format(token, self.testfile)
self.assertEqual(0, util.runcmd(cmd, outputs=[self.outfile], clobber=True))
fx = open(self.testfile)
line = fx.readline().strip()
#- command should have run, so tokens should be equal
self.assertEqual(token, line)
def test_existing_outputs(self):
token = uuid4().hex
cmd = 'echo {} > {}'.format(token, self.testfile)
self.assertEqual(0, util.runcmd(cmd, outputs=[self.outfile]))
fx = open(self.testfile)
line = fx.readline().strip()
#- command should not have run, so tokens should not be equal
self.assertNotEqual(token, line)
def test_compute_sky(self):
"""
Tests desi_compute_sky --infile frame.fits --fiberflat fiberflat.fits --outfile skymodel.fits
"""
self._write_frame(flavor='science', camera='b0') # MUST MATCH FLUXCALIB ABOVE
self._write_fiberflat()
self._write_fibermap()
cmd = "{} {}/desi_compute_sky --infile {} --fiberflat {} --outfile {} --qafile {} --qafig {}".format(
sys.executable, self.binDir, self.framefile, self.fiberflatfile, self.skyfile, self.qa_data_file, self.qafig)
inputs = [self.framefile, self.fiberflatfile]
outputs = [self.skyfile,self.qa_data_file,self.qafig,]
err = runcmd(cmd, inputs=inputs, outputs=outputs, clobber=True)
self.assertEqual(err, 0, 'FAILED: {}'.format(cmd))
#- Remove outputs and call again via function instead of system call
self._remove_files(outputs)
args = desispec.scripts.sky.parse(cmd.split()[2:])
err = runcmd(desispec.scripts.sky.main, args=[args,],
inputs=inputs, outputs=outputs, clobber=True)
self.assertEqual(err, None)
def test_zz(self):
"""
Even if clobber=False and outputs exist, run cmd if inputs are
newer than outputs. Run this test last since it alters timestamps.
"""
#- update input timestamp to be newer than output
fx = open(self.infile, 'w')
fx.write('This file is leftover from a test; you can remove it\n')
fx.close()
#- run a command
token = uuid4().hex
cmd = 'echo {} > {}'.format(token, self.testfile)
self.assertEqual(0, util.runcmd(cmd, outputs=[self.outfile], clobber=False))
#- command should have run even though outputs exist,
#- so tokens should be equal
fx = open(self.testfile)
line = fx.readline().strip()
self.assertNotEqual(token, line)
def test_missing_outputs(self):
cmd = 'echo hello > /dev/null'
self.assertNotEqual(0, util.runcmd(cmd, outputs=[uuid4().hex]))
def test_existing_inputs(self):
cmd = 'echo hello > /dev/null'
self.assertEqual(0, util.runcmd(cmd, inputs=[self.infile]))
def test_runcmd(self):
self.assertEqual(0, util.runcmd('echo hello > /dev/null'))
def main(args=None, comm=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
log = get_logger()
start_mpi_connect = time.time()
if comm is not None:
#- Use the provided comm to determine rank and size
rank = comm.rank
size = comm.size
else:
#- Check MPI flags and determine the comm, rank, and size given the arguments
comm, rank, size = assign_mpi(do_mpi=args.mpi,
do_batch=args.batch,
log=log)
stop_mpi_connect = time.time()
#- Start timer; only print log messages from rank 0 (others are silent)
timer = desiutil.timer.Timer(silent=(rank > 0))
#- Fill in timing information for steps before we had the timer created
if args.starttime is not None:
timer.start('startup', starttime=args.starttime)
timer.stop('startup', stoptime=start_imports)
timer.start('imports', starttime=start_imports)
timer.stop('imports', stoptime=stop_imports)
timer.start('mpi_connect', starttime=start_mpi_connect)
timer.stop('mpi_connect', stoptime=stop_mpi_connect)
#- Freeze IERS after parsing args so that it doesn't bother if only --help
timer.start('freeze_iers')
desiutil.iers.freeze_iers()
timer.stop('freeze_iers')
#- Preflight checks
timer.start('preflight')
# - Preflight checks
if rank > 0:
# - Let rank 0 fetch these, and then broadcast
args, hdr, camhdr = None, None, None
else:
if args.inputs is None:
if args.night is None or args.expids is None:
raise RuntimeError(
'Must specify --inputs or --night AND --expids')
else:
args.expids = np.array(
args.expids.strip(' \t').split(',')).astype(int)
args.inputs = []
for expid in args.expids:
infile = findfile('raw', night=args.night, expid=expid)
args.inputs.append(infile)
if not os.path.isfile(infile):
raise IOError('Missing input file: {}'.format(infile))
else:
args.inputs = np.array(args.inputs.strip(' \t').split(','))
#- args.night will be defined in update_args_with_headers,
#- but let's define the expids here
#- NOTE: inputs has priority. Overwriting expids if they existed.
args.expids = []
for infile in args.inputs:
hdr = load_raw_data_header(pathname=infile,
return_filehandle=False)
args.expids.append(int(hdr['EXPID']))
args.expids = np.sort(args.expids)
args.inputs = np.sort(args.inputs)
args.expid = args.expids[0]
args.input = args.inputs[0]
#- Use header information to fill in missing information in the arguments object
args, hdr, camhdr = update_args_with_headers(args)
#- If not a science observation, we don't need the hdr or camhdr objects,
#- So let's not broadcast them to all the ranks
if args.obstype != 'SCIENCE':
hdr, camhdr = None, None
if comm is not None:
args = comm.bcast(args, root=0)
hdr = comm.bcast(hdr, root=0)
camhdr = comm.bcast(camhdr, root=0)
known_obstype = ['SCIENCE', 'ARC', 'FLAT']
if args.obstype not in known_obstype:
raise RuntimeError('obstype {} not in {}'.format(
args.obstype, known_obstype))
timer.stop('preflight')
# -------------------------------------------------------------------------
# - Create and submit a batch job if requested
if args.batch:
#camword = create_camword(args.cameras)
#exp_str = '-'.join('{:08d}'.format(expid) for expid in args.expids)
if args.obstype.lower() == 'science':
jobdesc = 'stdstarfit'
elif args.obstype.lower() == 'arc':
jobdesc = 'psfnight'
elif args.obstype.lower() == 'flat':
jobdesc = 'nightlyflat'
else:
jobdesc = args.obstype.lower()
scriptfile = create_desi_proc_batch_script(night=args.night, exp=args.expids, cameras=args.cameras,\
jobdesc=jobdesc, queue=args.queue, runtime=args.runtime,\
batch_opts=args.batch_opts, timingfile=args.timingfile,
system_name=args.system_name)
err = 0
if not args.nosubmit:
err = subprocess.call(['sbatch', scriptfile])
sys.exit(err)
# -------------------------------------------------------------------------
# - Proceed with running
# - What are we going to do?
if rank == 0:
log.info('----------')
log.info('Input {}'.format(args.inputs))
log.info('Night {} expids {}'.format(args.night, args.expids))
log.info('Obstype {}'.format(args.obstype))
log.info('Cameras {}'.format(args.cameras))
log.info('Output root {}'.format(desispec.io.specprod_root()))
log.info('----------')
# - Wait for rank 0 to make directories before proceeding
if comm is not None:
comm.barrier()
# -------------------------------------------------------------------------
# - Merge PSF of night if applicable
if args.obstype in ['ARC']:
timer.start('psfnight')
num_cmd = num_err = 0
if rank == 0:
for camera in args.cameras:
psfnightfile = findfile('psfnight', args.night, args.expids[0],
camera)
if not os.path.isfile(
psfnightfile
): # we still don't have a psf night, see if we can compute it ...
psfs = [
findfile('psf', args.night, expid,
camera).replace("psf", "fit-psf")
for expid in args.expids
]
log.info(
"Number of PSF for night={} camera={} = {}".format(
args.night, camera, len(psfs)))
if len(psfs) > 4: # lets do it!
log.info("Computing psfnight ...")
dirname = os.path.dirname(psfnightfile)
if not os.path.isdir(dirname):
os.makedirs(dirname)
num_cmd += 1
#- generic try/except so that any failure doesn't leave
#- MPI rank 0 hanging while others are waiting for it
try:
desispec.scripts.specex.mean_psf(
psfs, psfnightfile)
except:
log.error('specex.meanpsf failed for {}'.format(
os.path.basename(psfnightfile)))
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(
exc_type, exc_value, exc_traceback)
log.error(''.join(lines))
sys.stdout.flush()
if not os.path.exists(psfnightfile):
log.error(f'Failed to create {psfnightfile}')
num_err += 1
else:
log.info(
"Fewer than 4 psfs were provided, can't compute psfnight. Exiting ..."
)
num_cmd += 1
num_err += 1
timer.stop('psfnight')
num_cmd, num_err = mpi_count_failures(num_cmd, num_err, comm=comm)
if rank == 0:
if num_err > 0:
log.error(f'{num_err}/{num_cmd} psfnight commands failed')
if num_err > 0 and num_err == num_cmd:
sys.stdout.flush()
if rank == 0:
log.critical('All psfnight commands failed')
sys.exit(1)
# -------------------------------------------------------------------------
# - Average and auto-calib fiberflats of night if applicable
if args.obstype in ['FLAT']:
timer.start('fiberflatnight')
#- Track number of commands run and number of errors for exit code
num_cmd = 0
num_err = 0
if rank == 0:
fiberflatnightfile = findfile('fiberflatnight', args.night,
args.expids[0], args.cameras[0])
fiberflatdirname = os.path.dirname(fiberflatnightfile)
if os.path.isfile(fiberflatnightfile):
log.info("Fiberflatnight already exists. Exitting ...")
elif len(
args.cameras
) < 6: # we still don't have them, see if we can compute them
# , but need at least 2 spectros ...
log.info(
"Fewer than 6 cameras were available, so couldn't perform joint fit. Exiting ..."
)
else:
flats = []
for camera in args.cameras:
for expid in args.expids:
flats.append(
findfile('fiberflat', args.night, expid, camera))
log.info("Number of fiberflat for night {} = {}".format(
args.night, len(flats)))
if len(flats) < 3 * 4 * len(args.cameras):
log.info(
"Fewer than 3 exposures with 4 lamps were available. Can't perform joint fit. Exiting..."
)
else:
log.info(
"Computing fiberflatnight per lamp and camera ...")
tmpdir = os.path.join(fiberflatdirname, "tmp")
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
log.info(
"First average measurements per camera and per lamp")
average_flats = dict()
for camera in args.cameras:
# list of flats for this camera
flats_for_this_camera = []
for flat in flats:
if flat.find(camera) >= 0:
flats_for_this_camera.append(flat)
# log.info("For camera {} , flats = {}".format(camera,flats_for_this_camera))
# sys.exit(12)
# average per lamp (and camera)
average_flats[camera] = list()
for lampbox in range(4):
ofile = os.path.join(
tmpdir,
"fiberflatnight-camera-{}-lamp-{}.fits".format(
camera, lampbox))
if not os.path.isfile(ofile):
log.info(
"Average flat for camera {} and lamp box #{}"
.format(camera, lampbox))
pg = "CALIB DESI-CALIB-0{} LEDs only".format(
lampbox)
cmd = "desi_average_fiberflat --program '{}' --outfile {} -i ".format(
pg, ofile)
for flat in flats_for_this_camera:
cmd += " {} ".format(flat)
num_cmd += 1
err = runcmd(cmd,
inputs=flats_for_this_camera,
outputs=[
ofile,
])
if err:
num_err += 1
if os.path.isfile(ofile):
average_flats[camera].append(ofile)
else:
log.error(
f"Generating {ofile} failed; proceeding with other flats"
)
else:
log.info("Will use existing {}".format(ofile))
average_flats[camera].append(ofile)
log.info(
"Auto-calibration across lamps and spectro per camera arm (b,r,z)"
)
for camera_arm in ["b", "r", "z"]:
cameras_for_this_arm = []
flats_for_this_arm = []
for camera in args.cameras:
if camera[0].lower() == camera_arm:
cameras_for_this_arm.append(camera)
if camera in average_flats:
for flat in average_flats[camera]:
flats_for_this_arm.append(flat)
if len(flats_for_this_arm) > 0:
cmd = "desi_autocalib_fiberflat --night {} --arm {} -i ".format(
args.night, camera_arm)
for flat in flats_for_this_arm:
cmd += " {} ".format(flat)
num_cmd += 1
err = runcmd(cmd,
inputs=flats_for_this_arm,
outputs=[])
if err:
num_err += 1
else:
log.error(f'No flats found for arm {camera_arm}')
log.info("Done with fiber flats per night")
timer.stop('fiberflatnight')
num_cmd, num_err = mpi_count_failures(num_cmd, num_err, comm=comm)
if comm is not None:
comm.barrier()
if rank == 0:
if num_err > 0:
log.error(f'{num_err}/{num_cmd} fiberflat commands failed')
if num_err > 0 and num_err == num_cmd:
if rank == 0:
log.critical('All fiberflat commands failed')
sys.exit(1)
##################### Note #############################
### Still for single exposure. Needs to be re-factored #
########################################################
if args.obstype in ['SCIENCE']:
#inputfile = findfile('raw', night=args.night, expid=args.expids[0])
#if not os.path.isfile(inputfile):
# raise IOError('Missing input file: {}'.format(inputfile))
## - Fill in values from raw data header if not overridden by command line
#fx = fitsio.FITS(inputfile)
#if 'SPEC' in fx: # - 20200225 onwards
# # hdr = fits.getheader(args.input, 'SPEC')
# hdr = fx['SPEC'].read_header()
#elif 'SPS' in fx: # - 20200224 and before
# # hdr = fits.getheader(args.input, 'SPS')
# hdr = fx['SPS'].read_header()
#else:
# # hdr = fits.getheader(args.input, 0)
# hdr = fx[0].read_header()
#
#camhdr = dict()
#for cam in args.cameras:
# camhdr[cam] = fx[cam].read_header()
#fx.close()
timer.start('stdstarfit')
num_err = num_cmd = 0
if rank == 0:
log.info('Starting stdstar fitting at {}'.format(time.asctime()))
# -------------------------------------------------------------------------
# - Get input fiberflat
input_fiberflat = dict()
if rank == 0:
for camera in args.cameras:
if args.fiberflat is not None:
input_fiberflat[camera] = args.fiberflat
elif args.calibnight is not None:
# look for a fiberflatnight for this calib night
fiberflatnightfile = findfile('fiberflatnight',
args.calibnight,
args.expids[0], camera)
if not os.path.isfile(fiberflatnightfile):
log.error("no {}".format(fiberflatnightfile))
raise IOError("no {}".format(fiberflatnightfile))
input_fiberflat[camera] = fiberflatnightfile
else:
# look for a fiberflatnight fiberflat
fiberflatnightfile = findfile('fiberflatnight', args.night,
args.expids[0], camera)
if os.path.isfile(fiberflatnightfile):
input_fiberflat[camera] = fiberflatnightfile
elif args.most_recent_calib:
# -- NOTE: Finding most recent only with respect to the first night
nightfile = find_most_recent(args.night,
file_type='fiberflatnight')
if nightfile is None:
input_fiberflat[camera] = findcalibfile(
[hdr, camhdr[camera]], 'FIBERFLAT')
else:
input_fiberflat[camera] = nightfile
else:
input_fiberflat[camera] = findcalibfile(
[hdr, camhdr[camera]], 'FIBERFLAT')
log.info("Will use input FIBERFLAT: {}".format(
input_fiberflat[camera]))
if comm is not None:
input_fiberflat = comm.bcast(input_fiberflat, root=0)
# - Group inputs by spectrograph
framefiles = dict()
skyfiles = dict()
fiberflatfiles = dict()
for camera in args.cameras:
sp = int(camera[1])
if sp not in framefiles:
framefiles[sp] = list()
skyfiles[sp] = list()
fiberflatfiles[sp] = list()
fiberflatfiles[sp].append(input_fiberflat[camera])
for expid in args.expids:
framefiles[sp].append(
findfile('frame', args.night, expid, camera))
skyfiles[sp].append(findfile('sky', args.night, expid, camera))
# - Hardcoded stdstar model version
starmodels = os.path.join(os.getenv('DESI_BASIS_TEMPLATES'),
'stdstar_templates_v2.2.fits')
# - Fit stdstars per spectrograph (not per-camera)
spectro_nums = sorted(framefiles.keys())
## for sp in spectro_nums[rank::size]:
for i in range(rank, len(spectro_nums), size):
sp = spectro_nums[i]
# - NOTE: Saving the joint fit file with only the name of the first exposure
stdfile = findfile('stdstars',
args.night,
args.expids[0],
spectrograph=sp)
#stdfile.replace('{:08d}'.format(args.expids[0]),'-'.join(['{:08d}'.format(eid) for eid in args.expids]))
cmd = "desi_fit_stdstars"
cmd += " --delta-color 0.1"
cmd += " --frames {}".format(' '.join(framefiles[sp]))
cmd += " --skymodels {}".format(' '.join(skyfiles[sp]))
cmd += " --fiberflats {}".format(' '.join(fiberflatfiles[sp]))
cmd += " --starmodels {}".format(starmodels)
cmd += " --outfile {}".format(stdfile)
if args.maxstdstars is not None:
cmd += " --maxstdstars {}".format(args.maxstdstars)
inputs = framefiles[sp] + skyfiles[sp] + fiberflatfiles[sp]
num_cmd += 1
err = runcmd(cmd, inputs=inputs, outputs=[stdfile])
if err:
num_err += 1
timer.stop('stdstarfit')
num_cmd, num_err = mpi_count_failures(num_cmd, num_err, comm=comm)
if comm is not None:
comm.barrier()
if rank == 0 and num_err > 0:
log.error(f'{num_err}/{num_cmd} stdstar commands failed')
sys.stdout.flush()
if num_err > 0 and num_err == num_cmd:
if rank == 0:
log.critical('All stdstar commands failed')
sys.exit(1)
if rank == 0 and len(args.expids) > 1:
for sp in spectro_nums:
saved_stdfile = findfile('stdstars',
args.night,
args.expids[0],
spectrograph=sp)
for expid in args.expids[1:]:
new_stdfile = findfile('stdstars',
args.night,
expid,
spectrograph=sp)
new_dirname, new_fname = os.path.split(new_stdfile)
log.debug(
"Path exists: {}, file exists: {}, link exists: {}".
format(os.path.exists(new_stdfile),
os.path.isfile(new_stdfile),
os.path.islink(new_stdfile)))
relpath_saved_std = os.path.relpath(
saved_stdfile, new_dirname)
log.debug(f'Sym Linking jointly fitted stdstar file: {new_stdfile} '+\
f'to existing file at rel. path {relpath_saved_std}')
runcmd(os.symlink, args=(relpath_saved_std, new_stdfile), \
inputs=[saved_stdfile, ], outputs=[new_stdfile, ])
log.debug(
"Path exists: {}, file exists: {}, link exists: {}".
format(os.path.exists(new_stdfile),
os.path.isfile(new_stdfile),
os.path.islink(new_stdfile)))
# -------------------------------------------------------------------------
# - Wrap up
# if rank == 0:
# report = timer.report()
# log.info('Rank 0 timing report:\n' + report)
if comm is not None:
timers = comm.gather(timer, root=0)
else:
timers = [
timer,
]
if rank == 0:
stats = desiutil.timer.compute_stats(timers)
log.info('Timing summary statistics:\n' + json.dumps(stats, indent=2))
if args.timingfile:
if os.path.exists(args.timingfile):
with open(args.timingfile) as fx:
previous_stats = json.load(fx)
#- augment previous_stats with new entries, but don't overwrite old
for name in stats:
if name not in previous_stats:
previous_stats[name] = stats[name]
stats = previous_stats
tmpfile = args.timingfile + '.tmp'
with open(tmpfile, 'w') as fx:
json.dump(stats, fx, indent=2)
os.rename(tmpfile, args.timingfile)
if rank == 0:
log.info('All done at {}'.format(time.asctime()))
def main(args=None, comm=None):
if args is None:
args = parse()
# elif isinstance(args, (list, tuple)):
# args = parse(args)
log = get_logger()
start_mpi_connect = time.time()
if comm is not None:
#- Use the provided comm to determine rank and size
rank = comm.rank
size = comm.size
else:
#- Check MPI flags and determine the comm, rank, and size given the arguments
comm, rank, size = assign_mpi(do_mpi=args.mpi,
do_batch=args.batch,
log=log)
stop_mpi_connect = time.time()
#- Start timer; only print log messages from rank 0 (others are silent)
timer = desiutil.timer.Timer(silent=(rank > 0))
#- Fill in timing information for steps before we had the timer created
if args.starttime is not None:
timer.start('startup', starttime=args.starttime)
timer.stop('startup', stoptime=start_imports)
timer.start('imports', starttime=start_imports)
timer.stop('imports', stoptime=stop_imports)
timer.start('mpi_connect', starttime=start_mpi_connect)
timer.stop('mpi_connect', stoptime=stop_mpi_connect)
#- Freeze IERS after parsing args so that it doesn't bother if only --help
timer.start('freeze_iers')
desiutil.iers.freeze_iers()
timer.stop('freeze_iers')
#- Preflight checks
timer.start('preflight')
if rank > 0:
#- Let rank 0 fetch these, and then broadcast
args, hdr, camhdr = None, None, None
else:
args, hdr, camhdr = update_args_with_headers(args)
## Make sure badamps is formatted properly
if comm is not None and rank == 0 and args.badamps is not None:
args.badamps = validate_badamps(args.badamps)
if comm is not None:
args = comm.bcast(args, root=0)
hdr = comm.bcast(hdr, root=0)
camhdr = comm.bcast(camhdr, root=0)
known_obstype = [
'SCIENCE', 'ARC', 'FLAT', 'ZERO', 'DARK', 'TESTARC', 'TESTFLAT',
'PIXFLAT', 'SKY', 'TWILIGHT', 'OTHER'
]
if args.obstype not in known_obstype:
raise RuntimeError('obstype {} not in {}'.format(
args.obstype, known_obstype))
timer.stop('preflight')
#-------------------------------------------------------------------------
#- Create and submit a batch job if requested
if args.batch:
#exp_str = '{:08d}'.format(args.expid)
jobdesc = args.obstype.lower()
if args.obstype == 'SCIENCE':
# if not doing pre-stdstar fitting or stdstar fitting and if there is
# no flag stopping flux calibration, set job to poststdstar
if args.noprestdstarfit and args.nostdstarfit and (
not args.nofluxcalib):
jobdesc = 'poststdstar'
# elif told not to do std or post stdstar but the flag for prestdstar isn't set,
# then perform prestdstar
elif (not args.noprestdstarfit
) and args.nostdstarfit and args.nofluxcalib:
jobdesc = 'prestdstar'
#elif (not args.noprestdstarfit) and (not args.nostdstarfit) and (not args.nofluxcalib):
# jobdesc = 'science'
scriptfile = create_desi_proc_batch_script(night=args.night, exp=args.expid, cameras=args.cameras,\
jobdesc=jobdesc, queue=args.queue, runtime=args.runtime,\
batch_opts=args.batch_opts, timingfile=args.timingfile,
system_name=args.system_name)
err = 0
if not args.nosubmit:
err = subprocess.call(['sbatch', scriptfile])
sys.exit(err)
#-------------------------------------------------------------------------
#- Proceeding with running
#- What are we going to do?
if rank == 0:
log.info('----------')
log.info('Input {}'.format(args.input))
log.info('Night {} expid {}'.format(args.night, args.expid))
log.info('Obstype {}'.format(args.obstype))
log.info('Cameras {}'.format(args.cameras))
log.info('Output root {}'.format(desispec.io.specprod_root()))
log.info('----------')
#- Create output directories if needed
if rank == 0:
preprocdir = os.path.dirname(
findfile('preproc', args.night, args.expid, 'b0'))
expdir = os.path.dirname(
findfile('frame', args.night, args.expid, 'b0'))
os.makedirs(preprocdir, exist_ok=True)
os.makedirs(expdir, exist_ok=True)
#- Wait for rank 0 to make directories before proceeding
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Preproc
#- All obstypes get preprocessed
timer.start('fibermap')
#- Assemble fibermap for science exposures
fibermap = None
fibermap_ok = None
if rank == 0 and args.obstype == 'SCIENCE':
fibermap = findfile('fibermap', args.night, args.expid)
if not os.path.exists(fibermap):
tmp = findfile('preproc', args.night, args.expid, 'b0')
preprocdir = os.path.dirname(tmp)
fibermap = os.path.join(preprocdir, os.path.basename(fibermap))
log.info('Creating fibermap {}'.format(fibermap))
cmd = 'assemble_fibermap -n {} -e {} -o {}'.format(
args.night, args.expid, fibermap)
if args.badamps is not None:
cmd += ' --badamps={}'.format(args.badamps)
runcmd(cmd, inputs=[], outputs=[fibermap])
fibermap_ok = os.path.exists(fibermap)
#- Some commissioning files didn't have coords* files that caused assemble_fibermap to fail
#- these are well known failures with no other solution, so for those, just force creation
#- of a fibermap with null coordinate information
if not fibermap_ok and int(args.night) < 20200310:
log.info(
"Since night is before 20200310, trying to force fibermap creation without coords file"
)
cmd += ' --force'
runcmd(cmd, inputs=[], outputs=[fibermap])
fibermap_ok = os.path.exists(fibermap)
#- If assemble_fibermap failed and obstype is SCIENCE, exit now
if comm is not None:
fibermap_ok = comm.bcast(fibermap_ok, root=0)
if args.obstype == 'SCIENCE' and not fibermap_ok:
sys.stdout.flush()
if rank == 0:
log.critical(
'assemble_fibermap failed for science exposure; exiting now')
sys.exit(13)
#- Wait for rank 0 to make fibermap if needed
if comm is not None:
fibermap = comm.bcast(fibermap, root=0)
timer.stop('fibermap')
if not (args.obstype in ['SCIENCE'] and args.noprestdstarfit):
timer.start('preproc')
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
outfile = findfile('preproc', args.night, args.expid, camera)
outdir = os.path.dirname(outfile)
cmd = "desi_preproc -i {} -o {} --outdir {} --cameras {}".format(
args.input, outfile, outdir, camera)
if args.scattered_light:
cmd += " --scattered-light"
if fibermap is not None:
cmd += " --fibermap {}".format(fibermap)
if not args.obstype in ['ARC']: # never model variance for arcs
if not args.no_model_pixel_variance:
cmd += " --model-variance"
runcmd(cmd, inputs=[args.input], outputs=[outfile])
timer.stop('preproc')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Get input PSFs
timer.start('findpsf')
input_psf = dict()
if rank == 0:
for camera in args.cameras:
if args.psf is not None:
input_psf[camera] = args.psf
elif args.calibnight is not None:
# look for a psfnight psf for this calib night
psfnightfile = findfile('psfnight', args.calibnight,
args.expid, camera)
if not os.path.isfile(psfnightfile):
log.error("no {}".format(psfnightfile))
raise IOError("no {}".format(psfnightfile))
input_psf[camera] = psfnightfile
else:
# look for a psfnight psf
psfnightfile = findfile('psfnight', args.night, args.expid,
camera)
if os.path.isfile(psfnightfile):
input_psf[camera] = psfnightfile
elif args.most_recent_calib:
nightfile = find_most_recent(args.night,
file_type='psfnight')
if nightfile is None:
input_psf[camera] = findcalibfile(
[hdr, camhdr[camera]], 'PSF')
else:
input_psf[camera] = nightfile
else:
input_psf[camera] = findcalibfile([hdr, camhdr[camera]],
'PSF')
log.info("Will use input PSF : {}".format(input_psf[camera]))
if comm is not None:
input_psf = comm.bcast(input_psf, root=0)
timer.stop('findpsf')
#-------------------------------------------------------------------------
#- Traceshift
if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT'] ) or \
( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ):
timer.start('traceshift')
if rank == 0 and args.traceshift:
log.info('Starting traceshift at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
preprocfile = findfile('preproc', args.night, args.expid, camera)
inpsf = input_psf[camera]
outpsf = findfile('psf', args.night, args.expid, camera)
if not os.path.isfile(outpsf):
if args.traceshift:
cmd = "desi_compute_trace_shifts"
cmd += " -i {}".format(preprocfile)
cmd += " --psf {}".format(inpsf)
cmd += " --outpsf {}".format(outpsf)
cmd += " --degxx 2 --degxy 0"
if args.obstype in ['FLAT', 'TESTFLAT', 'TWILIGHT']:
cmd += " --continuum"
else:
cmd += " --degyx 2 --degyy 0"
if args.obstype in ['SCIENCE', 'SKY']:
cmd += ' --sky'
else:
cmd = "ln -s {} {}".format(inpsf, outpsf)
runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf])
else:
log.info("PSF {} exists".format(outpsf))
timer.stop('traceshift')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- PSF
#- MPI parallelize this step
if args.obstype in ['ARC', 'TESTARC']:
timer.start('arc_traceshift')
if rank == 0:
log.info('Starting traceshift before specex PSF fit at {}'.format(
time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
preprocfile = findfile('preproc', args.night, args.expid, camera)
inpsf = input_psf[camera]
outpsf = findfile('psf', args.night, args.expid, camera)
outpsf = replace_prefix(outpsf, "psf", "shifted-input-psf")
if not os.path.isfile(outpsf):
cmd = "desi_compute_trace_shifts"
cmd += " -i {}".format(preprocfile)
cmd += " --psf {}".format(inpsf)
cmd += " --outpsf {}".format(outpsf)
cmd += " --degxx 0 --degxy 0 --degyx 0 --degyy 0"
cmd += ' --arc-lamps'
runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf])
else:
log.info("PSF {} exists".format(outpsf))
timer.stop('arc_traceshift')
if comm is not None:
comm.barrier()
timer.start('psf')
if rank == 0:
log.info('Starting specex PSF fitting at {}'.format(
time.asctime()))
if rank > 0:
cmds = inputs = outputs = None
else:
cmds = dict()
inputs = dict()
outputs = dict()
for camera in args.cameras:
preprocfile = findfile('preproc', args.night, args.expid,
camera)
tmpname = findfile('psf', args.night, args.expid, camera)
inpsf = replace_prefix(tmpname, "psf", "shifted-input-psf")
outpsf = replace_prefix(tmpname, "psf", "fit-psf")
log.info("now run specex psf fit")
cmd = 'desi_compute_psf'
cmd += ' --input-image {}'.format(preprocfile)
cmd += ' --input-psf {}'.format(inpsf)
cmd += ' --output-psf {}'.format(outpsf)
# look for fiber blacklist
cfinder = CalibFinder([hdr, camhdr[camera]])
blacklistkey = "FIBERBLACKLIST"
if not cfinder.haskey(blacklistkey) and cfinder.haskey(
"BROKENFIBERS"):
log.warning(
"BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST"
)
blacklistkey = "BROKENFIBERS"
if cfinder.haskey(blacklistkey):
blacklist = cfinder.value(blacklistkey)
cmd += ' --broken-fibers {}'.format(blacklist)
if rank == 0:
log.warning('broken fibers: {}'.format(blacklist))
if not os.path.exists(outpsf):
cmds[camera] = cmd
inputs[camera] = [preprocfile, inpsf]
outputs[camera] = [
outpsf,
]
if comm is not None:
cmds = comm.bcast(cmds, root=0)
inputs = comm.bcast(inputs, root=0)
outputs = comm.bcast(outputs, root=0)
#- split communicator by 20 (number of bundles)
group_size = 20
if (rank == 0) and (size % group_size != 0):
log.warning(
'MPI size={} should be evenly divisible by {}'.format(
size, group_size))
group = rank // group_size
num_groups = (size + group_size - 1) // group_size
comm_group = comm.Split(color=group)
if rank == 0:
log.info(
f'Fitting PSFs with {num_groups} sub-communicators of size {group_size}'
)
for i in range(group, len(args.cameras), num_groups):
camera = args.cameras[i]
if camera in cmds:
cmdargs = cmds[camera].split()[1:]
cmdargs = desispec.scripts.specex.parse(cmdargs)
if comm_group.rank == 0:
print('RUNNING: {}'.format(cmds[camera]))
t0 = time.time()
timestamp = time.asctime()
log.info(
f'MPI group {group} ranks {rank}-{rank+group_size-1} fitting PSF for {camera} at {timestamp}'
)
try:
desispec.scripts.specex.main(cmdargs, comm=comm_group)
except Exception as e:
if comm_group.rank == 0:
log.error(
f'FAILED: MPI group {group} ranks {rank}-{rank+group_size-1} camera {camera}'
)
log.error('FAILED: {}'.format(cmds[camera]))
log.error(e)
if comm_group.rank == 0:
specex_time = time.time() - t0
log.info(
f'specex fit for {camera} took {specex_time:.1f} seconds'
)
comm.barrier()
else:
log.warning(
'fitting PSFs without MPI parallelism; this will be SLOW')
for camera in args.cameras:
if camera in cmds:
runcmd(cmds[camera],
inputs=inputs[camera],
outputs=outputs[camera])
if comm is not None:
comm.barrier()
# loop on all cameras and interpolate bad fibers
for camera in args.cameras[rank::size]:
t0 = time.time()
log.info(f'Rank {rank} interpolating {camera} PSF over bad fibers')
# look for fiber blacklist
cfinder = CalibFinder([hdr, camhdr[camera]])
blacklistkey = "FIBERBLACKLIST"
if not cfinder.haskey(blacklistkey) and cfinder.haskey(
"BROKENFIBERS"):
log.warning(
"BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST"
)
blacklistkey = "BROKENFIBERS"
if cfinder.haskey(blacklistkey):
fiberblacklist = cfinder.value(blacklistkey)
tmpname = findfile('psf', args.night, args.expid, camera)
inpsf = replace_prefix(tmpname, "psf", "fit-psf")
outpsf = replace_prefix(tmpname, "psf",
"fit-psf-fixed-blacklisted")
if os.path.isfile(inpsf) and not os.path.isfile(outpsf):
cmd = 'desi_interpolate_fiber_psf'
cmd += ' --infile {}'.format(inpsf)
cmd += ' --outfile {}'.format(outpsf)
cmd += ' --fibers {}'.format(fiberblacklist)
log.info(
'For camera {} interpolating PSF for broken fibers: {}'
.format(camera, fiberblacklist))
runcmd(cmd, inputs=[inpsf], outputs=[outpsf])
if os.path.isfile(outpsf):
os.rename(
inpsf,
inpsf.replace("fit-psf",
"fit-psf-before-blacklisted-fix"))
subprocess.call('cp {} {}'.format(outpsf, inpsf),
shell=True)
dt = time.time() - t0
log.info(
f'Rank {rank} {camera} PSF interpolation took {dt:.1f} sec')
timer.stop('psf')
#-------------------------------------------------------------------------
#- Merge PSF of night if applicable
#if args.obstype in ['ARC']:
if False:
if rank == 0:
for camera in args.cameras:
psfnightfile = findfile('psfnight', args.night, args.expid,
camera)
if not os.path.isfile(
psfnightfile
): # we still don't have a psf night, see if we can compute it ...
psfs = glob.glob(
findfile('psf', args.night, args.expid,
camera).replace("psf", "fit-psf").replace(
str(args.expid), "*"))
log.info(
"Number of PSF for night={} camera={} = {}".format(
args.night, camera, len(psfs)))
if len(psfs) > 4: # lets do it!
log.info("Computing psfnight ...")
dirname = os.path.dirname(psfnightfile)
if not os.path.isdir(dirname):
os.makedirs(dirname)
desispec.scripts.specex.mean_psf(psfs, psfnightfile)
if os.path.isfile(psfnightfile): # now use this one
input_psf[camera] = psfnightfile
#-------------------------------------------------------------------------
#- Extract
#- This is MPI parallel so handle a bit differently
# maybe add ARC and TESTARC too
if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT'] ) or \
( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ):
timer.start('extract')
if rank == 0:
log.info('Starting extractions at {}'.format(time.asctime()))
if rank > 0:
cmds = inputs = outputs = None
else:
cmds = dict()
inputs = dict()
outputs = dict()
for camera in args.cameras:
cmd = 'desi_extract_spectra'
#- Based on data from SM1-SM8, looking at central and edge fibers
#- with in mind overlapping arc lamps lines
if camera.startswith('b'):
cmd += ' -w 3600.0,5800.0,0.8'
elif camera.startswith('r'):
cmd += ' -w 5760.0,7620.0,0.8'
elif camera.startswith('z'):
cmd += ' -w 7520.0,9824.0,0.8'
preprocfile = findfile('preproc', args.night, args.expid,
camera)
psffile = findfile('psf', args.night, args.expid, camera)
framefile = findfile('frame', args.night, args.expid, camera)
cmd += ' -i {}'.format(preprocfile)
cmd += ' -p {}'.format(psffile)
cmd += ' -o {}'.format(framefile)
cmd += ' --psferr 0.1'
if args.obstype == 'SCIENCE' or args.obstype == 'SKY':
if rank == 0:
log.info('Include barycentric correction')
cmd += ' --barycentric-correction'
if not os.path.exists(framefile):
cmds[camera] = cmd
inputs[camera] = [preprocfile, psffile]
outputs[camera] = [
framefile,
]
#- TODO: refactor/combine this with PSF comm splitting logic
if comm is not None:
cmds = comm.bcast(cmds, root=0)
inputs = comm.bcast(inputs, root=0)
outputs = comm.bcast(outputs, root=0)
#- split communicator by 20 (number of bundles)
extract_size = 20
if (rank == 0) and (size % extract_size != 0):
log.warning(
'MPI size={} should be evenly divisible by {}'.format(
size, extract_size))
extract_group = rank // extract_size
num_extract_groups = (size + extract_size - 1) // extract_size
comm_extract = comm.Split(color=extract_group)
for i in range(extract_group, len(args.cameras),
num_extract_groups):
camera = args.cameras[i]
if camera in cmds:
cmdargs = cmds[camera].split()[1:]
extract_args = desispec.scripts.extract.parse(cmdargs)
if comm_extract.rank == 0:
print('RUNNING: {}'.format(cmds[camera]))
desispec.scripts.extract.main_mpi(extract_args,
comm=comm_extract)
comm.barrier()
else:
log.warning(
'running extractions without MPI parallelism; this will be SLOW'
)
for camera in args.cameras:
if camera in cmds:
runcmd(cmds[camera],
inputs=inputs[camera],
outputs=outputs[camera])
timer.stop('extract')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Fiberflat
if args.obstype in ['FLAT', 'TESTFLAT']:
timer.start('fiberflat')
if rank == 0:
log.info('Starting fiberflats at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
framefile = findfile('frame', args.night, args.expid, camera)
fiberflatfile = findfile('fiberflat', args.night, args.expid,
camera)
cmd = "desi_compute_fiberflat"
cmd += " -i {}".format(framefile)
cmd += " -o {}".format(fiberflatfile)
runcmd(cmd, inputs=[
framefile,
], outputs=[
fiberflatfile,
])
timer.stop('fiberflat')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Average and auto-calib fiberflats of night if applicable
#if args.obstype in ['FLAT']:
if False:
if rank == 0:
fiberflatnightfile = findfile('fiberflatnight', args.night,
args.expid, args.cameras[0])
fiberflatdirname = os.path.dirname(fiberflatnightfile)
if not os.path.isfile(fiberflatnightfile) and len(
args.cameras
) >= 6: # we still don't have them, see if we can compute them, but need at least 2 spectros ...
flats = glob.glob(
findfile('fiberflat', args.night, args.expid,
"b0").replace(str(args.expid),
"*").replace("b0", "*"))
log.info("Number of fiberflat for night {} = {}".format(
args.night, len(flats)))
if len(flats) >= 3 * 4 * len(
args.cameras
): # lets do it! (3 exposures x 4 lamps x N cameras)
log.info(
"Computing fiberflatnight per lamp and camera ...")
tmpdir = os.path.join(fiberflatdirname, "tmp")
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
log.info(
"First average measurements per camera and per lamp")
average_flats = dict()
for camera in args.cameras:
# list of flats for this camera
flats_for_this_camera = []
for flat in flats:
if flat.find(camera) >= 0:
flats_for_this_camera.append(flat)
#log.info("For camera {} , flats = {}".format(camera,flats_for_this_camera))
#sys.exit(12)
# average per lamp (and camera)
average_flats[camera] = list()
for lampbox in range(4):
ofile = os.path.join(
tmpdir,
"fiberflatnight-camera-{}-lamp-{}.fits".format(
camera, lampbox))
if not os.path.isfile(ofile):
log.info(
"Average flat for camera {} and lamp box #{}"
.format(camera, lampbox))
pg = "CALIB DESI-CALIB-0{} LEDs only".format(
lampbox)
cmd = "desi_average_fiberflat --program '{}' --outfile {} -i ".format(
pg, ofile)
for flat in flats_for_this_camera:
cmd += " {} ".format(flat)
runcmd(cmd,
inputs=flats_for_this_camera,
outputs=[
ofile,
])
if os.path.isfile(ofile):
average_flats[camera].append(ofile)
else:
log.info("Will use existing {}".format(ofile))
average_flats[camera].append(ofile)
log.info(
"Auto-calibration across lamps and spectro per camera arm (b,r,z)"
)
for camera_arm in ["b", "r", "z"]:
cameras_for_this_arm = []
flats_for_this_arm = []
for camera in args.cameras:
if camera[0].lower() == camera_arm:
cameras_for_this_arm.append(camera)
if camera in average_flats:
for flat in average_flats[camera]:
flats_for_this_arm.append(flat)
cmd = "desi_autocalib_fiberflat --night {} --arm {} -i ".format(
args.night, camera_arm)
for flat in flats_for_this_arm:
cmd += " {} ".format(flat)
runcmd(cmd, inputs=flats_for_this_arm, outputs=[])
log.info("Done with fiber flats per night")
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Get input fiberflat
if args.obstype in ['SCIENCE', 'SKY'] and (not args.nofiberflat):
timer.start('find_fiberflat')
input_fiberflat = dict()
if rank == 0:
for camera in args.cameras:
if args.fiberflat is not None:
input_fiberflat[camera] = args.fiberflat
elif args.calibnight is not None:
# look for a fiberflatnight for this calib night
fiberflatnightfile = findfile('fiberflatnight',
args.calibnight, args.expid,
camera)
if not os.path.isfile(fiberflatnightfile):
log.error("no {}".format(fiberflatnightfile))
raise IOError("no {}".format(fiberflatnightfile))
input_fiberflat[camera] = fiberflatnightfile
else:
# look for a fiberflatnight fiberflat
fiberflatnightfile = findfile('fiberflatnight', args.night,
args.expid, camera)
if os.path.isfile(fiberflatnightfile):
input_fiberflat[camera] = fiberflatnightfile
elif args.most_recent_calib:
nightfile = find_most_recent(
args.night, file_type='fiberflatnight')
if nightfile is None:
input_fiberflat[camera] = findcalibfile(
[hdr, camhdr[camera]], 'FIBERFLAT')
else:
input_fiberflat[camera] = nightfile
else:
input_fiberflat[camera] = findcalibfile(
[hdr, camhdr[camera]], 'FIBERFLAT')
log.info("Will use input FIBERFLAT: {}".format(
input_fiberflat[camera]))
if comm is not None:
input_fiberflat = comm.bcast(input_fiberflat, root=0)
timer.stop('find_fiberflat')
#-------------------------------------------------------------------------
#- Apply fiberflat and write fframe file
if args.obstype in ['SCIENCE', 'SKY'] and args.fframe and \
( not args.nofiberflat ) and (not args.noprestdstarfit):
timer.start('apply_fiberflat')
if rank == 0:
log.info('Applying fiberflat at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
fframefile = findfile('fframe', args.night, args.expid, camera)
if not os.path.exists(fframefile):
framefile = findfile('frame', args.night, args.expid, camera)
fr = desispec.io.read_frame(framefile)
flatfilename = input_fiberflat[camera]
if flatfilename is not None:
ff = desispec.io.read_fiberflat(flatfilename)
fr.meta['FIBERFLT'] = desispec.io.shorten_filename(
flatfilename)
apply_fiberflat(fr, ff)
fframefile = findfile('fframe', args.night, args.expid,
camera)
desispec.io.write_frame(fframefile, fr)
else:
log.warning(
"Missing fiberflat for camera {}".format(camera))
timer.stop('apply_fiberflat')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Select random sky fibers (inplace update of frame file)
#- TODO: move this to a function somewhere
#- TODO: this assigns different sky fibers to each frame of same spectrograph
if (args.obstype in [
'SKY', 'SCIENCE'
]) and (not args.noskysub) and (not args.noprestdstarfit):
timer.start('picksky')
if rank == 0:
log.info('Picking sky fibers at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
framefile = findfile('frame', args.night, args.expid, camera)
orig_frame = desispec.io.read_frame(framefile)
#- Make a copy so that we can apply fiberflat
fr = deepcopy(orig_frame)
if np.any(fr.fibermap['OBJTYPE'] == 'SKY'):
log.info('{} sky fibers already set; skipping'.format(
os.path.basename(framefile)))
continue
#- Apply fiberflat then select random fibers below a flux cut
flatfilename = input_fiberflat[camera]
if flatfilename is None:
log.error("No fiberflat for {}".format(camera))
continue
ff = desispec.io.read_fiberflat(flatfilename)
apply_fiberflat(fr, ff)
sumflux = np.sum(fr.flux, axis=1)
fluxcut = np.percentile(sumflux, 30)
iisky = np.where(sumflux < fluxcut)[0]
iisky = np.random.choice(iisky, size=100, replace=False)
#- Update fibermap or original frame and write out
orig_frame.fibermap['OBJTYPE'][iisky] = 'SKY'
orig_frame.fibermap['DESI_TARGET'][iisky] |= desi_mask.SKY
desispec.io.write_frame(framefile, orig_frame)
timer.stop('picksky')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Sky subtraction
if args.obstype in [
'SCIENCE', 'SKY'
] and (not args.noskysub) and (not args.noprestdstarfit):
timer.start('skysub')
if rank == 0:
log.info('Starting sky subtraction at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
framefile = findfile('frame', args.night, args.expid, camera)
hdr = fitsio.read_header(framefile, 'FLUX')
fiberflatfile = input_fiberflat[camera]
if fiberflatfile is None:
log.error("No fiberflat for {}".format(camera))
continue
skyfile = findfile('sky', args.night, args.expid, camera)
cmd = "desi_compute_sky"
cmd += " -i {}".format(framefile)
cmd += " --fiberflat {}".format(fiberflatfile)
cmd += " --o {}".format(skyfile)
if args.no_extra_variance:
cmd += " --no-extra-variance"
if not args.no_sky_wavelength_adjustment:
cmd += " --adjust-wavelength"
if not args.no_sky_lsf_adjustment: cmd += " --adjust-lsf"
runcmd(cmd, inputs=[framefile, fiberflatfile], outputs=[
skyfile,
])
#- sframe = flatfielded sky-subtracted but not flux calibrated frame
#- Note: this re-reads and re-does steps previously done for picking
#- sky fibers; desi_proc is about human efficiency,
#- not I/O or CPU efficiency...
sframefile = desispec.io.findfile('sframe', args.night, args.expid,
camera)
if not os.path.exists(sframefile):
frame = desispec.io.read_frame(framefile)
fiberflat = desispec.io.read_fiberflat(fiberflatfile)
sky = desispec.io.read_sky(skyfile)
apply_fiberflat(frame, fiberflat)
subtract_sky(frame, sky, apply_throughput_correction=True)
frame.meta['IN_SKY'] = shorten_filename(skyfile)
frame.meta['FIBERFLT'] = shorten_filename(fiberflatfile)
desispec.io.write_frame(sframefile, frame)
timer.stop('skysub')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Standard Star Fitting
if args.obstype in ['SCIENCE',] and \
(not args.noskysub ) and \
(not args.nostdstarfit) :
timer.start('stdstarfit')
if rank == 0:
log.info('Starting flux calibration at {}'.format(time.asctime()))
#- Group inputs by spectrograph
framefiles = dict()
skyfiles = dict()
fiberflatfiles = dict()
night, expid = args.night, args.expid #- shorter
for camera in args.cameras:
sp = int(camera[1])
if sp not in framefiles:
framefiles[sp] = list()
skyfiles[sp] = list()
fiberflatfiles[sp] = list()
framefiles[sp].append(findfile('frame', night, expid, camera))
skyfiles[sp].append(findfile('sky', night, expid, camera))
fiberflatfiles[sp].append(input_fiberflat[camera])
#- Hardcoded stdstar model version
starmodels = os.path.join(os.getenv('DESI_BASIS_TEMPLATES'),
'stdstar_templates_v2.2.fits')
#- Fit stdstars per spectrograph (not per-camera)
spectro_nums = sorted(framefiles.keys())
## for sp in spectro_nums[rank::size]:
for i in range(rank, len(spectro_nums), size):
sp = spectro_nums[i]
stdfile = findfile('stdstars', night, expid, spectrograph=sp)
cmd = "desi_fit_stdstars"
cmd += " --frames {}".format(' '.join(framefiles[sp]))
cmd += " --skymodels {}".format(' '.join(skyfiles[sp]))
cmd += " --fiberflats {}".format(' '.join(fiberflatfiles[sp]))
cmd += " --starmodels {}".format(starmodels)
cmd += " --outfile {}".format(stdfile)
cmd += " --delta-color 0.1"
if args.maxstdstars is not None:
cmd += " --maxstdstars {}".format(args.maxstdstars)
inputs = framefiles[sp] + skyfiles[sp] + fiberflatfiles[sp]
runcmd(cmd, inputs=inputs, outputs=[stdfile])
timer.stop('stdstarfit')
if comm is not None:
comm.barrier()
# -------------------------------------------------------------------------
# - Flux calibration
if args.obstype in ['SCIENCE'] and \
(not args.noskysub) and \
(not args.nofluxcalib):
timer.start('fluxcalib')
night, expid = args.night, args.expid #- shorter
#- Compute flux calibration vectors per camera
for camera in args.cameras[rank::size]:
framefile = findfile('frame', night, expid, camera)
skyfile = findfile('sky', night, expid, camera)
spectrograph = int(camera[1])
stdfile = findfile('stdstars',
night,
expid,
spectrograph=spectrograph)
calibfile = findfile('fluxcalib', night, expid, camera)
fiberflatfile = input_fiberflat[camera]
cmd = "desi_compute_fluxcalibration"
cmd += " --infile {}".format(framefile)
cmd += " --sky {}".format(skyfile)
cmd += " --fiberflat {}".format(fiberflatfile)
cmd += " --models {}".format(stdfile)
cmd += " --outfile {}".format(calibfile)
cmd += " --delta-color-cut 0.1"
inputs = [framefile, skyfile, fiberflatfile, stdfile]
runcmd(cmd, inputs=inputs, outputs=[
calibfile,
])
timer.stop('fluxcalib')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Applying flux calibration
if args.obstype in [
'SCIENCE',
] and (not args.noskysub) and (not args.nofluxcalib):
night, expid = args.night, args.expid #- shorter
timer.start('applycalib')
if rank == 0:
log.info('Starting cframe file creation at {}'.format(
time.asctime()))
for camera in args.cameras[rank::size]:
framefile = findfile('frame', night, expid, camera)
skyfile = findfile('sky', night, expid, camera)
spectrograph = int(camera[1])
stdfile = findfile('stdstars',
night,
expid,
spectrograph=spectrograph)
calibfile = findfile('fluxcalib', night, expid, camera)
cframefile = findfile('cframe', night, expid, camera)
fiberflatfile = input_fiberflat[camera]
cmd = "desi_process_exposure"
cmd += " --infile {}".format(framefile)
cmd += " --fiberflat {}".format(fiberflatfile)
cmd += " --sky {}".format(skyfile)
cmd += " --calib {}".format(calibfile)
cmd += " --outfile {}".format(cframefile)
cmd += " --cosmics-nsig 6"
if args.no_xtalk:
cmd += " --no-xtalk"
inputs = [framefile, fiberflatfile, skyfile, calibfile]
runcmd(cmd, inputs=inputs, outputs=[
cframefile,
])
if comm is not None:
comm.barrier()
timer.stop('applycalib')
#-------------------------------------------------------------------------
#- Wrap up
# if rank == 0:
# report = timer.report()
# log.info('Rank 0 timing report:\n' + report)
if comm is not None:
timers = comm.gather(timer, root=0)
else:
timers = [
timer,
]
if rank == 0:
stats = desiutil.timer.compute_stats(timers)
log.info('Timing summary statistics:\n' + json.dumps(stats, indent=2))
if args.timingfile:
if os.path.exists(args.timingfile):
with open(args.timingfile) as fx:
previous_stats = json.load(fx)
#- augment previous_stats with new entries, but don't overwrite old
for name in stats:
if name not in previous_stats:
previous_stats[name] = stats[name]
stats = previous_stats
tmpfile = args.timingfile + '.tmp'
with open(tmpfile, 'w') as fx:
json.dump(stats, fx, indent=2)
os.rename(tmpfile, args.timingfile)
if rank == 0:
log.info('All done at {}'.format(time.asctime()))
def test_existing_inputs(self):
cmd = 'echo hello > /dev/null'
self.assertEqual(0, util.runcmd(cmd, inputs=[self.infile]))
def integration_test(args=None):
"""
Run an integration test from raw data simulations through QuickLook pipeline
Args:
night (str, optional): YEARMMDD
nspec (int, optional): number of spectra to simulate
overwrite (bool, otional) : overwrite existing files
delete (bool, optional) : delete all inputs and outputs
ql_data (bool, optional) : read data from $QL_SPEC_DATA
ql_redux (bool, optional) : write files to $QL_SPEC_REDUX
Raises:
RuntimeError if QuickLook fails
"""
#- Parse arguments and check environment variables
args = parse(args)
night = args.night
nspec = args.nspec
expid = 2
flat_expid = 1
if args.ql_data:
ql_data = 'Use $QL_SPEC_DATA'
raw_dir = os.environ['QL_SPEC_DATA']
else:
ql_data = 'Use $DESI_SPECTRO_DATA'
raw_dir = os.path.join(os.environ['DESI_SPECTRO_DATA'],'QL')
if args.ql_redux:
ql_redux = 'Use $QL_SPEC_REDUX'
output_dir = os.environ['QL_SPEC_REDUX']
else:
ql_redux = 'Use $DESI_SPECTRO_REDUX/QL/$SPECPROD'
output_dir = os.path.join(os.environ['DESI_SPECTRO_REDUX'],'QL',os.environ['SPECPROD'])
#- Simulate inputs
sim(night=night, nspec=nspec, ql_data=ql_data, ql_redux=ql_redux)
#- Get the configuration file from desispec/data/quicklook
from pkg_resources import resource_filename
configfile=resource_filename('desispec','data/quicklook/qlconfig_dark.yaml')
for camera in ['r0','z0']:
#- Verify that quicklook pipeline runs
if args.ql_data and args.ql_redux:
com = "desi_quicklook -i {} -n {} -c {} -e {} -f {}".format(configfile,night,camera,expid)
else:
com = "desi_quicklook -i {} -n {} -c {} -e {} -f {} --psfboot {} --fiberflat {} --rawdata_dir {} --specprod_dir {}".format(configfile,night,camera,expid,raw_dir,output_dir)
if runcmd(com) != 0:
raise RuntimeError('quicklook pipeline failed for camera {}'.format(camera))
#- Remove all output if desired
if args.delete:
sim_dir = os.path.join(os.environ['DESI_SPECTRO_SIM'],os.environ['PIXPROD'],args.night)
if os.path.exists(sim_dir):
sim_files = os.listdir(sim_dir)
for file in range(len(sim_files)):
sim_file = os.path.join(sim_dir,sim_files[file])
os.remove(sim_file)
os.rmdir(sim_dir)
data_dir = os.path.join(raw_dir,night)
if os.path.exists(data_dir):
data_files = os.listdir(data_dir)
for file in range(len(data_files)):
data_file = os.path.join(data_dir,data_files[file])
os.remove(data_file)
os.rmdir(data_dir)
if os.path.exists(output_dir):
exp_dir = os.path.join(output_dir,'exposures',night)
if os.path.exists(exp_dir):
id_dir = os.path.join(exp_dir,'00000002')
if os.path.exists(id_dir):
id_files = os.listdir(id_dir)
for file in range(len(id_files)):
id_file = os.path.join(id_dir,id_files[file])
os.remove(id_file)
os.rmdir(id_dir)
exp_files = os.listdir(exp_dir)
for file in range(len(exp_files)):
exp_file = os.path.join(exp_dir,exp_files[file])
os.remove(exp_file)
os.rmdir(exp_dir)
calib_dir=os.path.join(output_dir,'calib2d',night)
if os.path.exists(calib_dir):
calib_files = os.listdir(calib_dir)
for file in range(len(calib_files)):
calib_file = os.path.join(calib_dir,calib_files[file])
os.remove(calib_file)
os.rmdir(calib_dir)
psf_dir=os.path.join(output_dir,'calib2d','psf',night)
if os.path.exists(psf_dir):
psf_files = os.listdir(psf_dir)
for ii in range(len(psf_files)):
thisfile=os.path.join(psf_dir,psf_files[ii])
os.remove(thisfile)
os.rmdir(psf_dir)
def test_function(self):
def blat(*args):
return list(args)
self.assertEqual(util.runcmd(blat, args=[1,2,3]), [1,2,3])
self.assertEqual(util.runcmd(blat), [])
def sim(night,nspec):
"""
Simulate data as part of the QuickLook integration test.
Args:
night (str): YEARMMDD
nspec (int): number of spectra to simulate
Raises:
RuntimeError if any script fails
"""
psf_b = os.path.join(os.environ['DESIMODEL'],'data','specpsf','psf-b.fits')
psf_r = os.path.join(os.environ['DESIMODEL'],'data','specpsf','psf-r.fits')
psf_z = os.path.join(os.environ['DESIMODEL'],'data','specpsf','psf-z.fits')
#- Create files needed to run quicklook
sim_dir = os.path.join(os.environ['DESI_SPECTRO_SIM'],os.environ['PIXPROD'],night)
data_dir = os.path.join(os.environ['QL_SPEC_DATA'],night)
output_dir = os.environ['QL_SPEC_REDUX']
exp_dir = os.path.join(output_dir,'exposures',night)
calib_dir = os.path.join(output_dir,'calib2d',night)
psf_dir = os.path.join(output_dir,'calib2d','psf',night)
cmd = "newarc --nspec {} --night {} --expid 0".format(nspec,night)
if runcmd(cmd) != 0:
raise RuntimeError('newexp failed for arc exposure')
cmd = "newflat --nspec {} --night {} --expid 1".format(nspec,night)
if runcmd(cmd) != 0:
raise RuntimeError('newexp failed for flat exposure')
cmd = "newexp-random --program dark --nspec {} --night {} --expid 2".format(nspec,night)
if runcmd(cmd) != 0:
raise RuntimeError('newexp failed for dark exposure')
cmd = "pixsim --night {} --expid 0 --nspec {} --rawfile {}/desi-00000000.fits.fz --preproc --preproc_dir {}".format(night,nspec,data_dir,data_dir)
if runcmd(cmd) != 0:
raise RuntimeError('pixsim failed for arc exposure')
cmd = "pixsim --night {} --expid 1 --nspec {} --rawfile {}/desi-00000001.fits.fz --preproc --preproc_dir {}".format(night,nspec,data_dir,data_dir)
if runcmd(cmd) != 0:
raise RuntimeError('pixsim failed for flat exposure')
cmd = "pixsim --night {} --expid 2 --nspec {} --rawfile {}/desi-00000002.fits.fz".format(night,nspec,data_dir)
if runcmd(cmd) != 0:
raise RuntimeError('pixsim failed for dark exposure')
# cmd = "desi_extract_spectra -i {}/pix-b0-00000001.fits -o {}/frame-b0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 3550,5730,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_b,nspec)
# if runcmd(cmd) != 0:
# raise RuntimeError('desi_extract_spectra failed for camera b0')
cmd = "desi_extract_spectra -i {}/pix-r0-00000001.fits -o {}/frame-r0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 5630,7740,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_r,nspec)
if runcmd(cmd) != 0:
raise RuntimeError('desi_extract_spectra failed for camera r0')
cmd = "desi_extract_spectra -i {}/pix-z0-00000001.fits -o {}/frame-z0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 7650,9830,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_z,nspec)
if runcmd(cmd) != 0:
raise RuntimeError('desi_extract_spectra failed for camera z0')
copyfile(os.path.join(sim_dir,'fibermap-00000002.fits'),os.path.join(data_dir,'fibermap-00000002.fits'))
os.remove(os.path.join(data_dir,'simpix-00000002.fits'))
for camera in ['r0','z0']:
cmd = "desi_compute_fiberflat --infile {}/frame-{}-00000001.fits --outfile {}/fiberflat-{}-00000001.fits".format(exp_dir,camera,calib_dir,camera)
if runcmd(cmd) != 0:
raise RuntimeError('desi_compute_fiberflat failed for camera {}'.format(camera))
cmd = "desi_bootcalib --fiberflat {}/pix-{}-00000001.fits --arcfile {}/pix-{}-00000000.fits --outfile {}/psfboot-{}.fits".format(data_dir,camera,data_dir,camera,psf_dir,camera)
if runcmd(cmd) != 0:
raise RuntimeError('desi_bootcalib failed for camera {}'.format(camera))
return
def test_function(self):
def blat(*args):
return list(args)
self.assertEqual(util.runcmd(blat, args=[1, 2, 3]), [1, 2, 3])
self.assertEqual(util.runcmd(blat), [])
def test_missing_inputs(self):
cmd = 'echo hello > /dev/null'
self.assertNotEqual(0, util.runcmd(cmd, inputs=[uuid4().hex]))
def test_runcmd(self):
self.assertEqual(0, util.runcmd('echo hello > /dev/null'))
def test_mergeQA(self):
os.environ['QL_SPEC_REDUX'] = self.testDir
cmd = "{} {}/desi_quicklook -i {} -n {} -c {} -e {} --rawdata_dir {} --specprod_dir {} --mergeQA".format(sys.executable,self.binDir,self.configfile,self.night,self.camera,self.expid,self.testDir,self.testDir)
if runcmd(cmd) != 0:
raise RuntimeError('quicklook pipeline failed')
def integration_test(night="20160726", nspec=25, clobber=False):
"""Run an integration test from raw data simulations through redshifts
Args:
night (str, optional): YEARMMDD, defaults to current night
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
log.setLevel(logging.DEBUG)
flat_expid = 0
expid = 2
# check for required environment variables and simulate inputs
check_env()
sim(night, nspec=nspec, clobber=clobber)
for camera in ['b0', 'r0', 'z0']:
# find all necessary input and output files
framefile = desispec.io.findfile('frame', night, expid, camera)
fiberflatfile = desispec.io.findfile('fiberflat', night, flat_expid,
camera)
skyfile = desispec.io.findfile('sky', night, expid, camera)
skytestfile = desispec.io.findfile('sky', night, expid,
camera) + 'test'
calibfile = desispec.io.findfile('calib', night, expid, camera)
calibtestfile = desispec.io.findfile('calib', night, expid,
camera) + 'test'
stdstarsfile = desispec.io.findfile('stdstars',
night,
expid,
spectrograph=0)
cframetestfile = desispec.io.findfile('cframe', night, expid,
camera) + 'test'
# verify that quickgen output works for full pipeline
com = "desi_compute_sky --infile {} --fiberflat {} --outfile {}".format(
framefile, fiberflatfile, skytestfile)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError(
'desi_compute_sky failed for camera {}'.format(camera))
# only fit stdstars once
if camera == 'b0':
com = "desi_fit_stdstars --frames {} --skymodels {} --fiberflats {} --starmodels $DESI_BASIS_TEMPLATES/star_templates_v2.1.fits --outfile {}".format(
framefile, skyfile, fiberflatfile, stdstarsfile)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError(
'desi_fit_stdstars failed for camera {}'.format(camera))
com = "desi_compute_fluxcalibration --infile {} --fiberflat {} --sky {} --models {} --outfile {}".format(
framefile, fiberflatfile, skyfile, stdstarsfile, calibtestfile)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError(
'desi_compute_fluxcalibration failed for camera {}'.format(
camera))
com = "desi_process_exposure --infile {} --fiberflat {} --sky {} --calib {} --outfile {}".format(
framefile, fiberflatfile, skyfile, calibfile, cframetestfile)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError(
'desi_process_exposure failed for camera {}'.format(camera))
com = "desi_make_bricks --night {}".format(night)
if runcmd(com, clobber=clobber) != 0:
raise RuntimeError('desi_make_bricks failed')
def integration_test(args=None):
"""
Run an integration test from raw data simulations through QuickLook pipeline
Args:
night (str, optional): YEARMMDD
nspec (int, optional): number of spectra to simulate
overwrite (bool, otional) : overwrite existing files
delete (bool, optional) : delete all inputs and outputs
Raises:
RuntimeError if QuickLook fails
"""
#- Parse arguments and check environment variables
args = parse(args)
night = args.night
nspec = args.nspec
raw_dir = os.environ['QL_SPEC_DATA']
output_dir = os.environ['QL_SPEC_REDUX']
#- Simulate inputs
sim(night=night,nspec=nspec)
#- Get the configuration file from desispec/data/quicklook
from pkg_resources import resource_filename
configfile=resource_filename('desispec','data/quicklook/qlconfig_dark.yaml')
for camera in ['r0','z0']:
#- Verify that quicklook pipeline runs
com = "desi_quicklook -i {} -n {} -c {} -e 2".format(configfile,night,camera)
if runcmd(com) != 0:
raise RuntimeError('quicklook pipeline failed for camera {}'.format(camera))
#- Remove all output if desired
if args.delete:
sim_dir = os.path.join(os.environ['DESI_SPECTRO_SIM'],os.environ['PIXPROD'],args.night)
if os.path.exists(sim_dir):
sim_files = os.listdir(sim_dir)
for file in range(len(sim_files)):
sim_file = os.path.join(sim_dir,sim_files[file])
os.remove(sim_file)
os.rmdir(sim_dir)
data_dir = os.path.join(raw_dir,night)
if os.path.exists(data_dir):
data_files = os.listdir(data_dir)
for file in range(len(data_files)):
data_file = os.path.join(data_dir,data_files[file])
os.remove(data_file)
os.rmdir(data_dir)
if os.path.exists(output_dir):
exp_dir = os.path.join(output_dir,'exposures',night)
if os.path.exists(exp_dir):
id_dir = os.path.join(exp_dir,'00000002')
if os.path.exists(id_dir):
id_files = os.listdir(id_dir)
for file in range(len(id_files)):
id_file = os.path.join(id_dir,id_files[file])
os.remove(id_file)
os.rmdir(id_dir)
exp_files = os.listdir(exp_dir)
for file in range(len(exp_files)):
exp_file = os.path.join(exp_dir,exp_files[file])
os.remove(exp_file)
os.rmdir(exp_dir)
calib_dir=os.path.join(output_dir,'calib2d',night)
if os.path.exists(calib_dir):
calib_files = os.listdir(calib_dir)
for file in range(len(calib_files)):
calib_file = os.path.join(calib_dir,calib_files[file])
os.remove(calib_file)
os.rmdir(calib_dir)
psf_dir=os.path.join(output_dir,'calib2d','psf',night)
if os.path.exists(psf_dir):
psf_files = os.listdir(psf_dir)
for ii in range(len(psf_files)):
thisfile=os.path.join(psf_dir,psf_files[ii])
os.remove(thisfile)
os.rmdir(psf_dir)
def sim(night, nspec, ql_data, ql_redux):
"""
Simulate data as part of the QuickLook integration test.
Args:
night (str): YEARMMDD
nspec (int): number of spectra to simulate
ql_data (str) : data read from $QL_SPEC_DATA if specified
ql_redux (str) : output sent to $QL_SPEC_REDUX if specified
Raises:
RuntimeError if any script fails
"""
# psf_b = os.path.join(os.environ['DESIMODEL'],'data','specpsf','psf-b.fits')
psf_r = os.path.join(os.environ['DESIMODEL'],'data','specpsf','psf-r.fits')
psf_z = os.path.join(os.environ['DESIMODEL'],'data','specpsf','psf-z.fits')
#- Create files needed to run quicklook
sim_dir = os.path.join(os.environ['DESI_SPECTRO_SIM'],os.environ['PIXPROD'],night)
if ql_data == 'Use $QL_SPEC_DATA':
data_dir = os.path.join(os.environ['QL_SPEC_DATA'],night)
else:
data_dir = os.path.join(os.environ['DESI_SPECTRO_DATA'],'QL',night)
if ql_redux == 'Use $QL_SPEC_REDUX':
output_dir = os.environ['QL_SPEC_REDUX']
else:
output_dir = os.path.join(os.environ['DESI_SPECTRO_REDUX'],'QL',os.environ['SPECPROD'])
exp_dir = os.path.join(output_dir,'exposures',night)
calib_dir = os.path.join(output_dir,'calib2d',night)
psf_dir = os.path.join(output_dir,'calib2d','psf',night)
for expid, flavor in zip([0,1,2], ['arc', 'flat', 'dark']):
cmd = "newexp-desi --flavor {} --nspec {} --night {} --expid {}".format(flavor,nspec,night,expid)
if runcmd(cmd) != 0:
raise RuntimeError('newexp failed for {} exposure {}'.format(flavor, expid))
if flavor == 'dark':
cmd = "pixsim-desi --night {} --expid {} --nspec {} --rawfile {}/desi-{:08d}.fits.fz".format(night,expid,nspec,data_dir,expid)
if runcmd(cmd) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid))
if flavor == 'arc' or flavor == 'flat':
cmd = "pixsim-desi --night {} --expid {} --nspec {} --rawfile {}/desi-{:08d}.fits.fz --preproc --preproc_dir {}".format(night,expid,nspec,data_dir,expid,data_dir)
if runcmd(cmd) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid))
if flavor == 'flat':
# cmd = "desi_extract_spectra -i {}/pix-b0-00000001.fits -o {}/frame-b0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 3550,5730,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_b,nspec)
# if runcmd(cmd) != 0:
# raise RuntimeError('desi_extract_spectra failed for camera b0')
cmd = "desi_extract_spectra -i {}/pix-r0-00000001.fits -o {}/frame-r0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 5630,7740,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_r,nspec)
if runcmd(cmd) != 0:
raise RuntimeError('desi_extract_spectra failed for camera r0')
cmd = "desi_extract_spectra -i {}/pix-z0-00000001.fits -o {}/frame-z0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 7650,9830,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_z,nspec)
if runcmd(cmd) != 0:
raise RuntimeError('desi_extract_spectra failed for camera z0')
copyfile(os.path.join(sim_dir,'fibermap-{:08d}.fits'.format(expid)),os.path.join(data_dir,'fibermap-{:08d}.fits'.format(expid)))
os.remove(os.path.join(data_dir,'simpix-{:08d}.fits'.format(expid)))
for camera in ['r0','z0']:
cmd = "desi_compute_fiberflat --infile {}/frame-{}-00000001.fits --outfile {}/fiberflat-{}-00000001.fits".format(exp_dir,camera,calib_dir,camera)
if runcmd(cmd) != 0:
raise RuntimeError('desi_compute_fiberflat failed for camera {}'.format(camera))
cmd = "desi_bootcalib --fiberflat {}/pix-{}-00000001.fits --arcfile {}/pix-{}-00000000.fits --outfile {}/psfboot-{}.fits".format(data_dir,camera,data_dir,camera,psf_dir,camera)
if runcmd(cmd) != 0:
raise RuntimeError('desi_bootcalib failed for camera {}'.format(camera))
return
