def sim(night, nspec=5, clobber=False): """ Simulate data as part of the integration test. Args: night (str): YEARMMDD nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() # Create input fibermaps, spectra, and pixel-level raw data for expid, program in zip([0,1,2], ['flat', 'arc', 'dark']): cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format( expid=expid, program=program, nspec=nspec, night=night) fibermap = io.findfile('fibermap', night, expid) simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid) inputs = [] outputs = [fibermap, simspec] if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError('newexp-random failed for {} exposure {}'.format(program, expid)) cmd = "pixsim --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, nspec=nspec, night=night) inputs = [fibermap, simspec] outputs = list() outputs.append(fibermap.replace('fibermap-', 'simpix-')) outputs.append(io.findfile('raw', night, expid)) if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError('pixsim failed for {} exposure {}'.format(program, expid)) return
def test_compute_fiberflat(self): """ Tests desi_compute_fiberflat --infile frame.fits --outfile fiberflat.fits """ self._write_frame(flavor='flat') self._write_fibermap() # QA fig requires fibermapfile cmd = '{} {}/desi_compute_fiberflat --infile {} --outfile {} --qafile {} --qafig {}'.format( sys.executable, self.binDir, self.framefile, self.fiberflatfile, self.qa_calib_file, self.qafig) outputs = [self.fiberflatfile,self.qa_calib_file,self.qafig] inputs = [self.framefile,] err = runcmd(cmd, inputs=inputs, outputs=outputs, clobber=True) self.assertEqual(err, 0, 'FAILED: {}'.format(cmd)) #- Confirm that the output file can be read as a fiberflat ff1 = io.read_fiberflat(self.fiberflatfile) #- Remove outputs and call again via function instead of system call self._remove_files(outputs) args = desispec.scripts.fiberflat.parse(cmd.split()[2:]) err = runcmd(desispec.scripts.fiberflat.main, args=[args,], inputs=inputs, outputs=outputs, clobber=True) #- Confirm that the output file can be read as a fiberflat ff2 = io.read_fiberflat(self.fiberflatfile) self.assertTrue(np.all(ff1.fiberflat == ff2.fiberflat)) self.assertTrue(np.all(ff1.ivar == ff2.ivar)) self.assertTrue(np.all(ff1.mask == ff2.mask)) self.assertTrue(np.all(ff1.meanspec == ff2.meanspec)) self.assertTrue(np.all(ff1.wave == ff2.wave)) self.assertTrue(np.all(ff1.fibers == ff2.fibers))
def sim(night, nspec=25, clobber=False): """ Simulate data as part of the integration test. Args: night (str): YEARMMDD nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() output_dir = os.path.join('$DESI_SPECTRO_REDUX','calib2d') # Create input fibermaps, spectra, and quickgen data for expid, program in zip([0,1,2], ['flat', 'arc', 'dark']): cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, program=program, nspec=nspec, night=night) simspec = desisim.io.findfile('simspec', night, expid) fibermap = '{}/fibermap-{:08d}.fits'.format(os.path.dirname(simspec),expid) if runcmd(cmd, clobber=clobber) != 0: raise RuntimeError('newexp failed for {} exposure {}'.format(program, expid)) cmd = "quickgen --simspec {} --fibermap {}".format(simspec,fibermap) if runcmd(cmd, clobber=clobber) != 0: raise RuntimeError('quickgen failed for {} exposure {}'.format(program, expid)) return
def integration_test(night="20160726", nspec=25, clobber=False): """Run an integration test from raw data simulations through redshifts Args: night (str, optional): YEARMMDD, defaults to current night nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() log.setLevel(logging.DEBUG) flat_expid = 0 expid = 2 # check for required environment variables and simulate inputs check_env() sim(night, nspec=nspec, clobber=clobber) for camera in ['b0', 'r0', 'z0']: # find all necessary input and output files framefile = desispec.io.findfile('frame', night, expid, camera) fiberflatfile = desispec.io.findfile('fiberflat', night, flat_expid, camera) skyfile = desispec.io.findfile('sky', night, expid, camera) skytestfile = desispec.io.findfile('sky', night, expid, camera) + 'test' calibfile = desispec.io.findfile('calib', night, expid, camera) calibtestfile = desispec.io.findfile('calib', night, expid, camera) + 'test' stdstarsfile = desispec.io.findfile('stdstars', night, expid, spectrograph=0) cframetestfile = desispec.io.findfile('cframe', night, expid, camera) + 'test' # verify that quickgen output works for full pipeline com = "desi_compute_sky --infile {} --fiberflat {} --outfile {}".format(framefile, fiberflatfile, skytestfile) if runcmd(com, clobber=clobber) != 0: raise RuntimeError('desi_compute_sky failed for camera {}'.format(camera)) # only fit stdstars once if camera == 'b0': com = "desi_fit_stdstars --frames {} --skymodels {} --fiberflats {} --starmodels $DESI_BASIS_TEMPLATES/star_templates_v2.1.fits --outfile {}".format(framefile, skyfile, fiberflatfile, stdstarsfile) if runcmd(com, clobber=clobber) != 0: raise RuntimeError('desi_fit_stdstars failed for camera {}'.format(camera)) com = "desi_compute_fluxcalibration --infile {} --fiberflat {} --sky {} --models {} --outfile {}".format(framefile, fiberflatfile, skyfile, stdstarsfile, calibtestfile) if runcmd(com, clobber=clobber) != 0: raise RuntimeError('desi_compute_fluxcalibration failed for camera {}'.format(camera)) com = "desi_process_exposure --infile {} --fiberflat {} --sky {} --calib {} --outfile {}".format(framefile, fiberflatfile, skyfile, calibfile, cframetestfile) if runcmd(com, clobber=clobber) != 0: raise RuntimeError('desi_process_exposure failed for camera {}'.format(camera)) com = "desi_make_bricks --night {}".format(night) if runcmd(com, clobber=clobber) != 0: raise RuntimeError('desi_make_bricks failed')
def test_clobber(self): token = uuid4().hex cmd = 'echo {} > {}'.format(token, self.testfile) self.assertEqual(0, util.runcmd(cmd, outputs=[self.outfile], clobber=True)) fx = open(self.testfile) line = fx.readline().strip() #- command should have run, so tokens should be equal self.assertEqual(token, line)
def test_existing_outputs(self): token = uuid4().hex cmd = 'echo {} > {}'.format(token, self.testfile) self.assertEqual(0, util.runcmd(cmd, outputs=[self.outfile])) fx = open(self.testfile) line = fx.readline().strip() #- command should not have run, so tokens should not be equal self.assertNotEqual(token, line)
def test_compute_sky(self): """ Tests desi_compute_sky --infile frame.fits --fiberflat fiberflat.fits --outfile skymodel.fits """ self._write_frame(flavor='science', camera='b0') # MUST MATCH FLUXCALIB ABOVE self._write_fiberflat() self._write_fibermap() cmd = "{} {}/desi_compute_sky --infile {} --fiberflat {} --outfile {} --qafile {} --qafig {}".format( sys.executable, self.binDir, self.framefile, self.fiberflatfile, self.skyfile, self.qa_data_file, self.qafig) inputs = [self.framefile, self.fiberflatfile] outputs = [self.skyfile,self.qa_data_file,self.qafig,] err = runcmd(cmd, inputs=inputs, outputs=outputs, clobber=True) self.assertEqual(err, 0, 'FAILED: {}'.format(cmd)) #- Remove outputs and call again via function instead of system call self._remove_files(outputs) args = desispec.scripts.sky.parse(cmd.split()[2:]) err = runcmd(desispec.scripts.sky.main, args=[args,], inputs=inputs, outputs=outputs, clobber=True) self.assertEqual(err, None)
def test_zz(self): """ Even if clobber=False and outputs exist, run cmd if inputs are newer than outputs. Run this test last since it alters timestamps. """ #- update input timestamp to be newer than output fx = open(self.infile, 'w') fx.write('This file is leftover from a test; you can remove it\n') fx.close() #- run a command token = uuid4().hex cmd = 'echo {} > {}'.format(token, self.testfile) self.assertEqual(0, util.runcmd(cmd, outputs=[self.outfile], clobber=False)) #- command should have run even though outputs exist, #- so tokens should be equal fx = open(self.testfile) line = fx.readline().strip() self.assertNotEqual(token, line)
def test_missing_outputs(self): cmd = 'echo hello > /dev/null' self.assertNotEqual(0, util.runcmd(cmd, outputs=[uuid4().hex]))
def test_existing_inputs(self): cmd = 'echo hello > /dev/null' self.assertEqual(0, util.runcmd(cmd, inputs=[self.infile]))
def test_runcmd(self): self.assertEqual(0, util.runcmd('echo hello > /dev/null'))
def main(args=None, comm=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) log = get_logger() start_mpi_connect = time.time() if comm is not None: #- Use the provided comm to determine rank and size rank = comm.rank size = comm.size else: #- Check MPI flags and determine the comm, rank, and size given the arguments comm, rank, size = assign_mpi(do_mpi=args.mpi, do_batch=args.batch, log=log) stop_mpi_connect = time.time() #- Start timer; only print log messages from rank 0 (others are silent) timer = desiutil.timer.Timer(silent=(rank > 0)) #- Fill in timing information for steps before we had the timer created if args.starttime is not None: timer.start('startup', starttime=args.starttime) timer.stop('startup', stoptime=start_imports) timer.start('imports', starttime=start_imports) timer.stop('imports', stoptime=stop_imports) timer.start('mpi_connect', starttime=start_mpi_connect) timer.stop('mpi_connect', stoptime=stop_mpi_connect) #- Freeze IERS after parsing args so that it doesn't bother if only --help timer.start('freeze_iers') desiutil.iers.freeze_iers() timer.stop('freeze_iers') #- Preflight checks timer.start('preflight') # - Preflight checks if rank > 0: # - Let rank 0 fetch these, and then broadcast args, hdr, camhdr = None, None, None else: if args.inputs is None: if args.night is None or args.expids is None: raise RuntimeError( 'Must specify --inputs or --night AND --expids') else: args.expids = np.array( args.expids.strip(' \t').split(',')).astype(int) args.inputs = [] for expid in args.expids: infile = findfile('raw', night=args.night, expid=expid) args.inputs.append(infile) if not os.path.isfile(infile): raise IOError('Missing input file: {}'.format(infile)) else: args.inputs = np.array(args.inputs.strip(' \t').split(',')) #- args.night will be defined in update_args_with_headers, #- but let's define the expids here #- NOTE: inputs has priority. Overwriting expids if they existed. args.expids = [] for infile in args.inputs: hdr = load_raw_data_header(pathname=infile, return_filehandle=False) args.expids.append(int(hdr['EXPID'])) args.expids = np.sort(args.expids) args.inputs = np.sort(args.inputs) args.expid = args.expids[0] args.input = args.inputs[0] #- Use header information to fill in missing information in the arguments object args, hdr, camhdr = update_args_with_headers(args) #- If not a science observation, we don't need the hdr or camhdr objects, #- So let's not broadcast them to all the ranks if args.obstype != 'SCIENCE': hdr, camhdr = None, None if comm is not None: args = comm.bcast(args, root=0) hdr = comm.bcast(hdr, root=0) camhdr = comm.bcast(camhdr, root=0) known_obstype = ['SCIENCE', 'ARC', 'FLAT'] if args.obstype not in known_obstype: raise RuntimeError('obstype {} not in {}'.format( args.obstype, known_obstype)) timer.stop('preflight') # ------------------------------------------------------------------------- # - Create and submit a batch job if requested if args.batch: #camword = create_camword(args.cameras) #exp_str = '-'.join('{:08d}'.format(expid) for expid in args.expids) if args.obstype.lower() == 'science': jobdesc = 'stdstarfit' elif args.obstype.lower() == 'arc': jobdesc = 'psfnight' elif args.obstype.lower() == 'flat': jobdesc = 'nightlyflat' else: jobdesc = args.obstype.lower() scriptfile = create_desi_proc_batch_script(night=args.night, exp=args.expids, cameras=args.cameras,\ jobdesc=jobdesc, queue=args.queue, runtime=args.runtime,\ batch_opts=args.batch_opts, timingfile=args.timingfile, system_name=args.system_name) err = 0 if not args.nosubmit: err = subprocess.call(['sbatch', scriptfile]) sys.exit(err) # ------------------------------------------------------------------------- # - Proceed with running # - What are we going to do? if rank == 0: log.info('----------') log.info('Input {}'.format(args.inputs)) log.info('Night {} expids {}'.format(args.night, args.expids)) log.info('Obstype {}'.format(args.obstype)) log.info('Cameras {}'.format(args.cameras)) log.info('Output root {}'.format(desispec.io.specprod_root())) log.info('----------') # - Wait for rank 0 to make directories before proceeding if comm is not None: comm.barrier() # ------------------------------------------------------------------------- # - Merge PSF of night if applicable if args.obstype in ['ARC']: timer.start('psfnight') num_cmd = num_err = 0 if rank == 0: for camera in args.cameras: psfnightfile = findfile('psfnight', args.night, args.expids[0], camera) if not os.path.isfile( psfnightfile ): # we still don't have a psf night, see if we can compute it ... psfs = [ findfile('psf', args.night, expid, camera).replace("psf", "fit-psf") for expid in args.expids ] log.info( "Number of PSF for night={} camera={} = {}".format( args.night, camera, len(psfs))) if len(psfs) > 4: # lets do it! log.info("Computing psfnight ...") dirname = os.path.dirname(psfnightfile) if not os.path.isdir(dirname): os.makedirs(dirname) num_cmd += 1 #- generic try/except so that any failure doesn't leave #- MPI rank 0 hanging while others are waiting for it try: desispec.scripts.specex.mean_psf( psfs, psfnightfile) except: log.error('specex.meanpsf failed for {}'.format( os.path.basename(psfnightfile))) exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception( exc_type, exc_value, exc_traceback) log.error(''.join(lines)) sys.stdout.flush() if not os.path.exists(psfnightfile): log.error(f'Failed to create {psfnightfile}') num_err += 1 else: log.info( "Fewer than 4 psfs were provided, can't compute psfnight. Exiting ..." ) num_cmd += 1 num_err += 1 timer.stop('psfnight') num_cmd, num_err = mpi_count_failures(num_cmd, num_err, comm=comm) if rank == 0: if num_err > 0: log.error(f'{num_err}/{num_cmd} psfnight commands failed') if num_err > 0 and num_err == num_cmd: sys.stdout.flush() if rank == 0: log.critical('All psfnight commands failed') sys.exit(1) # ------------------------------------------------------------------------- # - Average and auto-calib fiberflats of night if applicable if args.obstype in ['FLAT']: timer.start('fiberflatnight') #- Track number of commands run and number of errors for exit code num_cmd = 0 num_err = 0 if rank == 0: fiberflatnightfile = findfile('fiberflatnight', args.night, args.expids[0], args.cameras[0]) fiberflatdirname = os.path.dirname(fiberflatnightfile) if os.path.isfile(fiberflatnightfile): log.info("Fiberflatnight already exists. Exitting ...") elif len( args.cameras ) < 6: # we still don't have them, see if we can compute them # , but need at least 2 spectros ... log.info( "Fewer than 6 cameras were available, so couldn't perform joint fit. Exiting ..." ) else: flats = [] for camera in args.cameras: for expid in args.expids: flats.append( findfile('fiberflat', args.night, expid, camera)) log.info("Number of fiberflat for night {} = {}".format( args.night, len(flats))) if len(flats) < 3 * 4 * len(args.cameras): log.info( "Fewer than 3 exposures with 4 lamps were available. Can't perform joint fit. Exiting..." ) else: log.info( "Computing fiberflatnight per lamp and camera ...") tmpdir = os.path.join(fiberflatdirname, "tmp") if not os.path.isdir(tmpdir): os.makedirs(tmpdir) log.info( "First average measurements per camera and per lamp") average_flats = dict() for camera in args.cameras: # list of flats for this camera flats_for_this_camera = [] for flat in flats: if flat.find(camera) >= 0: flats_for_this_camera.append(flat) # log.info("For camera {} , flats = {}".format(camera,flats_for_this_camera)) # sys.exit(12) # average per lamp (and camera) average_flats[camera] = list() for lampbox in range(4): ofile = os.path.join( tmpdir, "fiberflatnight-camera-{}-lamp-{}.fits".format( camera, lampbox)) if not os.path.isfile(ofile): log.info( "Average flat for camera {} and lamp box #{}" .format(camera, lampbox)) pg = "CALIB DESI-CALIB-0{} LEDs only".format( lampbox) cmd = "desi_average_fiberflat --program '{}' --outfile {} -i ".format( pg, ofile) for flat in flats_for_this_camera: cmd += " {} ".format(flat) num_cmd += 1 err = runcmd(cmd, inputs=flats_for_this_camera, outputs=[ ofile, ]) if err: num_err += 1 if os.path.isfile(ofile): average_flats[camera].append(ofile) else: log.error( f"Generating {ofile} failed; proceeding with other flats" ) else: log.info("Will use existing {}".format(ofile)) average_flats[camera].append(ofile) log.info( "Auto-calibration across lamps and spectro per camera arm (b,r,z)" ) for camera_arm in ["b", "r", "z"]: cameras_for_this_arm = [] flats_for_this_arm = [] for camera in args.cameras: if camera[0].lower() == camera_arm: cameras_for_this_arm.append(camera) if camera in average_flats: for flat in average_flats[camera]: flats_for_this_arm.append(flat) if len(flats_for_this_arm) > 0: cmd = "desi_autocalib_fiberflat --night {} --arm {} -i ".format( args.night, camera_arm) for flat in flats_for_this_arm: cmd += " {} ".format(flat) num_cmd += 1 err = runcmd(cmd, inputs=flats_for_this_arm, outputs=[]) if err: num_err += 1 else: log.error(f'No flats found for arm {camera_arm}') log.info("Done with fiber flats per night") timer.stop('fiberflatnight') num_cmd, num_err = mpi_count_failures(num_cmd, num_err, comm=comm) if comm is not None: comm.barrier() if rank == 0: if num_err > 0: log.error(f'{num_err}/{num_cmd} fiberflat commands failed') if num_err > 0 and num_err == num_cmd: if rank == 0: log.critical('All fiberflat commands failed') sys.exit(1) ##################### Note ############################# ### Still for single exposure. Needs to be re-factored # ######################################################## if args.obstype in ['SCIENCE']: #inputfile = findfile('raw', night=args.night, expid=args.expids[0]) #if not os.path.isfile(inputfile): # raise IOError('Missing input file: {}'.format(inputfile)) ## - Fill in values from raw data header if not overridden by command line #fx = fitsio.FITS(inputfile) #if 'SPEC' in fx: # - 20200225 onwards # # hdr = fits.getheader(args.input, 'SPEC') # hdr = fx['SPEC'].read_header() #elif 'SPS' in fx: # - 20200224 and before # # hdr = fits.getheader(args.input, 'SPS') # hdr = fx['SPS'].read_header() #else: # # hdr = fits.getheader(args.input, 0) # hdr = fx[0].read_header() # #camhdr = dict() #for cam in args.cameras: # camhdr[cam] = fx[cam].read_header() #fx.close() timer.start('stdstarfit') num_err = num_cmd = 0 if rank == 0: log.info('Starting stdstar fitting at {}'.format(time.asctime())) # ------------------------------------------------------------------------- # - Get input fiberflat input_fiberflat = dict() if rank == 0: for camera in args.cameras: if args.fiberflat is not None: input_fiberflat[camera] = args.fiberflat elif args.calibnight is not None: # look for a fiberflatnight for this calib night fiberflatnightfile = findfile('fiberflatnight', args.calibnight, args.expids[0], camera) if not os.path.isfile(fiberflatnightfile): log.error("no {}".format(fiberflatnightfile)) raise IOError("no {}".format(fiberflatnightfile)) input_fiberflat[camera] = fiberflatnightfile else: # look for a fiberflatnight fiberflat fiberflatnightfile = findfile('fiberflatnight', args.night, args.expids[0], camera) if os.path.isfile(fiberflatnightfile): input_fiberflat[camera] = fiberflatnightfile elif args.most_recent_calib: # -- NOTE: Finding most recent only with respect to the first night nightfile = find_most_recent(args.night, file_type='fiberflatnight') if nightfile is None: input_fiberflat[camera] = findcalibfile( [hdr, camhdr[camera]], 'FIBERFLAT') else: input_fiberflat[camera] = nightfile else: input_fiberflat[camera] = findcalibfile( [hdr, camhdr[camera]], 'FIBERFLAT') log.info("Will use input FIBERFLAT: {}".format( input_fiberflat[camera])) if comm is not None: input_fiberflat = comm.bcast(input_fiberflat, root=0) # - Group inputs by spectrograph framefiles = dict() skyfiles = dict() fiberflatfiles = dict() for camera in args.cameras: sp = int(camera[1]) if sp not in framefiles: framefiles[sp] = list() skyfiles[sp] = list() fiberflatfiles[sp] = list() fiberflatfiles[sp].append(input_fiberflat[camera]) for expid in args.expids: framefiles[sp].append( findfile('frame', args.night, expid, camera)) skyfiles[sp].append(findfile('sky', args.night, expid, camera)) # - Hardcoded stdstar model version starmodels = os.path.join(os.getenv('DESI_BASIS_TEMPLATES'), 'stdstar_templates_v2.2.fits') # - Fit stdstars per spectrograph (not per-camera) spectro_nums = sorted(framefiles.keys()) ## for sp in spectro_nums[rank::size]: for i in range(rank, len(spectro_nums), size): sp = spectro_nums[i] # - NOTE: Saving the joint fit file with only the name of the first exposure stdfile = findfile('stdstars', args.night, args.expids[0], spectrograph=sp) #stdfile.replace('{:08d}'.format(args.expids[0]),'-'.join(['{:08d}'.format(eid) for eid in args.expids])) cmd = "desi_fit_stdstars" cmd += " --delta-color 0.1" cmd += " --frames {}".format(' '.join(framefiles[sp])) cmd += " --skymodels {}".format(' '.join(skyfiles[sp])) cmd += " --fiberflats {}".format(' '.join(fiberflatfiles[sp])) cmd += " --starmodels {}".format(starmodels) cmd += " --outfile {}".format(stdfile) if args.maxstdstars is not None: cmd += " --maxstdstars {}".format(args.maxstdstars) inputs = framefiles[sp] + skyfiles[sp] + fiberflatfiles[sp] num_cmd += 1 err = runcmd(cmd, inputs=inputs, outputs=[stdfile]) if err: num_err += 1 timer.stop('stdstarfit') num_cmd, num_err = mpi_count_failures(num_cmd, num_err, comm=comm) if comm is not None: comm.barrier() if rank == 0 and num_err > 0: log.error(f'{num_err}/{num_cmd} stdstar commands failed') sys.stdout.flush() if num_err > 0 and num_err == num_cmd: if rank == 0: log.critical('All stdstar commands failed') sys.exit(1) if rank == 0 and len(args.expids) > 1: for sp in spectro_nums: saved_stdfile = findfile('stdstars', args.night, args.expids[0], spectrograph=sp) for expid in args.expids[1:]: new_stdfile = findfile('stdstars', args.night, expid, spectrograph=sp) new_dirname, new_fname = os.path.split(new_stdfile) log.debug( "Path exists: {}, file exists: {}, link exists: {}". format(os.path.exists(new_stdfile), os.path.isfile(new_stdfile), os.path.islink(new_stdfile))) relpath_saved_std = os.path.relpath( saved_stdfile, new_dirname) log.debug(f'Sym Linking jointly fitted stdstar file: {new_stdfile} '+\ f'to existing file at rel. path {relpath_saved_std}') runcmd(os.symlink, args=(relpath_saved_std, new_stdfile), \ inputs=[saved_stdfile, ], outputs=[new_stdfile, ]) log.debug( "Path exists: {}, file exists: {}, link exists: {}". format(os.path.exists(new_stdfile), os.path.isfile(new_stdfile), os.path.islink(new_stdfile))) # ------------------------------------------------------------------------- # - Wrap up # if rank == 0: # report = timer.report() # log.info('Rank 0 timing report:\n' + report) if comm is not None: timers = comm.gather(timer, root=0) else: timers = [ timer, ] if rank == 0: stats = desiutil.timer.compute_stats(timers) log.info('Timing summary statistics:\n' + json.dumps(stats, indent=2)) if args.timingfile: if os.path.exists(args.timingfile): with open(args.timingfile) as fx: previous_stats = json.load(fx) #- augment previous_stats with new entries, but don't overwrite old for name in stats: if name not in previous_stats: previous_stats[name] = stats[name] stats = previous_stats tmpfile = args.timingfile + '.tmp' with open(tmpfile, 'w') as fx: json.dump(stats, fx, indent=2) os.rename(tmpfile, args.timingfile) if rank == 0: log.info('All done at {}'.format(time.asctime()))
def main(args=None, comm=None): if args is None: args = parse() # elif isinstance(args, (list, tuple)): # args = parse(args) log = get_logger() start_mpi_connect = time.time() if comm is not None: #- Use the provided comm to determine rank and size rank = comm.rank size = comm.size else: #- Check MPI flags and determine the comm, rank, and size given the arguments comm, rank, size = assign_mpi(do_mpi=args.mpi, do_batch=args.batch, log=log) stop_mpi_connect = time.time() #- Start timer; only print log messages from rank 0 (others are silent) timer = desiutil.timer.Timer(silent=(rank > 0)) #- Fill in timing information for steps before we had the timer created if args.starttime is not None: timer.start('startup', starttime=args.starttime) timer.stop('startup', stoptime=start_imports) timer.start('imports', starttime=start_imports) timer.stop('imports', stoptime=stop_imports) timer.start('mpi_connect', starttime=start_mpi_connect) timer.stop('mpi_connect', stoptime=stop_mpi_connect) #- Freeze IERS after parsing args so that it doesn't bother if only --help timer.start('freeze_iers') desiutil.iers.freeze_iers() timer.stop('freeze_iers') #- Preflight checks timer.start('preflight') if rank > 0: #- Let rank 0 fetch these, and then broadcast args, hdr, camhdr = None, None, None else: args, hdr, camhdr = update_args_with_headers(args) ## Make sure badamps is formatted properly if comm is not None and rank == 0 and args.badamps is not None: args.badamps = validate_badamps(args.badamps) if comm is not None: args = comm.bcast(args, root=0) hdr = comm.bcast(hdr, root=0) camhdr = comm.bcast(camhdr, root=0) known_obstype = [ 'SCIENCE', 'ARC', 'FLAT', 'ZERO', 'DARK', 'TESTARC', 'TESTFLAT', 'PIXFLAT', 'SKY', 'TWILIGHT', 'OTHER' ] if args.obstype not in known_obstype: raise RuntimeError('obstype {} not in {}'.format( args.obstype, known_obstype)) timer.stop('preflight') #------------------------------------------------------------------------- #- Create and submit a batch job if requested if args.batch: #exp_str = '{:08d}'.format(args.expid) jobdesc = args.obstype.lower() if args.obstype == 'SCIENCE': # if not doing pre-stdstar fitting or stdstar fitting and if there is # no flag stopping flux calibration, set job to poststdstar if args.noprestdstarfit and args.nostdstarfit and ( not args.nofluxcalib): jobdesc = 'poststdstar' # elif told not to do std or post stdstar but the flag for prestdstar isn't set, # then perform prestdstar elif (not args.noprestdstarfit ) and args.nostdstarfit and args.nofluxcalib: jobdesc = 'prestdstar' #elif (not args.noprestdstarfit) and (not args.nostdstarfit) and (not args.nofluxcalib): # jobdesc = 'science' scriptfile = create_desi_proc_batch_script(night=args.night, exp=args.expid, cameras=args.cameras,\ jobdesc=jobdesc, queue=args.queue, runtime=args.runtime,\ batch_opts=args.batch_opts, timingfile=args.timingfile, system_name=args.system_name) err = 0 if not args.nosubmit: err = subprocess.call(['sbatch', scriptfile]) sys.exit(err) #------------------------------------------------------------------------- #- Proceeding with running #- What are we going to do? if rank == 0: log.info('----------') log.info('Input {}'.format(args.input)) log.info('Night {} expid {}'.format(args.night, args.expid)) log.info('Obstype {}'.format(args.obstype)) log.info('Cameras {}'.format(args.cameras)) log.info('Output root {}'.format(desispec.io.specprod_root())) log.info('----------') #- Create output directories if needed if rank == 0: preprocdir = os.path.dirname( findfile('preproc', args.night, args.expid, 'b0')) expdir = os.path.dirname( findfile('frame', args.night, args.expid, 'b0')) os.makedirs(preprocdir, exist_ok=True) os.makedirs(expdir, exist_ok=True) #- Wait for rank 0 to make directories before proceeding if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Preproc #- All obstypes get preprocessed timer.start('fibermap') #- Assemble fibermap for science exposures fibermap = None fibermap_ok = None if rank == 0 and args.obstype == 'SCIENCE': fibermap = findfile('fibermap', args.night, args.expid) if not os.path.exists(fibermap): tmp = findfile('preproc', args.night, args.expid, 'b0') preprocdir = os.path.dirname(tmp) fibermap = os.path.join(preprocdir, os.path.basename(fibermap)) log.info('Creating fibermap {}'.format(fibermap)) cmd = 'assemble_fibermap -n {} -e {} -o {}'.format( args.night, args.expid, fibermap) if args.badamps is not None: cmd += ' --badamps={}'.format(args.badamps) runcmd(cmd, inputs=[], outputs=[fibermap]) fibermap_ok = os.path.exists(fibermap) #- Some commissioning files didn't have coords* files that caused assemble_fibermap to fail #- these are well known failures with no other solution, so for those, just force creation #- of a fibermap with null coordinate information if not fibermap_ok and int(args.night) < 20200310: log.info( "Since night is before 20200310, trying to force fibermap creation without coords file" ) cmd += ' --force' runcmd(cmd, inputs=[], outputs=[fibermap]) fibermap_ok = os.path.exists(fibermap) #- If assemble_fibermap failed and obstype is SCIENCE, exit now if comm is not None: fibermap_ok = comm.bcast(fibermap_ok, root=0) if args.obstype == 'SCIENCE' and not fibermap_ok: sys.stdout.flush() if rank == 0: log.critical( 'assemble_fibermap failed for science exposure; exiting now') sys.exit(13) #- Wait for rank 0 to make fibermap if needed if comm is not None: fibermap = comm.bcast(fibermap, root=0) timer.stop('fibermap') if not (args.obstype in ['SCIENCE'] and args.noprestdstarfit): timer.start('preproc') for i in range(rank, len(args.cameras), size): camera = args.cameras[i] outfile = findfile('preproc', args.night, args.expid, camera) outdir = os.path.dirname(outfile) cmd = "desi_preproc -i {} -o {} --outdir {} --cameras {}".format( args.input, outfile, outdir, camera) if args.scattered_light: cmd += " --scattered-light" if fibermap is not None: cmd += " --fibermap {}".format(fibermap) if not args.obstype in ['ARC']: # never model variance for arcs if not args.no_model_pixel_variance: cmd += " --model-variance" runcmd(cmd, inputs=[args.input], outputs=[outfile]) timer.stop('preproc') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Get input PSFs timer.start('findpsf') input_psf = dict() if rank == 0: for camera in args.cameras: if args.psf is not None: input_psf[camera] = args.psf elif args.calibnight is not None: # look for a psfnight psf for this calib night psfnightfile = findfile('psfnight', args.calibnight, args.expid, camera) if not os.path.isfile(psfnightfile): log.error("no {}".format(psfnightfile)) raise IOError("no {}".format(psfnightfile)) input_psf[camera] = psfnightfile else: # look for a psfnight psf psfnightfile = findfile('psfnight', args.night, args.expid, camera) if os.path.isfile(psfnightfile): input_psf[camera] = psfnightfile elif args.most_recent_calib: nightfile = find_most_recent(args.night, file_type='psfnight') if nightfile is None: input_psf[camera] = findcalibfile( [hdr, camhdr[camera]], 'PSF') else: input_psf[camera] = nightfile else: input_psf[camera] = findcalibfile([hdr, camhdr[camera]], 'PSF') log.info("Will use input PSF : {}".format(input_psf[camera])) if comm is not None: input_psf = comm.bcast(input_psf, root=0) timer.stop('findpsf') #------------------------------------------------------------------------- #- Traceshift if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT'] ) or \ ( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ): timer.start('traceshift') if rank == 0 and args.traceshift: log.info('Starting traceshift at {}'.format(time.asctime())) for i in range(rank, len(args.cameras), size): camera = args.cameras[i] preprocfile = findfile('preproc', args.night, args.expid, camera) inpsf = input_psf[camera] outpsf = findfile('psf', args.night, args.expid, camera) if not os.path.isfile(outpsf): if args.traceshift: cmd = "desi_compute_trace_shifts" cmd += " -i {}".format(preprocfile) cmd += " --psf {}".format(inpsf) cmd += " --outpsf {}".format(outpsf) cmd += " --degxx 2 --degxy 0" if args.obstype in ['FLAT', 'TESTFLAT', 'TWILIGHT']: cmd += " --continuum" else: cmd += " --degyx 2 --degyy 0" if args.obstype in ['SCIENCE', 'SKY']: cmd += ' --sky' else: cmd = "ln -s {} {}".format(inpsf, outpsf) runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf]) else: log.info("PSF {} exists".format(outpsf)) timer.stop('traceshift') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- PSF #- MPI parallelize this step if args.obstype in ['ARC', 'TESTARC']: timer.start('arc_traceshift') if rank == 0: log.info('Starting traceshift before specex PSF fit at {}'.format( time.asctime())) for i in range(rank, len(args.cameras), size): camera = args.cameras[i] preprocfile = findfile('preproc', args.night, args.expid, camera) inpsf = input_psf[camera] outpsf = findfile('psf', args.night, args.expid, camera) outpsf = replace_prefix(outpsf, "psf", "shifted-input-psf") if not os.path.isfile(outpsf): cmd = "desi_compute_trace_shifts" cmd += " -i {}".format(preprocfile) cmd += " --psf {}".format(inpsf) cmd += " --outpsf {}".format(outpsf) cmd += " --degxx 0 --degxy 0 --degyx 0 --degyy 0" cmd += ' --arc-lamps' runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf]) else: log.info("PSF {} exists".format(outpsf)) timer.stop('arc_traceshift') if comm is not None: comm.barrier() timer.start('psf') if rank == 0: log.info('Starting specex PSF fitting at {}'.format( time.asctime())) if rank > 0: cmds = inputs = outputs = None else: cmds = dict() inputs = dict() outputs = dict() for camera in args.cameras: preprocfile = findfile('preproc', args.night, args.expid, camera) tmpname = findfile('psf', args.night, args.expid, camera) inpsf = replace_prefix(tmpname, "psf", "shifted-input-psf") outpsf = replace_prefix(tmpname, "psf", "fit-psf") log.info("now run specex psf fit") cmd = 'desi_compute_psf' cmd += ' --input-image {}'.format(preprocfile) cmd += ' --input-psf {}'.format(inpsf) cmd += ' --output-psf {}'.format(outpsf) # look for fiber blacklist cfinder = CalibFinder([hdr, camhdr[camera]]) blacklistkey = "FIBERBLACKLIST" if not cfinder.haskey(blacklistkey) and cfinder.haskey( "BROKENFIBERS"): log.warning( "BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST" ) blacklistkey = "BROKENFIBERS" if cfinder.haskey(blacklistkey): blacklist = cfinder.value(blacklistkey) cmd += ' --broken-fibers {}'.format(blacklist) if rank == 0: log.warning('broken fibers: {}'.format(blacklist)) if not os.path.exists(outpsf): cmds[camera] = cmd inputs[camera] = [preprocfile, inpsf] outputs[camera] = [ outpsf, ] if comm is not None: cmds = comm.bcast(cmds, root=0) inputs = comm.bcast(inputs, root=0) outputs = comm.bcast(outputs, root=0) #- split communicator by 20 (number of bundles) group_size = 20 if (rank == 0) and (size % group_size != 0): log.warning( 'MPI size={} should be evenly divisible by {}'.format( size, group_size)) group = rank // group_size num_groups = (size + group_size - 1) // group_size comm_group = comm.Split(color=group) if rank == 0: log.info( f'Fitting PSFs with {num_groups} sub-communicators of size {group_size}' ) for i in range(group, len(args.cameras), num_groups): camera = args.cameras[i] if camera in cmds: cmdargs = cmds[camera].split()[1:] cmdargs = desispec.scripts.specex.parse(cmdargs) if comm_group.rank == 0: print('RUNNING: {}'.format(cmds[camera])) t0 = time.time() timestamp = time.asctime() log.info( f'MPI group {group} ranks {rank}-{rank+group_size-1} fitting PSF for {camera} at {timestamp}' ) try: desispec.scripts.specex.main(cmdargs, comm=comm_group) except Exception as e: if comm_group.rank == 0: log.error( f'FAILED: MPI group {group} ranks {rank}-{rank+group_size-1} camera {camera}' ) log.error('FAILED: {}'.format(cmds[camera])) log.error(e) if comm_group.rank == 0: specex_time = time.time() - t0 log.info( f'specex fit for {camera} took {specex_time:.1f} seconds' ) comm.barrier() else: log.warning( 'fitting PSFs without MPI parallelism; this will be SLOW') for camera in args.cameras: if camera in cmds: runcmd(cmds[camera], inputs=inputs[camera], outputs=outputs[camera]) if comm is not None: comm.barrier() # loop on all cameras and interpolate bad fibers for camera in args.cameras[rank::size]: t0 = time.time() log.info(f'Rank {rank} interpolating {camera} PSF over bad fibers') # look for fiber blacklist cfinder = CalibFinder([hdr, camhdr[camera]]) blacklistkey = "FIBERBLACKLIST" if not cfinder.haskey(blacklistkey) and cfinder.haskey( "BROKENFIBERS"): log.warning( "BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST" ) blacklistkey = "BROKENFIBERS" if cfinder.haskey(blacklistkey): fiberblacklist = cfinder.value(blacklistkey) tmpname = findfile('psf', args.night, args.expid, camera) inpsf = replace_prefix(tmpname, "psf", "fit-psf") outpsf = replace_prefix(tmpname, "psf", "fit-psf-fixed-blacklisted") if os.path.isfile(inpsf) and not os.path.isfile(outpsf): cmd = 'desi_interpolate_fiber_psf' cmd += ' --infile {}'.format(inpsf) cmd += ' --outfile {}'.format(outpsf) cmd += ' --fibers {}'.format(fiberblacklist) log.info( 'For camera {} interpolating PSF for broken fibers: {}' .format(camera, fiberblacklist)) runcmd(cmd, inputs=[inpsf], outputs=[outpsf]) if os.path.isfile(outpsf): os.rename( inpsf, inpsf.replace("fit-psf", "fit-psf-before-blacklisted-fix")) subprocess.call('cp {} {}'.format(outpsf, inpsf), shell=True) dt = time.time() - t0 log.info( f'Rank {rank} {camera} PSF interpolation took {dt:.1f} sec') timer.stop('psf') #------------------------------------------------------------------------- #- Merge PSF of night if applicable #if args.obstype in ['ARC']: if False: if rank == 0: for camera in args.cameras: psfnightfile = findfile('psfnight', args.night, args.expid, camera) if not os.path.isfile( psfnightfile ): # we still don't have a psf night, see if we can compute it ... psfs = glob.glob( findfile('psf', args.night, args.expid, camera).replace("psf", "fit-psf").replace( str(args.expid), "*")) log.info( "Number of PSF for night={} camera={} = {}".format( args.night, camera, len(psfs))) if len(psfs) > 4: # lets do it! log.info("Computing psfnight ...") dirname = os.path.dirname(psfnightfile) if not os.path.isdir(dirname): os.makedirs(dirname) desispec.scripts.specex.mean_psf(psfs, psfnightfile) if os.path.isfile(psfnightfile): # now use this one input_psf[camera] = psfnightfile #------------------------------------------------------------------------- #- Extract #- This is MPI parallel so handle a bit differently # maybe add ARC and TESTARC too if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT'] ) or \ ( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ): timer.start('extract') if rank == 0: log.info('Starting extractions at {}'.format(time.asctime())) if rank > 0: cmds = inputs = outputs = None else: cmds = dict() inputs = dict() outputs = dict() for camera in args.cameras: cmd = 'desi_extract_spectra' #- Based on data from SM1-SM8, looking at central and edge fibers #- with in mind overlapping arc lamps lines if camera.startswith('b'): cmd += ' -w 3600.0,5800.0,0.8' elif camera.startswith('r'): cmd += ' -w 5760.0,7620.0,0.8' elif camera.startswith('z'): cmd += ' -w 7520.0,9824.0,0.8' preprocfile = findfile('preproc', args.night, args.expid, camera) psffile = findfile('psf', args.night, args.expid, camera) framefile = findfile('frame', args.night, args.expid, camera) cmd += ' -i {}'.format(preprocfile) cmd += ' -p {}'.format(psffile) cmd += ' -o {}'.format(framefile) cmd += ' --psferr 0.1' if args.obstype == 'SCIENCE' or args.obstype == 'SKY': if rank == 0: log.info('Include barycentric correction') cmd += ' --barycentric-correction' if not os.path.exists(framefile): cmds[camera] = cmd inputs[camera] = [preprocfile, psffile] outputs[camera] = [ framefile, ] #- TODO: refactor/combine this with PSF comm splitting logic if comm is not None: cmds = comm.bcast(cmds, root=0) inputs = comm.bcast(inputs, root=0) outputs = comm.bcast(outputs, root=0) #- split communicator by 20 (number of bundles) extract_size = 20 if (rank == 0) and (size % extract_size != 0): log.warning( 'MPI size={} should be evenly divisible by {}'.format( size, extract_size)) extract_group = rank // extract_size num_extract_groups = (size + extract_size - 1) // extract_size comm_extract = comm.Split(color=extract_group) for i in range(extract_group, len(args.cameras), num_extract_groups): camera = args.cameras[i] if camera in cmds: cmdargs = cmds[camera].split()[1:] extract_args = desispec.scripts.extract.parse(cmdargs) if comm_extract.rank == 0: print('RUNNING: {}'.format(cmds[camera])) desispec.scripts.extract.main_mpi(extract_args, comm=comm_extract) comm.barrier() else: log.warning( 'running extractions without MPI parallelism; this will be SLOW' ) for camera in args.cameras: if camera in cmds: runcmd(cmds[camera], inputs=inputs[camera], outputs=outputs[camera]) timer.stop('extract') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Fiberflat if args.obstype in ['FLAT', 'TESTFLAT']: timer.start('fiberflat') if rank == 0: log.info('Starting fiberflats at {}'.format(time.asctime())) for i in range(rank, len(args.cameras), size): camera = args.cameras[i] framefile = findfile('frame', args.night, args.expid, camera) fiberflatfile = findfile('fiberflat', args.night, args.expid, camera) cmd = "desi_compute_fiberflat" cmd += " -i {}".format(framefile) cmd += " -o {}".format(fiberflatfile) runcmd(cmd, inputs=[ framefile, ], outputs=[ fiberflatfile, ]) timer.stop('fiberflat') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Average and auto-calib fiberflats of night if applicable #if args.obstype in ['FLAT']: if False: if rank == 0: fiberflatnightfile = findfile('fiberflatnight', args.night, args.expid, args.cameras[0]) fiberflatdirname = os.path.dirname(fiberflatnightfile) if not os.path.isfile(fiberflatnightfile) and len( args.cameras ) >= 6: # we still don't have them, see if we can compute them, but need at least 2 spectros ... flats = glob.glob( findfile('fiberflat', args.night, args.expid, "b0").replace(str(args.expid), "*").replace("b0", "*")) log.info("Number of fiberflat for night {} = {}".format( args.night, len(flats))) if len(flats) >= 3 * 4 * len( args.cameras ): # lets do it! (3 exposures x 4 lamps x N cameras) log.info( "Computing fiberflatnight per lamp and camera ...") tmpdir = os.path.join(fiberflatdirname, "tmp") if not os.path.isdir(tmpdir): os.makedirs(tmpdir) log.info( "First average measurements per camera and per lamp") average_flats = dict() for camera in args.cameras: # list of flats for this camera flats_for_this_camera = [] for flat in flats: if flat.find(camera) >= 0: flats_for_this_camera.append(flat) #log.info("For camera {} , flats = {}".format(camera,flats_for_this_camera)) #sys.exit(12) # average per lamp (and camera) average_flats[camera] = list() for lampbox in range(4): ofile = os.path.join( tmpdir, "fiberflatnight-camera-{}-lamp-{}.fits".format( camera, lampbox)) if not os.path.isfile(ofile): log.info( "Average flat for camera {} and lamp box #{}" .format(camera, lampbox)) pg = "CALIB DESI-CALIB-0{} LEDs only".format( lampbox) cmd = "desi_average_fiberflat --program '{}' --outfile {} -i ".format( pg, ofile) for flat in flats_for_this_camera: cmd += " {} ".format(flat) runcmd(cmd, inputs=flats_for_this_camera, outputs=[ ofile, ]) if os.path.isfile(ofile): average_flats[camera].append(ofile) else: log.info("Will use existing {}".format(ofile)) average_flats[camera].append(ofile) log.info( "Auto-calibration across lamps and spectro per camera arm (b,r,z)" ) for camera_arm in ["b", "r", "z"]: cameras_for_this_arm = [] flats_for_this_arm = [] for camera in args.cameras: if camera[0].lower() == camera_arm: cameras_for_this_arm.append(camera) if camera in average_flats: for flat in average_flats[camera]: flats_for_this_arm.append(flat) cmd = "desi_autocalib_fiberflat --night {} --arm {} -i ".format( args.night, camera_arm) for flat in flats_for_this_arm: cmd += " {} ".format(flat) runcmd(cmd, inputs=flats_for_this_arm, outputs=[]) log.info("Done with fiber flats per night") if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Get input fiberflat if args.obstype in ['SCIENCE', 'SKY'] and (not args.nofiberflat): timer.start('find_fiberflat') input_fiberflat = dict() if rank == 0: for camera in args.cameras: if args.fiberflat is not None: input_fiberflat[camera] = args.fiberflat elif args.calibnight is not None: # look for a fiberflatnight for this calib night fiberflatnightfile = findfile('fiberflatnight', args.calibnight, args.expid, camera) if not os.path.isfile(fiberflatnightfile): log.error("no {}".format(fiberflatnightfile)) raise IOError("no {}".format(fiberflatnightfile)) input_fiberflat[camera] = fiberflatnightfile else: # look for a fiberflatnight fiberflat fiberflatnightfile = findfile('fiberflatnight', args.night, args.expid, camera) if os.path.isfile(fiberflatnightfile): input_fiberflat[camera] = fiberflatnightfile elif args.most_recent_calib: nightfile = find_most_recent( args.night, file_type='fiberflatnight') if nightfile is None: input_fiberflat[camera] = findcalibfile( [hdr, camhdr[camera]], 'FIBERFLAT') else: input_fiberflat[camera] = nightfile else: input_fiberflat[camera] = findcalibfile( [hdr, camhdr[camera]], 'FIBERFLAT') log.info("Will use input FIBERFLAT: {}".format( input_fiberflat[camera])) if comm is not None: input_fiberflat = comm.bcast(input_fiberflat, root=0) timer.stop('find_fiberflat') #------------------------------------------------------------------------- #- Apply fiberflat and write fframe file if args.obstype in ['SCIENCE', 'SKY'] and args.fframe and \ ( not args.nofiberflat ) and (not args.noprestdstarfit): timer.start('apply_fiberflat') if rank == 0: log.info('Applying fiberflat at {}'.format(time.asctime())) for i in range(rank, len(args.cameras), size): camera = args.cameras[i] fframefile = findfile('fframe', args.night, args.expid, camera) if not os.path.exists(fframefile): framefile = findfile('frame', args.night, args.expid, camera) fr = desispec.io.read_frame(framefile) flatfilename = input_fiberflat[camera] if flatfilename is not None: ff = desispec.io.read_fiberflat(flatfilename) fr.meta['FIBERFLT'] = desispec.io.shorten_filename( flatfilename) apply_fiberflat(fr, ff) fframefile = findfile('fframe', args.night, args.expid, camera) desispec.io.write_frame(fframefile, fr) else: log.warning( "Missing fiberflat for camera {}".format(camera)) timer.stop('apply_fiberflat') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Select random sky fibers (inplace update of frame file) #- TODO: move this to a function somewhere #- TODO: this assigns different sky fibers to each frame of same spectrograph if (args.obstype in [ 'SKY', 'SCIENCE' ]) and (not args.noskysub) and (not args.noprestdstarfit): timer.start('picksky') if rank == 0: log.info('Picking sky fibers at {}'.format(time.asctime())) for i in range(rank, len(args.cameras), size): camera = args.cameras[i] framefile = findfile('frame', args.night, args.expid, camera) orig_frame = desispec.io.read_frame(framefile) #- Make a copy so that we can apply fiberflat fr = deepcopy(orig_frame) if np.any(fr.fibermap['OBJTYPE'] == 'SKY'): log.info('{} sky fibers already set; skipping'.format( os.path.basename(framefile))) continue #- Apply fiberflat then select random fibers below a flux cut flatfilename = input_fiberflat[camera] if flatfilename is None: log.error("No fiberflat for {}".format(camera)) continue ff = desispec.io.read_fiberflat(flatfilename) apply_fiberflat(fr, ff) sumflux = np.sum(fr.flux, axis=1) fluxcut = np.percentile(sumflux, 30) iisky = np.where(sumflux < fluxcut)[0] iisky = np.random.choice(iisky, size=100, replace=False) #- Update fibermap or original frame and write out orig_frame.fibermap['OBJTYPE'][iisky] = 'SKY' orig_frame.fibermap['DESI_TARGET'][iisky] |= desi_mask.SKY desispec.io.write_frame(framefile, orig_frame) timer.stop('picksky') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Sky subtraction if args.obstype in [ 'SCIENCE', 'SKY' ] and (not args.noskysub) and (not args.noprestdstarfit): timer.start('skysub') if rank == 0: log.info('Starting sky subtraction at {}'.format(time.asctime())) for i in range(rank, len(args.cameras), size): camera = args.cameras[i] framefile = findfile('frame', args.night, args.expid, camera) hdr = fitsio.read_header(framefile, 'FLUX') fiberflatfile = input_fiberflat[camera] if fiberflatfile is None: log.error("No fiberflat for {}".format(camera)) continue skyfile = findfile('sky', args.night, args.expid, camera) cmd = "desi_compute_sky" cmd += " -i {}".format(framefile) cmd += " --fiberflat {}".format(fiberflatfile) cmd += " --o {}".format(skyfile) if args.no_extra_variance: cmd += " --no-extra-variance" if not args.no_sky_wavelength_adjustment: cmd += " --adjust-wavelength" if not args.no_sky_lsf_adjustment: cmd += " --adjust-lsf" runcmd(cmd, inputs=[framefile, fiberflatfile], outputs=[ skyfile, ]) #- sframe = flatfielded sky-subtracted but not flux calibrated frame #- Note: this re-reads and re-does steps previously done for picking #- sky fibers; desi_proc is about human efficiency, #- not I/O or CPU efficiency... sframefile = desispec.io.findfile('sframe', args.night, args.expid, camera) if not os.path.exists(sframefile): frame = desispec.io.read_frame(framefile) fiberflat = desispec.io.read_fiberflat(fiberflatfile) sky = desispec.io.read_sky(skyfile) apply_fiberflat(frame, fiberflat) subtract_sky(frame, sky, apply_throughput_correction=True) frame.meta['IN_SKY'] = shorten_filename(skyfile) frame.meta['FIBERFLT'] = shorten_filename(fiberflatfile) desispec.io.write_frame(sframefile, frame) timer.stop('skysub') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Standard Star Fitting if args.obstype in ['SCIENCE',] and \ (not args.noskysub ) and \ (not args.nostdstarfit) : timer.start('stdstarfit') if rank == 0: log.info('Starting flux calibration at {}'.format(time.asctime())) #- Group inputs by spectrograph framefiles = dict() skyfiles = dict() fiberflatfiles = dict() night, expid = args.night, args.expid #- shorter for camera in args.cameras: sp = int(camera[1]) if sp not in framefiles: framefiles[sp] = list() skyfiles[sp] = list() fiberflatfiles[sp] = list() framefiles[sp].append(findfile('frame', night, expid, camera)) skyfiles[sp].append(findfile('sky', night, expid, camera)) fiberflatfiles[sp].append(input_fiberflat[camera]) #- Hardcoded stdstar model version starmodels = os.path.join(os.getenv('DESI_BASIS_TEMPLATES'), 'stdstar_templates_v2.2.fits') #- Fit stdstars per spectrograph (not per-camera) spectro_nums = sorted(framefiles.keys()) ## for sp in spectro_nums[rank::size]: for i in range(rank, len(spectro_nums), size): sp = spectro_nums[i] stdfile = findfile('stdstars', night, expid, spectrograph=sp) cmd = "desi_fit_stdstars" cmd += " --frames {}".format(' '.join(framefiles[sp])) cmd += " --skymodels {}".format(' '.join(skyfiles[sp])) cmd += " --fiberflats {}".format(' '.join(fiberflatfiles[sp])) cmd += " --starmodels {}".format(starmodels) cmd += " --outfile {}".format(stdfile) cmd += " --delta-color 0.1" if args.maxstdstars is not None: cmd += " --maxstdstars {}".format(args.maxstdstars) inputs = framefiles[sp] + skyfiles[sp] + fiberflatfiles[sp] runcmd(cmd, inputs=inputs, outputs=[stdfile]) timer.stop('stdstarfit') if comm is not None: comm.barrier() # ------------------------------------------------------------------------- # - Flux calibration if args.obstype in ['SCIENCE'] and \ (not args.noskysub) and \ (not args.nofluxcalib): timer.start('fluxcalib') night, expid = args.night, args.expid #- shorter #- Compute flux calibration vectors per camera for camera in args.cameras[rank::size]: framefile = findfile('frame', night, expid, camera) skyfile = findfile('sky', night, expid, camera) spectrograph = int(camera[1]) stdfile = findfile('stdstars', night, expid, spectrograph=spectrograph) calibfile = findfile('fluxcalib', night, expid, camera) fiberflatfile = input_fiberflat[camera] cmd = "desi_compute_fluxcalibration" cmd += " --infile {}".format(framefile) cmd += " --sky {}".format(skyfile) cmd += " --fiberflat {}".format(fiberflatfile) cmd += " --models {}".format(stdfile) cmd += " --outfile {}".format(calibfile) cmd += " --delta-color-cut 0.1" inputs = [framefile, skyfile, fiberflatfile, stdfile] runcmd(cmd, inputs=inputs, outputs=[ calibfile, ]) timer.stop('fluxcalib') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Applying flux calibration if args.obstype in [ 'SCIENCE', ] and (not args.noskysub) and (not args.nofluxcalib): night, expid = args.night, args.expid #- shorter timer.start('applycalib') if rank == 0: log.info('Starting cframe file creation at {}'.format( time.asctime())) for camera in args.cameras[rank::size]: framefile = findfile('frame', night, expid, camera) skyfile = findfile('sky', night, expid, camera) spectrograph = int(camera[1]) stdfile = findfile('stdstars', night, expid, spectrograph=spectrograph) calibfile = findfile('fluxcalib', night, expid, camera) cframefile = findfile('cframe', night, expid, camera) fiberflatfile = input_fiberflat[camera] cmd = "desi_process_exposure" cmd += " --infile {}".format(framefile) cmd += " --fiberflat {}".format(fiberflatfile) cmd += " --sky {}".format(skyfile) cmd += " --calib {}".format(calibfile) cmd += " --outfile {}".format(cframefile) cmd += " --cosmics-nsig 6" if args.no_xtalk: cmd += " --no-xtalk" inputs = [framefile, fiberflatfile, skyfile, calibfile] runcmd(cmd, inputs=inputs, outputs=[ cframefile, ]) if comm is not None: comm.barrier() timer.stop('applycalib') #------------------------------------------------------------------------- #- Wrap up # if rank == 0: # report = timer.report() # log.info('Rank 0 timing report:\n' + report) if comm is not None: timers = comm.gather(timer, root=0) else: timers = [ timer, ] if rank == 0: stats = desiutil.timer.compute_stats(timers) log.info('Timing summary statistics:\n' + json.dumps(stats, indent=2)) if args.timingfile: if os.path.exists(args.timingfile): with open(args.timingfile) as fx: previous_stats = json.load(fx) #- augment previous_stats with new entries, but don't overwrite old for name in stats: if name not in previous_stats: previous_stats[name] = stats[name] stats = previous_stats tmpfile = args.timingfile + '.tmp' with open(tmpfile, 'w') as fx: json.dump(stats, fx, indent=2) os.rename(tmpfile, args.timingfile) if rank == 0: log.info('All done at {}'.format(time.asctime()))
def integration_test(args=None): """ Run an integration test from raw data simulations through QuickLook pipeline Args: night (str, optional): YEARMMDD nspec (int, optional): number of spectra to simulate overwrite (bool, otional) : overwrite existing files delete (bool, optional) : delete all inputs and outputs ql_data (bool, optional) : read data from $QL_SPEC_DATA ql_redux (bool, optional) : write files to $QL_SPEC_REDUX Raises: RuntimeError if QuickLook fails """ #- Parse arguments and check environment variables args = parse(args) night = args.night nspec = args.nspec expid = 2 flat_expid = 1 if args.ql_data: ql_data = 'Use $QL_SPEC_DATA' raw_dir = os.environ['QL_SPEC_DATA'] else: ql_data = 'Use $DESI_SPECTRO_DATA' raw_dir = os.path.join(os.environ['DESI_SPECTRO_DATA'],'QL') if args.ql_redux: ql_redux = 'Use $QL_SPEC_REDUX' output_dir = os.environ['QL_SPEC_REDUX'] else: ql_redux = 'Use $DESI_SPECTRO_REDUX/QL/$SPECPROD' output_dir = os.path.join(os.environ['DESI_SPECTRO_REDUX'],'QL',os.environ['SPECPROD']) #- Simulate inputs sim(night=night, nspec=nspec, ql_data=ql_data, ql_redux=ql_redux) #- Get the configuration file from desispec/data/quicklook from pkg_resources import resource_filename configfile=resource_filename('desispec','data/quicklook/qlconfig_dark.yaml') for camera in ['r0','z0']: #- Verify that quicklook pipeline runs if args.ql_data and args.ql_redux: com = "desi_quicklook -i {} -n {} -c {} -e {} -f {}".format(configfile,night,camera,expid) else: com = "desi_quicklook -i {} -n {} -c {} -e {} -f {} --psfboot {} --fiberflat {} --rawdata_dir {} --specprod_dir {}".format(configfile,night,camera,expid,raw_dir,output_dir) if runcmd(com) != 0: raise RuntimeError('quicklook pipeline failed for camera {}'.format(camera)) #- Remove all output if desired if args.delete: sim_dir = os.path.join(os.environ['DESI_SPECTRO_SIM'],os.environ['PIXPROD'],args.night) if os.path.exists(sim_dir): sim_files = os.listdir(sim_dir) for file in range(len(sim_files)): sim_file = os.path.join(sim_dir,sim_files[file]) os.remove(sim_file) os.rmdir(sim_dir) data_dir = os.path.join(raw_dir,night) if os.path.exists(data_dir): data_files = os.listdir(data_dir) for file in range(len(data_files)): data_file = os.path.join(data_dir,data_files[file]) os.remove(data_file) os.rmdir(data_dir) if os.path.exists(output_dir): exp_dir = os.path.join(output_dir,'exposures',night) if os.path.exists(exp_dir): id_dir = os.path.join(exp_dir,'00000002') if os.path.exists(id_dir): id_files = os.listdir(id_dir) for file in range(len(id_files)): id_file = os.path.join(id_dir,id_files[file]) os.remove(id_file) os.rmdir(id_dir) exp_files = os.listdir(exp_dir) for file in range(len(exp_files)): exp_file = os.path.join(exp_dir,exp_files[file]) os.remove(exp_file) os.rmdir(exp_dir) calib_dir=os.path.join(output_dir,'calib2d',night) if os.path.exists(calib_dir): calib_files = os.listdir(calib_dir) for file in range(len(calib_files)): calib_file = os.path.join(calib_dir,calib_files[file]) os.remove(calib_file) os.rmdir(calib_dir) psf_dir=os.path.join(output_dir,'calib2d','psf',night) if os.path.exists(psf_dir): psf_files = os.listdir(psf_dir) for ii in range(len(psf_files)): thisfile=os.path.join(psf_dir,psf_files[ii]) os.remove(thisfile) os.rmdir(psf_dir)
def test_function(self): def blat(*args): return list(args) self.assertEqual(util.runcmd(blat, args=[1,2,3]), [1,2,3]) self.assertEqual(util.runcmd(blat), [])
def sim(night,nspec): """ Simulate data as part of the QuickLook integration test. Args: night (str): YEARMMDD nspec (int): number of spectra to simulate Raises: RuntimeError if any script fails """ psf_b = os.path.join(os.environ['DESIMODEL'],'data','specpsf','psf-b.fits') psf_r = os.path.join(os.environ['DESIMODEL'],'data','specpsf','psf-r.fits') psf_z = os.path.join(os.environ['DESIMODEL'],'data','specpsf','psf-z.fits') #- Create files needed to run quicklook sim_dir = os.path.join(os.environ['DESI_SPECTRO_SIM'],os.environ['PIXPROD'],night) data_dir = os.path.join(os.environ['QL_SPEC_DATA'],night) output_dir = os.environ['QL_SPEC_REDUX'] exp_dir = os.path.join(output_dir,'exposures',night) calib_dir = os.path.join(output_dir,'calib2d',night) psf_dir = os.path.join(output_dir,'calib2d','psf',night) cmd = "newarc --nspec {} --night {} --expid 0".format(nspec,night) if runcmd(cmd) != 0: raise RuntimeError('newexp failed for arc exposure') cmd = "newflat --nspec {} --night {} --expid 1".format(nspec,night) if runcmd(cmd) != 0: raise RuntimeError('newexp failed for flat exposure') cmd = "newexp-random --program dark --nspec {} --night {} --expid 2".format(nspec,night) if runcmd(cmd) != 0: raise RuntimeError('newexp failed for dark exposure') cmd = "pixsim --night {} --expid 0 --nspec {} --rawfile {}/desi-00000000.fits.fz --preproc --preproc_dir {}".format(night,nspec,data_dir,data_dir) if runcmd(cmd) != 0: raise RuntimeError('pixsim failed for arc exposure') cmd = "pixsim --night {} --expid 1 --nspec {} --rawfile {}/desi-00000001.fits.fz --preproc --preproc_dir {}".format(night,nspec,data_dir,data_dir) if runcmd(cmd) != 0: raise RuntimeError('pixsim failed for flat exposure') cmd = "pixsim --night {} --expid 2 --nspec {} --rawfile {}/desi-00000002.fits.fz".format(night,nspec,data_dir) if runcmd(cmd) != 0: raise RuntimeError('pixsim failed for dark exposure') # cmd = "desi_extract_spectra -i {}/pix-b0-00000001.fits -o {}/frame-b0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 3550,5730,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_b,nspec) # if runcmd(cmd) != 0: # raise RuntimeError('desi_extract_spectra failed for camera b0') cmd = "desi_extract_spectra -i {}/pix-r0-00000001.fits -o {}/frame-r0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 5630,7740,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_r,nspec) if runcmd(cmd) != 0: raise RuntimeError('desi_extract_spectra failed for camera r0') cmd = "desi_extract_spectra -i {}/pix-z0-00000001.fits -o {}/frame-z0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 7650,9830,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_z,nspec) if runcmd(cmd) != 0: raise RuntimeError('desi_extract_spectra failed for camera z0') copyfile(os.path.join(sim_dir,'fibermap-00000002.fits'),os.path.join(data_dir,'fibermap-00000002.fits')) os.remove(os.path.join(data_dir,'simpix-00000002.fits')) for camera in ['r0','z0']: cmd = "desi_compute_fiberflat --infile {}/frame-{}-00000001.fits --outfile {}/fiberflat-{}-00000001.fits".format(exp_dir,camera,calib_dir,camera) if runcmd(cmd) != 0: raise RuntimeError('desi_compute_fiberflat failed for camera {}'.format(camera)) cmd = "desi_bootcalib --fiberflat {}/pix-{}-00000001.fits --arcfile {}/pix-{}-00000000.fits --outfile {}/psfboot-{}.fits".format(data_dir,camera,data_dir,camera,psf_dir,camera) if runcmd(cmd) != 0: raise RuntimeError('desi_bootcalib failed for camera {}'.format(camera)) return
def test_function(self): def blat(*args): return list(args) self.assertEqual(util.runcmd(blat, args=[1, 2, 3]), [1, 2, 3]) self.assertEqual(util.runcmd(blat), [])
def test_missing_inputs(self): cmd = 'echo hello > /dev/null' self.assertNotEqual(0, util.runcmd(cmd, inputs=[uuid4().hex]))
def test_mergeQA(self): os.environ['QL_SPEC_REDUX'] = self.testDir cmd = "{} {}/desi_quicklook -i {} -n {} -c {} -e {} --rawdata_dir {} --specprod_dir {} --mergeQA".format(sys.executable,self.binDir,self.configfile,self.night,self.camera,self.expid,self.testDir,self.testDir) if runcmd(cmd) != 0: raise RuntimeError('quicklook pipeline failed')
def integration_test(night="20160726", nspec=25, clobber=False): """Run an integration test from raw data simulations through redshifts Args: night (str, optional): YEARMMDD, defaults to current night nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() log.setLevel(logging.DEBUG) flat_expid = 0 expid = 2 # check for required environment variables and simulate inputs check_env() sim(night, nspec=nspec, clobber=clobber) for camera in ['b0', 'r0', 'z0']: # find all necessary input and output files framefile = desispec.io.findfile('frame', night, expid, camera) fiberflatfile = desispec.io.findfile('fiberflat', night, flat_expid, camera) skyfile = desispec.io.findfile('sky', night, expid, camera) skytestfile = desispec.io.findfile('sky', night, expid, camera) + 'test' calibfile = desispec.io.findfile('calib', night, expid, camera) calibtestfile = desispec.io.findfile('calib', night, expid, camera) + 'test' stdstarsfile = desispec.io.findfile('stdstars', night, expid, spectrograph=0) cframetestfile = desispec.io.findfile('cframe', night, expid, camera) + 'test' # verify that quickgen output works for full pipeline com = "desi_compute_sky --infile {} --fiberflat {} --outfile {}".format( framefile, fiberflatfile, skytestfile) if runcmd(com, clobber=clobber) != 0: raise RuntimeError( 'desi_compute_sky failed for camera {}'.format(camera)) # only fit stdstars once if camera == 'b0': com = "desi_fit_stdstars --frames {} --skymodels {} --fiberflats {} --starmodels $DESI_BASIS_TEMPLATES/star_templates_v2.1.fits --outfile {}".format( framefile, skyfile, fiberflatfile, stdstarsfile) if runcmd(com, clobber=clobber) != 0: raise RuntimeError( 'desi_fit_stdstars failed for camera {}'.format(camera)) com = "desi_compute_fluxcalibration --infile {} --fiberflat {} --sky {} --models {} --outfile {}".format( framefile, fiberflatfile, skyfile, stdstarsfile, calibtestfile) if runcmd(com, clobber=clobber) != 0: raise RuntimeError( 'desi_compute_fluxcalibration failed for camera {}'.format( camera)) com = "desi_process_exposure --infile {} --fiberflat {} --sky {} --calib {} --outfile {}".format( framefile, fiberflatfile, skyfile, calibfile, cframetestfile) if runcmd(com, clobber=clobber) != 0: raise RuntimeError( 'desi_process_exposure failed for camera {}'.format(camera)) com = "desi_make_bricks --night {}".format(night) if runcmd(com, clobber=clobber) != 0: raise RuntimeError('desi_make_bricks failed')
def integration_test(args=None): """ Run an integration test from raw data simulations through QuickLook pipeline Args: night (str, optional): YEARMMDD nspec (int, optional): number of spectra to simulate overwrite (bool, otional) : overwrite existing files delete (bool, optional) : delete all inputs and outputs Raises: RuntimeError if QuickLook fails """ #- Parse arguments and check environment variables args = parse(args) night = args.night nspec = args.nspec raw_dir = os.environ['QL_SPEC_DATA'] output_dir = os.environ['QL_SPEC_REDUX'] #- Simulate inputs sim(night=night,nspec=nspec) #- Get the configuration file from desispec/data/quicklook from pkg_resources import resource_filename configfile=resource_filename('desispec','data/quicklook/qlconfig_dark.yaml') for camera in ['r0','z0']: #- Verify that quicklook pipeline runs com = "desi_quicklook -i {} -n {} -c {} -e 2".format(configfile,night,camera) if runcmd(com) != 0: raise RuntimeError('quicklook pipeline failed for camera {}'.format(camera)) #- Remove all output if desired if args.delete: sim_dir = os.path.join(os.environ['DESI_SPECTRO_SIM'],os.environ['PIXPROD'],args.night) if os.path.exists(sim_dir): sim_files = os.listdir(sim_dir) for file in range(len(sim_files)): sim_file = os.path.join(sim_dir,sim_files[file]) os.remove(sim_file) os.rmdir(sim_dir) data_dir = os.path.join(raw_dir,night) if os.path.exists(data_dir): data_files = os.listdir(data_dir) for file in range(len(data_files)): data_file = os.path.join(data_dir,data_files[file]) os.remove(data_file) os.rmdir(data_dir) if os.path.exists(output_dir): exp_dir = os.path.join(output_dir,'exposures',night) if os.path.exists(exp_dir): id_dir = os.path.join(exp_dir,'00000002') if os.path.exists(id_dir): id_files = os.listdir(id_dir) for file in range(len(id_files)): id_file = os.path.join(id_dir,id_files[file]) os.remove(id_file) os.rmdir(id_dir) exp_files = os.listdir(exp_dir) for file in range(len(exp_files)): exp_file = os.path.join(exp_dir,exp_files[file]) os.remove(exp_file) os.rmdir(exp_dir) calib_dir=os.path.join(output_dir,'calib2d',night) if os.path.exists(calib_dir): calib_files = os.listdir(calib_dir) for file in range(len(calib_files)): calib_file = os.path.join(calib_dir,calib_files[file]) os.remove(calib_file) os.rmdir(calib_dir) psf_dir=os.path.join(output_dir,'calib2d','psf',night) if os.path.exists(psf_dir): psf_files = os.listdir(psf_dir) for ii in range(len(psf_files)): thisfile=os.path.join(psf_dir,psf_files[ii]) os.remove(thisfile) os.rmdir(psf_dir)
def sim(night, nspec, ql_data, ql_redux): """ Simulate data as part of the QuickLook integration test. Args: night (str): YEARMMDD nspec (int): number of spectra to simulate ql_data (str) : data read from $QL_SPEC_DATA if specified ql_redux (str) : output sent to $QL_SPEC_REDUX if specified Raises: RuntimeError if any script fails """ # psf_b = os.path.join(os.environ['DESIMODEL'],'data','specpsf','psf-b.fits') psf_r = os.path.join(os.environ['DESIMODEL'],'data','specpsf','psf-r.fits') psf_z = os.path.join(os.environ['DESIMODEL'],'data','specpsf','psf-z.fits') #- Create files needed to run quicklook sim_dir = os.path.join(os.environ['DESI_SPECTRO_SIM'],os.environ['PIXPROD'],night) if ql_data == 'Use $QL_SPEC_DATA': data_dir = os.path.join(os.environ['QL_SPEC_DATA'],night) else: data_dir = os.path.join(os.environ['DESI_SPECTRO_DATA'],'QL',night) if ql_redux == 'Use $QL_SPEC_REDUX': output_dir = os.environ['QL_SPEC_REDUX'] else: output_dir = os.path.join(os.environ['DESI_SPECTRO_REDUX'],'QL',os.environ['SPECPROD']) exp_dir = os.path.join(output_dir,'exposures',night) calib_dir = os.path.join(output_dir,'calib2d',night) psf_dir = os.path.join(output_dir,'calib2d','psf',night) for expid, flavor in zip([0,1,2], ['arc', 'flat', 'dark']): cmd = "newexp-desi --flavor {} --nspec {} --night {} --expid {}".format(flavor,nspec,night,expid) if runcmd(cmd) != 0: raise RuntimeError('newexp failed for {} exposure {}'.format(flavor, expid)) if flavor == 'dark': cmd = "pixsim-desi --night {} --expid {} --nspec {} --rawfile {}/desi-{:08d}.fits.fz".format(night,expid,nspec,data_dir,expid) if runcmd(cmd) != 0: raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid)) if flavor == 'arc' or flavor == 'flat': cmd = "pixsim-desi --night {} --expid {} --nspec {} --rawfile {}/desi-{:08d}.fits.fz --preproc --preproc_dir {}".format(night,expid,nspec,data_dir,expid,data_dir) if runcmd(cmd) != 0: raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid)) if flavor == 'flat': # cmd = "desi_extract_spectra -i {}/pix-b0-00000001.fits -o {}/frame-b0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 3550,5730,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_b,nspec) # if runcmd(cmd) != 0: # raise RuntimeError('desi_extract_spectra failed for camera b0') cmd = "desi_extract_spectra -i {}/pix-r0-00000001.fits -o {}/frame-r0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 5630,7740,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_r,nspec) if runcmd(cmd) != 0: raise RuntimeError('desi_extract_spectra failed for camera r0') cmd = "desi_extract_spectra -i {}/pix-z0-00000001.fits -o {}/frame-z0-00000001.fits -f {}/fibermap-00000001.fits -p {} -w 7650,9830,0.8 -n {}".format(data_dir,exp_dir,sim_dir,psf_z,nspec) if runcmd(cmd) != 0: raise RuntimeError('desi_extract_spectra failed for camera z0') copyfile(os.path.join(sim_dir,'fibermap-{:08d}.fits'.format(expid)),os.path.join(data_dir,'fibermap-{:08d}.fits'.format(expid))) os.remove(os.path.join(data_dir,'simpix-{:08d}.fits'.format(expid))) for camera in ['r0','z0']: cmd = "desi_compute_fiberflat --infile {}/frame-{}-00000001.fits --outfile {}/fiberflat-{}-00000001.fits".format(exp_dir,camera,calib_dir,camera) if runcmd(cmd) != 0: raise RuntimeError('desi_compute_fiberflat failed for camera {}'.format(camera)) cmd = "desi_bootcalib --fiberflat {}/pix-{}-00000001.fits --arcfile {}/pix-{}-00000000.fits --outfile {}/psfboot-{}.fits".format(data_dir,camera,data_dir,camera,psf_dir,camera) if runcmd(cmd) != 0: raise RuntimeError('desi_bootcalib failed for camera {}'.format(camera)) return