def test_run(self):
opts = {}
opts["spectrographs"] = "0"
opts["data"] = self.raw
opts["redux"] = self.redux
opts["prod"] = self.prod
opts["shifter"] = self.shifter
sopts = option_list(opts)
sargs = pipe_prod.parse(sopts)
pipe_prod.main(sargs)
# modify the options to use our No-op worker
rundir = io.get_pipe_rundir()
optfile = os.path.join(rundir, "options.yaml")
opts = {}
for step in step_types:
opts["{}_worker".format(step)] = "Noop"
opts["{}_worker_opts".format(step)] = {}
opts[step] = {}
yaml_write(optfile, opts)
envfile = os.path.join(rundir, "env.sh")
with open(envfile, "w") as f:
f.write("export DESIMODEL={}\n".format(rundir))
if "PATH" in os.environ:
f.write("export PATH={}\n".format(os.environ["PATH"]))
if "PYTHONPATH" in os.environ:
f.write("export PYTHONPATH={}\n".format(os.environ["PYTHONPATH"]))
if "LD_LIBRARY_PATH" in os.environ:
f.write("export LD_LIBRARY_PATH={}\n".format(os.environ["LD_LIBRARY_PATH"]))
com = ". {}; eval {}".format(envfile, os.path.join(rundir, "scripts", "run_shell_all.sh"))
print(com)
sp.call(com, shell=True, env=os.environ.copy())
def main(args, comm=None):
rank = 0
nproc = 1
if comm is not None:
rank = comm.rank
nproc = comm.size
# Determine which nights we are using
nights = None
if args.nights is not None:
nights = args.nights.split(",")
else:
if rank == 0:
rawdir = os.path.abspath(specio.rawdata_root())
nights = []
nightpat = re.compile(r"\d{8}")
for root, dirs, files in os.walk(rawdir, topdown=True):
for d in dirs:
nightmat = nightpat.match(d)
if nightmat is not None:
nights.append(d)
break
if comm is not None:
nights = comm.bcast(nights, root=0)
# Get the list of exposures for each night
night_expid = {}
all_expid = []
exp_to_night = {}
if rank == 0:
for nt in nights:
night_expid[nt] = specio.get_exposures(nt, raw=True)
all_expid.extend(night_expid[nt])
for ex in night_expid[nt]:
exp_to_night[ex] = nt
if comm is not None:
night_expid = comm.bcast(night_expid, root=0)
all_expid = comm.bcast(all_expid, root=0)
exp_to_night = comm.bcast(exp_to_night, root=0)
expids = np.array(all_expid, dtype=np.int32)
nexp = len(expids)
# Get the list of cameras
cams = None
if args.cameras is not None:
cams = args.cameras.split(",")
else:
cams = []
for band in ['b', 'r', 'z']:
for spec in range(10):
cams.append('{}{}'.format(band, spec))
# number of cameras
ncamera = len(cams)
# check that our communicator is an appropriate size
if comm is not None:
if ncamera * args.camera_procs > comm.size:
if comm.rank == 0:
print("Communicator size ({}) too small for {} cameras each with {} procs".format(comm.size, ncamera, args.camera_procs), flush=True)
comm.Abort()
# create a set of reproducible seeds for each exposure
np.random.seed(args.seed)
maxexp = np.max(expids)
allseeds = np.random.randint(2**32, size=(maxexp+1))
seeds = allseeds[-nexp:]
taskproc = ncamera * args.camera_procs
comm_group = comm
comm_rank = None
group = comm.rank
ngroup = comm.size
group_rank = 0
if comm is not None:
from mpi4py import MPI
if taskproc > 1:
ngroup = int(comm.size / taskproc)
group = int(comm.rank / taskproc)
group_rank = comm.rank % taskproc
comm_group = comm.Split(color=group, key=group_rank)
comm_rank = comm.Split(color=group_rank, key=group)
else:
comm_group = MPI.COMM_SELF
comm_rank = comm
myexpids = np.array_split(expids, ngroup)[group]
for ex in myexpids:
nt = exp_to_night[ex]
# path to raw file
simspecfile = simio.findfile('simspec', nt, ex)
rawfile = specio.findfile('raw', nt, ex)
rawfile = os.path.join(os.path.dirname(simspecfile), rawfile)
# Is this exposure already finished?
done = True
if group_rank == 0:
if not os.path.isfile(rawfile):
done = False
if args.preproc:
for c in cams:
pixfile = specio.findfile('pix', night=nt,
expid=ex, camera=c)
if not os.path.isfile(pixfile):
done = False
if comm_group is not None:
done = comm_group.bcast(done, root=0)
if done and not args.overwrite:
if group_rank == 0:
print("Skipping completed exposure {:08d} on night {}".format(ex, nt))
continue
# Write per-process logs to a separate directory,
# since there are so many of them.
logdir = "{}_logs".format(rawfile)
if group_rank == 0:
if not os.path.isdir(logdir):
os.makedirs(logdir)
if comm_group is not None:
comm_group.barrier()
tasklog = os.path.join(logdir, "pixsim")
with stdouterr_redirected(to=tasklog, comm=comm_group):
try:
options = {}
options["night"] = nt
options["expid"] = int(ex)
options["cosmics"] = args.cosmics
options["seed"] = seeds[ex]
options["cameras"] = ",".join(cams)
options["mpi_camera"] = args.camera_procs
options["verbose"] = args.verbose
options["preproc"] = args.preproc
optarray = option_list(options)
pixargs = pixsim.parse(optarray)
pixsim.main(pixargs, comm_group)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
print("".join(lines), flush=True)
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
t0 = time.time()
log = get_logger()
# guess if it is a preprocessed or a raw image
hdulist = fits.open(args.image)
is_input_preprocessed = ("IMAGE" in hdulist) & ("IVAR" in hdulist)
primary_header = hdulist[0].header
hdulist.close()
if is_input_preprocessed:
image = read_image(args.image)
else:
if args.camera is None:
print(
"ERROR: Need to specify camera to open a raw fits image (with all cameras in different fits HDUs)"
)
print(
"Try adding the option '--camera xx', with xx in {brz}{0-9}, like r7, or type 'desi_qproc --help' for more options"
)
sys.exit(12)
image = read_raw(args.image, args.camera, fill_header=[
1,
])
if args.auto:
log.debug("AUTOMATIC MODE")
try:
night = image.meta['NIGHT']
if not 'EXPID' in image.meta:
if 'EXPNUM' in image.meta:
log.warning('using EXPNUM {} for EXPID'.format(
image.meta['EXPNUM']))
image.meta['EXPID'] = image.meta['EXPNUM']
expid = image.meta['EXPID']
except KeyError as e:
log.error(
"Need at least NIGHT and EXPID (or EXPNUM) to run in auto mode. Retry without the --auto option."
)
log.error(str(e))
sys.exit(12)
indir = os.path.dirname(args.image)
if args.fibermap is None:
filename = '{}/fibermap-{:08d}.fits'.format(indir, expid)
if os.path.isfile(filename):
log.debug("auto-mode: found a fibermap, {}, using it!".format(
filename))
args.fibermap = filename
if args.output_preproc is None:
if not is_input_preprocessed:
args.output_preproc = '{}/preproc-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera.lower(), expid)
log.debug("auto-mode: will write preproc in " +
args.output_preproc)
else:
log.debug(
"auto-mode: will not write preproc because input is a preprocessed image"
)
if args.auto_output_dir != '.':
if not os.path.isdir(args.auto_output_dir):
log.debug("auto-mode: creating directory " +
args.auto_output_dir)
os.makedirs(args.auto_output_dir)
if args.output_preproc is not None:
write_image(args.output_preproc, image)
cfinder = None
if args.psf is None:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
args.psf = cfinder.findfile("PSF")
log.info(" Using PSF {}".format(args.psf))
tset = read_xytraceset(args.psf)
# add fibermap
if args.fibermap:
if os.path.isfile(args.fibermap):
fibermap = read_fibermap(args.fibermap)
else:
log.error("no fibermap file {}".format(args.fibermap))
fibermap = None
else:
fibermap = None
if "OBSTYPE" in image.meta:
obstype = image.meta["OBSTYPE"].upper()
image.meta["OBSTYPE"] = obstype # make sure it's upper case
qframe = None
else:
log.warning("No OBSTYPE keyword, trying to guess ...")
qframe = qproc_boxcar_extraction(tset,
image,
width=args.width,
fibermap=fibermap)
obstype = check_qframe_flavor(
qframe, input_flavor=image.meta["FLAVOR"]).upper()
image.meta["OBSTYPE"] = obstype
log.info("OBSTYPE = '{}'".format(obstype))
if args.auto:
# now set the things to do
if obstype == "SKY" or obstype == "TWILIGHT" or obstype == "SCIENCE":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.apply_fiberflat = True
args.skysub = True
args.output_skyframe = '{}/qsky-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.fluxcalib = True
args.outframe = '{}/qcframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
elif obstype == "ARC" or obstype == "TESTARC":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.compute_lsf_sigma = True
elif obstype == "FLAT" or obstype == "TESTFLAT":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.compute_fiberflat = '{}/qfiberflat-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
if args.shift_psf:
# using the trace shift script
if args.auto:
options = option_list({
"psf":
args.psf,
"image":
"dummy",
"outpsf":
"dummy",
"continuum": ((obstype == "FLAT") | (obstype == "TESTFLAT")),
"sky": ((obstype == "SCIENCE") | (obstype == "SKY"))
})
else:
options = option_list({
"psf": args.psf,
"image": "dummy",
"outpsf": "dummy"
})
tmp_args = trace_shifts_script.parse(options=options)
tset = trace_shifts_script.fit_trace_shifts(image=image, args=tmp_args)
qframe = qproc_boxcar_extraction(tset,
image,
width=args.width,
fibermap=fibermap)
if tset.meta is not None:
# add traceshift info in the qframe, this will be saved in the qframe header
if qframe.meta is None:
qframe.meta = dict()
for k in tset.meta.keys():
qframe.meta[k] = tset.meta[k]
if args.output_rawframe is not None:
write_qframe(args.output_rawframe, qframe)
log.info("wrote raw extracted frame in {}".format(
args.output_rawframe))
if args.compute_lsf_sigma:
tset = process_arc(qframe, tset, linelist=None, npoly=2, nbins=2)
if args.output_psf is not None:
for k in qframe.meta:
if k not in tset.meta:
tset.meta[k] = qframe.meta[k]
write_xytraceset(args.output_psf, tset)
if args.compute_fiberflat is not None:
fiberflat = qproc_compute_fiberflat(qframe)
#write_qframe(args.compute_fiberflat,qflat)
write_fiberflat(args.compute_fiberflat, fiberflat, header=qframe.meta)
log.info("wrote fiberflat in {}".format(args.compute_fiberflat))
if args.apply_fiberflat or args.input_fiberflat:
if args.input_fiberflat is None:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
try:
args.input_fiberflat = cfinder.findfile("FIBERFLAT")
except KeyError as e:
log.error("no FIBERFLAT for this spectro config")
sys.exit(12)
log.info("applying fiber flat {}".format(args.input_fiberflat))
flat = read_fiberflat(args.input_fiberflat)
qproc_apply_fiberflat(qframe, flat)
if args.skysub:
log.info("sky subtraction")
if args.output_skyframe is not None:
skyflux = qproc_sky_subtraction(qframe, return_skymodel=True)
sqframe = QFrame(qframe.wave, skyflux, np.ones(skyflux.shape))
write_qframe(args.output_skyframe, sqframe)
log.info("wrote sky model in {}".format(args.output_skyframe))
else:
qproc_sky_subtraction(qframe)
if args.fluxcalib:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
# check for flux calib
if cfinder.haskey("FLUXCALIB"):
fluxcalib_filename = cfinder.findfile("FLUXCALIB")
fluxcalib = read_average_flux_calibration(fluxcalib_filename)
log.info("read average calib in {}".format(fluxcalib_filename))
seeing = qframe.meta["SEEING"]
airmass = qframe.meta["AIRMASS"]
exptime = qframe.meta["EXPTIME"]
exposure_calib = fluxcalib.value(seeing=seeing, airmass=airmass)
for q in range(qframe.nspec):
fiber_calib = np.interp(qframe.wave[q], fluxcalib.wave,
exposure_calib) * exptime
inv_calib = (fiber_calib > 0) / (fiber_calib +
(fiber_calib == 0))
qframe.flux[q] *= inv_calib
qframe.ivar[q] *= fiber_calib**2 * (fiber_calib > 0)
# add keyword in header giving the calibration factor applied at a reference wavelength
band = qframe.meta["CAMERA"].upper()[0]
if band == "B":
refwave = 4500
elif band == "R":
refwave = 6500
else:
refwave = 8500
calvalue = np.interp(refwave, fluxcalib.wave,
exposure_calib) * exptime
qframe.meta["CALWAVE"] = refwave
qframe.meta["CALVALUE"] = calvalue
else:
log.error(
"Cannot calibrate fluxes because no FLUXCALIB keywork in calibration files"
)
fibers = parse_fibers(args.fibers)
if fibers is None:
fibers = qframe.flux.shape[0]
else:
ii = np.arange(qframe.fibers.size)[np.in1d(qframe.fibers, fibers)]
if ii.size == 0:
log.error("no such fibers in frame,")
log.error("fibers are in range [{}:{}]".format(
qframe.fibers[0], qframe.fibers[-1] + 1))
sys.exit(12)
qframe = qframe[ii]
if args.outframe is not None:
write_qframe(args.outframe, qframe)
log.info("wrote {}".format(args.outframe))
t1 = time.time()
log.info("all done in {:3.1f} sec".format(t1 - t0))
if args.plot:
log.info("plotting {} spectra".format(qframe.wave.shape[0]))
import matplotlib.pyplot as plt
fig = plt.figure()
for i in range(qframe.wave.shape[0]):
j = (qframe.ivar[i] > 0)
plt.plot(qframe.wave[i, j], qframe.flux[i, j])
plt.grid()
plt.xlabel("wavelength")
plt.ylabel("flux")
plt.show()
comm_rank = comm.Split(color=group_rank, key=group)
else:
comm_group = MPI.COMM_SELF
comm_rank = comm
log_root = "newexp_"
task = 0
for nt in nights:
for fl in flavors:
tasklog = "{}{}-{:08d}.log".format(log_root, nt, expids[task])
if task == group:
with stdouterr_redirected(to=tasklog, comm=comm_group):
try:
options = {}
options["program"] = fl
options["night"] = nt
options["expid"] = expids[task]
options["tileid"] = tileids[task]
options["seed"] = seeds[task]
#options["nproc"] = 1
optarray = option_list(options)
args = newexp.parse(optarray)
newexp.main(args)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
print("".join(lines), flush=True)
task += 1