def test_main(self):
if self.data_unavailable:
self.skipTest("Failed to download test data.")
argstr = [
'--fiberflat',
self.testflat,
'--arcfile',
self.testarc,
'--outfile',
self.testout,
'--qafile',
self.qafile,
]
args = bootscript.parse(options=argstr)
bootscript.main(args)
#- Ensure the PSF class can read that file
from desispec.quicklook.qlpsf import PSF
psf = PSF(self.testout)
#- While we're at it, test some PSF accessor functions
indices = np.array([0, 1])
waves = np.array([psf.wmin, psf.wmin + 1])
w = psf.wavelength()
w = psf.wavelength(ispec=0)
w = psf.wavelength(ispec=indices)
w = psf.wavelength(ispec=indices, y=0)
w = psf.wavelength(ispec=indices, y=indices)
x = psf.x()
x = psf.x(ispec=0)
x = psf.x(ispec=indices)
x = psf.x(ispec=None, wavelength=psf.wmin)
x = psf.x(ispec=1, wavelength=psf.wmin)
x = psf.x(ispec=indices, wavelength=psf.wmin)
x = psf.x(ispec=indices, wavelength=waves)
y = psf.y(ispec=None, wavelength=psf.wmin)
y = psf.y(ispec=0, wavelength=psf.wmin)
y = psf.y(ispec=indices, wavelength=psf.wmin)
y = psf.y(ispec=indices, wavelength=waves)
def test_main(self):
if self.data_unavailable:
self.skipTest("Failed to download test data.")
argstr = [
'--fiberflat', self.testflat,
'--arcfile', self.testarc,
'--outfile', self.testout,
'--qafile', self.qafile,
]
args = bootscript.parse(options=argstr)
bootscript.main(args)
#- Ensure the PSF class can read that file
from desispec.psf import PSF
psf = PSF(self.testout)
#- While we're at it, test some PSF accessor functions
w = psf.wavelength()
w = psf.wavelength(ispec=0)
w = psf.wavelength(ispec=[0,1])
w = psf.wavelength(ispec=[0,1], y=0)
w = psf.wavelength(ispec=[0,1], y=[0,1])
x = psf.x()
x = psf.x(ispec=0)
x = psf.x(ispec=[0,1])
x = psf.x(ispec=None, wavelength=psf.wmin)
x = psf.x(ispec=1, wavelength=psf.wmin)
x = psf.x(ispec=[0,1], wavelength=psf.wmin)
x = psf.x(ispec=[0,1], wavelength=[psf.wmin, psf.wmin+1])
y = psf.y(ispec=None, wavelength=psf.wmin)
y = psf.y(ispec=0, wavelength=psf.wmin)
y = psf.y(ispec=[0,1], wavelength=psf.wmin)
y = psf.y(ispec=[0,1], wavelength=[psf.wmin, psf.wmin+1])
t = psf.invert()
def run_task(step, rawdir, proddir, grph, opts, comm=None):
'''
Run a single pipeline task.
This function takes a truncated graph containing a single node
of the specified type and the nodes representing the inputs for
the task.
Args:
step (str): the pipeline step type.
rawdir (str): the path to the raw data directory.
proddir (str): the path to the production directory.
grph (dict): the truncated dependency graph.
opts (dict): the global options dictionary.
comm (mpi4py.Comm): the optional MPI communicator to use.
Returns:
Nothing.
'''
if step not in step_file_types.keys():
raise ValueError("step type {} not recognized".format(step))
log = get_logger()
# Verify that there is only a single node in the graph
# of the desired step. The graph should already have
# been sliced before calling this task.
nds = []
for name, nd in grph.items():
if nd['type'] in step_file_types[step]:
nds.append(name)
if len(nds) != 1:
raise RuntimeError("run_task should only be called with a graph containing a single node to process")
name = nds[0]
node = grph[name]
nproc = 1
rank = 0
if comm is not None:
nproc = comm.size
rank = comm.rank
# step-specific operations
if step == 'bootcalib':
# The inputs to this step include *all* the arcs and flats for the
# night. Here we sort them into the list of arcs and the list of
# flats, and simply choose the first one of each.
arcs = []
flats = []
for input in node['in']:
inode = grph[input]
if inode['flavor'] == 'arc':
arcs.append(input)
elif inode['flavor'] == 'flat':
flats.append(input)
if len(arcs) == 0:
raise RuntimeError("no arc images found!")
if len(flats) == 0:
raise RuntimeError("no flat images found!")
firstarc = sorted(arcs)[0]
firstflat = sorted(flats)[0]
# build list of options
arcpath = graph_path_pix(rawdir, firstarc)
flatpath = graph_path_pix(rawdir, firstflat)
outpath = graph_path_psfboot(proddir, name)
qafile, qafig = qa_path(outpath)
options = {}
options['fiberflat'] = flatpath
options['arcfile'] = arcpath
options['qafile'] = qafile
### options['qafig'] = qafig
options['outfile'] = outpath
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_bootcalib']
com.extend(optarray)
log.debug(" ".join(com))
args = bootcalib.parse(optarray)
sys.stdout.flush()
if rank == 0:
#print("proc {} call bootcalib main".format(rank))
#sys.stdout.flush()
bootcalib.main(args)
#print("proc {} returned from bootcalib main".format(rank))
#sys.stdout.flush()
#print("proc {} finish runtask bootcalib".format(rank))
#sys.stdout.flush()
elif step == 'specex':
# get input files
pix = []
boot = []
for input in node['in']:
inode = grph[input]
if inode['type'] == 'psfboot':
boot.append(input)
elif inode['type'] == 'pix':
pix.append(input)
if len(boot) != 1:
raise RuntimeError("specex needs exactly one psfboot file")
if len(pix) == 0:
raise RuntimeError("specex needs exactly one image file")
bootfile = graph_path_psfboot(proddir, boot[0])
imgfile = graph_path_pix(rawdir, pix[0])
outfile = graph_path_psf(proddir, name)
outdir = os.path.dirname(outfile)
options = {}
options['input'] = imgfile
options['bootfile'] = bootfile
options['output'] = outfile
if log.getEffectiveLevel() == desispec.log.DEBUG:
options['verbose'] = True
if len(opts) > 0:
extarray = option_list(opts)
options['extra'] = " ".join(extarray)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_compute_psf']
com.extend(optarray)
log.debug(" ".join(com))
args = specex.parse(optarray)
specex.main(args, comm=comm)
elif step == 'psfcombine':
outfile = graph_path_psfnight(proddir, name)
infiles = []
for input in node['in']:
infiles.append(graph_path_psf(proddir, input))
if rank == 0:
specex.mean_psf(infiles, outfile)
elif step == 'extract':
pix = []
psf = []
fm = []
band = None
for input in node['in']:
inode = grph[input]
if inode['type'] == 'psfnight':
psf.append(input)
elif inode['type'] == 'pix':
pix.append(input)
band = inode['band']
elif inode['type'] == 'fibermap':
fm.append(input)
if len(psf) != 1:
raise RuntimeError("extraction needs exactly one psfnight file")
if len(pix) != 1:
raise RuntimeError("extraction needs exactly one image file")
if len(fm) != 1:
raise RuntimeError("extraction needs exactly one fibermap file")
imgfile = graph_path_pix(rawdir, pix[0])
psffile = graph_path_psfnight(proddir, psf[0])
fmfile = graph_path_fibermap(rawdir, fm[0])
outfile = graph_path_frame(proddir, name)
options = {}
options['input'] = imgfile
options['fibermap'] = fmfile
options['psf'] = psffile
options['output'] = outfile
# extract the wavelength range from the options, depending on the band
optscopy = copy.deepcopy(opts)
wkey = "wavelength_{}".format(band)
wave = optscopy[wkey]
del optscopy['wavelength_b']
del optscopy['wavelength_r']
del optscopy['wavelength_z']
optscopy['wavelength'] = wave
options.update(optscopy)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_extract_spectra']
com.extend(optarray)
log.debug(" ".join(com))
args = extract.parse(optarray)
extract.main_mpi(args, comm=comm)
elif step == 'fiberflat':
if len(node['in']) != 1:
raise RuntimeError('fiberflat should have only one input frame')
framefile = graph_path_frame(proddir, node['in'][0])
outfile = graph_path_fiberflat(proddir, name)
qafile, qafig = qa_path(outfile)
options = {}
options['infile'] = framefile
options['qafile'] = qafile
options['qafig'] = qafig
options['outfile'] = outfile
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_compute_fiberflat']
com.extend(optarray)
log.debug(" ".join(com))
args = fiberflat.parse(optarray)
if rank == 0:
fiberflat.main(args)
elif step == 'sky':
frm = []
flat = []
for input in node['in']:
inode = grph[input]
if inode['type'] == 'frame':
frm.append(input)
elif inode['type'] == 'fiberflat':
flat.append(input)
if len(frm) != 1:
raise RuntimeError("sky needs exactly one frame file")
if len(flat) != 1:
raise RuntimeError("sky needs exactly one fiberflat file")
framefile = graph_path_frame(proddir, frm[0])
flatfile = graph_path_fiberflat(proddir, flat[0])
outfile = graph_path_sky(proddir, name)
qafile, qafig = qa_path(outfile)
options = {}
options['infile'] = framefile
options['fiberflat'] = flatfile
options['qafile'] = qafile
options['qafig'] = qafig
options['outfile'] = outfile
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_compute_sky']
com.extend(optarray)
log.debug(" ".join(com))
args = skypkg.parse(optarray)
if rank == 0:
skypkg.main(args)
elif step == 'stdstars':
frm = []
flat = []
sky = []
flatexp = None
specgrph = None
for input in node['in']:
inode = grph[input]
if inode['type'] == 'frame':
frm.append(input)
specgrph = inode['spec']
elif inode['type'] == 'fiberflat':
flat.append(input)
flatexp = inode['id']
elif inode['type'] == 'sky':
sky.append(input)
outfile = graph_path_stdstars(proddir, name)
qafile, qafig = qa_path(outfile)
framefiles = [graph_path_frame(proddir, x) for x in frm]
skyfiles = [graph_path_sky(proddir, x) for x in sky]
flatfiles = [graph_path_fiberflat(proddir, x) for x in flat]
options = {}
options['frames'] = framefiles
options['skymodels'] = skyfiles
options['fiberflats'] = flatfiles
options['outfile'] = outfile
options['ncpu'] = str(default_nproc)
#- TODO: no QA for fitting standard stars yet
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_fit_stdstars']
com.extend(optarray)
log.debug(" ".join(com))
args = stdstars.parse(optarray)
if rank == 0:
stdstars.main(args)
elif step == 'fluxcal':
frm = []
flat = []
sky = []
star = []
for input in node['in']:
inode = grph[input]
if inode['type'] == 'frame':
frm.append(input)
elif inode['type'] == 'fiberflat':
flat.append(input)
elif inode['type'] == 'sky':
sky.append(input)
elif inode['type'] == 'stdstars':
star.append(input)
if len(frm) != 1:
raise RuntimeError("fluxcal needs exactly one frame file")
if len(flat) != 1:
raise RuntimeError("fluxcal needs exactly one fiberflat file")
if len(sky) != 1:
raise RuntimeError("fluxcal needs exactly one sky file")
if len(star) != 1:
raise RuntimeError("fluxcal needs exactly one star file")
framefile = graph_path_frame(proddir, frm[0])
flatfile = graph_path_fiberflat(proddir, flat[0])
skyfile = graph_path_sky(proddir, sky[0])
starfile = graph_path_stdstars(proddir, star[0])
outfile = graph_path_calib(proddir, name)
qafile, qafig = qa_path(outfile)
options = {}
options['infile'] = framefile
options['fiberflat'] = flatfile
options['qafile'] = qafile
options['qafig'] = qafig
options['sky'] = skyfile
options['models'] = starfile
options['outfile'] = outfile
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_compute_fluxcalibration']
com.extend(optarray)
log.debug(" ".join(com))
args = fluxcal.parse(optarray)
if rank == 0:
fluxcal.main(args)
elif step == 'procexp':
frm = []
flat = []
sky = []
cal = []
for input in node['in']:
inode = grph[input]
if inode['type'] == 'frame':
frm.append(input)
elif inode['type'] == 'fiberflat':
flat.append(input)
elif inode['type'] == 'sky':
sky.append(input)
elif inode['type'] == 'calib':
cal.append(input)
if len(frm) != 1:
raise RuntimeError("procexp needs exactly one frame file")
if len(flat) != 1:
raise RuntimeError("procexp needs exactly one fiberflat file")
if len(sky) != 1:
raise RuntimeError("procexp needs exactly one sky file")
if len(cal) != 1:
raise RuntimeError("procexp needs exactly one calib file")
framefile = graph_path_frame(proddir, frm[0])
flatfile = graph_path_fiberflat(proddir, flat[0])
skyfile = graph_path_sky(proddir, sky[0])
calfile = graph_path_calib(proddir, cal[0])
outfile = graph_path_cframe(proddir, name)
options = {}
options['infile'] = framefile
options['fiberflat'] = flatfile
options['sky'] = skyfile
options['calib'] = calfile
options['outfile'] = outfile
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_process_exposure']
com.extend(optarray)
log.debug(" ".join(com))
args = procexp.parse(optarray)
if rank == 0:
procexp.main(args)
elif step == 'zfind':
brick = node['brick']
outfile = graph_path_zbest(proddir, name)
qafile, qafig = qa_path(outfile)
options = {}
options['brick'] = brick
options['outfile'] = outfile
#- TODO: no QA for desi_zfind yet
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_zfind']
com.extend(optarray)
log.debug(" ".join(com))
args = zfind.parse(optarray)
zfind.main(args, comm=comm)
else:
raise RuntimeError("Unknown pipeline step {}".format(step))
#sys.stdout.flush()
if comm is not None:
#print("proc {} hit runtask barrier".format(rank))
#sys.stdout.flush()
comm.barrier()
#print("proc {} finish runtask".format(rank))
#sys.stdout.flush()
return
def run_task(step, rawdir, proddir, grph, opts, comm=None):
if step not in step_file_types.keys():
raise ValueError("step type {} not recognized".format(step))
log = get_logger()
# Verify that there is only a single node in the graph
# of the desired step. The graph should already have
# been sliced before calling this task.
nds = []
for name, nd in grph.items():
if nd['type'] in step_file_types[step]:
nds.append(name)
if len(nds) != 1:
raise RuntimeError("run_task should only be called with a graph containing a single node to process")
name = nds[0]
node = grph[name]
nproc = 1
rank = 0
if comm is not None:
nproc = comm.size
rank = comm.rank
# step-specific operations
if step == 'bootcalib':
# The inputs to this step include *all* the arcs and flats for the
# night. Here we sort them into the list of arcs and the list of
# flats, and simply choose the first one of each.
arcs = []
flats = []
for input in node['in']:
inode = grph[input]
if inode['flavor'] == 'arc':
arcs.append(input)
elif inode['flavor'] == 'flat':
flats.append(input)
if len(arcs) == 0:
raise RuntimeError("no arc images found!")
if len(flats) == 0:
raise RuntimeError("no flat images found!")
firstarc = sorted(arcs)[0]
firstflat = sorted(flats)[0]
# build list of options
arcpath = graph_path_pix(rawdir, firstarc)
flatpath = graph_path_pix(rawdir, firstflat)
outpath = graph_path_psfboot(proddir, name)
qafile, qafig = qa_path(outpath)
options = {}
options['fiberflat'] = flatpath
options['arcfile'] = arcpath
options['qafile'] = qafile
options['qafig'] = qafig
options['outfile'] = outpath
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_bootcalib']
com.extend(optarray)
log.debug(" ".join(com))
args = bootcalib.parse(optarray)
sys.stdout.flush()
if rank == 0:
#print("proc {} call bootcalib main".format(rank))
#sys.stdout.flush()
bootcalib.main(args)
#print("proc {} returned from bootcalib main".format(rank))
#sys.stdout.flush()
#print("proc {} finish runtask bootcalib".format(rank))
#sys.stdout.flush()
elif step == 'specex':
# get input files
pix = []
boot = []
for input in node['in']:
inode = grph[input]
if inode['type'] == 'psfboot':
boot.append(input)
elif inode['type'] == 'pix':
pix.append(input)
if len(boot) != 1:
raise RuntimeError("specex needs exactly one psfboot file")
if len(pix) == 0:
raise RuntimeError("specex needs exactly one image file")
bootfile = graph_path_psfboot(proddir, boot[0])
imgfile = graph_path_pix(rawdir, pix[0])
outfile = graph_path_psf(proddir, name)
outdir = os.path.dirname(outfile)
options = {}
options['input'] = imgfile
options['bootfile'] = bootfile
options['output'] = outfile
if log.getEffectiveLevel() == desispec.log.DEBUG:
options['verbose'] = True
if len(opts) > 0:
extarray = option_list(opts)
options['extra'] = " ".join(extarray)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_compute_psf']
com.extend(optarray)
log.debug(" ".join(com))
args = specex.parse(optarray)
specex.main(args, comm=comm)
elif step == 'psfcombine':
outfile = graph_path_psfnight(proddir, name)
infiles = []
for input in node['in']:
infiles.append(graph_path_psf(proddir, input))
if rank == 0:
specex.mean_psf(infiles, outfile)
elif step == 'extract':
pix = []
psf = []
fm = []
band = None
for input in node['in']:
inode = grph[input]
if inode['type'] == 'psfnight':
psf.append(input)
elif inode['type'] == 'pix':
pix.append(input)
band = inode['band']
elif inode['type'] == 'fibermap':
fm.append(input)
if len(psf) != 1:
raise RuntimeError("extraction needs exactly one psfnight file")
if len(pix) != 1:
raise RuntimeError("extraction needs exactly one image file")
if len(fm) != 1:
raise RuntimeError("extraction needs exactly one fibermap file")
imgfile = graph_path_pix(rawdir, pix[0])
psffile = graph_path_psfnight(proddir, psf[0])
fmfile = graph_path_fibermap(rawdir, fm[0])
outfile = graph_path_frame(proddir, name)
options = {}
options['input'] = imgfile
options['fibermap'] = fmfile
options['psf'] = psffile
options['output'] = outfile
# extract the wavelength range from the options, depending on the band
optscopy = copy.deepcopy(opts)
wkey = "wavelength_{}".format(band)
wave = optscopy[wkey]
del optscopy['wavelength_b']
del optscopy['wavelength_r']
del optscopy['wavelength_z']
optscopy['wavelength'] = wave
options.update(optscopy)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_extract_spectra']
com.extend(optarray)
log.debug(" ".join(com))
args = extract.parse(optarray)
extract.main_mpi(args, comm=comm)
elif step == 'fiberflat':
if len(node['in']) != 1:
raise RuntimeError('fiberflat should have only one input frame')
framefile = graph_path_frame(proddir, node['in'][0])
outfile = graph_path_fiberflat(proddir, name)
qafile, qafig = qa_path(outfile)
options = {}
options['infile'] = framefile
options['qafile'] = qafile
options['qafig'] = qafig
options['outfile'] = outfile
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_compute_fiberflat']
com.extend(optarray)
log.debug(" ".join(com))
args = fiberflat.parse(optarray)
if rank == 0:
fiberflat.main(args)
elif step == 'sky':
frm = []
flat = []
for input in node['in']:
inode = grph[input]
if inode['type'] == 'frame':
frm.append(input)
elif inode['type'] == 'fiberflat':
flat.append(input)
if len(frm) != 1:
raise RuntimeError("sky needs exactly one frame file")
if len(flat) != 1:
raise RuntimeError("sky needs exactly one fiberflat file")
framefile = graph_path_frame(proddir, frm[0])
flatfile = graph_path_fiberflat(proddir, flat[0])
outfile = graph_path_sky(proddir, name)
qafile, qafig = qa_path(outfile)
options = {}
options['infile'] = framefile
options['fiberflat'] = flatfile
options['qafile'] = qafile
options['qafig'] = qafig
options['outfile'] = outfile
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_compute_sky']
com.extend(optarray)
log.debug(" ".join(com))
args = skypkg.parse(optarray)
if rank == 0:
skypkg.main(args)
elif step == 'stdstars':
frm = []
flat = []
sky = []
flatexp = None
specgrph = None
for input in node['in']:
inode = grph[input]
if inode['type'] == 'frame':
frm.append(input)
specgrph = inode['spec']
elif inode['type'] == 'fiberflat':
flat.append(input)
flatexp = inode['id']
elif inode['type'] == 'sky':
sky.append(input)
outfile = graph_path_stdstars(proddir, name)
qafile, qafig = qa_path(outfile)
framefiles = [graph_path_frame(proddir, x) for x in frm]
skyfiles = [graph_path_sky(proddir, x) for x in sky]
flatfiles = [graph_path_fiberflat(proddir, x) for x in flat]
options = {}
options['frames'] = framefiles
options['skymodels'] = skyfiles
options['fiberflats'] = flatfiles
options['outfile'] = outfile
options['ncpu'] = str(default_nproc)
#- TODO: no QA for fitting standard stars yet
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_fit_stdstars']
com.extend(optarray)
log.debug(" ".join(com))
args = stdstars.parse(optarray)
if rank == 0:
stdstars.main(args)
elif step == 'fluxcal':
frm = []
flat = []
sky = []
star = []
for input in node['in']:
inode = grph[input]
if inode['type'] == 'frame':
frm.append(input)
elif inode['type'] == 'fiberflat':
flat.append(input)
elif inode['type'] == 'sky':
sky.append(input)
elif inode['type'] == 'stdstars':
star.append(input)
if len(frm) != 1:
raise RuntimeError("fluxcal needs exactly one frame file")
if len(flat) != 1:
raise RuntimeError("fluxcal needs exactly one fiberflat file")
if len(sky) != 1:
raise RuntimeError("fluxcal needs exactly one sky file")
if len(star) != 1:
raise RuntimeError("fluxcal needs exactly one star file")
framefile = graph_path_frame(proddir, frm[0])
flatfile = graph_path_fiberflat(proddir, flat[0])
skyfile = graph_path_sky(proddir, sky[0])
starfile = graph_path_stdstars(proddir, star[0])
outfile = graph_path_calib(proddir, name)
qafile, qafig = qa_path(outfile)
options = {}
options['infile'] = framefile
options['fiberflat'] = flatfile
options['qafile'] = qafile
options['qafig'] = qafig
options['sky'] = skyfile
options['models'] = starfile
options['outfile'] = outfile
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_compute_fluxcalibration']
com.extend(optarray)
log.debug(" ".join(com))
args = fluxcal.parse(optarray)
if rank == 0:
fluxcal.main(args)
elif step == 'procexp':
frm = []
flat = []
sky = []
cal = []
for input in node['in']:
inode = grph[input]
if inode['type'] == 'frame':
frm.append(input)
elif inode['type'] == 'fiberflat':
flat.append(input)
elif inode['type'] == 'sky':
sky.append(input)
elif inode['type'] == 'calib':
cal.append(input)
if len(frm) != 1:
raise RuntimeError("procexp needs exactly one frame file")
if len(flat) != 1:
raise RuntimeError("procexp needs exactly one fiberflat file")
if len(sky) != 1:
raise RuntimeError("procexp needs exactly one sky file")
if len(cal) != 1:
raise RuntimeError("procexp needs exactly one calib file")
framefile = graph_path_frame(proddir, frm[0])
flatfile = graph_path_fiberflat(proddir, flat[0])
skyfile = graph_path_sky(proddir, sky[0])
calfile = graph_path_calib(proddir, cal[0])
outfile = graph_path_cframe(proddir, name)
options = {}
options['infile'] = framefile
options['fiberflat'] = flatfile
options['sky'] = skyfile
options['calib'] = calfile
options['outfile'] = outfile
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_process_exposure']
com.extend(optarray)
log.debug(" ".join(com))
args = procexp.parse(optarray)
if rank == 0:
procexp.main(args)
elif step == 'zfind':
brick = node['brick']
outfile = graph_path_zbest(proddir, name)
qafile, qafig = qa_path(outfile)
options = {}
options['brick'] = brick
options['outfile'] = outfile
#- TODO: no QA for desi_zfind yet
options.update(opts)
optarray = option_list(options)
# at debug level, write out the equivalent commandline
com = ['RUN', 'desi_zfind']
com.extend(optarray)
log.debug(" ".join(com))
args = zfind.parse(optarray)
zfind.main(args, comm=comm)
else:
raise RuntimeError("Unknown pipeline step {}".format(step))
#sys.stdout.flush()
if comm is not None:
#print("proc {} hit runtask barrier".format(rank))
#sys.stdout.flush()
comm.barrier()
#print("proc {} finish runtask".format(rank))
#sys.stdout.flush()
return