def dispatch_job(jobname, logfile, **kwargs):
"""Dispatch a single job
Parameters
----------
jobname : `str`
The command to run the job
logfile : `str`
The path to the logfile
Keywords
--------
run : `str`
The run number
batch : `str`
Send jobs to batch farm
batch_args : `str`
Arguments to pass to batch command
optstring : `str`
Additional arguments to pass to the command
dry_run : `bool`
Print command but do not run it
"""
run = kwargs.get('run', None)
batch_args = kwargs.get('batch_args', None)
optstring = kwargs.get('optstring', None)
dry_run = kwargs.get('dry_run', False)
disptach = 'native'
if kwargs.get('batch', False):
disptach = BATCH_SYSTEM
if disptach.find('lsf') == 0:
makedir_safe(logfile)
sub_com = "bsub -o %s" % logfile
if batch_args is not None:
sub_com += " %s " % batch_args
elif disptach.find('slurm') == 0:
batchfile = write_slurm_batchfile(jobname, logfile, **kwargs)
logfile_job = os.path.join('sbatch', logfile.replace('.fits', '.out'))
sub_com = "sbatch %s -o %s -e %s" % (batchfile, logfile_job,
logfile_job)
else:
sub_com = ""
if disptach.find('slurm') < 0:
if run is None:
sub_com += " %s" % jobname
else:
sub_com += " %s --run %s" % (jobname, run)
if optstring is not None:
sub_com += " %s" % optstring
if dry_run:
sys.stdout.write("%s\n" % sub_com)
else:
os.system(sub_com)
def make_superbias(self, butler, slot_data, **kwargs):
"""Stack the input data to make superbias frames
The superbias frames are stored as data members of this class
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
self._superbias_frame = None
mask_files = self.get_mask_files()
stat_type = self.config.stat
if stat_type is None:
stat_type = DEFAULT_STAT_TYPE
if stat_type == DEFAULT_STAT_TYPE:
output_file = self.tablefile_name() + '.fits'
else:
output_file = self.get_filename_from_format(
SUPERBIAS_STAT_FORMATTER, '.fits', **kwargs)
data_files = self.get_input_files(slot_data)
if not data_files:
return
makedir_safe(output_file)
if not self.config.skip:
out_data = self.extract(butler, slot_data)
if out_data is None:
self.log_warn_slot_msg(self.config, "extract() returned None.")
return
if butler is None:
template_file = data_files[0]
else:
template_file = get_filename_from_id(butler, data_files[0])
imutil.writeFits(out_data, output_file, template_file,
self.config.bitpix)
if butler is not None:
flip_data_in_place(output_file)
try:
self._superbias_frame = self.get_ccd(None, output_file, mask_files)
except Exception:
self._superbias_frame = None
def make_superflats(self, butler, data, **kwargs):
"""Stack the input data to make superflat frames
The superflats are stored as data members of this class
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
mask_files = self.get_mask_files()
output_file = self.tablefile_name()
makedir_safe(output_file)
if not self.config.skip:
sflats = self.extract(butler, data)
if sflats is None:
return
if butler is None:
template_file = data['SFLAT'][0]
else:
template_file = get_filename_from_id(butler, data['SFLAT'][0])
imutil.writeFits(sflats[0], output_file + '_l.fits', template_file,
self.config.bitpix)
imutil.writeFits(sflats[1], output_file + '_h.fits', template_file,
self.config.bitpix)
imutil.writeFits(sflats[2], output_file + '_r.fits', template_file,
self.config.bitpix)
if butler is not None:
flip_data_in_place(output_file + '_l.fits')
flip_data_in_place(output_file + '_h.fits')
flip_data_in_place(output_file + '_r.fits')
self._superflat_frame_l = self.get_ccd(None, output_file + '_l.fits',
mask_files)
self._superflat_frame_h = self.get_ccd(None, output_file + '_h.fits',
mask_files)
self._superflat_frame_r = self.get_ccd(None, output_file + '_r.fits',
mask_files)
def write_slurm_batchfile(jobname, logfile, **kwargs):
"""Dispatch a single job
Parameters
----------
jobname : `str`
The command to run the job
logfile : `str`
The path to the logfile
Keywords
--------
run : `str`
The run number
batch_args : `str`
Arguments to pass to batch command
optstring : `str`
Additional arguments to pass to the command
Returns
-------
batchfile : `str`
The name of the slurm submission script file
"""
run = kwargs.get('run', None)
batch_args = kwargs.get('batch_args', None)
optstring = kwargs.get('optstring', None)
batchfile = os.path.join('sbatch', logfile.replace('.log', '.sl'))
makedir_safe(batchfile)
fout = open(batchfile, 'w')
fout.write("#!/bin/bash -l\n")
tokens = batch_args.split()
for key, val in zip(tokens[0::2], tokens[1::2]):
fout.write("#SBATCH %s %s\n" % (key, val))
fout.write("\n")
sub_com = "srun --output %s" % os.path.join('sbatch', logfile)
if run is None:
sub_com += " %s" % jobname
else:
sub_com += " %s --run %s" % (jobname, run)
sub_com += " %s" % optstring
fout.write("%s\n" % sub_com)
fout.close()
return batchfile