def test_job_addition():
"""Test job addition and exist check"""
purge()
Status.add_job(STATUS_DIR, 'generation', 'test1')
status1 = Status(STATUS_DIR).data['generation']['test1']['job_status']
Status.add_job(STATUS_DIR, 'generation', 'test1',
job_attrs={'job_status': 'finished', 'additional': 'test'})
status2 = Status(STATUS_DIR).data['generation']['test1']['job_status']
assert status2 == status1
purge()
def test_job_replacement():
"""Test job addition and replacement"""
purge()
Status.add_job(STATUS_DIR, 'generation', 'test1',
job_attrs={'job_status': 'submitted'})
Status.add_job(STATUS_DIR, 'generation', 'test1',
job_attrs={'addition': 'test', 'job_status': 'finished'},
replace=True)
status = Status(STATUS_DIR).data['generation']['test1']['job_status']
addition = Status(STATUS_DIR).data['generation']['test1']['addition']
assert status == 'finished'
assert addition == 'test'
purge()
def parse_previous(status_dir, module, target='fpath', target_module=None):
"""Parse output file paths from the previous pipeline step.
Parameters
----------
status_dir : str
Directory containing the status file to parse.
module : str
Current module (i.e. current pipeline step).
target : str
Parsing target of previous module.
target_module : str | None
Optional name of module to pull target data from.
Returns
-------
out : list
Arguments parsed from the status file in status_dir from
the module preceding the input module arg.
"""
status = Status(status_dir)
msg = ('Could not parse data regarding "{}" from reV status file in '
'"{}".'.format(module, status_dir))
if module in status.data:
if 'pipeline_index' in status.data[module]:
msg = None
if msg:
raise KeyError(msg)
i1 = int(status.data[module]['pipeline_index'])
i0 = i1 - 1
if i0 < 0:
i0 = 0
warn('Module "{0}" is attempting to parse a previous pipeline '
'step, but it appears to be the first step. Attempting to '
'parse data from {0}.'.format(module))
if target_module is None:
module_status = Pipeline._get_module_status(status, i0)
job_statuses = Pipeline._get_job_status(module_status)
else:
if target_module not in status.data:
raise KeyError('Target module "{}" not found in pipeline '
'status dictionary.'.format(target_module))
else:
module_status = status.data[target_module]
job_statuses = Pipeline._get_job_status(module_status)
out = []
if target == 'fpath':
for status in job_statuses:
out.append(os.path.join(status['dirout'], status['fout']))
else:
for status in job_statuses:
out.append(status[target])
return out
def _get_status_obj(self):
"""Get a reV pipeline status object.
Returns
-------
status : reV.pipeline.status.Status
reV job status object.
"""
status = Status(self._config.dirout,
name=self._config.name,
hardware=self._config.hardware)
return status
