def submit(self, fn: Callable[..., Union[str, Tuple[str, Dict[str, Any]]]],
*args, **kwargs):
"""Submits a specific task to the QCG-PJ manager using template-based, executor-like interface.
Parameters
----------
fn : Callable
A callable that returns a tuple representing a task's template.
The first element of the tuple should be a string containing
a QCG-PilotJob task's description with placeholders
(identifiers preceded by $ symbol) and the second a dictionary
that assigns default values for selected placeholders.
*args: variable length list with dicts, optional
A set of dicts which contain parameters that will be used to substitute placeholders
defined in the template.
Note: *args overwrite defaults, but they are overwritten by **kwargs
**kwargs: arbitrary keyword arguments
A set of keyword arguments that will be used to substitute placeholders defined in
the template.
Note: **kwargs overwrite *args and defaults.
Returns
-------
QCGPJFuture
The QCGPJFuture object assigned with the submitted task
"""
template = fn()
if isinstance(template, tuple):
template_str = template[0]
defaults = template[1]
else:
template_str = template
defaults = {}
t = Template(textwrap.dedent(template_str))
substitutions = {}
for a in args:
if a is not None:
substitutions.update(a)
substitutions.update(kwargs)
td_str = t.substitute(defaults, **substitutions)
td = ast.literal_eval(td_str)
if 'env' not in td['execution']:
td['execution']['env'] = {}
td['execution']['env']['QCG_PM_EXEC_API_JOB_ID'] = '${jname}'
jobs = Jobs()
jobs.add_std(td)
jobs_ids = self._qcgpjm.submit(jobs)
return QCGPJFuture(jobs_ids, self._qcgpjm)
def test_resume_wflow(tmpdir):
try:
ncores = 4
m = LocalManager(['--log', 'debug', '--wd', tmpdir, '--report-format', 'json', '--nodes', str(ncores)],
{'wdir': str(tmpdir)})
its = 10
job_req_first = {
'name': 'first',
'execution': {
'exec': '/bin/sleep',
'args': [ '4s' ],
'stdout': 'sleep.${it}.out',
},
'iteration': { 'stop': its },
'resources': { 'numCores': { 'exact': 1 } }
}
job_req_second = {
'name': 'second',
'execution': {
'exec': '/bin/date',
'stdout': 'date.${it}.out',
},
'iteration': { 'stop': its },
'dependencies': { 'after': [ 'first' ] },
'resources': { 'numCores': { 'exact': 1 } }
}
jobs = Jobs()
jobs.add_std(job_req_first)
jobs.add_std(job_req_second)
job_ids = m.submit(jobs)
# because job iterations executes in order, after finish of 4th iteration, the three previous should also finish
m.wait4('first:3')
jinfos = m.info_parsed(job_ids, withChilds=True)
assert jinfos
jinfo = jinfos['first']
# only first 4 iterations should finish
assert all((jinfo.iterations, jinfo.iterations.get('start', -1) == 0,
jinfo.iterations.get('stop', 0) == its, jinfo.iterations.get('total', 0) == its,
jinfo.iterations.get('finished', 0) == ncores, jinfo.iterations.get('failed', -1) == 0)), str(jinfo)
assert len(jinfo.childs) == its
for iteration in range(its):
job_it = jinfo.childs[iteration]
exp_status = ['SUCCEED']
if iteration > 3:
exp_status = ['EXECUTING', 'SCHEDULED', 'QUEUED']
assert all((job_it.iteration == iteration,
job_it.name == '{}:{}'.format('first', iteration),
job_it.status in exp_status)), \
f"{job_it.iteration} != {iteration}, {job_it.name} != {'{}:{}'.format('first', iteration)}, {job_it.status} != {exp_status}"
# none of 'second' iterations should execute
jinfo = jinfos['second']
assert all((jinfo.iterations, jinfo.iterations.get('start', -1) == 0,
jinfo.iterations.get('stop', 0) == its, jinfo.iterations.get('total', 0) == its,
jinfo.iterations.get('finished', 0) == 0, jinfo.iterations.get('failed', -1) == 0)), str(jinfo)
assert len(jinfo.childs) == its
for iteration in range(its):
job_it = jinfo.childs[iteration]
exp_status = ['QUEUED']
assert all((job_it.iteration == iteration,
job_it.name == '{}:{}'.format('second', iteration),
job_it.status in exp_status)), \
f"{job_it.iteration} != {iteration}, {job_it.name} != {'{}:{}'.format('second', iteration)}, {job_it.status} != {exp_status}"
# kill process
m.kill_manager_process()
m.cleanup()
ncores = 4
m = LocalManager(['--log', 'debug', '--wd', tmpdir, '--report-format', 'json', '--nodes', str(ncores),
'--resume', tmpdir], {'wdir': str(tmpdir)})
m.wait4all()
jinfos = m.info_parsed(job_ids, withChilds=True)
assert jinfos
# all iterations of 'first' job should finish
jinfo = jinfos['first']
assert all((jinfo.iterations, jinfo.iterations.get('start', -1) == 0,
jinfo.iterations.get('stop', 0) == its, jinfo.iterations.get('total', 0) == its,
jinfo.iterations.get('finished', 0) == its, jinfo.iterations.get('failed', -1) == 0)), str(jinfo)
assert len(jinfo.childs) == its
for iteration in range(its):
job_it = jinfo.childs[iteration]
assert all((job_it.iteration == iteration,
job_it.name == '{}:{}'.format('first', iteration),
job_it.status == 'SUCCEED')), \
f"{job_it.iteration} != {iteration}, {job_it.name} != {'{}:{}'.format('first', iteration)}, {job_it.status} != SUCCEED"
# all iterations of 'second' job should finish
jinfo = jinfos['second']
assert all((jinfo.iterations, jinfo.iterations.get('start', -1) == 0,
jinfo.iterations.get('stop', 0) == its, jinfo.iterations.get('total', 0) == its,
jinfo.iterations.get('finished', 0) == its, jinfo.iterations.get('failed', -1) == 0)), str(jinfo)
assert len(jinfo.childs) == its
for iteration in range(its):
job_it = jinfo.childs[iteration]
assert all((job_it.iteration == iteration,
job_it.name == '{}:{}'.format('second', iteration),
job_it.status == 'SUCCEED')), \
f"{job_it.iteration} != {iteration}, {job_it.name} != {'{}:{}'.format('sleep', iteration)}, {job_it.status} != SUCCEED"
finally:
if m:
m.finish()
m.cleanup()
rmtree(tmpdir)