def submit(self, fn: Callable[..., Union[str, Tuple[str, Dict[str, Any]]]], *args, **kwargs): """Submits a specific task to the QCG-PJ manager using template-based, executor-like interface. Parameters ---------- fn : Callable A callable that returns a tuple representing a task's template. The first element of the tuple should be a string containing a QCG-PilotJob task's description with placeholders (identifiers preceded by $ symbol) and the second a dictionary that assigns default values for selected placeholders. *args: variable length list with dicts, optional A set of dicts which contain parameters that will be used to substitute placeholders defined in the template. Note: *args overwrite defaults, but they are overwritten by **kwargs **kwargs: arbitrary keyword arguments A set of keyword arguments that will be used to substitute placeholders defined in the template. Note: **kwargs overwrite *args and defaults. Returns ------- QCGPJFuture The QCGPJFuture object assigned with the submitted task """ template = fn() if isinstance(template, tuple): template_str = template[0] defaults = template[1] else: template_str = template defaults = {} t = Template(textwrap.dedent(template_str)) substitutions = {} for a in args: if a is not None: substitutions.update(a) substitutions.update(kwargs) td_str = t.substitute(defaults, **substitutions) td = ast.literal_eval(td_str) if 'env' not in td['execution']: td['execution']['env'] = {} td['execution']['env']['QCG_PM_EXEC_API_JOB_ID'] = '${jname}' jobs = Jobs() jobs.add_std(td) jobs_ids = self._qcgpjm.submit(jobs) return QCGPJFuture(jobs_ids, self._qcgpjm)
def test_resume_wflow(tmpdir): try: ncores = 4 m = LocalManager(['--log', 'debug', '--wd', tmpdir, '--report-format', 'json', '--nodes', str(ncores)], {'wdir': str(tmpdir)}) its = 10 job_req_first = { 'name': 'first', 'execution': { 'exec': '/bin/sleep', 'args': [ '4s' ], 'stdout': 'sleep.${it}.out', }, 'iteration': { 'stop': its }, 'resources': { 'numCores': { 'exact': 1 } } } job_req_second = { 'name': 'second', 'execution': { 'exec': '/bin/date', 'stdout': 'date.${it}.out', }, 'iteration': { 'stop': its }, 'dependencies': { 'after': [ 'first' ] }, 'resources': { 'numCores': { 'exact': 1 } } } jobs = Jobs() jobs.add_std(job_req_first) jobs.add_std(job_req_second) job_ids = m.submit(jobs) # because job iterations executes in order, after finish of 4th iteration, the three previous should also finish m.wait4('first:3') jinfos = m.info_parsed(job_ids, withChilds=True) assert jinfos jinfo = jinfos['first'] # only first 4 iterations should finish assert all((jinfo.iterations, jinfo.iterations.get('start', -1) == 0, jinfo.iterations.get('stop', 0) == its, jinfo.iterations.get('total', 0) == its, jinfo.iterations.get('finished', 0) == ncores, jinfo.iterations.get('failed', -1) == 0)), str(jinfo) assert len(jinfo.childs) == its for iteration in range(its): job_it = jinfo.childs[iteration] exp_status = ['SUCCEED'] if iteration > 3: exp_status = ['EXECUTING', 'SCHEDULED', 'QUEUED'] assert all((job_it.iteration == iteration, job_it.name == '{}:{}'.format('first', iteration), job_it.status in exp_status)), \ f"{job_it.iteration} != {iteration}, {job_it.name} != {'{}:{}'.format('first', iteration)}, {job_it.status} != {exp_status}" # none of 'second' iterations should execute jinfo = jinfos['second'] assert all((jinfo.iterations, jinfo.iterations.get('start', -1) == 0, jinfo.iterations.get('stop', 0) == its, jinfo.iterations.get('total', 0) == its, jinfo.iterations.get('finished', 0) == 0, jinfo.iterations.get('failed', -1) == 0)), str(jinfo) assert len(jinfo.childs) == its for iteration in range(its): job_it = jinfo.childs[iteration] exp_status = ['QUEUED'] assert all((job_it.iteration == iteration, job_it.name == '{}:{}'.format('second', iteration), job_it.status in exp_status)), \ f"{job_it.iteration} != {iteration}, {job_it.name} != {'{}:{}'.format('second', iteration)}, {job_it.status} != {exp_status}" # kill process m.kill_manager_process() m.cleanup() ncores = 4 m = LocalManager(['--log', 'debug', '--wd', tmpdir, '--report-format', 'json', '--nodes', str(ncores), '--resume', tmpdir], {'wdir': str(tmpdir)}) m.wait4all() jinfos = m.info_parsed(job_ids, withChilds=True) assert jinfos # all iterations of 'first' job should finish jinfo = jinfos['first'] assert all((jinfo.iterations, jinfo.iterations.get('start', -1) == 0, jinfo.iterations.get('stop', 0) == its, jinfo.iterations.get('total', 0) == its, jinfo.iterations.get('finished', 0) == its, jinfo.iterations.get('failed', -1) == 0)), str(jinfo) assert len(jinfo.childs) == its for iteration in range(its): job_it = jinfo.childs[iteration] assert all((job_it.iteration == iteration, job_it.name == '{}:{}'.format('first', iteration), job_it.status == 'SUCCEED')), \ f"{job_it.iteration} != {iteration}, {job_it.name} != {'{}:{}'.format('first', iteration)}, {job_it.status} != SUCCEED" # all iterations of 'second' job should finish jinfo = jinfos['second'] assert all((jinfo.iterations, jinfo.iterations.get('start', -1) == 0, jinfo.iterations.get('stop', 0) == its, jinfo.iterations.get('total', 0) == its, jinfo.iterations.get('finished', 0) == its, jinfo.iterations.get('failed', -1) == 0)), str(jinfo) assert len(jinfo.childs) == its for iteration in range(its): job_it = jinfo.childs[iteration] assert all((job_it.iteration == iteration, job_it.name == '{}:{}'.format('second', iteration), job_it.status == 'SUCCEED')), \ f"{job_it.iteration} != {iteration}, {job_it.name} != {'{}:{}'.format('sleep', iteration)}, {job_it.status} != SUCCEED" finally: if m: m.finish() m.cleanup() rmtree(tmpdir)