def __call__(resref, resref_type="auto"):
env_resource = get_manager().get_resource(resref, resref_type)
# Connect to resource environment
env_resource.connect()
with env_resource.get_session(pty=True):
pass
def _resurrect_orc(job):
resource = get_manager().get_resource(job["resource_id"], "id")
with chpwd(job["local_directory"]):
orchestrator_class = ORCHESTRATORS[job["orchestrator"]]
orc = orchestrator_class(resource, job["submitter"], job,
resurrection=True)
orc.submitter.submission_id = job.get("submission_id")
return orc
def _resurrect_orc(job):
resource = get_manager().get_resource(job["resource_id"], "id")
try:
# Create chpwd separately so that this try-except block doesn't cover
# the context manager suite below.
cd = chpwd(job["local_directory"])
except FileNotFoundError:
raise OrchestratorError(
"local directory for job {} no longer exists: {}"
.format(job["_jobid"], job["local_directory"]))
with cd:
orchestrator_class = ORCHESTRATORS[job["orchestrator"]]
orc = orchestrator_class(resource, job["submitter"], job,
resurrection=True)
orc.submitter.submission_id = job.get("_submission_id")
return orc
def __call__(command=None,
message=None,
resref=None,
resref_type="auto",
list_=None,
submitter=None,
orchestrator=None,
batch_spec=None,
batch_parameters=None,
job_specs=None,
job_parameters=None,
inputs=None,
outputs=None,
follow=False):
if list_ is not None:
wrapper = textwrap.TextWrapper(initial_indent=" ",
subsequent_indent=" ")
def get_doc(x):
doc = x if isinstance(x, str) else x.__doc__
paragraphs = doc.replace("\n\n", "\0").split("\0")
# Collapse whitespace.
paragraphs = (" ".join(p.strip().split()) for p in paragraphs)
return "\n\n".join(wrapper.fill(p) for p in paragraphs)
def fmt(d):
return [" {}\n{}".format(k, get_doc(v)) for k, v in d.items()]
# FIXME: We shouldn't bother calling fmt on items that aren't
# selected by list=X.
categories = [
("submitters", ["Submitters"] + fmt(SUBMITTERS)),
("orchestrators", ["Orchestrator"] + fmt(ORCHESTRATORS)),
("parameters", ["Job parameters"] + fmt(JOB_PARAMETERS)),
]
items = []
for c, lines in categories:
if not list_ or c == list_:
items.extend(lines)
items.append("")
print("\n".join(items))
return
# TODO: globbing for inputs/outputs and command string formatting is
# only supported for DataLad-based orchestrators.
# CLI things that can also be specified in spec.
cli_spec = {
k: v
for k, v in {
"message": message,
"submitter": submitter,
"orchestrator": orchestrator,
"batch_spec": batch_spec,
"batch_parameters": batch_parameters,
"inputs": inputs,
"outputs": outputs,
}.items() if v is not None
}
job_parameters = parse_kv_list(job_parameters)
# Precedence: CLI option > CLI job parameter > spec file
spec = _combine_job_specs(
_load_specs(job_specs or []) + [job_parameters, cli_spec])
spec["_resolved_batch_parameters"] = _resolve_batch_parameters(
spec.get("batch_spec"), spec.get("batch_parameters"))
# Treat "command" as a special case because it's a list and the
# template expects a string.
if not command and "command_str" in spec:
spec["_resolved_command_str"] = spec["command_str"]
elif not command and "command" not in spec:
raise InsufficientArgumentsError(
"No command specified via CLI or job spec")
else:
command = command or spec["command"]
# Unlike datalad run, we're only accepting a list form for now.
spec["command"] = command
spec["_resolved_command_str"] = " ".join(map(shlex_quote, command))
if resref is None:
if "resource_id" in spec:
resref = spec["resource_id"]
resref_type = "id"
elif "resource_name" in spec:
resref = spec["resource_name"]
resref_type = "name"
else:
raise InsufficientArgumentsError("No resource specified")
manager = get_manager()
resource = manager.get_resource(resref, resref_type)
if "orchestrator" not in spec:
# TODO: We could just set this as the default for the Parameter,
# but it probably makes sense to have the default configurable per
# resource.
lgr.debug("No orchestrator specified; setting to 'plain'")
spec["orchestrator"] = "plain"
orchestrator_class = ORCHESTRATORS[spec["orchestrator"]]
orc = orchestrator_class(resource, spec.get("submitter"), spec)
orc.prepare_remote()
# TODO: Add support for templates via CLI.
orc.submit()
lreg = LocalRegistry()
lreg.register(orc.jobid, orc.as_dict())
if follow:
orc.follow()
if follow is True:
remote_fn = None
else:
only_on_success = follow.endswith("-if-success")
do_delete = follow.split("-")[0] == "delete"
def remote_fn(res, failed):
if failed and only_on_success:
lgr.info(
"Not stopping%s resource '%s' "
"because there were failed jobs",
" or deleting" if do_delete else "", res.name)
else:
lgr.info("Stopping%s resource '%s' after %s run",
" and deleting" if do_delete else "",
res.name,
"failed" if failed else "successful")
manager.stop(res)
if do_delete:
manager.delete(res)
orc.fetch(on_remote_finish=remote_fn)
lreg.unregister(orc.jobid)
# TODO: this would duplicate what is done in each .fetch
# implementation above anyways. We might want to make
# fetch return a record with fetched content and failed subjobs
failed = orc.get_failed_subjobs()
if failed:
raise JobError(failed=failed)
def __call__(command=None,
message=None,
resref=None,
resref_type="auto",
list_=None,
submitter=None,
orchestrator=None,
job_specs=None,
job_parameters=None,
inputs=None,
outputs=None,
follow=False):
if list_ is not None:
wrapper = textwrap.TextWrapper(initial_indent=" ",
subsequent_indent=" ")
def get_doc(x):
doc = x if isinstance(x, str) else x.__doc__
paragraphs = doc.replace("\n\n", "\0").split("\0")
# Collapse whitespace.
paragraphs = (" ".join(p.strip().split()) for p in paragraphs)
return "\n\n".join(wrapper.fill(p) for p in paragraphs)
def fmt(d):
return [" {}\n{}".format(k, get_doc(v)) for k, v in d.items()]
# FIXME: We shouldn't bother calling fmt on items that aren't
# selected by list=X.
categories = [
("submitters", ["Submitters"] + fmt(SUBMITTERS)),
("orchestrators", ["Orchestrator"] + fmt(ORCHESTRATORS)),
("parameters", ["Job parameters"] + fmt(JOB_PARAMETERS)),
]
items = []
for c, lines in categories:
if not list_ or c == list_:
items.extend(lines)
items.append("")
print("\n".join(items))
return
# TODO: globbing for inputs/outputs and command string formatting is
# only supported for DataLad-based orchestrators.
# CLI things that can also be specified in spec.
cli_spec = {
k: v
for k, v in {
"message": message,
"submitter": submitter,
"orchestrator": orchestrator,
"inputs": inputs,
"outputs": outputs,
}.items() if v is not None
}
job_parameters = parse_kv_list(job_parameters)
# Precedence: CLI option > CLI job parameter > spec file
spec = _combine_job_specs(
_load_specs(job_specs or []) + [job_parameters, cli_spec])
# Treat "command" as a special case because it's a list and the
# template expects a string.
if not command and "command_str" in spec:
pass
elif not command and "command" not in spec:
raise ValueError("No command specified via CLI or job spec")
else:
command = command or spec["command"]
# Unlike datalad run, we're only accepting a list form for now.
command_str = " ".join(map(shlex_quote, command))
spec["command"] = command
spec["command_str"] = command_str
if resref is None:
if "resource_id" in spec:
resref = spec["resource_id"]
resref_type = "id"
elif "resource_name" in spec:
resref = spec["resource_name"]
resref_type = "name"
else:
raise ValueError("No resource specified")
resource = get_manager().get_resource(resref, resref_type)
if "orchestrator" not in spec:
# TODO: We could just set this as the default for the Parameter,
# but it probably makes sense to have the default configurable per
# resource.
lgr.debug("No orchestrator specified; setting to 'plain'")
spec["orchestrator"] = "plain"
orchestrator_class = ORCHESTRATORS[spec["orchestrator"]]
orc = orchestrator_class(resource, spec.get("submitter"), spec)
orc.prepare_remote()
# TODO: Add support for templates via CLI.
orc.submit()
lreg = LocalRegistry()
lreg.register(orc.jobid, orc.as_dict())
if follow:
orc.follow()
orc.fetch()
lreg.unregister(orc.jobid)