def arvExecutor(self, tool, job_order, input_basedir, args, **kwargs):
events = arvados.events.subscribe(
arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]],
self.on_message)
self.fs_access = CollectionFsAccess(input_basedir)
kwargs["fs_access"] = self.fs_access
kwargs["enable_reuse"] = args.enable_reuse
if kwargs.get("conformance_test"):
return cwltool.main.single_job_executor(tool, job_order,
input_basedir, args,
**kwargs)
else:
jobiter = tool.job(job_order, input_basedir, self.output_callback,
**kwargs)
for runnable in jobiter:
if runnable:
with self.lock:
runnable.run(**kwargs)
else:
if self.jobs:
try:
self.cond.acquire()
self.cond.wait()
finally:
self.cond.release()
else:
logger.error("Workflow cannot make any more progress.")
break
while self.jobs:
try:
self.cond.acquire()
self.cond.wait()
finally:
self.cond.release()
events.close()
if self.final_output is None:
raise cwltool.workflow.WorkflowException(
"Workflow did not return a result.")
return self.final_output
def arvExecutor(self, tool, job_order, input_basedir, args, **kwargs):
events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
self.fs_access = CollectionFsAccess(input_basedir)
kwargs["fs_access"] = self.fs_access
kwargs["enable_reuse"] = args.enable_reuse
kwargs["repository"] = args.repository
if kwargs.get("conformance_test"):
return cwltool.main.single_job_executor(tool, job_order, input_basedir, args, **kwargs)
else:
jobiter = tool.job(job_order,
input_basedir,
self.output_callback,
**kwargs)
for runnable in jobiter:
if runnable:
with self.lock:
runnable.run(**kwargs)
else:
if self.jobs:
try:
self.cond.acquire()
self.cond.wait()
finally:
self.cond.release()
else:
logger.error("Workflow cannot make any more progress.")
break
while self.jobs:
try:
self.cond.acquire()
self.cond.wait()
finally:
self.cond.release()
events.close()
if self.final_output is None:
raise cwltool.workflow.WorkflowException("Workflow did not return a result.")
return self.final_output
def arvExecutor(self, tool, job_order, input_basedir, args, **kwargs):
events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
try:
self.api.collections().get(uuid=crunchrunner_pdh).execute()
except arvados.errors.ApiError as e:
import httplib2
h = httplib2.Http(ca_certs=arvados.util.ca_certs_path())
resp, content = h.request(crunchrunner_download, "GET")
resp2, content2 = h.request(certs_download, "GET")
with arvados.collection.Collection() as col:
with col.open("crunchrunner", "w") as f:
f.write(content)
with col.open("ca-certificates.crt", "w") as f:
f.write(content2)
col.save_new("crunchrunner binary", ensure_unique_name=True)
self.fs_access = CollectionFsAccess(input_basedir)
kwargs["fs_access"] = self.fs_access
kwargs["enable_reuse"] = args.enable_reuse
kwargs["outdir"] = "$(task.outdir)"
kwargs["tmpdir"] = "$(task.tmpdir)"
useruuid = self.api.users().current().execute()["uuid"]
self.project_uuid = args.project_uuid if args.project_uuid else useruuid
if kwargs.get("conformance_test"):
return cwltool.main.single_job_executor(tool, job_order, input_basedir, args, **kwargs)
else:
self.pipeline = self.api.pipeline_instances().create(
body={
"owner_uuid": self.project_uuid,
"name": shortname(tool.tool["id"]),
"components": {},
"state": "RunningOnClient"}).execute(num_retries=self.num_retries)
logger.info("Pipeline instance %s", self.pipeline["uuid"])
jobiter = tool.job(job_order,
input_basedir,
self.output_callback,
docker_outdir="$(task.outdir)",
**kwargs)
try:
self.cond.acquire()
# Will continue to hold the lock for the duration of this code
# except when in cond.wait(), at which point on_message can update
# job state and process output callbacks.
for runnable in jobiter:
if runnable:
runnable.run(**kwargs)
else:
if self.jobs:
self.cond.wait(1)
else:
logger.error("Workflow is deadlocked, no runnable jobs and not waiting on any pending jobs.")
break
while self.jobs:
self.cond.wait(1)
events.close()
if self.final_output is None:
raise cwltool.workflow.WorkflowException("Workflow did not return a result.")
# create final output collection
except:
if sys.exc_info()[0] is KeyboardInterrupt:
logger.error("Interrupted, marking pipeline as failed")
else:
logger.exception("Caught unhandled exception, marking pipeline as failed")
self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
body={"state": "Failed"}).execute(num_retries=self.num_retries)
finally:
self.cond.release()
return self.final_output
def arvExecutor(self, tool, job_order, **kwargs):
self.debug = kwargs.get("debug")
if kwargs.get("quiet"):
logger.setLevel(logging.WARN)
logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
useruuid = self.api.users().current().execute()["uuid"]
self.project_uuid = kwargs.get("project_uuid") if kwargs.get(
"project_uuid") else useruuid
self.pipeline = None
if kwargs.get("create_template"):
tmpl = RunnerTemplate(self, tool, job_order,
kwargs.get("enable_reuse"))
tmpl.save()
# cwltool.main will write our return value to stdout.
return tmpl.uuid
if kwargs.get("submit"):
runnerjob = RunnerJob(self, tool, job_order,
kwargs.get("enable_reuse"))
if not kwargs.get("submit") and "cwl_runner_job" not in kwargs:
# Create pipeline for local run
self.pipeline = self.api.pipeline_instances().create(
body={
"owner_uuid": self.project_uuid,
"name": shortname(tool.tool["id"]),
"components": {},
"state": "RunningOnClient"
}).execute(num_retries=self.num_retries)
logger.info("Pipeline instance %s", self.pipeline["uuid"])
if kwargs.get("submit") and not kwargs.get("wait"):
runnerjob.run()
return runnerjob.uuid
events = arvados.events.subscribe(
arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]],
self.on_message)
self.debug = kwargs.get("debug")
self.ignore_docker_for_reuse = kwargs.get("ignore_docker_for_reuse")
self.fs_access = CollectionFsAccess(kwargs["basedir"])
kwargs["fs_access"] = self.fs_access
kwargs["enable_reuse"] = kwargs.get("enable_reuse")
kwargs["outdir"] = "$(task.outdir)"
kwargs["tmpdir"] = "$(task.tmpdir)"
if kwargs.get("conformance_test"):
return cwltool.main.single_job_executor(tool, job_order, **kwargs)
else:
if kwargs.get("submit"):
jobiter = iter((runnerjob, ))
else:
if "cwl_runner_job" in kwargs:
self.uuid = kwargs.get("cwl_runner_job").get('uuid')
jobiter = tool.job(job_order,
self.output_callback,
docker_outdir="$(task.outdir)",
**kwargs)
try:
self.cond.acquire()
# Will continue to hold the lock for the duration of this code
# except when in cond.wait(), at which point on_message can update
# job state and process output callbacks.
for runnable in jobiter:
if runnable:
runnable.run(**kwargs)
else:
if self.jobs:
self.cond.wait(1)
else:
logger.error(
"Workflow is deadlocked, no runnable jobs and not waiting on any pending jobs."
)
break
while self.jobs:
self.cond.wait(1)
events.close()
except:
if sys.exc_info()[0] is KeyboardInterrupt:
logger.error("Interrupted, marking pipeline as failed")
else:
logger.error(
"Caught unhandled exception, marking pipeline as failed. Error was: %s",
sys.exc_info()[0],
exc_info=(sys.exc_info()[1] if self.debug else False))
if self.pipeline:
self.api.pipeline_instances().update(
uuid=self.pipeline["uuid"], body={
"state": "Failed"
}).execute(num_retries=self.num_retries)
finally:
self.cond.release()
if self.final_output is None:
raise cwltool.workflow.WorkflowException(
"Workflow did not return a result.")
return self.final_output
def arvExecutor(self, tool, job_order, **kwargs):
self.debug = kwargs.get("debug")
if kwargs.get("quiet"):
logger.setLevel(logging.WARN)
logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
useruuid = self.api.users().current().execute()["uuid"]
self.project_uuid = kwargs.get("project_uuid") if kwargs.get("project_uuid") else useruuid
self.pipeline = None
self.fs_access = CollectionFsAccess(kwargs["basedir"], api_client=self.api)
if kwargs.get("create_template"):
tmpl = RunnerTemplate(self, tool, job_order, kwargs.get("enable_reuse"))
tmpl.save()
# cwltool.main will write our return value to stdout.
return tmpl.uuid
self.debug = kwargs.get("debug")
self.ignore_docker_for_reuse = kwargs.get("ignore_docker_for_reuse")
kwargs["fs_access"] = self.fs_access
kwargs["enable_reuse"] = kwargs.get("enable_reuse")
kwargs["use_container"] = True
kwargs["tmpdir_prefix"] = "tmp"
kwargs["on_error"] = "continue"
kwargs["compute_checksum"] = kwargs.get("compute_checksum")
if self.work_api == "containers":
kwargs["outdir"] = "/var/spool/cwl"
kwargs["docker_outdir"] = "/var/spool/cwl"
kwargs["tmpdir"] = "/tmp"
elif self.work_api == "jobs":
kwargs["outdir"] = "$(task.outdir)"
kwargs["docker_outdir"] = "$(task.outdir)"
kwargs["tmpdir"] = "$(task.tmpdir)"
runnerjob = None
if kwargs.get("submit"):
if self.work_api == "containers":
if tool.tool["class"] == "CommandLineTool":
runnerjob = tool.job(job_order,
self.output_callback,
**kwargs).next()
else:
runnerjob = RunnerContainer(self, tool, job_order, kwargs.get("enable_reuse"))
else:
runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"))
if not kwargs.get("submit") and "cwl_runner_job" not in kwargs and not self.work_api == "containers":
# Create pipeline for local run
self.pipeline = self.api.pipeline_instances().create(
body={
"owner_uuid": self.project_uuid,
"name": shortname(tool.tool["id"]),
"components": {},
"state": "RunningOnClient"}).execute(num_retries=self.num_retries)
logger.info("Pipeline instance %s", self.pipeline["uuid"])
if runnerjob and not kwargs.get("wait"):
runnerjob.run()
return runnerjob.uuid
arvados.config.settings()["ARVADOS_DISABLE_WEBSOCKETS"] = "1"
if self.work_api == "containers":
events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#container"]], self.on_message)
if self.work_api == "jobs":
events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
if runnerjob:
jobiter = iter((runnerjob,))
else:
if "cwl_runner_job" in kwargs:
self.uuid = kwargs.get("cwl_runner_job").get('uuid')
jobiter = tool.job(job_order,
self.output_callback,
**kwargs)
try:
self.cond.acquire()
# Will continue to hold the lock for the duration of this code
# except when in cond.wait(), at which point on_message can update
# job state and process output callbacks.
for runnable in jobiter:
if runnable:
runnable.run(**kwargs)
else:
if self.processes:
self.cond.wait(1)
else:
logger.error("Workflow is deadlocked, no runnable jobs and not waiting on any pending jobs.")
break
while self.processes:
self.cond.wait(1)
events.close()
except UnsupportedRequirement:
raise
except:
if sys.exc_info()[0] is KeyboardInterrupt:
logger.error("Interrupted, marking pipeline as failed")
else:
logger.error("Caught unhandled exception, marking pipeline as failed. Error was: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
if self.pipeline:
self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
body={"state": "Failed"}).execute(num_retries=self.num_retries)
if runnerjob and runnerjob.uuid and self.work_api == "containers":
self.api.container_requests().update(uuid=runnerjob.uuid,
body={"priority": "0"}).execute(num_retries=self.num_retries)
finally:
self.cond.release()
if self.final_status == "UnsupportedRequirement":
raise UnsupportedRequirement("Check log for details.")
if self.final_status != "success":
raise WorkflowException("Workflow failed.")
if self.final_output is None:
raise WorkflowException("Workflow did not return a result.")
if kwargs.get("compute_checksum"):
def compute_checksums(fileobj):
if "checksum" not in fileobj:
checksum = hashlib.sha1()
with self.fs_access.open(fileobj["location"], "rb") as f:
contents = f.read(1024*1024)
while contents != "":
checksum.update(contents)
contents = f.read(1024*1024)
fileobj["checksum"] = "sha1$%s" % checksum.hexdigest()
adjustFileObjs(self.final_output, compute_checksums)
return self.final_output
def arvExecutor(self, tool, job_order, **kwargs):
self.debug = kwargs.get("debug")
if kwargs.get("quiet"):
logger.setLevel(logging.WARN)
logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
useruuid = self.api.users().current().execute()["uuid"]
self.project_uuid = kwargs.get("project_uuid") if kwargs.get("project_uuid") else useruuid
self.pipeline = None
if kwargs.get("create_template"):
tmpl = RunnerTemplate(self, tool, job_order, kwargs.get("enable_reuse"))
tmpl.save()
# cwltool.main will write our return value to stdout.
return tmpl.uuid
if kwargs.get("submit"):
runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"))
if not kwargs.get("wait"):
runnerjob.run()
return runnerjob.uuid
events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
self.debug = kwargs.get("debug")
self.ignore_docker_for_reuse = kwargs.get("ignore_docker_for_reuse")
self.fs_access = CollectionFsAccess(kwargs["basedir"])
kwargs["fs_access"] = self.fs_access
kwargs["enable_reuse"] = kwargs.get("enable_reuse")
kwargs["outdir"] = "$(task.outdir)"
kwargs["tmpdir"] = "$(task.tmpdir)"
if kwargs.get("conformance_test"):
return cwltool.main.single_job_executor(tool, job_order, **kwargs)
else:
if kwargs.get("submit"):
jobiter = iter((runnerjob,))
else:
components = {}
if "cwl_runner_job" in kwargs:
components[os.path.basename(tool.tool["id"])] = {"job": kwargs["cwl_runner_job"]}
self.pipeline = self.api.pipeline_instances().create(
body={
"owner_uuid": self.project_uuid,
"name": shortname(tool.tool["id"]),
"components": components,
"state": "RunningOnClient"}).execute(num_retries=self.num_retries)
logger.info("Pipeline instance %s", self.pipeline["uuid"])
jobiter = tool.job(job_order,
self.output_callback,
docker_outdir="$(task.outdir)",
**kwargs)
try:
self.cond.acquire()
# Will continue to hold the lock for the duration of this code
# except when in cond.wait(), at which point on_message can update
# job state and process output callbacks.
for runnable in jobiter:
if runnable:
runnable.run(**kwargs)
else:
if self.jobs:
self.cond.wait(1)
else:
logger.error("Workflow is deadlocked, no runnable jobs and not waiting on any pending jobs.")
break
while self.jobs:
self.cond.wait(1)
events.close()
except:
if sys.exc_info()[0] is KeyboardInterrupt:
logger.error("Interrupted, marking pipeline as failed")
else:
logger.error("Caught unhandled exception, marking pipeline as failed. Error was: %s", sys.exc_info()[0], exc_info=(sys.exc_info()[1] if self.debug else False))
if self.pipeline:
self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
body={"state": "Failed"}).execute(num_retries=self.num_retries)
finally:
self.cond.release()
if self.final_output is None:
raise cwltool.workflow.WorkflowException("Workflow did not return a result.")
return self.final_output
def arvExecutor(self, tool, job_order, input_basedir, args, **kwargs):
events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
try:
self.api.collections().get(uuid=crunchrunner_pdh).execute()
except arvados.errors.ApiError as e:
import httplib2
h = httplib2.Http(ca_certs=arvados.util.ca_certs_path())
resp, content = h.request(crunchrunner_download, "GET")
resp2, content2 = h.request(certs_download, "GET")
with arvados.collection.Collection() as col:
with col.open("crunchrunner", "w") as f:
f.write(content)
with col.open("ca-certificates.crt", "w") as f:
f.write(content2)
col.save_new("crunchrunner binary", ensure_unique_name=True)
self.fs_access = CollectionFsAccess(input_basedir)
kwargs["fs_access"] = self.fs_access
kwargs["enable_reuse"] = args.enable_reuse
kwargs["outdir"] = "$(task.outdir)"
kwargs["tmpdir"] = "$(task.tmpdir)"
if kwargs.get("conformance_test"):
return cwltool.main.single_job_executor(tool, job_order, input_basedir, args, **kwargs)
else:
self.pipeline = self.api.pipeline_instances().create(body={"name": shortname(tool.tool["id"]),
"components": {},
"state": "RunningOnClient"}).execute(num_retries=self.num_retries)
jobiter = tool.job(job_order,
input_basedir,
self.output_callback,
docker_outdir="$(task.outdir)",
**kwargs)
try:
for runnable in jobiter:
if runnable:
with self.lock:
runnable.run(**kwargs)
else:
if self.jobs:
try:
self.cond.acquire()
self.cond.wait(1)
except RuntimeError:
pass
finally:
self.cond.release()
else:
logger.error("Workflow cannot make any more progress.")
break
while self.jobs:
try:
self.cond.acquire()
self.cond.wait(1)
except RuntimeError:
pass
finally:
self.cond.release()
events.close()
if self.final_output is None:
raise cwltool.workflow.WorkflowException("Workflow did not return a result.")
except:
if sys.exc_info()[0] is not KeyboardInterrupt:
logger.exception("Caught unhandled exception, marking pipeline as failed")
self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
body={"state": "Failed"}).execute(num_retries=self.num_retries)
return self.final_output