def invoke_cwl_runner(self, cr_uuid, workflow_url, workflow_params, env,
workflow_descriptor_file):
api = arvados.api_from_config(
version="v1",
apiconfig={
"ARVADOS_API_HOST": env["ARVADOS_API_HOST"],
"ARVADOS_API_TOKEN": env['ARVADOS_API_TOKEN'],
"ARVADOS_API_HOST_INSECURE":
env["ARVADOS_API_HOST_INSECURE"] # NOQA
})
try:
with tempfile.NamedTemporaryFile() as inputtemp:
json.dump(workflow_params, inputtemp)
inputtemp.flush()
# TODO: run submission process in a container to prevent
# a-c-r submission processes from seeing each other.
proc = subprocess.Popen(
[
"arvados-cwl-runner",
"--submit-request-uuid=" + cr_uuid, # NOQA
"--submit",
"--no-wait",
"--api=containers", # NOQA
workflow_url,
inputtemp.name
],
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) # NOQA
(stdoutdata, stderrdata) = proc.communicate()
if proc.returncode != 0:
api.container_requests().update(
uuid=cr_uuid,
body={
"priority": 0,
"properties": {
"arvados-cwl-runner-log": stderrdata
}
}).execute()
else:
api.container_requests().update(
uuid=cr_uuid,
body={
"properties": {
"arvados-cwl-runner-log": stderrdata
}
}).execute()
except subprocess.CalledProcessError as e:
api.container_requests().update(uuid=cr_uuid,
body={
"priority": 0,
"properties": {
"arvados-cwl-runner-log":
str(e)
}
}).execute()
finally:
if workflow_descriptor_file is not None:
workflow_descriptor_file.close()
def get_api():
if not connexion.request.headers.get('Authorization'):
raise MissingAuthorization()
authtoken = connexion.request.headers['Authorization']
if authtoken.startswith("Bearer ") or authtoken.startswith("OAuth2 "):
authtoken = authtoken[7:]
return arvados.api_from_config(version="v1", apiconfig={
"ARVADOS_API_HOST": os.environ["ARVADOS_API_HOST"],
"ARVADOS_API_TOKEN": authtoken,
"ARVADOS_API_HOST_INSECURE": os.environ.get("ARVADOS_API_HOST_INSECURE", "false"), # NOQA
})
def get_api():
return arvados.api_from_config(
version="v1",
apiconfig={
"ARVADOS_API_HOST":
os.environ["ARVADOS_API_HOST"],
"ARVADOS_API_TOKEN":
connexion.request.headers['Authorization'],
"ARVADOS_API_HOST_INSECURE":
os.environ.get("ARVADOS_API_HOST_INSECURE", "false"),
})
def invoke_cwl_runner(self, cr_uuid, workflow_url, workflow_params,
env, project_uuid,
tempdir):
api = arvados.api_from_config(version="v1", apiconfig={
"ARVADOS_API_HOST": env["ARVADOS_API_HOST"],
"ARVADOS_API_TOKEN": env['ARVADOS_API_TOKEN'],
"ARVADOS_API_HOST_INSECURE": env["ARVADOS_API_HOST_INSECURE"] # NOQA
})
try:
with tempfile.NamedTemporaryFile("wt", dir=tempdir, suffix=".json") as inputtemp:
json.dump(workflow_params, inputtemp)
inputtemp.flush()
msg = ""
for dirpath, dirs, files in os.walk(tempdir):
for f in files:
msg += " " + dirpath + "/" + f + "\n"
self.log_for_run(cr_uuid, "Contents of %s:\n%s" % (tempdir, msg),
env['ARVADOS_API_TOKEN'])
# TODO: run submission process in a container to prevent
# a-c-r submission processes from seeing each other.
cmd = ["arvados-cwl-runner", "--submit-request-uuid="+cr_uuid,
"--submit", "--no-wait", "--api=containers", "--debug"]
if project_uuid:
cmd.append("--project-uuid="+project_uuid)
cmd.append(workflow_url)
cmd.append(inputtemp.name)
self.log_for_run(cr_uuid, "Executing %s" % cmd, env['ARVADOS_API_TOKEN'])
proc = subprocess.Popen(cmd, env=env,
cwd=tempdir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdoutdata, stderrdata) = proc.communicate()
if proc.returncode != 0:
api.container_requests().update(uuid=cr_uuid, body={"priority": 0}).execute()
self.log_for_run(cr_uuid, stderrdata.decode("utf-8"), env['ARVADOS_API_TOKEN'])
if tempdir:
shutil.rmtree(tempdir)
except subprocess.CalledProcessError as e:
api.container_requests().update(uuid=cr_uuid, body={"priority": 0,
"name": "Cancelled container request",
"properties": {"arvados-cwl-runner-log": str(e)}}).execute()
def invoke_cwl_runner(self, cr_uuid, workflow_url, workflow_params,
env, workflow_descriptor_file, project_uuid,
tempdir):
api = arvados.api_from_config(version="v1", apiconfig={
"ARVADOS_API_HOST": env["ARVADOS_API_HOST"],
"ARVADOS_API_TOKEN": env['ARVADOS_API_TOKEN'],
"ARVADOS_API_HOST_INSECURE": env["ARVADOS_API_HOST_INSECURE"] # NOQA
})
try:
with tempfile.NamedTemporaryFile() as inputtemp:
json.dump(workflow_params, inputtemp)
inputtemp.flush()
# TODO: run submission process in a container to prevent
# a-c-r submission processes from seeing each other.
cmd = ["arvados-cwl-runner", "--submit-request-uuid="+cr_uuid,
"--submit", "--no-wait", "--api=containers"]
if project_uuid:
cmd.append("--project-uuid="+project_uuid)
cmd.append(workflow_url)
cmd.append(inputtemp.name)
proc = subprocess.Popen(cmd, env=env,
cwd=tempdir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdoutdata, stderrdata) = proc.communicate()
if proc.returncode != 0:
api.container_requests().update(uuid=cr_uuid, body={"priority": 0}).execute()
api.logs().create(body={"log": {"object_uuid": cr_uuid,
"event_type": "stderr",
"properties": {"text": stderrdata}}}).execute()
if tempdir:
shutil.rmtree(tempdir)
except subprocess.CalledProcessError as e:
api.container_requests().update(uuid=cr_uuid, body={"priority": 0,
"properties": {"arvados-cwl-runner-log": str(e)}}).execute()
finally:
if workflow_descriptor_file is not None:
workflow_descriptor_file.close()
def runTest(self):
os.environ['KEEP_LOCAL_STORE'] = keeptmp
with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2:
with collection2.open("file1.txt", "w") as f2:
f2.write("foo")
f.write("bar")
d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
self.assertEqual(len(d1), 2)
with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
self.assertEqual(f.read(), "bar")
assertRegex(self, d1[1],
r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
with open(os.path.join(mounttmp, d1[1]), "r") as f:
self.assertEqual(f.read(), "foo")
def localapi(self):
if 'api' not in self.local.__dict__:
self.local.api = arvados.api_from_config('v1',
apiconfig=self.apiconfig,
**self.api_params)
return self.local.api
def localapi(self):
if 'api' not in self.local.__dict__:
self.local.api = arvados.api_from_config('v1', apiconfig=self.apiconfig)
return self.local.api
