def invoke_cwl_runner(self, cr_uuid, workflow_url, workflow_params, env, workflow_descriptor_file): api = arvados.api_from_config( version="v1", apiconfig={ "ARVADOS_API_HOST": env["ARVADOS_API_HOST"], "ARVADOS_API_TOKEN": env['ARVADOS_API_TOKEN'], "ARVADOS_API_HOST_INSECURE": env["ARVADOS_API_HOST_INSECURE"] # NOQA }) try: with tempfile.NamedTemporaryFile() as inputtemp: json.dump(workflow_params, inputtemp) inputtemp.flush() # TODO: run submission process in a container to prevent # a-c-r submission processes from seeing each other. proc = subprocess.Popen( [ "arvados-cwl-runner", "--submit-request-uuid=" + cr_uuid, # NOQA "--submit", "--no-wait", "--api=containers", # NOQA workflow_url, inputtemp.name ], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # NOQA (stdoutdata, stderrdata) = proc.communicate() if proc.returncode != 0: api.container_requests().update( uuid=cr_uuid, body={ "priority": 0, "properties": { "arvados-cwl-runner-log": stderrdata } }).execute() else: api.container_requests().update( uuid=cr_uuid, body={ "properties": { "arvados-cwl-runner-log": stderrdata } }).execute() except subprocess.CalledProcessError as e: api.container_requests().update(uuid=cr_uuid, body={ "priority": 0, "properties": { "arvados-cwl-runner-log": str(e) } }).execute() finally: if workflow_descriptor_file is not None: workflow_descriptor_file.close()
def get_api(): if not connexion.request.headers.get('Authorization'): raise MissingAuthorization() authtoken = connexion.request.headers['Authorization'] if authtoken.startswith("Bearer ") or authtoken.startswith("OAuth2 "): authtoken = authtoken[7:] return arvados.api_from_config(version="v1", apiconfig={ "ARVADOS_API_HOST": os.environ["ARVADOS_API_HOST"], "ARVADOS_API_TOKEN": authtoken, "ARVADOS_API_HOST_INSECURE": os.environ.get("ARVADOS_API_HOST_INSECURE", "false"), # NOQA })
def get_api(): return arvados.api_from_config( version="v1", apiconfig={ "ARVADOS_API_HOST": os.environ["ARVADOS_API_HOST"], "ARVADOS_API_TOKEN": connexion.request.headers['Authorization'], "ARVADOS_API_HOST_INSECURE": os.environ.get("ARVADOS_API_HOST_INSECURE", "false"), })
def invoke_cwl_runner(self, cr_uuid, workflow_url, workflow_params, env, project_uuid, tempdir): api = arvados.api_from_config(version="v1", apiconfig={ "ARVADOS_API_HOST": env["ARVADOS_API_HOST"], "ARVADOS_API_TOKEN": env['ARVADOS_API_TOKEN'], "ARVADOS_API_HOST_INSECURE": env["ARVADOS_API_HOST_INSECURE"] # NOQA }) try: with tempfile.NamedTemporaryFile("wt", dir=tempdir, suffix=".json") as inputtemp: json.dump(workflow_params, inputtemp) inputtemp.flush() msg = "" for dirpath, dirs, files in os.walk(tempdir): for f in files: msg += " " + dirpath + "/" + f + "\n" self.log_for_run(cr_uuid, "Contents of %s:\n%s" % (tempdir, msg), env['ARVADOS_API_TOKEN']) # TODO: run submission process in a container to prevent # a-c-r submission processes from seeing each other. cmd = ["arvados-cwl-runner", "--submit-request-uuid="+cr_uuid, "--submit", "--no-wait", "--api=containers", "--debug"] if project_uuid: cmd.append("--project-uuid="+project_uuid) cmd.append(workflow_url) cmd.append(inputtemp.name) self.log_for_run(cr_uuid, "Executing %s" % cmd, env['ARVADOS_API_TOKEN']) proc = subprocess.Popen(cmd, env=env, cwd=tempdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdoutdata, stderrdata) = proc.communicate() if proc.returncode != 0: api.container_requests().update(uuid=cr_uuid, body={"priority": 0}).execute() self.log_for_run(cr_uuid, stderrdata.decode("utf-8"), env['ARVADOS_API_TOKEN']) if tempdir: shutil.rmtree(tempdir) except subprocess.CalledProcessError as e: api.container_requests().update(uuid=cr_uuid, body={"priority": 0, "name": "Cancelled container request", "properties": {"arvados-cwl-runner-log": str(e)}}).execute()
def invoke_cwl_runner(self, cr_uuid, workflow_url, workflow_params, env, workflow_descriptor_file, project_uuid, tempdir): api = arvados.api_from_config(version="v1", apiconfig={ "ARVADOS_API_HOST": env["ARVADOS_API_HOST"], "ARVADOS_API_TOKEN": env['ARVADOS_API_TOKEN'], "ARVADOS_API_HOST_INSECURE": env["ARVADOS_API_HOST_INSECURE"] # NOQA }) try: with tempfile.NamedTemporaryFile() as inputtemp: json.dump(workflow_params, inputtemp) inputtemp.flush() # TODO: run submission process in a container to prevent # a-c-r submission processes from seeing each other. cmd = ["arvados-cwl-runner", "--submit-request-uuid="+cr_uuid, "--submit", "--no-wait", "--api=containers"] if project_uuid: cmd.append("--project-uuid="+project_uuid) cmd.append(workflow_url) cmd.append(inputtemp.name) proc = subprocess.Popen(cmd, env=env, cwd=tempdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdoutdata, stderrdata) = proc.communicate() if proc.returncode != 0: api.container_requests().update(uuid=cr_uuid, body={"priority": 0}).execute() api.logs().create(body={"log": {"object_uuid": cr_uuid, "event_type": "stderr", "properties": {"text": stderrdata}}}).execute() if tempdir: shutil.rmtree(tempdir) except subprocess.CalledProcessError as e: api.container_requests().update(uuid=cr_uuid, body={"priority": 0, "properties": {"arvados-cwl-runner-log": str(e)}}).execute() finally: if workflow_descriptor_file is not None: workflow_descriptor_file.close()
def runTest(self): os.environ['KEEP_LOCAL_STORE'] = keeptmp with open(os.path.join(mounttmp, "file1.txt"), "w") as f: with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2: with collection2.open("file1.txt", "w") as f2: f2.write("foo") f.write("bar") d1 = sorted(llfuse.listdir(os.path.join(mounttmp))) self.assertEqual(len(d1), 2) with open(os.path.join(mounttmp, "file1.txt"), "r") as f: self.assertEqual(f.read(), "bar") assertRegex(self, d1[1], r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~') with open(os.path.join(mounttmp, d1[1]), "r") as f: self.assertEqual(f.read(), "foo")
def localapi(self): if 'api' not in self.local.__dict__: self.local.api = arvados.api_from_config('v1', apiconfig=self.apiconfig, **self.api_params) return self.local.api
def localapi(self): if 'api' not in self.local.__dict__: self.local.api = arvados.api_from_config('v1', apiconfig=self.apiconfig) return self.local.api