예제 #1
0
    def arvExecutor(self, tool, job_order, input_basedir, args, **kwargs):
        events = arvados.events.subscribe(
            arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]],
            self.on_message)

        self.fs_access = CollectionFsAccess(input_basedir)

        kwargs["fs_access"] = self.fs_access
        kwargs["enable_reuse"] = args.enable_reuse

        if kwargs.get("conformance_test"):
            return cwltool.main.single_job_executor(tool, job_order,
                                                    input_basedir, args,
                                                    **kwargs)
        else:
            jobiter = tool.job(job_order, input_basedir, self.output_callback,
                               **kwargs)

            for runnable in jobiter:
                if runnable:
                    with self.lock:
                        runnable.run(**kwargs)
                else:
                    if self.jobs:
                        try:
                            self.cond.acquire()
                            self.cond.wait()
                        finally:
                            self.cond.release()
                    else:
                        logger.error("Workflow cannot make any more progress.")
                        break

            while self.jobs:
                try:
                    self.cond.acquire()
                    self.cond.wait()
                finally:
                    self.cond.release()

            events.close()

            if self.final_output is None:
                raise cwltool.workflow.WorkflowException(
                    "Workflow did not return a result.")

            return self.final_output
예제 #2
0
파일: __init__.py 프로젝트: Kunde21/arvados
    def arvExecutor(self, tool, job_order, input_basedir, args, **kwargs):
        events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)

        self.fs_access = CollectionFsAccess(input_basedir)

        kwargs["fs_access"] = self.fs_access
        kwargs["enable_reuse"] = args.enable_reuse
        kwargs["repository"] = args.repository

        if kwargs.get("conformance_test"):
            return cwltool.main.single_job_executor(tool, job_order, input_basedir, args, **kwargs)
        else:
            jobiter = tool.job(job_order,
                            input_basedir,
                            self.output_callback,
                            **kwargs)

            for runnable in jobiter:
                if runnable:
                    with self.lock:
                        runnable.run(**kwargs)
                else:
                    if self.jobs:
                        try:
                            self.cond.acquire()
                            self.cond.wait()
                        finally:
                            self.cond.release()
                    else:
                        logger.error("Workflow cannot make any more progress.")
                        break

            while self.jobs:
                try:
                    self.cond.acquire()
                    self.cond.wait()
                finally:
                    self.cond.release()

            events.close()

            if self.final_output is None:
                raise cwltool.workflow.WorkflowException("Workflow did not return a result.")

            return self.final_output
예제 #3
0
파일: __init__.py 프로젝트: xiaojay/arvados
    def arvExecutor(self, tool, job_order, input_basedir, args, **kwargs):
        events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)

        try:
            self.api.collections().get(uuid=crunchrunner_pdh).execute()
        except arvados.errors.ApiError as e:
            import httplib2
            h = httplib2.Http(ca_certs=arvados.util.ca_certs_path())
            resp, content = h.request(crunchrunner_download, "GET")
            resp2, content2 = h.request(certs_download, "GET")
            with arvados.collection.Collection() as col:
                with col.open("crunchrunner", "w") as f:
                    f.write(content)
                with col.open("ca-certificates.crt", "w") as f:
                    f.write(content2)

                col.save_new("crunchrunner binary", ensure_unique_name=True)

        self.fs_access = CollectionFsAccess(input_basedir)

        kwargs["fs_access"] = self.fs_access
        kwargs["enable_reuse"] = args.enable_reuse

        kwargs["outdir"] = "$(task.outdir)"
        kwargs["tmpdir"] = "$(task.tmpdir)"

        useruuid = self.api.users().current().execute()["uuid"]
        self.project_uuid = args.project_uuid if args.project_uuid else useruuid

        if kwargs.get("conformance_test"):
            return cwltool.main.single_job_executor(tool, job_order, input_basedir, args, **kwargs)
        else:
            self.pipeline = self.api.pipeline_instances().create(
                body={
                    "owner_uuid": self.project_uuid,
                    "name": shortname(tool.tool["id"]),
                    "components": {},
                    "state": "RunningOnClient"}).execute(num_retries=self.num_retries)

            logger.info("Pipeline instance %s", self.pipeline["uuid"])

            jobiter = tool.job(job_order,
                               input_basedir,
                               self.output_callback,
                               docker_outdir="$(task.outdir)",
                               **kwargs)

            try:
                self.cond.acquire()
                # Will continue to hold the lock for the duration of this code
                # except when in cond.wait(), at which point on_message can update
                # job state and process output callbacks.

                for runnable in jobiter:
                    if runnable:
                        runnable.run(**kwargs)
                    else:
                        if self.jobs:
                            self.cond.wait(1)
                        else:
                            logger.error("Workflow is deadlocked, no runnable jobs and not waiting on any pending jobs.")
                            break

                while self.jobs:
                    self.cond.wait(1)

                events.close()

                if self.final_output is None:
                    raise cwltool.workflow.WorkflowException("Workflow did not return a result.")

                # create final output collection
            except:
                if sys.exc_info()[0] is KeyboardInterrupt:
                    logger.error("Interrupted, marking pipeline as failed")
                else:
                    logger.exception("Caught unhandled exception, marking pipeline as failed")
                self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
                                                     body={"state": "Failed"}).execute(num_retries=self.num_retries)
            finally:
                self.cond.release()

            return self.final_output
예제 #4
0
    def arvExecutor(self, tool, job_order, **kwargs):
        self.debug = kwargs.get("debug")

        if kwargs.get("quiet"):
            logger.setLevel(logging.WARN)
            logging.getLogger('arvados.arv-run').setLevel(logging.WARN)

        useruuid = self.api.users().current().execute()["uuid"]
        self.project_uuid = kwargs.get("project_uuid") if kwargs.get(
            "project_uuid") else useruuid
        self.pipeline = None

        if kwargs.get("create_template"):
            tmpl = RunnerTemplate(self, tool, job_order,
                                  kwargs.get("enable_reuse"))
            tmpl.save()
            # cwltool.main will write our return value to stdout.
            return tmpl.uuid

        if kwargs.get("submit"):
            runnerjob = RunnerJob(self, tool, job_order,
                                  kwargs.get("enable_reuse"))

        if not kwargs.get("submit") and "cwl_runner_job" not in kwargs:
            # Create pipeline for local run
            self.pipeline = self.api.pipeline_instances().create(
                body={
                    "owner_uuid": self.project_uuid,
                    "name": shortname(tool.tool["id"]),
                    "components": {},
                    "state": "RunningOnClient"
                }).execute(num_retries=self.num_retries)
            logger.info("Pipeline instance %s", self.pipeline["uuid"])

        if kwargs.get("submit") and not kwargs.get("wait"):
            runnerjob.run()
            return runnerjob.uuid

        events = arvados.events.subscribe(
            arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]],
            self.on_message)

        self.debug = kwargs.get("debug")
        self.ignore_docker_for_reuse = kwargs.get("ignore_docker_for_reuse")
        self.fs_access = CollectionFsAccess(kwargs["basedir"])

        kwargs["fs_access"] = self.fs_access
        kwargs["enable_reuse"] = kwargs.get("enable_reuse")

        kwargs["outdir"] = "$(task.outdir)"
        kwargs["tmpdir"] = "$(task.tmpdir)"

        if kwargs.get("conformance_test"):
            return cwltool.main.single_job_executor(tool, job_order, **kwargs)
        else:
            if kwargs.get("submit"):
                jobiter = iter((runnerjob, ))
            else:
                if "cwl_runner_job" in kwargs:
                    self.uuid = kwargs.get("cwl_runner_job").get('uuid')
                jobiter = tool.job(job_order,
                                   self.output_callback,
                                   docker_outdir="$(task.outdir)",
                                   **kwargs)

            try:
                self.cond.acquire()
                # Will continue to hold the lock for the duration of this code
                # except when in cond.wait(), at which point on_message can update
                # job state and process output callbacks.

                for runnable in jobiter:
                    if runnable:
                        runnable.run(**kwargs)
                    else:
                        if self.jobs:
                            self.cond.wait(1)
                        else:
                            logger.error(
                                "Workflow is deadlocked, no runnable jobs and not waiting on any pending jobs."
                            )
                            break

                while self.jobs:
                    self.cond.wait(1)

                events.close()
            except:
                if sys.exc_info()[0] is KeyboardInterrupt:
                    logger.error("Interrupted, marking pipeline as failed")
                else:
                    logger.error(
                        "Caught unhandled exception, marking pipeline as failed.  Error was: %s",
                        sys.exc_info()[0],
                        exc_info=(sys.exc_info()[1] if self.debug else False))
                if self.pipeline:
                    self.api.pipeline_instances().update(
                        uuid=self.pipeline["uuid"], body={
                            "state": "Failed"
                        }).execute(num_retries=self.num_retries)
            finally:
                self.cond.release()

            if self.final_output is None:
                raise cwltool.workflow.WorkflowException(
                    "Workflow did not return a result.")

            return self.final_output
예제 #5
0
    def arvExecutor(self, tool, job_order, **kwargs):
        self.debug = kwargs.get("debug")

        if kwargs.get("quiet"):
            logger.setLevel(logging.WARN)
            logging.getLogger('arvados.arv-run').setLevel(logging.WARN)

        useruuid = self.api.users().current().execute()["uuid"]
        self.project_uuid = kwargs.get("project_uuid") if kwargs.get("project_uuid") else useruuid
        self.pipeline = None
        self.fs_access = CollectionFsAccess(kwargs["basedir"], api_client=self.api)

        if kwargs.get("create_template"):
            tmpl = RunnerTemplate(self, tool, job_order, kwargs.get("enable_reuse"))
            tmpl.save()
            # cwltool.main will write our return value to stdout.
            return tmpl.uuid

        self.debug = kwargs.get("debug")
        self.ignore_docker_for_reuse = kwargs.get("ignore_docker_for_reuse")

        kwargs["fs_access"] = self.fs_access
        kwargs["enable_reuse"] = kwargs.get("enable_reuse")
        kwargs["use_container"] = True
        kwargs["tmpdir_prefix"] = "tmp"
        kwargs["on_error"] = "continue"
        kwargs["compute_checksum"] = kwargs.get("compute_checksum")

        if self.work_api == "containers":
            kwargs["outdir"] = "/var/spool/cwl"
            kwargs["docker_outdir"] = "/var/spool/cwl"
            kwargs["tmpdir"] = "/tmp"
        elif self.work_api == "jobs":
            kwargs["outdir"] = "$(task.outdir)"
            kwargs["docker_outdir"] = "$(task.outdir)"
            kwargs["tmpdir"] = "$(task.tmpdir)"

        runnerjob = None
        if kwargs.get("submit"):
            if self.work_api == "containers":
                if tool.tool["class"] == "CommandLineTool":
                    runnerjob = tool.job(job_order,
                                         self.output_callback,
                                         **kwargs).next()
                else:
                    runnerjob = RunnerContainer(self, tool, job_order, kwargs.get("enable_reuse"))
            else:
                runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"))

        if not kwargs.get("submit") and "cwl_runner_job" not in kwargs and not self.work_api == "containers":
            # Create pipeline for local run
            self.pipeline = self.api.pipeline_instances().create(
                body={
                    "owner_uuid": self.project_uuid,
                    "name": shortname(tool.tool["id"]),
                    "components": {},
                    "state": "RunningOnClient"}).execute(num_retries=self.num_retries)
            logger.info("Pipeline instance %s", self.pipeline["uuid"])

        if runnerjob and not kwargs.get("wait"):
            runnerjob.run()
            return runnerjob.uuid

        arvados.config.settings()["ARVADOS_DISABLE_WEBSOCKETS"] = "1"

        if self.work_api == "containers":
            events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#container"]], self.on_message)
        if self.work_api == "jobs":
            events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)

        if runnerjob:
            jobiter = iter((runnerjob,))
        else:
            if "cwl_runner_job" in kwargs:
                self.uuid = kwargs.get("cwl_runner_job").get('uuid')
            jobiter = tool.job(job_order,
                               self.output_callback,
                               **kwargs)

        try:
            self.cond.acquire()
            # Will continue to hold the lock for the duration of this code
            # except when in cond.wait(), at which point on_message can update
            # job state and process output callbacks.

            for runnable in jobiter:
                if runnable:
                    runnable.run(**kwargs)
                else:
                    if self.processes:
                        self.cond.wait(1)
                    else:
                        logger.error("Workflow is deadlocked, no runnable jobs and not waiting on any pending jobs.")
                        break

            while self.processes:
                self.cond.wait(1)

            events.close()
        except UnsupportedRequirement:
            raise
        except:
            if sys.exc_info()[0] is KeyboardInterrupt:
                logger.error("Interrupted, marking pipeline as failed")
            else:
                logger.error("Caught unhandled exception, marking pipeline as failed.  Error was: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
            if self.pipeline:
                self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
                                                     body={"state": "Failed"}).execute(num_retries=self.num_retries)
            if runnerjob and runnerjob.uuid and self.work_api == "containers":
                self.api.container_requests().update(uuid=runnerjob.uuid,
                                                     body={"priority": "0"}).execute(num_retries=self.num_retries)
        finally:
            self.cond.release()

        if self.final_status == "UnsupportedRequirement":
            raise UnsupportedRequirement("Check log for details.")

        if self.final_status != "success":
            raise WorkflowException("Workflow failed.")

        if self.final_output is None:
            raise WorkflowException("Workflow did not return a result.")

        if kwargs.get("compute_checksum"):
            def compute_checksums(fileobj):
                if "checksum" not in fileobj:
                    checksum = hashlib.sha1()
                    with self.fs_access.open(fileobj["location"], "rb") as f:
                        contents = f.read(1024*1024)
                        while contents != "":
                            checksum.update(contents)
                            contents = f.read(1024*1024)
                    fileobj["checksum"] = "sha1$%s" % checksum.hexdigest()

            adjustFileObjs(self.final_output, compute_checksums)

        return self.final_output
예제 #6
0
    def arvExecutor(self, tool, job_order, **kwargs):
        self.debug = kwargs.get("debug")

        if kwargs.get("quiet"):
            logger.setLevel(logging.WARN)
            logging.getLogger('arvados.arv-run').setLevel(logging.WARN)

        useruuid = self.api.users().current().execute()["uuid"]
        self.project_uuid = kwargs.get("project_uuid") if kwargs.get("project_uuid") else useruuid
        self.pipeline = None

        if kwargs.get("create_template"):
            tmpl = RunnerTemplate(self, tool, job_order, kwargs.get("enable_reuse"))
            tmpl.save()
            # cwltool.main will write our return value to stdout.
            return tmpl.uuid

        if kwargs.get("submit"):
            runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"))
            if not kwargs.get("wait"):
                runnerjob.run()
                return runnerjob.uuid

        events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)

        self.debug = kwargs.get("debug")
        self.ignore_docker_for_reuse = kwargs.get("ignore_docker_for_reuse")
        self.fs_access = CollectionFsAccess(kwargs["basedir"])

        kwargs["fs_access"] = self.fs_access
        kwargs["enable_reuse"] = kwargs.get("enable_reuse")

        kwargs["outdir"] = "$(task.outdir)"
        kwargs["tmpdir"] = "$(task.tmpdir)"

        if kwargs.get("conformance_test"):
            return cwltool.main.single_job_executor(tool, job_order, **kwargs)
        else:
            if kwargs.get("submit"):
                jobiter = iter((runnerjob,))
            else:
                components = {}
                if "cwl_runner_job" in kwargs:
                    components[os.path.basename(tool.tool["id"])] = {"job": kwargs["cwl_runner_job"]}

                self.pipeline = self.api.pipeline_instances().create(
                    body={
                        "owner_uuid": self.project_uuid,
                        "name": shortname(tool.tool["id"]),
                        "components": components,
                        "state": "RunningOnClient"}).execute(num_retries=self.num_retries)

                logger.info("Pipeline instance %s", self.pipeline["uuid"])

                jobiter = tool.job(job_order,
                                   self.output_callback,
                                   docker_outdir="$(task.outdir)",
                                   **kwargs)

            try:
                self.cond.acquire()
                # Will continue to hold the lock for the duration of this code
                # except when in cond.wait(), at which point on_message can update
                # job state and process output callbacks.

                for runnable in jobiter:
                    if runnable:
                        runnable.run(**kwargs)
                    else:
                        if self.jobs:
                            self.cond.wait(1)
                        else:
                            logger.error("Workflow is deadlocked, no runnable jobs and not waiting on any pending jobs.")
                            break

                while self.jobs:
                    self.cond.wait(1)

                events.close()
            except:
                if sys.exc_info()[0] is KeyboardInterrupt:
                    logger.error("Interrupted, marking pipeline as failed")
                else:
                    logger.error("Caught unhandled exception, marking pipeline as failed.  Error was: %s", sys.exc_info()[0], exc_info=(sys.exc_info()[1] if self.debug else False))
                if self.pipeline:
                    self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
                                                         body={"state": "Failed"}).execute(num_retries=self.num_retries)
            finally:
                self.cond.release()

            if self.final_output is None:
                raise cwltool.workflow.WorkflowException("Workflow did not return a result.")

            return self.final_output
예제 #7
0
파일: __init__.py 프로젝트: 2lenet/arvados
    def arvExecutor(self, tool, job_order, input_basedir, args, **kwargs):
        events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)

        try:
            self.api.collections().get(uuid=crunchrunner_pdh).execute()
        except arvados.errors.ApiError as e:
            import httplib2
            h = httplib2.Http(ca_certs=arvados.util.ca_certs_path())
            resp, content = h.request(crunchrunner_download, "GET")
            resp2, content2 = h.request(certs_download, "GET")
            with arvados.collection.Collection() as col:
                with col.open("crunchrunner", "w") as f:
                    f.write(content)
                with col.open("ca-certificates.crt", "w") as f:
                    f.write(content2)

                col.save_new("crunchrunner binary", ensure_unique_name=True)

        self.fs_access = CollectionFsAccess(input_basedir)

        kwargs["fs_access"] = self.fs_access
        kwargs["enable_reuse"] = args.enable_reuse

        kwargs["outdir"] = "$(task.outdir)"
        kwargs["tmpdir"] = "$(task.tmpdir)"

        if kwargs.get("conformance_test"):
            return cwltool.main.single_job_executor(tool, job_order, input_basedir, args, **kwargs)
        else:
            self.pipeline = self.api.pipeline_instances().create(body={"name": shortname(tool.tool["id"]),
                                                                   "components": {},
                                                                   "state": "RunningOnClient"}).execute(num_retries=self.num_retries)

            jobiter = tool.job(job_order,
                               input_basedir,
                               self.output_callback,
                               docker_outdir="$(task.outdir)",
                               **kwargs)

            try:
                for runnable in jobiter:
                    if runnable:
                        with self.lock:
                            runnable.run(**kwargs)
                    else:
                        if self.jobs:
                            try:
                                self.cond.acquire()
                                self.cond.wait(1)
                            except RuntimeError:
                                pass
                            finally:
                                self.cond.release()
                        else:
                            logger.error("Workflow cannot make any more progress.")
                            break

                while self.jobs:
                    try:
                        self.cond.acquire()
                        self.cond.wait(1)
                    except RuntimeError:
                        pass
                    finally:
                        self.cond.release()

                events.close()

                if self.final_output is None:
                    raise cwltool.workflow.WorkflowException("Workflow did not return a result.")

            except:
                if sys.exc_info()[0] is not KeyboardInterrupt:
                    logger.exception("Caught unhandled exception, marking pipeline as failed")
                self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
                                                     body={"state": "Failed"}).execute(num_retries=self.num_retries)

            return self.final_output