コード例 #1
0
ファイル: cwltoil.py プロジェクト: HPCBio/toil
    def run(self, fileStore):
        cwljob = resolve_indirect(self.cwljob)
        fillInDefaults(self.cwltool.tool["inputs"], cwljob)

        inpdir = os.path.join(fileStore.getLocalTempDir(), "inp")
        outdir = os.path.join(fileStore.getLocalTempDir(), "out")
        tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp")
        os.mkdir(inpdir)
        os.mkdir(outdir)
        os.mkdir(tmpdir)

        # Copy input files out of the global file store.
        index={}
        adjustFilesWithSecondary(cwljob, functools.partial(getFile, fileStore, inpdir, index=index))

        # Run the tool
        opts = copy.deepcopy(self.executor_options)
        # Exports temporary directory for batch systems that reset TMPDIR
        os.environ["TMPDIR"] = os.path.realpath(opts.pop("tmpdir", None) or tmpdir)
        output = cwltool.main.single_job_executor(self.cwltool, cwljob,
                                                  basedir=os.getcwd(),
                                                  outdir=outdir,
                                                  tmpdir=tmpdir,
                                                  tmpdir_prefix="tmp",
                                                  **opts)
        cwltool.builder.adjustDirObjs(output, locToPath)
        cwltool.builder.adjustFileObjs(output, locToPath)
        # Copy output files into the global file store.
        adjustFiles(output, functools.partial(writeFile, fileStore.writeGlobalFile, {}))

        return output
コード例 #2
0
ファイル: cwltoil.py プロジェクト: anukat2015/toil
    def run(self, fileStore):
        cwljob = resolve_indirect(self.cwljob)
        fillInDefaults(self.cwltool.tool["inputs"], cwljob)

        inpdir = os.path.join(fileStore.getLocalTempDir(), "inp")
        outdir = os.path.join(fileStore.getLocalTempDir(), "out")
        tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp")
        os.mkdir(inpdir)
        os.mkdir(outdir)
        os.mkdir(tmpdir)

        # Copy input files out of the global file store.
        index={}
        adjustFilesWithSecondary(cwljob, functools.partial(getFile, fileStore, inpdir, index=index))

        # Run the tool
        output = cwltool.main.single_job_executor(self.cwltool, cwljob,
                                                  os.getcwd(), None,
                                                  outdir=outdir,
                                                  tmpdir=tmpdir,
                                                  **self.executor_options)

        # Copy output files into the global file store.
        adjustFiles(output, functools.partial(writeFile, fileStore.writeGlobalFile, {}))

        return output
コード例 #3
0
ファイル: cwltoil.py プロジェクト: brainstorm/toil
    def run(self, fileStore):
        cwljob = resolve_indirect(self.cwljob)
        fillInDefaults(self.cwltool.tool["inputs"], cwljob)

        inpdir = os.path.join(fileStore.getLocalTempDir(), "inp")
        outdir = os.path.join(fileStore.getLocalTempDir(), "out")
        tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp")
        os.mkdir(inpdir)
        os.mkdir(outdir)
        os.mkdir(tmpdir)

        # Copy input files out of the global file store, ensure path/location synchronized
        index = {}
        existing = {}
        adjustFilesWithSecondary(cwljob, functools.partial(getFile, fileStore, inpdir, index=index,
                                                           existing=existing))
        cwltool.pathmapper.adjustFileObjs(cwljob, pathToLoc)
        cwltool.pathmapper.adjustFileObjs(cwljob, addFilePartRefs)

        # Run the tool
        opts = copy.deepcopy(self.executor_options)
        # Exports temporary directory for batch systems that reset TMPDIR
        os.environ["TMPDIR"] = os.path.realpath(opts.pop("tmpdir", None) or tmpdir)
        (output, status) = cwltool.main.single_job_executor(self.cwltool, cwljob,
                                                            basedir=os.getcwd(),
                                                            outdir=outdir,
                                                            tmpdir=tmpdir,
                                                            tmpdir_prefix="tmp",
                                                            make_fs_access=cwltool.stdfsaccess.StdFsAccess,
                                                            **opts)
        if status != "success":
            raise cwltool.errors.WorkflowException(status)
        cwltool.pathmapper.adjustDirObjs(output, locToPath)
        cwltool.pathmapper.adjustFileObjs(output, locToPath)
        cwltool.pathmapper.adjustFileObjs(output, functools.partial(computeFileChecksums,
                                                                    cwltool.stdfsaccess.StdFsAccess(outdir)))
        # Copy output files into the global file store.
        adjustFiles(output, functools.partial(writeFile, fileStore.writeGlobalFile, {}, existing))
        return output
コード例 #4
0
ファイル: cwltoil.py プロジェクト: brainstorm/toil
def main(args=None, stdout=sys.stdout):
    parser = ArgumentParser()
    Job.Runner.addToilOptions(parser)
    parser.add_argument("cwltool", type=str)
    parser.add_argument("cwljob", type=str, nargs="?", default=None)

    # Will override the "jobStore" positional argument, enables
    # user to select jobStore or get a default from logic one below.
    parser.add_argument("--jobStore", type=str)
    parser.add_argument("--conformance-test", action="store_true")
    parser.add_argument("--not-strict", action="store_true")
    parser.add_argument("--no-container", action="store_true")
    parser.add_argument("--quiet", dest="logLevel", action="store_const", const="ERROR")
    parser.add_argument("--basedir", type=str)
    parser.add_argument("--outdir", type=str, default=os.getcwd())
    parser.add_argument("--version", action='version', version=baseVersion)
    parser.add_argument("--preserve-environment", type=str, nargs='+',
                    help="Preserve specified environment variables when running CommandLineTools",
                    metavar=("VAR1 VAR2"),
                    default=("PATH",),
                    dest="preserve_environment")

    # mkdtemp actually creates the directory, but
    # toil requires that the directory not exist,
    # so make it and delete it and allow
    # toil to create it again (!)
    workdir = tempfile.mkdtemp()
    os.rmdir(workdir)

    if args is None:
        args = sys.argv[1:]

    options = parser.parse_args([workdir] + args)

    use_container = not options.no_container

    setLoggingFromOptions(options)
    if options.logLevel:
        cwllogger.setLevel(options.logLevel)

    useStrict = not options.not_strict
    try:
        t = cwltool.load_tool.load_tool(options.cwltool, cwltool.workflow.defaultMakeTool,
                                        resolver=cwltool.resolver.tool_resolver, strict=useStrict)
        unsupportedRequirementsCheck(t.requirements)
    except cwltool.process.UnsupportedRequirement as e:
        logging.error(e)
        return 33

    if options.conformance_test:
        loader = schema_salad.ref_resolver.Loader({})
    else:
        jobloaderctx = {"path": {"@type": "@id"}, "format": {"@type": "@id"}}
        jobloaderctx.update(t.metadata.get("$namespaces", {}))
        loader = schema_salad.ref_resolver.Loader(jobloaderctx)

    if options.cwljob:
        uri = (options.cwljob if urlparse.urlparse(options.cwljob).scheme
               else "file://" + os.path.abspath(options.cwljob))
        job, _ = loader.resolve_ref(uri, checklinks=False)
    else:
        job = {}

    try:
        cwltool.pathmapper.adjustDirObjs(job, unsupportedInputCheck)
        cwltool.pathmapper.adjustFileObjs(job, unsupportedInputCheck)
    except cwltool.process.UnsupportedRequirement as e:
        logging.error(e)
        return 33

    cwltool.pathmapper.adjustDirObjs(job, pathToLoc)
    cwltool.pathmapper.adjustFileObjs(job, pathToLoc)

    if type(t) == int:
        return t

    fillInDefaults(t.tool["inputs"], job)

    if options.conformance_test:
        adjustFiles(job, lambda x: x.replace("file://", ""))
        stdout.write(json.dumps(
            cwltool.main.single_job_executor(t, job, basedir=options.basedir,
                                             tmpdir_prefix="tmp",
                                             conformance_test=True, use_container=use_container,
                                             preserve_environment=options.preserve_environment), indent=4))
        return 0

    if not options.basedir:
        options.basedir = os.path.dirname(os.path.abspath(options.cwljob or options.cwltool))

    outdir = options.outdir

    with Toil(options) as toil:
        def importDefault(tool):
            cwltool.pathmapper.adjustDirObjs(tool, locToPath)
            cwltool.pathmapper.adjustFileObjs(tool, locToPath)
            adjustFiles(tool, lambda x: "file://%s" % x if not urlparse.urlparse(x).scheme else x)
            adjustFiles(tool, functools.partial(writeFile, toil.importFile, {}, {}))
        t.visit(importDefault)

        if options.restart:
            outobj = toil.restart()
        else:
            basedir = os.path.dirname(os.path.abspath(options.cwljob or options.cwltool))
            builder = t._init_job(job, basedir=basedir, use_container=use_container)
            (wf1, wf2) = makeJob(t, {}, use_container=use_container,
                    preserve_environment=options.preserve_environment,
                    tmpdir=os.path.realpath(outdir), builder=builder)
            try:
                if isinstance(wf1, CWLWorkflow):
                    [unsupportedDefaultCheck(s.tool) for s in wf1.cwlwf.steps]
            except cwltool.process.UnsupportedRequirement as e:
                logging.error(e)
                return 33

            cwltool.pathmapper.adjustDirObjs(builder.job, locToPath)
            cwltool.pathmapper.adjustFileObjs(builder.job, locToPath)
            adjustFiles(builder.job, lambda x: "file://%s" % os.path.abspath(os.path.join(basedir, x))
                        if not urlparse.urlparse(x).scheme else x)
            cwltool.pathmapper.adjustDirObjs(builder.job, pathToLoc)
            cwltool.pathmapper.adjustFileObjs(builder.job, pathToLoc)
            cwltool.pathmapper.adjustFileObjs(builder.job, addFilePartRefs)
            adjustFiles(builder.job, functools.partial(writeFile, toil.importFile, {}, {}))
            wf1.cwljob = builder.job
            outobj = toil.start(wf1)

        outobj = resolve_indirect(outobj)

        try:
            adjustFilesWithSecondary(outobj, functools.partial(getFile, toil, outdir, index={}, existing={},
                                                               export=True, rename_collision=True))
            cwltool.pathmapper.adjustFileObjs(outobj, pathToLoc)
        except cwltool.process.UnsupportedRequirement as e:
            logging.error(e)
            return 33

        stdout.write(json.dumps(outobj, indent=4))

    return 0