def resume(self): with self.conn.cursor() as cur: cur.execute("select * from job_store where namespace=%s", (self.namespace,)) if not bool(cur.rowcount): raise NoSuchJobStoreException(self.url) cur.execute("select * from file_store where namespace=%s", (self.namespace,)) if not bool(cur.rowcount): raise NoSuchJobStoreException(self.url) super(JobStore, self).resume() self.fileStore.resume() self.conn.commit()
def resume(self): try: self.files = self.uri.get_bucket(headers=self.headerValues, validate=True) except GSResponseError: raise NoSuchJobStoreException(self.locator) super(GoogleJobStore, self).resume()
def initialize(self, config=None): exists = True try: self.uri.get_bucket(headers=self.headerValues, validate=True) except GSResponseError: exists = False if exists: raise JobStoreExistsException(self.locator) from google.cloud import storage storage_client = storage.Client() bucket = storage_client.create_bucket(self.bucketName) #self.files = self._retryCreateBucket(self.uri, self.headerValues) try: self.files = self.uri.get_bucket(headers=self.headerValues, validate=True) except GSResponseError: raise NoSuchJobStoreException(self.locator) # functionally equivalent to dictionary1.update(dictionary2) but works with our immutable dicts self.encryptedHeaders = dict(self.encryptedHeaders, **self._resolveEncryptionHeaders(config)) super(GoogleJobStore, self).initialize(config)
def resume(self): if not self._jobStoreExists(): raise NoSuchJobStoreException(self.locator) logger.debug("Using existing job store at '%s'" % self.locator) self._bind(create=False) super(AzureJobStore, self).resume()
def resume(self): if not os.path.exists(self.jobStoreDir): raise NoSuchJobStoreException(self.jobStoreDir) require(os.path.isdir, "'%s' is not a directory", self.jobStoreDir) super(FileJobStore, self).resume()
def resume(self): if not os.path.isdir(self.jobStoreDir): raise NoSuchJobStoreException(self.jobStoreDir) super(FileJobStore, self).resume()
def resume(self): try: self.bucket = self.storageClient.get_bucket(self.bucketName) except exceptions.NotFound: raise NoSuchJobStoreException(self.locator) super(GoogleJobStore, self).resume()
def main(args=None, stdout=sys.stdout): """Main method for toil-cwl-runner.""" cwllogger.removeHandler(defaultStreamHandler) config = Config() config.cwl = True parser = argparse.ArgumentParser() addOptions(parser, config) parser.add_argument("cwltool", type=str) parser.add_argument("cwljob", nargs=argparse.REMAINDER) # Will override the "jobStore" positional argument, enables # user to select jobStore or get a default from logic one below. parser.add_argument("--jobStore", type=str) parser.add_argument("--not-strict", action="store_true") parser.add_argument("--quiet", dest="logLevel", action="store_const", const="ERROR") parser.add_argument("--basedir", type=str) parser.add_argument("--outdir", type=str, default=os.getcwd()) parser.add_argument("--version", action='version', version=baseVersion) dockergroup = parser.add_mutually_exclusive_group() dockergroup.add_argument( "--user-space-docker-cmd", help="(Linux/OS X only) Specify a user space docker command (like " "udocker or dx-docker) that will be used to call 'pull' and 'run'") dockergroup.add_argument( "--singularity", action="store_true", default=False, help="[experimental] Use Singularity runtime for running containers. " "Requires Singularity v2.3.2+ and Linux with kernel version v3.18+ or " "with overlayfs support backported.") dockergroup.add_argument( "--no-container", action="store_true", help="Do not execute jobs in a " "Docker container, even when `DockerRequirement` " "is specified under `hints`.") parser.add_argument( "--preserve-environment", type=str, nargs='+', help="Preserve specified environment variables when running" " CommandLineTools", metavar=("VAR1 VAR2"), default=("PATH",), dest="preserve_environment") parser.add_argument( "--destBucket", type=str, help="Specify a cloud bucket endpoint for output files.") parser.add_argument( "--beta-dependency-resolvers-configuration", default=None) parser.add_argument("--beta-dependencies-directory", default=None) parser.add_argument( "--beta-use-biocontainers", default=None, action="store_true") parser.add_argument( "--beta-conda-dependencies", default=None, action="store_true") parser.add_argument("--tmpdir-prefix", type=Text, help="Path prefix for temporary directories", default="tmp") parser.add_argument("--tmp-outdir-prefix", type=Text, help="Path prefix for intermediate output directories", default="tmp") parser.add_argument( "--force-docker-pull", action="store_true", default=False, dest="force_docker_pull", help="Pull latest docker image even if it is locally present") parser.add_argument( "--no-match-user", action="store_true", default=False, help="Disable passing the current uid to `docker run --user`") # mkdtemp actually creates the directory, but # toil requires that the directory not exist, # so make it and delete it and allow # toil to create it again (!) workdir = tempfile.mkdtemp() os.rmdir(workdir) if args is None: args = sys.argv[1:] # we use workdir as jobStore: options = parser.parse_args([workdir] + args) # if tmpdir_prefix is not the default value, set workDir too if options.tmpdir_prefix != 'tmp': options.workDir = options.tmpdir_prefix if options.provisioner and not options.jobStore: raise NoSuchJobStoreException( 'Please specify a jobstore with the --jobStore option when specifying a provisioner.') use_container = not options.no_container if options.logLevel: cwllogger.setLevel(options.logLevel) outdir = os.path.abspath(options.outdir) tmp_outdir_prefix = os.path.abspath(options.tmp_outdir_prefix) tmpdir_prefix = os.path.abspath(options.tmpdir_prefix) fileindex = {} existing = {} conf_file = getattr(options, "beta_dependency_resolvers_configuration", None) use_conda_dependencies = getattr(options, "beta_conda_dependencies", None) job_script_provider = None if conf_file or use_conda_dependencies: dependencies_configuration = DependenciesConfiguration(options) job_script_provider = dependencies_configuration options.default_container = None runtime_context = cwltool.context.RuntimeContext(vars(options)) runtime_context.find_default_container = functools.partial( find_default_container, options) runtime_context.workdir = workdir runtime_context.move_outputs = "leave" runtime_context.rm_tmpdir = False loading_context = cwltool.context.LoadingContext(vars(options)) with Toil(options) as toil: if options.restart: outobj = toil.restart() else: loading_context.hints = [{ "class": "ResourceRequirement", "coresMin": toil.config.defaultCores, "ramMin": toil.config.defaultMemory / (2**20), "outdirMin": toil.config.defaultDisk / (2**20), "tmpdirMin": 0 }] loading_context.construct_tool_object = toil_make_tool loading_context.resolver = cwltool.resolver.tool_resolver loading_context.strict = not options.not_strict options.workflow = options.cwltool options.job_order = options.cwljob uri, tool_file_uri = cwltool.load_tool.resolve_tool_uri( options.cwltool, loading_context.resolver, loading_context.fetcher_constructor) options.tool_help = None options.debug = options.logLevel == "DEBUG" job_order_object, options.basedir, jobloader = \ cwltool.main.load_job_order( options, sys.stdin, loading_context.fetcher_constructor, loading_context.overrides_list, tool_file_uri) document_loader, workflowobj, uri = \ cwltool.load_tool.fetch_document( uri, loading_context.resolver, loading_context.fetcher_constructor) document_loader, avsc_names, processobj, metadata, uri = \ cwltool.load_tool.validate_document( document_loader, workflowobj, uri, loading_context.enable_dev, loading_context.strict, False, loading_context.fetcher_constructor, False, loading_context.overrides_list, do_validate=loading_context.do_validate) loading_context.overrides_list.extend( metadata.get("cwltool:overrides", [])) try: tool = cwltool.load_tool.make_tool( document_loader, avsc_names, metadata, uri, loading_context) except cwltool.process.UnsupportedRequirement as err: logging.error(err) return 33 runtime_context.secret_store = SecretStore() initialized_job_order = cwltool.main.init_job_order( job_order_object, options, tool, jobloader, sys.stdout, secret_store=runtime_context.secret_store) fs_access = cwltool.stdfsaccess.StdFsAccess(options.basedir) fill_in_defaults( tool.tool["inputs"], initialized_job_order, fs_access) def path_to_loc(obj): if "location" not in obj and "path" in obj: obj["location"] = obj["path"] del obj["path"] def import_files(tool): visit_class(tool, ("File", "Directory"), path_to_loc) visit_class(tool, ("File", ), functools.partial( add_sizes, fs_access)) normalizeFilesDirs(tool) adjustDirObjs(tool, functools.partial( get_listing, fs_access, recursive=True)) adjustFileObjs(tool, functools.partial( uploadFile, toil.importFile, fileindex, existing, skip_broken=True)) tool.visit(import_files) for inp in tool.tool["inputs"]: def set_secondary(fileobj): if isinstance(fileobj, Mapping) \ and fileobj.get("class") == "File": if "secondaryFiles" not in fileobj: fileobj["secondaryFiles"] = [ {"location": cwltool.builder.substitute( fileobj["location"], sf), "class": "File"} for sf in inp["secondaryFiles"]] if isinstance(fileobj, MutableSequence): for entry in fileobj: set_secondary(entry) if shortname(inp["id"]) in initialized_job_order \ and inp.get("secondaryFiles"): set_secondary(initialized_job_order[shortname(inp["id"])]) import_files(initialized_job_order) visitSteps(tool, import_files) try: runtime_context.use_container = use_container runtime_context.tmpdir = os.path.realpath(tmpdir_prefix) runtime_context.tmp_outdir_prefix = os.path.realpath( tmp_outdir_prefix) runtime_context.job_script_provider = job_script_provider runtime_context.force_docker_pull = options.force_docker_pull runtime_context.no_match_user = options.no_match_user (wf1, _) = makeJob(tool, {}, None, runtime_context) except cwltool.process.UnsupportedRequirement as err: logging.error(err) return 33 wf1.cwljob = initialized_job_order if wf1 is CWLJob: # Clean up temporary directories only created with CWLJobs. wf1.addFollowOnFn(cleanTempDirs, wf1) outobj = toil.start(wf1) outobj = resolve_indirect(outobj) # Stage files. Specify destination bucket if specified in CLI # options. If destination bucket not passed in, # options.destBucket's value will be None. toilStageFiles( toil, outobj, outdir, fileindex, existing, export=True, destBucket=options.destBucket) if not options.destBucket: visit_class(outobj, ("File",), functools.partial( compute_checksums, cwltool.stdfsaccess.StdFsAccess(""))) visit_class(outobj, ("File", ), MutationManager().unset_generation) stdout.write(json.dumps(outobj, indent=4)) return 0