def secrets() -> Tuple[SecretStore, CWLObjectType]:
"""Fixture to return a secret store."""
sec_store = SecretStore()
job: CWLObjectType = {"foo": "bar", "baz": "quux"}
sec_store.store(["foo"], job)
return sec_store, job
def secrets():
sec_store = SecretStore()
job = {'foo': 'bar',
'baz': 'quux'}
sec_store.store(['foo'], job)
return sec_store, job
def secrets():
sec_store = SecretStore()
job = {'foo': 'bar',
'baz': 'quux'}
sec_store.store(['foo'], job)
return sec_store, job
def secrets() -> Tuple[SecretStore, Dict[str, str]]:
sec_store = SecretStore()
job = {"foo": "bar", "baz": "quux"}
sec_store.store(["foo"], job)
return sec_store, job
def main(args=None, stdout=sys.stdout):
"""Main method for toil-cwl-runner."""
cwllogger.removeHandler(defaultStreamHandler)
config = Config()
config.cwl = True
parser = argparse.ArgumentParser()
addOptions(parser, config)
parser.add_argument("cwltool", type=str)
parser.add_argument("cwljob", nargs=argparse.REMAINDER)
# Will override the "jobStore" positional argument, enables
# user to select jobStore or get a default from logic one below.
parser.add_argument("--jobStore", type=str)
parser.add_argument("--not-strict", action="store_true")
parser.add_argument("--quiet", dest="logLevel", action="store_const",
const="ERROR")
parser.add_argument("--basedir", type=str)
parser.add_argument("--outdir", type=str, default=os.getcwd())
parser.add_argument("--version", action='version', version=baseVersion)
dockergroup = parser.add_mutually_exclusive_group()
dockergroup.add_argument(
"--user-space-docker-cmd",
help="(Linux/OS X only) Specify a user space docker command (like "
"udocker or dx-docker) that will be used to call 'pull' and 'run'")
dockergroup.add_argument(
"--singularity", action="store_true", default=False,
help="[experimental] Use Singularity runtime for running containers. "
"Requires Singularity v2.3.2+ and Linux with kernel version v3.18+ or "
"with overlayfs support backported.")
dockergroup.add_argument(
"--no-container", action="store_true", help="Do not execute jobs in a "
"Docker container, even when `DockerRequirement` "
"is specified under `hints`.")
parser.add_argument(
"--preserve-environment", type=str, nargs='+',
help="Preserve specified environment variables when running"
" CommandLineTools", metavar=("VAR1 VAR2"), default=("PATH",),
dest="preserve_environment")
parser.add_argument(
"--destBucket", type=str,
help="Specify a cloud bucket endpoint for output files.")
parser.add_argument(
"--beta-dependency-resolvers-configuration", default=None)
parser.add_argument("--beta-dependencies-directory", default=None)
parser.add_argument(
"--beta-use-biocontainers", default=None, action="store_true")
parser.add_argument(
"--beta-conda-dependencies", default=None, action="store_true")
parser.add_argument("--tmpdir-prefix", type=Text,
help="Path prefix for temporary directories",
default="tmp")
parser.add_argument("--tmp-outdir-prefix", type=Text,
help="Path prefix for intermediate output directories",
default="tmp")
parser.add_argument(
"--force-docker-pull", action="store_true", default=False,
dest="force_docker_pull",
help="Pull latest docker image even if it is locally present")
parser.add_argument(
"--no-match-user", action="store_true", default=False,
help="Disable passing the current uid to `docker run --user`")
# mkdtemp actually creates the directory, but
# toil requires that the directory not exist,
# so make it and delete it and allow
# toil to create it again (!)
workdir = tempfile.mkdtemp()
os.rmdir(workdir)
if args is None:
args = sys.argv[1:]
# we use workdir as jobStore:
options = parser.parse_args([workdir] + args)
# if tmpdir_prefix is not the default value, set workDir too
if options.tmpdir_prefix != 'tmp':
options.workDir = options.tmpdir_prefix
if options.provisioner and not options.jobStore:
raise NoSuchJobStoreException(
'Please specify a jobstore with the --jobStore option when specifying a provisioner.')
use_container = not options.no_container
if options.logLevel:
cwllogger.setLevel(options.logLevel)
outdir = os.path.abspath(options.outdir)
tmp_outdir_prefix = os.path.abspath(options.tmp_outdir_prefix)
tmpdir_prefix = os.path.abspath(options.tmpdir_prefix)
fileindex = {}
existing = {}
conf_file = getattr(options,
"beta_dependency_resolvers_configuration", None)
use_conda_dependencies = getattr(options, "beta_conda_dependencies", None)
job_script_provider = None
if conf_file or use_conda_dependencies:
dependencies_configuration = DependenciesConfiguration(options)
job_script_provider = dependencies_configuration
options.default_container = None
runtime_context = cwltool.context.RuntimeContext(vars(options))
runtime_context.find_default_container = functools.partial(
find_default_container, options)
runtime_context.workdir = workdir
runtime_context.move_outputs = "leave"
runtime_context.rm_tmpdir = False
loading_context = cwltool.context.LoadingContext(vars(options))
with Toil(options) as toil:
if options.restart:
outobj = toil.restart()
else:
loading_context.hints = [{
"class": "ResourceRequirement",
"coresMin": toil.config.defaultCores,
"ramMin": toil.config.defaultMemory / (2**20),
"outdirMin": toil.config.defaultDisk / (2**20),
"tmpdirMin": 0
}]
loading_context.construct_tool_object = toil_make_tool
loading_context.resolver = cwltool.resolver.tool_resolver
loading_context.strict = not options.not_strict
options.workflow = options.cwltool
options.job_order = options.cwljob
uri, tool_file_uri = cwltool.load_tool.resolve_tool_uri(
options.cwltool, loading_context.resolver,
loading_context.fetcher_constructor)
options.tool_help = None
options.debug = options.logLevel == "DEBUG"
job_order_object, options.basedir, jobloader = \
cwltool.main.load_job_order(
options, sys.stdin, loading_context.fetcher_constructor,
loading_context.overrides_list, tool_file_uri)
document_loader, workflowobj, uri = \
cwltool.load_tool.fetch_document(
uri, loading_context.resolver,
loading_context.fetcher_constructor)
document_loader, avsc_names, processobj, metadata, uri = \
cwltool.load_tool.validate_document(
document_loader, workflowobj, uri,
loading_context.enable_dev, loading_context.strict, False,
loading_context.fetcher_constructor, False,
loading_context.overrides_list,
do_validate=loading_context.do_validate)
loading_context.overrides_list.extend(
metadata.get("cwltool:overrides", []))
try:
tool = cwltool.load_tool.make_tool(
document_loader, avsc_names, metadata, uri,
loading_context)
except cwltool.process.UnsupportedRequirement as err:
logging.error(err)
return 33
runtime_context.secret_store = SecretStore()
initialized_job_order = cwltool.main.init_job_order(
job_order_object, options, tool, jobloader, sys.stdout,
secret_store=runtime_context.secret_store)
fs_access = cwltool.stdfsaccess.StdFsAccess(options.basedir)
fill_in_defaults(
tool.tool["inputs"], initialized_job_order, fs_access)
def path_to_loc(obj):
if "location" not in obj and "path" in obj:
obj["location"] = obj["path"]
del obj["path"]
def import_files(tool):
visit_class(tool, ("File", "Directory"), path_to_loc)
visit_class(tool, ("File", ), functools.partial(
add_sizes, fs_access))
normalizeFilesDirs(tool)
adjustDirObjs(tool, functools.partial(
get_listing, fs_access, recursive=True))
adjustFileObjs(tool, functools.partial(
uploadFile, toil.importFile, fileindex, existing,
skip_broken=True))
tool.visit(import_files)
for inp in tool.tool["inputs"]:
def set_secondary(fileobj):
if isinstance(fileobj, Mapping) \
and fileobj.get("class") == "File":
if "secondaryFiles" not in fileobj:
fileobj["secondaryFiles"] = [
{"location": cwltool.builder.substitute(
fileobj["location"], sf), "class": "File"}
for sf in inp["secondaryFiles"]]
if isinstance(fileobj, MutableSequence):
for entry in fileobj:
set_secondary(entry)
if shortname(inp["id"]) in initialized_job_order \
and inp.get("secondaryFiles"):
set_secondary(initialized_job_order[shortname(inp["id"])])
import_files(initialized_job_order)
visitSteps(tool, import_files)
try:
runtime_context.use_container = use_container
runtime_context.tmpdir = os.path.realpath(tmpdir_prefix)
runtime_context.tmp_outdir_prefix = os.path.realpath(
tmp_outdir_prefix)
runtime_context.job_script_provider = job_script_provider
runtime_context.force_docker_pull = options.force_docker_pull
runtime_context.no_match_user = options.no_match_user
(wf1, _) = makeJob(tool, {}, None, runtime_context)
except cwltool.process.UnsupportedRequirement as err:
logging.error(err)
return 33
wf1.cwljob = initialized_job_order
if wf1 is CWLJob: # Clean up temporary directories only created with CWLJobs.
wf1.addFollowOnFn(cleanTempDirs, wf1)
outobj = toil.start(wf1)
outobj = resolve_indirect(outobj)
# Stage files. Specify destination bucket if specified in CLI
# options. If destination bucket not passed in,
# options.destBucket's value will be None.
toilStageFiles(
toil,
outobj,
outdir,
fileindex,
existing,
export=True,
destBucket=options.destBucket)
if not options.destBucket:
visit_class(outobj, ("File",), functools.partial(
compute_checksums, cwltool.stdfsaccess.StdFsAccess("")))
visit_class(outobj, ("File", ), MutationManager().unset_generation)
stdout.write(json.dumps(outobj, indent=4))
return 0
def cwlmain(
self,
argsl=None, # type: List[str]
args=None, # type: argparse.Namespace
job_order_object=None, # type: MutableMapping[Text, Any]
stdin=sys.stdin, # type: IO[Any]
stdout=None, # type: Union[TextIO, codecs.StreamWriter]
stderr=sys.stderr, # type: IO[Any]
versionfunc=versionstring, # type: Callable[[], Text]
logger_handler=None, #
custom_schema_callback=None, # type: Callable[[], None]
executor=None, # type: Callable[..., Tuple[Dict[Text, Any], Text]]
loadingContext=None, # type: LoadingContext
runtimeContext=None # type: RuntimeContext
): # type: (...) -> int
if not stdout:
stdout = codecs.getwriter('utf-8')(sys.stdout)
_logger.removeHandler(defaultStreamHandler)
if logger_handler:
stderr_handler = logger_handler
else:
stderr_handler = logging.StreamHandler(stderr)
_logger.addHandler(stderr_handler)
try:
if args is None:
args = arg_parser().parse_args(argsl)
if args.workflow and "--outdir" not in argsl:
outputPath = args.workflow.split('/')[-1].split('.')[0]
setattr(
args, "outdir",
os.getcwd() + "/" + outputPath + "/" +
datetime.datetime.now().strftime('%Y-%m-%d-%H%M'))
if runtimeContext is None:
runtimeContext = RuntimeContext(vars(args))
else:
runtimeContext = runtimeContext.copy()
rdflib_logger = logging.getLogger("rdflib.term")
rdflib_logger.addHandler(stderr_handler)
rdflib_logger.setLevel(logging.ERROR)
if args.quiet:
_logger.setLevel(logging.WARN)
if runtimeContext.debug:
_logger.setLevel(logging.DEBUG)
rdflib_logger.setLevel(logging.DEBUG)
if args.timestamps:
formatter = logging.Formatter("[%(asctime)s] %(message)s",
"%Y-%m-%d %H:%M:%S")
stderr_handler.setFormatter(formatter)
# version
if args.version:
return versionfunc(), 0
else:
_logger.info(versionfunc())
if args.print_supported_versions:
return "\n".join(supportedCWLversions(args.enable_dev)), 0
if not args.workflow:
if os.path.isfile("CWLFile"):
setattr(args, "workflow", "CWLFile")
else:
_logger.error("")
_logger.error(
"CWL document required, no input file was provided")
arg_parser().print_help()
return "CWL document required, no input file was provided", 1
if args.relax_path_checks:
command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE
if args.ga4gh_tool_registries:
ga4gh_tool_registries[:] = args.ga4gh_tool_registries
if not args.enable_ga4gh_tool_registry:
del ga4gh_tool_registries[:]
if custom_schema_callback:
custom_schema_callback()
elif args.enable_ext:
res = pkg_resources.resource_stream(__name__, 'extensions.yml')
use_custom_schema("v1.0", "http://commonwl.org/cwltool",
res.read())
res.close()
else:
use_standard_schema("v1.0")
if loadingContext is None:
loadingContext = LoadingContext(vars(args))
else:
loadingContext = loadingContext.copy()
loadingContext.disable_js_validation = \
args.disable_js_validation or (not args.do_validate)
loadingContext.construct_tool_object = getdefault(
loadingContext.construct_tool_object,
workflow.default_make_tool)
loadingContext.resolver = getdefault(loadingContext.resolver,
tool_resolver)
try:
uri, tool_file_uri = resolve_tool_uri(
args.workflow,
resolver=loadingContext.resolver,
fetcher_constructor=loadingContext.fetcher_constructor)
except:
return "Can't find file " + args.workflow, 0
try_again_msg = "" if args.debug else ", try again with --debug for more information"
try:
job_order_object, input_basedir, jobloader = load_job_order(
args, stdin, loadingContext.fetcher_constructor,
loadingContext.overrides_list, tool_file_uri)
if args.overrides:
loadingContext.overrides_list.extend(
load_overrides(
file_uri(os.path.abspath(args.overrides)),
tool_file_uri))
document_loader, workflowobj, uri = fetch_document(
uri,
resolver=loadingContext.resolver,
fetcher_constructor=loadingContext.fetcher_constructor)
if args.print_deps:
# printdeps(workflowobj, document_loader, stdout, args.relative_deps, uri)
result = returndeps(workflowobj, document_loader, stdout,
args.relative_deps, uri)
return result, 0
document_loader, avsc_names, processobj, metadata, uri \
= validate_document(document_loader, workflowobj, uri,
enable_dev=loadingContext.enable_dev,
strict=loadingContext.strict,
preprocess_only=(args.print_pre or args.pack),
fetcher_constructor=loadingContext.fetcher_constructor,
skip_schemas=args.skip_schemas,
overrides=loadingContext.overrides_list,
do_validate=loadingContext.do_validate)
if args.print_pre:
# stdout.write(json_dumps(processobj, indent=4))
return json_dumps(processobj, indent=4), 0
loadingContext.overrides_list.extend(
metadata.get("cwltool:overrides", []))
tool = make_tool(document_loader, avsc_names, metadata, uri,
loadingContext)
if args.make_template:
yaml.safe_dump(generate_input_template(tool),
sys.stdout,
default_flow_style=False,
indent=4,
block_seq_indent=2)
return yaml.safe_dump(generate_input_template(tool),
indent=4), 0
if args.validate:
_logger.info("Tool definition is valid")
return "Tool definition is valid", 0
if args.pack:
stdout.write(
print_pack(document_loader, processobj, uri, metadata))
return print_pack(document_loader, processobj, uri,
metadata), 0
if args.print_rdf:
stdout.write(
printrdf(tool, document_loader.ctx,
args.rdf_serializer))
return printrdf(tool, document_loader.ctx,
args.rdf_serializer), 0
if args.print_dot:
printdot(tool, document_loader.ctx, stdout)
return "args.print_dot still not solved", 0
except (validate.ValidationException) as exc:
_logger.error(u"Tool definition failed validation:\n%s",
exc,
exc_info=args.debug)
infor = "Tool definition failed validation:\n%s" + exc + args.debug
return infor, 1
except (RuntimeError, WorkflowException) as exc:
_logger.error(u"Tool definition failed initialization:\n%s",
exc,
exc_info=args.debug)
infor = "Tool definition failed initialization:\n%s" + exc + args.debug
return infor, 1
except Exception as exc:
_logger.error(
u"I'm sorry, I couldn't load this CWL file%s.\nThe error was: %s",
try_again_msg,
exc if not args.debug else "",
exc_info=args.debug)
return "I'm sorry, I couldn't load this CWL file", 1
if isinstance(tool, int):
return tool, 0
# If on MacOS platform, TMPDIR must be set to be under one of the
# shared volumes in Docker for Mac
# More info: https://dockstore.org/docs/faq
if sys.platform == "darwin":
default_mac_path = "/private/tmp/docker_tmp"
if runtimeContext.tmp_outdir_prefix == DEFAULT_TMP_PREFIX:
runtimeContext.tmp_outdir_prefix = default_mac_path
for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix",
"cachedir"):
if getattr(runtimeContext, dirprefix) and getattr(
runtimeContext, dirprefix) != DEFAULT_TMP_PREFIX:
sl = "/" if getattr(runtimeContext, dirprefix).endswith("/") or dirprefix == "cachedir" \
else ""
setattr(
runtimeContext, dirprefix,
os.path.abspath(getattr(runtimeContext, dirprefix)) +
sl)
if not os.path.exists(
os.path.dirname(getattr(runtimeContext,
dirprefix))):
try:
os.makedirs(
os.path.dirname(
getattr(runtimeContext, dirprefix)))
except Exception as e:
_logger.error("Failed to create directory: %s", e)
infor = "Failed to create directory: %s" + e + ""
return infor, 1
if args.cachedir:
if args.move_outputs == "move":
runtimeContext.move_outputs = "copy"
runtimeContext.tmp_outdir_prefix = args.cachedir
runtimeContext.secret_store = getdefault(
runtimeContext.secret_store, SecretStore())
try:
initialized_job_order_object = init_job_order(
job_order_object,
args,
tool,
jobloader,
stdout,
print_input_deps=args.print_input_deps,
relative_deps=args.relative_deps,
input_basedir=input_basedir,
secret_store=runtimeContext.secret_store)
except SystemExit as err:
return err.code
if not executor:
if args.parallel:
executor = MultithreadedJobExecutor()
else:
executor = SingleJobExecutor()
assert executor is not None
if isinstance(initialized_job_order_object, int):
return initialized_job_order_object
try:
runtimeContext.basedir = input_basedir
del args.workflow
del args.job_order
conf_file = getattr(args,
"beta_dependency_resolvers_configuration",
None) # Text
use_conda_dependencies = getattr(args,
"beta_conda_dependencies",
None) # Text
job_script_provider = None # type: Optional[DependenciesConfiguration]
if conf_file or use_conda_dependencies:
runtimeContext.job_script_provider = DependenciesConfiguration(
args)
runtimeContext.find_default_container = \
functools.partial(find_default_container, args)
runtimeContext.make_fs_access = getdefault(
runtimeContext.make_fs_access, StdFsAccess)
(out, status) = executor(tool,
initialized_job_order_object,
runtimeContext,
logger=_logger)
# This is the workflow output, it needs to be written
if out is not None:
def loc_to_path(obj):
for field in ("path", "nameext", "nameroot",
"dirname"):
if field in obj:
del obj[field]
if obj["location"].startswith("file://"):
obj["path"] = uri_file_path(obj["location"])
visit_class(out, ("File", "Directory"), loc_to_path)
# Unsetting the Generation fron final output object
visit_class(out, ("File", ),
MutationManager().unset_generation)
if isinstance(out, string_types):
stdout.write(out)
else:
stdout.write(
json_dumps(
out,
indent=4, # type: ignore
ensure_ascii=False))
stdout.write("\n")
if hasattr(stdout, "flush"):
stdout.flush() # type: ignore
if status != "success":
_logger.warning(u"Final process status is %s", status)
infor = "Final process status is %s" + status + ""
return infor, 1
_logger.info(u"Final process status is %s", status)
return out, status
except (validate.ValidationException) as exc:
_logger.error(u"Input object failed validation:\n%s",
exc,
exc_info=args.debug)
infor = "Input object failed validation:\n%s" + exc + args.debug
return infor, 1
except UnsupportedRequirement as exc:
_logger.error(
u"Workflow or tool uses unsupported feature:\n%s",
exc,
exc_info=args.debug)
infor = "Workflow or tool uses unsupported feature:\n%s" + exc + args.debug
return infor, 3
except WorkflowException as exc:
_logger.error(u"Workflow error%s:\n%s",
try_again_msg,
strip_dup_lineno(six.text_type(exc)),
exc_info=args.debug)
infor = "Workflow error%s:\n%s" + try_again_msg + strip_dup_lineno(
six.text_type(exc)) + args.debug
return infor, 1
except Exception as exc:
_logger.error(u"Unhandled error%s:\n %s",
try_again_msg,
exc,
exc_info=args.debug)
infor = "Unhandled error%s:\n %s" + try_again_msg + exc + args.debug
return infor, 1
finally:
_logger.removeHandler(stderr_handler)
_logger.addHandler(defaultStreamHandler)
def secrets():
sec_store = SecretStore()
job = {"foo": "bar", "baz": "quux"}
sec_store.store(["foo"], job)
return sec_store, job
def test_secrets(self):
secrets = SecretStore()
job = {"foo": "bar",
"baz": "quux"}
secrets.store(["foo"], job)
self.assertNotEquals(job["foo"], "bar")
self.assertEquals(job["baz"], "quux")
self.assertEquals(secrets.retrieve(job)["foo"], "bar")
hello = "hello %s" % job["foo"]
self.assertTrue(secrets.has_secret(hello))
self.assertNotEquals(hello, "hello bar")
self.assertEquals(secrets.retrieve(hello), "hello bar")
hello2 = ["echo", "hello %s" % job["foo"]]
self.assertTrue(secrets.has_secret(hello2))
self.assertNotEquals(hello2, ["echo", "hello bar"])
self.assertEquals(secrets.retrieve(hello2), ["echo", "hello bar"])
hello3 = {"foo": job["foo"]}
print(hello3)
self.assertTrue(secrets.has_secret(hello3))
self.assertNotEquals(hello3, {"foo": "bar"})
self.assertEquals(secrets.retrieve(hello3), {"foo": "bar"})
self.assertNotEquals(job["foo"], "bar")
self.assertEquals(job["baz"], "quux")