def execute(self,
targets=None,
dryrun=False,
touch=False,
cores=1,
nodes=1,
local_cores=1,
forcetargets=False,
forceall=False,
forcerun=None,
prioritytargets=None,
quiet=False,
keepgoing=False,
printshellcmds=False,
printreason=False,
printdag=False,
cluster=None,
cluster_config=None,
cluster_sync=None,
jobname=None,
immediate_submit=False,
ignore_ambiguity=False,
printrulegraph=False,
printd3dag=False,
drmaa=None,
stats=None,
force_incomplete=False,
ignore_incomplete=False,
list_version_changes=False,
list_code_changes=False,
list_input_changes=False,
list_params_changes=False,
summary=False,
detailed_summary=False,
latency_wait=3,
benchmark_repeats=3,
wait_for_files=None,
nolock=False,
unlock=False,
resources=None,
notemp=False,
nodeps=False,
cleanup_metadata=None,
subsnakemake=None,
updated_files=None,
keep_target_files=False,
allowed_rules=None,
greediness=1.0,
no_hooks=False):
self.global_resources = dict() if resources is None else resources
self.global_resources["_cores"] = cores
self.global_resources["_nodes"] = nodes
def rules(items):
return map(self._rules.__getitem__, filter(self.is_rule, items))
if keep_target_files:
def files(items):
return filterfalse(self.is_rule, items)
else:
def files(items):
return map(os.path.relpath, filterfalse(self.is_rule, items))
if not targets:
targets = [self.first_rule
] if self.first_rule is not None else list()
if prioritytargets is None:
prioritytargets = list()
if forcerun is None:
forcerun = list()
priorityrules = set(rules(prioritytargets))
priorityfiles = set(files(prioritytargets))
forcerules = set(rules(forcerun))
forcefiles = set(files(forcerun))
targetrules = set(
chain(rules(targets), filterfalse(Rule.has_wildcards,
priorityrules),
filterfalse(Rule.has_wildcards, forcerules)))
targetfiles = set(chain(files(targets), priorityfiles, forcefiles))
if forcetargets:
forcefiles.update(targetfiles)
forcerules.update(targetrules)
rules = self.rules
if allowed_rules:
rules = [rule for rule in rules if rule.name in set(allowed_rules)]
if wait_for_files is not None:
try:
snakemake.io.wait_for_files(wait_for_files,
latency_wait=latency_wait)
except IOError as e:
logger.error(str(e))
return False
dag = DAG(self,
rules,
dryrun=dryrun,
targetfiles=targetfiles,
targetrules=targetrules,
forceall=forceall,
forcefiles=forcefiles,
forcerules=forcerules,
priorityfiles=priorityfiles,
priorityrules=priorityrules,
ignore_ambiguity=ignore_ambiguity,
force_incomplete=force_incomplete,
ignore_incomplete=ignore_incomplete or printdag
or printrulegraph,
notemp=notemp)
self.persistence = Persistence(nolock=nolock,
dag=dag,
warn_only=dryrun or printrulegraph
or printdag or summary
or list_version_changes
or list_code_changes
or list_input_changes
or list_params_changes)
if cleanup_metadata:
for f in cleanup_metadata:
self.persistence.cleanup_metadata(f)
return True
dag.init()
dag.check_dynamic()
if unlock:
try:
self.persistence.cleanup_locks()
logger.info("Unlocking working directory.")
return True
except IOError:
logger.error("Error: Unlocking the directory {} failed. Maybe "
"you don't have the permissions?")
return False
try:
self.persistence.lock()
except IOError:
logger.error(
"Error: Directory cannot be locked. Please make "
"sure that no other Snakemake process is trying to create "
"the same files in the following directory:\n{}\n"
"If you are sure that no other "
"instances of snakemake are running on this directory, "
"the remaining lock was likely caused by a kill signal or "
"a power loss. It can be removed with "
"the --unlock argument.".format(os.getcwd()))
return False
if self.subworkflows and not printdag and not printrulegraph:
# backup globals
globals_backup = dict(self.globals)
# execute subworkflows
for subworkflow in self.subworkflows:
subworkflow_targets = subworkflow.targets(dag)
updated = list()
if subworkflow_targets:
logger.info("Executing subworkflow {}.".format(
subworkflow.name))
if not subsnakemake(subworkflow.snakefile,
workdir=subworkflow.workdir,
targets=subworkflow_targets,
updated_files=updated):
return False
dag.updated_subworkflow_files.update(
subworkflow.target(f) for f in updated)
else:
logger.info("Subworkflow {}: Nothing to be done.".format(
subworkflow.name))
if self.subworkflows:
logger.info("Executing main workflow.")
# rescue globals
self.globals.update(globals_backup)
dag.check_incomplete()
dag.postprocess()
if nodeps:
missing_input = [
f for job in dag.targetjobs for f in job.input
if dag.needrun(job) and not os.path.exists(f)
]
if missing_input:
logger.error(
"Dependency resolution disabled (--nodeps) "
"but missing input "
"files detected. If this happens on a cluster, please make sure "
"that you handle the dependencies yourself or turn of "
"--immediate-submit. Missing input files:\n{}".format(
"\n".join(missing_input)))
return False
updated_files.extend(f for job in dag.needrun_jobs for f in job.output)
if printd3dag:
dag.d3dag()
return True
elif printdag:
print(dag)
return True
elif printrulegraph:
print(dag.rule_dot())
return True
elif summary:
print("\n".join(dag.summary(detailed=False)))
return True
elif detailed_summary:
print("\n".join(dag.summary(detailed=True)))
return True
elif list_version_changes:
items = list(
chain(*map(self.persistence.version_changed, dag.jobs)))
if items:
print(*items, sep="\n")
return True
elif list_code_changes:
items = list(chain(*map(self.persistence.code_changed, dag.jobs)))
if items:
print(*items, sep="\n")
return True
elif list_input_changes:
items = list(chain(*map(self.persistence.input_changed, dag.jobs)))
if items:
print(*items, sep="\n")
return True
elif list_params_changes:
items = list(
chain(*map(self.persistence.params_changed, dag.jobs)))
if items:
print(*items, sep="\n")
return True
scheduler = JobScheduler(self,
dag,
cores,
local_cores=local_cores,
dryrun=dryrun,
touch=touch,
cluster=cluster,
cluster_config=cluster_config,
cluster_sync=cluster_sync,
jobname=jobname,
immediate_submit=immediate_submit,
quiet=quiet,
keepgoing=keepgoing,
drmaa=drmaa,
printreason=printreason,
printshellcmds=printshellcmds,
latency_wait=latency_wait,
benchmark_repeats=benchmark_repeats,
greediness=greediness)
if not dryrun and not quiet:
if len(dag):
if cluster or cluster_sync or drmaa:
logger.resources_info(
"Provided cluster nodes: {}".format(nodes))
else:
logger.resources_info("Provided cores: {}".format(cores))
logger.resources_info(
"Rules claiming more threads will be scaled down.")
provided_resources = format_resources(resources)
if provided_resources:
logger.resources_info("Provided resources: " +
provided_resources)
ignored_resources = format_resource_names(
set(resource for job in dag.needrun_jobs
for resource in job.resources_dict
if resource not in resources))
if ignored_resources:
logger.resources_info("Ignored resources: " +
ignored_resources)
logger.run_info("\n".join(dag.stats()))
else:
logger.info("Nothing to be done.")
if dryrun and not len(dag):
logger.info("Nothing to be done.")
success = scheduler.schedule()
if success:
if dryrun:
if not quiet and len(dag):
logger.run_info("\n".join(dag.stats()))
elif stats:
scheduler.stats.to_json(stats)
if not dryrun and not no_hooks:
self._onsuccess(logger.get_logfile())
return True
else:
if not dryrun and not no_hooks:
self._onerror(logger.get_logfile())
return False
def execute(self,
targets=None,
dryrun=False,
touch=False,
cores=1,
nodes=1,
local_cores=1,
forcetargets=False,
forceall=False,
forcerun=None,
prioritytargets=None,
quiet=False,
keepgoing=False,
printshellcmds=False,
printreason=False,
printdag=False,
cluster=None,
cluster_config=None,
cluster_sync=None,
jobname=None,
immediate_submit=False,
ignore_ambiguity=False,
printrulegraph=False,
printd3dag=False,
drmaa=None,
stats=None,
force_incomplete=False,
ignore_incomplete=False,
list_version_changes=False,
list_code_changes=False,
list_input_changes=False,
list_params_changes=False,
summary=False,
detailed_summary=False,
latency_wait=3,
benchmark_repeats=3,
wait_for_files=None,
nolock=False,
unlock=False,
resources=None,
notemp=False,
nodeps=False,
cleanup_metadata=None,
subsnakemake=None,
updated_files=None,
keep_target_files=False,
allowed_rules=None,
greediness=1.0,
no_hooks=False):
self.global_resources = dict() if resources is None else resources
self.global_resources["_cores"] = cores
self.global_resources["_nodes"] = nodes
def rules(items):
return map(self._rules.__getitem__, filter(self.is_rule, items))
if keep_target_files:
def files(items):
return filterfalse(self.is_rule, items)
else:
def files(items):
return map(os.path.relpath, filterfalse(self.is_rule, items))
if not targets:
targets = [self.first_rule
] if self.first_rule is not None else list()
if prioritytargets is None:
prioritytargets = list()
if forcerun is None:
forcerun = list()
priorityrules = set(rules(prioritytargets))
priorityfiles = set(files(prioritytargets))
forcerules = set(rules(forcerun))
forcefiles = set(files(forcerun))
targetrules = set(chain(rules(targets),
filterfalse(Rule.has_wildcards, priorityrules),
filterfalse(Rule.has_wildcards, forcerules)))
targetfiles = set(chain(files(targets), priorityfiles, forcefiles))
if forcetargets:
forcefiles.update(targetfiles)
forcerules.update(targetrules)
rules = self.rules
if allowed_rules:
rules = [rule for rule in rules if rule.name in set(allowed_rules)]
if wait_for_files is not None:
try:
snakemake.io.wait_for_files(wait_for_files,
latency_wait=latency_wait)
except IOError as e:
logger.error(str(e))
return False
dag = DAG(
self, rules,
dryrun=dryrun,
targetfiles=targetfiles,
targetrules=targetrules,
forceall=forceall,
forcefiles=forcefiles,
forcerules=forcerules,
priorityfiles=priorityfiles,
priorityrules=priorityrules,
ignore_ambiguity=ignore_ambiguity,
force_incomplete=force_incomplete,
ignore_incomplete=ignore_incomplete or printdag or printrulegraph,
notemp=notemp)
self.persistence = Persistence(
nolock=nolock,
dag=dag,
warn_only=dryrun or printrulegraph or printdag or summary or
list_version_changes or list_code_changes or list_input_changes or
list_params_changes)
if cleanup_metadata:
for f in cleanup_metadata:
self.persistence.cleanup_metadata(f)
return True
dag.init()
dag.check_dynamic()
if unlock:
try:
self.persistence.cleanup_locks()
logger.info("Unlocking working directory.")
return True
except IOError:
logger.error("Error: Unlocking the directory {} failed. Maybe "
"you don't have the permissions?")
return False
try:
self.persistence.lock()
except IOError:
logger.error(
"Error: Directory cannot be locked. Please make "
"sure that no other Snakemake process is trying to create "
"the same files in the following directory:\n{}\n"
"If you are sure that no other "
"instances of snakemake are running on this directory, "
"the remaining lock was likely caused by a kill signal or "
"a power loss. It can be removed with "
"the --unlock argument.".format(os.getcwd()))
return False
if self.subworkflows and not printdag and not printrulegraph:
# backup globals
globals_backup = dict(self.globals)
# execute subworkflows
for subworkflow in self.subworkflows:
subworkflow_targets = subworkflow.targets(dag)
updated = list()
if subworkflow_targets:
logger.info(
"Executing subworkflow {}.".format(subworkflow.name))
if not subsnakemake(subworkflow.snakefile,
workdir=subworkflow.workdir,
targets=subworkflow_targets,
updated_files=updated):
return False
dag.updated_subworkflow_files.update(subworkflow.target(f)
for f in updated)
else:
logger.info("Subworkflow {}: Nothing to be done.".format(
subworkflow.name))
if self.subworkflows:
logger.info("Executing main workflow.")
# rescue globals
self.globals.update(globals_backup)
dag.check_incomplete()
dag.postprocess()
if nodeps:
missing_input = [f for job in dag.targetjobs for f in job.input
if dag.needrun(job) and not os.path.exists(f)]
if missing_input:
logger.error(
"Dependency resolution disabled (--nodeps) "
"but missing input "
"files detected. If this happens on a cluster, please make sure "
"that you handle the dependencies yourself or turn of "
"--immediate-submit. Missing input files:\n{}".format(
"\n".join(missing_input)))
return False
updated_files.extend(f for job in dag.needrun_jobs for f in job.output)
if printd3dag:
dag.d3dag()
return True
elif printdag:
print(dag)
return True
elif printrulegraph:
print(dag.rule_dot())
return True
elif summary:
print("\n".join(dag.summary(detailed=False)))
return True
elif detailed_summary:
print("\n".join(dag.summary(detailed=True)))
return True
elif list_version_changes:
items = list(
chain(*map(self.persistence.version_changed, dag.jobs)))
if items:
print(*items, sep="\n")
return True
elif list_code_changes:
items = list(chain(*map(self.persistence.code_changed, dag.jobs)))
if items:
print(*items, sep="\n")
return True
elif list_input_changes:
items = list(chain(*map(self.persistence.input_changed, dag.jobs)))
if items:
print(*items, sep="\n")
return True
elif list_params_changes:
items = list(
chain(*map(self.persistence.params_changed, dag.jobs)))
if items:
print(*items, sep="\n")
return True
scheduler = JobScheduler(self, dag, cores,
local_cores=local_cores,
dryrun=dryrun,
touch=touch,
cluster=cluster,
cluster_config=cluster_config,
cluster_sync=cluster_sync,
jobname=jobname,
immediate_submit=immediate_submit,
quiet=quiet,
keepgoing=keepgoing,
drmaa=drmaa,
printreason=printreason,
printshellcmds=printshellcmds,
latency_wait=latency_wait,
benchmark_repeats=benchmark_repeats,
greediness=greediness)
if not dryrun and not quiet:
if len(dag):
if cluster or cluster_sync or drmaa:
logger.resources_info(
"Provided cluster nodes: {}".format(nodes))
else:
logger.resources_info("Provided cores: {}".format(cores))
logger.resources_info("Rules claiming more threads will be scaled down.")
provided_resources = format_resources(resources)
if provided_resources:
logger.resources_info(
"Provided resources: " + provided_resources)
ignored_resources = format_resource_names(
set(resource for job in dag.needrun_jobs for resource in
job.resources_dict if resource not in resources))
if ignored_resources:
logger.resources_info(
"Ignored resources: " + ignored_resources)
logger.run_info("\n".join(dag.stats()))
else:
logger.info("Nothing to be done.")
if dryrun and not len(dag):
logger.info("Nothing to be done.")
success = scheduler.schedule()
if success:
if dryrun:
if not quiet and len(dag):
logger.run_info("\n".join(dag.stats()))
elif stats:
scheduler.stats.to_json(stats)
if not dryrun and not no_hooks:
self._onsuccess(logger.get_logfile())
return True
else:
if not dryrun and not no_hooks:
self._onerror(logger.get_logfile())
return False
def execute(self,
targets=None,
dryrun=False,
touch=False,
cores=1,
nodes=1,
local_cores=1,
forcetargets=False,
forceall=False,
forcerun=None,
until=[],
omit_from=[],
prioritytargets=None,
quiet=False,
keepgoing=False,
printshellcmds=False,
printreason=False,
printdag=False,
cluster=None,
cluster_sync=None,
jobname=None,
immediate_submit=False,
ignore_ambiguity=False,
printrulegraph=False,
printd3dag=False,
drmaa=None,
drmaa_log_dir=None,
kubernetes=None,
kubernetes_envvars=None,
container_image=None,
stats=None,
force_incomplete=False,
ignore_incomplete=False,
list_version_changes=False,
list_code_changes=False,
list_input_changes=False,
list_params_changes=False,
list_untracked=False,
list_conda_envs=False,
summary=False,
archive=None,
delete_all_output=False,
delete_temp_output=False,
detailed_summary=False,
latency_wait=3,
wait_for_files=None,
nolock=False,
unlock=False,
resources=None,
notemp=False,
nodeps=False,
cleanup_metadata=None,
cleanup_conda=False,
cleanup_shadow=False,
subsnakemake=None,
updated_files=None,
keep_target_files=False,
keep_shadow=False,
keep_remote_local=False,
allowed_rules=None,
max_jobs_per_second=None,
max_status_checks_per_second=None,
greediness=1.0,
no_hooks=False,
force_use_threads=False,
create_envs_only=False,
assume_shared_fs=True,
cluster_status=None,
report=None):
self.check_localrules()
self.global_resources = dict() if resources is None else resources
self.global_resources["_cores"] = cores
self.global_resources["_nodes"] = nodes
self.immediate_submit = immediate_submit
def rules(items):
return map(self._rules.__getitem__, filter(self.is_rule, items))
if keep_target_files:
def files(items):
return filterfalse(self.is_rule, items)
else:
def files(items):
relpath = lambda f: f if os.path.isabs(f) else os.path.relpath(f)
return map(relpath, filterfalse(self.is_rule, items))
if not targets:
targets = [self.first_rule
] if self.first_rule is not None else list()
if prioritytargets is None:
prioritytargets = list()
if forcerun is None:
forcerun = list()
if until is None:
until = list()
if omit_from is None:
omit_from = list()
priorityrules = set(rules(prioritytargets))
priorityfiles = set(files(prioritytargets))
forcerules = set(rules(forcerun))
forcefiles = set(files(forcerun))
untilrules = set(rules(until))
untilfiles = set(files(until))
omitrules = set(rules(omit_from))
omitfiles = set(files(omit_from))
targetrules = set(chain(rules(targets),
filterfalse(Rule.has_wildcards, priorityrules),
filterfalse(Rule.has_wildcards, forcerules),
filterfalse(Rule.has_wildcards, untilrules)))
targetfiles = set(chain(files(targets), priorityfiles, forcefiles, untilfiles))
if forcetargets:
forcefiles.update(targetfiles)
forcerules.update(targetrules)
rules = self.rules
if allowed_rules:
rules = [rule for rule in rules if rule.name in set(allowed_rules)]
if wait_for_files is not None:
try:
snakemake.io.wait_for_files(wait_for_files,
latency_wait=latency_wait)
except IOError as e:
logger.error(str(e))
return False
dag = DAG(
self, rules,
dryrun=dryrun,
targetfiles=targetfiles,
targetrules=targetrules,
# when cleaning up conda, we should enforce all possible jobs
# since their envs shall not be deleted
forceall=forceall or cleanup_conda,
forcefiles=forcefiles,
forcerules=forcerules,
priorityfiles=priorityfiles,
priorityrules=priorityrules,
untilfiles=untilfiles,
untilrules=untilrules,
omitfiles=omitfiles,
omitrules=omitrules,
ignore_ambiguity=ignore_ambiguity,
force_incomplete=force_incomplete,
ignore_incomplete=ignore_incomplete or printdag or printrulegraph,
notemp=notemp,
keep_remote_local=keep_remote_local)
self.persistence = Persistence(
nolock=nolock,
dag=dag,
conda_prefix=self.conda_prefix,
singularity_prefix=self.singularity_prefix,
warn_only=dryrun or printrulegraph or printdag or summary or archive or
list_version_changes or list_code_changes or list_input_changes or
list_params_changes or list_untracked or delete_all_output or delete_temp_output)
if cleanup_metadata:
for f in cleanup_metadata:
self.persistence.cleanup_metadata(f)
return True
logger.info("Building DAG of jobs...")
dag.init()
dag.check_dynamic()
if unlock:
try:
self.persistence.cleanup_locks()
logger.info("Unlocking working directory.")
return True
except IOError:
logger.error("Error: Unlocking the directory {} failed. Maybe "
"you don't have the permissions?")
return False
try:
self.persistence.lock()
except IOError:
logger.error(
"Error: Directory cannot be locked. Please make "
"sure that no other Snakemake process is trying to create "
"the same files in the following directory:\n{}\n"
"If you are sure that no other "
"instances of snakemake are running on this directory, "
"the remaining lock was likely caused by a kill signal or "
"a power loss. It can be removed with "
"the --unlock argument.".format(os.getcwd()))
return False
if cleanup_shadow:
self.persistence.cleanup_shadow()
return True
if self.subworkflows and not printdag and not printrulegraph:
# backup globals
globals_backup = dict(self.globals)
# execute subworkflows
for subworkflow in self.subworkflows:
subworkflow_targets = subworkflow.targets(dag)
logger.debug("Files requested from subworkflow:\n {}".format(
"\n ".join(subworkflow_targets)))
updated = list()
if subworkflow_targets:
logger.info(
"Executing subworkflow {}.".format(subworkflow.name))
if not subsnakemake(subworkflow.snakefile,
workdir=subworkflow.workdir,
targets=subworkflow_targets,
configfile=subworkflow.configfile,
updated_files=updated):
return False
dag.updated_subworkflow_files.update(subworkflow.target(f)
for f in updated)
else:
logger.info("Subworkflow {}: Nothing to be done.".format(
subworkflow.name))
if self.subworkflows:
logger.info("Executing main workflow.")
# rescue globals
self.globals.update(globals_backup)
if not (cluster and cluster_status):
# no incomplete check needed because we use external jobids to handle
# this later in the executor
dag.check_incomplete()
dag.postprocess()
# deactivate IOCache such that from now on we always get updated
# size, existence and mtime information
# ATTENTION: this may never be removed without really good reason.
# Otherwise weird things may happen.
self.iocache.deactivate()
# clear and deactivate persistence cache, from now on we want to see updates
self.persistence.deactivate_cache()
if nodeps:
missing_input = [f for job in dag.targetjobs for f in job.input
if dag.needrun(job) and not os.path.exists(f)]
if missing_input:
logger.error(
"Dependency resolution disabled (--nodeps) "
"but missing input "
"files detected. If this happens on a cluster, please make sure "
"that you handle the dependencies yourself or turn of "
"--immediate-submit. Missing input files:\n{}".format(
"\n".join(missing_input)))
return False
updated_files.extend(f for job in dag.needrun_jobs for f in job.output)
if report:
from snakemake.report import auto_report
auto_report(dag, report)
return True
elif printd3dag:
dag.d3dag()
return True
elif printdag:
print(dag)
return True
elif printrulegraph:
print(dag.rule_dot())
return True
elif summary:
print("\n".join(dag.summary(detailed=False)))
return True
elif detailed_summary:
print("\n".join(dag.summary(detailed=True)))
return True
elif archive:
dag.archive(archive)
return True
elif delete_all_output:
dag.clean(only_temp=False, dryrun=dryrun)
return True
elif delete_temp_output:
dag.clean(only_temp=True, dryrun=dryrun)
return True
elif list_version_changes:
items = list(
chain(*map(self.persistence.version_changed, dag.jobs)))
if items:
print(*items, sep="\n")
return True
elif list_code_changes:
items = list(chain(*map(self.persistence.code_changed, dag.jobs)))
for j in dag.jobs:
items.extend(list(j.outputs_older_than_script()))
if items:
print(*items, sep="\n")
return True
elif list_input_changes:
items = list(chain(*map(self.persistence.input_changed, dag.jobs)))
if items:
print(*items, sep="\n")
return True
elif list_params_changes:
items = list(
chain(*map(self.persistence.params_changed, dag.jobs)))
if items:
print(*items, sep="\n")
return True
elif list_untracked:
dag.list_untracked()
return True
if self.use_singularity:
if assume_shared_fs:
dag.pull_singularity_imgs(dryrun=dryrun or list_conda_envs,
quiet=list_conda_envs)
if self.use_conda:
if assume_shared_fs:
dag.create_conda_envs(dryrun=dryrun or list_conda_envs or cleanup_conda,
quiet=list_conda_envs)
if create_envs_only:
return True
if list_conda_envs:
print("environment", "container", "location", sep="\t")
for env in set(job.conda_env for job in dag.jobs):
if env:
print(simplify_path(env.file),
env.singularity_img_url or "",
simplify_path(env.path),
sep="\t")
return True
if cleanup_conda:
self.persistence.cleanup_conda()
return True
scheduler = JobScheduler(self, dag, cores,
local_cores=local_cores,
dryrun=dryrun,
touch=touch,
cluster=cluster,
cluster_status=cluster_status,
cluster_config=cluster_config,
cluster_sync=cluster_sync,
jobname=jobname,
max_jobs_per_second=max_jobs_per_second,
max_status_checks_per_second=max_status_checks_per_second,
quiet=quiet,
keepgoing=keepgoing,
drmaa=drmaa,
drmaa_log_dir=drmaa_log_dir,
kubernetes=kubernetes,
kubernetes_envvars=kubernetes_envvars,
container_image=container_image,
printreason=printreason,
printshellcmds=printshellcmds,
latency_wait=latency_wait,
greediness=greediness,
force_use_threads=force_use_threads,
assume_shared_fs=assume_shared_fs)
if not dryrun:
if len(dag):
shell_exec = shell.get_executable()
if shell_exec is not None:
logger.info("Using shell: {}".format(shell_exec))
if cluster or cluster_sync or drmaa:
logger.resources_info(
"Provided cluster nodes: {}".format(nodes))
else:
logger.resources_info("Provided cores: {}".format(cores))
logger.resources_info("Rules claiming more threads will be scaled down.")
provided_resources = format_resources(resources)
if provided_resources:
logger.resources_info(
"Provided resources: " + provided_resources)
unlimited_resources = format_resource_names(set(
resource for job in dag.needrun_jobs
for resource in job.resources.keys()
if resource not in resources))
if unlimited_resources:
logger.resources_info(
"Unlimited resources: " + unlimited_resources)
logger.run_info("\n".join(dag.stats()))
else:
logger.info("Nothing to be done.")
else:
if len(dag):
logger.run_info("\n".join(dag.stats()))
else:
logger.info("Nothing to be done.")
if quiet:
# in case of dryrun and quiet, just print above info and exit
return True
if not dryrun and not no_hooks:
self._onstart(logger.get_logfile())
success = scheduler.schedule()
if success:
if dryrun:
if len(dag):
logger.run_info("\n".join(dag.stats()))
logger.remove_logfile()
else:
if stats:
scheduler.stats.to_json(stats)
logger.logfile_hint()
if not dryrun and not no_hooks:
self._onsuccess(logger.get_logfile())
return True
else:
if not dryrun and not no_hooks:
self._onerror(logger.get_logfile())
logger.logfile_hint()
return False