def _wait_for_jobs(self):
while True:
with self.lock:
if not self.wait:
return
active_jobs = self.active_jobs
self.active_jobs = list()
for active_job in active_jobs:
if os.path.exists(active_job.jobfinished):
os.remove(active_job.jobfinished)
os.remove(active_job.jobscript)
self.finish_job(active_job.job)
active_job.callback(active_job.job)
elif os.path.exists(active_job.jobfailed):
os.remove(active_job.jobfailed)
os.remove(active_job.jobscript)
self.print_job_error(active_job.job)
print_exception(
ClusterJobException(active_job.job,
self.dag.jobid(active_job.job),
active_job.jobscript),
self.workflow.linemaps)
active_job.error_callback(active_job.job)
else:
self.active_jobs.append(active_job)
time.sleep(1)
def _wait_for_jobs(self):
while True:
with self.lock:
if not self.wait:
return
active_jobs = self.active_jobs
self.active_jobs = list()
for active_job in active_jobs:
exitcode = active_job.process.poll()
if exitcode is None:
# job not yet finished
self.active_jobs.append(active_job)
elif exitcode == 0:
# job finished successfully
os.remove(active_job.jobscript)
self.finish_job(active_job.job)
active_job.callback(active_job.job)
else:
# job failed
os.remove(active_job.jobscript)
self.print_job_error(active_job.job)
print_exception(
ClusterJobException(active_job.job,
self.dag.jobid(active_job.job),
active_job.jobscript),
self.workflow.linemaps)
active_job.error_callback(active_job.job)
time.sleep(1)
def _wait_for_jobs(self):
import drmaa
while True:
with self.lock:
if not self.wait:
return
active_jobs = self.active_jobs
self.active_jobs = list()
for active_job in active_jobs:
try:
retval = self.session.wait(active_job.jobid,
drmaa.Session.TIMEOUT_NO_WAIT)
except drmaa.errors.ExitTimeoutException as e:
# job still active
self.active_jobs.append(active_job)
continue
except (drmaa.errors.InternalException, Exception) as e:
print_exception(WorkflowError("DRMAA Error: {}".format(e)),
self.workflow.linemaps)
os.remove(active_job.jobscript)
active_job.error_callback(active_job.job)
continue
# job exited
os.remove(active_job.jobscript)
if retval.hasExited and retval.exitStatus == 0:
self.finish_job(active_job.job)
active_job.callback(active_job.job)
else:
self.print_job_error(active_job.job)
print_exception(
ClusterJobException(active_job.job, self.dag.jobid(active_job.job), active_job.jobscript),
self.workflow.linemaps)
active_job.error_callback(active_job.job)
time.sleep(1)
def _wait_for_jobs(self):
while True:
with self.lock:
if not self.wait:
return
active_jobs = self.active_jobs
self.active_jobs = list()
for active_job in active_jobs:
exitcode = active_job.process.poll()
if exitcode is None:
# job not yet finished
self.active_jobs.append(active_job)
elif exitcode == 0:
# job finished successfully
os.remove(active_job.jobscript)
self.finish_job(active_job.job)
active_job.callback(active_job.job)
else:
# job failed
os.remove(active_job.jobscript)
self.print_job_error(active_job.job)
print_exception(ClusterJobException(active_job.job, self.dag.jobid(active_job.job),
active_job.jobscript),
self.workflow.linemaps)
active_job.error_callback(active_job.job)
time.sleep(1)
def _finish_jobs(self):
# must be called from within lock
# clear the global tofinish such that parallel calls do not interfere
for job in self._tofinish:
if self.handle_job_success:
try:
self.get_executor(job).handle_job_success(job)
except (RuleException, WorkflowError) as e:
# if an error occurs while processing job output,
# we do the same as in case of errors during execution
print_exception(e, self.workflow.linemaps)
self._handle_error(job)
continue
if self.update_resources:
# normal jobs have len=1, group jobs have len>1
self.finished_jobs += len(job)
self.running.remove(job)
self._free_resources(job)
if self.print_progress:
if job.is_group():
for j in job:
logger.job_finished(jobid=j.jobid)
else:
logger.job_finished(jobid=job.jobid)
self.progress()
self.dag.finish(job, update_dynamic=self.update_dynamic)
self._tofinish.clear()
def _wait_for_jobs(self):
import drmaa
while True:
with self.lock:
if not self.wait:
return
active_jobs = self.active_jobs
self.active_jobs = list()
for active_job in active_jobs:
try:
retval = self.session.wait(active_job.jobid,
drmaa.Session.TIMEOUT_NO_WAIT)
except drmaa.errors.ExitTimeoutException as e:
# job still active
self.active_jobs.append(active_job)
continue
except (drmaa.errors.InternalException, Exception) as e:
print_exception(WorkflowError("DRMAA Error: {}".format(e)),
self.workflow.linemaps)
os.remove(active_job.jobscript)
active_job.error_callback(active_job.job)
continue
# job exited
os.remove(active_job.jobscript)
if retval.hasExited and retval.exitStatus == 0:
self.finish_job(active_job.job)
active_job.callback(active_job.job)
else:
self.print_job_error(active_job.job)
print_exception(
ClusterJobException(active_job.job, self.dag.jobid(active_job.job), active_job.jobscript),
self.workflow.linemaps)
active_job.error_callback(active_job.job)
time.sleep(1)
def _proceed(
self,
job,
update_dynamic=True,
print_progress=False,
update_resources=True,
handle_job_success=True,
):
""" Do stuff after job is finished. """
with self._lock:
if handle_job_success:
# by calling this behind the lock, we avoid race conditions
try:
self.get_executor(job).handle_job_success(job)
except (RuleException, WorkflowError) as e:
# if an error occurs while processing job output,
# we do the same as in case of errors during execution
print_exception(e, self.workflow.linemaps)
self._handle_error(job)
return
try:
potential_new_ready_jobs = self.dag.finish(
job, update_dynamic=update_dynamic
)
except (RuleException, WorkflowError) as e:
# if an error occurs while processing job output,
# we do the same as in case of errors during execution
print_exception(e, self.workflow.linemaps)
self._handle_error(job)
return
if update_resources:
# normal jobs have len=1, group jobs have len>1
self.finished_jobs += len(job)
self.running.remove(job)
self._free_resources(job)
if print_progress:
if job.is_group():
for j in job:
logger.job_finished(jobid=j.jobid)
else:
logger.job_finished(jobid=job.jobid)
self.progress()
if self.dryrun:
if not self.running:
# During dryrun, only release when all running jobs are done.
# This saves a lot of time, as self.open_jobs has to be
# evaluated less frequently.
self._open_jobs.release()
elif (
not self.running
or potential_new_ready_jobs
or self.workflow.immediate_submit
):
# go on scheduling if open jobs are ready or no job is running
self._open_jobs.release()
def _callback(self, job, callback, error_callback, future):
try:
ex = future.exception()
if ex:
raise ex
self.finish_job(job)
callback(job)
except (Exception, BaseException) as ex:
print_exception(ex, self.workflow.linemaps)
job.cleanup()
self.workflow.persistence.cleanup(job)
error_callback(job)
def run(
self, job, callback=None, submit_callback=None, error_callback=None):
super()._run(job)
try:
for f in job.expanded_output:
f.touch()
time.sleep(0.1)
self.finish_job(job)
callback(job)
except OSError as ex:
print_exception(ex, self.workflow.linemaps)
error_callback(job)
def create_workflow(snakefile):
workflow = Workflow(snakefile=snakefile, use_conda=True)
try:
workflow.include(snakefile,
overwrite_first_rule=True,
print_compilation=False)
workflow.check()
except (Exception, BaseException) as ex:
print_exception(ex, workflow.linemaps)
success = False
return workflow, success
def run(self, job,
callback=None,
submit_callback=None,
error_callback=None):
super()._run(job)
try:
#Touching of output files will be done by finish_job
if job.benchmark:
job.benchmark.touch()
time.sleep(0.1)
self.finish_job(job)
callback(job)
except OSError as ex:
print_exception(ex, self.workflow.linemaps)
error_callback(job)
def run(self, job,
callback=None,
submit_callback=None,
error_callback=None):
super()._run(job)
try:
#Touching of output files will be done by finish_job
if job.benchmark:
job.benchmark.touch()
time.sleep(0.1)
self.finish_job(job)
callback(job)
except OSError as ex:
print_exception(ex, self.workflow.linemaps)
error_callback(job)
def _callback(self, job, callback, error_callback, future):
try:
ex = future.exception()
if ex:
raise ex
self.finish_job(job)
callback(job)
except _ProcessPoolExceptions:
job.cleanup()
self.workflow.persistence.cleanup(job)
# no error callback, just silently ignore the interrupt as the main scheduler is also killed
except (Exception, BaseException) as ex:
self.print_job_error(job)
print_exception(ex, self.workflow.linemaps)
job.cleanup()
self.workflow.persistence.cleanup(job)
error_callback(job)
def _callback(self, job, callback, error_callback, future):
try:
ex = future.exception()
if ex:
raise ex
self.finish_job(job)
callback(job)
except _ProcessPoolExceptions:
job.cleanup()
self.workflow.persistence.cleanup(job)
# no error callback, just silently ignore the interrupt as the main scheduler is also killed
except (Exception, BaseException) as ex:
self.print_job_error(job)
print_exception(ex, self.workflow.linemaps)
job.cleanup()
self.workflow.persistence.cleanup(job)
error_callback(job)
def _wait_for_job(
self, job, callback, error_callback,
jobscript, jobfinished, jobfailed):
while True:
if os.path.exists(jobfinished):
os.remove(jobfinished)
os.remove(jobscript)
self.finish_job(job)
callback(job)
return
if os.path.exists(jobfailed):
os.remove(jobfailed)
os.remove(jobscript)
print_exception(
ClusterJobException(job, self.dag.jobid(job), self.get_jobscript(job)), self.workflow.linemaps)
error_callback(job)
return
time.sleep(1)
def _proceed(self,
job,
update_dynamic=True,
print_progress=False,
update_resources=True,
handle_job_success=True):
""" Do stuff after job is finished. """
with self._lock:
if handle_job_success:
# by calling this behind the lock, we avoid race conditions
try:
self.get_executor(job).handle_job_success(job)
except (RuleException, WorkflowError) as e:
# if an error occurs while processing job output,
# we do the same as in case of errors during execution
print_exception(e, self.workflow.linemaps)
self._handle_error(job)
return
self.dag.finish(job, update_dynamic=update_dynamic)
if update_resources:
# normal jobs have len=1, group jobs have len>1
self.finished_jobs += len(job)
self.running.remove(job)
self._free_resources(job)
if print_progress:
if job.is_group():
for j in job:
logger.job_finished(jobid=j.jobid)
else:
logger.job_finished(jobid=job.jobid)
self.progress()
if any(self.open_jobs
) or not self.running or self.workflow.immediate_submit:
# go on scheduling if open jobs are ready or no job is running
self._open_jobs.release()
def run(self,
job,
callback=None,
submit_callback=None,
error_callback=None):
super()._run(job)
jobscript = self.get_jobscript(job)
self.spawn_jobscript(job, jobscript)
try:
drmaa_args = job.format_wildcards(
self.drmaa_args, cluster=self.cluster_wildcards(job))
except AttributeError as e:
raise WorkflowError(str(e), rule=job.rule)
import drmaa
try:
jt = self.session.createJobTemplate()
jt.remoteCommand = jobscript
jt.nativeSpecification = drmaa_args
jt.jobName = os.path.basename(jobscript)
jobid = self.session.runJob(jt)
except (drmaa.errors.InternalException,
drmaa.errors.InvalidAttributeValueException) as e:
print_exception(WorkflowError("DRMAA Error: {}".format(e)),
self.workflow.linemaps)
error_callback(job)
return
logger.info("Submitted DRMAA job (jobid {})".format(jobid))
self.submitted.append(jobid)
self.session.deleteJobTemplate(jt)
submit_callback(job)
with self.lock:
self.active_jobs.append(
DRMAAClusterJob(job, jobid, callback, error_callback,
jobscript))
def run(self, job,
callback=None,
submit_callback=None,
error_callback=None):
super()._run(job)
jobscript = self.get_jobscript(job)
self.spawn_jobscript(job, jobscript)
try:
drmaa_args = job.format_wildcards(
self.drmaa_args,
cluster=self.cluster_wildcards(job))
except AttributeError as e:
raise WorkflowError(str(e), rule=job.rule)
import drmaa
try:
jt = self.session.createJobTemplate()
jt.remoteCommand = jobscript
jt.nativeSpecification = drmaa_args
jt.jobName = os.path.basename(jobscript)
jobid = self.session.runJob(jt)
except (drmaa.errors.InternalException,
drmaa.errors.InvalidAttributeValueException) as e:
print_exception(WorkflowError("DRMAA Error: {}".format(e)),
self.workflow.linemaps)
error_callback(job)
return
logger.info("Submitted DRMAA job (jobid {})".format(jobid))
self.submitted.append(jobid)
self.session.deleteJobTemplate(jt)
submit_callback(job)
with self.lock:
self.active_jobs.append(DRMAAClusterJob(job, jobid, callback, error_callback, jobscript))
def _wait_for_jobs(self):
while True:
with self.lock:
if not self.wait:
return
active_jobs = self.active_jobs
self.active_jobs = list()
for active_job in active_jobs:
if os.path.exists(active_job.jobfinished):
os.remove(active_job.jobfinished)
os.remove(active_job.jobscript)
self.finish_job(active_job.job)
active_job.callback(active_job.job)
elif os.path.exists(active_job.jobfailed):
os.remove(active_job.jobfailed)
os.remove(active_job.jobscript)
self.print_job_error(active_job.job)
print_exception(ClusterJobException(active_job.job, self.dag.jobid(active_job.job),
active_job.jobscript),
self.workflow.linemaps)
active_job.error_callback(active_job.job)
else:
self.active_jobs.append(active_job)
time.sleep(1)
def _wait_for_jobs(self):
"""wait for jobs to complete. This means requesting their status,
and then marking them as finished when a "done" parameter
shows up. Even for finished jobs, the status should still return
"""
import googleapiclient
while True:
# always use self.lock to avoid race conditions
with self.lock:
if not self.wait:
return
active_jobs = self.active_jobs
self.active_jobs = list()
still_running = list()
# Loop through active jobs and act on status
for j in active_jobs:
# use self.status_rate_limiter to avoid too many API calls.
with self.status_rate_limiter:
# https://cloud.google.com/life-sciences/docs/reference/rest/v2beta/projects.locations.operations/get
# Get status from projects.locations.operations/get
operations = self._api.projects().locations().operations()
request = operations.get(name=j.jobname)
logger.debug("Checking status for operation {}".format(
j.jobid))
try:
status = self._retry_request(request)
except googleapiclient.errors.HttpError as ex:
# Operation name not found, even finished should be found
if ex.status == 404:
j.error_callback(j.job)
continue
# Unpredictable server (500) error
elif ex.status == 500:
logger.error(ex["content"].decode("utf-8"))
j.error_callback(j.job)
except WorkflowError as ex:
print_exception(ex, self.workflow.linemaps)
j.error_callback(j.job)
continue
# The operation is done
if status.get("done", False) == True:
# Derive success/failure from status codes (prints too)
if self._job_was_successful(status):
j.callback(j.job)
else:
self.print_job_error(j.job, jobid=j.jobid)
j.error_callback(j.job)
# The operation is still running
else:
still_running.append(j)
with self.lock:
self.active_jobs.extend(still_running)
sleep()
parser.add_argument('-d',
'--outdir',
action="store",
default=os.curdir,
help="Snakefile to import")
args = parser.parse_args()
snakefile = os.path.abspath(args.Snakefile)
workflow = Workflow(snakefile=snakefile)
try:
workflow.include(snakefile,
overwrite_first_rule=True,
print_compilation=False)
workflow.check()
except (Exception, BaseException) as ex:
print_exception(ex, workflow.linemaps)
success = False
# Map the rules included from snakemake_rules
DEST = args.outdir
rules = {
x: os.path.join(DEST, os.path.relpath(x, SNAKEMAKE_RULES_PATH))
for x in workflow.included if x.startswith(SNAKEMAKE_RULES_PATH)
}
# Copy rules to outdir
for k, v in rules.items():
sync_file(k, v, args.dry_run)
def snakemake(snakefile,
listrules=False,
cores=1,
resources=None,
workdir=None,
targets=None,
dryrun=False,
touch=False,
forcetargets=False,
forceall=False,
forcerun=None,
prioritytargets=None,
stats=None,
printreason=False,
printshellcmds=False,
printdag=False,
printrulegraph=False,
nocolor=False,
quiet=False,
keepgoing=False,
cluster=None,
immediate_submit=False,
standalone=False,
ignore_ambiguity=False,
snakemakepath=None,
lock=True,
unlock=False,
cleanup_metadata=None,
force_incomplete=False,
ignore_incomplete=False,
list_version_changes=False,
list_code_changes=False,
list_input_changes=False,
list_params_changes=False,
summary=False,
output_wait=3,
print_compilation=False,
debug=False,
notemp=False,
nodeps=False,
jobscript=None,
timestamp=False):
"""
Run snakemake on a given snakefile.
Note: at the moment, this function is not thread-safe!
Arguments
snakefile -- the snakefile.
list -- list rules.
jobs -- maximum number of parallel jobs (default: 1).
directory -- working directory (default: current directory).
rule -- execute this rule (default: first rule in snakefile).
dryrun -- print the rules that would be executed,
but do not execute them.
forcethis -- force the selected rule to be executed
forceall -- force all rules to be executed
time_measurements -- measure the running times of all rules
lock -- lock the working directory
"""
init_logger(nocolor=nocolor, stdout=dryrun, debug=debug, timestamp=timestamp)
if not os.path.exists(snakefile):
logger.error("Error: Snakefile \"{}\" not present.".format(snakefile))
return False
if workdir:
olddir = os.getcwd()
workflow = Workflow(
snakefile=snakefile, snakemakepath=snakemakepath,
jobscript=jobscript)
if standalone:
try:
# set the process group
os.setpgrp()
except:
# ignore: if it does not work we can still work without it
pass
success = True
try:
workflow.include(snakefile, workdir=workdir,
overwrite_first_rule=True, print_compilation=print_compilation)
workflow.check()
if not print_compilation:
if listrules:
workflow.list_rules()
else:
if not printdag and not printrulegraph:
# handle subworkflows
subsnakemake = partial(
snakemake,
cores=cores,
resources=resources,
dryrun=dryrun,
touch=touch,
printreason=printreason,
printshellcmds=printshellcmds,
nocolor=nocolor,
quiet=quiet,
keepgoing=keepgoing,
cluster=cluster,
immediate_submit=immediate_submit,
standalone=standalone,
ignore_ambiguity=ignore_ambiguity,
snakemakepath=snakemakepath,
lock=lock,
unlock=unlock,
cleanup_metadata=cleanup_metadata,
force_incomplete=force_incomplete,
ignore_incomplete=ignore_incomplete,
output_wait=output_wait,
debug=debug,
notemp=notemp,
nodeps=nodeps,
jobscript=jobscript,
timestamp=timestamp)
for subworkflow in workflow.subworkflows:
logger.warning("Executing subworkflow {}.".format(subworkflow.name))
if not subsnakemake(subworkflow.snakefile, workdir=subworkflow.workdir, targets=subworkflow.targets):
success = False
if workflow.subworkflows:
logger.warning("Executing main workflow.")
if success:
success = workflow.execute(
targets=targets, dryrun=dryrun, touch=touch,
cores=cores, forcetargets=forcetargets,
forceall=forceall, forcerun=forcerun,
prioritytargets=prioritytargets, quiet=quiet,
keepgoing=keepgoing, printshellcmds=printshellcmds,
printreason=printreason, printrulegraph=printrulegraph,
printdag=printdag, cluster=cluster,
immediate_submit=immediate_submit,
ignore_ambiguity=ignore_ambiguity,
workdir=workdir, stats=stats,
force_incomplete=force_incomplete,
ignore_incomplete=ignore_incomplete,
list_version_changes=list_version_changes,
list_code_changes=list_code_changes,
list_input_changes=list_input_changes,
list_params_changes=list_params_changes,
summary=summary,
output_wait=output_wait,
nolock=not lock,
unlock=unlock,
resources=resources,
notemp=notemp,
nodeps=nodeps,
cleanup_metadata=cleanup_metadata
)
except (Exception, BaseException) as ex:
print_exception(ex, workflow.linemaps)
success = False
if workdir:
os.chdir(olddir)
if workflow.persistence:
workflow.persistence.unlock()
return success
def schedule(self):
"""Schedule jobs that are ready, maximizing cpu usage."""
try:
while True:
# work around so that the wait does not prevent keyboard interrupts
# while not self._open_jobs.acquire(False):
# time.sleep(1)
self._open_jobs.acquire()
# obtain needrun and running jobs in a thread-safe way
with self._lock:
self._finish_jobs()
self._error_jobs()
needrun = set(self.open_jobs)
running = list(self.running)
errors = self._errors
executor_error = self._executor_error
user_kill = self._user_kill
# handle errors
if user_kill or (not self.keepgoing and errors) or executor_error:
if user_kill == "graceful":
logger.info(
"Will exit after finishing currently running jobs (scheduler)."
)
if executor_error:
print_exception(executor_error, self.workflow.linemaps)
if executor_error or not running:
logger.info("Shutting down, this might take some time.")
self._executor.shutdown()
if not user_kill:
logger.error(_ERROR_MSG_FINAL)
return False
continue
# all runnable jobs have finished, normal shutdown
if not needrun and (not running or self.workflow.immediate_submit):
self._executor.shutdown()
if errors:
logger.error(_ERROR_MSG_FINAL)
# we still have unfinished jobs. this is not good. direct
# user to github issue
if self.remaining_jobs and not self.keepgoing:
logger.error(_ERROR_MSG_ISSUE_823)
logger.error(
"Remaining jobs:\n"
+ "\n".join(
" - " + str(job) + ": " + ", ".join(job.output)
for job in self.remaining_jobs
)
)
return False
return not errors
# continue if no new job needs to be executed
if not needrun:
continue
# select jobs by solving knapsack problem (omit with dryrun)
if self.dryrun:
run = needrun
else:
# Reset params and resources because they might still contain TBDs
# or old values from before files have been regenerated.
# Now, they can be recalculated as all input is present and up to date.
for job in needrun:
job.reset_params_and_resources()
logger.debug(
"Resources before job selection: {}".format(self.resources)
)
logger.debug(
"Ready jobs ({}):\n\t".format(len(needrun))
+ "\n\t".join(map(str, needrun))
)
if not self._last_job_selection_empty:
logger.info("Select jobs to execute...")
run = self.job_selector(needrun)
self._last_job_selection_empty = not run
logger.debug(
"Selected jobs ({}):\n\t".format(len(run))
+ "\n\t".join(map(str, run))
)
logger.debug(
"Resources after job selection: {}".format(self.resources)
)
# update running jobs
with self._lock:
self.running.update(run)
# remove from ready_jobs
self.dag.register_running(run)
# actually run jobs
local_runjobs = [job for job in run if job.is_local]
runjobs = [job for job in run if not job.is_local]
self.run(local_runjobs, executor=self._local_executor or self._executor)
self.run(runjobs)
except (KeyboardInterrupt, SystemExit):
logger.info(
"Terminating processes on user request, this might take some time."
)
self._executor.cancel()
return False
def snakemake(snakefile,
listrules=False,
list_target_rules=False,
cores=1,
nodes=1,
local_cores=1,
resources=dict(),
config=dict(),
configfile=None,
config_args=None,
workdir=None,
targets=None,
dryrun=False,
touch=False,
forcetargets=False,
forceall=False,
forcerun=[],
prioritytargets=[],
stats=None,
printreason=False,
printshellcmds=False,
printdag=False,
printrulegraph=False,
printd3dag=False,
nocolor=False,
quiet=False,
keepgoing=False,
cluster=None,
cluster_config=None,
cluster_sync=None,
drmaa=None,
jobname="snakejob.{rulename}.{jobid}.sh",
immediate_submit=False,
standalone=False,
ignore_ambiguity=False,
snakemakepath=None,
lock=True,
unlock=False,
cleanup_metadata=None,
force_incomplete=False,
ignore_incomplete=False,
list_version_changes=False,
list_code_changes=False,
list_input_changes=False,
list_params_changes=False,
list_resources=False,
summary=False,
detailed_summary=False,
latency_wait=3,
benchmark_repeats=1,
wait_for_files=None,
print_compilation=False,
debug=False,
notemp=False,
nodeps=False,
keep_target_files=False,
allowed_rules=None,
jobscript=None,
timestamp=False,
greediness=None,
no_hooks=False,
overwrite_shellcmd=None,
updated_files=None,
log_handler=None,
keep_logger=False,
verbose=False):
"""Run snakemake on a given snakefile.
This function provides access to the whole snakemake functionality. It is not thread-safe.
Args:
snakefile (str): the path to the snakefile
listrules (bool): list rules (default False)
list_target_rules (bool): list target rules (default False)
cores (int): the number of provided cores (ignored when using cluster support) (default 1)
nodes (int): the number of provided cluster nodes (ignored without cluster support) (default 1)
local_cores (int): the number of provided local cores if in cluster mode (ignored without cluster support) (default 1)
resources (dict): provided resources, a dictionary assigning integers to resource names, e.g. {gpu=1, io=5} (default {})
config (dict): override values for workflow config
workdir (str): path to working directory (default None)
targets (list): list of targets, e.g. rule or file names (default None)
dryrun (bool): only dry-run the workflow (default False)
touch (bool): only touch all output files if present (default False)
forcetargets (bool): force given targets to be re-created (default False)
forceall (bool): force all output files to be re-created (default False)
forcerun (list): list of files and rules that shall be re-created/re-executed (default [])
prioritytargets (list): list of targets that shall be run with maximum priority (default [])
stats (str): path to file that shall contain stats about the workflow execution (default None)
printreason (bool): print the reason for the execution of each job (default false)
printshellcmds (bool): print the shell command of each job (default False)
printdag (bool): print the dag in the graphviz dot language (default False)
printrulegraph (bool): print the graph of rules in the graphviz dot language (default False)
printd3dag (bool): print a D3.js compatible JSON representation of the DAG (default False)
nocolor (bool): do not print colored output (default False)
quiet (bool): do not print any default job information (default False)
keepgoing (bool): keep goind upon errors (default False)
cluster (str): submission command of a cluster or batch system to use, e.g. qsub (default None)
cluster_config (str): configuration file for cluster options (default None)
cluster_sync (str): blocking cluster submission command (like SGE 'qsub -sync y') (default None)
drmaa (str): if not None use DRMAA for cluster support, str specifies native args passed to the cluster when submitting a job
jobname (str): naming scheme for cluster job scripts (default "snakejob.{rulename}.{jobid}.sh")
immediate_submit (bool): immediately submit all cluster jobs, regardless of dependencies (default False)
standalone (bool): kill all processes very rudely in case of failure (do not use this if you use this API) (default False)
ignore_ambiguity (bool): ignore ambiguous rules and always take the first possible one (default False)
snakemakepath (str): path to the snakemake executable (default None)
lock (bool): lock the working directory when executing the workflow (default True)
unlock (bool): just unlock the working directory (default False)
cleanup_metadata (bool): just cleanup metadata of output files (default False)
force_incomplete (bool): force the re-creation of incomplete files (default False)
ignore_incomplete (bool): ignore incomplete files (default False)
list_version_changes (bool): list output files with changed rule version (default False)
list_code_changes (bool): list output files with changed rule code (default False)
list_input_changes (bool): list output files with changed input files (default False)
list_params_changes (bool): list output files with changed params (default False)
summary (bool): list summary of all output files and their status (default False)
latency_wait (int): how many seconds to wait for an output file to appear after the execution of a job, e.g. to handle filesystem latency (default 3)
benchmark_repeats (int): number of repeated runs of a job if declared for benchmarking (default 1)
wait_for_files (list): wait for given files to be present before executing the workflow
list_resources (bool): list resources used in the workflow (default False)
summary (bool): list summary of all output files and their status (default False). If no option is specified a basic summary will be ouput. If 'detailed' is added as an option e.g --summary detailed, extra info about the input and shell commands will be included
detailed_summary (bool): list summary of all input and output files and their status (default False)
print_compilation (bool): print the compilation of the snakefile (default False)
debug (bool): allow to use the debugger within rules
notemp (bool): ignore temp file flags, e.g. do not delete output files marked as temp after use (default False)
nodeps (bool): ignore dependencies (default False)
keep_target_files (bool): Do not adjust the paths of given target files relative to the working directory.
allowed_rules (set): Restrict allowed rules to the given set. If None or empty, all rules are used.
jobscript (str): path to a custom shell script template for cluster jobs (default None)
timestamp (bool): print time stamps in front of any output (default False)
greediness (float): set the greediness of scheduling. This value between 0 and 1 determines how careful jobs are selected for execution. The default value (0.5 if prioritytargets are used, 1.0 else) provides the best speed and still acceptable scheduling quality.
overwrite_shellcmd (str): a shell command that shall be executed instead of those given in the workflow. This is for debugging purposes only.
updated_files(list): a list that will be filled with the files that are updated or created during the workflow execution
verbose(bool): show additional debug output (default False)
log_handler (function): redirect snakemake output to this custom log handler, a function that takes a log message dictionary (see below) as its only argument (default None). The log message dictionary for the log handler has to following entries:
:level:
the log level ("info", "error", "debug", "progress", "job_info")
:level="info", "error" or "debug":
:msg:
the log message
:level="progress":
:done:
number of already executed jobs
:total:
number of total jobs
:level="job_info":
:input:
list of input files of a job
:output:
list of output files of a job
:log:
path to log file of a job
:local:
whether a job is executed locally (i.e. ignoring cluster)
:msg:
the job message
:reason:
the job reason
:priority:
the job priority
:threads:
the threads of the job
Returns:
bool: True if workflow execution was successful.
"""
if updated_files is None:
updated_files = list()
if cluster or cluster_sync or drmaa:
cores = sys.maxsize
else:
nodes = sys.maxsize
if cluster_config:
cluster_config = load_configfile(cluster_config)
else:
cluster_config = dict()
if not keep_logger:
setup_logger(handler=log_handler,
quiet=quiet,
printreason=printreason,
printshellcmds=printshellcmds,
nocolor=nocolor,
stdout=dryrun,
debug=verbose,
timestamp=timestamp)
if greediness is None:
greediness = 0.5 if prioritytargets else 1.0
else:
if not (0 <= greediness <= 1.0):
logger.error("Error: greediness must be a float between 0 and 1.")
return False
if not os.path.exists(snakefile):
logger.error("Error: Snakefile \"{}\" not present.".format(snakefile))
return False
snakefile = os.path.abspath(snakefile)
cluster_mode = (cluster is not None) + (cluster_sync is not
None) + (drmaa is not None)
if cluster_mode > 1:
logger.error("Error: cluster and drmaa args are mutually exclusive")
return False
if debug and (cores > 1 or cluster_mode):
logger.error(
"Error: debug mode cannot be used with more than one core or cluster execution.")
return False
overwrite_config = dict()
if configfile:
overwrite_config.update(load_configfile(configfile))
if config:
overwrite_config.update(config)
if workdir:
olddir = os.getcwd()
if not os.path.exists(workdir):
logger.info(
"Creating specified working directory {}.".format(workdir))
os.makedirs(workdir)
workdir = os.path.abspath(workdir)
os.chdir(workdir)
workflow = Workflow(snakefile=snakefile,
snakemakepath=snakemakepath,
jobscript=jobscript,
overwrite_shellcmd=overwrite_shellcmd,
overwrite_config=overwrite_config,
overwrite_workdir=workdir,
overwrite_configfile=configfile,
config_args=config_args,
debug=debug)
if standalone:
try:
# set the process group
os.setpgrp()
except:
# ignore: if it does not work we can still work without it
pass
success = True
try:
workflow.include(snakefile,
overwrite_first_rule=True,
print_compilation=print_compilation)
workflow.check()
if not print_compilation:
if listrules:
workflow.list_rules()
elif list_target_rules:
workflow.list_rules(only_targets=True)
elif list_resources:
workflow.list_resources()
else:
# if not printdag and not printrulegraph:
# handle subworkflows
subsnakemake = partial(snakemake,
cores=cores,
nodes=nodes,
local_cores=local_cores,
resources=resources,
dryrun=dryrun,
touch=touch,
printreason=printreason,
printshellcmds=printshellcmds,
nocolor=nocolor,
quiet=quiet,
keepgoing=keepgoing,
cluster=cluster,
cluster_config=cluster_config,
cluster_sync=cluster_sync,
drmaa=drmaa,
jobname=jobname,
immediate_submit=immediate_submit,
standalone=standalone,
ignore_ambiguity=ignore_ambiguity,
snakemakepath=snakemakepath,
lock=lock,
unlock=unlock,
cleanup_metadata=cleanup_metadata,
force_incomplete=force_incomplete,
ignore_incomplete=ignore_incomplete,
latency_wait=latency_wait,
benchmark_repeats=benchmark_repeats,
verbose=verbose,
notemp=notemp,
nodeps=nodeps,
jobscript=jobscript,
timestamp=timestamp,
greediness=greediness,
no_hooks=no_hooks,
overwrite_shellcmd=overwrite_shellcmd,
config=config,
config_args=config_args,
keep_logger=True)
success = workflow.execute(
targets=targets,
dryrun=dryrun,
touch=touch,
cores=cores,
nodes=nodes,
local_cores=local_cores,
forcetargets=forcetargets,
forceall=forceall,
forcerun=forcerun,
prioritytargets=prioritytargets,
quiet=quiet,
keepgoing=keepgoing,
printshellcmds=printshellcmds,
printreason=printreason,
printrulegraph=printrulegraph,
printdag=printdag,
cluster=cluster,
cluster_config=cluster_config,
cluster_sync=cluster_sync,
jobname=jobname,
drmaa=drmaa,
printd3dag=printd3dag,
immediate_submit=immediate_submit,
ignore_ambiguity=ignore_ambiguity,
stats=stats,
force_incomplete=force_incomplete,
ignore_incomplete=ignore_incomplete,
list_version_changes=list_version_changes,
list_code_changes=list_code_changes,
list_input_changes=list_input_changes,
list_params_changes=list_params_changes,
summary=summary,
latency_wait=latency_wait,
benchmark_repeats=benchmark_repeats,
wait_for_files=wait_for_files,
detailed_summary=detailed_summary,
nolock=not lock,
unlock=unlock,
resources=resources,
notemp=notemp,
nodeps=nodeps,
keep_target_files=keep_target_files,
cleanup_metadata=cleanup_metadata,
subsnakemake=subsnakemake,
updated_files=updated_files,
allowed_rules=allowed_rules,
greediness=greediness,
no_hooks=no_hooks)
except BrokenPipeError:
# ignore this exception and stop. It occurs if snakemake output is piped into less and less quits before reading the whole output.
# in such a case, snakemake shall stop scheduling and quit with error 1
success = False
except (Exception, BaseException) as ex:
print_exception(ex, workflow.linemaps)
success = False
if workdir:
os.chdir(olddir)
if workflow.persistence:
workflow.persistence.unlock()
if not keep_logger:
logger.cleanup()
return success
