def _wait_for_jobs(self): while True: with self.lock: if not self.wait: return active_jobs = self.active_jobs self.active_jobs = list() for active_job in active_jobs: if os.path.exists(active_job.jobfinished): os.remove(active_job.jobfinished) os.remove(active_job.jobscript) self.finish_job(active_job.job) active_job.callback(active_job.job) elif os.path.exists(active_job.jobfailed): os.remove(active_job.jobfailed) os.remove(active_job.jobscript) self.print_job_error(active_job.job) print_exception( ClusterJobException(active_job.job, self.dag.jobid(active_job.job), active_job.jobscript), self.workflow.linemaps) active_job.error_callback(active_job.job) else: self.active_jobs.append(active_job) time.sleep(1)
def _wait_for_jobs(self): while True: with self.lock: if not self.wait: return active_jobs = self.active_jobs self.active_jobs = list() for active_job in active_jobs: exitcode = active_job.process.poll() if exitcode is None: # job not yet finished self.active_jobs.append(active_job) elif exitcode == 0: # job finished successfully os.remove(active_job.jobscript) self.finish_job(active_job.job) active_job.callback(active_job.job) else: # job failed os.remove(active_job.jobscript) self.print_job_error(active_job.job) print_exception( ClusterJobException(active_job.job, self.dag.jobid(active_job.job), active_job.jobscript), self.workflow.linemaps) active_job.error_callback(active_job.job) time.sleep(1)
def _wait_for_jobs(self): import drmaa while True: with self.lock: if not self.wait: return active_jobs = self.active_jobs self.active_jobs = list() for active_job in active_jobs: try: retval = self.session.wait(active_job.jobid, drmaa.Session.TIMEOUT_NO_WAIT) except drmaa.errors.ExitTimeoutException as e: # job still active self.active_jobs.append(active_job) continue except (drmaa.errors.InternalException, Exception) as e: print_exception(WorkflowError("DRMAA Error: {}".format(e)), self.workflow.linemaps) os.remove(active_job.jobscript) active_job.error_callback(active_job.job) continue # job exited os.remove(active_job.jobscript) if retval.hasExited and retval.exitStatus == 0: self.finish_job(active_job.job) active_job.callback(active_job.job) else: self.print_job_error(active_job.job) print_exception( ClusterJobException(active_job.job, self.dag.jobid(active_job.job), active_job.jobscript), self.workflow.linemaps) active_job.error_callback(active_job.job) time.sleep(1)
def _wait_for_jobs(self): while True: with self.lock: if not self.wait: return active_jobs = self.active_jobs self.active_jobs = list() for active_job in active_jobs: exitcode = active_job.process.poll() if exitcode is None: # job not yet finished self.active_jobs.append(active_job) elif exitcode == 0: # job finished successfully os.remove(active_job.jobscript) self.finish_job(active_job.job) active_job.callback(active_job.job) else: # job failed os.remove(active_job.jobscript) self.print_job_error(active_job.job) print_exception(ClusterJobException(active_job.job, self.dag.jobid(active_job.job), active_job.jobscript), self.workflow.linemaps) active_job.error_callback(active_job.job) time.sleep(1)
def _finish_jobs(self): # must be called from within lock # clear the global tofinish such that parallel calls do not interfere for job in self._tofinish: if self.handle_job_success: try: self.get_executor(job).handle_job_success(job) except (RuleException, WorkflowError) as e: # if an error occurs while processing job output, # we do the same as in case of errors during execution print_exception(e, self.workflow.linemaps) self._handle_error(job) continue if self.update_resources: # normal jobs have len=1, group jobs have len>1 self.finished_jobs += len(job) self.running.remove(job) self._free_resources(job) if self.print_progress: if job.is_group(): for j in job: logger.job_finished(jobid=j.jobid) else: logger.job_finished(jobid=job.jobid) self.progress() self.dag.finish(job, update_dynamic=self.update_dynamic) self._tofinish.clear()
def _proceed( self, job, update_dynamic=True, print_progress=False, update_resources=True, handle_job_success=True, ): """ Do stuff after job is finished. """ with self._lock: if handle_job_success: # by calling this behind the lock, we avoid race conditions try: self.get_executor(job).handle_job_success(job) except (RuleException, WorkflowError) as e: # if an error occurs while processing job output, # we do the same as in case of errors during execution print_exception(e, self.workflow.linemaps) self._handle_error(job) return try: potential_new_ready_jobs = self.dag.finish( job, update_dynamic=update_dynamic ) except (RuleException, WorkflowError) as e: # if an error occurs while processing job output, # we do the same as in case of errors during execution print_exception(e, self.workflow.linemaps) self._handle_error(job) return if update_resources: # normal jobs have len=1, group jobs have len>1 self.finished_jobs += len(job) self.running.remove(job) self._free_resources(job) if print_progress: if job.is_group(): for j in job: logger.job_finished(jobid=j.jobid) else: logger.job_finished(jobid=job.jobid) self.progress() if self.dryrun: if not self.running: # During dryrun, only release when all running jobs are done. # This saves a lot of time, as self.open_jobs has to be # evaluated less frequently. self._open_jobs.release() elif ( not self.running or potential_new_ready_jobs or self.workflow.immediate_submit ): # go on scheduling if open jobs are ready or no job is running self._open_jobs.release()
def _callback(self, job, callback, error_callback, future): try: ex = future.exception() if ex: raise ex self.finish_job(job) callback(job) except (Exception, BaseException) as ex: print_exception(ex, self.workflow.linemaps) job.cleanup() self.workflow.persistence.cleanup(job) error_callback(job)
def run( self, job, callback=None, submit_callback=None, error_callback=None): super()._run(job) try: for f in job.expanded_output: f.touch() time.sleep(0.1) self.finish_job(job) callback(job) except OSError as ex: print_exception(ex, self.workflow.linemaps) error_callback(job)
def create_workflow(snakefile): workflow = Workflow(snakefile=snakefile, use_conda=True) try: workflow.include(snakefile, overwrite_first_rule=True, print_compilation=False) workflow.check() except (Exception, BaseException) as ex: print_exception(ex, workflow.linemaps) success = False return workflow, success
def run(self, job, callback=None, submit_callback=None, error_callback=None): super()._run(job) try: #Touching of output files will be done by finish_job if job.benchmark: job.benchmark.touch() time.sleep(0.1) self.finish_job(job) callback(job) except OSError as ex: print_exception(ex, self.workflow.linemaps) error_callback(job)
def _callback(self, job, callback, error_callback, future): try: ex = future.exception() if ex: raise ex self.finish_job(job) callback(job) except _ProcessPoolExceptions: job.cleanup() self.workflow.persistence.cleanup(job) # no error callback, just silently ignore the interrupt as the main scheduler is also killed except (Exception, BaseException) as ex: self.print_job_error(job) print_exception(ex, self.workflow.linemaps) job.cleanup() self.workflow.persistence.cleanup(job) error_callback(job)
def _wait_for_job( self, job, callback, error_callback, jobscript, jobfinished, jobfailed): while True: if os.path.exists(jobfinished): os.remove(jobfinished) os.remove(jobscript) self.finish_job(job) callback(job) return if os.path.exists(jobfailed): os.remove(jobfailed) os.remove(jobscript) print_exception( ClusterJobException(job, self.dag.jobid(job), self.get_jobscript(job)), self.workflow.linemaps) error_callback(job) return time.sleep(1)
def _proceed(self, job, update_dynamic=True, print_progress=False, update_resources=True, handle_job_success=True): """ Do stuff after job is finished. """ with self._lock: if handle_job_success: # by calling this behind the lock, we avoid race conditions try: self.get_executor(job).handle_job_success(job) except (RuleException, WorkflowError) as e: # if an error occurs while processing job output, # we do the same as in case of errors during execution print_exception(e, self.workflow.linemaps) self._handle_error(job) return self.dag.finish(job, update_dynamic=update_dynamic) if update_resources: # normal jobs have len=1, group jobs have len>1 self.finished_jobs += len(job) self.running.remove(job) self._free_resources(job) if print_progress: if job.is_group(): for j in job: logger.job_finished(jobid=j.jobid) else: logger.job_finished(jobid=job.jobid) self.progress() if any(self.open_jobs ) or not self.running or self.workflow.immediate_submit: # go on scheduling if open jobs are ready or no job is running self._open_jobs.release()
def run(self, job, callback=None, submit_callback=None, error_callback=None): super()._run(job) jobscript = self.get_jobscript(job) self.spawn_jobscript(job, jobscript) try: drmaa_args = job.format_wildcards( self.drmaa_args, cluster=self.cluster_wildcards(job)) except AttributeError as e: raise WorkflowError(str(e), rule=job.rule) import drmaa try: jt = self.session.createJobTemplate() jt.remoteCommand = jobscript jt.nativeSpecification = drmaa_args jt.jobName = os.path.basename(jobscript) jobid = self.session.runJob(jt) except (drmaa.errors.InternalException, drmaa.errors.InvalidAttributeValueException) as e: print_exception(WorkflowError("DRMAA Error: {}".format(e)), self.workflow.linemaps) error_callback(job) return logger.info("Submitted DRMAA job (jobid {})".format(jobid)) self.submitted.append(jobid) self.session.deleteJobTemplate(jt) submit_callback(job) with self.lock: self.active_jobs.append( DRMAAClusterJob(job, jobid, callback, error_callback, jobscript))
def run(self, job, callback=None, submit_callback=None, error_callback=None): super()._run(job) jobscript = self.get_jobscript(job) self.spawn_jobscript(job, jobscript) try: drmaa_args = job.format_wildcards( self.drmaa_args, cluster=self.cluster_wildcards(job)) except AttributeError as e: raise WorkflowError(str(e), rule=job.rule) import drmaa try: jt = self.session.createJobTemplate() jt.remoteCommand = jobscript jt.nativeSpecification = drmaa_args jt.jobName = os.path.basename(jobscript) jobid = self.session.runJob(jt) except (drmaa.errors.InternalException, drmaa.errors.InvalidAttributeValueException) as e: print_exception(WorkflowError("DRMAA Error: {}".format(e)), self.workflow.linemaps) error_callback(job) return logger.info("Submitted DRMAA job (jobid {})".format(jobid)) self.submitted.append(jobid) self.session.deleteJobTemplate(jt) submit_callback(job) with self.lock: self.active_jobs.append(DRMAAClusterJob(job, jobid, callback, error_callback, jobscript))
def _wait_for_jobs(self): while True: with self.lock: if not self.wait: return active_jobs = self.active_jobs self.active_jobs = list() for active_job in active_jobs: if os.path.exists(active_job.jobfinished): os.remove(active_job.jobfinished) os.remove(active_job.jobscript) self.finish_job(active_job.job) active_job.callback(active_job.job) elif os.path.exists(active_job.jobfailed): os.remove(active_job.jobfailed) os.remove(active_job.jobscript) self.print_job_error(active_job.job) print_exception(ClusterJobException(active_job.job, self.dag.jobid(active_job.job), active_job.jobscript), self.workflow.linemaps) active_job.error_callback(active_job.job) else: self.active_jobs.append(active_job) time.sleep(1)
def _wait_for_jobs(self): """wait for jobs to complete. This means requesting their status, and then marking them as finished when a "done" parameter shows up. Even for finished jobs, the status should still return """ import googleapiclient while True: # always use self.lock to avoid race conditions with self.lock: if not self.wait: return active_jobs = self.active_jobs self.active_jobs = list() still_running = list() # Loop through active jobs and act on status for j in active_jobs: # use self.status_rate_limiter to avoid too many API calls. with self.status_rate_limiter: # https://cloud.google.com/life-sciences/docs/reference/rest/v2beta/projects.locations.operations/get # Get status from projects.locations.operations/get operations = self._api.projects().locations().operations() request = operations.get(name=j.jobname) logger.debug("Checking status for operation {}".format( j.jobid)) try: status = self._retry_request(request) except googleapiclient.errors.HttpError as ex: # Operation name not found, even finished should be found if ex.status == 404: j.error_callback(j.job) continue # Unpredictable server (500) error elif ex.status == 500: logger.error(ex["content"].decode("utf-8")) j.error_callback(j.job) except WorkflowError as ex: print_exception(ex, self.workflow.linemaps) j.error_callback(j.job) continue # The operation is done if status.get("done", False) == True: # Derive success/failure from status codes (prints too) if self._job_was_successful(status): j.callback(j.job) else: self.print_job_error(j.job, jobid=j.jobid) j.error_callback(j.job) # The operation is still running else: still_running.append(j) with self.lock: self.active_jobs.extend(still_running) sleep()
parser.add_argument('-d', '--outdir', action="store", default=os.curdir, help="Snakefile to import") args = parser.parse_args() snakefile = os.path.abspath(args.Snakefile) workflow = Workflow(snakefile=snakefile) try: workflow.include(snakefile, overwrite_first_rule=True, print_compilation=False) workflow.check() except (Exception, BaseException) as ex: print_exception(ex, workflow.linemaps) success = False # Map the rules included from snakemake_rules DEST = args.outdir rules = { x: os.path.join(DEST, os.path.relpath(x, SNAKEMAKE_RULES_PATH)) for x in workflow.included if x.startswith(SNAKEMAKE_RULES_PATH) } # Copy rules to outdir for k, v in rules.items(): sync_file(k, v, args.dry_run)
def snakemake(snakefile, listrules=False, cores=1, resources=None, workdir=None, targets=None, dryrun=False, touch=False, forcetargets=False, forceall=False, forcerun=None, prioritytargets=None, stats=None, printreason=False, printshellcmds=False, printdag=False, printrulegraph=False, nocolor=False, quiet=False, keepgoing=False, cluster=None, immediate_submit=False, standalone=False, ignore_ambiguity=False, snakemakepath=None, lock=True, unlock=False, cleanup_metadata=None, force_incomplete=False, ignore_incomplete=False, list_version_changes=False, list_code_changes=False, list_input_changes=False, list_params_changes=False, summary=False, output_wait=3, print_compilation=False, debug=False, notemp=False, nodeps=False, jobscript=None, timestamp=False): """ Run snakemake on a given snakefile. Note: at the moment, this function is not thread-safe! Arguments snakefile -- the snakefile. list -- list rules. jobs -- maximum number of parallel jobs (default: 1). directory -- working directory (default: current directory). rule -- execute this rule (default: first rule in snakefile). dryrun -- print the rules that would be executed, but do not execute them. forcethis -- force the selected rule to be executed forceall -- force all rules to be executed time_measurements -- measure the running times of all rules lock -- lock the working directory """ init_logger(nocolor=nocolor, stdout=dryrun, debug=debug, timestamp=timestamp) if not os.path.exists(snakefile): logger.error("Error: Snakefile \"{}\" not present.".format(snakefile)) return False if workdir: olddir = os.getcwd() workflow = Workflow( snakefile=snakefile, snakemakepath=snakemakepath, jobscript=jobscript) if standalone: try: # set the process group os.setpgrp() except: # ignore: if it does not work we can still work without it pass success = True try: workflow.include(snakefile, workdir=workdir, overwrite_first_rule=True, print_compilation=print_compilation) workflow.check() if not print_compilation: if listrules: workflow.list_rules() else: if not printdag and not printrulegraph: # handle subworkflows subsnakemake = partial( snakemake, cores=cores, resources=resources, dryrun=dryrun, touch=touch, printreason=printreason, printshellcmds=printshellcmds, nocolor=nocolor, quiet=quiet, keepgoing=keepgoing, cluster=cluster, immediate_submit=immediate_submit, standalone=standalone, ignore_ambiguity=ignore_ambiguity, snakemakepath=snakemakepath, lock=lock, unlock=unlock, cleanup_metadata=cleanup_metadata, force_incomplete=force_incomplete, ignore_incomplete=ignore_incomplete, output_wait=output_wait, debug=debug, notemp=notemp, nodeps=nodeps, jobscript=jobscript, timestamp=timestamp) for subworkflow in workflow.subworkflows: logger.warning("Executing subworkflow {}.".format(subworkflow.name)) if not subsnakemake(subworkflow.snakefile, workdir=subworkflow.workdir, targets=subworkflow.targets): success = False if workflow.subworkflows: logger.warning("Executing main workflow.") if success: success = workflow.execute( targets=targets, dryrun=dryrun, touch=touch, cores=cores, forcetargets=forcetargets, forceall=forceall, forcerun=forcerun, prioritytargets=prioritytargets, quiet=quiet, keepgoing=keepgoing, printshellcmds=printshellcmds, printreason=printreason, printrulegraph=printrulegraph, printdag=printdag, cluster=cluster, immediate_submit=immediate_submit, ignore_ambiguity=ignore_ambiguity, workdir=workdir, stats=stats, force_incomplete=force_incomplete, ignore_incomplete=ignore_incomplete, list_version_changes=list_version_changes, list_code_changes=list_code_changes, list_input_changes=list_input_changes, list_params_changes=list_params_changes, summary=summary, output_wait=output_wait, nolock=not lock, unlock=unlock, resources=resources, notemp=notemp, nodeps=nodeps, cleanup_metadata=cleanup_metadata ) except (Exception, BaseException) as ex: print_exception(ex, workflow.linemaps) success = False if workdir: os.chdir(olddir) if workflow.persistence: workflow.persistence.unlock() return success
def schedule(self): """Schedule jobs that are ready, maximizing cpu usage.""" try: while True: # work around so that the wait does not prevent keyboard interrupts # while not self._open_jobs.acquire(False): # time.sleep(1) self._open_jobs.acquire() # obtain needrun and running jobs in a thread-safe way with self._lock: self._finish_jobs() self._error_jobs() needrun = set(self.open_jobs) running = list(self.running) errors = self._errors executor_error = self._executor_error user_kill = self._user_kill # handle errors if user_kill or (not self.keepgoing and errors) or executor_error: if user_kill == "graceful": logger.info( "Will exit after finishing currently running jobs (scheduler)." ) if executor_error: print_exception(executor_error, self.workflow.linemaps) if executor_error or not running: logger.info("Shutting down, this might take some time.") self._executor.shutdown() if not user_kill: logger.error(_ERROR_MSG_FINAL) return False continue # all runnable jobs have finished, normal shutdown if not needrun and (not running or self.workflow.immediate_submit): self._executor.shutdown() if errors: logger.error(_ERROR_MSG_FINAL) # we still have unfinished jobs. this is not good. direct # user to github issue if self.remaining_jobs and not self.keepgoing: logger.error(_ERROR_MSG_ISSUE_823) logger.error( "Remaining jobs:\n" + "\n".join( " - " + str(job) + ": " + ", ".join(job.output) for job in self.remaining_jobs ) ) return False return not errors # continue if no new job needs to be executed if not needrun: continue # select jobs by solving knapsack problem (omit with dryrun) if self.dryrun: run = needrun else: # Reset params and resources because they might still contain TBDs # or old values from before files have been regenerated. # Now, they can be recalculated as all input is present and up to date. for job in needrun: job.reset_params_and_resources() logger.debug( "Resources before job selection: {}".format(self.resources) ) logger.debug( "Ready jobs ({}):\n\t".format(len(needrun)) + "\n\t".join(map(str, needrun)) ) if not self._last_job_selection_empty: logger.info("Select jobs to execute...") run = self.job_selector(needrun) self._last_job_selection_empty = not run logger.debug( "Selected jobs ({}):\n\t".format(len(run)) + "\n\t".join(map(str, run)) ) logger.debug( "Resources after job selection: {}".format(self.resources) ) # update running jobs with self._lock: self.running.update(run) # remove from ready_jobs self.dag.register_running(run) # actually run jobs local_runjobs = [job for job in run if job.is_local] runjobs = [job for job in run if not job.is_local] self.run(local_runjobs, executor=self._local_executor or self._executor) self.run(runjobs) except (KeyboardInterrupt, SystemExit): logger.info( "Terminating processes on user request, this might take some time." ) self._executor.cancel() return False
def snakemake(snakefile, listrules=False, list_target_rules=False, cores=1, nodes=1, local_cores=1, resources=dict(), config=dict(), configfile=None, config_args=None, workdir=None, targets=None, dryrun=False, touch=False, forcetargets=False, forceall=False, forcerun=[], prioritytargets=[], stats=None, printreason=False, printshellcmds=False, printdag=False, printrulegraph=False, printd3dag=False, nocolor=False, quiet=False, keepgoing=False, cluster=None, cluster_config=None, cluster_sync=None, drmaa=None, jobname="snakejob.{rulename}.{jobid}.sh", immediate_submit=False, standalone=False, ignore_ambiguity=False, snakemakepath=None, lock=True, unlock=False, cleanup_metadata=None, force_incomplete=False, ignore_incomplete=False, list_version_changes=False, list_code_changes=False, list_input_changes=False, list_params_changes=False, list_resources=False, summary=False, detailed_summary=False, latency_wait=3, benchmark_repeats=1, wait_for_files=None, print_compilation=False, debug=False, notemp=False, nodeps=False, keep_target_files=False, allowed_rules=None, jobscript=None, timestamp=False, greediness=None, no_hooks=False, overwrite_shellcmd=None, updated_files=None, log_handler=None, keep_logger=False, verbose=False): """Run snakemake on a given snakefile. This function provides access to the whole snakemake functionality. It is not thread-safe. Args: snakefile (str): the path to the snakefile listrules (bool): list rules (default False) list_target_rules (bool): list target rules (default False) cores (int): the number of provided cores (ignored when using cluster support) (default 1) nodes (int): the number of provided cluster nodes (ignored without cluster support) (default 1) local_cores (int): the number of provided local cores if in cluster mode (ignored without cluster support) (default 1) resources (dict): provided resources, a dictionary assigning integers to resource names, e.g. {gpu=1, io=5} (default {}) config (dict): override values for workflow config workdir (str): path to working directory (default None) targets (list): list of targets, e.g. rule or file names (default None) dryrun (bool): only dry-run the workflow (default False) touch (bool): only touch all output files if present (default False) forcetargets (bool): force given targets to be re-created (default False) forceall (bool): force all output files to be re-created (default False) forcerun (list): list of files and rules that shall be re-created/re-executed (default []) prioritytargets (list): list of targets that shall be run with maximum priority (default []) stats (str): path to file that shall contain stats about the workflow execution (default None) printreason (bool): print the reason for the execution of each job (default false) printshellcmds (bool): print the shell command of each job (default False) printdag (bool): print the dag in the graphviz dot language (default False) printrulegraph (bool): print the graph of rules in the graphviz dot language (default False) printd3dag (bool): print a D3.js compatible JSON representation of the DAG (default False) nocolor (bool): do not print colored output (default False) quiet (bool): do not print any default job information (default False) keepgoing (bool): keep goind upon errors (default False) cluster (str): submission command of a cluster or batch system to use, e.g. qsub (default None) cluster_config (str): configuration file for cluster options (default None) cluster_sync (str): blocking cluster submission command (like SGE 'qsub -sync y') (default None) drmaa (str): if not None use DRMAA for cluster support, str specifies native args passed to the cluster when submitting a job jobname (str): naming scheme for cluster job scripts (default "snakejob.{rulename}.{jobid}.sh") immediate_submit (bool): immediately submit all cluster jobs, regardless of dependencies (default False) standalone (bool): kill all processes very rudely in case of failure (do not use this if you use this API) (default False) ignore_ambiguity (bool): ignore ambiguous rules and always take the first possible one (default False) snakemakepath (str): path to the snakemake executable (default None) lock (bool): lock the working directory when executing the workflow (default True) unlock (bool): just unlock the working directory (default False) cleanup_metadata (bool): just cleanup metadata of output files (default False) force_incomplete (bool): force the re-creation of incomplete files (default False) ignore_incomplete (bool): ignore incomplete files (default False) list_version_changes (bool): list output files with changed rule version (default False) list_code_changes (bool): list output files with changed rule code (default False) list_input_changes (bool): list output files with changed input files (default False) list_params_changes (bool): list output files with changed params (default False) summary (bool): list summary of all output files and their status (default False) latency_wait (int): how many seconds to wait for an output file to appear after the execution of a job, e.g. to handle filesystem latency (default 3) benchmark_repeats (int): number of repeated runs of a job if declared for benchmarking (default 1) wait_for_files (list): wait for given files to be present before executing the workflow list_resources (bool): list resources used in the workflow (default False) summary (bool): list summary of all output files and their status (default False). If no option is specified a basic summary will be ouput. If 'detailed' is added as an option e.g --summary detailed, extra info about the input and shell commands will be included detailed_summary (bool): list summary of all input and output files and their status (default False) print_compilation (bool): print the compilation of the snakefile (default False) debug (bool): allow to use the debugger within rules notemp (bool): ignore temp file flags, e.g. do not delete output files marked as temp after use (default False) nodeps (bool): ignore dependencies (default False) keep_target_files (bool): Do not adjust the paths of given target files relative to the working directory. allowed_rules (set): Restrict allowed rules to the given set. If None or empty, all rules are used. jobscript (str): path to a custom shell script template for cluster jobs (default None) timestamp (bool): print time stamps in front of any output (default False) greediness (float): set the greediness of scheduling. This value between 0 and 1 determines how careful jobs are selected for execution. The default value (0.5 if prioritytargets are used, 1.0 else) provides the best speed and still acceptable scheduling quality. overwrite_shellcmd (str): a shell command that shall be executed instead of those given in the workflow. This is for debugging purposes only. updated_files(list): a list that will be filled with the files that are updated or created during the workflow execution verbose(bool): show additional debug output (default False) log_handler (function): redirect snakemake output to this custom log handler, a function that takes a log message dictionary (see below) as its only argument (default None). The log message dictionary for the log handler has to following entries: :level: the log level ("info", "error", "debug", "progress", "job_info") :level="info", "error" or "debug": :msg: the log message :level="progress": :done: number of already executed jobs :total: number of total jobs :level="job_info": :input: list of input files of a job :output: list of output files of a job :log: path to log file of a job :local: whether a job is executed locally (i.e. ignoring cluster) :msg: the job message :reason: the job reason :priority: the job priority :threads: the threads of the job Returns: bool: True if workflow execution was successful. """ if updated_files is None: updated_files = list() if cluster or cluster_sync or drmaa: cores = sys.maxsize else: nodes = sys.maxsize if cluster_config: cluster_config = load_configfile(cluster_config) else: cluster_config = dict() if not keep_logger: setup_logger(handler=log_handler, quiet=quiet, printreason=printreason, printshellcmds=printshellcmds, nocolor=nocolor, stdout=dryrun, debug=verbose, timestamp=timestamp) if greediness is None: greediness = 0.5 if prioritytargets else 1.0 else: if not (0 <= greediness <= 1.0): logger.error("Error: greediness must be a float between 0 and 1.") return False if not os.path.exists(snakefile): logger.error("Error: Snakefile \"{}\" not present.".format(snakefile)) return False snakefile = os.path.abspath(snakefile) cluster_mode = (cluster is not None) + (cluster_sync is not None) + (drmaa is not None) if cluster_mode > 1: logger.error("Error: cluster and drmaa args are mutually exclusive") return False if debug and (cores > 1 or cluster_mode): logger.error( "Error: debug mode cannot be used with more than one core or cluster execution.") return False overwrite_config = dict() if configfile: overwrite_config.update(load_configfile(configfile)) if config: overwrite_config.update(config) if workdir: olddir = os.getcwd() if not os.path.exists(workdir): logger.info( "Creating specified working directory {}.".format(workdir)) os.makedirs(workdir) workdir = os.path.abspath(workdir) os.chdir(workdir) workflow = Workflow(snakefile=snakefile, snakemakepath=snakemakepath, jobscript=jobscript, overwrite_shellcmd=overwrite_shellcmd, overwrite_config=overwrite_config, overwrite_workdir=workdir, overwrite_configfile=configfile, config_args=config_args, debug=debug) if standalone: try: # set the process group os.setpgrp() except: # ignore: if it does not work we can still work without it pass success = True try: workflow.include(snakefile, overwrite_first_rule=True, print_compilation=print_compilation) workflow.check() if not print_compilation: if listrules: workflow.list_rules() elif list_target_rules: workflow.list_rules(only_targets=True) elif list_resources: workflow.list_resources() else: # if not printdag and not printrulegraph: # handle subworkflows subsnakemake = partial(snakemake, cores=cores, nodes=nodes, local_cores=local_cores, resources=resources, dryrun=dryrun, touch=touch, printreason=printreason, printshellcmds=printshellcmds, nocolor=nocolor, quiet=quiet, keepgoing=keepgoing, cluster=cluster, cluster_config=cluster_config, cluster_sync=cluster_sync, drmaa=drmaa, jobname=jobname, immediate_submit=immediate_submit, standalone=standalone, ignore_ambiguity=ignore_ambiguity, snakemakepath=snakemakepath, lock=lock, unlock=unlock, cleanup_metadata=cleanup_metadata, force_incomplete=force_incomplete, ignore_incomplete=ignore_incomplete, latency_wait=latency_wait, benchmark_repeats=benchmark_repeats, verbose=verbose, notemp=notemp, nodeps=nodeps, jobscript=jobscript, timestamp=timestamp, greediness=greediness, no_hooks=no_hooks, overwrite_shellcmd=overwrite_shellcmd, config=config, config_args=config_args, keep_logger=True) success = workflow.execute( targets=targets, dryrun=dryrun, touch=touch, cores=cores, nodes=nodes, local_cores=local_cores, forcetargets=forcetargets, forceall=forceall, forcerun=forcerun, prioritytargets=prioritytargets, quiet=quiet, keepgoing=keepgoing, printshellcmds=printshellcmds, printreason=printreason, printrulegraph=printrulegraph, printdag=printdag, cluster=cluster, cluster_config=cluster_config, cluster_sync=cluster_sync, jobname=jobname, drmaa=drmaa, printd3dag=printd3dag, immediate_submit=immediate_submit, ignore_ambiguity=ignore_ambiguity, stats=stats, force_incomplete=force_incomplete, ignore_incomplete=ignore_incomplete, list_version_changes=list_version_changes, list_code_changes=list_code_changes, list_input_changes=list_input_changes, list_params_changes=list_params_changes, summary=summary, latency_wait=latency_wait, benchmark_repeats=benchmark_repeats, wait_for_files=wait_for_files, detailed_summary=detailed_summary, nolock=not lock, unlock=unlock, resources=resources, notemp=notemp, nodeps=nodeps, keep_target_files=keep_target_files, cleanup_metadata=cleanup_metadata, subsnakemake=subsnakemake, updated_files=updated_files, allowed_rules=allowed_rules, greediness=greediness, no_hooks=no_hooks) except BrokenPipeError: # ignore this exception and stop. It occurs if snakemake output is piped into less and less quits before reading the whole output. # in such a case, snakemake shall stop scheduling and quit with error 1 success = False except (Exception, BaseException) as ex: print_exception(ex, workflow.linemaps) success = False if workdir: os.chdir(olddir) if workflow.persistence: workflow.persistence.unlock() if not keep_logger: logger.cleanup() return success