def _expand(self, template, wc=None):
if wc is None:
wc = {}
if isinstance(template, str):
template = [template]
names = set()
for item in template:
names |= get_wildcard_names(item)
sources = [wc]
try:
ds = self.getDatasetFromDir(wc.dir)
sources += [ds]
except:
pass
sources += [self]
fields = {}
for name in names:
for source in sources:
if name in dir(source):
fields[name] = getattr(source, name)
break
if name not in fields:
fields[name] = "{{{}}}".format(name)
res = expand(template, **fields)
return res
def __init__(self, workflow, dag, cores,
jobname="snakejob.{rulename}.{jobid}.sh",
printreason=False,
quiet=False,
printshellcmds=False,
latency_wait=3,
benchmark_repeats=1,
cluster_config=None):
super().__init__(workflow, dag,
printreason=printreason,
quiet=quiet,
printshellcmds=printshellcmds,
latency_wait=latency_wait,
benchmark_repeats=benchmark_repeats)
if workflow.snakemakepath is None:
raise ValueError("Cluster executor needs to know the path "
"to the snakemake binary.")
jobscript = workflow.jobscript
if jobscript is None:
jobscript = os.path.join(os.path.dirname(__file__),
self.default_jobscript)
try:
with open(jobscript) as f:
self.jobscript = f.read()
except IOError as e:
raise WorkflowError(e)
if not "jobid" in get_wildcard_names(jobname):
raise WorkflowError(
"Defined jobname (\"{}\") has to contain the wildcard {jobid}.")
self.exec_job = (
'cd {workflow.workdir_init} && '
'{workflow.snakemakepath} --snakefile {workflow.snakefile} '
'--force -j{cores} --keep-target-files '
'--wait-for-files {job.input} --latency-wait {latency_wait} '
'--benchmark-repeats {benchmark_repeats} '
'{overwrite_workdir} {overwrite_config} --nocolor '
'--notemp --quiet --no-hooks --nolock {target}')
if printshellcmds:
self.exec_job += " --printshellcmds "
if not any(dag.dynamic_output_jobs):
# disable restiction to target rule in case of dynamic rules!
self.exec_job += " --allowed-rules {job.rule.name} "
self.jobname = jobname
self._tmpdir = None
self.cores = cores if cores else ""
self.cluster_config = cluster_config if cluster_config else dict()
self.active_jobs = list()
self.lock = threading.Lock()
self.wait = True
self.wait_thread = threading.Thread(target=self._wait_for_jobs)
self.wait_thread.daemon = True
self.wait_thread.start()
def __init__(self,
workflow,
dag,
cores,
jobname="snakejob.{rulename}.{jobid}.sh",
printreason=False,
quiet=False,
printshellcmds=False,
latency_wait=3,
benchmark_repeats=1,
cluster_config=None,
local_input=None,
max_jobs_per_second=None):
local_input = local_input or []
super().__init__(workflow,
dag,
printreason=printreason,
quiet=quiet,
printshellcmds=printshellcmds,
latency_wait=latency_wait,
benchmark_repeats=benchmark_repeats)
if workflow.snakemakepath is None:
raise ValueError("Cluster executor needs to know the path "
"to the snakemake binary.")
jobscript = workflow.jobscript
if jobscript is None:
jobscript = os.path.join(os.path.dirname(__file__),
self.default_jobscript)
try:
with open(jobscript) as f:
self.jobscript = f.read()
except IOError as e:
raise WorkflowError(e)
if not "jobid" in get_wildcard_names(jobname):
raise WorkflowError(
"Defined jobname (\"{}\") has to contain the wildcard {jobid}."
)
self.exec_job = '\\\n'.join((
'cd {workflow.workdir_init} && ',
'{workflow.snakemakepath} --snakefile {workflow.snakefile} ',
'--force -j{cores} --keep-target-files --keep-shadow --keep-remote ',
'--wait-for-files {wait_for_files} --latency-wait {latency_wait} ',
'--benchmark-repeats {benchmark_repeats} ',
'{overwrite_workdir} {overwrite_config} --nocolor ',
'--notemp --quiet --no-hooks --nolock {target}'))
if printshellcmds:
self.exec_job += " --printshellcmds "
# force threading.Lock() for cluster jobs
self.exec_job += " --force-use-threads "
if not any(dag.dynamic_output_jobs):
# disable restiction to target rule in case of dynamic rules!
self.exec_job += " --allowed-rules {job.rule.name} "
self.jobname = jobname
self._tmpdir = None
self.cores = cores if cores else ""
self.cluster_config = cluster_config if cluster_config else dict()
self.max_jobs_per_second = max_jobs_per_second
if self.max_jobs_per_second:
self.rate_lock = threading.RLock()
self.rate_interval = 1 / self.max_jobs_per_second
self.rate_last_called = 0
self.active_jobs = list()
self.lock = threading.Lock()
self.wait = True
self.wait_thread = threading.Thread(target=self._wait_for_jobs)
self.wait_thread.daemon = True
self.wait_thread.start()