def apply(name, res, threads=None):
if callable(res):
aux = dict(rulename=self.name)
if threads is not None:
aux["threads"] = threads
try:
res, _ = self.apply_input_function(
res,
wildcards,
input=input,
attempt=attempt,
incomplete_checkpoint_func=lambda e: 0,
raw_exceptions=True,
**aux)
except (Exception, BaseException) as e:
raise InputFunctionException(e,
rule=self,
wildcards=wildcards)
if isinstance(res, float):
# round to integer
res = int(round(res))
if not isinstance(res, int) and not isinstance(res, str):
raise WorkflowError(
"Resources function did not return int, float (floats are "
"rouded to the nearest integer), or str.",
rule=self,
)
if isinstance(res, int):
global_res = self.workflow.global_resources.get(name, res)
if global_res is not None:
res = min(global_res, res)
return res
def apply(name, res, threads=None):
if callable(res):
aux = dict(rulename=self.name)
if threads:
aux["threads"] = threads
try:
try:
res, _ = self.apply_input_function(
res,
wildcards,
input=input,
attempt=attempt,
incomplete_checkpoint_func=lambda e: 0,
raw_exceptions=True,
**aux)
except FileNotFoundError as e:
# Resources can depend on input files. Since expansion can happen during dryrun,
# where input files are not yet present, we need to skip such resources and
# mark them as [TBD].
if e.filename in input:
# use zero for resource if it cannot yet be determined
res = TBDInt(0)
else:
raise e
except e:
raise InputFunctionException(e,
rule=self,
wildcards=wildcards)
if not isinstance(res, int):
raise WorkflowError(
"Resources function did not return int.")
res = min(self.workflow.global_resources.get(name, res), res)
return res
def apply_input_function(self,
func,
wildcards,
incomplete_checkpoint_func=lambda e: None,
raw_exceptions=False,
**aux_params):
incomplete = False
if isinstance(func, _IOFile):
func = func._file.callable
elif isinstance(func, AnnotatedString):
func = func.callable
sig = inspect.signature(func)
_aux_params = {
k: v
for k, v in aux_params.items() if k in sig.parameters
}
try:
value = func(Wildcards(fromdict=wildcards), **_aux_params)
except IncompleteCheckpointException as e:
value = incomplete_checkpoint_func(e)
incomplete = True
except FileNotFoundError as e:
# Function evaluation can depend on input files. Since expansion can happen during dryrun,
# where input files are not yet present, we need to skip such cases and
# mark them as <TBD>.
if e.filename in aux_params["input"]:
value = TBDString()
else:
raise e
except (Exception, BaseException) as e:
if raw_exceptions:
raise e
else:
raise InputFunctionException(e, rule=self, wildcards=wildcards)
return value, incomplete
def _apply_wildcards(newitems,
olditems,
wildcards,
wildcards_obj,
concretize=apply_wildcards,
check_return_type=check_string_type,
ruleio=None,
no_flattening=False):
for name, item in olditems.allitems():
start = len(newitems)
is_iterable = True
if callable(item):
try:
item = item(wildcards_obj)
except (Exception, BaseException) as e:
raise InputFunctionException(e,
rule=self,
wildcards=wildcards)
if not_iterable(item) or no_flattening:
item = [item]
is_iterable = False
for item_ in item:
check_return_type(item_)
concrete = concretize(item_, wildcards)
newitems.append(concrete)
if ruleio is not None:
ruleio[concrete] = item_
if name:
newitems.set_name(
name,
start,
end=len(newitems) if is_iterable else None)
def apply_input_function(self,
func,
wildcards,
incomplete_checkpoint_func=lambda e: None,
raw_exceptions=False,
**aux_params):
incomplete = False
if isinstance(func, _IOFile):
func = func._file.callable
elif isinstance(func, AnnotatedString):
func = func.callable
sig = inspect.signature(func)
_aux_params = {
k: v
for k, v in aux_params.items() if k in sig.parameters
}
try:
value = func(Wildcards(fromdict=wildcards), **_aux_params)
except IncompleteCheckpointException as e:
value = incomplete_checkpoint_func(e)
incomplete = True
except (Exception, BaseException) as e:
if raw_exceptions:
raise e
else:
raise InputFunctionException(e, rule=self, wildcards=wildcards)
return value, incomplete
def apply_input_function(self, func, wildcards, **aux_params):
sig = inspect.signature(func)
_aux_params = {k: v for k, v in aux_params.items() if k in sig.parameters}
try:
value = func(Wildcards(fromdict=wildcards), **_aux_params)
except (Exception, BaseException) as e:
raise InputFunctionException(e, rule=self, wildcards=wildcards)
return value
def apply_input_function(
self,
func,
wildcards,
incomplete_checkpoint_func=lambda e: None,
raw_exceptions=False,
groupid=None,
**aux_params,
):
incomplete = False
if isinstance(func, _IOFile):
func = func._file.callable
elif isinstance(func, AnnotatedString):
func = func.callable
if "groupid" in get_function_params(func):
if groupid is not None:
aux_params["groupid"] = groupid
else:
# Return empty list of files and incomplete marker
# the job will be reevaluated once groupids have been determined
return [], True
_aux_params = get_input_function_aux_params(func, aux_params)
try:
value = func(Wildcards(fromdict=wildcards), **_aux_params)
if isinstance(value, types.GeneratorType):
# generators should be immediately collected here,
# otherwise we would miss any exceptions and
# would have to capture them again later.
value = list(value)
except IncompleteCheckpointException as e:
value = incomplete_checkpoint_func(e)
incomplete = True
except FileNotFoundError as e:
# Function evaluation can depend on input files. Since expansion can happen during dryrun,
# where input files are not yet present, we need to skip such cases and
# mark them as <TBD>.
if "input" in aux_params and e.filename in aux_params["input"]:
value = TBDString()
else:
raise e
except (Exception, BaseException) as e:
if raw_exceptions:
raise e
else:
raise InputFunctionException(e, rule=self, wildcards=wildcards)
return value, incomplete
def expand_report_argument(item, wildcards, job):
if is_callable(item):
aux_params = get_input_function_aux_params(item,
{"params": job.params})
try:
item = item(wildcards, **aux_params)
except Exception as e:
raise InputFunctionException(e, rule=job.rule, wildcards=wildcards)
if isinstance(item, str):
try:
return apply_wildcards(item, wildcards)
except AttributeError as e:
raise WorkflowError("Failed to resolve wildcards.",
e,
rule=job.rule)
else:
return item
def apply(name, res, threads=None):
if callable(res):
aux = dict(rulename=self.name)
if threads is not None:
aux["threads"] = threads
try:
res, _ = self.apply_input_function(
res,
wildcards,
input=input,
attempt=attempt,
incomplete_checkpoint_func=lambda e: 0,
raw_exceptions=True,
**aux,
)
except (Exception, BaseException) as e:
raise InputFunctionException(e,
rule=self,
wildcards=wildcards)
if isinstance(res, float):
# round to integer
res = int(round(res))
if not isinstance(res, int) and not isinstance(res, str):
raise WorkflowError(
f"Resource {name} is neither int, float(would be rounded to nearest int), or str.",
rule=self,
)
global_res = self.workflow.global_resources.get(name)
if global_res is not None:
if not isinstance(res,
TBDString) and type(res) != type(global_res):
global_type = ("an int" if isinstance(global_res, int) else
type(global_res))
raise WorkflowError(
f"Resource {name} is of type {type(res).__name__} but global resource constraint "
f"defines {global_type} with value {global_res}. "
"Resources with the same name need to have the same types (int, float, or str are allowed).",
rule=self,
)
if isinstance(res, int):
res = min(global_res, res)
return res
def _apply_wildcards(newitems,
olditems,
wildcards,
wildcards_obj,
concretize=apply_wildcards,
ruleio=None):
for name, item in olditems.allitems():
start = len(newitems)
is_iterable = True
if callable(item):
try:
item = item(wildcards_obj)
except (Exception, BaseException) as e:
raise InputFunctionException(e, rule=self)
if not_iterable(item):
item = [item]
is_iterable = False
for item_ in item:
if not isinstance(item_, str):
raise RuleException(
"Input function did not return str or list of str.",
rule=self)
concrete = concretize(item_, wildcards)
newitems.append(concrete)
if ruleio is not None:
ruleio[concrete] = item_
else:
if not_iterable(item):
item = [item]
is_iterable = False
for item_ in item:
concrete = concretize(item_, wildcards)
newitems.append(concrete)
if ruleio is not None:
ruleio[concrete] = item_
if name:
newitems.set_name(
name,
start,
end=len(newitems) if is_iterable else None)