def _run_callbacks(self, name: str, control: dict, _callbacks=None):
""" call the callbacks of type name. If a single callback is
defined just call it, else iterate sequence and call each """
ex_callbacks = [] if _callbacks is None else _callbacks.get(name, [])
func = ex_callbacks + list(iterate(self.get_option(name)) or [])
# set default raise if not on_failures are set
if not func and name == 'on_failure':
func = raise_exception
for callback in iterate(func):
# if this is a bound method, revert to unbound
# this enables using functions that havent defined self
with suppress(AttributeError):
callback = callback.__func__
# determine needed values from original task parameters
cb_sig = inspect.signature(callback) # callback signature
expected = set(cb_sig.parameters) & set(
control['signature'].parameters)
if expected:
signature = control['signature']
args, kwargs = control['args'], control['kwargs']
bound_args = signature.bind(*args, *kwargs).arguments
kwargs = {item: bound_args[item] for item in expected}
else:
kwargs = {}
# run callback
out = apply_partial(callback, partial_dict=control, **kwargs)
# if any output raise so task returns that
if out is not None:
control['outputs'] = out
raise ReturnCallbackValue
def __init__(self,
wraps: Optional[wrap_node_type] = None,
edges: Optional[edge_type] = None):
"""
Parameters
----------
wraps
Wrap objects that serve as nodes in the network.
edges
Directional edges to connect the nodes. Will add any nodes
included in the edge tuple that are not already in the network.
Attributes
----------
tasks
A mapping of each unique tasks to the wraps that cover it.
Eg task:[wrap, ...]
wraps
A mapping of wraps to the tasks they cover. Eg wrap: task
"""
# init data structures for representing graphs and dependencies
self.wraps = {}
self.edges = defaultdict(set)
self.tasks = defaultdict(list)
self.dependencies = defaultdict(set)
self.node_map = defaultdict(set)
# add wraps and edges
for wrap_ in iterate(wraps):
self.add_wrap(wrap_)
for edge in iterate(edges):
self.add_edge(edge)
def _queue_up(self,
inputs,
_meta,
que,
sending_wrap=None,
used_functions=None):
"""
Add this task onto que with given inputs.
Normally this wrap and inputs are simply appended to the que, unless
a special queue function is defined to allow custom behavior.
"""
# bail out early if nothing special needs to happen
if not (sending_wrap._after_task_funcs or self._before_task_funcs):
que.append((self, inputs))
return
# get the functions that should be executed. If None do normal queue
after_funcs = set(iterate(sending_wrap._after_task_funcs))
before_funcs = set(iterate(self._before_task_funcs))
used_funcs = set(used_functions) if used_functions else set()
# if there are funcs to call after operating on data
# current no after funcs should be un-used
assert not (after_funcs - used_funcs)
# if there are funcs to call before allowing task to operate on data
if before_funcs - used_funcs:
for func in before_funcs - used_funcs:
func(self, inputs, _meta, que, sending_wrap, used_funcs)
return
que.append((self, inputs)) # else just do the normal thing
def _validate_callbacks(self):
"""
Raise TypeError if not all wrap and task callbacks are valid.
"""
for name in CALLBACK_NAMES:
wrap_cbs = list(iterate(getattr(self, name, None)))
task_cbs = list(iterate(getattr(self.task, name, None)))
for cb in wrap_cbs + task_cbs:
self.task.validate_callback(cb)
def __init__(self, task: "Task", _fixtures, _callbacks, _predicates, args,
kwargs):
self.task = task
# get fixtures passed in from wraps/pypes or use empty dicts
_fixtures = _fixtures or EMPTY_FIXTURES
self.meta = _fixtures.get("meta", {}) or {
} # meta dict from pype or {}
# a proxy of outputs from previous tasks
self.task_outputs = self.meta.get("outputs", {})
# get a signature and determine if type checking should happen
self.sig = task.get_signature()
# get bound arguments raise Appropriate Exceptions if bad imputs
self.bound = task._bind(self.sig, args, kwargs, _fixtures,
self.task_outputs)
# create a dictionary of possible fixtures callbacks can ask for
self.control = dict(
task=task,
self=task,
signature=self.sig,
e=None,
outputs=None,
inputs=(args, kwargs),
args=args,
kwargs=kwargs,
)
self.wrap_callbacks = _callbacks if _callbacks is not None else {}
self.wrap_predicates = list(iterate(_predicates))
self.fixtures = ChainMap(self.control, _fixtures)
self.args = args
self.kwargs = kwargs
def test_add_single_node(self, digraph, nodes, expected):
""" ensure the values put into the digraph node pool show up as
expected """
digraph.add_wrap(nodes)
assert set(expected) == set(digraph.wraps)
tasks = {x.task for x in iterate(nodes)}
assert set(digraph.tasks) == tasks
def run_predicates(self):
""" run through each predicate and return None if not needed """
task_predicates = list(iterate(self.task.get_option("predicate")))
for pred in task_predicates + self.wrap_predicates:
# if a predicate returns a falsy value, bail out of task
if not self._hard_exit and not self._bind_and_run(pred):
self.final_output = None
def validate_callbacks(self) -> None:
"""
Iterate over all attached callbacks and raise TypeError if
any problems are detected.
"""
for name in CALLBACK_NAMES:
for cb in iterate(getattr(self, name, None)):
self.validate_callback(cb)
def add_wrap(self, wrap_: wrap_node_type) -> None:
"""
Add a single wrap or a sequence of wraps to the network.
Parameters
----------
wrap_
A Wrap object or sequence of wrap objects to add to network.
"""
for wrap_ in iterate(wrap_):
assert isinstance(wrap_, wrap.Wrap)
if wrap_ not in self.wraps:
self.wraps[wrap_] = wrap_.task
self.tasks[wrap_.task].append(wrap_)
# add dependencies
for dep in iterate(wrap_.features["dependencies"]):
self.dependencies[dep].add(wrap_)
def _iff(wrap: Wrap, inputs, _meta, que, sending_wrap=None,
used_functions=None):
"""
Function to ensure some condition(s) are true else dont put data on queue.
"""
for func in iterate(wrap.features['predicate']):
if not func(*inputs[0], **inputs[1]):
return # if a condition fails bail out
wrap._queue_up(inputs, _meta, que, sending_wrap, used_functions={_iff})
def _connect_to_pype(
pype: Pype,
other,
how: Union[str, "task.Task"] = "last",
inplace: bool = False,
wrap_func: Optional[Callable] = None,
):
"""
Add task or pype to the pype structure.
Parameters
----------
pype
Pype to join
other
Pype, Task, or Wrap instance to connect to pype.
how
How the connection should be done. Supported options are:
"first" : connect other to input_task of pype
"last" : connect other to last tasks in pype
Task instance : connect other to a specific task in pype
inplace
If False deepcopy pype before modfiying, else modify in place.
wrap_func
A function to call on the first wrap of other.
Returns
-------
Pype connectect
"""
pype1 = pype if inplace else deepcopy(pype)
# get attach points (where the other should be hooked)
attach_wraps = _get_attach_wraps(pype1, how)
# iterate items to be attached to pype
for oth in reversed(iterate(other)):
# handle route objects by converting them to pypes
if isinstance(oth, dict):
oth = _route_to_pype(oth)
# wrap or deepcopy to ensure data is ready for next step
oth = deepcopy(oth) if isinstance(oth, Pype) else _wrap_task(oth)
# apply task_func to other
if wrap_func is not None:
_apply_wrap_func(oth, wrap_func)
if isinstance(oth, Pype): # handle hooking up pypes
_pype_to_pype(pype1, attach_wraps, oth)
elif isinstance(oth,
(task.Task, wrap.Wrap)): # hook up everything else
_wrap_to_pype(pype1, attach_wraps, oth)
# ensure input task was handled correctly
assert len(pype1.flow.tasks[task.pype_input]) == 1
assert task.pype_input in pype1.flow.tasks
assert pype1.flow.get_input_wrap().task is task.pype_input
return pype1
def _run_callback(self, name: str):
""" call the callbacks of type name. If a single callback is
defined just call it, else iterate sequence and call each """
if self._hard_exit:
return
ex_callbacks = self.wrap_callbacks.get(name, [])
task_callbacks = self.task.get_option(name)
func = ex_callbacks + list(iterate(task_callbacks) or [])
# set default raise if not on_failures are set
if not func and name == "on_failure":
func = raise_exception
for callback in iterate(func):
# run callback
try:
out = self._bind_and_run(callback)
except ExitTask:
self.final_output = None
else:
if out is not None:
self.final_output = out
def add_callback(
self,
callback: callable,
callback_type: str,
tasks: Optional[Sequence["task.Task"]] = None,
) -> "Pype":
"""
Add a callback to all, or some, tasks in the pype. Return new Pype.
Parameters
----------
callback
The callable to attach to the tasks in the pype
callback_type
The type of callback: supported types are:
on_start, on_failure, on_success, and on_exception
tasks
A sequence of tasks to apply callback to, else apply to all tasks.
Returns
-------
A copy of Pype
"""
assert (callback_type
in CALLBACK_NAMES), f"unsported callback type {callback_type}"
pype = self.copy()
# get a list of wraps to apply callbacks to
if tasks is None:
wraps = pype.flow.wraps
else:
wraps_ = [
iterate(selected_wraps) for task in iterate(tasks)
for selected_wraps in pype.flow.tasks[task]
]
wraps = itertools.chain.from_iterable(wraps_)
# apply callbacks
for wrap_ in wraps:
setattr(wrap_, callback_type, callback)
return pype
def iff(self, predicate: Optional[conditional_type] = None) -> 'Wrap':
"""
Register a condition that must be true for data to continue in pype.
Parameters
----------
predicate
A function that takes the same inputs as the task and returns a
boolean.
Returns
-------
Wrap
"""
predicate_list = list(iterate(predicate))
if not predicate_list:
return self # do do anything for None
for func in iterate(predicate): # ensure compatible signatures
self._check_condtion(func)
self.features['predicate'] = predicate
self.features['is_conditional'] = True
self._before_task_funcs = _iff
return self
def _get_attach_wraps(pype, how):
""" return a list of Wraps which should be connected based on how arg """
out = [] # list of tasks to be connected to other
for arg in iterate(how):
# make sure input is valid
assert how in HOW_ARGS or isinstance(arg, task.Task)
if isinstance(arg, task.Task):
assert arg in pype.flow.tasks and len(pype.flow.tasks[arg]) == 1
out.append(pype.flow.tasks[arg][0])
elif how == "last":
out += pype._last_tasks
elif how == "first":
out.append(pype.flow.tasks[task.pype_input][0])
return out
def remove_wrap(self, wrap: wrap_node_type, edges=True):
"""
remove a wrap or sequence of wraps from the network.
Also removes all edges that use the removed wraps(s) if edges.
"""
for wr in iterate(wrap):
# pop out of wraps dict
self.wraps.pop(wr, None)
# pop out of map
self.node_map.pop(wr, None)
# remove from edges
if edges:
for edge in set(self.edges):
if wr in edge:
self.edges.pop(edge, None)
# remove tasks that are empty from task list
if wrap.task in set(self.tasks):
with suppress(ValueError):
self.tasks[wr.task].remove(wr)
if not len(self.tasks[wr.task]):
self.tasks.pop(wr.task, None)
def _get_return_type(bound):
""" based on bound signature get the expected return type, mainly
this function is to account for typevars """
# check if there are any typevars, swap these out for types bound
out = list(iterate(bound.signature.return_annotation))
if any([isinstance(x, TypeVar) for x in out]):
# get the expected value for typevars by transversing signature
params = bound.signature.parameters
vals = bound.arguments
new_types = {
val.annotation: type(vals[item])
for item, val in params.items()
if isinstance(val.annotation, TypeVar)
}
# swap out return annotations
for num, value in enumerate(out):
if value in new_types:
out[num] = new_types[value]
return tuple(out) if len(out) > 1 else out[0]
return bound.signature.return_annotation
