def _run_callbacks(self, name: str, control: dict, _callbacks=None): """ call the callbacks of type name. If a single callback is defined just call it, else iterate sequence and call each """ ex_callbacks = [] if _callbacks is None else _callbacks.get(name, []) func = ex_callbacks + list(iterate(self.get_option(name)) or []) # set default raise if not on_failures are set if not func and name == 'on_failure': func = raise_exception for callback in iterate(func): # if this is a bound method, revert to unbound # this enables using functions that havent defined self with suppress(AttributeError): callback = callback.__func__ # determine needed values from original task parameters cb_sig = inspect.signature(callback) # callback signature expected = set(cb_sig.parameters) & set( control['signature'].parameters) if expected: signature = control['signature'] args, kwargs = control['args'], control['kwargs'] bound_args = signature.bind(*args, *kwargs).arguments kwargs = {item: bound_args[item] for item in expected} else: kwargs = {} # run callback out = apply_partial(callback, partial_dict=control, **kwargs) # if any output raise so task returns that if out is not None: control['outputs'] = out raise ReturnCallbackValue
def __init__(self, wraps: Optional[wrap_node_type] = None, edges: Optional[edge_type] = None): """ Parameters ---------- wraps Wrap objects that serve as nodes in the network. edges Directional edges to connect the nodes. Will add any nodes included in the edge tuple that are not already in the network. Attributes ---------- tasks A mapping of each unique tasks to the wraps that cover it. Eg task:[wrap, ...] wraps A mapping of wraps to the tasks they cover. Eg wrap: task """ # init data structures for representing graphs and dependencies self.wraps = {} self.edges = defaultdict(set) self.tasks = defaultdict(list) self.dependencies = defaultdict(set) self.node_map = defaultdict(set) # add wraps and edges for wrap_ in iterate(wraps): self.add_wrap(wrap_) for edge in iterate(edges): self.add_edge(edge)
def _queue_up(self, inputs, _meta, que, sending_wrap=None, used_functions=None): """ Add this task onto que with given inputs. Normally this wrap and inputs are simply appended to the que, unless a special queue function is defined to allow custom behavior. """ # bail out early if nothing special needs to happen if not (sending_wrap._after_task_funcs or self._before_task_funcs): que.append((self, inputs)) return # get the functions that should be executed. If None do normal queue after_funcs = set(iterate(sending_wrap._after_task_funcs)) before_funcs = set(iterate(self._before_task_funcs)) used_funcs = set(used_functions) if used_functions else set() # if there are funcs to call after operating on data # current no after funcs should be un-used assert not (after_funcs - used_funcs) # if there are funcs to call before allowing task to operate on data if before_funcs - used_funcs: for func in before_funcs - used_funcs: func(self, inputs, _meta, que, sending_wrap, used_funcs) return que.append((self, inputs)) # else just do the normal thing
def _validate_callbacks(self): """ Raise TypeError if not all wrap and task callbacks are valid. """ for name in CALLBACK_NAMES: wrap_cbs = list(iterate(getattr(self, name, None))) task_cbs = list(iterate(getattr(self.task, name, None))) for cb in wrap_cbs + task_cbs: self.task.validate_callback(cb)
def __init__(self, task: "Task", _fixtures, _callbacks, _predicates, args, kwargs): self.task = task # get fixtures passed in from wraps/pypes or use empty dicts _fixtures = _fixtures or EMPTY_FIXTURES self.meta = _fixtures.get("meta", {}) or { } # meta dict from pype or {} # a proxy of outputs from previous tasks self.task_outputs = self.meta.get("outputs", {}) # get a signature and determine if type checking should happen self.sig = task.get_signature() # get bound arguments raise Appropriate Exceptions if bad imputs self.bound = task._bind(self.sig, args, kwargs, _fixtures, self.task_outputs) # create a dictionary of possible fixtures callbacks can ask for self.control = dict( task=task, self=task, signature=self.sig, e=None, outputs=None, inputs=(args, kwargs), args=args, kwargs=kwargs, ) self.wrap_callbacks = _callbacks if _callbacks is not None else {} self.wrap_predicates = list(iterate(_predicates)) self.fixtures = ChainMap(self.control, _fixtures) self.args = args self.kwargs = kwargs
def test_add_single_node(self, digraph, nodes, expected): """ ensure the values put into the digraph node pool show up as expected """ digraph.add_wrap(nodes) assert set(expected) == set(digraph.wraps) tasks = {x.task for x in iterate(nodes)} assert set(digraph.tasks) == tasks
def run_predicates(self): """ run through each predicate and return None if not needed """ task_predicates = list(iterate(self.task.get_option("predicate"))) for pred in task_predicates + self.wrap_predicates: # if a predicate returns a falsy value, bail out of task if not self._hard_exit and not self._bind_and_run(pred): self.final_output = None
def validate_callbacks(self) -> None: """ Iterate over all attached callbacks and raise TypeError if any problems are detected. """ for name in CALLBACK_NAMES: for cb in iterate(getattr(self, name, None)): self.validate_callback(cb)
def add_wrap(self, wrap_: wrap_node_type) -> None: """ Add a single wrap or a sequence of wraps to the network. Parameters ---------- wrap_ A Wrap object or sequence of wrap objects to add to network. """ for wrap_ in iterate(wrap_): assert isinstance(wrap_, wrap.Wrap) if wrap_ not in self.wraps: self.wraps[wrap_] = wrap_.task self.tasks[wrap_.task].append(wrap_) # add dependencies for dep in iterate(wrap_.features["dependencies"]): self.dependencies[dep].add(wrap_)
def _iff(wrap: Wrap, inputs, _meta, que, sending_wrap=None, used_functions=None): """ Function to ensure some condition(s) are true else dont put data on queue. """ for func in iterate(wrap.features['predicate']): if not func(*inputs[0], **inputs[1]): return # if a condition fails bail out wrap._queue_up(inputs, _meta, que, sending_wrap, used_functions={_iff})
def _connect_to_pype( pype: Pype, other, how: Union[str, "task.Task"] = "last", inplace: bool = False, wrap_func: Optional[Callable] = None, ): """ Add task or pype to the pype structure. Parameters ---------- pype Pype to join other Pype, Task, or Wrap instance to connect to pype. how How the connection should be done. Supported options are: "first" : connect other to input_task of pype "last" : connect other to last tasks in pype Task instance : connect other to a specific task in pype inplace If False deepcopy pype before modfiying, else modify in place. wrap_func A function to call on the first wrap of other. Returns ------- Pype connectect """ pype1 = pype if inplace else deepcopy(pype) # get attach points (where the other should be hooked) attach_wraps = _get_attach_wraps(pype1, how) # iterate items to be attached to pype for oth in reversed(iterate(other)): # handle route objects by converting them to pypes if isinstance(oth, dict): oth = _route_to_pype(oth) # wrap or deepcopy to ensure data is ready for next step oth = deepcopy(oth) if isinstance(oth, Pype) else _wrap_task(oth) # apply task_func to other if wrap_func is not None: _apply_wrap_func(oth, wrap_func) if isinstance(oth, Pype): # handle hooking up pypes _pype_to_pype(pype1, attach_wraps, oth) elif isinstance(oth, (task.Task, wrap.Wrap)): # hook up everything else _wrap_to_pype(pype1, attach_wraps, oth) # ensure input task was handled correctly assert len(pype1.flow.tasks[task.pype_input]) == 1 assert task.pype_input in pype1.flow.tasks assert pype1.flow.get_input_wrap().task is task.pype_input return pype1
def _run_callback(self, name: str): """ call the callbacks of type name. If a single callback is defined just call it, else iterate sequence and call each """ if self._hard_exit: return ex_callbacks = self.wrap_callbacks.get(name, []) task_callbacks = self.task.get_option(name) func = ex_callbacks + list(iterate(task_callbacks) or []) # set default raise if not on_failures are set if not func and name == "on_failure": func = raise_exception for callback in iterate(func): # run callback try: out = self._bind_and_run(callback) except ExitTask: self.final_output = None else: if out is not None: self.final_output = out
def add_callback( self, callback: callable, callback_type: str, tasks: Optional[Sequence["task.Task"]] = None, ) -> "Pype": """ Add a callback to all, or some, tasks in the pype. Return new Pype. Parameters ---------- callback The callable to attach to the tasks in the pype callback_type The type of callback: supported types are: on_start, on_failure, on_success, and on_exception tasks A sequence of tasks to apply callback to, else apply to all tasks. Returns ------- A copy of Pype """ assert (callback_type in CALLBACK_NAMES), f"unsported callback type {callback_type}" pype = self.copy() # get a list of wraps to apply callbacks to if tasks is None: wraps = pype.flow.wraps else: wraps_ = [ iterate(selected_wraps) for task in iterate(tasks) for selected_wraps in pype.flow.tasks[task] ] wraps = itertools.chain.from_iterable(wraps_) # apply callbacks for wrap_ in wraps: setattr(wrap_, callback_type, callback) return pype
def iff(self, predicate: Optional[conditional_type] = None) -> 'Wrap': """ Register a condition that must be true for data to continue in pype. Parameters ---------- predicate A function that takes the same inputs as the task and returns a boolean. Returns ------- Wrap """ predicate_list = list(iterate(predicate)) if not predicate_list: return self # do do anything for None for func in iterate(predicate): # ensure compatible signatures self._check_condtion(func) self.features['predicate'] = predicate self.features['is_conditional'] = True self._before_task_funcs = _iff return self
def _get_attach_wraps(pype, how): """ return a list of Wraps which should be connected based on how arg """ out = [] # list of tasks to be connected to other for arg in iterate(how): # make sure input is valid assert how in HOW_ARGS or isinstance(arg, task.Task) if isinstance(arg, task.Task): assert arg in pype.flow.tasks and len(pype.flow.tasks[arg]) == 1 out.append(pype.flow.tasks[arg][0]) elif how == "last": out += pype._last_tasks elif how == "first": out.append(pype.flow.tasks[task.pype_input][0]) return out
def remove_wrap(self, wrap: wrap_node_type, edges=True): """ remove a wrap or sequence of wraps from the network. Also removes all edges that use the removed wraps(s) if edges. """ for wr in iterate(wrap): # pop out of wraps dict self.wraps.pop(wr, None) # pop out of map self.node_map.pop(wr, None) # remove from edges if edges: for edge in set(self.edges): if wr in edge: self.edges.pop(edge, None) # remove tasks that are empty from task list if wrap.task in set(self.tasks): with suppress(ValueError): self.tasks[wr.task].remove(wr) if not len(self.tasks[wr.task]): self.tasks.pop(wr.task, None)
def _get_return_type(bound): """ based on bound signature get the expected return type, mainly this function is to account for typevars """ # check if there are any typevars, swap these out for types bound out = list(iterate(bound.signature.return_annotation)) if any([isinstance(x, TypeVar) for x in out]): # get the expected value for typevars by transversing signature params = bound.signature.parameters vals = bound.arguments new_types = { val.annotation: type(vals[item]) for item, val in params.items() if isinstance(val.annotation, TypeVar) } # swap out return annotations for num, value in enumerate(out): if value in new_types: out[num] = new_types[value] return tuple(out) if len(out) > 1 else out[0] return bound.signature.return_annotation