def test_and_joined_pypes(self, pype1, pype2, and_joined_pypes):
""" test that the first task of pype2 was hooked to the first
task of pype 1"""
last_task1 = list(pype1._last_tasks)[0].task
last_task2 = list(pype2._last_tasks)[0].task
# call joined pypes, make sure independent
and_joined_pypes(1)
t1 = de_args_kwargs(*and_joined_pypes.outputs[last_task1])
t2 = de_args_kwargs(*and_joined_pypes.outputs[last_task2])
assert t1 == pype1(1)
assert t2 == pype2(1)
def _aggregate(wrap: Wrap, inputs, _meta, que, sending_wrap=None,
used_functions=None, scope='object'):
""" aggregate outputs coming from sending wrap to call on warp """
# determine if aggregation has taken place yet
if not _meta[scope + '_scope_finished']:
# if not aggregate and exit
_meta[scope + '_scope_map'][wrap] = sending_wrap
puts = de_args_kwargs(*inputs)
_meta[scope + "_aggregates"][sending_wrap].append(puts)
return
def _fan(wrap: Wrap,
inputs,
_meta,
que,
sending_wrap=None,
used_functions=None):
""" fan out the output of sending task """
for val in reversed(de_args_kwargs(*inputs)):
wrap._queue_up(args_kwargs(val),
_meta,
que,
sending_wrap,
used_functions={_fan})
def _bind(self, signature, args, kwargs, fixtures, outputs):
"""
Bind args and kwargs to signature. If it fails, look for fixture that
may satisfy binding. If it does not have a value yet, raise an
UnresolvedDependency Exception.
"""
defaults = get_default_names(signature)
_kwargs = {key: fixtures[key] for key in (defaults & set(fixtures))}
try:
bind = signature.bind(*args, **{**kwargs, **_kwargs})
except TypeError: # need to look for fixtures
params = signature.parameters
# determine if any unresolved dependencies exist and raise if so
overlap_keys = set(params) & set(fixtures)
# get values that should be given to parameters.
values = {}
for key in overlap_keys:
if fixtures[key] in outputs:
args_, kwargs_ = outputs[fixtures[key]]
values[key] = de_args_kwargs(args_, kwargs_)
else:
if isinstance(fixtures[key], Task):
raise UnresolvedDependency
values[key] = fixtures[key]
try: # try binding with new inputs
bind = apply_partial(
signature.bind,
*args,
signature=signature,
partial_dict=values,
**kwargs,
)
except TypeError:
msg = (f"{args} and {kwargs} are not valid inputs for {self} "
f"which expects a signature of {signature}")
raise TypeError(msg)
return bind
def _run_queue(self, _meta, que):
""" run the queue until complete """
# run que until complete or all tasks are waiting agg results
assert self.flow.get_input_wrap().task is task.pype_input
fixtures = MapProxy({"meta": _meta, "pype": self, **self._partials})
while len(que):
wrap_, (args, kwargs) = que.pop()
wrap_: wrap.Wrap
try:
output = wrap_(*args, **kwargs, _pype_fixtures=fixtures)
except UnresolvedDependency: # task needs to be put back
_meta["defer_count"][wrap_] += 1 # up task deferment counter
que.appendleft((wrap_, (args, kwargs)))
continue
except TaskReturnedNone: # task returned None
continue
else: # everything went fine
_meta["outputs"][wrap_.task] = output
for neighbor in self.flow.neighbors(wrap_): # queue neighbors
neighbor._queue_up(output, _meta, que, sending_wrap=wrap_)
# run tasks that waited for object scoped aggregations
self._run_aggregations(_meta, que)
_meta["output"].append(de_args_kwargs(*output))
def test_good_pairs(self, args, kwargs, output):
""" ensure args_kwargs returns correct output """
assert de_args_kwargs(args, kwargs) == output
def test_deplayed_dep(self, pype_delayed_dep):
""" ensure a task that needs a dependency that is not yet calculated
can get it. """
pype_delayed_dep(1)
assert de_args_kwargs(*pype_delayed_dep.outputs[divide_numbers]) == 3.
