def funcs_to_operations(*funcs, exclude_names=()) -> Operation:
"""Get an operation from a callable"""
for func in funcs:
_func_name = func_name(func, exclude_names)
exclude_names = exclude_names + (_func_name, )
needs = arg_names(func, _func_name, exclude_names)
exclude_names = exclude_names + tuple(needs)
yield operation(
func,
name=_func_name,
needs=needs,
provides=_func_name,
)
def test_input_output_based_pruning():
# Tests to make sure we don't need to pass graph inputs if they're not
# needed to compute the requested outputs or of we're provided with
# inputs that are further downstream in the graph.
c = 2
sum2 = 5
# Set up a network such that we don't need to provide a or b d if we only
# request sum3 as output and if we provide sum2.
sum_op1 = operation(name='sum_op1', needs=['a', 'b'],
provides='sum1')(add)
sum_op2 = operation(name='sum_op2', needs=['c', 'd'],
provides='sum2')(add)
sum_op3 = operation(name='sum_op3',
needs=['c', 'sum2'],
provides='sum3')(add)
net = compose(name='test_net')(sum_op1, sum_op2, sum_op3)
results = net({'c': c, 'sum2': sum2}, outputs=['sum3'])
# Make sure we got expected result without having to pass a, b, or d.
assert 'sum3' in results
assert results['sum3'] == add(c, sum2)
def test_parallel_execution():
import time
def fn(x):
time.sleep(1)
print('fn %s' % (time.time() - t0))
return 1 + x
def fn2(a, b):
time.sleep(1)
print('fn2 %s' % (time.time() - t0))
return a + b
def fn3(z, k=1):
time.sleep(1)
print('fn3 %s' % (time.time() - t0))
return z + k
pipeline = compose(name='l', merge=True)(
# the following should execute in parallel under threaded execution mode
operation(name='a', needs='x', provides='ao')(fn),
operation(name='b', needs='x', provides='bo')(fn),
# this should execute after a and b have finished
operation(name='c', needs=['ao', 'bo'], provides='co')(fn2),
operation(name='d', needs=['ao', optional('k')],
provides='do')(fn3),
operation(name='e', needs=['ao', 'bo'], provides='eo')(fn2),
operation(name='f', needs='eo', provides='fo')(fn),
operation(name='g', needs='fo', provides='go')(fn),
)
t0 = time.time()
pipeline.set_execution_method('parallel')
result_threaded = pipeline({'x': 10}, ['co', 'go', 'do'])
print('threaded result')
print(result_threaded)
t0 = time.time()
pipeline.set_execution_method('sequential')
result_sequential = pipeline({'x': 10}, ['co', 'go', 'do'])
print('sequential result')
print(result_sequential)
# make sure results are the same using either method
assert result_sequential == result_threaded
def test_network():
# Sum operation, late-bind compute function
sum_op1 = operation(name='sum_op1',
needs=['a', 'b'],
provides='sum_ab')(add)
# sum_op1 is callable
print(sum_op1(1, 2))
# Multiply operation, decorate in-place
@operation(name='mul_op1',
needs=['sum_ab', 'b'],
provides='sum_ab_times_b')
def mul_op1(a, b):
return a * b
# mul_op1 is callable
print(mul_op1(1, 2))
# Pow operation
@operation(
name='pow_op1',
needs='sum_ab',
provides=['sum_ab_p1', 'sum_ab_p2', 'sum_ab_p3'],
params={'exponent': 3},
)
def pow_op1(a, exponent=2):
return [math.pow(a, y) for y in range(1, exponent + 1)]
print(pow_op1._compute({'sum_ab': 2}, ['sum_ab_p2']))
# Partial operation that is bound at a later time
partial_op = operation(name='sum_op2',
needs=['sum_ab_p1', 'sum_ab_p2'],
provides='p1_plus_p2')
# Bind the partial operation
sum_op2 = partial_op(add)
# Sum operation, early-bind compute function
sum_op_factory = operation(add)
sum_op3 = sum_op_factory(name='sum_op3',
needs=['a', 'b'],
provides='sum_ab2')
# sum_op3 is callable
print(sum_op3(5, 6))
# compose network
net = compose(name='my network')(sum_op1, mul_op1, pow_op1, sum_op2,
sum_op3)
#
# Running the network
#
# get all outputs
pprint(net({'a': 1, 'b': 2}))
# get specific outputs
pprint(net({'a': 1, 'b': 2}, outputs=['sum_ab_times_b']))
# start with inputs already computed
pprint(net({'sum_ab': 1, 'b': 2}, outputs=['sum_ab_times_b']))
def funcs_to_operators(*funcs, exclude_names=()) -> Operation:
"""Get an operation from a callable"""
for func, operation in zip(funcs,
funcs_to_operations(funcs, exclude_names)):
yield operation(func)