def pipeline():
return compose(name="netop")(
operation(name="add", needs=["a", "b1"], provides=["ab1"])(add),
operation(name="sub", needs=["a", optional("b2")],
provides=["ab2"])(lambda a, b=1: a - b),
operation(name="abb", needs=["ab1", "ab2"], provides=["asked"])(add),
)
def test_parallel_execution():
import time
def fn(x):
time.sleep(1)
print("fn %s" % (time.time() - t0))
return 1 + x
def fn2(a,b):
time.sleep(1)
print("fn2 %s" % (time.time() - t0))
return a+b
def fn3(z, k=1):
time.sleep(1)
print("fn3 %s" % (time.time() - t0))
return z + k
pipeline = compose(name="l", merge=True)(
# the following should execute in parallel under threaded execution mode
operation(name="a", needs="x", provides="ao")(fn),
operation(name="b", needs="x", provides="bo")(fn),
# this should execute after a and b have finished
operation(name="c", needs=["ao", "bo"], provides="co")(fn2),
operation(name="d",
needs=["ao", modifiers.optional("k")],
provides="do")(fn3),
operation(name="e", needs=["ao", "bo"], provides="eo")(fn2),
operation(name="f", needs="eo", provides="fo")(fn),
operation(name="g", needs="fo", provides="go")(fn)
)
t0 = time.time()
pipeline.set_execution_method("parallel")
result_threaded = pipeline({"x": 10}, ["co", "go", "do"])
print("threaded result")
print(result_threaded)
t0 = time.time()
pipeline.set_execution_method("sequential")
result_sequential = pipeline({"x": 10}, ["co", "go", "do"])
print("sequential result")
print(result_sequential)
# make sure results are the same using either method
assert result_sequential == result_threaded
def test_deleted_optional():
# Test that DeleteInstructions included for optionals do not raise
# exceptions when the corresponding input is not prodided.
# Function to add two values plus an optional third value.
def addplusplus(a, b, c=0):
return a + b + c
# Here, a DeleteInstruction will be inserted for the optional need 'c'.
sum_op1 = operation(name='sum_op1', needs=['a', 'b', modifiers.optional('c')], provides='sum1')(addplusplus)
sum_op2 = operation(name='sum_op2', needs=['sum1', 'sum1'], provides='sum2')(add)
net = compose(name='test_net')(sum_op1, sum_op2)
# DeleteInstructions are used only when a subset of outputs are requested.
results = net({'a': 4, 'b': 3}, outputs=['sum2'])
assert 'sum2' in results
def test_parallel_execution():
import time
def fn(x):
time.sleep(1)
print("fn %s" % (time.time() - t0))
return 1 + x
def fn2(a, b):
time.sleep(1)
print("fn2 %s" % (time.time() - t0))
return a + b
def fn3(z, k=1):
time.sleep(1)
print("fn3 %s" % (time.time() - t0))
return z + k
pipeline = compose(name="l", merge=True)(
# the following should execute in parallel under threaded execution mode
operation(name="a", needs="x", provides="ao")(fn),
operation(name="b", needs="x", provides="bo")(fn),
# this should execute after a and b have finished
operation(name="c", needs=["ao", "bo"], provides="co")(fn2),
operation(name="d",
needs=["ao", modifiers.optional("k")],
provides="do")(fn3),
operation(name="e", needs=["ao", "bo"], provides="eo")(fn2),
operation(name="f", needs="eo", provides="fo")(fn),
operation(name="g", needs="fo", provides="go")(fn))
t0 = time.time()
pipeline.set_execution_method("parallel")
result_threaded = pipeline({"x": 10}, ["co", "go", "do"])
print("threaded result")
print(result_threaded)
t0 = time.time()
pipeline.set_execution_method("sequential")
result_sequential = pipeline({"x": 10}, ["co", "go", "do"])
print("sequential result")
print(result_sequential)
# make sure results are the same using either method
assert result_sequential == result_threaded
def test_deleted_optional():
# Test that DeleteInstructions included for optionals do not raise
# exceptions when the corresponding input is not prodided.
# Function to add two values plus an optional third value.
def addplusplus(a, b, c=0):
return a + b + c
# Here, a DeleteInstruction will be inserted for the optional need 'c'.
sum_op1 = operation(name='sum_op1',
needs=['a', 'b', modifiers.optional('c')],
provides='sum1')(addplusplus)
sum_op2 = operation(name='sum_op2',
needs=['sum1', 'sum1'],
provides='sum2')(add)
net = compose(name='test_net')(sum_op1, sum_op2)
# DeleteInstructions are used only when a subset of outputs are requested.
results = net({'a': 4, 'b': 3}, outputs=['sum2'])
assert 'sum2' in results
def test_optional():
# Test that optional() needs work as expected.
# Function to add two values plus an optional third value.
def addplusplus(a, b, c=0):
return a + b + c
sum_op = operation(name='sum_op1', needs=['a', 'b', modifiers.optional('c')], provides='sum')(addplusplus)
net = compose(name='test_net')(sum_op)
# Make sure output with optional arg is as expected.
named_inputs = {'a': 4, 'b': 3, 'c': 2}
results = net(named_inputs)
assert 'sum' in results
assert results['sum'] == sum(named_inputs.values())
# Make sure output without optional arg is as expected.
named_inputs = {'a': 4, 'b': 3}
results = net(named_inputs)
assert 'sum' in results
assert results['sum'] == sum(named_inputs.values())
def test_optional():
# Test that optional() needs work as expected.
# Function to add two values plus an optional third value.
def addplusplus(a, b, c=0):
return a + b + c
sum_op = operation(name='sum_op1',
needs=['a', 'b', modifiers.optional('c')],
provides='sum')(addplusplus)
net = compose(name='test_net')(sum_op)
# Make sure output with optional arg is as expected.
named_inputs = {'a': 4, 'b': 3, 'c': 2}
results = net(named_inputs)
assert 'sum' in results
assert results['sum'] == sum(named_inputs.values())
# Make sure output without optional arg is as expected.
named_inputs = {'a': 4, 'b': 3}
results = net(named_inputs)
assert 'sum' in results
assert results['sum'] == sum(named_inputs.values())
