def pipeline(): return compose(name="netop")( operation(name="add", needs=["a", "b1"], provides=["ab1"])(add), operation(name="sub", needs=["a", optional("b2")], provides=["ab2"])(lambda a, b=1: a - b), operation(name="abb", needs=["ab1", "ab2"], provides=["asked"])(add), )
def test_parallel_execution(): import time def fn(x): time.sleep(1) print("fn %s" % (time.time() - t0)) return 1 + x def fn2(a,b): time.sleep(1) print("fn2 %s" % (time.time() - t0)) return a+b def fn3(z, k=1): time.sleep(1) print("fn3 %s" % (time.time() - t0)) return z + k pipeline = compose(name="l", merge=True)( # the following should execute in parallel under threaded execution mode operation(name="a", needs="x", provides="ao")(fn), operation(name="b", needs="x", provides="bo")(fn), # this should execute after a and b have finished operation(name="c", needs=["ao", "bo"], provides="co")(fn2), operation(name="d", needs=["ao", modifiers.optional("k")], provides="do")(fn3), operation(name="e", needs=["ao", "bo"], provides="eo")(fn2), operation(name="f", needs="eo", provides="fo")(fn), operation(name="g", needs="fo", provides="go")(fn) ) t0 = time.time() pipeline.set_execution_method("parallel") result_threaded = pipeline({"x": 10}, ["co", "go", "do"]) print("threaded result") print(result_threaded) t0 = time.time() pipeline.set_execution_method("sequential") result_sequential = pipeline({"x": 10}, ["co", "go", "do"]) print("sequential result") print(result_sequential) # make sure results are the same using either method assert result_sequential == result_threaded
def test_deleted_optional(): # Test that DeleteInstructions included for optionals do not raise # exceptions when the corresponding input is not prodided. # Function to add two values plus an optional third value. def addplusplus(a, b, c=0): return a + b + c # Here, a DeleteInstruction will be inserted for the optional need 'c'. sum_op1 = operation(name='sum_op1', needs=['a', 'b', modifiers.optional('c')], provides='sum1')(addplusplus) sum_op2 = operation(name='sum_op2', needs=['sum1', 'sum1'], provides='sum2')(add) net = compose(name='test_net')(sum_op1, sum_op2) # DeleteInstructions are used only when a subset of outputs are requested. results = net({'a': 4, 'b': 3}, outputs=['sum2']) assert 'sum2' in results
def test_parallel_execution(): import time def fn(x): time.sleep(1) print("fn %s" % (time.time() - t0)) return 1 + x def fn2(a, b): time.sleep(1) print("fn2 %s" % (time.time() - t0)) return a + b def fn3(z, k=1): time.sleep(1) print("fn3 %s" % (time.time() - t0)) return z + k pipeline = compose(name="l", merge=True)( # the following should execute in parallel under threaded execution mode operation(name="a", needs="x", provides="ao")(fn), operation(name="b", needs="x", provides="bo")(fn), # this should execute after a and b have finished operation(name="c", needs=["ao", "bo"], provides="co")(fn2), operation(name="d", needs=["ao", modifiers.optional("k")], provides="do")(fn3), operation(name="e", needs=["ao", "bo"], provides="eo")(fn2), operation(name="f", needs="eo", provides="fo")(fn), operation(name="g", needs="fo", provides="go")(fn)) t0 = time.time() pipeline.set_execution_method("parallel") result_threaded = pipeline({"x": 10}, ["co", "go", "do"]) print("threaded result") print(result_threaded) t0 = time.time() pipeline.set_execution_method("sequential") result_sequential = pipeline({"x": 10}, ["co", "go", "do"]) print("sequential result") print(result_sequential) # make sure results are the same using either method assert result_sequential == result_threaded
def test_optional(): # Test that optional() needs work as expected. # Function to add two values plus an optional third value. def addplusplus(a, b, c=0): return a + b + c sum_op = operation(name='sum_op1', needs=['a', 'b', modifiers.optional('c')], provides='sum')(addplusplus) net = compose(name='test_net')(sum_op) # Make sure output with optional arg is as expected. named_inputs = {'a': 4, 'b': 3, 'c': 2} results = net(named_inputs) assert 'sum' in results assert results['sum'] == sum(named_inputs.values()) # Make sure output without optional arg is as expected. named_inputs = {'a': 4, 'b': 3} results = net(named_inputs) assert 'sum' in results assert results['sum'] == sum(named_inputs.values())