Ejemplo n.º 1
0
def pipeline():
    return compose(name="netop")(
        operation(name="add", needs=["a", "b1"], provides=["ab1"])(add),
        operation(name="sub", needs=["a", optional("b2")],
                  provides=["ab2"])(lambda a, b=1: a - b),
        operation(name="abb", needs=["ab1", "ab2"], provides=["asked"])(add),
    )
Ejemplo n.º 2
0
def test_parallel_execution():
    import time

    def fn(x):
        time.sleep(1)
        print("fn %s" % (time.time() - t0))
        return 1 + x

    def fn2(a,b):
        time.sleep(1)
        print("fn2 %s" % (time.time() - t0))
        return a+b

    def fn3(z, k=1):
        time.sleep(1)
        print("fn3 %s" % (time.time() - t0))
        return z + k

    pipeline = compose(name="l", merge=True)(

        # the following should execute in parallel under threaded execution mode
        operation(name="a", needs="x", provides="ao")(fn),
        operation(name="b", needs="x", provides="bo")(fn),

        # this should execute after a and b have finished
        operation(name="c", needs=["ao", "bo"], provides="co")(fn2),

        operation(name="d",
                  needs=["ao", modifiers.optional("k")],
                  provides="do")(fn3),

        operation(name="e", needs=["ao", "bo"], provides="eo")(fn2),
        operation(name="f", needs="eo", provides="fo")(fn),
        operation(name="g", needs="fo", provides="go")(fn)


    )

    t0 = time.time()
    pipeline.set_execution_method("parallel")
    result_threaded = pipeline({"x": 10}, ["co", "go", "do"])
    print("threaded result")
    print(result_threaded)

    t0 = time.time()
    pipeline.set_execution_method("sequential")
    result_sequential = pipeline({"x": 10}, ["co", "go", "do"])
    print("sequential result")
    print(result_sequential)

    # make sure results are the same using either method
    assert result_sequential == result_threaded
Ejemplo n.º 3
0
def test_deleted_optional():
    # Test that DeleteInstructions included for optionals do not raise
    # exceptions when the corresponding input is not prodided.

    # Function to add two values plus an optional third value.
    def addplusplus(a, b, c=0):
        return a + b + c

    # Here, a DeleteInstruction will be inserted for the optional need 'c'.
    sum_op1 = operation(name='sum_op1', needs=['a', 'b', modifiers.optional('c')], provides='sum1')(addplusplus)
    sum_op2 = operation(name='sum_op2', needs=['sum1', 'sum1'], provides='sum2')(add)
    net = compose(name='test_net')(sum_op1, sum_op2)

    # DeleteInstructions are used only when a subset of outputs are requested.
    results = net({'a': 4, 'b': 3}, outputs=['sum2'])
    assert 'sum2' in results
Ejemplo n.º 4
0
def test_parallel_execution():
    import time

    def fn(x):
        time.sleep(1)
        print("fn %s" % (time.time() - t0))
        return 1 + x

    def fn2(a, b):
        time.sleep(1)
        print("fn2 %s" % (time.time() - t0))
        return a + b

    def fn3(z, k=1):
        time.sleep(1)
        print("fn3 %s" % (time.time() - t0))
        return z + k

    pipeline = compose(name="l", merge=True)(

        # the following should execute in parallel under threaded execution mode
        operation(name="a", needs="x", provides="ao")(fn),
        operation(name="b", needs="x", provides="bo")(fn),

        # this should execute after a and b have finished
        operation(name="c", needs=["ao", "bo"], provides="co")(fn2),
        operation(name="d",
                  needs=["ao", modifiers.optional("k")],
                  provides="do")(fn3),
        operation(name="e", needs=["ao", "bo"], provides="eo")(fn2),
        operation(name="f", needs="eo", provides="fo")(fn),
        operation(name="g", needs="fo", provides="go")(fn))

    t0 = time.time()
    pipeline.set_execution_method("parallel")
    result_threaded = pipeline({"x": 10}, ["co", "go", "do"])
    print("threaded result")
    print(result_threaded)

    t0 = time.time()
    pipeline.set_execution_method("sequential")
    result_sequential = pipeline({"x": 10}, ["co", "go", "do"])
    print("sequential result")
    print(result_sequential)

    # make sure results are the same using either method
    assert result_sequential == result_threaded
Ejemplo n.º 5
0
def test_deleted_optional():
    # Test that DeleteInstructions included for optionals do not raise
    # exceptions when the corresponding input is not prodided.

    # Function to add two values plus an optional third value.
    def addplusplus(a, b, c=0):
        return a + b + c

    # Here, a DeleteInstruction will be inserted for the optional need 'c'.
    sum_op1 = operation(name='sum_op1',
                        needs=['a', 'b', modifiers.optional('c')],
                        provides='sum1')(addplusplus)
    sum_op2 = operation(name='sum_op2',
                        needs=['sum1', 'sum1'],
                        provides='sum2')(add)
    net = compose(name='test_net')(sum_op1, sum_op2)

    # DeleteInstructions are used only when a subset of outputs are requested.
    results = net({'a': 4, 'b': 3}, outputs=['sum2'])
    assert 'sum2' in results
Ejemplo n.º 6
0
def test_optional():
    # Test that optional() needs work as expected.

    # Function to add two values plus an optional third value.
    def addplusplus(a, b, c=0):
        return a + b + c

    sum_op = operation(name='sum_op1', needs=['a', 'b', modifiers.optional('c')], provides='sum')(addplusplus)

    net = compose(name='test_net')(sum_op)

    # Make sure output with optional arg is as expected.
    named_inputs = {'a': 4, 'b': 3, 'c': 2}
    results = net(named_inputs)
    assert 'sum' in results
    assert results['sum'] == sum(named_inputs.values())

    # Make sure output without optional arg is as expected.
    named_inputs = {'a': 4, 'b': 3}
    results = net(named_inputs)
    assert 'sum' in results
    assert results['sum'] == sum(named_inputs.values())
Ejemplo n.º 7
0
def test_optional():
    # Test that optional() needs work as expected.

    # Function to add two values plus an optional third value.
    def addplusplus(a, b, c=0):
        return a + b + c

    sum_op = operation(name='sum_op1',
                       needs=['a', 'b', modifiers.optional('c')],
                       provides='sum')(addplusplus)

    net = compose(name='test_net')(sum_op)

    # Make sure output with optional arg is as expected.
    named_inputs = {'a': 4, 'b': 3, 'c': 2}
    results = net(named_inputs)
    assert 'sum' in results
    assert results['sum'] == sum(named_inputs.values())

    # Make sure output without optional arg is as expected.
    named_inputs = {'a': 4, 'b': 3}
    results = net(named_inputs)
    assert 'sum' in results
    assert results['sum'] == sum(named_inputs.values())