Python nesting_composite_pipeline Examples

Example #1

def dask_composite_pipeline():
    return nesting_composite_pipeline(
        6,
        2,
        mode_defs=[
            ModeDefinition(executor_defs=default_executors + [dask_executor])
        ])

Example #2

def test_composite_basic_execution():
    a_source = define_stub_solid('A_source', [input_set('A_input')])
    node_a = create_root_solid('A')
    node_b = create_solid_with_deps('B', node_a)
    node_c = create_solid_with_deps('C', node_a)
    node_d = create_solid_with_deps('D', node_b, node_c)

    @composite_solid
    def diamond_composite():
        a = node_a(a_source())
        node_d(B=node_b(a), C=node_c(a))

    @pipeline
    def test_pipeline_single():
        diamond_composite()

    result = execute_pipeline(test_pipeline_single)
    assert result.success

    @pipeline
    def test_pipeline_double():
        diamond_composite.alias('D1')()
        diamond_composite.alias('D2')()

    result = execute_pipeline(test_pipeline_double)
    assert result.success

    @composite_solid
    def wrapped_composite():
        diamond_composite()

    @pipeline
    def test_pipeline_mixed():
        diamond_composite()
        wrapped_composite()

    result = execute_pipeline(test_pipeline_mixed)
    assert result.success

    @composite_solid
    def empty():
        pass

    @pipeline
    def test_pipeline_empty():
        empty()

    result = execute_pipeline(test_pipeline_empty)
    assert result.success

    nested_pipeline = nesting_composite_pipeline(6, 2)

    result = execute_pipeline(nested_pipeline)
    assert result.success

Example #3

def dask_composite_pipeline():
    return nesting_composite_pipeline(
        6,
        2,
        mode_defs=[
            ModeDefinition(
                resource_defs={"io_manager": fs_io_manager},
                executor_defs=default_executors + [dask_executor],
            )
        ],
    )

Example #4

def test_composite_basic_execution(composition_decorator):
    a_source = define_stub_solid("A_source", [input_set("A_input")])
    node_a = create_root_solid("A")
    node_b = create_solid_with_deps("B", node_a)
    node_c = create_solid_with_deps("C", node_a)
    node_d = create_solid_with_deps("D", node_b, node_c)

    @composition_decorator
    def diamond_composite():
        a = node_a(a_source())
        node_d(B=node_b(a), C=node_c(a))

    res = execute_solid(diamond_composite)
    assert res.success

    @pipeline
    def test_pipeline_double():
        diamond_composite.alias("D1")()
        diamond_composite.alias("D2")()

    result = execute_pipeline(test_pipeline_double)
    assert result.success

    @composition_decorator
    def wrapped_composite():
        diamond_composite()

    @pipeline
    def test_pipeline_mixed():
        diamond_composite()
        wrapped_composite()

    result = execute_pipeline(test_pipeline_mixed)
    assert result.success

    @composition_decorator
    def empty():
        pass

    @pipeline
    def test_pipeline_empty():
        empty()

    result = execute_pipeline(test_pipeline_empty)
    assert result.success

    nested_pipeline = nesting_composite_pipeline(6, 2)

    result = execute_pipeline(nested_pipeline)
    assert result.success

Example #5

def test_execute_nested_composite_solids():
    nested_composite_pipeline = nesting_composite_pipeline(2, 2)
    nested_composite_solid = nested_composite_pipeline.solids[0].definition

    res = execute_solid(nested_composite_solid)

    assert res.success
    assert res.solid.name == 'wrap'

    assert res.output_values == {}

    with pytest.raises(
        DagsterInvariantViolationError,
        match=re.escape(
            'Output \'result\' not defined in composite solid \'wrap\': no output mappings were '
            'defined. If you were expecting this output to be present, you may be missing an '
            'output_mapping from an inner solid to its enclosing composite solid.'
        ),
    ):
        _ = res.output_value()

    assert len(res.solid_result_list) == 2

Example #6

File: test_execute.py Project: yetudada/dagster

def composite_pipeline():
    return nesting_composite_pipeline(6, 2, mode_defs=celery_mode_defs)

Example #7

def composite_pipeline():
    return nesting_composite_pipeline(COMPOSITE_DEPTH,
                                      2,
                                      mode_defs=celery_mode_defs)

Example #8

def define_large_pipeline_celery():
    return nesting_composite_pipeline(depth=1,
                                      num_children=6,
                                      mode_defs=celery_mode_defs(),
                                      name='large_pipeline_celery')

Example #9

def dask_composite_pipeline():
    return nesting_composite_pipeline(6, 2)

Example #10

File: repo.py Project: varokas/dagster-1

def define_large_pipeline():
    return nesting_composite_pipeline(depth=1,
                                      num_children=6,
                                      name='large_pipeline')