Esempio n. 1
0
def test_pipeline_step_key_subset_execution_wrong_step_key_in_step_output_handles(
):
    pipeline_def = define_addy_pipeline()
    old_run_id = str(uuid.uuid4())
    environment_dict = env_with_fs(
        {'solids': {
            'add_one': {
                'inputs': {
                    'num': {
                        'value': 3
                    }
                }
            }
        }})
    result = execute_pipeline(pipeline_def,
                              environment_dict=environment_dict,
                              run_config=RunConfig(run_id=old_run_id))
    assert result.success
    assert result.run_id == old_run_id

    new_run_id = str(uuid.uuid4())

    with pytest.raises(DagsterExecutionStepNotFoundError):
        execute_pipeline(
            pipeline_def,
            environment_dict=environment_dict,
            run_config=RunConfig(
                run_id=new_run_id,
                reexecution_config=ReexecutionConfig(
                    previous_run_id=result.run_id,
                    step_output_handles=[
                        StepOutputHandle('invalid_in_step_output_handles')
                    ],
                ),
                step_keys_to_execute=['add_two.compute'],
            ),
        )
Esempio n. 2
0
def test_execution_plan_reexecution_with_in_memory():
    pipeline_def = define_addy_pipeline()
    instance = DagsterInstance.ephemeral()
    old_run_id = str(uuid.uuid4())
    environment_dict = {'solids': {'add_one': {'inputs': {'num': {'value': 3}}}}}
    result = execute_pipeline(
        pipeline_def,
        environment_dict=environment_dict,
        run_config=RunConfig(run_id=old_run_id),
        instance=instance,
    )

    assert result.success

    ## re-execute add_two

    new_run_id = str(uuid.uuid4())

    in_memory_run_config = RunConfig(
        run_id=new_run_id,
        reexecution_config=ReexecutionConfig(
            previous_run_id=result.run_id, step_output_handles=[StepOutputHandle('add_one.compute')]
        ),
    )

    execution_plan = create_execution_plan(
        pipeline_def, environment_dict=environment_dict, run_config=in_memory_run_config
    )

    with pytest.raises(DagsterInvariantViolationError):
        execute_plan(
            execution_plan,
            environment_dict=environment_dict,
            run_config=in_memory_run_config,
            step_keys_to_execute=['add_two.compute'],
            instance=instance,
        )
Esempio n. 3
0
def _do_execute_plan(graphene_info, execution_params, dauphin_pipeline):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.inst_param(execution_params, 'execution_params', ExecutionParams)

    event_records = []

    run_config = RunConfig(
        run_id=execution_params.execution_metadata.run_id,
        mode=execution_params.mode,
        tags=execution_params.execution_metadata.tags,
        event_callback=event_records.append,
    )

    execution_plan = create_execution_plan(
        pipeline=dauphin_pipeline.get_dagster_pipeline(),
        environment_dict=execution_params.environment_dict,
        run_config=run_config,
    )

    if execution_params.step_keys:
        for step_key in execution_params.step_keys:
            if not execution_plan.has_step(step_key):
                raise UserFacingGraphQLError(
                    graphene_info.schema.type_named('InvalidStepError')(invalid_step_key=step_key)
                )

    execute_plan(
        execution_plan=execution_plan,
        environment_dict=execution_params.environment_dict,
        run_config=run_config,
        step_keys_to_execute=execution_params.step_keys,
    )

    def to_graphql_event(event_record):
        return from_dagster_event_record(
            graphene_info, event_record, dauphin_pipeline, execution_plan
        )

    return graphene_info.schema.type_named('ExecutePlanSuccess')(
        pipeline=dauphin_pipeline,
        has_failures=any(
            er
            for er in event_records
            if er.is_dagster_event and er.dagster_event.event_type == DagsterEventType.STEP_FAILURE
        ),
        step_events=list(
            map(to_graphql_event, filter(lambda er: er.is_dagster_event, event_records))
        ),
    )
Esempio n. 4
0
def test_using_gcs_for_subplan(gcs_bucket):
    pipeline_def = define_inty_pipeline()

    environment_dict = {'storage': {'gcs': {'config': {'gcs_bucket': gcs_bucket}}}}

    run_id = str(uuid.uuid4())

    execution_plan = create_execution_plan(
        pipeline_def, environment_dict=environment_dict, run_config=RunConfig(run_id=run_id)
    )

    assert execution_plan.get_step_by_key('return_one.compute')

    step_keys = ['return_one.compute']
    instance = DagsterInstance.ephemeral()
    pipeline_run = PipelineRun.create_empty_run(
        pipeline_def.name, run_id=run_id, environment_dict=environment_dict
    )

    return_one_step_events = list(
        execute_plan(
            execution_plan.build_subset_plan(step_keys),
            environment_dict=environment_dict,
            pipeline_run=pipeline_run,
            instance=instance,
        )
    )

    assert get_step_output(return_one_step_events, 'return_one.compute')
    with scoped_pipeline_context(pipeline_def, environment_dict, pipeline_run, instance) as context:
        store = GCSIntermediateStore(
            gcs_bucket, run_id, client=context.scoped_resources_builder.build().gcs.client
        )
        assert store.has_intermediate(context, 'return_one.compute')
        assert store.get_intermediate(context, 'return_one.compute', Int).obj == 1

    add_one_step_events = list(
        execute_plan(
            execution_plan.build_subset_plan(['add_one.compute']),
            environment_dict=environment_dict,
            pipeline_run=pipeline_run,
            instance=instance,
        )
    )

    assert get_step_output(add_one_step_events, 'add_one.compute')
    with scoped_pipeline_context(pipeline_def, environment_dict, pipeline_run, instance) as context:
        assert store.has_intermediate(context, 'add_one.compute')
        assert store.get_intermediate(context, 'add_one.compute', Int).obj == 2
Esempio n. 5
0
def test_multiproc_event_sink():
    pipeline = ExecutionTargetHandle.for_pipeline_python_file(
        __file__, 'define_diamond_pipeline'
    ).build_pipeline_definition()

    sink = InMemoryEventSink()

    result = execute_pipeline(
        pipeline,
        run_config=RunConfig(event_sink=sink),
        environment_dict={'storage': {'filesystem': {}}, 'execution': {'multiprocess': {}}},
    )

    assert result.success
    assert len(result.event_list) == len(sink.dagster_event_records)
Esempio n. 6
0
def do_execute_command(pipeline, env_file_list, mode=None):
    check.inst_param(pipeline, 'pipeline', PipelineDefinition)
    env_file_list = check.opt_list_param(env_file_list,
                                         'env_file_list',
                                         of_type=str)

    environment_dict = load_yaml_from_glob_list(
        env_file_list) if env_file_list else {}

    return execute_pipeline(
        pipeline,
        environment_dict=environment_dict,
        run_config=RunConfig(mode=mode),
        instance=DagsterInstance.get(),
    )
Esempio n. 7
0
def do_execute_command(pipeline, env_file_list, raise_on_error):
    check.inst_param(pipeline, 'pipeline', PipelineDefinition)
    env_file_list = check.opt_list_param(env_file_list,
                                         'env_file_list',
                                         of_type=str)

    environment_dict = load_yaml_from_glob_list(
        env_file_list) if env_file_list else {}

    return execute_pipeline(
        pipeline,
        environment_dict=environment_dict,
        run_config=RunConfig(executor_config=InProcessExecutorConfig(
            raise_on_error=raise_on_error)),
    )
Esempio n. 8
0
def test_ingest_pipeline_fast_filesystem_storage(postgres):
    ingest_config_dict = load_yaml_from_globs(
        config_path('local_base.yaml'),
        config_path('local_fast_ingest.yaml'),
        config_path('filesystem_storage.yaml'),
    )
    ingest_config_dict = enviroment_overrides(ingest_config_dict)
    result_ingest = execute_pipeline(
        ingest_pipeline_def,
        ingest_config_dict,
        run_config=RunConfig(mode='local'),
        instance=DagsterInstance.local_temp(),
    )

    assert result_ingest.success
Esempio n. 9
0
def test_no_repo_registration_error():
    with pytest.raises(
            DagstermillError,
            match=
            'Error occurred during the execution of Dagstermill solid no_repo_reg',
    ) as exc:
        execute_pipeline(define_no_repo_registration_error_pipeline())
    assert (
        'If Dagstermill solids have outputs that require serialization strategies'
        in exc.value.original_exc_info[1].args[0])

    with exec_for_test(
            define_no_repo_registration_error_pipeline(),
            run_config=RunConfig.nonthrowing_in_process()) as result:
        assert not result.success
def test_error_notebook():
    with pytest.raises(
        DagstermillError, match='Error occurred during the execution of Dagstermill solid'
    ) as exc:
        with exec_for_test('define_error_pipeline') as result:
            pass

    assert 'Someone set up us the bomb' in exc.value.original_exc_info[1].args[0]

    with exec_for_test(
        'define_error_pipeline', run_config=RunConfig.nonthrowing_in_process()
    ) as result:
        assert not result.success
        assert result.step_event_list[1].event_type.value == 'STEP_MATERIALIZATION'
        assert result.step_event_list[2].event_type.value == 'STEP_FAILURE'
Esempio n. 11
0
def test_execute_multi_mode_loggers_with_single_logger():
    pipeline_def, foo_logger_captured_results, bar_logger_captured_results = (
        define_multi_mode_with_loggers_pipeline()
    )

    execute_pipeline(
        pipeline_def,
        run_config=RunConfig(mode='foo_mode'),
        environment_dict={'loggers': {'foo': {'config': {'log_level': 'DEBUG'}}}},
    )

    assert not bar_logger_captured_results

    original_messages = parse_captured_results(foo_logger_captured_results)
    assert len([x for x in original_messages if 'Here we are' in x]) == 1
Esempio n. 12
0
def test_download_csv_locally_pipeline(mocker):
    # Setup download mocks
    mocker.patch('dagster_examples.bay_bikes.solids.requests')
    mocker.patch('dagster_examples.bay_bikes.solids._write_chunks_to_fp')
    mocker.patch('dagster_examples.bay_bikes.solids._unzip_file',
                 side_effect=mock_unzip_csv)

    # execute tests
    result = execute_pipeline_with_preset(monthly_bay_bike_etl_pipeline,
                                          preset_name='dev',
                                          run_config=RunConfig(mode='local'))
    assert result.success
    with open('/tmp/test_bucket/key_storage.json') as fp:
        key_storage = json.load(fp)
    assert len(key_storage.items()) == 1
Esempio n. 13
0
def test_filesystem_run_storage_from_run_config():
    @solid
    def check_run_storage(context):
        assert isinstance(context.get_system_context().run_storage,
                          FileSystemRunStorage)

    pipeline = PipelineDefinition(name='filesystem_run_storage_test',
                                  solids=[check_run_storage])

    result = execute_pipeline(
        pipeline, run_config=RunConfig(storage_mode=RunStorageMode.FILESYSTEM))

    assert result.success

    assert os.path.isdir(os.path.join(base_run_directory(), result.run_id))
def test_execution_plan_wrong_invalid_step_key():
    pipeline_def = define_addy_pipeline()

    old_run_id = str(uuid.uuid4())
    environment_dict = {'solids': {'add_one': {'inputs': {'num': {'value': 3}}}}}
    result = execute_pipeline(
        pipeline_def,
        environment_dict=environment_dict,
        run_config=RunConfig(storage_mode=RunStorageMode.FILESYSTEM, run_id=old_run_id),
    )

    new_run_id = str(uuid.uuid4())

    run_config = RunConfig(
        run_id=new_run_id,
        reexecution_config=ReexecutionConfig(
            previous_run_id=result.run_id,
            step_output_handles=[StepOutputHandle('not_valid.transform')],
        ),
        storage_mode=RunStorageMode.FILESYSTEM,
    )

    execution_plan = create_execution_plan(pipeline_def, environment_dict=environment_dict)

    with pytest.raises(DagsterExecutionStepNotFoundError) as exc_info:
        execute_plan(
            execution_plan,
            environment_dict=environment_dict,
            run_config=run_config,
            step_keys_to_execute=['add_two.transform'],
        )

    assert str(exc_info.value) == (
        'Step not_valid.transform was specified as a step from a previous run. '
        'It does not exist.'
    )
Esempio n. 15
0
def test_injected_tags():
    called = {}

    @solid
    def check_tags(context):
        assert context.get_tag('foo') == 'bar'
        called['yup'] = True

    pipeline_def = PipelineDefinition(name='injected_run_id',
                                      solid_defs=[check_tags])
    result = execute_pipeline(pipeline_def,
                              run_config=RunConfig(tags={'foo': 'bar'}))

    assert result.success
    assert called['yup']
Esempio n. 16
0
def test_pyspark_emr(mock_wait):
    client = boto3.client('emr', region_name='us-west-1')

    run_job_flow_args = dict(
        Instances={
            'InstanceCount': 1,
            'KeepJobFlowAliveWhenNoSteps': True,
            'MasterInstanceType': 'c3.medium',
            'Placement': {
                'AvailabilityZone': 'us-west-1a'
            },
            'SlaveInstanceType': 'c3.xlarge',
        },
        JobFlowRole='EMR_EC2_DefaultRole',
        LogUri='s3://mybucket/log',
        Name='cluster',
        ServiceRole='EMR_DefaultRole',
        VisibleToAllUsers=True,
    )

    job_flow_id = client.run_job_flow(**run_job_flow_args)['JobFlowId']
    result = execute_pipeline(
        example_pipe,
        environment_dict={
            'solids': {
                'blah': {
                    'config': {
                        'foo': 'a string',
                        'bar': 123
                    }
                }
            },
            'resources': {
                'pyspark': {
                    'config': {
                        'pipeline_file': __file__,
                        'pipeline_fn_name': 'example_pipe',
                        'job_flow_id': job_flow_id,
                        'staging_bucket': 'dagster-scratch-80542c2',
                        'region_name': 'us-west-1',
                    }
                }
            },
        },
        run_config=RunConfig(mode='prod'),
    )
    assert result.success
    assert mock_wait.called_once
Esempio n. 17
0
def _launch_pipeline_execution(graphene_info, execution_params, is_reexecuted=False):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.inst_param(execution_params, 'execution_params', ExecutionParams)

    if is_reexecuted:
        # required fields for re-execution
        execution_metadata = check.inst_param(
            execution_params.execution_metadata, 'execution_metadata', ExecutionMetadata
        )
        check.str_param(execution_metadata.root_run_id, 'root_run_id')
        check.str_param(execution_metadata.parent_run_id, 'parent_run_id')

    error_type = 'RunLauncherNotDefinedError'
    success_type = (
        'LaunchPipelineExecutionSuccess'
        if not is_reexecuted
        else 'LaunchPipelineReexecutionSuccess'
    )
    instance = graphene_info.context.instance
    run_launcher = instance.run_launcher

    if run_launcher is None:
        return graphene_info.schema.type_named(error_type)()

    pipeline_def = get_pipeline_def_from_selector(graphene_info, execution_params.selector)

    get_validated_config(
        graphene_info,
        pipeline_def,
        environment_dict=execution_params.environment_dict,
        mode=execution_params.mode,
    )

    execution_plan = create_execution_plan(
        pipeline_def,
        execution_params.environment_dict,
        run_config=RunConfig(
            mode=execution_params.mode, previous_run_id=execution_params.previous_run_id
        ),
    )

    _check_start_pipeline_execution_errors(graphene_info, execution_params, execution_plan)

    run = instance.launch_run(_create_pipeline_run(instance, pipeline_def, execution_params))

    return graphene_info.schema.type_named(success_type)(
        run=graphene_info.schema.type_named('PipelineRun')(run)
    )
Esempio n. 18
0
def start_pipeline_execution(graphene_info, execution_params,
                             reexecution_config):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.inst_param(execution_params, 'execution_params', ExecutionParams)
    check.opt_inst_param(reexecution_config, 'reexecution_config',
                         ReexecutionConfig)

    pipeline_run_storage = graphene_info.context.pipeline_runs

    dauphin_pipeline = get_dauphin_pipeline_from_selector(
        graphene_info, execution_params.selector)

    get_validated_config(
        graphene_info,
        dauphin_pipeline,
        environment_dict=execution_params.environment_dict,
        mode=execution_params.mode,
    )

    execution_plan = create_execution_plan(
        dauphin_pipeline.get_dagster_pipeline(),
        execution_params.environment_dict,
        run_config=RunConfig(mode=execution_params.mode),
    )

    _check_start_pipeline_execution_errors(graphene_info, execution_params,
                                           execution_plan, reexecution_config)

    run = pipeline_run_storage.create_run(
        run_id=execution_params.execution_metadata.run_id
        if execution_params.execution_metadata.run_id else make_new_run_id(),
        selector=execution_params.selector,
        env_config=execution_params.environment_dict,
        mode=execution_params.mode,
        reexecution_config=reexecution_config,
        step_keys_to_execute=execution_params.step_keys,
    )
    pipeline_run_storage.add_run(run)

    graphene_info.context.execution_manager.execute_pipeline(
        graphene_info.context.get_handle(),
        dauphin_pipeline.get_dagster_pipeline(),
        run,
        raise_on_error=graphene_info.context.raise_on_error,
    )

    return graphene_info.schema.type_named('StartPipelineExecutionSuccess')(
        run=graphene_info.schema.type_named('PipelineRun')(run))
Esempio n. 19
0
def test_failure_midstream():
    '''
    A
     \\
       C (fails) = D (skipped)
     //
    B
    '''

    solid_a = create_root_success_solid('A')
    solid_b = create_root_success_solid('B')

    def fail_fn(_context, inputs):
        check.failed('user error')
        return [inputs['A'], inputs['B'], {'C': 'compute_called'}]

    def success_fn(_context, inputs):
        return [inputs['C'], {'D': 'compute_called'}]

    solid_c = single_output_solid(
        name='C',
        input_defs=[InputDefinition(name='A'), InputDefinition(name='B')],
        compute_fn=fail_fn,
        output_def=OutputDefinition(),
    )

    solid_d = single_output_solid(
        name='D',
        input_defs=[InputDefinition(name='C')],
        compute_fn=success_fn,
        output_def=OutputDefinition(),
    )

    pipeline_def = PipelineDefinition(
        solid_defs=[solid_a, solid_b, solid_c, solid_d],
        dependencies={
            'C': {'A': DependencyDefinition(solid_a.name), 'B': DependencyDefinition(solid_b.name)},
            'D': {'C': DependencyDefinition(solid_c.name)},
        },
    )
    pipeline_result = execute_pipeline(pipeline_def, run_config=RunConfig.nonthrowing_in_process())

    assert pipeline_result.result_for_solid('A').success
    assert pipeline_result.result_for_solid('B').success
    assert not pipeline_result.result_for_solid('C').success
    assert pipeline_result.result_for_solid('C').failure_data.error.cls_name == 'CheckError'
    assert not pipeline_result.result_for_solid('D').success
    assert pipeline_result.result_for_solid('D').skipped
Esempio n. 20
0
def test_mem_storage_error_pipeline_multiprocess():
    with pytest.raises(DagsterInvariantViolationError) as exc_info:
        execute_pipeline(
            define_diamond_pipeline(),
            run_config=RunConfig(executor_config=MultiprocessExecutorConfig(
                ExecutionTargetHandle.for_pipeline_fn(define_error_pipeline))),
        )

    assert ('While invoking '
            'pipeline diamond_execution. You have attempted to use the '
            'multiprocessing executor while using system storage in_memory '
            'which does not persist intermediates. This means there would '
            'be no way to move data between different processes. Please '
            'configure your pipeline in the storage config section to use '
            'persistent system storage such as the filesystem.') in str(
                exc_info.value)
Esempio n. 21
0
def test_execute_multi_mode_loggers_with_multiple_loggers_single_config():
    pipeline_def, foo_logger_captured_results, bar_logger_captured_results = (
        define_multi_mode_with_loggers_pipeline()
    )

    execute_pipeline(
        pipeline_def,
        run_config=RunConfig(mode='foo_bar_mode'),
        environment_dict={'loggers': {'foo': {'config': {'log_level': 'DEBUG'}}}},
    )

    foo_original_messages = parse_captured_results(foo_logger_captured_results)

    assert len(list(filter(lambda x: x == '"Here we are"', foo_original_messages))) == 1

    assert not bar_logger_captured_results
Esempio n. 22
0
def test_local():
    result = execute_pipeline(
        example_pipe,
        environment_dict={
            'solids': {
                'blah': {
                    'config': {
                        'foo': 'a string',
                        'bar': 123
                    }
                }
            },
        },
        run_config=RunConfig(mode='local'),
    )
    assert result.success
def test_using_file_system_for_subplan_invalid_step():
    pipeline = define_inty_pipeline()

    environment_dict = {'storage': {'filesystem': {}}}

    execution_plan = create_execution_plan(pipeline, environment_dict=environment_dict)

    run_id = str(uuid.uuid4())

    with pytest.raises(DagsterExecutionStepNotFoundError):
        execute_plan(
            execution_plan,
            environment_dict=environment_dict,
            run_config=RunConfig(run_id=run_id),
            step_keys_to_execute=['nope'],
        )
Esempio n. 24
0
def test_airline_pipeline_1_warehouse(docker_compose_db):
    warehouse_config_object = load_yaml_from_globs(
        script_relative_path(
            '../../dagster_examples/airline_demo/environments/local_base.yaml'
        ),
        script_relative_path(
            '../../dagster_examples/airline_demo/environments/local_warehouse.yaml'
        ),
    )
    warehouse_config_object = enviroment_overrides(warehouse_config_object)
    result_warehouse = execute_pipeline(
        define_airline_demo_warehouse_pipeline(),
        warehouse_config_object,
        run_config=RunConfig(mode='local'),
    )
    assert result_warehouse.success
Esempio n. 25
0
def test_dask_cluster():
    result = execute_on_dask(
        ExecutionTargetHandle.for_pipeline_module(
            'dagster_examples.toys.hammer', 'define_hammer_pipeline'),
        env_config={
            'storage': {
                's3': {
                    's3_bucket': 'dagster-airflow-scratch'
                }
            }
        },
        run_config=RunConfig(storage_mode=RunStorageMode.S3),
        dask_config=DaskConfig(address='%s:8786' % os.getenv('DASK_ADDRESS')),
    )
    assert result.success
    assert result.result_for_solid('total').transformed_value() == 4
Esempio n. 26
0
def test_error_resource(snapshot):
    result = execute_pipeline(
        resource_error_pipeline,
        environment_dict={'storage': {
            'filesystem': {}
        }},
        run_config=RunConfig(executor_config=InProcessExecutorConfig(
            raise_on_error=False)),
    )

    assert not result.success
    assert len(result.event_list) == 1

    init_failure_event = result.event_list[0]
    assert init_failure_event.event_type_value == 'PIPELINE_INIT_FAILURE'
    snapshot.assert_match(init_failure_event.message)
Esempio n. 27
0
def test_s3_object_store_with_composite_type_storage_plugin():
    run_id = str(uuid.uuid4())

    # FIXME need a dedicated test bucket
    object_store = S3ObjectStore(
        run_id=run_id,
        s3_bucket='dagster-airflow-scratch',
        types_to_register={String.inst(): FancyStringS3TypeStoragePlugin},
    )

    with yield_pipeline_execution_context(PipelineDefinition([]), {},
                                          RunConfig(run_id=run_id)) as context:
        with pytest.raises(check.NotImplementedCheckError):
            object_store.set_value(['hello'], context,
                                   resolve_to_runtime_type(List_(String_)),
                                   ['obj_name'])
Esempio n. 28
0
def _create_pipeline_run(instance, pipeline, execution_params):
    step_keys_to_execute = execution_params.step_keys
    if not execution_params.step_keys and execution_params.previous_run_id:
        execution_plan = create_execution_plan(
            pipeline,
            execution_params.environment_dict,
            run_config=RunConfig(
                mode=execution_params.mode,
                previous_run_id=execution_params.previous_run_id,
                tags=execution_params.execution_metadata.tags,
            ),
        )
        step_keys_to_execute = get_retry_steps_from_execution_plan(
            instance, execution_plan)
    return pipeline_run_from_execution_params(execution_params,
                                              step_keys_to_execute)
def test_using_file_system_for_subplan_missing_input():
    pipeline = define_inty_pipeline()
    environment_dict = {'storage': {'filesystem': {}}}

    execution_plan = create_execution_plan(pipeline, environment_dict=environment_dict)

    run_id = str(uuid.uuid4())

    with pytest.raises(DagsterStepOutputNotFoundError):
        execute_plan(
            execution_plan,
            DagsterInstance.ephemeral(),
            environment_dict=environment_dict,
            run_config=RunConfig(run_id=run_id),
            step_keys_to_execute=['add_one.compute'],
        )
Esempio n. 30
0
def get_pipeline_run_observable(graphene_info, run_id, after=None):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.str_param(run_id, 'run_id')
    check.opt_int_param(after, 'after')
    instance = graphene_info.context.instance
    run = instance.get_run(run_id)

    if not run:

        def _get_error_observable(observer):
            observer.on_next(
                graphene_info.schema.
                type_named('PipelineRunLogsSubscriptionFailure')(
                    missingRunId=run_id,
                    message='Could not load run with id {}'.format(run_id)))

        return Observable.create(_get_error_observable)  # pylint: disable=E1101

    if not instance.can_watch_events:

        def _get_error_observable(observer):
            observer.on_next(
                graphene_info.schema.type_named(
                    'PipelineRunLogsSubscriptionFailure')
                (message=
                 'Event log storage on current DagsterInstance is not watchable.'
                 ))

        return Observable.create(_get_error_observable)  # pylint: disable=E1101

    pipeline = get_dauphin_pipeline_from_selector(graphene_info, run.selector)
    execution_plan = create_execution_plan(pipeline.get_dagster_pipeline(),
                                           run.environment_dict,
                                           RunConfig(mode=run.mode))

    # pylint: disable=E1101
    return Observable.create(
        PipelineRunObservableSubscribe(
            instance, run_id, after_cursor=after)).map(
                lambda events: graphene_info.schema.
                type_named('PipelineRunLogsSubscriptionSuccess')(
                    runId=run_id,
                    messages=[
                        from_event_record(graphene_info, event, pipeline,
                                          execution_plan) for event in events
                    ],
                ))