예제 #1
0
def get_dauphin_pipeline_from_pipeline_index(graphene_info, pipeline_index):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.inst_param(pipeline_index, 'pipeline_index', PipelineIndex)
    if isinstance(graphene_info.context, DagsterGraphQLContext):
        pipeline = get_pipeline_definition(graphene_info, pipeline_index.pipeline_snapshot.name)
        return DauphinPipeline(pipeline_index, presets=pipeline.get_presets())
    return DauphinPipeline(pipeline_index)
예제 #2
0
    def get_subset_external_pipeline(self, selector):
        check.inst_param(selector, "selector", PipelineSelector)
        # We have to grab the pipeline from the location instead of the repository directly
        # since we may have to request a subset we don't have in memory yet

        repository_location = self._repository_locations[
            selector.location_name]
        external_repository = repository_location.get_repository(
            selector.repository_name)

        subset_result = repository_location.get_subset_external_pipeline_result(
            selector)
        if not subset_result.success:
            error_info = subset_result.error
            raise UserFacingGraphQLError(
                DauphinInvalidSubsetError(
                    message="{message}{cause_message}".format(
                        message=error_info.message,
                        cause_message="\n{}".format(error_info.cause.message)
                        if error_info.cause else "",
                    ),
                    pipeline=DauphinPipeline(
                        self.get_full_external_pipeline(selector)),
                ))

        return ExternalPipeline(
            subset_result.external_pipeline_data,
            repository_handle=external_repository.handle,
        )
예제 #3
0
def get_dauphin_pipeline_from_selector(graphene_info, selector):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.inst_param(selector, 'selector', ExecutionSelector)

    return DauphinPipeline(
        get_external_pipeline_or_raise(graphene_info, selector.name,
                                       selector.solid_subset))
예제 #4
0
def get_used_solid_map(repo):
    check.inst_param(repo, "repo", ExternalRepository)

    inv_by_def_name = defaultdict(list)
    definitions = []

    for external_pipeline in repo.get_all_external_pipelines():
        for handle in build_dauphin_solid_handles(
            external_pipeline, external_pipeline.dep_structure_index
        ):
            definition = handle.solid.get_dauphin_solid_definition()
            if definition.name not in inv_by_def_name:
                definitions.append(definition)
            inv_by_def_name[definition.name].append(
                DauphinSolidInvocationSite(
                    pipeline=DauphinPipeline(external_pipeline), solidHandle=handle,
                )
            )

    return OrderedDict(
        (
            definition.name,
            DauphinUsedSolid(
                definition=definition,
                invocations=sorted(
                    inv_by_def_name[definition.name],
                    key=lambda i: i.solidHandle.handleID.to_string(),
                ),
            ),
        )
        for definition in sorted(definitions, key=lambda d: d.name)
    )
예제 #5
0
def validate_pipeline_config(graphene_info, selector, environment_dict, mode):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.inst_param(selector, 'selector', ExecutionSelector)
    check.opt_str_param(mode, 'mode')

    pipeline_def = get_pipeline_def_from_selector(graphene_info, selector)
    get_validated_config(pipeline_def, environment_dict, mode)
    return graphene_info.schema.type_named('PipelineConfigValidationValid')(
        DauphinPipeline.from_pipeline_def(pipeline_def))
예제 #6
0
 def for_validation_errors(pipeline, errors):
     return DauphinPipelineConfigValidationInvalid(
         pipeline=DauphinPipeline.from_pipeline_def(pipeline),
         errors=[
             DauphinPipelineConfigValidationError.from_dagster_error(
                 pipeline.get_config_schema_snapshot(), err)
             for err in errors
         ],
     )
예제 #7
0
def get_dauphin_pipeline_from_selector(graphene_info, selector):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.inst_param(selector, 'selector', ExecutionSelector)

    if isinstance(graphene_info.context, DagsterGraphQLContext):
        pipeline_definition = get_pipeline_def_from_selector(graphene_info, selector)
        return DauphinPipeline.from_pipeline_def(pipeline_definition)

    # TODO: Support solid sub selection.
    check.invariant(
        not selector.solid_subset,
        desc="DagsterSnapshotGraphQLContext doesn't support pipeline sub-selection.",
    )

    repository_index = graphene_info.context.get_repository_index()
    if not repository_index.has_pipeline_index(selector.name):
        raise UserFacingGraphQLError(
            graphene_info.schema.type_named('PipelineNotFoundError')(pipeline_name=selector.name)
        )
    return DauphinPipeline(repository_index.get_pipeline_index(selector.name))
예제 #8
0
def get_execution_plan(graphene_info, selector, environment_dict, mode):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.inst_param(selector, 'selector', ExecutionSelector)
    check.opt_str_param(mode, 'mode')

    pipeline_def = get_pipeline_def_from_selector(graphene_info, selector)
    get_validated_config(graphene_info, pipeline_def, environment_dict, mode)
    return graphene_info.schema.type_named('ExecutionPlan')(
        DauphinPipeline.from_pipeline_def(pipeline_def),
        create_execution_plan(pipeline_def, environment_dict,
                              RunConfig(mode=mode)),
    )
예제 #9
0
def get_dauphin_pipeline_from_selector(graphene_info, selector):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.inst_param(selector, 'selector', ExecutionSelector)

    if not isinstance(graphene_info.context,
                      DagsterGraphQLInProcessRepositoryContext):
        # TODO: Support solid sub selection.
        check.invariant(
            not selector.solid_subset,
            desc=
            "DagsterGraphQLOutOfProcessRepositoryContext doesn't support pipeline sub-selection.",
        )

    return DauphinPipeline(
        get_external_pipeline_subset_or_raise(graphene_info, selector.name,
                                              selector.solid_subset))
예제 #10
0
def get_validated_config(graphene_info, pipeline_def, environment_dict, mode):
    check.str_param(mode, 'mode')
    check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)

    environment_schema = create_environment_schema(pipeline_def, mode)

    validated_config = validate_config(environment_schema.environment_type,
                                       environment_dict)

    if not validated_config.success:
        raise UserFacingGraphQLError(
            graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
                pipeline=DauphinPipeline.from_pipeline_def(pipeline_def),
                errors=[
                    graphene_info.schema.type_named(
                        'PipelineConfigValidationError').from_dagster_error(
                            pipeline_def.get_config_schema_snapshot(), err)
                    for err in validated_config.errors
                ],
            ))

    return validated_config
예제 #11
0
def get_used_solid_map(graphene_info):
    inv_by_def_name = defaultdict(list)
    definitions = []

    for external_pipeline in graphene_info.context.get_all_external_pipelines(
    ):
        for handle in build_dauphin_solid_handles(
                external_pipeline, external_pipeline.dep_structure_index):
            definition = handle.solid.get_dauphin_solid_definition()
            if definition.name not in inv_by_def_name:
                definitions.append(definition)
            inv_by_def_name[definition.name].append(
                DauphinSolidInvocationSite(
                    pipeline=DauphinPipeline(external_pipeline),
                    solidHandle=handle,
                ))
    return OrderedDict((
        definition.name,
        DauphinUsedSolid(
            definition=definition,
            invocations=sorted(inv_by_def_name[definition.name],
                               key=lambda i: i.solidHandle.handleID),
        ),
    ) for definition in sorted(definitions, key=lambda d: d.name))
예제 #12
0
def get_dauphin_pipeline_from_selector(graphene_info, selector):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.inst_param(selector, 'selector', PipelineSelector)

    return DauphinPipeline(
        get_external_pipeline_or_raise(graphene_info, selector))
예제 #13
0
def _do_execute_plan(graphene_info, execution_params, external_pipeline,
                     retries):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.inst_param(execution_params, 'execution_params', ExecutionParams)
    check.inst_param(external_pipeline, 'external_pipeline', ExternalPipeline)
    check.opt_inst_param(retries, 'retries', Retries)

    run_id = execution_params.execution_metadata.run_id

    mode = execution_params.mode or external_pipeline.get_default_mode_name()

    pipeline_run = graphene_info.context.instance.get_run_by_id(run_id)

    external_execution_plan = graphene_info.context.get_external_execution_plan(
        external_pipeline=external_pipeline,
        run_config=execution_params.run_config,
        mode=mode,
        step_keys_to_execute=None,
    )

    if not pipeline_run:
        # TODO switch to raising a UserFacingError if the run_id cannot be found
        # https://github.com/dagster-io/dagster/issues/1876
        pipeline_run = graphene_info.context.instance.create_run(
            pipeline_name=external_pipeline.name,
            run_id=run_id,
            run_config=execution_params.run_config,
            mode=mode,
            solids_to_execute=None,
            step_keys_to_execute=None,
            status=None,
            tags=execution_params.execution_metadata.tags or {},
            root_run_id=None,
            parent_run_id=None,
            pipeline_snapshot=external_pipeline.pipeline_snapshot,
            execution_plan_snapshot=external_execution_plan.
            execution_plan_snapshot,
            parent_pipeline_snapshot=external_pipeline.
            parent_pipeline_snapshot,
        )

    ensure_valid_step_keys(external_execution_plan, execution_params.step_keys)

    if execution_params.step_keys:
        external_execution_plan = graphene_info.context.get_external_execution_plan(
            external_pipeline=external_pipeline,
            run_config=execution_params.run_config,
            mode=mode,
            step_keys_to_execute=execution_params.step_keys,
        )

    event_logs = []

    def _on_event_record(record):
        if record.is_dagster_event:
            event_logs.append(record)

    graphene_info.context.instance.add_event_listener(run_id, _on_event_record)

    graphene_info.context.execute_plan(
        external_pipeline=external_pipeline,
        run_config=execution_params.run_config,
        pipeline_run=pipeline_run,
        step_keys_to_execute=execution_params.step_keys,
        retries=retries,
    )

    def to_graphql_event(event_record):
        return from_dagster_event_record(event_record, external_pipeline.name)

    for event in event_logs:
        if event.dagster_event.is_pipeline_init_failure:
            return graphene_info.schema.type_named('PythonError')(
                event.dagster_event.pipeline_init_failure_data.error)

    return graphene_info.schema.type_named('ExecutePlanSuccess')(
        pipeline=DauphinPipeline(external_pipeline),
        has_failures=any(
            er for er in event_logs if er.is_dagster_event
            and er.dagster_event.event_type == DagsterEventType.STEP_FAILURE),
        step_events=list(map(to_graphql_event, event_logs)),
        raw_event_records=list(map(serialize_dagster_namedtuple, event_logs)),
    )
예제 #14
0
def _do_execute_plan(graphene_info, execution_params, pipeline_def):
    check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
    check.inst_param(execution_params, 'execution_params', ExecutionParams)

    run_id = execution_params.execution_metadata.run_id

    pipeline_run = graphene_info.context.instance.get_run_by_id(run_id)
    if not pipeline_run:
        # TODO switch to raising a UserFacingError if the run_id cannot be found
        # https://github.com/dagster-io/dagster/issues/1876
        pipeline_run = PipelineRun(
            pipeline_name=pipeline_def.name,
            run_id=run_id,
            environment_dict=execution_params.environment_dict,
            mode=execution_params.mode or pipeline_def.get_default_mode_name(),
            tags=execution_params.execution_metadata.tags or {},
        )

    execution_plan = create_execution_plan(
        pipeline=pipeline_def,
        environment_dict=execution_params.environment_dict,
        run_config=pipeline_run,
    )

    if execution_params.step_keys:
        for step_key in execution_params.step_keys:
            if not execution_plan.has_step(step_key):
                raise UserFacingGraphQLError(
                    graphene_info.schema.type_named('InvalidStepError')(invalid_step_key=step_key)
                )

        execution_plan = execution_plan.build_subset_plan(execution_params.step_keys)

    event_logs = []

    def _on_event_record(record):
        if record.is_dagster_event:
            event_logs.append(record)

    graphene_info.context.instance.add_event_listener(run_id, _on_event_record)

    execute_plan(
        execution_plan=execution_plan,
        environment_dict=execution_params.environment_dict,
        pipeline_run=pipeline_run,
        instance=graphene_info.context.instance,
    )

    dauphin_pipeline = DauphinPipeline.from_pipeline_def(pipeline_def)

    def to_graphql_event(event_record):
        return from_dagster_event_record(
            graphene_info, event_record, dauphin_pipeline, execution_plan
        )

    return graphene_info.schema.type_named('ExecutePlanSuccess')(
        pipeline=dauphin_pipeline,
        has_failures=any(
            er
            for er in event_logs
            if er.is_dagster_event and er.dagster_event.event_type == DagsterEventType.STEP_FAILURE
        ),
        step_events=list(map(to_graphql_event, event_logs)),
        raw_event_records=list(map(serialize_dagster_namedtuple, event_logs)),
    )
예제 #15
0
def get_pipeline_or_raise(graphene_info, selector):
    '''Returns a DauphinPipeline or raises a UserFacingGraphQLError if one cannot be retrieved
    from the selector, e.g., the pipeline is not present in the loaded repository.'''
    return DauphinPipeline(
        get_pipeline_def_from_selector(graphene_info, selector))
예제 #16
0
def get_pipeline_or_error(graphene_info, selector):
    '''Returns a DauphinPipelineOrError.'''
    return DauphinPipeline(
        get_pipeline_def_from_selector(graphene_info, selector))