def test_execute_execute_plan_mutation_out_of_process_fails(): pipeline_name = "sleepy_pipeline" instance = DagsterInstance.local_temp() pipeline = sleepy_recon_pipeline() workspace = load_sleepy_workspace(instance) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition()) variables = { "executionParams": { "runConfigData": {}, "mode": "default", "selector": { "repositoryLocationName": get_ephemeral_repository_name(pipeline_name), "repositoryName": get_ephemeral_repository_name(pipeline_name), "pipelineName": pipeline_name, }, "executionMetadata": { "runId": pipeline_run.run_id }, } } with pytest.raises( DagsterGraphQLClientError, match=re.escape( "execute_plan is not supported for out-of-process repository locations" ), ): execute_execute_plan_mutation(workspace, variables, instance_ref=instance.get_ref())
def test_execute_execute_plan_mutation(): pipeline_name = 'sleepy_pipeline' pipeline = ReconstructablePipeline.for_module( 'dagster_examples.toys.sleepy', pipeline_name) workspace = workspace_from_load_target( ModuleTarget('dagster_examples.toys.sleepy', pipeline_name)) instance = DagsterInstance.local_temp() pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition()) variables = { 'executionParams': { 'runConfigData': {}, 'mode': 'default', 'selector': { 'repositoryLocationName': pipeline_name, 'repositoryName': '<<unnamed>>', 'pipelineName': pipeline_name, }, 'executionMetadata': { 'runId': pipeline_run.run_id }, } } result = execute_execute_plan_mutation(workspace, variables, instance_ref=instance.get_ref()) seen_events = set() for event in result: seen_events.add((event.event_type_value, event.step_key)) assert seen_events == EXPECTED_EVENTS
def python_callable(ts, dag_run, **kwargs): # pylint: disable=unused-argument run_id = dag_run.run_id # TODO: https://github.com/dagster-io/dagster/issues/1342 redacted = construct_variables(mode, 'REDACTED', pipeline_name, run_id, ts, step_keys) logging.info('Executing GraphQL query: {query}\n'.format( query=EXECUTE_PLAN_MUTATION) + 'with variables:\n' + seven.json.dumps(redacted, indent=2)) instance = DagsterInstance.from_ref( instance_ref) if instance_ref else None if instance: instance.get_or_create_run( PipelineRun( pipeline_name=pipeline_name, run_id=run_id, environment_dict=environment_dict, mode=mode, selector=ExecutionSelector(pipeline_name), reexecution_config=None, step_keys_to_execute=None, tags=None, status=PipelineRunStatus.MANAGED, )) events = execute_execute_plan_mutation( handle, construct_variables(mode, environment_dict, pipeline_name, run_id, ts, step_keys), instance_ref=instance_ref, ) check_events_for_skips(events) return events
def test_execute_execute_plan_mutation(): pipeline_name = 'sleepy_pipeline' handle = ExecutionTargetHandle.for_pipeline_module( 'dagster_examples.toys.sleepy', pipeline_name) pipeline = handle.build_pipeline_definition() instance = DagsterInstance.local_temp() pipeline_run = instance.create_run_for_pipeline(pipeline=pipeline) variables = { 'executionParams': { 'environmentConfigData': {}, 'mode': 'default', 'selector': { 'name': pipeline_name }, 'executionMetadata': { 'runId': pipeline_run.run_id }, } } result = execute_execute_plan_mutation(handle, variables, instance_ref=instance.get_ref()) seen_events = set() for event in result: seen_events.add((event.event_type_value, event.step_key)) assert seen_events == EXPECTED_EVENTS
def test_execute_execute_plan_mutation(): pipeline_name = 'sleepy_pipeline' instance = DagsterInstance.local_temp() pipeline = sleepy_recon_pipeline() workspace = load_sleepy_workspace(instance) pipeline_run = instance.create_run_for_pipeline(pipeline_def=pipeline.get_definition()) variables = { 'executionParams': { 'runConfigData': {}, 'mode': 'default', 'selector': { 'repositoryLocationName': get_ephemeral_repository_name(pipeline_name), 'repositoryName': get_ephemeral_repository_name(pipeline_name), 'pipelineName': pipeline_name, }, 'executionMetadata': {'runId': pipeline_run.run_id}, } } result = execute_execute_plan_mutation(workspace, variables, instance_ref=instance.get_ref()) seen_events = set() for event in result: seen_events.add((event.event_type_value, event.step_key)) assert seen_events == EXPECTED_EVENTS
def query_on_dask_worker(handle, variables, dependencies, instance_ref=None): # pylint: disable=unused-argument '''Note that we need to pass "dependencies" to ensure Dask sequences futures during task scheduling, even though we do not use this argument within the function. ''' return execute_execute_plan_mutation(handle, variables, instance_ref=instance_ref)
def test_execute_execute_plan_mutation(): pipeline_name = "sleepy_pipeline" instance = DagsterInstance.local_temp() pipeline = sleepy_recon_pipeline() workspace = create_in_process_ephemeral_workspace( pointer=pipeline.repository.pointer) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition()) variables = { "executionParams": { "runConfigData": {}, "mode": "default", "selector": { "repositoryLocationName": IN_PROCESS_NAME, "repositoryName": get_ephemeral_repository_name(pipeline_name), "pipelineName": pipeline_name, }, "executionMetadata": { "runId": pipeline_run.run_id }, } } result = execute_execute_plan_mutation(workspace, variables, instance_ref=instance.get_ref()) seen_events = set() for event in result: seen_events.add((event.event_type_value, event.step_key)) assert seen_events == EXPECTED_EVENTS
def query_on_dask_worker(handle, variables, dependencies, instance_ref=None): # pylint: disable=unused-argument '''Note that we need to pass "dependencies" to ensure Dask sequences futures during task scheduling, even though we do not use this argument within the function. We also set 'raise_on_error' within pipeline execution, because otherwise (currently) very little information is propagated to the dask master from the workers about the state of execution; we should at least inform the user of exceptions. ''' return execute_execute_plan_mutation(handle, variables, instance_ref=instance_ref)
def _execute_query(_self, handle_dict, variables, instance_ref_dict): instance_ref = InstanceRef.from_dict(instance_ref_dict) handle = ExecutionTargetHandle.from_dict(handle_dict) events = execute_execute_plan_mutation( handle=handle, variables=variables, instance_ref=instance_ref, ) serialized_events = [ serialize_dagster_namedtuple(event) for event in events ] return serialized_events
def invoke_steps_within_python_operator(invocation_args, ts, dag_run, **kwargs): # pylint: disable=unused-argument mode = invocation_args.mode pipeline_name = invocation_args.pipeline_name step_keys = invocation_args.step_keys instance_ref = invocation_args.instance_ref run_config = invocation_args.run_config recon_repo = invocation_args.recon_repo pipeline_snapshot = invocation_args.pipeline_snapshot execution_plan_snapshot = invocation_args.execution_plan_snapshot parent_pipeline_snapshot = invocation_args.parent_pipeline_snapshot run_id = dag_run.run_id variables = construct_execute_plan_variables(recon_repo, mode, run_config, pipeline_name, run_id, step_keys) variables = add_airflow_tags(variables, ts) logging.info('Executing GraphQL query: {query}\n'.format( query=EXECUTE_PLAN_MUTATION) + 'with variables:\n' + seven.json.dumps(variables, indent=2)) instance = DagsterInstance.from_ref(instance_ref) if instance_ref else None if instance: instance.register_managed_run( pipeline_name=pipeline_name, run_id=run_id, run_config=run_config, mode=mode, solids_to_execute=None, step_keys_to_execute=None, tags=None, root_run_id=None, parent_run_id=None, pipeline_snapshot=pipeline_snapshot, execution_plan_snapshot=execution_plan_snapshot, parent_pipeline_snapshot=parent_pipeline_snapshot, ) workspace = create_in_process_ephemeral_workspace( pointer=recon_repo.pointer) events = execute_execute_plan_mutation( workspace, variables, instance_ref=instance_ref, ) check_events_for_failures(events) check_events_for_skips(events) return events
def invoke_steps_within_python_operator(invocation_args, ts, dag_run, **kwargs): # pylint: disable=unused-argument mode = invocation_args.mode pipeline_name = invocation_args.pipeline_name step_keys = invocation_args.step_keys instance_ref = invocation_args.instance_ref environment_dict = invocation_args.environment_dict handle = invocation_args.handle pipeline_snapshot = invocation_args.pipeline_snapshot execution_plan_snapshot = invocation_args.execution_plan_snapshot parent_pipeline_snapshot = invocation_args.parent_pipeline_snapshot run_id = dag_run.run_id variables = construct_variables(mode, environment_dict, pipeline_name, run_id, step_keys) variables = add_airflow_tags(variables, ts) logging.info('Executing GraphQL query: {query}\n'.format( query=EXECUTE_PLAN_MUTATION) + 'with variables:\n' + seven.json.dumps(variables, indent=2)) instance = DagsterInstance.from_ref(instance_ref) if instance_ref else None if instance: instance.register_managed_run( pipeline_name=pipeline_name, run_id=run_id, environment_dict=environment_dict, mode=mode, solid_subset=None, step_keys_to_execute=None, tags=None, root_run_id=None, parent_run_id=None, pipeline_snapshot=pipeline_snapshot, execution_plan_snapshot=execution_plan_snapshot, parent_pipeline_snapshot=parent_pipeline_snapshot, ) events = execute_execute_plan_mutation( handle, variables, instance_ref=instance_ref, ) check_events_for_failures(events) check_events_for_skips(events) return events
def invoke_steps_within_python_operator(invocation_args, ts, dag_run, **kwargs): # pylint: disable=unused-argument mode = invocation_args.mode pipeline_name = invocation_args.pipeline_name step_keys = invocation_args.step_keys instance_ref = invocation_args.instance_ref environment_dict = invocation_args.environment_dict handle = invocation_args.handle run_id = dag_run.run_id variables = construct_variables(mode, environment_dict, pipeline_name, run_id, step_keys) variables = add_airflow_tags(variables, ts) logging.info('Executing GraphQL query: {query}\n'.format( query=EXECUTE_PLAN_MUTATION) + 'with variables:\n' + seven.json.dumps(variables, indent=2)) instance = DagsterInstance.from_ref(instance_ref) if instance_ref else None if instance: instance.get_or_create_run( PipelineRun( pipeline_name=pipeline_name, run_id=run_id, environment_dict=environment_dict, mode=mode, selector=ExecutionSelector(pipeline_name), step_keys_to_execute=None, tags=None, status=PipelineRunStatus.MANAGED, )) events = execute_execute_plan_mutation( handle, variables, instance_ref=instance_ref, ) check_events_for_failures(events) check_events_for_skips(events) return events
def test_execute_execute_plan_mutation(): pipeline_name = 'sleepy_pipeline' handle = ExecutionTargetHandle.for_pipeline_module( 'dagster_examples.toys.sleepy', pipeline_name) variables = { 'executionParams': { 'environmentConfigData': {}, 'mode': 'default', 'selector': { 'name': pipeline_name }, 'executionMetadata': { 'runId': '12345' }, } } result = execute_execute_plan_mutation(handle, variables) seen_events = set() for event in result: seen_events.add((event.event_type_value, event.step_key)) assert seen_events == EXPECTED_EVENTS