def test_execute_execute_plan_mutation_out_of_process_fails():
pipeline_name = "sleepy_pipeline"
instance = DagsterInstance.local_temp()
pipeline = sleepy_recon_pipeline()
workspace = load_sleepy_workspace(instance)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition())
variables = {
"executionParams": {
"runConfigData": {},
"mode": "default",
"selector": {
"repositoryLocationName":
get_ephemeral_repository_name(pipeline_name),
"repositoryName":
get_ephemeral_repository_name(pipeline_name),
"pipelineName":
pipeline_name,
},
"executionMetadata": {
"runId": pipeline_run.run_id
},
}
}
with pytest.raises(
DagsterGraphQLClientError,
match=re.escape(
"execute_plan is not supported for out-of-process repository locations"
),
):
execute_execute_plan_mutation(workspace,
variables,
instance_ref=instance.get_ref())
def test_execute_execute_plan_mutation():
pipeline_name = 'sleepy_pipeline'
pipeline = ReconstructablePipeline.for_module(
'dagster_examples.toys.sleepy', pipeline_name)
workspace = workspace_from_load_target(
ModuleTarget('dagster_examples.toys.sleepy', pipeline_name))
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition())
variables = {
'executionParams': {
'runConfigData': {},
'mode': 'default',
'selector': {
'repositoryLocationName': pipeline_name,
'repositoryName': '<<unnamed>>',
'pipelineName': pipeline_name,
},
'executionMetadata': {
'runId': pipeline_run.run_id
},
}
}
result = execute_execute_plan_mutation(workspace,
variables,
instance_ref=instance.get_ref())
seen_events = set()
for event in result:
seen_events.add((event.event_type_value, event.step_key))
assert seen_events == EXPECTED_EVENTS
def python_callable(ts, dag_run, **kwargs): # pylint: disable=unused-argument
run_id = dag_run.run_id
# TODO: https://github.com/dagster-io/dagster/issues/1342
redacted = construct_variables(mode, 'REDACTED', pipeline_name,
run_id, ts, step_keys)
logging.info('Executing GraphQL query: {query}\n'.format(
query=EXECUTE_PLAN_MUTATION) + 'with variables:\n' +
seven.json.dumps(redacted, indent=2))
instance = DagsterInstance.from_ref(
instance_ref) if instance_ref else None
if instance:
instance.get_or_create_run(
PipelineRun(
pipeline_name=pipeline_name,
run_id=run_id,
environment_dict=environment_dict,
mode=mode,
selector=ExecutionSelector(pipeline_name),
reexecution_config=None,
step_keys_to_execute=None,
tags=None,
status=PipelineRunStatus.MANAGED,
))
events = execute_execute_plan_mutation(
handle,
construct_variables(mode, environment_dict, pipeline_name,
run_id, ts, step_keys),
instance_ref=instance_ref,
)
check_events_for_skips(events)
return events
def test_execute_execute_plan_mutation():
pipeline_name = 'sleepy_pipeline'
handle = ExecutionTargetHandle.for_pipeline_module(
'dagster_examples.toys.sleepy', pipeline_name)
pipeline = handle.build_pipeline_definition()
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run_for_pipeline(pipeline=pipeline)
variables = {
'executionParams': {
'environmentConfigData': {},
'mode': 'default',
'selector': {
'name': pipeline_name
},
'executionMetadata': {
'runId': pipeline_run.run_id
},
}
}
result = execute_execute_plan_mutation(handle,
variables,
instance_ref=instance.get_ref())
seen_events = set()
for event in result:
seen_events.add((event.event_type_value, event.step_key))
assert seen_events == EXPECTED_EVENTS
def test_execute_execute_plan_mutation():
pipeline_name = 'sleepy_pipeline'
instance = DagsterInstance.local_temp()
pipeline = sleepy_recon_pipeline()
workspace = load_sleepy_workspace(instance)
pipeline_run = instance.create_run_for_pipeline(pipeline_def=pipeline.get_definition())
variables = {
'executionParams': {
'runConfigData': {},
'mode': 'default',
'selector': {
'repositoryLocationName': get_ephemeral_repository_name(pipeline_name),
'repositoryName': get_ephemeral_repository_name(pipeline_name),
'pipelineName': pipeline_name,
},
'executionMetadata': {'runId': pipeline_run.run_id},
}
}
result = execute_execute_plan_mutation(workspace, variables, instance_ref=instance.get_ref())
seen_events = set()
for event in result:
seen_events.add((event.event_type_value, event.step_key))
assert seen_events == EXPECTED_EVENTS
def query_on_dask_worker(handle, variables, dependencies, instance_ref=None): # pylint: disable=unused-argument
'''Note that we need to pass "dependencies" to ensure Dask sequences futures during task
scheduling, even though we do not use this argument within the function.
'''
return execute_execute_plan_mutation(handle,
variables,
instance_ref=instance_ref)
def test_execute_execute_plan_mutation():
pipeline_name = "sleepy_pipeline"
instance = DagsterInstance.local_temp()
pipeline = sleepy_recon_pipeline()
workspace = create_in_process_ephemeral_workspace(
pointer=pipeline.repository.pointer)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition())
variables = {
"executionParams": {
"runConfigData": {},
"mode": "default",
"selector": {
"repositoryLocationName": IN_PROCESS_NAME,
"repositoryName": get_ephemeral_repository_name(pipeline_name),
"pipelineName": pipeline_name,
},
"executionMetadata": {
"runId": pipeline_run.run_id
},
}
}
result = execute_execute_plan_mutation(workspace,
variables,
instance_ref=instance.get_ref())
seen_events = set()
for event in result:
seen_events.add((event.event_type_value, event.step_key))
assert seen_events == EXPECTED_EVENTS
def query_on_dask_worker(handle, variables, dependencies, instance_ref=None): # pylint: disable=unused-argument
'''Note that we need to pass "dependencies" to ensure Dask sequences futures during task
scheduling, even though we do not use this argument within the function.
We also set 'raise_on_error' within pipeline execution, because otherwise (currently) very
little information is propagated to the dask master from the workers about the state of
execution; we should at least inform the user of exceptions.
'''
return execute_execute_plan_mutation(handle,
variables,
instance_ref=instance_ref)
def _execute_query(_self, handle_dict, variables, instance_ref_dict):
instance_ref = InstanceRef.from_dict(instance_ref_dict)
handle = ExecutionTargetHandle.from_dict(handle_dict)
events = execute_execute_plan_mutation(
handle=handle,
variables=variables,
instance_ref=instance_ref,
)
serialized_events = [
serialize_dagster_namedtuple(event) for event in events
]
return serialized_events
def invoke_steps_within_python_operator(invocation_args, ts, dag_run,
**kwargs): # pylint: disable=unused-argument
mode = invocation_args.mode
pipeline_name = invocation_args.pipeline_name
step_keys = invocation_args.step_keys
instance_ref = invocation_args.instance_ref
run_config = invocation_args.run_config
recon_repo = invocation_args.recon_repo
pipeline_snapshot = invocation_args.pipeline_snapshot
execution_plan_snapshot = invocation_args.execution_plan_snapshot
parent_pipeline_snapshot = invocation_args.parent_pipeline_snapshot
run_id = dag_run.run_id
variables = construct_execute_plan_variables(recon_repo, mode, run_config,
pipeline_name, run_id,
step_keys)
variables = add_airflow_tags(variables, ts)
logging.info('Executing GraphQL query: {query}\n'.format(
query=EXECUTE_PLAN_MUTATION) + 'with variables:\n' +
seven.json.dumps(variables, indent=2))
instance = DagsterInstance.from_ref(instance_ref) if instance_ref else None
if instance:
instance.register_managed_run(
pipeline_name=pipeline_name,
run_id=run_id,
run_config=run_config,
mode=mode,
solids_to_execute=None,
step_keys_to_execute=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=pipeline_snapshot,
execution_plan_snapshot=execution_plan_snapshot,
parent_pipeline_snapshot=parent_pipeline_snapshot,
)
workspace = create_in_process_ephemeral_workspace(
pointer=recon_repo.pointer)
events = execute_execute_plan_mutation(
workspace,
variables,
instance_ref=instance_ref,
)
check_events_for_failures(events)
check_events_for_skips(events)
return events
def invoke_steps_within_python_operator(invocation_args, ts, dag_run,
**kwargs): # pylint: disable=unused-argument
mode = invocation_args.mode
pipeline_name = invocation_args.pipeline_name
step_keys = invocation_args.step_keys
instance_ref = invocation_args.instance_ref
environment_dict = invocation_args.environment_dict
handle = invocation_args.handle
pipeline_snapshot = invocation_args.pipeline_snapshot
execution_plan_snapshot = invocation_args.execution_plan_snapshot
parent_pipeline_snapshot = invocation_args.parent_pipeline_snapshot
run_id = dag_run.run_id
variables = construct_variables(mode, environment_dict, pipeline_name,
run_id, step_keys)
variables = add_airflow_tags(variables, ts)
logging.info('Executing GraphQL query: {query}\n'.format(
query=EXECUTE_PLAN_MUTATION) + 'with variables:\n' +
seven.json.dumps(variables, indent=2))
instance = DagsterInstance.from_ref(instance_ref) if instance_ref else None
if instance:
instance.register_managed_run(
pipeline_name=pipeline_name,
run_id=run_id,
environment_dict=environment_dict,
mode=mode,
solid_subset=None,
step_keys_to_execute=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=pipeline_snapshot,
execution_plan_snapshot=execution_plan_snapshot,
parent_pipeline_snapshot=parent_pipeline_snapshot,
)
events = execute_execute_plan_mutation(
handle,
variables,
instance_ref=instance_ref,
)
check_events_for_failures(events)
check_events_for_skips(events)
return events
def invoke_steps_within_python_operator(invocation_args, ts, dag_run,
**kwargs): # pylint: disable=unused-argument
mode = invocation_args.mode
pipeline_name = invocation_args.pipeline_name
step_keys = invocation_args.step_keys
instance_ref = invocation_args.instance_ref
environment_dict = invocation_args.environment_dict
handle = invocation_args.handle
run_id = dag_run.run_id
variables = construct_variables(mode, environment_dict, pipeline_name,
run_id, step_keys)
variables = add_airflow_tags(variables, ts)
logging.info('Executing GraphQL query: {query}\n'.format(
query=EXECUTE_PLAN_MUTATION) + 'with variables:\n' +
seven.json.dumps(variables, indent=2))
instance = DagsterInstance.from_ref(instance_ref) if instance_ref else None
if instance:
instance.get_or_create_run(
PipelineRun(
pipeline_name=pipeline_name,
run_id=run_id,
environment_dict=environment_dict,
mode=mode,
selector=ExecutionSelector(pipeline_name),
step_keys_to_execute=None,
tags=None,
status=PipelineRunStatus.MANAGED,
))
events = execute_execute_plan_mutation(
handle,
variables,
instance_ref=instance_ref,
)
check_events_for_failures(events)
check_events_for_skips(events)
return events
def test_execute_execute_plan_mutation():
pipeline_name = 'sleepy_pipeline'
handle = ExecutionTargetHandle.for_pipeline_module(
'dagster_examples.toys.sleepy', pipeline_name)
variables = {
'executionParams': {
'environmentConfigData': {},
'mode': 'default',
'selector': {
'name': pipeline_name
},
'executionMetadata': {
'runId': '12345'
},
}
}
result = execute_execute_plan_mutation(handle, variables)
seen_events = set()
for event in result:
seen_events.add((event.event_type_value, event.step_key))
assert seen_events == EXPECTED_EVENTS
