def test_run_local():
events = []
def _event_callback(ev):
events.append(ev)
result = execute_pipeline(
define_resource_test_pipeline(),
environment_dict={
'context': {
'local': {}
},
'solids': {
'add_ints': {
'inputs': {
'num_one': {
'value': 2
},
'num_two': {
'value': 3
}
}
}
},
},
execution_metadata=ExecutionMetadata(event_callback=_event_callback),
)
assert result.success
assert result.result_for_solid('add_ints').transformed_value() == 5
assert has_message(events, 'Setting key=add value=5 in memory')
assert not has_message(events, 'Setting key=add value=5 in cloud')
def test_injected_run_id():
run_id = 'kdjfkdjfkd'
pipeline_def = PipelineDefinition(name='injected_run_id', solids=[])
result = execute_pipeline(pipeline_def, execution_metadata=ExecutionMetadata(run_id=run_id))
assert result.success
assert result.context.run_id == run_id
def test_user_injected_tags_collision():
called = {}
@solid
def check_tags(context):
assert context.get_tag('foo') == 'bar'
assert context.get_tag('quux') == 'baaz'
called['yup'] = True
def _create_context(_context):
return ExecutionContext(tags={'foo': 'baaz'})
pipeline_def = PipelineDefinition(
name='injected_run_id',
solids=[check_tags],
context_definitions={
'default': PipelineContextDefinition(context_fn=_create_context)
},
)
with pytest.raises(DagsterInvariantViolationError,
match='You have specified'):
execute_pipeline(
pipeline_def,
execution_metadata=ExecutionMetadata(tags={'foo': 'bar'}))
def test_user_injected_tags():
called = {}
@solid
def check_tags(context):
assert context.get_tag('foo') == 'bar'
assert context.get_tag('quux') == 'baaz'
called['yup'] = True
def _create_context(_context):
return ExecutionContext(tags={'quux': 'baaz'})
pipeline_def = PipelineDefinition(
name='injected_run_id',
solids=[check_tags],
context_definitions={
'default': PipelineContextDefinition(context_fn=_create_context)
},
)
result = execute_pipeline(
pipeline_def,
execution_metadata=ExecutionMetadata(tags={'foo': 'bar'}))
assert result.success
assert called['yup']
def test_create_subplan_middle_step():
pipeline_def = define_two_int_pipeline()
typed_environment = create_typed_environment(pipeline_def, None)
execution_plan = create_execution_plan(pipeline_def)
with yield_context(pipeline_def, typed_environment,
ExecutionMetadata()) as context:
subplan = create_subplan(
ExecutionPlanInfo(context=context,
pipeline=pipeline_def,
environment=typed_environment),
StepBuilderState(pipeline_name=pipeline_def.name),
execution_plan,
ExecutionPlanSubsetInfo(['add_one.transform'],
{'add_one.transform': {
'num': 2
}}),
)
assert subplan
steps = subplan.topological_steps()
assert len(steps) == 2
assert steps[0].key == 'add_one.transform.input.num.value'
assert not steps[0].step_inputs
assert len(steps[0].step_outputs) == 1
assert steps[1].key == 'add_one.transform'
assert len(steps[1].step_inputs) == 1
step_input = steps[1].step_inputs[0]
assert step_input.prev_output_handle.step.key == 'add_one.transform.input.num.value'
assert step_input.prev_output_handle.output_name == VALUE_OUTPUT
assert len(steps[1].step_outputs) == 1
assert len(subplan.topological_steps()) == 2
assert [step.key for step in subplan.topological_steps()] == [
'add_one.transform.input.num.value',
'add_one.transform',
]
def run_test_pipeline(pipeline):
typed_environment = create_typed_environment(pipeline, TEST_ENVIRONMENT)
execution_metadata = ExecutionMetadata(run_id=str(uuid.uuid4()))
with yield_context(pipeline, typed_environment,
execution_metadata) as context:
execution_plan = create_execution_plan_core(
ExecutionPlanInfo(context, pipeline, typed_environment))
return execute_plan(context, execution_plan)
def create_plan(pipeline):
config_or_error = _config_or_error_from_pipeline(
graphene_info, pipeline, config)
return config_or_error.chain(
lambda evaluate_value_result: graphene_info.schema.type_named(
'ExecutionPlan')(
pipeline,
create_execution_plan(
pipeline.get_dagster_pipeline(),
evaluate_value_result.value,
ExecutionMetadata(),
),
))
def create_plan(pipeline):
config_or_error = _config_or_error_from_pipeline(
info, pipeline, config)
return config_or_error.chain(
lambda evaluate_value_result: info.schema.
type_named('ExecutionPlan')(
pipeline,
create_execution_plan_with_typed_environment(
pipeline.get_dagster_pipeline(),
construct_environment_config(evaluate_value_result.value),
ExecutionMetadata(),
),
))
def test_injected_tags():
called = {}
@solid
def check_tags(info):
assert info.context.get_tag('foo') == 'bar'
called['yup'] = True
pipeline_def = PipelineDefinition(name='injected_run_id', solids=[check_tags])
result = execute_pipeline(
pipeline_def, execution_metadata=ExecutionMetadata(tags={'foo': 'bar'})
)
assert result.success
assert called['yup']
def _start_execution(validated_config_either):
new_run_id = str(uuid.uuid4())
execution_plan = create_execution_plan_with_typed_environment(
pipeline.get_dagster_pipeline(),
construct_environment_config(validated_config_either.value),
ExecutionMetadata(),
)
run = pipeline_run_storage.create_run(new_run_id, selector,
env_config, execution_plan)
pipeline_run_storage.add_run(run)
info.context.execution_manager.execute_pipeline(
info.context.repository_container,
pipeline.get_dagster_pipeline(), run)
return info.schema.type_named('StartPipelineExecutionSuccess')(
run=info.schema.type_named('PipelineRun')(run))
def test_compute_noop_node_core():
pipeline = silencing_pipeline(solids=[noop])
environment = EnvironmentConfig()
plan = create_execution_plan_core(
ExecutionPlanInfo(create_test_runtime_execution_context(), pipeline,
environment),
ExecutionMetadata(),
)
assert len(plan.steps) == 1
outputs = list(
execute_step(plan.steps[0], create_test_runtime_execution_context(),
{}))
assert outputs[0].success_data.value == 'foo'
def _start_execution(validated_config_either):
new_run_id = str(uuid.uuid4())
execution_plan = create_execution_plan(
pipeline.get_dagster_pipeline(), validated_config_either.value,
ExecutionMetadata())
run = pipeline_run_storage.create_run(new_run_id, selector,
env_config, execution_plan)
pipeline_run_storage.add_run(run)
graphene_info.context.execution_manager.execute_pipeline(
graphene_info.context.repository_container,
pipeline.get_dagster_pipeline(),
run,
throw_on_user_error=graphene_info.context.throw_on_user_error,
)
return graphene_info.schema.type_named(
'StartPipelineExecutionSuccess')(
run=graphene_info.schema.type_named('PipelineRun')(run))
def test_reentrant_execute_plan():
called = {}
@solid
def has_tag(context):
assert context.has_tag('foo')
assert context.get_tag('foo') == 'bar'
called['yup'] = True
pipeline_def = PipelineDefinition(name='has_tag_pipeline',
solids=[has_tag])
execution_plan = create_execution_plan(pipeline_def)
step_events = execute_plan(
execution_plan,
execution_metadata=ExecutionMetadata(tags={'foo': 'bar'}))
assert called['yup']
assert len(step_events) == 1
def execute_pipeline(self, repository_container, pipeline, pipeline_run):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
try:
return execute_pipeline_with_metadata(
pipeline,
create_typed_environment(pipeline, pipeline_run.config),
execution_metadata=ExecutionMetadata(
pipeline_run.run_id, event_callback=pipeline_run.handle_new_event
),
throw_on_user_error=False,
)
except: # pylint: disable=W0702
pipeline_run.handle_new_event(
build_synthetic_pipeline_error_record(
pipeline_run.run_id,
serializable_error_info_from_exc_info(sys.exc_info()),
pipeline.name,
)
)
def execute_pipeline_through_queue(
repository_info, pipeline_name, pipeline_solid_subset, config, run_id, message_queue
):
"""
Execute pipeline using message queue as a transport
"""
message_queue.put(ProcessStartedSentinel(os.getpid()))
execution_metadata = ExecutionMetadata(run_id, event_callback=message_queue.put)
from .app import RepositoryContainer
repository_container = RepositoryContainer(repository_info)
if repository_container.repo_error:
message_queue.put(
MultiprocessingError(
serializable_error_info_from_exc_info(repository_container.repo_error)
)
)
return
pipeline = repository_container.repository.get_pipeline(pipeline_name)
pipeline = get_subset_pipeline(pipeline, pipeline_solid_subset)
typed_environment = construct_environment_config(
evaluate_config_value(pipeline.environment_type, config).value
)
try:
result = execute_pipeline_with_metadata(
pipeline,
typed_environment,
execution_metadata=execution_metadata,
throw_on_user_error=False,
)
return result
except: # pylint: disable=W0702
message_queue.put(
MultiprocessingError(serializable_error_info_from_exc_info(sys.exc_info()))
)
finally:
message_queue.put(MultiprocessingDone())
def test_run_cloud():
events = []
def _event_callback(ev):
events.append(ev)
result = execute_pipeline(
define_resource_test_pipeline(),
environment_dict={
'context': {
'cloud': {
'resources': {
'store': {
'config': {
'username': '******',
'password': '******'
}
}
}
}
},
'solids': {
'add_ints': {
'inputs': {
'num_one': {
'value': 2
},
'num_two': {
'value': 6
}
}
}
},
},
execution_metadata=ExecutionMetadata(event_callback=_event_callback),
)
assert result.success
assert result.result_for_solid('add_ints').transformed_value() == 8
assert not has_message(events, 'Setting key=add value=8 in memory')
assert has_message(events, 'Setting key=add value=8 in cloud')
def test_create_subplan_source_step():
pipeline_def = define_two_int_pipeline()
typed_environment = create_typed_environment(pipeline_def, None)
execution_plan = create_execution_plan(pipeline_def)
with yield_context(pipeline_def, typed_environment,
ExecutionMetadata()) as context:
subplan = create_subplan(
ExecutionPlanInfo(context=context,
pipeline=pipeline_def,
environment=typed_environment),
StepBuilderState(pipeline_name=pipeline_def.name),
execution_plan,
ExecutionPlanSubsetInfo(['return_one.transform']),
)
assert subplan
assert len(subplan.steps) == 1
assert subplan.steps[0].key == 'return_one.transform'
assert not subplan.steps[0].step_inputs
assert len(subplan.steps[0].step_outputs) == 1
assert len(subplan.topological_steps()) == 1
def execute_pipeline(self, repository_container, pipeline, pipeline_run,
throw_on_user_error):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
try:
return execute_pipeline(
pipeline,
pipeline_run.config,
execution_metadata=ExecutionMetadata(
pipeline_run.run_id,
event_callback=pipeline_run.handle_new_event),
throw_on_user_error=throw_on_user_error,
)
except: # pylint: disable=W0702
if throw_on_user_error:
six.reraise(*sys.exc_info())
pipeline_run.handle_new_event(
build_synthetic_pipeline_error_record(
pipeline_run.run_id,
serializable_error_info_from_exc_info(sys.exc_info()),
pipeline.name,
))
def run_test_pipeline(pipeline):
execution_metadata = ExecutionMetadata(run_id=str(uuid.uuid4()))
with yield_pipeline_execution_context(pipeline, TEST_ENVIRONMENT,
execution_metadata) as context:
execution_plan = create_execution_plan_core(context)
return execute_plan(context, execution_plan)
