def test_run_local(): events = [] def _event_callback(ev): events.append(ev) result = execute_pipeline( define_resource_test_pipeline(), environment_dict={ 'context': { 'local': {} }, 'solids': { 'add_ints': { 'inputs': { 'num_one': { 'value': 2 }, 'num_two': { 'value': 3 } } } }, }, execution_metadata=ExecutionMetadata(event_callback=_event_callback), ) assert result.success assert result.result_for_solid('add_ints').transformed_value() == 5 assert has_message(events, 'Setting key=add value=5 in memory') assert not has_message(events, 'Setting key=add value=5 in cloud')
def test_injected_run_id(): run_id = 'kdjfkdjfkd' pipeline_def = PipelineDefinition(name='injected_run_id', solids=[]) result = execute_pipeline(pipeline_def, execution_metadata=ExecutionMetadata(run_id=run_id)) assert result.success assert result.context.run_id == run_id
def test_user_injected_tags_collision(): called = {} @solid def check_tags(context): assert context.get_tag('foo') == 'bar' assert context.get_tag('quux') == 'baaz' called['yup'] = True def _create_context(_context): return ExecutionContext(tags={'foo': 'baaz'}) pipeline_def = PipelineDefinition( name='injected_run_id', solids=[check_tags], context_definitions={ 'default': PipelineContextDefinition(context_fn=_create_context) }, ) with pytest.raises(DagsterInvariantViolationError, match='You have specified'): execute_pipeline( pipeline_def, execution_metadata=ExecutionMetadata(tags={'foo': 'bar'}))
def test_user_injected_tags(): called = {} @solid def check_tags(context): assert context.get_tag('foo') == 'bar' assert context.get_tag('quux') == 'baaz' called['yup'] = True def _create_context(_context): return ExecutionContext(tags={'quux': 'baaz'}) pipeline_def = PipelineDefinition( name='injected_run_id', solids=[check_tags], context_definitions={ 'default': PipelineContextDefinition(context_fn=_create_context) }, ) result = execute_pipeline( pipeline_def, execution_metadata=ExecutionMetadata(tags={'foo': 'bar'})) assert result.success assert called['yup']
def test_create_subplan_middle_step(): pipeline_def = define_two_int_pipeline() typed_environment = create_typed_environment(pipeline_def, None) execution_plan = create_execution_plan(pipeline_def) with yield_context(pipeline_def, typed_environment, ExecutionMetadata()) as context: subplan = create_subplan( ExecutionPlanInfo(context=context, pipeline=pipeline_def, environment=typed_environment), StepBuilderState(pipeline_name=pipeline_def.name), execution_plan, ExecutionPlanSubsetInfo(['add_one.transform'], {'add_one.transform': { 'num': 2 }}), ) assert subplan steps = subplan.topological_steps() assert len(steps) == 2 assert steps[0].key == 'add_one.transform.input.num.value' assert not steps[0].step_inputs assert len(steps[0].step_outputs) == 1 assert steps[1].key == 'add_one.transform' assert len(steps[1].step_inputs) == 1 step_input = steps[1].step_inputs[0] assert step_input.prev_output_handle.step.key == 'add_one.transform.input.num.value' assert step_input.prev_output_handle.output_name == VALUE_OUTPUT assert len(steps[1].step_outputs) == 1 assert len(subplan.topological_steps()) == 2 assert [step.key for step in subplan.topological_steps()] == [ 'add_one.transform.input.num.value', 'add_one.transform', ]
def run_test_pipeline(pipeline): typed_environment = create_typed_environment(pipeline, TEST_ENVIRONMENT) execution_metadata = ExecutionMetadata(run_id=str(uuid.uuid4())) with yield_context(pipeline, typed_environment, execution_metadata) as context: execution_plan = create_execution_plan_core( ExecutionPlanInfo(context, pipeline, typed_environment)) return execute_plan(context, execution_plan)
def create_plan(pipeline): config_or_error = _config_or_error_from_pipeline( graphene_info, pipeline, config) return config_or_error.chain( lambda evaluate_value_result: graphene_info.schema.type_named( 'ExecutionPlan')( pipeline, create_execution_plan( pipeline.get_dagster_pipeline(), evaluate_value_result.value, ExecutionMetadata(), ), ))
def create_plan(pipeline): config_or_error = _config_or_error_from_pipeline( info, pipeline, config) return config_or_error.chain( lambda evaluate_value_result: info.schema. type_named('ExecutionPlan')( pipeline, create_execution_plan_with_typed_environment( pipeline.get_dagster_pipeline(), construct_environment_config(evaluate_value_result.value), ExecutionMetadata(), ), ))
def test_injected_tags(): called = {} @solid def check_tags(info): assert info.context.get_tag('foo') == 'bar' called['yup'] = True pipeline_def = PipelineDefinition(name='injected_run_id', solids=[check_tags]) result = execute_pipeline( pipeline_def, execution_metadata=ExecutionMetadata(tags={'foo': 'bar'}) ) assert result.success assert called['yup']
def _start_execution(validated_config_either): new_run_id = str(uuid.uuid4()) execution_plan = create_execution_plan_with_typed_environment( pipeline.get_dagster_pipeline(), construct_environment_config(validated_config_either.value), ExecutionMetadata(), ) run = pipeline_run_storage.create_run(new_run_id, selector, env_config, execution_plan) pipeline_run_storage.add_run(run) info.context.execution_manager.execute_pipeline( info.context.repository_container, pipeline.get_dagster_pipeline(), run) return info.schema.type_named('StartPipelineExecutionSuccess')( run=info.schema.type_named('PipelineRun')(run))
def test_compute_noop_node_core(): pipeline = silencing_pipeline(solids=[noop]) environment = EnvironmentConfig() plan = create_execution_plan_core( ExecutionPlanInfo(create_test_runtime_execution_context(), pipeline, environment), ExecutionMetadata(), ) assert len(plan.steps) == 1 outputs = list( execute_step(plan.steps[0], create_test_runtime_execution_context(), {})) assert outputs[0].success_data.value == 'foo'
def _start_execution(validated_config_either): new_run_id = str(uuid.uuid4()) execution_plan = create_execution_plan( pipeline.get_dagster_pipeline(), validated_config_either.value, ExecutionMetadata()) run = pipeline_run_storage.create_run(new_run_id, selector, env_config, execution_plan) pipeline_run_storage.add_run(run) graphene_info.context.execution_manager.execute_pipeline( graphene_info.context.repository_container, pipeline.get_dagster_pipeline(), run, throw_on_user_error=graphene_info.context.throw_on_user_error, ) return graphene_info.schema.type_named( 'StartPipelineExecutionSuccess')( run=graphene_info.schema.type_named('PipelineRun')(run))
def test_reentrant_execute_plan(): called = {} @solid def has_tag(context): assert context.has_tag('foo') assert context.get_tag('foo') == 'bar' called['yup'] = True pipeline_def = PipelineDefinition(name='has_tag_pipeline', solids=[has_tag]) execution_plan = create_execution_plan(pipeline_def) step_events = execute_plan( execution_plan, execution_metadata=ExecutionMetadata(tags={'foo': 'bar'})) assert called['yup'] assert len(step_events) == 1
def execute_pipeline(self, repository_container, pipeline, pipeline_run): check.inst_param(pipeline, 'pipeline', PipelineDefinition) try: return execute_pipeline_with_metadata( pipeline, create_typed_environment(pipeline, pipeline_run.config), execution_metadata=ExecutionMetadata( pipeline_run.run_id, event_callback=pipeline_run.handle_new_event ), throw_on_user_error=False, ) except: # pylint: disable=W0702 pipeline_run.handle_new_event( build_synthetic_pipeline_error_record( pipeline_run.run_id, serializable_error_info_from_exc_info(sys.exc_info()), pipeline.name, ) )
def execute_pipeline_through_queue( repository_info, pipeline_name, pipeline_solid_subset, config, run_id, message_queue ): """ Execute pipeline using message queue as a transport """ message_queue.put(ProcessStartedSentinel(os.getpid())) execution_metadata = ExecutionMetadata(run_id, event_callback=message_queue.put) from .app import RepositoryContainer repository_container = RepositoryContainer(repository_info) if repository_container.repo_error: message_queue.put( MultiprocessingError( serializable_error_info_from_exc_info(repository_container.repo_error) ) ) return pipeline = repository_container.repository.get_pipeline(pipeline_name) pipeline = get_subset_pipeline(pipeline, pipeline_solid_subset) typed_environment = construct_environment_config( evaluate_config_value(pipeline.environment_type, config).value ) try: result = execute_pipeline_with_metadata( pipeline, typed_environment, execution_metadata=execution_metadata, throw_on_user_error=False, ) return result except: # pylint: disable=W0702 message_queue.put( MultiprocessingError(serializable_error_info_from_exc_info(sys.exc_info())) ) finally: message_queue.put(MultiprocessingDone())
def test_run_cloud(): events = [] def _event_callback(ev): events.append(ev) result = execute_pipeline( define_resource_test_pipeline(), environment_dict={ 'context': { 'cloud': { 'resources': { 'store': { 'config': { 'username': '******', 'password': '******' } } } } }, 'solids': { 'add_ints': { 'inputs': { 'num_one': { 'value': 2 }, 'num_two': { 'value': 6 } } } }, }, execution_metadata=ExecutionMetadata(event_callback=_event_callback), ) assert result.success assert result.result_for_solid('add_ints').transformed_value() == 8 assert not has_message(events, 'Setting key=add value=8 in memory') assert has_message(events, 'Setting key=add value=8 in cloud')
def test_create_subplan_source_step(): pipeline_def = define_two_int_pipeline() typed_environment = create_typed_environment(pipeline_def, None) execution_plan = create_execution_plan(pipeline_def) with yield_context(pipeline_def, typed_environment, ExecutionMetadata()) as context: subplan = create_subplan( ExecutionPlanInfo(context=context, pipeline=pipeline_def, environment=typed_environment), StepBuilderState(pipeline_name=pipeline_def.name), execution_plan, ExecutionPlanSubsetInfo(['return_one.transform']), ) assert subplan assert len(subplan.steps) == 1 assert subplan.steps[0].key == 'return_one.transform' assert not subplan.steps[0].step_inputs assert len(subplan.steps[0].step_outputs) == 1 assert len(subplan.topological_steps()) == 1
def execute_pipeline(self, repository_container, pipeline, pipeline_run, throw_on_user_error): check.inst_param(pipeline, 'pipeline', PipelineDefinition) try: return execute_pipeline( pipeline, pipeline_run.config, execution_metadata=ExecutionMetadata( pipeline_run.run_id, event_callback=pipeline_run.handle_new_event), throw_on_user_error=throw_on_user_error, ) except: # pylint: disable=W0702 if throw_on_user_error: six.reraise(*sys.exc_info()) pipeline_run.handle_new_event( build_synthetic_pipeline_error_record( pipeline_run.run_id, serializable_error_info_from_exc_info(sys.exc_info()), pipeline.name, ))
def run_test_pipeline(pipeline): execution_metadata = ExecutionMetadata(run_id=str(uuid.uuid4())) with yield_pipeline_execution_context(pipeline, TEST_ENVIRONMENT, execution_metadata) as context: execution_plan = create_execution_plan_core(context) return execute_plan(context, execution_plan)