def test_pipeline_init_failure(): stub_solid = define_stub_solid('stub', None) env_config = {} def failing_resource_fn(*args, **kwargs): raise Exception() pipeline_def = PipelineDefinition( [stub_solid], 'failing_init_pipeline', mode_definitions=[ ModeDefinition( resources={ 'failing': ResourceDefinition( resource_fn=failing_resource_fn) }) ], ) result = execute_pipeline( pipeline_def, environment_dict=env_config, run_config=RunConfig(executor_config=InProcessExecutorConfig( raise_on_error=False)), ) assert result.success is False assert len(result.event_list) == 1 event = result.event_list[0] assert event.event_type_value == 'PIPELINE_INIT_FAILURE' assert event.pipeline_init_failure_data
def test_error_pipeline(): pipeline = define_error_pipeline() result = execute_pipeline( pipeline, run_config=RunConfig(executor_config=InProcessExecutorConfig(raise_on_error=False)), ) assert not result.success
def execute_pipeline(self, repository_container, pipeline, pipeline_run, raise_on_error): check.inst_param(pipeline, 'pipeline', PipelineDefinition) try: return execute_pipeline( pipeline, pipeline_run.config, run_config=RunConfig( pipeline_run.run_id, event_callback=pipeline_run.handle_new_event, executor_config=InProcessExecutorConfig( raise_on_error=raise_on_error), reexecution_config=pipeline_run.reexecution_config, step_keys_to_execute=pipeline_run.step_keys_to_execute, ), ) except: # pylint: disable=W0702 if raise_on_error: six.reraise(*sys.exc_info()) pipeline_run.handle_new_event( build_synthetic_pipeline_error_record( pipeline_run.run_id, serializable_error_info_from_exc_info(sys.exc_info()), pipeline.name, ))
def test_resource_init_failure(): @resource def failing_resource(_init_context): raise Exception('Uh oh') @solid(required_resource_keys={'failing_resource'}) def failing_resource_solid(_context): pass pipeline = PipelineDefinition( name='test_resource_init_failure', solid_defs=[failing_resource_solid], mode_defs=[ ModeDefinition( resource_defs={'failing_resource': failing_resource}) ], ) run_config = RunConfig(executor_config=InProcessExecutorConfig( raise_on_error=False)) res = execute_pipeline(pipeline, run_config=run_config) assert res.event_list[0].event_type_value == 'PIPELINE_INIT_FAILURE' execution_plan = create_execution_plan(pipeline, run_config=run_config) step_events = execute_plan( execution_plan, run_config=run_config, step_keys_to_execute=[ step.key for step in execution_plan.topological_steps() ], ) assert step_events[0].event_type_value == 'PIPELINE_INIT_FAILURE'
def execute_execute_command_with_preset(preset, raise_on_error, cli_args, _mode): pipeline = handle_for_pipeline_cli_args( cli_args).build_pipeline_definition() cli_args.pop('pipeline_name') kwargs = pipeline.get_preset(preset) kwargs['run_config'] = kwargs['run_config'].with_executor_config( InProcessExecutorConfig(raise_on_error=raise_on_error)) return execute_pipeline(**kwargs)
def execute_execute_command_with_preset(preset, raise_on_error, cli_args): pipeline_target = load_pipeline_target_from_cli_args(cli_args) cli_args.pop('pipeline_name') repository_target_info = load_target_info_from_cli_args(cli_args) repository = load_repository_from_target_info(repository_target_info) return execute_pipeline( run_config=RunConfig(executor_config=InProcessExecutorConfig( raise_on_error=raise_on_error)), **(repository.get_preset_pipeline(pipeline_target.pipeline_name, preset)))
def execute_pipeline_through_queue( handle, pipeline_name, solid_subset, environment_dict, mode, run_id, message_queue, reexecution_config, step_keys_to_execute, ): """ Execute pipeline using message queue as a transport """ check.opt_str_param(mode, 'mode') message_queue.put(ProcessStartedSentinel(os.getpid())) run_config = RunConfig( run_id, mode=mode, event_callback=message_queue.put, executor_config=InProcessExecutorConfig(raise_on_error=False), reexecution_config=reexecution_config, step_keys_to_execute=step_keys_to_execute, ) try: repository = handle.build_repository_definition() except Exception: # pylint: disable=broad-except repo_error = sys.exc_info() message_queue.put( MultiprocessingError( serializable_error_info_from_exc_info(repo_error))) return try: result = execute_pipeline( repository.get_pipeline(pipeline_name).build_sub_pipeline( solid_subset), environment_dict, run_config=run_config, ) return result except Exception: # pylint: disable=broad-except error_info = serializable_error_info_from_exc_info(sys.exc_info()) message_queue.put(MultiprocessingError(error_info)) finally: message_queue.put(MultiprocessingDone()) message_queue.close()
def execute_execute_command_with_preset(preset, raise_on_error, cli_args, mode): pipeline = handle_for_pipeline_cli_args( cli_args).build_pipeline_definition() cli_args.pop('pipeline_name') repository = handle_for_repo_cli_args( cli_args).build_repository_definition() kwargs = repository.get_preset_pipeline(pipeline.name, preset) return execute_pipeline(run_config=RunConfig( mode=mode, executor_config=InProcessExecutorConfig( raise_on_error=raise_on_error)), **kwargs)
def do_execute_command(pipeline, env_file_list, raise_on_error): check.inst_param(pipeline, 'pipeline', PipelineDefinition) env_file_list = check.opt_list_param(env_file_list, 'env_file_list', of_type=str) environment_dict = load_yaml_from_glob_list( env_file_list) if env_file_list else {} return execute_pipeline( pipeline, environment_dict=environment_dict, run_config=RunConfig(executor_config=InProcessExecutorConfig( raise_on_error=raise_on_error)), )
def test_error_resource(snapshot): result = execute_pipeline( resource_error_pipeline, environment_dict={'storage': { 'filesystem': {} }}, run_config=RunConfig(executor_config=InProcessExecutorConfig( raise_on_error=False)), ) assert not result.success assert len(result.event_list) == 1 init_failure_event = result.event_list[0] assert init_failure_event.event_type_value == 'PIPELINE_INIT_FAILURE' snapshot.assert_match(init_failure_event.message)
def execute_pipeline_through_queue( repository_info, pipeline_name, solid_subset, environment_dict, run_id, message_queue, reexecution_config, step_keys_to_execute, ): """ Execute pipeline using message queue as a transport """ message_queue.put(ProcessStartedSentinel(os.getpid())) run_config = RunConfig( run_id, event_callback=message_queue.put, executor_config=InProcessExecutorConfig(raise_on_error=False), reexecution_config=reexecution_config, step_keys_to_execute=step_keys_to_execute, ) repository_container = RepositoryContainer(repository_info) if repository_container.repo_error: message_queue.put( MultiprocessingError( serializable_error_info_from_exc_info( repository_container.repo_error))) return try: result = execute_pipeline( repository_container.repository.get_pipeline( pipeline_name).build_sub_pipeline(solid_subset), environment_dict, run_config=run_config, ) return result except: # pylint: disable=W0702 error_info = serializable_error_info_from_exc_info(sys.exc_info()) message_queue.put(MultiprocessingError(error_info)) finally: message_queue.put(MultiprocessingDone()) message_queue.close()
def test_single_solid_pipeline_failure(): events = defaultdict(list) @lambda_solid def solid_one(): raise Exception('nope') def _event_callback(record): if record.is_dagster_event: events[record.dagster_event.event_type].append(record) pipeline_def = PipelineDefinition( name='single_solid_pipeline', solids=[solid_one], mode_definitions=[mode_def(_event_callback)], ) result = execute_pipeline( pipeline_def, {'loggers': { 'callback': {} }}, run_config=RunConfig(executor_config=InProcessExecutorConfig( raise_on_error=False)), ) assert not result.success start_event = single_dagster_event(events, DagsterEventType.STEP_START) assert start_event.pipeline_name == 'single_solid_pipeline' assert start_event.dagster_event.solid_name == 'solid_one' assert start_event.dagster_event.solid_definition_name == 'solid_one' assert start_event.level == logging.INFO failure_event = single_dagster_event(events, DagsterEventType.STEP_FAILURE) assert failure_event.pipeline_name == 'single_solid_pipeline' assert failure_event.dagster_event.solid_name == 'solid_one' assert failure_event.dagster_event.solid_definition_name == 'solid_one' assert failure_event.level == logging.ERROR
def execute_single_solid_in_isolation(context_params, solid_def, environment=None, raise_on_error=True): ''' Deprecated. Execute a solid outside the context of a pipeline, with an already-created context. Prefer execute_solid in dagster.utils.test ''' check.inst_param(context_params, 'context_params', ExecutionContext) check.inst_param(solid_def, 'solid_def', SolidDefinition) environment = check.opt_dict_param(environment, 'environment') check.bool_param(raise_on_error, 'raise_on_error') single_solid_environment = { 'expectations': environment.get('expectations'), 'context': environment.get('context'), 'solids': { solid_def.name: environment['solids'][solid_def.name] } if solid_def.name in environment.get('solids', {}) else None, } pipeline_result = execute_pipeline( PipelineDefinition( solids=[solid_def], context_definitions=PipelineContextDefinition. passthrough_context_definition(context_params), ), environment_dict=single_solid_environment, run_config=RunConfig(executor_config=InProcessExecutorConfig( raise_on_error=raise_on_error)), ) return pipeline_result
def execute_no_throw(pipeline_def): return execute_pipeline( pipeline_def, run_config=RunConfig(executor_config=InProcessExecutorConfig( raise_on_error=False)), )
def test_diamond_simple_execution(): result = execute_pipeline( define_diamond_pipeline(), run_config=RunConfig(executor_config=InProcessExecutorConfig()) ) assert result.success assert result.result_for_solid('adder').result_value() == 11
ModeDefinition, InProcessExecutorConfig, RunConfig, ) @resource(config_field=Field(Int, is_optional=True)) def a_resource(context): raise Exception("Bad Resource") resources = {'BadResource': a_resource} @solid(required_resource_keys={'BadResource'}) def one(_): return 1 @pipeline(mode_defs=[ModeDefinition(resource_defs=resources)]) def resource_error_pipeline(): one() if __name__ == '__main__': result = execute_pipeline( resource_error_pipeline, environment_dict={'storage': {'filesystem': {}}}, run_config=RunConfig(executor_config=InProcessExecutorConfig(raise_on_error=False)), )