def test_s3_pickle_io_manager_execution(mock_s3_bucket): pipeline_def = define_inty_pipeline() run_config = { "resources": { "io_manager": { "config": { "s3_bucket": mock_s3_bucket.name } } } } run_id = make_new_run_id() execution_plan = create_execution_plan(pipeline_def, run_config=run_config) assert execution_plan.get_step_by_key("return_one") step_keys = ["return_one"] instance = DagsterInstance.ephemeral() pipeline_run = PipelineRun(pipeline_name=pipeline_def.name, run_id=run_id, run_config=run_config) return_one_step_events = list( execute_plan( execution_plan.build_subset_plan(step_keys), run_config=run_config, pipeline_run=pipeline_run, instance=instance, )) assert get_step_output(return_one_step_events, "return_one") io_manager = PickledObjectS3IOManager(mock_s3_bucket.name, construct_s3_client(max_attempts=5), s3_prefix="dagster") step_output_handle = StepOutputHandle("return_one") context = InputContext( pipeline_name=pipeline_def.name, solid_def=pipeline_def.solid_def_named("return_one"), config={}, metadata={}, upstream_output=OutputContext( step_key=step_output_handle.step_key, name=step_output_handle.output_name, pipeline_name=pipeline_def.name, run_id=run_id, metadata={}, mapping_key=None, config=None, solid_def=pipeline_def.solid_def_named("return_one"), ), log_manager=DagsterLogManager(run_id=pipeline_run.run_id, logging_tags={}, loggers=[]), ) assert io_manager.load_input(context) == 1 add_one_step_events = list( execute_plan( execution_plan.build_subset_plan(["add_one"]), run_config=run_config, pipeline_run=pipeline_run, instance=instance, )) step_output_handle = StepOutputHandle("add_one") context = InputContext( pipeline_name=pipeline_def.name, solid_def=pipeline_def.solid_def_named("add_one"), config={}, metadata={}, upstream_output=OutputContext( step_key=step_output_handle.step_key, name=step_output_handle.output_name, pipeline_name=pipeline_def.name, run_id=run_id, metadata={}, mapping_key=None, config=None, solid_def=pipeline_def.solid_def_named("add_one"), ), log_manager=DagsterLogManager(run_id=pipeline_run.run_id, logging_tags={}, loggers=[]), ) assert get_step_output(add_one_step_events, "add_one") assert io_manager.load_input(context) == 2
def test_logging_basic(): with _setup_logger('test') as (captured_results, logger): dl = DagsterLogManager('123', {}, [logger]) dl.debug('test') dl.info('test') dl.warning('test') dl.error('test') dl.critical('test') kv_pairs = captured_results[0].replace(' ', '').split('\n')[1:] _validate_basic(kv_pairs)
def host_mode_execution_context_event_generator( pipeline, execution_plan, run_config, pipeline_run, instance, raise_on_error, executor_defs, output_capture, resume_from_failure: bool = False, ): check.inst_param(execution_plan, "execution_plan", ExecutionPlan) check.inst_param(pipeline, "pipeline", ReconstructablePipeline) check.dict_param(run_config, "run_config", key_type=str) check.inst_param(pipeline_run, "pipeline_run", PipelineRun) check.inst_param(instance, "instance", DagsterInstance) executor_defs = check.list_param(executor_defs, "executor_defs", of_type=ExecutorDefinition) check.bool_param(raise_on_error, "raise_on_error") check.invariant(output_capture is None) execution_context = None loggers = [] for (logger_def, logger_config) in default_system_loggers(): loggers.append( logger_def.logger_fn( InitLoggerContext( logger_config, pipeline_def=None, logger_def=logger_def, run_id=pipeline_run.run_id, ))) log_manager = DagsterLogManager.create(loggers=loggers, pipeline_run=pipeline_run, instance=instance) try: executor = _get_host_mode_executor(pipeline, run_config, executor_defs, instance) execution_context = PlanOrchestrationContext( plan_data=PlanData( pipeline=pipeline, pipeline_run=pipeline_run, instance=instance, execution_plan=execution_plan, raise_on_error=raise_on_error, retry_mode=executor.retries, ), log_manager=log_manager, executor=executor, output_capture=None, resume_from_failure=resume_from_failure, ) yield execution_context except DagsterError as dagster_error: if execution_context is None: user_facing_exc_info = ( # pylint does not know original_exc_info exists is is_user_code_error is true # pylint: disable=no-member dagster_error.original_exc_info # type: ignore if dagster_error.is_user_code_error else sys.exc_info()) error_info = serializable_error_info_from_exc_info( user_facing_exc_info) event = DagsterEvent.pipeline_failure( pipeline_context_or_name=pipeline_run.pipeline_name, context_msg= (f'Pipeline failure during initialization for pipeline "{pipeline_run.pipeline_name}". ' "This may be due to a failure in initializing the executor or one of the loggers." ), error_info=error_info, ) log_manager.log_dagster_event( level=logging.ERROR, msg=event.message, dagster_event=event # type: ignore ) yield event else: # pipeline teardown failure raise dagster_error if raise_on_error: raise dagster_error
def test_logging_bad_custom_log_levels(): with _setup_logger('test') as (_, logger): dl = DagsterLogManager('123', {}, [logger]) with pytest.raises(check.CheckError): dl._log('test', 'foobar', {}) # pylint: disable=protected-access
def test_multiline_logging_complex(): msg = 'DagsterEventType.STEP_FAILURE for step start.materialization.output.result.0' kwargs = { 'pipeline': 'example', 'pipeline_name': 'example', 'step_key': 'start.materialization.output.result.0', 'solid': 'start', 'solid_definition': 'emit_num', 'dagster_event': DagsterEvent( event_type_value='STEP_FAILURE', pipeline_name='error_monster', step_key='start.materialization.output.result.0', solid_handle=SolidHandle('start', 'emit_num', None), step_kind_value='MATERIALIZATION_THUNK', logging_tags={ 'pipeline': 'error_monster', 'step_key': 'start.materialization.output.result.0', 'solid': 'start', 'solid_definition': 'emit_num', }, event_specific_data=StepFailureData( error=SerializableErrorInfo( message= "FileNotFoundError: [Errno 2] No such file or directory: '/path/to/file'\n", stack=[ ' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/errors.py", line 186, in user_code_error_boundary\n yield\n', ' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/execution_plan/simple_engine.py", line 365, in _event_sequence_for_step_compute_fn\n for step_output in gen:\n', ' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/execution_plan/materialization_thunk.py", line 28, in _fn\n runtime_type.output_materialization_config.materialize_runtime_value(config_spec, runtime_value)\n', ' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/types/config_schema.py", line 93, in materialize_runtime_value\n return func(config_value, runtime_value)\n', ' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/types/config_schema.py", line 110, in _selector\n return func(selector_key, selector_value, runtime_value)\n', ' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/types/builtin_config_schemas.py", line 59, in _builtin_output_schema\n with open(json_file_path, \'w\') as ff:\n', ], cls_name='FileNotFoundError', ), user_failure_data=None, ), ), } with _setup_logger(DAGSTER_DEFAULT_LOGGER) as (captured_results, logger): dl = DagsterLogManager('123', {}, [logger]) dl.info(msg, **kwargs) kv_pairs = set(captured_results[0].split('\n')[1:]) expected_pairs = [ ' orig_message = "DagsterEventType.STEP_FAILURE for step start.materialization.output.result.0"', ' run_id = "123"', ' pipeline = "example"', ' solid_definition = "emit_num"', ' pipeline_name = "example"', ' solid = "start"', ' step_key = "start.materialization.output.result.0"', ] for e in expected_pairs: assert e in kv_pairs assert _regex_match_kv_pair( r' log_message_id = "{0}"'.format(REGEX_UUID), kv_pairs) assert _regex_match_kv_pair( r' log_timestamp = "{0}"'.format(REGEX_TS), kv_pairs) expected_dagster_event = { 'event_specific_data': [ [ "FileNotFoundError: [Errno 2] No such file or directory: '/path/to/file'\n", [ ' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/errors.py", line 186, in user_code_error_boundary\n yield\n', ' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/execution_plan/simple_engine.py", line 365, in _event_sequence_for_step_compute_fn\n for step_output in gen:\n', ' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/execution_plan/materialization_thunk.py", line 28, in _fn\n runtime_type.output_materialization_config.materialize_runtime_value(config_spec, runtime_value)\n', ' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/types/config_schema.py", line 93, in materialize_runtime_value\n return func(config_value, runtime_value)\n', ' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/types/config_schema.py", line 110, in _selector\n return func(selector_key, selector_value, runtime_value)\n', ' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/types/builtin_config_schemas.py", line 59, in _builtin_output_schema\n with open(json_file_path, \'w\') as ff:\n', ], 'FileNotFoundError', ], None, # user_failure_data ], 'event_type_value': 'STEP_FAILURE', 'message': None, 'pipeline_name': 'error_monster', 'solid_handle': ['start', 'emit_num', None], 'step_key': 'start.materialization.output.result.0', 'step_kind_value': 'MATERIALIZATION_THUNK', 'logging_tags': { 'pipeline': 'error_monster', 'solid': 'start', 'solid_definition': 'emit_num', 'step_key': 'start.materialization.output.result.0', }, } dagster_event = json.loads([ pair for pair in kv_pairs if 'dagster_event' in pair ][0].strip(' dagster_event = ')) assert dagster_event == expected_dagster_event
def test_logging_custom_log_levels(): with _setup_logger('test', {'FOO': 3}) as (_captured_results, logger): dl = DagsterLogManager('123', {}, [logger]) with pytest.raises(AttributeError): dl.foo('test') # pylint: disable=no-member
def host_mode_execution_context_event_generator( execution_plan, recon_pipeline, run_config, pipeline_run, instance, executor, raise_on_error ): check.inst_param(execution_plan, "execution_plan", ExecutionPlan) check.inst_param(recon_pipeline, "recon_pipeline", ReconstructablePipeline) check.dict_param(run_config, "run_config", key_type=str) check.inst_param(pipeline_run, "pipeline_run", PipelineRun) check.inst_param(instance, "instance", DagsterInstance) check.inst_param(executor, "executor", Executor) check.bool_param(raise_on_error, "raise_on_error") execution_context = None loggers = [] # Should these be configurable from the run config (without loading # the full EnvironmentConfig??) for (logger_def, logger_config) in default_system_loggers(): loggers.append( logger_def.logger_fn( InitLoggerContext( logger_config, pipeline_def=None, logger_def=logger_def, run_id=pipeline_run.run_id, ) ) ) loggers.append(instance.get_logger()) log_manager = DagsterLogManager( run_id=pipeline_run.run_id, logging_tags=_get_logging_tags(pipeline_run), loggers=loggers, ) # Create an executor (again how do we pull config from run_config # without going through the full EnvironmentConfig.build flow) try: execution_context = HostModeRunWorkerExecutionContext( execution_context_data=HostModeExecutionContextData( pipeline_run=pipeline_run, recon_pipeline=recon_pipeline, execution_plan=execution_plan, instance=instance, raise_on_error=raise_on_error, retry_mode=executor.retries, ), log_manager=log_manager, executor=executor, ) yield execution_context except DagsterError as dagster_error: if execution_context is None: user_facing_exc_info = ( # pylint does not know original_exc_info exists is is_user_code_error is true # pylint: disable=no-member dagster_error.original_exc_info if dagster_error.is_user_code_error else sys.exc_info() ) error_info = serializable_error_info_from_exc_info(user_facing_exc_info) yield DagsterEvent.pipeline_init_failure( pipeline_name=pipeline_run.pipeline_name, failure_data=PipelineInitFailureData(error=error_info), log_manager=log_manager, ) else: # pipeline teardown failure raise dagster_error if raise_on_error: raise dagster_error
def test_logging_basic(): with _setup_logger("test") as (captured_results, logger): dl = DagsterLogManager("123", {}, [logger]) dl.debug("test") dl.info("test") dl.warning("test") dl.error("test") dl.critical("test") assert captured_results == ["system - 123 - test"] * 5
def test_logging_integer_log_levels(): with _setup_logger("test", {"FOO": 3}) as (_captured_results, logger): dl = DagsterLogManager("123", {}, [logger]) dl.log(3, "test") # pylint: disable=no-member
def host_mode_execution_context_event_generator( pipeline, execution_plan, run_config, pipeline_run, instance, raise_on_error, get_executor_def_fn, output_capture, ): check.inst_param(execution_plan, "execution_plan", ExecutionPlan) check.inst_param(pipeline, "pipeline", ReconstructablePipeline) check.dict_param(run_config, "run_config", key_type=str) check.inst_param(pipeline_run, "pipeline_run", PipelineRun) check.inst_param(instance, "instance", DagsterInstance) get_executor_def_fn = check.opt_callable_param( get_executor_def_fn, "get_executor_def_fn", _default_get_executor_def_fn) check.bool_param(raise_on_error, "raise_on_error") check.invariant(output_capture is None) execution_context = None loggers = [] for (logger_def, logger_config) in default_system_loggers(): loggers.append( logger_def.logger_fn( InitLoggerContext( logger_config, pipeline_def=None, logger_def=logger_def, run_id=pipeline_run.run_id, ))) loggers.append(instance.get_logger()) log_manager = DagsterLogManager( run_id=pipeline_run.run_id, logging_tags=get_logging_tags(pipeline_run), loggers=loggers, ) try: executor = _get_host_mode_executor(pipeline, run_config, get_executor_def_fn, instance) execution_context = PlanOrchestrationContext( plan_data=PlanData( pipeline=pipeline, pipeline_run=pipeline_run, instance=instance, execution_plan=execution_plan, raise_on_error=raise_on_error, retry_mode=executor.retries, ), log_manager=log_manager, executor=executor, output_capture=None, ) yield execution_context except DagsterError as dagster_error: if execution_context is None: user_facing_exc_info = ( # pylint does not know original_exc_info exists is is_user_code_error is true # pylint: disable=no-member dagster_error.original_exc_info if dagster_error.is_user_code_error else sys.exc_info()) error_info = serializable_error_info_from_exc_info( user_facing_exc_info) yield DagsterEvent.pipeline_init_failure( pipeline_name=pipeline_run.pipeline_name, failure_data=PipelineInitFailureData(error=error_info), log_manager=log_manager, ) else: # pipeline teardown failure raise dagster_error if raise_on_error: raise dagster_error
def test_logging_no_loggers_registered(): dl = DagsterLogManager("none", {}, []) dl.debug("test") dl.info("test") dl.warning("test") dl.error("test") dl.critical("test")
def test_logging_integer_log_levels(): with _setup_logger("test", {"FOO": 3}) as (_captured_results, logger): dl = DagsterLogManager.create(loggers=[logger], pipeline_run=PipelineRun(run_id="123")) dl.log(3, "test") # pylint: disable=no-member
def test_adls2_pickle_io_manager_execution(storage_account, file_system, credential): pipeline_def = define_inty_pipeline() run_config = { "resources": { "io_manager": { "config": { "adls2_file_system": file_system } }, "adls2": { "config": { "storage_account": storage_account, "credential": { "key": credential } } }, } } run_id = make_new_run_id() execution_plan = create_execution_plan(pipeline_def, run_config=run_config) assert execution_plan.get_step_by_key("return_one") step_keys = ["return_one"] instance = DagsterInstance.ephemeral() pipeline_run = PipelineRun(pipeline_name=pipeline_def.name, run_id=run_id, run_config=run_config) return_one_step_events = list( execute_plan( execution_plan.build_subset_plan(step_keys), run_config=run_config, pipeline_run=pipeline_run, instance=instance, )) assert get_step_output(return_one_step_events, "return_one") step_output_handle = StepOutputHandle("return_one") context = InputContext( pipeline_name=pipeline_def.name, solid_def=pipeline_def.solid_def_named("return_one"), upstream_output=OutputContext( step_key=step_output_handle.step_key, name=step_output_handle.output_name, pipeline_name=pipeline_def.name, run_id=run_id, solid_def=pipeline_def.solid_def_named("return_one"), ), log_manager=DagsterLogManager(run_id=pipeline_run.run_id, logging_tags={}, loggers=[]), ) io_manager = PickledObjectADLS2IOManager( file_system=file_system, adls2_client=create_adls2_client(storage_account, credential), blob_client=create_blob_client(storage_account, credential), ) assert io_manager.load_input(context) == 1 add_one_step_events = list( execute_plan( execution_plan.build_subset_plan(["add_one"]), pipeline_run=pipeline_run, run_config=run_config, instance=instance, )) step_output_handle = StepOutputHandle("add_one") context = InputContext( pipeline_name=pipeline_def.name, solid_def=pipeline_def.solid_def_named("add_one"), upstream_output=OutputContext( step_key=step_output_handle.step_key, name=step_output_handle.output_name, pipeline_name=pipeline_def.name, run_id=run_id, solid_def=pipeline_def.solid_def_named("add_one"), ), log_manager=DagsterLogManager(run_id=pipeline_run.run_id, logging_tags={}, loggers=[]), ) assert get_step_output(add_one_step_events, "add_one") assert io_manager.load_input(context) == 2
def resource_initialization_event_generator( execution_plan, environment_config, pipeline_run, log_manager, resource_keys_to_init ): check.inst_param(execution_plan, 'execution_plan', ExecutionPlan) check.inst_param(environment_config, 'environment_config', EnvironmentConfig) check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) check.inst_param(log_manager, 'log_manager', DagsterLogManager) check.set_param(resource_keys_to_init, 'resource_keys_to_init', of_type=str) if execution_plan.step_key_for_single_step_plans(): step = execution_plan.get_step_by_key(execution_plan.step_key_for_single_step_plans()) resource_log_manager = DagsterLogManager( pipeline_run.run_id, merge_dicts(log_manager.logging_tags, step.logging_tags), log_manager.loggers, ) else: resource_log_manager = log_manager resource_instances = {} pipeline_def = execution_plan.pipeline_def mode_definition = pipeline_def.get_mode_definition(pipeline_run.mode) resource_managers = deque() generator_closed = False resource_init_times = {} try: if resource_keys_to_init: yield DagsterEvent.resource_init_start( execution_plan, resource_log_manager, resource_keys_to_init, ) for resource_name, resource_def in sorted(mode_definition.resource_defs.items()): if not resource_name in resource_keys_to_init: continue resource_context = InitResourceContext( pipeline_def=pipeline_def, resource_def=resource_def, resource_config=environment_config.resources.get(resource_name, {}).get('config'), run_id=pipeline_run.run_id, log_manager=resource_log_manager, ) manager = single_resource_generation_manager( resource_context, resource_name, resource_def ) for event in manager.generate_setup_events(): if event: yield event initialized_resource = check.inst(manager.get_object(), InitializedResource) resource_instances[resource_name] = initialized_resource.resource resource_init_times[resource_name] = initialized_resource.duration resource_managers.append(manager) if resource_keys_to_init: yield DagsterEvent.resource_init_success( execution_plan, resource_log_manager, resource_instances, resource_init_times ) yield ScopedResourcesBuilder(resource_instances) except GeneratorExit: # Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed # (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/). generator_closed = True raise except DagsterUserCodeExecutionError as dagster_user_error: yield DagsterEvent.resource_init_failure( execution_plan, resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info(dagster_user_error.original_exc_info), ) raise dagster_user_error finally: if not generator_closed: error = None while len(resource_managers) > 0: manager = resource_managers.pop() try: for event in manager.generate_teardown_events(): yield event except DagsterUserCodeExecutionError as dagster_user_error: error = dagster_user_error if error: yield DagsterEvent.resource_teardown_failure( execution_plan, resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info(error.original_exc_info), )
def test_logging_no_loggers_registered(): dl = DagsterLogManager('none', {}, []) dl.debug('test') dl.info('test') dl.warning('test') dl.error('test') dl.critical('test')
def resource_initialization_event_generator( resource_defs: Dict[str, ResourceDefinition], resource_configs: Dict[str, ResourceConfig], log_manager: DagsterLogManager, execution_plan: Optional[ExecutionPlan], pipeline_run: Optional[PipelineRun], resource_keys_to_init: Optional[Set[str]], instance: Optional[DagsterInstance], resource_instances_to_override: Optional[Dict[str, "InitializedResource"]], emit_persistent_events: Optional[bool], ): check.inst_param(log_manager, "log_manager", DagsterLogManager) resource_keys_to_init = check.opt_set_param(resource_keys_to_init, "resource_keys_to_init", of_type=str) check.opt_inst_param(execution_plan, "execution_plan", ExecutionPlan) check.opt_inst_param(pipeline_run, "pipeline_run", PipelineRun) check.opt_inst_param(instance, "instance", DagsterInstance) check.opt_dict_param(resource_instances_to_override, "resource_instances_to_override") if execution_plan and execution_plan.step_handle_for_single_step_plans(): step = execution_plan.get_step( cast( StepHandleUnion, cast(ExecutionPlan, execution_plan).step_handle_for_single_step_plans(), )) resource_log_manager = log_manager.with_tags( **cast(ExecutionStep, step).logging_tags) else: resource_log_manager = log_manager generator_closed = False resource_managers: Deque[EventGenerationManager] = deque() try: yield from _core_resource_initialization_event_generator( resource_defs=resource_defs, resource_configs=resource_configs, resource_log_manager=resource_log_manager, resource_managers=resource_managers, execution_plan=execution_plan, pipeline_run=pipeline_run, resource_keys_to_init=resource_keys_to_init, instance=instance, resource_instances_to_override=resource_instances_to_override, emit_persistent_events=emit_persistent_events, ) except GeneratorExit: # Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed # (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/). generator_closed = True raise finally: if not generator_closed: error = None while len(resource_managers) > 0: manager = resource_managers.pop() try: yield from manager.generate_teardown_events() except DagsterUserCodeExecutionError as dagster_user_error: error = dagster_user_error if error: yield DagsterEvent.resource_teardown_failure( execution_plan, resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info( error.original_exc_info), )
def test_logging_basic(): with _setup_logger('test') as (captured_results, logger): dl = DagsterLogManager('123', {}, [logger]) dl.debug('test') dl.info('test') dl.warning('test') dl.error('test') dl.critical('test') assert captured_results == ['system - 123 - test'] * 5
def _core_resource_initialization_event_generator( resource_defs: Dict[str, ResourceDefinition], resource_configs: Dict[str, ResourceConfig], resource_log_manager: DagsterLogManager, resource_managers: Deque[EventGenerationManager], execution_plan: Optional[ExecutionPlan], pipeline_run: Optional[PipelineRun], resource_keys_to_init: Optional[Set[str]], instance: Optional[DagsterInstance], resource_instances_to_override: Optional[Dict[str, "InitializedResource"]], emit_persistent_events: Optional[bool], ): if emit_persistent_events: check.invariant( execution_plan, "If emit_persistent_events is enabled, then execution_plan must be provided", ) resource_instances_to_override = check.opt_dict_param( resource_instances_to_override, "resource_instances_to_override") resource_keys_to_init = check.opt_set_param(resource_keys_to_init, "resource_keys_to_init") resource_instances: Dict[str, "InitializedResource"] = {} resource_init_times = {} try: if emit_persistent_events and resource_keys_to_init: yield DagsterEvent.resource_init_start( execution_plan, resource_log_manager, resource_keys_to_init, ) resource_dependencies = _resolve_resource_dependencies(resource_defs) for level in toposort(resource_dependencies): for resource_name in level: if resource_name in resource_instances_to_override: # use the given resource instances instead of re-initiating it from resource def resource_def = ResourceDefinition.hardcoded_resource( resource_instances_to_override[resource_name]) else: resource_def = resource_defs[resource_name] if not resource_name in resource_keys_to_init: continue resource_context = InitResourceContext( resource_def=resource_def, resource_config=resource_configs[resource_name].config, pipeline_run=pipeline_run, # Add tags with information about the resource log_manager=resource_log_manager.with_tags( resource_name=resource_name, resource_fn_name=str( resource_def.resource_fn.__name__), ), resource_instance_dict=resource_instances, required_resource_keys=resource_def.required_resource_keys, instance=instance, pipeline_def_for_backwards_compat=execution_plan. pipeline_def if execution_plan else None, ) manager = single_resource_generation_manager( resource_context, resource_name, resource_def) for event in manager.generate_setup_events(): if event: yield event initialized_resource = check.inst(manager.get_object(), InitializedResource) resource_instances[ resource_name] = initialized_resource.resource resource_init_times[ resource_name] = initialized_resource.duration resource_managers.append(manager) if emit_persistent_events and resource_keys_to_init: yield DagsterEvent.resource_init_success(execution_plan, resource_log_manager, resource_instances, resource_init_times) yield ScopedResourcesBuilder(resource_instances) except DagsterUserCodeExecutionError as dagster_user_error: # Can only end up in this state if we attempt to initialize a resource, so # resource_keys_to_init cannot be empty if emit_persistent_events: yield DagsterEvent.resource_init_failure( execution_plan, resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info( dagster_user_error.original_exc_info), ) raise dagster_user_error
def test_logging_integer_log_levels(): with _setup_logger('test', {'FOO': 3}) as (_captured_results, logger): dl = DagsterLogManager('123', {}, [logger]) dl.log(3, 'test') # pylint: disable=no-member
def _core_resource_initialization_event_generator( resource_defs: Dict[str, ResourceDefinition], resource_configs: Dict[str, ResourceConfig], resource_log_manager: DagsterLogManager, resource_managers: Deque[EventGenerationManager], execution_plan: Optional[ExecutionPlan], pipeline_run: Optional[PipelineRun], resource_keys_to_init: Optional[AbstractSet[str]], instance: Optional[DagsterInstance], emit_persistent_events: Optional[bool], pipeline_def_for_backwards_compat: Optional[PipelineDefinition], ): pipeline_name = None contains_generator = False if emit_persistent_events: check.invariant( pipeline_run and execution_plan, "If emit_persistent_events is enabled, then pipeline_run and execution_plan must be provided", ) pipeline_name = cast(PipelineRun, pipeline_run).pipeline_name resource_keys_to_init = check.opt_set_param(resource_keys_to_init, "resource_keys_to_init") resource_instances: Dict[str, "InitializedResource"] = {} resource_init_times = {} try: if emit_persistent_events and resource_keys_to_init: yield DagsterEvent.resource_init_start( cast(str, pipeline_name), cast(ExecutionPlan, execution_plan), resource_log_manager, resource_keys_to_init, ) resource_dependencies = _resolve_resource_dependencies(resource_defs) for level in toposort(resource_dependencies): for resource_name in level: resource_def = resource_defs[resource_name] if not resource_name in resource_keys_to_init: continue resource_fn = cast(Callable[[InitResourceContext], Any], resource_def.resource_fn) resources = ScopedResourcesBuilder(resource_instances).build( resource_def.required_resource_keys) resource_context = InitResourceContext( resource_def=resource_def, resource_config=resource_configs[resource_name].config, pipeline_run=pipeline_run, # Add tags with information about the resource log_manager=resource_log_manager.with_tags( resource_name=resource_name, resource_fn_name=str(resource_fn.__name__), ), resources=resources, instance=instance, pipeline_def_for_backwards_compat= pipeline_def_for_backwards_compat, ) manager = single_resource_generation_manager( resource_context, resource_name, resource_def) for event in manager.generate_setup_events(): if event: yield event initialized_resource = check.inst(manager.get_object(), InitializedResource) resource_instances[ resource_name] = initialized_resource.resource resource_init_times[ resource_name] = initialized_resource.duration contains_generator = contains_generator or initialized_resource.is_generator resource_managers.append(manager) if emit_persistent_events and resource_keys_to_init: yield DagsterEvent.resource_init_success( cast(str, pipeline_name), cast(ExecutionPlan, execution_plan), resource_log_manager, resource_instances, resource_init_times, ) yield ScopedResourcesBuilder(resource_instances, contains_generator) except DagsterUserCodeExecutionError as dagster_user_error: # Can only end up in this state if we attempt to initialize a resource, so # resource_keys_to_init cannot be empty if emit_persistent_events: yield DagsterEvent.resource_init_failure( cast(str, pipeline_name), cast(ExecutionPlan, execution_plan), resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info( dagster_user_error.original_exc_info), ) raise dagster_user_error