def test_s3_pickle_io_manager_execution(mock_s3_bucket):
pipeline_def = define_inty_pipeline()
run_config = {
"resources": {
"io_manager": {
"config": {
"s3_bucket": mock_s3_bucket.name
}
}
}
}
run_id = make_new_run_id()
execution_plan = create_execution_plan(pipeline_def, run_config=run_config)
assert execution_plan.get_step_by_key("return_one")
step_keys = ["return_one"]
instance = DagsterInstance.ephemeral()
pipeline_run = PipelineRun(pipeline_name=pipeline_def.name,
run_id=run_id,
run_config=run_config)
return_one_step_events = list(
execute_plan(
execution_plan.build_subset_plan(step_keys),
run_config=run_config,
pipeline_run=pipeline_run,
instance=instance,
))
assert get_step_output(return_one_step_events, "return_one")
io_manager = PickledObjectS3IOManager(mock_s3_bucket.name,
construct_s3_client(max_attempts=5),
s3_prefix="dagster")
step_output_handle = StepOutputHandle("return_one")
context = InputContext(
pipeline_name=pipeline_def.name,
solid_def=pipeline_def.solid_def_named("return_one"),
config={},
metadata={},
upstream_output=OutputContext(
step_key=step_output_handle.step_key,
name=step_output_handle.output_name,
pipeline_name=pipeline_def.name,
run_id=run_id,
metadata={},
mapping_key=None,
config=None,
solid_def=pipeline_def.solid_def_named("return_one"),
),
log_manager=DagsterLogManager(run_id=pipeline_run.run_id,
logging_tags={},
loggers=[]),
)
assert io_manager.load_input(context) == 1
add_one_step_events = list(
execute_plan(
execution_plan.build_subset_plan(["add_one"]),
run_config=run_config,
pipeline_run=pipeline_run,
instance=instance,
))
step_output_handle = StepOutputHandle("add_one")
context = InputContext(
pipeline_name=pipeline_def.name,
solid_def=pipeline_def.solid_def_named("add_one"),
config={},
metadata={},
upstream_output=OutputContext(
step_key=step_output_handle.step_key,
name=step_output_handle.output_name,
pipeline_name=pipeline_def.name,
run_id=run_id,
metadata={},
mapping_key=None,
config=None,
solid_def=pipeline_def.solid_def_named("add_one"),
),
log_manager=DagsterLogManager(run_id=pipeline_run.run_id,
logging_tags={},
loggers=[]),
)
assert get_step_output(add_one_step_events, "add_one")
assert io_manager.load_input(context) == 2
def test_logging_basic():
with _setup_logger('test') as (captured_results, logger):
dl = DagsterLogManager('123', {}, [logger])
dl.debug('test')
dl.info('test')
dl.warning('test')
dl.error('test')
dl.critical('test')
kv_pairs = captured_results[0].replace(' ', '').split('\n')[1:]
_validate_basic(kv_pairs)
def host_mode_execution_context_event_generator(
pipeline,
execution_plan,
run_config,
pipeline_run,
instance,
raise_on_error,
executor_defs,
output_capture,
resume_from_failure: bool = False,
):
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
check.inst_param(pipeline, "pipeline", ReconstructablePipeline)
check.dict_param(run_config, "run_config", key_type=str)
check.inst_param(pipeline_run, "pipeline_run", PipelineRun)
check.inst_param(instance, "instance", DagsterInstance)
executor_defs = check.list_param(executor_defs,
"executor_defs",
of_type=ExecutorDefinition)
check.bool_param(raise_on_error, "raise_on_error")
check.invariant(output_capture is None)
execution_context = None
loggers = []
for (logger_def, logger_config) in default_system_loggers():
loggers.append(
logger_def.logger_fn(
InitLoggerContext(
logger_config,
pipeline_def=None,
logger_def=logger_def,
run_id=pipeline_run.run_id,
)))
log_manager = DagsterLogManager.create(loggers=loggers,
pipeline_run=pipeline_run,
instance=instance)
try:
executor = _get_host_mode_executor(pipeline, run_config, executor_defs,
instance)
execution_context = PlanOrchestrationContext(
plan_data=PlanData(
pipeline=pipeline,
pipeline_run=pipeline_run,
instance=instance,
execution_plan=execution_plan,
raise_on_error=raise_on_error,
retry_mode=executor.retries,
),
log_manager=log_manager,
executor=executor,
output_capture=None,
resume_from_failure=resume_from_failure,
)
yield execution_context
except DagsterError as dagster_error:
if execution_context is None:
user_facing_exc_info = (
# pylint does not know original_exc_info exists is is_user_code_error is true
# pylint: disable=no-member
dagster_error.original_exc_info # type: ignore
if dagster_error.is_user_code_error else sys.exc_info())
error_info = serializable_error_info_from_exc_info(
user_facing_exc_info)
event = DagsterEvent.pipeline_failure(
pipeline_context_or_name=pipeline_run.pipeline_name,
context_msg=
(f'Pipeline failure during initialization for pipeline "{pipeline_run.pipeline_name}". '
"This may be due to a failure in initializing the executor or one of the loggers."
),
error_info=error_info,
)
log_manager.log_dagster_event(
level=logging.ERROR,
msg=event.message,
dagster_event=event # type: ignore
)
yield event
else:
# pipeline teardown failure
raise dagster_error
if raise_on_error:
raise dagster_error
def test_logging_bad_custom_log_levels():
with _setup_logger('test') as (_, logger):
dl = DagsterLogManager('123', {}, [logger])
with pytest.raises(check.CheckError):
dl._log('test', 'foobar', {}) # pylint: disable=protected-access
def test_multiline_logging_complex():
msg = 'DagsterEventType.STEP_FAILURE for step start.materialization.output.result.0'
kwargs = {
'pipeline':
'example',
'pipeline_name':
'example',
'step_key':
'start.materialization.output.result.0',
'solid':
'start',
'solid_definition':
'emit_num',
'dagster_event':
DagsterEvent(
event_type_value='STEP_FAILURE',
pipeline_name='error_monster',
step_key='start.materialization.output.result.0',
solid_handle=SolidHandle('start', 'emit_num', None),
step_kind_value='MATERIALIZATION_THUNK',
logging_tags={
'pipeline': 'error_monster',
'step_key': 'start.materialization.output.result.0',
'solid': 'start',
'solid_definition': 'emit_num',
},
event_specific_data=StepFailureData(
error=SerializableErrorInfo(
message=
"FileNotFoundError: [Errno 2] No such file or directory: '/path/to/file'\n",
stack=[
' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/errors.py", line 186, in user_code_error_boundary\n yield\n',
' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/execution_plan/simple_engine.py", line 365, in _event_sequence_for_step_compute_fn\n for step_output in gen:\n',
' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/execution_plan/materialization_thunk.py", line 28, in _fn\n runtime_type.output_materialization_config.materialize_runtime_value(config_spec, runtime_value)\n',
' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/types/config_schema.py", line 93, in materialize_runtime_value\n return func(config_value, runtime_value)\n',
' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/types/config_schema.py", line 110, in _selector\n return func(selector_key, selector_value, runtime_value)\n',
' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/types/builtin_config_schemas.py", line 59, in _builtin_output_schema\n with open(json_file_path, \'w\') as ff:\n',
],
cls_name='FileNotFoundError',
),
user_failure_data=None,
),
),
}
with _setup_logger(DAGSTER_DEFAULT_LOGGER) as (captured_results, logger):
dl = DagsterLogManager('123', {}, [logger])
dl.info(msg, **kwargs)
kv_pairs = set(captured_results[0].split('\n')[1:])
expected_pairs = [
' orig_message = "DagsterEventType.STEP_FAILURE for step start.materialization.output.result.0"',
' run_id = "123"',
' pipeline = "example"',
' solid_definition = "emit_num"',
' pipeline_name = "example"',
' solid = "start"',
' step_key = "start.materialization.output.result.0"',
]
for e in expected_pairs:
assert e in kv_pairs
assert _regex_match_kv_pair(
r' log_message_id = "{0}"'.format(REGEX_UUID), kv_pairs)
assert _regex_match_kv_pair(
r' log_timestamp = "{0}"'.format(REGEX_TS), kv_pairs)
expected_dagster_event = {
'event_specific_data': [
[
"FileNotFoundError: [Errno 2] No such file or directory: '/path/to/file'\n",
[
' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/errors.py", line 186, in user_code_error_boundary\n yield\n',
' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/execution_plan/simple_engine.py", line 365, in _event_sequence_for_step_compute_fn\n for step_output in gen:\n',
' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/execution_plan/materialization_thunk.py", line 28, in _fn\n runtime_type.output_materialization_config.materialize_runtime_value(config_spec, runtime_value)\n',
' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/types/config_schema.py", line 93, in materialize_runtime_value\n return func(config_value, runtime_value)\n',
' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/types/config_schema.py", line 110, in _selector\n return func(selector_key, selector_value, runtime_value)\n',
' File "/Users/nate/src/dagster/python_modules/dagster/dagster/core/types/builtin_config_schemas.py", line 59, in _builtin_output_schema\n with open(json_file_path, \'w\') as ff:\n',
],
'FileNotFoundError',
],
None, # user_failure_data
],
'event_type_value':
'STEP_FAILURE',
'message':
None,
'pipeline_name':
'error_monster',
'solid_handle': ['start', 'emit_num', None],
'step_key':
'start.materialization.output.result.0',
'step_kind_value':
'MATERIALIZATION_THUNK',
'logging_tags': {
'pipeline': 'error_monster',
'solid': 'start',
'solid_definition': 'emit_num',
'step_key': 'start.materialization.output.result.0',
},
}
dagster_event = json.loads([
pair for pair in kv_pairs if 'dagster_event' in pair
][0].strip(' dagster_event = '))
assert dagster_event == expected_dagster_event
def test_logging_custom_log_levels():
with _setup_logger('test', {'FOO': 3}) as (_captured_results, logger):
dl = DagsterLogManager('123', {}, [logger])
with pytest.raises(AttributeError):
dl.foo('test') # pylint: disable=no-member
def host_mode_execution_context_event_generator(
execution_plan, recon_pipeline, run_config, pipeline_run, instance, executor, raise_on_error
):
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
check.inst_param(recon_pipeline, "recon_pipeline", ReconstructablePipeline)
check.dict_param(run_config, "run_config", key_type=str)
check.inst_param(pipeline_run, "pipeline_run", PipelineRun)
check.inst_param(instance, "instance", DagsterInstance)
check.inst_param(executor, "executor", Executor)
check.bool_param(raise_on_error, "raise_on_error")
execution_context = None
loggers = []
# Should these be configurable from the run config (without loading
# the full EnvironmentConfig??)
for (logger_def, logger_config) in default_system_loggers():
loggers.append(
logger_def.logger_fn(
InitLoggerContext(
logger_config,
pipeline_def=None,
logger_def=logger_def,
run_id=pipeline_run.run_id,
)
)
)
loggers.append(instance.get_logger())
log_manager = DagsterLogManager(
run_id=pipeline_run.run_id,
logging_tags=_get_logging_tags(pipeline_run),
loggers=loggers,
)
# Create an executor (again how do we pull config from run_config
# without going through the full EnvironmentConfig.build flow)
try:
execution_context = HostModeRunWorkerExecutionContext(
execution_context_data=HostModeExecutionContextData(
pipeline_run=pipeline_run,
recon_pipeline=recon_pipeline,
execution_plan=execution_plan,
instance=instance,
raise_on_error=raise_on_error,
retry_mode=executor.retries,
),
log_manager=log_manager,
executor=executor,
)
yield execution_context
except DagsterError as dagster_error:
if execution_context is None:
user_facing_exc_info = (
# pylint does not know original_exc_info exists is is_user_code_error is true
# pylint: disable=no-member
dagster_error.original_exc_info
if dagster_error.is_user_code_error
else sys.exc_info()
)
error_info = serializable_error_info_from_exc_info(user_facing_exc_info)
yield DagsterEvent.pipeline_init_failure(
pipeline_name=pipeline_run.pipeline_name,
failure_data=PipelineInitFailureData(error=error_info),
log_manager=log_manager,
)
else:
# pipeline teardown failure
raise dagster_error
if raise_on_error:
raise dagster_error
def test_logging_basic():
with _setup_logger("test") as (captured_results, logger):
dl = DagsterLogManager("123", {}, [logger])
dl.debug("test")
dl.info("test")
dl.warning("test")
dl.error("test")
dl.critical("test")
assert captured_results == ["system - 123 - test"] * 5
def test_logging_integer_log_levels():
with _setup_logger("test", {"FOO": 3}) as (_captured_results, logger):
dl = DagsterLogManager("123", {}, [logger])
dl.log(3, "test") # pylint: disable=no-member
def host_mode_execution_context_event_generator(
pipeline,
execution_plan,
run_config,
pipeline_run,
instance,
raise_on_error,
get_executor_def_fn,
output_capture,
):
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
check.inst_param(pipeline, "pipeline", ReconstructablePipeline)
check.dict_param(run_config, "run_config", key_type=str)
check.inst_param(pipeline_run, "pipeline_run", PipelineRun)
check.inst_param(instance, "instance", DagsterInstance)
get_executor_def_fn = check.opt_callable_param(
get_executor_def_fn, "get_executor_def_fn",
_default_get_executor_def_fn)
check.bool_param(raise_on_error, "raise_on_error")
check.invariant(output_capture is None)
execution_context = None
loggers = []
for (logger_def, logger_config) in default_system_loggers():
loggers.append(
logger_def.logger_fn(
InitLoggerContext(
logger_config,
pipeline_def=None,
logger_def=logger_def,
run_id=pipeline_run.run_id,
)))
loggers.append(instance.get_logger())
log_manager = DagsterLogManager(
run_id=pipeline_run.run_id,
logging_tags=get_logging_tags(pipeline_run),
loggers=loggers,
)
try:
executor = _get_host_mode_executor(pipeline, run_config,
get_executor_def_fn, instance)
execution_context = PlanOrchestrationContext(
plan_data=PlanData(
pipeline=pipeline,
pipeline_run=pipeline_run,
instance=instance,
execution_plan=execution_plan,
raise_on_error=raise_on_error,
retry_mode=executor.retries,
),
log_manager=log_manager,
executor=executor,
output_capture=None,
)
yield execution_context
except DagsterError as dagster_error:
if execution_context is None:
user_facing_exc_info = (
# pylint does not know original_exc_info exists is is_user_code_error is true
# pylint: disable=no-member
dagster_error.original_exc_info
if dagster_error.is_user_code_error else sys.exc_info())
error_info = serializable_error_info_from_exc_info(
user_facing_exc_info)
yield DagsterEvent.pipeline_init_failure(
pipeline_name=pipeline_run.pipeline_name,
failure_data=PipelineInitFailureData(error=error_info),
log_manager=log_manager,
)
else:
# pipeline teardown failure
raise dagster_error
if raise_on_error:
raise dagster_error
def test_logging_no_loggers_registered():
dl = DagsterLogManager("none", {}, [])
dl.debug("test")
dl.info("test")
dl.warning("test")
dl.error("test")
dl.critical("test")
def test_logging_integer_log_levels():
with _setup_logger("test", {"FOO": 3}) as (_captured_results, logger):
dl = DagsterLogManager.create(loggers=[logger],
pipeline_run=PipelineRun(run_id="123"))
dl.log(3, "test") # pylint: disable=no-member
def test_adls2_pickle_io_manager_execution(storage_account, file_system,
credential):
pipeline_def = define_inty_pipeline()
run_config = {
"resources": {
"io_manager": {
"config": {
"adls2_file_system": file_system
}
},
"adls2": {
"config": {
"storage_account": storage_account,
"credential": {
"key": credential
}
}
},
}
}
run_id = make_new_run_id()
execution_plan = create_execution_plan(pipeline_def, run_config=run_config)
assert execution_plan.get_step_by_key("return_one")
step_keys = ["return_one"]
instance = DagsterInstance.ephemeral()
pipeline_run = PipelineRun(pipeline_name=pipeline_def.name,
run_id=run_id,
run_config=run_config)
return_one_step_events = list(
execute_plan(
execution_plan.build_subset_plan(step_keys),
run_config=run_config,
pipeline_run=pipeline_run,
instance=instance,
))
assert get_step_output(return_one_step_events, "return_one")
step_output_handle = StepOutputHandle("return_one")
context = InputContext(
pipeline_name=pipeline_def.name,
solid_def=pipeline_def.solid_def_named("return_one"),
upstream_output=OutputContext(
step_key=step_output_handle.step_key,
name=step_output_handle.output_name,
pipeline_name=pipeline_def.name,
run_id=run_id,
solid_def=pipeline_def.solid_def_named("return_one"),
),
log_manager=DagsterLogManager(run_id=pipeline_run.run_id,
logging_tags={},
loggers=[]),
)
io_manager = PickledObjectADLS2IOManager(
file_system=file_system,
adls2_client=create_adls2_client(storage_account, credential),
blob_client=create_blob_client(storage_account, credential),
)
assert io_manager.load_input(context) == 1
add_one_step_events = list(
execute_plan(
execution_plan.build_subset_plan(["add_one"]),
pipeline_run=pipeline_run,
run_config=run_config,
instance=instance,
))
step_output_handle = StepOutputHandle("add_one")
context = InputContext(
pipeline_name=pipeline_def.name,
solid_def=pipeline_def.solid_def_named("add_one"),
upstream_output=OutputContext(
step_key=step_output_handle.step_key,
name=step_output_handle.output_name,
pipeline_name=pipeline_def.name,
run_id=run_id,
solid_def=pipeline_def.solid_def_named("add_one"),
),
log_manager=DagsterLogManager(run_id=pipeline_run.run_id,
logging_tags={},
loggers=[]),
)
assert get_step_output(add_one_step_events, "add_one")
assert io_manager.load_input(context) == 2
def resource_initialization_event_generator(
execution_plan, environment_config, pipeline_run, log_manager, resource_keys_to_init
):
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.inst_param(environment_config, 'environment_config', EnvironmentConfig)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
check.inst_param(log_manager, 'log_manager', DagsterLogManager)
check.set_param(resource_keys_to_init, 'resource_keys_to_init', of_type=str)
if execution_plan.step_key_for_single_step_plans():
step = execution_plan.get_step_by_key(execution_plan.step_key_for_single_step_plans())
resource_log_manager = DagsterLogManager(
pipeline_run.run_id,
merge_dicts(log_manager.logging_tags, step.logging_tags),
log_manager.loggers,
)
else:
resource_log_manager = log_manager
resource_instances = {}
pipeline_def = execution_plan.pipeline_def
mode_definition = pipeline_def.get_mode_definition(pipeline_run.mode)
resource_managers = deque()
generator_closed = False
resource_init_times = {}
try:
if resource_keys_to_init:
yield DagsterEvent.resource_init_start(
execution_plan, resource_log_manager, resource_keys_to_init,
)
for resource_name, resource_def in sorted(mode_definition.resource_defs.items()):
if not resource_name in resource_keys_to_init:
continue
resource_context = InitResourceContext(
pipeline_def=pipeline_def,
resource_def=resource_def,
resource_config=environment_config.resources.get(resource_name, {}).get('config'),
run_id=pipeline_run.run_id,
log_manager=resource_log_manager,
)
manager = single_resource_generation_manager(
resource_context, resource_name, resource_def
)
for event in manager.generate_setup_events():
if event:
yield event
initialized_resource = check.inst(manager.get_object(), InitializedResource)
resource_instances[resource_name] = initialized_resource.resource
resource_init_times[resource_name] = initialized_resource.duration
resource_managers.append(manager)
if resource_keys_to_init:
yield DagsterEvent.resource_init_success(
execution_plan, resource_log_manager, resource_instances, resource_init_times
)
yield ScopedResourcesBuilder(resource_instances)
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
raise
except DagsterUserCodeExecutionError as dagster_user_error:
yield DagsterEvent.resource_init_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(dagster_user_error.original_exc_info),
)
raise dagster_user_error
finally:
if not generator_closed:
error = None
while len(resource_managers) > 0:
manager = resource_managers.pop()
try:
for event in manager.generate_teardown_events():
yield event
except DagsterUserCodeExecutionError as dagster_user_error:
error = dagster_user_error
if error:
yield DagsterEvent.resource_teardown_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(error.original_exc_info),
)
def test_logging_no_loggers_registered():
dl = DagsterLogManager('none', {}, [])
dl.debug('test')
dl.info('test')
dl.warning('test')
dl.error('test')
dl.critical('test')
def resource_initialization_event_generator(
resource_defs: Dict[str, ResourceDefinition],
resource_configs: Dict[str, ResourceConfig],
log_manager: DagsterLogManager,
execution_plan: Optional[ExecutionPlan],
pipeline_run: Optional[PipelineRun],
resource_keys_to_init: Optional[Set[str]],
instance: Optional[DagsterInstance],
resource_instances_to_override: Optional[Dict[str, "InitializedResource"]],
emit_persistent_events: Optional[bool],
):
check.inst_param(log_manager, "log_manager", DagsterLogManager)
resource_keys_to_init = check.opt_set_param(resource_keys_to_init,
"resource_keys_to_init",
of_type=str)
check.opt_inst_param(execution_plan, "execution_plan", ExecutionPlan)
check.opt_inst_param(pipeline_run, "pipeline_run", PipelineRun)
check.opt_inst_param(instance, "instance", DagsterInstance)
check.opt_dict_param(resource_instances_to_override,
"resource_instances_to_override")
if execution_plan and execution_plan.step_handle_for_single_step_plans():
step = execution_plan.get_step(
cast(
StepHandleUnion,
cast(ExecutionPlan,
execution_plan).step_handle_for_single_step_plans(),
))
resource_log_manager = log_manager.with_tags(
**cast(ExecutionStep, step).logging_tags)
else:
resource_log_manager = log_manager
generator_closed = False
resource_managers: Deque[EventGenerationManager] = deque()
try:
yield from _core_resource_initialization_event_generator(
resource_defs=resource_defs,
resource_configs=resource_configs,
resource_log_manager=resource_log_manager,
resource_managers=resource_managers,
execution_plan=execution_plan,
pipeline_run=pipeline_run,
resource_keys_to_init=resource_keys_to_init,
instance=instance,
resource_instances_to_override=resource_instances_to_override,
emit_persistent_events=emit_persistent_events,
)
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
raise
finally:
if not generator_closed:
error = None
while len(resource_managers) > 0:
manager = resource_managers.pop()
try:
yield from manager.generate_teardown_events()
except DagsterUserCodeExecutionError as dagster_user_error:
error = dagster_user_error
if error:
yield DagsterEvent.resource_teardown_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
error.original_exc_info),
)
def test_logging_basic():
with _setup_logger('test') as (captured_results, logger):
dl = DagsterLogManager('123', {}, [logger])
dl.debug('test')
dl.info('test')
dl.warning('test')
dl.error('test')
dl.critical('test')
assert captured_results == ['system - 123 - test'] * 5
def _core_resource_initialization_event_generator(
resource_defs: Dict[str, ResourceDefinition],
resource_configs: Dict[str, ResourceConfig],
resource_log_manager: DagsterLogManager,
resource_managers: Deque[EventGenerationManager],
execution_plan: Optional[ExecutionPlan],
pipeline_run: Optional[PipelineRun],
resource_keys_to_init: Optional[Set[str]],
instance: Optional[DagsterInstance],
resource_instances_to_override: Optional[Dict[str, "InitializedResource"]],
emit_persistent_events: Optional[bool],
):
if emit_persistent_events:
check.invariant(
execution_plan,
"If emit_persistent_events is enabled, then execution_plan must be provided",
)
resource_instances_to_override = check.opt_dict_param(
resource_instances_to_override, "resource_instances_to_override")
resource_keys_to_init = check.opt_set_param(resource_keys_to_init,
"resource_keys_to_init")
resource_instances: Dict[str, "InitializedResource"] = {}
resource_init_times = {}
try:
if emit_persistent_events and resource_keys_to_init:
yield DagsterEvent.resource_init_start(
execution_plan,
resource_log_manager,
resource_keys_to_init,
)
resource_dependencies = _resolve_resource_dependencies(resource_defs)
for level in toposort(resource_dependencies):
for resource_name in level:
if resource_name in resource_instances_to_override:
# use the given resource instances instead of re-initiating it from resource def
resource_def = ResourceDefinition.hardcoded_resource(
resource_instances_to_override[resource_name])
else:
resource_def = resource_defs[resource_name]
if not resource_name in resource_keys_to_init:
continue
resource_context = InitResourceContext(
resource_def=resource_def,
resource_config=resource_configs[resource_name].config,
pipeline_run=pipeline_run,
# Add tags with information about the resource
log_manager=resource_log_manager.with_tags(
resource_name=resource_name,
resource_fn_name=str(
resource_def.resource_fn.__name__),
),
resource_instance_dict=resource_instances,
required_resource_keys=resource_def.required_resource_keys,
instance=instance,
pipeline_def_for_backwards_compat=execution_plan.
pipeline_def if execution_plan else None,
)
manager = single_resource_generation_manager(
resource_context, resource_name, resource_def)
for event in manager.generate_setup_events():
if event:
yield event
initialized_resource = check.inst(manager.get_object(),
InitializedResource)
resource_instances[
resource_name] = initialized_resource.resource
resource_init_times[
resource_name] = initialized_resource.duration
resource_managers.append(manager)
if emit_persistent_events and resource_keys_to_init:
yield DagsterEvent.resource_init_success(execution_plan,
resource_log_manager,
resource_instances,
resource_init_times)
yield ScopedResourcesBuilder(resource_instances)
except DagsterUserCodeExecutionError as dagster_user_error:
# Can only end up in this state if we attempt to initialize a resource, so
# resource_keys_to_init cannot be empty
if emit_persistent_events:
yield DagsterEvent.resource_init_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
dagster_user_error.original_exc_info),
)
raise dagster_user_error
def test_logging_integer_log_levels():
with _setup_logger('test', {'FOO': 3}) as (_captured_results, logger):
dl = DagsterLogManager('123', {}, [logger])
dl.log(3, 'test') # pylint: disable=no-member
def _core_resource_initialization_event_generator(
resource_defs: Dict[str, ResourceDefinition],
resource_configs: Dict[str, ResourceConfig],
resource_log_manager: DagsterLogManager,
resource_managers: Deque[EventGenerationManager],
execution_plan: Optional[ExecutionPlan],
pipeline_run: Optional[PipelineRun],
resource_keys_to_init: Optional[AbstractSet[str]],
instance: Optional[DagsterInstance],
emit_persistent_events: Optional[bool],
pipeline_def_for_backwards_compat: Optional[PipelineDefinition],
):
pipeline_name = None
contains_generator = False
if emit_persistent_events:
check.invariant(
pipeline_run and execution_plan,
"If emit_persistent_events is enabled, then pipeline_run and execution_plan must be provided",
)
pipeline_name = cast(PipelineRun, pipeline_run).pipeline_name
resource_keys_to_init = check.opt_set_param(resource_keys_to_init,
"resource_keys_to_init")
resource_instances: Dict[str, "InitializedResource"] = {}
resource_init_times = {}
try:
if emit_persistent_events and resource_keys_to_init:
yield DagsterEvent.resource_init_start(
cast(str, pipeline_name),
cast(ExecutionPlan, execution_plan),
resource_log_manager,
resource_keys_to_init,
)
resource_dependencies = _resolve_resource_dependencies(resource_defs)
for level in toposort(resource_dependencies):
for resource_name in level:
resource_def = resource_defs[resource_name]
if not resource_name in resource_keys_to_init:
continue
resource_fn = cast(Callable[[InitResourceContext], Any],
resource_def.resource_fn)
resources = ScopedResourcesBuilder(resource_instances).build(
resource_def.required_resource_keys)
resource_context = InitResourceContext(
resource_def=resource_def,
resource_config=resource_configs[resource_name].config,
pipeline_run=pipeline_run,
# Add tags with information about the resource
log_manager=resource_log_manager.with_tags(
resource_name=resource_name,
resource_fn_name=str(resource_fn.__name__),
),
resources=resources,
instance=instance,
pipeline_def_for_backwards_compat=
pipeline_def_for_backwards_compat,
)
manager = single_resource_generation_manager(
resource_context, resource_name, resource_def)
for event in manager.generate_setup_events():
if event:
yield event
initialized_resource = check.inst(manager.get_object(),
InitializedResource)
resource_instances[
resource_name] = initialized_resource.resource
resource_init_times[
resource_name] = initialized_resource.duration
contains_generator = contains_generator or initialized_resource.is_generator
resource_managers.append(manager)
if emit_persistent_events and resource_keys_to_init:
yield DagsterEvent.resource_init_success(
cast(str, pipeline_name),
cast(ExecutionPlan, execution_plan),
resource_log_manager,
resource_instances,
resource_init_times,
)
yield ScopedResourcesBuilder(resource_instances, contains_generator)
except DagsterUserCodeExecutionError as dagster_user_error:
# Can only end up in this state if we attempt to initialize a resource, so
# resource_keys_to_init cannot be empty
if emit_persistent_events:
yield DagsterEvent.resource_init_failure(
cast(str, pipeline_name),
cast(ExecutionPlan, execution_plan),
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
dagster_user_error.original_exc_info),
)
raise dagster_user_error
