def test_log_level_filtering():
records = []
critical_records = []
debug_logger_def = construct_single_handler_logger('debug_handler',
'debug',
LogTestHandler(records))
critical_logger_def = construct_single_handler_logger(
'critical_handler', 'critical', LogTestHandler(critical_records))
loggers = [
logger_def.logger_fn(
InitLoggerContext({}, PipelineDefinition([]), logger_def, ''))
for logger_def in [debug_logger_def, critical_logger_def]
]
log_manager = DagsterLogManager('', {}, loggers)
log_manager.debug('Hello, there!')
messages = [x.dagster_meta['orig_message'] for x in records]
assert 'Hello, there!' in messages
critical_messages = [
x.dagster_meta['orig_message'] for x in critical_records
]
assert 'Hello, there!' not in critical_messages
def test_log_level_filtering():
records = []
critical_records = []
debug_logger_def = construct_single_handler_logger("debug_handler",
"debug",
LogTestHandler(records))
critical_logger_def = construct_single_handler_logger(
"critical_handler", "critical", LogTestHandler(critical_records))
loggers = [
logger_def.logger_fn(
InitLoggerContext(
{},
logger_def,
pipeline_def=PipelineDefinition([], "test"),
run_id="",
)) for logger_def in [debug_logger_def, critical_logger_def]
]
log_manager = DagsterLogManager.create(loggers=loggers)
log_manager.debug("Hello, there!")
messages = [x.dagster_meta["orig_message"] for x in records]
assert "Hello, there!" in messages
critical_messages = [
x.dagster_meta["orig_message"] for x in critical_records
]
assert "Hello, there!" not in critical_messages
def test_colored_console_logger_with_integer_log_level():
@pipeline
def pipe():
pass
colored_console_logger.logger_fn(
InitLoggerContext({"name": "dagster", "log_level": 4}, pipe, colored_console_logger, "")
)
def test_colored_console_logger_with_integer_log_level():
@pipeline
def pipe():
pass
colored_console_logger.logger_fn(
InitLoggerContext({
'name': 'dagster',
'log_level': 4
}, pipe, colored_console_logger, ''))
def setup_json_file_logger(tf_name, name="foo", level=logging.DEBUG):
logger_def = define_json_file_logger(name, tf_name, level)
init_logger_context = InitLoggerContext(
{},
logger_def,
pipeline_def=PipelineDefinition([], "test"),
run_id="",
)
return logger_def.logger_fn(init_logger_context)
def initialize_console_manager(
pipeline_run: Optional[PipelineRun]) -> DagsterLogManager:
# initialize default colored console logger
loggers = []
for logger_def, logger_config in default_system_loggers():
loggers.append(
logger_def.logger_fn(
InitLoggerContext(
logger_config,
logger_def,
run_id=pipeline_run.run_id if pipeline_run else None)))
return DagsterLogManager(
None, pipeline_run.tags if pipeline_run and pipeline_run.tags else {},
loggers)
def test_construct_event_record():
with construct_structured_logger(construct_event_record) as (logger_def, messages):
init_logger_context = InitLoggerContext({}, PipelineDefinition([]), logger_def, '')
logger = logger_def.logger_fn(init_logger_context)
context = create_test_pipeline_execution_context(
run_config_loggers=[logger], tags={'pipeline': 'some_pipeline'}
)
context.log.info('random message')
assert len(messages) == 1
message = messages[0]
assert isinstance(message, LogMessageRecord)
def test_colored_console_logger_with_integer_log_level():
colored_console_logger.logger_fn(
InitLoggerContext({
'name': 'dagster',
'log_level': 4
}, PipelineDefinition([]), colored_console_logger, ''))
def populate_context(
self,
run_id=None,
mode=None,
solid_def_name=None,
pipeline_name=None,
marshal_dir=None,
environment_config=None,
input_name_type_dict=None,
output_name_type_dict=None,
output_log_path=None,
**_kwargs
):
check.str_param(run_id, 'run_id')
check.str_param(mode, 'mode')
check.str_param(solid_def_name, 'solid_def_name')
check.str_param(pipeline_name, 'pipeline_name')
check.str_param(marshal_dir, 'marshal_dir')
check.dict_param(environment_config, 'environment_config')
check.dict_param(input_name_type_dict, 'input_name_type_dict')
check.dict_param(output_name_type_dict, 'output_name_type_dict')
check.str_param(output_log_path, 'output_log_path')
self.populated_by_papermill = True
self.solid_def_name = solid_def_name
self.marshal_dir = marshal_dir
logger_def = construct_logger(output_log_path)
loggers = {'dagstermill': logger_def}
if self.repository_def is None:
self.solid_def = None
self.pipeline_def = PipelineDefinition(
[],
mode_definitions=[ModeDefinition(loggers=loggers)],
name='Dummy Pipeline (No Repo Registration)',
)
self.input_name_type_dict = dict_to_enum(input_name_type_dict)
self.output_name_type_dict = dict_to_enum(output_name_type_dict)
for _, runtime_type_enum in self.input_name_type_dict.items():
if runtime_type_enum == SerializableRuntimeType.NONE:
raise DagstermillError(
'If Dagstermill solids have inputs that require serialization strategies '
'that are not pickling, then you must register a repository within '
'notebook by calling dagstermill.register_repository(repository_def)'
)
for _, runtime_type_enum in self.output_name_type_dict.items():
if runtime_type_enum == SerializableRuntimeType.NONE:
raise DagstermillError(
'If Dagstermill solids have outputs that require serialization strategies '
'that are not pickling, then you must register a repository within '
'notebook by calling dagstermill.register_repository(repository_def).'
)
environment_config = {'loggers': {'dagstermill': {}}}
run_config = RunConfig(run_id=run_id, mode=mode)
else:
self.pipeline_def = self.repository_def.get_pipeline(pipeline_name)
check.invariant(
self.pipeline_def.has_solid_def(solid_def_name),
'solid {} not found'.format(solid_def_name),
)
self.solid_def = self.pipeline_def.solid_def_named(solid_def_name)
logger = logger_def.logger_fn(
InitLoggerContext({}, self.pipeline_def, logger_def, run_id)
)
run_config = RunConfig(run_id, loggers=[logger], mode=mode)
with scoped_pipeline_context(
self.pipeline_def,
environment_config,
run_config,
scoped_resources_builder_cm=self.setup_resources,
) as pipeline_context:
self.context = DagstermillInNotebookExecutionContext(pipeline_context)
return self.context
def dummy_init_logger_context(logger_def, run_id):
return InitLoggerContext({}, PipelineDefinition([]), logger_def, run_id)
def populate_context(
self,
run_id,
mode,
solid_def_name,
pipeline_def_name,
marshal_dir,
environment_dict,
input_name_type_dict,
output_name_type_dict,
output_log_path,
):
check.dict_param(environment_dict, 'environment_dict')
self.populated_by_papermill = True
self.solid_def_name = solid_def_name
self.marshal_dir = marshal_dir
logger_def = construct_logger(output_log_path)
loggers = {'dagstermill': logger_def}
if self.repository_def is None:
self.solid_def = None
self.pipeline_def = PipelineDefinition(
[],
mode_definitions=[ModeDefinition(loggers=loggers)],
name='Dummy Pipeline (No Repo Registration)',
)
self.input_name_type_dict = dict_to_enum(input_name_type_dict)
self.output_name_type_dict = dict_to_enum(output_name_type_dict)
for _, runtime_type_enum in self.input_name_type_dict.items():
if runtime_type_enum == SerializableRuntimeType.NONE:
raise DagstermillError(
'If Dagstermill solids have inputs that require serialization strategies '
'that are not pickling, then you must register a repository within '
'notebook by calling dm.register_repository(repository_def)'
)
for _, runtime_type_enum in self.output_name_type_dict.items():
if runtime_type_enum == SerializableRuntimeType.NONE:
raise DagstermillError(
'If Dagstermill solids have outputs that require serialization strategies '
'that are not pickling, then you must register a repository within '
'notebook by calling dm.register_repository(repository_def).'
)
with scoped_pipeline_context(
self.pipeline_def,
{'loggers': {
'dagstermill': {}
}},
RunConfig(run_id=run_id, mode=mode),
) as pipeline_context:
self.context = DagstermillInNotebookExecutionContext(
pipeline_context)
else:
self.pipeline_def = self.repository_def.get_pipeline(
pipeline_def_name)
check.invariant(self.pipeline_def.has_solid_def(solid_def_name))
self.solid_def = self.pipeline_def.solid_def_named(solid_def_name)
logger = logger_def.logger_fn(
InitLoggerContext({}, self.pipeline_def, logger_def, run_id))
run_config = RunConfig(run_id, loggers=[logger], mode=mode)
# See block comment above referencing this issue
# See https://github.com/dagster-io/dagster/issues/796
with scoped_pipeline_context(self.pipeline_def, environment_dict,
run_config) as pipeline_context:
self.context = DagstermillInNotebookExecutionContext(
pipeline_context)
return self.context
def setup_json_file_logger(tf_name, name='foo', level=logging.DEBUG):
logger_def = define_json_file_logger(name, tf_name, level)
init_logger_context = InitLoggerContext({}, PipelineDefinition([]),
logger_def, '')
return logger_def.logger_fn(init_logger_context)
