def test_logging():
records = []
critical_records = []
pipeline_def = PipelineDefinition(
name='hello_logging_pipeline',
solids=[define_hello_logging_solid()],
mode_definitions=[
ModeDefinition(
loggers={
'test':
construct_single_handler_logger('test', 'debug',
LogTestHandler(records)),
'critical':
construct_single_handler_logger(
'critical', 'critical', LogTestHandler(
critical_records)),
})
],
)
execute_pipeline(pipeline_def, {'loggers': {'test': {}, 'critical': {}}})
messages = [x.dagster_meta['orig_message'] for x in records]
assert 'Hello, there!' in messages
critical_messages = [
x.dagster_meta['orig_message'] for x in critical_records
]
assert 'Hello, there!' not in critical_messages
def test_log_level_filtering():
records = []
critical_records = []
debug_logger_def = construct_single_handler_logger('debug_handler',
'debug',
LogTestHandler(records))
critical_logger_def = construct_single_handler_logger(
'critical_handler', 'critical', LogTestHandler(critical_records))
loggers = [
logger_def.logger_fn(
InitLoggerContext({}, PipelineDefinition([]), logger_def, ''))
for logger_def in [debug_logger_def, critical_logger_def]
]
log_manager = DagsterLogManager('', {}, loggers)
log_manager.debug('Hello, there!')
messages = [x.dagster_meta['orig_message'] for x in records]
assert 'Hello, there!' in messages
critical_messages = [
x.dagster_meta['orig_message'] for x in critical_records
]
assert 'Hello, there!' not in critical_messages
def test_log_level_filtering():
records = []
critical_records = []
debug_logger_def = construct_single_handler_logger("debug_handler",
"debug",
LogTestHandler(records))
critical_logger_def = construct_single_handler_logger(
"critical_handler", "critical", LogTestHandler(critical_records))
loggers = [
logger_def.logger_fn(
InitLoggerContext(
{},
logger_def,
pipeline_def=PipelineDefinition([], "test"),
run_id="",
)) for logger_def in [debug_logger_def, critical_logger_def]
]
log_manager = DagsterLogManager.create(loggers=loggers)
log_manager.debug("Hello, there!")
messages = [x.dagster_meta["orig_message"] for x in records]
assert "Hello, there!" in messages
critical_messages = [
x.dagster_meta["orig_message"] for x in critical_records
]
assert "Hello, there!" not in critical_messages
def test_error_during_logging(caplog):
run_id = str(uuid.uuid4())
with tempfile.NamedTemporaryFile() as sqlite3_db:
sqlite3_db_path = sqlite3_db.name
init_db(sqlite3_db_path)
sqlite3_handler = JsonSqlite3Handler(sqlite3_db_path)
def err_conn(*args, **kwargs):
raise Exception('Bailing!')
sqlite3_handler.connect = err_conn
sqlite3_logger_def = construct_single_handler_logger(
'sqlite3', 'debug', sqlite3_handler)
sqlite3_logger = sqlite3_logger_def.logger_fn(
dummy_init_logger_context(sqlite3_logger_def, run_id))
sqlite3_log_manager = DagsterLogManager(run_id, {}, [sqlite3_logger])
sqlite3_log_manager.info('Testing error handling')
assert caplog.record_tuples == [
('root', 50, 'Error during logging!'),
('root', 40, 'Bailing!'),
]
def check_thread_target(sqlite3_db_path, is_done, run_id, test_log_records):
test_handler = LogTestHandler(test_log_records)
test_logger_def = construct_single_handler_logger('test', 'debug', test_handler)
test_logger = test_logger_def.logger_fn(dummy_init_logger_context(test_logger_def, run_id))
test_log_manager = DagsterLogManager(run_id, {}, [test_logger])
test_log_watcher = JsonSqlite3LogWatcher(sqlite3_db_path, test_log_manager, is_done)
test_log_watcher.watch()
def test_json_sqlite3_watcher():
test_log_records = []
run_id = str(uuid.uuid4())
with tempfile.NamedTemporaryFile() as sqlite3_db:
sqlite3_db_path = sqlite3_db.name
init_db(sqlite3_db_path)
sqlite3_handler = JsonSqlite3Handler(sqlite3_db_path)
sqlite3_logger_def = construct_single_handler_logger(
'sqlite3', 'debug', sqlite3_handler)
sqlite3_logger = sqlite3_logger_def.logger_fn(
dummy_init_logger_context(sqlite3_logger_def, run_id))
sqlite3_log_manager = DagsterLogManager(run_id, {}, [sqlite3_logger])
for i in range(1000):
sqlite3_log_manager.info('Testing ' + str(i))
conn = sqlite3.connect(sqlite3_db_path)
cursor = conn.cursor()
count = cursor.execute('select count(1) from logs').fetchall()
assert count[0][0] == 1000
is_done = threading.Event()
is_done.set()
test_handler = LogTestHandler(test_log_records)
test_logger_def = construct_single_handler_logger(
'test', 'debug', test_handler)
test_logger = test_logger_def.logger_fn(
dummy_init_logger_context(test_logger_def, run_id))
sqlite3_watcher_log_manager = DagsterLogManager(
run_id, {}, [test_logger])
sqlite3_watcher = JsonSqlite3LogWatcher(sqlite3_db_path,
sqlite3_watcher_log_manager,
is_done)
sqlite3_watcher.watch()
assert len(test_log_records) == 1000
records = cursor.execute('select * from logs').fetchall()
for i, record in enumerate(records):
json_record = record[1]
assert json_record == seven.json.dumps(
test_log_records[i].__dict__)
def forward_logs(event_sink, wrap_it_up, run_id, test_log_records):
test_handler = LogTestHandler(test_log_records)
test_logger_def = construct_single_handler_logger('test', 'debug',
test_handler)
test_logger = test_logger_def.logger_fn(
dummy_init_logger_context(test_logger_def, run_id))
test_log_manager = DagsterLogManager(run_id, {}, [test_logger])
with event_sink.log_forwarding(test_log_manager):
while not wrap_it_up.is_set():
time.sleep(0.01)
def thread_target_source(sqlite3_db_path, run_id):
sqlite3_handler = JsonSqlite3Handler(sqlite3_db_path)
sqlite3_logger_def = construct_single_handler_logger('sqlite3', 'debug', sqlite3_handler)
sqlite3_logger = sqlite3_logger_def.logger_fn(
dummy_init_logger_context(sqlite3_logger_def, run_id)
)
sqlite3_log_manager = DagsterLogManager(run_id, {}, [sqlite3_logger])
for i in range(1000):
sqlite3_log_manager.info('Testing ' + str(i))
def construct_event_logger(event_record_callback):
"""
Callback receives a stream of event_records. Piggybacks on the logging machinery.
"""
check.callable_param(event_record_callback, "event_record_callback")
return construct_single_handler_logger(
"event-logger",
"debug",
StructuredLoggerHandler(lambda logger_message: event_record_callback(
construct_event_record(logger_message))),
)
def construct_event_logger(event_record_callback):
'''
Callback receives a stream of event_records. Piggybacks on the logging machinery.
'''
check.callable_param(event_record_callback, 'event_record_callback')
return construct_single_handler_logger(
'event-logger',
'debug',
StructuredLoggerHandler(lambda logger_message: event_record_callback(
construct_event_record(logger_message))),
)
def construct_json_event_logger(json_path):
"""Record a stream of event records to json"""
check.str_param(json_path, "json_path")
return construct_single_handler_logger(
"json-event-record-logger",
"debug",
JsonEventLoggerHandler(
json_path,
lambda record: construct_event_record(
StructuredLoggerMessage(
name=record.name,
message=record.msg,
level=record.levelno,
meta=record.dagster_meta,
record=record,
)),
),
)
def test_json_sqlite3_handler():
run_id = str(uuid.uuid4())
with tempfile.NamedTemporaryFile() as sqlite3_db:
sqlite3_db_path = sqlite3_db.name
init_db(sqlite3_db_path)
sqlite3_handler = JsonSqlite3Handler(sqlite3_db_path)
sqlite3_logger_def = construct_single_handler_logger(
'sqlite3', 'debug', sqlite3_handler)
sqlite3_logger = sqlite3_logger_def.logger_fn(
dummy_init_logger_context(sqlite3_logger_def, run_id))
sqlite3_log_manager = DagsterLogManager(run_id, {}, [sqlite3_logger])
for i in range(1000):
sqlite3_log_manager.info('Testing ' + str(i))
conn = sqlite3.connect(sqlite3_db_path)
cursor = conn.cursor()
count = cursor.execute('select count(1) from logs').fetchall()
assert count[0][0] == 1000
def test_sink_log_forwarding():
test_log_records = []
run_id = str(uuid.uuid4())
with safe_tempfile_path() as sqlite3_db_path:
sink = SqliteEventSink(sqlite3_db_path)
sqlite3_log_manager = DagsterLogManager(run_id, {},
[sink.get_logger()])
for i in range(1000):
sqlite3_log_manager.info('Testing ' + str(i))
with sqlite3.connect(sqlite3_db_path) as conn:
cursor = conn.cursor()
count = cursor.execute('select count(1) from logs').fetchall()
assert count[0][0] == 1000
test_handler = LogTestHandler(test_log_records)
test_logger_def = construct_single_handler_logger(
'test', 'debug', test_handler)
test_logger = test_logger_def.logger_fn(
dummy_init_logger_context(test_logger_def, run_id))
sqlite3_watcher_log_manager = DagsterLogManager(
run_id, {}, [test_logger])
with sink.log_forwarding(sqlite3_watcher_log_manager):
pass
assert len(test_log_records) == 1000
records = cursor.execute('select * from logs').fetchall()
for i, record in enumerate(records):
json_record = record[1]
assert json_record == seven.json.dumps(
test_log_records[i].__dict__)
conn.close()
sink.on_pipeline_teardown()
def construct_sqlite_logger(sqlite_db_path):
return construct_single_handler_logger('dagstermill', 'debug',
JsonSqlite3Handler(sqlite_db_path))
