def test_basic_event_store(): @solid def return_one(_): return 1 def _solids(): return_one() events, _result = gather_events(_solids) event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) for event in events: event_log_storage.store_event(event) rows = fetch_all_events(get_test_conn_string()) out_events = list( map(lambda r: deserialize_json_to_dagster_namedtuple(r[0]), rows)) assert list(map(lambda e: e.dagster_event.event_type, out_events)) == [ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ]
def test_basic_get_logs_for_run(): @solid def return_one(_): return 1 def _solids(): return_one() events, result = gather_events(_solids) event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) for event in events: event_log_storage.store_event(event) out_events = event_log_storage.get_logs_for_run(result.run_id) assert event_types(out_events) == [ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ]
def test_listen_notify_single_run_event(): event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) @solid def return_one(_): return 1 def _solids(): return_one() event_watcher = create_event_watcher(get_test_conn_string()) run_id = str(uuid.uuid4()) event_watcher.watch_run(run_id) try: events, result = gather_events(_solids, run_config=RunConfig(run_id=run_id)) for event in events: event_log_storage.store_event(event) event = event_watcher.queue.get(block=True) assert isinstance(event, EventWatcherStart) for _ in range(0, 5): watcher_event = event_watcher.queue.get(block=True) assert isinstance(watcher_event, EventWatcherEvent) assert watcher_event.payload.run_id == result.run_id finally: event_watcher.close()
def test_wipe_postgres_event_log(): @solid def return_one(_): return 1 def _solids(): return_one() # pylint: disable=no-value-for-parameter events, result = gather_events(_solids) event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) for event in events: event_log_storage.store_event(event) out_events = event_log_storage.get_logs_for_run(result.run_id) assert event_types(out_events) == [ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ] event_log_storage.wipe() assert event_log_storage.get_logs_for_run(result.run_id) == []
def test_basic_get_logs_for_run_multiple_runs_cursors(): event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) @solid def return_one(_): return 1 def _solids(): return_one() # pylint: disable=no-value-for-parameter events_one, result_one = gather_events(_solids) for event in events_one: event_log_storage.store_event(event) events_two, result_two = gather_events(_solids) for event in events_two: event_log_storage.store_event(event) out_events_one = event_log_storage.get_logs_for_run(result_one.run_id, cursor=1) assert len(out_events_one) == 5 assert event_types(out_events_one) == [ # DagsterEventType.PIPELINE_START, # DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ] assert set(map(lambda e: e.run_id, out_events_one)) == {result_one.run_id} out_events_two = event_log_storage.get_logs_for_run(result_two.run_id, cursor=2) assert len(out_events_two) == 4 assert event_types(out_events_two) == [ # DagsterEventType.PIPELINE_START, # DagsterEventType.ENGINE_EVENT, # DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_START, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ] assert set(map(lambda e: e.run_id, out_events_two)) == {result_two.run_id}
def test_basic_get_logs_for_run_multiple_runs(): event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) @solid def return_one(_): return 1 def _solids(): return_one() events_one, result_one = gather_events(_solids) for event in events_one: event_log_storage.store_event(event) events_two, result_two = gather_events(_solids) for event in events_two: event_log_storage.store_event(event) out_events_one = event_log_storage.get_logs_for_run(result_one.run_id) assert len(out_events_one) == 7 assert set(event_types(out_events_one)) == set([ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ]) assert set(map(lambda e: e.run_id, out_events_one)) == {result_one.run_id} out_events_two = event_log_storage.get_logs_for_run(result_two.run_id) assert len(out_events_two) == 7 assert set(event_types(out_events_two)) == set([ DagsterEventType.STEP_OUTPUT, DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ]) assert set(map(lambda e: e.run_id, out_events_two)) == {result_two.run_id}
def test_listen_notify_filter_two_runs_event(): event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) @solid def return_one(_): return 1 def _solids(): return_one() event_watcher = create_event_watcher(get_test_conn_string()) run_id_one = str(uuid.uuid4()) run_id_two = str(uuid.uuid4()) event_watcher.watch_run(run_id_one) event_watcher.watch_run(run_id_two) try: events_one, result_one = gather_events( _solids, run_config=RunConfig(run_id=run_id_one)) for event in events_one: event_log_storage.store_event(event) events_two, result_two = gather_events( _solids, run_config=RunConfig(run_id=run_id_two)) for event in events_two: event_log_storage.store_event(event) event = event_watcher.queue.get(block=True) assert isinstance(event, EventWatcherStart) for _ in range(0, 10): watcher_event = event_watcher.queue.get(block=True) assert isinstance(watcher_event, EventWatcherEvent) assert watcher_event.payload.run_id in { result_one.run_id, result_two.run_id } finally: event_watcher.close()
def test_listen_notify_filter_run_event(): event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) @solid def return_one(_): return 1 def _solids(): return_one() # pylint: disable=no-value-for-parameter event_watcher = create_event_watcher(get_test_conn_string()) run_id_one = str(uuid.uuid4()) run_id_two = str(uuid.uuid4()) # only watch one of the runs event_watcher.watch_run(run_id_two) try: events_one, _result_one = gather_events( _solids, run_config=RunConfig(run_id=run_id_one)) for event in events_one: event_log_storage.store_event(event) events_two, result_two = gather_events( _solids, run_config=RunConfig(run_id=run_id_two)) for event in events_two: event_log_storage.store_event(event) event = event_watcher.queue.get(block=True) assert isinstance(event, EventWatcherStart) for _ in range(0, 5): watcher_event = event_watcher.queue.get(block=True) assert isinstance(watcher_event, EventWatcherEvent) assert watcher_event.payload.run_id == result_two.run_id finally: event_watcher.close()
def test_basic_get_logs_for_run_cursor(): event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) @solid def return_one(_): return 1 def _solids(): return_one() # pylint: disable=no-value-for-parameter events, result = gather_events(_solids) for event in events: event_log_storage.store_event(event) assert event_types( event_log_storage.get_logs_for_run(result.run_id, cursor=0)) == [ # DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ] assert event_types( event_log_storage.get_logs_for_run(result.run_id, cursor=1)) == [ # DagsterEventType.PIPELINE_START, # DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ]