def test_listen_notify_filter_run_event(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage(conn_string) @solid def return_one(_): return 1 def _solids(): return_one() run_id_one = make_new_run_id() run_id_two = make_new_run_id() # only watch one of the runs event_list = [] event_log_storage.event_watcher.watch_run(run_id_two, 0, event_list.append) try: events_one, _result_one = synthesize_events(_solids, run_id=run_id_one) for event in events_one: event_log_storage.store_event(event) events_two, _result_two = synthesize_events(_solids, run_id=run_id_two) for event in events_two: event_log_storage.store_event(event) start = time.time() while len(event_list) < 7 and time.time() - start < TEST_TIMEOUT: pass assert len(event_list) == 7 assert all([isinstance(event, DagsterEventRecord) for event in event_list]) finally: del event_log_storage
def test_delete_postgres_event_log(conn_string): @solid def return_one(_): return 1 def _solids(): return_one() events, result = synthesize_events(_solids) event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) for event in events: event_log_storage.store_event(event) out_events = event_log_storage.get_logs_for_run(result.run_id) assert event_types(out_events) == [ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ] event_log_storage.delete_events(result.run_id) assert event_log_storage.get_logs_for_run(result.run_id) == []
def test_asset_materialization(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) asset_key = 'asset one' @solid def materialize_one(_): yield Materialization( label='one', asset_key=asset_key, metadata_entries=[ EventMetadataEntry.text('hello', 'text'), EventMetadataEntry.json({'hello': 'world'}, 'json'), EventMetadataEntry.float(1.0, 'one'), ], ) yield Output(1) def _solids(): materialize_one() events_one, _ = gather_events(_solids) for event in events_one: event_log_storage.store_event(event) assert asset_key in set(event_log_storage.get_all_asset_keys()) events = event_log_storage.get_asset_events(asset_key) assert len(events) == 1 event = events[0] assert isinstance(event, DagsterEventRecord) assert event.dagster_event.event_type_value == DagsterEventType.STEP_MATERIALIZATION.value
def event_log_storage(self, conn_string): # pylint: disable=arguments-differ storage = PostgresEventLogStorage.create_clean_storage(conn_string) assert storage try: yield storage finally: storage.dispose()
def test_basic_get_logs_for_run_multiple_runs_cursors(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) @solid def return_one(_): return 1 def _solids(): return_one() events_one, result_one = synthesize_events(_solids) for event in events_one: event_log_storage.store_event(event) events_two, result_two = synthesize_events(_solids) for event in events_two: event_log_storage.store_event(event) out_events_one = event_log_storage.get_logs_for_run(result_one.run_id, cursor=1) assert len(out_events_one) == len(events_one) assert set(event_types(out_events_one)) == set(event_types(events_one)) assert set(map(lambda e: e.run_id, out_events_one)) == {result_one.run_id} out_events_two = event_log_storage.get_logs_for_run(result_two.run_id, cursor=2) assert len(out_events_two) == len(events_two) assert set(event_types(out_events_two)) == set(event_types(events_one)) assert set(map(lambda e: e.run_id, out_events_two)) == {result_two.run_id}
def test_postgres_instance(multi_postgres): run_storage_conn_string, event_log_storage_conn_string = multi_postgres run_storage = PostgresRunStorage.create_clean_storage( run_storage_conn_string) event_storage = PostgresEventLogStorage.create_clean_storage( event_log_storage_conn_string) with seven.TemporaryDirectory() as temp_dir: instance = DagsterInstance( instance_type=InstanceType.PERSISTENT, local_artifact_storage=LocalArtifactStorage(temp_dir), run_storage=run_storage, event_storage=event_storage, compute_log_manager=LocalComputeLogManager(temp_dir), ) run = RunConfig() execute_pipeline(simple, run_config=run, instance=instance) assert run_storage.has_run(run.run_id) assert run_storage.get_run_by_id( run.run_id).status == PipelineRunStatus.SUCCESS assert DagsterEventType.PIPELINE_SUCCESS in [ event.dagster_event.event_type for event in event_storage.get_logs_for_run(run.run_id) if event.is_dagster_event ] stats = event_storage.get_stats_for_run(run.run_id) assert stats.steps_succeeded == 1 assert stats.end_time is not None
def test_basic_get_logs_for_run(): @solid def return_one(_): return 1 def _solids(): return_one() # pylint: disable=no-value-for-parameter events, result = gather_events(_solids) event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) for event in events: event_log_storage.store_event(event) out_events = event_log_storage.get_logs_for_run(result.run_id) assert event_types(out_events) == [ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ]
def test_basic_event_store(): @solid def return_one(_): return 1 def _solids(): return_one() # pylint: disable=no-value-for-parameter events, _result = gather_events(_solids) event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) for event in events: event_log_storage.store_event(event) rows = fetch_all_events(get_test_conn_string()) out_events = list( map(lambda r: deserialize_json_to_dagster_namedtuple(r[0]), rows)) assert list(map(lambda e: e.dagster_event.event_type, out_events)) == [ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ]
def test_listen_notify_single_run_event(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) @solid def return_one(_): return 1 def _solids(): return_one() event_list = [] run_id = make_new_run_id() event_log_storage.event_watcher.watch_run(run_id, 0, event_list.append) try: events, _ = synthesize_events(_solids, run_id=run_id) for event in events: event_log_storage.store_event(event) start = time.time() while len(event_list) < len( events) and time.time() - start < TEST_TIMEOUT: pass assert len(event_list) == len(events) # uncomment when https://github.com/dagster-io/dagster/issues/3368 is resolved with structured event # assert all([isinstance(event, EventRecord) for event in event_list]) finally: del event_log_storage
def test_correct_timezone(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) curr_time = time.time() event = EventRecord( None, "Message2", "debug", "", "foo", curr_time, dagster_event=DagsterEvent( DagsterEventType.PIPELINE_START.value, "nonce", event_specific_data=EngineEventData.in_process(999), ), ) event_log_storage.store_event(event) logs = event_log_storage.get_logs_for_run("foo") assert len(logs) == 1 log = logs[0] stats = event_log_storage.get_stats_for_run("foo") assert int(log.timestamp) == int(stats.start_time) assert int(log.timestamp) == int(curr_time)
def test_listen_notify_single_run_event(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) @solid def return_one(_): return 1 def _solids(): return_one() event_list = [] run_id = make_new_run_id() event_log_storage.event_watcher.watch_run(run_id, 0, event_list.append) try: events, _ = synthesize_events(_solids, run_id=run_id) for event in events: event_log_storage.store_event(event) start = time.time() while len(event_list) < 7 and time.time() - start < TEST_TIMEOUT: pass assert len(event_list) == 7 assert all( [isinstance(event, DagsterEventRecord) for event in event_list]) finally: del event_log_storage
def test_listen_notify_single_run_event(): event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) @solid def return_one(_): return 1 def _solids(): return_one() # pylint: disable=no-value-for-parameter event_watcher = create_event_watcher(get_test_conn_string()) run_id = str(uuid.uuid4()) event_watcher.watch_run(run_id) try: events, result = gather_events(_solids, run_config=RunConfig(run_id=run_id)) for event in events: event_log_storage.store_event(event) event = event_watcher.queue.get(block=True) assert isinstance(event, EventWatcherStart) for _ in range(0, 5): watcher_event = event_watcher.queue.get(block=True) assert isinstance(watcher_event, EventWatcherEvent) assert watcher_event.payload.run_id == result.run_id finally: event_watcher.close()
def test_basic_get_logs_for_run_cursor(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage(conn_string) @solid def return_one(_): return 1 def _solids(): return_one() events, result = synthesize_events(_solids) for event in events: event_log_storage.store_event(event) assert event_types(event_log_storage.get_logs_for_run(result.run_id, cursor=0)) == [ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ] assert event_types(event_log_storage.get_logs_for_run(result.run_id, cursor=1)) == [ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ]
def test_basic_event_store(conn_string): @solid def return_one(_): return 1 def _solids(): return_one() events, _result = synthesize_events(_solids) event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) for event in events: event_log_storage.store_event(event) rows = fetch_all_events(conn_string) out_events = list( map(lambda r: deserialize_json_to_dagster_namedtuple(r[0]), rows)) # messages can come out of order event_type_counts = Counter(event_types(out_events)) assert event_type_counts assert Counter(event_types(out_events)) == Counter(event_types(events))
def test_run_step_stats(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) @solid(input_defs=[InputDefinition("_input", str)], output_defs=[OutputDefinition(str)]) def should_fail(context, _input): context.log.info("fail") raise Exception("booo") def _one(): should_fail(should_succeed()) events, result = synthesize_events(_one, check_success=False) for event in events: event_log_storage.store_event(event) step_stats = sorted(event_log_storage.get_step_stats_for_run( result.run_id), key=lambda x: x.end_time) assert len(step_stats) == 2 assert step_stats[0].step_key == "should_succeed" assert step_stats[0].status == StepEventStatus.SUCCESS assert step_stats[0].end_time > step_stats[0].start_time assert step_stats[0].attempts == 1 assert step_stats[1].step_key == "should_fail" assert step_stats[1].status == StepEventStatus.FAILURE assert step_stats[1].end_time > step_stats[0].start_time assert step_stats[1].attempts == 1
def test_asset_materialization(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) asset_key = AssetKey(["path", "to", "asset_one"]) @solid def materialize_one(_): yield AssetMaterialization( asset_key=asset_key, metadata_entries=[ EventMetadataEntry.text("hello", "text"), EventMetadataEntry.json({"hello": "world"}, "json"), EventMetadataEntry.float(1.0, "one_float"), EventMetadataEntry.int(1, "one_int"), ], ) yield Output(1) def _solids(): materialize_one() events_one, _ = synthesize_events(_solids) for event in events_one: event_log_storage.store_event(event) assert asset_key in set(event_log_storage.get_all_asset_keys()) events = event_log_storage.get_asset_events(asset_key) assert len(events) == 1 event = events[0] assert isinstance(event, EventRecord) assert event.dagster_event.event_type_value == DagsterEventType.STEP_MATERIALIZATION.value
def test_basic_get_logs_for_run_multiple_runs(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage(conn_string) @solid def return_one(_): return 1 def _solids(): return_one() events_one, result_one = synthesize_events(_solids) for event in events_one: event_log_storage.store_event(event) events_two, result_two = synthesize_events(_solids) for event in events_two: event_log_storage.store_event(event) out_events_one = event_log_storage.get_logs_for_run(result_one.run_id) assert len(out_events_one) == 8 assert set(event_types(out_events_one)) == set( [ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.OBJECT_STORE_OPERATION, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ] ) assert set(map(lambda e: e.run_id, out_events_one)) == {result_one.run_id} stats_one = event_log_storage.get_stats_for_run(result_one.run_id) assert stats_one.steps_succeeded == 1 out_events_two = event_log_storage.get_logs_for_run(result_two.run_id) assert len(out_events_two) == 8 assert set(event_types(out_events_two)) == set( [ DagsterEventType.STEP_OUTPUT, DagsterEventType.OBJECT_STORE_OPERATION, DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ] ) assert set(map(lambda e: e.run_id, out_events_two)) == {result_two.run_id} stats_two = event_log_storage.get_stats_for_run(result_two.run_id) assert stats_two.steps_succeeded == 1
def event_log_storage(self, conn_string): # pylint: disable=arguments-differ storage = PostgresEventLogStorage.create_clean_storage(conn_string) assert storage yield storage # need to drop tables since PG tables are not run-sharded & some tests depend on a totally # fresh table (i.e.) autoincr. ids starting at 1 - this is related to cursor API, see # https://github.com/dagster-io/dagster/issues/3621 with storage.run_connection(run_id=None) as conn: SqlEventLogStorageTable.drop(conn) AssetKeyTable.drop(conn)
def clean_event_log_storage(conn_string): check.invariant( TestPostgresInstance.dagster_postgres_installed(), "dagster_postgres must be installed to test with postgres", ) from dagster_postgres.event_log import ( # pylint: disable=import-error PostgresEventLogStorage, ) storage = PostgresEventLogStorage.create_clean_storage(conn_string) assert storage return storage
def test_asset_events_error_parsing(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) _logs = [] def mock_log(msg): _logs.append(msg) asset_key = 'asset one' @solid def materialize_one(_): yield Materialization( label='one', asset_key=asset_key, ) yield Output(1) def _solids(): materialize_one() events_one, _ = synthesize_events(_solids) for event in events_one: event_log_storage.store_event(event) with mock.patch( 'dagster_postgres.event_log.event_log.logging.warning', side_effect=mock_log, ): with mock.patch( 'dagster_postgres.event_log.event_log.deserialize_json_to_dagster_namedtuple', return_value='not_an_event_record', ): assert asset_key in set(event_log_storage.get_all_asset_keys()) events = event_log_storage.get_asset_events(asset_key) assert len(events) == 0 assert len(_logs) == 1 assert re.match( 'Could not resolve asset event record as EventRecord', _logs[0]) _logs = [] # reset logs with mock.patch( 'dagster_postgres.event_log.event_log.deserialize_json_to_dagster_namedtuple', side_effect=seven.JSONDecodeError('error', '', 0), ): assert asset_key in set(event_log_storage.get_all_asset_keys()) events = event_log_storage.get_asset_events(asset_key) assert len(events) == 0 assert len(_logs) == 1 assert re.match('Could not parse asset event record id', _logs[0])
def test_basic_get_logs_for_run_multiple_runs_cursors(): event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) @solid def return_one(_): return 1 def _solids(): return_one() # pylint: disable=no-value-for-parameter events_one, result_one = gather_events(_solids) for event in events_one: event_log_storage.store_event(event) events_two, result_two = gather_events(_solids) for event in events_two: event_log_storage.store_event(event) out_events_one = event_log_storage.get_logs_for_run(result_one.run_id, cursor=1) assert len(out_events_one) == 5 assert event_types(out_events_one) == [ # DagsterEventType.PIPELINE_START, # DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ] assert set(map(lambda e: e.run_id, out_events_one)) == {result_one.run_id} out_events_two = event_log_storage.get_logs_for_run(result_two.run_id, cursor=2) assert len(out_events_two) == 4 assert event_types(out_events_two) == [ # DagsterEventType.PIPELINE_START, # DagsterEventType.ENGINE_EVENT, # DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_START, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ] assert set(map(lambda e: e.run_id, out_events_two)) == {result_two.run_id}
def test_asset_events_error_parsing(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) _logs = [] def mock_log(msg): _logs.append(msg) asset_key = AssetKey("asset_one") @solid def materialize_one(_): yield AssetMaterialization(asset_key=asset_key) yield Output(1) def _solids(): materialize_one() events_one, _ = synthesize_events(_solids) for event in events_one: event_log_storage.store_event(event) with mock.patch( "dagster.core.storage.event_log.sql_event_log.logging.warning", side_effect=mock_log, ): with mock.patch( "dagster.core.storage.event_log.sql_event_log.deserialize_json_to_dagster_namedtuple", return_value="not_an_event_record", ): assert asset_key in set(event_log_storage.get_all_asset_keys()) events = event_log_storage.get_asset_events(asset_key) assert len(events) == 0 assert len(_logs) == 1 assert re.match( "Could not resolve asset event record as EventRecord", _logs[0]) _logs = [] # reset logs with mock.patch( "dagster.core.storage.event_log.sql_event_log.deserialize_json_to_dagster_namedtuple", side_effect=seven.JSONDecodeError("error", "", 0), ): assert asset_key in set(event_log_storage.get_all_asset_keys()) events = event_log_storage.get_asset_events(asset_key) assert len(events) == 0 assert len(_logs) == 1 assert re.match("Could not parse asset event record id", _logs[0])
def test_listen_notify_filter_two_runs_event(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) @solid def return_one(_): return 1 def _solids(): return_one() event_list_one = [] event_list_two = [] run_id_one = make_new_run_id() run_id_two = make_new_run_id() event_log_storage.event_watcher.watch_run(run_id_one, 0, event_list_one.append) event_log_storage.event_watcher.watch_run(run_id_two, 0, event_list_two.append) try: events_one, _result_one = gather_events( _solids, run_config=RunConfig(run_id=run_id_one)) for event in events_one: event_log_storage.store_event(event) events_two, _result_two = gather_events( _solids, run_config=RunConfig(run_id=run_id_two)) for event in events_two: event_log_storage.store_event(event) start = time.time() while (len(event_list_one) < 7 or len(event_list_two) < 7) and time.time() - start < TEST_TIMEOUT: pass assert len(event_list_one) == 7 assert len(event_list_two) == 7 assert all([ isinstance(event, DagsterEventRecord) for event in event_list_one ]) assert all([ isinstance(event, DagsterEventRecord) for event in event_list_two ]) finally: del event_log_storage
def test_basic_get_logs_for_run_multiple_runs_cursors(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) @solid def return_one(_): return 1 def _solids(): return_one() events_one, result_one = gather_events(_solids) for event in events_one: event_log_storage.store_event(event) events_two, result_two = gather_events(_solids) for event in events_two: event_log_storage.store_event(event) out_events_one = event_log_storage.get_logs_for_run(result_one.run_id, cursor=1) assert len(out_events_one) == 7 assert set(event_types(out_events_one)) == set([ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ]) assert set(map(lambda e: e.run_id, out_events_one)) == {result_one.run_id} out_events_two = event_log_storage.get_logs_for_run(result_two.run_id, cursor=2) assert len(out_events_two) == 7 assert set(event_types(out_events_two)) == set([ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_OUTPUT, DagsterEventType.STEP_START, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ]) assert set(map(lambda e: e.run_id, out_events_two)) == {result_two.run_id}
def test_basic_get_logs_for_run(conn_string): @solid def return_one(_): return 1 def _solids(): return_one() events, result = synthesize_events(_solids) event_log_storage = PostgresEventLogStorage.create_clean_storage(conn_string) for event in events: event_log_storage.store_event(event) out_events = event_log_storage.get_logs_for_run(result.run_id) assert event_types(out_events) == event_types(events)
def test_listen_notify_filter_two_runs_event(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) @solid def return_one(_): return 1 def _solids(): return_one() event_list_one = [] event_list_two = [] run_id_one = make_new_run_id() run_id_two = make_new_run_id() event_log_storage.event_watcher.watch_run(run_id_one, 0, event_list_one.append) event_log_storage.event_watcher.watch_run(run_id_two, 0, event_list_two.append) try: events_one, _result_one = synthesize_events(_solids, run_id=run_id_one) for event in events_one: event_log_storage.store_event(event) events_two, _result_two = synthesize_events(_solids, run_id=run_id_two) for event in events_two: event_log_storage.store_event(event) start = time.time() while (len(event_list_one) < len(events_one) or len(event_list_two) < len(events_two)) and time.time() - start < TEST_TIMEOUT: pass assert len(event_list_one) == len(events_one) assert len(event_list_two) == len(events_two) # uncomment when https://github.com/dagster-io/dagster/issues/3368 is resolved with structured event # assert all([isinstance(event, DagsterEventRecord) for event in event_list_one]) # assert all([isinstance(event, DagsterEventRecord) for event in event_list_two]) finally: del event_log_storage
def test_listen_notify_filter_two_runs_event(): event_log_storage = PostgresEventLogStorage.create_nuked_storage( get_test_conn_string()) @solid def return_one(_): return 1 def _solids(): return_one() event_watcher = create_event_watcher(get_test_conn_string()) run_id_one = str(uuid.uuid4()) run_id_two = str(uuid.uuid4()) event_watcher.watch_run(run_id_one) event_watcher.watch_run(run_id_two) try: events_one, result_one = gather_events( _solids, run_config=RunConfig(run_id=run_id_one)) for event in events_one: event_log_storage.store_event(event) events_two, result_two = gather_events( _solids, run_config=RunConfig(run_id=run_id_two)) for event in events_two: event_log_storage.store_event(event) event = event_watcher.queue.get(block=True) assert isinstance(event, EventWatcherStart) for _ in range(0, 10): watcher_event = event_watcher.queue.get(block=True) assert isinstance(watcher_event, EventWatcherEvent) assert watcher_event.payload.run_id in { result_one.run_id, result_two.run_id } finally: event_watcher.close()
def test_basic_event_store(conn_string): @solid def return_one(_): return 1 def _solids(): return_one() events, _result = synthesize_events(_solids) event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) for event in events: event_log_storage.store_event(event) rows = fetch_all_events(conn_string) out_events = list( map(lambda r: deserialize_json_to_dagster_namedtuple(r[0]), rows)) # messages can come out of order assert Counter(event_types(out_events)) == Counter([ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_SUCCESS, DagsterEventType.PIPELINE_SUCCESS, DagsterEventType.STEP_OUTPUT, DagsterEventType.OBJECT_STORE_OPERATION, DagsterEventType.ENGINE_EVENT, ]) assert (sorted_event_types(out_events)) == [ DagsterEventType.PIPELINE_START, DagsterEventType.ENGINE_EVENT, DagsterEventType.STEP_START, DagsterEventType.STEP_OUTPUT, DagsterEventType.OBJECT_STORE_OPERATION, DagsterEventType.STEP_SUCCESS, DagsterEventType.ENGINE_EVENT, DagsterEventType.PIPELINE_SUCCESS, ]
def test_run_step_stats_with_retries(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage(conn_string) @solid(input_defs=[InputDefinition("_input", str)], output_defs=[OutputDefinition(str)]) def should_retry(context, _input): raise RetryRequested(max_retries=3) def _one(): should_retry(should_succeed()) events, result = synthesize_events(_one, check_success=False) for event in events: event_log_storage.store_event(event) step_stats = event_log_storage.get_step_stats_for_run(result.run_id, step_keys=["should_retry"]) assert len(step_stats) == 1 assert step_stats[0].step_key == "should_retry" assert step_stats[0].status == StepEventStatus.FAILURE assert step_stats[0].end_time > step_stats[0].start_time assert step_stats[0].attempts == 4
def test_secondary_index_asset_keys(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) asset_key_one = AssetKey(["one"]) asset_key_two = AssetKey(["two"]) @solid def materialize_one(_): yield AssetMaterialization(asset_key=asset_key_one) yield Output(1) @solid def materialize_two(_): yield AssetMaterialization(asset_key=asset_key_two) yield Output(1) def _one(): materialize_one() def _two(): materialize_two() events_one, _ = synthesize_events(_one) for event in events_one: event_log_storage.store_event(event) asset_keys = event_log_storage.get_all_asset_keys() assert len(asset_keys) == 1 assert asset_key_one in set(asset_keys) migrate_asset_key_data(event_log_storage) asset_keys = event_log_storage.get_all_asset_keys() assert len(asset_keys) == 1 assert asset_key_one in set(asset_keys) events_two, _ = synthesize_events(_two) for event in events_two: event_log_storage.store_event(event) asset_keys = event_log_storage.get_all_asset_keys() assert len(asset_keys) == 2 assert asset_key_one in set(asset_keys) assert asset_key_two in set(asset_keys)