def test_event_uploader2(self, MockCogniteClient): client: CogniteClient = MockCogniteClient() post_upload_test = {"value": False} def post(x): post_upload_test["value"] = True queue = EventUploadQueue(client, max_upload_interval=2, post_upload_function=post) queue.start() event1 = Event(start_time=1, end_time=2, description="test event") event2 = Event(start_time=3, end_time=4, description="another test event") queue.add_to_upload_queue(event1) queue.add_to_upload_queue(event2) time.sleep(2.1) client.events.create.ssert_called_with([event1, event2]) self.assertTrue(post_upload_test["value"]) queue.stop()
def test_make_objects_batch(): src_objects = [ Event(id=1, last_updated_time=1000), Event(id=2, last_updated_time=1000), Event(id=3, last_updated_time=1000), ] src_id_to_dst = { 1: Event(id=11, metadata={"_replicatedTime": 100}), 2: Event(id=12, metadata={"_replicatedTime": 10000}), } def dummy_update(src_obj, dst_obj, src_dst_ids_assets, project_src, replicated_runtime): return dst_obj def dummy_create(src_obj, src_dst_ids_assets, project_src, replicated_runtime): return src_obj created, updated, unchanged = make_objects_batch( src_objects, src_id_to_dst, {}, create=dummy_create, update=dummy_update, project_src="test project", replicated_runtime=10000, ) assert len(created) == len(updated) == len(unchanged) == 1 assert updated[0].id == 11 assert unchanged[0].id == 12 assert created[0].id == 3
def test_copy_events(): events_src = [ Event(metadata={}, id=1007, asset_ids=[3, 9], start_time=0, end_time=1), Event(metadata={}, id=2007, asset_ids=[7], start_time=1, end_time=2), Event(start_time=2, asset_ids=[5], end_time=4, metadata={}), ] id_mapping = {i: i * 111 for i in range(1, 10)} with monkeypatch_cognite_client() as client_dst: copy_events(events_src, {}, id_mapping, "src-project-name", 1000000, client_dst, None)
def update_event(src_event: Event, dst_event: Event, src_dst_ids_assets: Dict[int, int], project_src: str, runtime: int) -> Event: """ Makes an updated version of the destination event based on the corresponding source event. Args: src_event: The event from the source to be replicated. dst_event: The event from the destination that needs to be updated to reflect changes made to its source event. src_dst_ids_assets: A dictionary of all the mappings of source asset id to destination asset id. project_src: The name of the project the object is being replicated from. runtime: The timestamp to be used in the new replicated metadata. Returns: The updated event object for the replication destination. """ logging.debug( f"Updating existing event {dst_event.id} based on source event id {src_event.id}" ) dst_event.external_id = src_event.external_id dst_event.start_time = ( src_event.start_time if src_event.start_time and src_event.end_time and src_event.start_time < src_event.end_time else src_event.end_time) dst_event.end_time = src_event.end_time dst_event.type = src_event.type dst_event.subtype = src_event.subtype dst_event.description = src_event.description dst_event.metadata = replication.new_metadata(src_event, project_src, runtime) dst_event.asset_ids = replication.get_asset_ids(src_event.asset_ids, src_dst_ids_assets) dst_event.source = src_event.source return dst_event
def test_filter_objects(): time_series = [ TimeSeries(id=1, asset_id=100), TimeSeries(id=2), TimeSeries(id=3, asset_id=101) ] events = [ Event(id=10, asset_ids=[100, 101]), Event(id=11), Event(id=12, asset_ids=[101]) ] src_dst_asset_id_map = {100: 1000} dummy_filtered_events = filter_objects(events, src_dst_asset_id_map) dummy_filtered_ts = filter_objects(time_series, src_dst_asset_id_map) assert dummy_filtered_events == events assert dummy_filtered_ts == time_series asset_events = filter_objects(events, src_dst_asset_id_map, skip_nonasset=True) asset_ts = filter_objects(time_series, src_dst_asset_id_map, skip_nonasset=True) assert len(asset_events) == 2 assert len(asset_ts) == 2 for i in range(len(asset_ts)): assert asset_ts[i].asset_id is not None assert asset_events[i].asset_ids is not None linkable_events = filter_objects(events, src_dst_asset_id_map, skip_nonasset=True, skip_unlinkable=True) linkable_ts = filter_objects(time_series, src_dst_asset_id_map, skip_nonasset=True, skip_unlinkable=True) assert len(linkable_events) == 1 assert len(linkable_ts) == 1 assert linkable_events[0] == events[0] assert linkable_ts[0] == time_series[0] odd_id_events = filter_objects(events, src_dst_asset_id_map, filter_fn=lambda x: x.id % 2 == 1) assert len(odd_id_events) == 1 for event in odd_id_events: assert event.id % 2 == 1
def create_event(src_event: Event, src_dst_ids_assets: Dict[int, int], project_src: str, runtime: int) -> Event: """ Makes a new copy of the event to be replicated based on a source event. Args: src_event: The event from the source to be replicated to destination. src_dst_ids_assets: A dictionary of all the mappings of source asset id to destination asset id. project_src: The name of the project the object is being replicated from. runtime: The timestamp to be used in the new replicated metadata. Returns: The replicated event to be created in the destination. """ logging.debug( f"Creating a new event based on source event id {src_event.id}") return Event( external_id=src_event.external_id, start_time=src_event.start_time if src_event.start_time and src_event.end_time and src_event.start_time < src_event.end_time else src_event.end_time, end_time=src_event.end_time, type=src_event.type, subtype=src_event.subtype, description=src_event.description, metadata=replication.new_metadata(src_event, project_src, runtime), asset_ids=replication.get_asset_ids(src_event.asset_ids, src_dst_ids_assets), source=src_event.source, )
def test_event_uploader1(self, MockCogniteClient): client: CogniteClient = MockCogniteClient() queue = EventUploadQueue(client) event1 = Event(start_time=1, end_time=2, description="test event") event2 = Event(start_time=3, end_time=4, description="another test event") queue.add_to_upload_queue(event1) queue.add_to_upload_queue(event2) client.events.create.assert_not_called() queue.upload() client.events.create.ssert_called_with([event1, event2])
def test_create_event(): events_src = [ Event(metadata={}, id=1007, asset_ids=[3, 9], start_time=0, end_time=1), Event(metadata={}, id=2007, asset_ids=[7], start_time=1, end_time=2), Event(start_time=2, asset_ids=[5], end_time=4, metadata={}), Event(start_time=6, end_time=8, metadata={}), ] id_mapping = {3: 333, 7: 777, 5: 555, 9: 999} for i, event in enumerate(events_src): created_event = create_event(event, id_mapping, "src-project-name {}".format(i), 10000000) assert created_event.metadata["_replicatedInternalId"] == events_src[i].id assert created_event.metadata["_replicatedSource"] == "src-project-name {}".format(i) assert (created_event.asset_ids is None) == (events_src[i].asset_ids is None) if created_event.asset_ids is not None: assert len(created_event.asset_ids) == len(events_src[i].asset_ids) for j, asset_id in enumerate(created_event.asset_ids): assert asset_id == id_mapping[events_src[i].asset_ids[j]]
def test_remove_replication_metadata(): events = [ Event( metadata={ "_replicatedInternalId": 10, "_replicatedSource": "src_project", "_replicatedTime": 10000000 }), Event(metadata={}), Event(id=3, metadata={ "_replicatedInternalId": 10, "misc1": 16, "misc2": "text" }), ] remove_replication_metadata(events) for event in events: assert "_replicatedInternalId" not in event.metadata assert "_replicatedSource" not in event.metadata assert "_replicatedTime" not in event.metadata assert len(events[2].metadata.keys()) == 2
def test_update_event(): events_src = [ Event(metadata={}, id=1007, asset_ids=[3, 9], start_time=0, end_time=1), Event(metadata={}, id=2007, asset_ids=[7], start_time=1, end_time=2), Event(start_time=2, asset_ids=[5], end_time=4, metadata={}), ] events_dst = [ Event(asset_ids=[333], metadata={}), Event(asset_ids=[777], metadata={"_replicatedInternalId": 7}), Event(asset_ids=[555], metadata={"source": "None"}), ] id_mapping = {3: 333, 7: 777, 5: 555, 9: 999} for i in range(len(events_src)): updated_event = update_event(events_src[i], events_dst[i], id_mapping, "src-project-name", 1000000) assert updated_event.metadata["_replicatedInternalId"] == events_src[i].id assert updated_event.metadata["_replicatedSource"] == "src-project-name" assert len(updated_event.asset_ids) == len(events_src[i].asset_ids) for j, asset_id in enumerate(updated_event.asset_ids): assert asset_id == id_mapping[events_src[i].asset_ids[j]]
def new_event(): event = COGNITE_CLIENT.events.create(Event()) yield event COGNITE_CLIENT.events.delete(id=event.id) assert COGNITE_CLIENT.events.retrieve(event.id) is None
def test_delete_with_nonexisting(self): a = COGNITE_CLIENT.events.create(Event()) COGNITE_CLIENT.events.delete(id=a.id, external_id="this event does not exist", ignore_unknown_ids=True) assert COGNITE_CLIENT.events.retrieve(id=a.id) is None