def test_event_uploader2(self, MockCogniteClient):
client: CogniteClient = MockCogniteClient()
post_upload_test = {"value": False}
def post(x):
post_upload_test["value"] = True
queue = EventUploadQueue(client,
max_upload_interval=2,
post_upload_function=post)
queue.start()
event1 = Event(start_time=1, end_time=2, description="test event")
event2 = Event(start_time=3,
end_time=4,
description="another test event")
queue.add_to_upload_queue(event1)
queue.add_to_upload_queue(event2)
time.sleep(2.1)
client.events.create.ssert_called_with([event1, event2])
self.assertTrue(post_upload_test["value"])
queue.stop()
def test_make_objects_batch():
src_objects = [
Event(id=1, last_updated_time=1000),
Event(id=2, last_updated_time=1000),
Event(id=3, last_updated_time=1000),
]
src_id_to_dst = {
1: Event(id=11, metadata={"_replicatedTime": 100}),
2: Event(id=12, metadata={"_replicatedTime": 10000}),
}
def dummy_update(src_obj, dst_obj, src_dst_ids_assets, project_src,
replicated_runtime):
return dst_obj
def dummy_create(src_obj, src_dst_ids_assets, project_src,
replicated_runtime):
return src_obj
created, updated, unchanged = make_objects_batch(
src_objects,
src_id_to_dst,
{},
create=dummy_create,
update=dummy_update,
project_src="test project",
replicated_runtime=10000,
)
assert len(created) == len(updated) == len(unchanged) == 1
assert updated[0].id == 11
assert unchanged[0].id == 12
assert created[0].id == 3
def test_copy_events():
events_src = [
Event(metadata={}, id=1007, asset_ids=[3, 9], start_time=0, end_time=1),
Event(metadata={}, id=2007, asset_ids=[7], start_time=1, end_time=2),
Event(start_time=2, asset_ids=[5], end_time=4, metadata={}),
]
id_mapping = {i: i * 111 for i in range(1, 10)}
with monkeypatch_cognite_client() as client_dst:
copy_events(events_src, {}, id_mapping, "src-project-name", 1000000, client_dst, None)
def update_event(src_event: Event, dst_event: Event,
src_dst_ids_assets: Dict[int, int], project_src: str,
runtime: int) -> Event:
"""
Makes an updated version of the destination event based on the corresponding source event.
Args:
src_event: The event from the source to be replicated.
dst_event: The event from the destination that needs to be updated to reflect changes made to its source event.
src_dst_ids_assets: A dictionary of all the mappings of source asset id to destination asset id.
project_src: The name of the project the object is being replicated from.
runtime: The timestamp to be used in the new replicated metadata.
Returns:
The updated event object for the replication destination.
"""
logging.debug(
f"Updating existing event {dst_event.id} based on source event id {src_event.id}"
)
dst_event.external_id = src_event.external_id
dst_event.start_time = (
src_event.start_time if src_event.start_time and src_event.end_time
and src_event.start_time < src_event.end_time else src_event.end_time)
dst_event.end_time = src_event.end_time
dst_event.type = src_event.type
dst_event.subtype = src_event.subtype
dst_event.description = src_event.description
dst_event.metadata = replication.new_metadata(src_event, project_src,
runtime)
dst_event.asset_ids = replication.get_asset_ids(src_event.asset_ids,
src_dst_ids_assets)
dst_event.source = src_event.source
return dst_event
def test_filter_objects():
time_series = [
TimeSeries(id=1, asset_id=100),
TimeSeries(id=2),
TimeSeries(id=3, asset_id=101)
]
events = [
Event(id=10, asset_ids=[100, 101]),
Event(id=11),
Event(id=12, asset_ids=[101])
]
src_dst_asset_id_map = {100: 1000}
dummy_filtered_events = filter_objects(events, src_dst_asset_id_map)
dummy_filtered_ts = filter_objects(time_series, src_dst_asset_id_map)
assert dummy_filtered_events == events
assert dummy_filtered_ts == time_series
asset_events = filter_objects(events,
src_dst_asset_id_map,
skip_nonasset=True)
asset_ts = filter_objects(time_series,
src_dst_asset_id_map,
skip_nonasset=True)
assert len(asset_events) == 2
assert len(asset_ts) == 2
for i in range(len(asset_ts)):
assert asset_ts[i].asset_id is not None
assert asset_events[i].asset_ids is not None
linkable_events = filter_objects(events,
src_dst_asset_id_map,
skip_nonasset=True,
skip_unlinkable=True)
linkable_ts = filter_objects(time_series,
src_dst_asset_id_map,
skip_nonasset=True,
skip_unlinkable=True)
assert len(linkable_events) == 1
assert len(linkable_ts) == 1
assert linkable_events[0] == events[0]
assert linkable_ts[0] == time_series[0]
odd_id_events = filter_objects(events,
src_dst_asset_id_map,
filter_fn=lambda x: x.id % 2 == 1)
assert len(odd_id_events) == 1
for event in odd_id_events:
assert event.id % 2 == 1
def create_event(src_event: Event, src_dst_ids_assets: Dict[int, int],
project_src: str, runtime: int) -> Event:
"""
Makes a new copy of the event to be replicated based on a source event.
Args:
src_event: The event from the source to be replicated to destination.
src_dst_ids_assets: A dictionary of all the mappings of source asset id to destination asset id.
project_src: The name of the project the object is being replicated from.
runtime: The timestamp to be used in the new replicated metadata.
Returns:
The replicated event to be created in the destination.
"""
logging.debug(
f"Creating a new event based on source event id {src_event.id}")
return Event(
external_id=src_event.external_id,
start_time=src_event.start_time
if src_event.start_time and src_event.end_time
and src_event.start_time < src_event.end_time else src_event.end_time,
end_time=src_event.end_time,
type=src_event.type,
subtype=src_event.subtype,
description=src_event.description,
metadata=replication.new_metadata(src_event, project_src, runtime),
asset_ids=replication.get_asset_ids(src_event.asset_ids,
src_dst_ids_assets),
source=src_event.source,
)
def test_event_uploader1(self, MockCogniteClient):
client: CogniteClient = MockCogniteClient()
queue = EventUploadQueue(client)
event1 = Event(start_time=1, end_time=2, description="test event")
event2 = Event(start_time=3,
end_time=4,
description="another test event")
queue.add_to_upload_queue(event1)
queue.add_to_upload_queue(event2)
client.events.create.assert_not_called()
queue.upload()
client.events.create.ssert_called_with([event1, event2])
def test_create_event():
events_src = [
Event(metadata={}, id=1007, asset_ids=[3, 9], start_time=0, end_time=1),
Event(metadata={}, id=2007, asset_ids=[7], start_time=1, end_time=2),
Event(start_time=2, asset_ids=[5], end_time=4, metadata={}),
Event(start_time=6, end_time=8, metadata={}),
]
id_mapping = {3: 333, 7: 777, 5: 555, 9: 999}
for i, event in enumerate(events_src):
created_event = create_event(event, id_mapping, "src-project-name {}".format(i), 10000000)
assert created_event.metadata["_replicatedInternalId"] == events_src[i].id
assert created_event.metadata["_replicatedSource"] == "src-project-name {}".format(i)
assert (created_event.asset_ids is None) == (events_src[i].asset_ids is None)
if created_event.asset_ids is not None:
assert len(created_event.asset_ids) == len(events_src[i].asset_ids)
for j, asset_id in enumerate(created_event.asset_ids):
assert asset_id == id_mapping[events_src[i].asset_ids[j]]
def test_remove_replication_metadata():
events = [
Event(
metadata={
"_replicatedInternalId": 10,
"_replicatedSource": "src_project",
"_replicatedTime": 10000000
}),
Event(metadata={}),
Event(id=3,
metadata={
"_replicatedInternalId": 10,
"misc1": 16,
"misc2": "text"
}),
]
remove_replication_metadata(events)
for event in events:
assert "_replicatedInternalId" not in event.metadata
assert "_replicatedSource" not in event.metadata
assert "_replicatedTime" not in event.metadata
assert len(events[2].metadata.keys()) == 2
def test_update_event():
events_src = [
Event(metadata={}, id=1007, asset_ids=[3, 9], start_time=0, end_time=1),
Event(metadata={}, id=2007, asset_ids=[7], start_time=1, end_time=2),
Event(start_time=2, asset_ids=[5], end_time=4, metadata={}),
]
events_dst = [
Event(asset_ids=[333], metadata={}),
Event(asset_ids=[777], metadata={"_replicatedInternalId": 7}),
Event(asset_ids=[555], metadata={"source": "None"}),
]
id_mapping = {3: 333, 7: 777, 5: 555, 9: 999}
for i in range(len(events_src)):
updated_event = update_event(events_src[i], events_dst[i], id_mapping, "src-project-name", 1000000)
assert updated_event.metadata["_replicatedInternalId"] == events_src[i].id
assert updated_event.metadata["_replicatedSource"] == "src-project-name"
assert len(updated_event.asset_ids) == len(events_src[i].asset_ids)
for j, asset_id in enumerate(updated_event.asset_ids):
assert asset_id == id_mapping[events_src[i].asset_ids[j]]
def new_event():
event = COGNITE_CLIENT.events.create(Event())
yield event
COGNITE_CLIENT.events.delete(id=event.id)
assert COGNITE_CLIENT.events.retrieve(event.id) is None
def test_delete_with_nonexisting(self):
a = COGNITE_CLIENT.events.create(Event())
COGNITE_CLIENT.events.delete(id=a.id,
external_id="this event does not exist",
ignore_unknown_ids=True)
assert COGNITE_CLIENT.events.retrieve(id=a.id) is None