def test_decompress_ignores_if_not_enough_chunks(raw_snapshot_events):
raw_snapshot_data = [event["properties"]["$snapshot_data"] for event in raw_snapshot_events]
snapshot_data_list = [
event["properties"]["$snapshot_data"] for event in compress_and_chunk_snapshots(raw_snapshot_events, 1000)
]
window_id = "abc123"
snapshot_list = []
for snapshot_data in snapshot_data_list:
snapshot_list.append(SnapshotDataTaggedWithWindowId(window_id=window_id, snapshot_data=snapshot_data))
snapshot_list.append(
SnapshotDataTaggedWithWindowId(
snapshot_data={
"chunk_id": "unique_id",
"chunk_index": 1,
"chunk_count": 2,
"data": {},
"compression": "gzip",
"has_full_snapshot": False,
},
window_id=window_id,
)
)
assert (
decompress_chunked_snapshot_data(2, "someid", snapshot_list).snapshot_data_by_window_id[window_id]
== raw_snapshot_data
)
def compress_decompress_and_extract(events, chunk_size):
snapshot_data_list = [
event["properties"]["$snapshot_data"] for event in compress_and_chunk_snapshots(events, chunk_size)
]
window_id = "abc123"
snapshot_list = []
for snapshot_data in snapshot_data_list:
snapshot_list.append(SnapshotDataTaggedWithWindowId(window_id=window_id, snapshot_data=snapshot_data))
return decompress_chunked_snapshot_data(2, "someid", snapshot_list).snapshot_data_by_window_id[window_id]
def test_decompress_uncompressed_events_returns_unmodified_events(raw_snapshot_events):
snapshot_data_tagged_with_window_id = []
raw_snapshot_data = []
for event in raw_snapshot_events:
snapshot_data_tagged_with_window_id.append(
SnapshotDataTaggedWithWindowId(snapshot_data=event["properties"]["$snapshot_data"], window_id="1")
)
raw_snapshot_data.append(event["properties"]["$snapshot_data"])
assert (
decompress_chunked_snapshot_data(1, "someid", snapshot_data_tagged_with_window_id).snapshot_data_by_window_id[
"1"
]
== raw_snapshot_data
)
def get_snapshot_data(
self
) -> Tuple[Optional[DistinctId], Optional[datetime], Snapshots]:
events = self.query_recording_snapshots()
if len(events) == 0:
return None, None, []
return (
events[0].distinct_id,
events[0].timestamp,
list(
decompress_chunked_snapshot_data(
self._team.pk, self._session_recording_id,
[e.snapshot_data for e in events])),
)
def test_decompress_ignores_if_not_enough_chunks(snapshot_events):
snapshot_data = complete_snapshots = [event["properties"]["$snapshot_data"] for event in snapshot_events]
snapshot_data.append(
{
"$session_id": "1234",
"$snapshot_data": {
"chunk_id": "unique_id",
"chunk_index": 1,
"chunk_count": 2,
"data": {},
"compression": "gzip",
"has_full_snapshot": False,
},
"distinct_id": "abc123",
}
)
assert list(decompress_chunked_snapshot_data(1, "someid", snapshot_data)) == complete_snapshots
def test_decompress_data_returning_only_activity_info(chunked_and_compressed_snapshot_events):
snapshot_data = [
SnapshotDataTaggedWithWindowId(
snapshot_data=event["properties"]["$snapshot_data"], window_id=event["properties"].get("$window_id")
)
for event in chunked_and_compressed_snapshot_events
]
paginated_events = decompress_chunked_snapshot_data(1, "someid", snapshot_data, return_only_activity_data=True)
assert paginated_events.snapshot_data_by_window_id == {
None: [
{"timestamp": "2019-01-01T00:00:00.000Z", "is_active": False},
{"timestamp": "2019-01-01T00:00:00.000Z", "is_active": False},
],
"1": [
{"timestamp": "2019-01-01T00:00:00.000Z", "is_active": False},
{"timestamp": "2019-01-01T00:00:00.000Z", "is_active": True},
],
}
def run(self, team: Team, session_recording_id: str, *args,
**kwargs) -> Dict[str, Any]:
from posthog.api.person import PersonSerializer
distinct_id, start_time, snapshots = self.query_recording_snapshots(
team, session_recording_id)
snapshots = list(
decompress_chunked_snapshot_data(team.pk, session_recording_id,
snapshots))
person = (PersonSerializer(
Person.objects.get(team=team,
persondistinctid__distinct_id=distinct_id)).data
if distinct_id else None)
return {
"snapshots": snapshots,
"person": person,
"start_time": start_time,
}
def test_paginate_decompression(chunked_and_compressed_snapshot_events):
snapshot_data = [
SnapshotDataTaggedWithWindowId(
snapshot_data=event["properties"]["$snapshot_data"], window_id=event["properties"].get("$window_id")
)
for event in chunked_and_compressed_snapshot_events
]
# Get the first chunk
paginated_events = decompress_chunked_snapshot_data(1, "someid", snapshot_data, 1, 0)
assert paginated_events.has_next is True
assert paginated_events.snapshot_data_by_window_id[None][0]["type"] == 4
assert len(paginated_events.snapshot_data_by_window_id[None]) == 2 # 2 events in a chunk
# Get the second chunk
paginated_events = decompress_chunked_snapshot_data(1, "someid", snapshot_data, 1, 1)
assert paginated_events.has_next is False
assert paginated_events.snapshot_data_by_window_id["1"][0]["type"] == 3
assert len(paginated_events.snapshot_data_by_window_id["1"]) == 2 # 2 events in a chunk
# Limit exceeds the length
paginated_events = decompress_chunked_snapshot_data(1, "someid", snapshot_data, 10, 0)
assert paginated_events.has_next is False
assert len(paginated_events.snapshot_data_by_window_id["1"]) == 2
assert len(paginated_events.snapshot_data_by_window_id[None]) == 2
# Offset exceeds the length
paginated_events = decompress_chunked_snapshot_data(1, "someid", snapshot_data, 10, 2)
assert paginated_events.has_next is False
assert paginated_events.snapshot_data_by_window_id == {}
# Non sequential snapshots
snapshot_data = snapshot_data[-3:] + snapshot_data[0:-3]
paginated_events = decompress_chunked_snapshot_data(1, "someid", snapshot_data, 10, 0)
assert paginated_events.has_next is False
assert len(paginated_events.snapshot_data_by_window_id["1"]) == 2
assert len(paginated_events.snapshot_data_by_window_id[None]) == 2
# No limit or offset provided
paginated_events = decompress_chunked_snapshot_data(1, "someid", snapshot_data)
assert paginated_events.has_next is False
assert len(paginated_events.snapshot_data_by_window_id["1"]) == 2
assert len(paginated_events.snapshot_data_by_window_id[None]) == 2
def test_decompress_returns_unmodified_events(snapshot_events):
snapshot_data = [event["properties"]["$snapshot_data"] for event in snapshot_events]
assert list(decompress_chunked_snapshot_data(1, "someid", snapshot_data)) == snapshot_data
def compress_and_decompress(events, chunk_size):
snapshot_data = [
event["properties"]["$snapshot_data"] for event in compress_and_chunk_snapshots(events, chunk_size)
]
return list(decompress_chunked_snapshot_data(1, "someid", snapshot_data))
def test_decompress_empty_list(chunked_and_compressed_snapshot_events):
paginated_events = decompress_chunked_snapshot_data(1, "someid", [])
assert paginated_events.has_next is False
assert paginated_events.snapshot_data_by_window_id == {}
