def test_filter_outlier(self) -> None:
# outlier events must be returned, for the good of the collective federation
self._inject_room_member("@resident:remote_hs")
self._inject_visibility("@resident:remote_hs", "joined")
outlier = self._inject_outlier()
self.assertEqual(
self.get_success(
filter_events_for_server(self._storage_controllers,
"remote_hs", [outlier])),
[outlier],
)
# it should also work when there are other events in the list
evt = self._inject_message("@unerased:local_hs")
filtered = self.get_success(
filter_events_for_server(self._storage_controllers, "remote_hs",
[outlier, evt]))
self.assertEqual(len(filtered), 2,
f"expected 2 results, got: {filtered}")
self.assertEqual(filtered[0], outlier)
self.assertEqual(filtered[1].event_id, evt.event_id)
self.assertEqual(filtered[1].content, evt.content)
# ... but other servers should only be able to see the outlier (the other should
# be redacted)
filtered = self.get_success(
filter_events_for_server(self._storage_controllers, "other_server",
[outlier, evt]))
self.assertEqual(filtered[0], outlier)
self.assertEqual(filtered[1].event_id, evt.event_id)
self.assertNotIn("body", filtered[1].content)
def test_filtering(self):
#
# The events to be filtered consist of 10 membership events (it doesn't
# really matter if they are joins or leaves, so let's make them joins).
# One of those membership events is going to be for a user on the
# server we are filtering for (so we can check the filtering is doing
# the right thing).
#
# before we do that, we persist some other events to act as state.
self.inject_visibility("@admin:hs", "joined")
for i in range(0, 10):
yield self.inject_room_member("@resident%i:hs" % i)
events_to_filter = []
for i in range(0, 10):
user = "******" % (i,
"test_server" if i == 5 else "other_server")
evt = yield self.inject_room_member(user, extra_content={"a": "b"})
events_to_filter.append(evt)
filtered = yield filter_events_for_server(self.store, "test_server",
events_to_filter)
# the result should be 5 redacted events, and 5 unredacted events.
for i in range(0, 5):
self.assertEqual(events_to_filter[i].event_id,
filtered[i].event_id)
self.assertNotIn("a", filtered[i].content)
for i in range(5, 10):
self.assertEqual(events_to_filter[i].event_id,
filtered[i].event_id)
self.assertEqual(filtered[i].content["a"], "b")
def test_filtering(self):
#
# The events to be filtered consist of 10 membership events (it doesn't
# really matter if they are joins or leaves, so let's make them joins).
# One of those membership events is going to be for a user on the
# server we are filtering for (so we can check the filtering is doing
# the right thing).
#
# before we do that, we persist some other events to act as state.
self.inject_visibility("@admin:hs", "joined")
for i in range(0, 10):
yield self.inject_room_member("@resident%i:hs" % i)
events_to_filter = []
for i in range(0, 10):
user = "******" % (i, "test_server" if i == 5 else "other_server")
evt = yield self.inject_room_member(user, extra_content={"a": "b"})
events_to_filter.append(evt)
filtered = yield filter_events_for_server(
self.store, "test_server", events_to_filter
)
# the result should be 5 redacted events, and 5 unredacted events.
for i in range(0, 5):
self.assertEqual(events_to_filter[i].event_id, filtered[i].event_id)
self.assertNotIn("a", filtered[i].content)
for i in range(5, 10):
self.assertEqual(events_to_filter[i].event_id, filtered[i].event_id)
self.assertEqual(filtered[i].content["a"], "b")
def test_erased_user(self):
# 4 message events, from erased and unerased users, with a membership
# change in the middle of them.
events_to_filter = []
evt = yield self.inject_message("@unerased:local_hs")
events_to_filter.append(evt)
evt = yield self.inject_message("@erased:local_hs")
events_to_filter.append(evt)
evt = yield self.inject_room_member("@joiner:remote_hs")
events_to_filter.append(evt)
evt = yield self.inject_message("@unerased:local_hs")
events_to_filter.append(evt)
evt = yield self.inject_message("@erased:local_hs")
events_to_filter.append(evt)
# the erasey user gets erased
yield defer.ensureDeferred(
self.hs.get_datastore().mark_user_erased("@erased:local_hs")
)
# ... and the filtering happens.
filtered = yield defer.ensureDeferred(
filter_events_for_server(self.storage, "test_server", events_to_filter)
)
for i in range(0, len(events_to_filter)):
self.assertEqual(
events_to_filter[i].event_id,
filtered[i].event_id,
"Unexpected event at result position %i" % (i,),
)
for i in (0, 3):
self.assertEqual(
events_to_filter[i].content["body"],
filtered[i].content["body"],
"Unexpected event content at result position %i" % (i,),
)
for i in (1, 4):
self.assertNotIn("body", filtered[i].content)
def test_erased_user(self):
# 4 message events, from erased and unerased users, with a membership
# change in the middle of them.
events_to_filter = []
evt = yield self.inject_message("@unerased:local_hs")
events_to_filter.append(evt)
evt = yield self.inject_message("@erased:local_hs")
events_to_filter.append(evt)
evt = yield self.inject_room_member("@joiner:remote_hs")
events_to_filter.append(evt)
evt = yield self.inject_message("@unerased:local_hs")
events_to_filter.append(evt)
evt = yield self.inject_message("@erased:local_hs")
events_to_filter.append(evt)
# the erasey user gets erased
yield self.hs.get_datastore().mark_user_erased("@erased:local_hs")
# ... and the filtering happens.
filtered = yield filter_events_for_server(
self.store, "test_server", events_to_filter
)
for i in range(0, len(events_to_filter)):
self.assertEqual(
events_to_filter[i].event_id,
filtered[i].event_id,
"Unexpected event at result position %i" % (i,),
)
for i in (0, 3):
self.assertEqual(
events_to_filter[i].content["body"],
filtered[i].content["body"],
"Unexpected event content at result position %i" % (i,),
)
for i in (1, 4):
self.assertNotIn("body", filtered[i].content)
def test_large_room(self):
# see what happens when we have a large room with hundreds of thousands
# of membership events
# As above, the events to be filtered consist of 10 membership events,
# where one of them is for a user on the server we are filtering for.
import cProfile
import pstats
import time
# we stub out the store, because building up all that state the normal
# way is very slow.
test_store = _TestStore()
# our initial state is 100000 membership events and one
# history_visibility event.
room_state = []
history_visibility_evt = FrozenEvent({
"event_id": "$history_vis",
"type": "m.room.history_visibility",
"sender": "@resident_user_0:test.com",
"state_key": "",
"room_id": TEST_ROOM_ID,
"content": {
"history_visibility": "joined"
},
})
room_state.append(history_visibility_evt)
test_store.add_event(history_visibility_evt)
for i in range(0, 100000):
user = "******" % (i, )
evt = FrozenEvent({
"event_id": "$res_event_%i" % (i, ),
"type": "m.room.member",
"state_key": user,
"sender": user,
"room_id": TEST_ROOM_ID,
"content": {
"membership": "join",
"extra": "zzz,"
},
})
room_state.append(evt)
test_store.add_event(evt)
events_to_filter = []
for i in range(0, 10):
user = "******" % (i,
"test_server" if i == 5 else "other_server")
evt = FrozenEvent({
"event_id": "$evt%i" % (i, ),
"type": "m.room.member",
"state_key": user,
"sender": user,
"room_id": TEST_ROOM_ID,
"content": {
"membership": "join",
"extra": "zzz"
},
})
events_to_filter.append(evt)
room_state.append(evt)
test_store.add_event(evt)
test_store.set_state_ids_for_event(
evt, {(e.type, e.state_key): e.event_id
for e in room_state})
pr = cProfile.Profile()
pr.enable()
logger.info("Starting filtering")
start = time.time()
filtered = yield filter_events_for_server(test_store, "test_server",
events_to_filter)
logger.info("Filtering took %f seconds", time.time() - start)
pr.disable()
with open("filter_events_for_server.profile", "w+") as f:
ps = pstats.Stats(pr, stream=f).sort_stats("cumulative")
ps.print_stats()
# the result should be 5 redacted events, and 5 unredacted events.
for i in range(0, 5):
self.assertEqual(events_to_filter[i].event_id,
filtered[i].event_id)
self.assertNotIn("extra", filtered[i].content)
for i in range(5, 10):
self.assertEqual(events_to_filter[i].event_id,
filtered[i].event_id)
self.assertEqual(filtered[i].content["extra"], "zzz")
def test_large_room(self):
# see what happens when we have a large room with hundreds of thousands
# of membership events
# As above, the events to be filtered consist of 10 membership events,
# where one of them is for a user on the server we are filtering for.
import cProfile
import pstats
import time
# we stub out the store, because building up all that state the normal
# way is very slow.
test_store = _TestStore()
# our initial state is 100000 membership events and one
# history_visibility event.
room_state = []
history_visibility_evt = FrozenEvent(
{
"event_id": "$history_vis",
"type": "m.room.history_visibility",
"sender": "@resident_user_0:test.com",
"state_key": "",
"room_id": TEST_ROOM_ID,
"content": {"history_visibility": "joined"},
}
)
room_state.append(history_visibility_evt)
test_store.add_event(history_visibility_evt)
for i in range(0, 100000):
user = "******" % (i,)
evt = FrozenEvent(
{
"event_id": "$res_event_%i" % (i,),
"type": "m.room.member",
"state_key": user,
"sender": user,
"room_id": TEST_ROOM_ID,
"content": {"membership": "join", "extra": "zzz,"},
}
)
room_state.append(evt)
test_store.add_event(evt)
events_to_filter = []
for i in range(0, 10):
user = "******" % (i, "test_server" if i == 5 else "other_server")
evt = FrozenEvent(
{
"event_id": "$evt%i" % (i,),
"type": "m.room.member",
"state_key": user,
"sender": user,
"room_id": TEST_ROOM_ID,
"content": {"membership": "join", "extra": "zzz"},
}
)
events_to_filter.append(evt)
room_state.append(evt)
test_store.add_event(evt)
test_store.set_state_ids_for_event(
evt, {(e.type, e.state_key): e.event_id for e in room_state}
)
pr = cProfile.Profile()
pr.enable()
logger.info("Starting filtering")
start = time.time()
filtered = yield filter_events_for_server(
test_store, "test_server", events_to_filter
)
logger.info("Filtering took %f seconds", time.time() - start)
pr.disable()
with open("filter_events_for_server.profile", "w+") as f:
ps = pstats.Stats(pr, stream=f).sort_stats('cumulative')
ps.print_stats()
# the result should be 5 redacted events, and 5 unredacted events.
for i in range(0, 5):
self.assertEqual(events_to_filter[i].event_id, filtered[i].event_id)
self.assertNotIn("extra", filtered[i].content)
for i in range(5, 10):
self.assertEqual(events_to_filter[i].event_id, filtered[i].event_id)
self.assertEqual(filtered[i].content["extra"], "zzz")
