コード例 #1
0
    def setUp(self):
        self.mock_federation_resource = MockHttpResource()

        self.mock_http_client = Mock(spec=[])
        self.mock_http_client.put_json = DeferredMockCallable()

        hs = yield setup_test_homeserver(
            handlers=None,
            http_client=self.mock_http_client,
            keyring=Mock(),
        )

        self.filtering = hs.get_filtering()
        self.filter = Filter({})

        self.datastore = hs.get_datastore()
コード例 #2
0
 def test_definition_not_senders_works_with_literals(self):
     definition = {"not_senders": ["@flibble:wibble"]}
     event = MockEvent(sender="@flibble:wibble",
                       type="com.nom.nom.nom",
                       room_id="!foo:bar")
     self.assertFalse(Filter(definition).check(event))
コード例 #3
0
 def test_definition_senders_works_with_unknowns(self):
     definition = {"senders": ["@flibble:wibble"]}
     event = MockEvent(sender="@challenger:appears",
                       type="com.nom.nom.nom",
                       room_id="!foo:bar")
     self.assertFalse(Filter(definition).check(event))
コード例 #4
0
 def test_definition_not_types_works_with_unknowns(self):
     definition = {"not_types": ["m.*", "org.*"]}
     event = MockEvent(sender="@foo:bar",
                       type="com.nom.nom.nom",
                       room_id="!foo:bar")
     self.assertTrue(Filter(definition).check(event))
コード例 #5
0
 def test_definition_not_types_works_with_wildcards(self):
     definition = {"not_types": ["m.room.message", "org.matrix.*"]}
     event = MockEvent(sender="@foo:bar",
                       type="org.matrix.custom.event",
                       room_id="!foo:bar")
     self.assertFalse(Filter(definition).check(event))
コード例 #6
0
 def test_definition_types_works_with_wildcards(self):
     definition = {"types": ["m.*", "org.matrix.foo.bar"]}
     event = MockEvent(sender="@foo:bar",
                       type="m.room.message",
                       room_id="!foo:bar")
     self.assertTrue(Filter(definition).check(event))
コード例 #7
0
 def test_definition_rooms_works_with_literals(self):
     definition = {"rooms": ["!secretbase:unknown"]}
     event = MockEvent(sender="@foo:bar",
                       type="m.room.message",
                       room_id="!secretbase:unknown")
     self.assertTrue(Filter(definition).check(event))
コード例 #8
0
 def test_definition_types_works_with_unknowns(self):
     definition = {"types": ["m.room.message", "org.matrix.foo.bar"]}
     event = MockEvent(sender="@foo:bar",
                       type="now.for.something.completely.different",
                       room_id="!foo:bar")
     self.assertFalse(Filter(definition).check(event))
コード例 #9
0
class FilteringTestCase(unittest.TestCase):

    @defer.inlineCallbacks
    def setUp(self):
        self.mock_federation_resource = MockHttpResource()

        self.mock_http_client = Mock(spec=[])
        self.mock_http_client.put_json = DeferredMockCallable()

        hs = yield setup_test_homeserver(
            handlers=None,
            http_client=self.mock_http_client,
            keyring=Mock(),
        )

        self.filtering = hs.get_filtering()
        self.filter = Filter({})

        self.datastore = hs.get_datastore()

    def test_definition_types_works_with_literals(self):
        definition = {
            "types": ["m.room.message", "org.matrix.foo.bar"]
        }
        event = MockEvent(
            sender="@foo:bar",
            type="m.room.message",
            room_id="!foo:bar"
        )
        self.assertTrue(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_types_works_with_wildcards(self):
        definition = {
            "types": ["m.*", "org.matrix.foo.bar"]
        }
        event = MockEvent(
            sender="@foo:bar",
            type="m.room.message",
            room_id="!foo:bar"
        )
        self.assertTrue(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_types_works_with_unknowns(self):
        definition = {
            "types": ["m.room.message", "org.matrix.foo.bar"]
        }
        event = MockEvent(
            sender="@foo:bar",
            type="now.for.something.completely.different",
            room_id="!foo:bar"
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_not_types_works_with_literals(self):
        definition = {
            "not_types": ["m.room.message", "org.matrix.foo.bar"]
        }
        event = MockEvent(
            sender="@foo:bar",
            type="m.room.message",
            room_id="!foo:bar"
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_not_types_works_with_wildcards(self):
        definition = {
            "not_types": ["m.room.message", "org.matrix.*"]
        }
        event = MockEvent(
            sender="@foo:bar",
            type="org.matrix.custom.event",
            room_id="!foo:bar"
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_not_types_works_with_unknowns(self):
        definition = {
            "not_types": ["m.*", "org.*"]
        }
        event = MockEvent(
            sender="@foo:bar",
            type="com.nom.nom.nom",
            room_id="!foo:bar"
        )
        self.assertTrue(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_not_types_takes_priority_over_types(self):
        definition = {
            "not_types": ["m.*", "org.*"],
            "types": ["m.room.message", "m.room.topic"]
        }
        event = MockEvent(
            sender="@foo:bar",
            type="m.room.topic",
            room_id="!foo:bar"
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_senders_works_with_literals(self):
        definition = {
            "senders": ["@flibble:wibble"]
        }
        event = MockEvent(
            sender="@flibble:wibble",
            type="com.nom.nom.nom",
            room_id="!foo:bar"
        )
        self.assertTrue(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_senders_works_with_unknowns(self):
        definition = {
            "senders": ["@flibble:wibble"]
        }
        event = MockEvent(
            sender="@challenger:appears",
            type="com.nom.nom.nom",
            room_id="!foo:bar"
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_not_senders_works_with_literals(self):
        definition = {
            "not_senders": ["@flibble:wibble"]
        }
        event = MockEvent(
            sender="@flibble:wibble",
            type="com.nom.nom.nom",
            room_id="!foo:bar"
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_not_senders_works_with_unknowns(self):
        definition = {
            "not_senders": ["@flibble:wibble"]
        }
        event = MockEvent(
            sender="@challenger:appears",
            type="com.nom.nom.nom",
            room_id="!foo:bar"
        )
        self.assertTrue(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_not_senders_takes_priority_over_senders(self):
        definition = {
            "not_senders": ["@misspiggy:muppets"],
            "senders": ["@kermit:muppets", "@misspiggy:muppets"]
        }
        event = MockEvent(
            sender="@misspiggy:muppets",
            type="m.room.topic",
            room_id="!foo:bar"
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_rooms_works_with_literals(self):
        definition = {
            "rooms": ["!secretbase:unknown"]
        }
        event = MockEvent(
            sender="@foo:bar",
            type="m.room.message",
            room_id="!secretbase:unknown"
        )
        self.assertTrue(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_rooms_works_with_unknowns(self):
        definition = {
            "rooms": ["!secretbase:unknown"]
        }
        event = MockEvent(
            sender="@foo:bar",
            type="m.room.message",
            room_id="!anothersecretbase:unknown"
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_not_rooms_works_with_literals(self):
        definition = {
            "not_rooms": ["!anothersecretbase:unknown"]
        }
        event = MockEvent(
            sender="@foo:bar",
            type="m.room.message",
            room_id="!anothersecretbase:unknown"
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_not_rooms_works_with_unknowns(self):
        definition = {
            "not_rooms": ["!secretbase:unknown"]
        }
        event = MockEvent(
            sender="@foo:bar",
            type="m.room.message",
            room_id="!anothersecretbase:unknown"
        )
        self.assertTrue(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_not_rooms_takes_priority_over_rooms(self):
        definition = {
            "not_rooms": ["!secretbase:unknown"],
            "rooms": ["!secretbase:unknown"]
        }
        event = MockEvent(
            sender="@foo:bar",
            type="m.room.message",
            room_id="!secretbase:unknown"
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_combined_event(self):
        definition = {
            "not_senders": ["@misspiggy:muppets"],
            "senders": ["@kermit:muppets"],
            "rooms": ["!stage:unknown"],
            "not_rooms": ["!piggyshouse:muppets"],
            "types": ["m.room.message", "muppets.kermit.*"],
            "not_types": ["muppets.misspiggy.*"]
        }
        event = MockEvent(
            sender="@kermit:muppets",  # yup
            type="m.room.message",  # yup
            room_id="!stage:unknown"  # yup
        )
        self.assertTrue(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_combined_event_bad_sender(self):
        definition = {
            "not_senders": ["@misspiggy:muppets"],
            "senders": ["@kermit:muppets"],
            "rooms": ["!stage:unknown"],
            "not_rooms": ["!piggyshouse:muppets"],
            "types": ["m.room.message", "muppets.kermit.*"],
            "not_types": ["muppets.misspiggy.*"]
        }
        event = MockEvent(
            sender="@misspiggy:muppets",  # nope
            type="m.room.message",  # yup
            room_id="!stage:unknown"  # yup
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_combined_event_bad_room(self):
        definition = {
            "not_senders": ["@misspiggy:muppets"],
            "senders": ["@kermit:muppets"],
            "rooms": ["!stage:unknown"],
            "not_rooms": ["!piggyshouse:muppets"],
            "types": ["m.room.message", "muppets.kermit.*"],
            "not_types": ["muppets.misspiggy.*"]
        }
        event = MockEvent(
            sender="@kermit:muppets",  # yup
            type="m.room.message",  # yup
            room_id="!piggyshouse:muppets"  # nope
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    def test_definition_combined_event_bad_type(self):
        definition = {
            "not_senders": ["@misspiggy:muppets"],
            "senders": ["@kermit:muppets"],
            "rooms": ["!stage:unknown"],
            "not_rooms": ["!piggyshouse:muppets"],
            "types": ["m.room.message", "muppets.kermit.*"],
            "not_types": ["muppets.misspiggy.*"]
        }
        event = MockEvent(
            sender="@kermit:muppets",  # yup
            type="muppets.misspiggy.kisses",  # nope
            room_id="!stage:unknown"  # yup
        )
        self.assertFalse(
            self.filter._passes_definition(definition, event)
        )

    @defer.inlineCallbacks
    def test_filter_public_user_data_match(self):
        user_filter_json = {
            "public_user_data": {
                "types": ["m.*"]
            }
        }
        user = UserID.from_string("@" + user_localpart + ":test")
        filter_id = yield self.datastore.add_user_filter(
            user_localpart=user_localpart,
            user_filter=user_filter_json,
        )
        event = MockEvent(
            sender="@foo:bar",
            type="m.profile",
            room_id="!foo:bar"
        )
        events = [event]

        user_filter = yield self.filtering.get_user_filter(
            user_localpart=user_localpart,
            filter_id=filter_id,
        )

        results = user_filter.filter_public_user_data(events=events)
        self.assertEquals(events, results)

    @defer.inlineCallbacks
    def test_filter_public_user_data_no_match(self):
        user_filter_json = {
            "public_user_data": {
                "types": ["m.*"]
            }
        }
        user = UserID.from_string("@" + user_localpart + ":test")
        filter_id = yield self.datastore.add_user_filter(
            user_localpart=user_localpart,
            user_filter=user_filter_json,
        )
        event = MockEvent(
            sender="@foo:bar",
            type="custom.avatar.3d.crazy",
            room_id="!foo:bar"
        )
        events = [event]

        user_filter = yield self.filtering.get_user_filter(
            user_localpart=user_localpart,
            filter_id=filter_id,
        )

        results = user_filter.filter_public_user_data(events=events)
        self.assertEquals([], results)

    @defer.inlineCallbacks
    def test_filter_room_state_match(self):
        user_filter_json = {
            "room": {
                "state": {
                    "types": ["m.*"]
                }
            }
        }
        user = UserID.from_string("@" + user_localpart + ":test")
        filter_id = yield self.datastore.add_user_filter(
            user_localpart=user_localpart,
            user_filter=user_filter_json,
        )
        event = MockEvent(
            sender="@foo:bar",
            type="m.room.topic",
            room_id="!foo:bar"
        )
        events = [event]

        user_filter = yield self.filtering.get_user_filter(
            user_localpart=user_localpart,
            filter_id=filter_id,
        )

        results = user_filter.filter_room_state(events=events)
        self.assertEquals(events, results)

    @defer.inlineCallbacks
    def test_filter_room_state_no_match(self):
        user_filter_json = {
            "room": {
                "state": {
                    "types": ["m.*"]
                }
            }
        }
        user = UserID.from_string("@" + user_localpart + ":test")
        filter_id = yield self.datastore.add_user_filter(
            user_localpart=user_localpart,
            user_filter=user_filter_json,
        )
        event = MockEvent(
            sender="@foo:bar",
            type="org.matrix.custom.event",
            room_id="!foo:bar"
        )
        events = [event]

        user_filter = yield self.filtering.get_user_filter(
            user_localpart=user_localpart,
            filter_id=filter_id,
        )

        results = user_filter.filter_room_state(events)
        self.assertEquals([], results)

    @defer.inlineCallbacks
    def test_add_filter(self):
        user_filter_json = {
            "room": {
                "state": {
                    "types": ["m.*"]
                }
            }
        }

        filter_id = yield self.filtering.add_user_filter(
            user_localpart=user_localpart,
            user_filter=user_filter_json,
        )

        self.assertEquals(filter_id, 0)
        self.assertEquals(user_filter_json,
            (yield self.datastore.get_user_filter(
                user_localpart=user_localpart,
                filter_id=0,
            ))
        )

    @defer.inlineCallbacks
    def test_get_filter(self):
        user_filter_json = {
            "room": {
                "state": {
                    "types": ["m.*"]
                }
            }
        }

        filter_id = yield self.datastore.add_user_filter(
            user_localpart=user_localpart,
            user_filter=user_filter_json,
        )

        filter = yield self.filtering.get_user_filter(
            user_localpart=user_localpart,
            filter_id=filter_id,
        )

        self.assertEquals(filter.filter_json, user_filter_json)
コード例 #10
0
ファイル: search.py プロジェクト: Xe/synapse
    def search(self, user, content, batch=None):
        """Performs a full text search for a user.

        Args:
            user (UserID)
            content (dict): Search parameters
            batch (str): The next_batch parameter. Used for pagination.

        Returns:
            dict to be returned to the client with results of search
        """

        batch_group = None
        batch_group_key = None
        batch_token = None
        if batch:
            try:
                b = decode_base64(batch)
                batch_group, batch_group_key, batch_token = b.split("\n")

                assert batch_group is not None
                assert batch_group_key is not None
                assert batch_token is not None
            except:
                raise SynapseError(400, "Invalid batch")

        try:
            room_cat = content["search_categories"]["room_events"]

            # The actual thing to query in FTS
            search_term = room_cat["search_term"]

            # Which "keys" to search over in FTS query
            keys = room_cat.get("keys", [
                "content.body", "content.name", "content.topic",
            ])

            # Filter to apply to results
            filter_dict = room_cat.get("filter", {})

            # What to order results by (impacts whether pagination can be doen)
            order_by = room_cat.get("order_by", "rank")

            # Return the current state of the rooms?
            include_state = room_cat.get("include_state", False)

            # Include context around each event?
            event_context = room_cat.get(
                "event_context", None
            )

            # Group results together? May allow clients to paginate within a
            # group
            group_by = room_cat.get("groupings", {}).get("group_by", {})
            group_keys = [g["key"] for g in group_by]

            if event_context is not None:
                before_limit = int(event_context.get(
                    "before_limit", 5
                ))
                after_limit = int(event_context.get(
                    "after_limit", 5
                ))

                # Return the historic display name and avatar for the senders
                # of the events?
                include_profile = bool(event_context.get("include_profile", False))
        except KeyError:
            raise SynapseError(400, "Invalid search query")

        if order_by not in ("rank", "recent"):
            raise SynapseError(400, "Invalid order by: %r" % (order_by,))

        if set(group_keys) - {"room_id", "sender"}:
            raise SynapseError(
                400,
                "Invalid group by keys: %r" % (set(group_keys) - {"room_id", "sender"},)
            )

        search_filter = Filter(filter_dict)

        # TODO: Search through left rooms too
        rooms = yield self.store.get_rooms_for_user_where_membership_is(
            user.to_string(),
            membership_list=[Membership.JOIN],
            # membership_list=[Membership.JOIN, Membership.LEAVE, Membership.Ban],
        )
        room_ids = set(r.room_id for r in rooms)

        room_ids = search_filter.filter_rooms(room_ids)

        if batch_group == "room_id":
            room_ids.intersection_update({batch_group_key})

        rank_map = {}  # event_id -> rank of event
        allowed_events = []
        room_groups = {}  # Holds result of grouping by room, if applicable
        sender_group = {}  # Holds result of grouping by sender, if applicable

        # Holds the next_batch for the entire result set if one of those exists
        global_next_batch = None

        if order_by == "rank":
            results = yield self.store.search_msgs(
                room_ids, search_term, keys
            )

            results_map = {r["event"].event_id: r for r in results}

            rank_map.update({r["event"].event_id: r["rank"] for r in results})

            filtered_events = search_filter.filter([r["event"] for r in results])

            events = yield self._filter_events_for_client(
                user.to_string(), filtered_events
            )

            events.sort(key=lambda e: -rank_map[e.event_id])
            allowed_events = events[:search_filter.limit()]

            for e in allowed_events:
                rm = room_groups.setdefault(e.room_id, {
                    "results": [],
                    "order": rank_map[e.event_id],
                })
                rm["results"].append(e.event_id)

                s = sender_group.setdefault(e.sender, {
                    "results": [],
                    "order": rank_map[e.event_id],
                })
                s["results"].append(e.event_id)

        elif order_by == "recent":
            # In this case we specifically loop through each room as the given
            # limit applies to each room, rather than a global list.
            # This is not necessarilly a good idea.
            for room_id in room_ids:
                room_events = []
                if batch_group == "room_id" and batch_group_key == room_id:
                    pagination_token = batch_token
                else:
                    pagination_token = None
                i = 0

                # We keep looping and we keep filtering until we reach the limit
                # or we run out of things.
                # But only go around 5 times since otherwise synapse will be sad.
                while len(room_events) < search_filter.limit() and i < 5:
                    i += 1
                    results = yield self.store.search_room(
                        room_id, search_term, keys, search_filter.limit() * 2,
                        pagination_token=pagination_token,
                    )

                    results_map = {r["event"].event_id: r for r in results}

                    rank_map.update({r["event"].event_id: r["rank"] for r in results})

                    filtered_events = search_filter.filter([
                        r["event"] for r in results
                    ])

                    events = yield self._filter_events_for_client(
                        user.to_string(), filtered_events
                    )

                    room_events.extend(events)
                    room_events = room_events[:search_filter.limit()]

                    if len(results) < search_filter.limit() * 2:
                        pagination_token = None
                        break
                    else:
                        pagination_token = results[-1]["pagination_token"]

                if room_events:
                    res = results_map[room_events[-1].event_id]
                    pagination_token = res["pagination_token"]

                    group = room_groups.setdefault(room_id, {})
                    if pagination_token:
                        next_batch = encode_base64("%s\n%s\n%s" % (
                            "room_id", room_id, pagination_token
                        ))
                        group["next_batch"] = next_batch

                        if batch_token:
                            global_next_batch = next_batch

                    group["results"] = [e.event_id for e in room_events]
                    group["order"] = max(
                        e.origin_server_ts/1000 for e in room_events
                        if hasattr(e, "origin_server_ts")
                    )

                allowed_events.extend(room_events)

            # Normalize the group orders
            if room_groups:
                if len(room_groups) > 1:
                    mx = max(g["order"] for g in room_groups.values())
                    mn = min(g["order"] for g in room_groups.values())

                    for g in room_groups.values():
                        g["order"] = (g["order"] - mn) * 1.0 / (mx - mn)
                else:
                    room_groups.values()[0]["order"] = 1

        else:
            # We should never get here due to the guard earlier.
            raise NotImplementedError()

        # If client has asked for "context" for each event (i.e. some surrounding
        # events and state), fetch that
        if event_context is not None:
            now_token = yield self.hs.get_event_sources().get_current_token()

            contexts = {}
            for event in allowed_events:
                res = yield self.store.get_events_around(
                    event.room_id, event.event_id, before_limit, after_limit
                )

                res["events_before"] = yield self._filter_events_for_client(
                    user.to_string(), res["events_before"]
                )

                res["events_after"] = yield self._filter_events_for_client(
                    user.to_string(), res["events_after"]
                )

                res["start"] = now_token.copy_and_replace(
                    "room_key", res["start"]
                ).to_string()

                res["end"] = now_token.copy_and_replace(
                    "room_key", res["end"]
                ).to_string()

                if include_profile:
                    senders = set(
                        ev.sender
                        for ev in itertools.chain(
                            res["events_before"], [event], res["events_after"]
                        )
                    )

                    if res["events_after"]:
                        last_event_id = res["events_after"][-1].event_id
                    else:
                        last_event_id = event.event_id

                    state = yield self.store.get_state_for_event(
                        last_event_id,
                        types=[(EventTypes.Member, sender) for sender in senders]
                    )

                    res["profile_info"] = {
                        s.state_key: {
                            "displayname": s.content.get("displayname", None),
                            "avatar_url": s.content.get("avatar_url", None),
                        }
                        for s in state.values()
                        if s.type == EventTypes.Member and s.state_key in senders
                    }

                contexts[event.event_id] = res
        else:
            contexts = {}

        # TODO: Add a limit

        time_now = self.clock.time_msec()

        for context in contexts.values():
            context["events_before"] = [
                serialize_event(e, time_now)
                for e in context["events_before"]
            ]
            context["events_after"] = [
                serialize_event(e, time_now)
                for e in context["events_after"]
            ]

        state_results = {}
        if include_state:
            rooms = set(e.room_id for e in allowed_events)
            for room_id in rooms:
                state = yield self.state_handler.get_current_state(room_id)
                state_results[room_id] = state.values()

            state_results.values()

        # We're now about to serialize the events. We should not make any
        # blocking calls after this. Otherwise the 'age' will be wrong

        results = {
            e.event_id: {
                "rank": rank_map[e.event_id],
                "result": serialize_event(e, time_now),
                "context": contexts.get(e.event_id, {}),
            }
            for e in allowed_events
        }

        logger.info("Found %d results", len(results))

        rooms_cat_res = {
            "results": results,
            "count": len(results)
        }

        if state_results:
            rooms_cat_res["state"] = {
                room_id: [serialize_event(e, time_now) for e in state]
                for room_id, state in state_results.items()
            }

        if room_groups and "room_id" in group_keys:
            rooms_cat_res.setdefault("groups", {})["room_id"] = room_groups

        if sender_group and "sender" in group_keys:
            rooms_cat_res.setdefault("groups", {})["sender"] = sender_group

        if global_next_batch:
            rooms_cat_res["next_batch"] = global_next_batch

        defer.returnValue({
            "search_categories": {
                "room_events": rooms_cat_res
            }
        })
コード例 #11
0
    def search(self, user, content, batch=None):
        """Performs a full text search for a user.

        Args:
            user (UserID)
            content (dict): Search parameters
            batch (str): The next_batch parameter. Used for pagination.

        Returns:
            dict to be returned to the client with results of search
        """

        batch_group = None
        batch_group_key = None
        batch_token = None
        if batch:
            try:
                b = decode_base64(batch)
                batch_group, batch_group_key, batch_token = b.split("\n")

                assert batch_group is not None
                assert batch_group_key is not None
                assert batch_token is not None
            except:
                raise SynapseError(400, "Invalid batch")

        try:
            room_cat = content["search_categories"]["room_events"]

            # The actual thing to query in FTS
            search_term = room_cat["search_term"]

            # Which "keys" to search over in FTS query
            keys = room_cat.get("keys", [
                "content.body",
                "content.name",
                "content.topic",
            ])

            # Filter to apply to results
            filter_dict = room_cat.get("filter", {})

            # What to order results by (impacts whether pagination can be doen)
            order_by = room_cat.get("order_by", "rank")

            # Return the current state of the rooms?
            include_state = room_cat.get("include_state", False)

            # Include context around each event?
            event_context = room_cat.get("event_context", None)

            # Group results together? May allow clients to paginate within a
            # group
            group_by = room_cat.get("groupings", {}).get("group_by", {})
            group_keys = [g["key"] for g in group_by]

            if event_context is not None:
                before_limit = int(event_context.get("before_limit", 5))
                after_limit = int(event_context.get("after_limit", 5))

                # Return the historic display name and avatar for the senders
                # of the events?
                include_profile = bool(
                    event_context.get("include_profile", False))
        except KeyError:
            raise SynapseError(400, "Invalid search query")

        if order_by not in ("rank", "recent"):
            raise SynapseError(400, "Invalid order by: %r" % (order_by, ))

        if set(group_keys) - {"room_id", "sender"}:
            raise SynapseError(
                400, "Invalid group by keys: %r" %
                (set(group_keys) - {"room_id", "sender"}, ))

        search_filter = Filter(filter_dict)

        # TODO: Search through left rooms too
        rooms = yield self.store.get_rooms_for_user_where_membership_is(
            user.to_string(),
            membership_list=[Membership.JOIN],
            # membership_list=[Membership.JOIN, Membership.LEAVE, Membership.Ban],
        )
        room_ids = set(r.room_id for r in rooms)

        room_ids = search_filter.filter_rooms(room_ids)

        if batch_group == "room_id":
            room_ids.intersection_update({batch_group_key})

        if not room_ids:
            defer.returnValue({
                "search_categories": {
                    "room_events": {
                        "results": [],
                        "count": 0,
                        "highlights": [],
                    }
                }
            })

        rank_map = {}  # event_id -> rank of event
        allowed_events = []
        room_groups = {}  # Holds result of grouping by room, if applicable
        sender_group = {}  # Holds result of grouping by sender, if applicable

        # Holds the next_batch for the entire result set if one of those exists
        global_next_batch = None

        highlights = set()

        count = None

        if order_by == "rank":
            search_result = yield self.store.search_msgs(
                room_ids, search_term, keys)

            count = search_result["count"]

            if search_result["highlights"]:
                highlights.update(search_result["highlights"])

            results = search_result["results"]

            results_map = {r["event"].event_id: r for r in results}

            rank_map.update({r["event"].event_id: r["rank"] for r in results})

            filtered_events = search_filter.filter(
                [r["event"] for r in results])

            events = yield self._filter_events_for_client(
                user.to_string(), filtered_events)

            events.sort(key=lambda e: -rank_map[e.event_id])
            allowed_events = events[:search_filter.limit()]

            for e in allowed_events:
                rm = room_groups.setdefault(e.room_id, {
                    "results": [],
                    "order": rank_map[e.event_id],
                })
                rm["results"].append(e.event_id)

                s = sender_group.setdefault(e.sender, {
                    "results": [],
                    "order": rank_map[e.event_id],
                })
                s["results"].append(e.event_id)

        elif order_by == "recent":
            room_events = []
            i = 0

            pagination_token = batch_token

            # We keep looping and we keep filtering until we reach the limit
            # or we run out of things.
            # But only go around 5 times since otherwise synapse will be sad.
            while len(room_events) < search_filter.limit() and i < 5:
                i += 1
                search_result = yield self.store.search_rooms(
                    room_ids,
                    search_term,
                    keys,
                    search_filter.limit() * 2,
                    pagination_token=pagination_token,
                )

                if search_result["highlights"]:
                    highlights.update(search_result["highlights"])

                count = search_result["count"]

                results = search_result["results"]

                results_map = {r["event"].event_id: r for r in results}

                rank_map.update(
                    {r["event"].event_id: r["rank"]
                     for r in results})

                filtered_events = search_filter.filter(
                    [r["event"] for r in results])

                events = yield self._filter_events_for_client(
                    user.to_string(), filtered_events)

                room_events.extend(events)
                room_events = room_events[:search_filter.limit()]

                if len(results) < search_filter.limit() * 2:
                    pagination_token = None
                    break
                else:
                    pagination_token = results[-1]["pagination_token"]

            for event in room_events:
                group = room_groups.setdefault(event.room_id, {
                    "results": [],
                })
                group["results"].append(event.event_id)

            if room_events and len(room_events) >= search_filter.limit():
                last_event_id = room_events[-1].event_id
                pagination_token = results_map[last_event_id][
                    "pagination_token"]

                # We want to respect the given batch group and group keys so
                # that if people blindly use the top level `next_batch` token
                # it returns more from the same group (if applicable) rather
                # than reverting to searching all results again.
                if batch_group and batch_group_key:
                    global_next_batch = encode_base64(
                        "%s\n%s\n%s" %
                        (batch_group, batch_group_key, pagination_token))
                else:
                    global_next_batch = encode_base64(
                        "%s\n%s\n%s" % ("all", "", pagination_token))

                for room_id, group in room_groups.items():
                    group["next_batch"] = encode_base64(
                        "%s\n%s\n%s" % ("room_id", room_id, pagination_token))

            allowed_events.extend(room_events)

        else:
            # We should never get here due to the guard earlier.
            raise NotImplementedError()

        # If client has asked for "context" for each event (i.e. some surrounding
        # events and state), fetch that
        if event_context is not None:
            now_token = yield self.hs.get_event_sources().get_current_token()

            contexts = {}
            for event in allowed_events:
                res = yield self.store.get_events_around(
                    event.room_id, event.event_id, before_limit, after_limit)

                res["events_before"] = yield self._filter_events_for_client(
                    user.to_string(), res["events_before"])

                res["events_after"] = yield self._filter_events_for_client(
                    user.to_string(), res["events_after"])

                res["start"] = now_token.copy_and_replace(
                    "room_key", res["start"]).to_string()

                res["end"] = now_token.copy_and_replace(
                    "room_key", res["end"]).to_string()

                if include_profile:
                    senders = set(ev.sender for ev in itertools.chain(
                        res["events_before"], [event], res["events_after"]))

                    if res["events_after"]:
                        last_event_id = res["events_after"][-1].event_id
                    else:
                        last_event_id = event.event_id

                    state = yield self.store.get_state_for_event(
                        last_event_id,
                        types=[(EventTypes.Member, sender)
                               for sender in senders])

                    res["profile_info"] = {
                        s.state_key: {
                            "displayname": s.content.get("displayname", None),
                            "avatar_url": s.content.get("avatar_url", None),
                        }
                        for s in state.values() if s.type == EventTypes.Member
                        and s.state_key in senders
                    }

                contexts[event.event_id] = res
        else:
            contexts = {}

        # TODO: Add a limit

        time_now = self.clock.time_msec()

        for context in contexts.values():
            context["events_before"] = [
                serialize_event(e, time_now) for e in context["events_before"]
            ]
            context["events_after"] = [
                serialize_event(e, time_now) for e in context["events_after"]
            ]

        state_results = {}
        if include_state:
            rooms = set(e.room_id for e in allowed_events)
            for room_id in rooms:
                state = yield self.state_handler.get_current_state(room_id)
                state_results[room_id] = state.values()

            state_results.values()

        # We're now about to serialize the events. We should not make any
        # blocking calls after this. Otherwise the 'age' will be wrong

        results = [{
            "rank": rank_map[e.event_id],
            "result": serialize_event(e, time_now),
            "context": contexts.get(e.event_id, {}),
        } for e in allowed_events]

        rooms_cat_res = {
            "results": results,
            "count": count,
            "highlights": list(highlights),
        }

        if state_results:
            rooms_cat_res["state"] = {
                room_id: [serialize_event(e, time_now) for e in state]
                for room_id, state in state_results.items()
            }

        if room_groups and "room_id" in group_keys:
            rooms_cat_res.setdefault("groups", {})["room_id"] = room_groups

        if sender_group and "sender" in group_keys:
            rooms_cat_res.setdefault("groups", {})["sender"] = sender_group

        if global_next_batch:
            rooms_cat_res["next_batch"] = global_next_batch

        defer.returnValue(
            {"search_categories": {
                "room_events": rooms_cat_res
            }})
コード例 #12
0
ファイル: search.py プロジェクト: samuelyi/synapse
    async def _search(
        self,
        user: UserID,
        batch_group: Optional[str],
        batch_group_key: Optional[str],
        batch_token: Optional[str],
        search_term: str,
        keys: List[str],
        filter_dict: JsonDict,
        order_by: str,
        include_state: bool,
        group_keys: List[str],
        event_context: Optional[bool],
        before_limit: Optional[int],
        after_limit: Optional[int],
        include_profile: bool,
    ) -> JsonDict:
        """Performs a full text search for a user.

        Args:
            user: The user performing the search.
            batch_group: Pagination information.
            batch_group_key: Pagination information.
            batch_token: Pagination information.
            search_term: Search term to search for
            keys: List of keys to search in, currently supports
                "content.body", "content.name", "content.topic"
            filter_dict: The JSON to build a filter out of.
            order_by: How to order the results. Valid values ore "rank" and "recent".
            include_state: True if the state of the room at each result should
                be included.
            group_keys: A list of ways to group the results. Valid values are
                "room_id" and "sender".
            event_context: True to include contextual events around results.
            before_limit:
                The number of events before a result to include as context.

                Only used if event_context is True.
            after_limit:
                The number of events after a result to include as context.

                Only used if event_context is True.
            include_profile: True if historical profile information should be
                included in the event context.

                Only used if event_context is True.

        Returns:
            dict to be returned to the client with results of search
        """
        search_filter = Filter(self.hs, filter_dict)

        # TODO: Search through left rooms too
        rooms = await self.store.get_rooms_for_local_user_where_membership_is(
            user.to_string(),
            membership_list=[Membership.JOIN],
            # membership_list=[Membership.JOIN, Membership.LEAVE, Membership.Ban],
        )
        room_ids = {r.room_id for r in rooms}

        # If doing a subset of all rooms seearch, check if any of the rooms
        # are from an upgraded room, and search their contents as well
        if search_filter.rooms:
            historical_room_ids: List[str] = []
            for room_id in search_filter.rooms:
                # Add any previous rooms to the search if they exist
                ids = await self.get_old_rooms_from_upgraded_room(room_id)
                historical_room_ids += ids

            # Prevent any historical events from being filtered
            search_filter = search_filter.with_room_ids(historical_room_ids)

        room_ids = search_filter.filter_rooms(room_ids)

        if batch_group == "room_id":
            room_ids.intersection_update({batch_group_key})

        if not room_ids:
            return {
                "search_categories": {
                    "room_events": {
                        "results": [],
                        "count": 0,
                        "highlights": []
                    }
                }
            }

        sender_group: Optional[Dict[str, JsonDict]]

        if order_by == "rank":
            search_result, sender_group = await self._search_by_rank(
                user, room_ids, search_term, keys, search_filter)
            # Unused return values for rank search.
            global_next_batch = None
        elif order_by == "recent":
            search_result, global_next_batch = await self._search_by_recent(
                user,
                room_ids,
                search_term,
                keys,
                search_filter,
                batch_group,
                batch_group_key,
                batch_token,
            )
            # Unused return values for recent search.
            sender_group = None
        else:
            # We should never get here due to the guard earlier.
            raise NotImplementedError()

        logger.info("Found %d events to return",
                    len(search_result.allowed_events))

        # If client has asked for "context" for each event (i.e. some surrounding
        # events and state), fetch that
        if event_context is not None:
            # Note that before and after limit must be set in this case.
            assert before_limit is not None
            assert after_limit is not None

            contexts = await self._calculate_event_contexts(
                user,
                search_result.allowed_events,
                before_limit,
                after_limit,
                include_profile,
            )
        else:
            contexts = {}

        # TODO: Add a limit

        state_results = {}
        if include_state:
            for room_id in {e.room_id for e in search_result.allowed_events}:
                state = await self._storage_controllers.state.get_current_state(
                    room_id)
                state_results[room_id] = list(state.values())

        aggregations = await self._relations_handler.get_bundled_aggregations(
            # Generate an iterable of EventBase for all the events that will be
            # returned, including contextual events.
            itertools.chain(
                # The events_before and events_after for each context.
                itertools.chain.from_iterable(
                    itertools.chain(context["events_before"],
                                    context["events_after"])
                    for context in contexts.values()),
                # The returned events.
                search_result.allowed_events,
            ),
            user.to_string(),
        )

        # We're now about to serialize the events. We should not make any
        # blocking calls after this. Otherwise, the 'age' will be wrong.

        time_now = self.clock.time_msec()

        for context in contexts.values():
            context["events_before"] = self._event_serializer.serialize_events(
                context["events_before"],
                time_now,
                bundle_aggregations=aggregations)
            context["events_after"] = self._event_serializer.serialize_events(
                context["events_after"],
                time_now,
                bundle_aggregations=aggregations)

        results = [{
            "rank":
            search_result.rank_map[e.event_id],
            "result":
            self._event_serializer.serialize_event(
                e, time_now, bundle_aggregations=aggregations),
            "context":
            contexts.get(e.event_id, {}),
        } for e in search_result.allowed_events]

        rooms_cat_res: JsonDict = {
            "results": results,
            "count": search_result.count,
            "highlights": list(search_result.highlights),
        }

        if state_results:
            rooms_cat_res["state"] = {
                room_id: self._event_serializer.serialize_events(
                    state_events, time_now)
                for room_id, state_events in state_results.items()
            }

        if search_result.room_groups and "room_id" in group_keys:
            rooms_cat_res.setdefault("groups",
                                     {})["room_id"] = search_result.room_groups

        if sender_group and "sender" in group_keys:
            rooms_cat_res.setdefault("groups", {})["sender"] = sender_group

        if global_next_batch:
            rooms_cat_res["next_batch"] = global_next_batch

        return {"search_categories": {"room_events": rooms_cat_res}}
コード例 #13
0
 def test_definition_not_types_works_with_literals(self):
     definition = {"not_types": ["m.room.message", "org.matrix.foo.bar"]}
     event = MockEvent(sender="@foo:bar",
                       type="m.room.message",
                       room_id="!foo:bar")
     self.assertFalse(Filter(self.hs, definition)._check(event))
コード例 #14
0
ファイル: sync.py プロジェクト: casics/extractor
    def on_GET(self, request):
        user, client = yield self.auth.get_user_by_req(request)

        timeout = self.parse_integer(request, "timeout", default=0)
        limit = self.parse_integer(request, "limit", required=True)
        gap = self.parse_boolean(request, "gap", default=True)
        sort = self.parse_string(
            request, "sort", default="timeline,asc",
            allowed_values=self.ALLOWED_SORT
        )
        since = self.parse_string(request, "since")
        set_presence = self.parse_string(
            request, "set_presence", default="online",
            allowed_values=self.ALLOWED_PRESENCE
        )
        backfill = self.parse_boolean(request, "backfill", default=False)
        filter_id = self.parse_string(request, "filter", default=None)

        logger.info(
            "/sync: user=%r, timeout=%r, limit=%r, gap=%r, sort=%r, since=%r,"
            " set_presence=%r, backfill=%r, filter_id=%r" % (
                user, timeout, limit, gap, sort, since, set_presence,
                backfill, filter_id
            )
        )

        # TODO(mjark): Load filter and apply overrides.
        try:
            filter = yield self.filtering.get_user_filter(
                user.localpart, filter_id
            )
        except:
            filter = Filter({})
        # filter = filter.apply_overrides(http_request)
        # if filter.matches(event):
        #   # stuff

        sync_config = SyncConfig(
            user=user,
            client_info=client,
            gap=gap,
            limit=limit,
            sort=sort,
            backfill=backfill,
            filter=filter,
        )

        if since is not None:
            since_token = StreamToken.from_string(since)
        else:
            since_token = None

        sync_result = yield self.sync_handler.wait_for_sync_for_user(
            sync_config, since_token=since_token, timeout=timeout
        )

        time_now = self.clock.time_msec()

        response_content = {
            "public_user_data": self.encode_user_data(
                sync_result.public_user_data, filter, time_now
            ),
            "private_user_data": self.encode_user_data(
                sync_result.private_user_data, filter, time_now
            ),
            "rooms": self.encode_rooms(
                sync_result.rooms, filter, time_now, client.token_id
            ),
            "next_batch": sync_result.next_batch.to_string(),
        }

        defer.returnValue((200, response_content))
コード例 #15
0
ファイル: search.py プロジェクト: matrix-org/synapse
    def search(self, user, content, batch=None):
        """Performs a full text search for a user.

        Args:
            user (UserID)
            content (dict): Search parameters
            batch (str): The next_batch parameter. Used for pagination.

        Returns:
            dict to be returned to the client with results of search
        """

        if not self.hs.config.enable_search:
            raise SynapseError(400, "Search is disabled on this homeserver")

        batch_group = None
        batch_group_key = None
        batch_token = None
        if batch:
            try:
                b = decode_base64(batch).decode('ascii')
                batch_group, batch_group_key, batch_token = b.split("\n")

                assert batch_group is not None
                assert batch_group_key is not None
                assert batch_token is not None
            except Exception:
                raise SynapseError(400, "Invalid batch")

        logger.info(
            "Search batch properties: %r, %r, %r",
            batch_group, batch_group_key, batch_token,
        )

        logger.info("Search content: %s", content)

        try:
            room_cat = content["search_categories"]["room_events"]

            # The actual thing to query in FTS
            search_term = room_cat["search_term"]

            # Which "keys" to search over in FTS query
            keys = room_cat.get("keys", [
                "content.body", "content.name", "content.topic",
            ])

            # Filter to apply to results
            filter_dict = room_cat.get("filter", {})

            # What to order results by (impacts whether pagination can be doen)
            order_by = room_cat.get("order_by", "rank")

            # Return the current state of the rooms?
            include_state = room_cat.get("include_state", False)

            # Include context around each event?
            event_context = room_cat.get(
                "event_context", None
            )

            # Group results together? May allow clients to paginate within a
            # group
            group_by = room_cat.get("groupings", {}).get("group_by", {})
            group_keys = [g["key"] for g in group_by]

            if event_context is not None:
                before_limit = int(event_context.get(
                    "before_limit", 5
                ))
                after_limit = int(event_context.get(
                    "after_limit", 5
                ))

                # Return the historic display name and avatar for the senders
                # of the events?
                include_profile = bool(event_context.get("include_profile", False))
        except KeyError:
            raise SynapseError(400, "Invalid search query")

        if order_by not in ("rank", "recent"):
            raise SynapseError(400, "Invalid order by: %r" % (order_by,))

        if set(group_keys) - {"room_id", "sender"}:
            raise SynapseError(
                400,
                "Invalid group by keys: %r" % (set(group_keys) - {"room_id", "sender"},)
            )

        search_filter = Filter(filter_dict)

        # TODO: Search through left rooms too
        rooms = yield self.store.get_rooms_for_user_where_membership_is(
            user.to_string(),
            membership_list=[Membership.JOIN],
            # membership_list=[Membership.JOIN, Membership.LEAVE, Membership.Ban],
        )
        room_ids = set(r.room_id for r in rooms)

        # If doing a subset of all rooms seearch, check if any of the rooms
        # are from an upgraded room, and search their contents as well
        if search_filter.rooms:
            historical_room_ids = []
            for room_id in search_filter.rooms:
                # Add any previous rooms to the search if they exist
                ids = yield self.get_old_rooms_from_upgraded_room(room_id)
                historical_room_ids += ids

            # Prevent any historical events from being filtered
            search_filter = search_filter.with_room_ids(historical_room_ids)

        room_ids = search_filter.filter_rooms(room_ids)

        if batch_group == "room_id":
            room_ids.intersection_update({batch_group_key})

        if not room_ids:
            defer.returnValue({
                "search_categories": {
                    "room_events": {
                        "results": [],
                        "count": 0,
                        "highlights": [],
                    }
                }
            })

        rank_map = {}  # event_id -> rank of event
        allowed_events = []
        room_groups = {}  # Holds result of grouping by room, if applicable
        sender_group = {}  # Holds result of grouping by sender, if applicable

        # Holds the next_batch for the entire result set if one of those exists
        global_next_batch = None

        highlights = set()

        count = None

        if order_by == "rank":
            search_result = yield self.store.search_msgs(
                room_ids, search_term, keys
            )

            count = search_result["count"]

            if search_result["highlights"]:
                highlights.update(search_result["highlights"])

            results = search_result["results"]

            results_map = {r["event"].event_id: r for r in results}

            rank_map.update({r["event"].event_id: r["rank"] for r in results})

            filtered_events = search_filter.filter([r["event"] for r in results])

            events = yield filter_events_for_client(
                self.store, user.to_string(), filtered_events
            )

            events.sort(key=lambda e: -rank_map[e.event_id])
            allowed_events = events[:search_filter.limit()]

            for e in allowed_events:
                rm = room_groups.setdefault(e.room_id, {
                    "results": [],
                    "order": rank_map[e.event_id],
                })
                rm["results"].append(e.event_id)

                s = sender_group.setdefault(e.sender, {
                    "results": [],
                    "order": rank_map[e.event_id],
                })
                s["results"].append(e.event_id)

        elif order_by == "recent":
            room_events = []
            i = 0

            pagination_token = batch_token

            # We keep looping and we keep filtering until we reach the limit
            # or we run out of things.
            # But only go around 5 times since otherwise synapse will be sad.
            while len(room_events) < search_filter.limit() and i < 5:
                i += 1
                search_result = yield self.store.search_rooms(
                    room_ids, search_term, keys, search_filter.limit() * 2,
                    pagination_token=pagination_token,
                )

                if search_result["highlights"]:
                    highlights.update(search_result["highlights"])

                count = search_result["count"]

                results = search_result["results"]

                results_map = {r["event"].event_id: r for r in results}

                rank_map.update({r["event"].event_id: r["rank"] for r in results})

                filtered_events = search_filter.filter([
                    r["event"] for r in results
                ])

                events = yield filter_events_for_client(
                    self.store, user.to_string(), filtered_events
                )

                room_events.extend(events)
                room_events = room_events[:search_filter.limit()]

                if len(results) < search_filter.limit() * 2:
                    pagination_token = None
                    break
                else:
                    pagination_token = results[-1]["pagination_token"]

            for event in room_events:
                group = room_groups.setdefault(event.room_id, {
                    "results": [],
                })
                group["results"].append(event.event_id)

            if room_events and len(room_events) >= search_filter.limit():
                last_event_id = room_events[-1].event_id
                pagination_token = results_map[last_event_id]["pagination_token"]

                # We want to respect the given batch group and group keys so
                # that if people blindly use the top level `next_batch` token
                # it returns more from the same group (if applicable) rather
                # than reverting to searching all results again.
                if batch_group and batch_group_key:
                    global_next_batch = encode_base64(("%s\n%s\n%s" % (
                        batch_group, batch_group_key, pagination_token
                    )).encode('ascii'))
                else:
                    global_next_batch = encode_base64(("%s\n%s\n%s" % (
                        "all", "", pagination_token
                    )).encode('ascii'))

                for room_id, group in room_groups.items():
                    group["next_batch"] = encode_base64(("%s\n%s\n%s" % (
                        "room_id", room_id, pagination_token
                    )).encode('ascii'))

            allowed_events.extend(room_events)

        else:
            # We should never get here due to the guard earlier.
            raise NotImplementedError()

        logger.info("Found %d events to return", len(allowed_events))

        # If client has asked for "context" for each event (i.e. some surrounding
        # events and state), fetch that
        if event_context is not None:
            now_token = yield self.hs.get_event_sources().get_current_token()

            contexts = {}
            for event in allowed_events:
                res = yield self.store.get_events_around(
                    event.room_id, event.event_id, before_limit, after_limit,
                )

                logger.info(
                    "Context for search returned %d and %d events",
                    len(res["events_before"]), len(res["events_after"]),
                )

                res["events_before"] = yield filter_events_for_client(
                    self.store, user.to_string(), res["events_before"]
                )

                res["events_after"] = yield filter_events_for_client(
                    self.store, user.to_string(), res["events_after"]
                )

                res["start"] = now_token.copy_and_replace(
                    "room_key", res["start"]
                ).to_string()

                res["end"] = now_token.copy_and_replace(
                    "room_key", res["end"]
                ).to_string()

                if include_profile:
                    senders = set(
                        ev.sender
                        for ev in itertools.chain(
                            res["events_before"], [event], res["events_after"]
                        )
                    )

                    if res["events_after"]:
                        last_event_id = res["events_after"][-1].event_id
                    else:
                        last_event_id = event.event_id

                    state_filter = StateFilter.from_types(
                        [(EventTypes.Member, sender) for sender in senders]
                    )

                    state = yield self.store.get_state_for_event(
                        last_event_id, state_filter
                    )

                    res["profile_info"] = {
                        s.state_key: {
                            "displayname": s.content.get("displayname", None),
                            "avatar_url": s.content.get("avatar_url", None),
                        }
                        for s in state.values()
                        if s.type == EventTypes.Member and s.state_key in senders
                    }

                contexts[event.event_id] = res
        else:
            contexts = {}

        # TODO: Add a limit

        time_now = self.clock.time_msec()

        for context in contexts.values():
            context["events_before"] = (
                yield self._event_serializer.serialize_events(
                    context["events_before"], time_now,
                )
            )
            context["events_after"] = (
                yield self._event_serializer.serialize_events(
                    context["events_after"], time_now,
                )
            )

        state_results = {}
        if include_state:
            rooms = set(e.room_id for e in allowed_events)
            for room_id in rooms:
                state = yield self.state_handler.get_current_state(room_id)
                state_results[room_id] = list(state.values())

            state_results.values()

        # We're now about to serialize the events. We should not make any
        # blocking calls after this. Otherwise the 'age' will be wrong

        results = []
        for e in allowed_events:
            results.append({
                "rank": rank_map[e.event_id],
                "result": (yield self._event_serializer.serialize_event(e, time_now)),
                "context": contexts.get(e.event_id, {}),
            })

        rooms_cat_res = {
            "results": results,
            "count": count,
            "highlights": list(highlights),
        }

        if state_results:
            s = {}
            for room_id, state in state_results.items():
                s[room_id] = yield self._event_serializer.serialize_events(
                    state, time_now,
                )

            rooms_cat_res["state"] = s

        if room_groups and "room_id" in group_keys:
            rooms_cat_res.setdefault("groups", {})["room_id"] = room_groups

        if sender_group and "sender" in group_keys:
            rooms_cat_res.setdefault("groups", {})["sender"] = sender_group

        if global_next_batch:
            rooms_cat_res["next_batch"] = global_next_batch

        defer.returnValue({
            "search_categories": {
                "room_events": rooms_cat_res
            }
        })
コード例 #16
0
    async def search(self,
                     user: UserID,
                     content: JsonDict,
                     batch: Optional[str] = None) -> JsonDict:
        """Performs a full text search for a user.

        Args:
            user
            content: Search parameters
            batch: The next_batch parameter. Used for pagination.

        Returns:
            dict to be returned to the client with results of search
        """

        if not self.hs.config.server.enable_search:
            raise SynapseError(400, "Search is disabled on this homeserver")

        batch_group = None
        batch_group_key = None
        batch_token = None
        if batch:
            try:
                b = decode_base64(batch).decode("ascii")
                batch_group, batch_group_key, batch_token = b.split("\n")

                assert batch_group is not None
                assert batch_group_key is not None
                assert batch_token is not None
            except Exception:
                raise SynapseError(400, "Invalid batch")

        logger.info(
            "Search batch properties: %r, %r, %r",
            batch_group,
            batch_group_key,
            batch_token,
        )

        logger.info("Search content: %s", content)

        try:
            room_cat = content["search_categories"]["room_events"]

            # The actual thing to query in FTS
            search_term = room_cat["search_term"]

            # Which "keys" to search over in FTS query
            keys = room_cat.get(
                "keys", ["content.body", "content.name", "content.topic"])

            # Filter to apply to results
            filter_dict = room_cat.get("filter", {})

            # What to order results by (impacts whether pagination can be done)
            order_by = room_cat.get("order_by", "rank")

            # Return the current state of the rooms?
            include_state = room_cat.get("include_state", False)

            # Include context around each event?
            event_context = room_cat.get("event_context", None)

            # Group results together? May allow clients to paginate within a
            # group
            group_by = room_cat.get("groupings", {}).get("group_by", {})
            group_keys = [g["key"] for g in group_by]

            if event_context is not None:
                before_limit = int(event_context.get("before_limit", 5))
                after_limit = int(event_context.get("after_limit", 5))

                # Return the historic display name and avatar for the senders
                # of the events?
                include_profile = bool(
                    event_context.get("include_profile", False))
        except KeyError:
            raise SynapseError(400, "Invalid search query")

        if order_by not in ("rank", "recent"):
            raise SynapseError(400, "Invalid order by: %r" % (order_by, ))

        if set(group_keys) - {"room_id", "sender"}:
            raise SynapseError(
                400,
                "Invalid group by keys: %r" %
                (set(group_keys) - {"room_id", "sender"}, ),
            )

        search_filter = Filter(self.hs, filter_dict)

        # TODO: Search through left rooms too
        rooms = await self.store.get_rooms_for_local_user_where_membership_is(
            user.to_string(),
            membership_list=[Membership.JOIN],
            # membership_list=[Membership.JOIN, Membership.LEAVE, Membership.Ban],
        )
        room_ids = {r.room_id for r in rooms}

        # If doing a subset of all rooms seearch, check if any of the rooms
        # are from an upgraded room, and search their contents as well
        if search_filter.rooms:
            historical_room_ids: List[str] = []
            for room_id in search_filter.rooms:
                # Add any previous rooms to the search if they exist
                ids = await self.get_old_rooms_from_upgraded_room(room_id)
                historical_room_ids += ids

            # Prevent any historical events from being filtered
            search_filter = search_filter.with_room_ids(historical_room_ids)

        room_ids = search_filter.filter_rooms(room_ids)

        if batch_group == "room_id":
            room_ids.intersection_update({batch_group_key})

        if not room_ids:
            return {
                "search_categories": {
                    "room_events": {
                        "results": [],
                        "count": 0,
                        "highlights": []
                    }
                }
            }

        rank_map = {}  # event_id -> rank of event
        allowed_events = []
        # Holds result of grouping by room, if applicable
        room_groups: Dict[str, JsonDict] = {}
        # Holds result of grouping by sender, if applicable
        sender_group: Dict[str, JsonDict] = {}

        # Holds the next_batch for the entire result set if one of those exists
        global_next_batch = None

        highlights = set()

        count = None

        if order_by == "rank":
            search_result = await self.store.search_msgs(
                room_ids, search_term, keys)

            count = search_result["count"]

            if search_result["highlights"]:
                highlights.update(search_result["highlights"])

            results = search_result["results"]

            results_map = {r["event"].event_id: r for r in results}

            rank_map.update({r["event"].event_id: r["rank"] for r in results})

            filtered_events = await search_filter.filter(
                [r["event"] for r in results])

            events = await filter_events_for_client(self.storage,
                                                    user.to_string(),
                                                    filtered_events)

            events.sort(key=lambda e: -rank_map[e.event_id])
            allowed_events = events[:search_filter.limit]

            for e in allowed_events:
                rm = room_groups.setdefault(e.room_id, {
                    "results": [],
                    "order": rank_map[e.event_id]
                })
                rm["results"].append(e.event_id)

                s = sender_group.setdefault(e.sender, {
                    "results": [],
                    "order": rank_map[e.event_id]
                })
                s["results"].append(e.event_id)

        elif order_by == "recent":
            room_events: List[EventBase] = []
            i = 0

            pagination_token = batch_token

            # We keep looping and we keep filtering until we reach the limit
            # or we run out of things.
            # But only go around 5 times since otherwise synapse will be sad.
            while len(room_events) < search_filter.limit and i < 5:
                i += 1
                search_result = await self.store.search_rooms(
                    room_ids,
                    search_term,
                    keys,
                    search_filter.limit * 2,
                    pagination_token=pagination_token,
                )

                if search_result["highlights"]:
                    highlights.update(search_result["highlights"])

                count = search_result["count"]

                results = search_result["results"]

                results_map = {r["event"].event_id: r for r in results}

                rank_map.update(
                    {r["event"].event_id: r["rank"]
                     for r in results})

                filtered_events = await search_filter.filter(
                    [r["event"] for r in results])

                events = await filter_events_for_client(
                    self.storage, user.to_string(), filtered_events)

                room_events.extend(events)
                room_events = room_events[:search_filter.limit]

                if len(results) < search_filter.limit * 2:
                    pagination_token = None
                    break
                else:
                    pagination_token = results[-1]["pagination_token"]

            for event in room_events:
                group = room_groups.setdefault(event.room_id, {"results": []})
                group["results"].append(event.event_id)

            if room_events and len(room_events) >= search_filter.limit:
                last_event_id = room_events[-1].event_id
                pagination_token = results_map[last_event_id][
                    "pagination_token"]

                # We want to respect the given batch group and group keys so
                # that if people blindly use the top level `next_batch` token
                # it returns more from the same group (if applicable) rather
                # than reverting to searching all results again.
                if batch_group and batch_group_key:
                    global_next_batch = encode_base64(
                        ("%s\n%s\n%s" % (batch_group, batch_group_key,
                                         pagination_token)).encode("ascii"))
                else:
                    global_next_batch = encode_base64(
                        ("%s\n%s\n%s" %
                         ("all", "", pagination_token)).encode("ascii"))

                for room_id, group in room_groups.items():
                    group["next_batch"] = encode_base64(
                        ("%s\n%s\n%s" % ("room_id", room_id,
                                         pagination_token)).encode("ascii"))

            allowed_events.extend(room_events)

        else:
            # We should never get here due to the guard earlier.
            raise NotImplementedError()

        logger.info("Found %d events to return", len(allowed_events))

        # If client has asked for "context" for each event (i.e. some surrounding
        # events and state), fetch that
        if event_context is not None:
            now_token = self.hs.get_event_sources().get_current_token()

            contexts = {}
            for event in allowed_events:
                res = await self.store.get_events_around(
                    event.room_id, event.event_id, before_limit, after_limit)

                logger.info(
                    "Context for search returned %d and %d events",
                    len(res.events_before),
                    len(res.events_after),
                )

                events_before = await filter_events_for_client(
                    self.storage, user.to_string(), res.events_before)

                events_after = await filter_events_for_client(
                    self.storage, user.to_string(), res.events_after)

                context = {
                    "events_before":
                    events_before,
                    "events_after":
                    events_after,
                    "start":
                    await
                    now_token.copy_and_replace("room_key",
                                               res.start).to_string(self.store
                                                                    ),
                    "end":
                    await
                    now_token.copy_and_replace("room_key",
                                               res.end).to_string(self.store),
                }

                if include_profile:
                    senders = {
                        ev.sender
                        for ev in itertools.chain(events_before, [event],
                                                  events_after)
                    }

                    if events_after:
                        last_event_id = events_after[-1].event_id
                    else:
                        last_event_id = event.event_id

                    state_filter = StateFilter.from_types([
                        (EventTypes.Member, sender) for sender in senders
                    ])

                    state = await self.state_store.get_state_for_event(
                        last_event_id, state_filter)

                    context["profile_info"] = {
                        s.state_key: {
                            "displayname": s.content.get("displayname", None),
                            "avatar_url": s.content.get("avatar_url", None),
                        }
                        for s in state.values() if s.type == EventTypes.Member
                        and s.state_key in senders
                    }

                contexts[event.event_id] = context
        else:
            contexts = {}

        # TODO: Add a limit

        time_now = self.clock.time_msec()

        for context in contexts.values():
            context["events_before"] = self._event_serializer.serialize_events(
                context["events_before"],
                time_now  # type: ignore[arg-type]
            )
            context["events_after"] = self._event_serializer.serialize_events(
                context["events_after"],
                time_now  # type: ignore[arg-type]
            )

        state_results = {}
        if include_state:
            for room_id in {e.room_id for e in allowed_events}:
                state = await self.state_handler.get_current_state(room_id)
                state_results[room_id] = list(state.values())

        # We're now about to serialize the events. We should not make any
        # blocking calls after this. Otherwise the 'age' will be wrong

        results = []
        for e in allowed_events:
            results.append({
                "rank":
                rank_map[e.event_id],
                "result":
                self._event_serializer.serialize_event(e, time_now),
                "context":
                contexts.get(e.event_id, {}),
            })

        rooms_cat_res = {
            "results": results,
            "count": count,
            "highlights": list(highlights),
        }

        if state_results:
            s = {}
            for room_id, state_events in state_results.items():
                s[room_id] = self._event_serializer.serialize_events(
                    state_events, time_now)

            rooms_cat_res["state"] = s

        if room_groups and "room_id" in group_keys:
            rooms_cat_res.setdefault("groups", {})["room_id"] = room_groups

        if sender_group and "sender" in group_keys:
            rooms_cat_res.setdefault("groups", {})["sender"] = sender_group

        if global_next_batch:
            rooms_cat_res["next_batch"] = global_next_batch

        return {"search_categories": {"room_events": rooms_cat_res}}