def get_public_rooms(self, destinations): results_by_server = {} @defer.inlineCallbacks def _get_result(s): if s == self.server_name: defer.returnValue() try: result = yield self.transport_layer.get_public_rooms(s) results_by_server[s] = result except: logger.exception("Error getting room list from server %r", s) yield concurrently_execute(_get_result, destinations, 3) defer.returnValue(results_by_server)
def send_notification_mail(self, app_id, user_id, email_address, push_actions, reason): try: from_string = self.hs.config.email_notif_from % { "app": self.app_name } except TypeError: from_string = self.hs.config.email_notif_from raw_from = email.utils.parseaddr(from_string)[1] raw_to = email.utils.parseaddr(email_address)[1] if raw_to == '': raise RuntimeError("Invalid 'to' address") rooms_in_order = deduped_ordered_list( [pa['room_id'] for pa in push_actions]) notif_events = yield self.store.get_events( [pa['event_id'] for pa in push_actions]) notifs_by_room = {} for pa in push_actions: notifs_by_room.setdefault(pa["room_id"], []).append(pa) # collect the current state for all the rooms in which we have # notifications state_by_room = {} try: user_display_name = yield self.store.get_profile_displayname( UserID.from_string(user_id).localpart) if user_display_name is None: user_display_name = user_id except StoreError: user_display_name = user_id @defer.inlineCallbacks def _fetch_room_state(room_id): room_state = yield self.store.get_current_state_ids(room_id) state_by_room[room_id] = room_state # Run at most 3 of these at once: sync does 10 at a time but email # notifs are much less realtime than sync so we can afford to wait a bit. yield concurrently_execute(_fetch_room_state, rooms_in_order, 3) # actually sort our so-called rooms_in_order list, most recent room first rooms_in_order.sort( key=lambda r: -(notifs_by_room[r][-1]['received_ts'] or 0)) rooms = [] for r in rooms_in_order: roomvars = yield self.get_room_vars(r, user_id, notifs_by_room[r], notif_events, state_by_room[r]) rooms.append(roomvars) reason['room_name'] = yield calculate_room_name( self.store, state_by_room[reason['room_id']], user_id, fallback_to_members=True) summary_text = yield self.make_summary_text(notifs_by_room, state_by_room, notif_events, user_id, reason) template_vars = { "user_display_name": user_display_name, "unsubscribe_link": self.make_unsubscribe_link(user_id, app_id, email_address), "summary_text": summary_text, "app_name": self.app_name, "rooms": rooms, "reason": reason, } html_text = self.notif_template_html.render(**template_vars) html_part = MIMEText(html_text, "html", "utf8") plain_text = self.notif_template_text.render(**template_vars) text_part = MIMEText(plain_text, "plain", "utf8") multipart_msg = MIMEMultipart('alternative') multipart_msg['Subject'] = "[%s] %s" % (self.app_name, summary_text) multipart_msg['From'] = from_string multipart_msg['To'] = email_address multipart_msg['Date'] = email.utils.formatdate() multipart_msg['Message-ID'] = email.utils.make_msgid() multipart_msg.attach(text_part) multipart_msg.attach(html_part) logger.info("Sending email push notification to %s" % email_address) # logger.debug(html_text) yield sendmail( self.hs.config.email_smtp_host, raw_from, raw_to, multipart_msg.as_string(), port=self.hs.config.email_smtp_port, requireAuthentication=self.hs.config.email_smtp_user is not None, username=self.hs.config.email_smtp_user, password=self.hs.config.email_smtp_pass, requireTransportSecurity=self.hs.config.require_transport_security)
def _get_public_room_list( self, limit=None, since_token=None, search_filter=None, network_tuple=EMTPY_THIRD_PARTY_ID, ): if since_token and since_token != "END": since_token = RoomListNextBatch.from_token(since_token) else: since_token = None rooms_to_order_value = {} rooms_to_num_joined = {} newly_visible = [] newly_unpublished = [] if since_token: stream_token = since_token.stream_ordering current_public_id = yield self.store.get_current_public_room_stream_id( ) public_room_stream_id = since_token.public_room_stream_id newly_visible, newly_unpublished = yield self.store.get_public_room_changes( public_room_stream_id, current_public_id, network_tuple=network_tuple, ) else: stream_token = yield self.store.get_room_max_stream_ordering() public_room_stream_id = yield self.store.get_current_public_room_stream_id( ) room_ids = yield self.store.get_public_room_ids_at_stream_id( public_room_stream_id, network_tuple=network_tuple, ) # We want to return rooms in a particular order: the number of joined # users. We then arbitrarily use the room_id as a tie breaker. @defer.inlineCallbacks def get_order_for_room(room_id): # Most of the rooms won't have changed between the since token and # now (especially if the since token is "now"). So, we can ask what # the current users are in a room (that will hit a cache) and then # check if the room has changed since the since token. (We have to # do it in that order to avoid races). # If things have changed then fall back to getting the current state # at the since token. joined_users = yield self.store.get_users_in_room(room_id) if self.store.has_room_changed_since(room_id, stream_token): latest_event_ids = yield self.store.get_forward_extremeties_for_room( room_id, stream_token) if not latest_event_ids: return joined_users = yield self.state_handler.get_current_user_in_room( room_id, latest_event_ids, ) num_joined_users = len(joined_users) rooms_to_num_joined[room_id] = num_joined_users if num_joined_users == 0: return # We want larger rooms to be first, hence negating num_joined_users rooms_to_order_value[room_id] = (-num_joined_users, room_id) yield concurrently_execute(get_order_for_room, room_ids, 10) sorted_entries = sorted(rooms_to_order_value.items(), key=lambda e: e[1]) sorted_rooms = [room_id for room_id, _ in sorted_entries] # `sorted_rooms` should now be a list of all public room ids that is # stable across pagination. Therefore, we can use indices into this # list as our pagination tokens. # Filter out rooms that we don't want to return rooms_to_scan = [ r for r in sorted_rooms if r not in newly_unpublished and rooms_to_num_joined[room_id] > 0 ] total_room_count = len(rooms_to_scan) if since_token: # Filter out rooms we've already returned previously # `since_token.current_limit` is the index of the last room we # sent down, so we exclude it and everything before/after it. if since_token.direction_is_forward: rooms_to_scan = rooms_to_scan[since_token.current_limit + 1:] else: rooms_to_scan = rooms_to_scan[:since_token.current_limit] rooms_to_scan.reverse() # Actually generate the entries. _append_room_entry_to_chunk will append to # chunk but will stop if len(chunk) > limit chunk = [] if limit and not search_filter: step = limit + 1 for i in xrange(0, len(rooms_to_scan), step): # We iterate here because the vast majority of cases we'll stop # at first iteration, but occaisonally _append_room_entry_to_chunk # won't append to the chunk and so we need to loop again. # We don't want to scan over the entire range either as that # would potentially waste a lot of work. yield concurrently_execute( lambda r: self._append_room_entry_to_chunk( r, rooms_to_num_joined[r], chunk, limit, search_filter ), rooms_to_scan[i:i + step], 10) if len(chunk) >= limit + 1: break else: yield concurrently_execute( lambda r: self._append_room_entry_to_chunk( r, rooms_to_num_joined[r], chunk, limit, search_filter), rooms_to_scan, 5) chunk.sort(key=lambda e: (-e["num_joined_members"], e["room_id"])) # Work out the new limit of the batch for pagination, or None if we # know there are no more results that would be returned. # i.e., [since_token.current_limit..new_limit] is the batch of rooms # we've returned (or the reverse if we paginated backwards) # We tried to pull out limit + 1 rooms above, so if we have <= limit # then we know there are no more results to return new_limit = None if chunk and (not limit or len(chunk) > limit): if not since_token or since_token.direction_is_forward: if limit: chunk = chunk[:limit] last_room_id = chunk[-1]["room_id"] else: if limit: chunk = chunk[-limit:] last_room_id = chunk[0]["room_id"] new_limit = sorted_rooms.index(last_room_id) results = { "chunk": chunk, "total_room_count_estimate": total_room_count, } if since_token: results["new_rooms"] = bool(newly_visible) if not since_token or since_token.direction_is_forward: if new_limit is not None: results["next_batch"] = RoomListNextBatch( stream_ordering=stream_token, public_room_stream_id=public_room_stream_id, current_limit=new_limit, direction_is_forward=True, ).to_token() if since_token: results["prev_batch"] = since_token.copy_and_replace( direction_is_forward=False, current_limit=since_token.current_limit + 1, ).to_token() else: if new_limit is not None: results["prev_batch"] = RoomListNextBatch( stream_ordering=stream_token, public_room_stream_id=public_room_stream_id, current_limit=new_limit, direction_is_forward=False, ).to_token() if since_token: results["next_batch"] = since_token.copy_and_replace( direction_is_forward=True, current_limit=since_token.current_limit - 1, ).to_token() defer.returnValue(results)
def _generate_sync_entry_for_rooms(self, sync_result_builder, account_data_by_room): """Generates the rooms portion of the sync response. Populates the `sync_result_builder` with the result. Args: sync_result_builder(SyncResultBuilder) account_data_by_room(dict): Dictionary of per room account data Returns: Deferred(tuple): Returns a 2-tuple of `(newly_joined_rooms, newly_joined_users)` """ user_id = sync_result_builder.sync_config.user.to_string() now_token, ephemeral_by_room = yield self.ephemeral_by_room( sync_result_builder.sync_config, now_token=sync_result_builder.now_token, since_token=sync_result_builder.since_token, ) sync_result_builder.now_token = now_token ignored_account_data = yield self.store.get_global_account_data_by_type_for_user( "m.ignored_user_list", user_id=user_id, ) if ignored_account_data: ignored_users = ignored_account_data.get("ignored_users", {}).keys() else: ignored_users = frozenset() if sync_result_builder.since_token: res = yield self._get_rooms_changed(sync_result_builder, ignored_users) room_entries, invited, newly_joined_rooms = res tags_by_room = yield self.store.get_updated_tags( user_id, sync_result_builder.since_token.account_data_key, ) else: res = yield self._get_all_rooms(sync_result_builder, ignored_users) room_entries, invited, newly_joined_rooms = res tags_by_room = yield self.store.get_tags_for_user(user_id) def handle_room_entries(room_entry): return self._generate_room_entry( sync_result_builder, ignored_users, room_entry, ephemeral=ephemeral_by_room.get(room_entry.room_id, []), tags=tags_by_room.get(room_entry.room_id), account_data=account_data_by_room.get(room_entry.room_id, {}), always_include=sync_result_builder.full_state, ) yield concurrently_execute(handle_room_entries, room_entries, 10) sync_result_builder.invited.extend(invited) # Now we want to get any newly joined users newly_joined_users = set() if sync_result_builder.since_token: for joined_sync in sync_result_builder.joined: it = itertools.chain( joined_sync.timeline.events, joined_sync.state.values() ) for event in it: if event.type == EventTypes.Member: if event.membership == Membership.JOIN: newly_joined_users.add(event.state_key) defer.returnValue((newly_joined_rooms, newly_joined_users))
def _generate_sync_entry_for_rooms(self, sync_result_builder, account_data_by_room): """Generates the rooms portion of the sync response. Populates the `sync_result_builder` with the result. Args: sync_result_builder(SyncResultBuilder) account_data_by_room(dict): Dictionary of per room account data Returns: Deferred(tuple): Returns a 2-tuple of `(newly_joined_rooms, newly_joined_users)` """ user_id = sync_result_builder.sync_config.user.to_string() now_token, ephemeral_by_room = yield self.ephemeral_by_room( sync_result_builder.sync_config, now_token=sync_result_builder.now_token, since_token=sync_result_builder.since_token, ) sync_result_builder.now_token = now_token ignored_account_data = yield self.store.get_global_account_data_by_type_for_user( "m.ignored_user_list", user_id=user_id, ) if ignored_account_data: ignored_users = ignored_account_data.get("ignored_users", {}).keys() else: ignored_users = frozenset() if sync_result_builder.since_token: res = yield self._get_rooms_changed(sync_result_builder, ignored_users) room_entries, invited, newly_joined_rooms = res tags_by_room = yield self.store.get_updated_tags( user_id, sync_result_builder.since_token.account_data_key, ) else: res = yield self._get_all_rooms(sync_result_builder, ignored_users) room_entries, invited, newly_joined_rooms = res tags_by_room = yield self.store.get_tags_for_user(user_id) def handle_room_entries(room_entry): return self._generate_room_entry( sync_result_builder, ignored_users, room_entry, ephemeral=ephemeral_by_room.get(room_entry.room_id, []), tags=tags_by_room.get(room_entry.room_id), account_data=account_data_by_room.get(room_entry.room_id, {}), always_include=sync_result_builder.full_state, ) yield concurrently_execute(handle_room_entries, room_entries, 10) sync_result_builder.invited.extend(invited) # Now we want to get any newly joined users newly_joined_users = set() if sync_result_builder.since_token: for joined_sync in sync_result_builder.joined: it = itertools.chain(joined_sync.timeline.events, joined_sync.state.values()) for event in it: if event.type == EventTypes.Member: if event.membership == Membership.JOIN: newly_joined_users.add(event.state_key) defer.returnValue((newly_joined_rooms, newly_joined_users))
def send_notification_mail(self, app_id, user_id, email_address, push_actions, reason): try: from_string = self.hs.config.email_notif_from % { "app": self.app_name } except TypeError: from_string = self.hs.config.email_notif_from raw_from = email.utils.parseaddr(from_string)[1] raw_to = email.utils.parseaddr(email_address)[1] if raw_to == '': raise RuntimeError("Invalid 'to' address") rooms_in_order = deduped_ordered_list( [pa['room_id'] for pa in push_actions] ) notif_events = yield self.store.get_events( [pa['event_id'] for pa in push_actions] ) notifs_by_room = {} for pa in push_actions: notifs_by_room.setdefault(pa["room_id"], []).append(pa) # collect the current state for all the rooms in which we have # notifications state_by_room = {} try: user_display_name = yield self.store.get_profile_displayname( UserID.from_string(user_id).localpart ) if user_display_name is None: user_display_name = user_id except StoreError: user_display_name = user_id @defer.inlineCallbacks def _fetch_room_state(room_id): room_state = yield self.state_handler.get_current_state(room_id) state_by_room[room_id] = room_state # Run at most 3 of these at once: sync does 10 at a time but email # notifs are much less realtime than sync so we can afford to wait a bit. yield concurrently_execute(_fetch_room_state, rooms_in_order, 3) # actually sort our so-called rooms_in_order list, most recent room first rooms_in_order.sort( key=lambda r: -(notifs_by_room[r][-1]['received_ts'] or 0) ) rooms = [] for r in rooms_in_order: roomvars = yield self.get_room_vars( r, user_id, notifs_by_room[r], notif_events, state_by_room[r] ) rooms.append(roomvars) reason['room_name'] = calculate_room_name( state_by_room[reason['room_id']], user_id, fallback_to_members=True ) summary_text = self.make_summary_text( notifs_by_room, state_by_room, notif_events, user_id, reason ) template_vars = { "user_display_name": user_display_name, "unsubscribe_link": self.make_unsubscribe_link( user_id, app_id, email_address ), "summary_text": summary_text, "app_name": self.app_name, "rooms": rooms, "reason": reason, } html_text = self.notif_template_html.render(**template_vars) html_part = MIMEText(html_text, "html", "utf8") plain_text = self.notif_template_text.render(**template_vars) text_part = MIMEText(plain_text, "plain", "utf8") multipart_msg = MIMEMultipart('alternative') multipart_msg['Subject'] = "[%s] %s" % (self.app_name, summary_text) multipart_msg['From'] = from_string multipart_msg['To'] = email_address multipart_msg['Date'] = email.utils.formatdate() multipart_msg['Message-ID'] = email.utils.make_msgid() multipart_msg.attach(text_part) multipart_msg.attach(html_part) logger.info("Sending email push notification to %s" % email_address) # logger.debug(html_text) yield sendmail( self.hs.config.email_smtp_host, raw_from, raw_to, multipart_msg.as_string(), port=self.hs.config.email_smtp_port )
def _snapshot_all_rooms(self, user_id=None, pagin_config=None, as_client_event=True, include_archived=False): memberships = [Membership.INVITE, Membership.JOIN] if include_archived: memberships.append(Membership.LEAVE) room_list = yield self.store.get_rooms_for_user_where_membership_is( user_id=user_id, membership_list=memberships) user = UserID.from_string(user_id) rooms_ret = [] now_token = yield self.hs.get_event_sources().get_current_token() presence_stream = self.hs.get_event_sources().sources["presence"] pagination_config = PaginationConfig(from_token=now_token) presence, _ = yield presence_stream.get_pagination_rows( user, pagination_config.get_source_config("presence"), None) receipt_stream = self.hs.get_event_sources().sources["receipt"] receipt, _ = yield receipt_stream.get_pagination_rows( user, pagination_config.get_source_config("receipt"), None) tags_by_room = yield self.store.get_tags_for_user(user_id) account_data, account_data_by_room = ( yield self.store.get_account_data_for_user(user_id)) public_room_ids = yield self.store.get_public_room_ids() limit = pagin_config.limit if limit is None: limit = 10 @defer.inlineCallbacks def handle_room(event): d = { "room_id": event.room_id, "membership": event.membership, "visibility": ("public" if event.room_id in public_room_ids else "private"), } if event.membership == Membership.INVITE: time_now = self.clock.time_msec() d["inviter"] = event.sender invite_event = yield self.store.get_event(event.event_id) d["invite"] = serialize_event(invite_event, time_now, as_client_event) rooms_ret.append(d) if event.membership not in (Membership.JOIN, Membership.LEAVE): return try: if event.membership == Membership.JOIN: room_end_token = now_token.room_key deferred_room_state = self.state_handler.get_current_state( event.room_id) elif event.membership == Membership.LEAVE: room_end_token = "s%d" % (event.stream_ordering, ) deferred_room_state = self.store.get_state_for_events( [event.event_id], None) deferred_room_state.addCallback( lambda states: states[event.event_id]) (messages, token), current_state = yield make_deferred_yieldable( defer.gatherResults([ preserve_fn(self.store.get_recent_events_for_room)( event.room_id, limit=limit, end_token=room_end_token, ), deferred_room_state, ])).addErrback(unwrapFirstError) messages = yield filter_events_for_client( self.store, user_id, messages) start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) time_now = self.clock.time_msec() d["messages"] = { "chunk": [ serialize_event(m, time_now, as_client_event) for m in messages ], "start": start_token.to_string(), "end": end_token.to_string(), } d["state"] = [ serialize_event(c, time_now, as_client_event) for c in current_state.values() ] account_data_events = [] tags = tags_by_room.get(event.room_id) if tags: account_data_events.append({ "type": "m.tag", "content": { "tags": tags }, }) account_data = account_data_by_room.get(event.room_id, {}) for account_data_type, content in account_data.items(): account_data_events.append({ "type": account_data_type, "content": content, }) d["account_data"] = account_data_events except Exception: logger.exception("Failed to get snapshot") yield concurrently_execute(handle_room, room_list, 10) account_data_events = [] for account_data_type, content in account_data.items(): account_data_events.append({ "type": account_data_type, "content": content, }) now = self.clock.time_msec() ret = { "rooms": rooms_ret, "presence": [{ "type": "m.presence", "content": format_user_presence_state(event, now), } for event in presence], "account_data": account_data_events, "receipts": receipt, "end": now_token.to_string(), } defer.returnValue(ret)
def _get_public_room_list(self): room_ids = yield self.store.get_public_room_ids() results = [] @defer.inlineCallbacks def handle_room(room_id): # We pull each bit of state out indvidually to avoid pulling the # full state into memory. Due to how the caching works this should # be fairly quick, even if not originally in the cache. def get_state(etype, state_key): return self.state_handler.get_current_state(room_id, etype, state_key) # Double check that this is actually a public room. join_rules_event = yield get_state(EventTypes.JoinRules, "") if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) if join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) result = {"room_id": room_id} joined_users = yield self.store.get_users_in_room(room_id) if len(joined_users) == 0: return result["num_joined_members"] = len(joined_users) aliases = yield self.store.get_aliases_for_room(room_id) if aliases: result["aliases"] = aliases name_event = yield get_state(EventTypes.Name, "") if name_event: name = name_event.content.get("name", None) if name: result["name"] = name topic_event = yield get_state(EventTypes.Topic, "") if topic_event: topic = topic_event.content.get("topic", None) if topic: result["topic"] = topic canonical_event = yield get_state(EventTypes.CanonicalAlias, "") if canonical_event: canonical_alias = canonical_event.content.get("alias", None) if canonical_alias: result["canonical_alias"] = canonical_alias visibility_event = yield get_state(EventTypes.RoomHistoryVisibility, "") visibility = None if visibility_event: visibility = visibility_event.content.get("history_visibility", None) result["world_readable"] = visibility == "world_readable" guest_event = yield get_state(EventTypes.GuestAccess, "") guest = None if guest_event: guest = guest_event.content.get("guest_access", None) result["guest_can_join"] = guest == "can_join" avatar_event = yield get_state("m.room.avatar", "") if avatar_event: avatar_url = avatar_event.content.get("url", None) if avatar_url: result["avatar_url"] = avatar_url results.append(result) yield concurrently_execute(handle_room, room_ids, 10) # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": results})
def _snapshot_all_rooms(self, user_id=None, pagin_config=None, as_client_event=True, include_archived=False): memberships = [Membership.INVITE, Membership.JOIN] if include_archived: memberships.append(Membership.LEAVE) room_list = yield self.store.get_rooms_for_user_where_membership_is( user_id=user_id, membership_list=memberships ) user = UserID.from_string(user_id) rooms_ret = [] now_token = yield self.hs.get_event_sources().get_current_token() presence_stream = self.hs.get_event_sources().sources["presence"] pagination_config = PaginationConfig(from_token=now_token) presence, _ = yield presence_stream.get_pagination_rows( user, pagination_config.get_source_config("presence"), None ) receipt_stream = self.hs.get_event_sources().sources["receipt"] receipt, _ = yield receipt_stream.get_pagination_rows( user, pagination_config.get_source_config("receipt"), None ) tags_by_room = yield self.store.get_tags_for_user(user_id) account_data, account_data_by_room = ( yield self.store.get_account_data_for_user(user_id) ) public_room_ids = yield self.store.get_public_room_ids() limit = pagin_config.limit if limit is None: limit = 10 @defer.inlineCallbacks def handle_room(event): d = { "room_id": event.room_id, "membership": event.membership, "visibility": ( "public" if event.room_id in public_room_ids else "private" ), } if event.membership == Membership.INVITE: time_now = self.clock.time_msec() d["inviter"] = event.sender invite_event = yield self.store.get_event(event.event_id) d["invite"] = serialize_event(invite_event, time_now, as_client_event) rooms_ret.append(d) if event.membership not in (Membership.JOIN, Membership.LEAVE): return try: if event.membership == Membership.JOIN: room_end_token = now_token.room_key deferred_room_state = self.state_handler.get_current_state( event.room_id ) elif event.membership == Membership.LEAVE: room_end_token = "s%d" % (event.stream_ordering,) deferred_room_state = self.store.get_state_for_events( [event.event_id], None ) deferred_room_state.addCallback( lambda states: states[event.event_id] ) (messages, token), current_state = yield defer.gatherResults( [ self.store.get_recent_events_for_room( event.room_id, limit=limit, end_token=room_end_token, ), deferred_room_state, ] ).addErrback(unwrapFirstError) messages = yield filter_events_for_client( self.store, user_id, messages ) start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) time_now = self.clock.time_msec() d["messages"] = { "chunk": [ serialize_event(m, time_now, as_client_event) for m in messages ], "start": start_token.to_string(), "end": end_token.to_string(), } d["state"] = [ serialize_event(c, time_now, as_client_event) for c in current_state.values() ] account_data_events = [] tags = tags_by_room.get(event.room_id) if tags: account_data_events.append({ "type": "m.tag", "content": {"tags": tags}, }) account_data = account_data_by_room.get(event.room_id, {}) for account_data_type, content in account_data.items(): account_data_events.append({ "type": account_data_type, "content": content, }) d["account_data"] = account_data_events except: logger.exception("Failed to get snapshot") yield concurrently_execute(handle_room, room_list, 10) account_data_events = [] for account_data_type, content in account_data.items(): account_data_events.append({ "type": account_data_type, "content": content, }) ret = { "rooms": rooms_ret, "presence": presence, "account_data": account_data_events, "receipts": receipt, "end": now_token.to_string(), } defer.returnValue(ret)
def _get_public_room_list(self): room_ids = yield self.store.get_public_room_ids() results = [] @defer.inlineCallbacks def handle_room(room_id): current_state = yield self.state_handler.get_current_state(room_id) # Double check that this is actually a public room. join_rules_event = current_state.get((EventTypes.JoinRules, "")) if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) if join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) result = {"room_id": room_id} num_joined_users = len([ 1 for _, event in current_state.items() if event.type == EventTypes.Member and event.membership == Membership.JOIN ]) if num_joined_users == 0: return result["num_joined_members"] = num_joined_users aliases = yield self.store.get_aliases_for_room(room_id) if aliases: result["aliases"] = aliases name_event = yield current_state.get((EventTypes.Name, "")) if name_event: name = name_event.content.get("name", None) if name: result["name"] = name topic_event = current_state.get((EventTypes.Topic, "")) if topic_event: topic = topic_event.content.get("topic", None) if topic: result["topic"] = topic canonical_event = current_state.get((EventTypes.CanonicalAlias, "")) if canonical_event: canonical_alias = canonical_event.content.get("alias", None) if canonical_alias: result["canonical_alias"] = canonical_alias visibility_event = current_state.get((EventTypes.RoomHistoryVisibility, "")) visibility = None if visibility_event: visibility = visibility_event.content.get("history_visibility", None) result["world_readable"] = visibility == "world_readable" guest_event = current_state.get((EventTypes.GuestAccess, "")) guest = None if guest_event: guest = guest_event.content.get("guest_access", None) result["guest_can_join"] = guest == "can_join" avatar_event = current_state.get(("m.room.avatar", "")) if avatar_event: avatar_url = avatar_event.content.get("url", None) if avatar_url: result["avatar_url"] = avatar_url results.append(result) yield concurrently_execute(handle_room, room_ids, 10) # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": results})
def _get_public_room_list(self): room_ids = yield self.store.get_public_room_ids() results = [] @defer.inlineCallbacks def handle_room(room_id): # We pull each bit of state out indvidually to avoid pulling the # full state into memory. Due to how the caching works this should # be fairly quick, even if not originally in the cache. def get_state(etype, state_key): return self.state_handler.get_current_state( room_id, etype, state_key) # Double check that this is actually a public room. join_rules_event = yield get_state(EventTypes.JoinRules, "") if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) if join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) result = {"room_id": room_id} joined_users = yield self.store.get_users_in_room(room_id) if len(joined_users) == 0: return result["num_joined_members"] = len(joined_users) aliases = yield self.store.get_aliases_for_room(room_id) if aliases: result["aliases"] = aliases name_event = yield get_state(EventTypes.Name, "") if name_event: name = name_event.content.get("name", None) if name: result["name"] = name topic_event = yield get_state(EventTypes.Topic, "") if topic_event: topic = topic_event.content.get("topic", None) if topic: result["topic"] = topic canonical_event = yield get_state(EventTypes.CanonicalAlias, "") if canonical_event: canonical_alias = canonical_event.content.get("alias", None) if canonical_alias: result["canonical_alias"] = canonical_alias visibility_event = yield get_state( EventTypes.RoomHistoryVisibility, "") visibility = None if visibility_event: visibility = visibility_event.content.get( "history_visibility", None) result["world_readable"] = visibility == "world_readable" guest_event = yield get_state(EventTypes.GuestAccess, "") guest = None if guest_event: guest = guest_event.content.get("guest_access", None) result["guest_can_join"] = guest == "can_join" avatar_event = yield get_state("m.room.avatar", "") if avatar_event: avatar_url = avatar_event.content.get("url", None) if avatar_url: result["avatar_url"] = avatar_url results.append(result) yield concurrently_execute(handle_room, room_ids, 10) # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": results})
def _get_public_room_list(self, limit=None, since_token=None, search_filter=None): if since_token and since_token != "END": since_token = RoomListNextBatch.from_token(since_token) else: since_token = None rooms_to_order_value = {} rooms_to_num_joined = {} rooms_to_latest_event_ids = {} newly_visible = [] newly_unpublished = [] if since_token: stream_token = since_token.stream_ordering current_public_id = yield self.store.get_current_public_room_stream_id() public_room_stream_id = since_token.public_room_stream_id newly_visible, newly_unpublished = yield self.store.get_public_room_changes( public_room_stream_id, current_public_id ) else: stream_token = yield self.store.get_room_max_stream_ordering() public_room_stream_id = yield self.store.get_current_public_room_stream_id() room_ids = yield self.store.get_public_room_ids_at_stream_id( public_room_stream_id ) # We want to return rooms in a particular order: the number of joined # users. We then arbitrarily use the room_id as a tie breaker. @defer.inlineCallbacks def get_order_for_room(room_id): latest_event_ids = rooms_to_latest_event_ids.get(room_id, None) if not latest_event_ids: latest_event_ids = yield self.store.get_forward_extremeties_for_room( room_id, stream_token ) rooms_to_latest_event_ids[room_id] = latest_event_ids if not latest_event_ids: return joined_users = yield self.state_handler.get_current_user_in_room( room_id, latest_event_ids, ) num_joined_users = len(joined_users) rooms_to_num_joined[room_id] = num_joined_users if num_joined_users == 0: return # We want larger rooms to be first, hence negating num_joined_users rooms_to_order_value[room_id] = (-num_joined_users, room_id) yield concurrently_execute(get_order_for_room, room_ids, 10) sorted_entries = sorted(rooms_to_order_value.items(), key=lambda e: e[1]) sorted_rooms = [room_id for room_id, _ in sorted_entries] # `sorted_rooms` should now be a list of all public room ids that is # stable across pagination. Therefore, we can use indices into this # list as our pagination tokens. # Filter out rooms that we don't want to return rooms_to_scan = [ r for r in sorted_rooms if r not in newly_unpublished and rooms_to_num_joined[room_id] > 0 ] if since_token: # Filter out rooms we've already returned previously # `since_token.current_limit` is the index of the last room we # sent down, so we exclude it and everything before/after it. if since_token.direction_is_forward: rooms_to_scan = rooms_to_scan[since_token.current_limit + 1:] else: rooms_to_scan = rooms_to_scan[:since_token.current_limit] rooms_to_scan.reverse() # Actually generate the entries. _generate_room_entry will append to # chunk but will stop if len(chunk) > limit chunk = [] if limit and not search_filter: step = limit + 1 for i in xrange(0, len(rooms_to_scan), step): # We iterate here because the vast majority of cases we'll stop # at first iteration, but occaisonally _generate_room_entry # won't append to the chunk and so we need to loop again. # We don't want to scan over the entire range either as that # would potentially waste a lot of work. yield concurrently_execute( lambda r: self._generate_room_entry( r, rooms_to_num_joined[r], chunk, limit, search_filter ), rooms_to_scan[i:i + step], 10 ) if len(chunk) >= limit + 1: break else: yield concurrently_execute( lambda r: self._generate_room_entry( r, rooms_to_num_joined[r], chunk, limit, search_filter ), rooms_to_scan, 5 ) chunk.sort(key=lambda e: (-e["num_joined_members"], e["room_id"])) # Work out the new limit of the batch for pagination, or None if we # know there are no more results that would be returned. # i.e., [since_token.current_limit..new_limit] is the batch of rooms # we've returned (or the reverse if we paginated backwards) # We tried to pull out limit + 1 rooms above, so if we have <= limit # then we know there are no more results to return new_limit = None if chunk and (not limit or len(chunk) > limit): if not since_token or since_token.direction_is_forward: if limit: chunk = chunk[:limit] last_room_id = chunk[-1]["room_id"] else: if limit: chunk = chunk[-limit:] last_room_id = chunk[0]["room_id"] new_limit = sorted_rooms.index(last_room_id) results = { "chunk": chunk, } if since_token: results["new_rooms"] = bool(newly_visible) if not since_token or since_token.direction_is_forward: if new_limit is not None: results["next_batch"] = RoomListNextBatch( stream_ordering=stream_token, public_room_stream_id=public_room_stream_id, current_limit=new_limit, direction_is_forward=True, ).to_token() if since_token: results["prev_batch"] = since_token.copy_and_replace( direction_is_forward=False, current_limit=since_token.current_limit + 1, ).to_token() else: if new_limit is not None: results["prev_batch"] = RoomListNextBatch( stream_ordering=stream_token, public_room_stream_id=public_room_stream_id, current_limit=new_limit, direction_is_forward=False, ).to_token() if since_token: results["next_batch"] = since_token.copy_and_replace( direction_is_forward=True, current_limit=since_token.current_limit - 1, ).to_token() defer.returnValue(results)
def _get_public_room_list(self, limit=None, since_token=None, search_filter=None, network_tuple=EMTPY_THIRD_PARTY_ID,): if since_token and since_token != "END": since_token = RoomListNextBatch.from_token(since_token) else: since_token = None rooms_to_order_value = {} rooms_to_num_joined = {} newly_visible = [] newly_unpublished = [] if since_token: stream_token = since_token.stream_ordering current_public_id = yield self.store.get_current_public_room_stream_id() public_room_stream_id = since_token.public_room_stream_id newly_visible, newly_unpublished = yield self.store.get_public_room_changes( public_room_stream_id, current_public_id, network_tuple=network_tuple, ) else: stream_token = yield self.store.get_room_max_stream_ordering() public_room_stream_id = yield self.store.get_current_public_room_stream_id() room_ids = yield self.store.get_public_room_ids_at_stream_id( public_room_stream_id, network_tuple=network_tuple, ) # We want to return rooms in a particular order: the number of joined # users. We then arbitrarily use the room_id as a tie breaker. @defer.inlineCallbacks def get_order_for_room(room_id): # Most of the rooms won't have changed between the since token and # now (especially if the since token is "now"). So, we can ask what # the current users are in a room (that will hit a cache) and then # check if the room has changed since the since token. (We have to # do it in that order to avoid races). # If things have changed then fall back to getting the current state # at the since token. joined_users = yield self.store.get_users_in_room(room_id) if self.store.has_room_changed_since(room_id, stream_token): latest_event_ids = yield self.store.get_forward_extremeties_for_room( room_id, stream_token ) if not latest_event_ids: return joined_users = yield self.state_handler.get_current_user_in_room( room_id, latest_event_ids, ) num_joined_users = len(joined_users) rooms_to_num_joined[room_id] = num_joined_users if num_joined_users == 0: return # We want larger rooms to be first, hence negating num_joined_users rooms_to_order_value[room_id] = (-num_joined_users, room_id) logger.info("Getting ordering for %i rooms since %s", len(room_ids), stream_token) yield concurrently_execute(get_order_for_room, room_ids, 10) sorted_entries = sorted(rooms_to_order_value.items(), key=lambda e: e[1]) sorted_rooms = [room_id for room_id, _ in sorted_entries] # `sorted_rooms` should now be a list of all public room ids that is # stable across pagination. Therefore, we can use indices into this # list as our pagination tokens. # Filter out rooms that we don't want to return rooms_to_scan = [ r for r in sorted_rooms if r not in newly_unpublished and rooms_to_num_joined[room_id] > 0 ] total_room_count = len(rooms_to_scan) if since_token: # Filter out rooms we've already returned previously # `since_token.current_limit` is the index of the last room we # sent down, so we exclude it and everything before/after it. if since_token.direction_is_forward: rooms_to_scan = rooms_to_scan[since_token.current_limit + 1:] else: rooms_to_scan = rooms_to_scan[:since_token.current_limit] rooms_to_scan.reverse() logger.info("After sorting and filtering, %i rooms remain", len(rooms_to_scan)) # _append_room_entry_to_chunk will append to chunk but will stop if # len(chunk) > limit # # Normally we will generate enough results on the first iteration here, # but if there is a search filter, _append_room_entry_to_chunk may # filter some results out, in which case we loop again. # # We don't want to scan over the entire range either as that # would potentially waste a lot of work. # # XXX if there is no limit, we may end up DoSing the server with # calls to get_current_state_ids for every single room on the # server. Surely we should cap this somehow? # if limit: step = limit + 1 else: # step cannot be zero step = len(rooms_to_scan) if len(rooms_to_scan) != 0 else 1 chunk = [] for i in range(0, len(rooms_to_scan), step): batch = rooms_to_scan[i:i + step] logger.info("Processing %i rooms for result", len(batch)) yield concurrently_execute( lambda r: self._append_room_entry_to_chunk( r, rooms_to_num_joined[r], chunk, limit, search_filter ), batch, 5, ) logger.info("Now %i rooms in result", len(chunk)) if len(chunk) >= limit + 1: break chunk.sort(key=lambda e: (-e["num_joined_members"], e["room_id"])) # Work out the new limit of the batch for pagination, or None if we # know there are no more results that would be returned. # i.e., [since_token.current_limit..new_limit] is the batch of rooms # we've returned (or the reverse if we paginated backwards) # We tried to pull out limit + 1 rooms above, so if we have <= limit # then we know there are no more results to return new_limit = None if chunk and (not limit or len(chunk) > limit): if not since_token or since_token.direction_is_forward: if limit: chunk = chunk[:limit] last_room_id = chunk[-1]["room_id"] else: if limit: chunk = chunk[-limit:] last_room_id = chunk[0]["room_id"] new_limit = sorted_rooms.index(last_room_id) results = { "chunk": chunk, "total_room_count_estimate": total_room_count, } if since_token: results["new_rooms"] = bool(newly_visible) if not since_token or since_token.direction_is_forward: if new_limit is not None: results["next_batch"] = RoomListNextBatch( stream_ordering=stream_token, public_room_stream_id=public_room_stream_id, current_limit=new_limit, direction_is_forward=True, ).to_token() if since_token: results["prev_batch"] = since_token.copy_and_replace( direction_is_forward=False, current_limit=since_token.current_limit + 1, ).to_token() else: if new_limit is not None: results["prev_batch"] = RoomListNextBatch( stream_ordering=stream_token, public_room_stream_id=public_room_stream_id, current_limit=new_limit, direction_is_forward=False, ).to_token() if since_token: results["next_batch"] = since_token.copy_and_replace( direction_is_forward=True, current_limit=since_token.current_limit - 1, ).to_token() defer.returnValue(results)
def _get_public_room_list(self): room_ids = yield self.store.get_public_room_ids() results = [] @defer.inlineCallbacks def handle_room(room_id): current_state = yield self.state_handler.get_current_state(room_id) # Double check that this is actually a public room. join_rules_event = current_state.get((EventTypes.JoinRules, "")) if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) if join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) result = {"room_id": room_id} num_joined_users = len([ 1 for _, event in current_state.items() if event.type == EventTypes.Member and event.membership == Membership.JOIN ]) if num_joined_users == 0: return result["num_joined_members"] = num_joined_users aliases = yield self.store.get_aliases_for_room(room_id) if aliases: result["aliases"] = aliases name_event = yield current_state.get((EventTypes.Name, "")) if name_event: name = name_event.content.get("name", None) if name: result["name"] = name topic_event = current_state.get((EventTypes.Topic, "")) if topic_event: topic = topic_event.content.get("topic", None) if topic: result["topic"] = topic canonical_event = current_state.get( (EventTypes.CanonicalAlias, "")) if canonical_event: canonical_alias = canonical_event.content.get("alias", None) if canonical_alias: result["canonical_alias"] = canonical_alias visibility_event = current_state.get( (EventTypes.RoomHistoryVisibility, "")) visibility = None if visibility_event: visibility = visibility_event.content.get( "history_visibility", None) result["world_readable"] = visibility == "world_readable" guest_event = current_state.get((EventTypes.GuestAccess, "")) guest = None if guest_event: guest = guest_event.content.get("guest_access", None) result["guest_can_join"] = guest == "can_join" avatar_event = current_state.get(("m.room.avatar", "")) if avatar_event: avatar_url = avatar_event.content.get("url", None) if avatar_url: result["avatar_url"] = avatar_url results.append(result) yield concurrently_execute(handle_room, room_ids, 10) # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": results})