async def on_send_join_request(self, origin, content, room_id): logger.debug("on_send_join_request: content: %s", content) room_version = await self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) pdu = event_from_pdu_json(content, format_ver) origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, pdu.room_id) logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures) pdu = await self._check_sigs_and_hash(room_version, pdu) res_pdus = await self.handler.on_send_join_request(origin, pdu) time_now = self._clock.time_msec() return ( 200, { "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]], "auth_chain": [ p.get_pdu_json(time_now) for p in res_pdus["auth_chain"] ], }, )
def send_request(destination): ret = yield self.transport_layer.make_membership_event( destination, room_id, user_id, membership, params, ) # Note: If not supplied, the room version may be either v1 or v2, # however either way the event format version will be v1. room_version = ret.get("room_version", RoomVersions.V1.identifier) event_format = room_version_to_event_format(room_version) pdu_dict = ret.get("event", None) if not isinstance(pdu_dict, dict): raise InvalidResponseError("Bad 'event' field in response") logger.debug("Got response to make_%s: %s", membership, pdu_dict) pdu_dict["content"].update(content) # The protoevent received over the JSON wire may not have all # the required fields. Lets just gloss over that because # there's some we never care about if "prev_state" not in pdu_dict: pdu_dict["prev_state"] = [] ev = builder.create_local_event_from_event_dict( self._clock, self.hostname, self.signing_key, format_version=event_format, event_dict=pdu_dict, ) defer.returnValue( (destination, ev, event_format) )
def _add_new_user(self, room_id, user_id): """Add new user to the room by creating an event and poking the federation API. """ hostname = get_domain_from_id(user_id) room_version = self.get_success(self.store.get_room_version(room_id)) builder = EventBuilder( state=self.state, auth=self.auth, store=self.store, clock=self.clock, hostname=hostname, signing_key=self.random_signing_key, format_version=room_version_to_event_format(room_version), room_id=room_id, type=EventTypes.Member, sender=user_id, state_key=user_id, content={"membership": Membership.JOIN}, ) prev_event_ids = self.get_success( self.store.get_latest_event_ids_in_room(room_id) ) event = self.get_success(builder.build(prev_event_ids)) self.get_success(self.federation_handler.on_receive_pdu(hostname, event)) # Check that it was successfully persisted. self.get_success(self.store.get_event(event.event_id)) self.get_success(self.store.get_event(event.event_id))
def send_request(destination): ret = yield self.transport_layer.make_membership_event( destination, room_id, user_id, membership, params ) # Note: If not supplied, the room version may be either v1 or v2, # however either way the event format version will be v1. room_version = ret.get("room_version", RoomVersions.V1.identifier) event_format = room_version_to_event_format(room_version) pdu_dict = ret.get("event", None) if not isinstance(pdu_dict, dict): raise InvalidResponseError("Bad 'event' field in response") logger.debug("Got response to make_%s: %s", membership, pdu_dict) pdu_dict["content"].update(content) # The protoevent received over the JSON wire may not have all # the required fields. Lets just gloss over that because # there's some we never care about if "prev_state" not in pdu_dict: pdu_dict["prev_state"] = [] ev = builder.create_local_event_from_event_dict( self._clock, self.hostname, self.signing_key, format_version=event_format, event_dict=pdu_dict, ) return (destination, ev, event_format)
def _add_new_user(self, room_id, user_id): """Add new user to the room by creating an event and poking the federation API. """ hostname = get_domain_from_id(user_id) room_version = self.get_success(self.store.get_room_version(room_id)) builder = EventBuilder( state=self.state, auth=self.auth, store=self.store, clock=self.clock, hostname=hostname, signing_key=self.random_signing_key, format_version=room_version_to_event_format(room_version), room_id=room_id, type=EventTypes.Member, sender=user_id, state_key=user_id, content={"membership": Membership.JOIN}, ) prev_event_ids = self.get_success( self.store.get_latest_event_ids_in_room(room_id)) event = self.get_success(builder.build(prev_event_ids)) self.get_success( self.federation_handler.on_receive_pdu(hostname, event)) # Check that it was successfully persisted. self.get_success(self.store.get_event(event.event_id)) self.get_success(self.store.get_event(event.event_id))
def get_event_auth(self, destination, room_id, event_id): res = yield self.transport_layer.get_event_auth( destination, room_id, event_id, ) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) auth_chain = [ event_from_pdu_json(p, format_ver, outlier=True) for p in res["auth_chain"] ] signed_auth = yield self._check_sigs_and_hash_and_fetch( destination, auth_chain, outlier=True, room_version=room_version, ) signed_auth.sort(key=lambda e: e.depth) defer.returnValue(signed_auth)
def on_query_auth_request(self, origin, content, room_id, event_id): """ Content is a dict with keys:: auth_chain (list): A list of events that give the auth chain. missing (list): A list of event_ids indicating what the other side (`origin`) think we're missing. rejects (dict): A mapping from event_id to a 2-tuple of reason string and a proof (or None) of why the event was rejected. The keys of this dict give the list of events the `origin` has rejected. Args: origin (str) content (dict) event_id (str) Returns: Deferred: Results in `dict` with the same format as `content` """ with (yield self._server_linearizer.queue((origin, room_id))): origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, room_id) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) auth_chain = [ event_from_pdu_json(e, format_ver) for e in content["auth_chain"] ] signed_auth = yield self._check_sigs_and_hash_and_fetch( origin, auth_chain, outlier=True, room_version=room_version, ) ret = yield self.handler.on_query_auth( origin, event_id, room_id, signed_auth, content.get("rejects", []), content.get("missing", []), ) time_now = self._clock.time_msec() send_content = { "auth_chain": [ e.get_pdu_json(time_now) for e in ret["auth_chain"] ], "rejects": ret.get("rejects", []), "missing": ret.get("missing", []), } defer.returnValue( (200, send_content) )
def on_query_auth_request(self, origin, content, room_id, event_id): """ Content is a dict with keys:: auth_chain (list): A list of events that give the auth chain. missing (list): A list of event_ids indicating what the other side (`origin`) think we're missing. rejects (dict): A mapping from event_id to a 2-tuple of reason string and a proof (or None) of why the event was rejected. The keys of this dict give the list of events the `origin` has rejected. Args: origin (str) content (dict) event_id (str) Returns: Deferred: Results in `dict` with the same format as `content` """ with (yield self._server_linearizer.queue((origin, room_id))): origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, room_id) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) auth_chain = [ event_from_pdu_json(e, format_ver) for e in content["auth_chain"] ] signed_auth = yield self._check_sigs_and_hash_and_fetch( origin, auth_chain, outlier=True, room_version=room_version, ) ret = yield self.handler.on_query_auth( origin, event_id, room_id, signed_auth, content.get("rejects", []), content.get("missing", []), ) time_now = self._clock.time_msec() send_content = { "auth_chain": [ e.get_pdu_json(time_now) for e in ret["auth_chain"] ], "rejects": ret.get("rejects", []), "missing": ret.get("missing", []), } defer.returnValue( (200, send_content) )
def on_invite_request(self, origin, content, room_version): format_ver = room_version_to_event_format(room_version) pdu = event_from_pdu_json(content, format_ver) origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, pdu.room_id) ret_pdu = yield self.handler.on_invite_request(origin, pdu) time_now = self._clock.time_msec() defer.returnValue({"event": ret_pdu.get_pdu_json(time_now)})
def get_missing_events( self, destination, room_id, earliest_events_ids, latest_events, limit, min_depth, timeout, ): """Tries to fetch events we are missing. This is called when we receive an event without having received all of its ancestors. Args: destination (str) room_id (str) earliest_events_ids (list): List of event ids. Effectively the events we expected to receive, but haven't. `get_missing_events` should only return events that didn't happen before these. latest_events (list): List of events we have received that we don't have all previous events for. limit (int): Maximum number of events to return. min_depth (int): Minimum depth of events tor return. timeout (int): Max time to wait in ms """ try: content = yield self.transport_layer.get_missing_events( destination=destination, room_id=room_id, earliest_events=earliest_events_ids, latest_events=[e.event_id for e in latest_events], limit=limit, min_depth=min_depth, timeout=timeout, ) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) events = [ event_from_pdu_json(e, format_ver) for e in content.get("events", []) ] signed_events = yield self._check_sigs_and_hash_and_fetch( destination, events, outlier=False, room_version=room_version ) except HttpResponseException as e: if not e.code == 400: raise # We are probably hitting an old server that doesn't support # get_missing_events signed_events = [] return signed_events
def on_send_leave_request(self, origin, content, room_id): logger.debug("on_send_leave_request: content: %s", content) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) pdu = event_from_pdu_json(content, format_ver) origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, pdu.room_id) logger.debug("on_send_leave_request: pdu sigs: %s", pdu.signatures) yield self.handler.on_send_leave_request(origin, pdu) defer.returnValue((200, {}))
def on_send_leave_request(self, origin, content, room_id): logger.debug("on_send_leave_request: content: %s", content) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) pdu = event_from_pdu_json(content, format_ver) origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, pdu.room_id) logger.debug("on_send_leave_request: pdu sigs: %s", pdu.signatures) yield self.handler.on_send_leave_request(origin, pdu) defer.returnValue((200, {}))
def get_missing_events(self, destination, room_id, earliest_events_ids, latest_events, limit, min_depth, timeout): """Tries to fetch events we are missing. This is called when we receive an event without having received all of its ancestors. Args: destination (str) room_id (str) earliest_events_ids (list): List of event ids. Effectively the events we expected to receive, but haven't. `get_missing_events` should only return events that didn't happen before these. latest_events (list): List of events we have received that we don't have all previous events for. limit (int): Maximum number of events to return. min_depth (int): Minimum depth of events tor return. timeout (int): Max time to wait in ms """ try: content = yield self.transport_layer.get_missing_events( destination=destination, room_id=room_id, earliest_events=earliest_events_ids, latest_events=[e.event_id for e in latest_events], limit=limit, min_depth=min_depth, timeout=timeout, ) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) events = [ event_from_pdu_json(e, format_ver) for e in content.get("events", []) ] signed_events = yield self._check_sigs_and_hash_and_fetch( destination, events, outlier=False, room_version=room_version, ) except HttpResponseException as e: if not e.code == 400: raise # We are probably hitting an old server that doesn't support # get_missing_events signed_events = [] defer.returnValue(signed_events)
async def on_send_leave_request(self, origin, content, room_id): logger.debug("on_send_leave_request: content: %s", content) room_version = await self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) pdu = event_from_pdu_json(content, format_ver) origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, pdu.room_id) logger.debug("on_send_leave_request: pdu sigs: %s", pdu.signatures) pdu = await self._check_sigs_and_hash(room_version, pdu) await self.handler.on_send_leave_request(origin, pdu) return {}
def on_invite_request(self, origin, content, room_version): if room_version not in KNOWN_ROOM_VERSIONS: raise SynapseError( 400, "Homeserver does not support this room version", Codes.UNSUPPORTED_ROOM_VERSION, ) format_ver = room_version_to_event_format(room_version) pdu = event_from_pdu_json(content, format_ver) origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, pdu.room_id) ret_pdu = yield self.handler.on_invite_request(origin, pdu) time_now = self._clock.time_msec() defer.returnValue({"event": ret_pdu.get_pdu_json(time_now)})
def on_invite_request(self, origin, content, room_version): if room_version not in KNOWN_ROOM_VERSIONS: raise SynapseError( 400, "Homeserver does not support this room version", Codes.UNSUPPORTED_ROOM_VERSION, ) format_ver = room_version_to_event_format(room_version) pdu = event_from_pdu_json(content, format_ver) origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, pdu.room_id) ret_pdu = yield self.handler.on_invite_request(origin, pdu) time_now = self._clock.time_msec() defer.returnValue({"event": ret_pdu.get_pdu_json(time_now)})
def query_auth(self, destination, room_id, event_id, local_auth): """ Params: destination (str) event_it (str) local_auth (list) """ time_now = self._clock.time_msec() send_content = { "auth_chain": [e.get_pdu_json(time_now) for e in local_auth], } code, content = yield self.transport_layer.send_query_auth( destination=destination, room_id=room_id, event_id=event_id, content=send_content, ) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) auth_chain = [ event_from_pdu_json(e, format_ver) for e in content["auth_chain"] ] signed_auth = yield self._check_sigs_and_hash_and_fetch( destination, auth_chain, outlier=True, room_version=room_version, ) signed_auth.sort(key=lambda e: e.depth) ret = { "auth_chain": signed_auth, "rejects": content.get("rejects", []), "missing": content.get("missing", []), } defer.returnValue(ret)
def on_send_join_request(self, origin, content, room_id): logger.debug("on_send_join_request: content: %s", content) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) pdu = event_from_pdu_json(content, format_ver) origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, pdu.room_id) logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures) res_pdus = yield self.handler.on_send_join_request(origin, pdu) time_now = self._clock.time_msec() defer.returnValue((200, { "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]], "auth_chain": [ p.get_pdu_json(time_now) for p in res_pdus["auth_chain"] ], }))
def backfill(self, dest, room_id, limit, extremities): """Requests some more historic PDUs for the given context from the given destination server. Args: dest (str): The remote home server to ask. room_id (str): The room_id to backfill. limit (int): The maximum number of PDUs to return. extremities (list): List of PDU id and origins of the first pdus we have seen from the context Returns: Deferred: Results in the received PDUs. """ logger.debug("backfill extrem=%s", extremities) # If there are no extremeties then we've (probably) reached the start. if not extremities: return transaction_data = yield self.transport_layer.backfill( dest, room_id, extremities, limit ) logger.debug("backfill transaction_data=%s", repr(transaction_data)) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) pdus = [ event_from_pdu_json(p, format_ver, outlier=False) for p in transaction_data["pdus"] ] # FIXME: We should handle signature failures more gracefully. pdus[:] = yield make_deferred_yieldable( defer.gatherResults( self._check_sigs_and_hashes(room_version, pdus), consumeErrors=True ).addErrback(unwrapFirstError) ) return pdus
def send_invite(self, destination, room_id, event_id, pdu): room_version = yield self.store.get_room_version(room_id) content = yield self._do_send_invite(destination, pdu, room_version) pdu_dict = content["event"] logger.debug("Got response to send_invite: %s", pdu_dict) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) pdu = event_from_pdu_json(pdu_dict, format_ver) # Check signatures are correct. pdu = yield self._check_sigs_and_hash(room_version, pdu) # FIXME: We should handle signature failures more gracefully. return pdu
def send_invite(self, destination, room_id, event_id, pdu): room_version = yield self.store.get_room_version(room_id) content = yield self._do_send_invite(destination, pdu, room_version) pdu_dict = content["event"] logger.debug("Got response to send_invite: %s", pdu_dict) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) pdu = event_from_pdu_json(pdu_dict, format_ver) # Check signatures are correct. pdu = yield self._check_sigs_and_hash(room_version, pdu) # FIXME: We should handle signature failures more gracefully. defer.returnValue(pdu)
def query_auth(self, destination, room_id, event_id, local_auth): """ Params: destination (str) event_it (str) local_auth (list) """ time_now = self._clock.time_msec() send_content = { "auth_chain": [e.get_pdu_json(time_now) for e in local_auth], } code, content = yield self.transport_layer.send_query_auth( destination=destination, room_id=room_id, event_id=event_id, content=send_content, ) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) auth_chain = [ event_from_pdu_json(e, format_ver) for e in content["auth_chain"] ] signed_auth = yield self._check_sigs_and_hash_and_fetch( destination, auth_chain, outlier=True, room_version=room_version, ) signed_auth.sort(key=lambda e: e.depth) ret = { "auth_chain": signed_auth, "rejects": content.get("rejects", []), "missing": content.get("missing", []), } defer.returnValue(ret)
def get_event_auth(self, destination, room_id, event_id): res = yield self.transport_layer.get_event_auth( destination, room_id, event_id, ) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) auth_chain = [ event_from_pdu_json(p, format_ver, outlier=True) for p in res["auth_chain"] ] signed_auth = yield self._check_sigs_and_hash_and_fetch( destination, auth_chain, outlier=True, room_version=room_version, ) signed_auth.sort(key=lambda e: e.depth) defer.returnValue(signed_auth)
def backfill(self, dest, room_id, limit, extremities): """Requests some more historic PDUs for the given context from the given destination server. Args: dest (str): The remote home server to ask. room_id (str): The room_id to backfill. limit (int): The maximum number of PDUs to return. extremities (list): List of PDU id and origins of the first pdus we have seen from the context Returns: Deferred: Results in the received PDUs. """ logger.debug("backfill extrem=%s", extremities) # If there are no extremeties then we've (probably) reached the start. if not extremities: return transaction_data = yield self.transport_layer.backfill( dest, room_id, extremities, limit) logger.debug("backfill transaction_data=%s", repr(transaction_data)) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) pdus = [ event_from_pdu_json(p, format_ver, outlier=False) for p in transaction_data["pdus"] ] # FIXME: We should handle signature failures more gracefully. pdus[:] = yield logcontext.make_deferred_yieldable(defer.gatherResults( self._check_sigs_and_hashes(room_version, pdus), consumeErrors=True, ).addErrback(unwrapFirstError)) defer.returnValue(pdus)
def get_state_for_room(self, destination, room_id, event_id): """Requests all of the room state at a given event from a remote home server. Args: destination (str): The remote homeserver to query for the state. room_id (str): The id of the room we're interested in. event_id (str): The id of the event we want the state at. Returns: Deferred[Tuple[List[EventBase], List[EventBase]]]: A list of events in the state, and a list of events in the auth chain for the given event. """ try: # First we try and ask for just the IDs, as thats far quicker if # we have most of the state and auth_chain already. # However, this may 404 if the other side has an old synapse. result = yield self.transport_layer.get_room_state_ids( destination, room_id, event_id=event_id ) state_event_ids = result["pdu_ids"] auth_event_ids = result.get("auth_chain_ids", []) fetched_events, failed_to_fetch = yield self.get_events_from_store_or_dest( destination, room_id, set(state_event_ids + auth_event_ids) ) if failed_to_fetch: logger.warning( "Failed to fetch missing state/auth events for %s: %s", room_id, failed_to_fetch, ) event_map = {ev.event_id: ev for ev in fetched_events} pdus = [event_map[e_id] for e_id in state_event_ids if e_id in event_map] auth_chain = [ event_map[e_id] for e_id in auth_event_ids if e_id in event_map ] auth_chain.sort(key=lambda e: e.depth) return pdus, auth_chain except HttpResponseException as e: if e.code == 400 or e.code == 404: logger.info("Failed to use get_room_state_ids API, falling back") else: raise e result = yield self.transport_layer.get_room_state( destination, room_id, event_id=event_id ) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) pdus = [ event_from_pdu_json(p, format_ver, outlier=True) for p in result["pdus"] ] auth_chain = [ event_from_pdu_json(p, format_ver, outlier=True) for p in result.get("auth_chain", []) ] seen_events = yield self.store.get_events( [ev.event_id for ev in itertools.chain(pdus, auth_chain)] ) signed_pdus = yield self._check_sigs_and_hash_and_fetch( destination, [p for p in pdus if p.event_id not in seen_events], outlier=True, room_version=room_version, ) signed_pdus.extend( seen_events[p.event_id] for p in pdus if p.event_id in seen_events ) signed_auth = yield self._check_sigs_and_hash_and_fetch( destination, [p for p in auth_chain if p.event_id not in seen_events], outlier=True, room_version=room_version, ) signed_auth.extend( seen_events[p.event_id] for p in auth_chain if p.event_id in seen_events ) signed_auth.sort(key=lambda e: e.depth) return signed_pdus, signed_auth
def get_pdu( self, destinations, event_id, room_version, outlier=False, timeout=None ): """Requests the PDU with given origin and ID from the remote home servers. Will attempt to get the PDU from each destination in the list until one succeeds. Args: destinations (list): Which home servers to query event_id (str): event to fetch room_version (str): version of the room outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if it's from an arbitary point in the context as opposed to part of the current block of PDUs. Defaults to `False` timeout (int): How long to try (in ms) each destination for before moving to the next destination. None indicates no timeout. Returns: Deferred: Results in the requested PDU, or None if we were unable to find it. """ # TODO: Rate limit the number of times we try and get the same event. ev = self._get_pdu_cache.get(event_id) if ev: return ev pdu_attempts = self.pdu_destination_tried.setdefault(event_id, {}) format_ver = room_version_to_event_format(room_version) signed_pdu = None for destination in destinations: now = self._clock.time_msec() last_attempt = pdu_attempts.get(destination, 0) if last_attempt + PDU_RETRY_TIME_MS > now: continue try: transaction_data = yield self.transport_layer.get_event( destination, event_id, timeout=timeout ) logger.debug( "retrieved event id %s from %s: %r", event_id, destination, transaction_data, ) pdu_list = [ event_from_pdu_json(p, format_ver, outlier=outlier) for p in transaction_data["pdus"] ] if pdu_list and pdu_list[0]: pdu = pdu_list[0] # Check signatures are correct. signed_pdu = yield self._check_sigs_and_hash(room_version, pdu) break pdu_attempts[destination] = now except SynapseError as e: logger.info( "Failed to get PDU %s from %s because %s", event_id, destination, e ) continue except NotRetryingDestination as e: logger.info(str(e)) continue except FederationDeniedError as e: logger.info(str(e)) continue except Exception as e: pdu_attempts[destination] = now logger.info( "Failed to get PDU %s from %s because %s", event_id, destination, e ) continue if signed_pdu: self._get_pdu_cache[event_id] = signed_pdu return signed_pdu
def _handle_incoming_transaction(self, origin, transaction, request_time): """ Process an incoming transaction and return the HTTP response Args: origin (unicode): the server making the request transaction (Transaction): incoming transaction request_time (int): timestamp that the HTTP request arrived at Returns: Deferred[(int, object)]: http response code and body """ response = yield self.transaction_actions.have_responded( origin, transaction) if response: logger.debug( "[%s] We've already responded to this request", transaction.transaction_id, ) defer.returnValue(response) return logger.debug("[%s] Transaction is new", transaction.transaction_id) # Reject if PDU count > 50 and EDU count > 100 if len(transaction.pdus) > 50 or (hasattr(transaction, "edus") and len(transaction.edus) > 100): logger.info( "Transaction PDU or EDU count too large. Returning 400") response = {} yield self.transaction_actions.set_response( origin, transaction, 400, response) defer.returnValue((400, response)) received_pdus_counter.inc(len(transaction.pdus)) origin_host, _ = parse_server_name(origin) pdus_by_room = {} for p in transaction.pdus: if "unsigned" in p: unsigned = p["unsigned"] if "age" in unsigned: p["age"] = unsigned["age"] if "age" in p: p["age_ts"] = request_time - int(p["age"]) del p["age"] # We try and pull out an event ID so that if later checks fail we # can log something sensible. We don't mandate an event ID here in # case future event formats get rid of the key. possible_event_id = p.get("event_id", "<Unknown>") # Now we get the room ID so that we can check that we know the # version of the room. room_id = p.get("room_id") if not room_id: logger.info( "Ignoring PDU as does not have a room_id. Event ID: %s", possible_event_id, ) continue try: room_version = yield self.store.get_room_version(room_id) except NotFoundError: logger.info("Ignoring PDU for unknown room_id: %s", room_id) continue try: format_ver = room_version_to_event_format(room_version) except UnsupportedRoomVersionError: # this can happen if support for a given room version is withdrawn, # so that we still get events for said room. logger.info( "Ignoring PDU for room %s with unknown version %s", room_id, room_version, ) continue event = event_from_pdu_json(p, format_ver) pdus_by_room.setdefault(room_id, []).append(event) pdu_results = {} # we can process different rooms in parallel (which is useful if they # require callouts to other servers to fetch missing events), but # impose a limit to avoid going too crazy with ram/cpu. @defer.inlineCallbacks def process_pdus_for_room(room_id): logger.debug("Processing PDUs for %s", room_id) try: yield self.check_server_matches_acl(origin_host, room_id) except AuthError as e: logger.warn("Ignoring PDUs for room %s from banned server", room_id) for pdu in pdus_by_room[room_id]: event_id = pdu.event_id pdu_results[event_id] = e.error_dict() return for pdu in pdus_by_room[room_id]: event_id = pdu.event_id with nested_logging_context(event_id): try: yield self._handle_received_pdu(origin, pdu) pdu_results[event_id] = {} except FederationError as e: logger.warn("Error handling PDU %s: %s", event_id, e) pdu_results[event_id] = {"error": str(e)} except Exception as e: f = failure.Failure() pdu_results[event_id] = {"error": str(e)} logger.error( "Failed to handle PDU %s", event_id, exc_info=(f.type, f.value, f.getTracebackObject()), ) yield concurrently_execute(process_pdus_for_room, pdus_by_room.keys(), TRANSACTION_CONCURRENCY_LIMIT) if hasattr(transaction, "edus"): for edu in (Edu(**x) for x in transaction.edus): yield self.received_edu(origin, edu.edu_type, edu.content) response = {"pdus": pdu_results} logger.debug("Returning: %s", str(response)) yield self.transaction_actions.set_response(origin, transaction, 200, response) defer.returnValue((200, response))
def get_state_for_room(self, destination, room_id, event_id): """Requests all of the room state at a given event from a remote home server. Args: destination (str): The remote homeserver to query for the state. room_id (str): The id of the room we're interested in. event_id (str): The id of the event we want the state at. Returns: Deferred[Tuple[List[EventBase], List[EventBase]]]: A list of events in the state, and a list of events in the auth chain for the given event. """ try: # First we try and ask for just the IDs, as thats far quicker if # we have most of the state and auth_chain already. # However, this may 404 if the other side has an old synapse. result = yield self.transport_layer.get_room_state_ids( destination, room_id, event_id=event_id, ) state_event_ids = result["pdu_ids"] auth_event_ids = result.get("auth_chain_ids", []) fetched_events, failed_to_fetch = yield self.get_events( [destination], room_id, set(state_event_ids + auth_event_ids) ) if failed_to_fetch: logger.warn("Failed to get %r", failed_to_fetch) event_map = { ev.event_id: ev for ev in fetched_events } pdus = [event_map[e_id] for e_id in state_event_ids if e_id in event_map] auth_chain = [ event_map[e_id] for e_id in auth_event_ids if e_id in event_map ] auth_chain.sort(key=lambda e: e.depth) defer.returnValue((pdus, auth_chain)) except HttpResponseException as e: if e.code == 400 or e.code == 404: logger.info("Failed to use get_room_state_ids API, falling back") else: raise e result = yield self.transport_layer.get_room_state( destination, room_id, event_id=event_id, ) room_version = yield self.store.get_room_version(room_id) format_ver = room_version_to_event_format(room_version) pdus = [ event_from_pdu_json(p, format_ver, outlier=True) for p in result["pdus"] ] auth_chain = [ event_from_pdu_json(p, format_ver, outlier=True) for p in result.get("auth_chain", []) ] seen_events = yield self.store.get_events([ ev.event_id for ev in itertools.chain(pdus, auth_chain) ]) signed_pdus = yield self._check_sigs_and_hash_and_fetch( destination, [p for p in pdus if p.event_id not in seen_events], outlier=True, room_version=room_version, ) signed_pdus.extend( seen_events[p.event_id] for p in pdus if p.event_id in seen_events ) signed_auth = yield self._check_sigs_and_hash_and_fetch( destination, [p for p in auth_chain if p.event_id not in seen_events], outlier=True, room_version=room_version, ) signed_auth.extend( seen_events[p.event_id] for p in auth_chain if p.event_id in seen_events ) signed_auth.sort(key=lambda e: e.depth) defer.returnValue((signed_pdus, signed_auth))
async def _handle_pdus_in_txn(self, origin: str, transaction: Transaction, request_time: int) -> Dict[str, dict]: """Process the PDUs in a received transaction. Args: origin: the server making the request transaction: incoming transaction request_time: timestamp that the HTTP request arrived at Returns: A map from event ID of a processed PDU to any errors we should report back to the sending server. """ received_pdus_counter.inc(len(transaction.pdus)) origin_host, _ = parse_server_name(origin) pdus_by_room = {} for p in transaction.pdus: if "unsigned" in p: unsigned = p["unsigned"] if "age" in unsigned: p["age"] = unsigned["age"] if "age" in p: p["age_ts"] = request_time - int(p["age"]) del p["age"] # We try and pull out an event ID so that if later checks fail we # can log something sensible. We don't mandate an event ID here in # case future event formats get rid of the key. possible_event_id = p.get("event_id", "<Unknown>") # Now we get the room ID so that we can check that we know the # version of the room. room_id = p.get("room_id") if not room_id: logger.info( "Ignoring PDU as does not have a room_id. Event ID: %s", possible_event_id, ) continue try: room_version = await self.store.get_room_version(room_id) except NotFoundError: logger.info("Ignoring PDU for unknown room_id: %s", room_id) continue try: format_ver = room_version_to_event_format(room_version) except UnsupportedRoomVersionError: # this can happen if support for a given room version is withdrawn, # so that we still get events for said room. logger.info( "Ignoring PDU for room %s with unknown version %s", room_id, room_version, ) continue event = event_from_pdu_json(p, format_ver) pdus_by_room.setdefault(room_id, []).append(event) pdu_results = {} # we can process different rooms in parallel (which is useful if they # require callouts to other servers to fetch missing events), but # impose a limit to avoid going too crazy with ram/cpu. async def process_pdus_for_room(room_id): logger.debug("Processing PDUs for %s", room_id) try: await self.check_server_matches_acl(origin_host, room_id) except AuthError as e: logger.warning("Ignoring PDUs for room %s from banned server", room_id) for pdu in pdus_by_room[room_id]: event_id = pdu.event_id pdu_results[event_id] = e.error_dict() return for pdu in pdus_by_room[room_id]: event_id = pdu.event_id with nested_logging_context(event_id): try: await self._handle_received_pdu(origin, pdu) pdu_results[event_id] = {} except FederationError as e: logger.warning("Error handling PDU %s: %s", event_id, e) pdu_results[event_id] = {"error": str(e)} except Exception as e: f = failure.Failure() pdu_results[event_id] = {"error": str(e)} logger.error( "Failed to handle PDU %s", event_id, exc_info=(f.type, f.value, f.getTracebackObject()), ) await concurrently_execute(process_pdus_for_room, pdus_by_room.keys(), TRANSACTION_CONCURRENCY_LIMIT) return pdu_results
def get_pdu(self, destinations, event_id, room_version, outlier=False, timeout=None): """Requests the PDU with given origin and ID from the remote home servers. Will attempt to get the PDU from each destination in the list until one succeeds. Args: destinations (list): Which home servers to query event_id (str): event to fetch room_version (str): version of the room outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if it's from an arbitary point in the context as opposed to part of the current block of PDUs. Defaults to `False` timeout (int): How long to try (in ms) each destination for before moving to the next destination. None indicates no timeout. Returns: Deferred: Results in the requested PDU. """ # TODO: Rate limit the number of times we try and get the same event. ev = self._get_pdu_cache.get(event_id) if ev: defer.returnValue(ev) pdu_attempts = self.pdu_destination_tried.setdefault(event_id, {}) format_ver = room_version_to_event_format(room_version) signed_pdu = None for destination in destinations: now = self._clock.time_msec() last_attempt = pdu_attempts.get(destination, 0) if last_attempt + PDU_RETRY_TIME_MS > now: continue try: transaction_data = yield self.transport_layer.get_event( destination, event_id, timeout=timeout, ) logger.debug("transaction_data %r", transaction_data) pdu_list = [ event_from_pdu_json(p, format_ver, outlier=outlier) for p in transaction_data["pdus"] ] if pdu_list and pdu_list[0]: pdu = pdu_list[0] # Check signatures are correct. signed_pdu = yield self._check_sigs_and_hash(room_version, pdu) break pdu_attempts[destination] = now except SynapseError as e: logger.info( "Failed to get PDU %s from %s because %s", event_id, destination, e, ) except NotRetryingDestination as e: logger.info(str(e)) continue except FederationDeniedError as e: logger.info(str(e)) continue except Exception as e: pdu_attempts[destination] = now logger.info( "Failed to get PDU %s from %s because %s", event_id, destination, e, ) continue if signed_pdu: self._get_pdu_cache[event_id] = signed_pdu defer.returnValue(signed_pdu)
def _handle_incoming_transaction(self, origin, transaction, request_time): """ Process an incoming transaction and return the HTTP response Args: origin (unicode): the server making the request transaction (Transaction): incoming transaction request_time (int): timestamp that the HTTP request arrived at Returns: Deferred[(int, object)]: http response code and body """ response = yield self.transaction_actions.have_responded(origin, transaction) if response: logger.debug( "[%s] We've already responded to this request", transaction.transaction_id ) defer.returnValue(response) return logger.debug("[%s] Transaction is new", transaction.transaction_id) # Reject if PDU count > 50 and EDU count > 100 if (len(transaction.pdus) > 50 or (hasattr(transaction, "edus") and len(transaction.edus) > 100)): logger.info( "Transaction PDU or EDU count too large. Returning 400", ) response = {} yield self.transaction_actions.set_response( origin, transaction, 400, response ) defer.returnValue((400, response)) received_pdus_counter.inc(len(transaction.pdus)) origin_host, _ = parse_server_name(origin) pdus_by_room = {} for p in transaction.pdus: if "unsigned" in p: unsigned = p["unsigned"] if "age" in unsigned: p["age"] = unsigned["age"] if "age" in p: p["age_ts"] = request_time - int(p["age"]) del p["age"] # We try and pull out an event ID so that if later checks fail we # can log something sensible. We don't mandate an event ID here in # case future event formats get rid of the key. possible_event_id = p.get("event_id", "<Unknown>") # Now we get the room ID so that we can check that we know the # version of the room. room_id = p.get("room_id") if not room_id: logger.info( "Ignoring PDU as does not have a room_id. Event ID: %s", possible_event_id, ) continue try: room_version = yield self.store.get_room_version(room_id) except NotFoundError: logger.info("Ignoring PDU for unknown room_id: %s", room_id) continue try: format_ver = room_version_to_event_format(room_version) except UnsupportedRoomVersionError: # this can happen if support for a given room version is withdrawn, # so that we still get events for said room. logger.info( "Ignoring PDU for room %s with unknown version %s", room_id, room_version, ) continue event = event_from_pdu_json(p, format_ver) pdus_by_room.setdefault(room_id, []).append(event) pdu_results = {} # we can process different rooms in parallel (which is useful if they # require callouts to other servers to fetch missing events), but # impose a limit to avoid going too crazy with ram/cpu. @defer.inlineCallbacks def process_pdus_for_room(room_id): logger.debug("Processing PDUs for %s", room_id) try: yield self.check_server_matches_acl(origin_host, room_id) except AuthError as e: logger.warn( "Ignoring PDUs for room %s from banned server", room_id, ) for pdu in pdus_by_room[room_id]: event_id = pdu.event_id pdu_results[event_id] = e.error_dict() return for pdu in pdus_by_room[room_id]: event_id = pdu.event_id with nested_logging_context(event_id): try: yield self._handle_received_pdu( origin, pdu ) pdu_results[event_id] = {} except FederationError as e: logger.warn("Error handling PDU %s: %s", event_id, e) pdu_results[event_id] = {"error": str(e)} except Exception as e: f = failure.Failure() pdu_results[event_id] = {"error": str(e)} logger.error( "Failed to handle PDU %s", event_id, exc_info=(f.type, f.value, f.getTracebackObject()), ) yield concurrently_execute( process_pdus_for_room, pdus_by_room.keys(), TRANSACTION_CONCURRENCY_LIMIT, ) if hasattr(transaction, "edus"): for edu in (Edu(**x) for x in transaction.edus): yield self.received_edu( origin, edu.edu_type, edu.content ) response = { "pdus": pdu_results, } logger.debug("Returning: %s", str(response)) yield self.transaction_actions.set_response( origin, transaction, 200, response ) defer.returnValue((200, response))