async def on_incoming_transaction( self, origin: str, transaction_data: JsonDict ) -> Tuple[int, Dict[str, Any]]: # keep this as early as possible to make the calculated origin ts as # accurate as possible. request_time = self._clock.time_msec() transaction = Transaction(**transaction_data) if not transaction.transaction_id: # type: ignore raise Exception("Transaction missing transaction_id") logger.debug("[%s] Got transaction", transaction.transaction_id) # type: ignore # use a linearizer to ensure that we don't process the same transaction # multiple times in parallel. with ( await self._transaction_linearizer.queue( (origin, transaction.transaction_id) # type: ignore ) ): result = await self._handle_incoming_transaction( origin, transaction, request_time ) return result
def _transaction_from_pdus(self, pdu_list: List[EventBase]) -> Transaction: """Returns a new Transaction containing the given PDUs suitable for transmission. """ time_now = self._clock.time_msec() pdus = [p.get_pdu_json(time_now) for p in pdu_list] return Transaction( origin=self.server_name, pdus=pdus, origin_server_ts=int(time_now), destination=None, )
async def on_incoming_transaction( self, origin: str, transaction_data: JsonDict) -> Tuple[int, Dict[str, Any]]: # keep this as early as possible to make the calculated origin ts as # accurate as possible. request_time = self._clock.time_msec() transaction = Transaction(**transaction_data) transaction_id = transaction.transaction_id # type: ignore if not transaction_id: raise Exception("Transaction missing transaction_id") logger.debug("[%s] Got transaction", transaction_id) # Reject malformed transactions early: reject if too many PDUs/EDUs if len(transaction.pdus) > 50 or ( # type: ignore hasattr(transaction, "edus") and len(transaction.edus) > 100 # type: ignore ): logger.info( "Transaction PDU or EDU count too large. Returning 400") return 400, {} # we only process one transaction from each origin at a time. We need to do # this check here, rather than in _on_incoming_transaction_inner so that we # don't cache the rejection in _transaction_resp_cache (so that if the txn # arrives again later, we can process it). current_transaction = self._active_transactions.get(origin) if current_transaction and current_transaction != transaction_id: logger.warning( "Received another txn %s from %s while still processing %s", transaction_id, origin, current_transaction, ) return 429, { "errcode": Codes.UNKNOWN, "error": "Too many concurrent transactions", } # CRITICAL SECTION: we must now not await until we populate _active_transactions # in _on_incoming_transaction_inner. # We wrap in a ResponseCache so that we de-duplicate retried # transactions. return await self._transaction_resp_cache.wrap( (origin, transaction_id), self._on_incoming_transaction_inner, origin, transaction, request_time, )
def _transaction_dict_from_pdus(self, pdu_list: List[EventBase]) -> JsonDict: """Returns a new Transaction containing the given PDUs suitable for transmission. """ time_now = self._clock.time_msec() pdus = [p.get_pdu_json(time_now) for p in pdu_list] return Transaction( # Just need a dummy transaction ID and destination since it won't be used. transaction_id="", origin=self.server_name, pdus=pdus, origin_server_ts=int(time_now), destination="", ).get_dict()
async def send_transaction( self, transaction: Transaction, json_data_callback: Optional[Callable[[], JsonDict]] = None, ) -> JsonDict: """Sends the given Transaction to its destination Args: transaction Returns: Succeeds when we get a 2xx HTTP response. The result will be the decoded JSON body. Fails with ``HTTPRequestException`` if we get an HTTP response code >= 300. Fails with ``NotRetryingDestination`` if we are not yet ready to retry this server. Fails with ``FederationDeniedError`` if this destination is not on our federation whitelist """ logger.debug( "send_data dest=%s, txid=%s", transaction.destination, # type: ignore transaction.transaction_id, # type: ignore ) if transaction.destination == self.server_name: # type: ignore raise RuntimeError("Transport layer cannot send to itself!") # FIXME: This is only used by the tests. The actual json sent is # generated by the json_data_callback. json_data = transaction.get_dict() path = _create_v1_path("/send/%s", transaction.transaction_id) # type: ignore return await self.client.put_json( transaction.destination, # type: ignore path=path, data=json_data, json_data_callback=json_data_callback, long_retries=True, backoff_on_404=True, # If we get a 404 the other side has gone try_trailing_slash_on_400=True, )
def on_incoming_transaction(self, origin, transaction_data): # keep this as early as possible to make the calculated origin ts as # accurate as possible. request_time = self._clock.time_msec() transaction = Transaction(**transaction_data) if not transaction.transaction_id: raise Exception("Transaction missing transaction_id") logger.debug("[%s] Got transaction", transaction.transaction_id) # use a linearizer to ensure that we don't process the same transaction # multiple times in parallel. with (yield self._transaction_linearizer.queue( (origin, transaction.transaction_id))): result = yield self._handle_incoming_transaction( origin, transaction, request_time) defer.returnValue(result)
async def on_incoming_transaction( self, origin: str, transaction_data: JsonDict) -> Tuple[int, Dict[str, Any]]: # keep this as early as possible to make the calculated origin ts as # accurate as possible. request_time = self._clock.time_msec() transaction = Transaction(**transaction_data) transaction_id = transaction.transaction_id # type: ignore if not transaction_id: raise Exception("Transaction missing transaction_id") logger.debug("[%s] Got transaction", transaction_id) # We wrap in a ResponseCache so that we de-duplicate retried # transactions. return await self._transaction_resp_cache.wrap( (origin, transaction_id), self._on_incoming_transaction_inner, origin, transaction, request_time, )
def send_new_transaction(self, destination, pending_pdus, pending_edus): # Make a transaction-sending opentracing span. This span follows on from # all the edus in that transaction. This needs to be done since there is # no active span here, so if the edus were not received by the remote the # span would have no causality and it would be forgotten. # The span_contexts is a generator so that it won't be evaluated if # opentracing is disabled. (Yay speed!) span_contexts = [] keep_destination = whitelisted_homeserver(destination) for edu in pending_edus: context = edu.get_context() if context: span_contexts.append(extract_text_map(json.loads(context))) if keep_destination: edu.strip_context() with start_active_span_follows_from("send_transaction", span_contexts): # Sort based on the order field pending_pdus.sort(key=lambda t: t[1]) pdus = [x[0] for x in pending_pdus] edus = pending_edus success = True logger.debug("TX [%s] _attempt_new_transaction", destination) txn_id = str(self._next_txn_id) logger.debug( "TX [%s] {%s} Attempting new transaction (pdus: %d, edus: %d)", destination, txn_id, len(pdus), len(edus), ) transaction = Transaction.create_new( origin_server_ts=int(self.clock.time_msec()), transaction_id=txn_id, origin=self._server_name, destination=destination, pdus=pdus, edus=edus, ) self._next_txn_id += 1 logger.info( "TX [%s] {%s} Sending transaction [%s], (PDUs: %d, EDUs: %d)", destination, txn_id, transaction.transaction_id, len(pdus), len(edus), ) # Actually send the transaction # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work def json_data_cb(): data = transaction.get_dict() now = int(self.clock.time_msec()) if "pdus" in data: for p in data["pdus"]: if "age_ts" in p: unsigned = p.setdefault("unsigned", {}) unsigned["age"] = now - int(p["age_ts"]) del p["age_ts"] return data try: response = yield self._transport_layer.send_transaction( transaction, json_data_cb) code = 200 except HttpResponseException as e: code = e.code response = e.response if e.code in (401, 404, 429) or 500 <= e.code: logger.info("TX [%s] {%s} got %d response", destination, txn_id, code) raise e logger.info("TX [%s] {%s} got %d response", destination, txn_id, code) if code == 200: for e_id, r in response.get("pdus", {}).items(): if "error" in r: logger.warning( "TX [%s] {%s} Remote returned error for %s: %s", destination, txn_id, e_id, r, ) else: for p in pdus: logger.warning( "TX [%s] {%s} Failed to send event %s", destination, txn_id, p.event_id, ) success = False set_tag(tags.ERROR, not success) return success
async def send_new_transaction( self, destination: str, pdus: List[EventBase], edus: List[Edu], ) -> None: """ Args: destination: The destination to send to (e.g. 'example.org') pdus: In-order list of PDUs to send edus: List of EDUs to send """ # Make a transaction-sending opentracing span. This span follows on from # all the edus in that transaction. This needs to be done since there is # no active span here, so if the edus were not received by the remote the # span would have no causality and it would be forgotten. span_contexts = [] keep_destination = whitelisted_homeserver(destination) for edu in edus: context = edu.get_context() if context: span_contexts.append( extract_text_map(json_decoder.decode(context))) if keep_destination: edu.strip_context() with start_active_span_follows_from("send_transaction", span_contexts): logger.debug("TX [%s] _attempt_new_transaction", destination) txn_id = str(self._next_txn_id) logger.debug( "TX [%s] {%s} Attempting new transaction (pdus: %d, edus: %d)", destination, txn_id, len(pdus), len(edus), ) transaction = Transaction.create_new( origin_server_ts=int(self.clock.time_msec()), transaction_id=txn_id, origin=self._server_name, destination=destination, pdus=pdus, edus=edus, ) self._next_txn_id += 1 logger.info( "TX [%s] {%s} Sending transaction [%s], (PDUs: %d, EDUs: %d)", destination, txn_id, transaction.transaction_id, len(pdus), len(edus), ) # Actually send the transaction # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work # FIXME (richardv): I also believe it no longer works. We (now?) store # "age_ts" in "unsigned" rather than at the top level. See # https://github.com/matrix-org/synapse/issues/8429. def json_data_cb(): data = transaction.get_dict() now = int(self.clock.time_msec()) if "pdus" in data: for p in data["pdus"]: if "age_ts" in p: unsigned = p.setdefault("unsigned", {}) unsigned["age"] = now - int(p["age_ts"]) del p["age_ts"] return data try: response = await self._transport_layer.send_transaction( transaction, json_data_cb) except HttpResponseException as e: code = e.code response = e.response set_tag(tags.ERROR, True) logger.info("TX [%s] {%s} got %d response", destination, txn_id, code) raise logger.info("TX [%s] {%s} got 200 response", destination, txn_id) for e_id, r in response.get("pdus", {}).items(): if "error" in r: logger.warning( "TX [%s] {%s} Remote returned error for %s: %s", destination, txn_id, e_id, r, ) if pdus and destination in self._federation_metrics_domains: last_pdu = pdus[-1] last_pdu_ts_metric.labels(server_name=destination).set( last_pdu.origin_server_ts / 1000)
async def send_new_transaction( self, destination: str, pdus: List[EventBase], edus: List[Edu], ) -> bool: """ Args: destination: The destination to send to (e.g. 'example.org') pdus: In-order list of PDUs to send edus: List of EDUs to send Returns: True iff the transaction was successful """ # Make a transaction-sending opentracing span. This span follows on from # all the edus in that transaction. This needs to be done since there is # no active span here, so if the edus were not received by the remote the # span would have no causality and it would be forgotten. span_contexts = [] keep_destination = whitelisted_homeserver(destination) for edu in edus: context = edu.get_context() if context: span_contexts.append( extract_text_map(json_decoder.decode(context))) if keep_destination: edu.strip_context() with start_active_span_follows_from("send_transaction", span_contexts): success = True logger.debug("TX [%s] _attempt_new_transaction", destination) txn_id = str(self._next_txn_id) logger.debug( "TX [%s] {%s} Attempting new transaction (pdus: %d, edus: %d)", destination, txn_id, len(pdus), len(edus), ) transaction = Transaction.create_new( origin_server_ts=int(self.clock.time_msec()), transaction_id=txn_id, origin=self._server_name, destination=destination, pdus=pdus, edus=edus, ) self._next_txn_id += 1 logger.info( "TX [%s] {%s} Sending transaction [%s], (PDUs: %d, EDUs: %d)", destination, txn_id, transaction.transaction_id, len(pdus), len(edus), ) # Actually send the transaction # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work def json_data_cb(): data = transaction.get_dict() now = int(self.clock.time_msec()) if "pdus" in data: for p in data["pdus"]: if "age_ts" in p: unsigned = p.setdefault("unsigned", {}) unsigned["age"] = now - int(p["age_ts"]) del p["age_ts"] return data try: response = await self._transport_layer.send_transaction( transaction, json_data_cb) code = 200 except HttpResponseException as e: code = e.code response = e.response if e.code in (401, 404, 429) or 500 <= e.code: logger.info("TX [%s] {%s} got %d response", destination, txn_id, code) raise e logger.info("TX [%s] {%s} got %d response", destination, txn_id, code) if code == 200: for e_id, r in response.get("pdus", {}).items(): if "error" in r: logger.warning( "TX [%s] {%s} Remote returned error for %s: %s", destination, txn_id, e_id, r, ) else: for p in pdus: logger.warning( "TX [%s] {%s} Failed to send event %s", destination, txn_id, p.event_id, ) success = False set_tag(tags.ERROR, not success) return success
def send_new_transaction(self, destination, pending_pdus, pending_edus): # Sort based on the order field pending_pdus.sort(key=lambda t: t[1]) pdus = [x[0] for x in pending_pdus] edus = pending_edus success = True logger.debug("TX [%s] _attempt_new_transaction", destination) txn_id = str(self._next_txn_id) logger.debug( "TX [%s] {%s} Attempting new transaction" " (pdus: %d, edus: %d)", destination, txn_id, len(pdus), len(edus), ) logger.debug("TX [%s] Persisting transaction...", destination) transaction = Transaction.create_new( origin_server_ts=int(self.clock.time_msec()), transaction_id=txn_id, origin=self._server_name, destination=destination, pdus=pdus, edus=edus, ) self._next_txn_id += 1 yield self._transaction_actions.prepare_to_send(transaction) logger.debug("TX [%s] Persisted transaction", destination) logger.info( "TX [%s] {%s} Sending transaction [%s]," " (PDUs: %d, EDUs: %d)", destination, txn_id, transaction.transaction_id, len(pdus), len(edus), ) # Actually send the transaction # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work def json_data_cb(): data = transaction.get_dict() now = int(self.clock.time_msec()) if "pdus" in data: for p in data["pdus"]: if "age_ts" in p: unsigned = p.setdefault("unsigned", {}) unsigned["age"] = now - int(p["age_ts"]) del p["age_ts"] return data try: response = yield self._transport_layer.send_transaction( transaction, json_data_cb ) code = 200 except HttpResponseException as e: code = e.code response = e.response if e.code in (401, 404, 429) or 500 <= e.code: logger.info( "TX [%s] {%s} got %d response", destination, txn_id, code ) raise e logger.info( "TX [%s] {%s} got %d response", destination, txn_id, code ) yield self._transaction_actions.delivered( transaction, code, response ) logger.debug("TX [%s] {%s} Marked as delivered", destination, txn_id) if code == 200: for e_id, r in response.get("pdus", {}).items(): if "error" in r: logger.warn( "TX [%s] {%s} Remote returned error for %s: %s", destination, txn_id, e_id, r, ) else: for p in pdus: logger.warn( "TX [%s] {%s} Failed to send event %s", destination, txn_id, p.event_id, ) success = False defer.returnValue(success)
def send_new_transaction(self, destination, pending_pdus, pending_edus): # Sort based on the order field pending_pdus.sort(key=lambda t: t[1]) pdus = [x[0] for x in pending_pdus] edus = pending_edus success = True logger.debug("TX [%s] _attempt_new_transaction", destination) txn_id = str(self._next_txn_id) logger.debug( "TX [%s] {%s} Attempting new transaction" " (pdus: %d, edus: %d)", destination, txn_id, len(pdus), len(edus), ) logger.debug("TX [%s] Persisting transaction...", destination) transaction = Transaction.create_new( origin_server_ts=int(self.clock.time_msec()), transaction_id=txn_id, origin=self._server_name, destination=destination, pdus=pdus, edus=edus, ) self._next_txn_id += 1 yield self._transaction_actions.prepare_to_send(transaction) logger.debug("TX [%s] Persisted transaction", destination) logger.info( "TX [%s] {%s} Sending transaction [%s]," " (PDUs: %d, EDUs: %d)", destination, txn_id, transaction.transaction_id, len(pdus), len(edus), ) # Actually send the transaction # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work def json_data_cb(): data = transaction.get_dict() now = int(self.clock.time_msec()) if "pdus" in data: for p in data["pdus"]: if "age_ts" in p: unsigned = p.setdefault("unsigned", {}) unsigned["age"] = now - int(p["age_ts"]) del p["age_ts"] return data try: response = yield self._transport_layer.send_transaction( transaction, json_data_cb ) code = 200 except HttpResponseException as e: code = e.code response = e.response if e.code in (401, 404, 429) or 500 <= e.code: logger.info("TX [%s] {%s} got %d response", destination, txn_id, code) raise e logger.info("TX [%s] {%s} got %d response", destination, txn_id, code) yield self._transaction_actions.delivered(transaction, code, response) logger.debug("TX [%s] {%s} Marked as delivered", destination, txn_id) if code == 200: for e_id, r in response.get("pdus", {}).items(): if "error" in r: logger.warn( "TX [%s] {%s} Remote returned error for %s: %s", destination, txn_id, e_id, r, ) else: for p in pdus: logger.warn( "TX [%s] {%s} Failed to send event %s", destination, txn_id, p.event_id, ) success = False defer.returnValue(success)
async def on_incoming_transaction( self, origin: str, transaction_id: str, destination: str, transaction_data: JsonDict, ) -> Tuple[int, JsonDict]: # If we receive a transaction we should make sure that kick off handling # any old events in the staging area. if not self._started_handling_of_staged_events: self._started_handling_of_staged_events = True self._handle_old_staged_events() # keep this as early as possible to make the calculated origin ts as # accurate as possible. request_time = self._clock.time_msec() transaction = Transaction( transaction_id=transaction_id, destination=destination, origin=origin, origin_server_ts=transaction_data.get("origin_server_ts"), # type: ignore pdus=transaction_data.get("pdus"), # type: ignore edus=transaction_data.get("edus"), ) if not transaction_id: raise Exception("Transaction missing transaction_id") logger.debug("[%s] Got transaction", transaction_id) # Reject malformed transactions early: reject if too many PDUs/EDUs if len(transaction.pdus) > 50 or len(transaction.edus) > 100: logger.info("Transaction PDU or EDU count too large. Returning 400") return 400, {} # we only process one transaction from each origin at a time. We need to do # this check here, rather than in _on_incoming_transaction_inner so that we # don't cache the rejection in _transaction_resp_cache (so that if the txn # arrives again later, we can process it). current_transaction = self._active_transactions.get(origin) if current_transaction and current_transaction != transaction_id: logger.warning( "Received another txn %s from %s while still processing %s", transaction_id, origin, current_transaction, ) return 429, { "errcode": Codes.UNKNOWN, "error": "Too many concurrent transactions", } # CRITICAL SECTION: we must now not await until we populate _active_transactions # in _on_incoming_transaction_inner. # We wrap in a ResponseCache so that we de-duplicate retried # transactions. return await self._transaction_resp_cache.wrap( (origin, transaction_id), self._on_incoming_transaction_inner, origin, transaction, request_time, )