def _add_device_outbound_poke_to_stream_txn( self, txn, user_id, device_ids, hosts, stream_ids, context, ): for host in hosts: txn.call_after( self._device_list_federation_stream_cache.entity_has_changed, host, stream_ids[-1], ) now = self._clock.time_msec() next_stream_id = iter(stream_ids) self.db.simple_insert_many_txn( txn, table="device_lists_outbound_pokes", values=[ { "destination": destination, "stream_id": next(next_stream_id), "user_id": user_id, "device_id": device_id, "sent": False, "ts": now, "opentracing_context": json.dumps(context) if whitelisted_homeserver(destination) else "{}", } for destination in hosts for device_id in device_ids ], )
def _add_device_change_txn(self, txn, user_id, device_ids, hosts, stream_id): now = self._clock.time_msec() txn.call_after(self._device_list_stream_cache.entity_has_changed, user_id, stream_id) for host in hosts: txn.call_after( self._device_list_federation_stream_cache.entity_has_changed, host, stream_id, ) # Delete older entries in the table, as we really only care about # when the latest change happened. txn.executemany( """ DELETE FROM device_lists_stream WHERE user_id = ? AND device_id = ? AND stream_id < ? """, [(user_id, device_id, stream_id) for device_id in device_ids], ) self._simple_insert_many_txn( txn, table="device_lists_stream", values=[{ "stream_id": stream_id, "user_id": user_id, "device_id": device_id } for device_id in device_ids], ) context = get_active_span_text_map() self._simple_insert_many_txn( txn, table="device_lists_outbound_pokes", values=[{ "destination": destination, "stream_id": stream_id, "user_id": user_id, "device_id": device_id, "sent": False, "ts": now, "opentracing_context": json.dumps(context) if whitelisted_homeserver(destination) else "{}", } for destination in hosts for device_id in device_ids], )
async def new_func(request: SynapseRequest, *args: Any, **kwargs: str) -> Optional[Tuple[int, Any]]: """A callback which can be passed to HttpServer.RegisterPaths Args: request: *args: unused? **kwargs: the dict mapping keys to path components as specified in the path match regexp. Returns: (response code, response object) as returned by the callback method. None if the request has already been handled. """ content = None if request.method in [b"PUT", b"POST"]: # TODO: Handle other method types? other content types? content = parse_json_object_from_request(request) try: with start_active_span("authenticate_request"): origin: Optional[ str] = await authenticator.authenticate_request( request, content) except NoAuthenticationError: origin = None if self.REQUIRE_AUTH: logger.warning( "authenticate_request failed: missing authentication") raise except Exception as e: logger.warning("authenticate_request failed: %s", e) raise # update the active opentracing span with the authenticated entity set_tag("authenticated_entity", origin) # if the origin is authenticated and whitelisted, use its span context # as the parent. context = None if origin and whitelisted_homeserver(origin): context = span_context_from_request(request) if context: servlet_span = active_span() # a scope which uses the origin's context as a parent processing_start_time = time.time() scope = start_active_span_follows_from( "incoming-federation-request", child_of=context, contexts=(servlet_span, ), start_time=processing_start_time, ) else: # just use our context as a parent scope = start_active_span("incoming-federation-request", ) try: with scope: if origin and self.RATELIMIT: with ratelimiter.ratelimit(origin) as d: await d if request._disconnected: logger.warning( "client disconnected before we started processing " "request") return None response = await func(origin, content, request.args, *args, **kwargs) else: response = await func(origin, content, request.args, *args, **kwargs) finally: # if we used the origin's context as the parent, add a new span using # the servlet span as a parent, so that we have a link if context: scope2 = start_active_span_follows_from( "process-federation_request", contexts=(scope.span, ), start_time=processing_start_time, ) scope2.close() return response
def send_new_transaction(self, destination, pending_pdus, pending_edus): # Make a transaction-sending opentracing span. This span follows on from # all the edus in that transaction. This needs to be done since there is # no active span here, so if the edus were not received by the remote the # span would have no causality and it would be forgotten. # The span_contexts is a generator so that it won't be evaluated if # opentracing is disabled. (Yay speed!) span_contexts = [] keep_destination = whitelisted_homeserver(destination) for edu in pending_edus: context = edu.get_context() if context: span_contexts.append(extract_text_map(json.loads(context))) if keep_destination: edu.strip_context() with start_active_span_follows_from("send_transaction", span_contexts): # Sort based on the order field pending_pdus.sort(key=lambda t: t[1]) pdus = [x[0] for x in pending_pdus] edus = pending_edus success = True logger.debug("TX [%s] _attempt_new_transaction", destination) txn_id = str(self._next_txn_id) logger.debug( "TX [%s] {%s} Attempting new transaction (pdus: %d, edus: %d)", destination, txn_id, len(pdus), len(edus), ) transaction = Transaction.create_new( origin_server_ts=int(self.clock.time_msec()), transaction_id=txn_id, origin=self._server_name, destination=destination, pdus=pdus, edus=edus, ) self._next_txn_id += 1 logger.info( "TX [%s] {%s} Sending transaction [%s], (PDUs: %d, EDUs: %d)", destination, txn_id, transaction.transaction_id, len(pdus), len(edus), ) # Actually send the transaction # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work def json_data_cb(): data = transaction.get_dict() now = int(self.clock.time_msec()) if "pdus" in data: for p in data["pdus"]: if "age_ts" in p: unsigned = p.setdefault("unsigned", {}) unsigned["age"] = now - int(p["age_ts"]) del p["age_ts"] return data try: response = yield self._transport_layer.send_transaction( transaction, json_data_cb) code = 200 except HttpResponseException as e: code = e.code response = e.response if e.code in (401, 404, 429) or 500 <= e.code: logger.info("TX [%s] {%s} got %d response", destination, txn_id, code) raise e logger.info("TX [%s] {%s} got %d response", destination, txn_id, code) if code == 200: for e_id, r in response.get("pdus", {}).items(): if "error" in r: logger.warning( "TX [%s] {%s} Remote returned error for %s: %s", destination, txn_id, e_id, r, ) else: for p in pdus: logger.warning( "TX [%s] {%s} Failed to send event %s", destination, txn_id, p.event_id, ) success = False set_tag(tags.ERROR, not success) return success
async def send_new_transaction( self, destination: str, pdus: List[EventBase], edus: List[Edu], ) -> None: """ Args: destination: The destination to send to (e.g. 'example.org') pdus: In-order list of PDUs to send edus: List of EDUs to send """ # Make a transaction-sending opentracing span. This span follows on from # all the edus in that transaction. This needs to be done since there is # no active span here, so if the edus were not received by the remote the # span would have no causality and it would be forgotten. span_contexts = [] keep_destination = whitelisted_homeserver(destination) for edu in edus: context = edu.get_context() if context: span_contexts.append( extract_text_map(json_decoder.decode(context))) if keep_destination: edu.strip_context() with start_active_span_follows_from("send_transaction", span_contexts): logger.debug("TX [%s] _attempt_new_transaction", destination) txn_id = str(self._next_txn_id) logger.debug( "TX [%s] {%s} Attempting new transaction (pdus: %d, edus: %d)", destination, txn_id, len(pdus), len(edus), ) transaction = Transaction.create_new( origin_server_ts=int(self.clock.time_msec()), transaction_id=txn_id, origin=self._server_name, destination=destination, pdus=pdus, edus=edus, ) self._next_txn_id += 1 logger.info( "TX [%s] {%s} Sending transaction [%s], (PDUs: %d, EDUs: %d)", destination, txn_id, transaction.transaction_id, len(pdus), len(edus), ) # Actually send the transaction # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work # FIXME (richardv): I also believe it no longer works. We (now?) store # "age_ts" in "unsigned" rather than at the top level. See # https://github.com/matrix-org/synapse/issues/8429. def json_data_cb(): data = transaction.get_dict() now = int(self.clock.time_msec()) if "pdus" in data: for p in data["pdus"]: if "age_ts" in p: unsigned = p.setdefault("unsigned", {}) unsigned["age"] = now - int(p["age_ts"]) del p["age_ts"] return data try: response = await self._transport_layer.send_transaction( transaction, json_data_cb) except HttpResponseException as e: code = e.code response = e.response set_tag(tags.ERROR, True) logger.info("TX [%s] {%s} got %d response", destination, txn_id, code) raise logger.info("TX [%s] {%s} got 200 response", destination, txn_id) for e_id, r in response.get("pdus", {}).items(): if "error" in r: logger.warning( "TX [%s] {%s} Remote returned error for %s: %s", destination, txn_id, e_id, r, ) if pdus and destination in self._federation_metrics_domains: last_pdu = pdus[-1] last_pdu_ts_metric.labels(server_name=destination).set( last_pdu.origin_server_ts / 1000)
async def send_new_transaction( self, destination: str, pdus: List[EventBase], edus: List[Edu], ) -> bool: """ Args: destination: The destination to send to (e.g. 'example.org') pdus: In-order list of PDUs to send edus: List of EDUs to send Returns: True iff the transaction was successful """ # Make a transaction-sending opentracing span. This span follows on from # all the edus in that transaction. This needs to be done since there is # no active span here, so if the edus were not received by the remote the # span would have no causality and it would be forgotten. span_contexts = [] keep_destination = whitelisted_homeserver(destination) for edu in edus: context = edu.get_context() if context: span_contexts.append( extract_text_map(json_decoder.decode(context))) if keep_destination: edu.strip_context() with start_active_span_follows_from("send_transaction", span_contexts): success = True logger.debug("TX [%s] _attempt_new_transaction", destination) txn_id = str(self._next_txn_id) logger.debug( "TX [%s] {%s} Attempting new transaction (pdus: %d, edus: %d)", destination, txn_id, len(pdus), len(edus), ) transaction = Transaction.create_new( origin_server_ts=int(self.clock.time_msec()), transaction_id=txn_id, origin=self._server_name, destination=destination, pdus=pdus, edus=edus, ) self._next_txn_id += 1 logger.info( "TX [%s] {%s} Sending transaction [%s], (PDUs: %d, EDUs: %d)", destination, txn_id, transaction.transaction_id, len(pdus), len(edus), ) # Actually send the transaction # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work def json_data_cb(): data = transaction.get_dict() now = int(self.clock.time_msec()) if "pdus" in data: for p in data["pdus"]: if "age_ts" in p: unsigned = p.setdefault("unsigned", {}) unsigned["age"] = now - int(p["age_ts"]) del p["age_ts"] return data try: response = await self._transport_layer.send_transaction( transaction, json_data_cb) code = 200 except HttpResponseException as e: code = e.code response = e.response if e.code in (401, 404, 429) or 500 <= e.code: logger.info("TX [%s] {%s} got %d response", destination, txn_id, code) raise e logger.info("TX [%s] {%s} got %d response", destination, txn_id, code) if code == 200: for e_id, r in response.get("pdus", {}).items(): if "error" in r: logger.warning( "TX [%s] {%s} Remote returned error for %s: %s", destination, txn_id, e_id, r, ) else: for p in pdus: logger.warning( "TX [%s] {%s} Failed to send event %s", destination, txn_id, p.event_id, ) success = False set_tag(tags.ERROR, not success) return success
async def new_func(request, *args, **kwargs): """A callback which can be passed to HttpServer.RegisterPaths Args: request (twisted.web.http.Request): *args: unused? **kwargs (dict[unicode, unicode]): the dict mapping keys to path components as specified in the path match regexp. Returns: Tuple[int, object]|None: (response code, response object) as returned by the callback method. None if the request has already been handled. """ content = None if request.method in [b"PUT", b"POST"]: # TODO: Handle other method types? other content types? content = parse_json_object_from_request(request) try: origin = await authenticator.authenticate_request(request, content) except NoAuthenticationError: origin = None if self.REQUIRE_AUTH: logger.warn("authenticate_request failed: missing authentication") raise except Exception as e: logger.warn("authenticate_request failed: %s", e) raise request_tags = { "request_id": request.get_request_id(), tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER, tags.HTTP_METHOD: request.get_method(), tags.HTTP_URL: request.get_redacted_uri(), tags.PEER_HOST_IPV6: request.getClientIP(), "authenticated_entity": origin, "servlet_name": request.request_metrics.name, } # Only accept the span context if the origin is authenticated # and whitelisted if origin and whitelisted_homeserver(origin): scope = start_active_span_from_request( request, "incoming-federation-request", tags=request_tags ) else: scope = start_active_span( "incoming-federation-request", tags=request_tags ) with scope: if origin: with ratelimiter.ratelimit(origin) as d: await d response = await func( origin, content, request.args, *args, **kwargs ) else: response = await func( origin, content, request.args, *args, **kwargs ) return response
async def new_func(request: SynapseRequest, *args: Any, **kwargs: str) -> Optional[Tuple[int, Any]]: """A callback which can be passed to HttpServer.RegisterPaths Args: request: *args: unused? **kwargs: the dict mapping keys to path components as specified in the path match regexp. Returns: (response code, response object) as returned by the callback method. None if the request has already been handled. """ content = None if request.method in [b"PUT", b"POST"]: # TODO: Handle other method types? other content types? content = parse_json_object_from_request(request) try: origin: Optional[ str] = await authenticator.authenticate_request( request, content) except NoAuthenticationError: origin = None if self.REQUIRE_AUTH: logger.warning( "authenticate_request failed: missing authentication") raise except Exception as e: logger.warning("authenticate_request failed: %s", e) raise request_tags = { SynapseTags.REQUEST_ID: request.get_request_id(), tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER, tags.HTTP_METHOD: request.get_method(), tags.HTTP_URL: request.get_redacted_uri(), tags.PEER_HOST_IPV6: request.getClientIP(), "authenticated_entity": origin, "servlet_name": request.request_metrics.name, } # Only accept the span context if the origin is authenticated # and whitelisted if origin and whitelisted_homeserver(origin): scope = start_active_span_from_request( request, "incoming-federation-request", tags=request_tags) else: scope = start_active_span("incoming-federation-request", tags=request_tags) with scope: opentracing.inject_response_headers(request.responseHeaders) if origin and self.RATELIMIT: with ratelimiter.ratelimit(origin) as d: await d if request._disconnected: logger.warning( "client disconnected before we started processing " "request") return None response = await func(origin, content, request.args, *args, **kwargs) else: response = await func(origin, content, request.args, *args, **kwargs) return response