def _send_pdu(self, pdu, destinations): # We loop through all destinations to see whether we already have # a transaction in progress. If we do, stick it in the pending_pdus # table and we'll get back to it later. order = self._order self._order += 1 destinations = set(destinations) destinations = set(dest for dest in destinations if self.can_send_to(dest)) logger.debug("Sending to: %s", str(destinations)) if not destinations: return sent_pdus_destination_dist.inc_by(len(destinations)) for destination in destinations: self.pending_pdus_by_dest.setdefault(destination, []).append( (pdu, order)) preserve_context_over_fn(self._attempt_new_transaction, destination)
def get_user_by_req(self, request, allow_guest=False, rights="access"): """ Get a registered user's ID. Args: request - An HTTP request with an access_token query parameter. Returns: defer.Deferred: resolves to a ``synapse.types.Requester`` object Raises: AuthError if no user by that token exists or the token is invalid. """ # Can optionally look elsewhere in the request (e.g. headers) try: user_id = yield self._get_appservice_user_id(request) if user_id: request.authenticated_entity = user_id defer.returnValue(synapse.types.create_requester(user_id)) access_token = get_access_token_from_request( request, self.TOKEN_NOT_FOUND_HTTP_STATUS ) user_info = yield self.get_user_by_access_token(access_token, rights) user = user_info["user"] token_id = user_info["token_id"] is_guest = user_info["is_guest"] # device_id may not be present if get_user_by_access_token has been # stubbed out. device_id = user_info.get("device_id") ip_addr = self.hs.get_ip_from_request(request) user_agent = request.requestHeaders.getRawHeaders( "User-Agent", default=[""] )[0] if user and access_token and ip_addr: preserve_context_over_fn( self.store.insert_client_ip, user=user, access_token=access_token, ip=ip_addr, user_agent=user_agent, device_id=device_id, ) if is_guest and not allow_guest: raise AuthError( 403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN ) request.authenticated_entity = user.to_string() defer.returnValue(synapse.types.create_requester( user, token_id, is_guest, device_id)) except KeyError: raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", errcode=Codes.MISSING_TOKEN )
def enqueue_presence(self, destination, states): self.pending_presence_by_dest.setdefault(destination, {}).update({ state.user_id: state for state in states }) preserve_context_over_fn( self._attempt_new_transaction, destination )
def send_device_messages(self, destination): if destination == self.server_name or destination == "localhost": return if not self.can_send_to(destination): return preserve_context_over_fn(self._attempt_new_transaction, destination)
def get_user_by_req(self, request, allow_guest=False, rights="access"): """ Get a registered user's ID. Args: request - An HTTP request with an access_token query parameter. Returns: defer.Deferred: resolves to a ``synapse.types.Requester`` object Raises: AuthError if no user by that token exists or the token is invalid. """ # Can optionally look elsewhere in the request (e.g. headers) try: user_id = yield self._get_appservice_user_id(request) if user_id: request.authenticated_entity = user_id defer.returnValue(synapse.types.create_requester(user_id)) access_token = get_access_token_from_request( request, self.TOKEN_NOT_FOUND_HTTP_STATUS) user_info = yield self.get_user_by_access_token( access_token, rights) user = user_info["user"] token_id = user_info["token_id"] is_guest = user_info["is_guest"] # device_id may not be present if get_user_by_access_token has been # stubbed out. device_id = user_info.get("device_id") ip_addr = self.hs.get_ip_from_request(request) user_agent = request.requestHeaders.getRawHeaders("User-Agent", default=[""])[0] if user and access_token and ip_addr: preserve_context_over_fn( self.store.insert_client_ip, user=user, access_token=access_token, ip=ip_addr, user_agent=user_agent, device_id=device_id, ) if is_guest and not allow_guest: raise AuthError(403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN) request.authenticated_entity = user.to_string() defer.returnValue( synapse.types.create_requester(user, token_id, is_guest, device_id)) except KeyError: raise AuthError(self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", errcode=Codes.MISSING_TOKEN)
def send_presence(self, destination, states): if not self.can_send_to(destination): return self.pending_presence_by_dest.setdefault(destination, {}).update( {state.user_id: state for state in states}) preserve_context_over_fn(self._attempt_new_transaction, destination)
def enqueue_edu(self, edu): destination = edu.destination if not self.can_send_to(destination): return self.pending_edus_by_dest.setdefault(destination, []).append(edu) preserve_context_over_fn(self._attempt_new_transaction, destination)
def enqueue_device_messages(self, destination): if destination == self.server_name or destination == "localhost": return if not self.can_send_to(destination): return preserve_context_over_fn( self._attempt_new_transaction, destination )
def send_failure(self, failure, destination): if destination == self.server_name or destination == "localhost": return if not self.can_send_to(destination): return self.pending_failures_by_dest.setdefault(destination, []).append(failure) preserve_context_over_fn(self._attempt_new_transaction, destination)
def get_user_by_req(self, request, allow_guest=False): """ Get a registered user's ID. Args: request - An HTTP request with an access_token query parameter. Returns: tuple of: UserID (str) Access token ID (str) Raises: AuthError if no user by that token exists or the token is invalid. """ # Can optionally look elsewhere in the request (e.g. headers) try: user_id = yield self._get_appservice_user_id(request.args) if user_id: request.authenticated_entity = user_id defer.returnValue( Requester(UserID.from_string(user_id), "", False) ) access_token = request.args["access_token"][0] user_info = yield self._get_user_by_access_token(access_token) user = user_info["user"] token_id = user_info["token_id"] is_guest = user_info["is_guest"] ip_addr = self.hs.get_ip_from_request(request) user_agent = request.requestHeaders.getRawHeaders( "User-Agent", default=[""] )[0] if user and access_token and ip_addr: preserve_context_over_fn( self.store.insert_client_ip, user=user, access_token=access_token, ip=ip_addr, user_agent=user_agent ) if is_guest and not allow_guest: raise AuthError( 403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN ) request.authenticated_entity = user.to_string() defer.returnValue(Requester(user, token_id, is_guest)) except KeyError: raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", errcode=Codes.MISSING_TOKEN )
def enqueue_failure(self, failure, destination): if destination == self.server_name or destination == "localhost": return if not self.can_send_to(destination): return self.pending_failures_by_dest.setdefault( destination, [] ).append(failure) preserve_context_over_fn( self._attempt_new_transaction, destination )
def get_user_by_req(self, request, allow_guest=False, rights="access"): """ Get a registered user's ID. Args: request - An HTTP request with an access_token query parameter. Returns: tuple of: UserID (str) Access token ID (str) Raises: AuthError if no user by that token exists or the token is invalid. """ # Can optionally look elsewhere in the request (e.g. headers) try: user_id = yield self._get_appservice_user_id(request.args) if user_id: request.authenticated_entity = user_id defer.returnValue( Requester(UserID.from_string(user_id), "", False)) access_token = request.args["access_token"][0] user_info = yield self.get_user_by_access_token( access_token, rights) user = user_info["user"] token_id = user_info["token_id"] is_guest = user_info["is_guest"] ip_addr = self.hs.get_ip_from_request(request) user_agent = request.requestHeaders.getRawHeaders("User-Agent", default=[""])[0] if user and access_token and ip_addr: preserve_context_over_fn(self.store.insert_client_ip, user=user, access_token=access_token, ip=ip_addr, user_agent=user_agent) if is_guest and not allow_guest: raise AuthError(403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN) request.authenticated_entity = user.to_string() defer.returnValue(Requester(user, token_id, is_guest)) except KeyError: raise AuthError(self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", errcode=Codes.MISSING_TOKEN)
def enqueue_edu(self, edu, key=None): destination = edu.destination if not self.can_send_to(destination): return if key: self.pending_edus_keyed_by_dest.setdefault( destination, {} )[(edu.edu_type, key)] = edu else: self.pending_edus_by_dest.setdefault(destination, []).append(edu) preserve_context_over_fn( self._attempt_new_transaction, destination )
def request(self, method, uri, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach # counters to it outgoing_requests_counter.inc(method) d = preserve_context_over_fn( self.agent.request, method, uri, *args, **kwargs ) logger.info("Sending request %s %s", method, uri) def _cb(response): incoming_responses_counter.inc(method, response.code) logger.info( "Received response to %s %s: %s", method, uri, response.code ) return response def _eb(failure): incoming_responses_counter.inc(method, "ERR") logger.info( "Error sending request to %s %s: %s %s", method, uri, failure.type, failure.getErrorMessage() ) return failure d.addCallbacks(_cb, _eb) return d
def runInteraction(self, desc, func, *args, **kwargs): """Wraps the .runInteraction() method on the underlying db_pool.""" current_context = LoggingContext.current_context() start_time = time.time() * 1000 after_callbacks = [] def inner_func(conn, *args, **kwargs): with LoggingContext("runInteraction") as context: sql_scheduling_timer.inc_by(time.time() * 1000 - start_time) if self.database_engine.is_connection_closed(conn): logger.debug("Reconnecting closed database connection") conn.reconnect() current_context.copy_to(context) return self._new_transaction( conn, desc, after_callbacks, func, *args, **kwargs ) result = yield preserve_context_over_fn( self._db_pool.runWithConnection, inner_func, *args, **kwargs ) for after_callback, after_args in after_callbacks: after_callback(*after_args) defer.returnValue(result)
def get_raw(self, uri, args={}): """ Gets raw text from the given URI. Args: uri (str): The URI to request, not including query parameters args (dict): A dictionary used to create query strings, defaults to None. **Note**: The value of each key is assumed to be an iterable and *not* a string. Returns: Deferred: Succeeds when we get *any* 2xx HTTP response, with the HTTP body at text. Raises: On a non-2xx HTTP response. The response body will be used as the error message. """ if len(args): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) response = yield self.request( "GET", uri.encode("ascii"), headers=Headers({ b"User-Agent": [self.user_agent], }) ) body = yield preserve_context_over_fn(readBody, response) if 200 <= response.code < 300: defer.returnValue(body) else: raise CodeMessageException(response.code, body)
def fetch_server_key(server_name, ssl_context_factory, path=KEY_API_V1): """Fetch the keys for a remote server.""" factory = SynapseKeyClientFactory() factory.path = path factory.host = server_name endpoint = matrix_federation_endpoint(reactor, server_name, ssl_context_factory, timeout=30) for i in range(5): try: protocol = yield preserve_context_over_fn(endpoint.connect, factory) server_response, server_certificate = yield preserve_context_over_deferred( protocol.remote_key) defer.returnValue((server_response, server_certificate)) return except SynapseKeyClientError as e: logger.exception("Error getting key for %r" % (server_name, )) if e.status.startswith("4"): # Don't retry for 4xx responses. raise IOError("Cannot get key for %r" % server_name) except Exception as e: logger.exception(e) raise IOError("Cannot get key for %r" % server_name)
def runInteraction(self, desc, func, *args, **kwargs): """Wraps the .runInteraction() method on the underlying db_pool.""" current_context = LoggingContext.current_context() start_time = time.time() * 1000 after_callbacks = [] def inner_func(conn, *args, **kwargs): with LoggingContext("runInteraction") as context: sql_scheduling_timer.inc_by(time.time() * 1000 - start_time) if self.database_engine.is_connection_closed(conn): logger.debug("Reconnecting closed database connection") conn.reconnect() current_context.copy_to(context) return self._new_transaction(conn, desc, after_callbacks, func, *args, **kwargs) result = yield preserve_context_over_fn( self._db_pool.runWithConnection, inner_func, *args, **kwargs) for after_callback, after_args in after_callbacks: after_callback(*after_args) defer.returnValue(result)
def fetch_server_key(server_name, ssl_context_factory, path=KEY_API_V1): """Fetch the keys for a remote server.""" factory = SynapseKeyClientFactory() factory.path = path endpoint = matrix_federation_endpoint( reactor, server_name, ssl_context_factory, timeout=30 ) for i in range(5): try: protocol = yield preserve_context_over_fn( endpoint.connect, factory ) server_response, server_certificate = yield preserve_context_over_deferred( protocol.remote_key ) defer.returnValue((server_response, server_certificate)) return except SynapseKeyClientError as e: logger.exception("Error getting key for %r" % (server_name,)) if e.status.startswith("4"): # Don't retry for 4xx responses. raise IOError("Cannot get key for %r" % server_name) except Exception as e: logger.exception(e) raise IOError("Cannot get key for %r" % server_name)
def put_json(self, destination, path, data={}, json_data_callback=None, long_retries=False, timeout=None): """ Sends the specifed json data using PUT Args: destination (str): The remote server to send the HTTP request to. path (str): The HTTP path. data (dict): A dict containing the data that will be used as the request body. This will be encoded as JSON. json_data_callback (callable): A callable returning the dict to use as the request body. long_retries (bool): A boolean that indicates whether we should retry for a short or long time. timeout(int): How long to try (in ms) the destination for before giving up. None indicates no timeout. Returns: Deferred: Succeeds when we get a 2xx HTTP response. The result will be the decoded JSON body. On a 4xx or 5xx error response a CodeMessageException is raised. """ if not json_data_callback: def json_data_callback(): return data def body_callback(method, url_bytes, headers_dict): json_data = json_data_callback() self.sign_request(destination, method, url_bytes, headers_dict, json_data) producer = _JsonProducer(json_data) return producer response = yield self._create_request( destination.encode("ascii"), "PUT", path.encode("ascii"), body_callback=body_callback, headers_dict={"Content-Type": ["application/json"]}, long_retries=long_retries, timeout=timeout, ) if 200 <= response.code < 300: # We need to update the transactions table to say it was sent? c_type = response.headers.getRawHeaders("Content-Type") if "application/json" not in c_type: raise RuntimeError("Content-Type not application/json") body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body))
def send_request(): request_deferred = preserve_context_over_fn( self.agent.request, method, url_bytes, Headers(headers_dict), producer ) return self.clock.time_bound_deferred( request_deferred, time_out=timeout / 1000.0 if timeout else 60 )
def get_json(self, destination, path, args={}, retry_on_dns_fail=True, timeout=None): """ GETs some json from the given host homeserver and path Args: destination (str): The remote server to send the HTTP request to. path (str): The HTTP path. args (dict): A dictionary used to create query strings, defaults to None. timeout (int): How long to try (in ms) the destination for before giving up. None indicates no timeout and that the request will be retried. Returns: Deferred: Succeeds when we get *any* HTTP response. The result of the deferred is a tuple of `(code, response)`, where `response` is a dict representing the decoded JSON body. """ logger.debug("get_json args: %s", args) encoded_args = {} for k, vs in args.items(): if isinstance(vs, basestring): vs = [vs] encoded_args[k] = [v.encode("UTF-8") for v in vs] query_bytes = urllib.urlencode(encoded_args, True) logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) def body_callback(method, url_bytes, headers_dict): self.sign_request(destination, method, url_bytes, headers_dict) return None response = yield self._create_request( destination.encode("ascii"), "GET", path.encode("ascii"), query_bytes=query_bytes, body_callback=body_callback, retry_on_dns_fail=retry_on_dns_fail, timeout=timeout, ) if 200 <= response.code < 300: # We need to update the transactions table to say it was sent? c_type = response.headers.getRawHeaders("Content-Type") if "application/json" not in c_type: raise RuntimeError("Content-Type not application/json") body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body))
def send_request(): request_deferred = preserve_context_over_fn( self.agent.request, method, url_bytes, Headers(headers_dict), producer) return self.clock.time_bound_deferred( request_deferred, time_out=timeout / 1000. if timeout else 60, )
def _generate_remote_thumbnails(self, server_name, media_id, media_info): media_type = media_info["media_type"] file_id = media_info["filesystem_id"] requirements = self._get_thumbnail_requirements(media_type) if not requirements: return remote_thumbnails = [] input_path = self.filepaths.remote_media_filepath(server_name, file_id) thumbnailer = Thumbnailer(input_path) m_width = thumbnailer.width m_height = thumbnailer.height def generate_thumbnails(): if m_width * m_height >= self.max_image_pixels: logger.info("Image too large to thumbnail %r x %r > %r", m_width, m_height, self.max_image_pixels) return scales = set() crops = set() for r_width, r_height, r_method, r_type in requirements: if r_method == "scale": t_width, t_height = thumbnailer.aspect(r_width, r_height) scales.add((min(m_width, t_width), min(m_height, t_height), r_type)) elif r_method == "crop": crops.add((r_width, r_height, r_type)) for t_width, t_height, t_type in scales: t_method = "scale" t_path = self.filepaths.remote_media_thumbnail( server_name, file_id, t_width, t_height, t_type, t_method ) self._makedirs(t_path) t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) remote_thumbnails.append([server_name, media_id, file_id, t_width, t_height, t_type, t_method, t_len]) for t_width, t_height, t_type in crops: if (t_width, t_height, t_type) in scales: # If the aspect ratio of the cropped thumbnail matches a purely # scaled one then there is no point in calculating a separate # thumbnail. continue t_method = "crop" t_path = self.filepaths.remote_media_thumbnail( server_name, file_id, t_width, t_height, t_type, t_method ) self._makedirs(t_path) t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) remote_thumbnails.append([server_name, media_id, file_id, t_width, t_height, t_type, t_method, t_len]) yield preserve_context_over_fn(threads.deferToThread, generate_thumbnails) for r in remote_thumbnails: yield self.store.store_remote_media_thumbnail(*r) defer.returnValue({"width": m_width, "height": m_height})
def get_json(self, destination, path, args={}, retry_on_dns_fail=True, timeout=None): """ GETs some json from the given host homeserver and path Args: destination (str): The remote server to send the HTTP request to. path (str): The HTTP path. args (dict): A dictionary used to create query strings, defaults to None. timeout (int): How long to try (in ms) the destination for before giving up. None indicates no timeout and that the request will be retried. Returns: Deferred: Succeeds when we get *any* HTTP response. The result of the deferred is a tuple of `(code, response)`, where `response` is a dict representing the decoded JSON body. """ logger.debug("get_json args: %s", args) encoded_args = {} for k, vs in args.items(): if isinstance(vs, basestring): vs = [vs] encoded_args[k] = [v.encode("UTF-8") for v in vs] query_bytes = urllib.urlencode(encoded_args, True) logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) def body_callback(method, url_bytes, headers_dict): self.sign_request(destination, method, url_bytes, headers_dict) return None response = yield self._create_request( destination.encode("ascii"), "GET", path.encode("ascii"), query_bytes=query_bytes, body_callback=body_callback, retry_on_dns_fail=retry_on_dns_fail, timeout=timeout, ) if 200 <= response.code < 300: # We need to update the transactions table to say it was sent? c_type = response.headers.getRawHeaders("Content-Type") if "application/json" not in c_type: raise RuntimeError( "Content-Type not application/json" ) body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body))
def put_json(self, destination, path, data={}, json_data_callback=None, long_retries=False, timeout=None): """ Sends the specifed json data using PUT Args: destination (str): The remote server to send the HTTP request to. path (str): The HTTP path. data (dict): A dict containing the data that will be used as the request body. This will be encoded as JSON. json_data_callback (callable): A callable returning the dict to use as the request body. long_retries (bool): A boolean that indicates whether we should retry for a short or long time. timeout(int): How long to try (in ms) the destination for before giving up. None indicates no timeout. Returns: Deferred: Succeeds when we get a 2xx HTTP response. The result will be the decoded JSON body. On a 4xx or 5xx error response a CodeMessageException is raised. """ if not json_data_callback: def json_data_callback(): return data def body_callback(method, url_bytes, headers_dict): json_data = json_data_callback() self.sign_request( destination, method, url_bytes, headers_dict, json_data ) producer = _JsonProducer(json_data) return producer response = yield self._create_request( destination.encode("ascii"), "PUT", path.encode("ascii"), body_callback=body_callback, headers_dict={"Content-Type": ["application/json"]}, long_retries=long_retries, timeout=timeout, ) if 200 <= response.code < 300: # We need to update the transactions table to say it was sent? c_type = response.headers.getRawHeaders("Content-Type") if "application/json" not in c_type: raise RuntimeError( "Content-Type not application/json" ) body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body))
def get_file(self, url, output_stream, max_size=None): """GETs a file from a given URL Args: url (str): The URL to GET output_stream (file): File to write the response body to. Returns: A (int,dict,string,int) tuple of the file length, dict of the response headers, absolute URI of the response and HTTP response code. """ response = yield self.request( "GET", url.encode("ascii"), headers=Headers({ b"User-Agent": [self.user_agent], }) ) headers = dict(response.headers.getAllRawHeaders()) if 'Content-Length' in headers and headers['Content-Length'] > max_size: logger.warn("Requested URL is too large > %r bytes" % (self.max_size,)) raise SynapseError( 502, "Requested file is too large > %r bytes" % (self.max_size,), Codes.TOO_LARGE, ) if response.code > 299: logger.warn("Got %d when downloading %s" % (response.code, url)) raise SynapseError( 502, "Got error %d" % (response.code,), Codes.UNKNOWN, ) # TODO: if our Content-Type is HTML or something, just read the first # N bytes into RAM rather than saving it all to disk only to read it # straight back in again try: length = yield preserve_context_over_fn( _readBodyToFile, response, output_stream, max_size ) except Exception as e: logger.exception("Failed to download body") raise SynapseError( 502, ("Failed to download remote body: %s" % e), Codes.UNKNOWN, ) defer.returnValue((length, headers, response.request.absoluteURI, response.code))
def send_edu(self, destination, edu_type, content, key=None): edu = Edu( origin=self.server_name, destination=destination, edu_type=edu_type, content=content, ) if not self.can_send_to(destination): return sent_edus_counter.inc() if key: self.pending_edus_keyed_by_dest.setdefault(destination, {})[(edu.edu_type, key)] = edu else: self.pending_edus_by_dest.setdefault(destination, []).append(edu) preserve_context_over_fn(self._attempt_new_transaction, destination)
def generate_local_exact_thumbnail(self, media_id, t_width, t_height, t_method, t_type): input_path = self.filepaths.local_media_filepath(media_id) t_path = self.filepaths.local_media_thumbnail(media_id, t_width, t_height, t_type, t_method) self._makedirs(t_path) t_len = yield preserve_context_over_fn( threads.deferToThread, self._generate_thumbnail, input_path, t_path, t_width, t_height, t_method, t_type ) if t_len: yield self.store.store_local_thumbnail(media_id, t_width, t_height, t_type, t_method, t_len) defer.returnValue(t_path)
def enqueue_pdu(self, pdu, destinations, order): # We loop through all destinations to see whether we already have # a transaction in progress. If we do, stick it in the pending_pdus # table and we'll get back to it later. destinations = set(destinations) destinations = set( dest for dest in destinations if self.can_send_to(dest) ) logger.debug("Sending to: %s", str(destinations)) if not destinations: return for destination in destinations: self.pending_pdus_by_dest.setdefault(destination, []).append( (pdu, order) ) preserve_context_over_fn( self._attempt_new_transaction, destination )
def get_file(self, destination, path, output_stream, args={}, retry_on_dns_fail=True, max_size=None): """GETs a file from a given homeserver Args: destination (str): The remote server to send the HTTP request to. path (str): The HTTP path to GET. output_stream (file): File to write the response body to. args (dict): Optional dictionary used to create the query string. Returns: A (int,dict) tuple of the file length and a dict of the response headers. """ encoded_args = {} for k, vs in args.items(): if isinstance(vs, basestring): vs = [vs] encoded_args[k] = [v.encode("UTF-8") for v in vs] query_bytes = urllib.urlencode(encoded_args, True) logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) def body_callback(method, url_bytes, headers_dict): self.sign_request(destination, method, url_bytes, headers_dict) return None response = yield self._create_request( destination.encode("ascii"), "GET", path.encode("ascii"), query_bytes=query_bytes, body_callback=body_callback, retry_on_dns_fail=retry_on_dns_fail) headers = dict(response.headers.getAllRawHeaders()) try: length = yield preserve_context_over_fn(_readBodyToFile, response, output_stream, max_size) except: logger.exception("Failed to download body") raise defer.returnValue((length, headers))
def post_json_get_json(self, uri, post_json): json_str = encode_canonical_json(post_json) logger.debug("HTTP POST %s -> %s", json_str, uri) response = yield self.request( "POST", uri.encode("ascii"), headers=Headers({b"Content-Type": [b"application/json"], b"User-Agent": [self.user_agent]}), bodyProducer=FileBodyProducer(StringIO(json_str)), ) body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body))
def get_file(self, destination, path, output_stream, args={}, retry_on_dns_fail=True, max_size=None): """GETs a file from a given homeserver Args: destination (str): The remote server to send the HTTP request to. path (str): The HTTP path to GET. output_stream (file): File to write the response body to. args (dict): Optional dictionary used to create the query string. Returns: A (int,dict) tuple of the file length and a dict of the response headers. """ encoded_args = {} for k, vs in args.items(): if isinstance(vs, basestring): vs = [vs] encoded_args[k] = [v.encode("UTF-8") for v in vs] query_bytes = urllib.urlencode(encoded_args, True) logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) def body_callback(method, url_bytes, headers_dict): self.sign_request(destination, method, url_bytes, headers_dict) return None response = yield self._create_request( destination.encode("ascii"), "GET", path.encode("ascii"), query_bytes=query_bytes, body_callback=body_callback, retry_on_dns_fail=retry_on_dns_fail ) headers = dict(response.headers.getAllRawHeaders()) try: length = yield preserve_context_over_fn( _readBodyToFile, response, output_stream, max_size ) except: logger.exception("Failed to download body") raise defer.returnValue((length, headers))
def post_json_get_json(self, uri, post_json): json_str = encode_canonical_json(post_json) logger.debug("HTTP POST %s -> %s", json_str, uri) response = yield self.request( "POST", uri.encode("ascii"), headers=Headers({ "Content-Type": ["application/json"] }), bodyProducer=FileBodyProducer(StringIO(json_str)) ) body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body))
def post_json(self, destination, path, data={}, long_retries=True): """ Sends the specifed json data using POST Args: destination (str): The remote server to send the HTTP request to. path (str): The HTTP path. data (dict): A dict containing the data that will be used as the request body. This will be encoded as JSON. long_retries (bool): A boolean that indicates whether we should retry for a short or long time. Returns: Deferred: Succeeds when we get a 2xx HTTP response. The result will be the decoded JSON body. On a 4xx or 5xx error response a CodeMessageException is raised. """ def body_callback(method, url_bytes, headers_dict): self.sign_request( destination, method, url_bytes, headers_dict, data ) return _JsonProducer(data) response = yield self._create_request( destination.encode("ascii"), "POST", path.encode("ascii"), body_callback=body_callback, headers_dict={"Content-Type": ["application/json"]}, long_retries=True, ) if 200 <= response.code < 300: # We need to update the transactions table to say it was sent? c_type = response.headers.getRawHeaders("Content-Type") if "application/json" not in c_type: raise RuntimeError( "Content-Type not application/json" ) body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body))
def post_urlencoded_get_raw(self, url, args={}): query_bytes = urllib.urlencode(encode_urlencode_args(args), True) response = yield self.request( "POST", url.encode("ascii"), bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], b"User-Agent": [self.user_agent], })) try: body = yield preserve_context_over_fn(readBody, response) defer.returnValue(body) except PartialDownloadError as e: # twisted dislikes google's response, no content length. defer.returnValue(e.response)
def post_urlencoded_get_json(self, uri, args={}): # TODO: Do we ever want to log message contents? logger.debug("post_urlencoded_get_json args: %s", args) query_bytes = urllib.urlencode(args, True) response = yield self.request( "POST", uri.encode("ascii"), headers=Headers( {b"Content-Type": [b"application/x-www-form-urlencoded"], b"User-Agent": [self.user_agent]} ), bodyProducer=FileBodyProducer(StringIO(query_bytes)), ) body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body))
def post_urlencoded_get_json(self, uri, args={}): # TODO: Do we ever want to log message contents? logger.debug("post_urlencoded_get_json args: %s", args) query_bytes = urllib.urlencode(encode_urlencode_args(args), True) response = yield self.request( "POST", uri.encode("ascii"), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], b"User-Agent": [self.user_agent], }), bodyProducer=FileBodyProducer(StringIO(query_bytes))) body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body))
def post_urlencoded_get_raw(self, url, args={}): query_bytes = urllib.urlencode(args, True) response = yield self.request( "POST", url.encode("ascii"), bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers( {b"Content-Type": [b"application/x-www-form-urlencoded"], b"User-Agent": [self.user_agent]} ), ) try: body = yield preserve_context_over_fn(readBody, response) defer.returnValue(body) except PartialDownloadError as e: # twisted dislikes google's response, no content length. defer.returnValue(e.response)
def generate_local_exact_thumbnail(self, media_id, t_width, t_height, t_method, t_type): input_path = self.filepaths.local_media_filepath(media_id) t_path = self.filepaths.local_media_thumbnail(media_id, t_width, t_height, t_type, t_method) self._makedirs(t_path) t_len = yield preserve_context_over_fn(threads.deferToThread, self._generate_thumbnail, input_path, t_path, t_width, t_height, t_method, t_type) if t_len: yield self.store.store_local_thumbnail(media_id, t_width, t_height, t_type, t_method, t_len) defer.returnValue(t_path)
def post_json(self, destination, path, data={}): """ Sends the specifed json data using POST Args: destination (str): The remote server to send the HTTP request to. path (str): The HTTP path. data (dict): A dict containing the data that will be used as the request body. This will be encoded as JSON. Returns: Deferred: Succeeds when we get a 2xx HTTP response. The result will be the decoded JSON body. On a 4xx or 5xx error response a CodeMessageException is raised. """ def body_callback(method, url_bytes, headers_dict): self.sign_request( destination, method, url_bytes, headers_dict, data ) return _JsonProducer(data) response = yield self._create_request( destination.encode("ascii"), "POST", path.encode("ascii"), body_callback=body_callback, headers_dict={"Content-Type": ["application/json"]}, ) if 200 <= response.code < 300: # We need to update the transactions table to say it was sent? c_type = response.headers.getRawHeaders("Content-Type") if "application/json" not in c_type: raise RuntimeError( "Content-Type not application/json" ) body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body))
def request(self, method, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach # counters to it outgoing_requests_counter.inc(method) d = preserve_context_over_fn( self.agent.request, method, *args, **kwargs ) def _cb(response): incoming_responses_counter.inc(method, response.code) return response def _eb(failure): incoming_responses_counter.inc(method, "ERR") return failure d.addCallbacks(_cb, _eb) return d
def put_json(self, uri, json_body, args={}): """ Puts some json to the given URI. Args: uri (str): The URI to request, not including query parameters json_body (dict): The JSON to put in the HTTP body, args (dict): A dictionary used to create query strings, defaults to None. **Note**: The value of each key is assumed to be an iterable and *not* a string. Returns: Deferred: Succeeds when we get *any* 2xx HTTP response, with the HTTP body as JSON. Raises: On a non-2xx HTTP response. """ if len(args): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) json_str = encode_canonical_json(json_body) response = yield self.request( "PUT", uri.encode("ascii"), headers=Headers({ b"User-Agent": [self.user_agent], "Content-Type": ["application/json"] }), bodyProducer=FileBodyProducer(StringIO(json_str)) ) body = yield preserve_context_over_fn(readBody, response) if 200 <= response.code < 300: defer.returnValue(json.loads(body)) else: # NB: This is explicitly not json.loads(body)'d because the contract # of CodeMessageException is a *string* message. Callers can always # load it into JSON if they want. raise CodeMessageException(response.code, body)
def post_json_get_json(self, uri, post_json): json_str = encode_canonical_json(post_json) logger.debug("HTTP POST %s -> %s", json_str, uri) response = yield self.request("POST", uri.encode("ascii"), headers=Headers({ b"Content-Type": [b"application/json"], b"User-Agent": [self.user_agent], }), bodyProducer=FileBodyProducer( StringIO(json_str))) body = yield preserve_context_over_fn(readBody, response) if 200 <= response.code < 300: defer.returnValue(json.loads(body)) else: raise self._exceptionFromFailedRequest(response, body) defer.returnValue(json.loads(body))
def get_json(self, uri, args={}): """ Gets some json from the given URI. Args: uri (str): The URI to request, not including query parameters args (dict): A dictionary used to create query strings, defaults to None. **Note**: The value of each key is assumed to be an iterable and *not* a string. Returns: Deferred: Succeeds when we get *any* 2xx HTTP response, with the HTTP body as JSON. Raises: On a non-2xx HTTP response. The response body will be used as the error message. """ if len(args): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) response = yield self.request( "GET", uri.encode("ascii"), headers=Headers({ b"User-Agent": [self.version_string], }) ) body = yield preserve_context_over_fn(readBody, response) if 200 <= response.code < 300: defer.returnValue(json.loads(body)) else: # NB: This is explicitly not json.loads(body)'d because the contract # of CodeMessageException is a *string* message. Callers can always # load it into JSON if they want. raise CodeMessageException(response.code, body)
def user_joined_room(distributor, user, room_id): return preserve_context_over_fn(distributor.fire, "user_joined_room", user=user, room_id=room_id)
def verify_json_objects_for_server(self, server_and_json): """Bulk verfies signatures of json objects, bulk fetching keys as necessary. Args: server_and_json (list): List of pairs of (server_name, json_object) Returns: list of deferreds indicating success or failure to verify each json object's signature for the given server_name. """ verify_requests = [] for server_name, json_object in server_and_json: logger.debug("Verifying for %s", server_name) key_ids = signature_ids(json_object, server_name) if not key_ids: deferred = defer.fail(SynapseError( 400, "Not signed with a supported algorithm", Codes.UNAUTHORIZED, )) else: deferred = defer.Deferred() verify_request = VerifyKeyRequest( server_name, key_ids, json_object, deferred ) verify_requests.append(verify_request) @defer.inlineCallbacks def handle_key_deferred(verify_request): server_name = verify_request.server_name try: _, key_id, verify_key = yield verify_request.deferred except IOError as e: logger.warn( "Got IOError when downloading keys for %s: %s %s", server_name, type(e).__name__, str(e.message), ) raise SynapseError( 502, "Error downloading keys for %s" % (server_name,), Codes.UNAUTHORIZED, ) except Exception as e: logger.exception( "Got Exception when downloading keys for %s: %s %s", server_name, type(e).__name__, str(e.message), ) raise SynapseError( 401, "No key for %s with id %s" % (server_name, key_ids), Codes.UNAUTHORIZED, ) json_object = verify_request.json_object try: verify_signed_json(json_object, server_name, verify_key) except: raise SynapseError( 401, "Invalid signature for server %s with key %s:%s" % ( server_name, verify_key.alg, verify_key.version ), Codes.UNAUTHORIZED, ) server_to_deferred = { server_name: defer.Deferred() for server_name, _ in server_and_json } with PreserveLoggingContext(): # We want to wait for any previous lookups to complete before # proceeding. wait_on_deferred = self.wait_for_previous_lookups( [server_name for server_name, _ in server_and_json], server_to_deferred, ) # Actually start fetching keys. wait_on_deferred.addBoth( lambda _: self.get_server_verify_keys(verify_requests) ) # When we've finished fetching all the keys for a given server_name, # resolve the deferred passed to `wait_for_previous_lookups` so that # any lookups waiting will proceed. server_to_request_ids = {} def remove_deferreds(res, server_name, verify_request): request_id = id(verify_request) server_to_request_ids[server_name].discard(request_id) if not server_to_request_ids[server_name]: d = server_to_deferred.pop(server_name, None) if d: d.callback(None) return res for verify_request in verify_requests: server_name = verify_request.server_name request_id = id(verify_request) server_to_request_ids.setdefault(server_name, set()).add(request_id) deferred.addBoth(remove_deferreds, server_name, verify_request) # Pass those keys to handle_key_deferred so that the json object # signatures can be verified return [ preserve_context_over_fn(handle_key_deferred, verify_request) for verify_request in verify_requests ]
def user_joined_room(distributor, user, room_id): return preserve_context_over_fn( distributor.fire, "user_joined_room", user=user, room_id=room_id )
def stopped_user_eventstream(distributor, user): return preserve_context_over_fn( distributor.fire, "stopped_user_eventstream", user )
def _generate_remote_thumbnails(self, server_name, media_id, media_info): media_type = media_info["media_type"] file_id = media_info["filesystem_id"] requirements = self._get_thumbnail_requirements(media_type) if not requirements: return remote_thumbnails = [] input_path = self.filepaths.remote_media_filepath(server_name, file_id) thumbnailer = Thumbnailer(input_path) m_width = thumbnailer.width m_height = thumbnailer.height def generate_thumbnails(): if m_width * m_height >= self.max_image_pixels: logger.info( "Image too large to thumbnail %r x %r > %r", m_width, m_height, self.max_image_pixels ) return scales = set() crops = set() for r_width, r_height, r_method, r_type in requirements: if r_method == "scale": t_width, t_height = thumbnailer.aspect(r_width, r_height) scales.add(( min(m_width, t_width), min(m_height, t_height), r_type, )) elif r_method == "crop": crops.add((r_width, r_height, r_type)) for t_width, t_height, t_type in scales: t_method = "scale" t_path = self.filepaths.remote_media_thumbnail( server_name, file_id, t_width, t_height, t_type, t_method ) self._makedirs(t_path) t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) remote_thumbnails.append([ server_name, media_id, file_id, t_width, t_height, t_type, t_method, t_len ]) for t_width, t_height, t_type in crops: if (t_width, t_height, t_type) in scales: # If the aspect ratio of the cropped thumbnail matches a purely # scaled one then there is no point in calculating a separate # thumbnail. continue t_method = "crop" t_path = self.filepaths.remote_media_thumbnail( server_name, file_id, t_width, t_height, t_type, t_method ) self._makedirs(t_path) t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) remote_thumbnails.append([ server_name, media_id, file_id, t_width, t_height, t_type, t_method, t_len ]) yield preserve_context_over_fn(threads.deferToThread, generate_thumbnails) for r in remote_thumbnails: yield self.store.store_remote_media_thumbnail(*r) defer.returnValue({ "width": m_width, "height": m_height, })
def _create_request(self, destination, method, path_bytes, body_callback, headers_dict={}, param_bytes=b"", query_bytes=b"", retry_on_dns_fail=True, timeout=None): """ Creates and sends a request to the given url """ headers_dict[b"User-Agent"] = [self.version_string] headers_dict[b"Host"] = [destination] url_bytes = urlparse.urlunparse( ("", "", path_bytes, param_bytes, query_bytes, "",) ) logger.info("Sending request to %s: %s %s", destination, method, url_bytes) logger.debug( "Types: %s", [ type(destination), type(method), type(path_bytes), type(param_bytes), type(query_bytes) ] ) # XXX: Would be much nicer to retry only at the transaction-layer # (once we have reliable transactions in place) retries_left = 5 endpoint = self._getEndpoint(reactor, destination) while True: producer = None if body_callback: producer = body_callback(method, url_bytes, headers_dict) try: request_deferred = preserve_context_over_fn( self.agent.request, destination, endpoint, method, path_bytes, param_bytes, query_bytes, Headers(headers_dict), producer ) response = yield self.clock.time_bound_deferred( request_deferred, time_out=timeout/1000. if timeout else 60, ) logger.debug("Got response to %s", method) break except Exception as e: if not retry_on_dns_fail and isinstance(e, DNSLookupError): logger.warn( "DNS Lookup failed to %s with %s", destination, e ) raise logger.warn( "Sending request failed to %s: %s %s: %s - %s", destination, method, url_bytes, type(e).__name__, _flatten_response_never_received(e), ) if retries_left and not timeout: yield sleep(2 ** (5 - retries_left)) retries_left -= 1 else: raise logger.info( "Received response %d %s for %s: %s %s", response.code, response.phrase, destination, method, url_bytes ) if 200 <= response.code < 300: pass else: # :'( # Update transactions table? body = yield readBody(response) raise HttpResponseException( response.code, response.phrase, body ) defer.returnValue(response)
def _create_request(self, destination, method, path_bytes, body_callback, headers_dict={}, param_bytes=b"", query_bytes=b"", retry_on_dns_fail=True, timeout=None, long_retries=False): """ Creates and sends a request to the given url """ headers_dict[b"User-Agent"] = [self.version_string] headers_dict[b"Host"] = [destination] url_bytes = self._create_url(destination, path_bytes, param_bytes, query_bytes) txn_id = "%s-O-%s" % (method, self._next_id) self._next_id = (self._next_id + 1) % (sys.maxint - 1) outbound_logger.info("{%s} [%s] Sending request: %s %s", txn_id, destination, method, url_bytes) # XXX: Would be much nicer to retry only at the transaction-layer # (once we have reliable transactions in place) if long_retries: retries_left = MAX_LONG_RETRIES else: retries_left = MAX_SHORT_RETRIES http_url_bytes = urlparse.urlunparse( ("", "", path_bytes, param_bytes, query_bytes, "")) log_result = None try: while True: producer = None if body_callback: producer = body_callback(method, http_url_bytes, headers_dict) try: def send_request(): request_deferred = preserve_context_over_fn( self.agent.request, method, url_bytes, Headers(headers_dict), producer) return self.clock.time_bound_deferred( request_deferred, time_out=timeout / 1000. if timeout else 60, ) response = yield preserve_context_over_fn(send_request) log_result = "%d %s" % ( response.code, response.phrase, ) break except Exception as e: if not retry_on_dns_fail and isinstance(e, DNSLookupError): logger.warn("DNS Lookup failed to %s with %s", destination, e) log_result = "DNS Lookup failed to %s with %s" % ( destination, e) raise logger.warn( "{%s} Sending request failed to %s: %s %s: %s - %s", txn_id, destination, method, url_bytes, type(e).__name__, _flatten_response_never_received(e), ) log_result = "%s - %s" % ( type(e).__name__, _flatten_response_never_received(e), ) if retries_left and not timeout: if long_retries: delay = 4**(MAX_LONG_RETRIES + 1 - retries_left) delay = min(delay, 60) delay *= random.uniform(0.8, 1.4) else: delay = 0.5 * 2**(MAX_SHORT_RETRIES - retries_left) delay = min(delay, 2) delay *= random.uniform(0.8, 1.4) yield sleep(delay) retries_left -= 1 else: raise finally: outbound_logger.info( "{%s} [%s] Result: %s", txn_id, destination, log_result, ) if 200 <= response.code < 300: pass else: # :'( # Update transactions table? body = yield preserve_context_over_fn(readBody, response) raise HttpResponseException(response.code, response.phrase, body) defer.returnValue(response)
def _create_request(self, destination, method, path_bytes, body_callback, headers_dict={}, param_bytes=b"", query_bytes=b"", retry_on_dns_fail=True, timeout=None, long_retries=False): """ Creates and sends a request to the given url """ headers_dict[b"User-Agent"] = [self.version_string] headers_dict[b"Host"] = [destination] url_bytes = self._create_url( destination, path_bytes, param_bytes, query_bytes ) txn_id = "%s-O-%s" % (method, self._next_id) self._next_id = (self._next_id + 1) % (sys.maxint - 1) outbound_logger.info( "{%s} [%s] Sending request: %s %s", txn_id, destination, method, url_bytes ) # XXX: Would be much nicer to retry only at the transaction-layer # (once we have reliable transactions in place) if long_retries: retries_left = MAX_LONG_RETRIES else: retries_left = MAX_SHORT_RETRIES http_url_bytes = urlparse.urlunparse( ("", "", path_bytes, param_bytes, query_bytes, "") ) log_result = None try: while True: producer = None if body_callback: producer = body_callback(method, http_url_bytes, headers_dict) try: def send_request(): request_deferred = preserve_context_over_fn( self.agent.request, method, url_bytes, Headers(headers_dict), producer ) return self.clock.time_bound_deferred( request_deferred, time_out=timeout/1000. if timeout else 60, ) response = yield preserve_context_over_fn( send_request, ) log_result = "%d %s" % (response.code, response.phrase,) break except Exception as e: if not retry_on_dns_fail and isinstance(e, DNSLookupError): logger.warn( "DNS Lookup failed to %s with %s", destination, e ) log_result = "DNS Lookup failed to %s with %s" % ( destination, e ) raise logger.warn( "{%s} Sending request failed to %s: %s %s: %s - %s", txn_id, destination, method, url_bytes, type(e).__name__, _flatten_response_never_received(e), ) log_result = "%s - %s" % ( type(e).__name__, _flatten_response_never_received(e), ) if retries_left and not timeout: if long_retries: delay = 4 ** (MAX_LONG_RETRIES + 1 - retries_left) delay = min(delay, 60) delay *= random.uniform(0.8, 1.4) else: delay = 0.5 * 2 ** (MAX_SHORT_RETRIES - retries_left) delay = min(delay, 2) delay *= random.uniform(0.8, 1.4) yield sleep(delay) retries_left -= 1 else: raise finally: outbound_logger.info( "{%s} [%s] Result: %s", txn_id, destination, log_result, ) if 200 <= response.code < 300: pass else: # :'( # Update transactions table? body = yield preserve_context_over_fn(readBody, response) raise HttpResponseException( response.code, response.phrase, body ) defer.returnValue(response)