async def authenticate(host: str, port: str, method: str, path: str, authorization_headers: dict[str, str]) -> str: """Authenticate a request based on its headers. :param host: The authentication service host. :param port: The authentication Service port. :param method: The Authentication Service method. :param path: The Authentication Service path. :param authorization_headers: The headers that contain the authentication metadata. :return: The authenticated user identifier. """ authentication_url = URL(f"http://{host}:{port}{path}") authentication_method = method logger.info("Authenticating request...") try: async with ClientSession(headers=authorization_headers) as session: async with session.request(method=authentication_method, url=authentication_url) as response: if not response.ok: raise web.HTTPUnauthorized( text= "The given request does not have authorization to be forwarded." ) payload = await response.json() return payload["sub"] except ClientConnectorError: raise web.HTTPGatewayTimeout( text="The Authentication Service is not available.")
async def _precache(): try: return precache(request.app, old_run['id'], new_run['id']) except ArtifactsMissing as e: raise web.HTTPNotFound( text="No artifacts for run id: %r" % e, headers={"unavailable_run_id": e.args[0]}, ) except asyncio.TimeoutError: raise web.HTTPGatewayTimeout(text="Timeout retrieving artifacts")
async def precache(app, old_id, new_id): with ExitStack() as es: old_dir = es.enter_context(TemporaryDirectory()) new_dir = es.enter_context(TemporaryDirectory()) await asyncio.gather( app.artifact_manager.retrieve_artifacts( old_id, old_dir, filter_fn=is_binary, timeout=PRECACHE_RETRIEVE_TIMEOUT ), app.artifact_manager.retrieve_artifacts( new_id, new_dir, filter_fn=is_binary, timeout=PRECACHE_RETRIEVE_TIMEOUT ), ) old_binaries = find_binaries(old_dir) if not old_binaries: raise ArtifactsMissing(old_id) new_binaries = find_binaries(new_dir) if not new_binaries: raise ArtifactsMissing(new_id) debdiff_cache_path = app.debdiff_cache_path(old_id, new_id) if debdiff_cache_path and not os.path.exists(debdiff_cache_path): with open(debdiff_cache_path, "wb") as f: f.write( await run_debdiff( [p for (n, p) in old_binaries], [p for (n, p) in new_binaries] ) ) logging.info("Precached debdiff result for %s/%s", old_id, new_id) diffoscope_cache_path = app.diffoscope_cache_path(old_id, new_id) if diffoscope_cache_path and not os.path.exists(diffoscope_cache_path): try: diffoscope_diff = await asyncio.wait_for( run_diffoscope( old_binaries, new_binaries, lambda: _set_limits(app.task_memory_limit)), app.task_timeout ) except MemoryError: raise web.HTTPServiceUnavailable(text="diffoscope used too much memory") except asyncio.TimeoutError: raise web.HTTPGatewayTimeout(text="diffoscope timed out") try: with open(diffoscope_cache_path, "w") as f: json.dump(diffoscope_diff, f) except json.JSONDecodeError as e: raise web.HTTPServerError(text=str(e)) logging.info("Precached diffoscope result for %s/%s", old_id, new_id)
def _try_validator_request(self, message_type, content): """ Sends a protobuf message to the validator Handles a possible timeout if validator is unresponsive """ timeout = 5 timeout_msg = 'Could not reach validator, validator timed out' if isinstance(content, BaseMessage): content = content.SerializeToString() future = self._stream.send(message_type=message_type, content=content) try: response = future.result(timeout=timeout) except FutureTimeoutError as e: print(str(e)) raise web.HTTPGatewayTimeout(reason=timeout_msg) return response.content
async def discover(host: str, port: int, path: str, verb: str, endpoint: str) -> dict[str, Any]: """Call discovery service and get microservice connection data. :param host: Discovery host name. :param port: Discovery port. :param path: Discovery path. :param verb: Endpoint Verb. :param endpoint: Endpoint url. :return: The response of the discovery. """ url = URL.build(scheme="http", host=host, port=port, path=path, query={ "verb": verb, "path": endpoint }) try: async with ClientSession() as session: async with session.get(url=url) as response: if not response.ok: if response.status == 404: raise web.HTTPNotFound( text= f"The {endpoint!r} path is not available for {verb!r} method." ) raise web.HTTPBadGateway( text="The Discovery Service response is wrong.") data = await response.json() except ClientConnectorError: raise web.HTTPGatewayTimeout( text="The Discovery Service is not available.") data["port"] = int(data["port"]) return data
async def get_metrics(request: web.Request) -> web.Response: try: target = request.query['target'] except KeyError: raise web.HTTPBadRequest(text='target parameter omitted') cache = request.app['cache'] switch = cache.get(target) timeout = request.app['scrape_timeout'] try: with async_timeout.timeout(timeout), switch: counters = await switch.scrape() except asyncio.CancelledError: raise except asyncio.TimeoutError: raise web.HTTPGatewayTimeout( text='Scrape timed out after {}s'.format(timeout)) except Exception as exc: # Possibly a failed connection, so reset it switch.destroy() raise web.HTTPInternalServerError(text='Scrape failed: ' + str(exc)) else: content = prometheus_client.generate_latest(counters).decode() return web.Response(text=content)
async def _redirect(self, request, connector): if request.host in ( socket.getfqdn(), f'{self.config["hostname"]}:{self.config["port"]}', ): raise web.HTTPBadRequest( text=( "ERROR: Recursion error. " "Invalid 'Host' header specified.\n" ) ) self.logger.debug( f"Request (redirecting): method={request.method!r} " f"path={request.path!r}, query_string={request.query_string!r}, " f"headers={request.headers!r}, remote={request.remote!r}" ) req_headers = request.headers body = await request.read() # modify headers if self.config["num_forwarded"]: req_headers = copy.deepcopy(dict(req_headers)) req_headers["X-Forwarded-For"] = request.remote self.logger.debug( f"Request headers (redirecting, modified): {req_headers!r}" ) try: async with aiohttp.ClientSession( headers=req_headers, connector=connector, timeout=self.client_timeout, connector_owner=False, auto_decompress=False, ) as session: async with session.request( request.method, request.url, data=body, ) as resp: self.logger.debug( f"Response: {resp.reason}: resp.status={resp.status}, " f"resp.request_info={resp.request_info}, " f"resp.url={resp.url}, resp.headers={resp.headers}" ) proxied_response = web.StreamResponse( headers=resp.headers, status=resp.status ) if ( resp.headers.get("Transfer-Encoding", "").lower() == "chunked" ): proxied_response.enable_chunked_encoding() await proxied_response.prepare(request) async for data in resp.content.iter_any(): await proxied_response.write(data) await proxied_response.write_eof() return proxied_response except ConnectionResetError as err: self.logger.warning(f"Connection reset by peer: {err}") # TODO(damb): Would implementing a retry mechanism be an # alternative? raise web.HTTPNoContent() except asyncio.TimeoutError as err: self.logger.warning( f"Error while executing request: error={type(err)}, " f"url={request.url!r}, method={request.method!r}" ) raise web.HTTPGatewayTimeout(text=f"ERROR: {str(type(err))}\n") except aiohttp.ClientError as err: msg = ( f"Error while executing request: error={type(err)}, " f"msg={err}, url={request.url!r}, method={request.method!r}" ) if isinstance(err, aiohttp.ClientOSError): msg += f", errno={err.errno}" self.logger.warning(msg) raise web.HTTPServiceUnavailable(text=f"ERROR: {str(type(err))}\n")
async def handle_debdiff(request): old_id = request.match_info["old_id"] new_id = request.match_info["new_id"] old_run, new_run = await get_run_pair(request.app.db, old_id, new_id) cache_path = request.app.debdiff_cache_path(old_run['id'], new_run['id']) if cache_path: try: with open(cache_path, "rb") as f: debdiff = f.read() except FileNotFoundError: debdiff = None else: debdiff = None if debdiff is None: logging.info( "Generating debdiff between %s (%s/%s/%s) and %s (%s/%s/%s)", old_run['id'], old_run['package'], old_run['build_version'], old_run['suite'], new_run['id'], new_run['package'], new_run['build_version'], new_run['suite'], ) with ExitStack() as es: old_dir = es.enter_context(TemporaryDirectory()) new_dir = es.enter_context(TemporaryDirectory()) try: await asyncio.gather( request.app.artifact_manager.retrieve_artifacts( old_run['id'], old_dir, filter_fn=is_binary ), request.app.artifact_manager.retrieve_artifacts( new_run['id'], new_dir, filter_fn=is_binary ), ) except ArtifactsMissing as e: raise web.HTTPNotFound( text="No artifacts for run id: %r" % e, headers={"unavailable_run_id": e.args[0]}, ) except asyncio.TimeoutError: raise web.HTTPGatewayTimeout(text="Timeout retrieving artifacts") old_binaries = find_binaries(old_dir) if not old_binaries: raise web.HTTPNotFound( text="No artifacts for run id: %s" % old_run['id'], headers={"unavailable_run_id": old_run['id']}, ) new_binaries = find_binaries(new_dir) if not new_binaries: raise web.HTTPNotFound( text="No artifacts for run id: %s" % new_run['id'], headers={"unavailable_run_id": new_run['id']}, ) try: debdiff = await run_debdiff( [p for (n, p) in old_binaries], [p for (n, p) in new_binaries] ) except DebdiffError as e: return web.Response(status=400, text=e.args[0]) if cache_path: with open(cache_path, "wb") as f: f.write(debdiff) if "filter_boring" in request.query: debdiff = filter_debdiff_boring( debdiff.decode(), str(old_run['build_version']), str(new_run['build_version']) ).encode() for accept in request.headers.get("ACCEPT", "*/*").split(","): if accept in ("text/x-diff", "text/plain", "*/*"): return web.Response(body=debdiff, content_type="text/plain") if accept == "text/markdown": return web.Response( text=markdownify_debdiff(debdiff.decode("utf-8", "replace")), content_type="text/markdown", ) if accept == "text/html": return web.Response( text=htmlize_debdiff(debdiff.decode("utf-8", "replace")), content_type="text/html", ) raise web.HTTPNotAcceptable( text="Acceptable content types: " "text/html, text/plain, text/markdown" )
async def handle_diffoscope(request): for accept in request.headers.get("ACCEPT", "*/*").split(","): if accept in ("text/plain", "*/*"): content_type = "text/plain" break elif accept in ("text/html",): content_type = "text/html" break elif accept in ("application/json",): content_type = "application/json" break elif accept in ("text/markdown",): content_type = "text/markdown" break else: raise web.HTTPNotAcceptable( text="Acceptable content types: " "text/html, text/plain, application/json, " "application/markdown" ) old_id = request.match_info["old_id"] new_id = request.match_info["new_id"] old_run, new_run = await get_run_pair(request.app.db, old_id, new_id) cache_path = request.app.diffoscope_cache_path(old_run['id'], new_run['id']) if cache_path: try: with open(cache_path, "rb") as f: diffoscope_diff = json.load(f) except FileNotFoundError: diffoscope_diff = None else: diffoscope_diff = None if diffoscope_diff is None: logging.info( "Generating diffoscope between %s (%s/%s/%s) and %s (%s/%s/%s)", old_run['id'], old_run['package'], old_run['build_version'], old_run['suite'], new_run['id'], new_run['package'], new_run['build_version'], new_run['suite'], ) with ExitStack() as es: old_dir = es.enter_context(TemporaryDirectory()) new_dir = es.enter_context(TemporaryDirectory()) try: await asyncio.gather( request.app.artifact_manager.retrieve_artifacts( old_run['id'], old_dir, filter_fn=is_binary ), request.app.artifact_manager.retrieve_artifacts( new_run['id'], new_dir, filter_fn=is_binary ), ) except ArtifactsMissing as e: raise web.HTTPNotFound( text="No artifacts for run id: %r" % e, headers={"unavailable_run_id": e.args[0]}, ) except asyncio.TimeoutError: raise web.HTTPGatewayTimeout(text="Timeout retrieving artifacts") old_binaries = find_binaries(old_dir) if not old_binaries: raise web.HTTPNotFound( text="No artifacts for run id: %s" % old_run['id'], headers={"unavailable_run_id": old_run['id']}, ) new_binaries = find_binaries(new_dir) if not new_binaries: raise web.HTTPNotFound( text="No artifacts for run id: %s" % new_run['id'], headers={"unavailable_run_id": new_run['id']}, ) try: diffoscope_diff = await asyncio.wait_for( run_diffoscope( old_binaries, new_binaries, lambda: _set_limits(request.app.task_memory_limit)), request.app.task_timeout ) except MemoryError: raise web.HTTPServiceUnavailable(text="diffoscope used too much memory") except asyncio.TimeoutError: raise web.HTTPGatewayTimeout(text="diffoscope timed out") if cache_path is not None: with open(cache_path, "w") as f: json.dump(diffoscope_diff, f) diffoscope_diff["source1"] = "%s version %s (%s)" % ( old_run['package'], old_run['build_version'], old_run['suite'], ) diffoscope_diff["source2"] = "%s version %s (%s)" % ( new_run['package'], new_run['build_version'], new_run['suite'], ) filter_diffoscope_irrelevant(diffoscope_diff) title = "diffoscope for %s applied to %s" % (new_run['suite'], new_run['package']) if "filter_boring" in request.query: filter_diffoscope_boring( diffoscope_diff, str(old_run['build_version']), str(new_run['build_version']), old_run['suite'], new_run['suite'], ) title += " (filtered)" debdiff = await format_diffoscope( diffoscope_diff, content_type, title=title, jquery_url=request.query.get("jquery_url"), css_url=request.query.get("css_url"), ) return web.Response(text=debdiff, content_type=content_type)