async def _connect( hostname: str, port: int, use_ssl: bool, keepalive: Optional[int], handshake: BaseClientHandshake, ) -> Transport: try: stream = await trio.open_tcp_stream(hostname, port) except OSError as exc: logger.debug("Impossible to connect to backend", reason=exc) raise BackendNotAvailable(exc) from exc if use_ssl: stream = _upgrade_stream_to_ssl(stream, hostname) try: transport = await Transport.init_for_client(stream, host=hostname) transport.handshake = handshake transport.keepalive = keepalive except TransportError as exc: logger.debug("Connection lost during transport creation", reason=exc) raise BackendNotAvailable(exc) from exc try: await _do_handshake(transport, handshake) except Exception as exc: transport.logger.debug("Connection lost during handshake", reason=exc) await transport.aclose() raise return transport
async def _acquire_transport(self, force_fresh=False, ignore_status=False, allow_not_available=False): if not ignore_status: if self.status_exc: # Re-raising an already raised exception is bad practice # as its internal state gets mutated every time is raised. # Note that this copy preserves the __cause__ attribute. raise copy_exception(self.status_exc) try: async with self._transport_pool.acquire( force_fresh=force_fresh) as transport: yield transport except BackendNotAvailable as exc: if not allow_not_available: await self.set_status(BackendConnStatus.LOST, exc) self._cancel_manager_connect() raise except BackendConnectionRefused as exc: await self.set_status(BackendConnStatus.REFUSED, exc) self._cancel_manager_connect() raise except BackendOutOfBallparkError as exc: # Caller doesn't need to know about the desync, # simply pretend that we lost the connection instead new_exc = BackendNotAvailable() new_exc.__cause__ = exc await self.set_status(BackendConnStatus.DESYNC, new_exc) self._cancel_manager_connect() raise new_exc
async def _connect( hostname: str, port: int, use_ssl: bool, keepalive: Optional[int], handshake: BaseClientHandshake, ) -> Transport: stream = await maybe_connect_through_proxy(hostname, port, use_ssl) if not stream: logger.debug("Using direct (no proxy) connection", hostname=hostname, port=port) try: stream = await trio.open_tcp_stream(hostname, port) except OSError as exc: logger.debug("Impossible to connect to backend", hostname=hostname, port=port, exc_info=exc) raise BackendNotAvailable(exc) from exc if use_ssl: logger.debug("Using TLS to secure connection", hostname=hostname) stream = _upgrade_stream_to_ssl(stream, hostname) try: transport = await Transport.init_for_client(stream, host=hostname) transport.keepalive = keepalive except TransportError as exc: logger.warning("Connection lost during transport creation", exc_info=exc) raise BackendNotAvailable(exc) from exc try: await _do_handshake(transport, handshake) except BackendOutOfBallparkError: transport.logger.info( "Abort handshake due to the system clock being out of sync") await transport.aclose() raise except Exception as exc: transport.logger.warning("Connection lost during handshake", exc_info=exc) await transport.aclose() raise return transport
async def _connect( addr: Union[BackendAddr, BackendOrganizationBootstrapAddr, BackendOrganizationAddr], device_id: Optional[DeviceID] = None, signing_key: Optional[SigningKey] = None, administrator_token: Optional[str] = None, ): try: stream = await trio.open_tcp_stream(addr.hostname, addr.port) except OSError as exc: logger.debug("Impossible to connect to backend", reason=exc) raise BackendNotAvailable(exc) from exc if addr.scheme == "wss": stream = _upgrade_stream_to_ssl(stream, addr.hostname) try: transport = await Transport.init_for_client(stream, addr.hostname) except TransportError as exc: logger.debug("Connection lost during transport creation", reason=exc) raise BackendNotAvailable(exc) from exc if administrator_token: ch = AnonymousClientHandshake(administrator_token) elif not device_id: if isinstance(addr, BackendOrganizationBootstrapAddr): ch = AnonymousClientHandshake(addr.organization_id) else: assert isinstance(addr, BackendOrganizationAddr) ch = AnonymousClientHandshake(addr.organization_id, addr.root_verify_key) else: assert isinstance(addr, BackendOrganizationAddr) assert signing_key ch = ClientHandshake(addr.organization_id, device_id, signing_key, addr.root_verify_key) try: await _do_handshade(transport, ch) except Exception as exc: transport.logger.debug("Connection lost during handshake", reason=exc) await transport.aclose() raise return transport
async def backend_administrator_cmds_factory( addr: BackendAddr, token: str) -> BackendAdministratorCmds: try: async with administrator_transport_factory(addr, token) as transport: yield BackendAdministratorCmds(addr, transport) except TransportError as exc: raise BackendNotAvailable(exc) from exc
async def backend_anonymous_cmds_factory( addr: BackendOrganizationAddr) -> BackendAnonymousCmds: try: async with anonymous_transport_factory(addr) as transport: yield BackendAnonymousCmds(addr, transport) except TransportError as exc: raise BackendNotAvailable(exc) from exc
async def _do_handshake(transport: Transport, handshake): try: challenge_req = await transport.recv() answer_req = handshake.process_challenge_req(challenge_req) await transport.send(answer_req) result_req = await transport.recv() handshake.process_result_req(result_req) except TransportError as exc: raise BackendNotAvailable(exc) from exc except HandshakeOutOfBallparkError as exc: raise BackendOutOfBallparkError(exc) from exc except HandshakeError as exc: if str(exc) == "Invalid handshake: Invitation not found": raise BackendInvitationNotFound(str(exc)) from exc elif str(exc) == "Invalid handshake: Invitation already deleted": raise BackendInvitationAlreadyUsed(str(exc)) from exc else: raise BackendConnectionRefused(str(exc)) from exc except ProtocolError as exc: transport.logger.exception("Protocol error during handshake") raise BackendProtocolError(exc) from exc
async def _send_cmd(transport: Transport, serializer, **req) -> dict: """ Raises: Backend BackendNotAvailable BackendProtocolError BackendCmdsInvalidRequest BackendCmdsInvalidResponse BackendNotAvailable BackendCmdsBadResponse """ transport.logger.info("Request", cmd=req["cmd"]) try: raw_req = serializer.req_dumps(req) except ProtocolError as exc: transport.logger.exception("Invalid request data", cmd=req["cmd"], error=exc) raise BackendProtocolError("Invalid request data") from exc try: await transport.send(raw_req) raw_rep = await transport.recv() except TransportError as exc: transport.logger.debug("Request failed (backend not available)", cmd=req["cmd"]) raise BackendNotAvailable(exc) from exc try: rep = serializer.rep_loads(raw_rep) except ProtocolError as exc: transport.logger.exception("Invalid response data", cmd=req["cmd"], error=exc) raise BackendProtocolError("Invalid response data") from exc if rep["status"] == "invalid_msg_format": transport.logger.error("Invalid request data according to backend", cmd=req["cmd"], rep=rep) raise BackendProtocolError("Invalid request data according to backend") if rep["status"] == "bad_timestamp": raise BackendOutOfBallparkError(rep) # Backward compatibility with older backends (<= v2.3) if rep["status"] == "invalid_certification" and "timestamp" in rep[ "reason"]: raise BackendOutOfBallparkError(rep) return rep
async def _run_manager(self): while True: try: with trio.CancelScope() as self._manager_connect_cancel_scope: try: await self._manager_connect() except (BackendNotAvailable, BackendConnectionRefused): pass except Exception as exc: await self.set_status( BackendConnStatus.CRASHED, BackendNotAvailable( f"Backend connection manager has crashed: {exc}" ), ) logger.exception("Unhandled exception") finally: self._manager_connect_cancel_scope = None assert self.status not in (BackendConnStatus.READY, BackendConnStatus.INITIALIZING) if self.status == BackendConnStatus.LOST: # Start with a 0s cooldown and increase by power of 2 until # max cooldown every time the connection trial fails # (e.g. 0, 1, 2, 4, 8, 15, 15, 15 etc. if max cooldown is 15s) if self._backend_connection_failures < 1: # A cooldown time of 0 second is useful for the specific case of a # revocation when the event listener is the only running transport. # This way, we don't have to wait 1 second before the revocation is # detected. cooldown_time = 0 else: cooldown_time = 2**(self._backend_connection_failures - 1) if cooldown_time > self.max_cooldown: cooldown_time = self.max_cooldown self._backend_connection_failures += 1 logger.info("Backend offline", cooldown_time=cooldown_time) await trio.sleep(cooldown_time) if self.status == BackendConnStatus.REFUSED: # It's most likely useless to retry connection anyway logger.info("Backend connection refused", status=self.status) await trio.sleep_forever() if self.status == BackendConnStatus.CRASHED: # It's most likely useless to retry connection anyway logger.info("Backend connection has crashed", status=self.status) await trio.sleep_forever() if self.status == BackendConnStatus.DESYNC: # Try again in 10 seconds logger.info("Backend connection is desync", status=self.status) await trio.sleep(DESYNC_RETRY_TIME)
async def _send_cmd(transport: Transport, serializer, **req) -> dict: """ Raises: Backend BackendNotAvailable BackendProtocolError BackendCmdsInvalidRequest BackendCmdsInvalidResponse BackendNotAvailable BackendCmdsBadResponse """ transport.logger.info("Request", cmd=req["cmd"]) try: raw_req = serializer.req_dumps(req) except ProtocolError as exc: transport.logger.exception("Invalid request data", cmd=req["cmd"], error=exc) raise BackendProtocolError("Invalid request data") from exc try: await transport.send(raw_req) raw_rep = await transport.recv() except TransportError as exc: transport.logger.debug("Request failed (backend not available)", cmd=req["cmd"]) raise BackendNotAvailable(exc) from exc try: rep = serializer.rep_loads(raw_rep) except ProtocolError as exc: transport.logger.exception("Invalid response data", cmd=req["cmd"], error=exc) raise BackendProtocolError("Invalid response data") from exc if rep["status"] == "invalid_msg_format": transport.logger.error("Invalid request data according to backend", cmd=req["cmd"], rep=rep) raise BackendProtocolError("Invalid request data according to backend") return rep
async def _do_handshake(transport: Transport, handshake): try: challenge_req = await transport.recv() answer_req = handshake.process_challenge_req(challenge_req) await transport.send(answer_req) result_req = await transport.recv() handshake.process_result_req(result_req) except TransportError as exc: raise BackendNotAvailable(exc) from exc except HandshakeError as exc: raise BackendConnectionRefused(str(exc)) from exc except ProtocolError as exc: transport.logger.exception("Protocol error during handshake") raise BackendProtocolError(exc) from exc
async def _do_handshade(transport: Transport, ch): try: challenge_req = await transport.recv() answer_req = ch.process_challenge_req(challenge_req) await transport.send(answer_req) result_req = await transport.recv() ch.process_result_req(result_req) transport.logger.debug("Handshake done") except TransportError as exc: raise BackendNotAvailable(exc) from exc except HandshakeRevokedDevice as exc: transport.logger.warning("Handshake failed", reason=exc) raise BackendDeviceRevokedError(exc) from exc except ProtocoleError as exc: transport.logger.warning("Handshake failed", reason=exc) raise BackendHandshakeError(exc) from exc
async def _anonymous_cmd(serializer, addr: BackendAddr, organization_id: OrganizationID, **req) -> dict: """ Raises: BackendNotAvailable BackendProtocolError """ logger.info("Request", cmd=req["cmd"]) try: raw_req = serializer.req_dumps(req) except ProtocolError as exc: logger.exception("Invalid request data", cmd=req["cmd"], error=exc) raise BackendProtocolError("Invalid request data") from exc url = addr.to_http_domain_url(f"/anonymous/{organization_id}") try: raw_rep = await http_request(url=url, method="POST", data=raw_req) except OSError as exc: logger.debug("Request failed (backend not available)", cmd=req["cmd"], exc_info=exc) raise BackendNotAvailable(exc) from exc try: rep = serializer.rep_loads(raw_rep) except ProtocolError as exc: logger.exception("Invalid response data", cmd=req["cmd"], error=exc) raise BackendProtocolError("Invalid response data") from exc if rep["status"] == "invalid_msg_format": logger.error("Invalid request data according to backend", cmd=req["cmd"], rep=rep) raise BackendProtocolError("Invalid request data according to backend") if rep["status"] == "bad_timestamp": raise BackendOutOfBallparkError(rep) return rep
async def _send_cmd(transport, serializer, **req): transport.logger.info("Request", cmd=req["cmd"]) def _shorten_data(data): if len(req) > 300: return data[:150] + b"[...]" + data[-150:] else: return data try: raw_req = serializer.req_dumps(req) except ProtocoleError as exc: raise BackendCmdsInvalidRequest() from exc transport.logger.debug("send req", req=_shorten_data(raw_req)) try: await transport.send(raw_req) raw_rep = await transport.recv() except TransportError as exc: transport.logger.info("Request failed (backend not available)", cmd=req["cmd"]) raise BackendNotAvailable(exc) from exc transport.logger.debug("recv rep", req=_shorten_data(raw_rep)) try: rep = serializer.rep_loads(raw_rep) except ProtocoleError as exc: transport.logger.warning("Request failed (bad protocol)", cmd=req["cmd"], error=exc) raise BackendCmdsInvalidResponse(exc) from exc if rep["status"] == "invalid_msg_format": raise BackendCmdsInvalidRequest(rep) return rep
async def _run_manager(self): while True: try: with trio.CancelScope() as self._manager_connect_cancel_scope: try: await self._manager_connect() except (BackendNotAvailable, BackendConnectionRefused): pass except Exception as exc: self._status = BackendConnStatus.CRASHED self._status_exc = BackendNotAvailable( f"Backend connection manager has crashed: {exc}") logger.exception("Unhandled exception") finally: self._manager_connect_cancel_scope = None assert self._status not in (BackendConnStatus.READY, BackendConnStatus.INITIALIZING) if self._backend_connection_failures == 0: self.event_bus.send("backend.connection.changed", status=self._status, status_exc=self._status_exc) if self._status == BackendConnStatus.LOST: # Start with a 1s cooldown and increase by power of 2 until # max cooldown every time the connection trial fails # (e.g. 1, 2, 4, 8, 15, 15, 15 etc. if max cooldown is 15s) cooldown_time = 2**self._backend_connection_failures if cooldown_time > self.max_cooldown: cooldown_time = self.max_cooldown self._backend_connection_failures += 1 logger.info("Backend offline", cooldown_time=cooldown_time) await trio.sleep(cooldown_time) else: # It's most likely useless to retry connection anyway logger.info("Backend connection refused", status=self._status) await trio.sleep_forever()
async def maybe_connect_through_proxy( hostname: str, port: int, use_ssl: bool) -> Optional[trio.abc.Stream]: target_url = _build_http_url(hostname, port, use_ssl) proxy_url = await trio.to_thread.run_sync(blocking_io_get_proxy, target_url, hostname) if not proxy_url: return None logger.debug("Using proxy to connect", proxy_url=proxy_url, target_url=target_url) # A proxy has been retrieved, parse it url and handle potential auth try: proxy = urlsplit(proxy_url) # Typing helper, as result could be SplitResultBytes if we have provided bytes instead of str assert isinstance(proxy, SplitResult) except ValueError as exc: logger.warning( "Invalid proxy url, switching to no proxy", proxy_url=proxy_url, target_url=target_url, exc_info=exc, ) # Invalid url return None if proxy.port: proxy_port = proxy.port else: proxy_port = 443 if proxy.scheme == "https" else 80 if not proxy.hostname: logger.warning( "Missing required hostname in proxy url, switching to no proxy", proxy_url=proxy_url, target_url=target_url, ) return None proxy_headers: List[Tuple[str, str]] = [] if proxy.username is not None and proxy.password is not None: logger.debug("Using `Proxy-Authorization` header with proxy", username=proxy.username) proxy_headers.append(("Proxy-Authorization", cook_basic_auth_header(proxy.username, proxy.password))) # Connect to the proxy stream: trio.abc.Stream try: stream = await trio.open_tcp_stream(proxy.hostname, proxy_port) except OSError as exc: logger.warning( "Impossible to connect to proxy", proxy_hostname=proxy.hostname, proxy_port=proxy_port, target_url=target_url, exc_info=exc, ) raise BackendNotAvailable(exc) from exc if proxy.scheme == "https": logger.debug("Using TLS to secure proxy connection", hostname=proxy.hostname) from parsec.core.backend_connection.transport import _upgrade_stream_to_ssl stream = _upgrade_stream_to_ssl(stream, proxy.hostname) # Ask the proxy to connect the actual host conn = h11.Connection(our_role=h11.CLIENT) async def send(event): data = conn.send(event) await stream.send_all(data) async def next_event(): while True: event = conn.next_event() if event is h11.NEED_DATA: # Note there is no need to handle 100 continue here given we are client # (see https://h11.readthedocs.io/en/v0.10.0/api.htm1l#flow-control) data = await stream.receive_some(2048) conn.receive_data(data) continue return event host = f"{hostname}:{port}" try: req = h11.Request( method="CONNECT", target=host, headers=[ # According to RFC7230 (https://datatracker.ietf.org/doc/html/rfc7230#section-5.4) # Client must provide Host header, but the proxy must replace it # with the host information of the request-target. So in theory # we could set any dummy value for the Host header here ! ("Host", host), # User-Agent is not a mandatory header, however http proxy are # half-broken shenanigans and get suspicious if it is missing ("User-Agent", USER_AGENT), *proxy_headers, ], ) logger.debug("Sending CONNECT to proxy", proxy_url=proxy_url, req=req) await send(req) answer = await next_event() logger.debug("Receiving CONNECT answer from proxy", proxy_url=proxy_url, answer=answer) if not isinstance(answer, h11.Response) or not 200 <= answer.status_code < 300: logger.warning( "Bad answer from proxy to CONNECT request", proxy_url=proxy_url, target_host=host, target_url=target_url, answer_status=answer.status_code, ) raise BackendNotAvailable("Bad answer from proxy") # Successful CONNECT should reset the connection's statemachine answer = await next_event() while not isinstance(answer, h11.PAUSED): logger.debug( "Receiving additional answer to our CONNECT from proxy", proxy_url=proxy_url, answer=answer, ) answer = await next_event() except trio.BrokenResourceError as exc: logger.warning( "Proxy has unexpectedly closed the connection", proxy_url=proxy_url, target_host=host, target_url=target_url, exc_info=exc, ) raise BackendNotAvailable( "Proxy has unexpectedly closed the connection") from exc return stream
async def connect( addr: Union[BackendAddr, BackendOrganizationBootstrapAddr, BackendOrganizationAddr], device_id: Optional[DeviceID] = None, signing_key: Optional[SigningKey] = None, administration_token: Optional[str] = None, keepalive: Optional[int] = None, ) -> Transport: """ Raises: BackendConnectionError """ if administration_token: if not isinstance(addr, BackendAddr): raise BackendConnectionError(f"Invalid url format `{addr}`") handshake = AdministrationClientHandshake(administration_token) elif not device_id: if isinstance(addr, BackendOrganizationBootstrapAddr): handshake = AnonymousClientHandshake(addr.organization_id) elif isinstance(addr, BackendOrganizationAddr): handshake = AnonymousClientHandshake(addr.organization_id, addr.root_verify_key) else: raise BackendConnectionError( f"Invalid url format `{addr}` " "(should be an organization url or organization bootstrap url)" ) else: if not isinstance(addr, BackendOrganizationAddr): raise BackendConnectionError( f"Invalid url format `{addr}` (should be an organization url)") if not signing_key: raise BackendConnectionError( f"Missing signing_key to connect as `{device_id}`") handshake = AuthenticatedClientHandshake(addr.organization_id, device_id, signing_key, addr.root_verify_key) try: stream = await trio.open_tcp_stream(addr.hostname, addr.port) except OSError as exc: logger.debug("Impossible to connect to backend", reason=exc) raise BackendNotAvailable(exc) from exc if addr.use_ssl: stream = _upgrade_stream_to_ssl(stream, addr.hostname) try: transport = await Transport.init_for_client(stream, host=addr.hostname) transport.handshake = handshake transport.keepalive = keepalive except TransportError as exc: logger.debug("Connection lost during transport creation", reason=exc) raise BackendNotAvailable(exc) from exc try: await _do_handshake(transport, handshake) except Exception as exc: transport.logger.debug("Connection lost during handshake", reason=exc) await transport.aclose() raise return transport