async def _connect(
hostname: str,
port: int,
use_ssl: bool,
keepalive: Optional[int],
handshake: BaseClientHandshake,
) -> Transport:
try:
stream = await trio.open_tcp_stream(hostname, port)
except OSError as exc:
logger.debug("Impossible to connect to backend", reason=exc)
raise BackendNotAvailable(exc) from exc
if use_ssl:
stream = _upgrade_stream_to_ssl(stream, hostname)
try:
transport = await Transport.init_for_client(stream, host=hostname)
transport.handshake = handshake
transport.keepalive = keepalive
except TransportError as exc:
logger.debug("Connection lost during transport creation", reason=exc)
raise BackendNotAvailable(exc) from exc
try:
await _do_handshake(transport, handshake)
except Exception as exc:
transport.logger.debug("Connection lost during handshake", reason=exc)
await transport.aclose()
raise
return transport
async def _acquire_transport(self,
force_fresh=False,
ignore_status=False,
allow_not_available=False):
if not ignore_status:
if self.status_exc:
# Re-raising an already raised exception is bad practice
# as its internal state gets mutated every time is raised.
# Note that this copy preserves the __cause__ attribute.
raise copy_exception(self.status_exc)
try:
async with self._transport_pool.acquire(
force_fresh=force_fresh) as transport:
yield transport
except BackendNotAvailable as exc:
if not allow_not_available:
await self.set_status(BackendConnStatus.LOST, exc)
self._cancel_manager_connect()
raise
except BackendConnectionRefused as exc:
await self.set_status(BackendConnStatus.REFUSED, exc)
self._cancel_manager_connect()
raise
except BackendOutOfBallparkError as exc:
# Caller doesn't need to know about the desync,
# simply pretend that we lost the connection instead
new_exc = BackendNotAvailable()
new_exc.__cause__ = exc
await self.set_status(BackendConnStatus.DESYNC, new_exc)
self._cancel_manager_connect()
raise new_exc
async def _connect(
hostname: str,
port: int,
use_ssl: bool,
keepalive: Optional[int],
handshake: BaseClientHandshake,
) -> Transport:
stream = await maybe_connect_through_proxy(hostname, port, use_ssl)
if not stream:
logger.debug("Using direct (no proxy) connection",
hostname=hostname,
port=port)
try:
stream = await trio.open_tcp_stream(hostname, port)
except OSError as exc:
logger.debug("Impossible to connect to backend",
hostname=hostname,
port=port,
exc_info=exc)
raise BackendNotAvailable(exc) from exc
if use_ssl:
logger.debug("Using TLS to secure connection", hostname=hostname)
stream = _upgrade_stream_to_ssl(stream, hostname)
try:
transport = await Transport.init_for_client(stream, host=hostname)
transport.keepalive = keepalive
except TransportError as exc:
logger.warning("Connection lost during transport creation",
exc_info=exc)
raise BackendNotAvailable(exc) from exc
try:
await _do_handshake(transport, handshake)
except BackendOutOfBallparkError:
transport.logger.info(
"Abort handshake due to the system clock being out of sync")
await transport.aclose()
raise
except Exception as exc:
transport.logger.warning("Connection lost during handshake",
exc_info=exc)
await transport.aclose()
raise
return transport
async def _connect(
addr: Union[BackendAddr, BackendOrganizationBootstrapAddr,
BackendOrganizationAddr],
device_id: Optional[DeviceID] = None,
signing_key: Optional[SigningKey] = None,
administrator_token: Optional[str] = None,
):
try:
stream = await trio.open_tcp_stream(addr.hostname, addr.port)
except OSError as exc:
logger.debug("Impossible to connect to backend", reason=exc)
raise BackendNotAvailable(exc) from exc
if addr.scheme == "wss":
stream = _upgrade_stream_to_ssl(stream, addr.hostname)
try:
transport = await Transport.init_for_client(stream, addr.hostname)
except TransportError as exc:
logger.debug("Connection lost during transport creation", reason=exc)
raise BackendNotAvailable(exc) from exc
if administrator_token:
ch = AnonymousClientHandshake(administrator_token)
elif not device_id:
if isinstance(addr, BackendOrganizationBootstrapAddr):
ch = AnonymousClientHandshake(addr.organization_id)
else:
assert isinstance(addr, BackendOrganizationAddr)
ch = AnonymousClientHandshake(addr.organization_id,
addr.root_verify_key)
else:
assert isinstance(addr, BackendOrganizationAddr)
assert signing_key
ch = ClientHandshake(addr.organization_id, device_id, signing_key,
addr.root_verify_key)
try:
await _do_handshade(transport, ch)
except Exception as exc:
transport.logger.debug("Connection lost during handshake", reason=exc)
await transport.aclose()
raise
return transport
async def backend_administrator_cmds_factory(
addr: BackendAddr, token: str) -> BackendAdministratorCmds:
try:
async with administrator_transport_factory(addr, token) as transport:
yield BackendAdministratorCmds(addr, transport)
except TransportError as exc:
raise BackendNotAvailable(exc) from exc
async def backend_anonymous_cmds_factory(
addr: BackendOrganizationAddr) -> BackendAnonymousCmds:
try:
async with anonymous_transport_factory(addr) as transport:
yield BackendAnonymousCmds(addr, transport)
except TransportError as exc:
raise BackendNotAvailable(exc) from exc
async def _do_handshake(transport: Transport, handshake):
try:
challenge_req = await transport.recv()
answer_req = handshake.process_challenge_req(challenge_req)
await transport.send(answer_req)
result_req = await transport.recv()
handshake.process_result_req(result_req)
except TransportError as exc:
raise BackendNotAvailable(exc) from exc
except HandshakeOutOfBallparkError as exc:
raise BackendOutOfBallparkError(exc) from exc
except HandshakeError as exc:
if str(exc) == "Invalid handshake: Invitation not found":
raise BackendInvitationNotFound(str(exc)) from exc
elif str(exc) == "Invalid handshake: Invitation already deleted":
raise BackendInvitationAlreadyUsed(str(exc)) from exc
else:
raise BackendConnectionRefused(str(exc)) from exc
except ProtocolError as exc:
transport.logger.exception("Protocol error during handshake")
raise BackendProtocolError(exc) from exc
async def _send_cmd(transport: Transport, serializer, **req) -> dict:
"""
Raises:
Backend
BackendNotAvailable
BackendProtocolError
BackendCmdsInvalidRequest
BackendCmdsInvalidResponse
BackendNotAvailable
BackendCmdsBadResponse
"""
transport.logger.info("Request", cmd=req["cmd"])
try:
raw_req = serializer.req_dumps(req)
except ProtocolError as exc:
transport.logger.exception("Invalid request data",
cmd=req["cmd"],
error=exc)
raise BackendProtocolError("Invalid request data") from exc
try:
await transport.send(raw_req)
raw_rep = await transport.recv()
except TransportError as exc:
transport.logger.debug("Request failed (backend not available)",
cmd=req["cmd"])
raise BackendNotAvailable(exc) from exc
try:
rep = serializer.rep_loads(raw_rep)
except ProtocolError as exc:
transport.logger.exception("Invalid response data",
cmd=req["cmd"],
error=exc)
raise BackendProtocolError("Invalid response data") from exc
if rep["status"] == "invalid_msg_format":
transport.logger.error("Invalid request data according to backend",
cmd=req["cmd"],
rep=rep)
raise BackendProtocolError("Invalid request data according to backend")
if rep["status"] == "bad_timestamp":
raise BackendOutOfBallparkError(rep)
# Backward compatibility with older backends (<= v2.3)
if rep["status"] == "invalid_certification" and "timestamp" in rep[
"reason"]:
raise BackendOutOfBallparkError(rep)
return rep
async def _run_manager(self):
while True:
try:
with trio.CancelScope() as self._manager_connect_cancel_scope:
try:
await self._manager_connect()
except (BackendNotAvailable, BackendConnectionRefused):
pass
except Exception as exc:
await self.set_status(
BackendConnStatus.CRASHED,
BackendNotAvailable(
f"Backend connection manager has crashed: {exc}"
),
)
logger.exception("Unhandled exception")
finally:
self._manager_connect_cancel_scope = None
assert self.status not in (BackendConnStatus.READY,
BackendConnStatus.INITIALIZING)
if self.status == BackendConnStatus.LOST:
# Start with a 0s cooldown and increase by power of 2 until
# max cooldown every time the connection trial fails
# (e.g. 0, 1, 2, 4, 8, 15, 15, 15 etc. if max cooldown is 15s)
if self._backend_connection_failures < 1:
# A cooldown time of 0 second is useful for the specific case of a
# revocation when the event listener is the only running transport.
# This way, we don't have to wait 1 second before the revocation is
# detected.
cooldown_time = 0
else:
cooldown_time = 2**(self._backend_connection_failures - 1)
if cooldown_time > self.max_cooldown:
cooldown_time = self.max_cooldown
self._backend_connection_failures += 1
logger.info("Backend offline", cooldown_time=cooldown_time)
await trio.sleep(cooldown_time)
if self.status == BackendConnStatus.REFUSED:
# It's most likely useless to retry connection anyway
logger.info("Backend connection refused", status=self.status)
await trio.sleep_forever()
if self.status == BackendConnStatus.CRASHED:
# It's most likely useless to retry connection anyway
logger.info("Backend connection has crashed",
status=self.status)
await trio.sleep_forever()
if self.status == BackendConnStatus.DESYNC:
# Try again in 10 seconds
logger.info("Backend connection is desync", status=self.status)
await trio.sleep(DESYNC_RETRY_TIME)
async def _send_cmd(transport: Transport, serializer, **req) -> dict:
"""
Raises:
Backend
BackendNotAvailable
BackendProtocolError
BackendCmdsInvalidRequest
BackendCmdsInvalidResponse
BackendNotAvailable
BackendCmdsBadResponse
"""
transport.logger.info("Request", cmd=req["cmd"])
try:
raw_req = serializer.req_dumps(req)
except ProtocolError as exc:
transport.logger.exception("Invalid request data",
cmd=req["cmd"],
error=exc)
raise BackendProtocolError("Invalid request data") from exc
try:
await transport.send(raw_req)
raw_rep = await transport.recv()
except TransportError as exc:
transport.logger.debug("Request failed (backend not available)",
cmd=req["cmd"])
raise BackendNotAvailable(exc) from exc
try:
rep = serializer.rep_loads(raw_rep)
except ProtocolError as exc:
transport.logger.exception("Invalid response data",
cmd=req["cmd"],
error=exc)
raise BackendProtocolError("Invalid response data") from exc
if rep["status"] == "invalid_msg_format":
transport.logger.error("Invalid request data according to backend",
cmd=req["cmd"],
rep=rep)
raise BackendProtocolError("Invalid request data according to backend")
return rep
async def _do_handshake(transport: Transport, handshake):
try:
challenge_req = await transport.recv()
answer_req = handshake.process_challenge_req(challenge_req)
await transport.send(answer_req)
result_req = await transport.recv()
handshake.process_result_req(result_req)
except TransportError as exc:
raise BackendNotAvailable(exc) from exc
except HandshakeError as exc:
raise BackendConnectionRefused(str(exc)) from exc
except ProtocolError as exc:
transport.logger.exception("Protocol error during handshake")
raise BackendProtocolError(exc) from exc
async def _do_handshade(transport: Transport, ch):
try:
challenge_req = await transport.recv()
answer_req = ch.process_challenge_req(challenge_req)
await transport.send(answer_req)
result_req = await transport.recv()
ch.process_result_req(result_req)
transport.logger.debug("Handshake done")
except TransportError as exc:
raise BackendNotAvailable(exc) from exc
except HandshakeRevokedDevice as exc:
transport.logger.warning("Handshake failed", reason=exc)
raise BackendDeviceRevokedError(exc) from exc
except ProtocoleError as exc:
transport.logger.warning("Handshake failed", reason=exc)
raise BackendHandshakeError(exc) from exc
async def _anonymous_cmd(serializer, addr: BackendAddr,
organization_id: OrganizationID, **req) -> dict:
"""
Raises:
BackendNotAvailable
BackendProtocolError
"""
logger.info("Request", cmd=req["cmd"])
try:
raw_req = serializer.req_dumps(req)
except ProtocolError as exc:
logger.exception("Invalid request data", cmd=req["cmd"], error=exc)
raise BackendProtocolError("Invalid request data") from exc
url = addr.to_http_domain_url(f"/anonymous/{organization_id}")
try:
raw_rep = await http_request(url=url, method="POST", data=raw_req)
except OSError as exc:
logger.debug("Request failed (backend not available)",
cmd=req["cmd"],
exc_info=exc)
raise BackendNotAvailable(exc) from exc
try:
rep = serializer.rep_loads(raw_rep)
except ProtocolError as exc:
logger.exception("Invalid response data", cmd=req["cmd"], error=exc)
raise BackendProtocolError("Invalid response data") from exc
if rep["status"] == "invalid_msg_format":
logger.error("Invalid request data according to backend",
cmd=req["cmd"],
rep=rep)
raise BackendProtocolError("Invalid request data according to backend")
if rep["status"] == "bad_timestamp":
raise BackendOutOfBallparkError(rep)
return rep
async def _send_cmd(transport, serializer, **req):
transport.logger.info("Request", cmd=req["cmd"])
def _shorten_data(data):
if len(req) > 300:
return data[:150] + b"[...]" + data[-150:]
else:
return data
try:
raw_req = serializer.req_dumps(req)
except ProtocoleError as exc:
raise BackendCmdsInvalidRequest() from exc
transport.logger.debug("send req", req=_shorten_data(raw_req))
try:
await transport.send(raw_req)
raw_rep = await transport.recv()
except TransportError as exc:
transport.logger.info("Request failed (backend not available)",
cmd=req["cmd"])
raise BackendNotAvailable(exc) from exc
transport.logger.debug("recv rep", req=_shorten_data(raw_rep))
try:
rep = serializer.rep_loads(raw_rep)
except ProtocoleError as exc:
transport.logger.warning("Request failed (bad protocol)",
cmd=req["cmd"],
error=exc)
raise BackendCmdsInvalidResponse(exc) from exc
if rep["status"] == "invalid_msg_format":
raise BackendCmdsInvalidRequest(rep)
return rep
async def _run_manager(self):
while True:
try:
with trio.CancelScope() as self._manager_connect_cancel_scope:
try:
await self._manager_connect()
except (BackendNotAvailable, BackendConnectionRefused):
pass
except Exception as exc:
self._status = BackendConnStatus.CRASHED
self._status_exc = BackendNotAvailable(
f"Backend connection manager has crashed: {exc}")
logger.exception("Unhandled exception")
finally:
self._manager_connect_cancel_scope = None
assert self._status not in (BackendConnStatus.READY,
BackendConnStatus.INITIALIZING)
if self._backend_connection_failures == 0:
self.event_bus.send("backend.connection.changed",
status=self._status,
status_exc=self._status_exc)
if self._status == BackendConnStatus.LOST:
# Start with a 1s cooldown and increase by power of 2 until
# max cooldown every time the connection trial fails
# (e.g. 1, 2, 4, 8, 15, 15, 15 etc. if max cooldown is 15s)
cooldown_time = 2**self._backend_connection_failures
if cooldown_time > self.max_cooldown:
cooldown_time = self.max_cooldown
self._backend_connection_failures += 1
logger.info("Backend offline", cooldown_time=cooldown_time)
await trio.sleep(cooldown_time)
else:
# It's most likely useless to retry connection anyway
logger.info("Backend connection refused", status=self._status)
await trio.sleep_forever()
async def maybe_connect_through_proxy(
hostname: str, port: int, use_ssl: bool) -> Optional[trio.abc.Stream]:
target_url = _build_http_url(hostname, port, use_ssl)
proxy_url = await trio.to_thread.run_sync(blocking_io_get_proxy,
target_url, hostname)
if not proxy_url:
return None
logger.debug("Using proxy to connect",
proxy_url=proxy_url,
target_url=target_url)
# A proxy has been retrieved, parse it url and handle potential auth
try:
proxy = urlsplit(proxy_url)
# Typing helper, as result could be SplitResultBytes if we have provided bytes instead of str
assert isinstance(proxy, SplitResult)
except ValueError as exc:
logger.warning(
"Invalid proxy url, switching to no proxy",
proxy_url=proxy_url,
target_url=target_url,
exc_info=exc,
)
# Invalid url
return None
if proxy.port:
proxy_port = proxy.port
else:
proxy_port = 443 if proxy.scheme == "https" else 80
if not proxy.hostname:
logger.warning(
"Missing required hostname in proxy url, switching to no proxy",
proxy_url=proxy_url,
target_url=target_url,
)
return None
proxy_headers: List[Tuple[str, str]] = []
if proxy.username is not None and proxy.password is not None:
logger.debug("Using `Proxy-Authorization` header with proxy",
username=proxy.username)
proxy_headers.append(("Proxy-Authorization",
cook_basic_auth_header(proxy.username,
proxy.password)))
# Connect to the proxy
stream: trio.abc.Stream
try:
stream = await trio.open_tcp_stream(proxy.hostname, proxy_port)
except OSError as exc:
logger.warning(
"Impossible to connect to proxy",
proxy_hostname=proxy.hostname,
proxy_port=proxy_port,
target_url=target_url,
exc_info=exc,
)
raise BackendNotAvailable(exc) from exc
if proxy.scheme == "https":
logger.debug("Using TLS to secure proxy connection",
hostname=proxy.hostname)
from parsec.core.backend_connection.transport import _upgrade_stream_to_ssl
stream = _upgrade_stream_to_ssl(stream, proxy.hostname)
# Ask the proxy to connect the actual host
conn = h11.Connection(our_role=h11.CLIENT)
async def send(event):
data = conn.send(event)
await stream.send_all(data)
async def next_event():
while True:
event = conn.next_event()
if event is h11.NEED_DATA:
# Note there is no need to handle 100 continue here given we are client
# (see https://h11.readthedocs.io/en/v0.10.0/api.htm1l#flow-control)
data = await stream.receive_some(2048)
conn.receive_data(data)
continue
return event
host = f"{hostname}:{port}"
try:
req = h11.Request(
method="CONNECT",
target=host,
headers=[
# According to RFC7230 (https://datatracker.ietf.org/doc/html/rfc7230#section-5.4)
# Client must provide Host header, but the proxy must replace it
# with the host information of the request-target. So in theory
# we could set any dummy value for the Host header here !
("Host", host),
# User-Agent is not a mandatory header, however http proxy are
# half-broken shenanigans and get suspicious if it is missing
("User-Agent", USER_AGENT),
*proxy_headers,
],
)
logger.debug("Sending CONNECT to proxy", proxy_url=proxy_url, req=req)
await send(req)
answer = await next_event()
logger.debug("Receiving CONNECT answer from proxy",
proxy_url=proxy_url,
answer=answer)
if not isinstance(answer,
h11.Response) or not 200 <= answer.status_code < 300:
logger.warning(
"Bad answer from proxy to CONNECT request",
proxy_url=proxy_url,
target_host=host,
target_url=target_url,
answer_status=answer.status_code,
)
raise BackendNotAvailable("Bad answer from proxy")
# Successful CONNECT should reset the connection's statemachine
answer = await next_event()
while not isinstance(answer, h11.PAUSED):
logger.debug(
"Receiving additional answer to our CONNECT from proxy",
proxy_url=proxy_url,
answer=answer,
)
answer = await next_event()
except trio.BrokenResourceError as exc:
logger.warning(
"Proxy has unexpectedly closed the connection",
proxy_url=proxy_url,
target_host=host,
target_url=target_url,
exc_info=exc,
)
raise BackendNotAvailable(
"Proxy has unexpectedly closed the connection") from exc
return stream
async def connect(
addr: Union[BackendAddr, BackendOrganizationBootstrapAddr,
BackendOrganizationAddr],
device_id: Optional[DeviceID] = None,
signing_key: Optional[SigningKey] = None,
administration_token: Optional[str] = None,
keepalive: Optional[int] = None,
) -> Transport:
"""
Raises:
BackendConnectionError
"""
if administration_token:
if not isinstance(addr, BackendAddr):
raise BackendConnectionError(f"Invalid url format `{addr}`")
handshake = AdministrationClientHandshake(administration_token)
elif not device_id:
if isinstance(addr, BackendOrganizationBootstrapAddr):
handshake = AnonymousClientHandshake(addr.organization_id)
elif isinstance(addr, BackendOrganizationAddr):
handshake = AnonymousClientHandshake(addr.organization_id,
addr.root_verify_key)
else:
raise BackendConnectionError(
f"Invalid url format `{addr}` "
"(should be an organization url or organization bootstrap url)"
)
else:
if not isinstance(addr, BackendOrganizationAddr):
raise BackendConnectionError(
f"Invalid url format `{addr}` (should be an organization url)")
if not signing_key:
raise BackendConnectionError(
f"Missing signing_key to connect as `{device_id}`")
handshake = AuthenticatedClientHandshake(addr.organization_id,
device_id, signing_key,
addr.root_verify_key)
try:
stream = await trio.open_tcp_stream(addr.hostname, addr.port)
except OSError as exc:
logger.debug("Impossible to connect to backend", reason=exc)
raise BackendNotAvailable(exc) from exc
if addr.use_ssl:
stream = _upgrade_stream_to_ssl(stream, addr.hostname)
try:
transport = await Transport.init_for_client(stream, host=addr.hostname)
transport.handshake = handshake
transport.keepalive = keepalive
except TransportError as exc:
logger.debug("Connection lost during transport creation", reason=exc)
raise BackendNotAvailable(exc) from exc
try:
await _do_handshake(transport, handshake)
except Exception as exc:
transport.logger.debug("Connection lost during handshake", reason=exc)
await transport.aclose()
raise
return transport
