def run_secure_loop(cert):
creds = grpc.ssl_channel_credentials(cert)
with grpc.secure_channel("%s:%d" % (self.ip_address, self.port),
creds) as channel:
future = grpc.channel_ready_future(channel)
connect_retries = 0
while not self.ping_timer.is_set():
try:
future.result(timeout=2)
self.stub = warp_pb2_grpc.WarpStub(channel)
self.connected = True
break
except grpc.FutureTimeoutError:
if connect_retries < MAX_CONNECT_RETRIES:
# print("channel ready timeout, waiting 10s")
self.ping_timer.wait(self.ping_time)
connect_retries += 1
continue
else:
self.set_remote_status(RemoteStatus.UNREACHABLE)
# print("Trying to remake channel")
future.cancel()
return True
one_ping = False
while not self.ping_timer.is_set():
try:
self.stub.Ping(void, timeout=2)
if not one_ping:
# Wait
self.set_remote_status(
RemoteStatus.AWAITING_DUPLEX)
if self.check_duplex_connection():
self.set_remote_status(RemoteStatus.ONLINE)
self.update_remote_machine_info()
self.update_remote_machine_avatar()
self.ping_time = PING_TIME
one_ping = True
except grpc.RpcError as e:
if e.code() in (grpc.StatusCode.DEADLINE_EXCEEDED,
grpc.StatusCode.UNAVAILABLE):
one_ping = False
self.set_remote_status(RemoteStatus.UNREACHABLE)
self.ping_timer.wait(self.ping_time)
return False
def keep_channel():
with grpc.insecure_channel(
"%s:%d" % (self.ip_address, self.port)) as channel:
future = grpc.channel_ready_future(channel)
connect_retries = 0
while not self.need_shutdown:
try:
future.result(timeout=2)
self.stub = warp_pb2_grpc.WarpStub(channel)
break
except grpc.FutureTimeoutError:
if connect_retries < MAX_CONNECT_RETRIES:
print("channel ready timeout, waiting 10s")
time.sleep(PING_TIME)
connect_retries += 1
continue
else:
self.set_remote_status(RemoteStatus.UNREACHABLE)
print("Trying to remake channel")
return True
one_ping = False
while not self.need_shutdown:
try:
self.stub.Ping(void, timeout=2)
self.set_remote_status(RemoteStatus.ONLINE)
if not one_ping:
self.update_remote_machine_info()
self.update_remote_machine_avatar()
one_ping = True
except grpc.RpcError as e:
if e.code() in (grpc.StatusCode.DEADLINE_EXCEEDED,
grpc.StatusCode.UNAVAILABLE):
one_ping = False
self.set_remote_status(RemoteStatus.UNREACHABLE)
time.sleep(PING_TIME)
return False
def run_secure_loop():
logging.debug(
"Remote: Starting a new connection loop for %s (%s:%d)" %
(self.display_hostname, self.ip_address, self.port))
cert = auth.get_singleton().load_cert(self.hostname,
self.ip_address)
creds = grpc.ssl_channel_credentials(cert)
with grpc.secure_channel("%s:%d" % (self.ip_address, self.port),
creds) as channel:
future = grpc.channel_ready_future(channel)
try:
future.result(timeout=4)
self.stub = warp_pb2_grpc.WarpStub(channel)
except grpc.FutureTimeoutError:
self.set_remote_status(RemoteStatus.UNREACHABLE)
future.cancel()
if not self.ping_timer.is_set():
logging.debug(
"Remote: Unable to establish secure connection with %s (%s:%d). Trying again in %ds"
% (self.display_hostname, self.ip_address,
self.port, CHANNEL_RETRY_WAIT_TIME))
self.ping_timer.wait(CHANNEL_RETRY_WAIT_TIME)
return True # run_secure_loop()
return False # run_secure_loop()
duplex_fail_counter = 0
one_ping = False # A successful duplex response lets us finish setting things up.
while not self.ping_timer.is_set():
if self.busy:
logging.debug(
"Remote Ping: Skipping keepalive ping to %s (%s:%d) (busy)"
% (self.display_hostname, self.ip_address,
self.port))
self.busy = False
else:
try:
# t = GLib.get_monotonic_time()
logging.debug("Remote Ping: to %s (%s:%d)" %
(self.display_hostname,
self.ip_address, self.port))
self.stub.Ping(warp_pb2.LookupName(
id=self.local_ident,
readable_name=util.get_hostname()),
timeout=5)
# logging.debug("Latency: %s (%s)"
# % (util.precise_format_time_span(GLib.get_monotonic_time() - t), self.display_hostname))
if not one_ping:
self.set_remote_status(
RemoteStatus.AWAITING_DUPLEX)
if self.check_duplex_connection():
logging.debug(
"Remote: Connected to %s (%s:%d)" %
(self.display_hostname,
self.ip_address, self.port))
self.set_remote_status(RemoteStatus.ONLINE)
self.rpc_call(
self.update_remote_machine_info)
self.rpc_call(
self.update_remote_machine_avatar)
one_ping = True
else:
duplex_fail_counter += 1
if duplex_fail_counter > DUPLEX_MAX_FAILURES:
logging.debug(
"Remote: CheckDuplexConnection to %s (%s:%d) failed too many times"
% (self.display_hostname,
self.ip_address, self.port))
self.ping_timer.wait(
CHANNEL_RETRY_WAIT_TIME)
return True
except grpc.RpcError as e:
logging.debug(
"Remote: Ping failed, shutting down %s (%s:%d)"
% (self.display_hostname, self.ip_address,
self.port))
break
self.ping_timer.wait(
CONNECTED_PING_TIME if self.status ==
RemoteStatus.ONLINE else DUPLEX_WAIT_PING_TIME)
# This is reached by the RpcError break above. If the remote is still discoverable, start
# the secure loop over. This could have happened as a result of a quick disco/reconnect,
# And we don't notice until it has already come back. In this case, try a new connection.
if self.has_zc_presence and not self.ping_timer.is_set():
return True # run_secure_loop()
# The ping timer has been triggered, this is an orderly shutdown.
return False # run_secure_loop()
def run_secure_loop():
opts = (('grpc.keepalive_time_ms',
10000), ('grpc.keepalive_timeout_ms', 5000),
('grpc.keepalive_permit_without_calls',
True), ('grpc.http2.max_pings_without_data', 0),
('grpc.http2.min_time_between_pings_ms', 10000),
('grpc.http2.min_ping_interval_without_data_ms', 5000))
with grpc.secure_channel("%s:%d" % (self.ips, self.port),
creds,
options=opts) as channel:
def channel_state_changed(state):
if state != grpc.ChannelConnectivity.READY:
# The server may have already called shutdown
try:
self.shutdown()
except:
pass
intercepted_channel = grpc.intercept_channel(
channel, interceptors.ChunkDecompressor())
future = grpc.channel_ready_future(intercepted_channel)
try:
future.result(timeout=4)
channel.subscribe(channel_state_changed)
self.stub = warp_pb2_grpc.WarpStub(intercepted_channel)
self.set_remote_status(RemoteStatus.AWAITING_DUPLEX)
duplex = self.wait_for_duplex()
duplex.result(timeout=10)
self.set_remote_status(RemoteStatus.ONLINE)
self.rpc_call(self.update_remote_machine_info)
self.rpc_call(self.update_remote_machine_avatar)
# Online loop
while not self.channel_keepalive.is_set():
self.channel_keepalive.wait(.5)
##
except Exception as e:
print("exception")
self.set_remote_status(RemoteStatus.UNREACHABLE)
if isinstance(e, grpc.FutureTimeoutError):
future.cancel()
logging.critical(
"Problem while waiting for channel - api version 2: %s"
% e)
elif isinstance(e, grpc.RpcError):
logging.critical(
"Problem while awaiting duplex response - api version 2: %s - %s"
% (e.code(), e.details()))
else:
logging.critical(
"General error with remote channel connection - api version 2: %s"
% e)
self.channel_keepalive.wait(10)
finally:
channel.unsubscribe(channel_state_changed)