def run(self):
"""
This is executed when the thread is started. It gets a connection to
the cloud dispatcher, and calls its bidirectional streaming rpc
EstablishSyncRPCStream(). process_streams should never return, and
if it did, exception will be logged, and new connection to dispatcher
will be attempted after RETRY_DELAY_SECS seconds.
"""
while True:
try:
start_time = time.time()
chan = ServiceRegistry.get_rpc_channel('dispatcher',
ServiceRegistry.CLOUD)
client = SyncRPCServiceStub(chan)
self._set_connect_time()
self.process_streams(client)
except Exception as exp: # pylint: disable=broad-except
conn_time = time.time() - start_time
logging.error("[SyncRPC] Error after %ds: %s", conn_time, exp)
# If the connection is terminated, wait for a period of time
# before connecting back to the cloud.
# Also clear the conn closed table since cloud may reuse req IDs,
# and clear current proxy client connections
self._conn_closed_table.clear()
self._proxy_client.close_all_connections()
self._retry_connect_sleep()
def run(self):
"""
This is executed when the thread is started. It gets a connection to
the cloud dispatcher, and calls its bidirectional streaming rpc
EstablishSyncRPCStream(). process_streams should never return, and
if it did, exception will be logged, and new connection to dispatcher
will be attempted after RETRY_DELAY_SECS seconds.
"""
while True:
try:
start_time = time.time()
chan = ServiceRegistry.get_rpc_channel('dispatcher',
ServiceRegistry.CLOUD)
client = SyncRPCServiceStub(chan)
self._set_connect_time()
self.process_streams(client)
except grpc.RpcError as err:
if is_grpc_error_retryable(err):
logging.error(
"[SyncRPC] Transient gRPC error, retrying: %s",
err.details())
self._retry_connect_sleep()
continue
else:
logging.error(
"[SyncRPC] gRPC error: %s, reconnecting to "
"cloud.", err.details())
self._cleanup_and_reconnect()
except Exception as exp: # pylint: disable=broad-except
conn_time = time.time() - start_time
logging.error("[SyncRPC] Error after %ds: %s", conn_time, exp)
self._cleanup_and_reconnect()
def process_streams(self, client: SyncRPCServiceStub) -> None:
"""
Calls rpc function EstablishSyncRPCStream on the client to establish
a stream with dispatcher in the cloud, processes all requests from
the stream, and writes all responses to the stream.
Args:
client: a grpc client to dispatcher in the cloud.
Returns:
Should only return when an exception is encountered.
"""
# call to bidirectional streaming grpc takes in an iterator,
# and returns an iterator
sync_rpc_requests = client.EstablishSyncRPCStream(
self.send_sync_rpc_response())
# forward incoming requests from cloud to control_proxy
self.forward_requests(sync_rpc_requests)