def run(self): """ This is executed when the thread is started. It gets a connection to the cloud dispatcher, and calls its bidirectional streaming rpc EstablishSyncRPCStream(). process_streams should never return, and if it did, exception will be logged, and new connection to dispatcher will be attempted after RETRY_DELAY_SECS seconds. """ while True: try: start_time = time.time() chan = ServiceRegistry.get_rpc_channel('dispatcher', ServiceRegistry.CLOUD) client = SyncRPCServiceStub(chan) self._set_connect_time() self.process_streams(client) except Exception as exp: # pylint: disable=broad-except conn_time = time.time() - start_time logging.error("[SyncRPC] Error after %ds: %s", conn_time, exp) # If the connection is terminated, wait for a period of time # before connecting back to the cloud. # Also clear the conn closed table since cloud may reuse req IDs, # and clear current proxy client connections self._conn_closed_table.clear() self._proxy_client.close_all_connections() self._retry_connect_sleep()
def run(self): """ This is executed when the thread is started. It gets a connection to the cloud dispatcher, and calls its bidirectional streaming rpc EstablishSyncRPCStream(). process_streams should never return, and if it did, exception will be logged, and new connection to dispatcher will be attempted after RETRY_DELAY_SECS seconds. """ while True: try: start_time = time.time() chan = ServiceRegistry.get_rpc_channel('dispatcher', ServiceRegistry.CLOUD) client = SyncRPCServiceStub(chan) self._set_connect_time() self.process_streams(client) except grpc.RpcError as err: if is_grpc_error_retryable(err): logging.error( "[SyncRPC] Transient gRPC error, retrying: %s", err.details()) self._retry_connect_sleep() continue else: logging.error( "[SyncRPC] gRPC error: %s, reconnecting to " "cloud.", err.details()) self._cleanup_and_reconnect() except Exception as exp: # pylint: disable=broad-except conn_time = time.time() - start_time logging.error("[SyncRPC] Error after %ds: %s", conn_time, exp) self._cleanup_and_reconnect()
def process_streams(self, client: SyncRPCServiceStub) -> None: """ Calls rpc function EstablishSyncRPCStream on the client to establish a stream with dispatcher in the cloud, processes all requests from the stream, and writes all responses to the stream. Args: client: a grpc client to dispatcher in the cloud. Returns: Should only return when an exception is encountered. """ # call to bidirectional streaming grpc takes in an iterator, # and returns an iterator sync_rpc_requests = client.EstablishSyncRPCStream( self.send_sync_rpc_response()) # forward incoming requests from cloud to control_proxy self.forward_requests(sync_rpc_requests)