def _on_rpc_done(rpc_id: int, future: grpc.Future, method: str,
print_response: bool) -> None:
exception = future.exception()
hostname = ""
if exception is not None:
if exception.code() == grpc.StatusCode.DEADLINE_EXCEEDED:
logger.error(f"RPC {rpc_id} timed out")
else:
logger.error(exception)
else:
response = future.result()
hostname = None
for metadatum in future.initial_metadata():
if metadatum[0] == "hostname":
hostname = metadatum[1]
break
else:
hostname = response.hostname
if print_response:
if future.code() == grpc.StatusCode.OK:
logger.info("Successful response.")
else:
logger.info(f"RPC failed: {call}")
with _global_lock:
for watcher in _watchers:
watcher.on_rpc_complete(rpc_id, hostname, method)
def gf_terminated(gf: grpc.Future) -> None:
if gf.cancelled():
cf.cancel()
return
try:
result = gf.result(timeout=0)
cf.set_result(result)
except Exception as err:
cf.set_exception(err)
def _process_dispatched_task(task: grpc.Future, detect_anomalies: bool) -> Optional[Tuple[torch.Tensor]]:
if task.exception() or task.cancelled():
logger.warning(f"Task {task} failed: {type(task.exception())}")
return None
deserialized_outputs = []
for tensor in task.result().tensors:
deserialized_tensor = deserialize_torch_tensor(tensor)
if detect_anomalies and not deserialized_tensor.isfinite().all():
logger.error(f"Task {task} failed: output tensor contains nan/inf values")
return None
deserialized_outputs.append(deserialized_tensor)
return tuple(deserialized_outputs)
def grpc_future_to_concurrent_future(gf: grpc.Future) -> concurrent.futures.Future:
cf: concurrent.futures.Future = concurrent.futures.Future()
def gf_terminated(gf: grpc.Future) -> None:
if gf.cancelled():
cf.cancel()
return
try:
result = gf.result(timeout=0)
cf.set_result(result)
except Exception as err:
cf.set_exception(err)
if cf.set_running_or_notify_cancel():
gf.add_done_callback(gf_terminated)
else:
gf.cancel()
return cf
def _on_rpc_done(rpc_id: int, future: grpc.Future,
print_response: bool) -> None:
exception = future.exception()
hostname = ""
if exception is not None:
if exception.code() == grpc.StatusCode.DEADLINE_EXCEEDED:
logger.error(f"RPC {rpc_id} timed out")
else:
logger.error(exception)
else:
response = future.result()
logger.info(f"Got result {rpc_id}")
hostname = response.hostname
if print_response:
if future.code() == grpc.StatusCode.OK:
logger.info("Successful response.")
else:
logger.info(f"RPC failed: {call}")
with _global_lock:
for watcher in _watchers:
watcher.on_rpc_complete(rpc_id, hostname)
def inference_done(self, future: grpc.Future):
elapsed_time = time.time() - self.start_time_queue.pop(0)
try:
self.back_pressure_detection()
inference_result: grpc_def.InferenceResult = future.result(None)
num_processed_images = len(inference_result.result)
stats = InferenceStats(num_processed_images, elapsed_time)
self.stats_chan.on_next(stats)
# notify new detections
self.result_chan.on_next(inference_result)
except Exception as ex:
self.error_chan.on_next(f"Inference error: {ex}")