def recv(self, endpoint: ErEndpoint, tag, broker=FifoBroker()):
try:
@_exception_logger
def fill_broker(iterable: Iterable, broker):
iterator = iter(iterable)
for e in iterator:
broker.put(e)
broker.signal_write_finish()
channel = self.__grpc_channel_factory.create_channel(endpoint)
stub = transfer_pb2_grpc.TransferServiceStub(channel)
request = transfer_pb2.TransferBatch(
header=transfer_pb2.TransferHeader(id=1, tag=tag))
response_iter = stub.recv(request,
metadata=[(TRANSFER_BROKER_NAME, tag)])
if broker is None:
return response_iter
else:
t = Thread(target=fill_broker, args=[response_iter, broker])
t.start()
return broker
except Exception as e:
L.error(f'Error calling to {endpoint} in TransferClient.recv')
raise e
def recv(self, endpoint: ErEndpoint, tag, broker):
exception = None
cur_retry = 0
for cur_retry in range(3):
try:
L.trace(
f'TransferClient.recv for endpoint={endpoint}, tag={tag}')
@_exception_logger
def fill_broker(iterable: Iterable, broker):
try:
iterator = iter(iterable)
for e in iterator:
broker.put(e)
broker.signal_write_finish()
except Exception as e:
L.exception(
f'Fail to fill broker for tag: {tag}, endpoint: {endpoint}'
)
raise e
channel = self.__grpc_channel_factory.create_channel(endpoint)
stub = transfer_pb2_grpc.TransferServiceStub(channel)
request = transfer_pb2.TransferBatch(
header=transfer_pb2.TransferHeader(id=1, tag=tag))
response_iter = stub.recv(request,
metadata=[(TRANSFER_BROKER_NAME, tag)
])
if broker is None:
return response_iter
else:
t = Thread(target=fill_broker,
args=[response_iter, broker])
t.start()
return broker
except Exception as e:
L.warn(
f'Error calling to {endpoint} in TransferClient.recv, cur_retry={cur_retry}',
exc_info=e)
exception = e
cur_retry += 1
if exception is not None:
L.exception(
f'fail to {endpoint} in TransferClient.recv, cur_retry={cur_retry}',
exc_info=e)
raise exception
def send(self, broker, endpoint: ErEndpoint, tag):
try:
channel = self.__grpc_channel_factory.create_channel(endpoint)
stub = transfer_pb2_grpc.TransferServiceStub(channel)
import types
if isinstance(broker, types.GeneratorType):
requests = (transfer_pb2.TransferBatch(header=transfer_pb2.TransferHeader(id=i, tag=tag), data=d)
for i, d in enumerate(broker))
else:
requests = TransferService.transfer_batch_generator_from_broker(broker, tag)
future = stub.send.future(requests, metadata=[(TRANSFER_BROKER_NAME, tag)])
return future
except Exception as e:
L.error(f'Error calling to {endpoint} in TransferClient.send')
raise e