def send(self, request_iterator, context):
inited = False
response_header = None
broker = None
base_tag = None
for request in request_iterator:
if not inited:
base_tag = request.header.tag
L.info(f'GrpcTransferServicer send broker init. tag: {base_tag}')
broker = TransferService.get_broker(base_tag)
# response_header = request.header
# linux error:TypeError: Parameter to MergeFrom() must be instance of same class: expected TransferHeader got TransferHeader. for field TransferBatch.header
response_header = transfer_pb2.TransferHeader(tag=base_tag, id=request.header.id)
inited = True
broker.put(request)
if inited:
L.info(f'GrpcTransferServicer stream finished. tag: {base_tag}, remaining write count: {broker,broker.__dict__}, stream not empty')
result = transfer_pb2.TransferBatch(header=response_header)
else:
L.warn(f'broker is None. Getting tag from metadata')
metadata = dict(context.invocation_metadata())
base_tag = metadata[TRANSFER_BROKER_NAME]
broker = TransferService.get_broker(base_tag)
L.info(f"empty requests for tag: {base_tag}")
L.info(f'GrpcTransferServicer stream finished. tag: {base_tag}, remaining write count: {broker,broker.__dict__}, stream empty')
result = transfer_pb2.TransferBatch()
broker.signal_write_finish()
return result
def recv(self, endpoint: ErEndpoint, tag, broker=FifoBroker()):
try:
@_exception_logger
def fill_broker(iterable: Iterable, broker):
iterator = iter(iterable)
for e in iterator:
broker.put(e)
broker.signal_write_finish()
channel = self.__grpc_channel_factory.create_channel(endpoint)
stub = transfer_pb2_grpc.TransferServiceStub(channel)
request = transfer_pb2.TransferBatch(
header=transfer_pb2.TransferHeader(id=1, tag=tag))
response_iter = stub.recv(request,
metadata=[(TRANSFER_BROKER_NAME, tag)])
if broker is None:
return response_iter
else:
t = Thread(target=fill_broker, args=[response_iter, broker])
t.start()
return broker
except Exception as e:
L.error(f'Error calling to {endpoint} in TransferClient.recv')
raise e
def recv(self, endpoint: ErEndpoint, tag, broker):
exception = None
cur_retry = 0
for cur_retry in range(3):
try:
L.trace(
f'TransferClient.recv for endpoint={endpoint}, tag={tag}')
@_exception_logger
def fill_broker(iterable: Iterable, broker):
try:
iterator = iter(iterable)
for e in iterator:
broker.put(e)
broker.signal_write_finish()
except Exception as e:
L.exception(
f'Fail to fill broker for tag: {tag}, endpoint: {endpoint}'
)
raise e
channel = self.__grpc_channel_factory.create_channel(endpoint)
stub = transfer_pb2_grpc.TransferServiceStub(channel)
request = transfer_pb2.TransferBatch(
header=transfer_pb2.TransferHeader(id=1, tag=tag))
response_iter = stub.recv(request,
metadata=[(TRANSFER_BROKER_NAME, tag)
])
if broker is None:
return response_iter
else:
t = Thread(target=fill_broker,
args=[response_iter, broker])
t.start()
return broker
except Exception as e:
L.warn(
f'Error calling to {endpoint} in TransferClient.recv, cur_retry={cur_retry}',
exc_info=e)
exception = e
cur_retry += 1
if exception is not None:
L.exception(
f'fail to {endpoint} in TransferClient.recv, cur_retry={cur_retry}',
exc_info=e)
raise exception
def recv(self, request, context):
base_tag = request.header.tag
L.info(f'GrpcTransferServicer recv broker tag: {base_tag}')
callee_messages_broker = TransferService.get_broker(base_tag)
import types
if isinstance(callee_messages_broker, types.GeneratorType):
i = 0
for data in callee_messages_broker:
header = transfer_pb2.TransferHeader(id=i, tag=base_tag)
batch = transfer_pb2.TransferBatch(header=header, data=data)
i+=1
yield batch
else:
return TransferService.transfer_batch_generator_from_broker(callee_messages_broker, base_tag)
def send(self, broker, endpoint: ErEndpoint, tag):
try:
channel = self.__grpc_channel_factory.create_channel(endpoint)
stub = transfer_pb2_grpc.TransferServiceStub(channel)
import types
if isinstance(broker, types.GeneratorType):
requests = (transfer_pb2.TransferBatch(header=transfer_pb2.TransferHeader(id=i, tag=tag), data=d)
for i, d in enumerate(broker))
else:
requests = TransferService.transfer_batch_generator_from_broker(broker, tag)
future = stub.send.future(requests, metadata=[(TRANSFER_BROKER_NAME, tag)])
return future
except Exception as e:
L.error(f'Error calling to {endpoint} in TransferClient.send')
raise e
def transfer_batch_generator_from_broker(broker: FifoBroker, tag):
i = 0
while not broker.is_closable():
try:
data = broker.get(block=True, timeout=0.1)
if data:
header = transfer_pb2.TransferHeader(id=i, tag=tag)
batch = transfer_pb2.TransferBatch(header=header, data=data)
i += 1
yield batch
except queue.Empty as e:
# print("transfer client queue empty")
pass
except BrokerClosed as e:
break
except Exception as e:
print(e)
def send(self, request_iterator, context):
inited = False
response_header = None
broker = None
base_tag = None
try:
for request in request_iterator:
if not inited:
base_tag = request.header.tag
L.debug(
f'GrpcTransferServicer send broker init. tag={base_tag}'
)
broker = TransferService.get_broker(base_tag)
# response_header = request.header
# linux error:TypeError: Parameter to MergeFrom() must be instance of same class: expected TransferHeader got TransferHeader. for field TransferBatch.header
response_header = transfer_pb2.TransferHeader(
tag=base_tag,
id=request.header.id,
ext=request.header.ext)
inited = True
broker.put(request)
if inited:
L.trace(
f'GrpcTransferServicer stream finished. tag={base_tag}, remaining write count={broker,broker.__dict__}, stream not empty'
)
result = transfer_pb2.TransferBatch(header=response_header)
else:
raise ValueError(
'error in GrpcTransferServicer.send: empty request_iterator'
)
return result
except Exception as e:
TransferService.remove_broker(base_tag)
raise ValueError(f"error in processing {base_tag}", e)
finally:
if broker is not None:
broker.signal_write_finish()
def from_proto_string(pb_string):
pb_message = transfer_pb2.TransferBatch()
msg_len = pb_message.ParseFromString(pb_string)
return ErTransferBatch.from_proto(pb_message)
def to_proto(self):
return transfer_pb2.TransferBatch(header=self._header.to_proto(),
batchSize=self._batch_size,
data=self._data)
