def recv(self, endpoint: ErEndpoint, tag, broker=FifoBroker()): try: @_exception_logger def fill_broker(iterable: Iterable, broker): iterator = iter(iterable) for e in iterator: broker.put(e) broker.signal_write_finish() channel = self.__grpc_channel_factory.create_channel(endpoint) stub = transfer_pb2_grpc.TransferServiceStub(channel) request = transfer_pb2.TransferBatch( header=transfer_pb2.TransferHeader(id=1, tag=tag)) response_iter = stub.recv(request, metadata=[(TRANSFER_BROKER_NAME, tag)]) if broker is None: return response_iter else: t = Thread(target=fill_broker, args=[response_iter, broker]) t.start() return broker except Exception as e: L.error(f'Error calling to {endpoint} in TransferClient.recv') raise e
def send(self, request_iterator, context): inited = False response_header = None broker = None base_tag = None for request in request_iterator: if not inited: base_tag = request.header.tag L.info(f'GrpcTransferServicer send broker init. tag: {base_tag}') broker = TransferService.get_broker(base_tag) # response_header = request.header # linux error:TypeError: Parameter to MergeFrom() must be instance of same class: expected TransferHeader got TransferHeader. for field TransferBatch.header response_header = transfer_pb2.TransferHeader(tag=base_tag, id=request.header.id) inited = True broker.put(request) if inited: L.info(f'GrpcTransferServicer stream finished. tag: {base_tag}, remaining write count: {broker,broker.__dict__}, stream not empty') result = transfer_pb2.TransferBatch(header=response_header) else: L.warn(f'broker is None. Getting tag from metadata') metadata = dict(context.invocation_metadata()) base_tag = metadata[TRANSFER_BROKER_NAME] broker = TransferService.get_broker(base_tag) L.info(f"empty requests for tag: {base_tag}") L.info(f'GrpcTransferServicer stream finished. tag: {base_tag}, remaining write count: {broker,broker.__dict__}, stream empty') result = transfer_pb2.TransferBatch() broker.signal_write_finish() return result
def recv(self, request, context): base_tag = request.header.tag L.info(f'GrpcTransferServicer recv broker tag: {base_tag}') callee_messages_broker = TransferService.get_broker(base_tag) import types if isinstance(callee_messages_broker, types.GeneratorType): i = 0 for data in callee_messages_broker: header = transfer_pb2.TransferHeader(id=i, tag=base_tag) batch = transfer_pb2.TransferBatch(header=header, data=data) i+=1 yield batch else: return TransferService.transfer_batch_generator_from_broker(callee_messages_broker, base_tag)
def send(self, broker, endpoint: ErEndpoint, tag): try: channel = self.__grpc_channel_factory.create_channel(endpoint) stub = transfer_pb2_grpc.TransferServiceStub(channel) import types if isinstance(broker, types.GeneratorType): requests = (transfer_pb2.TransferBatch(header=transfer_pb2.TransferHeader(id=i, tag=tag), data=d) for i, d in enumerate(broker)) else: requests = TransferService.transfer_batch_generator_from_broker(broker, tag) future = stub.send.future(requests, metadata=[(TRANSFER_BROKER_NAME, tag)]) return future except Exception as e: L.error(f'Error calling to {endpoint} in TransferClient.send') raise e
def transfer_batch_generator_from_broker(broker: FifoBroker, tag): i = 0 while not broker.is_closable(): try: data = broker.get(block=True, timeout=0.1) if data: header = transfer_pb2.TransferHeader(id=i, tag=tag) batch = transfer_pb2.TransferBatch(header=header, data=data) i += 1 yield batch except queue.Empty as e: # print("transfer client queue empty") pass except BrokerClosed as e: break except Exception as e: print(e)
def send(self, request_iterator, context): inited = False response_header = None broker = None base_tag = None try: for request in request_iterator: if not inited: base_tag = request.header.tag L.debug( f'GrpcTransferServicer send broker init. tag={base_tag}' ) broker = TransferService.get_broker(base_tag) # response_header = request.header # linux error:TypeError: Parameter to MergeFrom() must be instance of same class: expected TransferHeader got TransferHeader. for field TransferBatch.header response_header = transfer_pb2.TransferHeader( tag=base_tag, id=request.header.id, ext=request.header.ext) inited = True broker.put(request) if inited: L.trace( f'GrpcTransferServicer stream finished. tag={base_tag}, remaining write count={broker,broker.__dict__}, stream not empty' ) result = transfer_pb2.TransferBatch(header=response_header) else: raise ValueError( 'error in GrpcTransferServicer.send: empty request_iterator' ) return result except Exception as e: TransferService.remove_broker(base_tag) raise ValueError(f"error in processing {base_tag}", e) finally: if broker is not None: broker.signal_write_finish()