def zmq_coroutine(context, dealer_url, doc_class=None, automagic=False): # FIXME: reduce overhead of reader/writer creation ostream = io.BytesIO() socket = context.socket(zmq.REP) socket.connect(dealer_url) while True: msg = socket.recv() istream = io.BytesIO(msg) istream.seek(0) reader = Reader(istream, doc_class, automagic) writer = Writer(ostream, reader.doc_schema) for doc in reader: res = yield (doc) writer.write(res or doc) ostream.seek(0) socket.send(ostream.getvalue()) ostream.truncate(0)
def stream_coroutine(istream, ostream, doc_class=None, automagic=False): reader = Reader(istream, doc_class, automagic) writer = Writer(ostream, reader.doc_schema) for doc in reader: res = yield (doc) writer.write(res or doc)
def stream_reader_writer(self): docs, schema = self.get_reader_and_schema() return docs, Writer(self.args.out_stream, schema)