def __iter__(self): while True: sb = self.stream.read(4) if sb == '': return size = struct.unpack('i', sb)[0] msg = self.stream.read(size) doc = document_pb2.document() doc.ParseFromString(msg) yield doc
def __iter__(self): for path in self.paths: with self.open_single(path) as stream: while True: sb = stream.read(4) if sb == '': break size = struct.unpack('i', sb)[0] msg = stream.read(size) doc = document_pb2.document() doc.ParseFromString(msg) yield doc
def __iter__(self): for file_ in self.files: self.stream = gzip.open(file_, 'rb') while True: sb = self.stream.read(4) if sb == '': break size = struct.unpack('i', sb)[0] msg = self.stream.read(size) doc = document_pb2.document() doc.ParseFromString(msg) yield doc