def _decode_one_batch_from_stream(self, in_stream: create_InputStream, size: int) -> List: self._resettable_io.set_input_bytes(in_stream.read(size)) # there is only one arrow batch in the underlying input stream return arrow_to_pandas(self._timezone, self._field_types, [next(self._batch_reader)])
def load_from_stream(self, stream): import pyarrow as pa reader = pa.ipc.open_stream(stream) for batch in reader: yield arrow_to_pandas(self._timezone, self._field_types, [batch])