def next(self) -> SampleBatchType:
# next() should not get called on None DatasetReader.
assert self._iter is not None
d = next(self._iter).as_pydict()
# Columns like obs are compressed when written by DatasetWriter.
d = from_json_data(d, self._ioctx.worker)
return d
def next(self) -> SampleBatchType:
# next() should not get called on None DatasetReader.
assert self._iter is not None
ret = []
count = 0
while count < self.batch_size:
d = next(self._iter).as_pydict()
# Columns like obs are compressed when written by DatasetWriter.
d = from_json_data(d, self._ioctx.worker)
count += d.count
ret.append(self._postprocess_if_needed(d))
ret = concat_samples(ret)
return ret