def _compact_if_needed(self) -> None:
assert self._columns
if self._uncompacted_size.size_bytes() < MAX_UNCOMPACTED_SIZE_BYTES:
return
block = self._table_from_pydict(self._columns)
self.add_block(block)
self._uncompacted_size = SizeEstimator()
self._columns.clear()
self._num_compactions += 1
def _compact_if_needed(self) -> None:
assert self._columns
if self._uncompacted_size.size_bytes() < MAX_UNCOMPACTED_SIZE_BYTES:
return
block = pyarrow.Table.from_pydict(self._columns)
self._tables.append(block)
self._tables_nbytes += block.nbytes
self._uncompacted_size = SizeEstimator()
self._columns.clear()
self._num_compactions += 1
def __init__(self, block_type):
# The set of uncompacted Python values buffered.
self._columns = collections.defaultdict(list)
# The set of compacted tables we have built so far.
self._tables: List[Any] = []
self._tables_size_bytes = 0
# Size estimator for un-compacted table values.
self._uncompacted_size = SizeEstimator()
self._num_rows = 0
self._num_compactions = 0
self._block_type = block_type
def __init__(self):
if pyarrow is None:
raise ImportError("Run `pip install pyarrow` for Arrow support")
# The set of uncompacted Python values buffered.
self._columns = collections.defaultdict(list)
# The set of compacted tables we have built so far.
self._tables: List["pyarrow.Table"] = []
self._tables_nbytes = 0
# Size estimator for un-compacted table values.
self._uncompacted_size = SizeEstimator()
self._num_rows = 0
self._num_compactions = 0
def __init__(self):
self._items = []
self._size_estimator = SizeEstimator()