class DelegatingBlockBuilder(BlockBuilder[T]): def __init__(self): self._builder = None self._empty_block = None def add(self, item: Any) -> None: if self._builder is None: # TODO (kfstorm): Maybe we can use Pandas block format for dict. if isinstance(item, dict) or isinstance(item, ArrowRow): import pyarrow try: check = ArrowBlockBuilder() check.add(item) check.build() self._builder = ArrowBlockBuilder() except (TypeError, pyarrow.lib.ArrowInvalid): self._builder = SimpleBlockBuilder() elif isinstance(item, PandasRow): self._builder = PandasBlockBuilder() else: self._builder = SimpleBlockBuilder() self._builder.add(item) def add_block(self, block: Block) -> None: accessor = BlockAccessor.for_block(block) if accessor.num_rows() == 0: # Don't infer types of empty lists. Store the block and use it if no # other data is added. https://github.com/ray-project/ray/issues/20290 self._empty_block = block return if self._builder is None: self._builder = accessor.builder() self._builder.add_block(block) def build(self) -> Block: if self._builder is None: if self._empty_block is not None: self._builder = BlockAccessor.for_block( self._empty_block).builder() else: self._builder = ArrowBlockBuilder() return self._builder.build() def num_rows(self) -> int: return self._builder.num_rows() if self._builder is not None else 0 def get_estimated_memory_usage(self) -> int: if self._builder is None: return 0 return self._builder.get_estimated_memory_usage()
class DelegatingBlockBuilder(BlockBuilder[T]): def __init__(self): self._builder = None def add(self, item: Any) -> None: if self._builder is None: # TODO (kfstorm): Maybe we can use Pandas block format for dict. if isinstance(item, dict) or isinstance(item, ArrowRow): import pyarrow try: check = ArrowBlockBuilder() check.add(item) check.build() self._builder = ArrowBlockBuilder() except (TypeError, pyarrow.lib.ArrowInvalid): self._builder = SimpleBlockBuilder() elif isinstance(item, PandasRow): self._builder = PandasBlockBuilder() else: self._builder = SimpleBlockBuilder() self._builder.add(item) def add_block(self, block: Block) -> None: if self._builder is None: self._builder = BlockAccessor.for_block(block).builder() self._builder.add_block(block) def build(self) -> Block: if self._builder is None: self._builder = ArrowBlockBuilder() return self._builder.build() def num_rows(self) -> int: return self._builder.num_rows() if self._builder is not None else 0 def get_estimated_memory_usage(self) -> int: if self._builder is None: return 0 return self._builder.get_estimated_memory_usage()