def _write_block(self, f: "pyarrow.NativeFile", block: BlockAccessor, writer_args_fn: Callable[[], Dict[str, Any]] = lambda: {}, **writer_args): import pyarrow.parquet as pq writer_args = _resolve_kwargs(writer_args_fn, **writer_args) pq.write_table(block.to_arrow(), f, **writer_args)
def _write_block(self, f: "pyarrow.NativeFile", block: BlockAccessor, writer_args_fn: Callable[[], Dict[str, Any]] = lambda: {}, **writer_args): from pyarrow import csv writer_args = _resolve_kwargs(writer_args_fn, **writer_args) write_options = writer_args.pop("write_options", None) csv.write_csv(block.to_arrow(), f, write_options, **writer_args)
def _zip(self, acc: BlockAccessor) -> "Block[T]": r = self.to_arrow() s = acc.to_arrow() for col_name in s.column_names: col = s.column(col_name) # Ensure the column names are unique after zip. if col_name in r.column_names: i = 1 new_name = col_name while new_name in r.column_names: new_name = "{}_{}".format(col_name, i) i += 1 col_name = new_name r = r.append_column(col_name, col) return r
def _write_block(self, f: "pyarrow.NativeFile", block: BlockAccessor, **writer_args): import pyarrow.parquet as pq pq.write_table(block.to_arrow(), f, **writer_args)
def _write_block(self, f: "pyarrow.NativeFile", block: BlockAccessor, **writer_args): from pyarrow import csv write_options = writer_args.pop("write_options", None) csv.write_csv(block.to_arrow(), f, write_options, **writer_args)
def _write_block(self, f: "pyarrow.NativeFile", block: BlockAccessor, **writer_args): np.save(f, block.to_arrow())