def _write_block(self,
f: "pyarrow.NativeFile",
block: BlockAccessor,
writer_args_fn: Callable[[], Dict[str, Any]] = lambda: {},
**writer_args):
import pyarrow.parquet as pq
writer_args = _resolve_kwargs(writer_args_fn, **writer_args)
pq.write_table(block.to_arrow(), f, **writer_args)
def _write_block(self,
f: "pyarrow.NativeFile",
block: BlockAccessor,
writer_args_fn: Callable[[], Dict[str, Any]] = lambda: {},
**writer_args):
from pyarrow import csv
writer_args = _resolve_kwargs(writer_args_fn, **writer_args)
write_options = writer_args.pop("write_options", None)
csv.write_csv(block.to_arrow(), f, write_options, **writer_args)
def _zip(self, acc: BlockAccessor) -> "Block[T]":
r = self.to_arrow()
s = acc.to_arrow()
for col_name in s.column_names:
col = s.column(col_name)
# Ensure the column names are unique after zip.
if col_name in r.column_names:
i = 1
new_name = col_name
while new_name in r.column_names:
new_name = "{}_{}".format(col_name, i)
i += 1
col_name = new_name
r = r.append_column(col_name, col)
return r
def _write_block(self, f: "pyarrow.NativeFile", block: BlockAccessor,
**writer_args):
import pyarrow.parquet as pq
pq.write_table(block.to_arrow(), f, **writer_args)
def _write_block(self, f: "pyarrow.NativeFile", block: BlockAccessor,
**writer_args):
from pyarrow import csv
write_options = writer_args.pop("write_options", None)
csv.write_csv(block.to_arrow(), f, write_options, **writer_args)
def _write_block(self, f: "pyarrow.NativeFile", block: BlockAccessor,
**writer_args):
np.save(f, block.to_arrow())
