def read_pandas(source, columns=None, nthreads=1, metadata=None):
"""
Read a Table from Parquet format, reconstructing the index values if
available.
Parameters
----------
source: str or pyarrow.io.NativeFile
Location of Parquet dataset. If a string passed, can be a single file
name. For passing Python file objects or byte buffers,
see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
columns: list
If not None, only these columns will be read from the file.
nthreads : int, default 1
Number of columns to read in parallel. Requires that the underlying
file source is threadsafe
metadata : FileMetaData
If separately computed
Returns
-------
pyarrow.Table
Content of the file as a Table of Columns, including DataFrame indexes
as Columns.
"""
if is_string(source):
fs = LocalFilesystem.get_instance()
if fs.isdir(source):
raise NotImplementedError(
'Reading a directory of Parquet files with DataFrame index '
'metadata is not yet supported'
)
pf = ParquetFile(source, metadata=metadata)
return pf.read_pandas(columns=columns, nthreads=nthreads)
def _generate_partition_directories(base_dir, partition_spec, df):
# partition_spec : list of lists, e.g. [['foo', [0, 1, 2],
# ['bar', ['a', 'b', 'c']]
# part_table : a pyarrow.Table to write to each partition
DEPTH = len(partition_spec)
fs = LocalFilesystem.get_instance()
def _visit_level(base_dir, level, part_keys):
name, values = partition_spec[level]
for value in values:
this_part_keys = part_keys + [(name, value)]
level_dir = pjoin(base_dir, '{0}={1}'.format(name, value))
fs.mkdir(level_dir)
if level == DEPTH - 1:
# Generate example data
file_path = pjoin(level_dir, 'data.parq')
filtered_df = _filter_partition(df, this_part_keys)
part_table = pa.Table.from_pandas(filtered_df)
_write_table(part_table, file_path)
else:
_visit_level(level_dir, level + 1, this_part_keys)
_visit_level(base_dir, 0, [])
def read_table(source, columns=None, nthreads=1, metadata=None):
"""
Read a Table from Parquet format
Parameters
----------
source: str or pyarrow.io.NativeFile
Location of Parquet dataset. If a string passed, can be a single file
name or directory name. For passing Python file objects or byte
buffers, see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
columns: list
If not None, only these columns will be read from the file.
nthreads : int, default 1
Number of columns to read in parallel. Requires that the underlying
file source is threadsafe
metadata : FileMetaData
If separately computed
Returns
-------
pyarrow.Table
Content of the file as a table (of columns)
"""
if is_string(source):
fs = LocalFilesystem.get_instance()
if fs.isdir(source):
return fs.read_parquet(source, columns=columns,
metadata=metadata)
pf = ParquetFile(source, metadata=metadata)
return pf.read(columns=columns, nthreads=nthreads)
def read_table(source, columns=None, nthreads=1, metadata=None):
"""
Read a Table from Parquet format
Parameters
----------
source: str or pyarrow.io.NativeFile
Location of Parquet dataset. If a string passed, can be a single file
name or directory name. For passing Python file objects or byte
buffers, see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
columns: list
If not None, only these columns will be read from the file.
nthreads : int, default 1
Number of columns to read in parallel. Requires that the underlying
file source is threadsafe
metadata : FileMetaData
If separately computed
Returns
-------
pyarrow.Table
Content of the file as a table (of columns)
"""
from pyarrow.filesystem import LocalFilesystem
if isinstance(source, six.string_types):
fs = LocalFilesystem.get_instance()
if fs.isdir(source):
return fs.read_parquet(source, columns=columns, metadata=metadata)
pf = ParquetFile(source, metadata=metadata)
return pf.read(columns=columns, nthreads=nthreads)
def _generate_partition_directories(base_dir, partition_spec, df):
# partition_spec : list of lists, e.g. [['foo', [0, 1, 2],
# ['bar', ['a', 'b', 'c']]
# part_table : a pyarrow.Table to write to each partition
DEPTH = len(partition_spec)
fs = LocalFilesystem.get_instance()
def _visit_level(base_dir, level, part_keys):
name, values = partition_spec[level]
for value in values:
this_part_keys = part_keys + [(name, value)]
level_dir = pjoin(base_dir, '{0}={1}'.format(name, value))
fs.mkdir(level_dir)
if level == DEPTH - 1:
# Generate example data
file_path = pjoin(level_dir, 'data.parq')
filtered_df = _filter_partition(df, this_part_keys)
part_table = pa.Table.from_pandas(filtered_df)
_write_table(part_table, file_path)
else:
_visit_level(level_dir, level + 1, this_part_keys)
_visit_level(base_dir, 0, [])
def read_pandas(source, columns=None, nthreads=1, metadata=None):
"""
Read a Table from Parquet format, reconstructing the index values if
available.
Parameters
----------
source: str or pyarrow.io.NativeFile
Location of Parquet dataset. If a string passed, can be a single file
name. For passing Python file objects or byte buffers,
see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader.
columns: list
If not None, only these columns will be read from the file.
nthreads : int, default 1
Number of columns to read in parallel. Requires that the underlying
file source is threadsafe
metadata : FileMetaData
If separately computed
Returns
-------
pyarrow.Table
Content of the file as a Table of Columns, including DataFrame indexes
as Columns.
"""
if is_string(source):
fs = LocalFilesystem.get_instance()
if fs.isdir(source):
raise NotImplementedError(
'Reading a directory of Parquet files with DataFrame index '
'metadata is not yet supported')
pf = ParquetFile(source, metadata=metadata)
return pf.read_pandas(columns=columns, nthreads=nthreads)
def __init__(self,
path_or_paths,
filesystem=None,
schema=None,
metadata=None,
split_row_groups=False,
validate_schema=True):
if filesystem is None:
self.fs = LocalFilesystem.get_instance()
else:
self.fs = filesystem
self.paths = path_or_paths
(self.pieces, self.partitions,
self.metadata_path) = _make_manifest(path_or_paths, self.fs)
self.metadata = metadata
self.schema = schema
self.split_row_groups = split_row_groups
if split_row_groups:
raise NotImplementedError("split_row_groups not yet implemented")
if validate_schema:
self.validate_schemas()
def __init__(self, dirpath, filesystem=None, pathsep='/',
partition_scheme='hive'):
self.filesystem = filesystem or LocalFilesystem.get_instance()
self.pathsep = pathsep
self.dirpath = dirpath
self.partition_scheme = partition_scheme
self.partitions = ParquetPartitions()
self.pieces = []
self.common_metadata_path = None
self.metadata_path = None
self._visit_level(0, self.dirpath, [])
def __init__(self, dirpath, filesystem=None, pathsep='/',
partition_scheme='hive'):
self.filesystem = filesystem or LocalFilesystem.get_instance()
self.pathsep = pathsep
self.dirpath = dirpath
self.partition_scheme = partition_scheme
self.partitions = ParquetPartitions()
self.pieces = []
self.common_metadata_path = None
self.metadata_path = None
self._visit_level(0, self.dirpath, [])
def __init__(self, path_or_paths, filesystem=None, schema=None,
metadata=None, split_row_groups=False, validate_schema=True):
if filesystem is None:
self.fs = LocalFilesystem.get_instance()
else:
self.fs = filesystem
self.paths = path_or_paths
(self.pieces, self.partitions,
self.metadata_path) = _make_manifest(path_or_paths, self.fs)
self.metadata = metadata
self.schema = schema
self.split_row_groups = split_row_groups
if split_row_groups:
raise NotImplementedError("split_row_groups not yet implemented")
if validate_schema:
self.validate_schemas()
ArrowNotImplementedError, ArrowTypeError)
from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
from pyarrow.io import (HdfsFile, NativeFile, PythonFileInterface, Buffer,
BufferReader, InMemoryOutputStream, MemoryMappedFile,
memory_map, frombuffer, read_tensor, write_tensor,
memory_map, create_memory_map, get_record_batch_size,
get_tensor_size)
from pyarrow.ipc import FileReader, FileWriter, StreamReader, StreamWriter
from pyarrow.memory import MemoryPool, total_allocated_bytes
from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType, BooleanValue,
Int8Value, Int16Value, Int32Value, Int64Value,
UInt8Value, UInt16Value, UInt32Value, UInt64Value,
FloatValue, DoubleValue, ListValue, BinaryValue,
StringValue, FixedSizeBinaryValue)
import pyarrow.schema as _schema
from pyarrow.schema import (null, bool_, int8, int16, int32, int64, uint8,
uint16, uint32, uint64, timestamp, date32, date64,
float16, float32, float64, binary, string, decimal,
list_, struct, dictionary, field, DataType,
FixedSizeBinaryType, Field, Schema, schema)
from pyarrow.table import Column, RecordBatch, Table, concat_tables
localfs = LocalFilesystem.get_instance()
pyarrow.set_memory_pool
"""
from pyarrow._jemalloc import default_pool
return default_pool()
from pyarrow.filesystem import Filesystem, HdfsClient, LocalFilesystem
from pyarrow.ipc import (RecordBatchFileReader, RecordBatchFileWriter,
RecordBatchStreamReader, RecordBatchStreamWriter,
open_stream,
open_file,
serialize_pandas, deserialize_pandas)
localfs = LocalFilesystem.get_instance()
# ----------------------------------------------------------------------
# 0.4.0 deprecations
import warnings
def _deprecate_class(old_name, new_name, klass, next_version='0.5.0'):
msg = ('pyarrow.{0} has been renamed to '
'{1}, will be removed in {2}'
.format(old_name, new_name, next_version))
def deprecated_factory(*args, **kwargs):
warnings.warn(msg, FutureWarning)
return klass(*args)
return deprecated_factory