result = _read_table(fs_protocol_obj,
use_legacy_dataset=use_legacy_dataset)
assert result.equals(table)
# combined with non-local filesystem raises
with pytest.raises(TypeError):
_read_table(fs_protocol_obj, filesystem=FileSystem())
@pytest.mark.dataset
@parametrize_legacy_dataset
@pytest.mark.parametrize(
"filesystem",
[None, fs.LocalFileSystem(),
LocalFileSystem._get_instance()])
def test_relative_paths(tempdir, use_legacy_dataset, filesystem):
# reading and writing from relative paths
table = pa.table({"a": [1, 2, 3]})
# reading
pq.write_table(table, str(tempdir / "data.parquet"))
with util.change_cwd(tempdir):
result = pq.read_table("data.parquet",
filesystem=filesystem,
use_legacy_dataset=use_legacy_dataset)
assert result.equals(table)
# writing
with util.change_cwd(tempdir):
pq.write_table(table, "data2.parquet", filesystem=filesystem)
from pyarrow.serialization import (default_serialization_context,
register_default_serialization_handlers,
register_torch_serialization_handlers)
import pyarrow.types as types
# deprecated top-level access
from pyarrow.filesystem import FileSystem as _FileSystem
from pyarrow.filesystem import LocalFileSystem as _LocalFileSystem
from pyarrow.hdfs import HadoopFileSystem as _HadoopFileSystem
from pyarrow.lib import SerializationContext as _SerializationContext
from pyarrow.lib import SerializedPyObject as _SerializedPyObject
_localfs = _LocalFileSystem._get_instance()
_msg = (
"pyarrow.{0} is deprecated as of 2.0.0, please use pyarrow.fs.{1} instead."
)
_serialization_msg = (
"'pyarrow.{0}' is deprecated and will be removed in a future version. "
"Use pickle or the pyarrow IPC functionality instead.")
_deprecated = {
"localfs": (_localfs, "LocalFileSystem"),
"FileSystem": (_FileSystem, "FileSystem"),
"LocalFileSystem": (_LocalFileSystem, "LocalFileSystem"),
"HadoopFileSystem": (_HadoopFileSystem, "HadoopFileSystem"),
}
fs_protocol_obj = util.FSProtocolClass(path)
result = _read_table(
fs_protocol_obj, use_legacy_dataset=use_legacy_dataset
)
assert result.equals(table)
# combined with non-local filesystem raises
with pytest.raises(TypeError):
_read_table(fs_protocol_obj, filesystem=FileSystem())
@pytest.mark.dataset
@parametrize_legacy_dataset
@pytest.mark.parametrize("filesystem", [
None, fs.LocalFileSystem(), LocalFileSystem._get_instance()
])
def test_relative_paths(tempdir, use_legacy_dataset, filesystem):
# reading and writing from relative paths
table = pa.table({"a": [1, 2, 3]})
# reading
pq.write_table(table, str(tempdir / "data.parquet"))
with util.change_cwd(tempdir):
result = pq.read_table("data.parquet", filesystem=filesystem,
use_legacy_dataset=use_legacy_dataset)
assert result.equals(table)
# writing
with util.change_cwd(tempdir):
pq.write_table(table, "data2.parquet", filesystem=filesystem)