result = _read_table(fs_protocol_obj, use_legacy_dataset=use_legacy_dataset) assert result.equals(table) # combined with non-local filesystem raises with pytest.raises(TypeError): _read_table(fs_protocol_obj, filesystem=FileSystem()) @pytest.mark.dataset @parametrize_legacy_dataset @pytest.mark.parametrize( "filesystem", [None, fs.LocalFileSystem(), LocalFileSystem._get_instance()]) def test_relative_paths(tempdir, use_legacy_dataset, filesystem): # reading and writing from relative paths table = pa.table({"a": [1, 2, 3]}) # reading pq.write_table(table, str(tempdir / "data.parquet")) with util.change_cwd(tempdir): result = pq.read_table("data.parquet", filesystem=filesystem, use_legacy_dataset=use_legacy_dataset) assert result.equals(table) # writing with util.change_cwd(tempdir): pq.write_table(table, "data2.parquet", filesystem=filesystem)
from pyarrow.serialization import (default_serialization_context, register_default_serialization_handlers, register_torch_serialization_handlers) import pyarrow.types as types # deprecated top-level access from pyarrow.filesystem import FileSystem as _FileSystem from pyarrow.filesystem import LocalFileSystem as _LocalFileSystem from pyarrow.hdfs import HadoopFileSystem as _HadoopFileSystem from pyarrow.lib import SerializationContext as _SerializationContext from pyarrow.lib import SerializedPyObject as _SerializedPyObject _localfs = _LocalFileSystem._get_instance() _msg = ( "pyarrow.{0} is deprecated as of 2.0.0, please use pyarrow.fs.{1} instead." ) _serialization_msg = ( "'pyarrow.{0}' is deprecated and will be removed in a future version. " "Use pickle or the pyarrow IPC functionality instead.") _deprecated = { "localfs": (_localfs, "LocalFileSystem"), "FileSystem": (_FileSystem, "FileSystem"), "LocalFileSystem": (_LocalFileSystem, "LocalFileSystem"), "HadoopFileSystem": (_HadoopFileSystem, "HadoopFileSystem"), }
fs_protocol_obj = util.FSProtocolClass(path) result = _read_table( fs_protocol_obj, use_legacy_dataset=use_legacy_dataset ) assert result.equals(table) # combined with non-local filesystem raises with pytest.raises(TypeError): _read_table(fs_protocol_obj, filesystem=FileSystem()) @pytest.mark.dataset @parametrize_legacy_dataset @pytest.mark.parametrize("filesystem", [ None, fs.LocalFileSystem(), LocalFileSystem._get_instance() ]) def test_relative_paths(tempdir, use_legacy_dataset, filesystem): # reading and writing from relative paths table = pa.table({"a": [1, 2, 3]}) # reading pq.write_table(table, str(tempdir / "data.parquet")) with util.change_cwd(tempdir): result = pq.read_table("data.parquet", filesystem=filesystem, use_legacy_dataset=use_legacy_dataset) assert result.equals(table) # writing with util.change_cwd(tempdir): pq.write_table(table, "data2.parquet", filesystem=filesystem)