def _get_filesystem_path(path, filesystem=None, storage_options=None):
"""
Get the filesystem and path for a given filesystem and path.
If the filesystem is not None then it's just returned as is.
"""
import pyarrow
if (isinstance(path, str) and storage_options is None
and filesystem is None
and Version(pyarrow.__version__) >= Version("5.0.0")):
# Use the native pyarrow filesystem if possible.
try:
from pyarrow.fs import FileSystem
filesystem, path = FileSystem.from_uri(path)
except Exception:
# fallback to use get_handle / fsspec for filesystems
# that pyarrow doesn't support
pass
if _is_fsspec_url(path) and filesystem is None:
fsspec = import_optional_dependency(
"fsspec", extra="fsspec is requred for 'storage_options'.")
filesystem, path = fsspec.core.url_to_fs(path, **(storage_options
or {}))
if filesystem is None and storage_options:
raise ValueError(
"Cannot provide 'storage_options' with non-fsspec path '{}'".
format(path))
return filesystem, path
def _to_parquet(df, path, index=None, compression="snappy", **kwargs):
"""
Write a GeoDataFrame to the Parquet format.
Any geometry columns present are serialized to WKB format in the file.
Requires 'pyarrow'.
This is an initial implementation of Parquet file support and
associated metadata. This is tracking version 0.1.0 of the metadata
specification at:
https://github.com/geopandas/geo-arrow-spec
.. versionadded:: 0.8
Parameters
----------
path : str, path object
index : bool, default None
If ``True``, always include the dataframe's index(es) as columns
in the file output.
If ``False``, the index(es) will not be written to the file.
If ``None``, the index(ex) will be included as columns in the file
output except `RangeIndex` which is stored as metadata only.
compression : {'snappy', 'gzip', 'brotli', None}, default 'snappy'
Name of the compression to use. Use ``None`` for no compression.
kwargs
Additional keyword arguments passed to pyarrow.parquet.write_table().
"""
parquet = import_optional_dependency(
"pyarrow.parquet", extra="pyarrow is required for Parquet support.")
path = _expand_user(path)
table = _geopandas_to_arrow(df, index=index)
parquet.write_table(table, path, compression=compression, **kwargs)
def _read_feather(path, columns=None, **kwargs):
"""
Load a Feather object from the file path, returning a GeoDataFrame.
You can read a subset of columns in the file using the ``columns`` parameter.
However, the structure of the returned GeoDataFrame will depend on which
columns you read:
* if no geometry columns are read, this will raise a ``ValueError`` - you
should use the pandas `read_feather` method instead.
* if the primary geometry column saved to this file is not included in
columns, the first available geometry column will be set as the geometry
column of the returned GeoDataFrame.
Requires 'pyarrow' >= 0.17.
.. versionadded:: 0.8
Parameters
----------
path : str, path object
columns : list-like of strings, default=None
If not None, only these columns will be read from the file. If
the primary geometry column is not included, the first secondary
geometry read from the file will be set as the geometry column
of the returned GeoDataFrame. If no geometry columns are present,
a ``ValueError`` will be raised.
**kwargs
Any additional kwargs passed to pyarrow.feather.read_table().
Returns
-------
GeoDataFrame
Examples
--------
>>> df = geopandas.read_feather("data.feather") # doctest: +SKIP
Specifying columns to read:
>>> df = geopandas.read_feather(
... "data.feather",
... columns=["geometry", "pop_est"]
... ) # doctest: +SKIP
"""
feather = import_optional_dependency(
"pyarrow.feather", extra="pyarrow is required for Feather support."
)
# TODO move this into `import_optional_dependency`
import pyarrow
if pyarrow.__version__ < LooseVersion("0.17.0"):
raise ImportError("pyarrow >= 0.17 required for Feather support")
table = feather.read_table(path, columns=columns, **kwargs)
return _arrow_to_geopandas(table)
def test_import_optional_dependency_present():
# pandas is not optional, but we know it is present
pandas = import_optional_dependency("pandas")
assert pandas is not None
# module imported normally must be same
import pandas as pd
assert pandas == pd
def _to_feather(df, path, index=None, compression=None, **kwargs):
"""
Write a GeoDataFrame to the Feather format.
Any geometry columns present are serialized to WKB format in the file.
Requires 'pyarrow' >= 0.17.
WARNING: this is an initial implementation of Feather file support and
associated metadata. This is tracking version 0.1.0 of the metadata
specification at:
https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. As such,
we do not yet recommend using this in a production setting unless you are
able to rewrite your Feather files.
.. versionadded:: 0.8
Parameters
----------
path : str, path object
index : bool, default None
If ``True``, always include the dataframe's index(es) as columns
in the file output.
If ``False``, the index(es) will not be written to the file.
If ``None``, the index(ex) will be included as columns in the file
output except `RangeIndex` which is stored as metadata only.
compression : {'zstd', 'lz4', 'uncompressed'}, optional
Name of the compression to use. Use ``"uncompressed"`` for no
compression. By default uses LZ4 if available, otherwise uncompressed.
kwargs
Additional keyword arguments passed to pyarrow.feather.write_feather().
"""
feather = import_optional_dependency(
"pyarrow.feather", extra="pyarrow is required for Feather support."
)
# TODO move this into `import_optional_dependency`
import pyarrow
if pyarrow.__version__ < LooseVersion("0.17.0"):
raise ImportError("pyarrow >= 0.17 required for Feather support")
table = _geopandas_to_arrow(df, index=index)
feather.write_feather(table, path, compression=compression, **kwargs)
def _read_parquet(path, columns=None, **kwargs):
"""
Load a Parquet object from the file path, returning a GeoDataFrame.
You can read a subset of columns in the file using the ``columns`` parameter.
However, the structure of the returned GeoDataFrame will depend on which
columns you read:
* if no geometry columns are read, this will raise a ``ValueError`` - you
should use the pandas `read_parquet` method instead.
* if the primary geometry column saved to this file is not included in
columns, the first available geometry column will be set as the geometry
column of the returned GeoDataFrame.
Requires 'pyarrow'.
.. versionadded:: 0.8
Parameters
----------
path : str, path object
columns : list-like of strings, default=None
If not None, only these columns will be read from the file. If
the primary geometry column is not included, the first secondary
geometry read from the file will be set as the geometry column
of the returned GeoDataFrame. If no geometry columns are present,
a ``ValueError`` will be raised.
**kwargs
Any additional kwargs passed to pyarrow.parquet.read_table().
Returns
-------
GeoDataFrame
"""
parquet = import_optional_dependency(
"pyarrow.parquet", extra="pyarrow is required for Parquet support.")
kwargs["use_pandas_metadata"] = True
table = parquet.read_table(path, columns=columns, **kwargs)
return _arrow_to_geopandas(table)
def _read_parquet(path, columns=None, storage_options=None, **kwargs):
"""
Load a Parquet object from the file path, returning a GeoDataFrame.
You can read a subset of columns in the file using the ``columns`` parameter.
However, the structure of the returned GeoDataFrame will depend on which
columns you read:
* if no geometry columns are read, this will raise a ``ValueError`` - you
should use the pandas `read_parquet` method instead.
* if the primary geometry column saved to this file is not included in
columns, the first available geometry column will be set as the geometry
column of the returned GeoDataFrame.
Requires 'pyarrow'.
.. versionadded:: 0.8
Parameters
----------
path : str, path object
columns : list-like of strings, default=None
If not None, only these columns will be read from the file. If
the primary geometry column is not included, the first secondary
geometry read from the file will be set as the geometry column
of the returned GeoDataFrame. If no geometry columns are present,
a ``ValueError`` will be raised.
storage_options : dict, optional
Extra options that make sense for a particular storage connection, e.g. host,
port, username, password, etc. For HTTP(S) URLs the key-value pairs are
forwarded to urllib as header options. For other URLs (e.g. starting with
"s3://", and "gcs://") the key-value pairs are forwarded to fsspec. Please
see fsspec and urllib for more details.
When no storage options are provided and a filesystem is implemented by
both ``pyarrow.fs`` and ``fsspec`` (e.g. "s3://") then the ``pyarrow.fs``
filesystem is preferred. Provide the instantiated fsspec filesystem using
the ``filesystem`` keyword if you wish to use its implementation.
**kwargs
Any additional kwargs passed to pyarrow.parquet.read_table().
Returns
-------
GeoDataFrame
Examples
--------
>>> df = geopandas.read_parquet("data.parquet") # doctest: +SKIP
Specifying columns to read:
>>> df = geopandas.read_parquet(
... "data.parquet",
... columns=["geometry", "pop_est"]
... ) # doctest: +SKIP
"""
parquet = import_optional_dependency(
"pyarrow.parquet", extra="pyarrow is required for Parquet support.")
# TODO(https://github.com/pandas-dev/pandas/pull/41194): see if pandas
# adds filesystem as a keyword and match that.
filesystem = kwargs.pop("filesystem", None)
filesystem, path = _get_filesystem_path(path,
filesystem=filesystem,
storage_options=storage_options)
path = _expand_user(path)
kwargs["use_pandas_metadata"] = True
table = parquet.read_table(path,
columns=columns,
filesystem=filesystem,
**kwargs)
return _arrow_to_geopandas(table)
def test_import_optional_dependency_invalid(bad_import):
with pytest.raises(ValueError, match="Invalid module name"):
import_optional_dependency(bad_import)
def test_import_optional_dependency_absent():
with pytest.raises(ImportError, match="Missing optional dependency 'foo'"):
import_optional_dependency("foo")
with pytest.raises(ImportError, match="foo is required"):
import_optional_dependency("foo", extra="foo is required")
def _read_parquet(path, columns=None, **kwargs):
"""
Load a Parquet object from the file path, returning a GeoDataFrame.
You can read a subset of columns in the file using the ``columns`` parameter.
However, the structure of the returned GeoDataFrame will depend on which
columns you read:
* if no geometry columns are read, this will raise a ``ValueError`` - you
should use the pandas `read_parquet` method instead.
* if the primary geometry column saved to this file is not included in
columns, the first available geometry column will be set as the geometry
column of the returned GeoDataFrame.
Requires 'pyarrow'.
.. versionadded:: 0.8
Parameters
----------
path : str, path object
columns : list-like of strings, default=None
If not None, only these columns will be read from the file. If
the primary geometry column is not included, the first secondary
geometry read from the file will be set as the geometry column
of the returned GeoDataFrame. If no geometry columns are present,
a ``ValueError`` will be raised.
**kwargs
Any additional kwargs passed to pyarrow.parquet.read_table().
Returns
-------
GeoDataFrame
"""
import_optional_dependency(
"pyarrow", extra="pyarrow is required for Parquet support.")
from pyarrow import parquet
kwargs["use_pandas_metadata"] = True
table = parquet.read_table(path, columns=columns, **kwargs)
df = table.to_pandas()
metadata = table.schema.metadata
if b"geo" not in metadata:
raise ValueError("""Missing geo metadata in Parquet file.
Use pandas.read_parquet() instead.""")
try:
metadata = _decode_metadata(metadata.get(b"geo", b""))
except (TypeError, json.decoder.JSONDecodeError):
raise ValueError("Missing or malformed geo metadata in Parquet file")
_validate_metadata(metadata)
# Find all geometry columns that were read from the file. May
# be a subset if 'columns' parameter is used.
geometry_columns = df.columns.intersection(metadata["columns"])
if not len(geometry_columns):
raise ValueError(
"""No geometry columns are included in the columns read from
the Parquet file. To read this file without geometry columns,
use pandas.read_parquet() instead.""")
geometry = metadata["primary_column"]
# Missing geometry likely indicates a subset of columns was read;
# promote the first available geometry to the primary geometry.
if len(geometry_columns) and geometry not in geometry_columns:
geometry = geometry_columns[0]
# if there are multiple non-primary geometry columns, raise a warning
if len(geometry_columns) > 1:
warnings.warn(
"Multiple non-primary geometry columns read from Parquet file. "
"The first column read was promoted to the primary geometry.")
# Convert the WKB columns that are present back to geometry.
for col in geometry_columns:
df[col] = from_wkb(df[col].values, crs=metadata["columns"][col]["crs"])
return GeoDataFrame(df, geometry=geometry)
def _to_parquet(df, path, compression="snappy", index=None, **kwargs):
"""
Write a GeoDataFrame to the Parquet format.
Any geometry columns present are serialized to WKB format in the file.
Requires 'pyarrow'.
WARNING: this is an initial implementation of Parquet file support and
associated metadata. This is tracking version 0.1.0 of the metadata
specification at:
https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. As such,
we do not yet recommend using this in a production setting unless you are
able to rewrite your Parquet files.
.. versionadded:: 0.8
Parameters
----------
path : str, path object
compression : {'snappy', 'gzip', 'brotli', None}, default 'snappy'
Name of the compression to use. Use ``None`` for no compression.
index : bool, default None
If ``True``, always include the dataframe's index(es) as columns
in the file output.
If ``False``, the index(es) will not be written to the file.
If ``None``, the index(ex) will be included as columns in the file
output except `RangeIndex` which is stored as metadata only.
kwargs
Additional keyword arguments passed to pyarrow.parquet.write_table().
"""
import_optional_dependency(
"pyarrow.parquet", extra="pyarrow is required for Parquet support.")
from pyarrow import parquet, Table
warnings.warn(
"this is an initial implementation of Parquet file support and "
"associated metadata. This is tracking version 0.1.0 of the metadata "
"specification at "
"https://github.com/geopandas/geo-arrow-spec\n\n"
"This metadata specification does not yet make stability promises. "
"We do not yet recommend using this in a production setting unless you "
"are able to rewrite your Parquet files.\n\n"
"To further ignore this warning, you can do: \n"
"import warnings; warnings.filterwarnings('ignore', "
"message='.*initial implementation of Parquet.*')",
UserWarning,
stacklevel=3,
)
_validate_dataframe(df)
# create geo metadata before altering incoming data frame
geo_metadata = _create_metadata(df)
df = _encode_wkb(df)
table = Table.from_pandas(df, preserve_index=index)
# Store geopandas specific file-level metadata
# This must be done AFTER creating the table or it is not persisted
metadata = table.schema.metadata
metadata.update({b"geo": _encode_metadata(geo_metadata)})
table = table.replace_schema_metadata(metadata)
parquet.write_table(table, path, compression=compression, **kwargs)
