def _get_filesystem_path(path, filesystem=None, storage_options=None): """ Get the filesystem and path for a given filesystem and path. If the filesystem is not None then it's just returned as is. """ import pyarrow if (isinstance(path, str) and storage_options is None and filesystem is None and Version(pyarrow.__version__) >= Version("5.0.0")): # Use the native pyarrow filesystem if possible. try: from pyarrow.fs import FileSystem filesystem, path = FileSystem.from_uri(path) except Exception: # fallback to use get_handle / fsspec for filesystems # that pyarrow doesn't support pass if _is_fsspec_url(path) and filesystem is None: fsspec = import_optional_dependency( "fsspec", extra="fsspec is requred for 'storage_options'.") filesystem, path = fsspec.core.url_to_fs(path, **(storage_options or {})) if filesystem is None and storage_options: raise ValueError( "Cannot provide 'storage_options' with non-fsspec path '{}'". format(path)) return filesystem, path
def _to_parquet(df, path, index=None, compression="snappy", **kwargs): """ Write a GeoDataFrame to the Parquet format. Any geometry columns present are serialized to WKB format in the file. Requires 'pyarrow'. This is an initial implementation of Parquet file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at: https://github.com/geopandas/geo-arrow-spec .. versionadded:: 0.8 Parameters ---------- path : str, path object index : bool, default None If ``True``, always include the dataframe's index(es) as columns in the file output. If ``False``, the index(es) will not be written to the file. If ``None``, the index(ex) will be included as columns in the file output except `RangeIndex` which is stored as metadata only. compression : {'snappy', 'gzip', 'brotli', None}, default 'snappy' Name of the compression to use. Use ``None`` for no compression. kwargs Additional keyword arguments passed to pyarrow.parquet.write_table(). """ parquet = import_optional_dependency( "pyarrow.parquet", extra="pyarrow is required for Parquet support.") path = _expand_user(path) table = _geopandas_to_arrow(df, index=index) parquet.write_table(table, path, compression=compression, **kwargs)
def _read_feather(path, columns=None, **kwargs): """ Load a Feather object from the file path, returning a GeoDataFrame. You can read a subset of columns in the file using the ``columns`` parameter. However, the structure of the returned GeoDataFrame will depend on which columns you read: * if no geometry columns are read, this will raise a ``ValueError`` - you should use the pandas `read_feather` method instead. * if the primary geometry column saved to this file is not included in columns, the first available geometry column will be set as the geometry column of the returned GeoDataFrame. Requires 'pyarrow' >= 0.17. .. versionadded:: 0.8 Parameters ---------- path : str, path object columns : list-like of strings, default=None If not None, only these columns will be read from the file. If the primary geometry column is not included, the first secondary geometry read from the file will be set as the geometry column of the returned GeoDataFrame. If no geometry columns are present, a ``ValueError`` will be raised. **kwargs Any additional kwargs passed to pyarrow.feather.read_table(). Returns ------- GeoDataFrame Examples -------- >>> df = geopandas.read_feather("data.feather") # doctest: +SKIP Specifying columns to read: >>> df = geopandas.read_feather( ... "data.feather", ... columns=["geometry", "pop_est"] ... ) # doctest: +SKIP """ feather = import_optional_dependency( "pyarrow.feather", extra="pyarrow is required for Feather support." ) # TODO move this into `import_optional_dependency` import pyarrow if pyarrow.__version__ < LooseVersion("0.17.0"): raise ImportError("pyarrow >= 0.17 required for Feather support") table = feather.read_table(path, columns=columns, **kwargs) return _arrow_to_geopandas(table)
def test_import_optional_dependency_present(): # pandas is not optional, but we know it is present pandas = import_optional_dependency("pandas") assert pandas is not None # module imported normally must be same import pandas as pd assert pandas == pd
def _to_feather(df, path, index=None, compression=None, **kwargs): """ Write a GeoDataFrame to the Feather format. Any geometry columns present are serialized to WKB format in the file. Requires 'pyarrow' >= 0.17. WARNING: this is an initial implementation of Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at: https://github.com/geopandas/geo-arrow-spec This metadata specification does not yet make stability promises. As such, we do not yet recommend using this in a production setting unless you are able to rewrite your Feather files. .. versionadded:: 0.8 Parameters ---------- path : str, path object index : bool, default None If ``True``, always include the dataframe's index(es) as columns in the file output. If ``False``, the index(es) will not be written to the file. If ``None``, the index(ex) will be included as columns in the file output except `RangeIndex` which is stored as metadata only. compression : {'zstd', 'lz4', 'uncompressed'}, optional Name of the compression to use. Use ``"uncompressed"`` for no compression. By default uses LZ4 if available, otherwise uncompressed. kwargs Additional keyword arguments passed to pyarrow.feather.write_feather(). """ feather = import_optional_dependency( "pyarrow.feather", extra="pyarrow is required for Feather support." ) # TODO move this into `import_optional_dependency` import pyarrow if pyarrow.__version__ < LooseVersion("0.17.0"): raise ImportError("pyarrow >= 0.17 required for Feather support") table = _geopandas_to_arrow(df, index=index) feather.write_feather(table, path, compression=compression, **kwargs)
def _read_parquet(path, columns=None, **kwargs): """ Load a Parquet object from the file path, returning a GeoDataFrame. You can read a subset of columns in the file using the ``columns`` parameter. However, the structure of the returned GeoDataFrame will depend on which columns you read: * if no geometry columns are read, this will raise a ``ValueError`` - you should use the pandas `read_parquet` method instead. * if the primary geometry column saved to this file is not included in columns, the first available geometry column will be set as the geometry column of the returned GeoDataFrame. Requires 'pyarrow'. .. versionadded:: 0.8 Parameters ---------- path : str, path object columns : list-like of strings, default=None If not None, only these columns will be read from the file. If the primary geometry column is not included, the first secondary geometry read from the file will be set as the geometry column of the returned GeoDataFrame. If no geometry columns are present, a ``ValueError`` will be raised. **kwargs Any additional kwargs passed to pyarrow.parquet.read_table(). Returns ------- GeoDataFrame """ parquet = import_optional_dependency( "pyarrow.parquet", extra="pyarrow is required for Parquet support.") kwargs["use_pandas_metadata"] = True table = parquet.read_table(path, columns=columns, **kwargs) return _arrow_to_geopandas(table)
def _read_parquet(path, columns=None, storage_options=None, **kwargs): """ Load a Parquet object from the file path, returning a GeoDataFrame. You can read a subset of columns in the file using the ``columns`` parameter. However, the structure of the returned GeoDataFrame will depend on which columns you read: * if no geometry columns are read, this will raise a ``ValueError`` - you should use the pandas `read_parquet` method instead. * if the primary geometry column saved to this file is not included in columns, the first available geometry column will be set as the geometry column of the returned GeoDataFrame. Requires 'pyarrow'. .. versionadded:: 0.8 Parameters ---------- path : str, path object columns : list-like of strings, default=None If not None, only these columns will be read from the file. If the primary geometry column is not included, the first secondary geometry read from the file will be set as the geometry column of the returned GeoDataFrame. If no geometry columns are present, a ``ValueError`` will be raised. storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc. For HTTP(S) URLs the key-value pairs are forwarded to urllib as header options. For other URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are forwarded to fsspec. Please see fsspec and urllib for more details. When no storage options are provided and a filesystem is implemented by both ``pyarrow.fs`` and ``fsspec`` (e.g. "s3://") then the ``pyarrow.fs`` filesystem is preferred. Provide the instantiated fsspec filesystem using the ``filesystem`` keyword if you wish to use its implementation. **kwargs Any additional kwargs passed to pyarrow.parquet.read_table(). Returns ------- GeoDataFrame Examples -------- >>> df = geopandas.read_parquet("data.parquet") # doctest: +SKIP Specifying columns to read: >>> df = geopandas.read_parquet( ... "data.parquet", ... columns=["geometry", "pop_est"] ... ) # doctest: +SKIP """ parquet = import_optional_dependency( "pyarrow.parquet", extra="pyarrow is required for Parquet support.") # TODO(https://github.com/pandas-dev/pandas/pull/41194): see if pandas # adds filesystem as a keyword and match that. filesystem = kwargs.pop("filesystem", None) filesystem, path = _get_filesystem_path(path, filesystem=filesystem, storage_options=storage_options) path = _expand_user(path) kwargs["use_pandas_metadata"] = True table = parquet.read_table(path, columns=columns, filesystem=filesystem, **kwargs) return _arrow_to_geopandas(table)
def test_import_optional_dependency_invalid(bad_import): with pytest.raises(ValueError, match="Invalid module name"): import_optional_dependency(bad_import)
def test_import_optional_dependency_absent(): with pytest.raises(ImportError, match="Missing optional dependency 'foo'"): import_optional_dependency("foo") with pytest.raises(ImportError, match="foo is required"): import_optional_dependency("foo", extra="foo is required")
def _read_parquet(path, columns=None, **kwargs): """ Load a Parquet object from the file path, returning a GeoDataFrame. You can read a subset of columns in the file using the ``columns`` parameter. However, the structure of the returned GeoDataFrame will depend on which columns you read: * if no geometry columns are read, this will raise a ``ValueError`` - you should use the pandas `read_parquet` method instead. * if the primary geometry column saved to this file is not included in columns, the first available geometry column will be set as the geometry column of the returned GeoDataFrame. Requires 'pyarrow'. .. versionadded:: 0.8 Parameters ---------- path : str, path object columns : list-like of strings, default=None If not None, only these columns will be read from the file. If the primary geometry column is not included, the first secondary geometry read from the file will be set as the geometry column of the returned GeoDataFrame. If no geometry columns are present, a ``ValueError`` will be raised. **kwargs Any additional kwargs passed to pyarrow.parquet.read_table(). Returns ------- GeoDataFrame """ import_optional_dependency( "pyarrow", extra="pyarrow is required for Parquet support.") from pyarrow import parquet kwargs["use_pandas_metadata"] = True table = parquet.read_table(path, columns=columns, **kwargs) df = table.to_pandas() metadata = table.schema.metadata if b"geo" not in metadata: raise ValueError("""Missing geo metadata in Parquet file. Use pandas.read_parquet() instead.""") try: metadata = _decode_metadata(metadata.get(b"geo", b"")) except (TypeError, json.decoder.JSONDecodeError): raise ValueError("Missing or malformed geo metadata in Parquet file") _validate_metadata(metadata) # Find all geometry columns that were read from the file. May # be a subset if 'columns' parameter is used. geometry_columns = df.columns.intersection(metadata["columns"]) if not len(geometry_columns): raise ValueError( """No geometry columns are included in the columns read from the Parquet file. To read this file without geometry columns, use pandas.read_parquet() instead.""") geometry = metadata["primary_column"] # Missing geometry likely indicates a subset of columns was read; # promote the first available geometry to the primary geometry. if len(geometry_columns) and geometry not in geometry_columns: geometry = geometry_columns[0] # if there are multiple non-primary geometry columns, raise a warning if len(geometry_columns) > 1: warnings.warn( "Multiple non-primary geometry columns read from Parquet file. " "The first column read was promoted to the primary geometry.") # Convert the WKB columns that are present back to geometry. for col in geometry_columns: df[col] = from_wkb(df[col].values, crs=metadata["columns"][col]["crs"]) return GeoDataFrame(df, geometry=geometry)
def _to_parquet(df, path, compression="snappy", index=None, **kwargs): """ Write a GeoDataFrame to the Parquet format. Any geometry columns present are serialized to WKB format in the file. Requires 'pyarrow'. WARNING: this is an initial implementation of Parquet file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at: https://github.com/geopandas/geo-arrow-spec This metadata specification does not yet make stability promises. As such, we do not yet recommend using this in a production setting unless you are able to rewrite your Parquet files. .. versionadded:: 0.8 Parameters ---------- path : str, path object compression : {'snappy', 'gzip', 'brotli', None}, default 'snappy' Name of the compression to use. Use ``None`` for no compression. index : bool, default None If ``True``, always include the dataframe's index(es) as columns in the file output. If ``False``, the index(es) will not be written to the file. If ``None``, the index(ex) will be included as columns in the file output except `RangeIndex` which is stored as metadata only. kwargs Additional keyword arguments passed to pyarrow.parquet.write_table(). """ import_optional_dependency( "pyarrow.parquet", extra="pyarrow is required for Parquet support.") from pyarrow import parquet, Table warnings.warn( "this is an initial implementation of Parquet file support and " "associated metadata. This is tracking version 0.1.0 of the metadata " "specification at " "https://github.com/geopandas/geo-arrow-spec\n\n" "This metadata specification does not yet make stability promises. " "We do not yet recommend using this in a production setting unless you " "are able to rewrite your Parquet files.\n\n" "To further ignore this warning, you can do: \n" "import warnings; warnings.filterwarnings('ignore', " "message='.*initial implementation of Parquet.*')", UserWarning, stacklevel=3, ) _validate_dataframe(df) # create geo metadata before altering incoming data frame geo_metadata = _create_metadata(df) df = _encode_wkb(df) table = Table.from_pandas(df, preserve_index=index) # Store geopandas specific file-level metadata # This must be done AFTER creating the table or it is not persisted metadata = table.schema.metadata metadata.update({b"geo": _encode_metadata(geo_metadata)}) table = table.replace_schema_metadata(metadata) parquet.write_table(table, path, compression=compression, **kwargs)