def test_read_where(naturalearth_lowres):
# empty filter should return full set of records
geometry, fields = read(naturalearth_lowres, where="")[1:]
assert len(geometry) == 177
assert len(fields) == 5
assert len(fields[0]) == 177
# should return singular item
geometry, fields = read(naturalearth_lowres, where="iso_a3 = 'CAN'")[1:]
assert len(geometry) == 1
assert len(fields) == 5
assert len(fields[0]) == 1
assert fields[3] == "CAN"
# should return items within range
geometry, fields = read(
naturalearth_lowres,
where="POP_EST >= 10000000 AND POP_EST < 100000000")[1:]
assert len(geometry) == 75
assert min(fields[0]) >= 10000000
assert max(fields[0]) < 100000000
# should match no items
with pytest.warns(UserWarning, match="does not have any features to read"):
geometry, fields = read(naturalearth_lowres,
where="iso_a3 = 'INVALID'")[1:]
assert len(geometry) == 0
def test_read_max_features(naturalearth_lowres):
expected_geometry, expected_fields = read(naturalearth_lowres)[1:]
geometry, fields = read(naturalearth_lowres, max_features=2)[1:]
assert len(geometry) == 2
assert len(fields[0]) == 2
assert np.array_equal(geometry, expected_geometry[:2])
assert np.array_equal(fields[-1], expected_fields[-1][:2])
def test_read_skip_features(naturalearth_lowres):
expected_geometry, expected_fields = read(naturalearth_lowres)[1:]
geometry, fields = read(naturalearth_lowres, skip_features=10)[1:]
assert len(geometry) == len(expected_geometry) - 10
assert len(fields[0]) == len(expected_fields[0]) - 10
assert np.array_equal(geometry, expected_geometry[10:])
# Last field has more variable data
assert np.array_equal(fields[-1], expected_fields[-1][10:])
def test_read_bbox(naturalearth_lowres):
# should return no features
with pytest.warns(UserWarning, match="does not have any features to read"):
geometry, fields = read(naturalearth_lowres,
bbox=(0, 0, 0.00001, 0.00001))[1:]
assert len(geometry) == 0
geometry, fields = read(naturalearth_lowres, bbox=(-140, 20, -100, 40))[1:]
assert len(geometry) == 2
assert np.array_equal(fields[3], ["USA", "MEX"])
def test_write_gpkg(tmpdir, naturalearth_lowres):
meta, geometry, field_data = read(naturalearth_lowres)
filename = os.path.join(str(tmpdir), "test.gpkg")
write(filename, geometry, field_data, driver="GPKG", **meta)
assert os.path.exists(filename)
def test_write_geojsonseq(tmpdir, naturalearth_lowres):
meta, geometry, field_data = read(naturalearth_lowres)
filename = os.path.join(str(tmpdir), "test.json")
write(filename, geometry, field_data, driver="GeoJSONSeq", **meta)
assert os.path.exists(filename)
def test_write_unsupported(tmpdir, naturalearth_lowres):
meta, geometry, field_data = read(naturalearth_lowres)
filename = os.path.join(str(tmpdir), "test.fgdb")
with pytest.raises(DriverError,
match="does not support write functionality"):
write(filename, geometry, field_data, driver="OpenFileGDB", **meta)
def test_write_lowres_shp(tmpdir, naturalearth_lowres, benchmark):
meta, geometry, field_data = read(naturalearth_lowres)
filename = os.path.join(str(tmpdir), "test.shp")
benchmark(write,
filename,
geometry,
field_data,
driver="ESRI Shapefile",
**meta)
def test_write(tmpdir, naturalearth_lowres):
meta, geometry, field_data = read(naturalearth_lowres)
filename = os.path.join(str(tmpdir), "test.shp")
write(filename, geometry, field_data, **meta)
assert os.path.exists(filename)
for ext in (".dbf", ".prj"):
assert os.path.exists(filename.replace(".shp", ext))
def test_write_lowres_geojsonseq(tmpdir, naturalearth_lowres, benchmark):
meta, geometry, field_data = read(naturalearth_lowres)
filename = os.path.join(str(tmpdir), "test.json")
benchmark(write,
filename,
geometry,
field_data,
driver="GeoJSONSeq",
**meta)
def test_read_columns(naturalearth_lowres):
# read no columns or geometry
meta, geometry, fields = read(naturalearth_lowres,
columns=[],
read_geometry=False)
assert geometry is None
assert len(fields) == 0
array_equal(meta["fields"], np.empty(shape=(0, 4), dtype="object"))
columns = ["NAME", "NAME_LONG"]
meta, geometry, fields = read(naturalearth_lowres,
columns=columns,
read_geometry=False)
array_equal(meta["fields"], columns)
# Repeats should be dropped
columns = ["NAME", "NAME_LONG", "NAME"]
meta, geometry, fields = read(naturalearth_lowres,
columns=columns,
read_geometry=False)
array_equal(meta["fields"], columns[:2])
def test_write_supported(tmpdir, naturalearth_lowres, driver):
"""Test drivers not specifically tested above"""
meta, geometry, field_data = read(naturalearth_lowres, columns=["iso_a3"])
# note: naturalearth_lowres contains mixed polygons / multipolygons, which
# are not supported in mixed form for all drivers. To get around this here
# we take the first record only.
meta["geometry_type"] = "MultiPolygon"
filename = tmpdir / "test"
write(filename,
geometry[:1],
field_data=[f[:1] for f in field_data],
driver=driver,
**meta)
assert filename.exists()
def test_write_geojson(tmpdir, naturalearth_lowres):
meta, geometry, field_data = read(naturalearth_lowres)
filename = os.path.join(str(tmpdir), "test.json")
write(filename, geometry, field_data, driver="GeoJSON", **meta)
assert os.path.exists(filename)
data = json.loads(open(filename).read())
assert data["type"] == "FeatureCollection"
assert data["name"] == "test"
assert "crs" in data
assert len(data["features"]) == len(geometry)
assert not len(
set(meta["fields"]).difference(
data["features"][0]["properties"].keys()))
def test_read(naturalearth_lowres):
meta, geometry, fields = read(naturalearth_lowres)
assert meta["crs"] == "EPSG:4326"
assert meta["geometry_type"] == "Polygon"
assert meta["encoding"] == "UTF-8"
assert meta["fields"].shape == (5, )
assert meta["fields"].tolist() == [
"pop_est",
"continent",
"name",
"iso_a3",
"gdp_md_est",
]
assert len(fields) == 5
assert len(geometry) == len(fields[0])
# quick test that WKB is a Polygon type
assert geometry[0][:6] == b"\x01\x06\x00\x00\x00\x03"
def test_write_lowres_gpkg(tmpdir, naturalearth_lowres, benchmark):
meta, geometry, field_data = read(naturalearth_lowres)
filename = os.path.join(str(tmpdir), "test.gpkg")
benchmark(write, filename, geometry, field_data, driver="GPKG", **meta)
def test_write_modres_shp(tmpdir, naturalearth_modres, benchmark):
meta, geometry, field_data = read(naturalearth_modres)
filename = os.path.join(str(tmpdir), "test.shp")
benchmark(write, filename, geometry, field_data, **meta)
def read_dataframe(
path,
layer=None,
encoding=None,
columns=None,
read_geometry=True,
force_2d=False,
skip_features=0,
max_features=None,
where=None,
bbox=None,
):
"""Read from an OGR data source to a GeoPandas GeoDataFrame or Pandas DataFrame.
If the data source does not have a geometry column or `read_geometry` is False,
a DataFrame will be returned.
Requires geopandas >= 0.8.
Parameters
----------
path : str
path to file
layer : int or str, optional (default: first layer)
If an integer is provided, it corresponds to the index of the layer
with the data source. If a string is provided, it must match the name
of the layer in the data source. Defaults to first layer in data source.
encoding : str, optional (default: None)
If present, will be used as the encoding for reading string values from
the data source, unless encoding can be inferred directly from the data
source.
columns : list-like, optional (default: all columns)
List of column names to import from the data source. Column names must
exactly match the names in the data source, and will be returned in
the order they occur in the data source. To avoid reading any columns,
pass an empty list-like.
read_geometry : bool, optional (default: True)
If True, will read geometry into a GeoSeries. If False, a Pandas DataFrame
will be returned instead.
force_2d : bool, optional (default: False)
If the geometry has Z values, setting this to True will cause those to
be ignored and 2D geometries to be returned
skip_features : int, optional (default: 0)
Number of features to skip from the beginning of the file before returning
features. Must be less than the total number of features in the file.
max_features : int, optional (default: None)
Number of features to read from the file. Must be less than the total
number of features in the file minus skip_features (if used).
where : str, optional (default: None)
Where clause to filter features in layer by attribute values. Uses a
restricted form of SQL WHERE clause, defined here:
http://ogdi.sourceforge.net/prop/6.2.CapabilitiesMetadata.html
Examples: "ISO_A3 = 'CAN'", "POP_EST > 10000000 AND POP_EST < 100000000"
bbox : tuple of (xmin, ymin, xmax, ymax) (default: None)
If present, will be used to filter records whose geometry intersects this
box. This must be in the same CRS as the dataset.
Returns
-------
GeoDataFrame or DataFrame (if no geometry is present)
"""
try:
with GDALEnv():
import pandas as pd
import geopandas as gp
from geopandas.array import from_wkb
except ImportError:
raise ImportError("geopandas is required to use pyogrio.read_dataframe()")
path = str(path)
if not "://" in path:
if not "/vsizip" in path.lower() and not os.path.exists(path):
raise ValueError(f"'{path}' does not exist")
meta, geometry, field_data = read(
path,
layer=layer,
encoding=encoding,
columns=columns,
read_geometry=read_geometry,
force_2d=force_2d,
skip_features=skip_features,
max_features=max_features,
where=where,
bbox=bbox,
)
columns = meta["fields"].tolist()
data = {columns[i]: field_data[i] for i in range(len(columns))}
df = pd.DataFrame(data, columns=columns)
if geometry is None or not read_geometry:
return df
geometry = from_wkb(geometry, crs=meta["crs"])
return gp.GeoDataFrame(df, geometry=geometry)
def test_read_no_geometry(naturalearth_lowres):
meta, geometry, fields = read(naturalearth_lowres, read_geometry=False)
assert geometry is None
def test_vsi_read_layers(naturalearth_lowres_vsi):
assert array_equal(list_layers(naturalearth_lowres_vsi),
[["naturalearth_lowres", "Polygon"]])
meta, geometry, fields = read(naturalearth_lowres_vsi)
assert geometry.shape == (177, )
def test_read_where_invalid(naturalearth_lowres):
with pytest.raises(ValueError, match="Invalid SQL"):
read(naturalearth_lowres, where="invalid")
def test_read_bbox_invalid(naturalearth_lowres, bbox):
with pytest.raises(ValueError, match="Invalid bbox"):
read(naturalearth_lowres, bbox=bbox)
