def internal_vector_to_memory(
vector: Union[str, ogr.DataSource],
memory_path: Optional[str] = None,
copy_if_already_in_memory: bool = True,
layer_to_extract: int = -1,
) -> str:
"""OBS: Internal. Single output.
Copies a vector source to memory.
"""
type_check(vector, [str, ogr.DataSource], "vector")
type_check(memory_path, [str], "memory_path", allow_none=True)
type_check(layer_to_extract, [int], "layer_to_extract")
ref = open_vector(vector)
path = get_vector_path(ref)
metadata = internal_vector_to_metadata(ref)
name = metadata["name"]
if not copy_if_already_in_memory and metadata["in_memory"]:
if layer_to_extract == -1:
return path
if memory_path is not None:
if memory_path[0:8] == "/vsimem/":
vector_name = memory_path
else:
vector_name = f"/vsimem/{memory_path}"
driver = ogr.GetDriverByName(path_to_driver_vector(memory_path))
else:
vector_name = f"/vsimem/{name}_{uuid4().int}.gpkg"
driver = ogr.GetDriverByName("GPKG")
if driver is None:
raise Exception(f"Error while parsing driver for: {vector}")
copy = driver.CreateDataSource(vector_name)
for layer_idx in range(metadata["layer_count"]):
if layer_to_extract is not None and layer_idx != layer_to_extract:
continue
layername = metadata["layers"][layer_idx]["layer_name"]
copy.CopyLayer(ref.GetLayer(layer_idx), layername, ["OVERWRITE=YES"])
return vector_name
def internal_singlepart_to_multipart(
vector: Union[str, ogr.DataSource],
out_path: Optional[str] = None,
overwrite: bool = True,
add_index: bool = True,
process_layer: int = -1,
) -> str:
type_check(vector, [str, ogr.DataSource], "vector")
type_check(out_path, [str], "out_path", allow_none=True)
type_check(overwrite, [bool], "overwrite")
type_check(add_index, [bool], "add_index")
type_check(process_layer, [int], "process_layer")
vector_list, path_list = ready_io_vector(vector, out_path, overwrite=overwrite)
ref = open_vector(vector_list[0])
out_name = path_list[0]
out_format = path_to_driver_vector(out_name)
driver = ogr.GetDriverByName(out_format)
overwrite_required(out_name, overwrite)
metadata = internal_vector_to_metadata(ref)
remove_if_overwrite(out_name, overwrite)
destination: ogr.DataSource = driver.CreateDataSource(out_name)
for index, layer_meta in enumerate(metadata["layers"]):
if process_layer != -1 and index != process_layer:
continue
name = layer_meta["layer_name"]
geom = layer_meta["column_geom"]
sql = f"SELECT ST_Collect({geom}) AS geom FROM {name};"
result = ref.ExecuteSQL(sql, dialect="SQLITE")
destination.CopyLayer(result, name, ["OVERWRITE=YES"])
if add_index:
vector_add_index(destination)
destination.FlushCache()
return out_name
def internal_vector_to_disk(
vector: Union[str, ogr.DataSource],
out_path: str,
overwrite: bool = True,
) -> str:
"""OBS: Internal. Single output.
Copies a vector source to disk.
"""
type_check(vector, [str, ogr.DataSource], "vector")
type_check(out_path, [str], "out_path")
type_check(overwrite, [bool], "overwrite")
overwrite_required(out_path, overwrite)
datasource = open_vector(vector)
metadata = internal_vector_to_metadata(vector)
if not os.path.dirname(os.path.abspath(out_path)):
raise ValueError(
f"Output folder does not exist. Please create first. {out_path}")
driver = ogr.GetDriverByName(path_to_driver_vector(out_path))
if driver is None:
raise Exception(f"Error while parsing driver for: {vector}")
remove_if_overwrite(out_path, overwrite)
copy = driver.CreateDataSource(out_path)
for layer_idx in range(metadata["layer_count"]):
layer_name = metadata["layers"][layer_idx]["layer_name"]
copy.CopyLayer(datasource.GetLayer(layer_idx), str(layer_name),
["OVERWRITE=YES"])
# Flush to disk
copy = None
return out_path
def merge_vectors(
vectors: List[Union[str, ogr.DataSource]],
out_path: Optional[str] = None,
preserve_fid: bool = True,
) -> str:
"""Merge vectors to a single geopackage."""
type_check(vectors, [list], "vector")
type_check(out_path, [str], "out_path", allow_none=True)
type_check(preserve_fid, [bool], "preserve_fid")
vector_list = to_vector_list(vectors)
out_driver = "GPKG"
out_format = ".gpkg"
out_target = f"/vsimem/clipped_{uuid4().int}{out_format}"
if out_path is not None:
out_target = out_path
out_driver = path_to_driver_vector(out_path)
out_format = path_to_ext(out_path)
driver = ogr.GetDriverByName(out_driver)
merged_ds: ogr.DataSource = driver.CreateDataSource(out_target)
for vector in vector_list:
ref = open_vector(vector)
metadata = internal_vector_to_metadata(ref)
for layer in metadata["layers"]:
name = layer["layer_name"]
merged_ds.CopyLayer(ref.GetLayer(name), name, ["OVERWRITE=YES"])
merged_ds.FlushCache()
return out_target
def internal_intersect_vector(
vector: Union[str, ogr.DataSource],
clip_geom: Union[str, ogr.DataSource, gdal.Dataset],
out_path: Optional[str] = None,
to_extent: bool = False,
process_layer: int = 0,
process_layer_clip: int = 0,
add_index: bool = True,
preserve_fid: bool = True,
overwrite: bool = True,
return_bool: bool = False,
) -> str:
"""Clips a vector to a geometry.
Returns:
A clipped ogr.Datasource or the path to one.
"""
type_check(vector, [ogr.DataSource, str, list], "vector")
type_check(clip_geom, [ogr.DataSource, gdal.Dataset, str, list, tuple], "clip_geom")
type_check(out_path, [str], "out_path", allow_none=True)
type_check(to_extent, [bool], "to_extent")
type_check(process_layer, [int], "process_layer")
type_check(process_layer_clip, [int], "process_layer_clip")
type_check(add_index, [bool], "add_index")
type_check(preserve_fid, [bool], "preserve_fid")
type_check(overwrite, [bool], "overwrite")
_vector_list, path_list = ready_io_vector(vector, out_path, overwrite=overwrite)
out_name = path_list[0]
match_projection = internal_reproject_vector(clip_geom, vector)
geometry_to_clip = open_vector(match_projection)
merged = open_vector(merge_vectors([vector, match_projection]))
if add_index:
vector_add_index(merged)
vector_metadata = internal_vector_to_metadata(vector)
vector_layername = vector_metadata["layers"][process_layer]["layer_name"]
vector_geom_col = vector_metadata["layers"][process_layer]["column_geom"]
clip_geom_metadata = internal_vector_to_metadata(geometry_to_clip)
clip_geom_layername = clip_geom_metadata["layers"][process_layer_clip]["layer_name"]
clip_geom_col = clip_geom_metadata["layers"][process_layer_clip]["column_geom"]
if return_bool:
sql = f"SELECT A.* FROM '{vector_layername}' A, '{clip_geom_layername}' B WHERE ST_INTERSECTS(A.{vector_geom_col}, B.{clip_geom_col});"
else:
sql = f"SELECT A.* FROM '{vector_layername}' A, '{clip_geom_layername}' B WHERE ST_INTERSECTS(A.{vector_geom_col}, B.{clip_geom_col});"
result = merged.ExecuteSQL(sql, dialect="SQLITE")
if return_bool:
if result.GetFeatureCount() == 0:
return False
else:
return True
driver = ogr.GetDriverByName(path_to_driver_vector(out_name))
destination: ogr.DataSource = driver.CreateDataSource(out_name)
destination.CopyLayer(result, vector_layername, ["OVERWRITE=YES"])
if destination is None:
raise Exception("Error while running intersect.")
destination.FlushCache()
return out_name
def internal_reproject_vector(
vector: Union[str, ogr.DataSource],
projection: Union[str, int, ogr.DataSource, gdal.Dataset,
osr.SpatialReference],
out_path: Optional[str] = None,
copy_if_same: bool = False,
overwrite: bool = True,
) -> str:
type_check(vector, [str, ogr.DataSource], "vector")
type_check(
projection,
[str, int, ogr.DataSource, gdal.Dataset, osr.SpatialReference],
"projection",
)
type_check(out_path, [str], "out_path", allow_none=True)
type_check(copy_if_same, [bool], "copy_if_same")
type_check(overwrite, [bool], "overwrite")
vector_list, path_list = ready_io_vector(vector,
out_path,
overwrite=overwrite)
origin = open_vector(vector_list[0])
metadata = internal_vector_to_metadata(origin)
out_name = path_list[0]
origin_projection = metadata["projection_osr"]
target_projection = parse_projection(projection)
if not isinstance(target_projection, osr.SpatialReference):
raise Exception("Error ")
if origin_projection.IsSame(target_projection):
if copy_if_same:
if out_path is None:
return internal_vector_to_memory(origin)
return internal_vector_to_disk(origin, out_name)
else:
return get_vector_path(vector)
# GDAL 3 changes axis order: https://github.com/OSGeo/gdal/issues/1546
if int(osgeo.__version__[0]) >= 3:
origin_projection.SetAxisMappingStrategy(
osr.OAMS_TRADITIONAL_GIS_ORDER)
target_projection.SetAxisMappingStrategy(
osr.OAMS_TRADITIONAL_GIS_ORDER)
coord_trans = osr.CoordinateTransformation(origin_projection,
target_projection)
remove_if_overwrite(out_path, overwrite)
driver = ogr.GetDriverByName(path_to_driver_vector(out_name))
destination: ogr.DataSource = driver.CreateDataSource(out_name)
for layer_idx in range(len(metadata["layers"])):
origin_layer = origin.GetLayerByIndex(layer_idx)
origin_layer_defn = origin_layer.GetLayerDefn()
layer_dict = metadata["layers"][layer_idx]
layer_name = layer_dict["layer_name"]
layer_geom_type = layer_dict["geom_type_ogr"]
destination_layer = destination.CreateLayer(layer_name,
target_projection,
layer_geom_type)
destination_layer_defn = destination_layer.GetLayerDefn()
# Copy field definitions
origin_layer_defn = origin_layer.GetLayerDefn()
for i in range(0, origin_layer_defn.GetFieldCount()):
field_defn = origin_layer_defn.GetFieldDefn(i)
destination_layer.CreateField(field_defn)
# Loop through the input features
for _ in range(origin_layer.GetFeatureCount()):
feature = origin_layer.GetNextFeature()
geom = feature.GetGeometryRef()
geom.Transform(coord_trans)
new_feature = ogr.Feature(destination_layer_defn)
new_feature.SetGeometry(geom)
# Copy field values
for i in range(0, destination_layer_defn.GetFieldCount()):
new_feature.SetField(
destination_layer_defn.GetFieldDefn(i).GetNameRef(),
feature.GetField(i),
)
destination_layer.CreateFeature(new_feature)
destination_layer.ResetReading()
destination_layer = None
destination.FlushCache()
return out_name
def internal_clip_vector(
vector: Union[str, ogr.DataSource],
clip_geom: Union[str, ogr.DataSource, gdal.Dataset],
out_path: Optional[str] = None,
process_layer: int = 0,
process_layer_clip: int = 0,
to_extent: bool = False,
target_projection: Optional[Union[str, ogr.DataSource, gdal.Dataset,
osr.SpatialReference, int]] = None,
preserve_fid: bool = True,
) -> str:
"""Clips a vector to a geometry.
Returns:
A clipped ogr.Datasource or the path to one.
"""
type_check(vector, [str, ogr.DataSource], "vector")
type_check(clip_geom, [ogr.DataSource, gdal.Dataset, str, list, tuple],
"clip_geom")
type_check(out_path, [str], "out_path", allow_none=True)
type_check(process_layer, [int], "process_layer")
type_check(process_layer_clip, [int], "process_layer_clip")
type_check(to_extent, [bool], "to_extent")
type_check(
target_projection,
[str, ogr.DataSource, gdal.Dataset, osr.SpatialReference, int],
"target_projection",
allow_none=True,
)
type_check(preserve_fid, [bool], "preserve_fid")
out_format = ".gpkg"
out_target = f"/vsimem/clipped_{uuid4().int}{out_format}"
if out_path is not None:
out_target = out_path
out_format = path_to_driver_vector(out_path)
options = []
geometry_to_clip = None
if is_vector(clip_geom):
if to_extent:
extent = internal_vector_to_metadata(
clip_geom, create_geometry=True)["extent_datasource"]
geometry_to_clip = internal_vector_to_memory(extent)
else:
geometry_to_clip = open_vector(clip_geom, layer=process_layer_clip)
elif is_raster(clip_geom):
extent = internal_raster_to_metadata(
clip_geom, create_geometry=True)["extent_datasource"]
geometry_to_clip = internal_vector_to_memory(extent)
else:
raise ValueError(
f"Invalid input in clip_geom, unable to parse: {clip_geom}")
clip_vector_path = internal_vector_to_metadata(geometry_to_clip)["path"]
options.append(f"-clipsrc {clip_vector_path}")
if preserve_fid:
options.append("-preserve_fid")
else:
options.append("-unsetFid")
out_projection = None
if target_projection is not None:
out_projection = parse_projection(target_projection, return_wkt=True)
options.append(f"-t_srs {out_projection}")
origin = open_vector(vector, layer=process_layer)
# dst # src
success = gdal.VectorTranslate(
out_target,
get_vector_path(origin),
format=out_format,
options=" ".join(options),
)
if success != 0:
return out_target
else:
raise Exception("Error while clipping geometry.")
def internal_dissolve_vector(
vector: Union[str, ogr.DataSource],
attribute: Optional[str] = None,
out_path: str = None,
overwrite: bool = True,
add_index: bool = True,
process_layer: int = -1,
) -> str:
"""Clips a vector to a geometry."""
type_check(vector, [str, ogr.DataSource], "vector")
type_check(attribute, [str], "attribute", allow_none=True)
type_check(out_path, [str], "out_path", allow_none=True)
type_check(overwrite, [bool], "overwrite")
type_check(add_index, [bool], "add_index")
type_check(process_layer, [int], "process_layer")
vector_list, path_list = ready_io_vector(vector, out_path)
out_name = path_list[0]
out_format = path_to_driver_vector(out_name)
driver = ogr.GetDriverByName(out_format)
ref = open_vector(vector_list[0])
metadata = internal_vector_to_metadata(ref)
Layer_info = TypedDict(
"Layer_info",
{
"name": str,
"geom": str,
"fields": List[str]
},
)
layers: List[Layer_info] = []
if process_layer == -1:
for index in range(len(metadata["layers"])):
layers.append({
"name": metadata["layers"][index]["layer_name"],
"geom": metadata["layers"][index]["column_geom"],
"fields": metadata["layers"][index]["field_names"],
})
else:
layers.append({
"name":
metadata["layers"][process_layer]["layer_name"],
"geom":
metadata["layers"][process_layer]["column_geom"],
"fields":
metadata["layers"][process_layer]["field_names"],
})
destination: ogr.DataSource = driver.CreateDataSource(out_name)
# Check if attribute table is valid
for index in range(len(metadata["layers"])):
layer = layers[index]
if attribute is not None and attribute not in layer["fields"]:
layer_fields = layer["fields"]
raise ValueError(
f"Invalid attribute for layer. Layers has the following fields: {layer_fields}"
)
geom_col = layer["geom"]
name = layer["name"]
sql = None
if attribute is None:
sql = f"SELECT ST_Union({geom_col}) AS geom FROM {name};"
else:
sql = f"SELECT {attribute}, ST_Union({geom_col}) AS geom FROM {name} GROUP BY {attribute};"
result = ref.ExecuteSQL(sql, dialect="SQLITE")
destination.CopyLayer(result, name, ["OVERWRITE=YES"])
if add_index:
vector_add_index(destination)
destination.FlushCache()
return out_name
def internal_multipart_to_singlepart(
vector: Union[str, ogr.DataSource],
out_path: Optional[str] = None,
copy_attributes: bool = False,
overwrite: bool = True,
add_index: bool = True,
process_layer: int = -1,
verbose: int = 1,
) -> str:
type_check(vector, [str, ogr.DataSource], "vector")
type_check(out_path, [str], "out_path", allow_none=True)
type_check(overwrite, [bool], "overwrite")
type_check(add_index, [bool], "add_index")
type_check(process_layer, [int], "process_layer")
type_check(verbose, [int], "verbose")
vector_list, path_list = ready_io_vector(vector, out_path, overwrite=overwrite)
ref = open_vector(vector_list[0])
out_name = path_list[0]
driver = ogr.GetDriverByName(path_to_driver_vector(out_name))
metadata = internal_vector_to_metadata(ref)
remove_if_overwrite(out_name, overwrite)
destination = driver.CreateDataSource(out_name)
for index, layer_meta in enumerate(metadata["layers"]):
if process_layer != -1 and index != process_layer:
continue
if verbose == 1:
layer_name = layer_meta["layer_name"]
print(f"Splitting layer: {layer_name}")
target_unknown = False
if layer_meta["geom_type_ogr"] == 4: # MultiPoint
target_type = 1 # Point
elif layer_meta["geom_type_ogr"] == 5: # MultiLineString
target_type = 2 # LineString
elif layer_meta["geom_type_ogr"] == 6: # MultiPolygon
target_type = 3 # Polygon
elif layer_meta["geom_type_ogr"] == 1004: # MultiPoint (z)
target_type = 1001 # Point (z)
elif layer_meta["geom_type_ogr"] == 1005: # MultiLineString (z)
target_type = 1002 # LineString (z)
elif layer_meta["geom_type_ogr"] == 1006: # MultiPolygon (z)
target_type = 1003 # Polygon (z)
elif layer_meta["geom_type_ogr"] == 2004: # MultiPoint (m)
target_type = 2001 # Point (m)
elif layer_meta["geom_type_ogr"] == 2005: # MultiLineString (m)
target_type = 2002 # LineString (m)
elif layer_meta["geom_type_ogr"] == 2006: # MultiPolygon (m)
target_type = 2003 # Polygon (m)
elif layer_meta["geom_type_ogr"] == 3004: # MultiPoint (zm)
target_type = 3001 # Point (m)
elif layer_meta["geom_type_ogr"] == 3005: # MultiLineString (zm)
target_type = 3002 # LineString (m)
elif layer_meta["geom_type_ogr"] == 3006: # MultiPolygon (zm)
target_type = 3003 # Polygon (m)
else:
target_unknown = True
target_type = layer_meta["geom_type_ogr"]
destination_layer = destination.CreateLayer(
layer_meta["layer_name"], layer_meta["projection_osr"], target_type
)
layer_defn = destination_layer.GetLayerDefn()
field_count = layer_meta["field_count"]
original_target = ref.GetLayerByIndex(index)
feature_count = original_target.GetFeatureCount()
if copy_attributes:
first_feature = original_target.GetNextFeature()
original_target.ResetReading()
if verbose == 1:
print("Creating attribute fields")
for field_id in range(field_count):
field_defn = first_feature.GetFieldDefnRef(field_id)
fname = field_defn.GetName()
ftype = field_defn.GetTypeName()
fwidth = field_defn.GetWidth()
fprecision = field_defn.GetPrecision()
if ftype == "String" or ftype == "Date":
fielddefn = ogr.FieldDefn(fname, ogr.OFTString)
fielddefn.SetWidth(fwidth)
elif ftype == "Real":
fielddefn = ogr.FieldDefn(fname, ogr.OFTReal)
fielddefn.SetWidth(fwidth)
fielddefn.SetPrecision(fprecision)
else:
fielddefn = ogr.FieldDefn(fname, ogr.OFTInteger)
destination_layer.CreateField(fielddefn)
for _ in range(feature_count):
feature = original_target.GetNextFeature()
geom = feature.GetGeometryRef()
if target_unknown:
out_feat = ogr.Feature(layer_defn)
out_feat.SetGeometry(geom)
if copy_attributes:
for field_id in range(field_count):
values = feature.GetField(field_id)
out_feat.SetField(field_id, values)
destination_layer.CreateFeature(out_feat)
for geom_part in geom:
out_feat = ogr.Feature(layer_defn)
out_feat.SetGeometry(geom_part)
if copy_attributes:
for field_id in range(field_count):
values = feature.GetField(field_id)
out_feat.SetField(field_id, values)
destination_layer.CreateFeature(out_feat)
if verbose == 1:
progress(_, feature_count - 1, "Splitting.")
if add_index:
vector_add_index(destination)
return out_name
