def load_data(self):
self.source = ogr.Open( self.config['file_name'], update = 0 )
self.layer = self.source.GetLayer(0)
if 'filter' in self.config and self.config['filter'] is not None:
self.layer.SetAttributeFilter( self.config['filter'].encode('ascii') )
self.layer_dfn = self.layer.GetLayerDefn()
self.fields = []
field_count = self.layer_dfn.GetFieldCount()
for field_index in range(field_count):
field = self.layer_dfn.GetFieldDefn( field_index )
self.fields.append({
'name': field.GetName(),
'type': field.GetType(),
'width': field.GetWidth(),
'precision': field.GetPrecision()
})
self.geometries = []
for feature in self.layer:
geometry = feature.GetGeometryRef()
geometry.TransformTo( self.spatialRef )
geometry = shapely.wkb.loads( geometry.ExportToWkb() )
if not geometry.is_valid:
geometry = geometry.buffer(0)
properties = {}
for field in self.fields:
properties[field['name']] = feature.GetFieldAsString(field['name']).decode('utf-8')
self.geometries.append( Geometry(geometry, properties) )
self.layer.ResetReading()
self.create_grammar()
def load_data(self):
self.source = ogr.Open(self.config['file_name'], update=0)
self.layer = self.source.GetLayer(0)
self.layer_dfn = self.layer.GetLayerDefn()
self.fields = []
field_count = self.layer_dfn.GetFieldCount()
for field_index in range(field_count):
field = self.layer_dfn.GetFieldDefn(field_index)
self.fields.append({
'name': field.GetName(),
'type': field.GetType(),
'width': field.GetWidth(),
'precision': field.GetPrecision()
})
self.geometries = []
for feature in self.layer:
geometry = feature.GetGeometryRef()
feature.GetFieldAsString
geometry = shapely.wkb.loads(geometry.ExportToWkb())
properties = {}
for field in self.fields:
properties[field['name']] = feature.GetFieldAsString(
field['name'])
self.geometries.append(Geometry(geometry, properties))
self.layer.ResetReading()
self.create_grammar()
def loadDataSource(self, sourceConfig):
source = ogr.Open( sourceConfig['input_file'] )
layer = source.GetLayer(0)
layer.SetAttributeFilter( sourceConfig['where'].encode('ascii') )
self.viewportRect = False
transformation = osr.CoordinateTransformation( layer.GetSpatialRef(), self.spatialRef )
if self.viewport:
layer.SetSpatialFilterRect( *self.viewport )
point1 = transformation.TransformPoint(self.viewport[0], self.viewport[1])
point2 = transformation.TransformPoint(self.viewport[2], self.viewport[3])
self.viewportRect = shapely.geometry.box(point1[0], point1[1], point2[0], point2[1])
layer.ResetReading()
codes = {}
if self.emulate_longitude0:
meridian = -180 + self.longitude0
p1 = transformation.TransformPoint(-180, 89)
p2 = transformation.TransformPoint(meridian, -89)
left = shapely.geometry.box(p1[0], p1[1], p2[0], p2[1])
p3 = transformation.TransformPoint(meridian, 89)
p4 = transformation.TransformPoint(180, -89)
right = shapely.geometry.box(p3[0], p3[1], p4[0], p4[1])
# load features
nextCode = 0
for feature in layer:
geometry = feature.GetGeometryRef()
geometryType = geometry.GetGeometryType()
if geometryType == ogr.wkbPolygon or geometryType == ogr.wkbMultiPolygon:
geometry.TransformTo( self.spatialRef )
shapelyGeometry = shapely.wkb.loads( geometry.ExportToWkb() )
if not shapelyGeometry.is_valid:
shapelyGeometry = shapelyGeometry.buffer(0, 1)
if self.emulate_longitude0:
leftPart = shapely.affinity.translate(shapelyGeometry.intersection(left), p4[0] - p3[0])
rightPart = shapely.affinity.translate(shapelyGeometry.intersection(right), p1[0] - p2[0])
shapelyGeometry = leftPart.buffer(0.1, 1).union(rightPart.buffer(0.1, 1)).buffer(-0.1, 1)
if not shapelyGeometry.is_valid:
shapelyGeometry = shapelyGeometry.buffer(0, 1)
shapelyGeometry = self.applyFilters(shapelyGeometry)
if shapelyGeometry:
name = feature.GetFieldAsString(str(sourceConfig.get('name_field'))).decode(sourceConfig.get('input_file_encoding'))
code = feature.GetFieldAsString(str(sourceConfig.get('code_field'))).decode(sourceConfig.get('input_file_encoding'))
if code in codes:
code = '_' + str(nextCode)
nextCode += 1
codes[code] = name
self.features[code] = {"geometry": shapelyGeometry, "name": name, "code": code}
else:
raise Exception, "Wrong geometry type: "+geometryType
def loadData(self):
source = ogr.Open( self.inputFile )
layer = source.GetLayer(0)
layer.SetAttributeFilter( self.where )
if self.viewport:
layer.SetSpatialFilterRect( *self.viewport )
transformation = osr.CoordinateTransformation( layer.GetSpatialRef(), self.spatialRef )
point1 = transformation.TransformPoint(self.viewport[0], self.viewport[1])
point2 = transformation.TransformPoint(self.viewport[2], self.viewport[3])
self.viewportRect = shapely.geometry.box(point1[0], point1[1], point2[0], point2[1])
else:
self.viewportRect = False
layer.ResetReading()
# load codes from external tsv file if present or geodata file otherwise
self.codes = {}
if self.codes_file:
for line in codecs.open(self.codes_file, 'r', "utf-8"):
row = map(lambda s: s.strip(), line.split('\t'))
self.codes[row[1]] = row[0]
else:
nextCode = 0
for feature in layer:
code = feature.GetFieldAsString(self.country_code_index)
if code == '-99':
code = '_'+str(nextCode)
nextCode += 1
name = feature.GetFieldAsString(self.country_name_index).decode(self.inputFileEncoding)
self.codes[name] = code
layer.ResetReading()
# load features
for feature in layer:
geometry = feature.GetGeometryRef()
geometryType = geometry.GetGeometryType()
if geometryType == ogr.wkbPolygon or geometryType == ogr.wkbMultiPolygon:
geometry.TransformTo( self.spatialRef )
shapelyGeometry = shapely.wkb.loads( geometry.ExportToWkb() )
if not shapelyGeometry.is_valid:
#buffer to fix selfcrosses
shapelyGeometry = shapelyGeometry.buffer(0, 1)
shapelyGeometry = self.applyFilters(shapelyGeometry)
if shapelyGeometry:
name = feature.GetFieldAsString(self.country_name_index).decode(self.inputFileEncoding)
code = self.codes[name]
self.features[code] = {"geometry": shapelyGeometry, "name": name, "code": code}
else:
raise Exception, "Wrong geomtry type: "+geometryType
in_field_count = in_defn.GetFieldCount()
for fld_index in range(in_field_count):
src_fd = in_defn.GetFieldDefn(fld_index)
fd = ogr.FieldDefn(src_fd.GetName(), src_fd.GetType())
fd.SetWidth(src_fd.GetWidth())
fd.SetPrecision(src_fd.GetPrecision())
shp_layer.CreateField(fd)
# Load geometries
geometries = []
for feature in in_layer:
geometry = feature.GetGeometryRef()
geometryType = geometry.GetGeometryType()
if geometryType == ogr.wkbPolygon or geometryType == ogr.wkbMultiPolygon:
shapelyGeometry = shapely.wkb.loads(geometry.ExportToWkb())
#if not shapelyGeometry.is_valid:
#buffer to fix selfcrosses
#shapelyGeometry = shapelyGeometry.buffer(0)
if shapelyGeometry:
geometries.append(shapelyGeometry)
in_layer.ResetReading()
start = int(round(time.time() * 1000))
# Simplification
points = []
connections = {}
counter = 0
for geom in geometries:
counter += 1
polygons = []
def test_watersheds_trivial(self):
"""PGP watersheds: test trivial delineation."""
flow_dir_array = numpy.array(
[[6, 6, 6, 6, 6, 6, 6, 6, 6, 6], [6, 6, 6, 6, 6, 6, 6, 6, 6, 6],
[6, 6, 6, 6, 6, 6, 6, 6, 6, 6], [6, 6, 6, 6, 6, 6, 6, 6, 6, 255],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 2, 2, 2, 2, 2, 2, 2, 2, 255],
[2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2],
[2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2]],
dtype=numpy.int8)
srs = osr.SpatialReference()
srs.ImportFromEPSG(32731) # WGS84 / UTM zone 31s
srs_wkt = srs.ExportToWkt()
flow_dir_path = os.path.join(self.workspace_dir, 'flow_dir.tif')
pygeoprocessing.numpy_array_to_raster(base_array=flow_dir_array,
target_nodata=255,
pixel_size=(2, -2),
origin=(2, -2),
projection_wkt=srs_wkt,
target_path=flow_dir_path)
# These geometries test:
# * Delineation works with varying geometry types
# * That we exclude seed pixels that are over nodata
# * That we exclude seed pixels off the bounds of the raster
horizontal_line = shapely.geometry.LineString([(19, -11), (25, -11)])
vertical_line = shapely.geometry.LineString([(21, -9), (21, -13)])
square = shapely.geometry.box(17, -13, 21, -9)
point = shapely.geometry.Point(21, -11)
outflow_vector_path = os.path.join(self.workspace_dir, 'outflow.gpkg')
pygeoprocessing.shapely_geometry_to_vector(
[horizontal_line, vertical_line, square, point],
outflow_vector_path,
srs_wkt,
'GPKG', {
'polygon_id': ogr.OFTInteger,
'field_string': ogr.OFTString,
'other': ogr.OFTReal
}, [{
'polygon_id': 1,
'field_string': 'hello world',
'other': 1.111
}, {
'polygon_id': 2,
'field_string': 'hello foo',
'other': 2.222
}, {
'polygon_id': 3,
'field_string': 'hello bar',
'other': 3.333
}, {
'polygon_id': 4,
'field_string': 'hello baz',
'other': 4.444
}],
ogr_geom_type=ogr.wkbUnknown)
target_watersheds_path = os.path.join(self.workspace_dir,
'watersheds.gpkg')
pygeoprocessing.routing.delineate_watersheds_d8(
(flow_dir_path, 1),
outflow_vector_path,
target_watersheds_path,
target_layer_name='watersheds_something')
watersheds_vector = gdal.OpenEx(target_watersheds_path, gdal.OF_VECTOR)
watersheds_layer = watersheds_vector.GetLayer('watersheds_something')
self.assertEqual(watersheds_layer.GetFeatureCount(), 4)
# All features should have the same watersheds, both in area and
# geometry.
flow_dir_bbox = pygeoprocessing.get_raster_info(
flow_dir_path)['bounding_box']
expected_watershed_geometry = shapely.geometry.box(*flow_dir_bbox)
expected_watershed_geometry = expected_watershed_geometry.difference(
shapely.geometry.box(20, -2, 22, -10))
expected_watershed_geometry = expected_watershed_geometry.difference(
shapely.geometry.box(20, -12, 22, -22))
pygeoprocessing.shapely_geometry_to_vector(
[expected_watershed_geometry],
os.path.join(self.workspace_dir, 'foo.gpkg'),
srs_wkt,
'GPKG',
ogr_geom_type=ogr.wkbGeometryCollection)
id_to_fields = {}
for feature in watersheds_layer:
geometry = feature.GetGeometryRef()
shapely_geom = shapely.wkb.loads(geometry.ExportToWkb())
self.assertEqual(shapely_geom.area,
expected_watershed_geometry.area)
self.assertEqual(
shapely_geom.intersection(expected_watershed_geometry).area,
expected_watershed_geometry.area)
self.assertEqual(
shapely_geom.difference(expected_watershed_geometry).area, 0)
field_values = feature.items()
id_to_fields[field_values['polygon_id']] = field_values
outflow_vector = gdal.OpenEx(outflow_vector_path, gdal.OF_VECTOR)
outflow_layer = outflow_vector.GetLayer()
try:
for feature in outflow_layer:
self.assertEqual(id_to_fields[feature.GetField('polygon_id')],
feature.items())
finally:
outflow_layer = None
outflow_vector = None
def test_watersheds_trivial(self):
"""PGP watersheds: test trivial delineation."""
import pygeoprocessing.testing
import pygeoprocessing.routing
flow_dir_array = numpy.array(
[[6, 6, 6, 6, 6, 6, 6, 6, 6, 6], [6, 6, 6, 6, 6, 6, 6, 6, 6, 6],
[6, 6, 6, 6, 6, 6, 6, 6, 6, 6], [6, 6, 6, 6, 6, 6, 6, 6, 6, 255],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 2, 2, 2, 2, 2, 2, 2, 2, 255],
[2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2],
[2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2]],
dtype=numpy.int8)
srs = osr.SpatialReference()
srs.ImportFromEPSG(32731) # WGS84 / UTM zone 31s
srs_wkt = srs.ExportToWkt()
flow_dir_path = os.path.join(self.workspace_dir, 'flow_dir.tif')
driver = gdal.GetDriverByName('GTiff')
flow_dir_raster = driver.Create(
flow_dir_path,
flow_dir_array.shape[1],
flow_dir_array.shape[0],
1,
gdal.GDT_Byte,
options=('TILED=YES', 'BIGTIFF=YES', 'COMPRESS=LZW',
'BLOCKXSIZE=256', 'BLOCKYSIZE=256'))
flow_dir_raster.SetProjection(srs_wkt)
flow_dir_band = flow_dir_raster.GetRasterBand(1)
flow_dir_band.WriteArray(flow_dir_array)
flow_dir_band.SetNoDataValue(255)
flow_dir_geotransform = [2, 2, 0, -2, 0, -2]
flow_dir_raster.SetGeoTransform(flow_dir_geotransform)
flow_dir_raster = None
# These geometries test:
# * Delineation works with varying geometry types
# * That we exclude seed pixels that are over nodata
# * That we exclude seed pixels off the bounds of the raster
horizontal_line = shapely.geometry.LineString([(19, -11), (25, -11)])
vertical_line = shapely.geometry.LineString([(21, -9), (21, -13)])
square = shapely.geometry.box(17, -13, 21, -9)
point = shapely.geometry.Point(21, -11)
outflow_vector_path = os.path.join(self.workspace_dir, 'outflow.gpkg')
pygeoprocessing.testing.create_vector_on_disk(
[horizontal_line, vertical_line, square, point],
projection=srs_wkt,
fields={
'polygon_id': 'int',
'field_string': 'string',
'other': 'real'
},
attributes=[{
'polygon_id': 1,
'field_string': 'hello world',
'other': 1.111
}, {
'polygon_id': 2,
'field_string': 'hello foo',
'other': 2.222
}, {
'polygon_id': 3,
'field_string': 'hello bar',
'other': 3.333
}, {
'polygon_id': 4,
'field_string': 'hello baz',
'other': 4.444
}],
vector_format='GPKG',
filename=outflow_vector_path)
target_watersheds_path = os.path.join(self.workspace_dir,
'watersheds.gpkg')
pygeoprocessing.routing.delineate_watersheds_d8(
(flow_dir_path, 1), outflow_vector_path, target_watersheds_path)
watersheds_vector = gdal.OpenEx(target_watersheds_path, gdal.OF_VECTOR)
watersheds_layer = watersheds_vector.GetLayer('watersheds')
self.assertEqual(watersheds_layer.GetFeatureCount(), 4)
# All features should have the same watersheds, both in area and
# geometry.
flow_dir_bbox = pygeoprocessing.get_raster_info(
flow_dir_path)['bounding_box']
expected_watershed_geometry = shapely.geometry.box(*flow_dir_bbox)
expected_watershed_geometry = expected_watershed_geometry.difference(
shapely.geometry.box(20, -2, 22, -10))
expected_watershed_geometry = expected_watershed_geometry.difference(
shapely.geometry.box(20, -12, 22, -22))
pygeoprocessing.testing.create_vector_on_disk(
[expected_watershed_geometry],
srs_wkt,
vector_format='GPKG',
filename=os.path.join(self.workspace_dir, 'foo.gpkg'))
id_to_fields = {}
for feature in watersheds_layer:
geometry = feature.GetGeometryRef()
shapely_geom = shapely.wkb.loads(geometry.ExportToWkb())
self.assertEqual(shapely_geom.area,
expected_watershed_geometry.area)
self.assertEqual(
shapely_geom.intersection(expected_watershed_geometry).area,
expected_watershed_geometry.area)
self.assertEqual(
shapely_geom.difference(expected_watershed_geometry).area, 0)
field_values = feature.items()
id_to_fields[field_values['polygon_id']] = field_values
outflow_vector = gdal.OpenEx(outflow_vector_path, gdal.OF_VECTOR)
outflow_layer = outflow_vector.GetLayer()
try:
for feature in outflow_layer:
self.assertEqual(id_to_fields[feature.GetField('polygon_id')],
feature.items())
finally:
outflow_layer = None
outflow_vector = None