def test04_wkbwriter(self):
wkb_w = WKBWriter()
# Representations of 'POINT (5 23)' in hex -- one normal and
# the other with the byte order changed.
g = GEOSGeometry("POINT (5 23)")
hex1 = b"010100000000000000000014400000000000003740"
wkb1 = memoryview(binascii.a2b_hex(hex1))
hex2 = b"000000000140140000000000004037000000000000"
wkb2 = memoryview(binascii.a2b_hex(hex2))
self.assertEqual(hex1, wkb_w.write_hex(g))
self.assertEqual(wkb1, wkb_w.write(g))
# Ensuring bad byteorders are not accepted.
for bad_byteorder in (-1, 2, 523, "foo", None):
# Equivalent of `wkb_w.byteorder = bad_byteorder`
with self.assertRaises(ValueError):
wkb_w._set_byteorder(bad_byteorder)
# Setting the byteorder to 0 (for Big Endian)
wkb_w.byteorder = 0
self.assertEqual(hex2, wkb_w.write_hex(g))
self.assertEqual(wkb2, wkb_w.write(g))
# Back to Little Endian
wkb_w.byteorder = 1
# Now, trying out the 3D and SRID flags.
g = GEOSGeometry("POINT (5 23 17)")
g.srid = 4326
hex3d = b"0101000080000000000000144000000000000037400000000000003140"
wkb3d = memoryview(binascii.a2b_hex(hex3d))
hex3d_srid = (
b"01010000A0E6100000000000000000144000000000000037400000000000003140"
)
wkb3d_srid = memoryview(binascii.a2b_hex(hex3d_srid))
# Ensuring bad output dimensions are not accepted
for bad_outdim in (-1, 0, 1, 4, 423, "foo", None):
with self.assertRaisesMessage(
ValueError, "WKB output dimension must be 2 or 3"):
wkb_w.outdim = bad_outdim
# Now setting the output dimensions to be 3
wkb_w.outdim = 3
self.assertEqual(hex3d, wkb_w.write_hex(g))
self.assertEqual(wkb3d, wkb_w.write(g))
# Telling the WKBWriter to include the srid in the representation.
wkb_w.srid = True
self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
self.assertEqual(wkb3d_srid, wkb_w.write(g))
def test04_wkbwriter(self):
wkb_w = WKBWriter()
# Representations of 'POINT (5 23)' in hex -- one normal and
# the other with the byte order changed.
g = GEOSGeometry('POINT (5 23)')
hex1 = b'010100000000000000000014400000000000003740'
wkb1 = memoryview(binascii.a2b_hex(hex1))
hex2 = b'000000000140140000000000004037000000000000'
wkb2 = memoryview(binascii.a2b_hex(hex2))
self.assertEqual(hex1, wkb_w.write_hex(g))
self.assertEqual(wkb1, wkb_w.write(g))
# Ensuring bad byteorders are not accepted.
for bad_byteorder in (-1, 2, 523, 'foo', None):
# Equivalent of `wkb_w.byteorder = bad_byteorder`
with self.assertRaises(ValueError):
wkb_w._set_byteorder(bad_byteorder)
# Setting the byteorder to 0 (for Big Endian)
wkb_w.byteorder = 0
self.assertEqual(hex2, wkb_w.write_hex(g))
self.assertEqual(wkb2, wkb_w.write(g))
# Back to Little Endian
wkb_w.byteorder = 1
# Now, trying out the 3D and SRID flags.
g = GEOSGeometry('POINT (5 23 17)')
g.srid = 4326
hex3d = b'0101000080000000000000144000000000000037400000000000003140'
wkb3d = memoryview(binascii.a2b_hex(hex3d))
hex3d_srid = b'01010000A0E6100000000000000000144000000000000037400000000000003140'
wkb3d_srid = memoryview(binascii.a2b_hex(hex3d_srid))
# Ensuring bad output dimensions are not accepted
for bad_outdim in (-1, 0, 1, 4, 423, 'foo', None):
with self.assertRaisesMessage(ValueError, 'WKB output dimension must be 2 or 3'):
wkb_w.outdim = bad_outdim
# Now setting the output dimensions to be 3
wkb_w.outdim = 3
self.assertEqual(hex3d, wkb_w.write_hex(g))
self.assertEqual(wkb3d, wkb_w.write(g))
# Telling the WKBWriter to include the srid in the representation.
wkb_w.srid = True
self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
self.assertEqual(wkb3d_srid, wkb_w.write(g))
def convert_to_2d(geom):
"""Convert a geometry from 3D to 2D"""
from django.contrib.gis.geos import WKBWriter, WKBReader
wkb_r = WKBReader()
wkb_w = WKBWriter()
wkb_w.outdim = 2
return wkb_r.read(wkb_w.write(geom))
def convert_to_2d(geom):
"""Convert a geometry from 3D to 2D"""
from django.contrib.gis.geos import WKBWriter, WKBReader
wkb_r = WKBReader()
wkb_w = WKBWriter()
wkb_w.outdim = 2
return wkb_r.read(wkb_w.write(geom))
def test04_wkbwriter(self):
wkb_w = WKBWriter()
# Representations of 'POINT (5 23)' in hex -- one normal and
# the other with the byte order changed.
g = GEOSGeometry('POINT (5 23)')
hex1 = b'010100000000000000000014400000000000003740'
wkb1 = memoryview(binascii.a2b_hex(hex1))
hex2 = b'000000000140140000000000004037000000000000'
wkb2 = memoryview(binascii.a2b_hex(hex2))
self.assertEqual(hex1, wkb_w.write_hex(g))
self.assertEqual(wkb1, wkb_w.write(g))
# Ensuring bad byteorders are not accepted.
for bad_byteorder in (-1, 2, 523, 'foo', None):
# Equivalent of `wkb_w.byteorder = bad_byteorder`
self.assertRaises(ValueError, wkb_w._set_byteorder, bad_byteorder)
# Setting the byteorder to 0 (for Big Endian)
wkb_w.byteorder = 0
self.assertEqual(hex2, wkb_w.write_hex(g))
self.assertEqual(wkb2, wkb_w.write(g))
# Back to Little Endian
wkb_w.byteorder = 1
# Now, trying out the 3D and SRID flags.
g = GEOSGeometry('POINT (5 23 17)')
g.srid = 4326
hex3d = b'0101000080000000000000144000000000000037400000000000003140'
wkb3d = memoryview(binascii.a2b_hex(hex3d))
hex3d_srid = b'01010000A0E6100000000000000000144000000000000037400000000000003140'
wkb3d_srid = memoryview(binascii.a2b_hex(hex3d_srid))
# Ensuring bad output dimensions are not accepted
for bad_outdim in (-1, 0, 1, 4, 423, 'foo', None):
# Equivalent of `wkb_w.outdim = bad_outdim`
self.assertRaises(ValueError, wkb_w._set_outdim, bad_outdim)
# These tests will fail on 3.0.0 because of a bug that was fixed in 3.1:
# http://trac.osgeo.org/geos/ticket/216
if not geos_version_info()['version'].startswith('3.0.'):
# Now setting the output dimensions to be 3
wkb_w.outdim = 3
self.assertEqual(hex3d, wkb_w.write_hex(g))
self.assertEqual(wkb3d, wkb_w.write(g))
# Telling the WKBWriter to include the srid in the representation.
wkb_w.srid = True
self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
self.assertEqual(wkb3d_srid, wkb_w.write(g))
def test04_wkbwriter(self):
wkb_w = WKBWriter()
# Representations of 'POINT (5 23)' in hex -- one normal and
# the other with the byte order changed.
g = GEOSGeometry('POINT (5 23)')
hex1 = '010100000000000000000014400000000000003740'
wkb1 = buffer(binascii.a2b_hex(hex1))
hex2 = '000000000140140000000000004037000000000000'
wkb2 = buffer(binascii.a2b_hex(hex2))
self.assertEqual(hex1, wkb_w.write_hex(g))
self.assertEqual(wkb1, wkb_w.write(g))
# Ensuring bad byteorders are not accepted.
for bad_byteorder in (-1, 2, 523, 'foo', None):
# Equivalent of `wkb_w.byteorder = bad_byteorder`
self.assertRaises(ValueError, wkb_w._set_byteorder, bad_byteorder)
# Setting the byteorder to 0 (for Big Endian)
wkb_w.byteorder = 0
self.assertEqual(hex2, wkb_w.write_hex(g))
self.assertEqual(wkb2, wkb_w.write(g))
# Back to Little Endian
wkb_w.byteorder = 1
# Now, trying out the 3D and SRID flags.
g = GEOSGeometry('POINT (5 23 17)')
g.srid = 4326
hex3d = '0101000080000000000000144000000000000037400000000000003140'
wkb3d = buffer(binascii.a2b_hex(hex3d))
hex3d_srid = '01010000A0E6100000000000000000144000000000000037400000000000003140'
wkb3d_srid = buffer(binascii.a2b_hex(hex3d_srid))
# Ensuring bad output dimensions are not accepted
for bad_outdim in (-1, 0, 1, 4, 423, 'foo', None):
# Equivalent of `wkb_w.outdim = bad_outdim`
self.assertRaises(ValueError, wkb_w._set_outdim, bad_outdim)
# These tests will fail on 3.0.0 because of a bug that was fixed in 3.1:
# http://trac.osgeo.org/geos/ticket/216
if not geos_version_info()['version'].startswith('3.0.'):
# Now setting the output dimensions to be 3
wkb_w.outdim = 3
self.assertEqual(hex3d, wkb_w.write_hex(g))
self.assertEqual(wkb3d, wkb_w.write(g))
# Telling the WKBWriter to inlcude the srid in the representation.
wkb_w.srid = True
self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
self.assertEqual(wkb3d_srid, wkb_w.write(g))
def _handle_geom(self, geometry):
""" Geometry processing (in place), depending on options """
# Optional force 2D
if self.options.get('force2d'):
wkb_w = WKBWriter()
wkb_w.outdim = 2
geometry = GEOSGeometry(wkb_w.write(geometry), srid=geometry.srid)
# Optional geometry simplification
simplify = self.options.get('simplify')
if simplify is not None:
geometry = geometry.simplify(tolerance=simplify, preserve_topology=True)
# Optional geometry reprojection
if self.srid != geometry.srid:
geometry.transform(self.srid)
return geometry
def _handle_geom(self, geometry):
""" Geometry processing (in place), depending on options """
# Optional force 2D
if self.options.get('force2d'):
wkb_w = WKBWriter()
wkb_w.outdim = 2
geometry = GEOSGeometry(wkb_w.write(geometry), srid=geometry.srid)
# Optional geometry simplification
simplify = self.options.get('simplify')
if simplify is not None:
geometry = geometry.simplify(tolerance=simplify,
preserve_topology=True)
# Optional geometry reprojection
if self.srid != geometry.srid:
geometry.transform(self.srid)
return geometry
def _handle_geom(self, value):
""" Geometry processing (in place), depending on options """
if value is None:
geometry = None
elif isinstance(value, dict) and 'type' in value:
geometry = value
else:
if isinstance(value, GEOSGeometry):
geometry = value
else:
try:
# this will handle string representations (e.g. ewkt, bwkt)
geometry = GEOSGeometry(value)
except ValueError:
# if the geometry couldn't be parsed.
# we can't generate valid geojson
error_msg = 'The field ["%s", "%s"] could not be parsed as a valid geometry' % (
self.geometry_field, value)
raise SerializationError(error_msg)
# Optional force 2D
if self.options.get('force2d'):
wkb_w = WKBWriter()
wkb_w.outdim = 2
geometry = GEOSGeometry(wkb_w.write(geometry),
srid=geometry.srid)
# Optional geometry simplification
simplify = self.options.get('simplify')
if simplify is not None:
geometry = geometry.simplify(tolerance=simplify,
preserve_topology=True)
# Optional geometry reprojection
if geometry.srid and geometry.srid != self.srid:
geometry.transform(self.srid)
# Optional bbox
if self.options.get('bbox_auto'):
self._current['bbox'] = geometry.extent
self._current['geometry'] = geometry
def _handle_geom(self, value):
""" Geometry processing (in place), depending on options """
if value is None:
geometry = None
elif isinstance(value, dict) and 'type' in value:
geometry = value
else:
if isinstance(value, GEOSGeometry):
geometry = value
else:
try:
# this will handle string representations (e.g. ewkt, bwkt)
geometry = GEOSGeometry(value)
except ValueError:
# if the geometry couldn't be parsed.
# we can't generate valid geojson
error_msg = 'The field ["%s", "%s"] could not be parsed as a valid geometry' % (
self.geometry_field, value
)
raise SerializationError(error_msg)
# Optional force 2D
if self.options.get('force2d'):
wkb_w = WKBWriter()
wkb_w.outdim = 2
geometry = GEOSGeometry(wkb_w.write(geometry), srid=geometry.srid)
# Optional geometry simplification
simplify = self.options.get('simplify')
if simplify is not None:
geometry = geometry.simplify(tolerance=simplify, preserve_topology=True)
# Optional geometry reprojection
if geometry.srid and geometry.srid != self.srid:
geometry.transform(self.srid)
# Optional bbox
if self.options.get('bbox_auto'):
self._current['bbox'] = geometry.extent
self._current['geometry'] = geometry
def run():
# Get data directory from environment
datadir = os.environ.get('CORINE_DATA_DIRECTORY', '')
if not datadir:
print('Datadir not found, please specify CORINE_DATA_DIRECTORY env var.')
return
wkb_w = WKBWriter()
wkb_w.outdim = 2
sources = sorted(glob.glob(os.path.join(datadir, '*.sqlite')))
print('Processing files', sources)
for source in sources:
# Detect file content either landcover or landcover change
change = re.findall(r'^cha([^\_*\.sqlite]+)', os.path.basename(source))
normal = re.findall(r'^clc([^\_*\.sqlite]+)', os.path.basename(source))
if len(normal):
# Select field mapping for landcover files
mapping = const.FIELD_MAPPING
# Get current year from regex match
year = normal[0]
# Set change flag
change = False
elif len(change):
# Select field mapping for change files
mapping = const.CHANGE_FIELD_MAPPING
# Get current year from regex match
year = change[0]
# Get previous year based on base year
previous = const.PREVIOUS_LOOKUP[year]
code_previous_mapping = 'code_' + previous
# Set change flag
change = True
else:
raise ValueError('Could not interpret source.')
# Mapping for the landcover code field source
code_mapping = 'code_' + year
# Convert regex match year to full year
year = const.YEAR_MAPPING[year]
Patch.objects.filter(year=year, change=change).delete()
print('Processing {}data for year {}.'.format('change ' if change else '', year))
# Get full nomenclature from nomenclature app. Convert to dict for speed.
nomenclature = {int(x.code): x.id for x in Nomenclature.objects.all()}
# Open datasource
ds = DataSource(source)
# Get layer from datasource
lyr = ds[0]
# Initiate counter and batch array
counter = 0
batch = []
# Process features in layer
for feat in lyr:
counter += 1
# Create patch instance without commiting
patch = Patch(
year=year,
change=change,
nomenclature_id=nomenclature[int(feat.get(code_mapping))],
)
try:
# Make sure geom is a multi polygon
multi = feat.geom.geos
if multi.geom_type != 'MultiPolygon':
multi = MultiPolygon(multi)
# If necessary, roundtrip through hex writer to drop z dim
if multi.hasz:
multi = GEOSGeometry(wkb_w.write_hex(multi))
patch.geom = multi
except (GDALException, GEOSException):
print(
'ERROR: Could not set geom for feature (objectid {}, id {}, counter {})'
.format(feat['OBJECTID'], feat['ID'], counter)
)
continue
# Set previous landcover for change patches
if change:
patch.nomenclature_previous_id = nomenclature[int(feat.get(code_previous_mapping))]
# Set fields that are common in both types
for k, v in mapping.items():
setattr(patch, k, feat.get(v))
# Apppend this patch to batch array
batch.append(patch)
if counter % 5000 == 0:
# Commit batch to database
Patch.objects.bulk_create(batch)
# Clear batch array
batch = []
# Log progress
now = '[{0}]'.format(datetime.datetime.now().strftime('%Y-%m-%d %T'))
print('{} Processed {} features'.format(now, counter))
# Commit remaining patches to database
if len(batch):
Patch.objects.bulk_create(batch)
