def test04_wkbwriter(self):
wkb_w = WKBWriter()
# Representations of 'POINT (5 23)' in hex -- one normal and
# the other with the byte order changed.
g = GEOSGeometry("POINT (5 23)")
hex1 = b"010100000000000000000014400000000000003740"
wkb1 = memoryview(binascii.a2b_hex(hex1))
hex2 = b"000000000140140000000000004037000000000000"
wkb2 = memoryview(binascii.a2b_hex(hex2))
self.assertEqual(hex1, wkb_w.write_hex(g))
self.assertEqual(wkb1, wkb_w.write(g))
# Ensuring bad byteorders are not accepted.
for bad_byteorder in (-1, 2, 523, "foo", None):
# Equivalent of `wkb_w.byteorder = bad_byteorder`
with self.assertRaises(ValueError):
wkb_w._set_byteorder(bad_byteorder)
# Setting the byteorder to 0 (for Big Endian)
wkb_w.byteorder = 0
self.assertEqual(hex2, wkb_w.write_hex(g))
self.assertEqual(wkb2, wkb_w.write(g))
# Back to Little Endian
wkb_w.byteorder = 1
# Now, trying out the 3D and SRID flags.
g = GEOSGeometry("POINT (5 23 17)")
g.srid = 4326
hex3d = b"0101000080000000000000144000000000000037400000000000003140"
wkb3d = memoryview(binascii.a2b_hex(hex3d))
hex3d_srid = (
b"01010000A0E6100000000000000000144000000000000037400000000000003140"
)
wkb3d_srid = memoryview(binascii.a2b_hex(hex3d_srid))
# Ensuring bad output dimensions are not accepted
for bad_outdim in (-1, 0, 1, 4, 423, "foo", None):
with self.assertRaisesMessage(
ValueError, "WKB output dimension must be 2 or 3"):
wkb_w.outdim = bad_outdim
# Now setting the output dimensions to be 3
wkb_w.outdim = 3
self.assertEqual(hex3d, wkb_w.write_hex(g))
self.assertEqual(wkb3d, wkb_w.write(g))
# Telling the WKBWriter to include the srid in the representation.
wkb_w.srid = True
self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
self.assertEqual(wkb3d_srid, wkb_w.write(g))
def test04_wkbwriter(self):
wkb_w = WKBWriter()
# Representations of 'POINT (5 23)' in hex -- one normal and
# the other with the byte order changed.
g = GEOSGeometry('POINT (5 23)')
hex1 = b'010100000000000000000014400000000000003740'
wkb1 = memoryview(binascii.a2b_hex(hex1))
hex2 = b'000000000140140000000000004037000000000000'
wkb2 = memoryview(binascii.a2b_hex(hex2))
self.assertEqual(hex1, wkb_w.write_hex(g))
self.assertEqual(wkb1, wkb_w.write(g))
# Ensuring bad byteorders are not accepted.
for bad_byteorder in (-1, 2, 523, 'foo', None):
# Equivalent of `wkb_w.byteorder = bad_byteorder`
self.assertRaises(ValueError, wkb_w._set_byteorder, bad_byteorder)
# Setting the byteorder to 0 (for Big Endian)
wkb_w.byteorder = 0
self.assertEqual(hex2, wkb_w.write_hex(g))
self.assertEqual(wkb2, wkb_w.write(g))
# Back to Little Endian
wkb_w.byteorder = 1
# Now, trying out the 3D and SRID flags.
g = GEOSGeometry('POINT (5 23 17)')
g.srid = 4326
hex3d = b'0101000080000000000000144000000000000037400000000000003140'
wkb3d = memoryview(binascii.a2b_hex(hex3d))
hex3d_srid = b'01010000A0E6100000000000000000144000000000000037400000000000003140'
wkb3d_srid = memoryview(binascii.a2b_hex(hex3d_srid))
# Ensuring bad output dimensions are not accepted
for bad_outdim in (-1, 0, 1, 4, 423, 'foo', None):
# Equivalent of `wkb_w.outdim = bad_outdim`
self.assertRaises(ValueError, wkb_w._set_outdim, bad_outdim)
# These tests will fail on 3.0.0 because of a bug that was fixed in 3.1:
# http://trac.osgeo.org/geos/ticket/216
if not geos_version_info()['version'].startswith('3.0.'):
# Now setting the output dimensions to be 3
wkb_w.outdim = 3
self.assertEqual(hex3d, wkb_w.write_hex(g))
self.assertEqual(wkb3d, wkb_w.write(g))
# Telling the WKBWriter to include the srid in the representation.
wkb_w.srid = True
self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
self.assertEqual(wkb3d_srid, wkb_w.write(g))
def test04_wkbwriter(self):
wkb_w = WKBWriter()
# Representations of 'POINT (5 23)' in hex -- one normal and
# the other with the byte order changed.
g = GEOSGeometry('POINT (5 23)')
hex1 = '010100000000000000000014400000000000003740'
wkb1 = buffer(binascii.a2b_hex(hex1))
hex2 = '000000000140140000000000004037000000000000'
wkb2 = buffer(binascii.a2b_hex(hex2))
self.assertEqual(hex1, wkb_w.write_hex(g))
self.assertEqual(wkb1, wkb_w.write(g))
# Ensuring bad byteorders are not accepted.
for bad_byteorder in (-1, 2, 523, 'foo', None):
# Equivalent of `wkb_w.byteorder = bad_byteorder`
self.assertRaises(ValueError, wkb_w._set_byteorder, bad_byteorder)
# Setting the byteorder to 0 (for Big Endian)
wkb_w.byteorder = 0
self.assertEqual(hex2, wkb_w.write_hex(g))
self.assertEqual(wkb2, wkb_w.write(g))
# Back to Little Endian
wkb_w.byteorder = 1
# Now, trying out the 3D and SRID flags.
g = GEOSGeometry('POINT (5 23 17)')
g.srid = 4326
hex3d = '0101000080000000000000144000000000000037400000000000003140'
wkb3d = buffer(binascii.a2b_hex(hex3d))
hex3d_srid = '01010000A0E6100000000000000000144000000000000037400000000000003140'
wkb3d_srid = buffer(binascii.a2b_hex(hex3d_srid))
# Ensuring bad output dimensions are not accepted
for bad_outdim in (-1, 0, 1, 4, 423, 'foo', None):
# Equivalent of `wkb_w.outdim = bad_outdim`
self.assertRaises(ValueError, wkb_w._set_outdim, bad_outdim)
# These tests will fail on 3.0.0 because of a bug that was fixed in 3.1:
# http://trac.osgeo.org/geos/ticket/216
if not geos_version_info()['version'].startswith('3.0.'):
# Now setting the output dimensions to be 3
wkb_w.outdim = 3
self.assertEqual(hex3d, wkb_w.write_hex(g))
self.assertEqual(wkb3d, wkb_w.write(g))
# Telling the WKBWriter to inlcude the srid in the representation.
wkb_w.srid = True
self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
self.assertEqual(wkb3d_srid, wkb_w.write(g))
def test04_wkbwriter(self):
wkb_w = WKBWriter()
# Representations of 'POINT (5 23)' in hex -- one normal and
# the other with the byte order changed.
g = GEOSGeometry('POINT (5 23)')
hex1 = b'010100000000000000000014400000000000003740'
wkb1 = memoryview(binascii.a2b_hex(hex1))
hex2 = b'000000000140140000000000004037000000000000'
wkb2 = memoryview(binascii.a2b_hex(hex2))
self.assertEqual(hex1, wkb_w.write_hex(g))
self.assertEqual(wkb1, wkb_w.write(g))
# Ensuring bad byteorders are not accepted.
for bad_byteorder in (-1, 2, 523, 'foo', None):
# Equivalent of `wkb_w.byteorder = bad_byteorder`
with self.assertRaises(ValueError):
wkb_w._set_byteorder(bad_byteorder)
# Setting the byteorder to 0 (for Big Endian)
wkb_w.byteorder = 0
self.assertEqual(hex2, wkb_w.write_hex(g))
self.assertEqual(wkb2, wkb_w.write(g))
# Back to Little Endian
wkb_w.byteorder = 1
# Now, trying out the 3D and SRID flags.
g = GEOSGeometry('POINT (5 23 17)')
g.srid = 4326
hex3d = b'0101000080000000000000144000000000000037400000000000003140'
wkb3d = memoryview(binascii.a2b_hex(hex3d))
hex3d_srid = b'01010000A0E6100000000000000000144000000000000037400000000000003140'
wkb3d_srid = memoryview(binascii.a2b_hex(hex3d_srid))
# Ensuring bad output dimensions are not accepted
for bad_outdim in (-1, 0, 1, 4, 423, 'foo', None):
# Equivalent of `wkb_w.outdim = bad_outdim`
with self.assertRaises(ValueError):
wkb_w._set_outdim(bad_outdim)
# Now setting the output dimensions to be 3
wkb_w.outdim = 3
self.assertEqual(hex3d, wkb_w.write_hex(g))
self.assertEqual(wkb3d, wkb_w.write(g))
# Telling the WKBWriter to include the srid in the representation.
wkb_w.srid = True
self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
self.assertEqual(wkb3d_srid, wkb_w.write(g))
def test_empty_polygon_wkb(self):
p = Polygon(srid=4326)
p_no_srid = Polygon()
wkb_w = WKBWriter()
wkb_w.srid = True
for byteorder, hexes in enumerate([
(b'000000000300000000', b'0020000003000010E600000000'),
(b'010300000000000000', b'0103000020E610000000000000'),
]):
wkb_w.byteorder = byteorder
for srid, hex in enumerate(hexes):
wkb_w.srid = srid
self.assertEqual(wkb_w.write_hex(p), hex)
self.assertEqual(GEOSGeometry(wkb_w.write_hex(p)), p if srid else p_no_srid)
self.assertEqual(wkb_w.write(p), memoryview(binascii.a2b_hex(hex)))
self.assertEqual(GEOSGeometry(wkb_w.write(p)), p if srid else p_no_srid)
def test_empty_polygon_wkb(self):
p = Polygon(srid=4326)
p_no_srid = Polygon()
wkb_w = WKBWriter()
wkb_w.srid = True
for byteorder, hexes in enumerate([
(b'000000000300000000', b'0020000003000010E600000000'),
(b'010300000000000000', b'0103000020E610000000000000'),
]):
wkb_w.byteorder = byteorder
for srid, hex in enumerate(hexes):
wkb_w.srid = srid
self.assertEqual(wkb_w.write_hex(p), hex)
self.assertEqual(GEOSGeometry(wkb_w.write_hex(p)), p if srid else p_no_srid)
self.assertEqual(wkb_w.write(p), memoryview(binascii.a2b_hex(hex)))
self.assertEqual(GEOSGeometry(wkb_w.write(p)), p if srid else p_no_srid)
def test_empty_point_wkb(self):
p = Point(srid=4326)
wkb_w = WKBWriter()
wkb_w.srid = False
with self.assertRaisesMessage(ValueError, 'Empty point is not representable in WKB.'):
wkb_w.write(p)
with self.assertRaisesMessage(ValueError, 'Empty point is not representable in WKB.'):
wkb_w.write_hex(p)
wkb_w.srid = True
for byteorder, hex in enumerate([
b'0020000001000010E67FF80000000000007FF8000000000000',
b'0101000020E6100000000000000000F87F000000000000F87F',
]):
wkb_w.byteorder = byteorder
self.assertEqual(wkb_w.write_hex(p), hex)
self.assertEqual(GEOSGeometry(wkb_w.write_hex(p)), p)
self.assertEqual(wkb_w.write(p), memoryview(binascii.a2b_hex(hex)))
self.assertEqual(GEOSGeometry(wkb_w.write(p)), p)
def test_empty_point_wkb(self):
p = Point(srid=4326)
wkb_w = WKBWriter()
wkb_w.srid = False
with self.assertRaisesMessage(ValueError, 'Empty point is not representable in WKB.'):
wkb_w.write(p)
with self.assertRaisesMessage(ValueError, 'Empty point is not representable in WKB.'):
wkb_w.write_hex(p)
wkb_w.srid = True
for byteorder, hex in enumerate([
b'0020000001000010E67FF80000000000007FF8000000000000',
b'0101000020E6100000000000000000F87F000000000000F87F',
]):
wkb_w.byteorder = byteorder
self.assertEqual(wkb_w.write_hex(p), hex)
self.assertEqual(GEOSGeometry(wkb_w.write_hex(p)), p)
self.assertEqual(wkb_w.write(p), memoryview(binascii.a2b_hex(hex)))
self.assertEqual(GEOSGeometry(wkb_w.write(p)), p)
def run():
# Get data directory from environment
datadir = os.environ.get('CORINE_DATA_DIRECTORY', '')
if not datadir:
print('Datadir not found, please specify CORINE_DATA_DIRECTORY env var.')
return
wkb_w = WKBWriter()
wkb_w.outdim = 2
sources = sorted(glob.glob(os.path.join(datadir, '*.sqlite')))
print('Processing files', sources)
for source in sources:
# Detect file content either landcover or landcover change
change = re.findall(r'^cha([^\_*\.sqlite]+)', os.path.basename(source))
normal = re.findall(r'^clc([^\_*\.sqlite]+)', os.path.basename(source))
if len(normal):
# Select field mapping for landcover files
mapping = const.FIELD_MAPPING
# Get current year from regex match
year = normal[0]
# Set change flag
change = False
elif len(change):
# Select field mapping for change files
mapping = const.CHANGE_FIELD_MAPPING
# Get current year from regex match
year = change[0]
# Get previous year based on base year
previous = const.PREVIOUS_LOOKUP[year]
code_previous_mapping = 'code_' + previous
# Set change flag
change = True
else:
raise ValueError('Could not interpret source.')
# Mapping for the landcover code field source
code_mapping = 'code_' + year
# Convert regex match year to full year
year = const.YEAR_MAPPING[year]
Patch.objects.filter(year=year, change=change).delete()
print('Processing {}data for year {}.'.format('change ' if change else '', year))
# Get full nomenclature from nomenclature app. Convert to dict for speed.
nomenclature = {int(x.code): x.id for x in Nomenclature.objects.all()}
# Open datasource
ds = DataSource(source)
# Get layer from datasource
lyr = ds[0]
# Initiate counter and batch array
counter = 0
batch = []
# Process features in layer
for feat in lyr:
counter += 1
# Create patch instance without commiting
patch = Patch(
year=year,
change=change,
nomenclature_id=nomenclature[int(feat.get(code_mapping))],
)
try:
# Make sure geom is a multi polygon
multi = feat.geom.geos
if multi.geom_type != 'MultiPolygon':
multi = MultiPolygon(multi)
# If necessary, roundtrip through hex writer to drop z dim
if multi.hasz:
multi = GEOSGeometry(wkb_w.write_hex(multi))
patch.geom = multi
except (GDALException, GEOSException):
print(
'ERROR: Could not set geom for feature (objectid {}, id {}, counter {})'
.format(feat['OBJECTID'], feat['ID'], counter)
)
continue
# Set previous landcover for change patches
if change:
patch.nomenclature_previous_id = nomenclature[int(feat.get(code_previous_mapping))]
# Set fields that are common in both types
for k, v in mapping.items():
setattr(patch, k, feat.get(v))
# Apppend this patch to batch array
batch.append(patch)
if counter % 5000 == 0:
# Commit batch to database
Patch.objects.bulk_create(batch)
# Clear batch array
batch = []
# Log progress
now = '[{0}]'.format(datetime.datetime.now().strftime('%Y-%m-%d %T'))
print('{} Processed {} features'.format(now, counter))
# Commit remaining patches to database
if len(batch):
Patch.objects.bulk_create(batch)
wkb_w.srid = True
print "writing output file"
with open("/tmp/allCountries.txt", "r") as f:
counter = 0
places_to_create = []
for line in f:
try:
counter += 1
geonameid, name, asciiname, alternatenames, latitude, longitude, feature_class, feature_code, country_code, cc2, admin1_code, admin2_code, admin3_code, admin4_code, population, elevation, dem, timezone, modification_date = line.split("\t")
if feature_code == "ADM1": admin_level = "1"
elif feature_code == "ADM2": admin_level = "2"
elif feature_code == "ADM3": admin_level = "3"
elif feature_code == "ADM4": admin_level = "4"
elif feature_code == "ADM5": admin_level = "5"
else: admin_level = "null"
point = wkb_w.write_hex(Point(float(longitude), float(latitude), srid=4326))
writer.writerow([ counter, admin_level or null, admin1_code or null, admin2_code or null, null, country_code or null, null, feature_class or null, feature_code or null, null, geonameid or null, null, null, name or null, null, point or null, population or null, null, null, timezone or null, null ])
if counter % 100000 == 0:
print counter, ":", str((datetime.now() - start).total_seconds()), "seconds so far"
if dry_run:
print "it's a dry_run so don't load all places"
break
except Exception as e:
print e
output_file.close()
print "closed output_file"
print "about to execute COPY"
# defaults to using text format with tab separator
call("""sudo -u postgres psql -c "COPY appfd_place FROM '/tmp/allCountriesCleaned.txt' WITH DELIMITER '""" + delimiter + """' NULL '""" + null + """';" dbfd""", shell=True)
print "done"
output_file = open("/tmp/allCountriesCleaned.txt", "wb")
writer = csv.writer(output_file)
wkb_w = WKBWriter()
wkb_w.srid = True
print "writing output file"
with open("/tmp/allCountries.txt", "r") as f:
counter = 0
places_to_create = []
for line in f:
try:
counter += 1
geonameid, name, asciiname, alternatenames, latitude, longitude, feature_class, feature_code, country_code, cc2, admin1_code, admin2_code, admin3_code, admin4_code, population, elevation, dem, timezone, modification_date = line.split("\t")
point = wkb_w.write_hex(Point(float(longitude), float(latitude), srid=4326))
writer.writerow([ counter, "", admin1_code, admin2_code, "", country_code, "", "", geonameid, "", "", name, "", point, population, "", "", timezone ])
if counter % 1000000 == 0:
print counter, ":", str((datetime.now() - start).total_seconds()), "seconds so far"
except Exception as e:
print e
output_file.close()
print "closed output_file"
print "about to execute COPY"
call("""sudo -u postgres psql -c "COPY appfd_place FROM '/tmp/allCountriesCleaned.txt' WITH CSV;" dbfd""", shell=True)
print "executed statement"
total_seconds = (datetime.now() - start).total_seconds()
message = "Loading geonames took " + str(total_seconds) + " seconds."
