def test_empty_polygon_wkb(self):
p = Polygon(srid=4326)
p_no_srid = Polygon()
wkb_w = WKBWriter()
wkb_w.srid = True
for byteorder, hexes in enumerate([
(b'000000000300000000', b'0020000003000010E600000000'),
(b'010300000000000000', b'0103000020E610000000000000'),
]):
wkb_w.byteorder = byteorder
for srid, hex in enumerate(hexes):
wkb_w.srid = srid
self.assertEqual(wkb_w.write_hex(p), hex)
self.assertEqual(GEOSGeometry(wkb_w.write_hex(p)), p if srid else p_no_srid)
self.assertEqual(wkb_w.write(p), memoryview(binascii.a2b_hex(hex)))
self.assertEqual(GEOSGeometry(wkb_w.write(p)), p if srid else p_no_srid)
def test_empty_polygon_wkb(self):
p = Polygon(srid=4326)
p_no_srid = Polygon()
wkb_w = WKBWriter()
wkb_w.srid = True
for byteorder, hexes in enumerate([
(b'000000000300000000', b'0020000003000010E600000000'),
(b'010300000000000000', b'0103000020E610000000000000'),
]):
wkb_w.byteorder = byteorder
for srid, hex in enumerate(hexes):
wkb_w.srid = srid
self.assertEqual(wkb_w.write_hex(p), hex)
self.assertEqual(GEOSGeometry(wkb_w.write_hex(p)), p if srid else p_no_srid)
self.assertEqual(wkb_w.write(p), memoryview(binascii.a2b_hex(hex)))
self.assertEqual(GEOSGeometry(wkb_w.write(p)), p if srid else p_no_srid)
def test04_wkbwriter(self):
wkb_w = WKBWriter()
# Representations of 'POINT (5 23)' in hex -- one normal and
# the other with the byte order changed.
g = GEOSGeometry("POINT (5 23)")
hex1 = b"010100000000000000000014400000000000003740"
wkb1 = memoryview(binascii.a2b_hex(hex1))
hex2 = b"000000000140140000000000004037000000000000"
wkb2 = memoryview(binascii.a2b_hex(hex2))
self.assertEqual(hex1, wkb_w.write_hex(g))
self.assertEqual(wkb1, wkb_w.write(g))
# Ensuring bad byteorders are not accepted.
for bad_byteorder in (-1, 2, 523, "foo", None):
# Equivalent of `wkb_w.byteorder = bad_byteorder`
with self.assertRaises(ValueError):
wkb_w._set_byteorder(bad_byteorder)
# Setting the byteorder to 0 (for Big Endian)
wkb_w.byteorder = 0
self.assertEqual(hex2, wkb_w.write_hex(g))
self.assertEqual(wkb2, wkb_w.write(g))
# Back to Little Endian
wkb_w.byteorder = 1
# Now, trying out the 3D and SRID flags.
g = GEOSGeometry("POINT (5 23 17)")
g.srid = 4326
hex3d = b"0101000080000000000000144000000000000037400000000000003140"
wkb3d = memoryview(binascii.a2b_hex(hex3d))
hex3d_srid = (
b"01010000A0E6100000000000000000144000000000000037400000000000003140"
)
wkb3d_srid = memoryview(binascii.a2b_hex(hex3d_srid))
# Ensuring bad output dimensions are not accepted
for bad_outdim in (-1, 0, 1, 4, 423, "foo", None):
with self.assertRaisesMessage(
ValueError, "WKB output dimension must be 2 or 3"):
wkb_w.outdim = bad_outdim
# Now setting the output dimensions to be 3
wkb_w.outdim = 3
self.assertEqual(hex3d, wkb_w.write_hex(g))
self.assertEqual(wkb3d, wkb_w.write(g))
# Telling the WKBWriter to include the srid in the representation.
wkb_w.srid = True
self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
self.assertEqual(wkb3d_srid, wkb_w.write(g))
def test_empty_point_wkb(self):
p = Point(srid=4326)
wkb_w = WKBWriter()
wkb_w.srid = False
with self.assertRaisesMessage(ValueError, 'Empty point is not representable in WKB.'):
wkb_w.write(p)
with self.assertRaisesMessage(ValueError, 'Empty point is not representable in WKB.'):
wkb_w.write_hex(p)
wkb_w.srid = True
for byteorder, hex in enumerate([
b'0020000001000010E67FF80000000000007FF8000000000000',
b'0101000020E6100000000000000000F87F000000000000F87F',
]):
wkb_w.byteorder = byteorder
self.assertEqual(wkb_w.write_hex(p), hex)
self.assertEqual(GEOSGeometry(wkb_w.write_hex(p)), p)
self.assertEqual(wkb_w.write(p), memoryview(binascii.a2b_hex(hex)))
self.assertEqual(GEOSGeometry(wkb_w.write(p)), p)
def test_empty_point_wkb(self):
p = Point(srid=4326)
wkb_w = WKBWriter()
wkb_w.srid = False
with self.assertRaisesMessage(ValueError, 'Empty point is not representable in WKB.'):
wkb_w.write(p)
with self.assertRaisesMessage(ValueError, 'Empty point is not representable in WKB.'):
wkb_w.write_hex(p)
wkb_w.srid = True
for byteorder, hex in enumerate([
b'0020000001000010E67FF80000000000007FF8000000000000',
b'0101000020E6100000000000000000F87F000000000000F87F',
]):
wkb_w.byteorder = byteorder
self.assertEqual(wkb_w.write_hex(p), hex)
self.assertEqual(GEOSGeometry(wkb_w.write_hex(p)), p)
self.assertEqual(wkb_w.write(p), memoryview(binascii.a2b_hex(hex)))
self.assertEqual(GEOSGeometry(wkb_w.write(p)), p)
def test04_wkbwriter(self):
wkb_w = WKBWriter()
# Representations of 'POINT (5 23)' in hex -- one normal and
# the other with the byte order changed.
g = GEOSGeometry('POINT (5 23)')
hex1 = b'010100000000000000000014400000000000003740'
wkb1 = memoryview(binascii.a2b_hex(hex1))
hex2 = b'000000000140140000000000004037000000000000'
wkb2 = memoryview(binascii.a2b_hex(hex2))
self.assertEqual(hex1, wkb_w.write_hex(g))
self.assertEqual(wkb1, wkb_w.write(g))
# Ensuring bad byteorders are not accepted.
for bad_byteorder in (-1, 2, 523, 'foo', None):
# Equivalent of `wkb_w.byteorder = bad_byteorder`
self.assertRaises(ValueError, wkb_w._set_byteorder, bad_byteorder)
# Setting the byteorder to 0 (for Big Endian)
wkb_w.byteorder = 0
self.assertEqual(hex2, wkb_w.write_hex(g))
self.assertEqual(wkb2, wkb_w.write(g))
# Back to Little Endian
wkb_w.byteorder = 1
# Now, trying out the 3D and SRID flags.
g = GEOSGeometry('POINT (5 23 17)')
g.srid = 4326
hex3d = b'0101000080000000000000144000000000000037400000000000003140'
wkb3d = memoryview(binascii.a2b_hex(hex3d))
hex3d_srid = b'01010000A0E6100000000000000000144000000000000037400000000000003140'
wkb3d_srid = memoryview(binascii.a2b_hex(hex3d_srid))
# Ensuring bad output dimensions are not accepted
for bad_outdim in (-1, 0, 1, 4, 423, 'foo', None):
# Equivalent of `wkb_w.outdim = bad_outdim`
self.assertRaises(ValueError, wkb_w._set_outdim, bad_outdim)
# These tests will fail on 3.0.0 because of a bug that was fixed in 3.1:
# http://trac.osgeo.org/geos/ticket/216
if not geos_version_info()['version'].startswith('3.0.'):
# Now setting the output dimensions to be 3
wkb_w.outdim = 3
self.assertEqual(hex3d, wkb_w.write_hex(g))
self.assertEqual(wkb3d, wkb_w.write(g))
# Telling the WKBWriter to include the srid in the representation.
wkb_w.srid = True
self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
self.assertEqual(wkb3d_srid, wkb_w.write(g))
def test04_wkbwriter(self):
wkb_w = WKBWriter()
# Representations of 'POINT (5 23)' in hex -- one normal and
# the other with the byte order changed.
g = GEOSGeometry('POINT (5 23)')
hex1 = '010100000000000000000014400000000000003740'
wkb1 = buffer(binascii.a2b_hex(hex1))
hex2 = '000000000140140000000000004037000000000000'
wkb2 = buffer(binascii.a2b_hex(hex2))
self.assertEqual(hex1, wkb_w.write_hex(g))
self.assertEqual(wkb1, wkb_w.write(g))
# Ensuring bad byteorders are not accepted.
for bad_byteorder in (-1, 2, 523, 'foo', None):
# Equivalent of `wkb_w.byteorder = bad_byteorder`
self.assertRaises(ValueError, wkb_w._set_byteorder, bad_byteorder)
# Setting the byteorder to 0 (for Big Endian)
wkb_w.byteorder = 0
self.assertEqual(hex2, wkb_w.write_hex(g))
self.assertEqual(wkb2, wkb_w.write(g))
# Back to Little Endian
wkb_w.byteorder = 1
# Now, trying out the 3D and SRID flags.
g = GEOSGeometry('POINT (5 23 17)')
g.srid = 4326
hex3d = '0101000080000000000000144000000000000037400000000000003140'
wkb3d = buffer(binascii.a2b_hex(hex3d))
hex3d_srid = '01010000A0E6100000000000000000144000000000000037400000000000003140'
wkb3d_srid = buffer(binascii.a2b_hex(hex3d_srid))
# Ensuring bad output dimensions are not accepted
for bad_outdim in (-1, 0, 1, 4, 423, 'foo', None):
# Equivalent of `wkb_w.outdim = bad_outdim`
self.assertRaises(ValueError, wkb_w._set_outdim, bad_outdim)
# These tests will fail on 3.0.0 because of a bug that was fixed in 3.1:
# http://trac.osgeo.org/geos/ticket/216
if not geos_version_info()['version'].startswith('3.0.'):
# Now setting the output dimensions to be 3
wkb_w.outdim = 3
self.assertEqual(hex3d, wkb_w.write_hex(g))
self.assertEqual(wkb3d, wkb_w.write(g))
# Telling the WKBWriter to inlcude the srid in the representation.
wkb_w.srid = True
self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
self.assertEqual(wkb3d_srid, wkb_w.write(g))
def test04_wkbwriter(self):
wkb_w = WKBWriter()
# Representations of 'POINT (5 23)' in hex -- one normal and
# the other with the byte order changed.
g = GEOSGeometry('POINT (5 23)')
hex1 = b'010100000000000000000014400000000000003740'
wkb1 = memoryview(binascii.a2b_hex(hex1))
hex2 = b'000000000140140000000000004037000000000000'
wkb2 = memoryview(binascii.a2b_hex(hex2))
self.assertEqual(hex1, wkb_w.write_hex(g))
self.assertEqual(wkb1, wkb_w.write(g))
# Ensuring bad byteorders are not accepted.
for bad_byteorder in (-1, 2, 523, 'foo', None):
# Equivalent of `wkb_w.byteorder = bad_byteorder`
with self.assertRaises(ValueError):
wkb_w._set_byteorder(bad_byteorder)
# Setting the byteorder to 0 (for Big Endian)
wkb_w.byteorder = 0
self.assertEqual(hex2, wkb_w.write_hex(g))
self.assertEqual(wkb2, wkb_w.write(g))
# Back to Little Endian
wkb_w.byteorder = 1
# Now, trying out the 3D and SRID flags.
g = GEOSGeometry('POINT (5 23 17)')
g.srid = 4326
hex3d = b'0101000080000000000000144000000000000037400000000000003140'
wkb3d = memoryview(binascii.a2b_hex(hex3d))
hex3d_srid = b'01010000A0E6100000000000000000144000000000000037400000000000003140'
wkb3d_srid = memoryview(binascii.a2b_hex(hex3d_srid))
# Ensuring bad output dimensions are not accepted
for bad_outdim in (-1, 0, 1, 4, 423, 'foo', None):
# Equivalent of `wkb_w.outdim = bad_outdim`
with self.assertRaises(ValueError):
wkb_w._set_outdim(bad_outdim)
# Now setting the output dimensions to be 3
wkb_w.outdim = 3
self.assertEqual(hex3d, wkb_w.write_hex(g))
self.assertEqual(wkb3d, wkb_w.write(g))
# Telling the WKBWriter to include the srid in the representation.
wkb_w.srid = True
self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
self.assertEqual(wkb3d_srid, wkb_w.write(g))
dry_run = len(argv) == 2 and argv[1] == "dry-run"
print "dry_run:", dry_run
print "deleting all places",
for table in ["appfd_alias", "appfd_alternatename", "appfd_place"]:
call("sudo -u postgres psql -c 'TRUNCATE " + table + " CASCADE;' dbfd", shell=True)
print "done"
delimiter = "\t"
null = "null"
output_file = open("/tmp/allCountriesCleaned.txt", "wb")
writer = csv.writer(output_file, delimiter=delimiter)
wkb_w = WKBWriter()
wkb_w.srid = True
print "writing output file"
with open("/tmp/allCountries.txt", "r") as f:
counter = 0
places_to_create = []
for line in f:
try:
counter += 1
geonameid, name, asciiname, alternatenames, latitude, longitude, feature_class, feature_code, country_code, cc2, admin1_code, admin2_code, admin3_code, admin4_code, population, elevation, dem, timezone, modification_date = line.split("\t")
if feature_code == "ADM1": admin_level = "1"
elif feature_code == "ADM2": admin_level = "2"
elif feature_code == "ADM3": admin_level = "3"
elif feature_code == "ADM4": admin_level = "4"
elif feature_code == "ADM5": admin_level = "5"
else: admin_level = "null"
point = wkb_w.write_hex(Point(float(longitude), float(latitude), srid=4326))
from subprocess import call, check_output
from sys import exit
start = datetime.now()
print "starting loadGeoNames at ", start
print "deleting all places",
for table in ["appfd_alias", "appfd_alternatename", "appfd_place"]:
call("sudo -u postgres psql -c 'TRUNCATE " + table + " CASCADE;' dbfd", shell=True)
print "done"
output_file = open("/tmp/allCountriesCleaned.txt", "wb")
writer = csv.writer(output_file)
wkb_w = WKBWriter()
wkb_w.srid = True
print "writing output file"
with open("/tmp/allCountries.txt", "r") as f:
counter = 0
places_to_create = []
for line in f:
try:
counter += 1
geonameid, name, asciiname, alternatenames, latitude, longitude, feature_class, feature_code, country_code, cc2, admin1_code, admin2_code, admin3_code, admin4_code, population, elevation, dem, timezone, modification_date = line.split("\t")
point = wkb_w.write_hex(Point(float(longitude), float(latitude), srid=4326))
writer.writerow([ counter, "", admin1_code, admin2_code, "", country_code, "", "", geonameid, "", "", name, "", point, population, "", "", timezone ])
if counter % 1000000 == 0:
print counter, ":", str((datetime.now() - start).total_seconds()), "seconds so far"
except Exception as e:
print e
