def create_index(data):
map_file = os.path.join("..", "appserver", "static", "data", data["file"])
"""
print("Change the line ending of %s" % data["file"])
with open(map_file) as inp, open(map_file+"-tmp", 'w') as out:
txt = inp.read()
txt = txt.replace('\r\n', '\n')
out.write(txt)
os.remove(map_file)
os.rename(map_file+"-tmp", map_file)
"""
rev = ReverseGeocoderShape()
rev.load_map_file("geojson", map_file)
rev.indexstep = data["step"]
print("Creating index for file: %s md5sum: %s" %
(data["file"], rev.map_md5))
sys.stdout.flush()
start = datetime.datetime.now()
rev.load_index("lib")
end = datetime.datetime.now()
delta = end - start
print("Index creation for %s took %s seconds for %s" %
(data["file"], delta.seconds, rev.map_md5))
sys.stdout.flush()
def create_index(data):
map_file = os.path.join("..", "appserver", "static", "data", data["file"])
"""
print("Change the line ending of %s" % data["file"])
with open(map_file) as inp, open(map_file+"-tmp", 'w') as out:
txt = inp.read()
txt = txt.replace('\r\n', '\n')
out.write(txt)
os.remove(map_file)
os.rename(map_file+"-tmp", map_file)
"""
rev = ReverseGeocoderShape()
rev.load_map_file("geojson", map_file)
rev.indexstep = data["step"]
print("Creating index for file: %s md5sum: %s" % (data["file"], rev.map_md5))
sys.stdout.flush()
start = datetime.datetime.now()
rev.load_index("lib")
end = datetime.datetime.now()
delta = end - start
print("Index creation for %s took %s seconds for %s" % (data["file"], delta.seconds, rev.map_md5))
sys.stdout.flush()
def stream(self, records):
basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps",
"heremaps")
rev = ReverseGeocoderShape()
map_file = os.path.join(basepath, "appserver", "static", "data",
self.filename)
rev.load_map_file(self.filetype, map_file)
self.logger.info("Loaded map file %s" % self.filename)
# Load map index file, this speeds up the command a lot
index_file = os.path.join(basepath, "bin", "lib")
rev.load_index(index_file)
self.logger.info("Loaded map index %s" % index_file)
# Load cached results
cache_file = os.path.join(
basepath, "bin", "lib",
"reversegeocodeshape-" + rev.map_md5 + ".cache")
self.cache = FileCache(1000000, 62)
self.cache.read_cache_file(cache_file)
self.logger.info("Loaded cached results %s" % cache_file)
# iterate over records
for record in records:
lat = round(float(record[self.lat]), 5)
lng = round(float(record[self.lng]), 5)
cache_key = "%s,%s" % (lat, lng)
try:
key = self.cache.get(cache_key)
except KeyError:
key = rev.reversegeocode(lat, lng)
self.cache.set(cache_key, key)
record[self.fieldname] = key
yield record
try:
self.cache.write_cache_file(cache_file)
except:
self.logger.error("Could not write cache file")
def stream(self, records):
basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps")
rev = ReverseGeocoderShape()
map_file = os.path.join(basepath, "appserver", "static", "data", self.filename)
rev.load_map_file(self.filetype, map_file)
self.logger.info("Loaded map file %s" % self.filename)
# Load map index file, this speeds up the command a lot
index_file = os.path.join(basepath, "bin", "lib")
rev.load_index(index_file)
self.logger.info("Loaded map index %s" % index_file)
# Load cached results
cache_file = os.path.join(basepath, "bin", "lib", "reversegeocodeshape-" + rev.map_md5 + ".cache")
self.cache = FileCache(1000000, 62)
self.cache.read_cache_file(cache_file)
self.logger.info("Loaded cached results %s" % cache_file)
# iterate over records
for record in records:
lat = round(float(record[self.lat]), 5)
lng = round(float(record[self.lng]), 5)
cache_key = "%s,%s" % (lat, lng)
try:
key = self.cache.get(cache_key)
except KeyError:
key = rev.reversegeocode(lat, lng)
self.cache.set(cache_key, key)
record[self.fieldname] = key
yield record
try:
self.cache.write_cache_file(cache_file)
except:
self.logger.error("Could not write cache file")
class ReverseGeocoderShapeMapTest(unittest.TestCase):
def setUp(self):
self.rev = ReverseGeocoderShape()
self.basepath = "."
self.filetype = "geojson"
def tearDown(self):
self.rev.stop()
del self.rev
def load_index(self):
index_file = os.path.join(
self.basepath, "bin", "lib",
"reversegeocodeshape-" + self.rev.map_md5 + ".index")
self.rev.load_index_file(index_file)
def test_map_custom(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"custom.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "area1")
def test_map_hexagonmap_display_2(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"hexagonmap_display_2.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "3350")
def test_map_hexagonmap_display_3(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"hexagonmap_display_3.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertIsNone(key)
def test_map_hexagonmap_regeo_2(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"hexagonmap_regeo_2.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "3350")
def test_map_hexagonmap_regeo_3(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"hexagonmap_regeo_3.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "1514")
def test_map_squaremap_2(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"squaremap_2.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "12691")
def test_map_squaremap_4(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"squaremap_4.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "3195")
def test_map_world2(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"world2.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "BE")
def test_map_world3(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"world3.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "BEL")
"""
Start testing for the different continents
"""
def test_map_continent_africa(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"continents/africa.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(17, 9)
self.assertEqual(key, "NER")
def test_map_continent_asia(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"continents/asia.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(19, 73)
self.assertEqual(key, "IND")
def test_map_continent_europe(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"continents/europe.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "BEL")
def test_map_continent_north_america(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"continents/north-america.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(45, -75)
self.assertEqual(key, "CAN")
def test_map_continent_oceania(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"continents/oceania.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(-33, 150)
self.assertEqual(key, "AUS")
def test_map_continent_south_america(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"continents/south-america.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(-31, -64)
self.assertEqual(key, "ARG")
"""
Start testing for the different countries
"""
def test_map_country_be(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/be.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "West Flanders")
def test_map_country_br(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/br.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(-8, -63)
self.assertEqual(key, "Amazonas")
def test_map_country_ca(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/ca.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(53, -113)
self.assertEqual(key, "Alberta")
def test_map_country_cn(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/cn.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(34, 108)
self.assertEqual(key, "Shaanxi")
def test_map_country_de(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/de.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(52, 13)
self.assertEqual(key, "Brandenburg")
def test_map_country_es(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/es.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(41, -1)
self.assertEqual(key, "Teruel")
def test_map_country_fr(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/fr.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(49, 0)
self.assertEqual(key, "Calvados")
def test_map_country_in(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/in.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(18, 75)
self.assertEqual(key, "Maharashtra")
def test_map_country_it(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/it.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(42, 12)
self.assertEqual(key, "Roma")
def test_map_country_ru(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/ru.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(55, 37)
self.assertEqual(key, "Kaluga")
def test_map_country_uk(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/uk.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 0)
self.assertEqual(key, "East Sussex")
def test_map_country_us(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/us.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(33, -84)
self.assertEqual(key, "GA")
"""
def test_map_country_us_counties(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/us_counties.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(33, -84)
self.assertEqual(key, "13207")
"""
def test_map_country_us_simplified(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data",
"countries/us_simplified.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(33, -84)
self.assertEqual(key, "GA")
reader = csv.reader(csvfile)
start = datetime.datetime.now()
count = 0
for row in reader:
count += 1
if count == 1:
continue
print(rev.reversegeocodeshape(Point((float(row[0]), float(row[1])))))
if count > csv_max_count:
break
stop = datetime.datetime.now()
delta = stop - start
print("Seconds: %s" % delta.seconds)
print("Without index")
with open("lookups/latlng-data.csv", 'rb') as csvfile:
rev = ReverseGeocoderShape()
rev.load_map_file("geojson", "appserver/static/data/world2.geojson")
performance_test(csvfile, rev)
print("With index")
with open("lookups/latlng-data.csv", 'rb') as csvfile:
rev = ReverseGeocoderShape()
rev.load_map_file("geojson", "appserver/static/data/world2.geojson")
rev.load_index("bin/lib")
performance_test(csvfile, rev)
class ReverseGeocoderShapeMapTest(unittest.TestCase):
def setUp(self):
self.rev = ReverseGeocoderShape()
self.basepath = "."
self.filetype = "geojson"
def tearDown(self):
self.rev.stop()
del self.rev
def load_index(self):
index_file = os.path.join(self.basepath, "bin", "lib", "reversegeocodeshape-" + self.rev.map_md5 + ".index")
self.rev.load_index_file(index_file)
def test_map_custom(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "custom.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "area1")
def test_map_hexagonmap_display_2(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "hexagonmap_display_2.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "3350")
def test_map_hexagonmap_display_3(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "hexagonmap_display_3.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertIsNone(key)
def test_map_hexagonmap_regeo_2(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "hexagonmap_regeo_2.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "3350")
def test_map_hexagonmap_regeo_3(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "hexagonmap_regeo_3.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "1514")
def test_map_squaremap_2(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "squaremap_2.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "12691")
def test_map_squaremap_4(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "squaremap_4.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "3195")
def test_map_world2(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "world2.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "BE")
def test_map_world3(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "world3.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "BEL")
"""
Start testing for the different continents
"""
def test_map_continent_africa(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/africa.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(17, 9)
self.assertEqual(key, "NER")
def test_map_continent_asia(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/asia.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(19, 73)
self.assertEqual(key, "IND")
def test_map_continent_europe(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/europe.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "BEL")
def test_map_continent_north_america(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/north-america.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(45, -75)
self.assertEqual(key, "CAN")
def test_map_continent_oceania(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/oceania.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(-33, 150)
self.assertEqual(key, "AUS")
def test_map_continent_south_america(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/south-america.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(-31, -64)
self.assertEqual(key, "ARG")
"""
Start testing for the different countries
"""
def test_map_country_be(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/be.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 3)
self.assertEqual(key, "West Flanders")
def test_map_country_br(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/br.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(-8, -63)
self.assertEqual(key, "Amazonas")
def test_map_country_ca(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/ca.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(53, -113)
self.assertEqual(key, "Alberta")
def test_map_country_cn(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/cn.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(34, 108)
self.assertEqual(key, "Shaanxi")
def test_map_country_de(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/de.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(52, 13)
self.assertEqual(key, "Brandenburg")
def test_map_country_es(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/es.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(41, -1)
self.assertEqual(key, "Teruel")
def test_map_country_fr(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/fr.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(49, 0)
self.assertEqual(key, "Calvados")
def test_map_country_in(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/in.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(18, 75)
self.assertEqual(key, "Maharashtra")
def test_map_country_it(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/it.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(42, 12)
self.assertEqual(key, "Roma")
def test_map_country_ru(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/ru.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(55, 37)
self.assertEqual(key, "Kaluga")
def test_map_country_uk(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/uk.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(51, 0)
self.assertEqual(key, "East Sussex")
def test_map_country_us(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/us.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(33, -84)
self.assertEqual(key, "GA")
"""
def test_map_country_us_counties(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/us_counties.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(33, -84)
self.assertEqual(key, "13207")
"""
def test_map_country_us_simplified(self):
map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/us_simplified.geojson")
self.rev.load_map_file(self.filetype, map_file)
self.load_index()
key = self.rev.reversegeocode(33, -84)
self.assertEqual(key, "GA")
def performance_test(csvfile, rev):
reader = csv.reader(csvfile)
start = datetime.datetime.now()
count = 0
for row in reader:
count += 1
if count == 1:
continue
print(rev.reversegeocodeshape(Point((float(row[0]), float(row[1])))))
if count > csv_max_count:
break
stop = datetime.datetime.now()
delta = stop - start
print("Seconds: %s" % delta.seconds)
print("Without index")
with open("lookups/latlng-data.csv", 'rb') as csvfile:
rev = ReverseGeocoderShape()
rev.load_map_file("geojson", "appserver/static/data/world2.geojson")
performance_test(csvfile, rev)
print("With index")
with open("lookups/latlng-data.csv", 'rb') as csvfile:
rev = ReverseGeocoderShape()
rev.load_map_file("geojson", "appserver/static/data/world2.geojson")
rev.load_index("bin/lib")
performance_test(csvfile, rev)
