def create_index(data): map_file = os.path.join("..", "appserver", "static", "data", data["file"]) """ print("Change the line ending of %s" % data["file"]) with open(map_file) as inp, open(map_file+"-tmp", 'w') as out: txt = inp.read() txt = txt.replace('\r\n', '\n') out.write(txt) os.remove(map_file) os.rename(map_file+"-tmp", map_file) """ rev = ReverseGeocoderShape() rev.load_map_file("geojson", map_file) rev.indexstep = data["step"] print("Creating index for file: %s md5sum: %s" % (data["file"], rev.map_md5)) sys.stdout.flush() start = datetime.datetime.now() rev.load_index("lib") end = datetime.datetime.now() delta = end - start print("Index creation for %s took %s seconds for %s" % (data["file"], delta.seconds, rev.map_md5)) sys.stdout.flush()
def stream(self, records): basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps") rev = ReverseGeocoderShape() map_file = os.path.join(basepath, "appserver", "static", "data", self.filename) rev.load_map_file(self.filetype, map_file) self.logger.info("Loaded map file %s" % self.filename) # Load map index file, this speeds up the command a lot index_file = os.path.join(basepath, "bin", "lib") rev.load_index(index_file) self.logger.info("Loaded map index %s" % index_file) # Load cached results cache_file = os.path.join( basepath, "bin", "lib", "reversegeocodeshape-" + rev.map_md5 + ".cache") self.cache = FileCache(1000000, 62) self.cache.read_cache_file(cache_file) self.logger.info("Loaded cached results %s" % cache_file) # iterate over records for record in records: lat = round(float(record[self.lat]), 5) lng = round(float(record[self.lng]), 5) cache_key = "%s,%s" % (lat, lng) try: key = self.cache.get(cache_key) except KeyError: key = rev.reversegeocode(lat, lng) self.cache.set(cache_key, key) record[self.fieldname] = key yield record try: self.cache.write_cache_file(cache_file) except: self.logger.error("Could not write cache file")
def stream(self, records): basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps") rev = ReverseGeocoderShape() map_file = os.path.join(basepath, "appserver", "static", "data", self.filename) rev.load_map_file(self.filetype, map_file) self.logger.info("Loaded map file %s" % self.filename) # Load map index file, this speeds up the command a lot index_file = os.path.join(basepath, "bin", "lib") rev.load_index(index_file) self.logger.info("Loaded map index %s" % index_file) # Load cached results cache_file = os.path.join(basepath, "bin", "lib", "reversegeocodeshape-" + rev.map_md5 + ".cache") self.cache = FileCache(1000000, 62) self.cache.read_cache_file(cache_file) self.logger.info("Loaded cached results %s" % cache_file) # iterate over records for record in records: lat = round(float(record[self.lat]), 5) lng = round(float(record[self.lng]), 5) cache_key = "%s,%s" % (lat, lng) try: key = self.cache.get(cache_key) except KeyError: key = rev.reversegeocode(lat, lng) self.cache.set(cache_key, key) record[self.fieldname] = key yield record try: self.cache.write_cache_file(cache_file) except: self.logger.error("Could not write cache file")
class ReverseGeocoderShapeMapTest(unittest.TestCase): def setUp(self): self.rev = ReverseGeocoderShape() self.basepath = "." self.filetype = "geojson" def tearDown(self): self.rev.stop() del self.rev def load_index(self): index_file = os.path.join( self.basepath, "bin", "lib", "reversegeocodeshape-" + self.rev.map_md5 + ".index") self.rev.load_index_file(index_file) def test_map_custom(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "custom.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "area1") def test_map_hexagonmap_display_2(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "hexagonmap_display_2.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "3350") def test_map_hexagonmap_display_3(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "hexagonmap_display_3.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertIsNone(key) def test_map_hexagonmap_regeo_2(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "hexagonmap_regeo_2.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "3350") def test_map_hexagonmap_regeo_3(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "hexagonmap_regeo_3.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "1514") def test_map_squaremap_2(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "squaremap_2.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "12691") def test_map_squaremap_4(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "squaremap_4.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "3195") def test_map_world2(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "world2.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "BE") def test_map_world3(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "world3.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "BEL") """ Start testing for the different continents """ def test_map_continent_africa(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/africa.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(17, 9) self.assertEqual(key, "NER") def test_map_continent_asia(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/asia.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(19, 73) self.assertEqual(key, "IND") def test_map_continent_europe(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/europe.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "BEL") def test_map_continent_north_america(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/north-america.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(45, -75) self.assertEqual(key, "CAN") def test_map_continent_oceania(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/oceania.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(-33, 150) self.assertEqual(key, "AUS") def test_map_continent_south_america(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/south-america.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(-31, -64) self.assertEqual(key, "ARG") """ Start testing for the different countries """ def test_map_country_be(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/be.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "West Flanders") def test_map_country_br(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/br.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(-8, -63) self.assertEqual(key, "Amazonas") def test_map_country_ca(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/ca.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(53, -113) self.assertEqual(key, "Alberta") def test_map_country_cn(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/cn.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(34, 108) self.assertEqual(key, "Shaanxi") def test_map_country_de(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/de.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(52, 13) self.assertEqual(key, "Brandenburg") def test_map_country_es(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/es.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(41, -1) self.assertEqual(key, "Teruel") def test_map_country_fr(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/fr.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(49, 0) self.assertEqual(key, "Calvados") def test_map_country_in(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/in.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(18, 75) self.assertEqual(key, "Maharashtra") def test_map_country_it(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/it.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(42, 12) self.assertEqual(key, "Roma") def test_map_country_ru(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/ru.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(55, 37) self.assertEqual(key, "Kaluga") def test_map_country_uk(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/uk.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 0) self.assertEqual(key, "East Sussex") def test_map_country_us(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/us.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(33, -84) self.assertEqual(key, "GA") """ def test_map_country_us_counties(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/us_counties.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(33, -84) self.assertEqual(key, "13207") """ def test_map_country_us_simplified(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/us_simplified.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(33, -84) self.assertEqual(key, "GA")
reader = csv.reader(csvfile) start = datetime.datetime.now() count = 0 for row in reader: count += 1 if count == 1: continue print(rev.reversegeocodeshape(Point((float(row[0]), float(row[1]))))) if count > csv_max_count: break stop = datetime.datetime.now() delta = stop - start print("Seconds: %s" % delta.seconds) print("Without index") with open("lookups/latlng-data.csv", 'rb') as csvfile: rev = ReverseGeocoderShape() rev.load_map_file("geojson", "appserver/static/data/world2.geojson") performance_test(csvfile, rev) print("With index") with open("lookups/latlng-data.csv", 'rb') as csvfile: rev = ReverseGeocoderShape() rev.load_map_file("geojson", "appserver/static/data/world2.geojson") rev.load_index("bin/lib") performance_test(csvfile, rev)
class ReverseGeocoderShapeMapTest(unittest.TestCase): def setUp(self): self.rev = ReverseGeocoderShape() self.basepath = "." self.filetype = "geojson" def tearDown(self): self.rev.stop() del self.rev def load_index(self): index_file = os.path.join(self.basepath, "bin", "lib", "reversegeocodeshape-" + self.rev.map_md5 + ".index") self.rev.load_index_file(index_file) def test_map_custom(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "custom.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "area1") def test_map_hexagonmap_display_2(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "hexagonmap_display_2.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "3350") def test_map_hexagonmap_display_3(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "hexagonmap_display_3.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertIsNone(key) def test_map_hexagonmap_regeo_2(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "hexagonmap_regeo_2.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "3350") def test_map_hexagonmap_regeo_3(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "hexagonmap_regeo_3.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "1514") def test_map_squaremap_2(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "squaremap_2.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "12691") def test_map_squaremap_4(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "squaremap_4.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "3195") def test_map_world2(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "world2.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "BE") def test_map_world3(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "world3.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "BEL") """ Start testing for the different continents """ def test_map_continent_africa(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/africa.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(17, 9) self.assertEqual(key, "NER") def test_map_continent_asia(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/asia.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(19, 73) self.assertEqual(key, "IND") def test_map_continent_europe(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/europe.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "BEL") def test_map_continent_north_america(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/north-america.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(45, -75) self.assertEqual(key, "CAN") def test_map_continent_oceania(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/oceania.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(-33, 150) self.assertEqual(key, "AUS") def test_map_continent_south_america(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "continents/south-america.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(-31, -64) self.assertEqual(key, "ARG") """ Start testing for the different countries """ def test_map_country_be(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/be.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 3) self.assertEqual(key, "West Flanders") def test_map_country_br(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/br.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(-8, -63) self.assertEqual(key, "Amazonas") def test_map_country_ca(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/ca.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(53, -113) self.assertEqual(key, "Alberta") def test_map_country_cn(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/cn.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(34, 108) self.assertEqual(key, "Shaanxi") def test_map_country_de(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/de.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(52, 13) self.assertEqual(key, "Brandenburg") def test_map_country_es(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/es.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(41, -1) self.assertEqual(key, "Teruel") def test_map_country_fr(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/fr.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(49, 0) self.assertEqual(key, "Calvados") def test_map_country_in(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/in.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(18, 75) self.assertEqual(key, "Maharashtra") def test_map_country_it(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/it.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(42, 12) self.assertEqual(key, "Roma") def test_map_country_ru(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/ru.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(55, 37) self.assertEqual(key, "Kaluga") def test_map_country_uk(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/uk.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(51, 0) self.assertEqual(key, "East Sussex") def test_map_country_us(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/us.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(33, -84) self.assertEqual(key, "GA") """ def test_map_country_us_counties(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/us_counties.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(33, -84) self.assertEqual(key, "13207") """ def test_map_country_us_simplified(self): map_file = os.path.join(self.basepath, "appserver", "static", "data", "countries/us_simplified.geojson") self.rev.load_map_file(self.filetype, map_file) self.load_index() key = self.rev.reversegeocode(33, -84) self.assertEqual(key, "GA")
def performance_test(csvfile, rev): reader = csv.reader(csvfile) start = datetime.datetime.now() count = 0 for row in reader: count += 1 if count == 1: continue print(rev.reversegeocodeshape(Point((float(row[0]), float(row[1]))))) if count > csv_max_count: break stop = datetime.datetime.now() delta = stop - start print("Seconds: %s" % delta.seconds) print("Without index") with open("lookups/latlng-data.csv", 'rb') as csvfile: rev = ReverseGeocoderShape() rev.load_map_file("geojson", "appserver/static/data/world2.geojson") performance_test(csvfile, rev) print("With index") with open("lookups/latlng-data.csv", 'rb') as csvfile: rev = ReverseGeocoderShape() rev.load_map_file("geojson", "appserver/static/data/world2.geojson") rev.load_index("bin/lib") performance_test(csvfile, rev)