def create_index(data):
map_file = os.path.join("..", "appserver", "static", "data", data["file"])
"""
print("Change the line ending of %s" % data["file"])
with open(map_file) as inp, open(map_file+"-tmp", 'w') as out:
txt = inp.read()
txt = txt.replace('\r\n', '\n')
out.write(txt)
os.remove(map_file)
os.rename(map_file+"-tmp", map_file)
"""
rev = ReverseGeocoderShape()
rev.load_map_file("geojson", map_file)
rev.indexstep = data["step"]
print("Creating index for file: %s md5sum: %s" %
(data["file"], rev.map_md5))
sys.stdout.flush()
start = datetime.datetime.now()
rev.load_index("lib")
end = datetime.datetime.now()
delta = end - start
print("Index creation for %s took %s seconds for %s" %
(data["file"], delta.seconds, rev.map_md5))
sys.stdout.flush()
def create_index(data):
map_file = os.path.join("..", "appserver", "static", "data", data["file"])
"""
print("Change the line ending of %s" % data["file"])
with open(map_file) as inp, open(map_file+"-tmp", 'w') as out:
txt = inp.read()
txt = txt.replace('\r\n', '\n')
out.write(txt)
os.remove(map_file)
os.rename(map_file+"-tmp", map_file)
"""
rev = ReverseGeocoderShape()
rev.load_map_file("geojson", map_file)
rev.indexstep = data["step"]
print("Creating index for file: %s md5sum: %s" % (data["file"], rev.map_md5))
sys.stdout.flush()
start = datetime.datetime.now()
rev.load_index("lib")
end = datetime.datetime.now()
delta = end - start
print("Index creation for %s took %s seconds for %s" % (data["file"], delta.seconds, rev.map_md5))
sys.stdout.flush()
def stream(self, records):
basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps",
"heremaps")
rev = ReverseGeocoderShape()
map_file = os.path.join(basepath, "appserver", "static", "data",
self.filename)
rev.load_map_file(self.filetype, map_file)
self.logger.info("Loaded map file %s" % self.filename)
# Load map index file, this speeds up the command a lot
index_file = os.path.join(basepath, "bin", "lib")
rev.load_index(index_file)
self.logger.info("Loaded map index %s" % index_file)
# Load cached results
cache_file = os.path.join(
basepath, "bin", "lib",
"reversegeocodeshape-" + rev.map_md5 + ".cache")
self.cache = FileCache(1000000, 62)
self.cache.read_cache_file(cache_file)
self.logger.info("Loaded cached results %s" % cache_file)
# iterate over records
for record in records:
lat = round(float(record[self.lat]), 5)
lng = round(float(record[self.lng]), 5)
cache_key = "%s,%s" % (lat, lng)
try:
key = self.cache.get(cache_key)
except KeyError:
key = rev.reversegeocode(lat, lng)
self.cache.set(cache_key, key)
record[self.fieldname] = key
yield record
try:
self.cache.write_cache_file(cache_file)
except:
self.logger.error("Could not write cache file")
def stream(self, records):
basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps")
rev = ReverseGeocoderShape()
map_file = os.path.join(basepath, "appserver", "static", "data", self.filename)
rev.load_map_file(self.filetype, map_file)
self.logger.info("Loaded map file %s" % self.filename)
# Load map index file, this speeds up the command a lot
index_file = os.path.join(basepath, "bin", "lib")
rev.load_index(index_file)
self.logger.info("Loaded map index %s" % index_file)
# Load cached results
cache_file = os.path.join(basepath, "bin", "lib", "reversegeocodeshape-" + rev.map_md5 + ".cache")
self.cache = FileCache(1000000, 62)
self.cache.read_cache_file(cache_file)
self.logger.info("Loaded cached results %s" % cache_file)
# iterate over records
for record in records:
lat = round(float(record[self.lat]), 5)
lng = round(float(record[self.lng]), 5)
cache_key = "%s,%s" % (lat, lng)
try:
key = self.cache.get(cache_key)
except KeyError:
key = rev.reversegeocode(lat, lng)
self.cache.set(cache_key, key)
record[self.fieldname] = key
yield record
try:
self.cache.write_cache_file(cache_file)
except:
self.logger.error("Could not write cache file")
reader = csv.reader(csvfile)
start = datetime.datetime.now()
count = 0
for row in reader:
count += 1
if count == 1:
continue
print(rev.reversegeocodeshape(Point((float(row[0]), float(row[1])))))
if count > csv_max_count:
break
stop = datetime.datetime.now()
delta = stop - start
print("Seconds: %s" % delta.seconds)
print("Without index")
with open("lookups/latlng-data.csv", 'rb') as csvfile:
rev = ReverseGeocoderShape()
rev.load_map_file("geojson", "appserver/static/data/world2.geojson")
performance_test(csvfile, rev)
print("With index")
with open("lookups/latlng-data.csv", 'rb') as csvfile:
rev = ReverseGeocoderShape()
rev.load_map_file("geojson", "appserver/static/data/world2.geojson")
rev.load_index("bin/lib")
performance_test(csvfile, rev)
def performance_test(csvfile, rev):
reader = csv.reader(csvfile)
start = datetime.datetime.now()
count = 0
for row in reader:
count += 1
if count == 1:
continue
print(rev.reversegeocodeshape(Point((float(row[0]), float(row[1])))))
if count > csv_max_count:
break
stop = datetime.datetime.now()
delta = stop - start
print("Seconds: %s" % delta.seconds)
print("Without index")
with open("lookups/latlng-data.csv", 'rb') as csvfile:
rev = ReverseGeocoderShape()
rev.load_map_file("geojson", "appserver/static/data/world2.geojson")
performance_test(csvfile, rev)
print("With index")
with open("lookups/latlng-data.csv", 'rb') as csvfile:
rev = ReverseGeocoderShape()
rev.load_map_file("geojson", "appserver/static/data/world2.geojson")
rev.load_index("bin/lib")
performance_test(csvfile, rev)
