def create_index(data): map_file = os.path.join("..", "appserver", "static", "data", data["file"]) """ print("Change the line ending of %s" % data["file"]) with open(map_file) as inp, open(map_file+"-tmp", 'w') as out: txt = inp.read() txt = txt.replace('\r\n', '\n') out.write(txt) os.remove(map_file) os.rename(map_file+"-tmp", map_file) """ rev = ReverseGeocoderShape() rev.load_map_file("geojson", map_file) rev.indexstep = data["step"] print("Creating index for file: %s md5sum: %s" % (data["file"], rev.map_md5)) sys.stdout.flush() start = datetime.datetime.now() rev.load_index("lib") end = datetime.datetime.now() delta = end - start print("Index creation for %s took %s seconds for %s" % (data["file"], delta.seconds, rev.map_md5)) sys.stdout.flush()
def create_index(data): map_file = os.path.join("..", "appserver", "static", "data", data["file"]) """ print("Change the line ending of %s" % data["file"]) with open(map_file) as inp, open(map_file+"-tmp", 'w') as out: txt = inp.read() txt = txt.replace('\r\n', '\n') out.write(txt) os.remove(map_file) os.rename(map_file+"-tmp", map_file) """ rev = ReverseGeocoderShape() rev.load_map_file("geojson", map_file) rev.indexstep = data["step"] print("Creating index for file: %s md5sum: %s" % (data["file"], rev.map_md5)) sys.stdout.flush() start = datetime.datetime.now() rev.load_index("lib") end = datetime.datetime.now() delta = end - start print("Index creation for %s took %s seconds for %s" % (data["file"], delta.seconds, rev.map_md5)) sys.stdout.flush()
def stream(self, records): basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps") rev = ReverseGeocoderShape() map_file = os.path.join(basepath, "appserver", "static", "data", self.filename) rev.load_map_file(self.filetype, map_file) self.logger.info("Loaded map file %s" % self.filename) # Load map index file, this speeds up the command a lot index_file = os.path.join(basepath, "bin", "lib") rev.load_index(index_file) self.logger.info("Loaded map index %s" % index_file) # Load cached results cache_file = os.path.join( basepath, "bin", "lib", "reversegeocodeshape-" + rev.map_md5 + ".cache") self.cache = FileCache(1000000, 62) self.cache.read_cache_file(cache_file) self.logger.info("Loaded cached results %s" % cache_file) # iterate over records for record in records: lat = round(float(record[self.lat]), 5) lng = round(float(record[self.lng]), 5) cache_key = "%s,%s" % (lat, lng) try: key = self.cache.get(cache_key) except KeyError: key = rev.reversegeocode(lat, lng) self.cache.set(cache_key, key) record[self.fieldname] = key yield record try: self.cache.write_cache_file(cache_file) except: self.logger.error("Could not write cache file")
def stream(self, records): basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps") rev = ReverseGeocoderShape() map_file = os.path.join(basepath, "appserver", "static", "data", self.filename) rev.load_map_file(self.filetype, map_file) self.logger.info("Loaded map file %s" % self.filename) # Load map index file, this speeds up the command a lot index_file = os.path.join(basepath, "bin", "lib") rev.load_index(index_file) self.logger.info("Loaded map index %s" % index_file) # Load cached results cache_file = os.path.join(basepath, "bin", "lib", "reversegeocodeshape-" + rev.map_md5 + ".cache") self.cache = FileCache(1000000, 62) self.cache.read_cache_file(cache_file) self.logger.info("Loaded cached results %s" % cache_file) # iterate over records for record in records: lat = round(float(record[self.lat]), 5) lng = round(float(record[self.lng]), 5) cache_key = "%s,%s" % (lat, lng) try: key = self.cache.get(cache_key) except KeyError: key = rev.reversegeocode(lat, lng) self.cache.set(cache_key, key) record[self.fieldname] = key yield record try: self.cache.write_cache_file(cache_file) except: self.logger.error("Could not write cache file")
reader = csv.reader(csvfile) start = datetime.datetime.now() count = 0 for row in reader: count += 1 if count == 1: continue print(rev.reversegeocodeshape(Point((float(row[0]), float(row[1]))))) if count > csv_max_count: break stop = datetime.datetime.now() delta = stop - start print("Seconds: %s" % delta.seconds) print("Without index") with open("lookups/latlng-data.csv", 'rb') as csvfile: rev = ReverseGeocoderShape() rev.load_map_file("geojson", "appserver/static/data/world2.geojson") performance_test(csvfile, rev) print("With index") with open("lookups/latlng-data.csv", 'rb') as csvfile: rev = ReverseGeocoderShape() rev.load_map_file("geojson", "appserver/static/data/world2.geojson") rev.load_index("bin/lib") performance_test(csvfile, rev)
def performance_test(csvfile, rev): reader = csv.reader(csvfile) start = datetime.datetime.now() count = 0 for row in reader: count += 1 if count == 1: continue print(rev.reversegeocodeshape(Point((float(row[0]), float(row[1]))))) if count > csv_max_count: break stop = datetime.datetime.now() delta = stop - start print("Seconds: %s" % delta.seconds) print("Without index") with open("lookups/latlng-data.csv", 'rb') as csvfile: rev = ReverseGeocoderShape() rev.load_map_file("geojson", "appserver/static/data/world2.geojson") performance_test(csvfile, rev) print("With index") with open("lookups/latlng-data.csv", 'rb') as csvfile: rev = ReverseGeocoderShape() rev.load_map_file("geojson", "appserver/static/data/world2.geojson") rev.load_index("bin/lib") performance_test(csvfile, rev)