def test_clean_old_2(self): cache = FileCache(10, 2) cache.set("key1", "value") cache.set("key2", "value") cache.set("key3", "value") cache.set("key4", "value") self.assertEqual(4, len(cache.objects)) cache.clean_old() self.assertEqual(4, len(cache.objects))
def test_clean_count_1(self): cache = FileCache(3, 1) cache.set("key1", "value") cache.set("key2", "value") cache.set("key3", "value") cache.set("key4", "value") self.assertEqual(4, len(cache.objects)) cache.clean_count() self.assertEqual(3, len(cache.objects))
def stream(self, records): basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps") rev = ReverseGeocoderShape() map_file = os.path.join(basepath, "appserver", "static", "data", self.filename) rev.load_map_file(self.filetype, map_file) self.logger.info("Loaded map file %s" % self.filename) # Load map index file, this speeds up the command a lot index_file = os.path.join(basepath, "bin", "lib") rev.load_index(index_file) self.logger.info("Loaded map index %s" % index_file) # Load cached results cache_file = os.path.join( basepath, "bin", "lib", "reversegeocodeshape-" + rev.map_md5 + ".cache") self.cache = FileCache(1000000, 62) self.cache.read_cache_file(cache_file) self.logger.info("Loaded cached results %s" % cache_file) # iterate over records for record in records: lat = round(float(record[self.lat]), 5) lng = round(float(record[self.lng]), 5) cache_key = "%s,%s" % (lat, lng) try: key = self.cache.get(cache_key) except KeyError: key = rev.reversegeocode(lat, lng) self.cache.set(cache_key, key) record[self.fieldname] = key yield record try: self.cache.write_cache_file(cache_file) except: self.logger.error("Could not write cache file")
def stream(self, records): basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps") rev = ReverseGeocoderShape() map_file = os.path.join(basepath, "appserver", "static", "data", self.filename) rev.load_map_file(self.filetype, map_file) self.logger.info("Loaded map file %s" % self.filename) # Load map index file, this speeds up the command a lot index_file = os.path.join(basepath, "bin", "lib") rev.load_index(index_file) self.logger.info("Loaded map index %s" % index_file) # Load cached results cache_file = os.path.join(basepath, "bin", "lib", "reversegeocodeshape-" + rev.map_md5 + ".cache") self.cache = FileCache(1000000, 62) self.cache.read_cache_file(cache_file) self.logger.info("Loaded cached results %s" % cache_file) # iterate over records for record in records: lat = round(float(record[self.lat]), 5) lng = round(float(record[self.lng]), 5) cache_key = "%s,%s" % (lat, lng) try: key = self.cache.get(cache_key) except KeyError: key = rev.reversegeocode(lat, lng) self.cache.set(cache_key, key) record[self.fieldname] = key yield record try: self.cache.write_cache_file(cache_file) except: self.logger.error("Could not write cache file")
def test_clean_old_1(self): cache = FileCache(10, 2) cache.get_today = mock.Mock( return_value=datetime.datetime.today().date()) cache.set("key1", "value") cache.get_today = mock.Mock( return_value=(datetime.datetime.today() - datetime.timedelta(days=1)).date()) cache.set("key2", "value") cache.get_today = mock.Mock( return_value=(datetime.datetime.today() - datetime.timedelta(days=2)).date()) cache.set("key3", "value") cache.get_today = mock.Mock( return_value=(datetime.datetime.today() - datetime.timedelta(days=3)).date()) cache.set("key4", "value") self.assertEqual(4, len(cache.objects)) cache.get_today = mock.Mock( return_value=datetime.datetime.today().date()) cache.clean_old() self.assertEqual(2, len(cache.objects))
def test_get_existent_key(self): cache = FileCache(1000, 1) cache.set("mykey", "myvalue") self.assertEqual("myvalue", cache.get("mykey"))
def test_get_non_existent_key(self): cache = FileCache(1000, 1) with self.assertRaises(KeyError): cache.get("mykey")
def test_empty_key(self): cache = FileCache(1000, 1) cache.set("mykey", "myvalue") cache.set("mykey", None) self.assertEquals(cache.objects["mykey"]["data"], None)
class ReverseGeocodeShapeCommand(StreamingCommand): lat = Option(doc=''' **Syntax:** **lat=***<fieldname>* **Description:** Name of the field that holds the latitude''', require=False, default="lat", validate=validators.Fieldname()) lng = Option(doc=''' **Syntax:** **lng=***<fieldname>* **Description:** Name of the field that holds the longitude''', require=False, default="lng", validate=validators.Fieldname()) filetype = Option(doc=''' **Syntax:** **filetype=***<filetype>* **Description:** Type of file: geojson or kml''', require=False, default="geojson") filename = Option(doc=''' **Syntax:** **filename=***<filename>* **Description:** Name of the file''', require=False, default="world2.geojson") fieldname = Option(doc=''' **Syntax:** **fieldname=***fieldname* **Description:** Name of the field that will contain the result''', require=False, default="key") def stream(self, records): basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps") rev = ReverseGeocoderShape() map_file = os.path.join(basepath, "appserver", "static", "data", self.filename) rev.load_map_file(self.filetype, map_file) self.logger.info("Loaded map file %s" % self.filename) # Load map index file, this speeds up the command a lot index_file = os.path.join(basepath, "bin", "lib") rev.load_index(index_file) self.logger.info("Loaded map index %s" % index_file) # Load cached results cache_file = os.path.join( basepath, "bin", "lib", "reversegeocodeshape-" + rev.map_md5 + ".cache") self.cache = FileCache(1000000, 62) self.cache.read_cache_file(cache_file) self.logger.info("Loaded cached results %s" % cache_file) # iterate over records for record in records: lat = round(float(record[self.lat]), 5) lng = round(float(record[self.lng]), 5) cache_key = "%s,%s" % (lat, lng) try: key = self.cache.get(cache_key) except KeyError: key = rev.reversegeocode(lat, lng) self.cache.set(cache_key, key) record[self.fieldname] = key yield record try: self.cache.write_cache_file(cache_file) except: self.logger.error("Could not write cache file")
def test_clean_old_1(self): cache = FileCache(10, 2) cache.get_today = mock.Mock(return_value=datetime.datetime.today().date()) cache.set("key1", "value") cache.get_today = mock.Mock(return_value=(datetime.datetime.today() - datetime.timedelta(days=1)).date()) cache.set("key2", "value") cache.get_today = mock.Mock(return_value=(datetime.datetime.today() - datetime.timedelta(days=2)).date()) cache.set("key3", "value") cache.get_today = mock.Mock(return_value=(datetime.datetime.today() - datetime.timedelta(days=3)).date()) cache.set("key4", "value") self.assertEqual(4, len(cache.objects)) cache.get_today = mock.Mock(return_value=datetime.datetime.today().date()) cache.clean_old() self.assertEqual(2, len(cache.objects))
def test_cache_creation(self): cache = FileCache(1000, 1) self.assertEqual(cache.max_count, 1000, "Is max count the same") self.assertEqual(cache.expiry, 1, "Is expiry the same") self.assertEqual(len(cache.objects), 0)
class ReverseGeocodeCommand(StreamingCommand): lat = Option(doc=''' **Syntax:** **lat=***<fieldname>* **Description:** Name of the field that holds the latitude''', require=False, default="lat", validate=validators.Fieldname()) lng = Option(doc=''' **Syntax:** **lng=***<fieldname>* **Description:** Name of the field that holds the longitude''', require=False, default="lng", validate=validators.Fieldname()) prefix = Option(doc=''' **Syntax:** **prefix=***customprefix* **Description:** Prefix for the fields that will contain the result''', require=False, default="regeo") app_id = "" app_code = "" cache = FileCache(1000000, 62) def get_app_id(self): try: response = self.service.get( "/servicesNS/nobody/heremaps/configs/conf-setup/heremaps") xml = response.body.read() root = et.fromstring(xml) self.app_id = root.findall( ".//{http://dev.splunk.com/ns/rest}key[@name='app_id']" )[0].text.strip() self.app_code = root.findall( ".//{http://dev.splunk.com/ns/rest}key[@name='app_code']" )[0].text.strip() except Exception as e: raise Exception( "Could not get app_id and app_code, is the app set up correctly?", e) def _parseJSON(self, obj): if isinstance(obj, dict): newobj = {} for key, value in obj.iteritems(): key = str(key) newobj[key] = self._parseJSON(value) elif isinstance(obj, list): newobj = [] for value in obj: newobj.append(self._parseJSON(value)) elif isinstance(obj, unicode): newobj = str(obj) else: newobj = obj return newobj def reverse_geocode(self, lat, lng): url = "http://reverse.geocoder.api.here.com/6.2/reversegeocode.json" + \ "?app_id=%s&app_code=%s&prox=%s,%s,10&" % (self.app_id, self.app_code, lat, lng) + \ "gen=7&mode=retrieveAreas&maxresults=1&responseattributes=-ps,-mq,-mt,-mc,-pr&" + \ "locationattributes=-mr,-mv,-ad,-ai" http = httplib2.Http() response, content = http.request(url, 'GET') if response["status"] == "200": result = json.loads(content) # Make sure we don't get unicode strings result = self._parseJSON(result) try: return result["Response"]["View"][0]["Result"][0]["Location"][ "Address"] except (KeyError, IndexError): self.logger.debug( "Could not find any results for lat=%s and lng=%s", lat, lng) return {} else: self.logger.error("Got an unexpected for ulr %s response: %s", url, response) def add_fields(self, record, location): record[self.prefix + "_country"] = "" if "Country" in location: record[self.prefix + "_country"] = location["Country"] record[self.prefix + "_city"] = "" if "City" in location: record[self.prefix + "_city"] = location["City"] record[self.prefix + "_postalcode"] = "" if "PostalCode" in location: record[self.prefix + "_postalcode"] = location["PostalCode"] record[self.prefix + "_region"] = "" if "Region" in location: record[self.prefix + "_region"] = location["Region"] record[self.prefix + "_district"] = "" if "District" in location: record[self.prefix + "_district"] = location["District"] record[self.prefix + "_label"] = "" if "Label" in location: record[self.prefix + "_label"] = location["Label"] record[self.prefix + "_state"] = "" if "State" in location: record[self.prefix + "_state"] = location["State"] record[self.prefix + "_county"] = "" if "County" in location: record[self.prefix + "_county"] = location["County"] def stream(self, records): basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps") cachefile = os.path.join(basepath, "bin", "lib", "reversegeocode.cache") if self.app_id is "" or self.app_code is "": self.get_app_id() self.cache.read_cache_file(cachefile) for record in records: lat = str(round(float(record[self.lat]), 5)) lng = str(round(float(record[self.lng]), 5)) cache_key = "%s,%s" % (lat, lng) try: location = self.cache.get(cache_key) except KeyError: location = self.reverse_geocode(lat, lng) self.cache.set(cache_key, location) self.add_fields(record, location) yield record try: self.cache.write_cache_file(cachefile) except: self.logger.error("Could not write cache file")
def test_overwrite_key(self): cache = FileCache(1000, 1) cache.set("mykey", "myvalue") cache.set("mykey", "NewValue") self.assertEquals(cache.objects["mykey"]["data"], "NewValue")
class ReverseGeocodeShapeCommand(StreamingCommand): lat = Option( doc=''' **Syntax:** **lat=***<fieldname>* **Description:** Name of the field that holds the latitude''', require=False, default="lat", validate=validators.Fieldname()) lng = Option( doc=''' **Syntax:** **lng=***<fieldname>* **Description:** Name of the field that holds the longitude''', require=False, default="lng", validate=validators.Fieldname()) filetype = Option( doc=''' **Syntax:** **filetype=***<filetype>* **Description:** Type of file: geojson or kml''', require=False, default="geojson") filename = Option( doc=''' **Syntax:** **filename=***<filename>* **Description:** Name of the file''', require=False, default="world2.geojson") fieldname = Option( doc=''' **Syntax:** **fieldname=***fieldname* **Description:** Name of the field that will contain the result''', require=False, default="key") def stream(self, records): basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps") rev = ReverseGeocoderShape() map_file = os.path.join(basepath, "appserver", "static", "data", self.filename) rev.load_map_file(self.filetype, map_file) self.logger.info("Loaded map file %s" % self.filename) # Load map index file, this speeds up the command a lot index_file = os.path.join(basepath, "bin", "lib") rev.load_index(index_file) self.logger.info("Loaded map index %s" % index_file) # Load cached results cache_file = os.path.join(basepath, "bin", "lib", "reversegeocodeshape-" + rev.map_md5 + ".cache") self.cache = FileCache(1000000, 62) self.cache.read_cache_file(cache_file) self.logger.info("Loaded cached results %s" % cache_file) # iterate over records for record in records: lat = round(float(record[self.lat]), 5) lng = round(float(record[self.lng]), 5) cache_key = "%s,%s" % (lat, lng) try: key = self.cache.get(cache_key) except KeyError: key = rev.reversegeocode(lat, lng) self.cache.set(cache_key, key) record[self.fieldname] = key yield record try: self.cache.write_cache_file(cache_file) except: self.logger.error("Could not write cache file")