def test_clean_old_2(self):
cache = FileCache(10, 2)
cache.set("key1", "value")
cache.set("key2", "value")
cache.set("key3", "value")
cache.set("key4", "value")
self.assertEqual(4, len(cache.objects))
cache.clean_old()
self.assertEqual(4, len(cache.objects))
def test_clean_count_1(self):
cache = FileCache(3, 1)
cache.set("key1", "value")
cache.set("key2", "value")
cache.set("key3", "value")
cache.set("key4", "value")
self.assertEqual(4, len(cache.objects))
cache.clean_count()
self.assertEqual(3, len(cache.objects))
def test_clean_old_2(self):
cache = FileCache(10, 2)
cache.set("key1", "value")
cache.set("key2", "value")
cache.set("key3", "value")
cache.set("key4", "value")
self.assertEqual(4, len(cache.objects))
cache.clean_old()
self.assertEqual(4, len(cache.objects))
def test_clean_count_1(self):
cache = FileCache(3, 1)
cache.set("key1", "value")
cache.set("key2", "value")
cache.set("key3", "value")
cache.set("key4", "value")
self.assertEqual(4, len(cache.objects))
cache.clean_count()
self.assertEqual(3, len(cache.objects))
def stream(self, records):
basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps",
"heremaps")
rev = ReverseGeocoderShape()
map_file = os.path.join(basepath, "appserver", "static", "data",
self.filename)
rev.load_map_file(self.filetype, map_file)
self.logger.info("Loaded map file %s" % self.filename)
# Load map index file, this speeds up the command a lot
index_file = os.path.join(basepath, "bin", "lib")
rev.load_index(index_file)
self.logger.info("Loaded map index %s" % index_file)
# Load cached results
cache_file = os.path.join(
basepath, "bin", "lib",
"reversegeocodeshape-" + rev.map_md5 + ".cache")
self.cache = FileCache(1000000, 62)
self.cache.read_cache_file(cache_file)
self.logger.info("Loaded cached results %s" % cache_file)
# iterate over records
for record in records:
lat = round(float(record[self.lat]), 5)
lng = round(float(record[self.lng]), 5)
cache_key = "%s,%s" % (lat, lng)
try:
key = self.cache.get(cache_key)
except KeyError:
key = rev.reversegeocode(lat, lng)
self.cache.set(cache_key, key)
record[self.fieldname] = key
yield record
try:
self.cache.write_cache_file(cache_file)
except:
self.logger.error("Could not write cache file")
def stream(self, records):
basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps")
rev = ReverseGeocoderShape()
map_file = os.path.join(basepath, "appserver", "static", "data", self.filename)
rev.load_map_file(self.filetype, map_file)
self.logger.info("Loaded map file %s" % self.filename)
# Load map index file, this speeds up the command a lot
index_file = os.path.join(basepath, "bin", "lib")
rev.load_index(index_file)
self.logger.info("Loaded map index %s" % index_file)
# Load cached results
cache_file = os.path.join(basepath, "bin", "lib", "reversegeocodeshape-" + rev.map_md5 + ".cache")
self.cache = FileCache(1000000, 62)
self.cache.read_cache_file(cache_file)
self.logger.info("Loaded cached results %s" % cache_file)
# iterate over records
for record in records:
lat = round(float(record[self.lat]), 5)
lng = round(float(record[self.lng]), 5)
cache_key = "%s,%s" % (lat, lng)
try:
key = self.cache.get(cache_key)
except KeyError:
key = rev.reversegeocode(lat, lng)
self.cache.set(cache_key, key)
record[self.fieldname] = key
yield record
try:
self.cache.write_cache_file(cache_file)
except:
self.logger.error("Could not write cache file")
def test_clean_old_1(self):
cache = FileCache(10, 2)
cache.get_today = mock.Mock(
return_value=datetime.datetime.today().date())
cache.set("key1", "value")
cache.get_today = mock.Mock(
return_value=(datetime.datetime.today() -
datetime.timedelta(days=1)).date())
cache.set("key2", "value")
cache.get_today = mock.Mock(
return_value=(datetime.datetime.today() -
datetime.timedelta(days=2)).date())
cache.set("key3", "value")
cache.get_today = mock.Mock(
return_value=(datetime.datetime.today() -
datetime.timedelta(days=3)).date())
cache.set("key4", "value")
self.assertEqual(4, len(cache.objects))
cache.get_today = mock.Mock(
return_value=datetime.datetime.today().date())
cache.clean_old()
self.assertEqual(2, len(cache.objects))
def test_get_existent_key(self):
cache = FileCache(1000, 1)
cache.set("mykey", "myvalue")
self.assertEqual("myvalue", cache.get("mykey"))
def test_get_non_existent_key(self):
cache = FileCache(1000, 1)
with self.assertRaises(KeyError):
cache.get("mykey")
def test_empty_key(self):
cache = FileCache(1000, 1)
cache.set("mykey", "myvalue")
cache.set("mykey", None)
self.assertEquals(cache.objects["mykey"]["data"], None)
def test_empty_key(self):
cache = FileCache(1000, 1)
cache.set("mykey", "myvalue")
cache.set("mykey", None)
self.assertEquals(cache.objects["mykey"]["data"], None)
class ReverseGeocodeShapeCommand(StreamingCommand):
lat = Option(doc='''
**Syntax:** **lat=***<fieldname>*
**Description:** Name of the field that holds the latitude''',
require=False,
default="lat",
validate=validators.Fieldname())
lng = Option(doc='''
**Syntax:** **lng=***<fieldname>*
**Description:** Name of the field that holds the longitude''',
require=False,
default="lng",
validate=validators.Fieldname())
filetype = Option(doc='''
**Syntax:** **filetype=***<filetype>*
**Description:** Type of file: geojson or kml''',
require=False,
default="geojson")
filename = Option(doc='''
**Syntax:** **filename=***<filename>*
**Description:** Name of the file''',
require=False,
default="world2.geojson")
fieldname = Option(doc='''
**Syntax:** **fieldname=***fieldname*
**Description:** Name of the field that will contain the result''',
require=False,
default="key")
def stream(self, records):
basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps",
"heremaps")
rev = ReverseGeocoderShape()
map_file = os.path.join(basepath, "appserver", "static", "data",
self.filename)
rev.load_map_file(self.filetype, map_file)
self.logger.info("Loaded map file %s" % self.filename)
# Load map index file, this speeds up the command a lot
index_file = os.path.join(basepath, "bin", "lib")
rev.load_index(index_file)
self.logger.info("Loaded map index %s" % index_file)
# Load cached results
cache_file = os.path.join(
basepath, "bin", "lib",
"reversegeocodeshape-" + rev.map_md5 + ".cache")
self.cache = FileCache(1000000, 62)
self.cache.read_cache_file(cache_file)
self.logger.info("Loaded cached results %s" % cache_file)
# iterate over records
for record in records:
lat = round(float(record[self.lat]), 5)
lng = round(float(record[self.lng]), 5)
cache_key = "%s,%s" % (lat, lng)
try:
key = self.cache.get(cache_key)
except KeyError:
key = rev.reversegeocode(lat, lng)
self.cache.set(cache_key, key)
record[self.fieldname] = key
yield record
try:
self.cache.write_cache_file(cache_file)
except:
self.logger.error("Could not write cache file")
def test_clean_old_1(self):
cache = FileCache(10, 2)
cache.get_today = mock.Mock(return_value=datetime.datetime.today().date())
cache.set("key1", "value")
cache.get_today = mock.Mock(return_value=(datetime.datetime.today() - datetime.timedelta(days=1)).date())
cache.set("key2", "value")
cache.get_today = mock.Mock(return_value=(datetime.datetime.today() - datetime.timedelta(days=2)).date())
cache.set("key3", "value")
cache.get_today = mock.Mock(return_value=(datetime.datetime.today() - datetime.timedelta(days=3)).date())
cache.set("key4", "value")
self.assertEqual(4, len(cache.objects))
cache.get_today = mock.Mock(return_value=datetime.datetime.today().date())
cache.clean_old()
self.assertEqual(2, len(cache.objects))
def test_get_existent_key(self):
cache = FileCache(1000, 1)
cache.set("mykey", "myvalue")
self.assertEqual("myvalue", cache.get("mykey"))
def test_get_non_existent_key(self):
cache = FileCache(1000, 1)
with self.assertRaises(KeyError):
cache.get("mykey")
def test_cache_creation(self):
cache = FileCache(1000, 1)
self.assertEqual(cache.max_count, 1000, "Is max count the same")
self.assertEqual(cache.expiry, 1, "Is expiry the same")
self.assertEqual(len(cache.objects), 0)
class ReverseGeocodeCommand(StreamingCommand):
lat = Option(doc='''
**Syntax:** **lat=***<fieldname>*
**Description:** Name of the field that holds the latitude''',
require=False,
default="lat",
validate=validators.Fieldname())
lng = Option(doc='''
**Syntax:** **lng=***<fieldname>*
**Description:** Name of the field that holds the longitude''',
require=False,
default="lng",
validate=validators.Fieldname())
prefix = Option(doc='''
**Syntax:** **prefix=***customprefix*
**Description:** Prefix for the fields that will contain the result''',
require=False,
default="regeo")
app_id = ""
app_code = ""
cache = FileCache(1000000, 62)
def get_app_id(self):
try:
response = self.service.get(
"/servicesNS/nobody/heremaps/configs/conf-setup/heremaps")
xml = response.body.read()
root = et.fromstring(xml)
self.app_id = root.findall(
".//{http://dev.splunk.com/ns/rest}key[@name='app_id']"
)[0].text.strip()
self.app_code = root.findall(
".//{http://dev.splunk.com/ns/rest}key[@name='app_code']"
)[0].text.strip()
except Exception as e:
raise Exception(
"Could not get app_id and app_code, is the app set up correctly?",
e)
def _parseJSON(self, obj):
if isinstance(obj, dict):
newobj = {}
for key, value in obj.iteritems():
key = str(key)
newobj[key] = self._parseJSON(value)
elif isinstance(obj, list):
newobj = []
for value in obj:
newobj.append(self._parseJSON(value))
elif isinstance(obj, unicode):
newobj = str(obj)
else:
newobj = obj
return newobj
def reverse_geocode(self, lat, lng):
url = "http://reverse.geocoder.api.here.com/6.2/reversegeocode.json" + \
"?app_id=%s&app_code=%s&prox=%s,%s,10&" % (self.app_id, self.app_code, lat, lng) + \
"gen=7&mode=retrieveAreas&maxresults=1&responseattributes=-ps,-mq,-mt,-mc,-pr&" + \
"locationattributes=-mr,-mv,-ad,-ai"
http = httplib2.Http()
response, content = http.request(url, 'GET')
if response["status"] == "200":
result = json.loads(content)
# Make sure we don't get unicode strings
result = self._parseJSON(result)
try:
return result["Response"]["View"][0]["Result"][0]["Location"][
"Address"]
except (KeyError, IndexError):
self.logger.debug(
"Could not find any results for lat=%s and lng=%s", lat,
lng)
return {}
else:
self.logger.error("Got an unexpected for ulr %s response: %s", url,
response)
def add_fields(self, record, location):
record[self.prefix + "_country"] = ""
if "Country" in location:
record[self.prefix + "_country"] = location["Country"]
record[self.prefix + "_city"] = ""
if "City" in location:
record[self.prefix + "_city"] = location["City"]
record[self.prefix + "_postalcode"] = ""
if "PostalCode" in location:
record[self.prefix + "_postalcode"] = location["PostalCode"]
record[self.prefix + "_region"] = ""
if "Region" in location:
record[self.prefix + "_region"] = location["Region"]
record[self.prefix + "_district"] = ""
if "District" in location:
record[self.prefix + "_district"] = location["District"]
record[self.prefix + "_label"] = ""
if "Label" in location:
record[self.prefix + "_label"] = location["Label"]
record[self.prefix + "_state"] = ""
if "State" in location:
record[self.prefix + "_state"] = location["State"]
record[self.prefix + "_county"] = ""
if "County" in location:
record[self.prefix + "_county"] = location["County"]
def stream(self, records):
basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps",
"heremaps")
cachefile = os.path.join(basepath, "bin", "lib",
"reversegeocode.cache")
if self.app_id is "" or self.app_code is "":
self.get_app_id()
self.cache.read_cache_file(cachefile)
for record in records:
lat = str(round(float(record[self.lat]), 5))
lng = str(round(float(record[self.lng]), 5))
cache_key = "%s,%s" % (lat, lng)
try:
location = self.cache.get(cache_key)
except KeyError:
location = self.reverse_geocode(lat, lng)
self.cache.set(cache_key, location)
self.add_fields(record, location)
yield record
try:
self.cache.write_cache_file(cachefile)
except:
self.logger.error("Could not write cache file")
def test_overwrite_key(self):
cache = FileCache(1000, 1)
cache.set("mykey", "myvalue")
cache.set("mykey", "NewValue")
self.assertEquals(cache.objects["mykey"]["data"], "NewValue")
class ReverseGeocodeShapeCommand(StreamingCommand):
lat = Option(
doc='''
**Syntax:** **lat=***<fieldname>*
**Description:** Name of the field that holds the latitude''',
require=False, default="lat", validate=validators.Fieldname())
lng = Option(
doc='''
**Syntax:** **lng=***<fieldname>*
**Description:** Name of the field that holds the longitude''',
require=False, default="lng", validate=validators.Fieldname())
filetype = Option(
doc='''
**Syntax:** **filetype=***<filetype>*
**Description:** Type of file: geojson or kml''',
require=False, default="geojson")
filename = Option(
doc='''
**Syntax:** **filename=***<filename>*
**Description:** Name of the file''',
require=False, default="world2.geojson")
fieldname = Option(
doc='''
**Syntax:** **fieldname=***fieldname*
**Description:** Name of the field that will contain the result''',
require=False, default="key")
def stream(self, records):
basepath = os.path.join(os.environ['SPLUNK_HOME'], "etc", "apps", "heremaps")
rev = ReverseGeocoderShape()
map_file = os.path.join(basepath, "appserver", "static", "data", self.filename)
rev.load_map_file(self.filetype, map_file)
self.logger.info("Loaded map file %s" % self.filename)
# Load map index file, this speeds up the command a lot
index_file = os.path.join(basepath, "bin", "lib")
rev.load_index(index_file)
self.logger.info("Loaded map index %s" % index_file)
# Load cached results
cache_file = os.path.join(basepath, "bin", "lib", "reversegeocodeshape-" + rev.map_md5 + ".cache")
self.cache = FileCache(1000000, 62)
self.cache.read_cache_file(cache_file)
self.logger.info("Loaded cached results %s" % cache_file)
# iterate over records
for record in records:
lat = round(float(record[self.lat]), 5)
lng = round(float(record[self.lng]), 5)
cache_key = "%s,%s" % (lat, lng)
try:
key = self.cache.get(cache_key)
except KeyError:
key = rev.reversegeocode(lat, lng)
self.cache.set(cache_key, key)
record[self.fieldname] = key
yield record
try:
self.cache.write_cache_file(cache_file)
except:
self.logger.error("Could not write cache file")
def test_overwrite_key(self):
cache = FileCache(1000, 1)
cache.set("mykey", "myvalue")
cache.set("mykey", "NewValue")
self.assertEquals(cache.objects["mykey"]["data"], "NewValue")
