def setUp(self):
self.conn = WithRedis.StrictRedis()
self.conn.flushdb()
# override with your own UA to verify test results in GA
self.my_ua = os.getenv('UA_ID', 'UA-116198943-3')
self.er = EventReporter(UA=self.my_ua, conn=self.conn)
class EventReporterTest(unittest.TestCase):
def setUp(self):
self.conn = WithRedis.StrictRedis()
self.conn.flushdb()
# override with your own UA to verify test results in GA
self.my_ua = os.getenv('UA_ID', 'UA-116198943-3')
self.er = EventReporter(UA=self.my_ua, conn=self.conn)
def test_base(self):
"""
Checks to see that the base EventReporter class loads
"""
pass
@unittest.skip('will fail if env var overrides UA_ID')
def test_args(self):
"""
Checks to see that EventReporter stores UA
"""
self.assertTrue(self.er.UA == 'UA-116198943-3')
def test_store_fetch_dispatch(self):
"""
Checks to see that the EventReporter stores expected data
"""
ar = self.er.store('ga',
'event',
'20538abc-a8af-46e0-b292-0999d94468e9',
category='user',
action='action_name',
aip='1',
uip='1.2.3.4',
ds='web',
ua='my-useragent-test')
self.assertTrue(ar == None)
expected = {
'handler': 'ga',
'etype': 'event',
'clientid': '20538abc-a8af-46e0-b292-0999d94468e9',
'ts': 1548546584914,
'args': {
'category': 'user',
'action': 'action_name',
'aip': '1',
'uip': '1.2.3.4',
'ds': 'web',
'ua': 'my-useragent-test'
}
}
r = self.er.fetch()
self.assertTrue(isinstance(r['ts'], int))
# ts varies
del expected['ts']
self.assertDictContainsSubset(expected, r)
# NOTE: live test.
self.assertTrue(self.er.dispatch(r))
# print(f'Data has been sent to {self.my_ua}. Please check real-time stats to confirm correctness.')
def test_store_fetch_oldest_double(self):
"""
Checks to see that the EventReporter fetch_oldest gets expected data
"""
ar = self.er.store('ga',
'event',
'20538abc-a8af-46e0-b292-0999d94468e9',
category='user',
action='action_name',
aip='1',
uip='1.2.3.4',
ds='web')
ar2 = self.er.store('ga',
'event',
'20538abc-a8af-46e0-b292-0999d94468e9',
category='user',
action='action_name_2',
aip='1',
uip='1.2.3.4',
ds='web')
self.assertTrue(ar == None)
expected = {
'handler': 'ga',
'etype': 'event',
'clientid': '20538abc-a8af-46e0-b292-0999d94468e9',
'ts': 1548546584914,
'args': {
'category': 'user',
'action': 'action_name',
'aip': '1',
'uip': '1.2.3.4',
'ds': 'web'
}
}
r = self.er.fetch_oldest()
self.assertTrue(isinstance(r['ts'], int))
# ts varies
del expected['ts']
self.assertDictContainsSubset(expected, r)
def test_store_fetch_oldest_single(self):
"""
Checks to see that the EventReporter fetch_oldest gets expected data
"""
ar = self.er.store('ga',
'event',
'20538abc-a8af-46e0-b292-0999d94468e9',
category='user',
action='action_name',
aip='1',
uip='1.2.3.4',
ds='web')
self.assertTrue(ar == None)
expected = {
'handler': 'ga',
'etype': 'event',
'clientid': '20538abc-a8af-46e0-b292-0999d94468e9',
'ts': 1548546584914,
'args': {
'category': 'user',
'action': 'action_name',
'aip': '1',
'uip': '1.2.3.4',
'ds': 'web'
}
}
r = self.er.fetch_oldest()
self.assertTrue(isinstance(r['ts'], int))
# ts varies
del expected['ts']
self.assertDictContainsSubset(expected, r)
def test_unsafe_store(self):
'''
Verify that e.g. a redis or argument failure throws an exception.
'''
with self.assertRaises(TypeError):
self.er.store(category='user',
action='action_name',
aip='1',
uip='1.2.3.4',
ds='web')
def test_safe_store_fail(self):
'''
Verify that e.g. a redis or argument failure does not throw an exception.
'''
r = self.er.safe_store(None,
None,
None,
action='action_name',
aip='1',
uip='1.2.3.4',
ds='web')
self.assertTrue(r == False)
def test_safe_store_success(self):
'''
Verify that e.g. a redis or argument failure does not throw an exception.
'''
r = self.er.safe_store('ga',
'event',
'20538abc-a8af-46e0-b292-0999d94468e9',
category='user',
action='action_name',
aip='1',
uip='1.2.3.4',
ds='web')
self.assertTrue(r == None)
def test_store_fetch_dispatch_referrer(self):
"""
Checks to see that the EventReporter stores expected data with referrer.
Looks like it's unnecessary to urlencode prior to handing it off.
"""
ar = self.er.store('ga',
'event',
'20538abc-a8af-46e0-b292-0999d94468e9',
category='user',
action='action_name',
aip='1',
uip='1.2.3.4',
ds='web',
dr='http://www.test.com')
self.assertTrue(ar == None)
expected = {
'handler': 'ga',
'etype': 'event',
'clientid': '20538abc-a8af-46e0-b292-0999d94468e9',
'ts': 1548546584914,
'args': {
'category': 'user',
'action': 'action_name',
'aip': '1',
'uip': '1.2.3.4',
'ds': 'web',
'dr': 'http://www.test.com'
}
}
r = self.er.fetch()
self.assertTrue(isinstance(r['ts'], int))
# ts varies
del expected['ts']
self.assertDictContainsSubset(expected, r)
# NOTE: live test.
self.assertTrue(self.er.dispatch(r))
def setup_reporter(self, cfg=None):
if self.__broker is None:
self.setup_broker(cfg)
self.__reporter = EventReporter(self.__broker)
def setup_reporter(self, cfg):
if self.__broker is None:
self.setup_broker(cfg)
self.__reporter = EventReporter(self.__broker)
class ScaleClient:
"""This class parses a configuration file and subsequently creates, configures, and manages each component of the
overall Scale Client software.
"""
def __init__(self, config_filename):
self.__broker = None
self.__reporter = None
self.__sensors = []
self.__applications = None
try:
log.info("Reading config file: %s" % config_filename)
with open(config_filename) as cfile:
cfg = yaml.load(cfile)
# lower-case all top-level headings to tolerate different capitalizations
cfg = {k.lower(): v for k, v in cfg.items()}
# verify required entries present
# required_entries = ('main',)
# for ent in required_entries:
# if ent not in cfg:
# raise Exception("Required entry %s not present in config file." % ent)
# call appropriate handlers for each section in the appropriate order
if "main" in cfg:
self.setup_broker(cfg["main"])
self.setup_reporter(cfg["main"])
else:
self.setup_broker({})
self.setup_reporter({})
if "eventsinks" in cfg:
self.setup_event_sinks(cfg["eventsinks"])
if "sensors" in cfg:
self.setup_sensors(cfg["sensors"])
if "networks" in cfg:
self.setup_network(cfg["networks"])
if "applications" in cfg:
self.setup_applications(cfg["applications"])
except IOError as e:
log.error("Error reading config file: %s" % e)
exit(1)
def setup_reporter(self, cfg):
if self.__broker is None:
self.setup_broker(cfg)
self.__reporter = EventReporter(self.__broker)
def setup_broker(self, cfg):
"""
Currently only creates a dummy Broker object for registering Applications to.
:param cfg:
:return:
"""
self.__broker = Broker()
def setup_event_sinks(self, sink_configurations):
for sink_config in (sink.values()[0] for sink in sink_configurations):
# need to get class definition to call constructor
if "class" not in sink_config:
log.warn("Skipping EventSink with no class definition: %s" % sink_config)
continue
try:
# this line lets us tolerate just e.g. mqtt_event_sink.MQTTEventSink as a relative pathname
cls_name = (
sink_config["class"]
if "scale_client.event_sinks" in sink_config["class"]
else "scale_client.event_sinks." + sink_config["class"]
)
cls = self._get_class_by_name(cls_name)
# make a copy of config so we can tweak it to expose only correct kwargs
new_sink_config = sink_config.copy()
new_sink_config.pop("class")
sink = cls(self.__broker, **new_sink_config)
self.__reporter.add_sink(sink)
log.info("EventSink created from config: %s" % sink_config)
except Exception as e:
log.error("Unexpected error while creating EventSink: %s" % e)
def setup_applications(self, application_configurations):
n_applications = 0
for application_config in (c.values()[0] for c in application_configurations):
# need to get class definition to call constructor
if "class" not in application_config:
log.warn("Skipping virtual application with no class definition: %s" % application_config)
continue
try:
cls_name = (
application_config["class"]
if "scale_client.applications" in application_config["class"]
else "scale_client.applications." + application_config["class"]
)
cls = self._get_class_by_name(cls_name)
# copy config s so we can tweak it as necessary to expose only correct kwargs
new_application_config = application_config.copy()
new_application_config.pop("class")
cls(self.__broker, **new_application_config)
n_applications += 1
log.info("Application created from config: %s" % application_config)
except Exception as e:
log.error("Unexpected error (%s) while creating application: %s" % (e, application_config))
continue
def setup_sensors(self, sensor_configurations):
n_sensors = 0
for sensor_config in (c.values()[0] for c in sensor_configurations):
# need to get class definition to call constructor
if "class" not in sensor_config:
log.warn("Skipping virtual sensor with no class definition: %s" % sensor_config)
continue
try:
cls_name = (
sensor_config["class"]
if "scale_client.sensors" in sensor_config["class"]
else "scale_client.sensors." + sensor_config["class"]
)
cls = self._get_class_by_name(cls_name)
# copy config so we can tweak it as necessary to expose only correct kwargs
new_sensor_config = sensor_config.copy()
new_sensor_config.pop("class")
dev_name = new_sensor_config.get("dev_name", "vs%i" % n_sensors)
new_sensor_config.pop("dev_name", dev_name)
cls(self.__broker, device=DeviceDescriptor(dev_name), **new_sensor_config)
n_sensors += 1
log.info("Virtual sensor created from config: %s" % sensor_config)
except Exception as e:
log.error("Unexpected error (%s) while creating virtual sensor: %s" % (e, sensor_config))
continue
def setup_network(self, network_configurations):
for network_config in (c.values()[0] for c in network_configurations):
# need to get class definition to call constructor
if "class" not in network_config:
log.warn("Skipping network config with no class definition: %s" % network_config)
continue
try:
cls_name = (
network_config["class"]
if "scale_client.network" in network_config["class"]
else "scale_client.network." + network_config["class"]
)
cls = self._get_class_by_name(cls_name)
# copy config s so we can tweak it as necessary to expose only correct kwargs
new_network_config = network_config.copy()
new_network_config.pop("class")
cls(self.__broker, **new_network_config)
log.info("Virtual sensor created from config: %s" % network_config)
except Exception as e:
log.error("Unexpected error (%s) while setting network class: %s" % (e, network_config))
def run(self):
"""Currently just loop forever to allow the other threads to do their work."""
# TODO: handle this with less overhead?
self.__broker.run()
def _get_class_by_name(self, kls):
"""Imports and returns a class reference for the full module name specified in regular Python import format"""
# TODO: download definition from some repository if necessary
# The following code taken from http://stackoverflow.com/questions/452969/does-python-have-an-equivalent-to-java-class-forname
parts = kls.split(".")
module = ".".join(parts[:-1])
m = __import__(module)
for comp in parts[1:]:
m = getattr(m, comp)
return m
import time
import json
import logging
import os
from redis import StrictRedis
from event_reporter import EventReporter
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
LOG = logging.getLogger('evrlistener')
LOCALCONN = StrictRedis()
ER = EventReporter(conn=LOCALCONN)
def process_item(item):
'''called for each new item'''
return ER.dispatch(item)
def fetch_func():
item = ER.fetch_oldest(blocking=False)
if item:
try:
process_item(item)
except Exception as e:
LOG.error("process_item failed:{}, item: {}".format(
e, item))
time.sleep(10) # sleep for 10 seconds
time.sleep(0.001) # be nice to the system :)
def main():
while True: