def test_schema_versions(self):
"""
Tests whether events formatted from different schema versions are compatible with the current data model.
:return:
"""
source_device = "scale-local:scale/devices/temperature"
v1_map = {"d" :
{"event" : "temperature",
"value" : 55.5,
"units" : "celsius",
"timestamp" : 12345678,
"device" : source_device,
"location" : {"lat" : 33.3, "lon" : "-71"},
"condition" : {"threshold" : {"operator" : ">", "value" : "95"}},
"prio_class": "high",
"prio_value": 2,
"schema" : "www.schema.org/scale_sensors.1.0.whatever",
}
}
v1_event = SensedEvent.from_map(v1_map)
self.assertEqual(v1_event.event_type, 'temperature')
self.assertEqual(v1_event.data, 55.5)
self.assertEqual(v1_event.priority, 2)
self.assertEqual(v1_event.source, source_device)
# Now the other way around: dumping to a map, first by looking at JSON encoding...
v1_json = json.dumps(v1_map, sort_keys=True)
manual_v1_json = json.dumps(json.loads(v1_event.to_json()), sort_keys=True)
self.assertEqual(manual_v1_json, v1_json)
new_v1_map = v1_event.to_map()
self.assertEqual(v1_map, new_v1_map, "making into v1.0 schema map didn't produce identical dict: %s" % new_v1_map)
def extract_columns(self, data, parse_metadata=True):
"""
Extracts the important columns from the given list of SensedEvents
:param data:
:type data: list[dict]
:param parse_metadata: if True (default), include columns for the metadata
:return:
"""
# QUESTION: how to handle empty results???
events = [SensedEvent.from_map(e) for e in data]
cols = {'topic': [ev.topic for ev in events],
'time_sent': [ev.timestamp for ev in events],
# TODO: might not even want this? what to do with it? the 'scale-local:/' part makes it less useful...
'source': [ev.source for ev in events],
'value': [ev.data for ev in events],
}
# Include the metadata in case it has something valuable for us.
# We have to gather up all unique keys first to ensure each row has all the needed columns so they line up.
metadata_keys = set()
for ev in events:
for k in ev.metadata:
metadata_keys.add(k)
cols.update({
k: [ev.metadata.get(k) for ev in events] for k in metadata_keys
})
return cols
def extract_columns(self, data, parse_metadata=True):
"""
Extracts the important columns from the given list of SensedEvents
:param data:
:type data: list[dict]
:param parse_metadata: if True (default), include columns for the metadata
:return:
"""
# QUESTION: how to handle empty results???
events = [SensedEvent.from_map(e) for e in data]
cols = {
'topic': [ev.topic for ev in events],
'time_sent': [ev.timestamp for ev in events],
# TODO: might not even want this? what to do with it? the 'scale-local:/' part makes it less useful...
'source': [ev.source for ev in events],
'value': [ev.data for ev in events],
}
# Include the metadata in case it has something valuable for us.
# We have to gather up all unique keys first to ensure each row has all the needed columns so they line up.
metadata_keys = set()
for ev in events:
for k in ev.metadata:
metadata_keys.add(k)
cols.update(
{k: [ev.metadata.get(k) for ev in events]
for k in metadata_keys})
return cols
def extract_columns(cls, data):
"""We need to extract the time each pick was received at the server for processing"""
cols = super(ServerOutput, cls).extract_columns(data)
# XXX: dummy data to ensure empty results parse okay and create a row
if not data:
cols['time_rcvd'] = [0]
else:
cols['time_rcvd'] = [
cls.get_aggregation_time(ev)
for ev in (SensedEvent.from_map(e) for e in data)
]
cols['src_ip'] = [
get_hostname_from_path(src) for src in cols.pop('source')
]
return cols
def test_schema_versions(self):
"""
Tests whether events formatted from different schema versions are compatible with the current data model.
:return:
"""
source_device = "scale-local:scale/devices/temperature"
v1_map = {
"d": {
"event": "temperature",
"value": 55.5,
"units": "celsius",
"timestamp": 12345678,
"device": source_device,
"location": {
"lat": 33.3,
"lon": "-71"
},
"condition": {
"threshold": {
"operator": ">",
"value": "95"
}
},
"prio_class": "high",
"prio_value": 2,
"schema": "www.schema.org/scale_sensors.1.0.whatever",
}
}
v1_event = SensedEvent.from_map(v1_map)
self.assertEqual(v1_event.event_type, 'temperature')
self.assertEqual(v1_event.data, 55.5)
self.assertEqual(v1_event.priority, 2)
self.assertEqual(v1_event.source, source_device)
# Now the other way around: dumping to a map, first by looking at JSON encoding...
v1_json = json.dumps(v1_map, sort_keys=True)
manual_v1_json = json.dumps(json.loads(v1_event.to_json()),
sort_keys=True)
self.assertEqual(manual_v1_json, v1_json)
new_v1_map = v1_event.to_map()
self.assertEqual(
v1_map, new_v1_map,
"making into v1.0 schema map didn't produce identical dict: %s" %
new_v1_map)
def extract_columns(cls, data):
"""
Extracts the important columns from the given list of SensedEvents
:param data:
:return:
"""
events = [SensedEvent.from_map(e) for e in data]
# XXX: to ensure this host shows up in at least one row, we add a dummy event when it had none
if not events:
events = [
SensedEvent(data=True,
source="no source",
event_type=EMPTY_OUTPUT_EVENT_TYPE)
]
cols = {
'topic': [ev.topic for ev in events],
'time_sent': [ev.timestamp for ev in events],
# TODO: might not even want this? what to do with it? the 'scale-local:/' part makes it less useful...
'source': [ev.source for ev in events],
# all the events these types receive are seq#s; could move this down to derived class if needed...
'seq': [ev.data for ev in events],
}
return cols
