def test_schema_versions(self): """ Tests whether events formatted from different schema versions are compatible with the current data model. :return: """ source_device = "scale-local:scale/devices/temperature" v1_map = {"d" : {"event" : "temperature", "value" : 55.5, "units" : "celsius", "timestamp" : 12345678, "device" : source_device, "location" : {"lat" : 33.3, "lon" : "-71"}, "condition" : {"threshold" : {"operator" : ">", "value" : "95"}}, "prio_class": "high", "prio_value": 2, "schema" : "www.schema.org/scale_sensors.1.0.whatever", } } v1_event = SensedEvent.from_map(v1_map) self.assertEqual(v1_event.event_type, 'temperature') self.assertEqual(v1_event.data, 55.5) self.assertEqual(v1_event.priority, 2) self.assertEqual(v1_event.source, source_device) # Now the other way around: dumping to a map, first by looking at JSON encoding... v1_json = json.dumps(v1_map, sort_keys=True) manual_v1_json = json.dumps(json.loads(v1_event.to_json()), sort_keys=True) self.assertEqual(manual_v1_json, v1_json) new_v1_map = v1_event.to_map() self.assertEqual(v1_map, new_v1_map, "making into v1.0 schema map didn't produce identical dict: %s" % new_v1_map)
def extract_columns(self, data, parse_metadata=True): """ Extracts the important columns from the given list of SensedEvents :param data: :type data: list[dict] :param parse_metadata: if True (default), include columns for the metadata :return: """ # QUESTION: how to handle empty results??? events = [SensedEvent.from_map(e) for e in data] cols = {'topic': [ev.topic for ev in events], 'time_sent': [ev.timestamp for ev in events], # TODO: might not even want this? what to do with it? the 'scale-local:/' part makes it less useful... 'source': [ev.source for ev in events], 'value': [ev.data for ev in events], } # Include the metadata in case it has something valuable for us. # We have to gather up all unique keys first to ensure each row has all the needed columns so they line up. metadata_keys = set() for ev in events: for k in ev.metadata: metadata_keys.add(k) cols.update({ k: [ev.metadata.get(k) for ev in events] for k in metadata_keys }) return cols
def extract_columns(self, data, parse_metadata=True): """ Extracts the important columns from the given list of SensedEvents :param data: :type data: list[dict] :param parse_metadata: if True (default), include columns for the metadata :return: """ # QUESTION: how to handle empty results??? events = [SensedEvent.from_map(e) for e in data] cols = { 'topic': [ev.topic for ev in events], 'time_sent': [ev.timestamp for ev in events], # TODO: might not even want this? what to do with it? the 'scale-local:/' part makes it less useful... 'source': [ev.source for ev in events], 'value': [ev.data for ev in events], } # Include the metadata in case it has something valuable for us. # We have to gather up all unique keys first to ensure each row has all the needed columns so they line up. metadata_keys = set() for ev in events: for k in ev.metadata: metadata_keys.add(k) cols.update( {k: [ev.metadata.get(k) for ev in events] for k in metadata_keys}) return cols
def extract_columns(cls, data): """We need to extract the time each pick was received at the server for processing""" cols = super(ServerOutput, cls).extract_columns(data) # XXX: dummy data to ensure empty results parse okay and create a row if not data: cols['time_rcvd'] = [0] else: cols['time_rcvd'] = [ cls.get_aggregation_time(ev) for ev in (SensedEvent.from_map(e) for e in data) ] cols['src_ip'] = [ get_hostname_from_path(src) for src in cols.pop('source') ] return cols
def test_schema_versions(self): """ Tests whether events formatted from different schema versions are compatible with the current data model. :return: """ source_device = "scale-local:scale/devices/temperature" v1_map = { "d": { "event": "temperature", "value": 55.5, "units": "celsius", "timestamp": 12345678, "device": source_device, "location": { "lat": 33.3, "lon": "-71" }, "condition": { "threshold": { "operator": ">", "value": "95" } }, "prio_class": "high", "prio_value": 2, "schema": "www.schema.org/scale_sensors.1.0.whatever", } } v1_event = SensedEvent.from_map(v1_map) self.assertEqual(v1_event.event_type, 'temperature') self.assertEqual(v1_event.data, 55.5) self.assertEqual(v1_event.priority, 2) self.assertEqual(v1_event.source, source_device) # Now the other way around: dumping to a map, first by looking at JSON encoding... v1_json = json.dumps(v1_map, sort_keys=True) manual_v1_json = json.dumps(json.loads(v1_event.to_json()), sort_keys=True) self.assertEqual(manual_v1_json, v1_json) new_v1_map = v1_event.to_map() self.assertEqual( v1_map, new_v1_map, "making into v1.0 schema map didn't produce identical dict: %s" % new_v1_map)
def extract_columns(cls, data): """ Extracts the important columns from the given list of SensedEvents :param data: :return: """ events = [SensedEvent.from_map(e) for e in data] # XXX: to ensure this host shows up in at least one row, we add a dummy event when it had none if not events: events = [ SensedEvent(data=True, source="no source", event_type=EMPTY_OUTPUT_EVENT_TYPE) ] cols = { 'topic': [ev.topic for ev in events], 'time_sent': [ev.timestamp for ev in events], # TODO: might not even want this? what to do with it? the 'scale-local:/' part makes it less useful... 'source': [ev.source for ev in events], # all the events these types receive are seq#s; could move this down to derived class if needed... 'seq': [ev.data for ev in events], } return cols