def get_sources(self, id_: str = None, query: Dict = None) -> Union[Source, List[Source]]: logger.debug("getting sources from server") if id_: try: source_data = self.http.get(f'{self.sources_url}/{id_}') return Source( id_=source_data['external_id'], tdmq_id=source_data['tdmq_id'], type_=EntityType(source_data['entity_type'], source_data['entity_category']), station_model=source_data.get('station_model', ''), geometry=Point( source_data['default_footprint']['coordinates'][1], source_data['default_footprint']['coordinates'][0])) except HTTPError as e: logger.error('error response from server with status code %s', e.response.status_code) raise_exception(e.response.status_code) try: return [ Source( id_=s['external_id'], tdmq_id=s['tdmq_id'], type_=EntityType(s['entity_type'], s['entity_category']), station_model=s.get('station_model', ''), geometry=Point(s['default_footprint']['coordinates'][1], s['default_footprint']['coordinates'][0])) for s in self.http.get(f'{self.sources_url}', params=query) ] except HTTPError as e: logger.error('error response from server with status code %s', e.response.status_code) raise_exception(e.response.status_code)
def _get_sensor_model(source_id: str, sensor_type: str) -> str: edge_name, station_name, sensor_name = source_id.split('.') if sensor_type == EntityType("EnergyConsumptionMonitor", "Station"): if 'emontx' in station_name.lower(): return 'emonTx' elif 'iotawatt' in station_name.lower(): return 'IoTaWatt' else: return 'IoTaWatt' elif sensor_type == EntityType("WeatherObserver", "Station"): if 'emontx' in station_name.lower(): return 'emonTx' elif 'esp8266-' in station_name.lower(): return 'airRohr' elif 'airrohr-' in station_name.lower(): return 'airRohr' elif 'edge' in station_name.lower(): return 'EdgeGateway' else: return 'Unknown' elif sensor_type == EntityType("DeviceStatusMonitor", "Station"): return 'EdgeGateway' return None
def test_get_source_by_id(self): """ Tests getting a source using the tdmq id """ expected_source = Source( id_=REST_SOURCE["external_id"], type_=EntityType(REST_SOURCE["entity_type"], REST_SOURCE["entity_category"]), geometry=Point(*REST_SOURCE["default_footprint"]["coordinates"][::-1]), # for some strange reason the points are inverted controlled_properties=None, tdmq_id=REST_SOURCE["tdmq_id"] ) client = Client(self.url) httpretty.register_uri(httpretty.GET, f'{client.sources_url}/{SENSORS[0].tdmq_id}', body=jsons.dumps(REST_SOURCE), match_querystring=False) res = client.get_sources(REST_SOURCE["tdmq_id"]) self.assertEqual(res.to_json(), expected_source.to_json())
def _test_convert_device(self, converter): timeseries_list = converter.convert([json.dumps(self.in_device_msg)]) self.assertEqual(len(timeseries_list), 1) self.assertEqual(timeseries_list[0].source.type, EntityType("DeviceStatusMonitor", "Station")) data = { attr["name"]: self.device_values.get(attr["name"], "") for attr in self.in_device_msg["body"]["attributes"] if attr['name'] not in self.ignored_attrs } self.assertEqual(timeseries_list[0].data, data) self.assertEqual( timeseries_list[0].time.strftime("%Y-%m-%dT%H:%M:%SZ"), "2019-12-18T14:05:05Z") self.assertEqual(str(timeseries_list[0].source.id_), "Edge-z.EDGE.HTU21D") self.assertEqual(str(timeseries_list[0].source.edge_id), "Edge-z") self.assertEqual(str(timeseries_list[0].source.station_id), "Edge-z.EDGE") self.assertEqual(str(timeseries_list[0].source.sensor_id), "HTU21D")
def _test_convert_energy(self, converter): timeseries_list = converter.convert([json.dumps(self.in_energy_msg)]) self.assertEqual(len(timeseries_list), 1) self.assertEqual(timeseries_list[0].source.type, EntityType("EnergyConsumptionMonitor", "Station")) data = { attr["name"]: self.energy_values.get(attr["name"], "") for attr in self.in_energy_msg["body"]["attributes"] if attr['name'] not in self.ignored_attrs } self.assertEqual(timeseries_list[0].data, data) self.assertEqual( timeseries_list[0].time.strftime("%Y-%m-%dT%H:%M:%SZ"), "2019-12-16T16:33:19Z") self.assertEqual(str(timeseries_list[0].source.id_), "Edge-y.emontx3.L3") self.assertEqual(str(timeseries_list[0].source.edge_id), "Edge-y") self.assertEqual(str(timeseries_list[0].source.station_id), "Edge-y.emontx3") self.assertEqual(str(timeseries_list[0].source.sensor_id), "L3")
def _test_convert_weather(self, converter): timeseries_list = converter.convert([json.dumps(self.in_weather_msg)]) self.assertEqual(len(timeseries_list), 1) self.assertEqual(timeseries_list[0].source.type, EntityType("WeatherObserver", "Station")) data = { attr["name"]: self.weather_values.get(attr["name"], "") for attr in self.in_weather_msg["body"]["attributes"] if attr['name'] not in self.ignored_attrs } self.assertEqual(timeseries_list[0].data, data) self.assertEqual( timeseries_list[0].time.strftime("%Y-%m-%dT%H:%M:%SZ"), "2019-12-16T16:31:34Z") self.assertEqual(str(timeseries_list[0].source.id_), "Edge-x.StationX.Davis") self.assertEqual(str(timeseries_list[0].source.edge_id), "Edge-x") self.assertEqual(str(timeseries_list[0].source.station_id), "Edge-x.StationX") self.assertEqual(str(timeseries_list[0].source.sensor_id), "Davis")
def test_get_all_sources(self): """ Tests getting all sources """ expected_sources = [ Source( id_=REST_SOURCE["external_id"], type_=EntityType(REST_SOURCE["entity_type"], REST_SOURCE["entity_category"]), geometry=Point(*REST_SOURCE["default_footprint"]["coordinates"][::-1]), controlled_properties=None, tdmq_id=REST_SOURCE["tdmq_id"] ) ] client = Client(self.url) httpretty.register_uri(httpretty.GET, client.sources_url, body=jsons.dumps([REST_SOURCE]), match_querystring=False) res = client.get_sources() self.assertEqual([s.to_json() for s in res], [s.to_json() for s in expected_sources])
class NgsiConverter: """ Class to convert json-formatted NGSI-Fiware messages to :class:`Record` instances """ non_properties = {"dateObserved", "location", "TimeInstant"} # Legacy attributes replaced by "location" _to_ignore = {"latitude", "longitude"} fiware_service_path_to_sensor_type = { "/cagliari/edge/meteo": EntityType("WeatherObserver", "Station"), "/cagliari/edge/energy": EntityType("EnergyConsumptionMonitor", "Station"), "/cagliari/edge/device": EntityType("DeviceStatusMonitor", "Station"), } #: Maps the ngsi attribute types to python types _type_mapper = { "String": str, "Float": float, "Integer": int, "geo:point": lambda v: Point(*v.replace(" ", "").split(",")), "ISO8601": isoparse } #: Maps the attributes with a specific handler. It has priority higher than type _attrs_mapper = { "timestamp": lambda v: datetime.datetime.fromtimestamp(int(v), datetime.timezone.utc ), "dateObserved": isoparse, "rssi": int, "dewpoint": float, "memoryFree": float } _message_id_regex = re.compile( r"(?P<Type>\w+):(?P<Edge>[a-zA-Z0-9_-]+)\.(?P<Station>[a-zA-Z0-9_-]+)\.(?P<Sensor>[a-zA-Z0-9_-]+)" ) @staticmethod def _get_fiware_service_path(msg: Dict): for header in msg["headers"]: try: return header["fiware-servicePath"] except KeyError: pass raise RuntimeError(f"fiware-servicePath not found in msg {msg}") @staticmethod def _get_source_id(msg: Dict) -> str: try: match = NgsiConverter._message_id_regex.search(msg["body"]["id"]) except KeyError: raise RuntimeError(f"invalid id {msg['body']['id']}") else: if match: _, edge_name, station_name, sensor_name = match.groups() source_id = "{}.{}.{}".format(edge_name, station_name, sensor_name) return source_id raise RuntimeError(f"invalid id {msg['body']['id']}") def _get_sensor_type(self, msg: Dict) -> str: try: service_path = self._get_fiware_service_path(msg) return self.fiware_service_path_to_sensor_type[service_path] except KeyError: raise RuntimeError(f"invalid message type {service_path}") @staticmethod def _create_sensor(sensor_name: str, sensor_type: EntityType, geometry: Geometry, properties: List[str]) -> Source: return Source(sensor_name, sensor_type, geometry, properties) def _create_record(self, msg: Dict) -> Record: source_id = self._get_source_id(msg) sensor_type = self._get_sensor_type(msg) logging.debug("Converting message of type %s", sensor_type.category) records: Dict = {} time = None geometry = None for attr in msg["body"]["attributes"]: name, value, type_ = attr["name"], attr["value"], attr["type"] if not name in self._to_ignore: if value is None or not str(value).strip(): converter = str value = '' else: # First check for a converter for the attribute try: converter = self._attrs_mapper[name] except KeyError: converter = self._type_mapper.get(type_, None) try: converted_value = converter(value) except (ValueError, TypeError): # FIXME: should we skip the message or just the property? logger.warning( "cannot read attribute %s of type %s with value %s", name, type_, value) continue else: if name == "timestamp": time = converted_value elif name == "location": geometry = converted_value elif name not in self.non_properties: records[name] = converted_value if not records: raise RuntimeError("conversion produced no useful data") if geometry is None: raise RuntimeError("missing latitude and/or longitude") sensor = self._create_sensor(source_id, sensor_type, geometry, records.keys()) return Record(time, sensor, geometry, records) def convert(self, messages: List[str]) -> List[Record]: """ Method that reads a list of ngsi messages and convert them in a list of :class:`Record` """ logger.debug("messages %s", len(messages)) timeseries_list: List = [] for m in messages: try: m_dict = json.loads(m) logger.debug("Message is %s", m_dict) timeseries_list.append(self._create_record(m_dict)) except JSONDecodeError: logger.error("exception decoding message %s", m) continue except RuntimeError as e: logger.error("exception occurred with message %s", m) logger.error(e) continue return timeseries_list
from datetime import datetime, timezone from tdm_ingestion.tdmq.models import EntityType, Point, Record, Source now = datetime.now(timezone.utc) SENSORS_TYPE = [EntityType("st1", "cat1"), EntityType("st2", "cat2")] SENSORS = [ Source("s1", SENSORS_TYPE[0], Point(0, 1), ["temperature"], "4d9ae10d-df9b-546c-a586-925e1e9ec049"), Source("s2", SENSORS_TYPE[1], Point(2, 3), ["humidity"], "6eb57b7e-43a3-5ad7-a4d1-d1ec54bb5520") ] TIME_SERIES = [ Record(now, SENSORS[0], Point(0, 1), {"temperature": 14.0}), Record(now, SENSORS[1], Point(2, 3), {"humidity": 95.0}) ] # the dictionary returned from the tdmq polystore rest api REST_SOURCE = { "default_footprint": { "coordinates": [SENSORS[0].geometry.latitude, SENSORS[0].geometry.longitude], "type": "Point" }, "entity_type": SENSORS_TYPE[0].name, "entity_category": SENSORS_TYPE[0].category, "external_id": SENSORS[0].id_,
from datetime import datetime, timezone from tdm_ingestion.tdmq.models import EntityType, Point, Record, Source now = datetime.now(timezone.utc) SENSORS_TYPE = [ EntityType("st1", "cat1"), EntityType("st2", "cat2") ] SENSORS = [ Source("s1", SENSORS_TYPE[0], 'model1', Point(0, 1), ["temperature"], "4d9ae10d-df9b-546c-a586-925e1e9ec049", "Edge1", "Station1", "Sensor1"), Source("s2", SENSORS_TYPE[1], 'model2', Point(2, 3), ["humidity"], "6eb57b7e-43a3-5ad7-a4d1-d1ec54bb5520", "Edge2", "Station1", "Sensor1"), ] TIME_SERIES = [ Record(now, SENSORS[0], Point(0, 1), {"temperature": 14.0}), Record(now, SENSORS[1], Point(2, 3), {"humidity": 95.0}) ] # the dictionary returned from the tdmq polystore rest api REST_SOURCE = { "default_footprint": { "coordinates": [ SENSORS[0].geometry.latitude, SENSORS[0].geometry.longitude
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('-d', help='debug', dest='debug', action='store_true', default=False) parser.add_argument('--tdmq_url', dest='tdmq_url', required=True) parser.add_argument('--bucket', dest='bucket', required=True, type=float) parser.add_argument('--op', dest='operation', required=True) time_delta_mapping = { 'today': TimeDelta.today, '1h': TimeDelta.one_hour, '1d': TimeDelta.one_day, '1w': TimeDelta.one_week, '1m': TimeDelta.one_month } parser.add_argument('--time_delta_before', dest='time_delta_before', choices=time_delta_mapping.keys()) parser.add_argument('--before', dest='before') parser.add_argument('--after', dest='after') parser.add_argument('--entity_type', dest='entity_type', required=True, choices=[ 'PointWeatherObserver', 'WeatherObserver', 'EnergyConsumptionMonitor', 'DeviceStatusMonitor' ]) parser.add_argument('--ckan_url', dest='ckan_url', required=True) parser.add_argument('--ckan_api_key', dest='ckan_api_key', required=True) parser.add_argument('--ckan_dataset', dest='ckan_dataset', required=True) parser.add_argument('--ckan_resource', dest='ckan_resource', required=True) parser.add_argument( '--ckan_description', dest='ckan_description', help=('description text; %%{after} and %%{before} are populated by ' 'first and last day (1w and 1m time deltas only)'), required=False, default="") parser.add_argument('--upsert', dest='upsert', default=False, action='store_true') parser.add_argument('--prune', dest='prune', default=False, help='when create a resource with time deltas of 1w or' '1m deletes the related daily reports.', action='store_true') args = parser.parse_args() logging_level = logging.DEBUG if args.debug else logging.INFO logging.basicConfig(level=logging_level) logging.info('running with args %s', args.__dict__) if args.time_delta_before: time_delta = time_delta_mapping[args.time_delta_before] before, after = time_delta.get_before_after() resource_name = after.strftime(args.ckan_resource) else: before, after = args.before, args.after resource_name = args.ckan_resource description_text = args.ckan_description or '' if args.ckan_description and args.time_delta_before: if "%{after}" in description_text: description_text = description_text.replace( "%{after}", after.strftime("%Y-%m-%d")) if "%{before}" in description_text: description_text = description_text.replace( "%{before}", before.strftime("%Y-%m-%d")) consumer = TDMQConsumer(Client(args.tdmq_url)) storage = CkanStorage( RemoteCkan(args.ckan_url, Requests(), args.ckan_api_key)) storage.write( consumer.poll(EntityType(args.entity_type), args.bucket, args.operation, before, after), args.ckan_dataset, resource_name, description_text, args.upsert) if args.prune: if args.time_delta_before == '1w': storage.prune_resources(args.ckan_dataset, resource_name, after, before, prune_weekly=False) elif args.time_delta_before == '1m': storage.prune_resources(args.ckan_dataset, resource_name, after, before, prune_weekly=True)