def get_sources(self,
id_: str = None,
query: Dict = None) -> Union[Source, List[Source]]:
logger.debug("getting sources from server")
if id_:
try:
source_data = self.http.get(f'{self.sources_url}/{id_}')
return Source(
id_=source_data['external_id'],
tdmq_id=source_data['tdmq_id'],
type_=EntityType(source_data['entity_type'],
source_data['entity_category']),
station_model=source_data.get('station_model', ''),
geometry=Point(
source_data['default_footprint']['coordinates'][1],
source_data['default_footprint']['coordinates'][0]))
except HTTPError as e:
logger.error('error response from server with status code %s',
e.response.status_code)
raise_exception(e.response.status_code)
try:
return [
Source(
id_=s['external_id'],
tdmq_id=s['tdmq_id'],
type_=EntityType(s['entity_type'], s['entity_category']),
station_model=s.get('station_model', ''),
geometry=Point(s['default_footprint']['coordinates'][1],
s['default_footprint']['coordinates'][0]))
for s in self.http.get(f'{self.sources_url}', params=query)
]
except HTTPError as e:
logger.error('error response from server with status code %s',
e.response.status_code)
raise_exception(e.response.status_code)
def get_time_series(self,
source: Source,
query: Dict[str, Any] = None) -> List[Record]:
try:
time_series = self.http.get(
f'{self.sources_url}/{source.tdmq_id}/timeseries',
params=query)
except HTTPError as e:
logger.error('error response from server with status code %s',
e.response.status_code)
raise_exception(e.response.status_code)
records: List[Record] = []
logger.debug('time_series %s', time_series)
for idx, time in enumerate(time_series['coords']['time']):
date_time = datetime.datetime.fromtimestamp(
time, datetime.timezone.utc)
try:
# No support for MultiPoint, just bring the last coordinate pair
footprint = time_series['coords']['footprint'][idx][
"coordinates"][-1]
except TypeError:
footprint = time_series['default_footprint']['coordinates']
# Point(latitude, longitude) but point are returned as [longitude, latitude]
records.append(
Record(
date_time, source, Point(footprint[1], footprint[0]), {
data: value_list[idx]
for data, value_list in time_series['data'].items()
if value_list
}))
return records
def test_get_source_by_id(self):
"""
Tests getting a source using the tdmq id
"""
expected_source = Source(
id_=REST_SOURCE["external_id"],
type_=EntityType(REST_SOURCE["entity_type"], REST_SOURCE["entity_category"]),
geometry=Point(*REST_SOURCE["default_footprint"]["coordinates"][::-1]), # for some strange reason the points are inverted
controlled_properties=None,
tdmq_id=REST_SOURCE["tdmq_id"]
)
client = Client(self.url)
httpretty.register_uri(httpretty.GET, f'{client.sources_url}/{SENSORS[0].tdmq_id}',
body=jsons.dumps(REST_SOURCE), match_querystring=False)
res = client.get_sources(REST_SOURCE["tdmq_id"])
self.assertEqual(res.to_json(), expected_source.to_json())
def test_get_all_sources(self):
"""
Tests getting all sources
"""
expected_sources = [
Source(
id_=REST_SOURCE["external_id"],
type_=EntityType(REST_SOURCE["entity_type"], REST_SOURCE["entity_category"]),
geometry=Point(*REST_SOURCE["default_footprint"]["coordinates"][::-1]),
controlled_properties=None,
tdmq_id=REST_SOURCE["tdmq_id"]
)
]
client = Client(self.url)
httpretty.register_uri(httpretty.GET, client.sources_url,
body=jsons.dumps([REST_SOURCE]), match_querystring=False)
res = client.get_sources()
self.assertEqual([s.to_json() for s in res],
[s.to_json() for s in expected_sources])
class NgsiConverter:
"""
Class to convert json-formatted NGSI-Fiware messages to :class:`Record` instances
"""
non_properties = {"dateObserved", "location", "TimeInstant"}
# Legacy attributes replaced by "location"
_to_ignore = {"latitude", "longitude"}
fiware_service_path_to_sensor_type = {
"/cagliari/edge/meteo": EntityType("WeatherObserver", "Station"),
"/cagliari/edge/energy": EntityType("EnergyConsumptionMonitor",
"Station"),
"/cagliari/edge/device": EntityType("DeviceStatusMonitor", "Station"),
}
#: Maps the ngsi attribute types to python types
_type_mapper = {
"String": str,
"Float": float,
"Integer": int,
"geo:point": lambda v: Point(*v.replace(" ", "").split(",")),
"ISO8601": isoparse
}
#: Maps the attributes with a specific handler. It has priority higher than type
_attrs_mapper = {
"timestamp":
lambda v: datetime.datetime.fromtimestamp(int(v), datetime.timezone.utc
),
"dateObserved":
isoparse,
"rssi":
int,
"dewpoint":
float,
"memoryFree":
float
}
_message_id_regex = re.compile(
r"(?P<Type>\w+):(?P<Edge>[a-zA-Z0-9_-]+)\.(?P<Station>[a-zA-Z0-9_-]+)\.(?P<Sensor>[a-zA-Z0-9_-]+)"
)
@staticmethod
def _get_fiware_service_path(msg: Dict):
for header in msg["headers"]:
try:
return header["fiware-servicePath"]
except KeyError:
pass
raise RuntimeError(f"fiware-servicePath not found in msg {msg}")
@staticmethod
def _get_source_id(msg: Dict) -> str:
try:
match = NgsiConverter._message_id_regex.search(msg["body"]["id"])
except KeyError:
raise RuntimeError(f"invalid id {msg['body']['id']}")
else:
if match:
_, edge_name, station_name, sensor_name = match.groups()
source_id = "{}.{}.{}".format(edge_name, station_name,
sensor_name)
return source_id
raise RuntimeError(f"invalid id {msg['body']['id']}")
def _get_sensor_type(self, msg: Dict) -> str:
try:
service_path = self._get_fiware_service_path(msg)
return self.fiware_service_path_to_sensor_type[service_path]
except KeyError:
raise RuntimeError(f"invalid message type {service_path}")
@staticmethod
def _create_sensor(sensor_name: str, sensor_type: EntityType,
geometry: Geometry, properties: List[str]) -> Source:
return Source(sensor_name, sensor_type, geometry, properties)
def _create_record(self, msg: Dict) -> Record:
source_id = self._get_source_id(msg)
sensor_type = self._get_sensor_type(msg)
logging.debug("Converting message of type %s", sensor_type.category)
records: Dict = {}
time = None
geometry = None
for attr in msg["body"]["attributes"]:
name, value, type_ = attr["name"], attr["value"], attr["type"]
if not name in self._to_ignore:
if value is None or not str(value).strip():
converter = str
value = ''
else:
# First check for a converter for the attribute
try:
converter = self._attrs_mapper[name]
except KeyError:
converter = self._type_mapper.get(type_, None)
try:
converted_value = converter(value)
except (ValueError, TypeError):
# FIXME: should we skip the message or just the property?
logger.warning(
"cannot read attribute %s of type %s with value %s",
name, type_, value)
continue
else:
if name == "timestamp":
time = converted_value
elif name == "location":
geometry = converted_value
elif name not in self.non_properties:
records[name] = converted_value
if not records:
raise RuntimeError("conversion produced no useful data")
if geometry is None:
raise RuntimeError("missing latitude and/or longitude")
sensor = self._create_sensor(source_id, sensor_type, geometry,
records.keys())
return Record(time, sensor, geometry, records)
def convert(self, messages: List[str]) -> List[Record]:
"""
Method that reads a list of ngsi messages and convert them in a list of :class:`Record`
"""
logger.debug("messages %s", len(messages))
timeseries_list: List = []
for m in messages:
try:
m_dict = json.loads(m)
logger.debug("Message is %s", m_dict)
timeseries_list.append(self._create_record(m_dict))
except JSONDecodeError:
logger.error("exception decoding message %s", m)
continue
except RuntimeError as e:
logger.error("exception occurred with message %s", m)
logger.error(e)
continue
return timeseries_list
from datetime import datetime, timezone
from tdm_ingestion.tdmq.models import EntityType, Point, Record, Source
now = datetime.now(timezone.utc)
SENSORS_TYPE = [EntityType("st1", "cat1"), EntityType("st2", "cat2")]
SENSORS = [
Source("s1", SENSORS_TYPE[0], Point(0, 1), ["temperature"],
"4d9ae10d-df9b-546c-a586-925e1e9ec049"),
Source("s2", SENSORS_TYPE[1], Point(2, 3), ["humidity"],
"6eb57b7e-43a3-5ad7-a4d1-d1ec54bb5520")
]
TIME_SERIES = [
Record(now, SENSORS[0], Point(0, 1), {"temperature": 14.0}),
Record(now, SENSORS[1], Point(2, 3), {"humidity": 95.0})
]
# the dictionary returned from the tdmq polystore rest api
REST_SOURCE = {
"default_footprint": {
"coordinates":
[SENSORS[0].geometry.latitude, SENSORS[0].geometry.longitude],
"type":
"Point"
},
"entity_type": SENSORS_TYPE[0].name,
"entity_category": SENSORS_TYPE[0].category,
"external_id": SENSORS[0].id_,