Example #1
0
 def get_sources(self,
                 id_: str = None,
                 query: Dict = None) -> Union[Source, List[Source]]:
     logger.debug("getting sources from server")
     if id_:
         try:
             source_data = self.http.get(f'{self.sources_url}/{id_}')
             return Source(
                 id_=source_data['external_id'],
                 tdmq_id=source_data['tdmq_id'],
                 type_=EntityType(source_data['entity_type'],
                                  source_data['entity_category']),
                 station_model=source_data.get('station_model', ''),
                 geometry=Point(
                     source_data['default_footprint']['coordinates'][1],
                     source_data['default_footprint']['coordinates'][0]))
         except HTTPError as e:
             logger.error('error response from server with status code %s',
                          e.response.status_code)
             raise_exception(e.response.status_code)
     try:
         return [
             Source(
                 id_=s['external_id'],
                 tdmq_id=s['tdmq_id'],
                 type_=EntityType(s['entity_type'], s['entity_category']),
                 station_model=s.get('station_model', ''),
                 geometry=Point(s['default_footprint']['coordinates'][1],
                                s['default_footprint']['coordinates'][0]))
             for s in self.http.get(f'{self.sources_url}', params=query)
         ]
     except HTTPError as e:
         logger.error('error response from server with status code %s',
                      e.response.status_code)
         raise_exception(e.response.status_code)
Example #2
0
    def get_time_series(self,
                        source: Source,
                        query: Dict[str, Any] = None) -> List[Record]:
        try:
            time_series = self.http.get(
                f'{self.sources_url}/{source.tdmq_id}/timeseries',
                params=query)
        except HTTPError as e:
            logger.error('error response from server with status code %s',
                         e.response.status_code)
            raise_exception(e.response.status_code)

        records: List[Record] = []
        logger.debug('time_series %s', time_series)
        for idx, time in enumerate(time_series['coords']['time']):
            date_time = datetime.datetime.fromtimestamp(
                time, datetime.timezone.utc)

            try:
                # No support for MultiPoint, just bring the last coordinate pair
                footprint = time_series['coords']['footprint'][idx][
                    "coordinates"][-1]
            except TypeError:
                footprint = time_series['default_footprint']['coordinates']
            # Point(latitude, longitude) but point are returned as [longitude, latitude]
            records.append(
                Record(
                    date_time, source, Point(footprint[1], footprint[0]), {
                        data: value_list[idx]
                        for data, value_list in time_series['data'].items()
                        if value_list
                    }))

        return records
Example #3
0
    def test_get_source_by_id(self):
        """
        Tests getting a source using the tdmq id
        """
        expected_source = Source(
            id_=REST_SOURCE["external_id"],
            type_=EntityType(REST_SOURCE["entity_type"], REST_SOURCE["entity_category"]),
            geometry=Point(*REST_SOURCE["default_footprint"]["coordinates"][::-1]),  # for some strange reason the points are inverted
            controlled_properties=None,
            tdmq_id=REST_SOURCE["tdmq_id"]
        )

        client = Client(self.url)
        httpretty.register_uri(httpretty.GET, f'{client.sources_url}/{SENSORS[0].tdmq_id}',
                               body=jsons.dumps(REST_SOURCE), match_querystring=False)

        res = client.get_sources(REST_SOURCE["tdmq_id"])
        self.assertEqual(res.to_json(), expected_source.to_json())
Example #4
0
    def test_get_all_sources(self):
        """
        Tests getting all sources
        """
        expected_sources = [
            Source(
                id_=REST_SOURCE["external_id"],
                type_=EntityType(REST_SOURCE["entity_type"], REST_SOURCE["entity_category"]),
                geometry=Point(*REST_SOURCE["default_footprint"]["coordinates"][::-1]),
                controlled_properties=None,
                tdmq_id=REST_SOURCE["tdmq_id"]
            )
        ]

        client = Client(self.url)
        httpretty.register_uri(httpretty.GET, client.sources_url,
                               body=jsons.dumps([REST_SOURCE]), match_querystring=False)

        res = client.get_sources()
        self.assertEqual([s.to_json() for s in res],
                         [s.to_json() for s in expected_sources])
Example #5
0
class NgsiConverter:
    """
    Class to convert json-formatted NGSI-Fiware messages to :class:`Record` instances
    """

    non_properties = {"dateObserved", "location", "TimeInstant"}
    # Legacy attributes replaced by "location"
    _to_ignore = {"latitude", "longitude"}

    fiware_service_path_to_sensor_type = {
        "/cagliari/edge/meteo": EntityType("WeatherObserver", "Station"),
        "/cagliari/edge/energy": EntityType("EnergyConsumptionMonitor",
                                            "Station"),
        "/cagliari/edge/device": EntityType("DeviceStatusMonitor", "Station"),
    }

    #: Maps the ngsi attribute types to python types
    _type_mapper = {
        "String": str,
        "Float": float,
        "Integer": int,
        "geo:point": lambda v: Point(*v.replace(" ", "").split(",")),
        "ISO8601": isoparse
    }

    #: Maps the attributes with a specific handler. It has priority higher than type
    _attrs_mapper = {
        "timestamp":
        lambda v: datetime.datetime.fromtimestamp(int(v), datetime.timezone.utc
                                                  ),
        "dateObserved":
        isoparse,
        "rssi":
        int,
        "dewpoint":
        float,
        "memoryFree":
        float
    }

    _message_id_regex = re.compile(
        r"(?P<Type>\w+):(?P<Edge>[a-zA-Z0-9_-]+)\.(?P<Station>[a-zA-Z0-9_-]+)\.(?P<Sensor>[a-zA-Z0-9_-]+)"
    )

    @staticmethod
    def _get_fiware_service_path(msg: Dict):
        for header in msg["headers"]:
            try:
                return header["fiware-servicePath"]
            except KeyError:
                pass
        raise RuntimeError(f"fiware-servicePath not found in msg {msg}")

    @staticmethod
    def _get_source_id(msg: Dict) -> str:
        try:
            match = NgsiConverter._message_id_regex.search(msg["body"]["id"])
        except KeyError:
            raise RuntimeError(f"invalid id {msg['body']['id']}")
        else:
            if match:
                _, edge_name, station_name, sensor_name = match.groups()
                source_id = "{}.{}.{}".format(edge_name, station_name,
                                              sensor_name)
                return source_id

        raise RuntimeError(f"invalid id {msg['body']['id']}")

    def _get_sensor_type(self, msg: Dict) -> str:
        try:
            service_path = self._get_fiware_service_path(msg)
            return self.fiware_service_path_to_sensor_type[service_path]
        except KeyError:
            raise RuntimeError(f"invalid message type {service_path}")

    @staticmethod
    def _create_sensor(sensor_name: str, sensor_type: EntityType,
                       geometry: Geometry, properties: List[str]) -> Source:
        return Source(sensor_name, sensor_type, geometry, properties)

    def _create_record(self, msg: Dict) -> Record:
        source_id = self._get_source_id(msg)
        sensor_type = self._get_sensor_type(msg)

        logging.debug("Converting message of type %s", sensor_type.category)

        records: Dict = {}
        time = None
        geometry = None
        for attr in msg["body"]["attributes"]:
            name, value, type_ = attr["name"], attr["value"], attr["type"]
            if not name in self._to_ignore:
                if value is None or not str(value).strip():
                    converter = str
                    value = ''
                else:
                    # First check for a converter for the attribute
                    try:
                        converter = self._attrs_mapper[name]
                    except KeyError:
                        converter = self._type_mapper.get(type_, None)

                try:
                    converted_value = converter(value)
                except (ValueError, TypeError):
                    # FIXME: should we skip the message or just the property?
                    logger.warning(
                        "cannot read attribute %s of type %s with value %s",
                        name, type_, value)
                    continue
                else:
                    if name == "timestamp":
                        time = converted_value
                    elif name == "location":
                        geometry = converted_value
                    elif name not in self.non_properties:
                        records[name] = converted_value

        if not records:
            raise RuntimeError("conversion produced no useful data")

        if geometry is None:
            raise RuntimeError("missing latitude and/or longitude")

        sensor = self._create_sensor(source_id, sensor_type, geometry,
                                     records.keys())

        return Record(time, sensor, geometry, records)

    def convert(self, messages: List[str]) -> List[Record]:
        """
        Method that reads a list of ngsi messages and convert them in a list of :class:`Record`
        """
        logger.debug("messages %s", len(messages))
        timeseries_list: List = []
        for m in messages:
            try:
                m_dict = json.loads(m)
                logger.debug("Message is %s", m_dict)
                timeseries_list.append(self._create_record(m_dict))
            except JSONDecodeError:
                logger.error("exception decoding message %s", m)
                continue
            except RuntimeError as e:
                logger.error("exception occurred with message %s", m)
                logger.error(e)
                continue
        return timeseries_list
Example #6
0
from datetime import datetime, timezone

from tdm_ingestion.tdmq.models import EntityType, Point, Record, Source

now = datetime.now(timezone.utc)

SENSORS_TYPE = [EntityType("st1", "cat1"), EntityType("st2", "cat2")]

SENSORS = [
    Source("s1", SENSORS_TYPE[0], Point(0, 1), ["temperature"],
           "4d9ae10d-df9b-546c-a586-925e1e9ec049"),
    Source("s2", SENSORS_TYPE[1], Point(2, 3), ["humidity"],
           "6eb57b7e-43a3-5ad7-a4d1-d1ec54bb5520")
]

TIME_SERIES = [
    Record(now, SENSORS[0], Point(0, 1), {"temperature": 14.0}),
    Record(now, SENSORS[1], Point(2, 3), {"humidity": 95.0})
]

# the dictionary returned from the tdmq polystore rest api
REST_SOURCE = {
    "default_footprint": {
        "coordinates":
        [SENSORS[0].geometry.latitude, SENSORS[0].geometry.longitude],
        "type":
        "Point"
    },
    "entity_type": SENSORS_TYPE[0].name,
    "entity_category": SENSORS_TYPE[0].category,
    "external_id": SENSORS[0].id_,