def get_sources(self,
id_: str = None,
query: Dict = None) -> Union[Source, List[Source]]:
logger.debug("getting sources from server")
if id_:
try:
source_data = self.http.get(f'{self.sources_url}/{id_}')
return Source(
id_=source_data['external_id'],
tdmq_id=source_data['tdmq_id'],
type_=EntityType(source_data['entity_type'],
source_data['entity_category']),
station_model=source_data.get('station_model', ''),
geometry=Point(
source_data['default_footprint']['coordinates'][1],
source_data['default_footprint']['coordinates'][0]))
except HTTPError as e:
logger.error('error response from server with status code %s',
e.response.status_code)
raise_exception(e.response.status_code)
try:
return [
Source(
id_=s['external_id'],
tdmq_id=s['tdmq_id'],
type_=EntityType(s['entity_type'], s['entity_category']),
station_model=s.get('station_model', ''),
geometry=Point(s['default_footprint']['coordinates'][1],
s['default_footprint']['coordinates'][0]))
for s in self.http.get(f'{self.sources_url}', params=query)
]
except HTTPError as e:
logger.error('error response from server with status code %s',
e.response.status_code)
raise_exception(e.response.status_code)
def _get_sensor_model(source_id: str, sensor_type: str) -> str:
edge_name, station_name, sensor_name = source_id.split('.')
if sensor_type == EntityType("EnergyConsumptionMonitor", "Station"):
if 'emontx' in station_name.lower():
return 'emonTx'
elif 'iotawatt' in station_name.lower():
return 'IoTaWatt'
else:
return 'IoTaWatt'
elif sensor_type == EntityType("WeatherObserver", "Station"):
if 'emontx' in station_name.lower():
return 'emonTx'
elif 'esp8266-' in station_name.lower():
return 'airRohr'
elif 'airrohr-' in station_name.lower():
return 'airRohr'
elif 'edge' in station_name.lower():
return 'EdgeGateway'
else:
return 'Unknown'
elif sensor_type == EntityType("DeviceStatusMonitor", "Station"):
return 'EdgeGateway'
return None
def test_get_source_by_id(self):
"""
Tests getting a source using the tdmq id
"""
expected_source = Source(
id_=REST_SOURCE["external_id"],
type_=EntityType(REST_SOURCE["entity_type"], REST_SOURCE["entity_category"]),
geometry=Point(*REST_SOURCE["default_footprint"]["coordinates"][::-1]), # for some strange reason the points are inverted
controlled_properties=None,
tdmq_id=REST_SOURCE["tdmq_id"]
)
client = Client(self.url)
httpretty.register_uri(httpretty.GET, f'{client.sources_url}/{SENSORS[0].tdmq_id}',
body=jsons.dumps(REST_SOURCE), match_querystring=False)
res = client.get_sources(REST_SOURCE["tdmq_id"])
self.assertEqual(res.to_json(), expected_source.to_json())
def _test_convert_device(self, converter):
timeseries_list = converter.convert([json.dumps(self.in_device_msg)])
self.assertEqual(len(timeseries_list), 1)
self.assertEqual(timeseries_list[0].source.type,
EntityType("DeviceStatusMonitor", "Station"))
data = {
attr["name"]: self.device_values.get(attr["name"], "")
for attr in self.in_device_msg["body"]["attributes"]
if attr['name'] not in self.ignored_attrs
}
self.assertEqual(timeseries_list[0].data, data)
self.assertEqual(
timeseries_list[0].time.strftime("%Y-%m-%dT%H:%M:%SZ"),
"2019-12-18T14:05:05Z")
self.assertEqual(str(timeseries_list[0].source.id_),
"Edge-z.EDGE.HTU21D")
self.assertEqual(str(timeseries_list[0].source.edge_id), "Edge-z")
self.assertEqual(str(timeseries_list[0].source.station_id),
"Edge-z.EDGE")
self.assertEqual(str(timeseries_list[0].source.sensor_id), "HTU21D")
def _test_convert_energy(self, converter):
timeseries_list = converter.convert([json.dumps(self.in_energy_msg)])
self.assertEqual(len(timeseries_list), 1)
self.assertEqual(timeseries_list[0].source.type,
EntityType("EnergyConsumptionMonitor", "Station"))
data = {
attr["name"]: self.energy_values.get(attr["name"], "")
for attr in self.in_energy_msg["body"]["attributes"]
if attr['name'] not in self.ignored_attrs
}
self.assertEqual(timeseries_list[0].data, data)
self.assertEqual(
timeseries_list[0].time.strftime("%Y-%m-%dT%H:%M:%SZ"),
"2019-12-16T16:33:19Z")
self.assertEqual(str(timeseries_list[0].source.id_),
"Edge-y.emontx3.L3")
self.assertEqual(str(timeseries_list[0].source.edge_id), "Edge-y")
self.assertEqual(str(timeseries_list[0].source.station_id),
"Edge-y.emontx3")
self.assertEqual(str(timeseries_list[0].source.sensor_id), "L3")
def _test_convert_weather(self, converter):
timeseries_list = converter.convert([json.dumps(self.in_weather_msg)])
self.assertEqual(len(timeseries_list), 1)
self.assertEqual(timeseries_list[0].source.type,
EntityType("WeatherObserver", "Station"))
data = {
attr["name"]: self.weather_values.get(attr["name"], "")
for attr in self.in_weather_msg["body"]["attributes"]
if attr['name'] not in self.ignored_attrs
}
self.assertEqual(timeseries_list[0].data, data)
self.assertEqual(
timeseries_list[0].time.strftime("%Y-%m-%dT%H:%M:%SZ"),
"2019-12-16T16:31:34Z")
self.assertEqual(str(timeseries_list[0].source.id_),
"Edge-x.StationX.Davis")
self.assertEqual(str(timeseries_list[0].source.edge_id), "Edge-x")
self.assertEqual(str(timeseries_list[0].source.station_id),
"Edge-x.StationX")
self.assertEqual(str(timeseries_list[0].source.sensor_id), "Davis")
def test_get_all_sources(self):
"""
Tests getting all sources
"""
expected_sources = [
Source(
id_=REST_SOURCE["external_id"],
type_=EntityType(REST_SOURCE["entity_type"], REST_SOURCE["entity_category"]),
geometry=Point(*REST_SOURCE["default_footprint"]["coordinates"][::-1]),
controlled_properties=None,
tdmq_id=REST_SOURCE["tdmq_id"]
)
]
client = Client(self.url)
httpretty.register_uri(httpretty.GET, client.sources_url,
body=jsons.dumps([REST_SOURCE]), match_querystring=False)
res = client.get_sources()
self.assertEqual([s.to_json() for s in res],
[s.to_json() for s in expected_sources])
class NgsiConverter:
"""
Class to convert json-formatted NGSI-Fiware messages to :class:`Record` instances
"""
non_properties = {"dateObserved", "location", "TimeInstant"}
# Legacy attributes replaced by "location"
_to_ignore = {"latitude", "longitude"}
fiware_service_path_to_sensor_type = {
"/cagliari/edge/meteo": EntityType("WeatherObserver", "Station"),
"/cagliari/edge/energy": EntityType("EnergyConsumptionMonitor",
"Station"),
"/cagliari/edge/device": EntityType("DeviceStatusMonitor", "Station"),
}
#: Maps the ngsi attribute types to python types
_type_mapper = {
"String": str,
"Float": float,
"Integer": int,
"geo:point": lambda v: Point(*v.replace(" ", "").split(",")),
"ISO8601": isoparse
}
#: Maps the attributes with a specific handler. It has priority higher than type
_attrs_mapper = {
"timestamp":
lambda v: datetime.datetime.fromtimestamp(int(v), datetime.timezone.utc
),
"dateObserved":
isoparse,
"rssi":
int,
"dewpoint":
float,
"memoryFree":
float
}
_message_id_regex = re.compile(
r"(?P<Type>\w+):(?P<Edge>[a-zA-Z0-9_-]+)\.(?P<Station>[a-zA-Z0-9_-]+)\.(?P<Sensor>[a-zA-Z0-9_-]+)"
)
@staticmethod
def _get_fiware_service_path(msg: Dict):
for header in msg["headers"]:
try:
return header["fiware-servicePath"]
except KeyError:
pass
raise RuntimeError(f"fiware-servicePath not found in msg {msg}")
@staticmethod
def _get_source_id(msg: Dict) -> str:
try:
match = NgsiConverter._message_id_regex.search(msg["body"]["id"])
except KeyError:
raise RuntimeError(f"invalid id {msg['body']['id']}")
else:
if match:
_, edge_name, station_name, sensor_name = match.groups()
source_id = "{}.{}.{}".format(edge_name, station_name,
sensor_name)
return source_id
raise RuntimeError(f"invalid id {msg['body']['id']}")
def _get_sensor_type(self, msg: Dict) -> str:
try:
service_path = self._get_fiware_service_path(msg)
return self.fiware_service_path_to_sensor_type[service_path]
except KeyError:
raise RuntimeError(f"invalid message type {service_path}")
@staticmethod
def _create_sensor(sensor_name: str, sensor_type: EntityType,
geometry: Geometry, properties: List[str]) -> Source:
return Source(sensor_name, sensor_type, geometry, properties)
def _create_record(self, msg: Dict) -> Record:
source_id = self._get_source_id(msg)
sensor_type = self._get_sensor_type(msg)
logging.debug("Converting message of type %s", sensor_type.category)
records: Dict = {}
time = None
geometry = None
for attr in msg["body"]["attributes"]:
name, value, type_ = attr["name"], attr["value"], attr["type"]
if not name in self._to_ignore:
if value is None or not str(value).strip():
converter = str
value = ''
else:
# First check for a converter for the attribute
try:
converter = self._attrs_mapper[name]
except KeyError:
converter = self._type_mapper.get(type_, None)
try:
converted_value = converter(value)
except (ValueError, TypeError):
# FIXME: should we skip the message or just the property?
logger.warning(
"cannot read attribute %s of type %s with value %s",
name, type_, value)
continue
else:
if name == "timestamp":
time = converted_value
elif name == "location":
geometry = converted_value
elif name not in self.non_properties:
records[name] = converted_value
if not records:
raise RuntimeError("conversion produced no useful data")
if geometry is None:
raise RuntimeError("missing latitude and/or longitude")
sensor = self._create_sensor(source_id, sensor_type, geometry,
records.keys())
return Record(time, sensor, geometry, records)
def convert(self, messages: List[str]) -> List[Record]:
"""
Method that reads a list of ngsi messages and convert them in a list of :class:`Record`
"""
logger.debug("messages %s", len(messages))
timeseries_list: List = []
for m in messages:
try:
m_dict = json.loads(m)
logger.debug("Message is %s", m_dict)
timeseries_list.append(self._create_record(m_dict))
except JSONDecodeError:
logger.error("exception decoding message %s", m)
continue
except RuntimeError as e:
logger.error("exception occurred with message %s", m)
logger.error(e)
continue
return timeseries_list
from datetime import datetime, timezone
from tdm_ingestion.tdmq.models import EntityType, Point, Record, Source
now = datetime.now(timezone.utc)
SENSORS_TYPE = [EntityType("st1", "cat1"), EntityType("st2", "cat2")]
SENSORS = [
Source("s1", SENSORS_TYPE[0], Point(0, 1), ["temperature"],
"4d9ae10d-df9b-546c-a586-925e1e9ec049"),
Source("s2", SENSORS_TYPE[1], Point(2, 3), ["humidity"],
"6eb57b7e-43a3-5ad7-a4d1-d1ec54bb5520")
]
TIME_SERIES = [
Record(now, SENSORS[0], Point(0, 1), {"temperature": 14.0}),
Record(now, SENSORS[1], Point(2, 3), {"humidity": 95.0})
]
# the dictionary returned from the tdmq polystore rest api
REST_SOURCE = {
"default_footprint": {
"coordinates":
[SENSORS[0].geometry.latitude, SENSORS[0].geometry.longitude],
"type":
"Point"
},
"entity_type": SENSORS_TYPE[0].name,
"entity_category": SENSORS_TYPE[0].category,
"external_id": SENSORS[0].id_,
from datetime import datetime, timezone
from tdm_ingestion.tdmq.models import EntityType, Point, Record, Source
now = datetime.now(timezone.utc)
SENSORS_TYPE = [
EntityType("st1", "cat1"),
EntityType("st2", "cat2")
]
SENSORS = [
Source("s1", SENSORS_TYPE[0], 'model1', Point(0, 1), ["temperature"],
"4d9ae10d-df9b-546c-a586-925e1e9ec049", "Edge1", "Station1",
"Sensor1"),
Source("s2", SENSORS_TYPE[1], 'model2', Point(2, 3), ["humidity"],
"6eb57b7e-43a3-5ad7-a4d1-d1ec54bb5520", "Edge2", "Station1",
"Sensor1"),
]
TIME_SERIES = [
Record(now, SENSORS[0], Point(0, 1), {"temperature": 14.0}),
Record(now, SENSORS[1], Point(2, 3), {"humidity": 95.0})
]
# the dictionary returned from the tdmq polystore rest api
REST_SOURCE = {
"default_footprint": {
"coordinates": [
SENSORS[0].geometry.latitude,
SENSORS[0].geometry.longitude
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-d',
help='debug',
dest='debug',
action='store_true',
default=False)
parser.add_argument('--tdmq_url', dest='tdmq_url', required=True)
parser.add_argument('--bucket', dest='bucket', required=True, type=float)
parser.add_argument('--op', dest='operation', required=True)
time_delta_mapping = {
'today': TimeDelta.today,
'1h': TimeDelta.one_hour,
'1d': TimeDelta.one_day,
'1w': TimeDelta.one_week,
'1m': TimeDelta.one_month
}
parser.add_argument('--time_delta_before',
dest='time_delta_before',
choices=time_delta_mapping.keys())
parser.add_argument('--before', dest='before')
parser.add_argument('--after', dest='after')
parser.add_argument('--entity_type',
dest='entity_type',
required=True,
choices=[
'PointWeatherObserver', 'WeatherObserver',
'EnergyConsumptionMonitor', 'DeviceStatusMonitor'
])
parser.add_argument('--ckan_url', dest='ckan_url', required=True)
parser.add_argument('--ckan_api_key', dest='ckan_api_key', required=True)
parser.add_argument('--ckan_dataset', dest='ckan_dataset', required=True)
parser.add_argument('--ckan_resource', dest='ckan_resource', required=True)
parser.add_argument(
'--ckan_description',
dest='ckan_description',
help=('description text; %%{after} and %%{before} are populated by '
'first and last day (1w and 1m time deltas only)'),
required=False,
default="")
parser.add_argument('--upsert',
dest='upsert',
default=False,
action='store_true')
parser.add_argument('--prune',
dest='prune',
default=False,
help='when create a resource with time deltas of 1w or'
'1m deletes the related daily reports.',
action='store_true')
args = parser.parse_args()
logging_level = logging.DEBUG if args.debug else logging.INFO
logging.basicConfig(level=logging_level)
logging.info('running with args %s', args.__dict__)
if args.time_delta_before:
time_delta = time_delta_mapping[args.time_delta_before]
before, after = time_delta.get_before_after()
resource_name = after.strftime(args.ckan_resource)
else:
before, after = args.before, args.after
resource_name = args.ckan_resource
description_text = args.ckan_description or ''
if args.ckan_description and args.time_delta_before:
if "%{after}" in description_text:
description_text = description_text.replace(
"%{after}", after.strftime("%Y-%m-%d"))
if "%{before}" in description_text:
description_text = description_text.replace(
"%{before}", before.strftime("%Y-%m-%d"))
consumer = TDMQConsumer(Client(args.tdmq_url))
storage = CkanStorage(
RemoteCkan(args.ckan_url, Requests(), args.ckan_api_key))
storage.write(
consumer.poll(EntityType(args.entity_type), args.bucket,
args.operation, before, after), args.ckan_dataset,
resource_name, description_text, args.upsert)
if args.prune:
if args.time_delta_before == '1w':
storage.prune_resources(args.ckan_dataset,
resource_name,
after,
before,
prune_weekly=False)
elif args.time_delta_before == '1m':
storage.prune_resources(args.ckan_dataset,
resource_name,
after,
before,
prune_weekly=True)