def get_time_series(self,
source: Source,
query: Dict[str, Any] = None) -> List[Record]:
try:
time_series = self.http.get(
f'{self.sources_url}/{source.tdmq_id}/timeseries',
params=query)
except HTTPError as e:
logger.error('error response from server with status code %s',
e.response.status_code)
raise_exception(e.response.status_code)
records: List[Record] = []
logger.debug('time_series %s', time_series)
for idx, time in enumerate(time_series['coords']['time']):
date_time = datetime.datetime.fromtimestamp(
time, datetime.timezone.utc)
try:
# No support for MultiPoint, just bring the last coordinate pair
footprint = time_series['coords']['footprint'][idx][
"coordinates"][-1]
except TypeError:
footprint = time_series['default_footprint']['coordinates']
# Point(latitude, longitude) but point are returned as [longitude, latitude]
records.append(
Record(
date_time, source, Point(footprint[1], footprint[0]), {
data: value_list[idx]
for data, value_list in time_series['data'].items()
if value_list
}))
return records
def convert(self, messages: List[str]) -> List[Record]:
series = []
for m in messages:
m = json.loads(m)
series.append(Record(datetime.datetime.now(), uuid.uuid4(),
random.random()))
return series
def _create_record(self, msg: Dict) -> Record:
source_id = self._get_source_id(msg)
sensor_type = self._get_sensor_type(msg)
logging.debug("Converting message of type %s", sensor_type.category)
records: Dict = {}
time = None
geometry = None
sensor_model = None
for attr in msg["body"]["attributes"]:
name, value, type_ = attr["name"], attr["value"], attr["type"]
if not name in self._to_ignore:
if value is None or not str(value).strip():
converter = str
value = ''
else:
# First check for a converter for the attribute
try:
converter = self._attrs_mapper[name]
except KeyError:
converter = self._type_mapper.get(type_, None)
try:
converted_value = converter(value)
except (ValueError, TypeError):
# FIXME: should we skip the message or just the property?
logger.warning(
"cannot read attribute %s of type %s with value %s",
name, type_, value)
continue
else:
if name == "timestamp":
time = converted_value
elif name == "location":
geometry = converted_value
elif name == "modelName":
sensor_model = converted_value
elif name not in self.non_properties:
records[name] = converted_value
if not records:
raise RuntimeError("conversion produced no useful data")
if geometry is None:
raise RuntimeError("missing latitude and/or longitude")
if sensor_model is None:
sensor_model = self._get_sensor_model(source_id, sensor_type)
sensor = self._create_sensor(source_id, sensor_type, sensor_model,
geometry, records.keys())
return Record(time, sensor, geometry, records)
from tdm_ingestion.tdmq.models import EntityType, Point, Record, Source
now = datetime.now(timezone.utc)
SENSORS_TYPE = [EntityType("st1", "cat1"), EntityType("st2", "cat2")]
SENSORS = [
Source("s1", SENSORS_TYPE[0], Point(0, 1), ["temperature"],
"4d9ae10d-df9b-546c-a586-925e1e9ec049"),
Source("s2", SENSORS_TYPE[1], Point(2, 3), ["humidity"],
"6eb57b7e-43a3-5ad7-a4d1-d1ec54bb5520")
]
TIME_SERIES = [
Record(now, SENSORS[0], Point(0, 1), {"temperature": 14.0}),
Record(now, SENSORS[1], Point(2, 3), {"humidity": 95.0})
]
# the dictionary returned from the tdmq polystore rest api
REST_SOURCE = {
"default_footprint": {
"coordinates":
[SENSORS[0].geometry.latitude, SENSORS[0].geometry.longitude],
"type":
"Point"
},
"entity_type": SENSORS_TYPE[0].name,
"entity_category": SENSORS_TYPE[0].category,
"external_id": SENSORS[0].id_,
"stationary": True,