def aggregate(self, data, sensordescription):
"""
this method is called when stream data has been annotated.
:param data:
:param sensordescription:
:return: The aggregated data
"""
if not data.fields:
return None
result = []
try:
aggregation_objs = self.aggregation_objs[sensordescription.uuid]
for key, agg in aggregation_objs.items():
agg_result = agg.control(data[key])
if agg_result:
g, start, end, size = agg_result
r = JSONObject()
r.graph = g
r.field = sensordescription.field[key]
r.sensorID = sensordescription.sensorID
r.propertyName = sensordescription.field[key].propertyName
r.category = sensordescription.sensorType
r.aggregationMethod = sensordescription.field[key].aggregationMethod
r.aggregationConfiguration = sensordescription.field[key].aggregationConfiguration
r.start = start
r.end = end
r.size = size
result.append(r)
return result
except Exception as e:
Log.e("aggregation failed due to Exception", e)
return None
def parse(self, data, clock):
if not data: # nothing received or nothing in the history -> nothing to parse
return None
if self.lenNames > len(data):
raise Exception("CSVParser: not enough data fields")
result = JOb()
sd = self.wrapper.getSensorDescription()
result.fields = sd.fields
# print data
for i in range(0, self.lenNames):
n = self.names[i]
if n in sd.fields:
result[n] = JOb()
result[n].propertyName = sd.field[n].propertyName
result[n].propertyURI = sd.field[n].propertyURI
if "unit" in sd.field[n]:
result[n].unit = sd.field[n].unit
result[n].observationSamplingTime = clock.timeAsString()
result[n].sensorID = sd.fullSensorID
if self.timestampcell >= 0:
result[n].observationResultTime = sd.parseTimestamp(data[self.timestampcell]).strftime(AbstractClock.format)
else:
result[n].observationResultTime = result[n].observationSamplingTime
if sd.field[n].dataType == "datetime.datetime":
result[n].value = dt.datetime.strptime(data[i], sd.field[n].format)
else:
result[n].value = str2Type(u(data[i]), sd.field[n].dataType)
return result
def aggregate(self, data, sensordescription):
result = []
try:
dftobjs = self.dftobjects[sensordescription.uuid]
for f in dftobjs:
g = dftobjs[f].control(data[f])
if g:
r = JSONObject()
r.graph = g
r.sensorID = sensordescription.sensorID
r.propertyType = sensordescription.field[f].propertyName
r.category = sensordescription.sensorType
result.append(r)
return result
except KeyError:
Log.e("Dft aggregation failed")
return None
def parse(self, data, clock):
if not data: # nothing received or nothing in the history -> nothing to parse
return None
if isinstance(data, str) or isinstance(data, unicode):
data = JOb(u(data))
result = JOb()
sd = self.wrapper.getSensorDescription()
result.fields = sd.fields
for fieldname in sd.fields:
result[fieldname] = JOb()
result[fieldname].propertyName = sd.field[fieldname].propertyName
result[fieldname].propertyURI = sd.field[fieldname].propertyURI
if "unit" in sd.field[fieldname]:
result[fieldname].unit = sd.field[fieldname].unit
result[fieldname].sensorID = sd.fullSensorID
result[fieldname].observationSamplingTime = clock.timeAsString()
if self.timestampfield and self.timestampfield in data:
result[fieldname].observationResultTime = sd.parseTimestamp(data[self.timestampfield]).strftime(
AbstractClock.format
)
else:
result[fieldname].observationResultTime = result[fieldname].observationSamplingTime
if sd.field[fieldname].dataType == "datetime.datetime":
if sd.field[fieldname].format.startswith("UNIX"):
result[fieldname].value = sd.parseTimestamp(data[fieldname]) if fieldname in data else None
else:
result[fieldname].value = (
dt.datetime.strptime(data[fieldname], sd.field[fieldname].format) if fieldname in data else None
)
else:
result[fieldname].value = (
str2Type(u(data[fieldname]), sd.field[fieldname].dataType) if fieldname in data else None
)
return result
def aggregate(self, data, sensordescription):
if not data.fields:
# Log.e("There was no data available so it could not be aggregated")
return None
result = []
try:
saxobjs = self.aggregation_objs[sensordescription.uuid]
for f in saxobjs:
g = saxobjs[f].control(data[f])
if g:
r = JSONObject()
r.graph = g
r.sensorID = sensordescription.sensorID
r.propertyType = sensordescription.field[f].propertyName
r.category = sensordescription.sensorType
result.append(r)
del data
if result is not None and len(result) is not 0:
Log.d2('SaxAggregator: Aggregation successful %s' % str(result))
return result
except Exception as e:
Log.e("aggregation failed due to Exception", e)
return None
def update(self):
from virtualisation.resourcemanagement.resourcemanagement import ResourceManagement
# print "time", self.clock.now()
latStart = datetime.now()
L.d("processing:", self.getSensorDescription().sensorID)
# L.d(self.clock.now())
if self.replaymode:
self.stats.startMeasurement("Update_replay")
# self.clock.pause()
if self.historyreader:
L.d2("abstractwrapper get data")
self.stats.startMeasurement("Update_replay.Historyreader")
data_raw = self.historyreader.tick(self.clock)
self.stats.stopMeasurement("Update_replay.Historyreader")
L.d2("abstractwrapper received data:", str(data_raw))
if data_raw:
data_list = [data_raw] if not self.historyreader.multiple_observations else data_raw
for data in data_list:
try:
L.d2("abstractwrapper parse data")
# print "data to parse", data
self.stats.startMeasurement("Update_replay.Historyparser")
parsed = self.historyparser.parse(data, self.clock)
self.stats.stopMeasurement("Update_replay.Historyparser")
L.d2("abstractwrapper parsed data:", str(parsed))
del data
if parsed:
self.stats.startMeasurement("Update_replay.Preparation")
ObservationIDGenerator.addObservationIDToFields(parsed)
parsed.producedInReplayMode = True
parsed.recovered = False
parsed.latency = (datetime.now() - latStart).total_seconds()
self.stats.stopMeasurement("Update_replay.Preparation")
# QoI Start
quality = None
if self.qoiSystem:
L.d2("abstractwrapper get quality")
self.stats.startMeasurement("Update_replay.Quality")
quality = self.qoiSystem.addData(self.getSensorDescription(), parsed, self.clock)
self.stats.stopMeasurement("Update_replay.Quality")
L.d2("abstractwrapper quality:", quality)
if self.faultRecoveryActive:
L.d2("abstractwrapper update fault recovery")
self.stats.startMeasurement("Update_replay.FaultRecoveryUpdate")
self.updateFaultRecoveries(parsed, quality)
self.stats.stopMeasurement("Update_replay.FaultRecoveryUpdate")
L.d2("abstractwrapper fault recovery updated")
self.stats.startMeasurement("Update_replay.Receiver")
for r in self.receiver:
L.d2("abstractwrapper start receiver", r)
r.receive(parsed, self.getSensorDescription(), self.clock, quality)
L.d2("abstractwrapper receiver", r, "finished")
self.stats.stopMeasurement("Update_replay.Receiver")
except Exception as e:
L.e("Error while updating sensor", self.getSensorDescription().fullSensorID, e)
finally:
if ResourceManagement.args.gentle:
self.clock.sleep()
else:
L.d("there is no data, ask fault recovery1")
# L.i(self.getSensorDescription().sensorID)
# L.i(self.clock.now())
try:
self.stats.startMeasurement("Update_replay.Recovery")
data = JSONObject()
data.latency = 0
data.producedInReplayMode = True
data.recovered = True
data.fields = []
for n in self.getSensorDescription().fields:
if n in self.faultRecoveries and self.faultRecoveries[n].isReady():
data.fields.append(n)
data[n] = JSONObject()
# at this point the dataType is in FAULT_RECOVERY_SUPPORTED_DATATYPES and we can safely use cast
data[n].value = self.faultRecoveryCast(
self.faultRecoveries[n].getEstimation(),
self.getSensorDescription().field[n].dataType,
)
data[n].propertyName = self.getSensorDescription().field[n].propertyName
data[n].propertyURI = self.getSensorDescription().field[n].propertyURI
if "unit" in self.getSensorDescription().field[n]:
data[n].unit = self.getSensorDescription().field[n].unit
data[n].sensorID = self.getSensorDescription().fullSensorID
data[n].observationSamplingTime = self.clock.timeAsString()
data[n].observationResultTime = data[n].observationSamplingTime
self.stats.stopMeasurement("Update_replay.Recovery")
self.stats.startMeasurement("Update_replay.ObservationIDGenerator")
ObservationIDGenerator.addObservationIDToFields(data)
self.stats.stopMeasurement("Update_replay.ObservationIDGenerator")
quality = None
if self.qoiSystem:
self.stats.startMeasurement("Update_replay.Quality")
quality = self.qoiSystem.addData(self.getSensorDescription(), data, self.clock)
self.stats.stopMeasurement("Update_replay.Quality")
self.stats.startMeasurement("Update_replay.Receiver")
for r in self.receiver:
r.receive(data, self.getSensorDescription(), self.clock, quality)
self.stats.stopMeasurement("Update_replay.Receiver")
except Exception as e:
L.e("Error while updating sensor", self.getSensorDescription().fullSensorID, e)
finally:
pass
# if ResourceManagement.args.gentle:
# self.clock.sleep()
else:
pass # no history reader - nothing to do
self.stats.stopMeasurement("Update_replay")
else: # no replay mode
self.stats.startMeasurement("Update_live")
if self.connection:
try:
self.stats.startMeasurement("Update_live.Connection")
data_raw = self.connection.next()
self.stats.stopMeasurement("Update_live.Connection")
if data_raw:
data_list = [data_raw] if not self.connection.multiple_observations else data_raw
for data in data_list:
self.stats.startMeasurement("Update_live.Parser")
parsed = self.parser.parse(data, self.clock)
self.stats.stopMeasurement("Update_live.Parser")
if parsed:
self.stats.startMeasurement("Update_live.Preparation")
ObservationIDGenerator.addObservationIDToFields(parsed)
parsed.producedInReplayMode = False
parsed.recovered = False
parsed.latency = (datetime.now() - latStart).total_seconds()
self.stats.stopMeasurement("Update_live.Preparation")
# QoI Start
quality = None
if self.qoiSystem:
# TODO update the timestamp
self.stats.startMeasurement("Update_live.Quality")
quality = self.qoiSystem.addData(self.getSensorDescription(), parsed, self.clock)
self.stats.stopMeasurement("Update_live.Quality")
if self.faultRecoveryActive:
L.d2("abstractwrapper update fault recovery")
self.stats.startMeasurement("Update_live.FaultRecoveryUpdate")
self.updateFaultRecoveries(parsed, quality)
self.stats.stopMeasurement("Update_live.FaultRecoveryUpdate")
L.d2("abstractwrapper fault recovery updated")
self.stats.startMeasurement("Update_live.Receiver")
for r in self.receiver:
r.receive(parsed, self.getSensorDescription(), self.clock, quality)
self.stats.stopMeasurement("Update_live.Receiver")
else:
# fault recovery
L.i("there is no data, ask fault recovery2")
try:
self.stats.startMeasurement("Update_live.Recovery")
data = JSONObject()
data.latency = 0
data.recovered = True
data.fields = []
for n in self.getSensorDescription().fields:
if n in self.faultRecoveries and self.faultRecoveries[n].isReady():
data.fields.append(n)
data[n] = JSONObject()
data[n].value = self.faultRecoveryCast(
self.faultRecoveries[n].getEstimation(),
self.getSensorDescription().field[n].dataType,
)
data[n].propertyName = self.getSensorDescription().field[n].propertyName
data[n].propertyURI = self.getSensorDescription().field[n].propertyURI
if "unit" in self.getSensorDescription().field[n]:
data[n].unit = self.getSensorDescription().field[n].unit
data[n].sensorID = self.getSensorDescription().fullSensorID
data[n].observationSamplingTime = self.clock.timeAsString()
data[n].observationResultTime = data[n].observationSamplingTime
self.stats.stopMeasurement("Update_live.Recovery")
ObservationIDGenerator.addObservationIDToFields(data)
quality = None
if self.qoiSystem:
self.stats.startMeasurement("Update_live.Quality")
quality = self.qoiSystem.addData(self.getSensorDescription(), data, self.clock)
self.stats.stopMeasurement("Update_live.Quality")
self.stats.startMeasurement("Update_live.Receiver")
for r in self.receiver:
r.receive(data, self.getSensorDescription(), self.clock, quality)
self.stats.stopMeasurement("Update_live.Receiver")
except Exception as e:
L.e(
"Error while updating sensor (fault recovery)",
self.getSensorDescription().fullSensorID,
str(e),
)
finally:
pass
# if ResourceManagement.args.gentle:
# self.clock.sleep()
except Exception as e:
L.e(
"Error while updating sensor (not fault recovery)",
self.getSensorDescription().fullSensorID,
str(e),
)
else:
pass # no live mode supported
self.stats.stopMeasurement("Update_live")