def deleteGraph(self, graphName):
queryString = "DEFINE sql:log-enable 3 DROP SILENT GRAPH <" + self.getGraphURI(graphName) + ">"
L.d("deleteGraph using query:", queryString)
sparql = self.getSparqlObject(graphName, queryString)
sparql.setTimeout(300)
try:
ret = sparql.query()
return True
except Exception as e:
L.e("Error in deleteGraph:", e.message)
return False
def checkTimeRelevantMetrics(self, lastUpdate):
L.d("ReputationSystem: checkTimeRelevantMetrics called for Stream", self.description.fullSensorID)
L.d("ReputationSystem:", lastUpdate, self.timestamp)
if (lastUpdate is not None) and (lastUpdate == self.timestamp): # check if there was an update in the meanwhile
L.d("ReputationSystem: There was no update, lets punish!")
qoiValues = {}
for metric in self.metrics:
value = metric.nonValueUpdate()
if value:
qoiValues[value[0]] = value[1]
self.avgQoIManager.calculateAvgQualities(qoiValues)
self.addClockJob()
def get_observations(self, uuid, start=None, end=None, format='json', onlyLast=False, fields=None, offset=0):
from virtualisation.resourcemanagement.resourcemanagement import ResourceManagement
w = self.rm.getWrapperByUUID(uuid)
if not w:
return None
sd = w.getSensorDescription()
# prepare query
_filter = ["sensor_uuid = '%s'" % uuid]
order = "ORDER BY sampling_time"
limitation = ""
if onlyLast:
order += " DESC"
else:
if start:
_filter.append("sampling_time >= TIMESTAMP '%s'" % start)
if end:
_filter.append("sampling_time <= TIMESTAMP '%s'" % end)
_filter = "WHERE " + " and ".join(_filter)
if fields:
fields = fields.split(',')
fields_ = []
for ft in fields:
fields_.append("data->'%s' AS %s" % (ft, ft))
fields_.append("quality")
else:
fields_ = SQL.cp_observation_fields
limitation = "LIMIT %d" % (1 if onlyLast else self.PAGINATION_LIMIT)
query = "SELECT %s FROM %s.cp_observations %s %s %s OFFSET %d;" % (",".join(fields_), SQL.SCHEMA, _filter, order, limitation, offset)
# query = "SELECT %s FROM %s.cp_observations %s %s;" % (",".join(fields_), SQL.SCHEMA, _filter, order)
L.d("SQL: executing query", query)
try:
# need a new cursor object to no interfere with the state of the class's inserting cursor
cursor = self.conn.cursor()
cursor.execute(query)
data = cursor.fetchall()
data2 = [list(x) for x in data]
del data
if format in ('n3', 'nt', 'xml', 'turtle', 'pretty-xml', 'trix'):
if ResourceManagement.args.messagebus or ResourceManagement.args.triplestore:
if fields:
observations = []
qualities = []
for x in data2:
tmp = JOb()
for i in range(0, len(fields)):
ft = fields[i]
tmp[ft] = JOb(x[i])
tmp.fields = fields
observations.append(tmp)
qualities.append(JOb(x[-1]))
else:
observations = [JOb(x[3]) for x in data2]
qualities = [JOb(x[4]) for x in data2]
g = self.rm.annotator.annotateObservation(observations, sd, None, qualities)
del observations
del qualities
del query
return g.serialize(format=format)
else:
return "Error: requires messagebus or triplestore to be enabled"
else:
# search in all columns in each row for a datetime.datetime and parse it
for i in range(0, len(data2)):
data2[i] = map(lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if isinstance(x, datetime.datetime) else x, data2[i])
json_list = []
for x in data2:
if fields:
# y = JOb({})
y = {}
for i in range(0, len(fields)):
# ft = fields[i]
# y[ft] = JOb(x[i])
y[fields[i]] = x[i]
# y.quality = JOb(x[-1])
# y.fields = fields
y["fields"] = fields
y["quality"] = x[-1]
else:
# y = JOb(x[3])
# y.quality = JOb(x[4])
y = x[3]
y["quality"] = x[4]
json_list.append(y)
del query
del data2
# return JOb(json_list).dumps()
return json_list
except Exception as e:
L.e("SQL:", e)
L.e("SQL query used:", query)
return "Error: " + str(e)
def update(self, data):
self.updatecounter += 1
# special case when no fields are in data
# (fault recovery is not ready yet)
if len(data.fields) == 0:
self.rewardAndPunishment.update(False)
self.absoluteValue = float("inf")
self.ratedValue = self.rewardAndPunishment.value()
return
wrongFieldList = []
for field in data.fields:
if field not in data:
wrongFieldList.append(field)
continue
dataTypeStr = self.repsys.description.field[field].dataType
dataType = utils.getType(dataTypeStr)
minValue, maxValue = self.getMinMaxValue(field, data)
value = data[field].value
# print "field:", field, "value:", value, "min:", minValue, "max:", maxValue, "dataType:", dataTypeStr, dataType, "value type:", type(value)
if minValue and maxValue:
if dataTypeStr == "datetime.datetime":
minValue = datetime.datetime.strptime(minValue, AbstractClock.parserformat)
maxValue = datetime.datetime.strptime(maxValue, AbstractClock.parserformat)
else:
maxValue = dataType(maxValue)
minValue = dataType(minValue)
# everything might be a string => first check for type, then try to cast, afterwards check min and max
wrongValue = False
if not isinstance(value, dataType): # type(value) is not dataType:
try:
# special handling for datetime as format is needed
if dataTypeStr == "datetime.datetime":
value = datetime.datetime.strptime(value, self.repsys.description.field[field].format)
else:
value = dataType(value)
except ValueError:
wrongFieldList.append(field)
wrongValue = True
if not wrongValue:
# now check if value is within min max interval
if minValue and minValue is not "":
if value < minValue:
wrongFieldList.append(field)
elif maxValue and maxValue is not "":
if value > maxValue:
wrongFieldList.append(field)
# print "Correctness for", self.repsys.description.fullSensorID, len(wrongFieldList), value, minValue, maxValue
nrWrongFields = len(wrongFieldList)
if nrWrongFields > 0:
L.d("Correctness wrong fields:", nrWrongFields, "(", ",".join(wrongFieldList), ")")
if data.recovered or (nrWrongFields >= 1):
self.rewardAndPunishment.update(False)
else:
self.rewardAndPunishment.update(True)
self.ratedValue = self.rewardAndPunishment.value()
self.absoluteValue = 1 - nrWrongFields / len(data.fields)
self.min = min(self.min, self.absoluteValue)
self.mean = ((self.updatecounter - 1) * self.mean) / self.updatecounter + float(
self.absoluteValue) / self.updatecounter
correctness = JSONObject()
correctness.wrongFields = wrongFieldList
correctness.absoluteValue = self.absoluteValue
correctness.ratedValue = self.ratedValue
correctness.unit = self.unit
# print "correctness:", self.ratedValue, self.absoluteValue
return (self.name, correctness)
def update(self, data):
# special case when no fields are in data
# (fault recovery is not ready yet)
if len(data.fields) == 0:
self.rewardAndPunishment.update(False)
self.absoluteValue = float("inf")
self.ratedValue = self.rewardAndPunishment.value()
return
# look for expected fields in sensor description, look only for non optional fields
fields = self.repsys.description.fields
fields = [x for x in fields if not self.repsys.description.field[x].optional]
receivedFields = data.fields
# check if expected and received identical, how to handle received fields with no values?
nrOfMissingFields = 0
missingFields = set()
if set(fields).difference(set(receivedFields)):
# lists are different
missingFields = set(fields).difference(set(receivedFields))
nrOfMissingFields = len(missingFields)
# now go through all fields and check for NULL, NA,...
nrOfWrongFields = 0
wrongFields = set()
wrongValues = ['None', 'Null', '', 'NA'] #TODO make the list of wrong values configurable
for field in data.fields:
if field in data:
value = data[field].value
if value is None or value in wrongValues:
nrOfWrongFields += 1
wrongFields.add(field)
else:
nrOfWrongFields += 1
wrongFields.add(field)
if nrOfMissingFields > 0:
L.d("Completeness missing fields:", nrOfMissingFields, "(", ",".join(missingFields), ")")
if nrOfWrongFields > 0:
L.d("Completeness wrong fields:", nrOfWrongFields, "(", ",".join(wrongFields), ")")
length = len(self.repsys.description.fields)
currentLength = length - nrOfMissingFields - nrOfWrongFields
self.updatecounter += 1
if not self.goal:
self.goal = length
self.min = float(length)
self.mean = float(length)
# return (length, self.rewardAndPunishment.value())
else:
self.min = min(self.min, currentLength)
self.mean = ((self.updatecounter - 1) * self.mean) / self.updatecounter + float(
currentLength) / self.updatecounter
if data.recovered:
self.rewardAndPunishment.update(False)
else:
self.rewardAndPunishment.update(self.goal == currentLength)
self.absoluteValue = currentLength
self.ratedValue = self.rewardAndPunishment.value()
completeness = JSONObject()
completeness.missingFields = list(missingFields | wrongFields)
completeness.absoluteValue = self.absoluteValue
completeness.ratedValue = self.ratedValue
completeness.unit = self.unit
# print completeness.dumps()
# print "completeness:", self.name, completeness
# print (self.name, missingFields)
return (self.name, completeness)
def update(self):
from virtualisation.resourcemanagement.resourcemanagement import ResourceManagement
# print "time", self.clock.now()
latStart = datetime.now()
L.d("processing:", self.getSensorDescription().sensorID)
# L.d(self.clock.now())
if self.replaymode:
self.stats.startMeasurement("Update_replay")
# self.clock.pause()
if self.historyreader:
L.d2("abstractwrapper get data")
self.stats.startMeasurement("Update_replay.Historyreader")
data_raw = self.historyreader.tick(self.clock)
self.stats.stopMeasurement("Update_replay.Historyreader")
L.d2("abstractwrapper received data:", str(data_raw))
if data_raw:
data_list = [data_raw] if not self.historyreader.multiple_observations else data_raw
for data in data_list:
try:
L.d2("abstractwrapper parse data")
# print "data to parse", data
self.stats.startMeasurement("Update_replay.Historyparser")
parsed = self.historyparser.parse(data, self.clock)
self.stats.stopMeasurement("Update_replay.Historyparser")
L.d2("abstractwrapper parsed data:", str(parsed))
del data
if parsed:
self.stats.startMeasurement("Update_replay.Preparation")
ObservationIDGenerator.addObservationIDToFields(parsed)
parsed.producedInReplayMode = True
parsed.recovered = False
parsed.latency = (datetime.now() - latStart).total_seconds()
self.stats.stopMeasurement("Update_replay.Preparation")
# QoI Start
quality = None
if self.qoiSystem:
L.d2("abstractwrapper get quality")
self.stats.startMeasurement("Update_replay.Quality")
quality = self.qoiSystem.addData(self.getSensorDescription(), parsed, self.clock)
self.stats.stopMeasurement("Update_replay.Quality")
L.d2("abstractwrapper quality:", quality)
if self.faultRecoveryActive:
L.d2("abstractwrapper update fault recovery")
self.stats.startMeasurement("Update_replay.FaultRecoveryUpdate")
self.updateFaultRecoveries(parsed, quality)
self.stats.stopMeasurement("Update_replay.FaultRecoveryUpdate")
L.d2("abstractwrapper fault recovery updated")
self.stats.startMeasurement("Update_replay.Receiver")
for r in self.receiver:
L.d2("abstractwrapper start receiver", r)
r.receive(parsed, self.getSensorDescription(), self.clock, quality)
L.d2("abstractwrapper receiver", r, "finished")
self.stats.stopMeasurement("Update_replay.Receiver")
except Exception as e:
L.e("Error while updating sensor", self.getSensorDescription().fullSensorID, e)
finally:
if ResourceManagement.args.gentle:
self.clock.sleep()
else:
L.d("there is no data, ask fault recovery1")
# L.i(self.getSensorDescription().sensorID)
# L.i(self.clock.now())
try:
self.stats.startMeasurement("Update_replay.Recovery")
data = JSONObject()
data.latency = 0
data.producedInReplayMode = True
data.recovered = True
data.fields = []
for n in self.getSensorDescription().fields:
if n in self.faultRecoveries and self.faultRecoveries[n].isReady():
data.fields.append(n)
data[n] = JSONObject()
# at this point the dataType is in FAULT_RECOVERY_SUPPORTED_DATATYPES and we can safely use cast
data[n].value = self.faultRecoveryCast(
self.faultRecoveries[n].getEstimation(),
self.getSensorDescription().field[n].dataType,
)
data[n].propertyName = self.getSensorDescription().field[n].propertyName
data[n].propertyURI = self.getSensorDescription().field[n].propertyURI
if "unit" in self.getSensorDescription().field[n]:
data[n].unit = self.getSensorDescription().field[n].unit
data[n].sensorID = self.getSensorDescription().fullSensorID
data[n].observationSamplingTime = self.clock.timeAsString()
data[n].observationResultTime = data[n].observationSamplingTime
self.stats.stopMeasurement("Update_replay.Recovery")
self.stats.startMeasurement("Update_replay.ObservationIDGenerator")
ObservationIDGenerator.addObservationIDToFields(data)
self.stats.stopMeasurement("Update_replay.ObservationIDGenerator")
quality = None
if self.qoiSystem:
self.stats.startMeasurement("Update_replay.Quality")
quality = self.qoiSystem.addData(self.getSensorDescription(), data, self.clock)
self.stats.stopMeasurement("Update_replay.Quality")
self.stats.startMeasurement("Update_replay.Receiver")
for r in self.receiver:
r.receive(data, self.getSensorDescription(), self.clock, quality)
self.stats.stopMeasurement("Update_replay.Receiver")
except Exception as e:
L.e("Error while updating sensor", self.getSensorDescription().fullSensorID, e)
finally:
pass
# if ResourceManagement.args.gentle:
# self.clock.sleep()
else:
pass # no history reader - nothing to do
self.stats.stopMeasurement("Update_replay")
else: # no replay mode
self.stats.startMeasurement("Update_live")
if self.connection:
try:
self.stats.startMeasurement("Update_live.Connection")
data_raw = self.connection.next()
self.stats.stopMeasurement("Update_live.Connection")
if data_raw:
data_list = [data_raw] if not self.connection.multiple_observations else data_raw
for data in data_list:
self.stats.startMeasurement("Update_live.Parser")
parsed = self.parser.parse(data, self.clock)
self.stats.stopMeasurement("Update_live.Parser")
if parsed:
self.stats.startMeasurement("Update_live.Preparation")
ObservationIDGenerator.addObservationIDToFields(parsed)
parsed.producedInReplayMode = False
parsed.recovered = False
parsed.latency = (datetime.now() - latStart).total_seconds()
self.stats.stopMeasurement("Update_live.Preparation")
# QoI Start
quality = None
if self.qoiSystem:
# TODO update the timestamp
self.stats.startMeasurement("Update_live.Quality")
quality = self.qoiSystem.addData(self.getSensorDescription(), parsed, self.clock)
self.stats.stopMeasurement("Update_live.Quality")
if self.faultRecoveryActive:
L.d2("abstractwrapper update fault recovery")
self.stats.startMeasurement("Update_live.FaultRecoveryUpdate")
self.updateFaultRecoveries(parsed, quality)
self.stats.stopMeasurement("Update_live.FaultRecoveryUpdate")
L.d2("abstractwrapper fault recovery updated")
self.stats.startMeasurement("Update_live.Receiver")
for r in self.receiver:
r.receive(parsed, self.getSensorDescription(), self.clock, quality)
self.stats.stopMeasurement("Update_live.Receiver")
else:
# fault recovery
L.i("there is no data, ask fault recovery2")
try:
self.stats.startMeasurement("Update_live.Recovery")
data = JSONObject()
data.latency = 0
data.recovered = True
data.fields = []
for n in self.getSensorDescription().fields:
if n in self.faultRecoveries and self.faultRecoveries[n].isReady():
data.fields.append(n)
data[n] = JSONObject()
data[n].value = self.faultRecoveryCast(
self.faultRecoveries[n].getEstimation(),
self.getSensorDescription().field[n].dataType,
)
data[n].propertyName = self.getSensorDescription().field[n].propertyName
data[n].propertyURI = self.getSensorDescription().field[n].propertyURI
if "unit" in self.getSensorDescription().field[n]:
data[n].unit = self.getSensorDescription().field[n].unit
data[n].sensorID = self.getSensorDescription().fullSensorID
data[n].observationSamplingTime = self.clock.timeAsString()
data[n].observationResultTime = data[n].observationSamplingTime
self.stats.stopMeasurement("Update_live.Recovery")
ObservationIDGenerator.addObservationIDToFields(data)
quality = None
if self.qoiSystem:
self.stats.startMeasurement("Update_live.Quality")
quality = self.qoiSystem.addData(self.getSensorDescription(), data, self.clock)
self.stats.stopMeasurement("Update_live.Quality")
self.stats.startMeasurement("Update_live.Receiver")
for r in self.receiver:
r.receive(data, self.getSensorDescription(), self.clock, quality)
self.stats.stopMeasurement("Update_live.Receiver")
except Exception as e:
L.e(
"Error while updating sensor (fault recovery)",
self.getSensorDescription().fullSensorID,
str(e),
)
finally:
pass
# if ResourceManagement.args.gentle:
# self.clock.sleep()
except Exception as e:
L.e(
"Error while updating sensor (not fault recovery)",
self.getSensorDescription().fullSensorID,
str(e),
)
else:
pass # no live mode supported
self.stats.stopMeasurement("Update_live")
