def update(self, data):
# special case when no fields are in data
# (fault recovery is not ready yet)
if len(data.fields) == 0:
self.rewardAndPunishment.update(False)
self.absoluteValue = float("inf")
self.ratedValue = self.rewardAndPunishment.value()
return
# look for expected fields in sensor description, look only for non optional fields
fields = self.repsys.description.fields
fields = [x for x in fields if not self.repsys.description.field[x].optional]
receivedFields = data.fields
# check if expected and received identical, how to handle received fields with no values?
nrOfMissingFields = 0
missingFields = set()
if set(fields).difference(set(receivedFields)):
# lists are different
missingFields = set(fields).difference(set(receivedFields))
nrOfMissingFields = len(missingFields)
# now go through all fields and check for NULL, NA,...
nrOfWrongFields = 0
wrongFields = set()
wrongValues = ['None', 'Null', '', 'NA'] #TODO make the list of wrong values configurable
for field in data.fields:
if field in data:
value = data[field].value
if value is None or value in wrongValues:
nrOfWrongFields += 1
wrongFields.add(field)
else:
nrOfWrongFields += 1
wrongFields.add(field)
if nrOfMissingFields > 0:
L.d("Completeness missing fields:", nrOfMissingFields, "(", ",".join(missingFields), ")")
if nrOfWrongFields > 0:
L.d("Completeness wrong fields:", nrOfWrongFields, "(", ",".join(wrongFields), ")")
length = len(self.repsys.description.fields)
currentLength = length - nrOfMissingFields - nrOfWrongFields
self.updatecounter += 1
if not self.goal:
self.goal = length
self.min = float(length)
self.mean = float(length)
# return (length, self.rewardAndPunishment.value())
else:
self.min = min(self.min, currentLength)
self.mean = ((self.updatecounter - 1) * self.mean) / self.updatecounter + float(
currentLength) / self.updatecounter
if data.recovered:
self.rewardAndPunishment.update(False)
else:
self.rewardAndPunishment.update(self.goal == currentLength)
self.absoluteValue = currentLength
self.ratedValue = self.rewardAndPunishment.value()
completeness = JSONObject()
completeness.missingFields = list(missingFields | wrongFields)
completeness.absoluteValue = self.absoluteValue
completeness.ratedValue = self.ratedValue
completeness.unit = self.unit
# print completeness.dumps()
# print "completeness:", self.name, completeness
# print (self.name, missingFields)
return (self.name, completeness)
