def get_result(self):
metric_result = MetricResult()
metric_result.name = self.name
metric_result.started = self.started # FIXME remove
metric_result.finished = self.finished # FIXME remove
metric_result.series = []
metric_result.data = None
metric_result.groundtruth = self.groundtruth
metric_result.groundtruth_epsilon = self.groundtruth_epsilon
# assign default value
metric_result.groundtruth_result = None
metric_result.groundtruth_error_message = None
if metric_result.started and metric_result.finished: # we check if the testblock was ever started and stopped
# calculate metric data
if self.series_mode != None:
metric_result.series = self.series
metric_result.data = self.series[
-1] # take last element from self.series
metric_result.min = metrics_helper.get_min(self.series)
metric_result.max = metrics_helper.get_max(self.series)
metric_result.mean = metrics_helper.get_mean(self.series)
metric_result.std = metrics_helper.get_std(self.series)
# fill details as KeyValue messages
details = []
details.append(KeyValue("root_frame", self.root_frame))
details.append(KeyValue("measured_frame", self.measured_frame))
metric_result.details = details
# evaluate metric data
if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None:
if math.fabs(metric_result.groundtruth -
metric_result.data.data
) <= metric_result.groundtruth_epsilon:
metric_result.groundtruth_result = True
metric_result.groundtruth_error_message = "all OK"
else:
metric_result.groundtruth_result = False
metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f" % (
metric_result.data.data, metric_result.groundtruth,
metric_result.groundtruth_epsilon)
if metric_result.data == None:
metric_result.groundtruth_result = False
metric_result.groundtruth_error_message = "no result"
return metric_result
def aggregate_results(self, atf_result):
test_list = self.configuration_parser.get_test_list()
ret = self.configuration_parser.get_sorted_plot_dicts(
atf_result, "", "", "")
mbt = ret['mbt']
mbt_aggregated = {}
for metric in list(mbt.keys()):
#print "m=", metric
if metric not in list(mbt_aggregated.keys()):
mbt_aggregated[metric] = {}
for testblock in list(mbt[metric].keys()):
#print " b=", testblock
if testblock not in list(mbt_aggregated[metric].keys()):
mbt_aggregated[metric][testblock] = {}
for tl_tests in test_list:
#print "tl_tests=", tl_tests
for tl_test in list(tl_tests.keys()):
#print " tl_test=", tl_test
metric_result = MetricResult()
status = TestblockStatus.SUCCEEDED
groundtruth_result = Groundtruth.SUCCEEDED
groundtruth_error_message = ""
details = []
for test in list(mbt[metric][testblock].keys()):
if test.startswith(tl_test):
# aggregate status SUCCEEDED from every metric_result
if mbt[metric][testblock][
test].status != TestblockStatus.SUCCEEDED:
status = TestblockStatus.ERROR
# aggregate data from every metric_result
data = mbt[metric][testblock][test].data
stamp = data.stamp
# check if data is set (not all default values anymore)
if data.stamp == rospy.Time(
0) and data.data == 0:
stamp = rospy.Time(
0
) # mark metric result as invalid by settimg timestamp to zero
metric_result.series.append(data)
# aggregate groundtruth from every metric_result
groundtruth = mbt[metric][testblock][
test].groundtruth
if groundtruth.result != Groundtruth.SUCCEEDED:
groundtruth_result = Groundtruth.FAILED
if groundtruth_error_message != "":
groundtruth_error_message += "\n"
groundtruth_error_message += "groundtruth missmatch in subtest %s" % (
test)
# aggregate details from every metric_result
details = details + mbt[metric][testblock][
test].details
if len(metric_result.series
) == 0: # no matching substest found
continue
metric_result.groundtruth = groundtruth
metric_result.groundtruth.result = groundtruth_result
metric_result.groundtruth.error_message = groundtruth_error_message
metric_result.name = mbt[metric][testblock][test].name
metric_result.unit = mbt[metric][testblock][test].unit
metric_result.mode = MetricResult.SPAN_MEAN # aggregated metrics are always SPAN_MEAN
metric_result.status = status
# metric_result.series is set above
metric_result.data.stamp = stamp
metric_result.data.data = metrics_helper.get_mean(
metric_result.series)
metric_result.min = metrics_helper.get_min(
metric_result.series)
metric_result.max = metrics_helper.get_max(
metric_result.series)
metric_result.mean = metric_result.data.data
metric_result.std = metrics_helper.get_std(
metric_result.series)
# metric_result.groundtruth is set above
metric_result.details = details
mbt_aggregated[metric][testblock][
tl_test] = metric_result
# convert mbt to tbm
tbm = {}
for metric in list(mbt_aggregated.keys()):
#print "m=", metric
for testblock in list(mbt_aggregated[metric].keys()):
#print " b=", testblock
for test in list(mbt_aggregated[metric][testblock].keys()):
#print " t=", test
if test not in list(tbm.keys()):
tbm[test] = {}
if testblock not in list(tbm[test].keys()):
tbm[test][testblock] = {}
tbm[test][testblock][metric] = mbt_aggregated[metric][
testblock][test]
# convert tbm to atf_result_aggregated
atf_result_aggregated = AtfResult()
atf_result_aggregated.header = atf_result.header
atf_result_aggregated.name = atf_result.name
atf_result_aggregated.result = True
for test in sorted(tbm.keys()):
test_result = TestResult()
test_result.name = test
test_result.result = True
# find test metadata in atf_result
for t in atf_result.results:
if t.name.startswith(test):
test_result.test_config = t.test_config
test_result.robot = t.robot
test_result.env = t.env
test_result.testblockset = t.testblockset
break
for testblock in sorted(tbm[test].keys()):
testblock_result = TestblockResult()
testblock_result.name = testblock
testblock_result.result = True
for metric in sorted(tbm[test][testblock].keys()):
metric_result = tbm[test][testblock][metric]
testblock_result.results.append(metric_result)
# aggregate metric result
if metric_result.groundtruth.result != Groundtruth.SUCCEEDED:
testblock_result.result = False
testblock_result.error_message += "\n - metric '%s': %s" % (
metric_result.name,
metric_result.groundtruth.error_message)
test_result.results.append(testblock_result)
# aggregate testblock result
if testblock_result.result == False:
test_result.result = False
test_result.error_message += "\n - testblock '%s': %s" % (
testblock_result.name, testblock_result.error_message)
atf_result_aggregated.results.append(test_result)
# aggregate test result
if test_result.result == False:
atf_result_aggregated.result = False
atf_result_aggregated.error_message += "\n - test '%s' (%s, %s, %s, %s): %s" % (
test_result.name, test_result.test_config,
test_result.robot, test_result.env,
test_result.testblockset, test_result.error_message)
return atf_result_aggregated
def get_result(self):
metric_result = MetricResult()
metric_result.name = self.name
metric_result.mode = self.mode
metric_result.started = self.started # FIXME remove
metric_result.finished = self.finished # FIXME remove
metric_result.series = []
metric_result.groundtruth.available = self.groundtruth.available
metric_result.groundtruth.data = self.groundtruth.data
metric_result.groundtruth.epsilon = self.groundtruth.epsilon
# assign default value
metric_result.groundtruth.result = None
metric_result.groundtruth.error_message = None
if metric_result.started and metric_result.finished and len(
self.series
) != 0: # we check if the testblock was ever started and stopped and if result data is available
# calculate metric data
if self.series_mode != None:
metric_result.series = self.series
if metric_result.mode == MetricResult.SNAP:
metric_result.data = self.series[
-1] # take last element from self.series for data and stamp
metric_result.min = metric_result.data
metric_result.max = metric_result.data
metric_result.mean = metric_result.data.data
metric_result.std = 0.0
elif metric_result.mode == MetricResult.SPAN_MEAN:
metric_result.min = metrics_helper.get_min(self.series)
metric_result.max = metrics_helper.get_max(self.series)
metric_result.mean = metrics_helper.get_mean(self.series)
metric_result.std = metrics_helper.get_std(self.series)
metric_result.data.data = metric_result.mean # take mean for data
metric_result.data.stamp = self.series[
-1].stamp # take stamp from last element in self.series for stamp
elif metric_result.mode == MetricResult.SPAN_MIN:
metric_result.min = metrics_helper.get_min(self.series)
metric_result.max = metrics_helper.get_max(self.series)
metric_result.mean = metrics_helper.get_mean(self.series)
metric_result.std = metrics_helper.get_std(self.series)
metric_result.data = metric_result.min
elif metric_result.mode == MetricResult.SPAN_ABSMIN:
metric_result.min = metrics_helper.get_absmin(self.series)
metric_result.max = metrics_helper.get_absmax(self.series)
metric_result.mean = metrics_helper.get_mean(self.series)
metric_result.std = metrics_helper.get_std(self.series)
metric_result.data = metric_result.min
elif metric_result.mode == MetricResult.SPAN_MAX:
metric_result.min = metrics_helper.get_min(self.series)
metric_result.max = metrics_helper.get_max(self.series)
metric_result.mean = metrics_helper.get_mean(self.series)
metric_result.std = metrics_helper.get_std(self.series)
metric_result.data = metric_result.max
elif metric_result.mode == MetricResult.SPAN_ABSMAX:
metric_result.min = metrics_helper.get_absmin(self.series)
metric_result.max = metrics_helper.get_absmax(self.series)
metric_result.mean = metrics_helper.get_mean(self.series)
metric_result.std = metrics_helper.get_std(self.series)
metric_result.data = metric_result.max
else: # invalid mode
raise ATFAnalyserError(
"Analysing failed, invalid mode '%s' for metric '%s'." %
(metric_result.mode, metric_result.name))
# fill details as KeyValue messages
details = []
details.append(KeyValue("root_frame", self.root_frame))
details.append(KeyValue("measured_frame", self.measured_frame))
metric_result.details = details
# evaluate metric data
if not metric_result.groundtruth.available: # no groundtruth given
metric_result.groundtruth.result = True
metric_result.groundtruth.error_message = "all OK (no groundtruth available)"
else: # groundtruth available
if math.fabs(metric_result.groundtruth.data -
metric_result.data.data
) <= metric_result.groundtruth.epsilon:
metric_result.groundtruth.result = True
metric_result.groundtruth.error_message = "all OK"
else:
metric_result.groundtruth.result = False
metric_result.groundtruth.error_message = "groundtruth missmatch: %f not within %f+-%f" % (
metric_result.data.data,
metric_result.groundtruth.data,
metric_result.groundtruth.epsilon)
else: # testblock did not start and/or finish
metric_result.groundtruth.result = False
metric_result.groundtruth.error_message = "no result"
if metric_result.groundtruth.result == None:
raise ATFAnalyserError(
"Analysing failed, metric result is None for metric '%s'." %
metric_result.name)
return metric_result
