def get_result(self): metric_result = MetricResult() metric_result.name = self.name metric_result.started = self.started # FIXME remove metric_result.finished = self.finished # FIXME remove # check if user result is set if self.metric_result != None and not (self.metric_result.groundtruth_result == False\ and self.metric_result.groundtruth_error_message == ""\ and self.metric_result.groundtruth == 0.0\ and self.metric_result.groundtruth_epsilon == 0.0): #print "groundtruth data is set from user within atf application for testblock %s. Skipping groundtruth evaluation from test_config"%self.testblock_name # use data from user result metric_result = self.metric_result return metric_result metric_result.data = None metric_result.groundtruth = self.groundtruth metric_result.groundtruth_epsilon = self.groundtruth_epsilon # assign default value metric_result.groundtruth_result = None metric_result.groundtruth_error_message = None if metric_result.started and metric_result.finished: # we check if the testblock was ever started and stopped # calculate metric data if self.metric_result == None: print "ERROR user result for testblock %s not set" % self.testblock_name metric_result.data = None else: metric_result.data = self.metric_result.data # fill details as KeyValue messages metric_result.details = self.metric_result.details # evaluate metric data if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None: if math.fabs(metric_result.groundtruth - metric_result.data.data ) <= metric_result.groundtruth_epsilon: metric_result.groundtruth_result = True metric_result.groundtruth_error_message = "all OK" else: metric_result.groundtruth_result = False metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f" % ( metric_result.data.data, metric_result.groundtruth, metric_result.groundtruth_epsilon) if metric_result.data == None: metric_result.groundtruth_result = False metric_result.groundtruth_error_message = "no result" return metric_result
def get_result(self): metric_result = MetricResult() metric_result.name = self.name metric_result.started = self.started # FIXME remove metric_result.finished = self.finished # FIXME remove metric_result.series = [] metric_result.data = None metric_result.groundtruth = self.groundtruth metric_result.groundtruth_epsilon = self.groundtruth_epsilon # assign default value metric_result.groundtruth_result = None metric_result.groundtruth_error_message = None if metric_result.started and metric_result.finished: # we check if the testblock was ever started and stopped # calculate metric data if self.series_mode != None: metric_result.series = self.series metric_result.data = self.series[ -1] # take last element from self.series metric_result.min = metrics_helper.get_min(self.series) metric_result.max = metrics_helper.get_max(self.series) metric_result.mean = metrics_helper.get_mean(self.series) metric_result.std = metrics_helper.get_std(self.series) # fill details as KeyValue messages details = [] details.append(KeyValue("root_frame", self.root_frame)) details.append(KeyValue("measured_frame", self.measured_frame)) metric_result.details = details # evaluate metric data if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None: if math.fabs(metric_result.groundtruth - metric_result.data.data ) <= metric_result.groundtruth_epsilon: metric_result.groundtruth_result = True metric_result.groundtruth_error_message = "all OK" else: metric_result.groundtruth_result = False metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f" % ( metric_result.data.data, metric_result.groundtruth, metric_result.groundtruth_epsilon) if metric_result.data == None: metric_result.groundtruth_result = False metric_result.groundtruth_error_message = "no result" return metric_result
def get_result(self): interface_data, interface_details = self.calculate_data_and_details() groundtruth_result = None # interface metric not usable without groundtruth groundtruth = 100 # this is the max score groundtruth_epsilon = 0 # no deviation from max score allowed metric_result = MetricResult() metric_result.name = "interface" metric_result.started = self.started # FIXME remove metric_result.finished = self.finished # FIXME remove metric_result.data = None metric_result.groundtruth = self.groundtruth metric_result.groundtruth_epsilon = self.groundtruth_epsilon # assign default value metric_result.groundtruth_result = None metric_result.groundtruth_error_message = None if metric_result.started and metric_result.finished: # we check if the testblock was ever started and stopped # calculate metric data metric_result.data = interface_data # fill details as KeyValue messages details = [] details.append(KeyValue("api_status", interface_details)) metric_result.details = details # evaluate metric data if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None: if math.fabs(metric_result.groundtruth - metric_result.data ) <= metric_result.groundtruth_epsilon: metric_result.groundtruth_result = True metric_result.groundtruth_error_message = "all OK" else: metric_result.groundtruth_result = False metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f" % ( metric_result.data, metric_result.groundtruth, metric_result.groundtruth_epsilon) #print metric_result.groundtruth_error_message if metric_result.data == None: metric_result.groundtruth_result = False metric_result.groundtruth_error_message = "no result" #print "\nmetric_result:\n", metric_result return metric_result
def get_result(self): metric_result = MetricResult() metric_result.name = self.name metric_result.status = self.status.status if self.status.status != TestblockStatus.SUCCEEDED: metric_result.groundtruth.result = Groundtruth.FAILED metric_result.groundtruth.error_message = metrics_helper.extract_error_message( self.status) return metric_result # check if result is available if self.metric_result.groundtruth.result == Groundtruth.UNSET and not self.groundtruth.available: # let the analyzer know that this test failed metric_result.groundtruth.result = Groundtruth.FAILED metric_result.groundtruth.error_message = "testblock %s stopped without user_result" % self.testblock_name return metric_result # at this point we're sure that any user_result is available # overwrite user_result data with mandatory ATF fields metric_result = self.metric_result metric_result.name = self.name metric_result.status = self.status.status # evaluate metric data if self.groundtruth.available: # groundtruth available # overwrite grundtruth with data from yaml file metric_result.groundtruth = self.groundtruth if math.fabs(metric_result.groundtruth.data - metric_result.data.data ) <= metric_result.groundtruth.epsilon: metric_result.groundtruth.result = Groundtruth.SUCCEEDED metric_result.groundtruth.error_message = "all OK" else: metric_result.groundtruth.result = Groundtruth.FAILED metric_result.groundtruth.error_message = "groundtruth missmatch: %f not within %f+-%f" % ( metric_result.data.data, metric_result.groundtruth.data, metric_result.groundtruth.epsilon) else: # groundtruth not available # we'll keep what is set in self.metric_result: user_result set by user pass return metric_result
def get_result(self): metric_result = MetricResult() metric_result.name = "publish_rate" metric_result.started = self.started # FIXME remove metric_result.finished = self.finished # FIXME remove metric_result.data = None metric_result.groundtruth = self.groundtruth metric_result.groundtruth_epsilon = self.groundtruth_epsilon # assign default value metric_result.groundtruth_result = None metric_result.groundtruth_error_message = None if metric_result.started and metric_result.finished: # we check if the testblock was ever started and stopped # calculate metric data metric_result.data = round( self.counter / (self.stop_time - self.start_time).to_sec(), 3) # fill details as KeyValue messages details = [] details.append(KeyValue("topic", self.topic)) metric_result.details = details # evaluate metric data if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None: if math.fabs(metric_result.groundtruth - metric_result.data ) <= metric_result.groundtruth_epsilon: metric_result.groundtruth_result = True metric_result.groundtruth_error_message = "all OK" else: metric_result.groundtruth_result = False metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f" % ( metric_result.data, metric_result.groundtruth, metric_result.groundtruth_epsilon) #print metric_result.groundtruth_error_message if metric_result.data == None: metric_result.groundtruth_result = False metric_result.groundtruth_error_message = "no result" #print "\nmetric_result:\n", metric_result return metric_result
def aggregate_results(self, atf_result): test_list = self.configuration_parser.get_test_list() ret = self.configuration_parser.get_sorted_plot_dicts( atf_result, "", "", "") mbt = ret['mbt'] mbt_aggregated = {} for metric in list(mbt.keys()): #print "m=", metric if metric not in list(mbt_aggregated.keys()): mbt_aggregated[metric] = {} for testblock in list(mbt[metric].keys()): #print " b=", testblock if testblock not in list(mbt_aggregated[metric].keys()): mbt_aggregated[metric][testblock] = {} for tl_tests in test_list: #print "tl_tests=", tl_tests for tl_test in list(tl_tests.keys()): #print " tl_test=", tl_test metric_result = MetricResult() status = TestblockStatus.SUCCEEDED groundtruth_result = Groundtruth.SUCCEEDED groundtruth_error_message = "" details = [] for test in list(mbt[metric][testblock].keys()): if test.startswith(tl_test): # aggregate status SUCCEEDED from every metric_result if mbt[metric][testblock][ test].status != TestblockStatus.SUCCEEDED: status = TestblockStatus.ERROR # aggregate data from every metric_result data = mbt[metric][testblock][test].data stamp = data.stamp # check if data is set (not all default values anymore) if data.stamp == rospy.Time( 0) and data.data == 0: stamp = rospy.Time( 0 ) # mark metric result as invalid by settimg timestamp to zero metric_result.series.append(data) # aggregate groundtruth from every metric_result groundtruth = mbt[metric][testblock][ test].groundtruth if groundtruth.result != Groundtruth.SUCCEEDED: groundtruth_result = Groundtruth.FAILED if groundtruth_error_message != "": groundtruth_error_message += "\n" groundtruth_error_message += "groundtruth missmatch in subtest %s" % ( test) # aggregate details from every metric_result details = details + mbt[metric][testblock][ test].details if len(metric_result.series ) == 0: # no matching substest found continue metric_result.groundtruth = groundtruth metric_result.groundtruth.result = groundtruth_result metric_result.groundtruth.error_message = groundtruth_error_message metric_result.name = mbt[metric][testblock][test].name metric_result.unit = mbt[metric][testblock][test].unit metric_result.mode = MetricResult.SPAN_MEAN # aggregated metrics are always SPAN_MEAN metric_result.status = status # metric_result.series is set above metric_result.data.stamp = stamp metric_result.data.data = metrics_helper.get_mean( metric_result.series) metric_result.min = metrics_helper.get_min( metric_result.series) metric_result.max = metrics_helper.get_max( metric_result.series) metric_result.mean = metric_result.data.data metric_result.std = metrics_helper.get_std( metric_result.series) # metric_result.groundtruth is set above metric_result.details = details mbt_aggregated[metric][testblock][ tl_test] = metric_result # convert mbt to tbm tbm = {} for metric in list(mbt_aggregated.keys()): #print "m=", metric for testblock in list(mbt_aggregated[metric].keys()): #print " b=", testblock for test in list(mbt_aggregated[metric][testblock].keys()): #print " t=", test if test not in list(tbm.keys()): tbm[test] = {} if testblock not in list(tbm[test].keys()): tbm[test][testblock] = {} tbm[test][testblock][metric] = mbt_aggregated[metric][ testblock][test] # convert tbm to atf_result_aggregated atf_result_aggregated = AtfResult() atf_result_aggregated.header = atf_result.header atf_result_aggregated.name = atf_result.name atf_result_aggregated.result = True for test in sorted(tbm.keys()): test_result = TestResult() test_result.name = test test_result.result = True # find test metadata in atf_result for t in atf_result.results: if t.name.startswith(test): test_result.test_config = t.test_config test_result.robot = t.robot test_result.env = t.env test_result.testblockset = t.testblockset break for testblock in sorted(tbm[test].keys()): testblock_result = TestblockResult() testblock_result.name = testblock testblock_result.result = True for metric in sorted(tbm[test][testblock].keys()): metric_result = tbm[test][testblock][metric] testblock_result.results.append(metric_result) # aggregate metric result if metric_result.groundtruth.result != Groundtruth.SUCCEEDED: testblock_result.result = False testblock_result.error_message += "\n - metric '%s': %s" % ( metric_result.name, metric_result.groundtruth.error_message) test_result.results.append(testblock_result) # aggregate testblock result if testblock_result.result == False: test_result.result = False test_result.error_message += "\n - testblock '%s': %s" % ( testblock_result.name, testblock_result.error_message) atf_result_aggregated.results.append(test_result) # aggregate test result if test_result.result == False: atf_result_aggregated.result = False atf_result_aggregated.error_message += "\n - test '%s' (%s, %s, %s, %s): %s" % ( test_result.name, test_result.test_config, test_result.robot, test_result.env, test_result.testblockset, test_result.error_message) return atf_result_aggregated