def get_result(self):
        metric_result = MetricResult()
        metric_result.name = self.name
        metric_result.started = self.started  # FIXME remove
        metric_result.finished = self.finished  # FIXME remove
        metric_result.series = []
        metric_result.data = None
        metric_result.groundtruth = self.groundtruth
        metric_result.groundtruth_epsilon = self.groundtruth_epsilon

        # assign default value
        metric_result.groundtruth_result = None
        metric_result.groundtruth_error_message = None

        if metric_result.started and metric_result.finished:  #  we check if the testblock was ever started and stopped
            # calculate metric data
            if self.series_mode != None:
                metric_result.series = self.series
            metric_result.data = self.series[
                -1]  # take last element from self.series
            metric_result.min = metrics_helper.get_min(self.series)
            metric_result.max = metrics_helper.get_max(self.series)
            metric_result.mean = metrics_helper.get_mean(self.series)
            metric_result.std = metrics_helper.get_std(self.series)

            # fill details as KeyValue messages
            details = []
            details.append(KeyValue("root_frame", self.root_frame))
            details.append(KeyValue("measured_frame", self.measured_frame))
            metric_result.details = details

            # evaluate metric data
            if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None:
                if math.fabs(metric_result.groundtruth -
                             metric_result.data.data
                             ) <= metric_result.groundtruth_epsilon:
                    metric_result.groundtruth_result = True
                    metric_result.groundtruth_error_message = "all OK"
                else:
                    metric_result.groundtruth_result = False
                    metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f" % (
                        metric_result.data.data, metric_result.groundtruth,
                        metric_result.groundtruth_epsilon)

        if metric_result.data == None:
            metric_result.groundtruth_result = False
            metric_result.groundtruth_error_message = "no result"

        return metric_result
Esempio n. 2
0
    def aggregate_results(self, atf_result):
        test_list = self.configuration_parser.get_test_list()

        ret = self.configuration_parser.get_sorted_plot_dicts(
            atf_result, "", "", "")

        mbt = ret['mbt']
        mbt_aggregated = {}
        for metric in list(mbt.keys()):
            #print "m=", metric
            if metric not in list(mbt_aggregated.keys()):
                mbt_aggregated[metric] = {}
            for testblock in list(mbt[metric].keys()):
                #print "  b=", testblock
                if testblock not in list(mbt_aggregated[metric].keys()):
                    mbt_aggregated[metric][testblock] = {}
                for tl_tests in test_list:
                    #print "tl_tests=", tl_tests
                    for tl_test in list(tl_tests.keys()):
                        #print "    tl_test=", tl_test
                        metric_result = MetricResult()
                        status = TestblockStatus.SUCCEEDED
                        groundtruth_result = Groundtruth.SUCCEEDED
                        groundtruth_error_message = ""
                        details = []
                        for test in list(mbt[metric][testblock].keys()):
                            if test.startswith(tl_test):
                                # aggregate status SUCCEEDED from every metric_result
                                if mbt[metric][testblock][
                                        test].status != TestblockStatus.SUCCEEDED:
                                    status = TestblockStatus.ERROR

                                # aggregate data from every metric_result
                                data = mbt[metric][testblock][test].data
                                stamp = data.stamp
                                # check if data is set (not all default values anymore)
                                if data.stamp == rospy.Time(
                                        0) and data.data == 0:
                                    stamp = rospy.Time(
                                        0
                                    )  # mark metric result as invalid by settimg timestamp to zero
                                metric_result.series.append(data)

                                # aggregate groundtruth from every metric_result
                                groundtruth = mbt[metric][testblock][
                                    test].groundtruth
                                if groundtruth.result != Groundtruth.SUCCEEDED:
                                    groundtruth_result = Groundtruth.FAILED
                                    if groundtruth_error_message != "":
                                        groundtruth_error_message += "\n"
                                    groundtruth_error_message += "groundtruth missmatch in subtest %s" % (
                                        test)

                                # aggregate details from every metric_result
                                details = details + mbt[metric][testblock][
                                    test].details

                        if len(metric_result.series
                               ) == 0:  # no matching substest found
                            continue

                        metric_result.groundtruth = groundtruth
                        metric_result.groundtruth.result = groundtruth_result
                        metric_result.groundtruth.error_message = groundtruth_error_message

                        metric_result.name = mbt[metric][testblock][test].name
                        metric_result.unit = mbt[metric][testblock][test].unit
                        metric_result.mode = MetricResult.SPAN_MEAN  # aggregated metrics are always SPAN_MEAN
                        metric_result.status = status
                        # metric_result.series is set above
                        metric_result.data.stamp = stamp
                        metric_result.data.data = metrics_helper.get_mean(
                            metric_result.series)
                        metric_result.min = metrics_helper.get_min(
                            metric_result.series)
                        metric_result.max = metrics_helper.get_max(
                            metric_result.series)
                        metric_result.mean = metric_result.data.data
                        metric_result.std = metrics_helper.get_std(
                            metric_result.series)
                        # metric_result.groundtruth is set above
                        metric_result.details = details
                        mbt_aggregated[metric][testblock][
                            tl_test] = metric_result

        # convert mbt to tbm
        tbm = {}
        for metric in list(mbt_aggregated.keys()):
            #print "m=", metric
            for testblock in list(mbt_aggregated[metric].keys()):
                #print "  b=", testblock
                for test in list(mbt_aggregated[metric][testblock].keys()):
                    #print "    t=", test
                    if test not in list(tbm.keys()):
                        tbm[test] = {}
                    if testblock not in list(tbm[test].keys()):
                        tbm[test][testblock] = {}
                    tbm[test][testblock][metric] = mbt_aggregated[metric][
                        testblock][test]

        # convert tbm to atf_result_aggregated
        atf_result_aggregated = AtfResult()
        atf_result_aggregated.header = atf_result.header
        atf_result_aggregated.name = atf_result.name
        atf_result_aggregated.result = True
        for test in sorted(tbm.keys()):
            test_result = TestResult()
            test_result.name = test
            test_result.result = True

            # find test metadata in atf_result
            for t in atf_result.results:
                if t.name.startswith(test):
                    test_result.test_config = t.test_config
                    test_result.robot = t.robot
                    test_result.env = t.env
                    test_result.testblockset = t.testblockset
                    break

            for testblock in sorted(tbm[test].keys()):
                testblock_result = TestblockResult()
                testblock_result.name = testblock
                testblock_result.result = True
                for metric in sorted(tbm[test][testblock].keys()):
                    metric_result = tbm[test][testblock][metric]
                    testblock_result.results.append(metric_result)
                    # aggregate metric result
                    if metric_result.groundtruth.result != Groundtruth.SUCCEEDED:
                        testblock_result.result = False
                        testblock_result.error_message += "\n     - metric '%s': %s" % (
                            metric_result.name,
                            metric_result.groundtruth.error_message)

                test_result.results.append(testblock_result)
                # aggregate testblock result
                if testblock_result.result == False:
                    test_result.result = False
                    test_result.error_message += "\n   - testblock '%s': %s" % (
                        testblock_result.name, testblock_result.error_message)

            atf_result_aggregated.results.append(test_result)
            # aggregate test result
            if test_result.result == False:
                atf_result_aggregated.result = False
                atf_result_aggregated.error_message += "\n - test '%s' (%s, %s, %s, %s): %s" % (
                    test_result.name, test_result.test_config,
                    test_result.robot, test_result.env,
                    test_result.testblockset, test_result.error_message)

        return atf_result_aggregated
    def get_result(self):
        metric_result = MetricResult()
        metric_result.name = self.name
        metric_result.mode = self.mode
        metric_result.started = self.started  # FIXME remove
        metric_result.finished = self.finished  # FIXME remove
        metric_result.series = []
        metric_result.groundtruth.available = self.groundtruth.available
        metric_result.groundtruth.data = self.groundtruth.data
        metric_result.groundtruth.epsilon = self.groundtruth.epsilon

        # assign default value
        metric_result.groundtruth.result = None
        metric_result.groundtruth.error_message = None

        if metric_result.started and metric_result.finished and len(
                self.series
        ) != 0:  #  we check if the testblock was ever started and stopped and if result data is available
            # calculate metric data
            if self.series_mode != None:
                metric_result.series = self.series
            if metric_result.mode == MetricResult.SNAP:
                metric_result.data = self.series[
                    -1]  # take last element from self.series for data and stamp
                metric_result.min = metric_result.data
                metric_result.max = metric_result.data
                metric_result.mean = metric_result.data.data
                metric_result.std = 0.0
            elif metric_result.mode == MetricResult.SPAN_MEAN:
                metric_result.min = metrics_helper.get_min(self.series)
                metric_result.max = metrics_helper.get_max(self.series)
                metric_result.mean = metrics_helper.get_mean(self.series)
                metric_result.std = metrics_helper.get_std(self.series)
                metric_result.data.data = metric_result.mean  # take mean for data
                metric_result.data.stamp = self.series[
                    -1].stamp  # take stamp from last element in self.series for stamp
            elif metric_result.mode == MetricResult.SPAN_MIN:
                metric_result.min = metrics_helper.get_min(self.series)
                metric_result.max = metrics_helper.get_max(self.series)
                metric_result.mean = metrics_helper.get_mean(self.series)
                metric_result.std = metrics_helper.get_std(self.series)
                metric_result.data = metric_result.min
            elif metric_result.mode == MetricResult.SPAN_ABSMIN:
                metric_result.min = metrics_helper.get_absmin(self.series)
                metric_result.max = metrics_helper.get_absmax(self.series)
                metric_result.mean = metrics_helper.get_mean(self.series)
                metric_result.std = metrics_helper.get_std(self.series)
                metric_result.data = metric_result.min
            elif metric_result.mode == MetricResult.SPAN_MAX:
                metric_result.min = metrics_helper.get_min(self.series)
                metric_result.max = metrics_helper.get_max(self.series)
                metric_result.mean = metrics_helper.get_mean(self.series)
                metric_result.std = metrics_helper.get_std(self.series)
                metric_result.data = metric_result.max
            elif metric_result.mode == MetricResult.SPAN_ABSMAX:
                metric_result.min = metrics_helper.get_absmin(self.series)
                metric_result.max = metrics_helper.get_absmax(self.series)
                metric_result.mean = metrics_helper.get_mean(self.series)
                metric_result.std = metrics_helper.get_std(self.series)
                metric_result.data = metric_result.max
            else:  # invalid mode
                raise ATFAnalyserError(
                    "Analysing failed, invalid mode '%s' for metric '%s'." %
                    (metric_result.mode, metric_result.name))

            # fill details as KeyValue messages
            details = []
            details.append(KeyValue("root_frame", self.root_frame))
            details.append(KeyValue("measured_frame", self.measured_frame))
            metric_result.details = details

            # evaluate metric data
            if not metric_result.groundtruth.available:  # no groundtruth given
                metric_result.groundtruth.result = True
                metric_result.groundtruth.error_message = "all OK (no groundtruth available)"
            else:  # groundtruth available
                if math.fabs(metric_result.groundtruth.data -
                             metric_result.data.data
                             ) <= metric_result.groundtruth.epsilon:
                    metric_result.groundtruth.result = True
                    metric_result.groundtruth.error_message = "all OK"
                else:
                    metric_result.groundtruth.result = False
                    metric_result.groundtruth.error_message = "groundtruth missmatch: %f not within %f+-%f" % (
                        metric_result.data.data,
                        metric_result.groundtruth.data,
                        metric_result.groundtruth.epsilon)

        else:  # testblock did not start and/or finish
            metric_result.groundtruth.result = False
            metric_result.groundtruth.error_message = "no result"

        if metric_result.groundtruth.result == None:
            raise ATFAnalyserError(
                "Analysing failed, metric result is None for metric '%s'." %
                metric_result.name)

        return metric_result