コード例 #1
0
    def get_result(self):
        metric_result = MetricResult()
        metric_result.name = self.name
        metric_result.unit = self.unit
        metric_result.mode = self.mode
        metric_result.status = self.status.status
        metric_result.series = []
        metric_result.groundtruth.available = self.groundtruth.available
        metric_result.groundtruth.data = self.groundtruth.data
        metric_result.groundtruth.epsilon = self.groundtruth.epsilon

        if self.status.status != TestblockStatus.SUCCEEDED:
            metric_result.groundtruth.result = Groundtruth.FAILED
            metric_result.groundtruth.error_message = metrics_helper.extract_error_message(self.status)
            return metric_result

        # check if result is available
        if len(self.series) == 0:
            # let the analyzer know that this test failed
            metric_result.groundtruth.result = Groundtruth.FAILED
            metric_result.groundtruth.error_message = "testblock {} stopped without result. " \
                                                      "No transforms found between {} & {}".format(self.testblock_name,
                                                                                                   self.root_frame,
                                                                                                   self.measured_frame)
            return metric_result

        # at this point we're sure that any result is available

        # set series
        if self.series_mode != None:
            metric_result.series = self.series

        # calculate metric data
        [metric_result.data, metric_result.min, metric_result.max, metric_result.mean, metric_result.std] = metrics_helper.calculate_metric_data(metric_result.name, metric_result.mode, self.series)

        # fill details as KeyValue messages
        details = []
        details.append(KeyValue("root_frame", self.root_frame))
        details.append(KeyValue("measured_frame", self.measured_frame))
        metric_result.details = details

        # evaluate metric data
        if metric_result.groundtruth.available: # groundtruth available
            if math.fabs(metric_result.groundtruth.data - metric_result.data.data) <= metric_result.groundtruth.epsilon:
                metric_result.groundtruth.result = Groundtruth.SUCCEEDED
                metric_result.groundtruth.error_message = "all OK"
            else:
                metric_result.groundtruth.result = Groundtruth.FAILED
                metric_result.groundtruth.error_message = "groundtruth missmatch: %f not within %f+-%f"%(metric_result.data.data, metric_result.groundtruth.data, metric_result.groundtruth.epsilon)

        else: # groundtruth not available
            metric_result.groundtruth.result = Groundtruth.SUCCEEDED
            metric_result.groundtruth.error_message = "all OK (no groundtruth available)"

        return metric_result
コード例 #2
0
    def get_result(self):
        metric_result = MetricResult()
        metric_result.name = self.name
        metric_result.started = self.started  # FIXME remove
        metric_result.finished = self.finished  # FIXME remove
        metric_result.series = []
        metric_result.data = None
        metric_result.groundtruth = self.groundtruth
        metric_result.groundtruth_epsilon = self.groundtruth_epsilon

        # assign default value
        metric_result.groundtruth_result = None
        metric_result.groundtruth_error_message = None

        if metric_result.started and metric_result.finished:  #  we check if the testblock was ever started and stopped
            # calculate metric data
            if self.series_mode != None:
                metric_result.series = self.series
            metric_result.data = self.series[
                -1]  # take last element from self.series
            metric_result.min = metrics_helper.get_min(self.series)
            metric_result.max = metrics_helper.get_max(self.series)
            metric_result.mean = metrics_helper.get_mean(self.series)
            metric_result.std = metrics_helper.get_std(self.series)

            # fill details as KeyValue messages
            details = []
            details.append(KeyValue("root_frame", self.root_frame))
            details.append(KeyValue("measured_frame", self.measured_frame))
            metric_result.details = details

            # evaluate metric data
            if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None:
                if math.fabs(metric_result.groundtruth -
                             metric_result.data.data
                             ) <= metric_result.groundtruth_epsilon:
                    metric_result.groundtruth_result = True
                    metric_result.groundtruth_error_message = "all OK"
                else:
                    metric_result.groundtruth_result = False
                    metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f" % (
                        metric_result.data.data, metric_result.groundtruth,
                        metric_result.groundtruth_epsilon)

        if metric_result.data == None:
            metric_result.groundtruth_result = False
            metric_result.groundtruth_error_message = "no result"

        return metric_result
コード例 #3
0
ファイル: calculate_path_length.py プロジェクト: Acuda/atf
    def get_result(self):
        metric_result = MetricResult()
        metric_result.name = "path_length"
        metric_result.started = self.started # FIXME remove
        metric_result.finished = self.finished # FIXME remove
        metric_result.data = None
        metric_result.groundtruth = self.groundtruth
        metric_result.groundtruth_epsilon = self.groundtruth_epsilon
        
        # assign default value
        metric_result.groundtruth_result = None
        metric_result.groundtruth_error_message = None

        if metric_result.started and metric_result.finished: #  we check if the testblock was ever started and stopped
            # calculate metric data
            metric_result.data = round(self.path_length, 3)

            # fill details as KeyValue messages
            details = []
            details.append(KeyValue("root_frame", self.root_frame))
            details.append(KeyValue("measured_frame", self.measured_frame))
            metric_result.details = details

            # evaluate metric data
            if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None:
                if math.fabs(metric_result.groundtruth - metric_result.data) <= metric_result.groundtruth_epsilon:
                    metric_result.groundtruth_result = True
                    metric_result.groundtruth_error_message = "all OK"
                else:
                    metric_result.groundtruth_result = False
                    metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f"%(metric_result.data, metric_result.groundtruth, metric_result.groundtruth_epsilon)
                    #print metric_result.groundtruth_error_message

        if metric_result.data == None:
            metric_result.groundtruth_result = False
            metric_result.groundtruth_error_message = "no result"

        #print "\nmetric_result:\n", metric_result
        return metric_result
コード例 #4
0
ファイル: calculate_interface.py プロジェクト: Acuda/atf
    def get_result(self):
        interface_data, interface_details = self.calculate_data_and_details()
        groundtruth_result = None  # interface metric not usable without groundtruth
        groundtruth = 100  # this is the max score
        groundtruth_epsilon = 0  # no deviation from max score allowed

        metric_result = MetricResult()
        metric_result.name = "interface"
        metric_result.started = self.started  # FIXME remove
        metric_result.finished = self.finished  # FIXME remove
        metric_result.data = None
        metric_result.groundtruth = self.groundtruth
        metric_result.groundtruth_epsilon = self.groundtruth_epsilon

        # assign default value
        metric_result.groundtruth_result = None
        metric_result.groundtruth_error_message = None

        if metric_result.started and metric_result.finished:  #  we check if the testblock was ever started and stopped
            # calculate metric data
            metric_result.data = interface_data

            # fill details as KeyValue messages
            details = []
            details.append(KeyValue("api_status", interface_details))
            metric_result.details = details

            # evaluate metric data
            if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None:
                if math.fabs(metric_result.groundtruth - metric_result.data
                             ) <= metric_result.groundtruth_epsilon:
                    metric_result.groundtruth_result = True
                    metric_result.groundtruth_error_message = "all OK"
                else:
                    metric_result.groundtruth_result = False
                    metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f" % (
                        metric_result.data, metric_result.groundtruth,
                        metric_result.groundtruth_epsilon)
                    #print metric_result.groundtruth_error_message

        if metric_result.data == None:
            metric_result.groundtruth_result = False
            metric_result.groundtruth_error_message = "no result"

        #print "\nmetric_result:\n", metric_result
        return metric_result
コード例 #5
0
    def get_result(self):
        metric_result = MetricResult()
        metric_result.name = self.name
        metric_result.mode = self.mode
        metric_result.started = self.started  # FIXME remove
        metric_result.finished = self.finished  # FIXME remove
        metric_result.series = []
        metric_result.groundtruth.available = self.groundtruth.available
        metric_result.groundtruth.data = self.groundtruth.data
        metric_result.groundtruth.epsilon = self.groundtruth.epsilon

        # assign default value
        metric_result.groundtruth.result = None
        metric_result.groundtruth.error_message = None

        if metric_result.started and metric_result.finished and len(
                self.series
        ) != 0:  #  we check if the testblock was ever started and stopped and if result data is available
            # calculate metric data
            if self.series_mode != None:
                metric_result.series = self.series
            if metric_result.mode == MetricResult.SNAP:
                metric_result.data = self.series[
                    -1]  # take last element from self.series for data and stamp
                metric_result.min = metric_result.data
                metric_result.max = metric_result.data
                metric_result.mean = metric_result.data.data
                metric_result.std = 0.0
            elif metric_result.mode == MetricResult.SPAN_MEAN:
                metric_result.min = metrics_helper.get_min(self.series)
                metric_result.max = metrics_helper.get_max(self.series)
                metric_result.mean = metrics_helper.get_mean(self.series)
                metric_result.std = metrics_helper.get_std(self.series)
                metric_result.data.data = metric_result.mean  # take mean for data
                metric_result.data.stamp = self.series[
                    -1].stamp  # take stamp from last element in self.series for stamp
            elif metric_result.mode == MetricResult.SPAN_MIN:
                metric_result.min = metrics_helper.get_min(self.series)
                metric_result.max = metrics_helper.get_max(self.series)
                metric_result.mean = metrics_helper.get_mean(self.series)
                metric_result.std = metrics_helper.get_std(self.series)
                metric_result.data = metric_result.min
            elif metric_result.mode == MetricResult.SPAN_ABSMIN:
                metric_result.min = metrics_helper.get_absmin(self.series)
                metric_result.max = metrics_helper.get_absmax(self.series)
                metric_result.mean = metrics_helper.get_mean(self.series)
                metric_result.std = metrics_helper.get_std(self.series)
                metric_result.data = metric_result.min
            elif metric_result.mode == MetricResult.SPAN_MAX:
                metric_result.min = metrics_helper.get_min(self.series)
                metric_result.max = metrics_helper.get_max(self.series)
                metric_result.mean = metrics_helper.get_mean(self.series)
                metric_result.std = metrics_helper.get_std(self.series)
                metric_result.data = metric_result.max
            elif metric_result.mode == MetricResult.SPAN_ABSMAX:
                metric_result.min = metrics_helper.get_absmin(self.series)
                metric_result.max = metrics_helper.get_absmax(self.series)
                metric_result.mean = metrics_helper.get_mean(self.series)
                metric_result.std = metrics_helper.get_std(self.series)
                metric_result.data = metric_result.max
            else:  # invalid mode
                raise ATFAnalyserError(
                    "Analysing failed, invalid mode '%s' for metric '%s'." %
                    (metric_result.mode, metric_result.name))

            # fill details as KeyValue messages
            details = []
            details.append(KeyValue("root_frame", self.root_frame))
            details.append(KeyValue("measured_frame", self.measured_frame))
            metric_result.details = details

            # evaluate metric data
            if not metric_result.groundtruth.available:  # no groundtruth given
                metric_result.groundtruth.result = True
                metric_result.groundtruth.error_message = "all OK (no groundtruth available)"
            else:  # groundtruth available
                if math.fabs(metric_result.groundtruth.data -
                             metric_result.data.data
                             ) <= metric_result.groundtruth.epsilon:
                    metric_result.groundtruth.result = True
                    metric_result.groundtruth.error_message = "all OK"
                else:
                    metric_result.groundtruth.result = False
                    metric_result.groundtruth.error_message = "groundtruth missmatch: %f not within %f+-%f" % (
                        metric_result.data.data,
                        metric_result.groundtruth.data,
                        metric_result.groundtruth.epsilon)

        else:  # testblock did not start and/or finish
            metric_result.groundtruth.result = False
            metric_result.groundtruth.error_message = "no result"

        if metric_result.groundtruth.result == None:
            raise ATFAnalyserError(
                "Analysing failed, metric result is None for metric '%s'." %
                metric_result.name)

        return metric_result