def execute(self):
self.atf.start("testblock_8s")
self.atf.start("testblock_3s")
rospy.sleep(3)
# user result
metric_result = MetricResult()
metric_result.data = 0.8
metric_result.groundtruth_result = True
metric_result.groundtruth_error_message = "all ok in application of atf_test"
self.atf.stop("testblock_3s",
metric_result) # user result with groundtruth result
self.atf.start("testblock_5s")
rospy.sleep(5)
# user result
metric_result = MetricResult()
metric_result.data = 0.7
self.atf.stop("testblock_5s",
metric_result) # user result without groundtruth result
self.atf.stop("testblock_8s") # no user result
# shutdown atf
self.atf.shutdown()
def get_result(self):
metric_result = MetricResult()
metric_result.name = self.name
metric_result.started = self.started # FIXME remove
metric_result.finished = self.finished # FIXME remove
# check if user result is set
if self.metric_result != None and not (self.metric_result.groundtruth_result == False\
and self.metric_result.groundtruth_error_message == ""\
and self.metric_result.groundtruth == 0.0\
and self.metric_result.groundtruth_epsilon == 0.0):
#print "groundtruth data is set from user within atf application for testblock %s. Skipping groundtruth evaluation from test_config"%self.testblock_name
# use data from user result
metric_result = self.metric_result
return metric_result
metric_result.data = None
metric_result.groundtruth = self.groundtruth
metric_result.groundtruth_epsilon = self.groundtruth_epsilon
# assign default value
metric_result.groundtruth_result = None
metric_result.groundtruth_error_message = None
if metric_result.started and metric_result.finished: # we check if the testblock was ever started and stopped
# calculate metric data
if self.metric_result == None:
print "ERROR user result for testblock %s not set" % self.testblock_name
metric_result.data = None
else:
metric_result.data = self.metric_result.data
# fill details as KeyValue messages
metric_result.details = self.metric_result.details
# evaluate metric data
if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None:
if math.fabs(metric_result.groundtruth -
metric_result.data.data
) <= metric_result.groundtruth_epsilon:
metric_result.groundtruth_result = True
metric_result.groundtruth_error_message = "all OK"
else:
metric_result.groundtruth_result = False
metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f" % (
metric_result.data.data, metric_result.groundtruth,
metric_result.groundtruth_epsilon)
if metric_result.data == None:
metric_result.groundtruth_result = False
metric_result.groundtruth_error_message = "no result"
return metric_result
def get_result(self):
metric_result = MetricResult()
metric_result.name = self.name
metric_result.started = self.started # FIXME remove
metric_result.finished = self.finished # FIXME remove
metric_result.series = []
metric_result.data = None
metric_result.groundtruth = self.groundtruth
metric_result.groundtruth_epsilon = self.groundtruth_epsilon
# assign default value
metric_result.groundtruth_result = None
metric_result.groundtruth_error_message = None
if metric_result.started and metric_result.finished: # we check if the testblock was ever started and stopped
# calculate metric data
if self.series_mode != None:
metric_result.series = self.series
metric_result.data = self.series[
-1] # take last element from self.series
metric_result.min = metrics_helper.get_min(self.series)
metric_result.max = metrics_helper.get_max(self.series)
metric_result.mean = metrics_helper.get_mean(self.series)
metric_result.std = metrics_helper.get_std(self.series)
# fill details as KeyValue messages
details = []
details.append(KeyValue("root_frame", self.root_frame))
details.append(KeyValue("measured_frame", self.measured_frame))
metric_result.details = details
# evaluate metric data
if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None:
if math.fabs(metric_result.groundtruth -
metric_result.data.data
) <= metric_result.groundtruth_epsilon:
metric_result.groundtruth_result = True
metric_result.groundtruth_error_message = "all OK"
else:
metric_result.groundtruth_result = False
metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f" % (
metric_result.data.data, metric_result.groundtruth,
metric_result.groundtruth_epsilon)
if metric_result.data == None:
metric_result.groundtruth_result = False
metric_result.groundtruth_error_message = "no result"
return metric_result
def get_result(self):
interface_data, interface_details = self.calculate_data_and_details()
groundtruth_result = None # interface metric not usable without groundtruth
groundtruth = 100 # this is the max score
groundtruth_epsilon = 0 # no deviation from max score allowed
metric_result = MetricResult()
metric_result.name = "interface"
metric_result.started = self.started # FIXME remove
metric_result.finished = self.finished # FIXME remove
metric_result.data = None
metric_result.groundtruth = self.groundtruth
metric_result.groundtruth_epsilon = self.groundtruth_epsilon
# assign default value
metric_result.groundtruth_result = None
metric_result.groundtruth_error_message = None
if metric_result.started and metric_result.finished: # we check if the testblock was ever started and stopped
# calculate metric data
metric_result.data = interface_data
# fill details as KeyValue messages
details = []
details.append(KeyValue("api_status", interface_details))
metric_result.details = details
# evaluate metric data
if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None:
if math.fabs(metric_result.groundtruth - metric_result.data
) <= metric_result.groundtruth_epsilon:
metric_result.groundtruth_result = True
metric_result.groundtruth_error_message = "all OK"
else:
metric_result.groundtruth_result = False
metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f" % (
metric_result.data, metric_result.groundtruth,
metric_result.groundtruth_epsilon)
#print metric_result.groundtruth_error_message
if metric_result.data == None:
metric_result.groundtruth_result = False
metric_result.groundtruth_error_message = "no result"
#print "\nmetric_result:\n", metric_result
return metric_result
def get_result(self):
metric_result = MetricResult()
metric_result.name = "publish_rate"
metric_result.started = self.started # FIXME remove
metric_result.finished = self.finished # FIXME remove
metric_result.data = None
metric_result.groundtruth = self.groundtruth
metric_result.groundtruth_epsilon = self.groundtruth_epsilon
# assign default value
metric_result.groundtruth_result = None
metric_result.groundtruth_error_message = None
if metric_result.started and metric_result.finished: # we check if the testblock was ever started and stopped
# calculate metric data
metric_result.data = round(
self.counter / (self.stop_time - self.start_time).to_sec(), 3)
# fill details as KeyValue messages
details = []
details.append(KeyValue("topic", self.topic))
metric_result.details = details
# evaluate metric data
if metric_result.data != None and metric_result.groundtruth != None and metric_result.groundtruth_epsilon != None:
if math.fabs(metric_result.groundtruth - metric_result.data
) <= metric_result.groundtruth_epsilon:
metric_result.groundtruth_result = True
metric_result.groundtruth_error_message = "all OK"
else:
metric_result.groundtruth_result = False
metric_result.groundtruth_error_message = "groundtruth missmatch: %f not within %f+-%f" % (
metric_result.data, metric_result.groundtruth,
metric_result.groundtruth_epsilon)
#print metric_result.groundtruth_error_message
if metric_result.data == None:
metric_result.groundtruth_result = False
metric_result.groundtruth_error_message = "no result"
#print "\nmetric_result:\n", metric_result
return metric_result
def execute(self):
# small testblock (circle r=0.5, time=3)
self.atf.start("testblock_small")
self.pub_tf_circle("link1", "link2", radius=1, time=3)
# user result
metric_result = MetricResult()
metric_result.data = 0.8
metric_result.groundtruth_result = True
metric_result.groundtruth_error_message = "all ok in application of atf_test"
self.atf.stop("testblock_small", metric_result)
# large testblock (circle r=1, time=5)
self.atf.start("testblock_large")
self.pub_tf_circle("link1", "link2", radius=2, time=5)
# user result
metric_result = MetricResult()
metric_result.data = 0.7
self.atf.stop("testblock_large", metric_result)
# shutdown atf
self.atf.shutdown()
def get_result(self):
metric_result = MetricResult()
metric_result.name = self.name
metric_result.mode = self.mode
metric_result.started = self.started # FIXME remove
metric_result.finished = self.finished # FIXME remove
metric_result.series = []
metric_result.groundtruth.available = self.groundtruth.available
metric_result.groundtruth.data = self.groundtruth.data
metric_result.groundtruth.epsilon = self.groundtruth.epsilon
# assign default value
metric_result.groundtruth.result = None
metric_result.groundtruth.error_message = None
if metric_result.started and metric_result.finished and len(
self.series
) != 0: # we check if the testblock was ever started and stopped and if result data is available
# calculate metric data
if self.series_mode != None:
metric_result.series = self.series
if metric_result.mode == MetricResult.SNAP:
metric_result.data = self.series[
-1] # take last element from self.series for data and stamp
metric_result.min = metric_result.data
metric_result.max = metric_result.data
metric_result.mean = metric_result.data.data
metric_result.std = 0.0
elif metric_result.mode == MetricResult.SPAN_MEAN:
metric_result.min = metrics_helper.get_min(self.series)
metric_result.max = metrics_helper.get_max(self.series)
metric_result.mean = metrics_helper.get_mean(self.series)
metric_result.std = metrics_helper.get_std(self.series)
metric_result.data.data = metric_result.mean # take mean for data
metric_result.data.stamp = self.series[
-1].stamp # take stamp from last element in self.series for stamp
elif metric_result.mode == MetricResult.SPAN_MIN:
metric_result.min = metrics_helper.get_min(self.series)
metric_result.max = metrics_helper.get_max(self.series)
metric_result.mean = metrics_helper.get_mean(self.series)
metric_result.std = metrics_helper.get_std(self.series)
metric_result.data = metric_result.min
elif metric_result.mode == MetricResult.SPAN_ABSMIN:
metric_result.min = metrics_helper.get_absmin(self.series)
metric_result.max = metrics_helper.get_absmax(self.series)
metric_result.mean = metrics_helper.get_mean(self.series)
metric_result.std = metrics_helper.get_std(self.series)
metric_result.data = metric_result.min
elif metric_result.mode == MetricResult.SPAN_MAX:
metric_result.min = metrics_helper.get_min(self.series)
metric_result.max = metrics_helper.get_max(self.series)
metric_result.mean = metrics_helper.get_mean(self.series)
metric_result.std = metrics_helper.get_std(self.series)
metric_result.data = metric_result.max
elif metric_result.mode == MetricResult.SPAN_ABSMAX:
metric_result.min = metrics_helper.get_absmin(self.series)
metric_result.max = metrics_helper.get_absmax(self.series)
metric_result.mean = metrics_helper.get_mean(self.series)
metric_result.std = metrics_helper.get_std(self.series)
metric_result.data = metric_result.max
else: # invalid mode
raise ATFAnalyserError(
"Analysing failed, invalid mode '%s' for metric '%s'." %
(metric_result.mode, metric_result.name))
# fill details as KeyValue messages
details = []
details.append(KeyValue("root_frame", self.root_frame))
details.append(KeyValue("measured_frame", self.measured_frame))
metric_result.details = details
# evaluate metric data
if not metric_result.groundtruth.available: # no groundtruth given
metric_result.groundtruth.result = True
metric_result.groundtruth.error_message = "all OK (no groundtruth available)"
else: # groundtruth available
if math.fabs(metric_result.groundtruth.data -
metric_result.data.data
) <= metric_result.groundtruth.epsilon:
metric_result.groundtruth.result = True
metric_result.groundtruth.error_message = "all OK"
else:
metric_result.groundtruth.result = False
metric_result.groundtruth.error_message = "groundtruth missmatch: %f not within %f+-%f" % (
metric_result.data.data,
metric_result.groundtruth.data,
metric_result.groundtruth.epsilon)
else: # testblock did not start and/or finish
metric_result.groundtruth.result = False
metric_result.groundtruth.error_message = "no result"
if metric_result.groundtruth.result == None:
raise ATFAnalyserError(
"Analysing failed, metric result is None for metric '%s'." %
metric_result.name)
return metric_result
def get_result(self):
metric_result = MetricResult()
metric_result.name = self.name
metric_result.started = self.started # FIXME remove
metric_result.finished = self.finished # FIXME remove
# check if user result is set at all
if self.metric_result == None:
#print "no user result set"
metric_result.groundtruth.result = False
metric_result.groundtruth.error_message = "no result"
#print "EXIT 0", metric_result.data.data, metric_result.groundtruth.result, metric_result.groundtruth.error_message
return metric_result
# check if groundtruth is set via user result (not all default values anymore)
if self.metric_result.groundtruth.result\
or self.metric_result.groundtruth.available\
or self.metric_result.groundtruth.error_message != ""\
or self.metric_result.groundtruth.data != 0\
or self.metric_result.groundtruth.epsilon != 0:
#print "groundtruth data is set from user within atf application for testblock %s. Skipping groundtruth evaluation from test_config"%self.testblock_name
# use data from user result
metric_result = self.metric_result
# overwrite user_result data with mandatory ATF filds
metric_result.name = self.name
metric_result.started = True
metric_result.finished = True
metric_result.groundtruth.available = True
#print "EXIT 1", metric_result.data.data, metric_result.groundtruth.result, metric_result.groundtruth.error_message
return metric_result
#print "no groundtruth set via user_result", self.metric_result.groundtruth
metric_result.groundtruth.available = self.groundtruth.available
metric_result.groundtruth.data = self.groundtruth.data
metric_result.groundtruth.epsilon = self.groundtruth.epsilon
# assign default value
metric_result.groundtruth.result = None
metric_result.groundtruth.error_message = None
if metric_result.started and metric_result.finished: # we check if the testblock was ever started and stopped
# calculate metric data
# check if user has set any metric_result (not all default values anymore)
if not self.metric_result.started\
and not self.metric_result.finished\
and len(self.metric_result.series) == 0\
and self.metric_result.data.stamp == rospy.Time(0)\
and self.metric_result.data.data == 0:
# let the analyzer know that this test failed
metric_result.groundtruth.result = False
metric_result.groundtruth.error_message = "user result for testblock %s not set" % self.testblock_name
#print "EXIT 2", metric_result.data.data, metric_result.groundtruth.result, metric_result.groundtruth.error_message
return metric_result
metric_result.data = self.metric_result.data
# fill details as KeyValue messages
metric_result.details = self.metric_result.details
# evaluate metric data
if not metric_result.groundtruth.available: # no groundtruth given
metric_result.groundtruth.result = True
metric_result.groundtruth.error_message = "all OK (no groundtruth available)"
else: # groundtruth available
if math.fabs(metric_result.groundtruth.data -
metric_result.data.data
) <= metric_result.groundtruth.epsilon:
metric_result.groundtruth.result = True
metric_result.groundtruth.error_message = "all OK"
else:
metric_result.groundtruth.result = False
metric_result.groundtruth.error_message = "groundtruth missmatch: %f not within %f+-%f" % (
metric_result.data.data,
metric_result.groundtruth.data,
metric_result.groundtruth.epsilon)
else: # testblock did not start and/or finish
metric_result.groundtruth.result = False
metric_result.groundtruth.error_message = "no result"
if metric_result.groundtruth.result == None:
raise ATFAnalyserError(
"Analysing failed, metric result is None for metric '%s'." %
metric_result.name)
#print "EXIT 3", metric_result.data.data, metric_result.groundtruth.result, metric_result.groundtruth.error_message
return metric_result