def pt_add_test(self, job, job_n, test_obj):
if not test_obj.avg_score:
return
self.data.append((job.product_ver if job.product_ver is not None else '?', pt_float2human(test_obj.avg_score)))
if not self.title or test_obj.less_better != self.less_better:
self.title = test_obj.metrics + " " + ("[lower is better]" if test_obj.less_better else "[bigger is better]")
self.less_better = test_obj.less_better
def table_row(self):
if not len(self.tests):
return []
t = self.tests
ret = ['', self.id, self.seq_num, self.title]
for n in range(0, len(t)):
if t[n]:
ret.append(pt_float2human(t[n].avg_score))
ret.append(
int(round(100 * t[n].avg_dev /
t[n].avg_score, 0)) if t[n].avg_score else 0)
ret.append(test_errors2str(t[n]))
else:
ret.append("-")
ret.append("-")
ret.append("")
for prev in range(0, n):
if t[prev] is None or t[n] is None:
ret.append("- 0")
continue
if not t[n].avg_score or not t[prev].avg_score:
d = 0
elif t[prev].avg_score < t[n].avg_score:
d = 100 * (t[n].avg_score / t[prev].avg_score - 1)
elif t[prev].avg_score > t[n].avg_score:
d = -100 * (t[prev].avg_score / t[n].avg_score - 1)
else:
d = 0
diff = 0
try:
s = stats.ttest_ind_from_stats(t[prev].avg_score,
t[prev].avg_dev,
t[prev].samples,
t[n].avg_score,
t[n].avg_dev, t[n].samples)
if s[1] < 0.1:
diff = 1
except ZeroDivisionError:
diff = 1
if diff:
if t[prev].avg_score < t[n].avg_score:
diff = (-1 if t[prev].less_better else 1)
elif t[prev].avg_score > t[n].avg_score:
diff = (1 if t[prev].less_better else -1)
elif t[prev].avg_score == t[n].avg_score:
diff = 0
ret.append(str(int(round(d, 0))) + " " + str(diff))
return ret
def _init_scores(self):
if self._scores:
return
self._scores = [None] * len(self.sect.x_axis_categories)
self._errors = [None] * len(self.sect.x_axis_categories)
maxi = 0
for t in self.tests:
if t.category not in self.sect.test_cat_to_axis_cat_seqnum:
print("WARNING: test category '%s' is not found in %s" % (t.category, str(self.sect.test_cat_to_axis_cat_seqnum)))
continue
i = self.sect.test_cat_to_axis_cat_seqnum[t.category]
maxi = max(maxi, i)
self._scores[i] = pt_float2human(t.avg_score)
self._errors[i] = t.errors or ((t.loops or "all") if t.status == 'FAILED' else 0)
self._scores = self._scores[:maxi + 1]
self._errors = self._errors[:maxi + 1]
def ptAddTest(self, job, job_n, test_obj):
self.series.append(pt_float2human(test_obj.avg_score))