def construct_results(self, vals, testname):
if 'responsiveness' in testname:
return filter.responsiveness_Metric([val for (val, page) in vals])
elif testname.startswith('v8_7'):
return self.v8_Metric(vals)
elif testname.startswith('kraken'):
return self.JS_Metric(vals)
elif testname.startswith('tcanvasmark'):
return self.CanvasMark_Metric(vals)
elif len(vals) > 1:
return filter.geometric_mean([i for i, j in vals])
else:
return filter.mean([i for i, j in vals])
def construct_results(self, vals, testname):
if 'responsiveness' in testname:
return filter.responsiveness_Metric([val for (val, page) in vals])
elif testname.startswith('v8_7'):
return self.v8_Metric(vals)
elif testname.startswith('kraken'):
return self.JS_Metric(vals)
elif testname.startswith('tcanvasmark'):
return self.CanvasMark_Metric(vals)
elif len(vals) > 1:
return filter.geometric_mean([i for i, j in vals])
else:
return filter.mean([i for i, j in vals])
def construct_results(self, vals, testname):
if 'responsiveness' in testname:
return filter.responsiveness_Metric([val for (val, page) in vals])
elif testname.startswith('v8_7'):
return self.v8_Metric(vals)
elif testname.startswith('kraken'):
return self.JS_Metric(vals)
elif testname.startswith('speedometer'):
return self.speedometer_score(vals)
elif testname.startswith('stylebench'):
return self.stylebench_score(vals)
elif len(vals) > 1:
return filter.geometric_mean([i for i, j in vals])
else:
return filter.mean([i for i, j in vals])
def __call__(self):
suites = []
test_results = {
'framework': {
'name': self.results.results[0].framework,
},
'suites': suites,
}
for test in self.results.results:
# serialize test results
tsresult = None
if not test.using_xperf:
subtests = []
suite = {
'name': test.name(),
'subtests': subtests,
}
if self.results.extra_options:
suite['extraOptions'] = self.results.extra_options
suites.append(suite)
vals = []
replicates = {}
# TODO: counters!!!! we don't have any, but they suffer the
# same
for result in test.results:
# XXX this will not work for manifests which list
# the same page name twice. It also ignores cycles
for page, val in result.raw_values():
if page == 'NULL':
page = test.name()
if tsresult is None:
tsresult = r = TalosResults.Results()
r.results = [{
'index': 0,
'page': test.name(),
'runs': val
}]
else:
r = tsresult.results[0]
if r['page'] == test.name():
r['runs'].extend(val)
replicates.setdefault(page, []).extend(val)
tresults = [tsresult] if tsresult else test.results
for result in tresults:
filtered_results = \
result.values(suite['name'],
test.test_config['filters'])
vals.extend([[i['value'], j] for i, j in filtered_results])
for val, page in filtered_results:
if page == 'NULL':
# no real subtests
page = test.name()
subtest = {
'name': page,
'value': val['filtered'],
'replicates': replicates[page],
}
subtests.append(subtest)
if test.test_config.get('lower_is_better') is not None:
subtest['lowerIsBetter'] = \
test.test_config['lower_is_better']
if test.test_config.get('alert_threshold') is not None:
subtest['alertThreshold'] = \
test.test_config['alert_threshold']
if test.test_config.get('unit'):
subtest['unit'] = test.test_config['unit']
# if there is more than one subtest, calculate a summary result
if len(subtests) > 1:
suite['value'] = self.construct_results(
vals, testname=test.name())
if test.test_config.get('lower_is_better') is not None:
suite['lowerIsBetter'] = \
test.test_config['lower_is_better']
if test.test_config.get('alert_threshold') is not None:
suite['alertThreshold'] = \
test.test_config['alert_threshold']
# counters results_aux data
counter_subtests = []
for cd in test.all_counter_results:
for name, vals in cd.items():
# We want to add the xperf data as talos_counters
# exclude counters whose values are tuples (bad for
# graphserver)
if len(vals) > 0 and isinstance(vals[0], list):
continue
# mainthread IO is a list of filenames and accesses, we do
# not report this as a counter
if 'mainthreadio' in name:
continue
# responsiveness has it's own metric, not the mean
# TODO: consider doing this for all counters
if 'responsiveness' is name:
subtest = {
'name': name,
'value': filter.responsiveness_Metric(vals)
}
counter_subtests.append(subtest)
continue
subtest = {
'name': name,
'value': 0.0,
}
counter_subtests.append(subtest)
if test.using_xperf:
if len(vals) > 0:
subtest['value'] = vals[0]
else:
# calculate mean value
if len(vals) > 0:
varray = [float(v) for v in vals]
subtest['value'] = filter.mean(varray)
if counter_subtests:
suites.append({
'name': test.name(),
'subtests': counter_subtests
})
return test_results
def __call__(self):
suites = []
test_results = {
'framework': {
'name': self.results.results[0].framework,
},
'suites': suites,
}
for test in self.results.results:
# serialize test results
tsresult = None
if not test.using_xperf:
subtests = []
suite = {
'name': test.name(),
'subtests': subtests,
}
if self.results.extra_options:
suite['extraOptions'] = self.results.extra_options
suites.append(suite)
vals = []
replicates = {}
# TODO: counters!!!! we don't have any, but they suffer the
# same
for result in test.results:
# XXX this will not work for manifests which list
# the same page name twice. It also ignores cycles
for page, val in result.raw_values():
if page == 'NULL':
page = test.name()
if tsresult is None:
tsresult = r = TalosResults.Results()
r.results = [{'index': 0, 'page': test.name(),
'runs': val}]
else:
r = tsresult.results[0]
if r['page'] == test.name():
r['runs'].extend(val)
replicates.setdefault(page, []).extend(val)
tresults = [tsresult] if tsresult else test.results
for result in tresults:
filtered_results = \
result.values(suite['name'],
test.test_config['filters'])
vals.extend([[i['value'], j] for i, j in filtered_results])
for val, page in filtered_results:
if page == 'NULL':
# no real subtests
page = test.name()
subtest = {
'name': page,
'value': val['filtered'],
'replicates': replicates[page],
}
subtests.append(subtest)
if test.test_config.get('lower_is_better') is not None:
subtest['lowerIsBetter'] = \
test.test_config['lower_is_better']
if test.test_config.get('alert_threshold') is not None:
subtest['alertThreshold'] = \
test.test_config['alert_threshold']
if test.test_config.get('unit'):
subtest['unit'] = test.test_config['unit']
# if there is more than one subtest, calculate a summary result
if len(subtests) > 1:
suite['value'] = self.construct_results(
vals, testname=test.name())
if test.test_config.get('lower_is_better') is not None:
suite['lowerIsBetter'] = \
test.test_config['lower_is_better']
if test.test_config.get('alert_threshold') is not None:
suite['alertThreshold'] = \
test.test_config['alert_threshold']
# counters results_aux data
counter_subtests = []
for cd in test.all_counter_results:
for name, vals in cd.items():
# We want to add the xperf data as talos_counters
# exclude counters whose values are tuples (bad for
# graphserver)
if len(vals) > 0 and isinstance(vals[0], list):
continue
# mainthread IO is a list of filenames and accesses, we do
# not report this as a counter
if 'mainthreadio' in name:
continue
# responsiveness has it's own metric, not the mean
# TODO: consider doing this for all counters
if 'responsiveness' is name:
subtest = {
'name': name,
'value': filter.responsiveness_Metric(vals)
}
counter_subtests.append(subtest)
continue
subtest = {
'name': name,
'value': 0.0,
}
counter_subtests.append(subtest)
if test.using_xperf:
if len(vals) > 0:
subtest['value'] = vals[0]
else:
# calculate mean value
if len(vals) > 0:
varray = [float(v) for v in vals]
subtest['value'] = filter.mean(varray)
if counter_subtests:
suites.append({'name': test.name(),
'subtests': counter_subtests})
return test_results
