def run_tests(configurator):
"""Runs the talos tests on the given configuration and generates a report.
Args:
config: dictionary of configuration, as generated by PerfConfigurator
"""
config=configurator.config
# data filters
filters = config['filters']
try:
filters = filter.filters_args(filters)
except AssertionError, e:
raise talosError(str(e))
def run_tests(configurator):
"""Runs the talos tests on the given configuration and generates a report.
Args:
config: dictionary of configuration, as generated by PerfConfigurator
"""
config = configurator.config
# data filters
filters = config['filters']
try:
filters = filter.filters_args(filters)
except AssertionError, e:
raise talosError(str(e))
# https://bugzilla.mozilla.org/show_bug.cgi?id=727711
# Build command line from config
for path in paths:
if test.get(path):
test[path] = utils.interpolatePath(test[path])
if test.get('tpmanifest'):
test['tpmanifest'] = os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'], '/\\t:\\')))
if not test.get('url'):
# build 'url' for tptest
test['url'] = buildCommandLine(test)
test['url'] = utils.interpolatePath(test['url'])
# ensure test-specific filters are valid
if 'filters' in test:
try:
filter.filters_args(test['filters'])
except AssertionError, e:
raise talosError(str(e))
except IndexError, e:
raise talosError(str(e))
# set browser_config
browser_config=configurator.browser_config()
#set defaults
title = config.get('title', '')
testdate = config.get('testdate', '')
# get the process name from the path to the browser
if not browser_config['process']:
def __call__(self):
"""
results to send to graphserver:
construct all the strings of data, one string per test and one string per counter
"""
result_strings = []
info_dict = dict(title=self.results.title,
date=self.results.date,
branch_name=self.results.browser_config['branch_name'],
sourcestamp=self.results.browser_config['sourcestamp'],
buildid=self.results.browser_config['buildid'],
browser_name=self.results.browser_config['browser_name'],
browser_version=self.results.browser_config['browser_version']
)
for test in self.results.results:
utils.debug("Working with test: %s", test.name())
# get full name of test
testname = test.name()
if test.format == 'tpformat':
# for some reason, we append the test extension to tp results but not ts
# http://hg.mozilla.org/build/talos/file/170c100911b6/talos/run_tests.py#l176
testname += self.results.test_name_extension
utils.stamped_msg("Generating results file: %s" % test.name(), "Started")
# HACK: when running xperf, we upload xperf counters to the graph server but we do not want to
# upload the test results as they will confuse the graph server
if not test.using_xperf:
vals = []
for result in test.results:
# per test filters
_filters = self.results.filters
if 'filters' in test.test_config:
try:
_filters = filter.filters_args(test.test_config['filters'])
except AssertionError, e:
raise utils.talosError(str(e))
vals.extend(result.values(_filters))
result_strings.append(self.construct_results(vals, testname=testname, **info_dict))
utils.stamped_msg("Generating results file: %s" % test.name(), "Stopped")
# counter results
for cd in test.all_counter_results:
for counter_type, values in cd.items():
# get the counter name
counterName = '%s_%s' % (test.name() , self.shortName(counter_type))
if not values:
# failed to collect any data for this counter
utils.stamped_msg("No results collected for: " + counterName, "Error")
# NOTE: we are not going to enforce this warning for now as this happens too frequently: bugs 803413, 802475, 805925
# raise utils.talosError("Unable to proceed with missing counter '%s'" % counterName)
# (jhammel: we probably should do this in e.g. results.py vs in graphserver-specific code anyway)
# exclude counters whose values are tuples (bad for graphserver)
if len(values) > 0 and isinstance(values[0], list):
continue
# counter values
vals = [[x, 'NULL'] for x in values]
# append test name extension but only for tpformat tests
if test.format == 'tpformat':
counterName += self.results.test_name_extension
info = info_dict.copy()
info['testname'] = counterName
# append the counter string
utils.stamped_msg("Generating results file: %s" % counterName, "Started")
result_strings.append(self.construct_results(vals, **info))
utils.stamped_msg("Generating results file: %s" % counterName, "Stopped")
if test.get(path):
test[path] = utils.interpolatePath(test[path])
if test.get('tpmanifest'):
test['tpmanifest'] = os.path.normpath(
'file:/%s' % (urllib.quote(test['tpmanifest'], '/\\t:\\')))
if not test.get('url'):
# build 'url' for tptest
test['url'] = buildCommandLine(test)
test['url'] = utils.interpolatePath(test['url'])
test['setup'] = utils.interpolatePath(test['setup'])
test['cleanup'] = utils.interpolatePath(test['cleanup'])
# ensure test-specific filters are valid
if 'filters' in test:
try:
filter.filters_args(test['filters'])
except AssertionError, e:
raise talosError(str(e))
except IndexError, e:
raise talosError(str(e))
# set browser_config
browser_config = configurator.browser_config()
#set defaults
title = config.get('title', '')
testdate = config.get('testdate', '')
# Bug 940690 - Get existing metrofx talos tests running on release/project
# branches:
# Identify when we do a win8 metro talos run, and a non metro run. We do
def __call__(self):
"""
results to send to graphserver:
construct all the strings of data, one string per test and one string per counter
"""
result_strings = []
info_dict = dict(title=self.results.title,
date=self.results.date,
branch_name=self.results.browser_config['branch_name'],
sourcestamp=self.results.browser_config['sourcestamp'],
buildid=self.results.browser_config['buildid'],
browser_name=self.results.browser_config['browser_name'],
browser_version=self.results.browser_config['browser_version']
)
for test in self.results.results:
utils.debug("Working with test: %s", test.name())
# get full name of test
testname = test.name()
if test.format == 'tpformat':
# for some reason, we append the test extension to tp results but not ts
# http://hg.mozilla.org/build/talos/file/170c100911b6/talos/run_tests.py#l176
testname += self.results.test_name_extension
utils.stamped_msg("Generating results file: %s" % test.name(), "Started")
# HACK: when running xperf, we upload xperf counters to the graph server but we do not want to
# upload the test results as they will confuse the graph server
if not test.using_xperf:
vals = []
for result in test.results:
# per test filters
_filters = self.results.filters
if 'filters' in test.test_config:
try:
_filters = filter.filters_args(test.test_config['filters'])
except AssertionError, e:
raise utils.talosError(str(e))
vals.extend(result.values(_filters))
result_strings.append(self.construct_results(vals, testname=testname, **info_dict))
utils.stamped_msg("Generating results file: %s" % test.name(), "Stopped")
# counter results
for cd in test.all_counter_results:
for counter_type, values in cd.items():
# get the counter name
counterName = '%s_%s' % (test.name() , self.shortName(counter_type))
if not values:
# failed to collect any data for this counter
utils.stamped_msg("No results collected for: " + counterName, "Error")
# NOTE: we are not going to enforce this warning for now as this happens too frequently: bugs 803413, 802475, 805925
# raise utils.talosError("Unable to proceed with missing counter '%s'" % counterName)
# (jhammel: we probably should do this in e.g. results.py vs in graphserver-specific code anyway)
# exclude counters whose values are tuples (bad for graphserver)
if len(values) > 0 and isinstance(values[0], list):
continue
# counter values
vals = [[x, 'NULL'] for x in values]
# append test name extension but only for tpformat tests
if test.format == 'tpformat':
counterName += self.results.test_name_extension
info = info_dict.copy()
info['testname'] = counterName
# append the counter string
utils.stamped_msg("Generating results file: %s" % counterName, "Started")
result_strings.append(self.construct_results(vals, **info))
utils.stamped_msg("Generating results file: %s" % counterName, "Stopped")