def run_tests(config, browser_config):
"""Runs the talos tests on the given configuration and generates a report.
"""
# get the test data
tests = config['tests']
tests = useBaseTestDefaults(config.get('basetest', {}), tests)
paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup']
for test in tests:
# Check for profile_path, tpmanifest and interpolate based on Talos
# root https://bugzilla.mozilla.org/show_bug.cgi?id=727711
# Build command line from config
for path in paths:
if test.get(path):
test[path] = utils.interpolate(test[path])
if test.get('tpmanifest'):
test['tpmanifest'] = \
os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'],
'/\\t:\\')))
if not test.get('url'):
# build 'url' for tptest
test['url'] = buildCommandLine(test)
test['url'] = utils.interpolate(test['url'])
test['setup'] = utils.interpolate(test['setup'])
test['cleanup'] = utils.interpolate(test['cleanup'])
# pass --no-remote to firefox launch, if --develop is specified
# we do that to allow locally the user to have another running firefox
# instance
if browser_config['develop']:
browser_config['extra_args'] = '--no-remote'
# with addon signing for production talos, we want to develop without it
if browser_config['develop'] or browser_config['branch_name'] == 'Try':
browser_config['preferences']['xpinstall.signatures.required'] = False
browser_config['extensions'] = [
os.path.dirname(i) for i in browser_config['extensions']
]
# set defaults
title = config.get('title', '')
testdate = config.get('testdate', '')
if browser_config['e10s'] and not title.endswith(".e"):
# we are running in e10s mode
title = "%s.e" % (title, )
# get the process name from the path to the browser
if not browser_config['process']:
browser_config['process'] = \
os.path.basename(browser_config['browser_path'])
# fix paths to substitute
# `os.path.dirname(os.path.abspath(__file__))` for ${talos}
# https://bugzilla.mozilla.org/show_bug.cgi?id=705809
browser_config['extensions'] = [
utils.interpolate(i) for i in browser_config['extensions']
]
browser_config['bcontroller_config'] = \
utils.interpolate(browser_config['bcontroller_config'])
# normalize browser path to work across platforms
browser_config['browser_path'] = \
os.path.normpath(browser_config['browser_path'])
binary = browser_config["browser_path"]
version_info = mozversion.get_version(binary=binary)
browser_config['browser_name'] = version_info['application_name']
browser_config['browser_version'] = version_info['application_version']
browser_config['buildid'] = version_info['application_buildid']
try:
browser_config['repository'] = version_info['application_repository']
browser_config['sourcestamp'] = version_info['application_changeset']
except KeyError:
if not browser_config['develop']:
print "unable to find changeset or repository: %s" % version_info
sys.exit()
else:
browser_config['repository'] = 'develop'
browser_config['sourcestamp'] = 'develop'
# get test date in seconds since epoch
if testdate:
date = int(
time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT')))
else:
date = int(time.time())
LOG.debug("using testdate: %d" % date)
LOG.debug("actual date: %d" % int(time.time()))
# results container
talos_results = TalosResults()
# results links
if not browser_config['develop']:
results_urls = dict(
# another hack; datazilla stands for Perfherder
# and do not require url, but a non empty dict is required...
datazilla_urls=['local.json'], )
else:
# local mode, output to files
results_urls = dict(datazilla_urls=[os.path.abspath('local.json')])
talos_results.check_output_formats(results_urls)
httpd = setup_webserver(browser_config['webserver'])
httpd.start()
testname = None
# run the tests
timer = utils.Timer()
LOG.suite_start(tests=[test['name'] for test in tests])
try:
for test in tests:
testname = test['name']
LOG.test_start(testname)
mytest = TTest()
talos_results.add(mytest.runTest(browser_config, test))
LOG.test_end(testname, status='OK')
except TalosRegression as exc:
LOG.error("Detected a regression for %s" % testname)
# by returning 1, we report an orange to buildbot
# http://docs.buildbot.net/latest/developer/results.html
LOG.test_end(testname,
status='FAIL',
message=unicode(exc),
stack=traceback.format_exc())
return 1
except Exception as exc:
# NOTE: if we get into this condition, talos has an internal
# problem and cannot continue
# this will prevent future tests from running
LOG.test_end(testname,
status='ERROR',
message=unicode(exc),
stack=traceback.format_exc())
# indicate a failure to buildbot, turn the job red
return 2
finally:
LOG.suite_end()
httpd.stop()
LOG.info("Completed test suite (%s)" % timer.elapsed())
# output results
if results_urls:
talos_results.output(results_urls)
if browser_config['develop']:
print("Thanks for running Talos locally. Results are in %s" %
(results_urls['datazilla_urls']))
# we will stop running tests on a failed test, or we will return 0 for
# green
return 0
def run_tests(config, browser_config):
"""Runs the talos tests on the given configuration and generates a report.
"""
# get the test data
tests = config['tests']
tests = useBaseTestDefaults(config.get('basetest', {}), tests)
paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup']
for test in tests:
# Check for profile_path, tpmanifest and interpolate based on Talos
# root https://bugzilla.mozilla.org/show_bug.cgi?id=727711
# Build command line from config
for path in paths:
if test.get(path):
test[path] = utils.interpolate(test[path])
if test.get('tpmanifest'):
test['tpmanifest'] = \
os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'],
'/\\t:\\')))
if not test.get('url'):
# build 'url' for tptest
test['url'] = buildCommandLine(test)
test['url'] = utils.interpolate(test['url'])
test['setup'] = utils.interpolate(test['setup'])
test['cleanup'] = utils.interpolate(test['cleanup'])
# pass --no-remote to firefox launch, if --develop is specified
if browser_config['develop']:
browser_config['extra_args'] = '--no-remote'
# set defaults
title = config.get('title', '')
testdate = config.get('testdate', '')
if browser_config['e10s'] and not title.endswith(".e"):
# we are running in e10s mode
title = "%s.e" % (title, )
# get the process name from the path to the browser
if not browser_config['process']:
browser_config['process'] = \
os.path.basename(browser_config['browser_path'])
# fix paths to substitute
# `os.path.dirname(os.path.abspath(__file__))` for ${talos}
# https://bugzilla.mozilla.org/show_bug.cgi?id=705809
browser_config['extensions'] = [
utils.interpolate(i) for i in browser_config['extensions']
]
browser_config['bcontroller_config'] = \
utils.interpolate(browser_config['bcontroller_config'])
# normalize browser path to work across platforms
browser_config['browser_path'] = \
os.path.normpath(browser_config['browser_path'])
binary = browser_config["browser_path"]
version_info = mozversion.get_version(binary=binary)
browser_config['browser_name'] = version_info['application_name']
browser_config['browser_version'] = version_info['application_version']
browser_config['buildid'] = version_info['application_buildid']
try:
browser_config['repository'] = version_info['application_repository']
browser_config['sourcestamp'] = version_info['application_changeset']
except KeyError:
if not browser_config['develop']:
print "unable to find changeset or repository: %s" % version_info
sys.exit()
else:
browser_config['repository'] = 'develop'
browser_config['sourcestamp'] = 'develop'
# get test date in seconds since epoch
if testdate:
date = int(
time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT')))
else:
date = int(time.time())
logging.debug("using testdate: %d", date)
logging.debug("actual date: %d", int(time.time()))
# results container
talos_results = TalosResults(title=title,
date=date,
browser_config=browser_config)
# results links
if not browser_config['develop']:
results_urls = dict(
# hardcoded, this will be removed soon anyway.
results_urls=['http://graphs.mozilla.org/server/collect.cgi'],
# another hack; datazilla stands for Perfherder
# and do not require url, but a non empty dict is required...
datazilla_urls=['local.json'],
)
else:
# local mode, output to files
results_urls = dict(results_urls=[os.path.abspath('local.out')],
datazilla_urls=[os.path.abspath('local.json')])
talos_results.check_output_formats(results_urls)
# setup a webserver, if --develop is specified
httpd = None
if browser_config['develop']:
httpd = setup_webserver(browser_config['webserver'])
if httpd:
httpd.start()
# run the tests
timer = utils.Timer()
logging.info("Starting test suite %s", title)
for test in tests:
testname = test['name']
testtimer = utils.Timer()
logging.info("Starting test %s", testname)
try:
mytest = TTest()
if mytest:
talos_results.add(mytest.runTest(browser_config, test))
else:
logging.error("Error found while running %s", testname)
except TalosRegression:
logging.error("Detected a regression for %s", testname)
if httpd:
httpd.stop()
# by returning 1, we report an orange to buildbot
# http://docs.buildbot.net/latest/developer/results.html
return 1
except (TalosCrash, TalosError):
# NOTE: if we get into this condition, talos has an internal
# problem and cannot continue
# this will prevent future tests from running
traceback.print_exception(*sys.exc_info())
if httpd:
httpd.stop()
# indicate a failure to buildbot, turn the job red
return 2
logging.info("Completed test %s (%s)", testname, testtimer.elapsed())
logging.info("Completed test suite (%s)", timer.elapsed())
# stop the webserver if running
if httpd:
httpd.stop()
# output results
if results_urls:
talos_results.output(results_urls)
if browser_config['develop']:
print
print(
"Thanks for running Talos locally. Results are in"
" %s and %s" %
(results_urls['results_urls'], results_urls['datazilla_urls']))
# we will stop running tests on a failed test, or we will return 0 for
# green
return 0
def run_tests(config, browser_config):
"""Runs the talos tests on the given configuration and generates a report.
"""
# get the test data
tests = config['tests']
tests = useBaseTestDefaults(config.get('basetest', {}), tests)
paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup']
for test in tests:
# Check for profile_path, tpmanifest and interpolate based on Talos
# root https://bugzilla.mozilla.org/show_bug.cgi?id=727711
# Build command line from config
for path in paths:
if test.get(path):
test[path] = utils.interpolate(test[path])
if test.get('tpmanifest'):
test['tpmanifest'] = \
os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'],
'/\\t:\\')))
if not test.get('url'):
# build 'url' for tptest
test['url'] = buildCommandLine(test)
test['url'] = utils.interpolate(test['url'])
test['setup'] = utils.interpolate(test['setup'])
test['cleanup'] = utils.interpolate(test['cleanup'])
# pass --no-remote to firefox launch, if --develop is specified
# we do that to allow locally the user to have another running firefox
# instance
if browser_config['develop']:
browser_config['extra_args'] = '--no-remote'
# with addon signing for production talos, we want to develop without it
if browser_config['develop'] or browser_config['branch_name'] == 'Try':
browser_config['preferences']['xpinstall.signatures.required'] = False
browser_config['extensions'] = [os.path.dirname(i)
for i in browser_config['extensions']]
# set defaults
title = config.get('title', '')
testdate = config.get('testdate', '')
if browser_config['e10s'] and not title.endswith(".e"):
# we are running in e10s mode
title = "%s.e" % (title,)
# get the process name from the path to the browser
if not browser_config['process']:
browser_config['process'] = \
os.path.basename(browser_config['browser_path'])
# fix paths to substitute
# `os.path.dirname(os.path.abspath(__file__))` for ${talos}
# https://bugzilla.mozilla.org/show_bug.cgi?id=705809
browser_config['extensions'] = [utils.interpolate(i)
for i in browser_config['extensions']]
browser_config['bcontroller_config'] = \
utils.interpolate(browser_config['bcontroller_config'])
# normalize browser path to work across platforms
browser_config['browser_path'] = \
os.path.normpath(browser_config['browser_path'])
binary = browser_config["browser_path"]
version_info = mozversion.get_version(binary=binary)
browser_config['browser_name'] = version_info['application_name']
browser_config['browser_version'] = version_info['application_version']
browser_config['buildid'] = version_info['application_buildid']
try:
browser_config['repository'] = version_info['application_repository']
browser_config['sourcestamp'] = version_info['application_changeset']
except KeyError:
if not browser_config['develop']:
print "unable to find changeset or repository: %s" % version_info
sys.exit()
else:
browser_config['repository'] = 'develop'
browser_config['sourcestamp'] = 'develop'
# get test date in seconds since epoch
if testdate:
date = int(time.mktime(time.strptime(testdate,
'%a, %d %b %Y %H:%M:%S GMT')))
else:
date = int(time.time())
LOG.debug("using testdate: %d" % date)
LOG.debug("actual date: %d" % int(time.time()))
# results container
talos_results = TalosResults()
# results links
if not browser_config['develop']:
results_urls = dict(
# another hack; datazilla stands for Perfherder
# and do not require url, but a non empty dict is required...
datazilla_urls=['local.json'],
)
else:
# local mode, output to files
results_urls = dict(datazilla_urls=[os.path.abspath('local.json')])
talos_results.check_output_formats(results_urls)
httpd = setup_webserver(browser_config['webserver'])
httpd.start()
testname = None
# run the tests
timer = utils.Timer()
LOG.suite_start(tests=[test['name'] for test in tests])
try:
for test in tests:
testname = test['name']
LOG.test_start(testname)
mytest = TTest()
talos_results.add(mytest.runTest(browser_config, test))
LOG.test_end(testname, status='OK')
except TalosRegression as exc:
LOG.error("Detected a regression for %s" % testname)
# by returning 1, we report an orange to buildbot
# http://docs.buildbot.net/latest/developer/results.html
LOG.test_end(testname, status='FAIL', message=unicode(exc),
stack=traceback.format_exc())
return 1
except Exception as exc:
# NOTE: if we get into this condition, talos has an internal
# problem and cannot continue
# this will prevent future tests from running
LOG.test_end(testname, status='ERROR', message=unicode(exc),
stack=traceback.format_exc())
# indicate a failure to buildbot, turn the job red
return 2
finally:
LOG.suite_end()
httpd.stop()
LOG.info("Completed test suite (%s)" % timer.elapsed())
# output results
if results_urls:
talos_results.output(results_urls)
if browser_config['develop']:
print ("Thanks for running Talos locally. Results are in %s"
% (results_urls['datazilla_urls']))
# we will stop running tests on a failed test, or we will return 0 for
# green
return 0
def run_tests(config, browser_config):
"""Runs the talos tests on the given configuration and generates a report.
"""
# get the test data
tests = config['tests']
tests = useBaseTestDefaults(config.get('basetest', {}), tests)
paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup']
for test in tests:
# Check for profile_path, tpmanifest and interpolate based on Talos
# root https://bugzilla.mozilla.org/show_bug.cgi?id=727711
# Build command line from config
for path in paths:
if test.get(path):
test[path] = utils.interpolate(test[path])
if test.get('tpmanifest'):
test['tpmanifest'] = \
os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'],
'/\\t:\\')))
if not test.get('url'):
# build 'url' for tptest
test['url'] = buildCommandLine(test)
test['url'] = utils.interpolate(test['url'])
test['setup'] = utils.interpolate(test['setup'])
test['cleanup'] = utils.interpolate(test['cleanup'])
# pass --no-remote to firefox launch, if --develop is specified
if browser_config['develop']:
browser_config['extra_args'] = '--no-remote'
# set defaults
title = config.get('title', '')
testdate = config.get('testdate', '')
if browser_config['e10s'] and not title.endswith(".e"):
# we are running in e10s mode
title = "%s.e" % (title,)
# get the process name from the path to the browser
if not browser_config['process']:
browser_config['process'] = \
os.path.basename(browser_config['browser_path'])
# fix paths to substitute
# `os.path.dirname(os.path.abspath(__file__))` for ${talos}
# https://bugzilla.mozilla.org/show_bug.cgi?id=705809
browser_config['extensions'] = [utils.interpolate(i)
for i in browser_config['extensions']]
browser_config['bcontroller_config'] = \
utils.interpolate(browser_config['bcontroller_config'])
# normalize browser path to work across platforms
browser_config['browser_path'] = \
os.path.normpath(browser_config['browser_path'])
binary = browser_config["browser_path"]
version_info = mozversion.get_version(binary=binary)
browser_config['browser_name'] = version_info['application_name']
browser_config['browser_version'] = version_info['application_version']
browser_config['buildid'] = version_info['application_buildid']
try:
browser_config['repository'] = version_info['application_repository']
browser_config['sourcestamp'] = version_info['application_changeset']
except KeyError:
if not browser_config['develop']:
print "unable to find changeset or repository: %s" % version_info
sys.exit()
else:
browser_config['repository'] = 'develop'
browser_config['sourcestamp'] = 'develop'
# get test date in seconds since epoch
if testdate:
date = int(time.mktime(time.strptime(testdate,
'%a, %d %b %Y %H:%M:%S GMT')))
else:
date = int(time.time())
logging.debug("using testdate: %d", date)
logging.debug("actual date: %d", int(time.time()))
# results container
talos_results = TalosResults(title=title,
date=date,
browser_config=browser_config)
# results links
if not browser_config['develop']:
results_urls = dict(
# hardcoded, this will be removed soon anyway.
results_urls=['http://graphs.mozilla.org/server/collect.cgi'],
# another hack; datazilla stands for Perfherder
# and do not require url, but a non empty dict is required...
datazilla_urls=['local.json'],
)
else:
# local mode, output to files
results_urls = dict(
results_urls=[os.path.abspath('local.out')],
datazilla_urls=[os.path.abspath('local.json')]
)
talos_results.check_output_formats(results_urls)
# setup a webserver, if --develop is specified
httpd = None
if browser_config['develop']:
httpd = setup_webserver(browser_config['webserver'])
if httpd:
httpd.start()
# run the tests
timer = utils.Timer()
logging.info("Starting test suite %s", title)
for test in tests:
testname = test['name']
testtimer = utils.Timer()
logging.info("Starting test %s", testname)
try:
mytest = TTest()
if mytest:
talos_results.add(mytest.runTest(browser_config, test))
else:
logging.error("Error found while running %s", testname)
except TalosRegression:
logging.error("Detected a regression for %s", testname)
if httpd:
httpd.stop()
# by returning 1, we report an orange to buildbot
# http://docs.buildbot.net/latest/developer/results.html
return 1
except (TalosCrash, TalosError):
# NOTE: if we get into this condition, talos has an internal
# problem and cannot continue
# this will prevent future tests from running
traceback.print_exception(*sys.exc_info())
if httpd:
httpd.stop()
# indicate a failure to buildbot, turn the job red
return 2
logging.info("Completed test %s (%s)", testname, testtimer.elapsed())
logging.info("Completed test suite (%s)", timer.elapsed())
# stop the webserver if running
if httpd:
httpd.stop()
# output results
if results_urls:
talos_results.output(results_urls)
if browser_config['develop']:
print
print ("Thanks for running Talos locally. Results are in"
" %s and %s" % (results_urls['results_urls'],
results_urls['datazilla_urls']))
# we will stop running tests on a failed test, or we will return 0 for
# green
return 0
