def run_tests(config, browser_config): """Runs the talos tests on the given configuration and generates a report. """ # get the test data tests = config['tests'] tests = useBaseTestDefaults(config.get('basetest', {}), tests) paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup'] for test in tests: # Check for profile_path, tpmanifest and interpolate based on Talos # root https://bugzilla.mozilla.org/show_bug.cgi?id=727711 # Build command line from config for path in paths: if test.get(path): test[path] = utils.interpolate(test[path]) if test.get('tpmanifest'): test['tpmanifest'] = \ os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'], '/\\t:\\'))) if not test.get('url'): # build 'url' for tptest test['url'] = buildCommandLine(test) test['url'] = utils.interpolate(test['url']) test['setup'] = utils.interpolate(test['setup']) test['cleanup'] = utils.interpolate(test['cleanup']) # pass --no-remote to firefox launch, if --develop is specified # we do that to allow locally the user to have another running firefox # instance if browser_config['develop']: browser_config['extra_args'] = '--no-remote' # with addon signing for production talos, we want to develop without it if browser_config['develop'] or browser_config['branch_name'] == 'Try': browser_config['preferences']['xpinstall.signatures.required'] = False browser_config['extensions'] = [ os.path.dirname(i) for i in browser_config['extensions'] ] # set defaults title = config.get('title', '') testdate = config.get('testdate', '') if browser_config['e10s'] and not title.endswith(".e"): # we are running in e10s mode title = "%s.e" % (title, ) # get the process name from the path to the browser if not browser_config['process']: browser_config['process'] = \ os.path.basename(browser_config['browser_path']) # fix paths to substitute # `os.path.dirname(os.path.abspath(__file__))` for ${talos} # https://bugzilla.mozilla.org/show_bug.cgi?id=705809 browser_config['extensions'] = [ utils.interpolate(i) for i in browser_config['extensions'] ] browser_config['bcontroller_config'] = \ utils.interpolate(browser_config['bcontroller_config']) # normalize browser path to work across platforms browser_config['browser_path'] = \ os.path.normpath(browser_config['browser_path']) binary = browser_config["browser_path"] version_info = mozversion.get_version(binary=binary) browser_config['browser_name'] = version_info['application_name'] browser_config['browser_version'] = version_info['application_version'] browser_config['buildid'] = version_info['application_buildid'] try: browser_config['repository'] = version_info['application_repository'] browser_config['sourcestamp'] = version_info['application_changeset'] except KeyError: if not browser_config['develop']: print "unable to find changeset or repository: %s" % version_info sys.exit() else: browser_config['repository'] = 'develop' browser_config['sourcestamp'] = 'develop' # get test date in seconds since epoch if testdate: date = int( time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT'))) else: date = int(time.time()) LOG.debug("using testdate: %d" % date) LOG.debug("actual date: %d" % int(time.time())) # results container talos_results = TalosResults() # results links if not browser_config['develop']: results_urls = dict( # another hack; datazilla stands for Perfherder # and do not require url, but a non empty dict is required... datazilla_urls=['local.json'], ) else: # local mode, output to files results_urls = dict(datazilla_urls=[os.path.abspath('local.json')]) talos_results.check_output_formats(results_urls) httpd = setup_webserver(browser_config['webserver']) httpd.start() testname = None # run the tests timer = utils.Timer() LOG.suite_start(tests=[test['name'] for test in tests]) try: for test in tests: testname = test['name'] LOG.test_start(testname) mytest = TTest() talos_results.add(mytest.runTest(browser_config, test)) LOG.test_end(testname, status='OK') except TalosRegression as exc: LOG.error("Detected a regression for %s" % testname) # by returning 1, we report an orange to buildbot # http://docs.buildbot.net/latest/developer/results.html LOG.test_end(testname, status='FAIL', message=unicode(exc), stack=traceback.format_exc()) return 1 except Exception as exc: # NOTE: if we get into this condition, talos has an internal # problem and cannot continue # this will prevent future tests from running LOG.test_end(testname, status='ERROR', message=unicode(exc), stack=traceback.format_exc()) # indicate a failure to buildbot, turn the job red return 2 finally: LOG.suite_end() httpd.stop() LOG.info("Completed test suite (%s)" % timer.elapsed()) # output results if results_urls: talos_results.output(results_urls) if browser_config['develop']: print("Thanks for running Talos locally. Results are in %s" % (results_urls['datazilla_urls'])) # we will stop running tests on a failed test, or we will return 0 for # green return 0
def run_tests(config, browser_config): """Runs the talos tests on the given configuration and generates a report. """ # get the test data tests = config['tests'] tests = useBaseTestDefaults(config.get('basetest', {}), tests) paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup'] for test in tests: # Check for profile_path, tpmanifest and interpolate based on Talos # root https://bugzilla.mozilla.org/show_bug.cgi?id=727711 # Build command line from config for path in paths: if test.get(path): test[path] = utils.interpolate(test[path]) if test.get('tpmanifest'): test['tpmanifest'] = \ os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'], '/\\t:\\'))) if not test.get('url'): # build 'url' for tptest test['url'] = buildCommandLine(test) test['url'] = utils.interpolate(test['url']) test['setup'] = utils.interpolate(test['setup']) test['cleanup'] = utils.interpolate(test['cleanup']) # pass --no-remote to firefox launch, if --develop is specified if browser_config['develop']: browser_config['extra_args'] = '--no-remote' # set defaults title = config.get('title', '') testdate = config.get('testdate', '') if browser_config['e10s'] and not title.endswith(".e"): # we are running in e10s mode title = "%s.e" % (title, ) # get the process name from the path to the browser if not browser_config['process']: browser_config['process'] = \ os.path.basename(browser_config['browser_path']) # fix paths to substitute # `os.path.dirname(os.path.abspath(__file__))` for ${talos} # https://bugzilla.mozilla.org/show_bug.cgi?id=705809 browser_config['extensions'] = [ utils.interpolate(i) for i in browser_config['extensions'] ] browser_config['bcontroller_config'] = \ utils.interpolate(browser_config['bcontroller_config']) # normalize browser path to work across platforms browser_config['browser_path'] = \ os.path.normpath(browser_config['browser_path']) binary = browser_config["browser_path"] version_info = mozversion.get_version(binary=binary) browser_config['browser_name'] = version_info['application_name'] browser_config['browser_version'] = version_info['application_version'] browser_config['buildid'] = version_info['application_buildid'] try: browser_config['repository'] = version_info['application_repository'] browser_config['sourcestamp'] = version_info['application_changeset'] except KeyError: if not browser_config['develop']: print "unable to find changeset or repository: %s" % version_info sys.exit() else: browser_config['repository'] = 'develop' browser_config['sourcestamp'] = 'develop' # get test date in seconds since epoch if testdate: date = int( time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT'))) else: date = int(time.time()) logging.debug("using testdate: %d", date) logging.debug("actual date: %d", int(time.time())) # results container talos_results = TalosResults(title=title, date=date, browser_config=browser_config) # results links if not browser_config['develop']: results_urls = dict( # hardcoded, this will be removed soon anyway. results_urls=['http://graphs.mozilla.org/server/collect.cgi'], # another hack; datazilla stands for Perfherder # and do not require url, but a non empty dict is required... datazilla_urls=['local.json'], ) else: # local mode, output to files results_urls = dict(results_urls=[os.path.abspath('local.out')], datazilla_urls=[os.path.abspath('local.json')]) talos_results.check_output_formats(results_urls) # setup a webserver, if --develop is specified httpd = None if browser_config['develop']: httpd = setup_webserver(browser_config['webserver']) if httpd: httpd.start() # run the tests timer = utils.Timer() logging.info("Starting test suite %s", title) for test in tests: testname = test['name'] testtimer = utils.Timer() logging.info("Starting test %s", testname) try: mytest = TTest() if mytest: talos_results.add(mytest.runTest(browser_config, test)) else: logging.error("Error found while running %s", testname) except TalosRegression: logging.error("Detected a regression for %s", testname) if httpd: httpd.stop() # by returning 1, we report an orange to buildbot # http://docs.buildbot.net/latest/developer/results.html return 1 except (TalosCrash, TalosError): # NOTE: if we get into this condition, talos has an internal # problem and cannot continue # this will prevent future tests from running traceback.print_exception(*sys.exc_info()) if httpd: httpd.stop() # indicate a failure to buildbot, turn the job red return 2 logging.info("Completed test %s (%s)", testname, testtimer.elapsed()) logging.info("Completed test suite (%s)", timer.elapsed()) # stop the webserver if running if httpd: httpd.stop() # output results if results_urls: talos_results.output(results_urls) if browser_config['develop']: print print( "Thanks for running Talos locally. Results are in" " %s and %s" % (results_urls['results_urls'], results_urls['datazilla_urls'])) # we will stop running tests on a failed test, or we will return 0 for # green return 0
def run_tests(config, browser_config): """Runs the talos tests on the given configuration and generates a report. """ # get the test data tests = config['tests'] tests = useBaseTestDefaults(config.get('basetest', {}), tests) paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup'] for test in tests: # Check for profile_path, tpmanifest and interpolate based on Talos # root https://bugzilla.mozilla.org/show_bug.cgi?id=727711 # Build command line from config for path in paths: if test.get(path): test[path] = utils.interpolate(test[path]) if test.get('tpmanifest'): test['tpmanifest'] = \ os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'], '/\\t:\\'))) if not test.get('url'): # build 'url' for tptest test['url'] = buildCommandLine(test) test['url'] = utils.interpolate(test['url']) test['setup'] = utils.interpolate(test['setup']) test['cleanup'] = utils.interpolate(test['cleanup']) # pass --no-remote to firefox launch, if --develop is specified # we do that to allow locally the user to have another running firefox # instance if browser_config['develop']: browser_config['extra_args'] = '--no-remote' # with addon signing for production talos, we want to develop without it if browser_config['develop'] or browser_config['branch_name'] == 'Try': browser_config['preferences']['xpinstall.signatures.required'] = False browser_config['extensions'] = [os.path.dirname(i) for i in browser_config['extensions']] # set defaults title = config.get('title', '') testdate = config.get('testdate', '') if browser_config['e10s'] and not title.endswith(".e"): # we are running in e10s mode title = "%s.e" % (title,) # get the process name from the path to the browser if not browser_config['process']: browser_config['process'] = \ os.path.basename(browser_config['browser_path']) # fix paths to substitute # `os.path.dirname(os.path.abspath(__file__))` for ${talos} # https://bugzilla.mozilla.org/show_bug.cgi?id=705809 browser_config['extensions'] = [utils.interpolate(i) for i in browser_config['extensions']] browser_config['bcontroller_config'] = \ utils.interpolate(browser_config['bcontroller_config']) # normalize browser path to work across platforms browser_config['browser_path'] = \ os.path.normpath(browser_config['browser_path']) binary = browser_config["browser_path"] version_info = mozversion.get_version(binary=binary) browser_config['browser_name'] = version_info['application_name'] browser_config['browser_version'] = version_info['application_version'] browser_config['buildid'] = version_info['application_buildid'] try: browser_config['repository'] = version_info['application_repository'] browser_config['sourcestamp'] = version_info['application_changeset'] except KeyError: if not browser_config['develop']: print "unable to find changeset or repository: %s" % version_info sys.exit() else: browser_config['repository'] = 'develop' browser_config['sourcestamp'] = 'develop' # get test date in seconds since epoch if testdate: date = int(time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT'))) else: date = int(time.time()) LOG.debug("using testdate: %d" % date) LOG.debug("actual date: %d" % int(time.time())) # results container talos_results = TalosResults() # results links if not browser_config['develop']: results_urls = dict( # another hack; datazilla stands for Perfherder # and do not require url, but a non empty dict is required... datazilla_urls=['local.json'], ) else: # local mode, output to files results_urls = dict(datazilla_urls=[os.path.abspath('local.json')]) talos_results.check_output_formats(results_urls) httpd = setup_webserver(browser_config['webserver']) httpd.start() testname = None # run the tests timer = utils.Timer() LOG.suite_start(tests=[test['name'] for test in tests]) try: for test in tests: testname = test['name'] LOG.test_start(testname) mytest = TTest() talos_results.add(mytest.runTest(browser_config, test)) LOG.test_end(testname, status='OK') except TalosRegression as exc: LOG.error("Detected a regression for %s" % testname) # by returning 1, we report an orange to buildbot # http://docs.buildbot.net/latest/developer/results.html LOG.test_end(testname, status='FAIL', message=unicode(exc), stack=traceback.format_exc()) return 1 except Exception as exc: # NOTE: if we get into this condition, talos has an internal # problem and cannot continue # this will prevent future tests from running LOG.test_end(testname, status='ERROR', message=unicode(exc), stack=traceback.format_exc()) # indicate a failure to buildbot, turn the job red return 2 finally: LOG.suite_end() httpd.stop() LOG.info("Completed test suite (%s)" % timer.elapsed()) # output results if results_urls: talos_results.output(results_urls) if browser_config['develop']: print ("Thanks for running Talos locally. Results are in %s" % (results_urls['datazilla_urls'])) # we will stop running tests on a failed test, or we will return 0 for # green return 0
def run_tests(config, browser_config): """Runs the talos tests on the given configuration and generates a report. """ # get the test data tests = config['tests'] tests = useBaseTestDefaults(config.get('basetest', {}), tests) paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup'] for test in tests: # Check for profile_path, tpmanifest and interpolate based on Talos # root https://bugzilla.mozilla.org/show_bug.cgi?id=727711 # Build command line from config for path in paths: if test.get(path): test[path] = utils.interpolate(test[path]) if test.get('tpmanifest'): test['tpmanifest'] = \ os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'], '/\\t:\\'))) if not test.get('url'): # build 'url' for tptest test['url'] = buildCommandLine(test) test['url'] = utils.interpolate(test['url']) test['setup'] = utils.interpolate(test['setup']) test['cleanup'] = utils.interpolate(test['cleanup']) # pass --no-remote to firefox launch, if --develop is specified if browser_config['develop']: browser_config['extra_args'] = '--no-remote' # set defaults title = config.get('title', '') testdate = config.get('testdate', '') if browser_config['e10s'] and not title.endswith(".e"): # we are running in e10s mode title = "%s.e" % (title,) # get the process name from the path to the browser if not browser_config['process']: browser_config['process'] = \ os.path.basename(browser_config['browser_path']) # fix paths to substitute # `os.path.dirname(os.path.abspath(__file__))` for ${talos} # https://bugzilla.mozilla.org/show_bug.cgi?id=705809 browser_config['extensions'] = [utils.interpolate(i) for i in browser_config['extensions']] browser_config['bcontroller_config'] = \ utils.interpolate(browser_config['bcontroller_config']) # normalize browser path to work across platforms browser_config['browser_path'] = \ os.path.normpath(browser_config['browser_path']) binary = browser_config["browser_path"] version_info = mozversion.get_version(binary=binary) browser_config['browser_name'] = version_info['application_name'] browser_config['browser_version'] = version_info['application_version'] browser_config['buildid'] = version_info['application_buildid'] try: browser_config['repository'] = version_info['application_repository'] browser_config['sourcestamp'] = version_info['application_changeset'] except KeyError: if not browser_config['develop']: print "unable to find changeset or repository: %s" % version_info sys.exit() else: browser_config['repository'] = 'develop' browser_config['sourcestamp'] = 'develop' # get test date in seconds since epoch if testdate: date = int(time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT'))) else: date = int(time.time()) logging.debug("using testdate: %d", date) logging.debug("actual date: %d", int(time.time())) # results container talos_results = TalosResults(title=title, date=date, browser_config=browser_config) # results links if not browser_config['develop']: results_urls = dict( # hardcoded, this will be removed soon anyway. results_urls=['http://graphs.mozilla.org/server/collect.cgi'], # another hack; datazilla stands for Perfherder # and do not require url, but a non empty dict is required... datazilla_urls=['local.json'], ) else: # local mode, output to files results_urls = dict( results_urls=[os.path.abspath('local.out')], datazilla_urls=[os.path.abspath('local.json')] ) talos_results.check_output_formats(results_urls) # setup a webserver, if --develop is specified httpd = None if browser_config['develop']: httpd = setup_webserver(browser_config['webserver']) if httpd: httpd.start() # run the tests timer = utils.Timer() logging.info("Starting test suite %s", title) for test in tests: testname = test['name'] testtimer = utils.Timer() logging.info("Starting test %s", testname) try: mytest = TTest() if mytest: talos_results.add(mytest.runTest(browser_config, test)) else: logging.error("Error found while running %s", testname) except TalosRegression: logging.error("Detected a regression for %s", testname) if httpd: httpd.stop() # by returning 1, we report an orange to buildbot # http://docs.buildbot.net/latest/developer/results.html return 1 except (TalosCrash, TalosError): # NOTE: if we get into this condition, talos has an internal # problem and cannot continue # this will prevent future tests from running traceback.print_exception(*sys.exc_info()) if httpd: httpd.stop() # indicate a failure to buildbot, turn the job red return 2 logging.info("Completed test %s (%s)", testname, testtimer.elapsed()) logging.info("Completed test suite (%s)", timer.elapsed()) # stop the webserver if running if httpd: httpd.stop() # output results if results_urls: talos_results.output(results_urls) if browser_config['develop']: print print ("Thanks for running Talos locally. Results are in" " %s and %s" % (results_urls['results_urls'], results_urls['datazilla_urls'])) # we will stop running tests on a failed test, or we will return 0 for # green return 0