def _runTest(self, browser_config, test_config, setup): minidump_dir = os.path.join(setup.profile_dir, 'minidumps') counters = test_config.get(self.platform_type + 'counters', []) resolution = test_config['resolution'] # add the mainthread_io to the environment variable, as defined # in test.py configs here = os.path.dirname(os.path.realpath(__file__)) if test_config['mainthread']: mainthread_io = os.path.join(here, "mainthread_io.log") setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io test_config['url'] = utils.interpolate( test_config['url'], profile=setup.profile_dir, firefox=browser_config['browser_path']) # setup global (cross-cycle) counters: # shutdown, responsiveness global_counters = {} if browser_config.get('xperf_path'): for c in test_config.get('xperf_counters', []): global_counters[c] = [] if test_config['shutdown']: global_counters['shutdown'] = [] if test_config.get('responsiveness') and \ platform.system() != "Linux": # ignore responsiveness tests on linux until we fix # Bug 710296 setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10' global_counters['responsiveness'] = [] # instantiate an object to hold test results test_results = results.TestResults(test_config, global_counters) for i in range(test_config['cycles']): LOG.info("Running cycle %d/%d for %s test..." % (i + 1, test_config['cycles'], test_config['name'])) # remove the browser error file mozfile.remove(browser_config['error_filename']) # reinstall any file whose stability we need to ensure across # the cycles if test_config.get('reinstall', ''): for keep in test_config['reinstall']: origin = os.path.join(test_config['profile_path'], keep) dest = os.path.join(setup.profile_dir, keep) LOG.debug("Reinstalling %s on top of %s" % (origin, dest)) shutil.copy(origin, dest) # Run the test timeout = test_config.get('timeout', 7200) # 2 hours default if setup.sps_profile: # When profiling, give the browser some extra time # to dump the profile. timeout += 5 * 60 command_args = utils.GenerateBrowserCommandLine( browser_config["browser_path"], browser_config["extra_args"], setup.profile_dir, test_config['url'], profiling_info=(setup.sps_profile.profiling_info if setup.sps_profile else None)) mainthread_error_count = 0 if test_config['setup']: # Generate bcontroller.json for xperf talosconfig.generateTalosConfig(command_args, browser_config, test_config) subprocess.call(['python'] + test_config['setup'].split(), ) mm_httpd = None if test_config['name'] == 'media_tests': from startup_test.media import media_manager mm_httpd = media_manager.run_server( os.path.dirname(os.path.realpath(__file__))) counter_management = None if counters: counter_management = CounterManagement( browser_config['process'], counters, resolution) try: pcontext = run_browser( command_args, minidump_dir, timeout=timeout, env=setup.env, # start collecting counters as soon as possible on_started=(counter_management.start if counter_management else None), ) finally: if counter_management: counter_management.stop() if mm_httpd: mm_httpd.stop() if test_config['mainthread']: rawlog = os.path.join(here, "mainthread_io.log") if os.path.exists(rawlog): processedlog = \ os.path.join(here, 'mainthread_io.json') xre_path = \ os.path.dirname(browser_config['browser_path']) mtio_py = os.path.join(here, 'mainthreadio.py') command = [ 'python', mtio_py, rawlog, processedlog, xre_path ] mtio = subprocess.Popen(command, env=os.environ.copy(), stdout=subprocess.PIPE) output, stderr = mtio.communicate() for line in output.split('\n'): if line.strip() == "": continue print line mainthread_error_count += 1 mozfile.remove(rawlog) if test_config['cleanup']: # HACK: add the pid to support xperf where we require # the pid in post processing talosconfig.generateTalosConfig(command_args, browser_config, test_config, pid=pcontext.pid) subprocess.call([sys.executable] + test_config['cleanup'].split()) # For startup tests, we launch the browser multiple times # with the same profile for fname in ('sessionstore.js', '.parentlock', 'sessionstore.bak'): mozfile.remove(os.path.join(setup.profile_dir, fname)) # check for xperf errors if os.path.exists(browser_config['error_filename']) or \ mainthread_error_count > 0: raise TalosRegression( "Talos has found a regression, if you have questions" " ask for help in irc on #perf") # add the results from the browser output test_results.add('\n'.join(pcontext.output), counter_results=(counter_management.results() if counter_management else None)) if setup.sps_profile: setup.sps_profile.symbolicate(i) self.check_for_crashes(browser_config, minidump_dir, test_config['name']) # include global (cross-cycle) counters test_results.all_counter_results.extend([{ key: value } for key, value in global_counters.items()]) for c in test_results.all_counter_results: for key, value in c.items(): LOG.debug("COUNTER %r: %s" % (key, value)) # return results return test_results
def _runTest(self, browser_config, test_config, setup): minidump_dir = os.path.join(setup.profile_dir, 'minidumps') counters = test_config.get(self.platform_type + 'counters', []) resolution = test_config['resolution'] # add the mainthread_io to the environment variable, as defined # in test.py configs here = os.path.dirname(os.path.realpath(__file__)) if test_config['mainthread']: mainthread_io = os.path.join(here, "mainthread_io.log") setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io test_config['url'] = utils.interpolate( test_config['url'], profile=setup.profile_dir, firefox=browser_config['browser_path'] ) # setup global (cross-cycle) counters: # shutdown, responsiveness global_counters = {} if browser_config.get('xperf_path'): for c in test_config.get('xperf_counters', []): global_counters[c] = [] if test_config['shutdown']: global_counters['shutdown'] = [] if test_config.get('responsiveness') and \ platform.system() != "Darwin": # ignore osx for now as per bug 1245793 setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10' global_counters['responsiveness'] = [] # instantiate an object to hold test results test_results = results.TestResults( test_config, global_counters ) for i in range(test_config['cycles']): LOG.info("Running cycle %d/%d for %s test..." % (i+1, test_config['cycles'], test_config['name'])) # remove the browser error file mozfile.remove(browser_config['error_filename']) # reinstall any file whose stability we need to ensure across # the cycles if test_config.get('reinstall', ''): for keep in test_config['reinstall']: origin = os.path.join(test_config['profile_path'], keep) dest = os.path.join(setup.profile_dir, keep) LOG.debug("Reinstalling %s on top of %s" % (origin, dest)) shutil.copy(origin, dest) # Run the test timeout = test_config.get('timeout', 7200) # 2 hours default if setup.sps_profile: # When profiling, give the browser some extra time # to dump the profile. timeout += 5 * 60 command_args = utils.GenerateBrowserCommandLine( browser_config["browser_path"], browser_config["extra_args"], setup.profile_dir, test_config['url'], profiling_info=(setup.sps_profile.profiling_info if setup.sps_profile else None) ) mainthread_error_count = 0 if test_config['setup']: # Generate bcontroller.json for xperf talosconfig.generateTalosConfig(command_args, browser_config, test_config) subprocess.call( ['python'] + test_config['setup'].split(), ) counter_management = None if counters: counter_management = CounterManagement( browser_config['process'], counters, resolution ) try: pcontext = run_browser( command_args, minidump_dir, timeout=timeout, env=setup.env, # start collecting counters as soon as possible on_started=(counter_management.start if counter_management else None), ) finally: if counter_management: counter_management.stop() if test_config['mainthread']: rawlog = os.path.join(here, "mainthread_io.log") if os.path.exists(rawlog): processedlog = \ os.path.join(here, 'mainthread_io.json') xre_path = \ os.path.dirname(browser_config['browser_path']) mtio_py = os.path.join(here, 'mainthreadio.py') command = ['python', mtio_py, rawlog, processedlog, xre_path] mtio = subprocess.Popen(command, env=os.environ.copy(), stdout=subprocess.PIPE) output, stderr = mtio.communicate() for line in output.split('\n'): if line.strip() == "": continue print line mainthread_error_count += 1 mozfile.remove(rawlog) if test_config['cleanup']: # HACK: add the pid to support xperf where we require # the pid in post processing talosconfig.generateTalosConfig(command_args, browser_config, test_config, pid=pcontext.pid) subprocess.call( [sys.executable] + test_config['cleanup'].split() ) # For startup tests, we launch the browser multiple times # with the same profile for fname in ('sessionstore.js', '.parentlock', 'sessionstore.bak'): mozfile.remove(os.path.join(setup.profile_dir, fname)) # check for xperf errors if os.path.exists(browser_config['error_filename']) or \ mainthread_error_count > 0: raise TalosRegression( "Talos has found a regression, if you have questions" " ask for help in irc on #perf" ) # add the results from the browser output test_results.add( '\n'.join(pcontext.output), counter_results=(counter_management.results() if counter_management else None) ) if setup.sps_profile: setup.sps_profile.symbolicate(i) self.check_for_crashes(browser_config, minidump_dir, test_config['name']) # include global (cross-cycle) counters test_results.all_counter_results.extend( [{key: value} for key, value in global_counters.items()] ) for c in test_results.all_counter_results: for key, value in c.items(): LOG.debug("COUNTER %r: %s" % (key, value)) # return results return test_results
def _runTest(self, browser_config, test_config, setup): minidump_dir = os.path.join(setup.profile_dir, 'minidumps') counters = test_config.get('%s_counters' % self._get_counter_prefix(), []) resolution = test_config['resolution'] # add the mainthread_io to the environment variable, as defined # in test.py configs here = os.path.dirname(os.path.realpath(__file__)) if test_config['mainthread']: mainthread_io = os.path.join(here, "mainthread_io.log") setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io if browser_config['disable_stylo']: if browser_config['stylothreads']: raise TalosError( "--disable-stylo conflicts with --stylo-threads") if browser_config['enable_stylo']: raise TalosError( "--disable-stylo conflicts with --enable-stylo") # As we transition to Stylo, we need to set env vars and output data properly if browser_config['enable_stylo']: setup.env['STYLO_FORCE_ENABLED'] = '1' if browser_config['disable_stylo']: setup.env['STYLO_FORCE_DISABLED'] = '1' # During the Stylo transition, measure different number of threads if browser_config.get('stylothreads', 0) > 0: setup.env['STYLO_THREADS'] = str(browser_config['stylothreads']) test_config['url'] = utils.interpolate( test_config['url'], profile=setup.profile_dir, firefox=browser_config['browser_path']) # setup global (cross-cycle) counters: # shutdown, responsiveness global_counters = {} if browser_config.get('xperf_path'): for c in test_config.get('xperf_counters', []): global_counters[c] = [] if test_config['shutdown']: global_counters['shutdown'] = [] if test_config.get('responsiveness') and \ platform.system() != "Darwin": # ignore osx for now as per bug 1245793 setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10' global_counters['responsiveness'] = [] setup.env['JSGC_DISABLE_POISONING'] = '1' setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1' # if using mitmproxy we must allow access to 'external' sites if browser_config.get('mitmproxy', False): LOG.info( "Using mitmproxy so setting MOZ_DISABLE_NONLOCAL_CONNECTIONS to 0" ) setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '0' # instantiate an object to hold test results test_results = results.TestResults(test_config, global_counters, browser_config.get('framework')) for i in range(test_config['cycles']): LOG.info("Running cycle %d/%d for %s test..." % (i + 1, test_config['cycles'], test_config['name'])) # remove the browser error file mozfile.remove(browser_config['error_filename']) # reinstall any file whose stability we need to ensure across # the cycles if test_config.get('reinstall', ''): for keep in test_config['reinstall']: origin = os.path.join(test_config['profile_path'], keep) dest = os.path.join(setup.profile_dir, keep) LOG.debug("Reinstalling %s on top of %s" % (origin, dest)) shutil.copy(origin, dest) # Run the test timeout = test_config.get('timeout', 7200) # 2 hours default if setup.gecko_profile: # When profiling, give the browser some extra time # to dump the profile. timeout += 5 * 60 command_args = utils.GenerateBrowserCommandLine( browser_config["browser_path"], browser_config["extra_args"], setup.profile_dir, test_config['url'], profiling_info=(setup.gecko_profile.profiling_info if setup.gecko_profile else None)) mainthread_error_count = 0 if test_config['setup']: # Generate bcontroller.json for xperf talosconfig.generateTalosConfig(command_args, browser_config, test_config) subprocess.call(['python'] + test_config['setup'].split(), ) counter_management = None if counters: counter_management = CounterManagement( browser_config['process'], counters, resolution) try: pcontext = run_browser( command_args, minidump_dir, timeout=timeout, env=setup.env, # start collecting counters as soon as possible on_started=(counter_management.start if counter_management else None), ) except: self.check_for_crashes(browser_config, minidump_dir, test_config['name']) raise finally: if counter_management: counter_management.stop() if test_config['mainthread']: rawlog = os.path.join(here, "mainthread_io.log") if os.path.exists(rawlog): processedlog = \ os.path.join(here, 'mainthread_io.json') xre_path = \ os.path.dirname(browser_config['browser_path']) mtio_py = os.path.join(here, 'mainthreadio.py') command = [ 'python', mtio_py, rawlog, processedlog, xre_path ] mtio = subprocess.Popen(command, env=os.environ.copy(), stdout=subprocess.PIPE) output, stderr = mtio.communicate() for line in output.split('\n'): if line.strip() == "": continue print(line) mainthread_error_count += 1 mozfile.remove(rawlog) if test_config['cleanup']: # HACK: add the pid to support xperf where we require # the pid in post processing talosconfig.generateTalosConfig(command_args, browser_config, test_config, pid=pcontext.pid) subprocess.call([sys.executable] + test_config['cleanup'].split()) # For startup tests, we launch the browser multiple times # with the same profile for fname in ('sessionstore.js', '.parentlock', 'sessionstore.bak'): mozfile.remove(os.path.join(setup.profile_dir, fname)) # check for xperf errors if os.path.exists(browser_config['error_filename']) or \ mainthread_error_count > 0: raise TalosRegression( "Talos has found a regression, if you have questions" " ask for help in irc on #perf") # add the results from the browser output test_results.add('\n'.join(pcontext.output), counter_results=(counter_management.results() if counter_management else None)) if setup.gecko_profile: setup.gecko_profile.symbolicate(i) self.check_for_crashes(browser_config, minidump_dir, test_config['name']) # include global (cross-cycle) counters test_results.all_counter_results.extend([{ key: value } for key, value in global_counters.items()]) for c in test_results.all_counter_results: for key, value in c.items(): LOG.debug("COUNTER %r: %s" % (key, value)) # return results return test_results
def _runTest(self, browser_config, test_config, setup): minidump_dir = os.path.join(setup.profile_dir, 'minidumps') counters = test_config.get('%s_counters' % self._get_counter_prefix(), []) resolution = test_config['resolution'] # add the mainthread_io to the environment variable, as defined # in test.py configs here = os.path.dirname(os.path.realpath(__file__)) if test_config['mainthread']: mainthread_io = os.path.join(here, 'mainthread_io.log') setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io if browser_config['disable_stylo']: if browser_config['stylothreads']: raise TalosError('--disable-stylo conflicts with --stylo-threads') if browser_config['enable_stylo']: raise TalosError('--disable-stylo conflicts with --enable-stylo') # As we transition to Stylo, we need to set env vars and output data properly if browser_config['enable_stylo']: setup.env['STYLO_FORCE_ENABLED'] = '1' if browser_config['disable_stylo']: setup.env['STYLO_FORCE_DISABLED'] = '1' # During the Stylo transition, measure different number of threads if browser_config.get('stylothreads', 0) > 0: setup.env['STYLO_THREADS'] = str(browser_config['stylothreads']) # set url if there is one (i.e. receiving a test page, not a manifest/pageloader test) if test_config.get('url', None) is not None: test_config['url'] = utils.interpolate( test_config['url'], profile=setup.profile_dir, firefox=browser_config['browser_path'] ) # setup global (cross-cycle) responsiveness counters global_counters = {} if browser_config.get('xperf_path'): for c in test_config.get('xperf_counters', []): global_counters[c] = [] if test_config.get('responsiveness') and \ platform.system() != 'Darwin': # ignore osx for now as per bug 1245793 setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10' global_counters['responsiveness'] = [] setup.env['JSGC_DISABLE_POISONING'] = '1' setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1' # if using mitmproxy we must allow access to 'external' sites if browser_config.get('mitmproxy', False): LOG.info('Using mitmproxy so setting MOZ_DISABLE_NONLOCAL_CONNECTIONS to 0') setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '0' # instantiate an object to hold test results test_results = results.TestResults( test_config, global_counters, browser_config.get('framework') ) for i in range(test_config['cycles']): LOG.info('Running cycle %d/%d for %s test...' % (i+1, test_config['cycles'], test_config['name'])) # remove the browser error file mozfile.remove(browser_config['error_filename']) # reinstall any file whose stability we need to ensure across # the cycles if test_config.get('reinstall', ''): for keep in test_config['reinstall']: origin = os.path.join(test_config['profile_path'], keep) dest = os.path.join(setup.profile_dir, keep) LOG.debug('Reinstalling %s on top of %s' % (origin, dest)) shutil.copy(origin, dest) # Run the test timeout = test_config.get('timeout', 7200) # 2 hours default if setup.gecko_profile: # When profiling, give the browser some extra time # to dump the profile. timeout += 5 * 60 # store profiling info for pageloader; too late to add it as browser pref setup.env["TPPROFILINGINFO"] = json.dumps(setup.gecko_profile.profiling_info) command_args = utils.GenerateBrowserCommandLine( browser_config['browser_path'], browser_config['extra_args'], setup.profile_dir, test_config['url'], profiling_info=(setup.gecko_profile.profiling_info if setup.gecko_profile else None) ) mainthread_error_count = 0 if test_config['setup']: # Generate bcontroller.json for xperf talosconfig.generateTalosConfig(command_args, browser_config, test_config) subprocess.call( ['python'] + test_config['setup'].split(), ) counter_management = None if counters: counter_management = CounterManagement( browser_config['process'], counters, resolution ) try: pcontext = run_browser( command_args, minidump_dir, timeout=timeout, env=setup.env, # start collecting counters as soon as possible on_started=(counter_management.start if counter_management else None), debug=browser_config['debug'], debugger=browser_config['debugger'], debugger_args=browser_config['debugger_args'] ) except Exception: self.check_for_crashes(browser_config, minidump_dir, test_config['name']) raise finally: if counter_management: counter_management.stop() if test_config['mainthread']: rawlog = os.path.join(here, 'mainthread_io.log') if os.path.exists(rawlog): processedlog = \ os.path.join(here, 'mainthread_io.json') xre_path = \ os.path.dirname(browser_config['browser_path']) mtio_py = os.path.join(here, 'mainthreadio.py') command = ['python', mtio_py, rawlog, processedlog, xre_path] mtio = subprocess.Popen(command, env=os.environ.copy(), stdout=subprocess.PIPE) output, stderr = mtio.communicate() for line in output.split('\n'): if line.strip() == '': continue print(line) mainthread_error_count += 1 mozfile.remove(rawlog) if test_config['cleanup']: # HACK: add the pid to support xperf where we require # the pid in post processing talosconfig.generateTalosConfig(command_args, browser_config, test_config, pid=pcontext.pid) subprocess.call( [sys.executable] + test_config['cleanup'].split() ) # For startup tests, we launch the browser multiple times # with the same profile for fname in ('sessionstore.js', '.parentlock', 'sessionstore.bak'): mozfile.remove(os.path.join(setup.profile_dir, fname)) # check for xperf errors if os.path.exists(browser_config['error_filename']) or \ mainthread_error_count > 0: raise TalosRegression( 'Talos has found a regression, if you have questions' ' ask for help in irc on #perf' ) # add the results from the browser output if not run_in_debug_mode(browser_config): test_results.add( '\n'.join(pcontext.output), counter_results=(counter_management.results() if counter_management else None) ) if setup.gecko_profile: setup.gecko_profile.symbolicate(i) self.check_for_crashes(browser_config, minidump_dir, test_config['name']) # include global (cross-cycle) counters test_results.all_counter_results.extend( [{key: value} for key, value in global_counters.items()] ) for c in test_results.all_counter_results: for key, value in c.items(): LOG.debug('COUNTER %r: %s' % (key, value)) # if running against a code-coverage instrumented build, move the # produced gcda files to a folder where they will be collected later if browser_config.get('code_coverage', False): setup.collect_or_clean_ccov() # return results return test_results
def _runTest(self, browser_config, test_config, setup): counters = test_config.get(self.platform_type + "counters", []) resolution = test_config["resolution"] # add the mainthread_io to the environment variable, as defined # in test.py configs here = os.path.dirname(os.path.realpath(__file__)) if test_config["mainthread"]: mainthread_io = os.path.join(here, "mainthread_io.log") setup.env["MOZ_MAIN_THREAD_IO_LOG"] = mainthread_io test_config["url"] = utils.interpolate( test_config["url"], profile=setup.profile_dir, firefox=browser_config["browser_path"] ) # setup global (cross-cycle) counters: # shutdown, responsiveness global_counters = {} if browser_config.get("xperf_path"): for c in test_config.get("xperf_counters", []): global_counters[c] = [] if test_config["shutdown"]: global_counters["shutdown"] = [] if test_config.get("responsiveness") and platform.system() != "Linux": # ignore responsiveness tests on linux until we fix # Bug 710296 setup.env["MOZ_INSTRUMENT_EVENT_LOOP"] = "1" setup.env["MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD"] = "20" setup.env["MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL"] = "10" global_counters["responsiveness"] = [] # instantiate an object to hold test results test_results = results.TestResults(test_config, global_counters) for i in range(test_config["cycles"]): logging.info("Running cycle %d/%d for %s test...", i + 1, test_config["cycles"], test_config["name"]) # remove the browser error file mozfile.remove(browser_config["error_filename"]) # reinstall any file whose stability we need to ensure across # the cycles if test_config.get("reinstall", ""): for keep in test_config["reinstall"]: origin = os.path.join(test_config["profile_path"], keep) dest = os.path.join(setup.profile_dir, keep) logging.debug("Reinstalling %s on top of %s", origin, dest) shutil.copy(origin, dest) # Run the test timeout = test_config.get("timeout", 7200) # 2 hours default if setup.sps_profile: # When profiling, give the browser some extra time # to dump the profile. timeout += 5 * 60 command_args = utils.GenerateBrowserCommandLine( browser_config["browser_path"], browser_config["extra_args"], setup.profile_dir, test_config["url"], profiling_info=(setup.sps_profile.profiling_info if setup.sps_profile else None), ) mainthread_error_count = 0 if test_config["setup"]: # Generate bcontroller.json for xperf talosconfig.generateTalosConfig(command_args, browser_config, test_config) subprocess.call(["python"] + test_config["setup"].split()) mm_httpd = None if test_config["name"] == "media_tests": from startup_test.media import media_manager mm_httpd = media_manager.run_server(os.path.dirname(os.path.realpath(__file__))) counter_management = None if counters: counter_management = CounterManagement(browser_config["process"], counters, resolution) try: pcontext = run_browser( command_args, timeout=timeout, env=setup.env, # start collecting counters as soon as possible on_started=(counter_management.start if counter_management else None), ) finally: if counter_management: counter_management.stop() if mm_httpd: mm_httpd.stop() if test_config["mainthread"]: rawlog = os.path.join(here, "mainthread_io.log") if os.path.exists(rawlog): processedlog = os.path.join(here, "mainthread_io.json") xre_path = os.path.dirname(browser_config["browser_path"]) mtio_py = os.path.join(here, "mainthreadio.py") command = ["python", mtio_py, rawlog, processedlog, xre_path] mtio = subprocess.Popen(command, env=os.environ.copy(), stdout=subprocess.PIPE) output, stderr = mtio.communicate() for line in output.split("\n"): if line.strip() == "": continue print line mainthread_error_count += 1 mozfile.remove(rawlog) if test_config["cleanup"]: # HACK: add the pid to support xperf where we require # the pid in post processing talosconfig.generateTalosConfig(command_args, browser_config, test_config, pid=pcontext.pid) subprocess.call([sys.executable] + test_config["cleanup"].split()) # For startup tests, we launch the browser multiple times # with the same profile for fname in ("sessionstore.js", ".parentlock", "sessionstore.bak"): mozfile.remove(os.path.join(setup.profile_dir, fname)) # check for xperf errors if os.path.exists(browser_config["error_filename"]) or mainthread_error_count > 0: raise TalosRegression( "Talos has found a regression, if you have questions" " ask for help in irc on #perf" ) # add the results from the browser output test_results.add( "\n".join(pcontext.output), counter_results=(counter_management.results() if counter_management else None), ) if setup.sps_profile: setup.sps_profile.symbolicate(i) self.check_for_crashes(browser_config, setup.profile_dir, test_config["name"]) # include global (cross-cycle) counters test_results.all_counter_results.extend([{key: value} for key, value in global_counters.items()]) for c in test_results.all_counter_results: for key, value in c.items(): logging.debug("COUNTER %r: %s", key, value) # return results return test_results
def _runTest(self, browser_config, test_config, setup): minidump_dir = os.path.join(setup.profile_dir, 'minidumps') counters = test_config.get('%s_counters' % self._get_counter_prefix(), []) resolution = test_config['resolution'] # add the mainthread_io to the environment variable, as defined # in test.py configs here = os.path.dirname(os.path.realpath(__file__)) if test_config['mainthread']: mainthread_io = os.path.join(here, 'mainthread_io.log') setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io # Stylo is on by default setup.env['STYLO_FORCE_ENABLED'] = '1' # During the Stylo transition, measure different number of threads if browser_config.get('stylothreads', 0) > 0: setup.env['STYLO_THREADS'] = str(browser_config['stylothreads']) # set url if there is one (i.e. receiving a test page, not a manifest/pageloader test) if test_config.get('url', None) is not None: test_config['url'] = utils.interpolate( test_config['url'], profile=setup.profile_dir, firefox=browser_config['browser_path']) else: setup.env['MOZ_USE_PAGELOADER'] = '1' # setup global (cross-cycle) responsiveness counters global_counters = {} if browser_config.get('xperf_path'): for c in test_config.get('xperf_counters', []): global_counters[c] = [] if test_config.get('responsiveness') and \ platform.system() != 'Darwin': # ignore osx for now as per bug 1245793 setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10' global_counters['responsiveness'] = [] setup.env['JSGC_DISABLE_POISONING'] = '1' setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1' # if using mitmproxy we must allow access to 'external' sites if browser_config.get('mitmproxy', False): LOG.info( 'Using mitmproxy so setting MOZ_DISABLE_NONLOCAL_CONNECTIONS to 0' ) setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '0' # instantiate an object to hold test results test_results = results.TestResults(test_config, global_counters, browser_config.get('framework')) for i in range(test_config['cycles']): time.sleep(0.25) LOG.info('Running cycle %d/%d for %s test...' % (i + 1, test_config['cycles'], test_config['name'])) # remove the browser error file mozfile.remove(browser_config['error_filename']) # individual tests can have different frameworks # TODO: ensure that we don't run >1 test with custom frameworks if test_config.get('perfherder_framework', None) is not None: test_results.framework = test_config['perfherder_framework'] # reinstall any file whose stability we need to ensure across # the cycles if test_config.get('reinstall', ''): for keep in test_config['reinstall']: origin = os.path.join(test_config['profile_path'], keep) dest = os.path.join(setup.profile_dir, keep) LOG.debug('Reinstalling %s on top of %s' % (origin, dest)) shutil.copy(origin, dest) # Run the test timeout = test_config.get('timeout', 7200) # 2 hours default if setup.gecko_profile: # When profiling, give the browser some extra time # to dump the profile. timeout += 5 * 60 # store profiling info for pageloader; too late to add it as browser pref setup.env["TPPROFILINGINFO"] = json.dumps( setup.gecko_profile.profiling_info) command_args = utils.GenerateBrowserCommandLine( browser_config['browser_path'], browser_config['extra_args'], setup.profile_dir, test_config['url'], profiling_info=(setup.gecko_profile.profiling_info if setup.gecko_profile else None)) mainthread_error_count = 0 if test_config['setup']: # Generate bcontroller.json for xperf talosconfig.generateTalosConfig(command_args, browser_config, test_config) subprocess.call(['python'] + test_config['setup'].split(), ) counter_management = None if counters: counter_management = CounterManagement( browser_config['process'], counters, resolution) try: pcontext = run_browser( command_args, minidump_dir, timeout=timeout, env=setup.env, # start collecting counters as soon as possible on_started=(counter_management.start if counter_management else None), debug=browser_config['debug'], debugger=browser_config['debugger'], debugger_args=browser_config['debugger_args']) except Exception: self.check_for_crashes(browser_config, minidump_dir, test_config['name']) raise finally: if counter_management: counter_management.stop() if test_config['mainthread']: rawlog = os.path.join(here, 'mainthread_io.log') if os.path.exists(rawlog): processedlog = \ os.path.join(here, 'mainthread_io.json') xre_path = \ os.path.dirname(browser_config['browser_path']) mtio_py = os.path.join(here, 'mainthreadio.py') command = [ 'python', mtio_py, rawlog, processedlog, xre_path ] mtio = subprocess.Popen(command, env=os.environ.copy(), stdout=subprocess.PIPE) output, stderr = mtio.communicate() for line in output.split('\n'): if line.strip() == '': continue print(line) mainthread_error_count += 1 mozfile.remove(rawlog) if test_config['cleanup']: # HACK: add the pid to support xperf where we require # the pid in post processing talosconfig.generateTalosConfig(command_args, browser_config, test_config, pid=pcontext.pid) subprocess.call([sys.executable] + test_config['cleanup'].split()) # For startup tests, we launch the browser multiple times # with the same profile for fname in ('sessionstore.js', '.parentlock', 'sessionstore.bak'): mozfile.remove(os.path.join(setup.profile_dir, fname)) # check for xperf errors if os.path.exists(browser_config['error_filename']) or \ mainthread_error_count > 0: raise TalosRegression( 'Talos has found a regression, if you have questions' ' ask for help in irc on #perf') # add the results from the browser output if not run_in_debug_mode(browser_config): test_results.add( '\n'.join(pcontext.output), counter_results=(counter_management.results() if counter_management else None)) if setup.gecko_profile: setup.gecko_profile.symbolicate(i) self.check_for_crashes(browser_config, minidump_dir, test_config['name']) # include global (cross-cycle) counters test_results.all_counter_results.extend([{ key: value } for key, value in global_counters.items()]) for c in test_results.all_counter_results: for key, value in c.items(): LOG.debug('COUNTER %r: %s' % (key, value)) # if running against a code-coverage instrumented build, move the # produced gcda files to a folder where they will be collected later if browser_config.get('code_coverage', False): setup.collect_or_clean_ccov() # return results return test_results