def _runTest(self, browser_config, test_config, setup): minidump_dir = os.path.join(setup.profile_dir, 'minidumps') counters = test_config.get(self.platform_type + 'counters', []) resolution = test_config['resolution'] # add the mainthread_io to the environment variable, as defined # in test.py configs here = os.path.dirname(os.path.realpath(__file__)) if test_config['mainthread']: mainthread_io = os.path.join(here, "mainthread_io.log") setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io test_config['url'] = utils.interpolate( test_config['url'], profile=setup.profile_dir, firefox=browser_config['browser_path']) # setup global (cross-cycle) counters: # shutdown, responsiveness global_counters = {} if browser_config.get('xperf_path'): for c in test_config.get('xperf_counters', []): global_counters[c] = [] if test_config['shutdown']: global_counters['shutdown'] = [] if test_config.get('responsiveness') and \ platform.system() != "Linux": # ignore responsiveness tests on linux until we fix # Bug 710296 setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10' global_counters['responsiveness'] = [] # instantiate an object to hold test results test_results = results.TestResults(test_config, global_counters) for i in range(test_config['cycles']): LOG.info("Running cycle %d/%d for %s test..." % (i + 1, test_config['cycles'], test_config['name'])) # remove the browser error file mozfile.remove(browser_config['error_filename']) # reinstall any file whose stability we need to ensure across # the cycles if test_config.get('reinstall', ''): for keep in test_config['reinstall']: origin = os.path.join(test_config['profile_path'], keep) dest = os.path.join(setup.profile_dir, keep) LOG.debug("Reinstalling %s on top of %s" % (origin, dest)) shutil.copy(origin, dest) # Run the test timeout = test_config.get('timeout', 7200) # 2 hours default if setup.sps_profile: # When profiling, give the browser some extra time # to dump the profile. timeout += 5 * 60 command_args = utils.GenerateBrowserCommandLine( browser_config["browser_path"], browser_config["extra_args"], setup.profile_dir, test_config['url'], profiling_info=(setup.sps_profile.profiling_info if setup.sps_profile else None)) mainthread_error_count = 0 if test_config['setup']: # Generate bcontroller.json for xperf talosconfig.generateTalosConfig(command_args, browser_config, test_config) subprocess.call(['python'] + test_config['setup'].split(), ) mm_httpd = None if test_config['name'] == 'media_tests': from startup_test.media import media_manager mm_httpd = media_manager.run_server( os.path.dirname(os.path.realpath(__file__))) counter_management = None if counters: counter_management = CounterManagement( browser_config['process'], counters, resolution) try: pcontext = run_browser( command_args, minidump_dir, timeout=timeout, env=setup.env, # start collecting counters as soon as possible on_started=(counter_management.start if counter_management else None), ) finally: if counter_management: counter_management.stop() if mm_httpd: mm_httpd.stop() if test_config['mainthread']: rawlog = os.path.join(here, "mainthread_io.log") if os.path.exists(rawlog): processedlog = \ os.path.join(here, 'mainthread_io.json') xre_path = \ os.path.dirname(browser_config['browser_path']) mtio_py = os.path.join(here, 'mainthreadio.py') command = [ 'python', mtio_py, rawlog, processedlog, xre_path ] mtio = subprocess.Popen(command, env=os.environ.copy(), stdout=subprocess.PIPE) output, stderr = mtio.communicate() for line in output.split('\n'): if line.strip() == "": continue print line mainthread_error_count += 1 mozfile.remove(rawlog) if test_config['cleanup']: # HACK: add the pid to support xperf where we require # the pid in post processing talosconfig.generateTalosConfig(command_args, browser_config, test_config, pid=pcontext.pid) subprocess.call([sys.executable] + test_config['cleanup'].split()) # For startup tests, we launch the browser multiple times # with the same profile for fname in ('sessionstore.js', '.parentlock', 'sessionstore.bak'): mozfile.remove(os.path.join(setup.profile_dir, fname)) # check for xperf errors if os.path.exists(browser_config['error_filename']) or \ mainthread_error_count > 0: raise TalosRegression( "Talos has found a regression, if you have questions" " ask for help in irc on #perf") # add the results from the browser output test_results.add('\n'.join(pcontext.output), counter_results=(counter_management.results() if counter_management else None)) if setup.sps_profile: setup.sps_profile.symbolicate(i) self.check_for_crashes(browser_config, minidump_dir, test_config['name']) # include global (cross-cycle) counters test_results.all_counter_results.extend([{ key: value } for key, value in global_counters.items()]) for c in test_results.all_counter_results: for key, value in c.items(): LOG.debug("COUNTER %r: %s" % (key, value)) # return results return test_results
def runTest(self, browser_config, test_config): """ Runs an url based test on the browser as specified in the browser_config dictionary Args: browser_config: Dictionary of configuration options for the browser (paths, prefs, etc) test_config : Dictionary of configuration for the given test (url, cycles, counters, etc) """ self.initializeLibraries(browser_config) utils.debug("operating with platform_type : %s", self.platform_type) counters = test_config.get(self.platform_type + 'counters', []) resolution = test_config['resolution'] utils.setEnvironmentVars(browser_config['env']) utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'}) if browser_config['symbols_path']: utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'}) else: utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'}) utils.setEnvironmentVars({ "LD_LIBRARY_PATH": os.path.dirname(browser_config['browser_path']) }) profile_dir = None temp_dir = None try: running_processes = self._ffprocess.checkAllProcesses( browser_config['process'], browser_config['child_process']) if running_processes: msg = " already running before testing started (unclean system)" utils.debug("%s%s", browser_config['process'], msg) running_processes_str = ", ".join([ ('[%s] %s' % (pid, process_name)) for pid, process_name in running_processes ]) raise talosError( "Found processes still running: %s. Please close them before running talos." % running_processes_str) # add any provided directories to the installed browser for dir in browser_config['dirs']: self._ffsetup.InstallInBrowser(browser_config['browser_path'], browser_config['dirs'][dir]) # make profile path work cross-platform test_config['profile_path'] = os.path.normpath( test_config['profile_path']) preferences = copy.deepcopy(browser_config['preferences']) if 'preferences' in test_config and test_config['preferences']: testPrefs = dict([ (i, utils.parsePref(j)) for i, j in test_config['preferences'].items() ]) preferences.update(testPrefs) extensions = copy.deepcopy(browser_config['extensions']) if 'extensions' in test_config and test_config['extensions']: extensions.append(test_config['extensions']) profile_dir, temp_dir = self.createProfile( test_config['profile_path'], preferences, extensions, browser_config['webserver']) self.initializeProfile(profile_dir, browser_config) if browser_config['fennecIDs']: # This pushes environment variables to the device, be careful of placement self.setupRobocopTests(browser_config, profile_dir) utils.debug("initialized %s", browser_config['process']) # setup global (cross-cycle) counters: # shutdown, responsiveness global_counters = {} if browser_config.get('xperf_path'): for c in test_config.get('xperf_counters', []): global_counters[c] = [] if test_config['shutdown']: global_counters['shutdown'] = [] if test_config.get( 'responsiveness') and platform.system() != "Linux": # ignore responsiveness tests on linux until we fix Bug 710296 utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP': '1'}) utils.setEnvironmentVars( {'MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD': '20'}) utils.setEnvironmentVars( {'MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL': '10'}) global_counters['responsiveness'] = [] # instantiate an object to hold test results test_results = results.TestResults( test_config, global_counters, extensions=self._ffsetup.extensions) for i in range(test_config['cycles']): # remove the browser log file if os.path.isfile(browser_config['browser_log']): os.chmod(browser_config['browser_log'], 0777) os.remove(browser_config['browser_log']) # remove the error file if it exists if os.path.exists(browser_config['error_filename']): os.chmod(browser_config['error_filename'], 0777) os.remove(browser_config['error_filename']) # on remote devices we do not have the fast launch/shutdown as we do on desktop if not browser_config['remote']: time.sleep(browser_config['browser_wait'] ) #wait out the browser closing # check to see if the previous cycle is still hanging around if (i > 0) and self._ffprocess.checkAllProcesses( browser_config['process'], browser_config['child_process']): raise talosError("previous cycle still running") # Run the test timeout = test_config.get('timeout', 7200) # 2 hours default total_time = 0 url = test_config['url'] command_line = self._ffprocess.GenerateBrowserCommandLine( browser_config['browser_path'], browser_config['extra_args'], browser_config['deviceroot'], profile_dir, url) utils.debug("command line: %s", command_line) b_log = browser_config['browser_log'] if self.remote == True: b_log = browser_config[ 'deviceroot'] + '/' + browser_config['browser_log'] self._ffprocess.removeFile(b_log) # bug 816719, remove sessionstore.js so we don't interfere with talos self._ffprocess.testAgent.removeFile( os.path.join(self._ffprocess.testAgent.getDeviceRoot(), "profile/sessionstore.js")) b_cmd = self._ffprocess.GenerateBControllerCommandLine( command_line, browser_config, test_config) try: process = subprocess.Popen(b_cmd, universal_newlines=True, bufsize=0, env=os.environ) except: raise talosError( "error executing browser command line '%s': %s" % (subprocess.list2cmdline(b_cmd), sys.exc_info()[0])) #give browser a chance to open # this could mean that we are losing the first couple of data points # as the tests starts, but if we don't provide # some time for the browser to start we have trouble connecting the CounterManager to it # on remote devices we do not have the fast launch/shutdown as we do on desktop if not browser_config['remote']: time.sleep(browser_config['browser_wait']) #set up the counters for this test counter_results = None if counters: cm = self.CounterManager(self._ffprocess, browser_config['process'], counters) counter_results = dict([(counter, []) for counter in counters]) #the main test loop, monitors counters and checks for browser output dumpResult = "" while total_time < timeout: # Sleep for [resolution] seconds time.sleep(resolution) total_time += resolution fileData = self._ffprocess.getFile(b_log) if fileData and len(fileData) > 0: newResults = fileData.replace(dumpResult, '') if len(newResults.strip()) > 0: utils.info(newResults) dumpResult = fileData # Get the output from all the possible counters for count_type in counters: val = cm.getCounterValue(count_type) if val: counter_results[count_type].append(val) if process.poll( ) != None: #browser_controller completed, file now full break if hasattr(process, 'kill'): # BBB python 2.4 does not have Popen.kill(); see # https://bugzilla.mozilla.org/show_bug.cgi?id=752951#c6 try: process.kill() except OSError, e: if (not mozinfo.isWin) and (e.errno != 3): # 3 == No such process in Linux and Mac (errno.h) raise if total_time >= timeout: raise talosError("timeout exceeded") #stop the counter manager since this test is complete if counters: cm.stopMonitor() # ensure the browser log exists browser_log_filename = browser_config['browser_log'] if not os.path.isfile(browser_log_filename): raise talosError("no output from browser [%s]" % browser_log_filename) # ensure the browser log exists if os.path.exists(browser_config['error_filename']): raise talosRegression( "Talos has found a regression, if you have questions ask for help in irc on #perf" ) # add the results from the browser output test_results.add(browser_log_filename, counter_results=counter_results) # on remote devices we do not have the fast launch/shutdown as we do on desktop if not browser_config['remote']: time.sleep(browser_config['browser_wait']) #clean up any stray browser processes self.cleanupAndCheckForCrashes(browser_config, profile_dir, test_config['name']) #clean up the bcontroller process timer = 0 while ((process.poll() is None) and timer < browser_config['browser_wait']): time.sleep(1) timer += 1 # cleanup self.cleanupProfile(temp_dir) utils.restoreEnvironmentVars() # include global (cross-cycle) counters test_results.all_counter_results.extend([{ key: value } for key, value in global_counters.items()]) # return results return test_results
def runTest(self, browser_config, test_config): """ Runs an url based test on the browser as specified in the browser_config dictionary Args: browser_config: Dictionary of configuration options for the browser (paths, prefs, etc) test_config : Dictionary of configuration for the given test (url, cycles, counters, etc) """ self.initializeLibraries(browser_config) utils.debug("operating with platform_type : %s", self.platform_type) self.counters = test_config.get(self.platform_type + 'counters', []) self.resolution = test_config['resolution'] utils.setEnvironmentVars(browser_config['env']) utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'}) if browser_config['symbols_path']: utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'}) else: utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'}) utils.setEnvironmentVars({ "LD_LIBRARY_PATH": os.path.dirname(browser_config['browser_path']) }) profile_dir = None temp_dir = None try: running_processes = self._ffprocess.checkAllProcesses( browser_config['process'], browser_config['child_process']) if running_processes: msg = " already running before testing started (unclean system)" utils.debug("%s%s", browser_config['process'], msg) running_processes_str = ", ".join([ ('[%s] %s' % (pid, process_name)) for pid, process_name in running_processes ]) raise talosError( "Found processes still running: %s. Please close them before running talos." % running_processes_str) # add any provided directories to the installed browser for dir in browser_config['dirs']: self._ffsetup.InstallInBrowser(browser_config['browser_path'], browser_config['dirs'][dir]) # make profile path work cross-platform test_config['profile_path'] = os.path.normpath( test_config['profile_path']) preferences = copy.deepcopy(browser_config['preferences']) if 'preferences' in test_config and test_config['preferences']: testPrefs = dict([ (i, utils.parsePref(j)) for i, j in test_config['preferences'].items() ]) preferences.update(testPrefs) extensions = copy.deepcopy(browser_config['extensions']) if 'extensions' in test_config and test_config['extensions']: extensions.append(test_config['extensions']) profile_dir, temp_dir = self.createProfile( test_config['profile_path'], preferences, extensions, browser_config['webserver']) self.initializeProfile(profile_dir, browser_config) test_config['url'] = utils.interpolatePath( test_config['url'], profile_dir=profile_dir, firefox_path=browser_config['browser_path']) if browser_config['fennecIDs']: # This pushes environment variables to the device, be careful of placement self.setupRobocopTests(browser_config, profile_dir) utils.debug("initialized %s", browser_config['process']) # setup global (cross-cycle) counters: # shutdown, responsiveness global_counters = {} if browser_config.get('xperf_path'): for c in test_config.get('xperf_counters', []): global_counters[c] = [] if test_config['shutdown']: global_counters['shutdown'] = [] if test_config.get( 'responsiveness') and platform.system() != "Linux": # ignore responsiveness tests on linux until we fix Bug 710296 utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP': '1'}) utils.setEnvironmentVars( {'MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD': '20'}) utils.setEnvironmentVars( {'MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL': '10'}) global_counters['responsiveness'] = [] # instantiate an object to hold test results test_results = results.TestResults( test_config, global_counters, extensions=self._ffsetup.extensions) for i in range(test_config['cycles']): # remove the browser log file if os.path.isfile(browser_config['browser_log']): os.chmod(browser_config['browser_log'], 0777) os.remove(browser_config['browser_log']) # remove the error file if it exists if os.path.exists(browser_config['error_filename']): os.chmod(browser_config['error_filename'], 0777) os.remove(browser_config['error_filename']) # check to see if the previous cycle is still hanging around if (i > 0) and self._ffprocess.checkAllProcesses( browser_config['process'], browser_config['child_process']): raise talosError("previous cycle still running") # Run the test timeout = test_config.get('timeout', 7200) # 2 hours default total_time = 0 command_args = utils.GenerateBrowserCommandLine( browser_config["browser_path"], browser_config["extra_args"], browser_config["deviceroot"], profile_dir, test_config['url']) self.counter_results = None if not browser_config['remote']: if test_config['setup']: # Generate bcontroller.yml for xperf utils.GenerateTalosConfig(command_args, browser_config, test_config) setup = talosProcess.talosProcess( ['python'] + test_config['setup'].split(), env=os.environ.copy()) setup.run() setup.wait() self.isFinished = False browser = talosProcess.talosProcess( command_args, env=os.environ.copy(), logfile=browser_config['browser_log']) browser.run(timeout=timeout) self.pid = browser.pid if self.counters: self.cm = self.CounterManager( browser_config['process'], self.counters) self.counter_results = dict([ (counter, []) for counter in self.counters ]) cmthread = Thread(target=self.collectCounters) cmthread.setDaemon(True) # don't hang on quit cmthread.start() # todo: ctrl+c doesn't close the browser windows browser.wait() browser = None self.isFinished = True if test_config['cleanup']: #HACK: add the pid to support xperf where we require the pid in post processing utils.GenerateTalosConfig(command_args, browser_config, test_config, pid=self.pid) cleanup = talosProcess.talosProcess( ['python'] + test_config['cleanup'].split(), env=os.environ.copy()) cleanup.run() cleanup.wait() # allow mozprocess to terminate fully. It appears our log file is partial unless we wait time.sleep(5) else: self._ffprocess.runProgram(browser_config, command_args, timeout=timeout) # check if we found results from our webserver if os.path.isfile(browser_config['results_log']): shutil.move(browser_config['results_log'], browser_config['browser_log']) # ensure the browser log exists browser_log_filename = browser_config['browser_log'] if not os.path.isfile(browser_log_filename): raise talosError("no output from browser [%s]" % browser_log_filename) # ensure the browser log exists if os.path.exists(browser_config['error_filename']): raise talosRegression( "Talos has found a regression, if you have questions ask for help in irc on #perf" ) # add the results from the browser output test_results.add(browser_log_filename, counter_results=self.counter_results) #clean up any stray browser processes self.cleanupAndCheckForCrashes(browser_config, profile_dir, test_config['name']) #clean up the bcontroller process timer = 0 # cleanup self.cleanupProfile(temp_dir) utils.restoreEnvironmentVars() # include global (cross-cycle) counters test_results.all_counter_results.extend([{ key: value } for key, value in global_counters.items()]) # return results return test_results except Exception, e: self.counters = vars().get('cm', self.counters) self.testCleanup(browser_config, profile_dir, test_config, self.counters, temp_dir) raise
def _runTest(self, browser_config, test_config, setup): minidump_dir = os.path.join(setup.profile_dir, 'minidumps') counters = test_config.get('%s_counters' % self._get_counter_prefix(), []) resolution = test_config['resolution'] # add the mainthread_io to the environment variable, as defined # in test.py configs here = os.path.dirname(os.path.realpath(__file__)) if test_config['mainthread']: mainthread_io = os.path.join(here, "mainthread_io.log") setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io if browser_config['disable_stylo']: if browser_config['stylothreads']: raise TalosError( "--disable-stylo conflicts with --stylo-threads") if browser_config['enable_stylo']: raise TalosError( "--disable-stylo conflicts with --enable-stylo") # As we transition to Stylo, we need to set env vars and output data properly if browser_config['enable_stylo']: setup.env['STYLO_FORCE_ENABLED'] = '1' if browser_config['disable_stylo']: setup.env['STYLO_FORCE_DISABLED'] = '1' # During the Stylo transition, measure different number of threads if browser_config.get('stylothreads', 0) > 0: setup.env['STYLO_THREADS'] = str(browser_config['stylothreads']) test_config['url'] = utils.interpolate( test_config['url'], profile=setup.profile_dir, firefox=browser_config['browser_path']) # setup global (cross-cycle) counters: # shutdown, responsiveness global_counters = {} if browser_config.get('xperf_path'): for c in test_config.get('xperf_counters', []): global_counters[c] = [] if test_config['shutdown']: global_counters['shutdown'] = [] if test_config.get('responsiveness') and \ platform.system() != "Darwin": # ignore osx for now as per bug 1245793 setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10' global_counters['responsiveness'] = [] setup.env['JSGC_DISABLE_POISONING'] = '1' setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1' # if using mitmproxy we must allow access to 'external' sites if browser_config.get('mitmproxy', False): LOG.info( "Using mitmproxy so setting MOZ_DISABLE_NONLOCAL_CONNECTIONS to 0" ) setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '0' # instantiate an object to hold test results test_results = results.TestResults(test_config, global_counters, browser_config.get('framework')) for i in range(test_config['cycles']): LOG.info("Running cycle %d/%d for %s test..." % (i + 1, test_config['cycles'], test_config['name'])) # remove the browser error file mozfile.remove(browser_config['error_filename']) # reinstall any file whose stability we need to ensure across # the cycles if test_config.get('reinstall', ''): for keep in test_config['reinstall']: origin = os.path.join(test_config['profile_path'], keep) dest = os.path.join(setup.profile_dir, keep) LOG.debug("Reinstalling %s on top of %s" % (origin, dest)) shutil.copy(origin, dest) # Run the test timeout = test_config.get('timeout', 7200) # 2 hours default if setup.gecko_profile: # When profiling, give the browser some extra time # to dump the profile. timeout += 5 * 60 command_args = utils.GenerateBrowserCommandLine( browser_config["browser_path"], browser_config["extra_args"], setup.profile_dir, test_config['url'], profiling_info=(setup.gecko_profile.profiling_info if setup.gecko_profile else None)) mainthread_error_count = 0 if test_config['setup']: # Generate bcontroller.json for xperf talosconfig.generateTalosConfig(command_args, browser_config, test_config) subprocess.call(['python'] + test_config['setup'].split(), ) counter_management = None if counters: counter_management = CounterManagement( browser_config['process'], counters, resolution) try: pcontext = run_browser( command_args, minidump_dir, timeout=timeout, env=setup.env, # start collecting counters as soon as possible on_started=(counter_management.start if counter_management else None), ) except: self.check_for_crashes(browser_config, minidump_dir, test_config['name']) raise finally: if counter_management: counter_management.stop() if test_config['mainthread']: rawlog = os.path.join(here, "mainthread_io.log") if os.path.exists(rawlog): processedlog = \ os.path.join(here, 'mainthread_io.json') xre_path = \ os.path.dirname(browser_config['browser_path']) mtio_py = os.path.join(here, 'mainthreadio.py') command = [ 'python', mtio_py, rawlog, processedlog, xre_path ] mtio = subprocess.Popen(command, env=os.environ.copy(), stdout=subprocess.PIPE) output, stderr = mtio.communicate() for line in output.split('\n'): if line.strip() == "": continue print(line) mainthread_error_count += 1 mozfile.remove(rawlog) if test_config['cleanup']: # HACK: add the pid to support xperf where we require # the pid in post processing talosconfig.generateTalosConfig(command_args, browser_config, test_config, pid=pcontext.pid) subprocess.call([sys.executable] + test_config['cleanup'].split()) # For startup tests, we launch the browser multiple times # with the same profile for fname in ('sessionstore.js', '.parentlock', 'sessionstore.bak'): mozfile.remove(os.path.join(setup.profile_dir, fname)) # check for xperf errors if os.path.exists(browser_config['error_filename']) or \ mainthread_error_count > 0: raise TalosRegression( "Talos has found a regression, if you have questions" " ask for help in irc on #perf") # add the results from the browser output test_results.add('\n'.join(pcontext.output), counter_results=(counter_management.results() if counter_management else None)) if setup.gecko_profile: setup.gecko_profile.symbolicate(i) self.check_for_crashes(browser_config, minidump_dir, test_config['name']) # include global (cross-cycle) counters test_results.all_counter_results.extend([{ key: value } for key, value in global_counters.items()]) for c in test_results.all_counter_results: for key, value in c.items(): LOG.debug("COUNTER %r: %s" % (key, value)) # return results return test_results
def _runTest(self, browser_config, test_config, setup): minidump_dir = os.path.join(setup.profile_dir, 'minidumps') counters = test_config.get('%s_counters' % self._get_counter_prefix(), []) resolution = test_config['resolution'] # add the mainthread_io to the environment variable, as defined # in test.py configs here = os.path.dirname(os.path.realpath(__file__)) if test_config['mainthread']: mainthread_io = os.path.join(here, 'mainthread_io.log') setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io # Stylo is on by default setup.env['STYLO_FORCE_ENABLED'] = '1' # During the Stylo transition, measure different number of threads if browser_config.get('stylothreads', 0) > 0: setup.env['STYLO_THREADS'] = str(browser_config['stylothreads']) # set url if there is one (i.e. receiving a test page, not a manifest/pageloader test) if test_config.get('url', None) is not None: test_config['url'] = utils.interpolate( test_config['url'], profile=setup.profile_dir, firefox=browser_config['browser_path']) else: setup.env['MOZ_USE_PAGELOADER'] = '1' # setup global (cross-cycle) responsiveness counters global_counters = {} if browser_config.get('xperf_path'): for c in test_config.get('xperf_counters', []): global_counters[c] = [] if test_config.get('responsiveness') and \ platform.system() != 'Darwin': # ignore osx for now as per bug 1245793 setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20' setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10' global_counters['responsiveness'] = [] setup.env['JSGC_DISABLE_POISONING'] = '1' setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1' # if using mitmproxy we must allow access to 'external' sites if browser_config.get('mitmproxy', False): LOG.info( 'Using mitmproxy so setting MOZ_DISABLE_NONLOCAL_CONNECTIONS to 0' ) setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '0' # instantiate an object to hold test results test_results = results.TestResults(test_config, global_counters, browser_config.get('framework')) for i in range(test_config['cycles']): time.sleep(0.25) LOG.info('Running cycle %d/%d for %s test...' % (i + 1, test_config['cycles'], test_config['name'])) # remove the browser error file mozfile.remove(browser_config['error_filename']) # individual tests can have different frameworks # TODO: ensure that we don't run >1 test with custom frameworks if test_config.get('perfherder_framework', None) is not None: test_results.framework = test_config['perfherder_framework'] # reinstall any file whose stability we need to ensure across # the cycles if test_config.get('reinstall', ''): for keep in test_config['reinstall']: origin = os.path.join(test_config['profile_path'], keep) dest = os.path.join(setup.profile_dir, keep) LOG.debug('Reinstalling %s on top of %s' % (origin, dest)) shutil.copy(origin, dest) # Run the test timeout = test_config.get('timeout', 7200) # 2 hours default if setup.gecko_profile: # When profiling, give the browser some extra time # to dump the profile. timeout += 5 * 60 # store profiling info for pageloader; too late to add it as browser pref setup.env["TPPROFILINGINFO"] = json.dumps( setup.gecko_profile.profiling_info) command_args = utils.GenerateBrowserCommandLine( browser_config['browser_path'], browser_config['extra_args'], setup.profile_dir, test_config['url'], profiling_info=(setup.gecko_profile.profiling_info if setup.gecko_profile else None)) mainthread_error_count = 0 if test_config['setup']: # Generate bcontroller.json for xperf talosconfig.generateTalosConfig(command_args, browser_config, test_config) subprocess.call(['python'] + test_config['setup'].split(), ) counter_management = None if counters: counter_management = CounterManagement( browser_config['process'], counters, resolution) try: pcontext = run_browser( command_args, minidump_dir, timeout=timeout, env=setup.env, # start collecting counters as soon as possible on_started=(counter_management.start if counter_management else None), debug=browser_config['debug'], debugger=browser_config['debugger'], debugger_args=browser_config['debugger_args']) except Exception: self.check_for_crashes(browser_config, minidump_dir, test_config['name']) raise finally: if counter_management: counter_management.stop() if test_config['mainthread']: rawlog = os.path.join(here, 'mainthread_io.log') if os.path.exists(rawlog): processedlog = \ os.path.join(here, 'mainthread_io.json') xre_path = \ os.path.dirname(browser_config['browser_path']) mtio_py = os.path.join(here, 'mainthreadio.py') command = [ 'python', mtio_py, rawlog, processedlog, xre_path ] mtio = subprocess.Popen(command, env=os.environ.copy(), stdout=subprocess.PIPE) output, stderr = mtio.communicate() for line in output.split('\n'): if line.strip() == '': continue print(line) mainthread_error_count += 1 mozfile.remove(rawlog) if test_config['cleanup']: # HACK: add the pid to support xperf where we require # the pid in post processing talosconfig.generateTalosConfig(command_args, browser_config, test_config, pid=pcontext.pid) subprocess.call([sys.executable] + test_config['cleanup'].split()) # For startup tests, we launch the browser multiple times # with the same profile for fname in ('sessionstore.js', '.parentlock', 'sessionstore.bak'): mozfile.remove(os.path.join(setup.profile_dir, fname)) # check for xperf errors if os.path.exists(browser_config['error_filename']) or \ mainthread_error_count > 0: raise TalosRegression( 'Talos has found a regression, if you have questions' ' ask for help in irc on #perf') # add the results from the browser output if not run_in_debug_mode(browser_config): test_results.add( '\n'.join(pcontext.output), counter_results=(counter_management.results() if counter_management else None)) if setup.gecko_profile: setup.gecko_profile.symbolicate(i) self.check_for_crashes(browser_config, minidump_dir, test_config['name']) # include global (cross-cycle) counters test_results.all_counter_results.extend([{ key: value } for key, value in global_counters.items()]) for c in test_results.all_counter_results: for key, value in c.items(): LOG.debug('COUNTER %r: %s' % (key, value)) # if running against a code-coverage instrumented build, move the # produced gcda files to a folder where they will be collected later if browser_config.get('code_coverage', False): setup.collect_or_clean_ccov() # return results return test_results