def _runTest(self, browser_config, test_config, setup):
minidump_dir = os.path.join(setup.profile_dir, 'minidumps')
counters = test_config.get(self.platform_type + 'counters', [])
resolution = test_config['resolution']
# add the mainthread_io to the environment variable, as defined
# in test.py configs
here = os.path.dirname(os.path.realpath(__file__))
if test_config['mainthread']:
mainthread_io = os.path.join(here, "mainthread_io.log")
setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io
test_config['url'] = utils.interpolate(
test_config['url'],
profile=setup.profile_dir,
firefox=browser_config['browser_path'])
# setup global (cross-cycle) counters:
# shutdown, responsiveness
global_counters = {}
if browser_config.get('xperf_path'):
for c in test_config.get('xperf_counters', []):
global_counters[c] = []
if test_config['shutdown']:
global_counters['shutdown'] = []
if test_config.get('responsiveness') and \
platform.system() != "Linux":
# ignore responsiveness tests on linux until we fix
# Bug 710296
setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10'
global_counters['responsiveness'] = []
# instantiate an object to hold test results
test_results = results.TestResults(test_config, global_counters)
for i in range(test_config['cycles']):
LOG.info("Running cycle %d/%d for %s test..." %
(i + 1, test_config['cycles'], test_config['name']))
# remove the browser error file
mozfile.remove(browser_config['error_filename'])
# reinstall any file whose stability we need to ensure across
# the cycles
if test_config.get('reinstall', ''):
for keep in test_config['reinstall']:
origin = os.path.join(test_config['profile_path'], keep)
dest = os.path.join(setup.profile_dir, keep)
LOG.debug("Reinstalling %s on top of %s" % (origin, dest))
shutil.copy(origin, dest)
# Run the test
timeout = test_config.get('timeout', 7200) # 2 hours default
if setup.sps_profile:
# When profiling, give the browser some extra time
# to dump the profile.
timeout += 5 * 60
command_args = utils.GenerateBrowserCommandLine(
browser_config["browser_path"],
browser_config["extra_args"],
setup.profile_dir,
test_config['url'],
profiling_info=(setup.sps_profile.profiling_info
if setup.sps_profile else None))
mainthread_error_count = 0
if test_config['setup']:
# Generate bcontroller.json for xperf
talosconfig.generateTalosConfig(command_args, browser_config,
test_config)
subprocess.call(['python'] + test_config['setup'].split(), )
mm_httpd = None
if test_config['name'] == 'media_tests':
from startup_test.media import media_manager
mm_httpd = media_manager.run_server(
os.path.dirname(os.path.realpath(__file__)))
counter_management = None
if counters:
counter_management = CounterManagement(
browser_config['process'], counters, resolution)
try:
pcontext = run_browser(
command_args,
minidump_dir,
timeout=timeout,
env=setup.env,
# start collecting counters as soon as possible
on_started=(counter_management.start
if counter_management else None),
)
finally:
if counter_management:
counter_management.stop()
if mm_httpd:
mm_httpd.stop()
if test_config['mainthread']:
rawlog = os.path.join(here, "mainthread_io.log")
if os.path.exists(rawlog):
processedlog = \
os.path.join(here, 'mainthread_io.json')
xre_path = \
os.path.dirname(browser_config['browser_path'])
mtio_py = os.path.join(here, 'mainthreadio.py')
command = [
'python', mtio_py, rawlog, processedlog, xre_path
]
mtio = subprocess.Popen(command,
env=os.environ.copy(),
stdout=subprocess.PIPE)
output, stderr = mtio.communicate()
for line in output.split('\n'):
if line.strip() == "":
continue
print line
mainthread_error_count += 1
mozfile.remove(rawlog)
if test_config['cleanup']:
# HACK: add the pid to support xperf where we require
# the pid in post processing
talosconfig.generateTalosConfig(command_args,
browser_config,
test_config,
pid=pcontext.pid)
subprocess.call([sys.executable] +
test_config['cleanup'].split())
# For startup tests, we launch the browser multiple times
# with the same profile
for fname in ('sessionstore.js', '.parentlock',
'sessionstore.bak'):
mozfile.remove(os.path.join(setup.profile_dir, fname))
# check for xperf errors
if os.path.exists(browser_config['error_filename']) or \
mainthread_error_count > 0:
raise TalosRegression(
"Talos has found a regression, if you have questions"
" ask for help in irc on #perf")
# add the results from the browser output
test_results.add('\n'.join(pcontext.output),
counter_results=(counter_management.results()
if counter_management else None))
if setup.sps_profile:
setup.sps_profile.symbolicate(i)
self.check_for_crashes(browser_config, minidump_dir,
test_config['name'])
# include global (cross-cycle) counters
test_results.all_counter_results.extend([{
key: value
} for key, value in global_counters.items()])
for c in test_results.all_counter_results:
for key, value in c.items():
LOG.debug("COUNTER %r: %s" % (key, value))
# return results
return test_results
def runTest(self, browser_config, test_config):
"""
Runs an url based test on the browser as specified in the browser_config dictionary
Args:
browser_config: Dictionary of configuration options for the browser (paths, prefs, etc)
test_config : Dictionary of configuration for the given test (url, cycles, counters, etc)
"""
self.initializeLibraries(browser_config)
utils.debug("operating with platform_type : %s", self.platform_type)
counters = test_config.get(self.platform_type + 'counters', [])
resolution = test_config['resolution']
utils.setEnvironmentVars(browser_config['env'])
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})
if browser_config['symbols_path']:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
else:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})
utils.setEnvironmentVars({
"LD_LIBRARY_PATH":
os.path.dirname(browser_config['browser_path'])
})
profile_dir = None
temp_dir = None
try:
running_processes = self._ffprocess.checkAllProcesses(
browser_config['process'], browser_config['child_process'])
if running_processes:
msg = " already running before testing started (unclean system)"
utils.debug("%s%s", browser_config['process'], msg)
running_processes_str = ", ".join([
('[%s] %s' % (pid, process_name))
for pid, process_name in running_processes
])
raise talosError(
"Found processes still running: %s. Please close them before running talos."
% running_processes_str)
# add any provided directories to the installed browser
for dir in browser_config['dirs']:
self._ffsetup.InstallInBrowser(browser_config['browser_path'],
browser_config['dirs'][dir])
# make profile path work cross-platform
test_config['profile_path'] = os.path.normpath(
test_config['profile_path'])
preferences = copy.deepcopy(browser_config['preferences'])
if 'preferences' in test_config and test_config['preferences']:
testPrefs = dict([
(i, utils.parsePref(j))
for i, j in test_config['preferences'].items()
])
preferences.update(testPrefs)
extensions = copy.deepcopy(browser_config['extensions'])
if 'extensions' in test_config and test_config['extensions']:
extensions.append(test_config['extensions'])
profile_dir, temp_dir = self.createProfile(
test_config['profile_path'], preferences, extensions,
browser_config['webserver'])
self.initializeProfile(profile_dir, browser_config)
if browser_config['fennecIDs']:
# This pushes environment variables to the device, be careful of placement
self.setupRobocopTests(browser_config, profile_dir)
utils.debug("initialized %s", browser_config['process'])
# setup global (cross-cycle) counters:
# shutdown, responsiveness
global_counters = {}
if browser_config.get('xperf_path'):
for c in test_config.get('xperf_counters', []):
global_counters[c] = []
if test_config['shutdown']:
global_counters['shutdown'] = []
if test_config.get(
'responsiveness') and platform.system() != "Linux":
# ignore responsiveness tests on linux until we fix Bug 710296
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP': '1'})
utils.setEnvironmentVars(
{'MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD': '20'})
utils.setEnvironmentVars(
{'MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL': '10'})
global_counters['responsiveness'] = []
# instantiate an object to hold test results
test_results = results.TestResults(
test_config,
global_counters,
extensions=self._ffsetup.extensions)
for i in range(test_config['cycles']):
# remove the browser log file
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
# remove the error file if it exists
if os.path.exists(browser_config['error_filename']):
os.chmod(browser_config['error_filename'], 0777)
os.remove(browser_config['error_filename'])
# on remote devices we do not have the fast launch/shutdown as we do on desktop
if not browser_config['remote']:
time.sleep(browser_config['browser_wait']
) #wait out the browser closing
# check to see if the previous cycle is still hanging around
if (i > 0) and self._ffprocess.checkAllProcesses(
browser_config['process'],
browser_config['child_process']):
raise talosError("previous cycle still running")
# Run the test
timeout = test_config.get('timeout', 7200) # 2 hours default
total_time = 0
url = test_config['url']
command_line = self._ffprocess.GenerateBrowserCommandLine(
browser_config['browser_path'],
browser_config['extra_args'], browser_config['deviceroot'],
profile_dir, url)
utils.debug("command line: %s", command_line)
b_log = browser_config['browser_log']
if self.remote == True:
b_log = browser_config[
'deviceroot'] + '/' + browser_config['browser_log']
self._ffprocess.removeFile(b_log)
# bug 816719, remove sessionstore.js so we don't interfere with talos
self._ffprocess.testAgent.removeFile(
os.path.join(self._ffprocess.testAgent.getDeviceRoot(),
"profile/sessionstore.js"))
b_cmd = self._ffprocess.GenerateBControllerCommandLine(
command_line, browser_config, test_config)
try:
process = subprocess.Popen(b_cmd,
universal_newlines=True,
bufsize=0,
env=os.environ)
except:
raise talosError(
"error executing browser command line '%s': %s" %
(subprocess.list2cmdline(b_cmd), sys.exc_info()[0]))
#give browser a chance to open
# this could mean that we are losing the first couple of data points
# as the tests starts, but if we don't provide
# some time for the browser to start we have trouble connecting the CounterManager to it
# on remote devices we do not have the fast launch/shutdown as we do on desktop
if not browser_config['remote']:
time.sleep(browser_config['browser_wait'])
#set up the counters for this test
counter_results = None
if counters:
cm = self.CounterManager(self._ffprocess,
browser_config['process'],
counters)
counter_results = dict([(counter, [])
for counter in counters])
#the main test loop, monitors counters and checks for browser output
dumpResult = ""
while total_time < timeout:
# Sleep for [resolution] seconds
time.sleep(resolution)
total_time += resolution
fileData = self._ffprocess.getFile(b_log)
if fileData and len(fileData) > 0:
newResults = fileData.replace(dumpResult, '')
if len(newResults.strip()) > 0:
utils.info(newResults)
dumpResult = fileData
# Get the output from all the possible counters
for count_type in counters:
val = cm.getCounterValue(count_type)
if val:
counter_results[count_type].append(val)
if process.poll(
) != None: #browser_controller completed, file now full
break
if hasattr(process, 'kill'):
# BBB python 2.4 does not have Popen.kill(); see
# https://bugzilla.mozilla.org/show_bug.cgi?id=752951#c6
try:
process.kill()
except OSError, e:
if (not mozinfo.isWin) and (e.errno != 3):
# 3 == No such process in Linux and Mac (errno.h)
raise
if total_time >= timeout:
raise talosError("timeout exceeded")
#stop the counter manager since this test is complete
if counters:
cm.stopMonitor()
# ensure the browser log exists
browser_log_filename = browser_config['browser_log']
if not os.path.isfile(browser_log_filename):
raise talosError("no output from browser [%s]" %
browser_log_filename)
# ensure the browser log exists
if os.path.exists(browser_config['error_filename']):
raise talosRegression(
"Talos has found a regression, if you have questions ask for help in irc on #perf"
)
# add the results from the browser output
test_results.add(browser_log_filename,
counter_results=counter_results)
# on remote devices we do not have the fast launch/shutdown as we do on desktop
if not browser_config['remote']:
time.sleep(browser_config['browser_wait'])
#clean up any stray browser processes
self.cleanupAndCheckForCrashes(browser_config, profile_dir,
test_config['name'])
#clean up the bcontroller process
timer = 0
while ((process.poll() is None)
and timer < browser_config['browser_wait']):
time.sleep(1)
timer += 1
# cleanup
self.cleanupProfile(temp_dir)
utils.restoreEnvironmentVars()
# include global (cross-cycle) counters
test_results.all_counter_results.extend([{
key: value
} for key, value in global_counters.items()])
# return results
return test_results
def runTest(self, browser_config, test_config):
"""
Runs an url based test on the browser as specified in the browser_config dictionary
Args:
browser_config: Dictionary of configuration options for the browser (paths, prefs, etc)
test_config : Dictionary of configuration for the given test (url, cycles, counters, etc)
"""
self.initializeLibraries(browser_config)
utils.debug("operating with platform_type : %s", self.platform_type)
self.counters = test_config.get(self.platform_type + 'counters', [])
self.resolution = test_config['resolution']
utils.setEnvironmentVars(browser_config['env'])
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})
if browser_config['symbols_path']:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
else:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})
utils.setEnvironmentVars({
"LD_LIBRARY_PATH":
os.path.dirname(browser_config['browser_path'])
})
profile_dir = None
temp_dir = None
try:
running_processes = self._ffprocess.checkAllProcesses(
browser_config['process'], browser_config['child_process'])
if running_processes:
msg = " already running before testing started (unclean system)"
utils.debug("%s%s", browser_config['process'], msg)
running_processes_str = ", ".join([
('[%s] %s' % (pid, process_name))
for pid, process_name in running_processes
])
raise talosError(
"Found processes still running: %s. Please close them before running talos."
% running_processes_str)
# add any provided directories to the installed browser
for dir in browser_config['dirs']:
self._ffsetup.InstallInBrowser(browser_config['browser_path'],
browser_config['dirs'][dir])
# make profile path work cross-platform
test_config['profile_path'] = os.path.normpath(
test_config['profile_path'])
preferences = copy.deepcopy(browser_config['preferences'])
if 'preferences' in test_config and test_config['preferences']:
testPrefs = dict([
(i, utils.parsePref(j))
for i, j in test_config['preferences'].items()
])
preferences.update(testPrefs)
extensions = copy.deepcopy(browser_config['extensions'])
if 'extensions' in test_config and test_config['extensions']:
extensions.append(test_config['extensions'])
profile_dir, temp_dir = self.createProfile(
test_config['profile_path'], preferences, extensions,
browser_config['webserver'])
self.initializeProfile(profile_dir, browser_config)
test_config['url'] = utils.interpolatePath(
test_config['url'],
profile_dir=profile_dir,
firefox_path=browser_config['browser_path'])
if browser_config['fennecIDs']:
# This pushes environment variables to the device, be careful of placement
self.setupRobocopTests(browser_config, profile_dir)
utils.debug("initialized %s", browser_config['process'])
# setup global (cross-cycle) counters:
# shutdown, responsiveness
global_counters = {}
if browser_config.get('xperf_path'):
for c in test_config.get('xperf_counters', []):
global_counters[c] = []
if test_config['shutdown']:
global_counters['shutdown'] = []
if test_config.get(
'responsiveness') and platform.system() != "Linux":
# ignore responsiveness tests on linux until we fix Bug 710296
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP': '1'})
utils.setEnvironmentVars(
{'MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD': '20'})
utils.setEnvironmentVars(
{'MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL': '10'})
global_counters['responsiveness'] = []
# instantiate an object to hold test results
test_results = results.TestResults(
test_config,
global_counters,
extensions=self._ffsetup.extensions)
for i in range(test_config['cycles']):
# remove the browser log file
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
# remove the error file if it exists
if os.path.exists(browser_config['error_filename']):
os.chmod(browser_config['error_filename'], 0777)
os.remove(browser_config['error_filename'])
# check to see if the previous cycle is still hanging around
if (i > 0) and self._ffprocess.checkAllProcesses(
browser_config['process'],
browser_config['child_process']):
raise talosError("previous cycle still running")
# Run the test
timeout = test_config.get('timeout', 7200) # 2 hours default
total_time = 0
command_args = utils.GenerateBrowserCommandLine(
browser_config["browser_path"],
browser_config["extra_args"], browser_config["deviceroot"],
profile_dir, test_config['url'])
self.counter_results = None
if not browser_config['remote']:
if test_config['setup']:
# Generate bcontroller.yml for xperf
utils.GenerateTalosConfig(command_args, browser_config,
test_config)
setup = talosProcess.talosProcess(
['python'] + test_config['setup'].split(),
env=os.environ.copy())
setup.run()
setup.wait()
self.isFinished = False
browser = talosProcess.talosProcess(
command_args,
env=os.environ.copy(),
logfile=browser_config['browser_log'])
browser.run(timeout=timeout)
self.pid = browser.pid
if self.counters:
self.cm = self.CounterManager(
browser_config['process'], self.counters)
self.counter_results = dict([
(counter, []) for counter in self.counters
])
cmthread = Thread(target=self.collectCounters)
cmthread.setDaemon(True) # don't hang on quit
cmthread.start()
# todo: ctrl+c doesn't close the browser windows
browser.wait()
browser = None
self.isFinished = True
if test_config['cleanup']:
#HACK: add the pid to support xperf where we require the pid in post processing
utils.GenerateTalosConfig(command_args,
browser_config,
test_config,
pid=self.pid)
cleanup = talosProcess.talosProcess(
['python'] + test_config['cleanup'].split(),
env=os.environ.copy())
cleanup.run()
cleanup.wait()
# allow mozprocess to terminate fully. It appears our log file is partial unless we wait
time.sleep(5)
else:
self._ffprocess.runProgram(browser_config,
command_args,
timeout=timeout)
# check if we found results from our webserver
if os.path.isfile(browser_config['results_log']):
shutil.move(browser_config['results_log'],
browser_config['browser_log'])
# ensure the browser log exists
browser_log_filename = browser_config['browser_log']
if not os.path.isfile(browser_log_filename):
raise talosError("no output from browser [%s]" %
browser_log_filename)
# ensure the browser log exists
if os.path.exists(browser_config['error_filename']):
raise talosRegression(
"Talos has found a regression, if you have questions ask for help in irc on #perf"
)
# add the results from the browser output
test_results.add(browser_log_filename,
counter_results=self.counter_results)
#clean up any stray browser processes
self.cleanupAndCheckForCrashes(browser_config, profile_dir,
test_config['name'])
#clean up the bcontroller process
timer = 0
# cleanup
self.cleanupProfile(temp_dir)
utils.restoreEnvironmentVars()
# include global (cross-cycle) counters
test_results.all_counter_results.extend([{
key: value
} for key, value in global_counters.items()])
# return results
return test_results
except Exception, e:
self.counters = vars().get('cm', self.counters)
self.testCleanup(browser_config, profile_dir, test_config,
self.counters, temp_dir)
raise
def _runTest(self, browser_config, test_config, setup):
minidump_dir = os.path.join(setup.profile_dir, 'minidumps')
counters = test_config.get('%s_counters' % self._get_counter_prefix(),
[])
resolution = test_config['resolution']
# add the mainthread_io to the environment variable, as defined
# in test.py configs
here = os.path.dirname(os.path.realpath(__file__))
if test_config['mainthread']:
mainthread_io = os.path.join(here, "mainthread_io.log")
setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io
if browser_config['disable_stylo']:
if browser_config['stylothreads']:
raise TalosError(
"--disable-stylo conflicts with --stylo-threads")
if browser_config['enable_stylo']:
raise TalosError(
"--disable-stylo conflicts with --enable-stylo")
# As we transition to Stylo, we need to set env vars and output data properly
if browser_config['enable_stylo']:
setup.env['STYLO_FORCE_ENABLED'] = '1'
if browser_config['disable_stylo']:
setup.env['STYLO_FORCE_DISABLED'] = '1'
# During the Stylo transition, measure different number of threads
if browser_config.get('stylothreads', 0) > 0:
setup.env['STYLO_THREADS'] = str(browser_config['stylothreads'])
test_config['url'] = utils.interpolate(
test_config['url'],
profile=setup.profile_dir,
firefox=browser_config['browser_path'])
# setup global (cross-cycle) counters:
# shutdown, responsiveness
global_counters = {}
if browser_config.get('xperf_path'):
for c in test_config.get('xperf_counters', []):
global_counters[c] = []
if test_config['shutdown']:
global_counters['shutdown'] = []
if test_config.get('responsiveness') and \
platform.system() != "Darwin":
# ignore osx for now as per bug 1245793
setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10'
global_counters['responsiveness'] = []
setup.env['JSGC_DISABLE_POISONING'] = '1'
setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1'
# if using mitmproxy we must allow access to 'external' sites
if browser_config.get('mitmproxy', False):
LOG.info(
"Using mitmproxy so setting MOZ_DISABLE_NONLOCAL_CONNECTIONS to 0"
)
setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '0'
# instantiate an object to hold test results
test_results = results.TestResults(test_config, global_counters,
browser_config.get('framework'))
for i in range(test_config['cycles']):
LOG.info("Running cycle %d/%d for %s test..." %
(i + 1, test_config['cycles'], test_config['name']))
# remove the browser error file
mozfile.remove(browser_config['error_filename'])
# reinstall any file whose stability we need to ensure across
# the cycles
if test_config.get('reinstall', ''):
for keep in test_config['reinstall']:
origin = os.path.join(test_config['profile_path'], keep)
dest = os.path.join(setup.profile_dir, keep)
LOG.debug("Reinstalling %s on top of %s" % (origin, dest))
shutil.copy(origin, dest)
# Run the test
timeout = test_config.get('timeout', 7200) # 2 hours default
if setup.gecko_profile:
# When profiling, give the browser some extra time
# to dump the profile.
timeout += 5 * 60
command_args = utils.GenerateBrowserCommandLine(
browser_config["browser_path"],
browser_config["extra_args"],
setup.profile_dir,
test_config['url'],
profiling_info=(setup.gecko_profile.profiling_info
if setup.gecko_profile else None))
mainthread_error_count = 0
if test_config['setup']:
# Generate bcontroller.json for xperf
talosconfig.generateTalosConfig(command_args, browser_config,
test_config)
subprocess.call(['python'] + test_config['setup'].split(), )
counter_management = None
if counters:
counter_management = CounterManagement(
browser_config['process'], counters, resolution)
try:
pcontext = run_browser(
command_args,
minidump_dir,
timeout=timeout,
env=setup.env,
# start collecting counters as soon as possible
on_started=(counter_management.start
if counter_management else None),
)
except:
self.check_for_crashes(browser_config, minidump_dir,
test_config['name'])
raise
finally:
if counter_management:
counter_management.stop()
if test_config['mainthread']:
rawlog = os.path.join(here, "mainthread_io.log")
if os.path.exists(rawlog):
processedlog = \
os.path.join(here, 'mainthread_io.json')
xre_path = \
os.path.dirname(browser_config['browser_path'])
mtio_py = os.path.join(here, 'mainthreadio.py')
command = [
'python', mtio_py, rawlog, processedlog, xre_path
]
mtio = subprocess.Popen(command,
env=os.environ.copy(),
stdout=subprocess.PIPE)
output, stderr = mtio.communicate()
for line in output.split('\n'):
if line.strip() == "":
continue
print(line)
mainthread_error_count += 1
mozfile.remove(rawlog)
if test_config['cleanup']:
# HACK: add the pid to support xperf where we require
# the pid in post processing
talosconfig.generateTalosConfig(command_args,
browser_config,
test_config,
pid=pcontext.pid)
subprocess.call([sys.executable] +
test_config['cleanup'].split())
# For startup tests, we launch the browser multiple times
# with the same profile
for fname in ('sessionstore.js', '.parentlock',
'sessionstore.bak'):
mozfile.remove(os.path.join(setup.profile_dir, fname))
# check for xperf errors
if os.path.exists(browser_config['error_filename']) or \
mainthread_error_count > 0:
raise TalosRegression(
"Talos has found a regression, if you have questions"
" ask for help in irc on #perf")
# add the results from the browser output
test_results.add('\n'.join(pcontext.output),
counter_results=(counter_management.results()
if counter_management else None))
if setup.gecko_profile:
setup.gecko_profile.symbolicate(i)
self.check_for_crashes(browser_config, minidump_dir,
test_config['name'])
# include global (cross-cycle) counters
test_results.all_counter_results.extend([{
key: value
} for key, value in global_counters.items()])
for c in test_results.all_counter_results:
for key, value in c.items():
LOG.debug("COUNTER %r: %s" % (key, value))
# return results
return test_results
def _runTest(self, browser_config, test_config, setup):
minidump_dir = os.path.join(setup.profile_dir, 'minidumps')
counters = test_config.get('%s_counters' % self._get_counter_prefix(),
[])
resolution = test_config['resolution']
# add the mainthread_io to the environment variable, as defined
# in test.py configs
here = os.path.dirname(os.path.realpath(__file__))
if test_config['mainthread']:
mainthread_io = os.path.join(here, 'mainthread_io.log')
setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io
# Stylo is on by default
setup.env['STYLO_FORCE_ENABLED'] = '1'
# During the Stylo transition, measure different number of threads
if browser_config.get('stylothreads', 0) > 0:
setup.env['STYLO_THREADS'] = str(browser_config['stylothreads'])
# set url if there is one (i.e. receiving a test page, not a manifest/pageloader test)
if test_config.get('url', None) is not None:
test_config['url'] = utils.interpolate(
test_config['url'],
profile=setup.profile_dir,
firefox=browser_config['browser_path'])
else:
setup.env['MOZ_USE_PAGELOADER'] = '1'
# setup global (cross-cycle) responsiveness counters
global_counters = {}
if browser_config.get('xperf_path'):
for c in test_config.get('xperf_counters', []):
global_counters[c] = []
if test_config.get('responsiveness') and \
platform.system() != 'Darwin':
# ignore osx for now as per bug 1245793
setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10'
global_counters['responsiveness'] = []
setup.env['JSGC_DISABLE_POISONING'] = '1'
setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1'
# if using mitmproxy we must allow access to 'external' sites
if browser_config.get('mitmproxy', False):
LOG.info(
'Using mitmproxy so setting MOZ_DISABLE_NONLOCAL_CONNECTIONS to 0'
)
setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '0'
# instantiate an object to hold test results
test_results = results.TestResults(test_config, global_counters,
browser_config.get('framework'))
for i in range(test_config['cycles']):
time.sleep(0.25)
LOG.info('Running cycle %d/%d for %s test...' %
(i + 1, test_config['cycles'], test_config['name']))
# remove the browser error file
mozfile.remove(browser_config['error_filename'])
# individual tests can have different frameworks
# TODO: ensure that we don't run >1 test with custom frameworks
if test_config.get('perfherder_framework', None) is not None:
test_results.framework = test_config['perfherder_framework']
# reinstall any file whose stability we need to ensure across
# the cycles
if test_config.get('reinstall', ''):
for keep in test_config['reinstall']:
origin = os.path.join(test_config['profile_path'], keep)
dest = os.path.join(setup.profile_dir, keep)
LOG.debug('Reinstalling %s on top of %s' % (origin, dest))
shutil.copy(origin, dest)
# Run the test
timeout = test_config.get('timeout', 7200) # 2 hours default
if setup.gecko_profile:
# When profiling, give the browser some extra time
# to dump the profile.
timeout += 5 * 60
# store profiling info for pageloader; too late to add it as browser pref
setup.env["TPPROFILINGINFO"] = json.dumps(
setup.gecko_profile.profiling_info)
command_args = utils.GenerateBrowserCommandLine(
browser_config['browser_path'],
browser_config['extra_args'],
setup.profile_dir,
test_config['url'],
profiling_info=(setup.gecko_profile.profiling_info
if setup.gecko_profile else None))
mainthread_error_count = 0
if test_config['setup']:
# Generate bcontroller.json for xperf
talosconfig.generateTalosConfig(command_args, browser_config,
test_config)
subprocess.call(['python'] + test_config['setup'].split(), )
counter_management = None
if counters:
counter_management = CounterManagement(
browser_config['process'], counters, resolution)
try:
pcontext = run_browser(
command_args,
minidump_dir,
timeout=timeout,
env=setup.env,
# start collecting counters as soon as possible
on_started=(counter_management.start
if counter_management else None),
debug=browser_config['debug'],
debugger=browser_config['debugger'],
debugger_args=browser_config['debugger_args'])
except Exception:
self.check_for_crashes(browser_config, minidump_dir,
test_config['name'])
raise
finally:
if counter_management:
counter_management.stop()
if test_config['mainthread']:
rawlog = os.path.join(here, 'mainthread_io.log')
if os.path.exists(rawlog):
processedlog = \
os.path.join(here, 'mainthread_io.json')
xre_path = \
os.path.dirname(browser_config['browser_path'])
mtio_py = os.path.join(here, 'mainthreadio.py')
command = [
'python', mtio_py, rawlog, processedlog, xre_path
]
mtio = subprocess.Popen(command,
env=os.environ.copy(),
stdout=subprocess.PIPE)
output, stderr = mtio.communicate()
for line in output.split('\n'):
if line.strip() == '':
continue
print(line)
mainthread_error_count += 1
mozfile.remove(rawlog)
if test_config['cleanup']:
# HACK: add the pid to support xperf where we require
# the pid in post processing
talosconfig.generateTalosConfig(command_args,
browser_config,
test_config,
pid=pcontext.pid)
subprocess.call([sys.executable] +
test_config['cleanup'].split())
# For startup tests, we launch the browser multiple times
# with the same profile
for fname in ('sessionstore.js', '.parentlock',
'sessionstore.bak'):
mozfile.remove(os.path.join(setup.profile_dir, fname))
# check for xperf errors
if os.path.exists(browser_config['error_filename']) or \
mainthread_error_count > 0:
raise TalosRegression(
'Talos has found a regression, if you have questions'
' ask for help in irc on #perf')
# add the results from the browser output
if not run_in_debug_mode(browser_config):
test_results.add(
'\n'.join(pcontext.output),
counter_results=(counter_management.results()
if counter_management else None))
if setup.gecko_profile:
setup.gecko_profile.symbolicate(i)
self.check_for_crashes(browser_config, minidump_dir,
test_config['name'])
# include global (cross-cycle) counters
test_results.all_counter_results.extend([{
key: value
} for key, value in global_counters.items()])
for c in test_results.all_counter_results:
for key, value in c.items():
LOG.debug('COUNTER %r: %s' % (key, value))
# if running against a code-coverage instrumented build, move the
# produced gcda files to a folder where they will be collected later
if browser_config.get('code_coverage', False):
setup.collect_or_clean_ccov()
# return results
return test_results
