def runTest(self, browser_config, test_config):
"""
Runs an url based test on the browser as specified in the browser_config dictionary
Args:
browser_config: Dictionary of configuration options for the browser (paths, prefs, etc)
test_config : Dictionary of configuration for the given test (url, cycles, counters, etc)
"""
self.initializeLibraries(browser_config)
utils.debug("operating with platform_type : %s", self.platform_type)
counters = test_config.get(self.platform_type + 'counters', [])
resolution = test_config['resolution']
utils.setEnvironmentVars(browser_config['env'])
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})
if browser_config['symbols_path']:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
else:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})
utils.setEnvironmentVars({
"LD_LIBRARY_PATH":
os.path.dirname(browser_config['browser_path'])
})
profile_dir = None
temp_dir = None
try:
running_processes = self._ffprocess.checkAllProcesses(
browser_config['process'], browser_config['child_process'])
if running_processes:
msg = " already running before testing started (unclean system)"
utils.debug("%s%s", browser_config['process'], msg)
running_processes_str = ", ".join([
('[%s] %s' % (pid, process_name))
for pid, process_name in running_processes
])
raise talosError(
"Found processes still running: %s. Please close them before running talos."
% running_processes_str)
# add any provided directories to the installed browser
for dir in browser_config['dirs']:
self._ffsetup.InstallInBrowser(browser_config['browser_path'],
browser_config['dirs'][dir])
# make profile path work cross-platform
test_config['profile_path'] = os.path.normpath(
test_config['profile_path'])
preferences = copy.deepcopy(browser_config['preferences'])
if 'preferences' in test_config and test_config['preferences']:
testPrefs = dict([
(i, utils.parsePref(j))
for i, j in test_config['preferences'].items()
])
preferences.update(testPrefs)
extensions = copy.deepcopy(browser_config['extensions'])
if 'extensions' in test_config and test_config['extensions']:
extensions.append(test_config['extensions'])
profile_dir, temp_dir = self.createProfile(
test_config['profile_path'], preferences, extensions,
browser_config['webserver'])
self.initializeProfile(profile_dir, browser_config)
if browser_config['fennecIDs']:
# This pushes environment variables to the device, be careful of placement
self.setupRobocopTests(browser_config, profile_dir)
utils.debug("initialized %s", browser_config['process'])
# setup global (cross-cycle) counters:
# shutdown, responsiveness
global_counters = {}
if browser_config.get('xperf_path'):
for c in test_config.get('xperf_counters', []):
global_counters[c] = []
if test_config['shutdown']:
global_counters['shutdown'] = []
if test_config.get(
'responsiveness') and platform.system() != "Linux":
# ignore responsiveness tests on linux until we fix Bug 710296
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP': '1'})
utils.setEnvironmentVars(
{'MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD': '20'})
utils.setEnvironmentVars(
{'MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL': '10'})
global_counters['responsiveness'] = []
# instantiate an object to hold test results
test_results = results.TestResults(
test_config,
global_counters,
extensions=self._ffsetup.extensions)
for i in range(test_config['cycles']):
# remove the browser log file
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
# remove the error file if it exists
if os.path.exists(browser_config['error_filename']):
os.chmod(browser_config['error_filename'], 0777)
os.remove(browser_config['error_filename'])
# on remote devices we do not have the fast launch/shutdown as we do on desktop
if not browser_config['remote']:
time.sleep(browser_config['browser_wait']
) #wait out the browser closing
# check to see if the previous cycle is still hanging around
if (i > 0) and self._ffprocess.checkAllProcesses(
browser_config['process'],
browser_config['child_process']):
raise talosError("previous cycle still running")
# Run the test
timeout = test_config.get('timeout', 7200) # 2 hours default
total_time = 0
url = test_config['url']
command_line = self._ffprocess.GenerateBrowserCommandLine(
browser_config['browser_path'],
browser_config['extra_args'], browser_config['deviceroot'],
profile_dir, url)
utils.debug("command line: %s", command_line)
b_log = browser_config['browser_log']
if self.remote == True:
b_log = browser_config[
'deviceroot'] + '/' + browser_config['browser_log']
self._ffprocess.removeFile(b_log)
# bug 816719, remove sessionstore.js so we don't interfere with talos
self._ffprocess.testAgent.removeFile(
os.path.join(self._ffprocess.testAgent.getDeviceRoot(),
"profile/sessionstore.js"))
b_cmd = self._ffprocess.GenerateBControllerCommandLine(
command_line, browser_config, test_config)
try:
process = subprocess.Popen(b_cmd,
universal_newlines=True,
bufsize=0,
env=os.environ)
except:
raise talosError(
"error executing browser command line '%s': %s" %
(subprocess.list2cmdline(b_cmd), sys.exc_info()[0]))
#give browser a chance to open
# this could mean that we are losing the first couple of data points
# as the tests starts, but if we don't provide
# some time for the browser to start we have trouble connecting the CounterManager to it
# on remote devices we do not have the fast launch/shutdown as we do on desktop
if not browser_config['remote']:
time.sleep(browser_config['browser_wait'])
#set up the counters for this test
counter_results = None
if counters:
cm = self.CounterManager(self._ffprocess,
browser_config['process'],
counters)
counter_results = dict([(counter, [])
for counter in counters])
#the main test loop, monitors counters and checks for browser output
dumpResult = ""
while total_time < timeout:
# Sleep for [resolution] seconds
time.sleep(resolution)
total_time += resolution
fileData = self._ffprocess.getFile(b_log)
if fileData and len(fileData) > 0:
newResults = fileData.replace(dumpResult, '')
if len(newResults.strip()) > 0:
utils.info(newResults)
dumpResult = fileData
# Get the output from all the possible counters
for count_type in counters:
val = cm.getCounterValue(count_type)
if val:
counter_results[count_type].append(val)
if process.poll(
) != None: #browser_controller completed, file now full
break
if hasattr(process, 'kill'):
# BBB python 2.4 does not have Popen.kill(); see
# https://bugzilla.mozilla.org/show_bug.cgi?id=752951#c6
try:
process.kill()
except OSError, e:
if (not mozinfo.isWin) and (e.errno != 3):
# 3 == No such process in Linux and Mac (errno.h)
raise
if total_time >= timeout:
raise talosError("timeout exceeded")
#stop the counter manager since this test is complete
if counters:
cm.stopMonitor()
# ensure the browser log exists
browser_log_filename = browser_config['browser_log']
if not os.path.isfile(browser_log_filename):
raise talosError("no output from browser [%s]" %
browser_log_filename)
# ensure the browser log exists
if os.path.exists(browser_config['error_filename']):
raise talosRegression(
"Talos has found a regression, if you have questions ask for help in irc on #perf"
)
# add the results from the browser output
test_results.add(browser_log_filename,
counter_results=counter_results)
# on remote devices we do not have the fast launch/shutdown as we do on desktop
if not browser_config['remote']:
time.sleep(browser_config['browser_wait'])
#clean up any stray browser processes
self.cleanupAndCheckForCrashes(browser_config, profile_dir,
test_config['name'])
#clean up the bcontroller process
timer = 0
while ((process.poll() is None)
and timer < browser_config['browser_wait']):
time.sleep(1)
timer += 1
# cleanup
self.cleanupProfile(temp_dir)
utils.restoreEnvironmentVars()
# include global (cross-cycle) counters
test_results.all_counter_results.extend([{
key: value
} for key, value in global_counters.items()])
# return results
return test_results
def runTest(self, browser_config, test_config):
"""
Runs an url based test on the browser as specified in the browser_config dictionary
Args:
browser_config: Dictionary of configuration options for the browser (paths, prefs, etc)
test_config : Dictionary of configuration for the given test (url, cycles, counters, etc)
"""
self.initializeLibraries(browser_config)
utils.debug("operating with platform_type : %s", self.platform_type)
counters = test_config.get(self.platform_type + 'counters', [])
resolution = test_config['resolution']
utils.setEnvironmentVars(browser_config['env'])
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})
if browser_config['symbols_path']:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
else:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})
utils.setEnvironmentVars({"LD_LIBRARY_PATH" : os.path.dirname(browser_config['browser_path'])})
profile_dir = None
temp_dir = None
try:
running_processes = self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process'])
if running_processes:
msg = " already running before testing started (unclean system)"
utils.debug("%s%s", browser_config['process'], msg)
running_processes_str = ", ".join([('[%s] %s' % (pid, process_name)) for pid, process_name in running_processes])
raise talosError("Found processes still running: %s. Please close them before running talos." % running_processes_str)
# add any provided directories to the installed browser
for dir in browser_config['dirs']:
self._ffsetup.InstallInBrowser(browser_config['browser_path'],
browser_config['dirs'][dir])
# make profile path work cross-platform
test_config['profile_path'] = os.path.normpath(test_config['profile_path'])
preferences = copy.deepcopy(browser_config['preferences'])
if 'preferences' in test_config and test_config['preferences']:
testPrefs = dict([(i, utils.parsePref(j)) for i, j in test_config['preferences'].items()])
preferences.update(testPrefs)
extensions = copy.deepcopy(browser_config['extensions'])
if 'extensions' in test_config and test_config['extensions']:
extensions.append(test_config['extensions'])
profile_dir, temp_dir = self.createProfile(test_config['profile_path'],
preferences,
extensions,
browser_config['webserver'])
self.initializeProfile(profile_dir, browser_config)
if browser_config['fennecIDs']:
# This pushes environment variables to the device, be careful of placement
self.setupRobocopTests(browser_config, profile_dir)
utils.debug("initialized %s", browser_config['process'])
# setup global (cross-cycle) counters:
# shutdown, responsiveness
global_counters = {}
if browser_config.get('xperf_path'):
for c in test_config.get('xperf_counters', []):
global_counters[c] = []
if test_config['shutdown']:
global_counters['shutdown'] = []
if test_config.get('responsiveness') and platform.system() != "Linux":
# ignore responsiveness tests on linux until we fix Bug 710296
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP': '1'})
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD': '20'})
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL': '10'})
global_counters['responsiveness'] = []
# instantiate an object to hold test results
test_results = results.TestResults(test_config, global_counters, extensions=self._ffsetup.extensions)
for i in range(test_config['cycles']):
# remove the browser log file
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
# remove the error file if it exists
if os.path.exists(browser_config['error_filename']):
os.chmod(browser_config['error_filename'], 0777)
os.remove(browser_config['error_filename'])
# on remote devices we do not have the fast launch/shutdown as we do on desktop
if not browser_config['remote']:
time.sleep(browser_config['browser_wait']) #wait out the browser closing
# check to see if the previous cycle is still hanging around
if (i > 0) and self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process']):
raise talosError("previous cycle still running")
# Run the test
timeout = test_config.get('timeout', 7200) # 2 hours default
total_time = 0
url = test_config['url']
command_line = self._ffprocess.GenerateBrowserCommandLine(browser_config['browser_path'],
browser_config['extra_args'],
browser_config['deviceroot'],
profile_dir,
url)
utils.debug("command line: %s", command_line)
b_log = browser_config['browser_log']
if self.remote == True:
b_log = browser_config['deviceroot'] + '/' + browser_config['browser_log']
self._ffprocess.removeFile(b_log)
# bug 816719, remove sessionstore.js so we don't interfere with talos
self._ffprocess.testAgent.removeFile(os.path.join(self._ffprocess.testAgent.getDeviceRoot(), "profile/sessionstore.js"))
b_cmd = self._ffprocess.GenerateBControllerCommandLine(command_line, browser_config, test_config)
try:
process = subprocess.Popen(b_cmd, universal_newlines=True, bufsize=0, env=os.environ)
except:
raise talosError("error executing browser command line '%s': %s" % (subprocess.list2cmdline(b_cmd), sys.exc_info()[0]))
#give browser a chance to open
# this could mean that we are losing the first couple of data points
# as the tests starts, but if we don't provide
# some time for the browser to start we have trouble connecting the CounterManager to it
# on remote devices we do not have the fast launch/shutdown as we do on desktop
if not browser_config['remote']:
time.sleep(browser_config['browser_wait'])
#set up the counters for this test
counter_results = None
if counters:
cm = self.CounterManager(self._ffprocess, browser_config['process'], counters)
counter_results = dict([(counter, []) for counter in counters])
#the main test loop, monitors counters and checks for browser output
dumpResult = ""
while total_time < timeout:
# Sleep for [resolution] seconds
time.sleep(resolution)
total_time += resolution
fileData = self._ffprocess.getFile(b_log)
if fileData and len(fileData) > 0:
newResults = fileData.replace(dumpResult, '')
if len(newResults.strip()) > 0:
utils.info(newResults)
dumpResult = fileData
# Get the output from all the possible counters
for count_type in counters:
val = cm.getCounterValue(count_type)
if val:
counter_results[count_type].append(val)
if process.poll() != None: #browser_controller completed, file now full
break
if hasattr(process, 'kill'):
# BBB python 2.4 does not have Popen.kill(); see
# https://bugzilla.mozilla.org/show_bug.cgi?id=752951#c6
try:
process.kill()
except OSError, e:
if (not mozinfo.isWin) and (e.errno != 3):
# 3 == No such process in Linux and Mac (errno.h)
raise
if total_time >= timeout:
raise talosError("timeout exceeded")
#stop the counter manager since this test is complete
if counters:
cm.stopMonitor()
# ensure the browser log exists
browser_log_filename = browser_config['browser_log']
if not os.path.isfile(browser_log_filename):
raise talosError("no output from browser [%s]" % browser_log_filename)
# ensure the browser log exists
if os.path.exists(browser_config['error_filename']):
raise talosRegression("Talos has found a regression, if you have questions ask for help in irc on #perf")
# add the results from the browser output
test_results.add(browser_log_filename, counter_results=counter_results)
# on remote devices we do not have the fast launch/shutdown as we do on desktop
if not browser_config['remote']:
time.sleep(browser_config['browser_wait'])
#clean up any stray browser processes
self.cleanupAndCheckForCrashes(browser_config, profile_dir, test_config['name'])
#clean up the bcontroller process
timer = 0
while ((process.poll() is None) and timer < browser_config['browser_wait']):
time.sleep(1)
timer+=1
# cleanup
self.cleanupProfile(temp_dir)
utils.restoreEnvironmentVars()
# include global (cross-cycle) counters
test_results.all_counter_results.extend([{key: value} for key, value in global_counters.items()])
# return results
return test_results
def runTest(self, browser_config, test_config):
"""
Runs an url based test on the browser as specified in the browser_config dictionary
Args:
browser_config: Dictionary of configuration options for the browser (paths, prefs, etc)
test_config : Dictionary of configuration for the given test (url, cycles, counters, etc)
"""
self.initializeLibraries(browser_config)
utils.debug("operating with platform_type : " + self.platform_type)
counters = test_config[self.platform_type + 'counters']
resolution = test_config['resolution']
all_browser_results = []
all_counter_results = []
format = ""
utils.setEnvironmentVars(browser_config['env'])
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})
if browser_config['symbols_path']:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
else:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})
utils.setEnvironmentVars({"LD_LIBRARY_PATH" : os.path.dirname(browser_config['browser_path'])})
profile_dir = None
try:
running_processes = self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process'])
if running_processes:
msg = " already running before testing started (unclean system)"
utils.debug(browser_config['process'] + msg)
raise talosError("Found processes still running: %s. Please close them before running talos." % ", ".join(running_processes))
for bundlename in browser_config['bundles']:
self._ffsetup.InstallBundleInBrowser(browser_config['browser_path'],
bundlename,
browser_config['bundles'][bundlename])
# add any provided directories to the installed browser
for dir in browser_config['dirs']:
self._ffsetup.InstallInBrowser(browser_config['browser_path'],
browser_config['dirs'][dir])
# make profile path work cross-platform
test_config['profile_path'] = os.path.normpath(test_config['profile_path'])
profile_dir, temp_dir = self.createProfile(test_config['profile_path'], browser_config)
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
self.initializeProfile(profile_dir, browser_config)
utils.debug("initialized " + browser_config['process'])
if test_config['shutdown']:
shutdown = []
if 'responsiveness' in test_config and test_config['responsiveness']:
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP': '1'})
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD': '20'})
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL': '10'})
responsiveness = []
for i in range(test_config['cycles']):
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
time.sleep(browser_config['browser_wait']) #wait out the browser closing
# check to see if the previous cycle is still hanging around
if (i > 0) and self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process']):
raise talosError("previous cycle still running")
# Execute the test's head script if there is one
if 'head' in test_config:
try:
subprocess.call(['python', test_config['head']])
except:
raise talosError("error executing head script: %s" % sys.exc_info()[0])
# Run the test
browser_results = ""
if 'timeout' in test_config:
timeout = test_config['timeout']
else:
timeout = 7200 # 2 hours
total_time = 0
output = ''
url = test_config['url']
command_line = self._ffprocess.GenerateBrowserCommandLine(browser_config['browser_path'],
browser_config['extra_args'],
profile_dir,
url)
utils.debug("command line: " + command_line)
b_log = browser_config['browser_log']
if (self.remote == True):
b_log = browser_config['deviceroot'] + '/' + browser_config['browser_log']
b_cmd = self._ffprocess.GenerateBControllerCommandLine(command_line, browser_config, test_config)
process = subprocess.Popen(b_cmd, universal_newlines=True, shell=True, bufsize=0, env=os.environ)
#give browser a chance to open
# this could mean that we are losing the first couple of data points
# as the tests starts, but if we don't provide
# some time for the browser to start we have trouble connecting the CounterManager to it
time.sleep(browser_config['browser_wait'])
#set up the counters for this test
if counters:
cm = self.cmanager.CounterManager(self._ffprocess, browser_config['process'], counters)
counter_results = {}
for counter in counters:
counter_results[counter] = []
startTime = -1
dumpResult = ""
#the main test loop, monitors counters and checks for browser output
while total_time < timeout:
# Sleep for [resolution] seconds
time.sleep(resolution)
total_time += resolution
fileData = self._ffprocess.getFile(b_log)
if (len(fileData) > 0):
utils.noisy(fileData.replace(dumpResult, ''))
dumpResult = fileData
# Get the output from all the possible counters
for count_type in counters:
val = cm.getCounterValue(count_type)
if (val):
counter_results[count_type].append(val)
if process.poll() != None: #browser_controller completed, file now full
timed_out = False
break
if total_time >= timeout:
raise talosError("timeout exceeded")
else:
#stop the counter manager since this test is complete
if counters:
cm.stopMonitor()
if not os.path.isfile(browser_config['browser_log']):
raise talosError("no output from browser")
results_file = open(browser_config['browser_log'], "r")
results_raw = results_file.read()
results_file.close()
match = self.RESULTS_REGEX.search(results_raw)
tpmatch = self.RESULTS_TP_REGEX.search(results_raw)
failmatch = self.RESULTS_REGEX_FAIL.search(results_raw)
if match:
browser_results += match.group(1)
startTime = int(match.group(2))
endTime = int(match.group(4))
format = "tsformat"
#TODO: this a stop gap until all of the tests start outputting the same format
elif tpmatch:
match = tpmatch
browser_results += match.group(1)
startTime = int(match.group(2))
endTime = int(match.group(4))
format = "tpformat"
elif failmatch:
match = failmatch
browser_results += match.group(1)
raise talosError(match.group(1))
else:
raise talosError("unrecognized output format")
time.sleep(browser_config['browser_wait'])
#clean up any stray browser processes
self.cleanupAndCheckForCrashes(browser_config, profile_dir)
#clean up the bcontroller process
timer = 0
while ((process.poll() is None) and timer < browser_config['browser_wait']):
time.sleep(1)
timer+=1
if test_config['shutdown']:
shutdown.append(endTime - startTime)
if 'responsiveness' in test_config and test_config['responsiveness']:
responsiveness = self.RESULTS_RESPONSIVENESS_REGEX.findall(results_raw)
all_browser_results.append(browser_results)
all_counter_results.append(counter_results)
# Execute the test's tail script if there is one
if 'tail' in test_config:
try:
subprocess.call(['python', test_config['tail']])
except:
raise talosError("error executing tail script: %s" % sys.exc_info()[0])
self.cleanupProfile(temp_dir)
utils.restoreEnvironmentVars()
if test_config['shutdown']:
all_counter_results.append({'shutdown' : shutdown})
if 'responsiveness' in test_config and test_config['responsiveness']:
all_counter_results.append({'responsiveness' : responsiveness})
return (all_browser_results, all_counter_results, format)
except:
try:
if 'cm' in vars():
cm.stopMonitor()
if os.path.isfile(browser_config['browser_log']):
results_file = open(browser_config['browser_log'], "r")
results_raw = results_file.read()
results_file.close()
utils.noisy(results_raw)
if profile_dir:
try:
self.cleanupAndCheckForCrashes(browser_config, profile_dir)
except talosError:
pass
if vars().has_key('temp_dir'):
self.cleanupProfile(temp_dir)
except talosError, te:
utils.debug("cleanup error: " + te.msg)
except:
utils.debug("unknown error during cleanup")
raise
def runTest(browser_config, test_config):
"""
Runs an url based test on the browser as specified in the browser_config dictionary
Args:
browser_config: Dictionary of configuration options for the browser (paths, prefs, etc)
test_config : Dictionary of configuration for the given test (url, cycles, counters, etc)
"""
utils.debug("operating with platform_type : " + platform_type)
counters = test_config[platform_type + 'counters']
resolution = test_config['resolution']
all_browser_results = []
all_counter_results = []
format = ""
utils.setEnvironmentVars(browser_config['env'])
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})
if browser_config['symbols_path']:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
else:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})
utils.setEnvironmentVars({"LD_LIBRARY_PATH" : os.path.dirname(browser_config['browser_path'])})
profile_dir = None
try:
if ffprocess.checkAllProcesses(browser_config['process']):
utils.debug(browser_config['process'] + " already running before testing started (unclean system)")
raise talosError("system not clean")
# add any provided directories to the installed browser
for dir in browser_config['dirs']:
ffsetup.InstallInBrowser(browser_config['browser_path'], browser_config['dirs'][dir])
# make profile path work cross-platform
test_config['profile_path'] = os.path.normpath(test_config['profile_path'])
profile_dir, temp_dir = test_config['profile_path'], test_config['profile_path']
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
#initializeProfile(profile_dir, browser_config)
utils.debug("initialized " + browser_config['process'])
if test_config['shutdown']:
shutdown = []
for i in range(test_config['cycles']):
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
time.sleep(browser_config['browser_wait']) #wait out the browser closing
# check to see if the previous cycle is still hanging around
if (i > 0) and ffprocess.checkAllProcesses(browser_config['process']):
raise talosError("previous cycle still running")
# Run the test
browser_results = ""
if 'timeout' in test_config:
timeout = test_config['timeout']
else:
timeout = 7200 # 2 hours
total_time = 0
output = ''
url = test_config['url']
command_line = ffprocess.GenerateBrowserCommandLine(browser_config['browser_path'], browser_config['extra_args'], profile_dir, url)
utils.debug("command line: " + command_line)
bcontroller = path('talos/bcontroller.py')
if 'url_mod' in test_config:
process = subprocess.Popen('python %s --command "%s" --mod "%s" --name %s --timeout %d --log %s' % (bcontroller, command_line, test_config['url_mod'], browser_config['process'], browser_config['browser_wait'], browser_config['browser_log']), universal_newlines=True, shell=True, bufsize=0, env=os.environ)
else:
process = subprocess.Popen('python %s --command "%s" --name %s --timeout %d --log %s' % (bcontroller, command_line, browser_config['process'], browser_config['browser_wait'], browser_config['browser_log']), universal_newlines=True, shell=True, bufsize=0, env=os.environ)
#give browser a chance to open
# this could mean that we are losing the first couple of data points as the tests starts, but if we don't provide
# some time for the browser to start we have trouble connecting the CounterManager to it
time.sleep(browser_config['browser_wait'])
#set up the counters for this test
if counters:
cm = CounterManager(browser_config['process'], counters)
cm.startMonitor()
counter_results = {}
for counter in counters:
counter_results[counter] = []
startTime = -1
while total_time < timeout: #the main test loop, monitors counters and checks for browser ouptut
# Sleep for [resolution] seconds
time.sleep(resolution)
total_time += resolution
# Get the output from all the possible counters
for count_type in counters:
val = cm.getCounterValue(count_type)
if (val):
counter_results[count_type].append(val)
if process.poll() != None: #browser_controller completed, file now full
#stop the counter manager since this test is complete
if counters:
cm.stopMonitor()
if not os.path.isfile(browser_config['browser_log']):
raise talosError("no output from browser")
results_file = open(browser_config['browser_log'], "r")
results_raw = results_file.read()
results_file.close()
utils.noisy(results_raw)
match = RESULTS_REGEX.search(results_raw)
if match:
browser_results += match.group(1)
startTime = int(match.group(2))
endTime = int(match.group(3))
utils.debug("Matched basic results: " + browser_results)
format = "tsformat"
break
#TODO: this a stop gap until all of the tests start outputting the same format
match = RESULTS_TP_REGEX.search(results_raw)
if match:
browser_results += match.group(1)
startTime = int(match.group(2))
endTime = int(match.group(3))
utils.debug("Matched tp results: " + browser_results)
format = "tpformat"
break
match = RESULTS_REGEX_FAIL.search(results_raw)
if match:
browser_results += match.group(1)
utils.debug("Matched fail results: " + browser_results)
raise talosError(match.group(1))
raise talosError("unrecognized output format")
if total_time >= timeout:
raise talosError("timeout exceeded")
time.sleep(browser_config['browser_wait'])
#clean up the process
timer = 0
while ((process.poll() is None) and timer < browser_config['browser_wait']):
time.sleep(1)
timer+=1
if test_config['shutdown']:
shutdown.append(endTime - startTime)
checkForCrashes(browser_config, profile_dir)
utils.debug("Completed test with: " + browser_results)
all_browser_results.append(browser_results)
all_counter_results.append(counter_results)
ffprocess.cleanupProcesses(browser_config['process'], browser_config['browser_wait'])
#cleanupProfile(temp_dir)
utils.restoreEnvironmentVars()
if test_config['shutdown']:
all_counter_results.append({'shutdown' : shutdown})
return (all_browser_results, all_counter_results, format)
except:
try:
if 'cm' in vars():
cm.stopMonitor()
if os.path.isfile(browser_config['browser_log']):
results_file = open(browser_config['browser_log'], "r")
results_raw = results_file.read()
results_file.close()
utils.noisy(results_raw)
ffprocess.cleanupProcesses(browser_config['process'], browser_config['browser_wait'])
if profile_dir:
try:
checkForCrashes(browser_config, profile_dir)
except talosError:
pass
if vars().has_key('temp_dir'):
pass
#cleanupProfile(temp_dir)
except talosError, te:
utils.debug("cleanup error: " + te.msg)
except:
utils.debug("unknown error during cleanup")
raise
def runTest(self, browser_config, test_config):
"""
Runs an url based test on the browser as specified in the browser_config dictionary
Args:
browser_config: Dictionary of configuration options for the browser (paths, prefs, etc)
test_config : Dictionary of configuration for the given test (url, cycles, counters, etc)
"""
self.initializeLibraries(browser_config)
utils.debug("operating with platform_type : %s", self.platform_type)
self.counters = test_config.get(self.platform_type + 'counters', [])
self.resolution = test_config['resolution']
utils.setEnvironmentVars(browser_config['env'])
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})
if browser_config['symbols_path']:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
else:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})
utils.setEnvironmentVars({
"LD_LIBRARY_PATH":
os.path.dirname(browser_config['browser_path'])
})
profile_dir = None
temp_dir = None
try:
running_processes = self._ffprocess.checkAllProcesses(
browser_config['process'], browser_config['child_process'])
if running_processes:
msg = " already running before testing started (unclean system)"
utils.debug("%s%s", browser_config['process'], msg)
running_processes_str = ", ".join([
('[%s] %s' % (pid, process_name))
for pid, process_name in running_processes
])
raise talosError(
"Found processes still running: %s. Please close them before running talos."
% running_processes_str)
# add any provided directories to the installed browser
for dir in browser_config['dirs']:
self._ffsetup.InstallInBrowser(browser_config['browser_path'],
browser_config['dirs'][dir])
# make profile path work cross-platform
test_config['profile_path'] = os.path.normpath(
test_config['profile_path'])
preferences = copy.deepcopy(browser_config['preferences'])
if 'preferences' in test_config and test_config['preferences']:
testPrefs = dict([
(i, utils.parsePref(j))
for i, j in test_config['preferences'].items()
])
preferences.update(testPrefs)
extensions = copy.deepcopy(browser_config['extensions'])
if 'extensions' in test_config and test_config['extensions']:
extensions.append(test_config['extensions'])
profile_dir, temp_dir = self.createProfile(
test_config['profile_path'], preferences, extensions,
browser_config['webserver'])
self.initializeProfile(profile_dir, browser_config)
test_config['url'] = utils.interpolatePath(
test_config['url'],
profile_dir=profile_dir,
firefox_path=browser_config['browser_path'])
if browser_config['fennecIDs']:
# This pushes environment variables to the device, be careful of placement
self.setupRobocopTests(browser_config, profile_dir)
utils.debug("initialized %s", browser_config['process'])
# setup global (cross-cycle) counters:
# shutdown, responsiveness
global_counters = {}
if browser_config.get('xperf_path'):
for c in test_config.get('xperf_counters', []):
global_counters[c] = []
if test_config['shutdown']:
global_counters['shutdown'] = []
if test_config.get(
'responsiveness') and platform.system() != "Linux":
# ignore responsiveness tests on linux until we fix Bug 710296
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP': '1'})
utils.setEnvironmentVars(
{'MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD': '20'})
utils.setEnvironmentVars(
{'MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL': '10'})
global_counters['responsiveness'] = []
# instantiate an object to hold test results
test_results = results.TestResults(
test_config,
global_counters,
extensions=self._ffsetup.extensions)
for i in range(test_config['cycles']):
# remove the browser log file
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
# remove the error file if it exists
if os.path.exists(browser_config['error_filename']):
os.chmod(browser_config['error_filename'], 0777)
os.remove(browser_config['error_filename'])
# check to see if the previous cycle is still hanging around
if (i > 0) and self._ffprocess.checkAllProcesses(
browser_config['process'],
browser_config['child_process']):
raise talosError("previous cycle still running")
# Run the test
timeout = test_config.get('timeout', 7200) # 2 hours default
total_time = 0
command_args = utils.GenerateBrowserCommandLine(
browser_config["browser_path"],
browser_config["extra_args"], browser_config["deviceroot"],
profile_dir, test_config['url'])
self.counter_results = None
if not browser_config['remote']:
if test_config['setup']:
# Generate bcontroller.yml for xperf
utils.GenerateTalosConfig(command_args, browser_config,
test_config)
setup = talosProcess.talosProcess(
['python'] + test_config['setup'].split(),
env=os.environ.copy())
setup.run()
setup.wait()
self.isFinished = False
browser = talosProcess.talosProcess(
command_args,
env=os.environ.copy(),
logfile=browser_config['browser_log'])
browser.run(timeout=timeout)
self.pid = browser.pid
if self.counters:
self.cm = self.CounterManager(
browser_config['process'], self.counters)
self.counter_results = dict([
(counter, []) for counter in self.counters
])
cmthread = Thread(target=self.collectCounters)
cmthread.setDaemon(True) # don't hang on quit
cmthread.start()
# todo: ctrl+c doesn't close the browser windows
browser.wait()
browser = None
self.isFinished = True
if test_config['cleanup']:
#HACK: add the pid to support xperf where we require the pid in post processing
utils.GenerateTalosConfig(command_args,
browser_config,
test_config,
pid=self.pid)
cleanup = talosProcess.talosProcess(
['python'] + test_config['cleanup'].split(),
env=os.environ.copy())
cleanup.run()
cleanup.wait()
# allow mozprocess to terminate fully. It appears our log file is partial unless we wait
time.sleep(5)
else:
self._ffprocess.runProgram(browser_config,
command_args,
timeout=timeout)
# check if we found results from our webserver
if os.path.isfile(browser_config['results_log']):
shutil.move(browser_config['results_log'],
browser_config['browser_log'])
# ensure the browser log exists
browser_log_filename = browser_config['browser_log']
if not os.path.isfile(browser_log_filename):
raise talosError("no output from browser [%s]" %
browser_log_filename)
# ensure the browser log exists
if os.path.exists(browser_config['error_filename']):
raise talosRegression(
"Talos has found a regression, if you have questions ask for help in irc on #perf"
)
# add the results from the browser output
test_results.add(browser_log_filename,
counter_results=self.counter_results)
#clean up any stray browser processes
self.cleanupAndCheckForCrashes(browser_config, profile_dir,
test_config['name'])
#clean up the bcontroller process
timer = 0
# cleanup
self.cleanupProfile(temp_dir)
utils.restoreEnvironmentVars()
# include global (cross-cycle) counters
test_results.all_counter_results.extend([{
key: value
} for key, value in global_counters.items()])
# return results
return test_results
except Exception, e:
self.counters = vars().get('cm', self.counters)
self.testCleanup(browser_config, profile_dir, test_config,
self.counters, temp_dir)
raise
def runTest(browser_config, test_config):
"""
Runs an url based test on the browser as specified in the browser_config dictionary
Args:
browser_config: Dictionary of configuration options for the browser (paths, prefs, etc)
test_config : Dictionary of configuration for the given test (url, cycles, counters, etc)
"""
utils.debug("operating with platform_type : " + platform_type)
counters = test_config[platform_type + 'counters']
resolution = test_config['resolution']
all_browser_results = []
all_counter_results = []
utils.setEnvironmentVars(browser_config['env'])
try:
if checkAllProcesses():
utils.debug("firefox already running before testing started (unclean system)")
raise talosError("system not clean")
# add any provided directories to the installed firefox
for dir in browser_config['dirs']:
ffsetup.InstallInBrowser(browser_config['firefox'], browser_config['dirs'][dir])
profile_dir, temp_dir = createProfile(browser_config)
initializeProfile(profile_dir, browser_config)
utils.debug("initialized firefox")
ffprocess.Sleep()
for i in range(test_config['cycles']):
# check to see if the previous cycle is still hanging around
if (i > 0) and checkAllProcesses():
raise talosError("previous cycle still running")
# Run the test
browser_results = ""
if 'timeout' in test_config:
timeout = test_config['timeout']
else:
timeout = 28800 # 8 hours
if 'pagetimeout' in test_config:
pagetimeout = test_config['pagetimeout']
else:
pagetimeout = 28800 # 8 hours
total_time = 0
output = ''
url = test_config['url']
if 'url_mod' in test_config:
url += eval(test_config['url_mod'])
command_line = ffprocess.GenerateFirefoxCommandLine(browser_config['firefox'], profile_dir, url)
utils.debug("command line: " + command_line)
process = subprocess.Popen(command_line, stdout=subprocess.PIPE, universal_newlines=True, shell=True, bufsize=0, env=os.environ)
handle = process.stdout
#give firefox a chance to open
# this could mean that we are losing the first couple of data points as the tests starts, but if we don't provide
# some time for the browser to start we have trouble connecting the CounterManager to it
ffprocess.Sleep()
#set up the counters for this test
if counters:
cm = CounterManager("firefox", counters)
cm.startMonitor()
counter_results = {}
for counter in counters:
counter_results[counter] = []
busted = False
lastNoise = 0
while total_time < timeout:
# Sleep for [resolution] seconds
time.sleep(resolution)
total_time += resolution
lastNoise += resolution
# Get the output from all the possible counters
for count_type in counters:
val = cm.getCounterValue(count_type)
if (val):
counter_results[count_type].append(val)
# Check to see if page load times were outputted
(bytes, current_output) = ffprocess.NonBlockingReadProcessOutput(handle)
output += current_output
match = RESULTS_GENERIC.search(current_output)
if match:
if match.group(1):
utils.noisy(match.group(1))
lastNoise = 0
match = RESULTS_REGEX.search(output)
if match:
browser_results += match.group(1)
utils.debug("Matched basic results: " + browser_results)
break
#TODO: this a stop gap until all of the tests start outputting the same format
match = RESULTS_TP_REGEX.search(output)
if match:
browser_results += match.group(1)
utils.debug("Matched tp results: " + browser_results)
break
match = RESULTS_REGEX_FAIL.search(output)
if match:
browser_results += match.group(1)
utils.debug("Matched fail results: " + browser_results)
raise talosError(match.group(1))
#ensure that the browser is still running
#check at intervals of 60 - this is just to cut down on load
#use the busted check to ensure that we aren't catching a bad time slice where the browser has
# completed the test and closed but we haven't picked up the result yet
if busted:
raise talosError("browser crash")
if (total_time % 60 == 0):
if not checkBrowserAlive():
busted = True
if lastNoise > pagetimeout:
raise talosError("browser frozen")
if total_time >= timeout:
raise talosError("timeout exceeded")
#stop the counter manager since this test is complete
if counters:
cm.stopMonitor()
utils.debug("Completed test with: " + browser_results)
all_browser_results.append(browser_results)
all_counter_results.append(counter_results)
cleanupProcesses()
cleanupProfile(temp_dir, browser_config)
utils.restoreEnvironmentVars()
return (all_browser_results, all_counter_results)
except:
try:
cleanupProcesses()
if vars().has_key('temp_dir'):
cleanupProfile(temp_dir, browser_config)
except talosError, te:
utils.debug("cleanup error: " + te.msg)
except:
utils.debug("unknown error during cleanup")
raise
def runTest(self, browser_config, test_config):
"""
Runs an url based test on the browser as specified in the browser_config dictionary
Args:
browser_config: Dictionary of configuration options for the browser (paths, prefs, etc)
test_config : Dictionary of configuration for the given test (url, cycles, counters, etc)
"""
self.initializeLibraries(browser_config)
utils.debug("operating with platform_type : %s", self.platform_type)
self.counters = test_config.get(self.platform_type + 'counters', [])
self.resolution = test_config['resolution']
utils.setEnvironmentVars(browser_config['env'])
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})
if browser_config['symbols_path']:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
else:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})
utils.setEnvironmentVars({"LD_LIBRARY_PATH" : os.path.dirname(browser_config['browser_path'])})
profile_dir = None
temp_dir = None
try:
running_processes = self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process'])
if running_processes:
msg = " already running before testing started (unclean system)"
utils.debug("%s%s", browser_config['process'], msg)
running_processes_str = ", ".join([('[%s] %s' % (pid, process_name)) for pid, process_name in running_processes])
raise talosError("Found processes still running: %s. Please close them before running talos." % running_processes_str)
# add any provided directories to the installed browser
for dir in browser_config['dirs']:
self._ffsetup.InstallInBrowser(browser_config['browser_path'],
browser_config['dirs'][dir])
# make profile path work cross-platform
test_config['profile_path'] = os.path.normpath(test_config['profile_path'])
preferences = copy.deepcopy(browser_config['preferences'])
if 'preferences' in test_config and test_config['preferences']:
testPrefs = dict([(i, utils.parsePref(j)) for i, j in test_config['preferences'].items()])
preferences.update(testPrefs)
extensions = copy.deepcopy(browser_config['extensions'])
if 'extensions' in test_config and test_config['extensions']:
extensions.append(test_config['extensions'])
profile_dir, temp_dir = self.createProfile(test_config['profile_path'],
preferences,
extensions,
browser_config['webserver'])
self.initializeProfile(profile_dir, browser_config)
test_config['url'] = utils.interpolatePath(test_config['url'], profile_dir=profile_dir, firefox_path=browser_config['browser_path'])
if browser_config['fennecIDs']:
# This pushes environment variables to the device, be careful of placement
self.setupRobocopTests(browser_config, profile_dir)
utils.debug("initialized %s", browser_config['process'])
# setup global (cross-cycle) counters:
# shutdown, responsiveness
global_counters = {}
if browser_config.get('xperf_path'):
for c in test_config.get('xperf_counters', []):
global_counters[c] = []
if test_config['shutdown']:
global_counters['shutdown'] = []
if test_config.get('responsiveness') and platform.system() != "Linux":
# ignore responsiveness tests on linux until we fix Bug 710296
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP': '1'})
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD': '20'})
utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL': '10'})
global_counters['responsiveness'] = []
# instantiate an object to hold test results
test_results = results.TestResults(test_config, global_counters, extensions=self._ffsetup.extensions)
for i in range(test_config['cycles']):
# remove the browser log file
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
# remove the error file if it exists
if os.path.exists(browser_config['error_filename']):
os.chmod(browser_config['error_filename'], 0777)
os.remove(browser_config['error_filename'])
# check to see if the previous cycle is still hanging around
if (i > 0) and self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process']):
raise talosError("previous cycle still running")
# Run the test
timeout = test_config.get('timeout', 7200) # 2 hours default
total_time = 0
command_args = utils.GenerateBrowserCommandLine(browser_config["browser_path"],
browser_config["extra_args"],
browser_config["deviceroot"],
profile_dir,
test_config['url'])
self.counter_results = None
if not browser_config['remote']:
if test_config['setup']:
# Generate bcontroller.yml for xperf
utils.GenerateTalosConfig(command_args, browser_config, test_config)
setup = talosProcess.talosProcess(['python'] + test_config['setup'].split(), env=os.environ.copy())
setup.run()
setup.wait()
self.isFinished = False
browser = talosProcess.talosProcess(command_args, env=os.environ.copy(), logfile=browser_config['browser_log'])
browser.run(timeout=timeout)
self.pid = browser.pid
if self.counters:
self.cm = self.CounterManager(browser_config['process'], self.counters)
self.counter_results = dict([(counter, []) for counter in self.counters])
cmthread = Thread(target=self.collectCounters)
cmthread.setDaemon(True) # don't hang on quit
cmthread.start()
# todo: ctrl+c doesn't close the browser windows
browser.wait()
browser = None
self.isFinished = True
if test_config['cleanup']:
#HACK: add the pid to support xperf where we require the pid in post processing
utils.GenerateTalosConfig(command_args, browser_config, test_config, pid=self.pid)
cleanup = talosProcess.talosProcess(['python'] + test_config['cleanup'].split(), env=os.environ.copy())
cleanup.run()
cleanup.wait()
# allow mozprocess to terminate fully. It appears our log file is partial unless we wait
time.sleep(5)
else:
self._ffprocess.runProgram(browser_config, command_args, timeout=timeout)
# check if we found results from our webserver
if os.path.isfile(browser_config['results_log']):
shutil.move(browser_config['results_log'], browser_config['browser_log'])
# ensure the browser log exists
browser_log_filename = browser_config['browser_log']
if not os.path.isfile(browser_log_filename):
raise talosError("no output from browser [%s]" % browser_log_filename)
# ensure the browser log exists
if os.path.exists(browser_config['error_filename']):
raise talosRegression("Talos has found a regression, if you have questions ask for help in irc on #perf")
# add the results from the browser output
test_results.add(browser_log_filename, counter_results=self.counter_results)
#clean up any stray browser processes
self.cleanupAndCheckForCrashes(browser_config, profile_dir, test_config['name'])
#clean up the bcontroller process
timer = 0
# cleanup
self.cleanupProfile(temp_dir)
utils.restoreEnvironmentVars()
# include global (cross-cycle) counters
test_results.all_counter_results.extend([{key: value} for key, value in global_counters.items()])
# return results
return test_results
except Exception, e:
self.counters = vars().get('cm', self.counters)
self.testCleanup(browser_config, profile_dir, test_config, self.counters, temp_dir)
raise
def runTest(self, browser_config, test_config):
"""
Runs an url based test on the browser as specified in the browser_config dictionary
Args:
browser_config: Dictionary of configuration options for the browser (paths, prefs, etc)
test_config : Dictionary of configuration for the given test (url, cycles, counters, etc)
"""
self.initializeLibraries(browser_config)
utils.debug("operating with platform_type : " + self.platform_type)
counters = test_config[self.platform_type + 'counters']
resolution = test_config['resolution']
all_browser_results = []
all_counter_results = []
format = ""
utils.setEnvironmentVars(browser_config['env'])
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})
if browser_config['symbols_path']:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
else:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})
utils.setEnvironmentVars({"LD_LIBRARY_PATH" : os.path.dirname(browser_config['browser_path'])})
profile_dir = None
try:
if self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process']):
msg = " already running before testing started (unclean system)"
utils.debug(browser_config['process'] + msg)
raise talosError("system not clean")
for bundlename in browser_config['bundles']:
self._ffsetup.InstallBundleInBrowser(browser_config['browser_path'],
bundlename,
browser_config['bundles'][bundlename])
# add any provided directories to the installed browser
for dir in browser_config['dirs']:
self._ffsetup.InstallInBrowser(browser_config['browser_path'],
browser_config['dirs'][dir])
# make profile path work cross-platform
test_config['profile_path'] = os.path.normpath(test_config['profile_path'])
profile_dir, temp_dir = self.createProfile(test_config['profile_path'], browser_config)
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
self.initializeProfile(profile_dir, browser_config)
utils.debug("initialized " + browser_config['process'])
if test_config['shutdown']:
shutdown = []
for i in range(test_config['cycles']):
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
time.sleep(browser_config['browser_wait']) #wait out the browser closing
# check to see if the previous cycle is still hanging around
if (i > 0) and self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process']):
raise talosError("previous cycle still running")
# Execute the test's head script if there is one
if 'head' in test_config:
try:
subprocess.call(['python', test_config['head']])
except:
raise talosError("error executing head script: %s" % sys.exc_info()[0])
# Run the test
browser_results = ""
if 'timeout' in test_config:
timeout = test_config['timeout']
else:
timeout = 7200 # 2 hours
total_time = 0
output = ''
url = test_config['url']
command_line = self._ffprocess.GenerateBrowserCommandLine(browser_config['browser_path'],
browser_config['extra_args'],
profile_dir,
url)
utils.debug("command line: " + command_line)
b_cmd = 'python bcontroller.py --command "%s"' % (command_line)
b_cmd += ' --child_process %s ' % (browser_config['child_process'])
b_cmd += ' --name %s ' % (browser_config['process'])
b_cmd += ' --timeout %d ' % (browser_config['browser_wait'])
b_cmd += ' --log %s ' % (browser_config['browser_log'])
if 'url_mod' in test_config:
b_cmd += ' --mod "%s" ' % (test_config['url_mod'])
if (self.remote == True):
b_cmd += ' --host "%s" ' % (browser_config['host'])
b_cmd += ' --port "%s" ' % (browser_config['port'])
b_cmd += ' --deviceRoot "%s" ' % (browser_config['deviceroot'])
process = subprocess.Popen(b_cmd, universal_newlines=True, shell=True, bufsize=0, env=os.environ)
#give browser a chance to open
# this could mean that we are losing the first couple of data points
# as the tests starts, but if we don't provide
# some time for the browser to start we have trouble connecting the CounterManager to it
time.sleep(browser_config['browser_wait'])
#set up the counters for this test
if counters:
cm = self.cmanager.CounterManager(self._ffprocess, browser_config['process'], counters)
counter_results = {}
for counter in counters:
counter_results[counter] = []
startTime = -1
dumpResult = ""
#the main test loop, monitors counters and checks for browser ouptut
while total_time < timeout:
# Sleep for [resolution] seconds
time.sleep(resolution)
total_time += resolution
fileData = self._ffprocess.getFile(browser_config['browser_log'])
if (len(fileData) > 0):
utils.noisy(fileData.replace(dumpResult, ''))
dumpResult = fileData
# Get the output from all the possible counters
for count_type in counters:
val = cm.getCounterValue(count_type)
if (val):
counter_results[count_type].append(val)
if process.poll() != None: #browser_controller completed, file now full
timed_out = False
break
if total_time >= timeout:
raise talosError("timeout exceeded")
else:
#stop the counter manager since this test is complete
if counters:
cm.stopMonitor()
if not os.path.isfile(browser_config['browser_log']):
raise talosError("no output from browser")
results_file = open(browser_config['browser_log'], "r")
results_raw = results_file.read()
results_file.close()
match = self.RESULTS_REGEX.search(results_raw)
tpmatch = self.RESULTS_TP_REGEX.search(results_raw)
failmatch = self.RESULTS_REGEX_FAIL.search(results_raw)
if match:
browser_results += match.group(1)
startTime = int(match.group(2))
endTime = int(match.group(3))
format = "tsformat"
#TODO: this a stop gap until all of the tests start outputting the same format
elif tpmatch:
match = tpmatch
browser_results += match.group(1)
startTime = int(match.group(2))
endTime = int(match.group(3))
format = "tpformat"
elif failmatch:
match = failmatch
browser_results += match.group(1)
raise talosError(match.group(1))
else:
raise talosError("unrecognized output format")
time.sleep(browser_config['browser_wait'])
#clean up the process
timer = 0
while ((process.poll() is None) and timer < browser_config['browser_wait']):
time.sleep(1)
timer+=1
if test_config['shutdown']:
shutdown.append(endTime - startTime)
self.checkForCrashes(browser_config, profile_dir)
all_browser_results.append(browser_results)
all_counter_results.append(counter_results)
# Execute the test's tail script if there is one
if 'tail' in test_config:
try:
subprocess.call(['python', test_config['tail']])
except:
raise talosError("error executing tail script: %s" % sys.exc_info()[0])
self._ffprocess.cleanupProcesses(browser_config['process'], browser_config['child_process'], browser_config['browser_wait'])
self.cleanupProfile(temp_dir)
utils.restoreEnvironmentVars()
if test_config['shutdown']:
all_counter_results.append({'shutdown' : shutdown})
return (all_browser_results, all_counter_results, format)
except:
try:
if 'cm' in vars():
cm.stopMonitor()
if os.path.isfile(browser_config['browser_log']):
results_file = open(browser_config['browser_log'], "r")
results_raw = results_file.read()
results_file.close()
utils.noisy(results_raw)
self._ffprocess.cleanupProcesses(browser_config['process'], browser_config['child_process'],
browser_config['browser_wait'])
if profile_dir:
try:
self.checkForCrashes(browser_config, profile_dir)
except talosError:
pass
if vars().has_key('temp_dir'):
self.cleanupProfile(temp_dir)
except talosError, te:
utils.debug("cleanup error: " + te.msg)
except:
utils.debug("unknown error during cleanup")
raise
