def runTest(browser_config, test_config):
"""
Runs an url based test on the browser as specified in the browser_config dictionary
Args:
browser_config: Dictionary of configuration options for the browser (paths, prefs, etc)
test_config : Dictionary of configuration for the given test (url, cycles, counters, etc)
"""
utils.debug("operating with platform_type : " + platform_type)
counters = test_config[platform_type + 'counters']
resolution = test_config['resolution']
all_browser_results = []
all_counter_results = []
format = ""
utils.setEnvironmentVars(browser_config['env'])
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})
if browser_config['symbols_path']:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
else:
utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})
utils.setEnvironmentVars({"LD_LIBRARY_PATH" : os.path.dirname(browser_config['browser_path'])})
profile_dir = None
try:
if ffprocess.checkAllProcesses(browser_config['process']):
utils.debug(browser_config['process'] + " already running before testing started (unclean system)")
raise talosError("system not clean")
# add any provided directories to the installed browser
for dir in browser_config['dirs']:
ffsetup.InstallInBrowser(browser_config['browser_path'], browser_config['dirs'][dir])
# make profile path work cross-platform
test_config['profile_path'] = os.path.normpath(test_config['profile_path'])
profile_dir, temp_dir = test_config['profile_path'], test_config['profile_path']
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
#initializeProfile(profile_dir, browser_config)
utils.debug("initialized " + browser_config['process'])
if test_config['shutdown']:
shutdown = []
for i in range(test_config['cycles']):
if os.path.isfile(browser_config['browser_log']):
os.chmod(browser_config['browser_log'], 0777)
os.remove(browser_config['browser_log'])
time.sleep(browser_config['browser_wait']) #wait out the browser closing
# check to see if the previous cycle is still hanging around
if (i > 0) and ffprocess.checkAllProcesses(browser_config['process']):
raise talosError("previous cycle still running")
# Run the test
browser_results = ""
if 'timeout' in test_config:
timeout = test_config['timeout']
else:
timeout = 7200 # 2 hours
total_time = 0
output = ''
url = test_config['url']
command_line = ffprocess.GenerateBrowserCommandLine(browser_config['browser_path'], browser_config['extra_args'], profile_dir, url)
utils.debug("command line: " + command_line)
bcontroller = path('talos/bcontroller.py')
if 'url_mod' in test_config:
process = subprocess.Popen('python %s --command "%s" --mod "%s" --name %s --timeout %d --log %s' % (bcontroller, command_line, test_config['url_mod'], browser_config['process'], browser_config['browser_wait'], browser_config['browser_log']), universal_newlines=True, shell=True, bufsize=0, env=os.environ)
else:
process = subprocess.Popen('python %s --command "%s" --name %s --timeout %d --log %s' % (bcontroller, command_line, browser_config['process'], browser_config['browser_wait'], browser_config['browser_log']), universal_newlines=True, shell=True, bufsize=0, env=os.environ)
#give browser a chance to open
# this could mean that we are losing the first couple of data points as the tests starts, but if we don't provide
# some time for the browser to start we have trouble connecting the CounterManager to it
time.sleep(browser_config['browser_wait'])
#set up the counters for this test
if counters:
cm = CounterManager(browser_config['process'], counters)
cm.startMonitor()
counter_results = {}
for counter in counters:
counter_results[counter] = []
startTime = -1
while total_time < timeout: #the main test loop, monitors counters and checks for browser ouptut
# Sleep for [resolution] seconds
time.sleep(resolution)
total_time += resolution
# Get the output from all the possible counters
for count_type in counters:
val = cm.getCounterValue(count_type)
if (val):
counter_results[count_type].append(val)
if process.poll() != None: #browser_controller completed, file now full
#stop the counter manager since this test is complete
if counters:
cm.stopMonitor()
if not os.path.isfile(browser_config['browser_log']):
raise talosError("no output from browser")
results_file = open(browser_config['browser_log'], "r")
results_raw = results_file.read()
results_file.close()
utils.noisy(results_raw)
match = RESULTS_REGEX.search(results_raw)
if match:
browser_results += match.group(1)
startTime = int(match.group(2))
endTime = int(match.group(3))
utils.debug("Matched basic results: " + browser_results)
format = "tsformat"
break
#TODO: this a stop gap until all of the tests start outputting the same format
match = RESULTS_TP_REGEX.search(results_raw)
if match:
browser_results += match.group(1)
startTime = int(match.group(2))
endTime = int(match.group(3))
utils.debug("Matched tp results: " + browser_results)
format = "tpformat"
break
match = RESULTS_REGEX_FAIL.search(results_raw)
if match:
browser_results += match.group(1)
utils.debug("Matched fail results: " + browser_results)
raise talosError(match.group(1))
raise talosError("unrecognized output format")
if total_time >= timeout:
raise talosError("timeout exceeded")
time.sleep(browser_config['browser_wait'])
#clean up the process
timer = 0
while ((process.poll() is None) and timer < browser_config['browser_wait']):
time.sleep(1)
timer+=1
if test_config['shutdown']:
shutdown.append(endTime - startTime)
checkForCrashes(browser_config, profile_dir)
utils.debug("Completed test with: " + browser_results)
all_browser_results.append(browser_results)
all_counter_results.append(counter_results)
ffprocess.cleanupProcesses(browser_config['process'], browser_config['browser_wait'])
#cleanupProfile(temp_dir)
utils.restoreEnvironmentVars()
if test_config['shutdown']:
all_counter_results.append({'shutdown' : shutdown})
return (all_browser_results, all_counter_results, format)
except:
try:
if 'cm' in vars():
cm.stopMonitor()
if os.path.isfile(browser_config['browser_log']):
results_file = open(browser_config['browser_log'], "r")
results_raw = results_file.read()
results_file.close()
utils.noisy(results_raw)
ffprocess.cleanupProcesses(browser_config['process'], browser_config['browser_wait'])
if profile_dir:
try:
checkForCrashes(browser_config, profile_dir)
except talosError:
pass
if vars().has_key('temp_dir'):
pass
#cleanupProfile(temp_dir)
except talosError, te:
utils.debug("cleanup error: " + te.msg)
except:
utils.debug("unknown error during cleanup")
raise
def initializeProfile(profile_dir, browser_config):
if not (ffsetup.InitializeNewProfile(browser_config['browser_path'], browser_config['process'], browser_config['browser_wait'], browser_config['extra_args'], profile_dir, browser_config['init_url'], browser_config['browser_log'])):
raise talosError("failed to initialize browser")
time.sleep(browser_config['browser_wait'])
if ffprocess.checkAllProcesses(browser_config['process']):
raise talosError("browser failed to close after being initialized")
def test_file(filename):
"""Runs the talos tests on the given config file and generates a report.
Args:
filename: the name of the file to run the tests on
"""
browser_config = []
tests = []
title = ''
testdate = ''
csv_dir = ''
results_server = ''
results_link = ''
results = {}
# Read in the profile info from the YAML config file
config_file = open(filename, 'r')
yaml_config = yaml.load(config_file)
config_file.close()
for item in yaml_config:
if item == 'title':
title = yaml_config[item]
elif item == 'testdate':
testdate = yaml_config[item]
elif item == 'csv_dir':
csv_dir = os.path.normpath(yaml_config[item])
if not os.path.exists(csv_dir):
print "FAIL: path \"" + csv_dir + "\" does not exist"
sys.exit(0)
elif item == 'results_server':
results_server = yaml_config[item]
elif item == 'results_link' :
results_link = yaml_config[item]
if (results_link != results_server != ''):
if not post_file.link_exists(results_server, results_link):
sys.exit(0)
browser_config = {'preferences' : yaml_config['preferences'],
'extensions' : yaml_config['extensions'],
'browser_path' : yaml_config['browser_path'],
'browser_log' : yaml_config['browser_log'],
'symbols_path' : yaml_config.get('symbols_path', None),
'browser_wait' : yaml_config['browser_wait'],
'process' : yaml_config['process'],
'extra_args' : yaml_config['extra_args'],
'branch' : yaml_config['branch'],
'buildid' : yaml_config['buildid'],
'env' : yaml_config['env'],
'dirs' : yaml_config['dirs'],
'init_url' : yaml_config['init_url']}
if 'branch_name' in yaml_config:
browser_config['branch_name'] = yaml_config['branch_name']
if 'test_name_extension' in yaml_config:
browser_config['test_name_extension'] = yaml_config['test_name_extension']
else:
browser_config['test_name_extension'] = ''
#normalize paths to work accross platforms
browser_config['browser_path'] = os.path.normpath(browser_config['browser_path'])
for dir in browser_config['dirs']:
browser_config['dirs'][dir] = os.path.normpath(browser_config['dirs'][dir])
tests = yaml_config['tests']
config_file.close()
if (testdate != ''):
date = int(time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT')))
else:
date = int(time.time()) #TODO get this into own file
utils.debug("using testdate: %d" % date)
utils.debug("actual date: %d" % int(time.time()))
print 'RETURN:s: %s' % title
#pull buildid & sourcestamp from browser
browser_config = browserInfo(browser_config)
if ffprocess.checkAllProcesses(browser_config['process']):
print "FAIL: all firefox processes must be closed before tests can be run"
sys.exit(0)
utils.startTimer()
utils.stamped_msg(title, "Started")
for test in tests:
testname = test['name']
utils.stamped_msg("Running test " + testname, "Started")
try:
browser_dump, counter_dump, print_format = ttest.runTest(browser_config, test)
utils.debug("Received test results: " + " ".join(browser_dump))
results[testname] = [browser_dump, counter_dump, print_format]
# If we're doing CSV, write this test immediately (bug 419367)
if csv_dir != '':
send_to_csv(csv_dir, {testname : results[testname]})
except talosError, e:
utils.stamped_msg("Failed " + testname, "Stopped")
print 'FAIL: Busted: ' + testname
print 'FAIL: ' + e.msg.replace('\n','\nRETURN:')
utils.stamped_msg("Completed test " + testname, "Stopped")