def run_tests(self):
"""
Run the tests
"""
dirs = self.query_abs_dirs()
error_list = self.error_list
error_list.extend(BaseErrorList)
suite = self.config['test_suite']
if suite not in self.test_suites:
self.fatal("Don't know how to run --test-suite '%s'!" % suite)
cmd = self._query_abs_base_cmd(suite)
cwd = dirs['abs_%s_dir' % suite]
# TODO we probably have to move some of the code in
# scripts/desktop_unittest.py and scripts/marionette.py to
# mozharness.mozilla.testing.unittest so we can share it.
# In the short term, I'm ok with some duplication of code if it
# expedites things; please file bugs to merge if that happens.
suite_name = [x for x in self.test_suites if x in self.config['test_suite']][0]
if self.config.get('this_chunk'):
suite = '%s-%s' % (suite_name, self.config['this_chunk'])
else:
suite = suite_name
# bug 773703
success_codes = None
if suite_name == 'xpcshell':
success_codes = [0, 1]
env = {}
if self.query_minidump_stackwalk():
env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir']
if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
env = self.query_env(partial_env=env)
parser = DesktopUnittestOutputParser(suite_category=suite_name,
config=self.config,
log_obj=self.log_obj,
error_list=error_list)
return_code = self.run_command(cmd, cwd=cwd, env=env,
output_timeout=1000,
output_parser=parser,
success_codes=success_codes)
tbpl_status, log_level = parser.evaluate_parser(return_code)
parser.append_tinderboxprint_line(suite_name)
self.buildbot_status(tbpl_status, level=log_level)
self.log("The %s suite: %s ran with return status: %s" %
(suite_name, suite, tbpl_status), level=log_level)
def run_tests(self):
"""
Run the tests
"""
dirs = self.query_abs_dirs()
error_list = self.error_list
error_list.extend(BaseErrorList)
suite = self.config['test_suite']
if suite not in self.test_suites:
self.fatal("Don't know how to run --test-suite '%s'!" % suite)
cmd = self._query_abs_base_cmd(suite)
cwd = dirs['abs_%s_dir' % suite]
# TODO we probably have to move some of the code in
# scripts/desktop_unittest.py and scripts/marionette.py to
# mozharness.mozilla.testing.unittest so we can share it.
# In the short term, I'm ok with some duplication of code if it
# expedites things; please file bugs to merge if that happens.
suite_name = [x for x in self.test_suites if x in self.config['test_suite']][0]
if self.config.get('this_chunk'):
suite = '%s-%s' % (suite_name, self.config['this_chunk'])
else:
suite = suite_name
# bug 773703
success_codes = None
if suite_name == 'xpcshell':
success_codes = [0, 1]
env = {}
if self.query_minidump_stackwalk():
env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
env = self.query_env(partial_env=env)
parser = DesktopUnittestOutputParser(suite_category=suite_name,
config=self.config,
log_obj=self.log_obj,
error_list=error_list)
return_code = self.run_command(cmd, cwd=cwd, env=env,
output_timeout=1000,
output_parser=parser,
success_codes=success_codes)
self._dump_logcat(parser)
tbpl_status, log_level = parser.evaluate_parser(return_code)
parser.append_tinderboxprint_line(suite_name)
self.buildbot_status(tbpl_status, level=log_level)
self.log("The %s suite: %s ran with return status: %s" %
(suite_name, suite, tbpl_status), level=log_level)
def run_tests(self):
"""
Run the tests
"""
procs = []
emulator_index = 0
for suite_name in self.test_suites:
procs.append(self._trigger_test(suite_name, emulator_index))
emulator_index+=1
joint_tbpl_status = None
joint_log_level = None
start_time = int(time.time())
while True:
for p in procs:
return_code = p["process"].poll()
if return_code!=None:
suite_name = p["suite_name"]
# To make reading the log of the suite not mix with the previous line
sys.stdout.write('\n')
self.info("##### %s log begins" % p["suite_name"])
# Let's close the stdout
p["tmp_stdout"].close()
# Let's read the file that now has the output
output = self.read_from_file(p["tmp_file"].name, verbose=False)
# Let's output all the log
self.info(output)
# Let's parse the output and determine what the results should be
parser = DesktopUnittestOutputParser(
suite_category=self.test_suite_definitions[p["suite_name"]]["category"],
config=self.config,
log_obj=self.log_obj,
error_list=self.error_list)
for line in output.splitlines():
parser.parse_single_line(line)
# After parsing each line we should know what the summary for this suite should be
tbpl_status, log_level = parser.evaluate_parser(return_code)
parser.append_tinderboxprint_line(p["suite_name"])
# After running all jobs we will report the worst status of all emulator runs
joint_tbpl_status = self.worst_level(tbpl_status, joint_tbpl_status, TBPL_WORST_LEVEL_TUPLE)
joint_log_level = self.worst_level(log_level, joint_log_level)
self.info("##### %s log ends" % p["suite_name"])
procs.remove(p)
if procs == []:
break
else:
# Every 5 minutes let's print something to stdout
# so buildbot won't kill the process due to lack of output
if int(time.time()) - start_time > 5 * 60:
self.info('#')
start_time = int(time.time())
time.sleep(30)
self.buildbot_status(joint_tbpl_status, level=joint_log_level)
def get_test_output_parser(self,
suite_category,
flavor=None,
strict=False,
**kwargs):
if not self.structured_output(suite_category, flavor):
return DesktopUnittestOutputParser(suite_category=suite_category,
**kwargs)
self.info("Structured output parser in use for %s." % suite_category)
return StructuredOutputParser(suite_category=suite_category,
strict=strict,
**kwargs)
def get_test_output_parser(self, suite_category, strict=False,
fallback_parser_class=DesktopUnittestOutputParser,
**kwargs):
"""Derive and return an appropriate output parser, either the structured
output parser or a fallback based on the type of logging in use as determined by
configuration.
"""
if not self.structured_output(suite_category):
if fallback_parser_class is DesktopUnittestOutputParser:
return DesktopUnittestOutputParser(suite_category=suite_category, **kwargs)
return fallback_parser_class(**kwargs)
self.info("Structured output parser in use for %s." % suite_category)
return StructuredOutputParser(suite_category=suite_category, strict=strict, **kwargs)
def run_tests(self):
"""
Run the tests
"""
procs = []
emulator_index = 0
for suite_name in self.test_suites:
procs.append(self._trigger_test(suite_name, emulator_index))
emulator_index += 1
joint_tbpl_status = None
joint_log_level = None
start_time = int(time.time())
while True:
for p in procs:
emulator_index = p["emulator_index"]
return_code = p["process"].poll()
if return_code is not None:
suite_name = p["suite_name"]
# To make reading the log of the suite not mix with the previous line
sys.stdout.write('\n')
self.info("##### %s log begins" % p["suite_name"])
# Let's close the stdout
p["tmp_stdout"].close()
# Let's read the file that now has the output
output = self.read_from_file(p["tmp_file"].name, verbose=False)
# Let's parse the output (which also prints it)
# and determine what the results should be
parser = DesktopUnittestOutputParser(
suite_category=self.test_suite_definitions[p["suite_name"]]["category"],
config=self.config,
log_obj=self.log_obj,
error_list=self.error_list)
for line in output.splitlines():
parser.parse_single_line(line)
# After parsing each line we should know what the summary for this suite should be
tbpl_status, log_level = parser.evaluate_parser(return_code)
parser.append_tinderboxprint_line(p["suite_name"])
# After running all jobs we will report the worst status of all emulator runs
joint_tbpl_status = self.worst_level(tbpl_status, joint_tbpl_status, TBPL_WORST_LEVEL_TUPLE)
joint_log_level = self.worst_level(log_level, joint_log_level)
self.info("##### %s log ends" % p["suite_name"])
self._dump_emulator_log(emulator_index)
procs.remove(p)
if procs == []:
break
else:
# Every 5 minutes let's print something to stdout
# so buildbot won't kill the process due to lack of output
if int(time.time()) - start_time > 5 * 60:
self.info('#')
start_time = int(time.time())
time.sleep(30)
self.buildbot_status(joint_tbpl_status, level=joint_log_level)
def _run_category_suites(self, suite_category, preflight_run_method=None):
"""run suite(s) to a specific category"""
c = self.config
dirs = self.query_abs_dirs()
abs_base_cmd = self._query_abs_base_cmd(suite_category)
suites = self._query_specified_suites(suite_category)
abs_app_dir = self.query_abs_app_dir()
if preflight_run_method:
preflight_run_method(suites)
if suites:
self.info('#### Running %s suites' % suite_category)
for suite in suites:
cmd = abs_base_cmd[:]
replace_dict = {
'abs_app_dir': abs_app_dir,
}
options_list = []
env = {}
if isinstance(suites[suite], dict):
options_list = suites[suite]['options']
env = copy.deepcopy(suites[suite]['env'])
else:
options_list = suites[suite]
for arg in options_list:
cmd.append(arg % replace_dict)
suite_name = suite_category + '-' + suite
tbpl_status, log_level = None, None
error_list = BaseErrorList + [{
'regex': re.compile(r'''PROCESS-CRASH.*application crashed'''),
'level': ERROR,
}]
parser = DesktopUnittestOutputParser(suite_category,
config=self.config,
error_list=error_list,
log_obj=self.log_obj)
if c.get('minidump_stackwalk_path'):
env['MINIDUMP_STACKWALK'] = c['minidump_stackwalk_path']
env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
env = self.query_env(partial_env=env, log_level=INFO)
return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'],
output_timeout=1000,
output_parser=parser,
env=env)
# mochitest, reftest, and xpcshell suites do not return
# appropriate return codes. Therefore, we must parse the output
# to determine what the tbpl_status and worst_log_level must
# be. We do this by:
# 1) checking to see if our mozharness script ran into any
# errors itself with 'num_errors' <- OutputParser
# 2) if num_errors is 0 then we look in the subclassed 'parser'
# findings for harness/suite errors <- DesktopUnittestOutputParser
tbpl_status, log_level = parser.evaluate_parser(return_code)
parser.append_tinderboxprint_line(suite_name)
self.buildbot_status(tbpl_status, level=log_level)
self.log("The %s suite: %s ran with return status: %s" %
(suite_category, suite, tbpl_status), level=log_level)
else:
self.debug('There were no suites to run for %s' % suite_category)
def run_tests(self):
"""
Run the tests
"""
dirs = self.query_abs_dirs()
error_list = self.error_list
error_list.extend(BaseErrorList)
suite = self.config['test_suite']
if suite not in self.test_suites:
self.fatal("Don't know how to run --test-suite '%s'!" % suite)
cmd = self._query_abs_base_cmd(suite)
cwd = dirs['abs_%s_dir' % suite]
# TODO we probably have to move some of the code in
# scripts/desktop_unittest.py and scripts/marionette.py to
# mozharness.mozilla.testing.unittest so we can share it.
# In the short term, I'm ok with some duplication of code if it
# expedites things; please file bugs to merge if that happens.
suite_name = [x for x in self.test_suites if x in self.config['test_suite']][0]
if self.config.get('this_chunk'):
suite = '%s-%s' % (suite_name, self.config['this_chunk'])
else:
suite = suite_name
# bug 773703
success_codes = None
if suite_name == 'xpcshell':
success_codes = [0, 1]
env = {}
if self.query_minidump_stackwalk():
env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir']
if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
env = self.query_env(partial_env=env)
parser = DesktopUnittestOutputParser(suite_category=suite_name,
config=self.config,
log_obj=self.log_obj,
error_list=error_list)
return_code = self.run_command(cmd, cwd=cwd, env=env,
output_timeout=1000,
output_parser=parser,
success_codes=success_codes)
logcat = os.path.join(dirs['abs_work_dir'], 'emulator-5554.log')
if os.path.isfile(logcat):
if parser.fail_count != 0 or parser.crashed or parser.leaked:
# On failure, dump logcat, check if the emulator is still
# running, and if it is still accessible via adb.
self.info('dumping logcat')
self.run_command(['cat', logcat], error_list=LogcatErrorList)
self.run_command(['ps', '-C', 'emulator'])
self.run_command([self.adb_path, 'devices'])
# upload logcat to blobber
self.copyfile(logcat, os.path.join(env['MOZ_UPLOAD_DIR'],
os.path.basename(logcat)))
else:
self.info('no logcat file found')
qemu = os.path.join(dirs['abs_work_dir'], 'qemu.log')
if os.path.isfile(qemu):
self.copyfile(qemu, os.path.join(env['MOZ_UPLOAD_DIR'],
os.path.basename(qemu)))
tbpl_status, log_level = parser.evaluate_parser(return_code)
parser.append_tinderboxprint_line(suite_name)
self.buildbot_status(tbpl_status, level=log_level)
self.log("The %s suite: %s ran with return status: %s" %
(suite_name, suite, tbpl_status), level=log_level)
def _run_category_suites(self, suite_category, preflight_run_method=None):
"""run suite(s) to a specific category"""
env = self.query_env(partial_env={'DM_TRANS': "sut", 'TEST_DEVICE': self.mozpool_device})
self.info("Running tests...")
suites = self._query_specified_suites(suite_category)
level = INFO
if preflight_run_method:
preflight_run_method(suites)
if suites:
self.info('#### Running %s suites' % suite_category)
for suite in suites:
dirs = self.query_abs_dirs()
self._download_unzip_hostutils()
abs_base_cmd = self._query_abs_base_cmd(suite_category)
should_install_app = True
if 'cppunittest' in suite:
should_install_app = False
if 'robocop' in suite:
self._download_robocop_apk()
if 'jittest' in suite:
should_install_app = False
self._download_unzip(self.query_jsshell_url(), dirs['abs_test_bin_dir'])
if should_install_app:
self._install_app()
cmd = abs_base_cmd[:]
replace_dict = {}
for arg in suites[suite]:
cmd.append(arg % replace_dict)
if 'mochitest-gl' in suite:
cmd.remove("--run-only-tests=android.json")
tbpl_status, log_level = None, None
error_list = BaseErrorList + [{
'regex': re.compile(r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \| (application crashed|missing output line for total leaks!|negative leaks caught!|\d+ bytes leaked)"),
'level': ERROR,
}]
c = self.config
if c.get('minidump_stackwalk_path'):
env['MINIDUMP_STACKWALK'] = c['minidump_stackwalk_path']
env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
env = self.query_env(partial_env=env, log_level=INFO)
if env.has_key('PYTHONPATH'):
del env['PYTHONPATH']
test_summary_parser = DesktopUnittestOutputParser(suite_category,
config=self.config,
error_list=error_list,
log_obj=self.log_obj)
return_code = self.run_command(cmd, cwd=dirs['abs_test_install_dir'], env=env, output_parser=test_summary_parser)
tbpl_status, log_level = test_summary_parser.evaluate_parser(return_code)
if tbpl_status != TBPL_SUCCESS:
self.info("Output logcat...")
try:
lines = self.get_logcat()
self.info("*** STARTING LOGCAT ***")
for l in lines:
self.info(l)
self.info("*** END LOGCAT ***")
except Exception, e:
self.warning("We failed to run logcat: str(%s)" % str(e))
test_summary_parser.append_tinderboxprint_line(suite)
self.buildbot_status(tbpl_status, level=level)
self.log("The %s suite: %s ran with return status: %s" %
(suite_category, suite, tbpl_status), level=log_level)
def _run_category_suites(self, suite_category, preflight_run_method=None):
"""run suite(s) to a specific category"""
c = self.config
dirs = self.query_abs_dirs()
abs_base_cmd = self._query_abs_base_cmd(suite_category)
suites = self._query_specified_suites(suite_category)
abs_app_dir = self.query_abs_app_dir()
if preflight_run_method:
preflight_run_method(suites)
if suites:
self.info('#### Running %s suites' % suite_category)
for suite in suites:
cmd = abs_base_cmd[:]
replace_dict = {
'abs_app_dir': abs_app_dir,
}
options_list = []
env = {}
if isinstance(suites[suite], dict):
options_list = suites[suite]['options']
env = copy.deepcopy(suites[suite]['env'])
else:
options_list = suites[suite]
for arg in options_list:
cmd.append(arg % replace_dict)
suite_name = suite_category + '-' + suite
tbpl_status, log_level = None, None
error_list = BaseErrorList + [{
'regex':
re.compile(r'''PROCESS-CRASH.*application crashed'''),
'level':
ERROR,
}]
parser = DesktopUnittestOutputParser(suite_category,
config=self.config,
error_list=error_list,
log_obj=self.log_obj)
if c.get('minidump_stackwalk_path'):
env['MINIDUMP_STACKWALK'] = c['minidump_stackwalk_path']
env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs(
)['abs_blob_upload_dir']
env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs(
)['abs_blob_upload_dir']
if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
env = self.query_env(partial_env=env, log_level=INFO)
return_code = self.run_command(cmd,
cwd=dirs['abs_work_dir'],
output_timeout=1000,
output_parser=parser,
env=env)
# mochitest, reftest, and xpcshell suites do not return
# appropriate return codes. Therefore, we must parse the output
# to determine what the tbpl_status and worst_log_level must
# be. We do this by:
# 1) checking to see if our mozharness script ran into any
# errors itself with 'num_errors' <- OutputParser
# 2) if num_errors is 0 then we look in the subclassed 'parser'
# findings for harness/suite errors <- DesktopUnittestOutputParser
tbpl_status, log_level = parser.evaluate_parser(return_code)
parser.append_tinderboxprint_line(suite_name)
self.buildbot_status(tbpl_status, level=log_level)
self.log("The %s suite: %s ran with return status: %s" %
(suite_category, suite, tbpl_status),
level=log_level)
else:
self.debug('There were no suites to run for %s' % suite_category)
def _run_category_suites(self, suite_category, preflight_run_method=None):
"""run suite(s) to a specific category"""
env = self.query_env(partial_env={
'DM_TRANS': "sut",
'TEST_DEVICE': self.mozpool_device
})
self.info("Running tests...")
suites = self._query_specified_suites(suite_category)
level = INFO
if preflight_run_method:
preflight_run_method(suites)
if suites:
self.info('#### Running %s suites' % suite_category)
for suite in suites:
dirs = self.query_abs_dirs()
self._download_unzip_hostutils()
abs_base_cmd = self._query_abs_base_cmd(suite_category)
should_install_app = True
if 'cppunittest' in suite:
should_install_app = False
if 'robocop' in suite:
self._download_robocop_apk()
if 'jittest' in suite:
should_install_app = False
self._download_unzip(self.query_jsshell_url(),
dirs['abs_test_bin_dir'])
if should_install_app:
self._install_app()
cmd = abs_base_cmd[:]
replace_dict = {}
for arg in suites[suite]:
cmd.append(arg % replace_dict)
if 'mochitest-gl' in suite:
cmd.remove("--run-only-tests=android.json")
tbpl_status, log_level = None, None
error_list = BaseErrorList + [{
'regex':
re.compile(
r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \| (application crashed|missing output line for total leaks!|negative leaks caught!|\d+ bytes leaked)"
),
'level':
ERROR,
}]
c = self.config
if c.get('minidump_stackwalk_path'):
env['MINIDUMP_STACKWALK'] = c['minidump_stackwalk_path']
env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs(
)['abs_blob_upload_dir']
env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs(
)['abs_blob_upload_dir']
if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
env = self.query_env(partial_env=env, log_level=INFO)
if env.has_key('PYTHONPATH'):
del env['PYTHONPATH']
test_summary_parser = DesktopUnittestOutputParser(
suite_category,
config=self.config,
error_list=error_list,
log_obj=self.log_obj)
return_code = self.run_command(
cmd,
cwd=dirs['abs_test_install_dir'],
env=env,
output_parser=test_summary_parser)
tbpl_status, log_level = test_summary_parser.evaluate_parser(
return_code)
if tbpl_status != TBPL_SUCCESS:
self.info("Output logcat...")
try:
lines = self.get_logcat()
self.info("*** STARTING LOGCAT ***")
for l in lines:
self.info(l)
self.info("*** END LOGCAT ***")
except Exception, e:
self.warning("We failed to run logcat: str(%s)" %
str(e))
test_summary_parser.append_tinderboxprint_line(suite)
self.buildbot_status(tbpl_status, level=level)
self.log("The %s suite: %s ran with return status: %s" %
(suite_category, suite, tbpl_status),
level=log_level)
