def process_gtest_json_output(exit_code, output_dir): # summary.json is produced by swarming.py itself. We are mostly interested # in the number of shards. try: with open(os.path.join(output_dir, 'summary.json')) as f: summary = json.load(f) except (IOError, ValueError): emit_warning('summary.json is missing or can not be read', traceback.format_exc()) return # For each shard load its JSON output if available and feed it to the parser. parser = gtest_utils.GTestJSONParser() missing_shards = [] for index, result in enumerate(summary['shards']): if result is not None: json_data = load_shard_json(output_dir, index) if json_data: parser.ProcessJSONData(json_data) continue missing_shards.append(index) # If some shards are missing, make it known. Continue parsing anyway. Step # should be red anyway, since swarming.py return non-zero exit code in that # case. if missing_shards: as_str = ' ,'.join(map(str, missing_shards)) emit_warning( 'missing results from some shards', 'Missing results from the following shard(s): %s' % as_str) # Emit annotations with a summary of test execution. annotation_utils.annotate('', exit_code, parser)
def testFlakyTests(self): parser = gtest_utils.GTestJSONParser() parser.ProcessJSONData({ 'disabled_tests': [], 'global_tags': [], 'per_iteration_data': [{ 'Test.One': [{ 'status': 'FAILURE', 'output_snippet': '' }], 'Test.Two': [ { 'status': 'FAILURE', 'output_snippet': '' }, { 'status': 'SUCCESS', 'output_snippet': '' }, ], }] }) self.assertEqual(['Test.Two'], parser.PassedTests()) self.assertEqual(['Test.One'], parser.FailedTests()) self.assertEqual(1, parser.FlakyTests()) self.assertEqual(0, parser.DisabledTests()) self.assertEqual(['FAILURE'], parser.TriesForTest('Test.One')) self.assertEqual(['FAILURE', 'SUCCESS'], parser.TriesForTest('Test.Two'))
def testIngoredFailedTests(self): TEST_IGNORED_FAILED_TESTS_SPEC = """ # A comment. crbug.com/12345 [ OS_WIN , OS_LINUX] Test.One crbug.com/12345 [OS_WIN CPU_64_BITS MODE_RELEASE] Test.Two/2 crbug.com/12345 [,OS_MACOSX, OS_WIN CPU_64_BITS, ] Perf/Test.Three crbug.com/12345 [ invalid.platform.spec ] Test.Four crbug.com/12345 [ OS_WIN CPU_32_BITS MODE_RELEASE ] Test.Five invalid line """ _, spec_filename = tempfile.mkstemp() spec_fd = open(spec_filename, 'w') spec_fd.write(TEST_IGNORED_FAILED_TESTS_SPEC) spec_fd.close() self.mock(chromium_utils, 'FindUpward', lambda *_: spec_filename) parser = gtest_utils.GTestJSONParser() try: parser.ProcessJSONData( { 'disabled_tests': ['Test.Six'], 'per_iteration_data': [{ 'Test.One': [{ 'status': 'FAILURE', 'output_snippet': '' }], 'Test.Two/2': [{ 'status': 'FAILURE', 'output_snippet': '' }], 'Perf/Test.Three': [{ 'status': 'FAILURE', 'output_snippet': '' }], 'Test.Four': [{ 'status': 'FAILURE', 'output_snippet': '' }], 'Test.Five': [{ 'status': 'FAILURE', 'output_snippet': '' }], }], 'global_tags': ['OS_WIN', 'CPU_64_BITS', 'MODE_RELEASE', 'OTHER_FLAG'] }, '/fake/path/to/build') finally: os.remove(spec_filename) self.assertEqual(['Test.Five', 'Test.Four'], parser.FailedTests()) self.assertEqual(['Perf/Test.Three', 'Test.One', 'Test.Two/2'], parser.IgnoredFailedTests()) self.assertEqual(['FAILURE'], parser.TriesForTest('Test.One')) self.assertEqual(['FAILURE'], parser.TriesForTest('Test.Two/2')) self.assertEqual(['FAILURE'], parser.TriesForTest('Perf/Test.Three')) self.assertEqual(['FAILURE'], parser.TriesForTest('Test.Four')) self.assertEqual(['FAILURE'], parser.TriesForTest('Test.Five'))
def testInvalidEscape_crbug632047(self): parser = gtest_utils.GTestJSONParser() parser.ProcessJSONData({ 'disabled_tests': [], 'global_tags': [], 'per_iteration_data': [{ 'Test.One': [{ 'status': 'FAILURE', # Use 'loads' to make sure we get exactly what the parser returns. 'output_snippet': json.loads( r'"\tcontent::BrowserMainLoop::PreMainMessageLoopRun ' + r'(C:\\b\\c\\b\\CrWinAsan_dll_\\src\\content\\browser\\' + r'browser_main_loop.cc:1172)\r\n"'), }], }] }) self.assertEqual(['Test.One'], parser.FailedTests()) # '\\b' and '\\s' MUST be preserved. self.assertEqual([ 'Test.One (run #1):', '\tcontent::BrowserMainLoop::PreMainMessageLoopRun ' '(C:\\b\\c\\b\\CrWinAsan_dll_\\src\\content\\browser\\' 'browser_main_loop.cc:1172)\r', '' ], parser.FailureDescription('Test.One'))
def testDoesNotThrowExceptionOnMissingIgnoredFailedTestsFile(self): parser = gtest_utils.GTestJSONParser() parser.ProcessJSONData( { 'disabled_tests': [], 'global_tags': [], 'per_iteration_data': [] }, tempfile.gettempdir())
def testRetriedTests(self): parser = gtest_utils.GTestJSONParser() parser.ProcessJSONData({ 'disabled_tests': [], 'global_tags': [], 'per_iteration_data': [{ 'Test.One': [ { 'status': 'FAILURE', 'output_snippet': '' }, { 'status': 'FAILURE', 'output_snippet': '' }, ], 'Test.Two': [ { 'status': 'FAILURE', 'output_snippet': '' }, { 'status': 'FAILURE_ON_EXIT', 'output_snippet': '' }, { 'status': 'CRASH', 'output_snippet': '' }, { 'status': 'TIMEOUT', 'output_snippet': '' }, { 'status': 'SKIPPED', 'output_snippet': '' }, { 'status': 'SUCCESS', 'output_snippet': '' }, ], }] }) expected_tries_test_two = [ 'FAILURE', 'FAILURE_ON_EXIT', 'CRASH', 'TIMEOUT', 'SKIPPED', 'SUCCESS' ] self.assertEqual(['Test.Two'], parser.PassedTests()) self.assertEqual(['Test.One'], parser.FailedTests()) self.assertEqual(1, parser.FlakyTests()) self.assertEqual(0, parser.DisabledTests()) self.assertEqual(['FAILURE', 'FAILURE'], parser.TriesForTest('Test.One')) self.assertEqual(expected_tries_test_two, parser.TriesForTest('Test.Two'))
def testDisabledTests(self): parser = gtest_utils.GTestJSONParser() parser.ProcessJSONData({ 'disabled_tests': ['Test.Two'], 'per_iteration_data': [{ 'Test.One': [{ 'status': 'SUCCESS', 'output_snippet': '' }], }] }) self.assertEqual(['Test.One'], parser.PassedTests()) self.assertEqual([], parser.FailedTests()) self.assertEqual(0, parser.FlakyTests()) self.assertEqual(1, parser.DisabledTests())
def emit_test_annotations(exit_code, json_data): """Emits annotations with logs of failed tests.""" parser = gtest_utils.GTestJSONParser() if json_data: parser.ProcessJSONData(json_data) annotation_utils.annotate('', exit_code, parser)
def _Main(options, args, extra_env): """Using the target build configuration, run the executable given in the first non-option argument, passing any following arguments to that executable. Args: options: Command-line options for this invocation of runtest.py. args: Command and arguments for the test. extra_env: A dictionary of extra environment variables to set. Returns: Exit status code. """ if len(args) < 1: raise chromium_utils.MissingArgument('Usage: %s' % USAGE) xvfb_path = os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'third_party', 'xvfb', platform.architecture()[0]) special_xvfb_dir = None fp_chromeos = options.factory_properties.get('chromeos', None) if (fp_chromeos or slave_utils.GypFlagIsOn(options, 'use_aura') or slave_utils.GypFlagIsOn(options, 'chromeos')): special_xvfb_dir = xvfb_path build_dir = os.path.normpath(os.path.abspath(options.build_dir)) bin_dir = os.path.join(build_dir, options.target) slave_name = options.slave_name or slave_utils.SlaveBuildName(build_dir) test_exe = args[0] if options.run_python_script: test_exe_path = test_exe else: test_exe_path = os.path.join(bin_dir, test_exe) if not os.path.exists(test_exe_path): if options.factory_properties.get('succeed_on_missing_exe', False): print '%s missing but succeed_on_missing_exe used, exiting' % ( test_exe_path) return 0 raise chromium_utils.PathNotFound('Unable to find %s' % test_exe_path) if sys.platform == 'linux2': # Unset http_proxy and HTTPS_PROXY environment variables. When set, this # causes some tests to hang. See http://crbug.com/139638 for more info. if 'http_proxy' in os.environ: del os.environ['http_proxy'] print 'Deleted http_proxy environment variable.' if 'HTTPS_PROXY' in os.environ: del os.environ['HTTPS_PROXY'] print 'Deleted HTTPS_PROXY environment variable.' # Path to SUID sandbox binary. This must be installed on all bots. extra_env['CHROME_DEVEL_SANDBOX'] = CHROME_SANDBOX_PATH extra_env['LD_LIBRARY_PATH'] = '' if options.enable_lsan: # Use the debug version of libstdc++ under LSan. If we don't, there will # be a lot of incomplete stack traces in the reports. extra_env['LD_LIBRARY_PATH'] += '/usr/lib/x86_64-linux-gnu/debug:' extra_env['LD_LIBRARY_PATH'] += '%s:%s/lib:%s/lib.target' % ( bin_dir, bin_dir, bin_dir) if options.run_shell_script: command = ['bash', test_exe_path] elif options.run_python_script: command = [sys.executable, test_exe] else: command = _BuildTestBinaryCommand(build_dir, test_exe_path, options) command.extend(args[1:]) # Nuke anything that appears to be stale chrome items in the temporary # directory from previous test runs (i.e.- from crashes or unittest leaks). slave_utils.RemoveChromeTemporaryFiles() log_processor = None if _UsingGtestJson(options): log_processor = gtest_utils.GTestJSONParser( options.build_properties.get('mastername')) if options.generate_json_file: if os.path.exists(options.test_output_xml): # remove the old XML output file. os.remove(options.test_output_xml) try: # TODO(dpranke): checking on test_exe is a temporary hack until we # can change the buildbot master to pass --xvfb instead of --no-xvfb # for these two steps. See # https://code.google.com/p/chromium/issues/detail?id=179814 start_xvfb = (sys.platform == 'linux2' and (options.xvfb or 'layout_test_wrapper' in test_exe or 'devtools_perf_test_wrapper' in test_exe)) if start_xvfb: xvfb.StartVirtualX(slave_name, bin_dir, with_wm=(options.factory_properties.get( 'window_manager', 'True') == 'True'), server_dir=special_xvfb_dir) if _UsingGtestJson(options): json_file_name = log_processor.PrepareJSONFile( options.test_launcher_summary_output) command.append('--test-launcher-summary-output=%s' % json_file_name) pipes = [] # See the comment in main() regarding offline symbolization. if options.use_symbolization_script: symbolize_command = _GetSanitizerSymbolizeCommand( strip_path_prefix=options.strip_path_prefix) pipes = [symbolize_command] command = _GenerateRunIsolatedCommand(build_dir, test_exe_path, options, command) result = _RunGTestCommand(options, command, extra_env, pipes=pipes) finally: if start_xvfb: xvfb.StopVirtualX(slave_name) if _UsingGtestJson(options): if options.use_symbolization_script: _SymbolizeSnippetsInJSON(options, json_file_name) log_processor.ProcessJSONFile(options.build_dir) if options.generate_json_file: if not _GenerateJSONForTestResults(options, log_processor): return 1 if options.annotate: annotation_utils.annotate(options.test_type, result, log_processor) return result
def _Main(options, args, extra_env): """Using the target build configuration, run the executable given in the first non-option argument, passing any following arguments to that executable. Args: options: Command-line options for this invocation of runtest.py. args: Command and arguments for the test. extra_env: A dictionary of extra environment variables to set. Returns: Exit status code. """ if len(args) < 1: raise chromium_utils.MissingArgument('Usage: %s' % USAGE) xvfb_path = os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'third_party', 'xvfb', platform.architecture()[0]) build_dir = os.path.normpath(os.path.abspath(options.build_dir)) bin_dir = os.path.join(build_dir, options.target) test_exe = args[0] if options.run_python_script: test_exe_path = test_exe else: test_exe_path = os.path.join(bin_dir, test_exe) if not os.path.exists(test_exe_path): if options.factory_properties.get('succeed_on_missing_exe', False): print '%s missing but succeed_on_missing_exe used, exiting' % ( test_exe_path) return 0 raise chromium_utils.PathNotFound('Unable to find %s' % test_exe_path) if sys.platform == 'linux2': # Unset http_proxy and HTTPS_PROXY environment variables. When set, this # causes some tests to hang. See http://crbug.com/139638 for more info. if 'http_proxy' in os.environ: del os.environ['http_proxy'] print 'Deleted http_proxy environment variable.' if 'HTTPS_PROXY' in os.environ: del os.environ['HTTPS_PROXY'] print 'Deleted HTTPS_PROXY environment variable.' # Path to SUID sandbox binary. This must be installed on all bots. extra_env['CHROME_DEVEL_SANDBOX'] = CHROME_SANDBOX_PATH extra_env['LD_LIBRARY_PATH'] = '' if options.enable_lsan: # Use the debug version of libstdc++ under LSan. If we don't, there will # be a lot of incomplete stack traces in the reports. extra_env['LD_LIBRARY_PATH'] += '/usr/lib/x86_64-linux-gnu/debug:' extra_env['LD_LIBRARY_PATH'] += '%s:%s/lib:%s/lib.target' % ( bin_dir, bin_dir, bin_dir) if options.run_shell_script: command = ['bash', test_exe_path] elif options.run_python_script: command = [sys.executable, test_exe] else: command = _BuildTestBinaryCommand(build_dir, test_exe_path, options) command.extend(args[1:]) log_processor = None if _UsingGtestJson(options): log_processor = gtest_utils.GTestJSONParser( options.build_properties.get('mastername')) try: # TODO(dpranke): checking on test_exe is a temporary hack until we # can change the buildbot master to pass --xvfb instead of --no-xvfb # for these two steps. See # https://code.google.com/p/chromium/issues/detail?id=179814 start_xvfb = (sys.platform == 'linux2' and (options.xvfb or 'layout_test_wrapper' in test_exe or 'devtools_perf_test_wrapper' in test_exe)) if start_xvfb: xvfb.StartVirtualX(None, bin_dir, with_wm=(options.factory_properties.get( 'window_manager', 'True') == 'True')) if _UsingGtestJson(options): json_file_name = log_processor.PrepareJSONFile( options.test_launcher_summary_output) command.append('--test-launcher-summary-output=%s' % json_file_name) command = _GenerateRunIsolatedCommand(build_dir, test_exe_path, options, command) env = os.environ.copy() if extra_env: print 'Additional test environment:' for k, v in sorted(extra_env.items()): print ' %s=%s' % (k, v) env.update(extra_env or {}) # Trigger bot mode (test retries, redirection of stdio, possibly faster, # etc.) - using an environment variable instead of command-line flags # because some internal waterfalls run this for totally non-gtest code. # TODO(phajdan.jr): Clean this up when internal waterfalls are fixed. env.update({'CHROMIUM_TEST_LAUNCHER_BOT_MODE': '1'}) if options.use_symbolization_script: symbolize_command = _GetSanitizerSymbolizeCommand( strip_path_prefix=options.strip_path_prefix) command_process = subprocess.Popen(command, env=env, stdout=subprocess.PIPE) symbolize_process = subprocess.Popen(symbolize_command, env=env, stdin=command_process.stdout) command_process.stdout.close() command_process.wait() symbolize_process.wait() result = command_process.returncode if result == 0: result = symbolize_process.returncode else: result = subprocess.call(command, env=env) finally: if start_xvfb: xvfb.StopVirtualX(None) if _UsingGtestJson(options): if options.use_symbolization_script: _SymbolizeSnippetsInJSON(options, json_file_name) log_processor.ProcessJSONFile(options.build_dir) if options.annotate: annotation_utils.annotate(options.test_type, result, log_processor) return result