def GetGomaLogDirectory(): """Get goma's log directory. Returns: a string of a directory name where goma's log may exist. Raises: chromium_utils.PathNotFound if it cannot find an available log directory. """ candidates = ['GLOG_log_dir', 'GOOGLE_LOG_DIR', 'TEST_TMPDIR'] default_dir = None if chromium_utils.IsWindows(): candidates.extend(['TMP', 'TEMP', 'USERPROFILE']) # Note: I believe one of environment variables is set for usual Windows # environment, let me avoid to check the Windows directory, which we # need to use win32api on Python. else: candidates.extend(['TMPDIR', 'TMP']) default_dir = '/tmp' for candidate in candidates: value = os.environ.get(candidate) if value and os.path.isdir(value): return value if default_dir: return default_dir raise chromium_utils.PathNotFound('Cannot find Goma log directory.')
def run_benchmark(options, use_refbuild, benchmark_results): result = 0 build_dir = os.path.abspath(options.build_dir) if not use_refbuild: if chromium_utils.IsMac(): build_dir = os.path.join(os.path.dirname(build_dir), 'xcodebuild') elif chromium_utils.IsLinux(): build_dir = os.path.join(os.path.dirname(build_dir), 'sconsbuild') build_dir = os.path.join(build_dir, options.target) else: build_dir = os.path.join(os.path.dirname(build_dir), 'chrome', 'tools', 'test', 'reference_build') if chromium_utils.IsMac(): build_dir = os.path.join(build_dir, 'chrome_mac') elif chromium_utils.IsLinux(): build_dir = os.path.join(build_dir, 'chrome_linux') else: build_dir = os.path.join(build_dir, 'chrome_win') if chromium_utils.IsWindows(): chrome_exe_name = 'chrome.exe' elif chromium_utils.IsLinux(): chrome_exe_name = 'chrome' else: chrome_exe_name = 'Chromium' chrome_exe_path = os.path.join(build_dir, chrome_exe_name) if not os.path.exists(chrome_exe_path): raise chromium_utils.PathNotFound('Unable to find %s' % chrome_exe_path) temp_dir = tempfile.mkdtemp() command = [ chrome_exe_path, '--user-data-dir=%s' % temp_dir, '--no-first-run', '--no-default-browser-check', START_URL ] print "Executing: " print command browser_process = subprocess.Popen(command) benchmark_results['ready'].wait() if benchmark_results['ready'].isSet(): results = json.loads(benchmark_results['results'])[0] print_result(True, 'Total', results['score'], use_refbuild) for child in results['children']: print_result(False, child['name'], child['score'], use_refbuild) benchmark_results['ready'].clear() if chromium_utils.IsWindows(): subprocess.call('taskkill /f /pid %i /t' % browser_process.pid) else: os.system('kill -15 %i' % browser_process.pid) browser_process.wait() shutil.rmtree(temp_dir) return result
def main(): """Using the target build configuration, run the crash_service.exe executable. """ option_parser = optparse.OptionParser(usage=USAGE) option_parser.add_option('--target', default='Release', help='build target (Debug or Release)') option_parser.add_option('--build-dir', help='ignored') options, args = option_parser.parse_args() options.build_dir = build_directory.GetBuildOutputDirectory() if args: option_parser.error('No args are supported') build_dir = os.path.abspath(options.build_dir) exe_path = os.path.join(build_dir, options.target, 'crash_service.exe') if not os.path.exists(exe_path): raise chromium_utils.PathNotFound('Unable to find %s' % exe_path) # crash_service's window can interfere with interactive ui tests. cmd = exe_path + ' --no-window' print '\n' + cmd + '\n', # We cannot use Popen or os.spawn here because buildbot will wait until # the process terminates before going to the next step. Since we want to # keep the process alive, we need to explicitly say that we want the # process to be detached and that we don't want to inherit handles from # the parent process. si = win32process.STARTUPINFO() details = win32process.CreateProcess(None, cmd, None, None, 0, win32process.DETACHED_PROCESS, None, None, si) print '\nCreated with process id %d\n' % details[2] return 0
def SlaveBaseDir(chrome_dir): """Finds the full path to the build slave's base directory (e.g. 'c:/b/chrome/chrome-release'). This is assumed to be the parent of the shallowest 'build' directory in the chrome_dir path. Raises chromium_utils.PathNotFound if there is no such directory. """ result = '' prev_dir = '' curr_dir = chrome_dir while prev_dir != curr_dir: (parent, leaf) = os.path.split(curr_dir) if leaf == 'build': # Remember this one and keep looking for something shallower. result = parent if leaf == 'slave': # We are too deep, stop now. break prev_dir = curr_dir curr_dir = parent if not result: raise chromium_utils.PathNotFound('Unable to find slave base dir above %s' % chrome_dir) return result
def _Main(options, args, extra_env): """Using the target build configuration, run the executable given in the first non-option argument, passing any following arguments to that executable. Args: options: Command-line options for this invocation of runtest.py. args: Command and arguments for the test. extra_env: A dictionary of extra environment variables to set. Returns: Exit status code. """ if len(args) < 1: raise chromium_utils.MissingArgument('Usage: %s' % USAGE) xvfb_path = os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'third_party', 'xvfb', platform.architecture()[0]) build_dir = os.path.normpath(os.path.abspath(options.build_dir)) bin_dir = os.path.join(build_dir, options.target) test_exe = args[0] if options.run_python_script: test_exe_path = test_exe else: test_exe_path = os.path.join(bin_dir, test_exe) if not os.path.exists(test_exe_path): if options.factory_properties.get('succeed_on_missing_exe', False): print '%s missing but succeed_on_missing_exe used, exiting' % ( test_exe_path) return 0 raise chromium_utils.PathNotFound('Unable to find %s' % test_exe_path) if sys.platform == 'linux2': # Unset http_proxy and HTTPS_PROXY environment variables. When set, this # causes some tests to hang. See http://crbug.com/139638 for more info. if 'http_proxy' in os.environ: del os.environ['http_proxy'] print 'Deleted http_proxy environment variable.' if 'HTTPS_PROXY' in os.environ: del os.environ['HTTPS_PROXY'] print 'Deleted HTTPS_PROXY environment variable.' # Path to SUID sandbox binary. This must be installed on all bots. extra_env['CHROME_DEVEL_SANDBOX'] = CHROME_SANDBOX_PATH extra_env['LD_LIBRARY_PATH'] = '' if options.enable_lsan: # Use the debug version of libstdc++ under LSan. If we don't, there will # be a lot of incomplete stack traces in the reports. extra_env['LD_LIBRARY_PATH'] += '/usr/lib/x86_64-linux-gnu/debug:' extra_env['LD_LIBRARY_PATH'] += '%s:%s/lib:%s/lib.target' % ( bin_dir, bin_dir, bin_dir) if options.run_shell_script: command = ['bash', test_exe_path] elif options.run_python_script: command = [sys.executable, test_exe] else: command = _BuildTestBinaryCommand(build_dir, test_exe_path, options) command.extend(args[1:]) log_processor = None try: # TODO(dpranke): checking on test_exe is a temporary hack until we # can change the buildbot master to pass --xvfb instead of --no-xvfb # for these two steps. See # https://code.google.com/p/chromium/issues/detail?id=179814 start_xvfb = (sys.platform == 'linux2' and (options.xvfb or 'layout_test_wrapper' in test_exe or 'devtools_perf_test_wrapper' in test_exe)) if start_xvfb: xvfb.StartVirtualX(None, bin_dir, with_wm=(options.factory_properties.get( 'window_manager', 'True') == 'True')) if options.test_launcher_summary_output: command.append('--test-launcher-summary-output=%s' % options.test_launcher_summary_output) command = _GenerateRunIsolatedCommand(build_dir, test_exe_path, options, command) env = os.environ.copy() if extra_env: print 'Additional test environment:' for k, v in sorted(extra_env.items()): print ' %s=%s' % (k, v) env.update(extra_env or {}) # Trigger bot mode (test retries, redirection of stdio, possibly faster, # etc.) - using an environment variable instead of command-line flags # because some internal waterfalls run this for totally non-gtest code. # TODO(phajdan.jr): Clean this up when internal waterfalls are fixed. env.update({'CHROMIUM_TEST_LAUNCHER_BOT_MODE': '1'}) if options.use_symbolization_script: symbolize_command = _GetSanitizerSymbolizeCommand( strip_path_prefix=options.strip_path_prefix) command_process = subprocess.Popen(command, env=env, stdout=subprocess.PIPE) symbolize_process = subprocess.Popen(symbolize_command, env=env, stdin=command_process.stdout) command_process.stdout.close() command_process.wait() symbolize_process.wait() result = command_process.returncode if result == 0: result = symbolize_process.returncode else: result = subprocess.call(command, env=env) finally: if start_xvfb: xvfb.StopVirtualX(None) return result
def dom_perf(options, args): """Using the target build configuration, run the dom perf test.""" build_dir = os.path.abspath(options.build_dir) if chromium_utils.IsWindows(): test_exe_name = 'performance_ui_tests.exe' else: test_exe_name = 'performance_ui_tests' if chromium_utils.IsMac(): is_make_or_ninja = (options.factory_properties.get( "gclient_env", {}).get('GYP_GENERATORS', '') in ('ninja', 'make')) if is_make_or_ninja: build_dir = os.path.join(os.path.dirname(build_dir), 'out') else: build_dir = os.path.join(os.path.dirname(build_dir), 'xcodebuild') elif chromium_utils.IsLinux(): build_dir = os.path.join(os.path.dirname(build_dir), 'sconsbuild') test_exe_path = os.path.join(build_dir, options.target, test_exe_name) if not os.path.exists(test_exe_path): raise chromium_utils.PathNotFound('Unable to find %s' % test_exe_path) # Find the current revision to pass to the test. build_revision = slave_utils.SubversionRevision(build_dir) # Compute the path to the test data. src_dir = os.path.dirname(build_dir) data_dir = os.path.join(src_dir, 'data') dom_perf_dir = os.path.join(data_dir, 'dom_perf') iterations = '' # Default if options.target == 'Debug': iterations = '&minIterations=1' def run_and_print(use_refbuild): # Windows used to write to the root of C:, but that doesn't work # on Vista so we write into the build folder instead. suffix = '' if (use_refbuild): suffix = '_ref' output_file = os.path.join( build_dir, options.target, 'dom_perf_result_%s%s.txt' % (build_revision, suffix)) result = 0 compiled_data = [] for test in TESTS: url = URL % (dom_perf_dir, test, iterations, build_revision) url_flag = '--url=%s' % url command = [ test_exe_path, '--wait_cookie_name=__domperf_finished', '--jsvar=__domperf_result', '--jsvar_output=%s' % output_file, '--gtest_filter=UrlFetchTest.UrlFetch', url_flag ] if use_refbuild: command.append('--reference_build') print "Executing: " print command result |= chromium_utils.RunCommand(command) # Open the resulting file and display it. data = json.load(open(output_file, 'r')) for suite in data['BenchmarkSuites']: # Skip benchmarks that we didn't actually run this time around. if len(suite['Benchmarks']) == 0 and suite['score'] == 0: continue compiled_data.append(suite) # Now give the geometric mean as the total for the combined runs. total = geometric_mean([s['score'] for s in compiled_data]) print_result(True, 'Total', total, use_refbuild) for suite in compiled_data: print_result(False, suite['name'], suite['score'], use_refbuild) return result try: if chromium_utils.IsLinux(): xvfb.StartVirtualX(options.target, os.path.join(build_dir, options.target)) result = run_and_print(False) result |= run_and_print(True) finally: if chromium_utils.IsLinux(): xvfb.StopVirtualX(options.target) return result
def main_win(options, args): """Using the target build configuration, run the executable given in the first non-option argument, passing any following arguments to that executable. """ if len(args) < 1: raise chromium_utils.MissingArgument('Usage: %s' % USAGE) test_exe = args[0] build_dir = os.path.abspath(options.build_dir) test_exe_path = os.path.join(build_dir, options.target, test_exe) if not os.path.exists(test_exe_path): if options.factory_properties.get('succeed_on_missing_exe', False): print '%s missing but succeed_on_missing_exe used, exiting' % ( test_exe_path) return 0 raise chromium_utils.PathNotFound('Unable to find %s' % test_exe_path) if options.enable_pageheap: slave_utils.SetPageHeap(build_dir, 'chrome.exe', True) if options.parallel: command = _BuildParallelCommand(build_dir, test_exe_path, options) elif options.run_python_script: command = [sys.executable, test_exe] else: command = [test_exe_path] command.extend(args[1:]) # Nuke anything that appears to be stale chrome items in the temporary # directory from previous test runs (i.e.- from crashes or unittest leaks). slave_utils.RemoveChromeTemporaryFiles() results_tracker = None if options.generate_json_file: results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker() if os.path.exists(options.test_output_xml): # remove the old XML output file. os.remove(options.test_output_xml) try: http_server = None if options.document_root: http_server = start_http_server( 'win', build_dir=build_dir, test_exe_path=test_exe_path, document_root=options.document_root) result = _RunGTestCommand(command, results_tracker) finally: if http_server: http_server.StopServer() if options.enable_pageheap: slave_utils.SetPageHeap(build_dir, 'chrome.exe', False) if options.generate_json_file: _GenerateJSONForTestResults(options, results_tracker) return result
def main_linux(options, args): if len(args) < 1: raise chromium_utils.MissingArgument('Usage: %s' % USAGE) build_dir = os.path.normpath(os.path.abspath(options.build_dir)) slave_name = slave_utils.SlaveBuildName(build_dir) # If this is a sub-project build (i.e. there's a 'sconsbuild' in build_dir), # look for the test binaries there, otherwise look for the top-level build # output. # This assumes we never pass a build_dir which might contain build output that # we're not trying to test. This is currently a safe assumption since we don't # have any builders that do both sub-project and top-level builds (only # Modules builders do sub-project builds), so they shouldn't ever have both # 'build_dir/sconsbuild' and 'build_dir/../sconsbuild'. outdir = None if os.path.exists(os.path.join(build_dir, 'sconsbuild')): outdir = 'sconsbuild' elif os.path.exists(os.path.join(build_dir, 'out')): outdir = 'out' if outdir: bin_dir = os.path.join(build_dir, outdir, options.target) src_dir = os.path.join(slave_utils.SlaveBaseDir(build_dir), 'build', 'src') os.environ['CR_SOURCE_ROOT'] = src_dir else: if os.path.exists(os.path.join(build_dir, '..', 'sconsbuild')): bin_dir = os.path.join(build_dir, '..', 'sconsbuild', options.target) else: bin_dir = os.path.join(build_dir, '..', 'out', options.target) # Figure out what we want for a special frame buffer directory. special_xvfb_dir = None if options.special_xvfb == 'auto': fp_special_xvfb = options.factory_properties.get('special_xvfb', None) fp_chromeos = options.factory_properties.get('chromeos', None) if fp_special_xvfb or ( fp_special_xvfb is None and (fp_chromeos or slave_utils.GypFlagIsOn(options, 'use_aura') or slave_utils.GypFlagIsOn(options, 'chromeos'))): special_xvfb_dir = options.special_xvfb_dir elif options.special_xvfb: special_xvfb_dir = options.special_xvfb_dir test_exe = args[0] test_exe_path = os.path.join(bin_dir, test_exe) if not os.path.exists(test_exe_path): if options.factory_properties.get('succeed_on_missing_exe', False): print '%s missing but succeed_on_missing_exe used, exiting' % ( test_exe_path) return 0 msg = 'Unable to find %s' % test_exe_path raise chromium_utils.PathNotFound(msg) # Decide whether to enable the suid sandbox for Chrome. if should_enable_sandbox(CHROME_SANDBOX_PATH): print 'Enabling sandbox. Setting environment variable:' print ' CHROME_DEVEL_SANDBOX="%s"' % CHROME_SANDBOX_PATH os.environ['CHROME_DEVEL_SANDBOX'] = CHROME_SANDBOX_PATH else: print 'Disabling sandbox. Setting environment variable:' print ' CHROME_DEVEL_SANDBOX=""' os.environ['CHROME_DEVEL_SANDBOX'] = '' # Nuke anything that appears to be stale chrome items in the temporary # directory from previous test runs (i.e.- from crashes or unittest leaks). slave_utils.RemoveChromeTemporaryFiles() os.environ['LD_LIBRARY_PATH'] = '%s:%s/lib:%s/lib.target' % ( bin_dir, bin_dir, bin_dir) # Figure out what we want for a special llvmpipe directory. if (options.llvmpipe_dir and os.path.exists(options.llvmpipe_dir)): os.environ['LD_LIBRARY_PATH'] += ':' + options.llvmpipe_dir if options.parallel: command = _BuildParallelCommand(build_dir, test_exe_path, options) elif options.run_shell_script: command = ['bash', test_exe_path] elif options.run_python_script: command = [sys.executable, test_exe] else: command = [test_exe_path] command.extend(args[1:]) results_tracker = None if options.generate_json_file: results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker() if os.path.exists(options.test_output_xml): # remove the old XML output file. os.remove(options.test_output_xml) try: http_server = None if options.document_root: http_server = start_http_server( 'linux', build_dir=build_dir, test_exe_path=test_exe_path, document_root=options.document_root) if options.xvfb: xvfb.StartVirtualX(slave_name, bin_dir, with_wm=options.factory_properties.get( 'window_manager', True), server_dir=special_xvfb_dir) if options.factory_properties.get('asan', False): symbolize = os.path.abspath( os.path.join('src', 'tools', 'valgrind', 'asan', 'asan_symbolize.py')) pipes = [[sys.executable, symbolize], ['c++filt']] result = _RunGTestCommand(command, pipes=pipes) else: result = _RunGTestCommand(command, results_tracker) finally: if http_server: http_server.StopServer() if options.xvfb: xvfb.StopVirtualX(slave_name) if options.generate_json_file: _GenerateJSONForTestResults(options, results_tracker) return result
def main_mac(options, args): if len(args) < 1: raise chromium_utils.MissingArgument('Usage: %s' % USAGE) test_exe = args[0] build_dir = os.path.normpath(os.path.abspath(options.build_dir)) test_exe_path = os.path.join(build_dir, options.target, test_exe) if not os.path.exists(test_exe_path): pre = 'Unable to find %s\n' % test_exe_path build_dir = os.path.dirname(build_dir) outdir = 'xcodebuild' is_make_or_ninja = (options.factory_properties.get( "gclient_env", {}).get('GYP_GENERATORS', '') in ('ninja', 'make')) if is_make_or_ninja: outdir = 'out' build_dir = os.path.join(build_dir, outdir) test_exe_path = os.path.join(build_dir, options.target, test_exe) if not os.path.exists(test_exe_path): msg = pre + 'Unable to find %s' % test_exe_path if options.factory_properties.get('succeed_on_missing_exe', False): print '%s missing but succeed_on_missing_exe used, exiting' % ( test_exe_path) return 0 raise chromium_utils.PathNotFound(msg) # Nuke anything that appears to be stale chrome items in the temporary # directory from previous test runs (i.e.- from crashes or unittest leaks). slave_utils.RemoveChromeTemporaryFiles() if options.parallel: command = _BuildParallelCommand(build_dir, test_exe_path, options) elif options.run_shell_script: command = ['bash', test_exe_path] elif options.run_python_script: command = [sys.executable, test_exe] else: command = [test_exe_path] command.extend(args[1:]) results_tracker = None if options.generate_json_file: results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker() if os.path.exists(options.test_output_xml): # remove the old XML output file. os.remove(options.test_output_xml) try: http_server = None if options.document_root: http_server = start_http_server( 'mac', build_dir=build_dir, test_exe_path=test_exe_path, document_root=options.document_root) if options.factory_properties.get('asan', False): symbolize = os.path.abspath( os.path.join('src', 'tools', 'valgrind', 'asan', 'asan_symbolize.py')) pipes = [[sys.executable, symbolize], ['c++filt']] result = _RunGTestCommand(command, pipes=pipes) else: result = _RunGTestCommand(command, results_tracker) finally: if http_server: http_server.StopServer() if options.generate_json_file: _GenerateJSONForTestResults(options, results_tracker) return result
def _Main(options, args, extra_env): """Using the target build configuration, run the executable given in the first non-option argument, passing any following arguments to that executable. Args: options: Command-line options for this invocation of runtest.py. args: Command and arguments for the test. extra_env: A dictionary of extra environment variables to set. Returns: Exit status code. """ if len(args) < 1: raise chromium_utils.MissingArgument('Usage: %s' % USAGE) xvfb_path = os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'third_party', 'xvfb', platform.architecture()[0]) special_xvfb_dir = None fp_chromeos = options.factory_properties.get('chromeos', None) if (fp_chromeos or slave_utils.GypFlagIsOn(options, 'use_aura') or slave_utils.GypFlagIsOn(options, 'chromeos')): special_xvfb_dir = xvfb_path build_dir = os.path.normpath(os.path.abspath(options.build_dir)) bin_dir = os.path.join(build_dir, options.target) slave_name = options.slave_name or slave_utils.SlaveBuildName(build_dir) test_exe = args[0] if options.run_python_script: test_exe_path = test_exe else: test_exe_path = os.path.join(bin_dir, test_exe) if not os.path.exists(test_exe_path): if options.factory_properties.get('succeed_on_missing_exe', False): print '%s missing but succeed_on_missing_exe used, exiting' % ( test_exe_path) return 0 raise chromium_utils.PathNotFound('Unable to find %s' % test_exe_path) if sys.platform == 'linux2': # Unset http_proxy and HTTPS_PROXY environment variables. When set, this # causes some tests to hang. See http://crbug.com/139638 for more info. if 'http_proxy' in os.environ: del os.environ['http_proxy'] print 'Deleted http_proxy environment variable.' if 'HTTPS_PROXY' in os.environ: del os.environ['HTTPS_PROXY'] print 'Deleted HTTPS_PROXY environment variable.' # Path to SUID sandbox binary. This must be installed on all bots. extra_env['CHROME_DEVEL_SANDBOX'] = CHROME_SANDBOX_PATH extra_env['LD_LIBRARY_PATH'] = '' if options.enable_lsan: # Use the debug version of libstdc++ under LSan. If we don't, there will # be a lot of incomplete stack traces in the reports. extra_env['LD_LIBRARY_PATH'] += '/usr/lib/x86_64-linux-gnu/debug:' extra_env['LD_LIBRARY_PATH'] += '%s:%s/lib:%s/lib.target' % ( bin_dir, bin_dir, bin_dir) if options.run_shell_script: command = ['bash', test_exe_path] elif options.run_python_script: command = [sys.executable, test_exe] else: command = _BuildTestBinaryCommand(build_dir, test_exe_path, options) command.extend(args[1:]) # Nuke anything that appears to be stale chrome items in the temporary # directory from previous test runs (i.e.- from crashes or unittest leaks). slave_utils.RemoveChromeTemporaryFiles() log_processor = None if _UsingGtestJson(options): log_processor = gtest_utils.GTestJSONParser( options.build_properties.get('mastername')) if options.generate_json_file: if os.path.exists(options.test_output_xml): # remove the old XML output file. os.remove(options.test_output_xml) try: # TODO(dpranke): checking on test_exe is a temporary hack until we # can change the buildbot master to pass --xvfb instead of --no-xvfb # for these two steps. See # https://code.google.com/p/chromium/issues/detail?id=179814 start_xvfb = (sys.platform == 'linux2' and (options.xvfb or 'layout_test_wrapper' in test_exe or 'devtools_perf_test_wrapper' in test_exe)) if start_xvfb: xvfb.StartVirtualX(slave_name, bin_dir, with_wm=(options.factory_properties.get( 'window_manager', 'True') == 'True'), server_dir=special_xvfb_dir) if _UsingGtestJson(options): json_file_name = log_processor.PrepareJSONFile( options.test_launcher_summary_output) command.append('--test-launcher-summary-output=%s' % json_file_name) pipes = [] # See the comment in main() regarding offline symbolization. if options.use_symbolization_script: symbolize_command = _GetSanitizerSymbolizeCommand( strip_path_prefix=options.strip_path_prefix) pipes = [symbolize_command] command = _GenerateRunIsolatedCommand(build_dir, test_exe_path, options, command) result = _RunGTestCommand(options, command, extra_env, pipes=pipes) finally: if start_xvfb: xvfb.StopVirtualX(slave_name) if _UsingGtestJson(options): if options.use_symbolization_script: _SymbolizeSnippetsInJSON(options, json_file_name) log_processor.ProcessJSONFile(options.build_dir) if options.generate_json_file: if not _GenerateJSONForTestResults(options, log_processor): return 1 if options.annotate: annotation_utils.annotate(options.test_type, result, log_processor) return result