def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--output', required=True) parser.add_argument('args', nargs=argparse.REMAINDER) args = parser.parse_args(argv) passthrough_args = args.args if passthrough_args[0] == '--': passthrough_args = passthrough_args[1:] results = {} for filename in os.listdir(common.SCRIPT_DIR): if not filename.endswith('.py'): continue if filename in ('common.py', 'get_compile_targets.py'): continue with common.temporary_file() as tempfile_path: rc = common.run_command( [sys.executable, os.path.join(common.SCRIPT_DIR, filename)] + passthrough_args + ['compile_targets', '--output', tempfile_path]) if rc != 0: return rc with open(tempfile_path) as f: results[filename] = json.load(f) with open(args.output, 'w') as f: json.dump(results, f) return 0
def main_run(args): filter_tests = [] if args.filter_file: filter_tests = json.load(args.filter_file) test_args = ['--retry-limit', '3'] if 'android' == args.properties.get('target_platform'): test_args += ['--browser', 'android-chromium', '--device', 'android'] else: test_args += ['--browser', args.build_config_fs.lower()] with common.temporary_file() as tempfile_path: test_args += ['--write-full-results-to', tempfile_path] rc = common.run_runtest(args, [ '--test-type', 'telemetry_perf_unittests', '--run-python-script', os.path.join(common.SRC_DIR, 'tools', 'perf', 'run_tests') ] + test_args + filter_tests) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results, test_separator='.') failures = parsed_results['unexpected_failures'] json.dump( { 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.output) return rc
def _run_tests(args, tests, extra_flags, env, screenshot_dir, results, test_results): keys = get_skia_gold_keys(args) with temporary_dir('angle_skia_gold_') as skia_gold_temp_dir: gold_properties = angle_skia_gold_properties.ANGLESkiaGoldProperties( args) gold_session_manager = angle_skia_gold_session_manager.ANGLESkiaGoldSessionManager( skia_gold_temp_dir, gold_properties) gold_session = gold_session_manager.GetSkiaGoldSession(keys) traces = [trace.split(' ')[0] for trace in tests] for test in traces: # Apply test filter if present. if args.isolated_script_test_filter: full_name = 'angle_restricted_trace_gold_tests.%s' % test if not fnmatch.fnmatch(full_name, args.isolated_script_test_filter): logging.info( 'Skipping test %s because it does not match filter %s' % (full_name, args.isolated_script_test_filter)) continue with common.temporary_file() as tempfile_path: cmd = [ args.test_suite, DEFAULT_TEST_PREFIX + test, '--render-test-output-dir=%s' % screenshot_dir, '--one-frame-only', '--verbose-logging', ] + extra_flags result = None for iteration in range(0, args.flaky_retries + 1): if result != PASS: if iteration > 0: logging.info('Retrying flaky test: "%s"...' % test) result = PASS if run_wrapper( args, cmd, env, tempfile_path) == 0 else FAIL artifacts = {} if result == PASS: result = upload_test_result_to_skia_gold( args, gold_session_manager, gold_session, gold_properties, screenshot_dir, test, artifacts) expected_result = SKIP if result == SKIP else PASS test_results[test] = { 'expected': expected_result, 'actual': result } if result == FAIL: test_results[test]['is_unexpected'] = True if len(artifacts) > 0: test_results[test]['artifacts'] = artifacts results['num_failures_by_type'][result] += 1 return results['num_failures_by_type'][FAIL] == 0
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command([ 'vpython', os.path.join(common.SRC_DIR, 'testing', 'test_env.py'), os.path.join(common.SRC_DIR, 'tools', 'metrics', 'metrics_python_tests.py'), '--isolated-script-test-output', tempfile_path, '--skip-set-lpac-acls=1', ], cwd=os.path.join(common.SRC_DIR, 'out', args.build_config_fs)) with open(tempfile_path) as f: isolated_results = json.load(f) results = common.parse_common_test_results(isolated_results, test_separator='.') failures = [ '%s: %s' % (k, v) for k, v in results['unexpected_failures'].items() ] common.record_local_script_results('metrics_python_tests', args.output, failures, True) return rc
def main(): parser = argparse.ArgumentParser() parser.add_argument('--test-launcher-summary-output', type=argparse.FileType('w'), required=True) args, rest_args = parser.parse_known_args() with common.temporary_file() as tempfile_path: rc = common.run_command([sys.executable] + rest_args + [ '--write-full-results-to', tempfile_path, ]) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results, test_separator='.') failures = parsed_results['unexpected_failures'] json.dump( { 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.test_launcher_summary_output) return rc
def get_device_info(args, failures): """Parses the device info for each attached device, and returns a summary of the device info and any mismatches. Returns: A dict indicating the result. """ if not is_linux(): return {} with common.temporary_file() as tempfile_path: rc = common.run_command([ sys.executable, os.path.join(args.paths['checkout'], 'third_party', 'catapult', 'devil', 'devil', 'android', 'tools', 'device_status.py'), '--json-output', tempfile_path, '--blacklist-file', os.path.join( args.paths['checkout'], 'out', 'bad_devices.json')]) if rc: failures.append('device_status') return {} with open(tempfile_path, 'r') as src: device_info = json.load(src) results = {} results['devices'] = sorted(v['serial'] for v in device_info) details = [ v['ro.build.fingerprint'] for v in device_info if not v['blacklisted']] def unique_build_details(index): return sorted(list(set([v.split(':')[index] for v in details]))) parsed_details = { 'device_names': unique_build_details(0), 'build_versions': unique_build_details(1), 'build_types': unique_build_details(2), } for k, v in parsed_details.iteritems(): if len(v) == 1: results[k] = v[0] else: results[k] = 'MISMATCH' results['%s_list' % k] = v failures.append(k) for v in device_info: if v['blacklisted']: failures.append('Device %s blacklisted' % v['serial']) return results
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command([ sys.executable, os.path.join(common.SRC_DIR, 'build', 'check_gn_headers.py'), '--out-dir', os.path.join(args.paths['checkout'], 'out', args.build_config_fs), '--whitelist', os.path.join(common.SRC_DIR, 'build', 'check_gn_headers_whitelist.txt'), '--json', tempfile_path, '--verbose', ], cwd=common.SRC_DIR) with open(tempfile_path) as f: failures = json.load(f) json.dump({ 'valid': True, 'failures': failures, }, args.output) return rc
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command([ sys.executable, os.path.join(common.SRC_DIR, 'third_party', 'WebKit', 'Tools', 'Scripts', 'test-webkitpy'), '--write-full-results-to', tempfile_path, ], cwd=args.paths['checkout']) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results) failures = parsed_results['unexpected_failures'] json.dump( { 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.output) return rc
def main_run(args): filter_tests = [] if args.filter_file: filter_tests = json.load(args.filter_file) perf_id = args.properties.get("perf-id") script_args = args.args if IsWindows(): script_args[0] += ".exe" test_suite = script_args[0] with common.temporary_file() as tempfile_path: gtest_args = [ "--target", args.build_config_fs, "--annotate", "graphing", "--perf-id", perf_id, "--perf-dashboard-id", test_suite, "--results-url", args.properties.get("results-url"), "--slave-name", args.properties.get("slavename"), "--builder-name", args.properties.get("buildername"), "--build-number", str(args.properties.get("buildnumber")), "--log-processor-output-file", tempfile_path, "--test-type", test_suite, ] if "android" == args.properties.get("target_platform"): gtest_args.extend( [ "--no-xvfb", "--run-python-script", os.path.join(args.paths["checkout"], "build", "android", "test_runner.py"), "gtest", "--release", "--suite", test_suite, "--verbose", ] ) else: gtest_args.extend(["--xvfb"]) gtest_args.extend(script_args) rc = common.run_runtest(args, gtest_args + filter_tests) with open(tempfile_path) as f: results = json.load(f) json.dump({"valid": bool(rc == 0), "failures": results["failed"]}, args.output) return rc
def main_run(args): filter_tests = [] if args.filter_file: filter_tests = json.load(args.filter_file) test_args = ['--retry-limit', '3'] if 'android' == args.properties.get('target_platform'): test_args += ['--browser', 'android-chrome-shell', '--device', 'android'] else: test_args += ['--browser', args.build_config_fs.lower()] with common.temporary_file() as tempfile_path: test_args += ['--write-full-results-to', tempfile_path] rc = common.run_runtest(args, [ '--test-type', 'telemetry_perf_unittests', '--run-python-script', os.path.join(common.SRC_DIR, 'tools', 'perf', 'run_tests') ] + test_args + filter_tests) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results, test_separator='.') failures = parsed_results['unexpected_failures'] json.dump({ 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.output) return rc
def main_run(args): filter_tests = [] if args.filter_file: filter_tests = json.load(args.filter_file) script_args = args.args test_suite = script_args[0] with common.temporary_file() as tempfile_path: cmd = [ os.path.join( args.paths['checkout'], 'build', 'android', 'test_runner.py'), 'gtest', '--release' if 'release' in args.build_config_fs.lower() else '--debug', '--suite', test_suite, '--verbose', '--flakiness-dashboard-server=http://test-results.appspot.com', '--json-results-file', tempfile_path, ] if filter_tests: cmd.extend(['--gtest-filter', ':'.join(filter_tests)]) rc = common.run_command(cmd) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_gtest_test_results(results) json.dump({ 'valid': True, 'failures': parsed_results['failures'], }, args.output) return rc
def main_run(args): filter_tests = [] if args.filter_file: filter_tests = json.load(args.filter_file) perf_id = args.properties.get('perf-id') script_args = args.args test_suite = script_args[0] if IsWindows(): script_args[0] += '.exe' with common.temporary_file() as tempfile_path: gtest_args = [ '--target', args.build_config_fs, '--annotate', 'graphing', '--perf-id', perf_id, '--perf-dashboard-id', test_suite, '--results-url', args.properties.get('results-url'), '--slave-name', args.properties.get('slavename'), '--builder-name', args.properties.get('buildername'), '--build-number', str(args.properties.get('buildnumber')), '--log-processor-output-file', tempfile_path, '--test-type', test_suite, ] if 'android' == args.properties.get('target_platform'): gtest_args.extend([ '--no-xvfb', '--run-python-script', os.path.join(args.paths['checkout'], 'out', args.build_config_fs, 'bin', 'run_%s' % test_suite), '--verbose', ]) gtest_args.extend(script_args[1:]) else: gtest_args.extend(['--xvfb']) gtest_args.extend(script_args) rc = common.run_runtest(args, gtest_args + filter_tests) with open(tempfile_path) as f: results = json.load(f) json.dump({ 'valid': bool(rc == 0), 'failures': results['failed'], }, args.output) return rc
def main_run(args): typ_path = os.path.abspath( os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir, 'third_party', 'catapult', 'third_party', 'typ')) _AddToPathIfNeeded(typ_path) import typ top_level_dir = os.path.join(common.SRC_DIR, 'headless', 'lib', 'browser', 'devtools_api') with common.temporary_file() as tempfile_path: rc = typ.main(argv=[], top_level_dir=top_level_dir, write_full_results_to=tempfile_path, coverage_source=[top_level_dir]) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results, test_separator='.') failures = parsed_results['unexpected_failures'] json.dump( { 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.output) return rc
def main(argv): parser = argparse.ArgumentParser() parser.add_argument("--output", required=True) parser.add_argument("args", nargs=argparse.REMAINDER) args = parser.parse_args(argv) passthrough_args = args.args if passthrough_args[0] == "--": passthrough_args = passthrough_args[1:] results = {} for filename in os.listdir(common.SCRIPT_DIR): if not filename.endswith(".py"): continue if filename in ("common.py", "get_compile_targets.py"): continue with common.temporary_file() as tempfile_path: rc = common.run_command( [sys.executable, os.path.join(common.SCRIPT_DIR, filename)] + passthrough_args + ["compile_targets", "--output", tempfile_path] ) if rc != 0: return rc with open(tempfile_path) as f: results[filename] = json.load(f) with open(args.output, "w") as f: json.dump(results, f) return 0
def main_run(args): filter_tests = [] if args.filter_file: filter_tests = json.load(args.filter_file) with common.temporary_file() as tempfile_path: rc = common.run_runtest(args, [ '--annotate', 'gtest', '--test-type', 'telemetry_unittests', '--run-python-script', os.path.join(common.SRC_DIR, 'tools', 'telemetry', 'run_tests'), '--browser', args.build_config_fs.lower(), '--retry-limit', '3', '--write-full-results-to', tempfile_path, ] + filter_tests) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results, test_separator='.') failures = parsed_results['unexpected_failures'] json.dump({ 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.output) return rc
def main_run(args): runner = os.path.join(common.SRC_DIR, 'mojo', 'tools', 'apptest_runner.py') build_dir = os.path.join(common.SRC_DIR, 'out', args.build_config_fs) with common.temporary_file() as tempfile_path: rc = common.run_command([ runner, build_dir, '--verbose', '--write-full-results-to', tempfile_path ]) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results, test_separator='.') failures = parsed_results['unexpected_failures'] json.dump( { 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.output) return rc
def main(): parser = argparse.ArgumentParser() parser.add_argument('--isolated-script-test-output', type=str) args, _ = parser.parse_known_args() if sys.platform == 'win32': exe = os.path.join('.', 'flatbuffers_unittests.exe') else: exe = os.path.join('.', 'flatbuffers_unittests') env = os.environ.copy() failures = [] with common.temporary_file() as tempfile_path: rc = xvfb.run_executable([exe], env, stdoutfile=tempfile_path) # The flatbuffer tests do not really conform to anything parsable, except # that they will succeed with "ALL TESTS PASSED". with open(tempfile_path) as f: output = f.read() if output != "ALL TESTS PASSED\n": failures = [output] if args.isolated_script_test_output: with open(args.isolated_script_test_output, 'w') as fp: json.dump({'valid': True,'failures': failures}, fp) return rc
def main_run(args): filter_tests = [] if args.filter_file: filter_tests = json.load(args.filter_file) with common.temporary_file() as tempfile_path: rc = common.run_command([ sys.executable, os.path.join(common.SRC_DIR, 'chrome', 'test', 'nacl_test_injection', 'buildbot_nacl_integration.py'), '--mode', args.build_config_fs, '--json_build_results_output_file', tempfile_path, ] + filter_tests) with open(tempfile_path) as f: results = json.load(f) json.dump({ 'valid': True, 'failures': [f['raw_name'] for f in results], }, args.output) return rc
def main(): parser = argparse.ArgumentParser() parser.add_argument("--isolated-script-test-output", type=argparse.FileType("w"), required=True) parser.add_argument("--xvfb", help="Start xvfb.", action="store_true") args, rest_args = parser.parse_known_args() xvfb_proc = None openbox_proc = None env = os.environ.copy() if args.xvfb and xvfb.should_start_xvfb(env): xvfb_proc, openbox_proc = xvfb.start_xvfb(env=env, build_dir=".") assert xvfb_proc and openbox_proc, "Failed to start xvfb" try: with common.temporary_file() as tempfile_path: rc = common.run_command([sys.executable] + rest_args + ["--write-full-results-to", tempfile_path], env=env) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results, test_separator=".") failures = parsed_results["unexpected_failures"] json.dump( { "valid": bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), "failures": failures.keys(), }, args.isolated_script_test_output, ) return rc finally: xvfb.kill(xvfb_proc) xvfb.kill(openbox_proc)
def main_run(args): typ_path = os.path.abspath(os.path.join( os.path.dirname(__file__), os.path.pardir, os.path.pardir, 'third_party', 'typ')) _AddToPathIfNeeded(typ_path) import typ top_level_dir = os.path.join( common.SRC_DIR, 'headless', 'lib', 'browser', 'devtools_api') with common.temporary_file() as tempfile_path: rc = typ.main( argv=[], top_level_dir=top_level_dir, write_full_results_to=tempfile_path, coverage_source=[top_level_dir]) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results, test_separator='.') failures = parsed_results['unexpected_failures'] json.dump({ 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.output) return rc
def main(): parser = argparse.ArgumentParser() parser.add_argument('executable', help='Test executable.') parser.add_argument('--isolated-script-test-output', type=str) parser.add_argument('--isolated-script-test-filter', type=str) parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') # Kept for compatiblity. # TODO(jmadill): Remove when removed from the recipes. http://crbug.com/954415 parser.add_argument('--isolated-script-test-perf-output', type=str) args, extra_flags = parser.parse_known_args() env = os.environ.copy() if 'GTEST_TOTAL_SHARDS' in env: extra_flags += ['--shard-count=' + env['GTEST_TOTAL_SHARDS']] env.pop('GTEST_TOTAL_SHARDS') if 'GTEST_SHARD_INDEX' in env: extra_flags += ['--shard-index=' + env['GTEST_SHARD_INDEX']] env.pop('GTEST_SHARD_INDEX') # Assume we want to set up the sandbox environment variables all the # time; doing so is harmless on non-Linux platforms and is needed # all the time on Linux. env[CHROME_SANDBOX_ENV] = CHROME_SANDBOX_PATH rc = 0 try: # Consider adding stdio control flags. if args.isolated_script_test_output: extra_flags.append('--isolated-script-test-output=%s' % args.isolated_script_test_output) if args.isolated_script_test_filter: filter_list = common.extract_filter_list( args.isolated_script_test_filter) extra_flags.append('--gtest_filter=' + ':'.join(filter_list)) if IsWindows(): args.executable = '.\\%s.exe' % args.executable else: args.executable = './%s' % args.executable with common.temporary_file() as tempfile_path: env['CHROME_HEADLESS'] = '1' cmd = [args.executable] + extra_flags if args.xvfb: rc = xvfb.run_executable(cmd, env, stdoutfile=tempfile_path) else: rc = test_env.run_command_with_output(cmd, env=env, stdoutfile=tempfile_path) except Exception: traceback.print_exc() rc = 1 return rc
def execute_gtest_perf_test(args, rest_args): env = os.environ.copy() # Assume we want to set up the sandbox environment variables all the # time; doing so is harmless on non-Linux platforms and is needed # all the time on Linux. env[CHROME_SANDBOX_ENV] = CHROME_SANDBOX_PATH rc = 0 try: executable = rest_args[0] extra_flags = [] if len(rest_args) > 1: extra_flags = rest_args[1:] # These flags are to make sure that test output perf metrics in the log. if not '--verbose' in extra_flags: extra_flags.append('--verbose') if not '--test-launcher-print-test-stdio=always' in extra_flags: extra_flags.append('--test-launcher-print-test-stdio=always') if args.isolated_script_test_filter: filter_list = common.extract_filter_list( args.isolated_script_test_filter) extra_flags.append('--gtest_filter=' + ':'.join(filter_list)) if IsWindows(): executable = '.\%s.exe' % executable else: executable = './%s' % executable with common.temporary_file() as tempfile_path: env['CHROME_HEADLESS'] = '1' cmd = [executable] + extra_flags if args.xvfb: rc = xvfb.run_executable(cmd, env, stdoutfile=tempfile_path) else: rc = test_env.run_command_with_output(cmd, env=env, stdoutfile=tempfile_path) # Now get the correct json format from the stdout to write to the perf # results file results_processor = ( generate_legacy_perf_dashboard_json.LegacyResultsProcessor()) charts = results_processor.GenerateJsonResults(tempfile_path) except Exception: traceback.print_exc() rc = 1 valid = (rc == 0) failures = [] if valid else ['(entire test suite)'] output_json = { 'valid': valid, 'failures': failures, } return rc, charts, output_json
def _run_and_get_output(args, cmd, env): lines = [] logging.debug(' '.join(cmd)) with common.temporary_file() as tempfile_path: if args.xvfb: exit_code = xvfb.run_executable(cmd, env, stdoutfile=tempfile_path) else: exit_code = run_command_with_output(cmd, env=env, stdoutfile=tempfile_path, log=True) with open(tempfile_path) as f: for line in f: lines.append(line.strip()) return exit_code, lines
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_runtest(args, [ '--test-type', 'sizes', '--run-python-script', os.path.join(common.SRC_DIR, 'infra', 'scripts', 'legacy', 'scripts', 'slave', 'chromium', 'sizes.py'), '--json', tempfile_path ]) with open(tempfile_path) as f: results = json.load(f) with open( os.path.join(common.SRC_DIR, 'tools', 'perf_expectations', 'perf_expectations.json')) as f: perf_expectations = json.load(f) prefix = args.args[0] valid = (rc == 0) failures = [] for name, result in results.iteritems(): fqtn = '%s/%s/%s' % (prefix, name, result['identifier']) if fqtn not in perf_expectations: continue if perf_expectations[fqtn]['type'] != 'absolute': print 'ERROR: perf expectation %r is not yet supported' % fqtn valid = False continue actual = result['value'] expected = perf_expectations[fqtn]['regress'] better = perf_expectations[fqtn]['better'] check_result = ((actual <= expected) if better == 'lower' else (actual >= expected)) if not check_result: failures.append(fqtn) print 'FAILED %s: actual %s, expected %s, better %s' % ( fqtn, actual, expected, better) json.dump({ 'valid': valid, 'failures': failures, }, args.output) # sizes.py itself doesn't fail on regressions. if failures and rc == 0: rc = 1 return rc
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--isolated-script-test-output', type=argparse.FileType('w'), required=True) parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') args, rest_args = parser.parse_known_args() xvfb_proc = None openbox_proc = None xcompmgr_proc = None env = os.environ.copy() if args.xvfb and xvfb.should_start_xvfb(env): xvfb_proc, openbox_proc, xcompmgr_proc = xvfb.start_xvfb(env=env, build_dir='.') assert xvfb_proc and openbox_proc and xcompmgr_proc, 'Failed to start xvfb' # Compatibility with gtest-based sharding. total_shards = None shard_index = None if 'GTEST_TOTAL_SHARDS' in env: total_shards = int(env['GTEST_TOTAL_SHARDS']) del env['GTEST_TOTAL_SHARDS'] if 'GTEST_SHARD_INDEX' in env: shard_index = int(env['GTEST_SHARD_INDEX']) del env['GTEST_SHARD_INDEX'] sharding_args = [] if total_shards is not None and shard_index is not None: sharding_args = [ '--total-shards=%d' % total_shards, '--shard-index=%d' % shard_index ] try: with common.temporary_file() as tempfile_path: rc = common.run_command([sys.executable] + rest_args + sharding_args + [ '--write-full-results-to', tempfile_path, ], env=env) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results, test_separator='.') failures = parsed_results['unexpected_failures'] json.dump({ 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.isolated_script_test_output) return rc finally: xvfb.kill(xvfb_proc) xvfb.kill(openbox_proc) xvfb.kill(xcompmgr_proc)
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_runtest(args, [ '--test-type', 'sizes', '--run-python-script', os.path.join( common.SRC_DIR, 'infra', 'scripts', 'legacy', 'scripts', 'slave', 'chromium', 'sizes.py'), '--json', tempfile_path]) with open(tempfile_path) as f: results = json.load(f) with open(os.path.join(common.SRC_DIR, 'tools', 'perf_expectations', 'perf_expectations.json')) as f: perf_expectations = json.load(f) prefix = args.args[0] valid = (rc == 0) failures = [] for name, result in results.iteritems(): fqtn = '%s/%s/%s' % (prefix, name, result['identifier']) if fqtn not in perf_expectations: continue if perf_expectations[fqtn]['type'] != 'absolute': print 'ERROR: perf expectation %r is not yet supported' % fqtn valid = False continue actual = result['value'] expected = perf_expectations[fqtn]['regress'] better = perf_expectations[fqtn]['better'] check_result = ((actual <= expected) if better == 'lower' else (actual >= expected)) if not check_result: failures.append(fqtn) print 'FAILED %s: actual %s, expected %s, better %s' % ( fqtn, actual, expected, better) json.dump({ 'valid': valid, 'failures': failures, }, args.output) # sizes.py itself doesn't fail on regressions. if failures and rc == 0: rc = 1 return rc
def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--isolated-script-test-output', type=str, required=True) parser.add_argument('--isolated-script-test-chartjson-output', type=str, required=False) parser.add_argument('--isolated-script-test-perf-output', type=str, required=False) parser.add_argument('--isolated-script-test-filter', type=str, required=False) args = parser.parse_args(argv) env = os.environ.copy() # Assume we want to set up the sandbox environment variables all the # time; doing so is harmless on non-Linux platforms and is needed # all the time on Linux. env[CHROME_SANDBOX_ENV] = CHROME_SANDBOX_PATH if sys.platform == 'win32': exe = os.path.join('.', 'content_shell.exe') elif sys.platform == 'darwin': exe = os.path.join('.', 'Content Shell.app', 'Contents', 'MacOS', 'Content Shell') else: exe = os.path.join('.', 'content_shell') with common.temporary_file() as tempfile_path: env['CHROME_HEADLESS'] = '1' rc = xvfb.run_executable([ sys.executable, os.path.join(common.SRC_DIR, 'content', 'shell', 'tools', 'breakpad_integration_test.py'), '--verbose', '--build-dir', '.', '--binary', exe, '--json', tempfile_path ], env) with open(tempfile_path) as f: failures = json.load(f) with open(args.isolated_script_test_output, 'w') as fp: json.dump({ 'valid': True, 'failures': failures, }, fp) return rc
def main_run(args): if not sys.platform.startswith('win'): json.dump( { 'valid': False, 'failures': ['This script should only be called on win32.'], }, args.output) with common.temporary_file() as tempfile_path: syzyasan_integration_test_res = common.run_integration_test( SYZYASAN_INTEGRATION_TEST, ['--log-to-json', tempfile_path], tempfile_path, args.output) return syzyasan_integration_test_res
def main_run(args): if not sys.platform.startswith('win'): json.dump({ 'valid': False, 'failures': ['This script should only be called on win32.'], }, args.output) with common.temporary_file() as tempfile_path: kasko_integration_test_res = common.run_integration_test( KASKO_INTEGRATION_TEST, ['--log-to-json', tempfile_path], tempfile_path, args.output) return kasko_integration_test_res
def main_run(args): filter_tests = [] if args.filter_file: filter_tests = json.load(args.filter_file) perf_id = args.properties.get('perf-id') script_args = args.args test_suite = script_args[0] if IsWindows(): script_args[0] += '.exe' with common.temporary_file() as tempfile_path: gtest_args = [ '--target', args.build_config_fs, '--annotate', 'graphing', '--perf-id', perf_id, '--perf-dashboard-id', test_suite, '--results-url', args.properties.get('results-url'), '--slave-name', args.properties.get('slavename'), '--builder-name', args.properties.get('buildername'), '--build-number', str(args.properties.get('buildnumber')), '--log-processor-output-file', tempfile_path, '--test-type', test_suite, ] if 'android' == args.properties.get('target_platform'): gtest_args.extend([ '--no-xvfb', '--run-python-script', os.path.join( args.paths['checkout'], 'out', args.build_config_fs, 'bin', 'run_%s' % test_suite), '--verbose', ]) gtest_args.extend(script_args[1:]) else: gtest_args.extend(['--xvfb']) gtest_args.extend(script_args) rc = common.run_runtest(args, gtest_args + filter_tests) with open(tempfile_path) as f: results = json.load(f) json.dump({ 'valid': bool(rc == 0), 'failures': results['failed'], }, args.output) return rc
def main(argv): parser = argparse.ArgumentParser() parser.add_argument( '--isolated-script-test-output', type=str, required=True) parser.add_argument( '--isolated-script-test-chartjson-output', type=str, required=True) parser.add_argument( '--isolated-script-test-perf-output', type=str, required=False) args = parser.parse_args(argv) env = os.environ.copy() # Assume we want to set up the sandbox environment variables all the # time; doing so is harmless on non-Linux platforms and is needed # all the time on Linux. env[CHROME_SANDBOX_ENV] = CHROME_SANDBOX_PATH if sys.platform == 'win32': exe = os.path.join('.', 'content_shell.exe') elif sys.platform == 'darwin': exe = os.path.join('.', 'Content Shell.app', 'Contents', 'MacOS', 'Content Shell') else: exe = os.path.join('.', 'content_shell') with common.temporary_file() as tempfile_path: rc = xvfb.run_executable([ sys.executable, os.path.join(common.SRC_DIR, 'content', 'shell', 'tools', 'breakpad_integration_test.py'), '--verbose', '--build-dir', '.', '--binary', exe, '--json', tempfile_path ], env) with open(tempfile_path) as f: failures = json.load(f) with open(args.isolated_script_test_output, 'w') as fp: json.dump({ 'valid': True, 'failures': failures, }, fp) return rc
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command([ sys.executable, os.path.join(common.SRC_DIR, 'third_party', 'blink', 'tools', 'lint_test_expectations.py'), '--json', tempfile_path ]) with open(tempfile_path) as f: failures = json.load(f) common.record_local_script_results('blink_lint_expectations', args.output, failures, True) return rc
def _run_and_get_output(args, cmd, env): lines = [] with common.temporary_file() as tempfile_path: if args.xvfb: ret = xvfb.run_executable(cmd, env, stdoutfile=tempfile_path) else: ret = test_env.run_command_with_output(cmd, env=env, stdoutfile=tempfile_path) if ret: logging.error('Error running test suite.') return None with open(tempfile_path) as f: for line in f: lines.append(line.strip()) return lines
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command([ os.path.join(common.SRC_DIR, 'android_webview', 'tools', 'webview_licenses.py'), 'scan', '--json', tempfile_path ]) with open(tempfile_path) as f: results = json.load(f) json.dump({ 'valid': True, 'failures': results, }, args.output) return rc
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command([ sys.executable, os.path.join(common.SRC_DIR, 'third_party', 'blink', 'tools', 'lint_test_expectations.py'), '--json', tempfile_path ]) with open(tempfile_path) as f: failures = json.load(f) json.dump({ 'valid': True, 'failures': failures, }, args.output) return rc
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command( [ sys.executable, os.path.join(common.SRC_DIR, "third_party", "WebKit", "Tools", "Scripts", "lint-test-expectations"), "--json", tempfile_path, ] ) with open(tempfile_path) as f: failures = json.load(f) json.dump({"valid": True, "failures": failures}, args.output) return rc
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command([ sys.executable, os.path.join(common.SRC_DIR, 'tools', 'checkbins', 'checkbins.py'), '--verbose', '--json', tempfile_path, os.path.join(args.paths['checkout'], 'out', args.build_config_fs), ]) with open(tempfile_path) as f: checkbins_results = json.load(f) json.dump({ 'valid': True, 'failures': checkbins_results, }, args.output) return rc
def main_run(args): print(sys.executable) with common.temporary_file() as tempfile_path: rc = common.run_command([ with_python3(), os.path.join(common.SRC_DIR, 'tools', 'checkbins', 'checkbins.py'), '--verbose', '--json', tempfile_path, os.path.join(args.paths['checkout'], 'out', args.build_config_fs), ]) with open(tempfile_path) as f: checkbins_results = json.load(f) common.record_local_script_results('checkbins', args.output, checkbins_results, True) return rc
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command([ os.path.join(common.SRC_DIR, 'tools', 'checkperms', 'checkperms.py'), '--root', args.paths['checkout'], '--json', tempfile_path ]) with open(tempfile_path) as f: checkperms_results = json.load(f) result_set = set() for result in checkperms_results: result_set.add((result['rel_path'], result['error'])) failures = ['%s: %s' % (r[0], r[1]) for r in result_set] common.record_local_script_results( 'checkperms', args.output, failures, True) return rc
def main_run(args): runner = os.path.join(common.SRC_DIR, 'mojo', 'tools', 'apptest_runner.py') build_dir = os.path.join(common.SRC_DIR, 'out', args.build_config_fs) with common.temporary_file() as tempfile_path: rc = common.run_command([runner, build_dir, '--verbose', '--write-full-results-to', tempfile_path]) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results, test_separator='.') failures = parsed_results['unexpected_failures'] json.dump({ 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.output) return rc
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command([ os.path.join(common.SRC_DIR, 'buildtools', 'checkdeps', 'checkdeps.py'), '--json', tempfile_path ]) with open(tempfile_path) as f: checkdeps_results = json.load(f) result_set = set() for result in checkdeps_results: for violation in result['violations']: result_set.add((result['dependee_path'], violation['include_path'])) json.dump({ 'valid': True, 'failures': ['%s: %s' % (r[0], r[1]) for r in result_set], }, args.output) return rc
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command([ os.path.join(common.SRC_DIR, 'tools', 'checkperms', 'checkperms.py'), '--root', args.paths['checkout'], '--json', tempfile_path ]) with open(tempfile_path) as f: checkperms_results = json.load(f) result_set = set() for result in checkperms_results: result_set.add((result['rel_path'], result['error'])) json.dump({ 'valid': True, 'failures': ['%s: %s' % (r[0], r[1]) for r in result_set], }, args.output) return rc
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command([ os.path.join(common.SRC_DIR, 'buildtools', 'checkdeps', 'checkdeps.py'), '--json', tempfile_path ]) with open(tempfile_path) as f: checkdeps_results = json.load(f) result_set = set() for result in checkdeps_results: for violation in result['violations']: result_set.add( (result['dependee_path'], violation['include_path'])) failures = ['%s: %s' % (r[0], r[1]) for r in result_set] common.record_local_script_results('checkdeps', args.output, failures, True) return rc
def run_tests(args, tests, extra_flags, env, screenshot_dir): keys = get_skia_gold_keys(args) with temporary_dir('angle_skia_gold_') as skia_gold_temp_dir: gold_properties = angle_skia_gold_properties.ANGLESkiaGoldProperties( args) gold_session_manager = angle_skia_gold_session_manager.ANGLESkiaGoldSessionManager( skia_gold_temp_dir, gold_properties) gold_session = gold_session_manager.GetSkiaGoldSession(keys) for test in tests['traces']: with common.temporary_file() as tempfile_path: cmd = [ args.test_suite, DEFAULT_TEST_PREFIX + test, '--render-test-output-dir=%s' % screenshot_dir, '--one-frame-only', '--verbose-logging', ] + extra_flags result = PASS if run_wrapper(args, cmd, env, tempfile_path) == 0 else FAIL artifacts = {} if result == PASS: result = upload_test_result_to_skia_gold( args, gold_session_manager, gold_session, gold_properties, screenshot_dir, test, artifacts) expected_result = SKIP if result == SKIP else PASS result_tests[test] = { 'expected': expected_result, 'actual': result } if len(artifacts) > 0: result_tests[test]['artifacts'] = artifacts results['num_failures_by_type'][result] += 1 return results['num_failures_by_type'][FAIL] == 0
def main_run(args): with common.temporary_file() as tempfile_path: rc = common.run_command([ sys.executable, os.path.join(common.SRC_DIR, 'third_party', 'WebKit', 'Tools', 'Scripts', 'test-webkitpy'), '--write-full-results-to', tempfile_path, ], cwd=args.paths['checkout']) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results) failures = parsed_results['unexpected_failures'] json.dump({ 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.output) return rc
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--isolated-script-test-output', type=argparse.FileType('w'), required=True) parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') args, rest_args = parser.parse_known_args() xvfb_proc = None openbox_proc = None xcompmgr_proc = None env = os.environ.copy() if args.xvfb and xvfb.should_start_xvfb(env): xvfb_proc, openbox_proc, xcompmgr_proc = xvfb.start_xvfb(env=env, build_dir='.') assert xvfb_proc and openbox_proc and xcompmgr_proc, 'Failed to start xvfb' try: with common.temporary_file() as tempfile_path: rc = common.run_command([sys.executable] + rest_args + [ '--write-full-results-to', tempfile_path, ], env=env) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results, test_separator='.') failures = parsed_results['unexpected_failures'] json.dump({ 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.isolated_script_test_output) return rc finally: xvfb.kill(xvfb_proc) xvfb.kill(openbox_proc) xvfb.kill(xcompmgr_proc)
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--isolated-script-test-output', type=argparse.FileType('w'), required=True) args, rest_args = parser.parse_known_args() with common.temporary_file() as tempfile_path: rc = common.run_command([sys.executable] + rest_args + [ '--write-full-results-to', tempfile_path, ]) with open(tempfile_path) as f: results = json.load(f) parsed_results = common.parse_common_test_results(results, test_separator='.') failures = parsed_results['unexpected_failures'] json.dump({ 'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and ((rc == 0) or failures)), 'failures': failures.keys(), }, args.isolated_script_test_output) return rc
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--isolated-script-test-output', type=str, required=True) parser.add_argument( '--isolated-script-test-chartjson-output', type=str, required=True) parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') args, rest_args = parser.parse_known_args() xvfb_proc = None openbox_proc = None xcompmgr_proc = None env = os.environ.copy() # Assume we want to set up the sandbox environment variables all the # time; doing so is harmless on non-Linux platforms and is needed # all the time on Linux. env[CHROME_SANDBOX_ENV] = CHROME_SANDBOX_PATH if args.xvfb and xvfb.should_start_xvfb(env): xvfb_proc, openbox_proc, xcompmgr_proc = xvfb.start_xvfb(env=env, build_dir='.') assert xvfb_proc and openbox_proc and xcompmgr_proc, 'Failed to start xvfb' try: valid = True rc = 0 try: executable = rest_args[0] if IsWindows(): executable = '.\%s.exe' % executable else: executable = './%s' % executable with common.temporary_file() as tempfile_path: valid = (common.run_command_with_output([executable], env=env, stdoutfile=tempfile_path) == 0) # Now get the correct json format from the stdout to write to the # perf results file results_processor = ( generate_legacy_perf_dashboard_json.LegacyResultsProcessor()) charts = results_processor.GenerateJsonResults(tempfile_path) # Write the returned encoded json to a the charts output file with open(args.isolated_script_test_chartjson_output, 'w') as f: f.write(charts) except Exception: traceback.print_exc() valid = False failures = [] if valid else ['(entire test suite)'] with open(args.isolated_script_test_output, 'w') as fp: json.dump({ 'valid': valid, 'failures': failures, }, fp) return rc finally: xvfb.kill(xvfb_proc) xvfb.kill(openbox_proc) xvfb.kill(xcompmgr_proc)