def main_run(args):
filter_tests = []
if args.filter_file:
filter_tests = json.load(args.filter_file)
test_args = ['--retry-limit', '3']
if 'android' == args.properties.get('target_platform'):
test_args += ['--browser', 'android-chromium', '--device', 'android']
else:
test_args += ['--browser', args.build_config_fs.lower()]
with common.temporary_file() as tempfile_path:
test_args += ['--write-full-results-to', tempfile_path]
rc = common.run_runtest(args, [
'--test-type', 'telemetry_perf_unittests', '--run-python-script',
os.path.join(common.SRC_DIR, 'tools', 'perf', 'run_tests')
] + test_args + filter_tests)
with open(tempfile_path) as f:
results = json.load(f)
parsed_results = common.parse_common_test_results(results,
test_separator='.')
failures = parsed_results['unexpected_failures']
json.dump(
{
'valid':
bool(rc <= common.MAX_FAILURES_EXIT_STATUS and
((rc == 0) or failures)),
'failures':
failures.keys(),
}, args.output)
return rc
def main_run(args):
filter_tests = []
if args.filter_file:
filter_tests = json.load(args.filter_file)
with common.temporary_file() as tempfile_path:
rc = common.run_runtest(args, [
'--annotate', 'gtest',
'--test-type', 'telemetry_unittests',
'--run-python-script',
os.path.join(common.SRC_DIR, 'tools', 'telemetry', 'run_tests'),
'--browser', args.build_config_fs.lower(),
'--retry-limit', '3',
'--write-full-results-to', tempfile_path,
] + filter_tests)
with open(tempfile_path) as f:
results = json.load(f)
parsed_results = common.parse_common_test_results(results, test_separator='.')
failures = parsed_results['unexpected_failures']
json.dump({
'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and
((rc == 0) or failures)),
'failures': failures.keys(),
}, args.output)
return rc
def main_run(script_args):
parser = create_argparser()
args, sizes_args = parser.parse_known_args(script_args.args)
runtest_args = [
'--test-type',
'sizes',
'--run-python-script',
]
if args.perf_id:
runtest_args.extend([
'--perf-id',
args.perf_id,
'--results-url=%s' % args.results_url,
'--perf-dashboard-id=sizes',
'--annotate=graphing',
])
sizes_cmd = [
os.path.join(common.SRC_DIR, 'infra', 'scripts', 'legacy', 'scripts',
'slave', 'chromium', 'sizes.py')
]
sizes_cmd.extend(sizes_args)
rc = common.run_runtest(script_args, runtest_args + sizes_cmd)
json.dump({
'valid': rc == 0,
'failures': [],
}, script_args.output)
return rc
def main_run(args):
filter_tests = []
if args.filter_file:
filter_tests = json.load(args.filter_file)
perf_id = args.properties.get('perf-id')
script_args = args.args
test_suite = script_args[0]
if IsWindows():
script_args[0] += '.exe'
with common.temporary_file() as tempfile_path:
gtest_args = [
'--target',
args.build_config_fs,
'--annotate',
'graphing',
'--perf-id',
perf_id,
'--perf-dashboard-id',
test_suite,
'--results-url',
args.properties.get('results-url'),
'--slave-name',
args.properties.get('slavename'),
'--builder-name',
args.properties.get('buildername'),
'--build-number',
str(args.properties.get('buildnumber')),
'--log-processor-output-file',
tempfile_path,
'--test-type',
test_suite,
]
if 'android' == args.properties.get('target_platform'):
gtest_args.extend([
'--no-xvfb',
'--run-python-script',
os.path.join(args.paths['checkout'], 'out',
args.build_config_fs, 'bin',
'run_%s' % test_suite),
'--verbose',
])
gtest_args.extend(script_args[1:])
else:
gtest_args.extend(['--xvfb'])
gtest_args.extend(script_args)
rc = common.run_runtest(args, gtest_args + filter_tests)
with open(tempfile_path) as f:
results = json.load(f)
json.dump({
'valid': bool(rc == 0),
'failures': results['failed'],
}, args.output)
return rc
def main_run(args):
filter_tests = []
if args.filter_file:
filter_tests = json.load(args.filter_file)
test_args = ['--retry-limit', '3']
if 'android' == args.properties.get('target_platform'):
test_args += ['--browser', 'android-chrome-shell', '--device', 'android']
else:
test_args += ['--browser', args.build_config_fs.lower()]
with common.temporary_file() as tempfile_path:
test_args += ['--write-full-results-to', tempfile_path]
rc = common.run_runtest(args, [
'--test-type', 'telemetry_perf_unittests',
'--run-python-script',
os.path.join(common.SRC_DIR, 'tools', 'perf', 'run_tests')
] + test_args + filter_tests)
with open(tempfile_path) as f:
results = json.load(f)
parsed_results = common.parse_common_test_results(results, test_separator='.')
failures = parsed_results['unexpected_failures']
json.dump({
'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and
((rc == 0) or failures)),
'failures': failures.keys(),
}, args.output)
return rc
def main_run(args):
filter_tests = []
if args.filter_file:
filter_tests = json.load(args.filter_file)
perf_id = args.properties.get("perf-id")
script_args = args.args
if IsWindows():
script_args[0] += ".exe"
test_suite = script_args[0]
with common.temporary_file() as tempfile_path:
gtest_args = [
"--target",
args.build_config_fs,
"--annotate",
"graphing",
"--perf-id",
perf_id,
"--perf-dashboard-id",
test_suite,
"--results-url",
args.properties.get("results-url"),
"--slave-name",
args.properties.get("slavename"),
"--builder-name",
args.properties.get("buildername"),
"--build-number",
str(args.properties.get("buildnumber")),
"--log-processor-output-file",
tempfile_path,
"--test-type",
test_suite,
]
if "android" == args.properties.get("target_platform"):
gtest_args.extend(
[
"--no-xvfb",
"--run-python-script",
os.path.join(args.paths["checkout"], "build", "android", "test_runner.py"),
"gtest",
"--release",
"--suite",
test_suite,
"--verbose",
]
)
else:
gtest_args.extend(["--xvfb"])
gtest_args.extend(script_args)
rc = common.run_runtest(args, gtest_args + filter_tests)
with open(tempfile_path) as f:
results = json.load(f)
json.dump({"valid": bool(rc == 0), "failures": results["failed"]}, args.output)
return rc
def main_run(args):
with common.temporary_file() as tempfile_path:
rc = common.run_runtest(args, [
'--test-type', 'sizes', '--run-python-script',
os.path.join(common.SRC_DIR, 'infra', 'scripts', 'legacy',
'scripts', 'slave', 'chromium', 'sizes.py'), '--json',
tempfile_path
])
with open(tempfile_path) as f:
results = json.load(f)
with open(
os.path.join(common.SRC_DIR, 'tools', 'perf_expectations',
'perf_expectations.json')) as f:
perf_expectations = json.load(f)
prefix = args.args[0]
valid = (rc == 0)
failures = []
for name, result in results.iteritems():
fqtn = '%s/%s/%s' % (prefix, name, result['identifier'])
if fqtn not in perf_expectations:
continue
if perf_expectations[fqtn]['type'] != 'absolute':
print 'ERROR: perf expectation %r is not yet supported' % fqtn
valid = False
continue
actual = result['value']
expected = perf_expectations[fqtn]['regress']
better = perf_expectations[fqtn]['better']
check_result = ((actual <= expected) if better == 'lower' else
(actual >= expected))
if not check_result:
failures.append(fqtn)
print 'FAILED %s: actual %s, expected %s, better %s' % (
fqtn, actual, expected, better)
json.dump({
'valid': valid,
'failures': failures,
}, args.output)
# sizes.py itself doesn't fail on regressions.
if failures and rc == 0:
rc = 1
return rc
def main_run(args):
with common.temporary_file() as tempfile_path:
rc = common.run_runtest(args, [
'--test-type', 'sizes',
'--run-python-script',
os.path.join(
common.SRC_DIR, 'infra', 'scripts', 'legacy', 'scripts', 'slave',
'chromium', 'sizes.py'),
'--json', tempfile_path])
with open(tempfile_path) as f:
results = json.load(f)
with open(os.path.join(common.SRC_DIR, 'tools', 'perf_expectations',
'perf_expectations.json')) as f:
perf_expectations = json.load(f)
prefix = args.args[0]
valid = (rc == 0)
failures = []
for name, result in results.iteritems():
fqtn = '%s/%s/%s' % (prefix, name, result['identifier'])
if fqtn not in perf_expectations:
continue
if perf_expectations[fqtn]['type'] != 'absolute':
print 'ERROR: perf expectation %r is not yet supported' % fqtn
valid = False
continue
actual = result['value']
expected = perf_expectations[fqtn]['regress']
better = perf_expectations[fqtn]['better']
check_result = ((actual <= expected) if better == 'lower'
else (actual >= expected))
if not check_result:
failures.append(fqtn)
print 'FAILED %s: actual %s, expected %s, better %s' % (
fqtn, actual, expected, better)
json.dump({
'valid': valid,
'failures': failures,
}, args.output)
# sizes.py itself doesn't fail on regressions.
if failures and rc == 0:
rc = 1
return rc
def main_run(args):
rc = common.run_runtest(args, [
os.path.join(common.SRC_DIR, 'tools', 'valgrind', 'chrome_tests.sh'),
'--tool', 'memcheck',
'--build-dir', os.path.join(common.SRC_DIR, 'out', args.build_config_fs),
] + args.args)
json.dump({
'valid': True,
'failures': ['failed'] if rc else []
}, args.output)
return rc
def main_run(args):
rc = common.run_runtest(args, [
'--test-type', 'sizes', '--run-python-script',
os.path.join(common.SRC_DIR, 'infra', 'scripts', 'legacy', 'scripts',
'slave', 'chromium', 'sizes.py')
])
# TODO(phajdan.jr): Implement more granular failures.
json.dump({
'valid': True,
'failures': ['sizes_failed'] if rc else [],
}, args.output)
return rc
def main_run(args):
filter_tests = []
if args.filter_file:
filter_tests = json.load(args.filter_file)
perf_id = args.properties.get('perf-id')
script_args = args.args
test_suite = script_args[0]
if IsWindows():
script_args[0] += '.exe'
with common.temporary_file() as tempfile_path:
gtest_args = [
'--target', args.build_config_fs,
'--annotate', 'graphing',
'--perf-id', perf_id,
'--perf-dashboard-id', test_suite,
'--results-url', args.properties.get('results-url'),
'--slave-name', args.properties.get('slavename'),
'--builder-name', args.properties.get('buildername'),
'--build-number', str(args.properties.get('buildnumber')),
'--log-processor-output-file', tempfile_path,
'--test-type', test_suite,
]
if 'android' == args.properties.get('target_platform'):
gtest_args.extend([
'--no-xvfb',
'--run-python-script', os.path.join(
args.paths['checkout'], 'out', args.build_config_fs, 'bin',
'run_%s' % test_suite),
'--verbose',
])
gtest_args.extend(script_args[1:])
else:
gtest_args.extend(['--xvfb'])
gtest_args.extend(script_args)
rc = common.run_runtest(args, gtest_args + filter_tests)
with open(tempfile_path) as f:
results = json.load(f)
json.dump({
'valid': bool(rc == 0),
'failures': results['failed'],
}, args.output)
return rc
def main_run(args):
rc = common.run_runtest(args, [
os.path.join(common.SRC_DIR, 'tools', 'valgrind', 'chrome_tests.sh'),
'--tool',
'memcheck',
'--build-dir',
os.path.join(common.SRC_DIR, 'out', args.build_config_fs),
] + args.args)
json.dump({
'valid': True,
'failures': ['failed'] if rc else []
}, args.output)
return rc
def main_run(args):
rc = common.run_runtest(args, [
'--test-type', 'sizes',
'--run-python-script',
os.path.join(
common.SRC_DIR, 'infra', 'scripts', 'legacy', 'scripts', 'slave',
'chromium', 'sizes.py')])
# TODO(phajdan.jr): Implement more granular failures.
json.dump({
'valid': True,
'failures': ['sizes_failed'] if rc else [],
}, args.output)
return rc
def main_run(script_args):
parser = create_argparser()
parser.add_argument('prefix')
args = parser.parse_args(script_args.args)
with common.temporary_file() as tempfile_path:
runtest_args = [
'--test-type',
'sizes',
'--run-python-script',
]
if args.perf_id:
runtest_args.extend([
'--perf-id',
args.perf_id,
'--results-url=%s' % args.results_url,
'--perf-dashboard-id=sizes',
'--annotate=graphing',
])
sizes_cmd = [
os.path.join(common.SRC_DIR, 'infra', 'scripts', 'legacy',
'scripts', 'slave', 'chromium', 'sizes.py'),
'--failures', tempfile_path
]
if args.platform:
sizes_cmd.extend(['--platform', args.platform])
rc = common.run_runtest(script_args, runtest_args + sizes_cmd)
with open(tempfile_path) as f:
failures = json.load(f)
json.dump({
'valid': (rc == 0 or rc == 125),
'failures': failures,
}, script_args.output)
return rc
def main_run(script_args):
parser = create_argparser()
parser.add_argument('prefix')
args = parser.parse_args(script_args.args)
with common.temporary_file() as tempfile_path:
runtest_args = [
'--test-type',
'sizes',
'--run-python-script',
]
if args.perf_id:
runtest_args.extend([
'--perf-id',
args.perf_id,
'--results-url=%s' % args.results_url,
'--perf-dashboard-id=sizes',
'--annotate=graphing',
])
sizes_cmd = [
os.path.join(common.SRC_DIR, 'infra', 'scripts', 'legacy',
'scripts', 'slave', 'chromium', 'sizes.py'), '--json',
tempfile_path
]
if args.platform:
sizes_cmd.extend(['--platform', args.platform])
rc = common.run_runtest(script_args, runtest_args + sizes_cmd)
with open(tempfile_path) as f:
results = json.load(f)
with open(
os.path.join(common.SRC_DIR, 'tools', 'perf_expectations',
'perf_expectations.json')) as f:
perf_expectations = json.load(f)
valid = (rc == 0)
failures = []
for name, result in results.iteritems():
fqtn = '%s/%s/%s' % (args.prefix, name, result['identifier'])
if fqtn not in perf_expectations:
continue
if perf_expectations[fqtn]['type'] != 'absolute':
print 'ERROR: perf expectation %r is not yet supported' % fqtn
valid = False
continue
actual = result['value']
expected = perf_expectations[fqtn]['regress']
better = perf_expectations[fqtn]['better']
check_result = ((actual <= expected) if better == 'lower' else
(actual >= expected))
if not check_result:
failures.append(fqtn)
print 'FAILED %s: actual %s, expected %s, better %s' % (
fqtn, actual, expected, better)
json.dump({
'valid': valid,
'failures': failures,
}, script_args.output)
# sizes.py itself doesn't fail on regressions.
if failures and rc == 0:
rc = 1
return rc
