def run(tool,
args,
build=None,
debug=True,
profile=False,
track_memory_file=None,
extra_args=None,
stderr=None,
stdout=None,
return_stdout=False,
timeout=0,
quiet=False):
if build is None:
build, args = extract_build_from_args(args)
if build:
gradle.RunGradle(['r8lib' if tool.startswith('r8lib') else 'r8'])
cmd = []
if track_memory_file:
cmd.extend(['tools/track_memory.sh', track_memory_file])
cmd.append(jdk.GetJavaExecutable())
if extra_args:
cmd.extend(extra_args)
if debug:
cmd.append('-ea')
if profile:
cmd.append('-agentlib:hprof=cpu=samples,interval=1,depth=8')
if tool == 'r8lib-d8':
cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.D8'])
elif tool == 'r8lib-r8':
cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.R8'])
else:
cmd.extend(['-jar', utils.R8_JAR, tool])
lib, args = extract_lib_from_args(args)
if lib:
cmd.extend(["--lib", lib])
cmd.extend(args)
utils.PrintCmd(cmd, quiet=quiet)
if timeout > 0:
kill = lambda process: process.kill()
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
timer = Timer(timeout, kill, [proc])
try:
timer.start()
stdout, stderr = proc.communicate()
finally:
timer.cancel()
return stdout if return_stdout else proc.returncode
else:
if return_stdout:
return subprocess.check_output(cmd)
return subprocess.call(cmd, stdout=stdout, stderr=stderr)
def run_once(archive):
failed = False
git_hash = utils.get_HEAD_sha1()
log('Running once with hash %s' % git_hash)
env = os.environ.copy()
# Bot does not have a lot of memory.
env['R8_GRADLE_CORES_PER_FORK'] = '16'
failed = any([execute(cmd, archive, env) for cmd in TEST_COMMANDS])
# Gradle daemon occasionally leaks memory, stop it.
gradle.RunGradle('--stop')
archive_status(1 if failed else 0)
return failed
def Main():
gradle.RunGradle(BUILD_TARGETS)
utils.DownloadFromX20(
os.path.join(utils.THIRD_PARTY, 'gradle-plugin') + '.tar.gz.sha1')
utils.DownloadFromX20(
os.path.join(utils.THIRD_PARTY, 'benchmarks', 'android-sdk') +
'.tar.gz.sha1')
utils.DownloadFromGoogleCloudStorage(utils.SAMPLE_LIBRARIES_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.OPENSOURCE_APPS_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.ANDROID_SDK + '.tar.gz.sha1',
bucket='r8-deps-internal',
auth=True)
def build_r8lib(target, exclude_deps, no_relocate, keep_rules_path,
output_path, **kwargs):
# Clean the build directory to ensure no repackaging of any existing
# lib or deps.
gradle.RunGradle(['clean'])
lib_args = [target]
deps_args = ['repackageDeps']
if exclude_deps:
lib_args.append('-Pexclude_deps')
if no_relocate:
lib_args.append('-Plib_no_relocate')
deps_args.append('-Plib_no_relocate')
# Produce the r8lib target to be processed later.
gradle.RunGradle(lib_args)
target_lib = os.path.join(utils.LIBS, target + '.jar')
temp_lib = os.path.join(utils.LIBS, target + '_to_process.jar')
os.rename(target_lib, temp_lib)
# Produce the dependencies needed for running r8 on lib.jar.
gradle.RunGradle(deps_args)
temp_deps = os.path.join(utils.LIBS, target + 'lib_deps.jar')
os.rename(DEPS_JAR, temp_deps)
# Produce R8 for compiling lib
if output_path is None:
output_path = target + 'lib.jar'
output_map_path = output_path + '.map'
toolhelper.run(
'r8',
('--release',
'--classfile',
'--lib', utils.RT_JAR,
'--lib', temp_deps,
temp_lib,
'--output', output_path,
'--pg-conf', keep_rules_path,
'--pg-map-output', output_map_path),
**kwargs)
if exclude_deps:
return [output_path, temp_deps]
else:
return [output_path]
def main():
args = sys.argv[1:]
if len(args) == 0:
gradle.RunGradle(['r8lib'])
r8lib = utils.R8LIB_JAR
r8map = utils.R8LIB + '.map'
elif len(args) == 2:
r8lib = args[0]
r8map = args[1]
elif len(args) == 1 and args[0] == '--help':
print(
'Usage: test_self_retrace.py [<path-to-r8lib-jar> <path-to-r8lib-map]'
)
print('If the path is missing the script builds and uses ' +
utils.R8LIB_JAR)
return
else:
raise Exception("Only two argument allowed, see '--help'.")
# Run 'r8 --help' which throws an exception.
cmd = [
jdk.GetJavaExecutable(), '-cp', r8lib, 'com.android.tools.r8.R8',
'--help'
]
os.environ["R8_THROW_EXCEPTION_FOR_TESTING_RETRACE"] = "1"
utils.PrintCmd(cmd)
p = subprocess.Popen(cmd, stderr=subprocess.PIPE)
_, stacktrace = p.communicate()
assert (p.returncode != 0)
assert (EXCEPTION_LINE in stacktrace)
# r8lib must be minified, original class names must not be present.
assert ('SelfRetraceTest' not in stacktrace)
# Run the retrace tool.
cmd = [jdk.GetJavaExecutable(), '-jar', utils.RETRACE_JAR, r8map]
utils.PrintCmd(cmd)
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
retrace_stdout, _ = p.communicate(stacktrace)
assert p.returncode == 0
retrace_lines = retrace_stdout.splitlines()
line_index = -1
for line in retrace_lines:
if line_index < 0:
if 'java.lang.RuntimeException' in line:
assert (EXCEPTION_LINE in line)
line_index = 0
else:
assert EXPECTED_LINES[line_index] in line
line_index += 1
if line_index >= len(EXPECTED_LINES):
break
assert (line_index >= 0)
def run(args, build=True):
if build:
gradle.RunGradle(['copyMavenDeps'])
cmd = []
cmd.append('java')
cp = ":".join([os.path.join(utils.REPO_ROOT, 'build/deps/asm-6.2.1.jar'),
os.path.join(utils.REPO_ROOT, 'build/deps/asm-util-6.2.1.jar')])
cmd.extend(['-cp', cp])
cmd.append('org.objectweb.asm.util.ASMifier')
cmd.extend(args)
utils.PrintCmd(cmd)
result = subprocess.check_output(cmd)
print(result)
return result
def run(args, build=True, debug=True, profile=False, track_memory_file=None):
if build:
gradle.RunGradle(['CompatDX'])
cmd = []
if track_memory_file:
cmd.extend(['tools/track_memory.sh', track_memory_file])
cmd.append('java')
if debug:
cmd.append('-ea')
if profile:
cmd.append('-agentlib:hprof=cpu=samples,interval=1,depth=8')
cmd.extend(['-jar', COMPATDX_JAR])
cmd.extend(args)
subprocess.check_call(cmd)
def run(args, build=True):
if build:
gradle.RunGradle(['copyMavenDeps'])
cmd = []
cmd.append(jdk.GetJavaExecutable())
cp = ":".join([
os.path.join(utils.REPO_ROOT, 'build/deps/' + ASM_JAR),
os.path.join(utils.REPO_ROOT, 'build/deps/' + ASM_UTIL_JAR)
])
cmd.extend(['-cp', cp])
cmd.append('org.objectweb.asm.util.ASMifier')
cmd.extend(args)
utils.PrintCmd(cmd)
result = subprocess.check_output(cmd)
print(result)
return result
def Main():
args = parse_arguments()
if not args.no_build:
gradle.RunGradle(['CompatDx'])
cmd = [REPLAY_SCRIPT, 'java', '-jar', utils.COMPATDX_JAR]
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
# collect dex files below OUT_DIR
dex_files = (chain.from_iterable(
glob(join(x[0], '*.dex')) for x in os.walk(OUT_DIR)))
for dex_file in dex_files:
dex2oat.run(dex_file)
def generate_r8_maven_zip(out, is_r8lib=False):
# Build the R8 no deps artifact.
if not is_r8lib:
gradle.RunGradleExcludeDeps([utils.R8])
else:
gradle.RunGradle([utils.R8LIB, '-Pno_internal'])
version = determine_version()
# Generate the pom file.
pom_file = 'r8.pom'
write_pom_file(R8_POMTEMPLATE, pom_file, version,
"" if is_r8lib else generate_dependencies(),
generate_library_licenses() if is_r8lib else "")
# Write the maven zip file.
generate_maven_zip('r8', version, pom_file,
utils.R8LIB_JAR if is_r8lib else utils.R8_JAR, out)
def Main():
(options, args) = ParseOptions()
gradle_args = ['cleanTest', 'test']
if len(args) > 1:
print("test.py takes at most one argument, the pattern for tests to run")
return -1
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.jctf:
gradle_args.append('-Pjctf')
if options.only_jctf:
gradle_args.append('-Ponly_jctf')
if options.jctf_compile_only:
gradle_args.append('-Pjctf_compile_only')
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if len(args) > 0:
gradle_args.append('--tests')
gradle_args.append(args[0])
if os.name == 'nt':
# temporary hack
gradle_args.append('-Pno_internal')
gradle_args.append('-x')
gradle_args.append('createJctfTests')
gradle_args.append('-x')
gradle_args.append('jctfCommonJar')
gradle_args.append('-x')
gradle_args.append('jctfTestsClasses')
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
for art_vm in vms_to_test:
return_code = gradle.RunGradle(gradle_args + ['-Pdex_vm=%s' % art_vm],
throw_on_failure=False)
if return_code != 0:
if options.archive_failures:
archive_failures()
return return_code
def run(args, build=True, debug=True, profile=False, track_memory_file=None):
if build:
gradle.RunGradle(['D8'])
cmd = []
if track_memory_file:
cmd.extend(['tools/track_memory.sh', track_memory_file])
cmd.append('java')
if debug:
cmd.append('-ea')
if profile:
cmd.append('-agentlib:hprof=cpu=samples,interval=1,depth=8')
cmd.extend(['-jar', utils.D8_JAR])
cmd.extend(args)
utils.PrintCmd(cmd)
result = subprocess.check_output(cmd)
print(result)
return result
def Main():
args = parse_arguments()
target_root = args.android_root[0]
if args.commit_hash == None and args.version == None:
gradle.RunGradle(JAR_TARGETS)
copy_jar_targets(utils.LIBS, target_root)
copy_other_targets(utils.GENERATED_LICENSE_DIR, target_root)
else:
assert args.commit_hash == None or args.version == None
targets = map((lambda t: t + '.jar'), JAR_TARGETS) + OTHER_TARGETS
with utils.TempDir() as root:
for target in targets:
if args.commit_hash:
download_hash(root, args.commit_hash, target)
else:
assert args.version
download_version(root, args.version, target)
copy_jar_targets(root, target_root)
copy_other_targets(root, target_root)
def run_in_tests(tool, args, build=None, debug=True, extra_args=None):
if build is None:
build, args = extract_build_from_args(args)
if build:
gradle.RunGradle([
'copyMavenDeps',
'compileTestJava',
])
cmd = []
cmd.append(jdk.GetJavaExecutable())
if extra_args:
cmd.extend(extra_args)
if debug:
cmd.append('-ea')
cmd.extend(['-cp', ':'.join([
utils.BUILD_MAIN_DIR,
utils.BUILD_TEST_DIR,
] + glob.glob('%s/*.jar' % utils.BUILD_DEPS_DIR))])
cmd.extend([tool])
cmd.extend(args)
utils.PrintCmd(cmd)
return subprocess.call(cmd)
def main(argv):
global SHRINKERS
(options, args) = ParseOptions(argv)
with utils.TempDir() as temp_dir:
if options.disable_tot:
# Cannot run r8 lib without adding r8lib.jar as an dependency
SHRINKERS = [
shrinker for shrinker in SHRINKERS
if 'minified' not in shrinker
]
else:
if not options.no_build:
gradle.RunGradle(['r8', 'r8lib'])
assert os.path.isfile(
utils.R8_JAR), ('Cannot build from ToT without r8.jar')
assert os.path.isfile(
utils.R8LIB_JAR), ('Cannot build from ToT without r8lib.jar')
# Make a copy of r8.jar and r8lib.jar such that they stay the same for
# the entire execution of this script.
shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
shutil.copyfile(utils.R8LIB_JAR,
os.path.join(temp_dir, 'r8lib.jar'))
result_per_shrinker_per_app = {}
if options.app:
result_per_shrinker_per_app[options.app] = GetResultsForApp(
options.app, APPS.get(options.app), options, temp_dir)
else:
for app, config in sorted(APPS.iteritems()):
if not config.get('skip', False):
result_per_shrinker_per_app[app] = GetResultsForApp(
app, config, options, temp_dir)
LogResultsForApps(result_per_shrinker_per_app, options)
def run(tool,
args,
build=None,
debug=True,
profile=False,
track_memory_file=None,
extra_args=None,
stderr=None,
stdout=None,
return_stdout=False):
if build is None:
build, args = extract_build_from_args(args)
if build:
gradle.RunGradle(['r8lib' if tool.startswith('r8lib') else 'r8'])
cmd = []
if track_memory_file:
cmd.extend(['tools/track_memory.sh', track_memory_file])
cmd.append('java')
if extra_args:
cmd.extend(extra_args)
if debug:
cmd.append('-ea')
if profile:
cmd.append('-agentlib:hprof=cpu=samples,interval=1,depth=8')
if tool == 'r8lib-d8':
cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.D8'])
elif tool == 'r8lib-r8':
cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.R8'])
else:
cmd.extend(['-jar', utils.R8_JAR, tool])
lib, args = extract_lib_from_args(args)
if lib:
cmd.extend(["--lib", lib])
cmd.extend(args)
utils.PrintCmd(cmd)
if return_stdout:
return subprocess.check_output(cmd)
return subprocess.call(cmd, stdout=stdout, stderr=stderr)
def Main():
gradle.RunGradle(BUILD_TARGETS)
utils.DownloadFromX20(utils.OPENSOURCE_APPS_SHA_FILE)
utils.DownloadFromX20(utils.ANDROID_SDK + '.tar.gz.sha1')
def Main():
(options, args) = ParseOptions()
if 'BUILDBOT_BUILDERNAME' in os.environ:
gradle.RunGradle(['clean'])
# Build R8lib with dependencies for bootstrapping tests before adding test sources
gradle.RunGradle(['r8libwithdeps'])
gradle_args = ['--stacktrace']
# Set all necessary Gradle properties and options first.
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.jctf:
gradle_args.append('-Pjctf')
if options.only_jctf:
gradle_args.append('-Ponly_jctf')
if options.jctf_compile_only:
gradle_args.append('-Pjctf_compile_only')
if options.aosp_jar:
gradle_args.append('-Paosp_jar')
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if options.with_code_coverage:
gradle_args.append('-Pwith_code_coverage')
if os.name == 'nt':
# temporary hack
gradle_args.append('-Pno_internal')
gradle_args.append('-x')
gradle_args.append('createJctfTests')
gradle_args.append('-x')
gradle_args.append('jctfCommonJar')
gradle_args.append('-x')
gradle_args.append('jctfTestsClasses')
if options.test_dir:
gradle_args.append('-Ptest_dir=' + options.test_dir)
if not os.path.exists(options.test_dir):
os.makedirs(options.test_dir)
if options.java_home:
gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
if options.generate_golden_files_to:
gradle_args.append('-Pgenerate_golden_files_to=' +
options.generate_golden_files_to)
if not os.path.exists(options.generate_golden_files_to):
os.makedirs(options.generate_golden_files_to)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.use_golden_files_in:
gradle_args.append('-Puse_golden_files_in=' +
options.use_golden_files_in)
if not os.path.exists(options.use_golden_files_in):
os.makedirs(options.use_golden_files_in)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
# Add Gradle tasks
gradle_args.append('cleanTest')
gradle_args.append('test')
# Test filtering. Must always follow the 'test' task.
for testFilter in args:
gradle_args.append('--tests')
gradle_args.append(testFilter)
if options.with_code_coverage:
# Create Jacoco report after tests.
gradle_args.append('jacocoTestReport')
if options.use_golden_files_in:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.use_golden_files_in):
utils.download_file_from_cloud_storage(
'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
# Now run tests on selected runtime(s).
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
for art_vm in vms_to_test:
vm_kind_to_test = "_" + options.dex_vm_kind if art_vm != "default" else ""
return_code = gradle.RunGradle(
gradle_args + ['-Pdex_vm=%s' % (art_vm + vm_kind_to_test)],
throw_on_failure=False)
if options.generate_golden_files_to:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(
options.generate_golden_files_to):
archive = utils.create_archive(sha1)
utils.upload_file_to_cloud_storage(
archive, 'gs://r8-test-results/golden-files/' + archive)
if return_code != 0:
if options.archive_failures and os.name != 'nt':
archive_failures()
return return_code
return 0
def Main():
args = parse_arguments()
if args.d8log and args.tool != 'd8':
print("The '--d8log' option works only with '--tool=d8'.",
file=sys.stderr)
return EXIT_FAILURE
assert args.tool in ['jack', 'dx', 'd8']
jack_option = 'ANDROID_COMPILE_WITH_JACK=' \
+ ('true' if args.tool == 'jack' else 'false')
# DX_ALT_JAR need to be cleared if not set, for 'make' to work properly
alt_jar_option = 'DX_ALT_JAR='
if args.tool == 'd8':
if args.d8log:
alt_jar_option += D8LOGGER_JAR
os.environ['D8LOGGER_OUTPUT'] = args.d8log
else:
alt_jar_option += utils.COMPATDX_JAR
gradle.RunGradle(['d8', 'd8logger', 'compatdx'])
setup_and_clean(args.tool == 'd8', args.clean_dex)
checkout_aosp.checkout_aosp(AOSP_ROOT, AOSP_MANIFEST_XML, J_DEFAULT)
# activate OUT_CTS and build Android CTS
# AOSP has no clean way to set the output directory.
# In order to do incremental builds we apply the following symlink-based
# workaround.
# Note: this does not work on windows, but the AOSP
# doesn't build, either
if not remove_aosp_out():
return EXIT_FAILURE
print("-- Building CTS with 'make {} cts'.".format(J_OPTION))
os.symlink(OUT_CTS, AOSP_OUT)
check_call([AOSP_HELPER_SH, AOSP_PRESET, 'make', J_OPTION, 'cts'],
cwd=AOSP_ROOT)
# activate OUT_IMG and build the Android image
if not remove_aosp_out():
return EXIT_FAILURE
print("-- Building Android image with 'make {} {} {}'." \
.format(J_OPTION, jack_option, alt_jar_option))
os.symlink(OUT_IMG, AOSP_OUT)
check_call([
AOSP_HELPER_SH, AOSP_PRESET, 'make', J_OPTION, jack_option,
alt_jar_option
],
cwd=AOSP_ROOT)
emulator_proc = Popen([
AOSP_HELPER_SH, AOSP_PRESET, 'emulator', '-partition-size', '4096',
'-wipe-data'
],
cwd=AOSP_ROOT)
if emulator_proc.poll() is not None:
print("Can't start Android Emulator.", file=sys.stderr)
check_call(
[AOSP_HELPER_SH, AOSP_PRESET, 'run-cts', CTS_TRADEFED, 'run', 'cts'],
cwd=AOSP_ROOT)
emulator_proc.terminate()
time.sleep(6) # aosp_helper waits to be killed in looping 'sleep 5'
# find the newest test_result.xml
result_dirs = \
[f for f in glob(join(RESULTS_DIR_BASE, '*')) if os.path.isdir(f)]
if len(result_dirs) == 0:
print("Can't find result directories in ", RESULTS_DIR_BASE)
return EXIT_FAILURE
result_dirs.sort(key=os.path.getmtime)
results_xml = join(result_dirs[-1], 'test_result.xml')
# print summaries
re_summary = re.compile('<Summary ')
summaries = [('Summary from current test results: ', results_xml)]
for (title, result_file) in summaries:
print(title, result_file)
with open(result_file) as f:
for line in f:
if re_summary.search(line):
print(line)
break
if args.no_baseline:
r = 0
else:
print('Comparing test results to baseline:\n')
passing_tests = consistently_passing_tests_from_test_results(
[results_xml])
baseline_results = glob(join(CTS_BASELINE_FILES_DIR, '*.xml'))
assert len(baseline_results) != 0
passing_tests_in_baseline = \
consistently_passing_tests_from_test_results(baseline_results)
missing_or_failing_tests = \
set(passing_tests_in_baseline) - set(passing_tests)
num_tests = len(missing_or_failing_tests)
if num_tests != 0:
if num_tests > 1:
text = '{} tests that consistently pass in the baseline' \
' are missing or failing in the current test:'.format(num_tests)
else:
text = '1 test that consistently passes in the baseline' \
' is missing or failing in the current test:'
print(text)
for t in missing_or_failing_tests:
print(t)
r = EXIT_FAILURE
else:
r = 0
if args.save_result:
copy2(results_xml, args.save_result)
return r
def Main():
(options, args) = ParseOptions()
gradle_args = []
# Set all necessary Gradle properties and options first.
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.jctf:
gradle_args.append('-Pjctf')
if options.only_jctf:
gradle_args.append('-Ponly_jctf')
if options.jctf_compile_only:
gradle_args.append('-Pjctf_compile_only')
if options.aosp_jar:
gradle_args.append('-Paosp_jar')
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if options.with_code_coverage:
gradle_args.append('-Pwith_code_coverage')
if os.name == 'nt':
# temporary hack
gradle_args.append('-Pno_internal')
gradle_args.append('-x')
gradle_args.append('createJctfTests')
gradle_args.append('-x')
gradle_args.append('jctfCommonJar')
gradle_args.append('-x')
gradle_args.append('jctfTestsClasses')
if options.test_dir:
gradle_args.append('-Ptest_dir=' + options.test_dir)
if not os.path.exists(options.test_dir):
os.makedirs(options.test_dir)
# Add Gradle tasks
gradle_args.append('cleanTest')
gradle_args.append('test')
# Test filtering. Must always follow the 'test' task.
for testFilter in args:
gradle_args.append('--tests')
gradle_args.append(testFilter)
if options.with_code_coverage:
# Create Jacoco report after tests.
gradle_args.append('jacocoTestReport')
# Now run tests on selected runtime(s).
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
for art_vm in vms_to_test:
return_code = gradle.RunGradle(gradle_args + ['-Pdex_vm=%s' % art_vm],
throw_on_failure=False)
if return_code != 0:
if options.archive_failures and os.name != 'nt':
archive_failures()
return return_code
return 0
def main(argv):
(options, args) = ParseOptions(argv)
if options.golem:
golem.link_third_party()
if os.path.exists(WORKING_DIR):
shutil.rmtree(WORKING_DIR)
shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR)
os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join(
utils.ANDROID_SDK)
os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3'
options.disable_assertions = True
options.ignore_versions = True
options.no_build = True
options.r8_compilation_steps = 1
options.quiet = True
options.gradle_pre_runs = 2
options.use_daemon = True
if not os.path.exists(WORKING_DIR):
os.makedirs(WORKING_DIR)
if options.download_only:
clone_repositories(options.quiet)
return
with utils.TempDir() as temp_dir:
if not (options.no_build or options.golem):
gradle.RunGradle(['r8', 'r8lib'])
if options.version:
# Download r8-<version>.jar from
# http://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.version)
update_prebuilds_in_android.download_version(
temp_dir, 'com/android/tools/r8/' + options.version, target)
as_utils.MoveFile(os.path.join(temp_dir, target),
os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
else:
# Make a copy of r8.jar and r8lib.jar such that they stay the same for
# the entire execution of this script.
if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker:
assert os.path.isfile(
utils.R8_JAR), 'Cannot build without r8.jar'
shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
if 'r8' in options.shrinker or 'r8-full' in options.shrinker:
assert os.path.isfile(
utils.R8LIB_JAR), 'Cannot build without r8lib.jar'
shutil.copyfile(utils.R8LIB_JAR,
os.path.join(temp_dir, 'r8lib.jar'))
result_per_shrinker_per_app = []
for (app, repo) in options.apps:
if app.skip:
continue
result_per_shrinker_per_app.append(
(app, GetResultsForApp(app, repo, options, temp_dir)))
LogResultsForApps(result_per_shrinker_per_app, options)
def Main():
(options, args) = ParseOptions()
if utils.is_bot():
gradle.RunGradle(['--no-daemon', 'clean'])
gradle_args = ['--stacktrace']
if utils.is_bot():
# Bots don't like dangling processes
gradle_args.append('--no-daemon')
# Set all necessary Gradle properties and options first.
if options.shard_count:
assert options.shard_number
gradle_args.append('-Pshard_count=%s' % options.shard_count)
gradle_args.append('-Pshard_number=%s' % options.shard_number)
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.jctf:
gradle_args.append('-Pjctf')
if options.only_jctf:
gradle_args.append('-Ponly_jctf')
if options.jctf_compile_only:
gradle_args.append('-Pjctf_compile_only')
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if options.with_code_coverage:
gradle_args.append('-Pwith_code_coverage')
if os.name == 'nt':
# temporary hack
gradle_args.append('-Pno_internal')
gradle_args.append('-x')
gradle_args.append('createJctfTests')
gradle_args.append('-x')
gradle_args.append('jctfCommonJar')
gradle_args.append('-x')
gradle_args.append('jctfTestsClasses')
if options.test_dir:
gradle_args.append('-Ptest_dir=' + options.test_dir)
if not os.path.exists(options.test_dir):
os.makedirs(options.test_dir)
if options.java_home:
gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
if options.java_max_memory_size:
gradle_args.append('-Dorg.gradle.jvmargs=-Xmx' +
options.java_max_memory_size)
if options.generate_golden_files_to:
gradle_args.append('-Pgenerate_golden_files_to=' +
options.generate_golden_files_to)
if not os.path.exists(options.generate_golden_files_to):
os.makedirs(options.generate_golden_files_to)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.use_golden_files_in:
gradle_args.append('-Puse_golden_files_in=' +
options.use_golden_files_in)
if not os.path.exists(options.use_golden_files_in):
os.makedirs(options.use_golden_files_in)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.alternative_frontend:
gradle_args.append('-Palternative_frontend')
if (not options.no_r8lib) and options.r8lib_no_deps:
print(
'Cannot run tests on r8lib with and without deps. R8lib is now default target.'
)
exit(1)
if not options.no_r8lib:
gradle_args.append('-Pr8lib')
# Force gradle to build a version of r8lib without dependencies for
# BootstrapCurrentEqualityTest.
gradle_args.append('R8LibNoDeps')
if options.r8lib_no_deps:
gradle_args.append('-Pr8lib_no_deps')
# Build an R8 with dependencies for bootstrapping tests before adding test sources.
gradle_args.append('r8WithRelocatedDeps')
gradle_args.append('r8WithRelocatedDeps11')
# Add Gradle tasks
gradle_args.append('cleanTest')
gradle_args.append('test')
if options.fail_fast:
gradle_args.append('--fail-fast')
if options.failed:
args = compute_failed_tests(args)
if args is None:
return 1
if len(args) == 0:
print "No failing tests"
return 0
# Test filtering. Must always follow the 'test' task.
for testFilter in args:
gradle_args.append('--tests')
gradle_args.append(testFilter)
if options.with_code_coverage:
# Create Jacoco report after tests.
gradle_args.append('jacocoTestReport')
if options.use_golden_files_in:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.use_golden_files_in):
utils.download_file_from_cloud_storage(
'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
if utils.is_bot() and not utils.IsWindows():
timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
if os.path.exists(timestamp_file):
os.remove(timestamp_file)
gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
thread.start_new_thread(timeout_handler, (timestamp_file, ))
rotate_test_reports()
if options.only_jctf:
# Note: not setting -Pruntimes will run with all available runtimes.
return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
return archive_and_return(return_code, options)
# Now run tests on selected runtime(s).
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
for art_vm in vms_to_test:
vm_suffix = "_" + options.dex_vm_kind if art_vm != "default" else ""
runtimes = ['dex-' + art_vm]
# Only append the "none" runtime and JVMs if running on the "default" DEX VM.
if art_vm == "default":
runtimes.extend(['jdk8', 'jdk9', 'jdk11', 'none'])
return_code = gradle.RunGradle(gradle_args + [
'-Pdex_vm=%s' % art_vm + vm_suffix,
'-Pruntimes=%s' % ':'.join(runtimes),
],
throw_on_failure=False)
if options.generate_golden_files_to:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(
options.generate_golden_files_to):
archive = utils.create_archive(sha1)
utils.upload_file_to_cloud_storage(
archive, 'gs://r8-test-results/golden-files/' + archive)
if return_code != 0:
return archive_and_return(return_code, options)
return 0
def main(argv):
(options, args) = ParseOptions(argv)
if options.bot:
utils.DownloadFromGoogleCloudStorage(utils.OPENSOURCE_APPS_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.ANDROID_SDK + '.tar.gz.sha1',
bucket='r8-deps-internal',
auth=True)
if os.path.exists(WORKING_DIR):
shutil.rmtree(WORKING_DIR)
shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR)
os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join(
utils.ANDROID_SDK)
os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3'
# TODO(b/141081520): Set to True once fixed.
options.no_logging = False
# TODO(b/141081520): Remove logging filter once fixed.
options.app_logging_filter = ['sqldelight']
options.shrinker = ['r8', 'r8-full']
print(options.shrinker)
if options.golem:
golem.link_third_party()
if os.path.exists(WORKING_DIR):
shutil.rmtree(WORKING_DIR)
shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR)
os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join(
utils.ANDROID_SDK)
os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3'
options.disable_assertions = True
options.ignore_versions = True
options.no_build = True
options.r8_compilation_steps = 1
options.quiet = True
options.gradle_pre_runs = 2
options.use_daemon = True
options.no_logging = True
if not os.path.exists(WORKING_DIR):
os.makedirs(WORKING_DIR)
if options.download_only:
clone_repositories(options.quiet)
return
with utils.TempDir() as temp_dir:
if not (options.no_build or options.golem):
gradle.RunGradle(['r8', '-Pno_internal'])
build_r8lib = False
for shrinker in options.shrinker:
if IsMinifiedR8(shrinker):
build_r8lib = True
if build_r8lib:
gradle.RunGradle(['r8lib', '-Pno_internal'])
if options.hash:
# Download r8-<hash>.jar from
# https://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.hash)
update_prebuilds_in_android.download_hash(
temp_dir, 'com/android/tools/r8/' + options.hash, target)
as_utils.MoveFile(
os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
elif options.version:
# Download r8-<version>.jar from
# https://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.version)
update_prebuilds_in_android.download_version(
temp_dir, 'com/android/tools/r8/' + options.version, target)
as_utils.MoveFile(
os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
else:
# Make a copy of r8.jar and r8lib.jar such that they stay the same for
# the entire execution of this script.
if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker:
assert os.path.isfile(utils.R8_JAR), 'Cannot build without r8.jar'
shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
if 'r8' in options.shrinker or 'r8-full' in options.shrinker:
assert os.path.isfile(utils.R8LIB_JAR), 'Cannot build without r8lib.jar'
shutil.copyfile(utils.R8LIB_JAR, os.path.join(temp_dir, 'r8lib.jar'))
result_per_shrinker_per_app = []
# If we are running on golem we kill all java processes after the run
# to ensure no hanging gradle daemons.
with EnsureNoGradleAlive(options.golem):
for (app, repo) in options.apps:
if app.skip:
continue
result_per_shrinker_per_app.append(
(app, GetResultsForApp(app, repo, options, temp_dir)))
return LogResultsForApps(result_per_shrinker_per_app, options)
def Main():
gradle.RunGradle(GRADLE_ARGS + BUILD_TARGETS)
def Main():
(options, args) = ParseOptions()
if utils.is_bot():
gradle.RunGradle(['--no-daemon', 'clean'])
gradle_args = ['--stacktrace']
if utils.is_bot():
# Bots don't like dangling processes.
gradle_args.append('--no-daemon')
# Set all necessary Gradle properties and options first.
if options.shard_count:
assert options.shard_number
gradle_args.append('-Pshard_count=%s' % options.shard_count)
gradle_args.append('-Pshard_number=%s' % options.shard_number)
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.slow_tests:
gradle_args.append('-Pslow_tests=1')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.jctf:
gradle_args.append('-Pjctf')
if options.only_jctf:
gradle_args.append('-Ponly_jctf')
if options.test_namespace:
gradle_args.append('-Ptest_namespace=%s' % options.test_namespace)
if options.jctf_compile_only:
gradle_args.append('-Pjctf_compile_only')
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if options.with_code_coverage:
gradle_args.append('-Pwith_code_coverage')
if os.name == 'nt':
# temporary hack
gradle_args.append('-Pno_internal')
gradle_args.append('-x')
gradle_args.append('createJctfTests')
gradle_args.append('-x')
gradle_args.append('jctfCommonJar')
gradle_args.append('-x')
gradle_args.append('jctfTestsClasses')
if options.test_dir:
gradle_args.append('-Ptest_dir=' + options.test_dir)
if not os.path.exists(options.test_dir):
os.makedirs(options.test_dir)
if options.java_home:
gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
if options.java_max_memory_size:
gradle_args.append('-Ptest_xmx=' + options.java_max_memory_size)
if options.generate_golden_files_to:
gradle_args.append('-Pgenerate_golden_files_to=' +
options.generate_golden_files_to)
if not os.path.exists(options.generate_golden_files_to):
os.makedirs(options.generate_golden_files_to)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.use_golden_files_in:
gradle_args.append('-Puse_golden_files_in=' +
options.use_golden_files_in)
if not os.path.exists(options.use_golden_files_in):
os.makedirs(options.use_golden_files_in)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if (not options.no_r8lib) and options.r8lib_no_deps:
print(
'Cannot run tests on r8lib with and without deps. R8lib is now default target.'
)
exit(1)
if not options.no_r8lib:
gradle_args.append('-Pr8lib')
# Force gradle to build a version of r8lib without dependencies for
# BootstrapCurrentEqualityTest.
gradle_args.append('R8LibNoDeps')
if options.r8lib_no_deps:
gradle_args.append('-Pr8lib_no_deps')
if options.worktree:
gradle_args.append('-g=' +
os.path.join(utils.REPO_ROOT, ".gradle_user_home"))
gradle_args.append('--no-daemon')
# Build an R8 with dependencies for bootstrapping tests before adding test sources.
gradle_args.append('r8WithDeps')
gradle_args.append('r8WithDeps11')
gradle_args.append('r8WithRelocatedDeps')
gradle_args.append('r8WithRelocatedDeps11')
# Add Gradle tasks
gradle_args.append('cleanTest')
gradle_args.append('test')
if options.fail_fast:
gradle_args.append('--fail-fast')
if options.failed:
args = compute_failed_tests(args)
if args is None:
return 1
if len(args) == 0:
print "No failing tests"
return 0
# Test filtering. Must always follow the 'test' task.
for testFilter in args:
gradle_args.append('--tests')
gradle_args.append(testFilter)
if options.with_code_coverage:
# Create Jacoco report after tests.
gradle_args.append('jacocoTestReport')
if options.use_golden_files_in:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.use_golden_files_in):
utils.download_file_from_cloud_storage(
'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
print_stacks_timeout = options.print_hanging_stacks
if (utils.is_bot() and not utils.IsWindows()) or print_stacks_timeout > -1:
timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
if os.path.exists(timestamp_file):
os.remove(timestamp_file)
gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
print_stacks_timeout = (print_stacks_timeout
if print_stacks_timeout != -1 else
TIMEOUT_HANDLER_PERIOD)
thread.start_new_thread(timeout_handler, (
timestamp_file,
print_stacks_timeout,
))
rotate_test_reports()
if options.only_jctf:
# Note: not setting -Pruntimes will run with all available runtimes.
return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
return archive_and_return(return_code, options)
if options.horizontal_class_merging:
gradle_args.append('-PhorizontalClassMerging')
# Now run tests on selected runtime(s).
if options.runtimes:
if options.dex_vm != 'default':
print 'Unexpected runtimes and dex_vm argument: ' + options.dex_vm
sys.exit(1)
if options.runtimes == 'empty':
# Set runtimes with no content will configure no runtimes.
gradle_args.append('-Pruntimes=')
elif options.runtimes == 'all':
# An unset runtimes will configure all runtimes
pass
else:
prefixes = [
prefix.strip() for prefix in options.runtimes.split(':')
]
runtimes = []
for prefix in prefixes:
matches = [
rt for rt in VALID_RUNTIMES if rt.startswith(prefix)
]
if len(matches) == 0:
print "Invalid runtime prefix '%s'." % prefix
print "Must be just 'all', 'empty'," \
" or a prefix of %s" % ', '.join(VALID_RUNTIMES)
sys.exit(1)
runtimes.extend(matches)
gradle_args.append('-Pruntimes=%s' % ':'.join(runtimes))
return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
return archive_and_return(return_code, options)
# Legacy testing populates the runtimes based on dex_vm.
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
for art_vm in vms_to_test:
vm_suffix = "_" + options.dex_vm_kind if art_vm != "default" else ""
runtimes = ['dex-' + art_vm]
# Only append the "none" runtime and JVMs if running on the "default" DEX VM.
if art_vm == "default":
# TODO(b/170454076): Remove special casing for bot when rex-script has
# been migrated to account for runtimes.
if utils.is_bot():
runtimes.extend(['jdk11', 'none'])
else:
runtimes.extend(['jdk8', 'jdk9', 'jdk11', 'none'])
return_code = gradle.RunGradle(gradle_args + [
'-Pdex_vm=%s' % art_vm + vm_suffix,
'-Pruntimes=%s' % ':'.join(runtimes),
],
throw_on_failure=False)
if options.generate_golden_files_to:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(
options.generate_golden_files_to):
archive = utils.create_archive(sha1)
utils.upload_file_to_cloud_storage(
archive, 'gs://r8-test-results/golden-files/' + archive)
if return_code != 0:
return archive_and_return(return_code, options)
return 0
def Main():
(options, args) = ParseOptions()
if not utils.is_bot() and not options.dry_run:
raise Exception('You are not a bot, don\'t archive builds. ' +
'Use --dry-run to test locally')
if (options.dry_run_output
and (not os.path.exists(options.dry_run_output)
or not os.path.isdir(options.dry_run_output))):
raise Exception(options.dry_run_output +
' does not exist or is not a directory')
if utils.is_bot() and not utils.IsWindows():
SetRLimitToMax()
if not utils.IsWindows():
PrintResourceInfo()
# Create maven release which uses a build that exclude dependencies.
create_maven_release.generate_r8_maven_zip(utils.MAVEN_ZIP)
create_maven_release.generate_r8_maven_zip(utils.MAVEN_ZIP_LIB,
is_r8lib=True)
# Generate and copy a full build without dependencies.
gradle.RunGradleExcludeDeps([utils.R8, utils.R8_SRC])
shutil.copyfile(utils.R8_JAR, utils.R8_FULL_EXCLUDE_DEPS_JAR)
# Ensure all archived artifacts has been built before archiving.
# The target tasks postfixed by 'lib' depend on the actual target task so
# building it invokes the original task first.
# The '-Pno_internal' flag is important because we generate the lib based on uses in tests.
gradle.RunGradle([
utils.R8, utils.D8, utils.R8LIB, utils.R8LIB_NO_DEPS, utils.R8RETRACE,
utils.R8RETRACE_NO_DEPS, utils.LIBRARY_DESUGAR_CONVERSIONS,
'-Pno_internal'
])
# Create maven release of the desuage_jdk_libs configuration. This require
# an r8.jar with dependencies to have been built.
create_maven_release.generate_desugar_configuration_maven_zip(
utils.DESUGAR_CONFIGURATION_MAVEN_ZIP)
version = GetVersion()
is_master = IsMaster(version)
if is_master:
# On master we use the git hash to archive with
print('On master, using git hash for archiving')
version = GetGitHash()
destination = GetVersionDestination('gs://', version, is_master)
if utils.cloud_storage_exists(destination) and not options.dry_run:
raise Exception('Target archive directory %s already exists' %
destination)
with utils.TempDir() as temp:
# Create pom file for our maven repository that we build for testing.
default_pom_file = os.path.join(temp, 'r8.pom')
create_maven_release.write_default_r8_pom_file(default_pom_file,
version)
version_file = os.path.join(temp, 'r8-version.properties')
with open(version_file, 'w') as version_writer:
version_writer.write('version.sha=' + GetGitHash() + '\n')
if not os.environ.get('SWARMING_BOT_ID') and not options.dry_run:
raise Exception('Environment variable SWARMING_BOT_ID not set')
releaser = \
("<local developer build>" if options.dry_run
else 'releaser=go/r8bot ('
+ os.environ.get('SWARMING_BOT_ID') + ')\n')
version_writer.write(releaser)
version_writer.write('version-file.version.code=1\n')
for file in [
utils.D8_JAR,
utils.R8_JAR,
utils.R8LIB_JAR,
utils.R8LIB_JAR + '.map',
utils.R8_SRC_JAR,
utils.R8_FULL_EXCLUDE_DEPS_JAR,
utils.R8LIB_EXCLUDE_DEPS_JAR,
utils.R8LIB_EXCLUDE_DEPS_JAR + '.map',
utils.R8RETRACE_JAR,
utils.R8RETRACE_EXCLUDE_DEPS_JAR,
utils.MAVEN_ZIP,
utils.MAVEN_ZIP_LIB,
utils.DESUGAR_CONFIGURATION,
utils.DESUGAR_CONFIGURATION_MAVEN_ZIP,
utils.GENERATED_LICENSE,
]:
file_name = os.path.basename(file)
tagged_jar = os.path.join(temp, file_name)
shutil.copyfile(file, tagged_jar)
if file_name.endswith(
'.jar') and not file_name.endswith('-src.jar'):
with zipfile.ZipFile(tagged_jar, 'a') as zip:
zip.write(version_file, os.path.basename(version_file))
destination = GetUploadDestination(version, file_name, is_master)
print('Uploading %s to %s' % (tagged_jar, destination))
if options.dry_run:
if options.dry_run_output:
dry_run_destination = os.path.join(options.dry_run_output,
file_name)
print('Dry run, not actually uploading. Copying to ' +
dry_run_destination)
shutil.copyfile(tagged_jar, dry_run_destination)
else:
print('Dry run, not actually uploading')
else:
utils.upload_file_to_cloud_storage(tagged_jar, destination)
print('File available at: %s' %
GetUrl(version, file_name, is_master))
# Upload R8 to a maven compatible location.
if file == utils.R8_JAR:
maven_dst = GetUploadDestination(
utils.get_maven_path('r8', version), 'r8-%s.jar' % version,
is_master)
maven_pom_dst = GetUploadDestination(
utils.get_maven_path('r8', version), 'r8-%s.pom' % version,
is_master)
if options.dry_run:
print('Dry run, not actually creating maven repo for R8')
else:
utils.upload_file_to_cloud_storage(tagged_jar, maven_dst)
utils.upload_file_to_cloud_storage(default_pom_file,
maven_pom_dst)
print('Maven repo root available at: %s' %
GetMavenUrl(is_master))
# Upload desugar_jdk_libs configuration to a maven compatible location.
if file == utils.DESUGAR_CONFIGURATION:
jar_basename = 'desugar_jdk_libs_configuration.jar'
jar_version_name = 'desugar_jdk_libs_configuration-%s.jar' % version
maven_dst = GetUploadDestination(
utils.get_maven_path('desugar_jdk_libs_configuration',
version), jar_version_name, is_master)
with utils.TempDir() as tmp_dir:
desugar_jdk_libs_configuration_jar = os.path.join(
tmp_dir, jar_version_name)
create_maven_release.generate_jar_with_desugar_configuration(
utils.DESUGAR_CONFIGURATION,
utils.DESUGAR_IMPLEMENTATION,
utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP,
desugar_jdk_libs_configuration_jar)
if options.dry_run:
print(
'Dry run, not actually creating maven repo for ' +
'desugar configuration.')
if options.dry_run_output:
shutil.copyfile(
desugar_jdk_libs_configuration_jar,
os.path.join(options.dry_run_output,
jar_version_name))
else:
utils.upload_file_to_cloud_storage(
desugar_jdk_libs_configuration_jar, maven_dst)
print('Maven repo root available at: %s' %
GetMavenUrl(is_master))
# Also archive the jar as non maven destination for Google3
jar_destination = GetUploadDestination(
version, jar_basename, is_master)
utils.upload_file_to_cloud_storage(
desugar_jdk_libs_configuration_jar,
jar_destination)
def main(argv):
(options, args) = parse_options(argv)
if options.bot:
options.no_logging = True
options.shrinker = ['r8', 'r8-full']
print(options.shrinker)
if options.golem:
options.disable_assertions = True
options.no_build = True
options.r8_compilation_steps = 1
options.quiet = True
options.no_logging = True
if options.generate_golem_config:
print_golem_config(options)
return 0
with utils.TempDir() as temp_dir:
if options.hash:
# Download r8-<hash>.jar from
# https://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.hash)
update_prebuilds_in_android.download_hash(
temp_dir, 'com/android/tools/r8/' + options.hash, target)
as_utils.MoveFile(os.path.join(temp_dir, target),
os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
elif version_is_built_jar(options.version):
# Download r8-<version>.jar from
# https://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.version)
update_prebuilds_in_android.download_version(
temp_dir, 'com/android/tools/r8/' + options.version, target)
as_utils.MoveFile(os.path.join(temp_dir, target),
os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
elif options.version == 'master':
if not (options.no_build or options.golem):
gradle.RunGradle(['r8', '-Pno_internal'])
build_r8lib = False
for shrinker in options.shrinker:
if is_minified_r8(shrinker):
build_r8lib = True
if build_r8lib:
gradle.RunGradle(['r8lib', '-Pno_internal'])
# Make a copy of r8.jar and r8lib.jar such that they stay the same for
# the entire execution of this script.
if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker:
assert os.path.isfile(
utils.R8_JAR), 'Cannot build without r8.jar'
shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
if 'r8' in options.shrinker or 'r8-full' in options.shrinker:
assert os.path.isfile(
utils.R8LIB_JAR), 'Cannot build without r8lib.jar'
shutil.copyfile(utils.R8LIB_JAR,
os.path.join(temp_dir, 'r8lib.jar'))
result_per_shrinker_per_app = []
for app in options.apps:
if app.skip:
continue
result_per_shrinker_per_app.append(
(app, get_results_for_app(app, options, temp_dir)))
errors = log_results_for_apps(result_per_shrinker_per_app, options)
if errors > 0:
dest = 'gs://r8-test-results/r8-libs/' + str(int(time.time()))
utils.upload_file_to_cloud_storage(
os.path.join(temp_dir, 'r8lib.jar'), dest)
print('R8lib saved to %s' % dest)
return errors
def Main():
(options, args) = ParseOptions()
if not utils.is_bot() and not options.dry_run:
raise Exception('You are not a bot, don\'t archive builds')
if utils.is_bot():
SetRLimitToMax()
PrintResourceInfo()
# Create maven release which uses a build that exclude dependencies.
create_maven_release.run(utils.MAVEN_ZIP)
create_maven_release.run(utils.MAVEN_ZIP_LIB, is_r8lib=True)
# Generate and copy a full build without dependencies.
gradle.RunGradleExcludeDeps([utils.R8, utils.R8_SRC])
shutil.copyfile(utils.R8_JAR, utils.R8_FULL_EXCLUDE_DEPS_JAR)
# Ensure all archived artifacts has been built before archiving.
# The target tasks postfixed by 'lib' depend on the actual target task so
# building it invokes the original task first.
# The '-Pno_internal' flag is important because we generate the lib based on uses in tests.
gradle.RunGradle([
utils.R8, utils.D8, utils.COMPATDX, utils.COMPATPROGUARD, utils.R8LIB,
utils.R8LIB_NO_DEPS, utils.COMPATDXLIB, utils.COMPATPROGUARDLIB,
'-Pno_internal'
])
version = GetVersion()
is_master = IsMaster(version)
if is_master:
# On master we use the git hash to archive with
print 'On master, using git hash for archiving'
version = GetGitHash()
destination = GetVersionDestination('gs://', version, is_master)
if utils.cloud_storage_exists(destination) and not options.dry_run:
raise Exception('Target archive directory %s already exists' %
destination)
with utils.TempDir() as temp:
version_file = os.path.join(temp, 'r8-version.properties')
with open(version_file, 'w') as version_writer:
version_writer.write('version.sha=' + GetGitHash() + '\n')
version_writer.write('releaser=go/r8bot (' +
os.environ.get('SWARMING_BOT_ID') + ')\n')
version_writer.write('version-file.version.code=1\n')
for file in [
utils.D8_JAR,
utils.R8_JAR,
utils.R8LIB_JAR,
utils.R8LIB_JAR + '.map',
utils.R8_SRC_JAR,
utils.R8_FULL_EXCLUDE_DEPS_JAR,
utils.R8LIB_EXCLUDE_DEPS_JAR,
utils.R8LIB_EXCLUDE_DEPS_JAR + '.map',
utils.COMPATDX_JAR,
utils.COMPATDXLIB_JAR,
utils.COMPATDXLIB_JAR + '.map',
utils.COMPATPROGUARD_JAR,
utils.COMPATPROGUARDLIB_JAR,
utils.COMPATPROGUARDLIB_JAR + '.map',
utils.MAVEN_ZIP,
utils.MAVEN_ZIP_LIB,
utils.GENERATED_LICENSE,
]:
file_name = os.path.basename(file)
tagged_jar = os.path.join(temp, file_name)
shutil.copyfile(file, tagged_jar)
if file_name.endswith(
'.jar') and not file_name.endswith('-src.jar'):
with zipfile.ZipFile(tagged_jar, 'a') as zip:
zip.write(version_file, os.path.basename(version_file))
destination = GetUploadDestination(version, file_name, is_master)
print('Uploading %s to %s' % (tagged_jar, destination))
if options.dry_run:
print('Dry run, not actually uploading')
else:
utils.upload_file_to_cloud_storage(tagged_jar, destination)
print('File available at: %s' %
GetUrl(version, file_name, is_master))
if file == utils.R8_JAR:
# Upload R8 to a maven compatible location.
maven_dst = GetUploadDestination(utils.get_maven_path(version),
'r8-%s.jar' % version,
is_master)
if options.dry_run:
print('Dry run, not actually creating maven repo')
else:
utils.upload_file_to_cloud_storage(tagged_jar, maven_dst)
print('Maven repo root available at: %s' %
GetMavenUrl(is_master))
def Main():
(options, args) = ParseOptions()
if utils.is_bot():
print "Result of 'java -version':"
print subprocess.check_output(['java', '-version'])
gradle.RunGradle(['clean'])
gradle_args = ['--stacktrace']
# Set all necessary Gradle properties and options first.
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.jctf:
gradle_args.append('-Pjctf')
if options.only_jctf:
gradle_args.append('-Ponly_jctf')
if options.jctf_compile_only:
gradle_args.append('-Pjctf_compile_only')
if options.aosp_jar:
gradle_args.append('-Paosp_jar')
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if options.with_code_coverage:
gradle_args.append('-Pwith_code_coverage')
if os.name == 'nt':
# temporary hack
gradle_args.append('-Pno_internal')
gradle_args.append('-x')
gradle_args.append('createJctfTests')
gradle_args.append('-x')
gradle_args.append('jctfCommonJar')
gradle_args.append('-x')
gradle_args.append('jctfTestsClasses')
if options.test_dir:
gradle_args.append('-Ptest_dir=' + options.test_dir)
if not os.path.exists(options.test_dir):
os.makedirs(options.test_dir)
if options.java_home:
gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
if options.generate_golden_files_to:
gradle_args.append('-Pgenerate_golden_files_to=' +
options.generate_golden_files_to)
if not os.path.exists(options.generate_golden_files_to):
os.makedirs(options.generate_golden_files_to)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.use_golden_files_in:
gradle_args.append('-Puse_golden_files_in=' +
options.use_golden_files_in)
if not os.path.exists(options.use_golden_files_in):
os.makedirs(options.use_golden_files_in)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if (not options.no_r8lib) and options.r8lib_no_deps:
print(
'Cannot run tests on r8lib with and without deps. R8lib is now default target.'
)
exit(1)
if not options.no_r8lib:
gradle_args.append('-Pr8lib')
# Force gradle to build a version of r8lib without dependencies for
# BootstrapCurrentEqualityTest.
gradle_args.append('R8LibNoDeps')
if options.r8lib_no_deps:
gradle_args.append('-Pr8lib_no_deps')
# Build an R8 with dependencies for bootstrapping tests before adding test sources.
gradle_args.append('r8WithRelocatedDeps')
# Add Gradle tasks
gradle_args.append('cleanTest')
gradle_args.append('test')
# Test filtering. Must always follow the 'test' task.
for testFilter in args:
gradle_args.append('--tests')
gradle_args.append(testFilter)
if options.with_code_coverage:
# Create Jacoco report after tests.
gradle_args.append('jacocoTestReport')
if options.use_golden_files_in:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.use_golden_files_in):
utils.download_file_from_cloud_storage(
'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
if utils.is_bot() and not utils.IsWindows():
timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
if os.path.exists(timestamp_file):
os.remove(timestamp_file)
gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
thread.start_new_thread(timeout_handler, (timestamp_file, ))
# Now run tests on selected runtime(s).
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
for art_vm in vms_to_test:
vm_kind_to_test = "_" + options.dex_vm_kind if art_vm != "default" else ""
return_code = gradle.RunGradle(
gradle_args + ['-Pdex_vm=%s' % (art_vm + vm_kind_to_test)],
throw_on_failure=False)
if options.generate_golden_files_to:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(
options.generate_golden_files_to):
archive = utils.create_archive(sha1)
utils.upload_file_to_cloud_storage(
archive, 'gs://r8-test-results/golden-files/' + archive)
if return_code != 0:
if options.archive_failures and os.name != 'nt':
archive_failures()
return return_code
return 0
