def unpack(self):
if os.path.exists(self.source_dir):
utils.output("%s already unpacked. Skipping." % BASENAME)
return
utils.output("Unpacking %s." % BASENAME)
utils.unpack_archive(self.archive_path)
def unpack(self):
if os.path.exists(self.source_dir):
utils.output("%s already unpacked. Skipping." % BASENAME)
return
utils.output("Unpacking %s." % BASENAME)
utils.unpack_archive(self.archive_path)
def main():
args = parser.parse_args()
src = Path(args.source_path)
dest = Path(args.destination_dir)
print("Unpacking archive...")
utils.unpack_archive(src, dest)
print("Done unpacking.\n")
def Main():
(options, args) = parse_options()
assert len(args) == 1
sha1_file = args[0]
dest = sha1_file[:-5]
print 'Ensuring %s' % dest
with open(sha1_file, 'r') as input_sha:
sha1 = input_sha.readline()
if os.path.exists(dest) and utils.get_sha1(dest) == sha1:
print 'sha1 matches, not downloading'
dest_dir = utils.extract_dir(dest)
if os.path.exists(dest_dir):
print 'destination directory exists, no extraction'
else:
utils.unpack_archive(dest)
return
src = os.path.join(GMSCORE_DEPS, sha1)
if not os.path.exists(src):
print 'File (%s) does not exist on x20' % src
print 'Maybe pass -Pno_internal to your gradle invocation'
return 42
download(src, dest)
def Main():
(options, args) = ParseOptions()
if utils.is_bot():
gradle.RunGradle(['--no-daemon', 'clean'])
gradle_args = ['--stacktrace']
if utils.is_bot():
# Bots don't like dangling processes
gradle_args.append('--no-daemon')
# Set all necessary Gradle properties and options first.
if options.shard_count:
assert options.shard_number
gradle_args.append('-Pshard_count=%s' % options.shard_count)
gradle_args.append('-Pshard_number=%s' % options.shard_number)
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.jctf:
gradle_args.append('-Pjctf')
if options.only_jctf:
gradle_args.append('-Ponly_jctf')
if options.jctf_compile_only:
gradle_args.append('-Pjctf_compile_only')
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if options.with_code_coverage:
gradle_args.append('-Pwith_code_coverage')
if os.name == 'nt':
# temporary hack
gradle_args.append('-Pno_internal')
gradle_args.append('-x')
gradle_args.append('createJctfTests')
gradle_args.append('-x')
gradle_args.append('jctfCommonJar')
gradle_args.append('-x')
gradle_args.append('jctfTestsClasses')
if options.test_dir:
gradle_args.append('-Ptest_dir=' + options.test_dir)
if not os.path.exists(options.test_dir):
os.makedirs(options.test_dir)
if options.java_home:
gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
if options.java_max_memory_size:
gradle_args.append('-Dorg.gradle.jvmargs=-Xmx' +
options.java_max_memory_size)
if options.generate_golden_files_to:
gradle_args.append('-Pgenerate_golden_files_to=' +
options.generate_golden_files_to)
if not os.path.exists(options.generate_golden_files_to):
os.makedirs(options.generate_golden_files_to)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.use_golden_files_in:
gradle_args.append('-Puse_golden_files_in=' +
options.use_golden_files_in)
if not os.path.exists(options.use_golden_files_in):
os.makedirs(options.use_golden_files_in)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.alternative_frontend:
gradle_args.append('-Palternative_frontend')
if (not options.no_r8lib) and options.r8lib_no_deps:
print(
'Cannot run tests on r8lib with and without deps. R8lib is now default target.'
)
exit(1)
if not options.no_r8lib:
gradle_args.append('-Pr8lib')
# Force gradle to build a version of r8lib without dependencies for
# BootstrapCurrentEqualityTest.
gradle_args.append('R8LibNoDeps')
if options.r8lib_no_deps:
gradle_args.append('-Pr8lib_no_deps')
# Build an R8 with dependencies for bootstrapping tests before adding test sources.
gradle_args.append('r8WithRelocatedDeps')
gradle_args.append('r8WithRelocatedDeps11')
# Add Gradle tasks
gradle_args.append('cleanTest')
gradle_args.append('test')
if options.fail_fast:
gradle_args.append('--fail-fast')
if options.failed:
args = compute_failed_tests(args)
if args is None:
return 1
if len(args) == 0:
print "No failing tests"
return 0
# Test filtering. Must always follow the 'test' task.
for testFilter in args:
gradle_args.append('--tests')
gradle_args.append(testFilter)
if options.with_code_coverage:
# Create Jacoco report after tests.
gradle_args.append('jacocoTestReport')
if options.use_golden_files_in:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.use_golden_files_in):
utils.download_file_from_cloud_storage(
'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
if utils.is_bot() and not utils.IsWindows():
timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
if os.path.exists(timestamp_file):
os.remove(timestamp_file)
gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
thread.start_new_thread(timeout_handler, (timestamp_file, ))
rotate_test_reports()
if options.only_jctf:
# Note: not setting -Pruntimes will run with all available runtimes.
return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
return archive_and_return(return_code, options)
# Now run tests on selected runtime(s).
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
for art_vm in vms_to_test:
vm_suffix = "_" + options.dex_vm_kind if art_vm != "default" else ""
runtimes = ['dex-' + art_vm]
# Only append the "none" runtime and JVMs if running on the "default" DEX VM.
if art_vm == "default":
runtimes.extend(['jdk8', 'jdk9', 'jdk11', 'none'])
return_code = gradle.RunGradle(gradle_args + [
'-Pdex_vm=%s' % art_vm + vm_suffix,
'-Pruntimes=%s' % ':'.join(runtimes),
],
throw_on_failure=False)
if options.generate_golden_files_to:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(
options.generate_golden_files_to):
archive = utils.create_archive(sha1)
utils.upload_file_to_cloud_storage(
archive, 'gs://r8-test-results/golden-files/' + archive)
if return_code != 0:
return archive_and_return(return_code, options)
return 0
def Main():
(options, args) = ParseOptions()
if 'BUILDBOT_BUILDERNAME' in os.environ:
gradle.RunGradle(['clean'])
# Build R8lib with dependencies for bootstrapping tests before adding test sources
gradle.RunGradle(['r8libwithdeps'])
gradle_args = ['--stacktrace']
# Set all necessary Gradle properties and options first.
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.jctf:
gradle_args.append('-Pjctf')
if options.only_jctf:
gradle_args.append('-Ponly_jctf')
if options.jctf_compile_only:
gradle_args.append('-Pjctf_compile_only')
if options.aosp_jar:
gradle_args.append('-Paosp_jar')
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if options.with_code_coverage:
gradle_args.append('-Pwith_code_coverage')
if os.name == 'nt':
# temporary hack
gradle_args.append('-Pno_internal')
gradle_args.append('-x')
gradle_args.append('createJctfTests')
gradle_args.append('-x')
gradle_args.append('jctfCommonJar')
gradle_args.append('-x')
gradle_args.append('jctfTestsClasses')
if options.test_dir:
gradle_args.append('-Ptest_dir=' + options.test_dir)
if not os.path.exists(options.test_dir):
os.makedirs(options.test_dir)
if options.java_home:
gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
if options.generate_golden_files_to:
gradle_args.append('-Pgenerate_golden_files_to=' +
options.generate_golden_files_to)
if not os.path.exists(options.generate_golden_files_to):
os.makedirs(options.generate_golden_files_to)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.use_golden_files_in:
gradle_args.append('-Puse_golden_files_in=' +
options.use_golden_files_in)
if not os.path.exists(options.use_golden_files_in):
os.makedirs(options.use_golden_files_in)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
# Add Gradle tasks
gradle_args.append('cleanTest')
gradle_args.append('test')
# Test filtering. Must always follow the 'test' task.
for testFilter in args:
gradle_args.append('--tests')
gradle_args.append(testFilter)
if options.with_code_coverage:
# Create Jacoco report after tests.
gradle_args.append('jacocoTestReport')
if options.use_golden_files_in:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.use_golden_files_in):
utils.download_file_from_cloud_storage(
'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
# Now run tests on selected runtime(s).
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
for art_vm in vms_to_test:
vm_kind_to_test = "_" + options.dex_vm_kind if art_vm != "default" else ""
return_code = gradle.RunGradle(
gradle_args + ['-Pdex_vm=%s' % (art_vm + vm_kind_to_test)],
throw_on_failure=False)
if options.generate_golden_files_to:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(
options.generate_golden_files_to):
archive = utils.create_archive(sha1)
utils.upload_file_to_cloud_storage(
archive, 'gs://r8-test-results/golden-files/' + archive)
if return_code != 0:
if options.archive_failures and os.name != 'nt':
archive_failures()
return return_code
return 0
def Main():
(options, args) = ParseOptions()
if utils.is_bot():
gradle.RunGradle(['--no-daemon', 'clean'])
gradle_args = ['--stacktrace']
if utils.is_bot():
# Bots don't like dangling processes.
gradle_args.append('--no-daemon')
# Set all necessary Gradle properties and options first.
if options.shard_count:
assert options.shard_number
gradle_args.append('-Pshard_count=%s' % options.shard_count)
gradle_args.append('-Pshard_number=%s' % options.shard_number)
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.slow_tests:
gradle_args.append('-Pslow_tests=1')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.jctf:
gradle_args.append('-Pjctf')
if options.only_jctf:
gradle_args.append('-Ponly_jctf')
if options.test_namespace:
gradle_args.append('-Ptest_namespace=%s' % options.test_namespace)
if options.jctf_compile_only:
gradle_args.append('-Pjctf_compile_only')
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if options.with_code_coverage:
gradle_args.append('-Pwith_code_coverage')
if os.name == 'nt':
# temporary hack
gradle_args.append('-Pno_internal')
gradle_args.append('-x')
gradle_args.append('createJctfTests')
gradle_args.append('-x')
gradle_args.append('jctfCommonJar')
gradle_args.append('-x')
gradle_args.append('jctfTestsClasses')
if options.test_dir:
gradle_args.append('-Ptest_dir=' + options.test_dir)
if not os.path.exists(options.test_dir):
os.makedirs(options.test_dir)
if options.java_home:
gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
if options.java_max_memory_size:
gradle_args.append('-Ptest_xmx=' + options.java_max_memory_size)
if options.generate_golden_files_to:
gradle_args.append('-Pgenerate_golden_files_to=' +
options.generate_golden_files_to)
if not os.path.exists(options.generate_golden_files_to):
os.makedirs(options.generate_golden_files_to)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.use_golden_files_in:
gradle_args.append('-Puse_golden_files_in=' +
options.use_golden_files_in)
if not os.path.exists(options.use_golden_files_in):
os.makedirs(options.use_golden_files_in)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if (not options.no_r8lib) and options.r8lib_no_deps:
print(
'Cannot run tests on r8lib with and without deps. R8lib is now default target.'
)
exit(1)
if not options.no_r8lib:
gradle_args.append('-Pr8lib')
# Force gradle to build a version of r8lib without dependencies for
# BootstrapCurrentEqualityTest.
gradle_args.append('R8LibNoDeps')
if options.r8lib_no_deps:
gradle_args.append('-Pr8lib_no_deps')
if options.worktree:
gradle_args.append('-g=' +
os.path.join(utils.REPO_ROOT, ".gradle_user_home"))
gradle_args.append('--no-daemon')
# Build an R8 with dependencies for bootstrapping tests before adding test sources.
gradle_args.append('r8WithDeps')
gradle_args.append('r8WithDeps11')
gradle_args.append('r8WithRelocatedDeps')
gradle_args.append('r8WithRelocatedDeps11')
# Add Gradle tasks
gradle_args.append('cleanTest')
gradle_args.append('test')
if options.fail_fast:
gradle_args.append('--fail-fast')
if options.failed:
args = compute_failed_tests(args)
if args is None:
return 1
if len(args) == 0:
print "No failing tests"
return 0
# Test filtering. Must always follow the 'test' task.
for testFilter in args:
gradle_args.append('--tests')
gradle_args.append(testFilter)
if options.with_code_coverage:
# Create Jacoco report after tests.
gradle_args.append('jacocoTestReport')
if options.use_golden_files_in:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.use_golden_files_in):
utils.download_file_from_cloud_storage(
'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
print_stacks_timeout = options.print_hanging_stacks
if (utils.is_bot() and not utils.IsWindows()) or print_stacks_timeout > -1:
timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
if os.path.exists(timestamp_file):
os.remove(timestamp_file)
gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
print_stacks_timeout = (print_stacks_timeout
if print_stacks_timeout != -1 else
TIMEOUT_HANDLER_PERIOD)
thread.start_new_thread(timeout_handler, (
timestamp_file,
print_stacks_timeout,
))
rotate_test_reports()
if options.only_jctf:
# Note: not setting -Pruntimes will run with all available runtimes.
return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
return archive_and_return(return_code, options)
if options.horizontal_class_merging:
gradle_args.append('-PhorizontalClassMerging')
# Now run tests on selected runtime(s).
if options.runtimes:
if options.dex_vm != 'default':
print 'Unexpected runtimes and dex_vm argument: ' + options.dex_vm
sys.exit(1)
if options.runtimes == 'empty':
# Set runtimes with no content will configure no runtimes.
gradle_args.append('-Pruntimes=')
elif options.runtimes == 'all':
# An unset runtimes will configure all runtimes
pass
else:
prefixes = [
prefix.strip() for prefix in options.runtimes.split(':')
]
runtimes = []
for prefix in prefixes:
matches = [
rt for rt in VALID_RUNTIMES if rt.startswith(prefix)
]
if len(matches) == 0:
print "Invalid runtime prefix '%s'." % prefix
print "Must be just 'all', 'empty'," \
" or a prefix of %s" % ', '.join(VALID_RUNTIMES)
sys.exit(1)
runtimes.extend(matches)
gradle_args.append('-Pruntimes=%s' % ':'.join(runtimes))
return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
return archive_and_return(return_code, options)
# Legacy testing populates the runtimes based on dex_vm.
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
for art_vm in vms_to_test:
vm_suffix = "_" + options.dex_vm_kind if art_vm != "default" else ""
runtimes = ['dex-' + art_vm]
# Only append the "none" runtime and JVMs if running on the "default" DEX VM.
if art_vm == "default":
# TODO(b/170454076): Remove special casing for bot when rex-script has
# been migrated to account for runtimes.
if utils.is_bot():
runtimes.extend(['jdk11', 'none'])
else:
runtimes.extend(['jdk8', 'jdk9', 'jdk11', 'none'])
return_code = gradle.RunGradle(gradle_args + [
'-Pdex_vm=%s' % art_vm + vm_suffix,
'-Pruntimes=%s' % ':'.join(runtimes),
],
throw_on_failure=False)
if options.generate_golden_files_to:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(
options.generate_golden_files_to):
archive = utils.create_archive(sha1)
utils.upload_file_to_cloud_storage(
archive, 'gs://r8-test-results/golden-files/' + archive)
if return_code != 0:
return archive_and_return(return_code, options)
return 0
def download(src, dest): print 'Downloading %s to %s' % (src, dest) shutil.copyfile(src, dest) utils.unpack_archive(dest)
def Main():
(options, args) = ParseOptions()
if utils.is_bot():
print "Result of 'java -version':"
print subprocess.check_output(['java', '-version'])
gradle.RunGradle(['clean'])
gradle_args = ['--stacktrace']
# Set all necessary Gradle properties and options first.
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.jctf:
gradle_args.append('-Pjctf')
if options.only_jctf:
gradle_args.append('-Ponly_jctf')
if options.jctf_compile_only:
gradle_args.append('-Pjctf_compile_only')
if options.aosp_jar:
gradle_args.append('-Paosp_jar')
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if options.with_code_coverage:
gradle_args.append('-Pwith_code_coverage')
if os.name == 'nt':
# temporary hack
gradle_args.append('-Pno_internal')
gradle_args.append('-x')
gradle_args.append('createJctfTests')
gradle_args.append('-x')
gradle_args.append('jctfCommonJar')
gradle_args.append('-x')
gradle_args.append('jctfTestsClasses')
if options.test_dir:
gradle_args.append('-Ptest_dir=' + options.test_dir)
if not os.path.exists(options.test_dir):
os.makedirs(options.test_dir)
if options.java_home:
gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
if options.generate_golden_files_to:
gradle_args.append('-Pgenerate_golden_files_to=' +
options.generate_golden_files_to)
if not os.path.exists(options.generate_golden_files_to):
os.makedirs(options.generate_golden_files_to)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.use_golden_files_in:
gradle_args.append('-Puse_golden_files_in=' +
options.use_golden_files_in)
if not os.path.exists(options.use_golden_files_in):
os.makedirs(options.use_golden_files_in)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if (not options.no_r8lib) and options.r8lib_no_deps:
print(
'Cannot run tests on r8lib with and without deps. R8lib is now default target.'
)
exit(1)
if not options.no_r8lib:
gradle_args.append('-Pr8lib')
# Force gradle to build a version of r8lib without dependencies for
# BootstrapCurrentEqualityTest.
gradle_args.append('R8LibNoDeps')
if options.r8lib_no_deps:
gradle_args.append('-Pr8lib_no_deps')
# Build an R8 with dependencies for bootstrapping tests before adding test sources.
gradle_args.append('r8WithRelocatedDeps')
# Add Gradle tasks
gradle_args.append('cleanTest')
gradle_args.append('test')
# Test filtering. Must always follow the 'test' task.
for testFilter in args:
gradle_args.append('--tests')
gradle_args.append(testFilter)
if options.with_code_coverage:
# Create Jacoco report after tests.
gradle_args.append('jacocoTestReport')
if options.use_golden_files_in:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.use_golden_files_in):
utils.download_file_from_cloud_storage(
'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
if utils.is_bot() and not utils.IsWindows():
timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
if os.path.exists(timestamp_file):
os.remove(timestamp_file)
gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
thread.start_new_thread(timeout_handler, (timestamp_file, ))
# Now run tests on selected runtime(s).
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
for art_vm in vms_to_test:
vm_kind_to_test = "_" + options.dex_vm_kind if art_vm != "default" else ""
return_code = gradle.RunGradle(
gradle_args + ['-Pdex_vm=%s' % (art_vm + vm_kind_to_test)],
throw_on_failure=False)
if options.generate_golden_files_to:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(
options.generate_golden_files_to):
archive = utils.create_archive(sha1)
utils.upload_file_to_cloud_storage(
archive, 'gs://r8-test-results/golden-files/' + archive)
if return_code != 0:
if options.archive_failures and os.name != 'nt':
archive_failures()
return return_code
return 0