def Main(): gradle.RunGradle(BUILD_TARGETS) utils.DownloadFromX20( os.path.join(utils.THIRD_PARTY, 'gradle-plugin') + '.tar.gz.sha1') utils.DownloadFromX20( os.path.join(utils.THIRD_PARTY, 'benchmarks', 'android-sdk') + '.tar.gz.sha1') utils.DownloadFromGoogleCloudStorage(utils.SAMPLE_LIBRARIES_SHA_FILE) utils.DownloadFromGoogleCloudStorage(utils.OPENSOURCE_APPS_SHA_FILE) utils.DownloadFromGoogleCloudStorage(utils.ANDROID_SDK + '.tar.gz.sha1', bucket='r8-deps-internal', auth=True)
def EnsureGradle(): if not os.path.exists(GRADLE) or os.path.getmtime(GRADLE_TGZ) < os.path.getmtime(GRADLE_SHA1): # Bootstrap or update gradle, everything else is controlled using gradle. utils.DownloadFromGoogleCloudStorage(GRADLE_SHA1) # Update the mtime of the tar file to make sure we do not run again unless # there is an update. os.utime(GRADLE_TGZ, None) else: print 'gradle.py: Gradle binary present'
def Main(argv): (options, args) = ParseOptions(argv) if (len(args) > 0): raise Exception('Unsupported arguments') if not utils.is_bot() and not (options.dry_run or options.build_only): raise Exception('You are not a bot, don\'t archive builds. ' + 'Use --dry-run or --build-only to test locally') if options.dry_run_output: MustBeExistingDirectory(options.dry_run_output) if options.build_only: MustBeExistingDirectory(options.build_only) if utils.is_bot(): archive.SetRLimitToMax() # Make sure bazel is extracted in third_party. utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE) utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE) utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE) if options.build_only: with utils.TempDir() as checkout_dir: CloneDesugaredLibrary(options.github_account, checkout_dir) (library_jar, maven_zip) = BuildDesugaredLibrary(checkout_dir, "jdk8") shutil.copyfile( library_jar, os.path.join(options.build_only, os.path.basename(library_jar))) shutil.copyfile( maven_zip, os.path.join(options.build_only, os.path.basename(maven_zip))) return # Only handling versioned desugar_jdk_libs. is_master = False with utils.TempDir() as checkout_dir: CloneDesugaredLibrary(options.github_account, checkout_dir) version = GetVersion(os.path.join(checkout_dir, VERSION_FILE)) destination = archive.GetVersionDestination( 'gs://', LIBRARY_NAME + '/' + version, is_master) if utils.cloud_storage_exists(destination) and not options.dry_run: raise Exception('Target archive directory %s already exists' % destination) (library_jar, maven_zip) = BuildDesugaredLibrary(checkout_dir, "jdk8") storage_path = LIBRARY_NAME + '/' + version # Upload the jar file with the library. destination = archive.GetUploadDestination(storage_path, LIBRARY_NAME + '.jar', is_master) Upload(options, library_jar, storage_path, destination, is_master) # Upload the maven zip file with the library. destination = archive.GetUploadDestination(storage_path, LIBRARY_NAME + '.zip', is_master) Upload(options, maven_zip, storage_path, destination, is_master) # Upload the jar file for accessing GCS as a maven repro. maven_destination = archive.GetUploadDestination( utils.get_maven_path('desugar_jdk_libs', version), 'desugar_jdk_libs-%s.jar' % version, is_master) if options.dry_run: print('Dry run, not actually creating maven repo') else: utils.upload_file_to_cloud_storage(library_jar, maven_destination) print('Maven repo root available at: %s' % archive.GetMavenUrl(is_master))
def main(argv): (options, args) = ParseOptions(argv) if options.bot: utils.DownloadFromGoogleCloudStorage(utils.OPENSOURCE_APPS_SHA_FILE) utils.DownloadFromGoogleCloudStorage(utils.ANDROID_SDK + '.tar.gz.sha1', bucket='r8-deps-internal', auth=True) if os.path.exists(WORKING_DIR): shutil.rmtree(WORKING_DIR) shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR) os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join( utils.ANDROID_SDK) os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3' # TODO(b/141081520): Set to True once fixed. options.no_logging = False # TODO(b/141081520): Remove logging filter once fixed. options.app_logging_filter = ['sqldelight'] options.shrinker = ['r8', 'r8-full'] print(options.shrinker) if options.golem: golem.link_third_party() if os.path.exists(WORKING_DIR): shutil.rmtree(WORKING_DIR) shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR) os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join( utils.ANDROID_SDK) os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3' options.disable_assertions = True options.ignore_versions = True options.no_build = True options.r8_compilation_steps = 1 options.quiet = True options.gradle_pre_runs = 2 options.use_daemon = True options.no_logging = True if not os.path.exists(WORKING_DIR): os.makedirs(WORKING_DIR) if options.download_only: clone_repositories(options.quiet) return with utils.TempDir() as temp_dir: if not (options.no_build or options.golem): gradle.RunGradle(['r8', '-Pno_internal']) build_r8lib = False for shrinker in options.shrinker: if IsMinifiedR8(shrinker): build_r8lib = True if build_r8lib: gradle.RunGradle(['r8lib', '-Pno_internal']) if options.hash: # Download r8-<hash>.jar from # https://storage.googleapis.com/r8-releases/raw/. target = 'r8-{}.jar'.format(options.hash) update_prebuilds_in_android.download_hash( temp_dir, 'com/android/tools/r8/' + options.hash, target) as_utils.MoveFile( os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'), quiet=options.quiet) elif options.version: # Download r8-<version>.jar from # https://storage.googleapis.com/r8-releases/raw/. target = 'r8-{}.jar'.format(options.version) update_prebuilds_in_android.download_version( temp_dir, 'com/android/tools/r8/' + options.version, target) as_utils.MoveFile( os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'), quiet=options.quiet) else: # Make a copy of r8.jar and r8lib.jar such that they stay the same for # the entire execution of this script. if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker: assert os.path.isfile(utils.R8_JAR), 'Cannot build without r8.jar' shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar')) if 'r8' in options.shrinker or 'r8-full' in options.shrinker: assert os.path.isfile(utils.R8LIB_JAR), 'Cannot build without r8lib.jar' shutil.copyfile(utils.R8LIB_JAR, os.path.join(temp_dir, 'r8lib.jar')) result_per_shrinker_per_app = [] # If we are running on golem we kill all java processes after the run # to ensure no hanging gradle daemons. with EnsureNoGradleAlive(options.golem): for (app, repo) in options.apps: if app.skip: continue result_per_shrinker_per_app.append( (app, GetResultsForApp(app, repo, options, temp_dir))) return LogResultsForApps(result_per_shrinker_per_app, options)
def download_sha(app_sha, internal, quiet=False): if internal: utils.DownloadFromX20(app_sha) else: utils.DownloadFromGoogleCloudStorage(app_sha, quiet=quiet)
def Main(argv): (options, args) = ParseOptions(argv) if (len(args) > 0): raise Exception('Unsupported arguments') if not utils.is_bot() and not options.dry_run: raise Exception('You are not a bot, don\'t archive builds. ' + 'Use --dry-run to test locally') if (options.dry_run_output and (not os.path.exists(options.dry_run_output) or not os.path.isdir(options.dry_run_output))): raise Exception(options.dry_run_output + ' does not exist or is not a directory') if utils.is_bot(): archive.SetRLimitToMax() # Make sure bazel is extracted in third_party. utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE) utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE) # Only handling versioned desugar_jdk_libs. is_master = False with utils.TempDir() as checkout_dir: git_utils.GitClone( 'https://github.com/' + options.github_account + '/' + LIBRARY_NAME, checkout_dir) with utils.ChangedWorkingDirectory(checkout_dir): version = GetVersion() destination = archive.GetVersionDestination( 'gs://', LIBRARY_NAME + '/' + version, is_master) if utils.cloud_storage_exists(destination) and not options.dry_run: raise Exception('Target archive directory %s already exists' % destination) bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin', 'bazel') cmd = [bazel, 'build', '--host_force_python=PY2', 'maven_release'] utils.PrintCmd(cmd) subprocess.check_call(cmd) cmd = [bazel, 'shutdown'] utils.PrintCmd(cmd) subprocess.check_call(cmd) # Compile the stubs for conversion files compilation. stub_compiled_folder = os.path.join(checkout_dir, 'stubs') os.mkdir(stub_compiled_folder) all_stubs = GetFilesInFolder( os.path.join(CONVERSION_FOLDER, 'stubs')) cmd = [JDK8_JAVAC, '-d', stub_compiled_folder] + all_stubs utils.PrintCmd(cmd) subprocess.check_call(cmd) # Compile the conversion files. conversions_compiled_folder = os.path.join(checkout_dir, 'conversions') os.mkdir(conversions_compiled_folder) all_conversions = GetFilesInFolder( os.path.join(CONVERSION_FOLDER, 'conversions')) cmd = [ JDK8_JAVAC, '-cp', stub_compiled_folder, '-d', conversions_compiled_folder ] + all_conversions utils.PrintCmd(cmd) subprocess.check_call(cmd) # Locate the library jar and the maven zip with the jar from the # bazel build. library_jar = os.path.join('bazel-bin', 'src', 'share', 'classes', 'java', 'libjava.jar') maven_zip = os.path.join('bazel-bin', LIBRARY_NAME + '.zip') # Make a writable copy of the jar. jar_folder = os.path.join(checkout_dir, 'jar') os.mkdir(jar_folder) shutil.copy(library_jar, jar_folder) library_jar = os.path.join(jar_folder, 'libjava.jar') os.chmod(library_jar, 0o777) # Add conversion classes into the jar. all_compiled_conversions = GetFilesInFolder( conversions_compiled_folder) with zipfile.ZipFile(library_jar, mode='a', allowZip64=True) as jar: for clazz in all_compiled_conversions: jar.write(clazz, arcname=os.path.relpath( clazz, conversions_compiled_folder), compress_type=zipfile.ZIP_DEFLATED) storage_path = LIBRARY_NAME + '/' + version # Upload the jar file with the library. destination = archive.GetUploadDestination(storage_path, LIBRARY_NAME + '.jar', is_master) Upload(options, library_jar, storage_path, destination, is_master) # Upload the maven zip file with the library. destination = archive.GetUploadDestination(storage_path, LIBRARY_NAME + '.zip', is_master) Upload(options, maven_zip, storage_path, destination, is_master) # Upload the jar file for accessing GCS as a maven repro. maven_destination = archive.GetUploadDestination( utils.get_maven_path('desugar_jdk_libs', version), 'desugar_jdk_libs-%s.jar' % version, is_master) if options.dry_run: print('Dry run, not actually creating maven repo') else: utils.upload_file_to_cloud_storage(library_jar, maven_destination) print('Maven repo root available at: %s' % archive.GetMavenUrl(is_master))
def Main(argv): (options, args) = ParseOptions(argv) if (len(args) > 0): raise Exception('Unsupported arguments') if not utils.is_bot() and not options.dry_run: raise Exception('You are not a bot, don\'t archive builds. ' + 'Use --dry-run to test locally') if (options.dry_run_output and (not os.path.exists(options.dry_run_output) or not os.path.isdir(options.dry_run_output))): raise Exception(options.dry_run_output + ' does not exist or is not a directory') if utils.is_bot(): archive.SetRLimitToMax() # Make sure bazel is extracted in third_party. utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE) # Only handling versioned desugar_jdk_libs. is_master = False with utils.TempDir() as checkout_dir: git_utils.GitClone( 'https://github.com/' + options.github_account + '/' + LIBRARY_NAME, checkout_dir) with utils.ChangedWorkingDirectory(checkout_dir): version = GetVersion() destination = archive.GetVersionDestination( 'gs://', LIBRARY_NAME + '/' + version, is_master) if utils.cloud_storage_exists(destination) and not options.dry_run: raise Exception('Target archive directory %s already exists' % destination) bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin', 'bazel') cmd = [bazel, 'build', 'maven_release'] utils.PrintCmd(cmd) subprocess.check_call(cmd) cmd = [bazel, 'shutdown'] utils.PrintCmd(cmd) subprocess.check_call(cmd) # Locate the library jar and the maven zip with the jar from the # bazel build. library_jar = os.path.join('bazel-bin', 'src', 'share', 'classes', 'java', 'libjava.jar') maven_zip = os.path.join('bazel-bin', LIBRARY_NAME + '.zip') storage_path = LIBRARY_NAME + '/' + version # Upload the jar file with the library. destination = archive.GetUploadDestination(storage_path, LIBRARY_NAME + '.jar', is_master) Upload(options, library_jar, storage_path, destination, is_master) # Upload the maven zip file with the library. destination = archive.GetUploadDestination(storage_path, LIBRARY_NAME + '.zip', is_master) Upload(options, maven_zip, storage_path, destination, is_master) # Upload the jar file for accessing GCS as a maven repro. maven_destination = archive.GetUploadDestination( utils.get_maven_path('desugar_jdk_libs', version), 'desugar_jdk_libs-%s.jar' % version, is_master) if options.dry_run: print('Dry run, not actually creating maven repo') else: utils.upload_file_to_cloud_storage(library_jar, maven_destination) print('Maven repo root available at: %s' % archive.GetMavenUrl(is_master))
def Main(): (options, args) = ParseOptions() if utils.is_bot(): gradle.RunGradle(['--no-daemon', 'clean']) desugar_jdk_json_dir = None if options.desugared_library_configuration: if options.desugared_library_configuration != 'jdk11': print("Only value supported for --desugared-library is 'jdk11'") exit(1) desugar_jdk_json_dir = 'src/library_desugar/jdk11' desugar_jdk_libs = None if options.desugared_library: if options.desugared_library != 'HEAD': print("Only value supported for --desugared-library is 'HEAD'") exit(1) desugar_jdk_libs_dir = 'build/desugar_jdk_libs' shutil.rmtree(desugar_jdk_libs_dir, ignore_errors=True) os.makedirs(desugar_jdk_libs_dir) print('Building desugared library.') with utils.TempDir() as checkout_dir: archive_desugar_jdk_libs.CloneDesugaredLibrary( 'google', checkout_dir) # Make sure bazel is extracted in third_party. utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE) utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE) utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE) (library_jar, maven_zip) = archive_desugar_jdk_libs.BuildDesugaredLibrary( checkout_dir, 'jdk11' if options.desugared_library_configuration == 'jdk11' else 'jdk8') desugar_jdk_libs = os.path.join(desugar_jdk_libs_dir, os.path.basename(library_jar)) shutil.copyfile(library_jar, desugar_jdk_libs) print('Desugared library for test in ' + desugar_jdk_libs) gradle_args = ['--stacktrace'] if utils.is_bot(): # Bots don't like dangling processes. gradle_args.append('--no-daemon') # Set all necessary Gradle properties and options first. if options.shard_count: assert options.shard_number gradle_args.append('-Pshard_count=%s' % options.shard_count) gradle_args.append('-Pshard_number=%s' % options.shard_number) if options.verbose: gradle_args.append('-Pprint_test_stdout') if options.no_internal: gradle_args.append('-Pno_internal') if options.only_internal: gradle_args.append('-Ponly_internal') if options.all_tests: gradle_args.append('-Pall_tests') if options.slow_tests: gradle_args.append('-Pslow_tests=1') if options.tool: gradle_args.append('-Ptool=%s' % options.tool) if options.one_line_per_test: gradle_args.append('-Pone_line_per_test') if options.jctf: gradle_args.append('-Pjctf') if options.only_jctf: gradle_args.append('-Ponly_jctf') if options.test_namespace: gradle_args.append('-Ptest_namespace=%s' % options.test_namespace) if options.jctf_compile_only: gradle_args.append('-Pjctf_compile_only') if options.disable_assertions: gradle_args.append('-Pdisable_assertions') if options.with_code_coverage: gradle_args.append('-Pwith_code_coverage') if options.print_full_stacktraces: gradle_args.append('-Pprint_full_stacktraces') if options.print_obfuscated_stacktraces: gradle_args.append('-Pprint_obfuscated_stacktraces') if os.name == 'nt': # temporary hack gradle_args.append('-Pno_internal') gradle_args.append('-x') gradle_args.append('createJctfTests') gradle_args.append('-x') gradle_args.append('jctfCommonJar') gradle_args.append('-x') gradle_args.append('jctfTestsClasses') if options.test_dir: gradle_args.append('-Ptest_dir=' + options.test_dir) if not os.path.exists(options.test_dir): os.makedirs(options.test_dir) if options.java_home: gradle_args.append('-Dorg.gradle.java.home=' + options.java_home) if options.java_max_memory_size: gradle_args.append('-Ptest_xmx=' + options.java_max_memory_size) if options.generate_golden_files_to: gradle_args.append('-Pgenerate_golden_files_to=' + options.generate_golden_files_to) if not os.path.exists(options.generate_golden_files_to): os.makedirs(options.generate_golden_files_to) gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1()) if options.use_golden_files_in: gradle_args.append('-Puse_golden_files_in=' + options.use_golden_files_in) if not os.path.exists(options.use_golden_files_in): os.makedirs(options.use_golden_files_in) gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1()) if (not options.no_r8lib) and options.r8lib_no_deps: print( 'Cannot run tests on r8lib with and without deps. R8lib is now default target.' ) exit(1) if not options.no_r8lib: gradle_args.append('-Pr8lib') # Force gradle to build a version of r8lib without dependencies for # BootstrapCurrentEqualityTest. gradle_args.append('R8LibNoDeps') gradle_args.append('R8Retrace') if options.r8lib_no_deps: gradle_args.append('-Pr8lib_no_deps') if options.worktree: gradle_args.append('-g=' + os.path.join(utils.REPO_ROOT, ".gradle_user_home")) gradle_args.append('--no-daemon') if options.debug_agent: gradle_args.append('--no-daemon') if desugar_jdk_json_dir: gradle_args.append('-Pdesugar_jdk_json_dir=' + desugar_jdk_json_dir) if desugar_jdk_libs: gradle_args.append('-Pdesugar_jdk_libs=' + desugar_jdk_libs) # Build an R8 with dependencies for bootstrapping tests before adding test sources. gradle_args.append('r8WithDeps') gradle_args.append('r8WithDeps11') gradle_args.append('r8WithRelocatedDeps') gradle_args.append('r8WithRelocatedDeps11') # Add Gradle tasks gradle_args.append('cleanTest') gradle_args.append('test') if options.debug_agent: gradle_args.append('--debug-jvm') if options.fail_fast: gradle_args.append('--fail-fast') if options.failed: args = compute_failed_tests(args) if args is None: return 1 if len(args) == 0: print "No failing tests" return 0 # Test filtering. Must always follow the 'test' task. for testFilter in args: gradle_args.append('--tests') gradle_args.append(testFilter) if options.with_code_coverage: # Create Jacoco report after tests. gradle_args.append('jacocoTestReport') if options.use_golden_files_in: sha1 = '%s' % utils.get_HEAD_sha1() with utils.ChangedWorkingDirectory(options.use_golden_files_in): utils.download_file_from_cloud_storage( 'gs://r8-test-results/golden-files/%s.tar.gz' % sha1, '%s.tar.gz' % sha1) utils.unpack_archive('%s.tar.gz' % sha1) print_stacks_timeout = options.print_hanging_stacks if (utils.is_bot() and not utils.IsWindows()) or print_stacks_timeout > -1: timestamp_file = os.path.join(utils.BUILD, 'last_test_time') if os.path.exists(timestamp_file): os.remove(timestamp_file) gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file) print_stacks_timeout = (print_stacks_timeout if print_stacks_timeout != -1 else TIMEOUT_HANDLER_PERIOD) thread.start_new_thread(timeout_handler, ( timestamp_file, print_stacks_timeout, )) rotate_test_reports() if options.only_jctf: # Note: not setting -Pruntimes will run with all available runtimes. return_code = gradle.RunGradle(gradle_args, throw_on_failure=False) return archive_and_return(return_code, options) # Now run tests on selected runtime(s). if options.runtimes: if options.dex_vm != 'default': print 'Unexpected runtimes and dex_vm argument: ' + options.dex_vm sys.exit(1) if options.runtimes == 'empty': # Set runtimes with no content will configure no runtimes. gradle_args.append('-Pruntimes=') elif options.runtimes == 'all': # An unset runtimes will configure all runtimes pass else: prefixes = [ prefix.strip() for prefix in options.runtimes.split(':') ] runtimes = [] for prefix in prefixes: matches = [ rt for rt in VALID_RUNTIMES if rt.startswith(prefix) ] if len(matches) == 0: print "Invalid runtime prefix '%s'." % prefix print "Must be just 'all', 'empty'," \ " or a prefix of %s" % ', '.join(VALID_RUNTIMES) sys.exit(1) runtimes.extend(matches) gradle_args.append('-Pruntimes=%s' % ':'.join(runtimes)) return_code = gradle.RunGradle(gradle_args, throw_on_failure=False) return archive_and_return(return_code, options) # Legacy testing populates the runtimes based on dex_vm. vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS for art_vm in vms_to_test: vm_suffix = "_" + options.dex_vm_kind if art_vm != "default" else "" runtimes = ['dex-' + art_vm] # Only append the "none" runtime and JVMs if running on the "default" DEX VM. if art_vm == "default": # TODO(b/170454076): Remove special casing for bot when rex-script has # been migrated to account for runtimes. if utils.is_bot(): runtimes.extend(['jdk11', 'none']) else: runtimes.extend(['jdk8', 'jdk9', 'jdk11', 'none']) return_code = gradle.RunGradle(gradle_args + [ '-Pdex_vm=%s' % art_vm + vm_suffix, '-Pruntimes=%s' % ':'.join(runtimes), ], throw_on_failure=False) if options.generate_golden_files_to: sha1 = '%s' % utils.get_HEAD_sha1() with utils.ChangedWorkingDirectory( options.generate_golden_files_to): archive = utils.create_archive(sha1) utils.upload_file_to_cloud_storage( archive, 'gs://r8-test-results/golden-files/' + archive) if return_code != 0: return archive_and_return(return_code, options) return 0
def download_benchmarks(): utils.DownloadFromGoogleCloudStorage( os.path.join(utils.THIRD_PARTY, 'retrace_benchmark') + '.tar.gz.sha1')