def run(out, is_r8lib=False): if out == None: print 'Need to supply output zip with --out.' exit(1) # Build the R8 no deps artifact. if not is_r8lib: gradle.RunGradleExcludeDeps([utils.R8]) else: gradle.RunGradle([utils.R8LIB, '-Pno_internal']) # Create directory structure for this version. version = determine_version() with utils.TempDir() as tmp_dir: version_dir = join(tmp_dir, utils.get_maven_path('r8', version)) makedirs(version_dir) # Write the pom file. pom_file = join(version_dir, 'r8-' + version + '.pom') write_pom_file(version, pom_file, is_r8lib) # Copy the jar to the output. target_jar = join(version_dir, 'r8-' + version + '.jar') copyfile(utils.R8LIB_JAR if is_r8lib else utils.R8_JAR, target_jar) # Create check sums. write_md5_for(target_jar) write_md5_for(pom_file) write_sha1_for(target_jar) write_sha1_for(pom_file) # Zip it up - make_archive will append zip to the file, so remove. assert out.endswith('.zip') base_no_zip = out[0:len(out) - 4] make_archive(base_no_zip, 'zip', tmp_dir)
def main(argv): options = parse_options(argv) outdir = options.out if outdir == None: print 'Need to supply output dir with --out.' exit(1) # Build the R8 no deps artifact. gradle.RunGradleExcludeDeps([utils.R8]) # Create directory structure for this version. version = determine_version() with utils.TempDir() as tmp_dir: version_dir = join(tmp_dir, utils.get_maven_path(version)) makedirs(version_dir) # Write the pom file. pom_file = join(version_dir, 'r8-' + version + '.pom') write_pom_file(version, pom_file) # Copy the jar to the output. target_jar = join(version_dir, 'r8-' + version + '.jar') copyfile(utils.R8_JAR, target_jar) # Create check sums. write_md5_for(target_jar) write_md5_for(pom_file) write_sha1_for(target_jar) write_sha1_for(pom_file) # Zip it up. make_archive(join(outdir, 'r8'), 'zip', tmp_dir)
def Main(): if not 'BUILDBOT_BUILDERNAME' in os.environ: raise Exception('You are not a bot, don\'t archive builds') # Create maven release first which uses a build that exclude dependencies. create_maven_release.main(["--out", utils.LIBS]) # Generate and copy the build that exclude dependencies. gradle.RunGradleExcludeDeps([utils.R8, utils.R8_SRC]) shutil.copyfile(utils.R8_JAR, utils.R8_EXCLUDE_DEPS_JAR) # Ensure all archived artifacts has been built before archiving. gradle.RunGradle([utils.D8, utils.R8, utils.COMPATDX, utils.COMPATPROGUARD]) version = GetVersion() is_master = IsMaster(version) if is_master: # On master we use the git hash to archive with print 'On master, using git hash for archiving' version = GetGitHash() destination = GetVersionDestination('gs://', version, is_master) if utils.cloud_storage_exists(destination): raise Exception('Target archive directory %s already exists' % destination) with utils.TempDir() as temp: version_file = os.path.join(temp, 'r8-version.properties') with open(version_file,'w') as version_writer: version_writer.write('version.sha=' + GetGitHash() + '\n') version_writer.write( 'releaser=go/r8bot (' + os.environ.get('BUILDBOT_SLAVENAME') + ')\n') version_writer.write('version-file.version.code=1\n') for file in [utils.D8_JAR, utils.R8_JAR, utils.R8_SRC_JAR, utils.R8_EXCLUDE_DEPS_JAR, utils.COMPATDX_JAR, utils.COMPATPROGUARD_JAR, utils.MAVEN_ZIP, utils.GENERATED_LICENSE]: file_name = os.path.basename(file) tagged_jar = os.path.join(temp, file_name) shutil.copyfile(file, tagged_jar) if file_name.endswith('.jar') and not file_name.endswith('-src.jar'): with zipfile.ZipFile(tagged_jar, 'a') as zip: zip.write(version_file, os.path.basename(version_file)) destination = GetUploadDestination(version, file_name, is_master) print('Uploading %s to %s' % (tagged_jar, destination)) utils.upload_file_to_cloud_storage(tagged_jar, destination) print('File available at: %s' % GetUrl(version, file_name, is_master)) if file == utils.R8_JAR: # Upload R8 to a maven compatible location. maven_dst = GetUploadDestination(utils.get_maven_path(version), 'r8-%s.jar' % version, is_master) utils.upload_file_to_cloud_storage(tagged_jar, maven_dst) print('Maven repo root available at: %s' % GetMavenUrl(is_master))
def generate_maven_zip(name, version, pom_file, jar_file, out): with utils.TempDir() as tmp_dir: # Create the base maven version directory version_dir = join(tmp_dir, utils.get_maven_path(name, version)) makedirs(version_dir) # Write the pom file. pom_file_location = join(version_dir, name + '-' + version + '.pom') copyfile(pom_file, pom_file_location) # Write the jar file. jar_file_location = join(version_dir, name + '-' + version + '.jar') copyfile(jar_file, jar_file_location) # Create check sums. write_md5_for(jar_file_location) write_md5_for(pom_file_location) write_sha1_for(jar_file_location) write_sha1_for(pom_file_location) # Zip it up - make_archive will append zip to the file, so remove. assert out.endswith('.zip') base_no_zip = out[0:len(out) - 4] make_archive(base_no_zip, 'zip', tmp_dir)
def Main(): (options, args) = ParseOptions() if not utils.is_bot() and not options.dry_run: raise Exception('You are not a bot, don\'t archive builds') if utils.is_bot(): SetRLimitToMax() PrintResourceInfo() # Create maven release which uses a build that exclude dependencies. create_maven_release.run(utils.MAVEN_ZIP) create_maven_release.run(utils.MAVEN_ZIP_LIB, is_r8lib=True) # Generate and copy a full build without dependencies. gradle.RunGradleExcludeDeps([utils.R8, utils.R8_SRC]) shutil.copyfile(utils.R8_JAR, utils.R8_FULL_EXCLUDE_DEPS_JAR) # Ensure all archived artifacts has been built before archiving. # The target tasks postfixed by 'lib' depend on the actual target task so # building it invokes the original task first. # The '-Pno_internal' flag is important because we generate the lib based on uses in tests. gradle.RunGradle([ utils.R8, utils.D8, utils.COMPATDX, utils.COMPATPROGUARD, utils.R8LIB, utils.R8LIB_NO_DEPS, utils.COMPATDXLIB, utils.COMPATPROGUARDLIB, '-Pno_internal' ]) version = GetVersion() is_master = IsMaster(version) if is_master: # On master we use the git hash to archive with print 'On master, using git hash for archiving' version = GetGitHash() destination = GetVersionDestination('gs://', version, is_master) if utils.cloud_storage_exists(destination) and not options.dry_run: raise Exception('Target archive directory %s already exists' % destination) with utils.TempDir() as temp: version_file = os.path.join(temp, 'r8-version.properties') with open(version_file, 'w') as version_writer: version_writer.write('version.sha=' + GetGitHash() + '\n') version_writer.write('releaser=go/r8bot (' + os.environ.get('SWARMING_BOT_ID') + ')\n') version_writer.write('version-file.version.code=1\n') for file in [ utils.D8_JAR, utils.R8_JAR, utils.R8LIB_JAR, utils.R8LIB_JAR + '.map', utils.R8_SRC_JAR, utils.R8_FULL_EXCLUDE_DEPS_JAR, utils.R8LIB_EXCLUDE_DEPS_JAR, utils.R8LIB_EXCLUDE_DEPS_JAR + '.map', utils.COMPATDX_JAR, utils.COMPATDXLIB_JAR, utils.COMPATDXLIB_JAR + '.map', utils.COMPATPROGUARD_JAR, utils.COMPATPROGUARDLIB_JAR, utils.COMPATPROGUARDLIB_JAR + '.map', utils.MAVEN_ZIP, utils.MAVEN_ZIP_LIB, utils.GENERATED_LICENSE, ]: file_name = os.path.basename(file) tagged_jar = os.path.join(temp, file_name) shutil.copyfile(file, tagged_jar) if file_name.endswith( '.jar') and not file_name.endswith('-src.jar'): with zipfile.ZipFile(tagged_jar, 'a') as zip: zip.write(version_file, os.path.basename(version_file)) destination = GetUploadDestination(version, file_name, is_master) print('Uploading %s to %s' % (tagged_jar, destination)) if options.dry_run: print('Dry run, not actually uploading') else: utils.upload_file_to_cloud_storage(tagged_jar, destination) print('File available at: %s' % GetUrl(version, file_name, is_master)) if file == utils.R8_JAR: # Upload R8 to a maven compatible location. maven_dst = GetUploadDestination(utils.get_maven_path(version), 'r8-%s.jar' % version, is_master) if options.dry_run: print('Dry run, not actually creating maven repo') else: utils.upload_file_to_cloud_storage(tagged_jar, maven_dst) print('Maven repo root available at: %s' % GetMavenUrl(is_master))
def Main(argv): (options, args) = ParseOptions(argv) if (len(args) > 0): raise Exception('Unsupported arguments') if not utils.is_bot() and not (options.dry_run or options.build_only): raise Exception('You are not a bot, don\'t archive builds. ' + 'Use --dry-run or --build-only to test locally') if options.dry_run_output: MustBeExistingDirectory(options.dry_run_output) if options.build_only: MustBeExistingDirectory(options.build_only) if utils.is_bot(): archive.SetRLimitToMax() # Make sure bazel is extracted in third_party. utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE) utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE) utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE) if options.build_only: with utils.TempDir() as checkout_dir: CloneDesugaredLibrary(options.github_account, checkout_dir) (library_jar, maven_zip) = BuildDesugaredLibrary(checkout_dir, "jdk8") shutil.copyfile( library_jar, os.path.join(options.build_only, os.path.basename(library_jar))) shutil.copyfile( maven_zip, os.path.join(options.build_only, os.path.basename(maven_zip))) return # Only handling versioned desugar_jdk_libs. is_master = False with utils.TempDir() as checkout_dir: CloneDesugaredLibrary(options.github_account, checkout_dir) version = GetVersion(os.path.join(checkout_dir, VERSION_FILE)) destination = archive.GetVersionDestination( 'gs://', LIBRARY_NAME + '/' + version, is_master) if utils.cloud_storage_exists(destination) and not options.dry_run: raise Exception('Target archive directory %s already exists' % destination) (library_jar, maven_zip) = BuildDesugaredLibrary(checkout_dir, "jdk8") storage_path = LIBRARY_NAME + '/' + version # Upload the jar file with the library. destination = archive.GetUploadDestination(storage_path, LIBRARY_NAME + '.jar', is_master) Upload(options, library_jar, storage_path, destination, is_master) # Upload the maven zip file with the library. destination = archive.GetUploadDestination(storage_path, LIBRARY_NAME + '.zip', is_master) Upload(options, maven_zip, storage_path, destination, is_master) # Upload the jar file for accessing GCS as a maven repro. maven_destination = archive.GetUploadDestination( utils.get_maven_path('desugar_jdk_libs', version), 'desugar_jdk_libs-%s.jar' % version, is_master) if options.dry_run: print('Dry run, not actually creating maven repo') else: utils.upload_file_to_cloud_storage(library_jar, maven_destination) print('Maven repo root available at: %s' % archive.GetMavenUrl(is_master))
def Main(): (options, args) = ParseOptions() if not utils.is_bot() and not options.dry_run: raise Exception('You are not a bot, don\'t archive builds. ' + 'Use --dry-run to test locally') if (options.dry_run_output and (not os.path.exists(options.dry_run_output) or not os.path.isdir(options.dry_run_output))): raise Exception(options.dry_run_output + ' does not exist or is not a directory') if utils.is_bot() and not utils.IsWindows(): SetRLimitToMax() if not utils.IsWindows(): PrintResourceInfo() # Create maven release which uses a build that exclude dependencies. create_maven_release.generate_r8_maven_zip(utils.MAVEN_ZIP) create_maven_release.generate_r8_maven_zip(utils.MAVEN_ZIP_LIB, is_r8lib=True) # Generate and copy a full build without dependencies. gradle.RunGradleExcludeDeps([utils.R8, utils.R8_SRC]) shutil.copyfile(utils.R8_JAR, utils.R8_FULL_EXCLUDE_DEPS_JAR) # Ensure all archived artifacts has been built before archiving. # The target tasks postfixed by 'lib' depend on the actual target task so # building it invokes the original task first. # The '-Pno_internal' flag is important because we generate the lib based on uses in tests. gradle.RunGradle([ utils.R8, utils.D8, utils.R8LIB, utils.R8LIB_NO_DEPS, utils.R8RETRACE, utils.R8RETRACE_NO_DEPS, utils.LIBRARY_DESUGAR_CONVERSIONS, '-Pno_internal' ]) # Create maven release of the desuage_jdk_libs configuration. This require # an r8.jar with dependencies to have been built. create_maven_release.generate_desugar_configuration_maven_zip( utils.DESUGAR_CONFIGURATION_MAVEN_ZIP) version = GetVersion() is_master = IsMaster(version) if is_master: # On master we use the git hash to archive with print('On master, using git hash for archiving') version = GetGitHash() destination = GetVersionDestination('gs://', version, is_master) if utils.cloud_storage_exists(destination) and not options.dry_run: raise Exception('Target archive directory %s already exists' % destination) with utils.TempDir() as temp: # Create pom file for our maven repository that we build for testing. default_pom_file = os.path.join(temp, 'r8.pom') create_maven_release.write_default_r8_pom_file(default_pom_file, version) version_file = os.path.join(temp, 'r8-version.properties') with open(version_file, 'w') as version_writer: version_writer.write('version.sha=' + GetGitHash() + '\n') if not os.environ.get('SWARMING_BOT_ID') and not options.dry_run: raise Exception('Environment variable SWARMING_BOT_ID not set') releaser = \ ("<local developer build>" if options.dry_run else 'releaser=go/r8bot (' + os.environ.get('SWARMING_BOT_ID') + ')\n') version_writer.write(releaser) version_writer.write('version-file.version.code=1\n') for file in [ utils.D8_JAR, utils.R8_JAR, utils.R8LIB_JAR, utils.R8LIB_JAR + '.map', utils.R8_SRC_JAR, utils.R8_FULL_EXCLUDE_DEPS_JAR, utils.R8LIB_EXCLUDE_DEPS_JAR, utils.R8LIB_EXCLUDE_DEPS_JAR + '.map', utils.R8RETRACE_JAR, utils.R8RETRACE_EXCLUDE_DEPS_JAR, utils.MAVEN_ZIP, utils.MAVEN_ZIP_LIB, utils.DESUGAR_CONFIGURATION, utils.DESUGAR_CONFIGURATION_MAVEN_ZIP, utils.GENERATED_LICENSE, ]: file_name = os.path.basename(file) tagged_jar = os.path.join(temp, file_name) shutil.copyfile(file, tagged_jar) if file_name.endswith( '.jar') and not file_name.endswith('-src.jar'): with zipfile.ZipFile(tagged_jar, 'a') as zip: zip.write(version_file, os.path.basename(version_file)) destination = GetUploadDestination(version, file_name, is_master) print('Uploading %s to %s' % (tagged_jar, destination)) if options.dry_run: if options.dry_run_output: dry_run_destination = os.path.join(options.dry_run_output, file_name) print('Dry run, not actually uploading. Copying to ' + dry_run_destination) shutil.copyfile(tagged_jar, dry_run_destination) else: print('Dry run, not actually uploading') else: utils.upload_file_to_cloud_storage(tagged_jar, destination) print('File available at: %s' % GetUrl(version, file_name, is_master)) # Upload R8 to a maven compatible location. if file == utils.R8_JAR: maven_dst = GetUploadDestination( utils.get_maven_path('r8', version), 'r8-%s.jar' % version, is_master) maven_pom_dst = GetUploadDestination( utils.get_maven_path('r8', version), 'r8-%s.pom' % version, is_master) if options.dry_run: print('Dry run, not actually creating maven repo for R8') else: utils.upload_file_to_cloud_storage(tagged_jar, maven_dst) utils.upload_file_to_cloud_storage(default_pom_file, maven_pom_dst) print('Maven repo root available at: %s' % GetMavenUrl(is_master)) # Upload desugar_jdk_libs configuration to a maven compatible location. if file == utils.DESUGAR_CONFIGURATION: jar_basename = 'desugar_jdk_libs_configuration.jar' jar_version_name = 'desugar_jdk_libs_configuration-%s.jar' % version maven_dst = GetUploadDestination( utils.get_maven_path('desugar_jdk_libs_configuration', version), jar_version_name, is_master) with utils.TempDir() as tmp_dir: desugar_jdk_libs_configuration_jar = os.path.join( tmp_dir, jar_version_name) create_maven_release.generate_jar_with_desugar_configuration( utils.DESUGAR_CONFIGURATION, utils.DESUGAR_IMPLEMENTATION, utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP, desugar_jdk_libs_configuration_jar) if options.dry_run: print( 'Dry run, not actually creating maven repo for ' + 'desugar configuration.') if options.dry_run_output: shutil.copyfile( desugar_jdk_libs_configuration_jar, os.path.join(options.dry_run_output, jar_version_name)) else: utils.upload_file_to_cloud_storage( desugar_jdk_libs_configuration_jar, maven_dst) print('Maven repo root available at: %s' % GetMavenUrl(is_master)) # Also archive the jar as non maven destination for Google3 jar_destination = GetUploadDestination( version, jar_basename, is_master) utils.upload_file_to_cloud_storage( desugar_jdk_libs_configuration_jar, jar_destination)
def Main(argv): (options, args) = ParseOptions(argv) if (len(args) > 0): raise Exception('Unsupported arguments') if not utils.is_bot() and not options.dry_run: raise Exception('You are not a bot, don\'t archive builds. ' + 'Use --dry-run to test locally') if (options.dry_run_output and (not os.path.exists(options.dry_run_output) or not os.path.isdir(options.dry_run_output))): raise Exception(options.dry_run_output + ' does not exist or is not a directory') if utils.is_bot(): archive.SetRLimitToMax() # Make sure bazel is extracted in third_party. utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE) utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE) # Only handling versioned desugar_jdk_libs. is_master = False with utils.TempDir() as checkout_dir: git_utils.GitClone( 'https://github.com/' + options.github_account + '/' + LIBRARY_NAME, checkout_dir) with utils.ChangedWorkingDirectory(checkout_dir): version = GetVersion() destination = archive.GetVersionDestination( 'gs://', LIBRARY_NAME + '/' + version, is_master) if utils.cloud_storage_exists(destination) and not options.dry_run: raise Exception('Target archive directory %s already exists' % destination) bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin', 'bazel') cmd = [bazel, 'build', '--host_force_python=PY2', 'maven_release'] utils.PrintCmd(cmd) subprocess.check_call(cmd) cmd = [bazel, 'shutdown'] utils.PrintCmd(cmd) subprocess.check_call(cmd) # Compile the stubs for conversion files compilation. stub_compiled_folder = os.path.join(checkout_dir, 'stubs') os.mkdir(stub_compiled_folder) all_stubs = GetFilesInFolder( os.path.join(CONVERSION_FOLDER, 'stubs')) cmd = [JDK8_JAVAC, '-d', stub_compiled_folder] + all_stubs utils.PrintCmd(cmd) subprocess.check_call(cmd) # Compile the conversion files. conversions_compiled_folder = os.path.join(checkout_dir, 'conversions') os.mkdir(conversions_compiled_folder) all_conversions = GetFilesInFolder( os.path.join(CONVERSION_FOLDER, 'conversions')) cmd = [ JDK8_JAVAC, '-cp', stub_compiled_folder, '-d', conversions_compiled_folder ] + all_conversions utils.PrintCmd(cmd) subprocess.check_call(cmd) # Locate the library jar and the maven zip with the jar from the # bazel build. library_jar = os.path.join('bazel-bin', 'src', 'share', 'classes', 'java', 'libjava.jar') maven_zip = os.path.join('bazel-bin', LIBRARY_NAME + '.zip') # Make a writable copy of the jar. jar_folder = os.path.join(checkout_dir, 'jar') os.mkdir(jar_folder) shutil.copy(library_jar, jar_folder) library_jar = os.path.join(jar_folder, 'libjava.jar') os.chmod(library_jar, 0o777) # Add conversion classes into the jar. all_compiled_conversions = GetFilesInFolder( conversions_compiled_folder) with zipfile.ZipFile(library_jar, mode='a', allowZip64=True) as jar: for clazz in all_compiled_conversions: jar.write(clazz, arcname=os.path.relpath( clazz, conversions_compiled_folder), compress_type=zipfile.ZIP_DEFLATED) storage_path = LIBRARY_NAME + '/' + version # Upload the jar file with the library. destination = archive.GetUploadDestination(storage_path, LIBRARY_NAME + '.jar', is_master) Upload(options, library_jar, storage_path, destination, is_master) # Upload the maven zip file with the library. destination = archive.GetUploadDestination(storage_path, LIBRARY_NAME + '.zip', is_master) Upload(options, maven_zip, storage_path, destination, is_master) # Upload the jar file for accessing GCS as a maven repro. maven_destination = archive.GetUploadDestination( utils.get_maven_path('desugar_jdk_libs', version), 'desugar_jdk_libs-%s.jar' % version, is_master) if options.dry_run: print('Dry run, not actually creating maven repo') else: utils.upload_file_to_cloud_storage(library_jar, maven_destination) print('Maven repo root available at: %s' % archive.GetMavenUrl(is_master))
def Main(argv): (options, args) = ParseOptions(argv) if (len(args) > 0): raise Exception('Unsupported arguments') if not utils.is_bot() and not options.dry_run: raise Exception('You are not a bot, don\'t archive builds. ' + 'Use --dry-run to test locally') if (options.dry_run_output and (not os.path.exists(options.dry_run_output) or not os.path.isdir(options.dry_run_output))): raise Exception(options.dry_run_output + ' does not exist or is not a directory') if utils.is_bot(): archive.SetRLimitToMax() # Make sure bazel is extracted in third_party. utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE) # Only handling versioned desugar_jdk_libs. is_master = False with utils.TempDir() as checkout_dir: git_utils.GitClone( 'https://github.com/' + options.github_account + '/' + LIBRARY_NAME, checkout_dir) with utils.ChangedWorkingDirectory(checkout_dir): version = GetVersion() destination = archive.GetVersionDestination( 'gs://', LIBRARY_NAME + '/' + version, is_master) if utils.cloud_storage_exists(destination) and not options.dry_run: raise Exception('Target archive directory %s already exists' % destination) bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin', 'bazel') cmd = [bazel, 'build', 'maven_release'] utils.PrintCmd(cmd) subprocess.check_call(cmd) cmd = [bazel, 'shutdown'] utils.PrintCmd(cmd) subprocess.check_call(cmd) # Locate the library jar and the maven zip with the jar from the # bazel build. library_jar = os.path.join('bazel-bin', 'src', 'share', 'classes', 'java', 'libjava.jar') maven_zip = os.path.join('bazel-bin', LIBRARY_NAME + '.zip') storage_path = LIBRARY_NAME + '/' + version # Upload the jar file with the library. destination = archive.GetUploadDestination(storage_path, LIBRARY_NAME + '.jar', is_master) Upload(options, library_jar, storage_path, destination, is_master) # Upload the maven zip file with the library. destination = archive.GetUploadDestination(storage_path, LIBRARY_NAME + '.zip', is_master) Upload(options, maven_zip, storage_path, destination, is_master) # Upload the jar file for accessing GCS as a maven repro. maven_destination = archive.GetUploadDestination( utils.get_maven_path('desugar_jdk_libs', version), 'desugar_jdk_libs-%s.jar' % version, is_master) if options.dry_run: print('Dry run, not actually creating maven repo') else: utils.upload_file_to_cloud_storage(library_jar, maven_destination) print('Maven repo root available at: %s' % archive.GetMavenUrl(is_master))