def download_distribution(args, version, temp):
if version == 'master':
return utils.R8_JAR if args.nolib else utils.R8LIB_JAR
name = 'r8.jar' if args.nolib else 'r8lib.jar'
source = archive.GetUploadDestination(version, name, is_hash(version))
dest = os.path.join(temp, 'r8.jar')
utils.download_file_from_cloud_storage(source, dest)
return dest
def main():
args = parse_arguments()
r8lib_map_path = args.map
hashOrVersion = args.commit_hash or args.version
if hashOrVersion:
download_path = archive.GetUploadDestination(
hashOrVersion, 'r8lib.jar.map', args.commit_hash is not None)
if utils.file_exists_on_cloud_storage(download_path):
r8lib_map_path = tempfile.NamedTemporaryFile().name
utils.download_file_from_cloud_storage(download_path,
r8lib_map_path)
else:
print('Could not find map file from argument: %s.' % hashOrVersion)
return 1
retrace_args = ['java', '-jar', utils.RETRACE_JAR, r8lib_map_path]
if args.stacktrace:
retrace_args.append(args.stacktrace)
return subprocess.call(retrace_args)
def run(map_path,
hash_or_version,
stacktrace,
is_hash,
no_r8lib,
quiet=False,
debug=False):
if hash_or_version:
download_path = archive.GetUploadDestination(hash_or_version,
'r8lib.jar.map', is_hash)
if utils.file_exists_on_cloud_storage(download_path):
map_path = tempfile.NamedTemporaryFile().name
utils.download_file_from_cloud_storage(download_path, map_path)
else:
print('Could not find map file from argument: %s.' %
hash_or_version)
return 1
retrace_args = [jdk.GetJavaExecutable()]
if debug:
retrace_args.append(
'-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005'
)
retrace_args += [
'-cp', utils.R8_JAR if no_r8lib else utils.R8LIB_JAR,
'com.android.tools.r8.retrace.Retrace', map_path
]
if quiet:
retrace_args.append('--quiet')
if stacktrace:
retrace_args.append(stacktrace)
utils.PrintCmd(retrace_args, quiet=quiet)
return subprocess.call(retrace_args)
def run(map_path, hash_or_version, stacktrace, is_hash, no_r8lib):
if hash_or_version:
download_path = archive.GetUploadDestination(hash_or_version,
'r8lib.jar.map', is_hash)
if utils.file_exists_on_cloud_storage(download_path):
map_path = tempfile.NamedTemporaryFile().name
utils.download_file_from_cloud_storage(download_path, map_path)
else:
print('Could not find map file from argument: %s.' %
hash_or_version)
return 1
retrace_args = [
jdk.GetJavaExecutable(), '-cp',
utils.R8_JAR if no_r8lib else utils.R8LIB_JAR,
'com.android.tools.r8.retrace.Retrace', map_path
]
if stacktrace:
retrace_args.append(stacktrace)
utils.PrintCmd(retrace_args)
return subprocess.call(retrace_args)
def Main(argv):
(options, args) = ParseOptions(argv)
if (len(args) > 0):
raise Exception('Unsupported arguments')
if not utils.is_bot() and not (options.dry_run or options.build_only):
raise Exception('You are not a bot, don\'t archive builds. ' +
'Use --dry-run or --build-only to test locally')
if options.dry_run_output:
MustBeExistingDirectory(options.dry_run_output)
if options.build_only:
MustBeExistingDirectory(options.build_only)
if utils.is_bot():
archive.SetRLimitToMax()
# Make sure bazel is extracted in third_party.
utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE)
if options.build_only:
with utils.TempDir() as checkout_dir:
CloneDesugaredLibrary(options.github_account, checkout_dir)
(library_jar,
maven_zip) = BuildDesugaredLibrary(checkout_dir, "jdk8")
shutil.copyfile(
library_jar,
os.path.join(options.build_only,
os.path.basename(library_jar)))
shutil.copyfile(
maven_zip,
os.path.join(options.build_only, os.path.basename(maven_zip)))
return
# Only handling versioned desugar_jdk_libs.
is_master = False
with utils.TempDir() as checkout_dir:
CloneDesugaredLibrary(options.github_account, checkout_dir)
version = GetVersion(os.path.join(checkout_dir, VERSION_FILE))
destination = archive.GetVersionDestination(
'gs://', LIBRARY_NAME + '/' + version, is_master)
if utils.cloud_storage_exists(destination) and not options.dry_run:
raise Exception('Target archive directory %s already exists' %
destination)
(library_jar, maven_zip) = BuildDesugaredLibrary(checkout_dir, "jdk8")
storage_path = LIBRARY_NAME + '/' + version
# Upload the jar file with the library.
destination = archive.GetUploadDestination(storage_path,
LIBRARY_NAME + '.jar',
is_master)
Upload(options, library_jar, storage_path, destination, is_master)
# Upload the maven zip file with the library.
destination = archive.GetUploadDestination(storage_path,
LIBRARY_NAME + '.zip',
is_master)
Upload(options, maven_zip, storage_path, destination, is_master)
# Upload the jar file for accessing GCS as a maven repro.
maven_destination = archive.GetUploadDestination(
utils.get_maven_path('desugar_jdk_libs', version),
'desugar_jdk_libs-%s.jar' % version, is_master)
if options.dry_run:
print('Dry run, not actually creating maven repo')
else:
utils.upload_file_to_cloud_storage(library_jar, maven_destination)
print('Maven repo root available at: %s' %
archive.GetMavenUrl(is_master))
def download_target(root, target, hash_or_version, is_hash, quiet=False):
download_path = os.path.join(root, target)
url = archive.GetUploadDestination(hash_or_version, target, is_hash)
if not quiet:
print('Downloading: ' + url + ' -> ' + download_path)
utils.download_file_from_cloud_storage(url, download_path, quiet=quiet)
def Main(argv):
(options, args) = ParseOptions(argv)
if (len(args) > 0):
raise Exception('Unsupported arguments')
if not utils.is_bot() and not options.dry_run:
raise Exception('You are not a bot, don\'t archive builds. ' +
'Use --dry-run to test locally')
if (options.dry_run_output
and (not os.path.exists(options.dry_run_output)
or not os.path.isdir(options.dry_run_output))):
raise Exception(options.dry_run_output +
' does not exist or is not a directory')
if utils.is_bot():
archive.SetRLimitToMax()
# Make sure bazel is extracted in third_party.
utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
# Only handling versioned desugar_jdk_libs.
is_master = False
with utils.TempDir() as checkout_dir:
git_utils.GitClone(
'https://github.com/' + options.github_account + '/' +
LIBRARY_NAME, checkout_dir)
with utils.ChangedWorkingDirectory(checkout_dir):
version = GetVersion()
destination = archive.GetVersionDestination(
'gs://', LIBRARY_NAME + '/' + version, is_master)
if utils.cloud_storage_exists(destination) and not options.dry_run:
raise Exception('Target archive directory %s already exists' %
destination)
bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin',
'bazel')
cmd = [bazel, 'build', '--host_force_python=PY2', 'maven_release']
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
cmd = [bazel, 'shutdown']
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
# Compile the stubs for conversion files compilation.
stub_compiled_folder = os.path.join(checkout_dir, 'stubs')
os.mkdir(stub_compiled_folder)
all_stubs = GetFilesInFolder(
os.path.join(CONVERSION_FOLDER, 'stubs'))
cmd = [JDK8_JAVAC, '-d', stub_compiled_folder] + all_stubs
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
# Compile the conversion files.
conversions_compiled_folder = os.path.join(checkout_dir,
'conversions')
os.mkdir(conversions_compiled_folder)
all_conversions = GetFilesInFolder(
os.path.join(CONVERSION_FOLDER, 'conversions'))
cmd = [
JDK8_JAVAC, '-cp', stub_compiled_folder, '-d',
conversions_compiled_folder
] + all_conversions
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
# Locate the library jar and the maven zip with the jar from the
# bazel build.
library_jar = os.path.join('bazel-bin', 'src', 'share', 'classes',
'java', 'libjava.jar')
maven_zip = os.path.join('bazel-bin', LIBRARY_NAME + '.zip')
# Make a writable copy of the jar.
jar_folder = os.path.join(checkout_dir, 'jar')
os.mkdir(jar_folder)
shutil.copy(library_jar, jar_folder)
library_jar = os.path.join(jar_folder, 'libjava.jar')
os.chmod(library_jar, 0o777)
# Add conversion classes into the jar.
all_compiled_conversions = GetFilesInFolder(
conversions_compiled_folder)
with zipfile.ZipFile(library_jar, mode='a',
allowZip64=True) as jar:
for clazz in all_compiled_conversions:
jar.write(clazz,
arcname=os.path.relpath(
clazz, conversions_compiled_folder),
compress_type=zipfile.ZIP_DEFLATED)
storage_path = LIBRARY_NAME + '/' + version
# Upload the jar file with the library.
destination = archive.GetUploadDestination(storage_path,
LIBRARY_NAME + '.jar',
is_master)
Upload(options, library_jar, storage_path, destination, is_master)
# Upload the maven zip file with the library.
destination = archive.GetUploadDestination(storage_path,
LIBRARY_NAME + '.zip',
is_master)
Upload(options, maven_zip, storage_path, destination, is_master)
# Upload the jar file for accessing GCS as a maven repro.
maven_destination = archive.GetUploadDestination(
utils.get_maven_path('desugar_jdk_libs', version),
'desugar_jdk_libs-%s.jar' % version, is_master)
if options.dry_run:
print('Dry run, not actually creating maven repo')
else:
utils.upload_file_to_cloud_storage(library_jar,
maven_destination)
print('Maven repo root available at: %s' %
archive.GetMavenUrl(is_master))
def run_with_options(options, args, extra_args=None, stdout=None, quiet=False):
if extra_args is None:
extra_args = []
app_provided_pg_conf = False
# todo(121018500): remove when memory is under control
if not any('-Xmx' in arg for arg in extra_args):
if options.max_memory:
extra_args.append('-Xmx%sM' % options.max_memory)
else:
extra_args.append('-Xmx8G')
if options.golem:
golem.link_third_party()
options.out = os.getcwd()
if not options.ignore_java_version:
utils.check_java_version()
if options.print_times:
extra_args.append('-Dcom.android.tools.r8.printtimes=1')
outdir = options.out
(version_id, data) = get_version_and_data(options)
if options.compiler not in COMPILERS:
raise Exception("You need to specify '--compiler={}'".format(
'|'.join(COMPILERS)))
if options.compiler_build not in COMPILER_BUILDS:
raise Exception("You need to specify '--compiler-build={}'".format(
'|'.join(COMPILER_BUILDS)))
if not version_id in data.VERSIONS.keys():
print('No version {} for application {}'.format(
version_id, options.app))
print('Valid versions are {}'.format(data.VERSIONS.keys()))
return 1
version = data.VERSIONS[version_id]
type = get_type(options)
if type not in version:
print('No type {} for version {}'.format(type, version))
print('Valid types are {}'.format(version.keys()))
return 1
values = version[type]
inputs = []
# For R8 'deploy' the JAR is located using the Proguard configuration
# -injars option. For chrome and nest we don't have the injars in the
# proguard files.
if 'inputs' in values and (options.compiler != 'r8' or type != 'deploy'
or options.app == 'chrome'
or options.app == 'nest' or options.app == 'r8'
or options.app == 'iosched'
or options.app == 'tachiyomi'):
inputs = values['inputs']
args.extend(['--output', outdir])
if 'min-api' in values:
args.extend(['--min-api', values['min-api']])
if 'main-dex-list' in values:
args.extend(['--main-dex-list', values['main-dex-list']])
libraries = values['libraries'] if 'libraries' in values else []
if options.compiler == 'r8':
if 'pgconf' in values and not options.k:
if has_injars_and_libraryjars(values['pgconf']):
sanitized_lib_path = os.path.join(os.path.abspath(outdir),
'sanitized_lib.jar')
sanitized_pgconf_path = os.path.join(os.path.abspath(outdir),
'sanitized.config')
SanitizeLibrariesInPgconf(sanitized_lib_path,
sanitized_pgconf_path,
values['pgconf'])
libraries = [sanitized_lib_path]
args.extend(['--pg-conf', sanitized_pgconf_path])
else:
# -injars without -libraryjars or vice versa is not supported.
check_no_injars_and_no_libraryjars(values['pgconf'])
for pgconf in values['pgconf']:
args.extend(['--pg-conf', pgconf])
if 'sanitize_libraries' in values and values[
'sanitize_libraries']:
sanitized_lib_path = os.path.join(os.path.abspath(outdir),
'sanitized_lib.jar')
SanitizeLibraries(sanitized_lib_path, values['libraries'],
values['inputs'])
libraries = [sanitized_lib_path]
inputs = values['inputs']
app_provided_pg_conf = True
if options.k:
args.extend(['--pg-conf', options.k])
if 'maindexrules' in values:
for rules in values['maindexrules']:
args.extend(['--main-dex-rules', rules])
if 'allow-type-errors' in values:
extra_args.append('-Dcom.android.tools.r8.allowTypeErrors=1')
extra_args.append(
'-Dcom.android.tools.r8.disallowClassInlinerGracefulExit=1')
if options.debug_agent:
if not options.compiler_build == 'full':
print(
'WARNING: Running debugging agent on r8lib is questionable...')
extra_args.append(
'-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
)
if not options.no_libraries:
for lib in libraries:
args.extend(['--lib', lib])
if not outdir.endswith('.zip') and not outdir.endswith('.jar') \
and not os.path.exists(outdir):
os.makedirs(outdir)
if options.hash:
# Download r8-<hash>.jar from
# https://storage.googleapis.com/r8-releases/raw/<hash>/.
download_path = archive.GetUploadDestination(options.hash, 'r8.jar',
True)
assert utils.file_exists_on_cloud_storage(download_path), (
'Could not find r8.jar file from provided hash: %s' % options.hash)
destination = os.path.join(utils.LIBS, 'r8-' + options.hash + '.jar')
utils.download_file_from_cloud_storage(download_path,
destination,
quiet=quiet)
# Additional flags for the compiler from the configuration file.
if 'flags' in values:
args.extend(values['flags'].split(' '))
if options.compiler == 'r8':
if 'r8-flags' in values:
args.extend(values['r8-flags'].split(' '))
# Additional flags for the compiler from the command line.
if options.compiler_flags:
args.extend(options.compiler_flags.split(' '))
if options.r8_flags:
args.extend(options.r8_flags.split(' '))
# Feature jars.
features = values['features'] if 'features' in values else []
for i, feature in enumerate(features, start=1):
feature_out = os.path.join(outdir, 'feature-%d.zip' % i)
for feature_jar in feature['inputs']:
args.extend(['--feature', feature_jar, feature_out])
args.extend(inputs)
t0 = time.time()
if options.dump_args_file:
with open(options.dump_args_file, 'w') as args_file:
args_file.writelines([arg + os.linesep for arg in args])
else:
with utils.TempDir() as temp:
if options.print_memoryuse and not options.track_memory_to_file:
options.track_memory_to_file = os.path.join(
temp, utils.MEMORY_USE_TMP_FILE)
if options.compiler == 'r8' and app_provided_pg_conf:
# Ensure that output of -printmapping and -printseeds go to the output
# location and not where the app Proguard configuration places them.
if outdir.endswith('.zip') or outdir.endswith('.jar'):
pg_outdir = os.path.dirname(outdir)
else:
pg_outdir = outdir
if not options.no_extra_pgconf:
additional_pg_conf = GenerateAdditionalProguardConfiguration(
temp, os.path.abspath(pg_outdir))
args.extend(['--pg-conf', additional_pg_conf])
build = not options.no_build and not options.golem
stderr_path = os.path.join(temp, 'stderr')
with open(stderr_path, 'w') as stderr:
jar = None
main = None
if options.compiler_build == 'full':
tool = options.compiler
else:
assert (options.compiler_build == 'lib')
tool = 'r8lib-' + options.compiler
if options.hash:
jar = os.path.join(utils.LIBS,
'r8-' + options.hash + '.jar')
main = 'com.android.tools.r8.' + options.compiler.upper()
exit_code = toolhelper.run(
tool,
args,
build=build,
debug=not options.no_debug,
profile=options.profile,
track_memory_file=options.track_memory_to_file,
extra_args=extra_args,
stdout=stdout,
stderr=stderr,
timeout=options.timeout,
quiet=quiet,
cmd_prefix=['taskset', '-c', options.cpu_list]
if options.cpu_list else [],
jar=jar,
main=main)
if exit_code != 0:
with open(stderr_path) as stderr:
stderr_text = stderr.read()
if not quiet:
print(stderr_text)
if 'java.lang.OutOfMemoryError' in stderr_text:
if not quiet:
print('Failure was OOM')
return OOM_EXIT_CODE
return exit_code
if options.print_memoryuse:
print('{}(MemoryUse): {}'.format(
options.print_memoryuse,
utils.grep_memoryuse(options.track_memory_to_file)))
if options.print_runtimeraw:
print('{}(RunTimeRaw): {} ms'.format(options.print_runtimeraw,
1000.0 * (time.time() - t0)))
if options.print_dexsegments:
dex_files = glob(os.path.join(outdir, '*.dex'))
utils.print_dexsegments(options.print_dexsegments, dex_files)
return 0
def Main(argv):
(options, args) = ParseOptions(argv)
if (len(args) > 0):
raise Exception('Unsupported arguments')
if not utils.is_bot() and not options.dry_run:
raise Exception('You are not a bot, don\'t archive builds. ' +
'Use --dry-run to test locally')
if (options.dry_run_output
and (not os.path.exists(options.dry_run_output)
or not os.path.isdir(options.dry_run_output))):
raise Exception(options.dry_run_output +
' does not exist or is not a directory')
if utils.is_bot():
archive.SetRLimitToMax()
# Make sure bazel is extracted in third_party.
utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
# Only handling versioned desugar_jdk_libs.
is_master = False
with utils.TempDir() as checkout_dir:
git_utils.GitClone(
'https://github.com/' + options.github_account + '/' +
LIBRARY_NAME, checkout_dir)
with utils.ChangedWorkingDirectory(checkout_dir):
version = GetVersion()
destination = archive.GetVersionDestination(
'gs://', LIBRARY_NAME + '/' + version, is_master)
if utils.cloud_storage_exists(destination) and not options.dry_run:
raise Exception('Target archive directory %s already exists' %
destination)
bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin',
'bazel')
cmd = [bazel, 'build', 'maven_release']
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
cmd = [bazel, 'shutdown']
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
# Locate the library jar and the maven zip with the jar from the
# bazel build.
library_jar = os.path.join('bazel-bin', 'src', 'share', 'classes',
'java', 'libjava.jar')
maven_zip = os.path.join('bazel-bin', LIBRARY_NAME + '.zip')
storage_path = LIBRARY_NAME + '/' + version
# Upload the jar file with the library.
destination = archive.GetUploadDestination(storage_path,
LIBRARY_NAME + '.jar',
is_master)
Upload(options, library_jar, storage_path, destination, is_master)
# Upload the maven zip file with the library.
destination = archive.GetUploadDestination(storage_path,
LIBRARY_NAME + '.zip',
is_master)
Upload(options, maven_zip, storage_path, destination, is_master)
# Upload the jar file for accessing GCS as a maven repro.
maven_destination = archive.GetUploadDestination(
utils.get_maven_path('desugar_jdk_libs', version),
'desugar_jdk_libs-%s.jar' % version, is_master)
if options.dry_run:
print('Dry run, not actually creating maven repo')
else:
utils.upload_file_to_cloud_storage(library_jar,
maven_destination)
print('Maven repo root available at: %s' %
archive.GetMavenUrl(is_master))