def _SplitChildFeatures(options, feature_contexts, tmp_dir, mapping_path,
print_stdout):
feature_map = {f.name: f for f in feature_contexts}
parent_to_child = defaultdict(list)
for child, parent in options.uses_split.items():
parent_to_child[parent].append(child)
for parent, children in parent_to_child.items():
split_output = os.path.join(tmp_dir, 'split_%s' % parent)
os.mkdir(split_output)
# DexSplitter is not perfect and can cause issues related to inlining and
# class merging (see crbug.com/1032609). If strange class loading errors
# happen in DFMs specifying uses_split, this may be the cause.
split_cmd = build_utils.JavaCmd(options.warnings_as_errors) + [
'-cp',
options.r8_path,
'com.android.tools.r8.dexsplitter.DexSplitter',
'--output',
split_output,
'--proguard-map',
mapping_path,
]
parent_jars = set(feature_map[parent].input_paths)
for base_jar in sorted(parent_jars):
split_cmd += ['--base-jar', base_jar]
for child in children:
for feature_jar in feature_map[child].input_paths:
if feature_jar not in parent_jars:
split_cmd += [
'--feature-jar',
'%s:%s' % (feature_jar, child)
]
# The inputs are the outputs for the parent from the original R8 call.
parent_dir = feature_map[parent].staging_dir
for file_name in os.listdir(parent_dir):
split_cmd += ['--input', os.path.join(parent_dir, file_name)]
logging.debug('Running R8 DexSplitter')
build_utils.CheckOutput(split_cmd,
print_stdout=print_stdout,
fail_on_output=options.warnings_as_errors)
# Copy the parent dex back into the parent's staging dir.
base_split_output = os.path.join(split_output, 'base')
shutil.rmtree(parent_dir)
os.mkdir(parent_dir)
for dex_file in os.listdir(base_split_output):
shutil.move(os.path.join(base_split_output, dex_file),
os.path.join(parent_dir, dex_file))
# Copy each child dex back into the child's staging dir.
for child in children:
child_split_output = os.path.join(split_output, child)
child_staging_dir = feature_map[child].staging_dir
shutil.rmtree(child_staging_dir)
os.mkdir(child_staging_dir)
for dex_file in os.listdir(child_split_output):
shutil.move(os.path.join(child_split_output, dex_file),
os.path.join(child_staging_dir, dex_file))
def FinalizeApk(apksigner_path,
zipalign_path,
unsigned_apk_path,
final_apk_path,
key_path,
key_passwd,
key_name,
min_sdk_version,
warnings_as_errors=False):
# Use a tempfile so that Ctrl-C does not leave the file with a fresh mtime
# and a corrupted state.
with tempfile.NamedTemporaryFile() as staging_file:
if zipalign_path:
# v2 signing requires that zipalign happen first.
logging.debug('Running zipalign')
zipalign_cmd = [
zipalign_path, '-p', '-f', '4', unsigned_apk_path,
staging_file.name
]
build_utils.CheckOutput(zipalign_cmd,
print_stdout=True,
fail_on_output=warnings_as_errors)
signer_input_path = staging_file.name
else:
signer_input_path = unsigned_apk_path
sign_cmd = build_utils.JavaCmd(warnings_as_errors) + [
'-jar',
apksigner_path,
'sign',
'--in',
signer_input_path,
'--out',
staging_file.name,
'--ks',
key_path,
'--ks-key-alias',
key_name,
'--ks-pass',
'pass:'******'--v3-signing-enabled', 'false']
if min_sdk_version >= 24:
# Disable v1 signatures when v2 signing can be used (it's much faster).
# By default, both v1 and v2 signing happen.
sign_cmd += ['--v1-signing-enabled', 'false']
else:
# Force SHA-1 (makes signing faster; insecure is fine for local builds).
# Leave v2 signing enabled since it verifies faster on device when
# supported.
sign_cmd += ['--min-sdk-version', '1']
logging.debug('Signing apk')
build_utils.CheckOutput(sign_cmd,
print_stdout=True,
fail_on_output=warnings_as_errors)
shutil.move(staging_file.name, final_apk_path)
staging_file.delete = False
def _OnStaleMd5(options, javac_cmd, javac_args, java_files):
logging.info('Starting _OnStaleMd5')
if options.enable_kythe_annotations:
# Kythe requires those env variables to be set and compile_java.py does the
# same
if not os.environ.get('KYTHE_ROOT_DIRECTORY') or \
not os.environ.get('KYTHE_OUTPUT_DIRECTORY'):
raise Exception('--enable-kythe-annotations requires '
'KYTHE_ROOT_DIRECTORY and KYTHE_OUTPUT_DIRECTORY '
'environment variables to be set.')
javac_extractor_cmd = build_utils.JavaCmd() + [
'-jar',
_JAVAC_EXTRACTOR,
]
try:
_RunCompiler(options,
javac_extractor_cmd + javac_args,
java_files,
options.classpath,
options.jar_path + '.javac_extractor',
save_outputs=False),
except build_utils.CalledProcessError as e:
# Having no index for particular target is better than failing entire
# codesearch. Log and error and move on.
logging.error('Could not generate kzip: %s', e)
# Compiles with Error Prone take twice as long to run as pure javac. Thus GN
# rules run both in parallel, with Error Prone only used for checks.
_RunCompiler(options,
javac_cmd + javac_args,
java_files,
options.classpath,
options.jar_path,
save_outputs=not options.enable_errorprone)
logging.info('Completed all steps in _OnStaleMd5')
def _RunDexsplitter(options, output_dir):
cmd = build_utils.JavaCmd() + [
'-cp',
options.r8_path,
'com.android.tools.r8.dexsplitter.DexSplitter',
'--output',
output_dir,
'--proguard-map',
options.proguard_mapping_file,
]
for base_jar in options.features['base']:
cmd += ['--base-jar', base_jar]
base_jars_lookup = set(options.features['base'])
for feature in options.features:
if feature == 'base':
continue
for feature_jar in options.features[feature]:
if feature_jar not in base_jars_lookup:
cmd += ['--feature-jar', feature_jar + ':' + feature]
with build_utils.TempDir() as temp_dir:
unzipped_files = build_utils.ExtractAll(options.input_dex_zip,
temp_dir)
for file_name in unzipped_files:
cmd += ['--input', file_name]
build_utils.CheckOutput(cmd)
def MergeDexForIncrementalInstall(r8_jar_path, src_paths, dest_dex_jar):
dex_cmd = build_utils.JavaCmd(verify=False) + [
'-cp',
r8_jar_path,
'com.android.tools.r8.D8',
]
with build_utils.TempDir() as tmp_dir:
_CreateFinalDex(src_paths, dest_dex_jar, tmp_dir, dex_cmd)
def main():
args = build_utils.ExpandFileArgs(sys.argv[1:])
parser = argparse.ArgumentParser()
build_utils.AddDepfileOption(parser)
parser.add_argument('--desugar-jar',
required=True,
help='Path to Desugar.jar.')
parser.add_argument('--input-jar',
required=True,
help='Jar input path to include .class files from.')
parser.add_argument('--output-jar', required=True, help='Jar output path.')
parser.add_argument('--classpath',
action='append',
required=True,
help='Classpath.')
parser.add_argument('--bootclasspath',
required=True,
help='Path to javac bootclasspath interface jar.')
parser.add_argument('--warnings-as-errors',
action='store_true',
help='Treat all warnings as errors.')
options = parser.parse_args(args)
options.bootclasspath = build_utils.ParseGnList(options.bootclasspath)
options.classpath = build_utils.ParseGnList(options.classpath)
cmd = build_utils.JavaCmd(options.warnings_as_errors) + [
'-jar',
options.desugar_jar,
'--input',
options.input_jar,
'--output',
options.output_jar,
'--generate_base_classes_for_default_methods',
# Don't include try-with-resources files in every .jar. Instead, they
# are included via //third_party/bazel/desugar:desugar_runtime_java.
'--desugar_try_with_resources_omit_runtime_classes',
]
for path in options.bootclasspath:
cmd += ['--bootclasspath_entry', path]
for path in options.classpath:
cmd += ['--classpath_entry', path]
build_utils.CheckOutput(
cmd,
print_stdout=False,
stderr_filter=build_utils.FilterReflectiveAccessJavaWarnings,
fail_on_output=options.warnings_as_errors)
if options.depfile:
build_utils.WriteDepfile(options.depfile,
options.output_jar,
inputs=options.bootclasspath +
options.classpath)
def _RunInstrumentCommand(parser):
"""Instruments class or Jar files using JaCoCo.
Args:
parser: ArgumentParser object.
Returns:
An exit code.
"""
args = parser.parse_args()
source_files = []
if args.java_sources_file:
source_files.extend(build_utils.ReadSourcesList(
args.java_sources_file))
with build_utils.TempDir() as temp_dir:
instrument_cmd = build_utils.JavaCmd() + [
'-jar', args.jacococli_jar, 'instrument'
]
if not args.files_to_instrument:
_InstrumentClassFiles(instrument_cmd, args.input_path,
args.output_path, temp_dir)
else:
affected_files = build_utils.ReadSourcesList(
args.files_to_instrument)
source_set = set(source_files)
affected_source_files = [
f for f in affected_files if f in source_set
]
# Copy input_path to output_path and return if no source file affected.
if not affected_source_files:
shutil.copyfile(args.input_path, args.output_path)
# Create a dummy sources_json_file.
_CreateSourcesJsonFile([], None, args.sources_json_file,
build_utils.DIR_SOURCE_ROOT)
return 0
else:
_InstrumentClassFiles(instrument_cmd, args.input_path,
args.output_path, temp_dir,
affected_source_files)
source_dirs = _GetSourceDirsFromSourceFiles(source_files)
# TODO(GYP): In GN, we are passed the list of sources, detecting source
# directories, then walking them to re-establish the list of sources.
# This can obviously be simplified!
_CreateSourcesJsonFile(source_dirs, args.input_path,
args.sources_json_file, build_utils.DIR_SOURCE_ROOT)
return 0
def RunBundleTool(args, warnings_as_errors=()):
# Use () instead of None because command-line flags are None by default.
verify = warnings_as_errors == () or warnings_as_errors
# ASAN builds failed with the default of 1GB (crbug.com/1120202).
# Bug for bundletool: https://issuetracker.google.com/issues/165911616
cmd = build_utils.JavaCmd(verify, xmx='4G')
cmd += ['-jar', BUNDLETOOL_JAR_PATH]
cmd += args
logging.debug(' '.join(cmd))
return build_utils.CheckOutput(
cmd,
print_stderr=True,
fail_on_output=False,
stderr_filter=build_utils.FilterReflectiveAccessJavaWarnings)
def _OnStaleMd5(changes, options, javac_cmd, javac_args, java_files):
logging.info('Starting _OnStaleMd5')
if options.enable_kythe_annotations:
# Kythe requires those env variables to be set and compile_java.py does the
# same
if not os.environ.get('KYTHE_ROOT_DIRECTORY') or \
not os.environ.get('KYTHE_OUTPUT_DIRECTORY'):
raise Exception('--enable-kythe-annotations requires '
'KYTHE_ROOT_DIRECTORY and KYTHE_OUTPUT_DIRECTORY '
'environment variables to be set.')
javac_extractor_cmd = build_utils.JavaCmd() + [
'-jar',
_JAVAC_EXTRACTOR,
]
try:
# _RunCompiler()'s partial javac implementation does not support
# generating outputs in $KYTHE_OUTPUT_DIRECTORY.
_RunCompiler(changes,
options,
javac_extractor_cmd + javac_args,
java_files,
options.jar_path + '.javac_extractor',
enable_partial_javac=False)
except build_utils.CalledProcessError as e:
# Having no index for particular target is better than failing entire
# codesearch. Log and error and move on.
logging.error('Could not generate kzip: %s', e)
intermediates_out_dir = None
jar_info_path = None
if not options.enable_errorprone:
# Delete any stale files in the generated directory. The purpose of
# options.generated_dir is for codesearch.
shutil.rmtree(options.generated_dir, True)
intermediates_out_dir = options.generated_dir
jar_info_path = options.jar_path + '.info'
# Compiles with Error Prone take twice as long to run as pure javac. Thus GN
# rules run both in parallel, with Error Prone only used for checks.
_RunCompiler(changes,
options,
javac_cmd + javac_args,
java_files,
options.jar_path,
jar_info_path=jar_info_path,
intermediates_out_dir=intermediates_out_dir,
enable_partial_javac=True)
logging.info('Completed all steps in _OnStaleMd5')
def _OutputKeepRules(r8_path, input_paths, classpath, targets_re_string,
keep_rules_output):
cmd = build_utils.JavaCmd(False) + [
'-cp', r8_path, 'com.android.tools.r8.tracereferences.TraceReferences',
'--map-diagnostics:MissingDefinitionsDiagnostic', 'error', 'warning',
'--keep-rules', '--output', keep_rules_output
]
targets_re = re.compile(targets_re_string)
for path in input_paths:
if targets_re.search(path):
cmd += ['--target', path]
else:
cmd += ['--source', path]
for path in classpath:
cmd += ['--lib', path]
build_utils.CheckOutput(cmd, print_stderr=False, fail_on_output=False)
def DexJdkLibJar(r8_path,
min_api,
desugar_jdk_libs_json,
desugar_jdk_libs_jar,
desugar_jdk_libs_configuration_jar,
output,
warnings_as_errors,
config_paths=None):
# TODO(agrieve): Spews a lot of stderr about missing classes.
with build_utils.TempDir() as tmp_dir:
cmd = build_utils.JavaCmd(warnings_as_errors) + [
'-cp',
r8_path,
'com.android.tools.r8.L8',
'--min-api',
min_api,
'--lib',
build_utils.JAVA_HOME,
'--desugared-lib',
desugar_jdk_libs_json,
]
# If no desugaring is required, no keep rules are generated, and the keep
# file will not be created.
if config_paths is not None:
for path in config_paths:
cmd += ['--pg-conf', path]
cmd += [
'--output', tmp_dir, desugar_jdk_libs_jar,
desugar_jdk_libs_configuration_jar
]
build_utils.CheckOutput(cmd,
print_stdout=True,
fail_on_output=warnings_as_errors)
if os.path.exists(os.path.join(tmp_dir, 'classes2.dex')):
raise Exception(
'Achievement unlocked: desugar_jdk_libs is multidex!')
# classes.dex might not exists if the "desugar_jdk_libs_jar" is not used
# at all.
if os.path.exists(os.path.join(tmp_dir, 'classes.dex')):
shutil.move(os.path.join(tmp_dir, 'classes.dex'), output)
return True
return False
def main(argv):
build_utils.InitLogging('TURBINE_DEBUG')
argv = build_utils.ExpandFileArgs(argv[1:])
parser = argparse.ArgumentParser()
build_utils.AddDepfileOption(parser)
parser.add_argument('--turbine-jar-path',
required=True,
help='Path to the turbine jar file.')
parser.add_argument('--java-srcjars',
action='append',
default=[],
help='List of srcjars to include in compilation.')
parser.add_argument(
'--bootclasspath',
action='append',
default=[],
help='Boot classpath for javac. If this is specified multiple times, '
'they will all be appended to construct the classpath.')
parser.add_argument(
'--java-version',
help=
'Java language version to use in -source and -target args to javac.')
parser.add_argument('--classpath',
action='append',
help='Classpath to use.')
parser.add_argument('--processors',
action='append',
help='GN list of annotation processor main classes.')
parser.add_argument(
'--processorpath',
action='append',
help='GN list of jars that comprise the classpath used for Annotation '
'Processors.')
parser.add_argument(
'--processor-args',
action='append',
help='key=value arguments for the annotation processors.')
parser.add_argument('--jar-path', help='Jar output path.', required=True)
parser.add_argument('--generated-jar-path',
required=True,
help='Output path for generated source files.')
parser.add_argument('--warnings-as-errors',
action='store_true',
help='Treat all warnings as errors.')
options, unknown_args = parser.parse_known_args(argv)
options.bootclasspath = build_utils.ParseGnList(options.bootclasspath)
options.classpath = build_utils.ParseGnList(options.classpath)
options.processorpath = build_utils.ParseGnList(options.processorpath)
options.processors = build_utils.ParseGnList(options.processors)
options.java_srcjars = build_utils.ParseGnList(options.java_srcjars)
files = []
for arg in unknown_args:
# Interpret a path prefixed with @ as a file containing a list of sources.
if arg.startswith('@'):
files.extend(build_utils.ReadSourcesList(arg[1:]))
cmd = build_utils.JavaCmd(options.warnings_as_errors) + [
'-classpath', options.turbine_jar_path, 'com.google.turbine.main.Main'
]
javac_cmd = []
# Turbine reads lists from command line args by consuming args until one
# starts with double dash (--). Thus command line args should be grouped
# together and passed in together.
if options.processors:
cmd += ['--processors']
cmd += options.processors
if options.java_version:
javac_cmd.extend([
'-source',
options.java_version,
'-target',
options.java_version,
])
if options.java_version == '1.8':
# Android's boot jar doesn't contain all java 8 classes.
options.bootclasspath.append(build_utils.RT_JAR_PATH)
if options.bootclasspath:
cmd += ['--bootclasspath']
for bootclasspath in options.bootclasspath:
cmd += bootclasspath.split(':')
if options.processorpath:
cmd += ['--processorpath']
cmd += options.processorpath
if options.processor_args:
for arg in options.processor_args:
javac_cmd.extend(['-A%s' % arg])
if options.classpath:
cmd += ['--classpath']
cmd += options.classpath
if options.java_srcjars:
cmd += ['--source_jars']
cmd += options.java_srcjars
if files:
# Use jar_path to ensure paths are relative (needed for goma).
files_rsp_path = options.jar_path + '.files_list.txt'
with open(files_rsp_path, 'w') as f:
f.write(' '.join(files))
# Pass source paths as response files to avoid extremely long command lines
# that are tedius to debug.
cmd += ['--sources']
cmd += ['@' + files_rsp_path]
if javac_cmd:
cmd.append('--javacopts')
cmd += javac_cmd
cmd.append('--') # Terminate javacopts
# Use AtomicOutput so that output timestamps are not updated when outputs
# are not changed.
with build_utils.AtomicOutput(options.jar_path) as output_jar, \
build_utils.AtomicOutput(options.generated_jar_path) as generated_jar:
cmd += [
'--output', output_jar.name, '--gensrc_output', generated_jar.name
]
logging.debug('Command: %s', cmd)
start = time.time()
build_utils.CheckOutput(cmd,
print_stdout=True,
fail_on_output=options.warnings_as_errors)
end = time.time() - start
logging.info('Header compilation took %ss', end)
if options.depfile:
# GN already knows of the java files, so avoid listing individual java files
# in the depfile.
depfile_deps = (options.bootclasspath + options.classpath +
options.processorpath + options.java_srcjars)
build_utils.WriteDepfile(options.depfile, options.jar_path,
depfile_deps)
def _OptimizeWithR8(options,
config_paths,
libraries,
dynamic_config_data,
print_stdout=False):
with build_utils.TempDir() as tmp_dir:
if dynamic_config_data:
tmp_config_path = os.path.join(tmp_dir, 'proguard_config.txt')
with open(tmp_config_path, 'w') as f:
f.write(dynamic_config_data)
config_paths = config_paths + [tmp_config_path]
tmp_mapping_path = os.path.join(tmp_dir, 'mapping.txt')
# If there is no output (no classes are kept), this prevents this script
# from failing.
build_utils.Touch(tmp_mapping_path)
tmp_output = os.path.join(tmp_dir, 'r8out')
os.mkdir(tmp_output)
feature_contexts = []
if options.feature_names:
for name, dest_dex, input_paths in zip(options.feature_names,
options.dex_dests,
options.feature_jars):
feature_context = _DexPathContext(name, dest_dex, input_paths,
tmp_output)
if name == 'base':
base_dex_context = feature_context
else:
feature_contexts.append(feature_context)
else:
base_dex_context = _DexPathContext('base', options.output_path,
options.input_paths, tmp_output)
cmd = build_utils.JavaCmd(options.warnings_as_errors) + [
'-Dcom.android.tools.r8.allowTestProguardOptions=1',
'-Dcom.android.tools.r8.verticalClassMerging=1',
]
if options.disable_outlining:
cmd += ['-Dcom.android.tools.r8.disableOutlining=1']
cmd += [
'-cp',
options.r8_path,
'com.android.tools.r8.R8',
'--no-data-resources',
'--output',
base_dex_context.staging_dir,
'--pg-map-output',
tmp_mapping_path,
]
if options.disable_checkdiscard:
# Info level priority logs are not printed by default.
cmd += [
'--map-diagnostics:'
'com.android.tools.r8.errors.CheckDiscardDiagnostic',
'error',
'info',
]
if options.desugar_jdk_libs_json:
cmd += [
'--desugared-lib',
options.desugar_jdk_libs_json,
'--desugared-lib-pg-conf-output',
options.desugared_library_keep_rule_output,
]
if options.min_api:
cmd += ['--min-api', options.min_api]
if options.force_enable_assertions:
cmd += ['--force-enable-assertions']
for lib in libraries:
cmd += ['--lib', lib]
for config_file in config_paths:
cmd += ['--pg-conf', config_file]
if options.main_dex_rules_path:
for main_dex_rule in options.main_dex_rules_path:
cmd += ['--main-dex-rules', main_dex_rule]
base_jars = set(base_dex_context.input_paths)
# If a jar is present in multiple features, it should be moved to the base
# module.
all_feature_jars = set()
for feature in feature_contexts:
base_jars.update(all_feature_jars.intersection(
feature.input_paths))
all_feature_jars.update(feature.input_paths)
module_input_jars = base_jars.copy()
for feature in feature_contexts:
feature_input_jars = [
p for p in feature.input_paths if p not in module_input_jars
]
module_input_jars.update(feature_input_jars)
for in_jar in feature_input_jars:
cmd += ['--feature', in_jar, feature.staging_dir]
cmd += sorted(base_jars)
# Add any extra input jars to the base module (e.g. desugar runtime).
extra_jars = set(options.input_paths) - module_input_jars
cmd += sorted(extra_jars)
try:
stderr_filter = dex.CreateStderrFilter(
options.show_desugar_default_interface_warnings)
logging.debug('Running R8')
build_utils.CheckOutput(cmd,
print_stdout=print_stdout,
stderr_filter=stderr_filter,
fail_on_output=options.warnings_as_errors)
except build_utils.CalledProcessError as err:
debugging_link = ('\n\nR8 failed. Please see {}.'.format(
'https://chromium.googlesource.com/chromium/src/+/HEAD/build/'
'android/docs/java_optimization.md#Debugging-common-failures\n'
))
raise build_utils.CalledProcessError(err.cwd, err.args,
err.output + debugging_link)
base_has_imported_lib = False
if options.desugar_jdk_libs_json:
logging.debug('Running L8')
existing_files = build_utils.FindInDirectory(
base_dex_context.staging_dir)
jdk_dex_output = os.path.join(
base_dex_context.staging_dir,
'classes%d.dex' % (len(existing_files) + 1))
base_has_imported_lib = dex_jdk_libs.DexJdkLibJar(
options.r8_path, options.min_api,
options.desugar_jdk_libs_json, options.desugar_jdk_libs_jar,
options.desugar_jdk_libs_configuration_jar,
options.desugared_library_keep_rule_output, jdk_dex_output,
options.warnings_as_errors)
logging.debug('Collecting ouputs')
base_dex_context.CreateOutput(
base_has_imported_lib, options.desugared_library_keep_rule_output)
for feature in feature_contexts:
feature.CreateOutput()
with open(options.mapping_output, 'w') as out_file, \
open(tmp_mapping_path) as in_file:
# Mapping files generated by R8 include comments that may break
# some of our tooling so remove those (specifically: apkanalyzer).
out_file.writelines(l for l in in_file if not l.startswith('#'))
def main(args):
build_utils.InitLogging('DEX_DEBUG')
options = _ParseArgs(args)
options.class_inputs += options.class_inputs_filearg
options.dex_inputs += options.dex_inputs_filearg
input_paths = options.class_inputs + options.dex_inputs
if options.multi_dex and options.main_dex_list_path:
input_paths.append(options.main_dex_list_path)
input_paths.append(options.r8_jar_path)
input_paths.append(options.custom_d8_jar_path)
depfile_deps = options.class_inputs_filearg + options.dex_inputs_filearg
output_paths = [options.output]
if options.incremental_dir:
final_dex_inputs = _IntermediateDexFilePathsFromInputJars(
options.class_inputs, options.incremental_dir)
output_paths += final_dex_inputs
track_subpaths_allowlist = options.class_inputs
else:
final_dex_inputs = list(options.class_inputs)
track_subpaths_allowlist = None
final_dex_inputs += options.dex_inputs
dex_cmd = build_utils.JavaCmd(options.warnings_as_errors) + [
'-cp',
'{}:{}'.format(options.r8_jar_path, options.custom_d8_jar_path),
'org.chromium.build.CustomD8',
]
if options.release:
dex_cmd += ['--release']
if options.min_api:
dex_cmd += ['--min-api', options.min_api]
if not options.desugar:
dex_cmd += ['--no-desugaring']
elif options.classpath:
# The classpath is used by D8 to for interface desugaring.
classpath_paths = options.classpath
if options.desugar_dependencies:
dex_cmd += ['--desugar-dependencies', options.desugar_dependencies]
if os.path.exists(options.desugar_dependencies):
with open(options.desugar_dependencies, 'r') as f:
lines = [line.strip() for line in f.readlines()]
# Use a set to deduplicate entries.
desugar_dependencies = set(dep for dep in lines if dep)
# Desugar dependencies are a subset of classpath.
classpath_paths = list(desugar_dependencies)
depfile_deps += classpath_paths
input_paths += classpath_paths
dex_cmd += ['--lib', build_utils.JAVA_HOME]
for path in options.bootclasspath:
dex_cmd += ['--lib', path]
# Still pass the entire classpath in case a new dependency is needed by
# desugar, so that desugar_dependencies will be updated for the next build.
for path in options.classpath:
dex_cmd += ['--classpath', path]
depfile_deps += options.bootclasspath
input_paths += options.bootclasspath
if options.desugar_jdk_libs_json:
dex_cmd += ['--desugared-lib', options.desugar_jdk_libs_json]
if options.force_enable_assertions:
dex_cmd += ['--force-enable-assertions']
md5_check.CallAndWriteDepfileIfStale(
lambda changes: _OnStaleMd5(changes, options, final_dex_inputs, dex_cmd
),
options,
input_paths=input_paths,
input_strings=dex_cmd + [bool(options.incremental_dir)],
output_paths=output_paths,
pass_changes=True,
track_subpaths_allowlist=track_subpaths_allowlist,
depfile_deps=depfile_deps)
def _CheckForMissingSymbols(r8_path, dex_files, classpath, warnings_as_errors,
error_title):
cmd = build_utils.JavaCmd(warnings_as_errors) + [
'-cp', r8_path, 'com.android.tools.r8.tracereferences.TraceReferences',
'--map-diagnostics:MissingDefinitionsDiagnostic', 'error', 'warning',
'--check'
]
for path in classpath:
cmd += ['--lib', path]
for path in dex_files:
cmd += ['--source', path]
def stderr_filter(stderr):
ignored_lines = [
# Summary contains warning count, which our filtering makes wrong.
'Warning: Tracereferences found',
# TODO(agrieve): Create interface jars for these missing classes rather
# than allowlisting here.
'dalvik.system',
'libcore.io',
'sun.misc.Unsafe',
# Found in: com/facebook/fbui/textlayoutbuilder/StaticLayoutHelper
'android.text.StaticLayout.<init>',
# Explicictly guarded by try (NoClassDefFoundError) in Flogger's
# PlatformProvider.
'com.google.common.flogger.backend.google.GooglePlatform',
'com.google.common.flogger.backend.system.DefaultPlatform',
# trichrome_webview_google_bundle contains this missing reference.
# TODO(crbug.com/1142530): Fix this missing reference properly.
'org.chromium.build.NativeLibraries',
# TODO(agrieve): Exclude these only when use_jacoco_coverage=true.
'java.lang.instrument.ClassFileTransformer',
'java.lang.instrument.IllegalClassFormatException',
'java.lang.instrument.Instrumentation',
'java.lang.management.ManagementFactory',
'javax.management.MBeanServer',
'javax.management.ObjectInstance',
'javax.management.ObjectName',
'javax.management.StandardMBean',
# Explicitly guarded by try (NoClassDefFoundError) in Firebase's
# KotlinDetector: com.google.firebase.platforminfo.KotlinDetector.
'kotlin.KotlinVersion',
]
had_unfiltered_items = ' ' in stderr
stderr = build_utils.FilterLines(
stderr, '|'.join(re.escape(x) for x in ignored_lines))
if stderr:
if ' ' in stderr:
stderr = error_title + """
Tip: Build with:
is_java_debug=false
treat_warnings_as_errors=false
enable_proguard_obfuscation=false
and then use dexdump to see which class(s) reference them.
E.g.:
third_party/android_sdk/public/build-tools/*/dexdump -d \
out/Release/apks/YourApk.apk > dex.txt
""" + stderr
if 'FragmentActivity' in stderr:
stderr += """
You may need to update build configs to run FragmentActivityReplacer for
additional targets. See
https://chromium.googlesource.com/chromium/src.git/+/master/docs/ui/android/bytecode_rewriting.md.
"""
elif had_unfiltered_items:
# Left only with empty headings. All indented items filtered out.
stderr = ''
return stderr
logging.debug('cmd: %s', ' '.join(cmd))
build_utils.CheckOutput(cmd,
print_stdout=True,
stderr_filter=stderr_filter,
fail_on_output=warnings_as_errors)
def _OptimizeWithR8(options,
config_paths,
libraries,
dynamic_config_data,
print_stdout=False):
with build_utils.TempDir() as tmp_dir:
if dynamic_config_data:
dynamic_config_path = os.path.join(tmp_dir, 'dynamic_config.flags')
with open(dynamic_config_path, 'w') as f:
f.write(dynamic_config_data)
config_paths = config_paths + [dynamic_config_path]
tmp_mapping_path = os.path.join(tmp_dir, 'mapping.txt')
# If there is no output (no classes are kept), this prevents this script
# from failing.
build_utils.Touch(tmp_mapping_path)
tmp_output = os.path.join(tmp_dir, 'r8out')
os.mkdir(tmp_output)
split_contexts_by_name = {}
if options.feature_names:
for name, dest_dex, input_jars in zip(options.feature_names,
options.dex_dests,
options.feature_jars):
parent_name = options.uses_split.get(name)
if parent_name is None and name != 'base':
parent_name = 'base'
split_context = _SplitContext(name,
dest_dex,
input_jars,
tmp_output,
parent_name=parent_name)
split_contexts_by_name[name] = split_context
else:
# Base context will get populated via "extra_jars" below.
split_contexts_by_name['base'] = _SplitContext('base',
options.output_path, [],
tmp_output)
base_context = split_contexts_by_name['base']
# R8 OOMs with the default xmx=1G.
cmd = build_utils.JavaCmd(options.warnings_as_errors, xmx='2G') + [
'-Dcom.android.tools.r8.allowTestProguardOptions=1',
'-Dcom.android.tools.r8.disableHorizontalClassMerging=1',
]
if options.disable_outlining:
cmd += ['-Dcom.android.tools.r8.disableOutlining=1']
if options.dump_inputs:
cmd += ['-Dcom.android.tools.r8.dumpinputtofile=r8inputs.zip']
cmd += [
'-cp',
options.r8_path,
'com.android.tools.r8.R8',
'--no-data-resources',
'--output',
base_context.staging_dir,
'--pg-map-output',
tmp_mapping_path,
]
if options.disable_checks:
# Info level priority logs are not printed by default.
cmd += ['--map-diagnostics:CheckDiscardDiagnostic', 'error', 'info']
if options.desugar_jdk_libs_json:
cmd += [
'--desugared-lib',
options.desugar_jdk_libs_json,
'--desugared-lib-pg-conf-output',
options.desugared_library_keep_rule_output,
]
if options.min_api:
cmd += ['--min-api', options.min_api]
if options.force_enable_assertions:
cmd += ['--force-enable-assertions']
for lib in libraries:
cmd += ['--lib', lib]
for config_file in config_paths:
cmd += ['--pg-conf', config_file]
if options.main_dex_rules_path:
for main_dex_rule in options.main_dex_rules_path:
cmd += ['--main-dex-rules', main_dex_rule]
_DeDupeInputJars(split_contexts_by_name)
# Add any extra inputs to the base context (e.g. desugar runtime).
extra_jars = set(options.input_paths)
for split_context in split_contexts_by_name.values():
extra_jars -= split_context.input_jars
base_context.input_jars.update(extra_jars)
for split_context in split_contexts_by_name.values():
if split_context is base_context:
continue
for in_jar in sorted(split_context.input_jars):
cmd += ['--feature', in_jar, split_context.staging_dir]
cmd += sorted(base_context.input_jars)
try:
stderr_filter = dex.CreateStderrFilter(
options.show_desugar_default_interface_warnings)
logging.debug('Running R8')
build_utils.CheckOutput(cmd,
print_stdout=print_stdout,
stderr_filter=stderr_filter,
fail_on_output=options.warnings_as_errors)
except build_utils.CalledProcessError as err:
debugging_link = ('\n\nR8 failed. Please see {}.'.format(
'https://chromium.googlesource.com/chromium/src/+/HEAD/build/'
'android/docs/java_optimization.md#Debugging-common-failures\n'))
raise build_utils.CalledProcessError(err.cwd, err.args,
err.output + debugging_link)
base_has_imported_lib = False
if options.desugar_jdk_libs_json:
logging.debug('Running L8')
existing_files = build_utils.FindInDirectory(base_context.staging_dir)
jdk_dex_output = os.path.join(base_context.staging_dir,
'classes%d.dex' % (len(existing_files) + 1))
# Use -applymapping to avoid name collisions.
l8_dynamic_config_path = os.path.join(tmp_dir, 'l8_dynamic_config.flags')
with open(l8_dynamic_config_path, 'w') as f:
f.write("-applymapping '{}'\n".format(tmp_mapping_path))
# Pass the dynamic config so that obfuscation options are picked up.
l8_config_paths = [dynamic_config_path, l8_dynamic_config_path]
if os.path.exists(options.desugared_library_keep_rule_output):
l8_config_paths.append(options.desugared_library_keep_rule_output)
base_has_imported_lib = dex_jdk_libs.DexJdkLibJar(
options.r8_path, options.min_api, options.desugar_jdk_libs_json,
options.desugar_jdk_libs_jar,
options.desugar_jdk_libs_configuration_jar, jdk_dex_output,
options.warnings_as_errors, l8_config_paths)
if int(options.min_api) >= 24 and base_has_imported_lib:
with open(jdk_dex_output, 'rb') as f:
dexfile = dex_parser.DexFile(bytearray(f.read()))
for m in dexfile.IterMethodSignatureParts():
print('{}#{}'.format(m[0], m[2]))
assert False, (
'Desugared JDK libs are disabled on Monochrome and newer - see '
'crbug.com/1159984 for details, and see above list for desugared '
'classes and methods.')
logging.debug('Collecting ouputs')
base_context.CreateOutput(base_has_imported_lib,
options.desugared_library_keep_rule_output)
for split_context in split_contexts_by_name.values():
if split_context is not base_context:
split_context.CreateOutput()
with open(options.mapping_output, 'w') as out_file, \
open(tmp_mapping_path) as in_file:
# Mapping files generated by R8 include comments that may break
# some of our tooling so remove those (specifically: apkanalyzer).
out_file.writelines(l for l in in_file if not l.startswith('#'))
return base_context
def _CheckForMissingSymbols(r8_path, dex_files, classpath, warnings_as_errors):
cmd = build_utils.JavaCmd(warnings_as_errors) + [
'-cp', r8_path, 'com.android.tools.r8.tracereferences.TraceReferences',
'--map-diagnostics:MissingDefinitionsDiagnostic', 'error', 'warning',
'--check'
]
for path in classpath:
cmd += ['--lib', path]
for path in dex_files:
cmd += ['--source', path]
def stderr_filter(stderr):
ignored_lines = [
# Summary contains warning count, which our filtering makes wrong.
'Warning: Tracereferences found',
# TODO(agrieve): Create interface jars for these missing classes rather
# than allowlisting here.
'dalvik/system',
'libcore/io',
'sun/misc/Unsafe',
# Found in: com/facebook/fbui/textlayoutbuilder/StaticLayoutHelper
('android/text/StaticLayout;<init>(Ljava/lang/CharSequence;IILandroid'
'/text/TextPaint;ILandroid/text/Layout$Alignment;Landroid/text/'
'TextDirectionHeuristic;FFZLandroid/text/TextUtils$TruncateAt;II)V'
),
# Found in
# com/google/android/gms/cast/framework/media/internal/ResourceProvider
# Missing due to setting "strip_resources = true".
'com/google/android/gms/cast/framework/R',
# Found in com/google/android/gms/common/GoogleApiAvailability
# Missing due to setting "strip_drawables = true".
'com/google/android/gms/base/R$drawable',
# Explicictly guarded by try (NoClassDefFoundError) in Flogger's
# PlatformProvider.
'com/google/common/flogger/backend/google/GooglePlatform',
'com/google/common/flogger/backend/system/DefaultPlatform',
# trichrome_webview_google_bundle contains this missing reference.
# TODO(crbug.com/1142530): Fix this missing reference properly.
'org/chromium/base/library_loader/NativeLibraries',
# TODO(agrieve): Exclude these only when use_jacoco_coverage=true.
'Ljava/lang/instrument/ClassFileTransformer',
'Ljava/lang/instrument/IllegalClassFormatException',
'Ljava/lang/instrument/Instrumentation',
'Ljava/lang/management/ManagementFactory',
'Ljavax/management/MBeanServer',
'Ljavax/management/ObjectInstance',
'Ljavax/management/ObjectName',
'Ljavax/management/StandardMBean',
# Explicitly guarded by try (NoClassDefFoundError) in Firebase's
# KotlinDetector: com.google.firebase.platforminfo.KotlinDetector.
'Lkotlin/KotlinVersion',
]
had_unfiltered_items = ' ' in stderr
stderr = build_utils.FilterLines(
stderr, '|'.join(re.escape(x) for x in ignored_lines))
if stderr:
if ' ' in stderr:
stderr = """
DEX contains references to non-existent symbols after R8 optimization.
Tip: Build with:
is_java_debug=false
treat_warnings_as_errors=false
enable_proguard_obfuscation=false
and then use dexdump to see which class(s) reference them.
E.g.:
third_party/android_sdk/public/build-tools/*/dexdump -d \
out/Release/apks/YourApk.apk > dex.txt
""" + stderr
if 'FragmentActivity' in stderr:
stderr += """
You may need to update build configs to run FragmentActivityReplacer for
additional targets. See
https://chromium.googlesource.com/chromium/src.git/+/master/docs/ui/android/bytecode_rewriting.md.
"""
elif had_unfiltered_items:
# Left only with empty headings. All indented items filtered out.
stderr = ''
return stderr
logging.debug('cmd: %s', ' '.join(cmd))
build_utils.CheckOutput(cmd,
print_stdout=True,
stderr_filter=stderr_filter,
fail_on_output=warnings_as_errors)
def main(argv):
argv = build_utils.ExpandFileArgs(argv)
parser = argparse.ArgumentParser(description=__doc__)
build_utils.AddDepfileOption(parser)
parser.add_argument('--android-sdk-cmdline-tools',
help='Path to SDK\'s cmdline-tools folder.',
required=True)
parser.add_argument('--root-manifest',
help='Root manifest which to merge into',
required=True)
parser.add_argument('--output', help='Output manifest path', required=True)
parser.add_argument('--extras',
help='GN list of additional manifest to merge')
parser.add_argument('--min-sdk-version',
required=True,
help='android:minSdkVersion for merging.')
parser.add_argument('--target-sdk-version',
required=True,
help='android:targetSdkVersion for merging.')
parser.add_argument('--max-sdk-version',
help='android:maxSdkVersion for merging.')
parser.add_argument('--manifest-package',
help='Package name of the merged AndroidManifest.xml.')
parser.add_argument('--warnings-as-errors',
action='store_true',
help='Treat all warnings as errors.')
args = parser.parse_args(argv)
classpath = _BuildManifestMergerClasspath(args.android_sdk_cmdline_tools)
with build_utils.AtomicOutput(args.output) as output:
cmd = build_utils.JavaCmd(args.warnings_as_errors) + [
'-cp',
classpath,
_MANIFEST_MERGER_MAIN_CLASS,
'--out',
output.name,
'--property',
'MIN_SDK_VERSION=' + args.min_sdk_version,
'--property',
'TARGET_SDK_VERSION=' + args.target_sdk_version,
]
if args.max_sdk_version:
cmd += [
'--property',
'MAX_SDK_VERSION=' + args.max_sdk_version,
]
extras = build_utils.ParseGnList(args.extras)
if extras:
cmd += ['--libs', ':'.join(extras)]
with _ProcessManifest(args.root_manifest, args.min_sdk_version,
args.target_sdk_version, args.max_sdk_version,
args.manifest_package) as tup:
root_manifest, package = tup
cmd += [
'--main',
root_manifest,
'--property',
'PACKAGE=' + package,
'--remove-tools-declarations',
]
build_utils.CheckOutput(
cmd,
# https://issuetracker.google.com/issues/63514300:
# The merger doesn't set a nonzero exit code for failures.
fail_func=lambda returncode, stderr: returncode != 0 or
build_utils.IsTimeStale(output.name, [root_manifest] + extras),
fail_on_output=args.warnings_as_errors)
# Check for correct output.
_, manifest, _ = manifest_utils.ParseManifest(output.name)
manifest_utils.AssertUsesSdk(manifest, args.min_sdk_version,
args.target_sdk_version)
manifest_utils.AssertPackage(manifest, package)
if args.depfile:
inputs = extras + classpath.split(':')
build_utils.WriteDepfile(args.depfile, args.output, inputs=inputs)
def main(argv):
build_utils.InitLogging('TURBINE_DEBUG')
argv = build_utils.ExpandFileArgs(argv[1:])
parser = argparse.ArgumentParser()
build_utils.AddDepfileOption(parser)
parser.add_argument('--turbine-jar-path',
required=True,
help='Path to the turbine jar file.')
parser.add_argument('--java-srcjars',
action='append',
default=[],
help='List of srcjars to include in compilation.')
parser.add_argument(
'--bootclasspath',
action='append',
default=[],
help='Boot classpath for javac. If this is specified multiple times, '
'they will all be appended to construct the classpath.')
parser.add_argument(
'--java-version',
help=
'Java language version to use in -source and -target args to javac.')
parser.add_argument('--classpath',
action='append',
help='Classpath to use.')
parser.add_argument('--processors',
action='append',
help='GN list of annotation processor main classes.')
parser.add_argument(
'--processorpath',
action='append',
help='GN list of jars that comprise the classpath used for Annotation '
'Processors.')
parser.add_argument(
'--processor-args',
action='append',
help='key=value arguments for the annotation processors.')
parser.add_argument('--jar-path', help='Jar output path.', required=True)
parser.add_argument('--generated-jar-path',
required=True,
help='Output path for generated source files.')
parser.add_argument('--warnings-as-errors',
action='store_true',
help='Treat all warnings as errors.')
options, unknown_args = parser.parse_known_args(argv)
options.bootclasspath = build_utils.ParseGnList(options.bootclasspath)
options.classpath = build_utils.ParseGnList(options.classpath)
options.processorpath = build_utils.ParseGnList(options.processorpath)
options.processors = build_utils.ParseGnList(options.processors)
options.java_srcjars = build_utils.ParseGnList(options.java_srcjars)
files = []
for arg in unknown_args:
# Interpret a path prefixed with @ as a file containing a list of sources.
if arg.startswith('@'):
files.extend(build_utils.ReadSourcesList(arg[1:]))
cmd = build_utils.JavaCmd(options.warnings_as_errors) + [
'-classpath', options.turbine_jar_path, 'com.google.turbine.main.Main'
]
javac_cmd = []
# Turbine reads lists from command line args by consuming args until one
# starts with double dash (--). Thus command line args should be grouped
# together and passed in together.
if options.processors:
cmd += ['--processors']
cmd += options.processors
if options.java_version:
javac_cmd.extend([
'-source',
options.java_version,
'-target',
options.java_version,
])
if options.java_version == '1.8':
# Android's boot jar doesn't contain all java 8 classes.
options.bootclasspath.append(build_utils.RT_JAR_PATH)
if options.bootclasspath:
cmd += ['--bootclasspath']
for bootclasspath in options.bootclasspath:
cmd += bootclasspath.split(':')
if options.processorpath:
cmd += ['--processorpath']
cmd += options.processorpath
if options.processor_args:
for arg in options.processor_args:
javac_cmd.extend(['-A%s' % arg])
classpath_inputs = (options.bootclasspath + options.classpath +
options.processorpath)
# GN already knows of the java files, so avoid listing individual java files
# in the depfile.
depfile_deps = classpath_inputs + options.java_srcjars
input_paths = depfile_deps + files
output_paths = [
options.jar_path,
options.generated_jar_path,
]
input_strings = cmd + options.classpath + files
md5_check.CallAndWriteDepfileIfStale(
lambda: _OnStaleMd5(options, cmd, javac_cmd, files, options.classpath),
options,
depfile_deps=depfile_deps,
input_paths=input_paths,
input_strings=input_strings,
output_paths=output_paths)
def main(args):
build_utils.InitLogging('DEX_DEBUG')
options = _ParseArgs(args)
options.class_inputs += options.class_inputs_filearg
options.dex_inputs += options.dex_inputs_filearg
input_paths = options.class_inputs + options.dex_inputs
if options.multi_dex and options.main_dex_list_path:
input_paths.append(options.main_dex_list_path)
input_paths.append(options.r8_jar_path)
depfile_deps = options.class_inputs_filearg + options.dex_inputs_filearg
output_paths = [options.output]
if options.incremental_dir:
final_dex_inputs = _IntermediateDexFilePathsFromInputJars(
options.class_inputs, options.incremental_dir)
output_paths += final_dex_inputs
track_subpaths_allowlist = options.class_inputs
else:
final_dex_inputs = list(options.class_inputs)
track_subpaths_allowlist = None
final_dex_inputs += options.dex_inputs
dex_cmd = build_utils.JavaCmd(options.warnings_as_errors) + [
'-cp',
options.r8_jar_path,
'com.android.tools.r8.D8',
]
if options.release:
dex_cmd += ['--release']
if options.min_api:
dex_cmd += ['--min-api', options.min_api]
if not options.desugar:
dex_cmd += ['--no-desugaring']
elif options.classpath:
# Don't pass classpath when Desugar.jar is doing interface desugaring.
dex_cmd += ['--lib', build_utils.JAVA_HOME]
for path in options.bootclasspath:
dex_cmd += ['--lib', path]
for path in options.classpath:
dex_cmd += ['--classpath', path]
depfile_deps += options.classpath
depfile_deps += options.bootclasspath
input_paths += options.classpath
input_paths += options.bootclasspath
if options.desugar_jdk_libs_json:
dex_cmd += ['--desugared-lib', options.desugar_jdk_libs_json]
if options.force_enable_assertions:
dex_cmd += ['--force-enable-assertions']
md5_check.CallAndWriteDepfileIfStale(
lambda changes: _OnStaleMd5(changes, options, final_dex_inputs, dex_cmd
),
options,
input_paths=input_paths,
input_strings=dex_cmd + [bool(options.incremental_dir)],
output_paths=output_paths,
pass_changes=True,
track_subpaths_allowlist=track_subpaths_allowlist,
depfile_deps=depfile_deps)
def main(args):
args = build_utils.ExpandFileArgs(args)
options = _ParseArgs(args)
split_dimensions = []
if options.split_dimensions:
split_dimensions = [x.upper() for x in options.split_dimensions]
with build_utils.TempDir() as tmp_dir:
module_zips = [
_SplitModuleForAssetTargeting(module, tmp_dir, split_dimensions) \
for module in options.module_zips]
base_master_resource_ids = None
if options.base_module_rtxt_path:
base_master_resource_ids = _GenerateBaseResourcesAllowList(
options.base_module_rtxt_path,
options.base_allowlist_rtxt_path)
bundle_config = _GenerateBundleConfigJson(
options.uncompressed_assets, options.compress_shared_libraries,
split_dimensions, base_master_resource_ids)
tmp_bundle = os.path.join(tmp_dir, 'tmp_bundle')
tmp_unsigned_bundle = tmp_bundle
if options.keystore_path:
tmp_unsigned_bundle = tmp_bundle + '.unsigned'
# Important: bundletool requires that the bundle config file is
# named with a .pb.json extension.
tmp_bundle_config = tmp_bundle + '.BundleConfig.pb.json'
with open(tmp_bundle_config, 'w') as f:
f.write(bundle_config)
cmd_args = build_utils.JavaCmd(options.warnings_as_errors) + [
'-jar',
bundletool.BUNDLETOOL_JAR_PATH,
'build-bundle',
'--modules=' + ','.join(module_zips),
'--output=' + tmp_unsigned_bundle,
'--config=' + tmp_bundle_config,
]
build_utils.CheckOutput(
cmd_args,
print_stdout=True,
print_stderr=True,
stderr_filter=build_utils.FilterReflectiveAccessJavaWarnings,
fail_on_output=options.warnings_as_errors)
if options.validate_services:
# TODO(crbug.com/1126301): This step takes 0.4s locally for bundles with
# isolated splits disabled and 2s for bundles with isolated splits
# enabled. Consider making this run in parallel or move into a separate
# step before enabling isolated splits by default.
_MaybeCheckServicesPresentInBase(tmp_unsigned_bundle, module_zips)
if options.keystore_path:
# NOTE: As stated by the public documentation, apksigner cannot be used
# to sign the bundle (because it rejects anything that isn't an APK).
# The signature and digest algorithm selection come from the internal
# App Bundle documentation. There is no corresponding public doc :-(
signing_cmd_args = [
'jarsigner',
'-sigalg',
'SHA256withRSA',
'-digestalg',
'SHA-256',
'-keystore',
'file:' + options.keystore_path,
'-storepass',
options.keystore_password,
'-signedjar',
tmp_bundle,
tmp_unsigned_bundle,
options.key_name,
]
build_utils.CheckOutput(signing_cmd_args,
print_stderr=True,
fail_on_output=options.warnings_as_errors)
shutil.move(tmp_bundle, options.out_bundle)
if options.rtxt_out_path:
_ConcatTextFiles(options.rtxt_in_paths, options.rtxt_out_path)
if options.pathmap_out_path:
_WriteBundlePathmap(options.pathmap_in_paths, options.module_names,
options.pathmap_out_path)
def main():
args = _ParseArgs()
proguard_cmd = build_utils.JavaCmd(args.warnings_as_errors) + [
'-cp',
args.r8_path,
'com.android.tools.r8.R8',
'--classfile',
'--no-desugaring',
'--lib',
args.shrinked_android_path,
]
for m in args.main_dex_rules_paths:
proguard_cmd.extend(['--pg-conf', m])
proguard_flags = [
'-forceprocessing',
'-dontwarn',
'-dontoptimize',
'-dontobfuscate',
'-dontpreverify',
]
if args.negative_main_dex_globs:
for glob in args.negative_main_dex_globs:
# Globs come with 1 asterix, but we want 2 to match subpackages.
proguard_flags.append('-checkdiscard class ' +
glob.replace('*', '**').replace('/', '.'))
main_dex_list = ''
try:
with tempfile.NamedTemporaryFile(suffix='.jar') as temp_jar:
# Step 1: Use R8 to find all @MainDex code, and all code reachable
# from @MainDex code (recursive).
proguard_cmd += ['--output', temp_jar.name]
with tempfile.NamedTemporaryFile() as proguard_flags_file:
for flag in proguard_flags:
proguard_flags_file.write(flag + '\n')
proguard_flags_file.flush()
proguard_cmd += ['--pg-conf', proguard_flags_file.name]
for injar in args.class_inputs:
proguard_cmd.append(injar)
build_utils.CheckOutput(proguard_cmd,
print_stderr=False,
fail_on_output=args.warnings_as_errors)
# Record the classes kept by ProGuard. Not used by the build, but useful
# for debugging what classes are kept by ProGuard vs. MainDexListBuilder.
with zipfile.ZipFile(temp_jar.name) as z:
kept_classes = [
p for p in z.namelist() if p.endswith('.class')
]
with open(args.main_dex_list_path + '.partial', 'w') as f:
f.write('\n'.join(kept_classes) + '\n')
# Step 2: Expand inclusion list to all classes referenced by the .class
# files of kept classes (non-recursive).
main_dex_list_cmd = build_utils.JavaCmd() + [
'-cp',
args.dx_path,
'com.android.multidex.MainDexListBuilder',
# This workaround increases main dex size and does not seem to
# be needed by Chrome. See comment in the source:
# https://android.googlesource.com/platform/dalvik/+/master/dx/src/com/android/multidex/MainDexListBuilder.java
'--disable-annotation-resolution-workaround',
temp_jar.name,
':'.join(args.class_inputs)
]
main_dex_list = build_utils.CheckOutput(
main_dex_list_cmd, fail_on_output=args.warnings_as_errors)
except build_utils.CalledProcessError as e:
if 'output jar is empty' in e.output:
pass
elif "input doesn't contain any classes" in e.output:
pass
else:
raise
with build_utils.AtomicOutput(args.main_dex_list_path) as f:
f.write(main_dex_list)
if args.depfile:
build_utils.WriteDepfile(args.depfile,
args.main_dex_list_path,
inputs=args.class_inputs_filearg)
def _CheckForMissingSymbols(r8_path, dex_files, classpath, warnings_as_errors):
cmd = build_utils.JavaCmd(warnings_as_errors) + [
'-cp', r8_path, 'com.android.tools.r8.tracereferences.TraceReferences',
'--map-diagnostics:MissingDefinitionsDiagnostic', 'error', 'warning'
]
for path in classpath:
cmd += ['--lib', path]
for path in dex_files:
cmd += ['--source', path]
def stderr_filter(stderr):
ignored_lines = [
# Summary contains warning count, which our filtering makes wrong.
'Warning: Tracereferences found',
# TODO(agrieve): Create interface jars for these missing classes rather
# than allowlisting here.
'dalvik/system',
'libcore/io',
'sun/misc/Unsafe',
# Found in: com/facebook/fbui/textlayoutbuilder/StaticLayoutHelper
('android/text/StaticLayout;<init>(Ljava/lang/CharSequence;IILandroid'
'/text/TextPaint;ILandroid/text/Layout$Alignment;Landroid/text/'
'TextDirectionHeuristic;FFZLandroid/text/TextUtils$TruncateAt;II)V'),
# Found in
# com/google/android/gms/cast/framework/media/internal/ResourceProvider
# Missing due to setting "strip_resources = true".
'com/google/android/gms/cast/framework/R',
# Found in com/google/android/gms/common/GoogleApiAvailability
# Missing due to setting "strip_drawables = true".
'com/google/android/gms/base/R$drawable',
# Explicictly guarded by try (NoClassDefFoundError) in Flogger's
# PlatformProvider.
'com/google/common/flogger/backend/google/GooglePlatform',
'com/google/common/flogger/backend/system/DefaultPlatform',
# trichrome_webview_google_bundle contains this missing reference.
# TODO(crbug.com/1142530): Fix this missing reference properly.
'org/chromium/base/library_loader/NativeLibraries',
# Currently required when enable_chrome_android_internal=true.
'com/google/protos/research/ink/InkEventProto',
'ink_sdk/com/google/protobuf/Internal$EnumVerifier',
'ink_sdk/com/google/protobuf/MessageLite',
'com/google/protobuf/GeneratedMessageLite$GeneratedExtension',
# Referenced from GeneratedExtensionRegistryLite.
# Exists only for Chrome Modern (not Monochrome nor Trichrome).
# TODO(agrieve): Figure out why. Perhaps related to Feed V2.
('com/google/wireless/android/play/playlog/proto/ClientAnalytics$'
'ClientInfo'),
# TODO(agrieve): Exclude these only when use_jacoco_coverage=true.
'Ljava/lang/instrument/ClassFileTransformer',
'Ljava/lang/instrument/IllegalClassFormatException',
'Ljava/lang/instrument/Instrumentation',
'Ljava/lang/management/ManagementFactory',
'Ljavax/management/MBeanServer',
'Ljavax/management/ObjectInstance',
'Ljavax/management/ObjectName',
'Ljavax/management/StandardMBean',
]
had_unfiltered_items = ' ' in stderr
stderr = build_utils.FilterLines(
stderr, '|'.join(re.escape(x) for x in ignored_lines))
if stderr:
if ' ' in stderr:
stderr = """
DEX contains references to non-existent symbols after R8 optimization.
Tip: Build with:
is_java_debug=false
treat_warnings_as_errors=false
enable_proguard_obfuscation=false
and then use dexdump to see which class(s) reference them.
E.g.:
third_party/android_sdk/public/build-tools/*/dexdump -d \
out/Release/apks/YourApk.apk > dex.txt
""" + stderr
elif had_unfiltered_items:
# Left only with empty headings. All indented items filtered out.
stderr = ''
return stderr
logging.debug('cmd: %s', ' '.join(cmd))
build_utils.CheckOutput(cmd,
print_stdout=True,
stderr_filter=stderr_filter,
fail_on_output=warnings_as_errors)
def main(args):
build_utils.InitLogging('DEX_DEBUG')
options = _ParseArgs(args)
options.class_inputs += options.class_inputs_filearg
options.dex_inputs += options.dex_inputs_filearg
input_paths = options.class_inputs + options.dex_inputs
if options.multi_dex and options.main_dex_list_path:
input_paths.append(options.main_dex_list_path)
input_paths.append(options.r8_jar_path)
input_paths.append(options.custom_d8_jar_path)
depfile_deps = options.class_inputs_filearg + options.dex_inputs_filearg
output_paths = [options.output]
track_subpaths_allowlist = []
if options.incremental_dir:
final_dex_inputs = _IntermediateDexFilePathsFromInputJars(
options.class_inputs, options.incremental_dir)
output_paths += final_dex_inputs
track_subpaths_allowlist += options.class_inputs
else:
final_dex_inputs = list(options.class_inputs)
final_dex_inputs += options.dex_inputs
dex_cmd = build_utils.JavaCmd(options.warnings_as_errors) + [
'-cp',
'{}:{}'.format(options.r8_jar_path, options.custom_d8_jar_path),
'org.chromium.build.CustomD8',
]
if options.release:
dex_cmd += ['--release']
if options.min_api:
dex_cmd += ['--min-api', options.min_api]
if not options.desugar:
dex_cmd += ['--no-desugaring']
elif options.classpath:
# The classpath is used by D8 to for interface desugaring.
if options.desugar_dependencies:
dex_cmd += ['--desugar-dependencies', options.desugar_dependencies]
if track_subpaths_allowlist:
track_subpaths_allowlist += options.classpath
depfile_deps += options.classpath
input_paths += options.classpath
dex_cmd += ['--lib', build_utils.JAVA_HOME]
for path in options.bootclasspath:
dex_cmd += ['--lib', path]
# Still pass the entire classpath in case a new dependency is needed by
# desugar, so that desugar_dependencies will be updated for the next build.
for path in options.classpath:
dex_cmd += ['--classpath', path]
depfile_deps += options.bootclasspath
input_paths += options.bootclasspath
if options.desugar_jdk_libs_json:
dex_cmd += ['--desugared-lib', options.desugar_jdk_libs_json]
if options.force_enable_assertions:
dex_cmd += ['--force-enable-assertions']
# The changes feature from md5_check allows us to only re-dex the class files
# that have changed and the class files that need to be re-desugared by D8.
md5_check.CallAndWriteDepfileIfStale(
lambda changes: _OnStaleMd5(changes, options, final_dex_inputs, dex_cmd
),
options,
input_paths=input_paths,
input_strings=dex_cmd + [bool(options.incremental_dir)],
output_paths=output_paths,
pass_changes=True,
track_subpaths_allowlist=track_subpaths_allowlist,
depfile_deps=depfile_deps)
def main(args):
args = build_utils.ExpandFileArgs(args)
options = _ParseArgs(args)
split_dimensions = []
if options.split_dimensions:
split_dimensions = [x.upper() for x in options.split_dimensions]
with build_utils.TempDir() as tmp_dir:
module_zips = [
_SplitModuleForAssetTargeting(module, tmp_dir, split_dimensions) \
for module in options.module_zips]
base_master_resource_ids = None
if options.base_module_rtxt_path:
base_master_resource_ids = _GenerateBaseResourcesAllowList(
options.base_module_rtxt_path,
options.base_allowlist_rtxt_path)
bundle_config = _GenerateBundleConfigJson(
options.uncompressed_assets, options.compress_shared_libraries,
split_dimensions, base_master_resource_ids)
tmp_bundle = os.path.join(tmp_dir, 'tmp_bundle')
# Important: bundletool requires that the bundle config file is
# named with a .pb.json extension.
tmp_bundle_config = tmp_bundle + '.BundleConfig.pb.json'
with open(tmp_bundle_config, 'w') as f:
f.write(bundle_config)
cmd_args = build_utils.JavaCmd(options.warnings_as_errors) + [
'-jar',
bundletool.BUNDLETOOL_JAR_PATH,
'build-bundle',
'--modules=' + ','.join(module_zips),
'--output=' + tmp_bundle,
'--config=' + tmp_bundle_config,
]
build_utils.CheckOutput(
cmd_args,
print_stdout=True,
print_stderr=True,
stderr_filter=build_utils.FilterReflectiveAccessJavaWarnings,
fail_on_output=options.warnings_as_errors)
if options.validate_services:
# TODO(crbug.com/1126301): This step takes 0.4s locally for bundles with
# isolated splits disabled and 2s for bundles with isolated splits
# enabled. Consider making this run in parallel or move into a separate
# step before enabling isolated splits by default.
_MaybeCheckServicesAndProvidersPresentInBase(
tmp_bundle, module_zips)
shutil.move(tmp_bundle, options.out_bundle)
if options.rtxt_out_path:
_ConcatTextFiles(options.rtxt_in_paths, options.rtxt_out_path)
if options.pathmap_out_path:
_WriteBundlePathmap(options.pathmap_in_paths, options.module_names,
options.pathmap_out_path)