def compile_partial(directory):
dirname = os.path.basename(directory)
partial_path = os.path.join(partials_dir, dirname + '.zip')
compile_command = (partial_compile_command +
['--dir', directory, '-o', partial_path])
build_utils.CheckOutput(
compile_command,
stderr_filter=lambda output: build_utils.FilterLines(
output, r'ignoring configuration .* for styleable'))
# Sorting the files in the partial ensures deterministic output from the
# aapt2 link step which uses order of files in the partial.
sorted_partial_path = os.path.join(partials_dir,
dirname + '.sorted.zip')
_SortZip(partial_path, sorted_partial_path)
return sorted_partial_path
def compile_partial(params):
index, dep_path = params
basename = os.path.basename(dep_path)
unique_name = '{}_{}'.format(index, basename)
partial_path = os.path.join(partials_dir, '{}.zip'.format(unique_name))
compile_command = (
partial_compile_command + ['--dir', dep_path, '-o', partial_path])
# There are resources targeting API-versions lower than our minapi. For
# various reasons it's easier to let aapt2 ignore these than for us to
# remove them from our build (e.g. it's from a 3rd party library).
build_utils.CheckOutput(
compile_command,
stderr_filter=lambda output:
build_utils.FilterLines(
output, r'ignoring configuration .* for (styleable|attribute)'))
return partial_path
def stderr_filter(output):
# Filter out warnings caused by our fake main dex list used to enable
# multidex on library targets.
# Warning: Application does not contain `Foo` as referenced in main-dex-list
pattern = r'does not contain `Foo`'
pattern += '|' + '|'.join(re.escape(p) for p in _IGNORE_WARNINGS)
output = build_utils.FilterLines(output, pattern)
# Each warning has a prefix line of tthe file it's from. If we've filtered
# out the warning, then also filter out the file header.
# E.g.:
# Warning in path/to/Foo.class:
# Error message #1 indented here.
# Error message #2 indented here.
output = re.sub(r'^Warning in .*?:\n(?! )',
'',
output,
flags=re.MULTILINE)
return output
def compile_partial(directory):
dirname = os.path.basename(directory)
partial_path = os.path.join(partials_dir, dirname + '.zip')
compile_command = (partial_compile_command +
['--dir', directory, '-o', partial_path])
# There are resources targeting API-versions lower than our minapi. For
# various reasons it's easier to let aapt2 ignore these than for us to
# remove them from our build (e.g. it's from a 3rd party library).
build_utils.CheckOutput(
compile_command,
stderr_filter=lambda output:
build_utils.FilterLines(
output, r'ignoring configuration .* for (styleable|attribute)'))
# Sorting the files in the partial ensures deterministic output from the
# aapt2 link step which uses order of files in the partial.
sorted_partial_path = os.path.join(partials_dir, dirname + '.sorted.zip')
_SortZip(partial_path, sorted_partial_path)
return sorted_partial_path
def filter_stderr(output):
patterns = _IGNORE_WARNINGS
# When using Bazel's Desugar tool to desugar lambdas and interface methods,
# we do not provide D8 with a classpath, which causes a lot of warnings
# from D8's default interface desugaring pass.
# Not having a classpath makes incremental dexing much more effective.
# D8 still does backported method desugaring.
if not show_desugar_default_interface_warnings:
patterns = list(patterns) + ['default or static interface methods']
combined_pattern = '|'.join(re.escape(p) for p in patterns)
output = build_utils.FilterLines(output, combined_pattern)
# Each warning has a prefix line of the file it's from. If we've filtered
# out the warning, then also filter out the file header.
# E.g.:
# Warning in path/to/Foo.class:
# Error message #1 indented here.
# Error message #2 indented here.
output = re.sub(r'^Warning in .*?:\n(?! )', '', output, flags=re.MULTILINE)
return output
def filter_stderr(output):
patterns = list(_IGNORE_WARNINGS)
# When using Bazel's Desugar tool to desugar lambdas and interface methods,
# we do not provide D8 with a classpath, which causes a lot of warnings from
# D8's default interface desugaring pass. Not having a classpath makes
# incremental dexing much more effective. D8 still does backported method
# desugaring.
# These warnings are also turned off when bytecode checks are turned off.
if not show_desugar_default_interface_warnings:
patterns += ['default or static interface methods']
combined_pattern = '|'.join(re.escape(p) for p in patterns)
output = build_utils.FilterLines(output, combined_pattern)
# Each warning has a prefix line of the file it's from. If we've filtered
# out the warning, then also filter out the file header.
# E.g.:
# Warning in path/to/Foo.class:
# Error message #1 indented here.
# Error message #2 indented here.
output = re.sub(r'^Warning in .*?:\n(?! )',
'',
output,
flags=re.MULTILINE)
# Caused by protobuf runtime using -identifiernamestring in a way that
# doesn't work with R8. Looks like:
# Rule matches ... (very long line) {
# static java.lang.String CONTAINING_TYPE_*;
# }
output = re.sub(
r'Rule matches the static final field `java\.lang\.String '
'com\.google\.protobuf.*\{\n.*?\n\}\n?',
'',
output,
flags=re.DOTALL)
return output
def _RunLint(lint_binary_path,
backported_methods_path,
config_path,
manifest_path,
extra_manifest_paths,
sources,
classpath,
cache_dir,
android_sdk_version,
aars,
srcjars,
min_sdk_version,
resource_sources,
resource_zips,
android_sdk_root,
lint_gen_dir,
baseline,
expected_warnings,
testonly_target=False,
warnings_as_errors=False):
logging.info('Lint starting')
cmd = [
lint_binary_path,
'--quiet', # Silences lint's "." progress updates.
'--disable',
','.join(_DISABLED_ALWAYS),
]
if baseline:
cmd.extend(['--baseline', baseline])
if testonly_target:
cmd.extend(['--disable', ','.join(_DISABLED_FOR_TESTS)])
if not manifest_path:
manifest_path = os.path.join(build_utils.DIR_SOURCE_ROOT, 'build',
'android', 'AndroidManifest.xml')
logging.info('Generating config.xml')
backported_methods = _RetrieveBackportedMethods(backported_methods_path)
config_xml_node = _GenerateConfigXmlTree(config_path, backported_methods)
generated_config_path = os.path.join(lint_gen_dir, 'config.xml')
_WriteXmlFile(config_xml_node, generated_config_path)
cmd.extend(['--config', generated_config_path])
logging.info('Generating Android manifest file')
android_manifest_tree = _GenerateAndroidManifest(manifest_path,
extra_manifest_paths,
min_sdk_version,
android_sdk_version)
# Include the rebased manifest_path in the lint generated path so that it is
# clear in error messages where the original AndroidManifest.xml came from.
lint_android_manifest_path = os.path.join(lint_gen_dir, manifest_path)
_WriteXmlFile(android_manifest_tree.getroot(), lint_android_manifest_path)
resource_root_dir = os.path.join(lint_gen_dir, _RES_ZIP_DIR)
# These are zip files with generated resources (e. g. strings from GRD).
logging.info('Extracting resource zips')
for resource_zip in resource_zips:
# Use a consistent root and name rather than a temporary file so that
# suppressions can be local to the lint target and the resource target.
resource_dir = os.path.join(resource_root_dir, resource_zip)
shutil.rmtree(resource_dir, True)
os.makedirs(resource_dir)
resource_sources.extend(
build_utils.ExtractAll(resource_zip, path=resource_dir))
logging.info('Extracting aars')
aar_root_dir = os.path.join(lint_gen_dir, _AAR_DIR)
custom_lint_jars = []
custom_annotation_zips = []
if aars:
for aar in aars:
# Use relative source for aar files since they are not generated.
aar_dir = os.path.join(aar_root_dir,
os.path.splitext(_SrcRelative(aar))[0])
shutil.rmtree(aar_dir, True)
os.makedirs(aar_dir)
aar_files = build_utils.ExtractAll(aar, path=aar_dir)
for f in aar_files:
if f.endswith('lint.jar'):
custom_lint_jars.append(f)
elif f.endswith('annotations.zip'):
custom_annotation_zips.append(f)
logging.info('Extracting srcjars')
srcjar_root_dir = os.path.join(lint_gen_dir, _SRCJAR_DIR)
srcjar_sources = []
if srcjars:
for srcjar in srcjars:
# Use path without extensions since otherwise the file name includes
# .srcjar and lint treats it as a srcjar.
srcjar_dir = os.path.join(srcjar_root_dir, os.path.splitext(srcjar)[0])
shutil.rmtree(srcjar_dir, True)
os.makedirs(srcjar_dir)
# Sadly lint's srcjar support is broken since it only considers the first
# srcjar. Until we roll a lint version with that fixed, we need to extract
# it ourselves.
srcjar_sources.extend(build_utils.ExtractAll(srcjar, path=srcjar_dir))
logging.info('Generating project file')
project_file_root = _GenerateProjectFile(lint_android_manifest_path,
android_sdk_root, cache_dir, sources,
classpath, srcjar_sources,
resource_sources, custom_lint_jars,
custom_annotation_zips,
android_sdk_version)
project_xml_path = os.path.join(lint_gen_dir, 'project.xml')
_WriteXmlFile(project_file_root, project_xml_path)
cmd += ['--project', project_xml_path]
logging.info('Preparing environment variables')
env = os.environ.copy()
# It is important that lint uses the checked-in JDK11 as it is almost 50%
# faster than JDK8.
env['JAVA_HOME'] = build_utils.JAVA_HOME
# This is necessary so that lint errors print stack traces in stdout.
env['LINT_PRINT_STACKTRACE'] = 'true'
if baseline and not os.path.exists(baseline):
# Generating new baselines is only done locally, and requires more memory to
# avoid OOMs.
env['LINT_OPTS'] = '-Xmx4g'
else:
# The default set in the wrapper script is 1g, but it seems not enough :(
env['LINT_OPTS'] = '-Xmx2g'
# This filter is necessary for JDK11.
stderr_filter = build_utils.FilterReflectiveAccessJavaWarnings
stdout_filter = lambda x: build_utils.FilterLines(x, 'No issues found')
if expected_warnings:
stdout_filter = functools.partial(_CheckLintWarning, expected_warnings)
start = time.time()
logging.debug('Lint command %s', ' '.join(cmd))
failed = True
try:
failed = bool(
build_utils.CheckOutput(cmd,
env=env,
print_stdout=True,
stdout_filter=stdout_filter,
stderr_filter=stderr_filter,
fail_on_output=warnings_as_errors))
finally:
# When not treating warnings as errors, display the extra footer.
is_debug = os.environ.get('LINT_DEBUG', '0') != '0'
if failed:
print('- For more help with lint in Chrome:', _LINT_MD_URL)
if is_debug:
print('- DEBUG MODE: Here is the project.xml: {}'.format(
_SrcRelative(project_xml_path)))
else:
print('- Run with LINT_DEBUG=1 to enable lint configuration debugging')
end = time.time() - start
logging.info('Lint command took %ss', end)
if not is_debug:
shutil.rmtree(aar_root_dir, ignore_errors=True)
shutil.rmtree(resource_root_dir, ignore_errors=True)
shutil.rmtree(srcjar_root_dir, ignore_errors=True)
os.unlink(project_xml_path)
logging.info('Lint completed')
def stderr_filter(stderr):
ignored_lines = [
# Summary contains warning count, which our filtering makes wrong.
'Warning: Tracereferences found',
# TODO(agrieve): Create interface jars for these missing classes rather
# than allowlisting here.
'dalvik.system',
'libcore.io',
'sun.misc.Unsafe',
# Found in: com/facebook/fbui/textlayoutbuilder/StaticLayoutHelper
'android.text.StaticLayout.<init>',
# Explicictly guarded by try (NoClassDefFoundError) in Flogger's
# PlatformProvider.
'com.google.common.flogger.backend.google.GooglePlatform',
'com.google.common.flogger.backend.system.DefaultPlatform',
# trichrome_webview_google_bundle contains this missing reference.
# TODO(crbug.com/1142530): Fix this missing reference properly.
'org.chromium.build.NativeLibraries',
# TODO(agrieve): Exclude these only when use_jacoco_coverage=true.
'java.lang.instrument.ClassFileTransformer',
'java.lang.instrument.IllegalClassFormatException',
'java.lang.instrument.Instrumentation',
'java.lang.management.ManagementFactory',
'javax.management.MBeanServer',
'javax.management.ObjectInstance',
'javax.management.ObjectName',
'javax.management.StandardMBean',
# Explicitly guarded by try (NoClassDefFoundError) in Firebase's
# KotlinDetector: com.google.firebase.platforminfo.KotlinDetector.
'kotlin.KotlinVersion',
]
had_unfiltered_items = ' ' in stderr
stderr = build_utils.FilterLines(
stderr, '|'.join(re.escape(x) for x in ignored_lines))
if stderr:
if ' ' in stderr:
stderr = error_title + """
Tip: Build with:
is_java_debug=false
treat_warnings_as_errors=false
enable_proguard_obfuscation=false
and then use dexdump to see which class(s) reference them.
E.g.:
third_party/android_sdk/public/build-tools/*/dexdump -d \
out/Release/apks/YourApk.apk > dex.txt
""" + stderr
if 'FragmentActivity' in stderr:
stderr += """
You may need to update build configs to run FragmentActivityReplacer for
additional targets. See
https://chromium.googlesource.com/chromium/src.git/+/master/docs/ui/android/bytecode_rewriting.md.
"""
elif had_unfiltered_items:
# Left only with empty headings. All indented items filtered out.
stderr = ''
return stderr
def stderr_filter(stderr):
ignored_lines = [
# Summary contains warning count, which our filtering makes wrong.
'Warning: Tracereferences found',
# TODO(agrieve): Create interface jars for these missing classes rather
# than allowlisting here.
'dalvik/system',
'libcore/io',
'sun/misc/Unsafe',
# Found in: com/facebook/fbui/textlayoutbuilder/StaticLayoutHelper
('android/text/StaticLayout;<init>(Ljava/lang/CharSequence;IILandroid'
'/text/TextPaint;ILandroid/text/Layout$Alignment;Landroid/text/'
'TextDirectionHeuristic;FFZLandroid/text/TextUtils$TruncateAt;II)V'),
# Found in
# com/google/android/gms/cast/framework/media/internal/ResourceProvider
# Missing due to setting "strip_resources = true".
'com/google/android/gms/cast/framework/R',
# Found in com/google/android/gms/common/GoogleApiAvailability
# Missing due to setting "strip_drawables = true".
'com/google/android/gms/base/R$drawable',
# Explicictly guarded by try (NoClassDefFoundError) in Flogger's
# PlatformProvider.
'com/google/common/flogger/backend/google/GooglePlatform',
'com/google/common/flogger/backend/system/DefaultPlatform',
# trichrome_webview_google_bundle contains this missing reference.
# TODO(crbug.com/1142530): Fix this missing reference properly.
'org/chromium/base/library_loader/NativeLibraries',
# Currently required when enable_chrome_android_internal=true.
'com/google/protos/research/ink/InkEventProto',
'ink_sdk/com/google/protobuf/Internal$EnumVerifier',
'ink_sdk/com/google/protobuf/MessageLite',
'com/google/protobuf/GeneratedMessageLite$GeneratedExtension',
# Referenced from GeneratedExtensionRegistryLite.
# Exists only for Chrome Modern (not Monochrome nor Trichrome).
# TODO(agrieve): Figure out why. Perhaps related to Feed V2.
('com/google/wireless/android/play/playlog/proto/ClientAnalytics$'
'ClientInfo'),
# TODO(agrieve): Exclude these only when use_jacoco_coverage=true.
'Ljava/lang/instrument/ClassFileTransformer',
'Ljava/lang/instrument/IllegalClassFormatException',
'Ljava/lang/instrument/Instrumentation',
'Ljava/lang/management/ManagementFactory',
'Ljavax/management/MBeanServer',
'Ljavax/management/ObjectInstance',
'Ljavax/management/ObjectName',
'Ljavax/management/StandardMBean',
]
had_unfiltered_items = ' ' in stderr
stderr = build_utils.FilterLines(
stderr, '|'.join(re.escape(x) for x in ignored_lines))
if stderr:
if ' ' in stderr:
stderr = """
DEX contains references to non-existent symbols after R8 optimization.
Tip: Build with:
is_java_debug=false
treat_warnings_as_errors=false
enable_proguard_obfuscation=false
and then use dexdump to see which class(s) reference them.
E.g.:
third_party/android_sdk/public/build-tools/*/dexdump -d \
out/Release/apks/YourApk.apk > dex.txt
""" + stderr
elif had_unfiltered_items:
# Left only with empty headings. All indented items filtered out.
stderr = ''
return stderr
def stderr_filter(stderr):
ignored_lines = [
# Summary contains warning count, which our filtering makes wrong.
'Warning: Tracereferences found',
# TODO(agrieve): Create interface jars for these missing classes rather
# than allowlisting here.
'dalvik/system',
'libcore/io',
'sun/misc/Unsafe',
# Found in: com/facebook/fbui/textlayoutbuilder/StaticLayoutHelper
('android/text/StaticLayout;<init>(Ljava/lang/CharSequence;IILandroid'
'/text/TextPaint;ILandroid/text/Layout$Alignment;Landroid/text/'
'TextDirectionHeuristic;FFZLandroid/text/TextUtils$TruncateAt;II)V'
),
# Found in
# com/google/android/gms/cast/framework/media/internal/ResourceProvider
# Missing due to setting "strip_resources = true".
'com/google/android/gms/cast/framework/R',
# Found in com/google/android/gms/common/GoogleApiAvailability
# Missing due to setting "strip_drawables = true".
'com/google/android/gms/base/R$drawable',
# Explicictly guarded by try (NoClassDefFoundError) in Flogger's
# PlatformProvider.
'com/google/common/flogger/backend/google/GooglePlatform',
'com/google/common/flogger/backend/system/DefaultPlatform',
# trichrome_webview_google_bundle contains this missing reference.
# TODO(crbug.com/1142530): Fix this missing reference properly.
'org/chromium/base/library_loader/NativeLibraries',
# TODO(agrieve): Exclude these only when use_jacoco_coverage=true.
'Ljava/lang/instrument/ClassFileTransformer',
'Ljava/lang/instrument/IllegalClassFormatException',
'Ljava/lang/instrument/Instrumentation',
'Ljava/lang/management/ManagementFactory',
'Ljavax/management/MBeanServer',
'Ljavax/management/ObjectInstance',
'Ljavax/management/ObjectName',
'Ljavax/management/StandardMBean',
# Explicitly guarded by try (NoClassDefFoundError) in Firebase's
# KotlinDetector: com.google.firebase.platforminfo.KotlinDetector.
'Lkotlin/KotlinVersion',
]
had_unfiltered_items = ' ' in stderr
stderr = build_utils.FilterLines(
stderr, '|'.join(re.escape(x) for x in ignored_lines))
if stderr:
if ' ' in stderr:
stderr = """
DEX contains references to non-existent symbols after R8 optimization.
Tip: Build with:
is_java_debug=false
treat_warnings_as_errors=false
enable_proguard_obfuscation=false
and then use dexdump to see which class(s) reference them.
E.g.:
third_party/android_sdk/public/build-tools/*/dexdump -d \
out/Release/apks/YourApk.apk > dex.txt
""" + stderr
if 'FragmentActivity' in stderr:
stderr += """
You may need to update build configs to run FragmentActivityReplacer for
additional targets. See
https://chromium.googlesource.com/chromium/src.git/+/master/docs/ui/android/bytecode_rewriting.md.
"""
elif had_unfiltered_items:
# Left only with empty headings. All indented items filtered out.
stderr = ''
return stderr
def main(argv):
parser = optparse.OptionParser()
build_utils.AddDepfileOption(parser)
parser.add_option("--protoc", help="Path to protoc binary.")
parser.add_option("--proto-path", help="Path to proto directory.")
parser.add_option("--java-out-dir",
help="Path to output directory for java files.")
parser.add_option("--srcjar", help="Path to output srcjar.")
parser.add_option("--stamp", help="File to touch on success.")
parser.add_option("--nano",
help="Use to generate nano protos.",
action='store_true')
parser.add_option(
"--import-dir",
action="append",
default=[],
help="Extra import directory for protos, can be repeated.")
options, args = parser.parse_args(argv)
build_utils.CheckOptions(options, parser, ['protoc', 'proto_path'])
if not options.java_out_dir and not options.srcjar:
print('One of --java-out-dir or --srcjar must be specified.')
return 1
proto_path_args = ['--proto_path', options.proto_path]
for path in options.import_dir:
proto_path_args += ["--proto_path", path]
with build_utils.TempDir() as temp_dir:
if options.nano:
# Specify arguments to the generator.
generator_args = [
'optional_field_style=reftypes', 'store_unknown_fields=true'
]
out_arg = '--javanano_out=' + ','.join(
generator_args) + ':' + temp_dir
else:
out_arg = '--java_out=lite:' + temp_dir
# Generate Java files using protoc.
build_utils.CheckOutput(
[options.protoc] + proto_path_args + [out_arg] + args,
# protoc generates superfluous warnings about LITE_RUNTIME deprecation
# even though we are using the new non-deprecated method.
stderr_filter=lambda output: build_utils.FilterLines(
output, '|'.join(
[r'optimize_for = LITE_RUNTIME', r'java/lite\.md'])))
if options.java_out_dir:
build_utils.DeleteDirectory(options.java_out_dir)
shutil.copytree(temp_dir, options.java_out_dir)
else:
build_utils.ZipDir(options.srcjar, temp_dir)
if options.depfile:
assert options.srcjar
deps = args + [options.protoc]
build_utils.WriteDepfile(options.depfile,
options.srcjar,
deps,
add_pydeps=False)
if options.stamp:
build_utils.Touch(options.stamp)
