def compile_partial(directory): dirname = os.path.basename(directory) partial_path = os.path.join(partials_dir, dirname + '.zip') compile_command = (partial_compile_command + ['--dir', directory, '-o', partial_path]) build_utils.CheckOutput( compile_command, stderr_filter=lambda output: build_utils.FilterLines( output, r'ignoring configuration .* for styleable')) # Sorting the files in the partial ensures deterministic output from the # aapt2 link step which uses order of files in the partial. sorted_partial_path = os.path.join(partials_dir, dirname + '.sorted.zip') _SortZip(partial_path, sorted_partial_path) return sorted_partial_path
def compile_partial(params): index, dep_path = params basename = os.path.basename(dep_path) unique_name = '{}_{}'.format(index, basename) partial_path = os.path.join(partials_dir, '{}.zip'.format(unique_name)) compile_command = ( partial_compile_command + ['--dir', dep_path, '-o', partial_path]) # There are resources targeting API-versions lower than our minapi. For # various reasons it's easier to let aapt2 ignore these than for us to # remove them from our build (e.g. it's from a 3rd party library). build_utils.CheckOutput( compile_command, stderr_filter=lambda output: build_utils.FilterLines( output, r'ignoring configuration .* for (styleable|attribute)')) return partial_path
def stderr_filter(output): # Filter out warnings caused by our fake main dex list used to enable # multidex on library targets. # Warning: Application does not contain `Foo` as referenced in main-dex-list pattern = r'does not contain `Foo`' pattern += '|' + '|'.join(re.escape(p) for p in _IGNORE_WARNINGS) output = build_utils.FilterLines(output, pattern) # Each warning has a prefix line of tthe file it's from. If we've filtered # out the warning, then also filter out the file header. # E.g.: # Warning in path/to/Foo.class: # Error message #1 indented here. # Error message #2 indented here. output = re.sub(r'^Warning in .*?:\n(?! )', '', output, flags=re.MULTILINE) return output
def compile_partial(directory): dirname = os.path.basename(directory) partial_path = os.path.join(partials_dir, dirname + '.zip') compile_command = (partial_compile_command + ['--dir', directory, '-o', partial_path]) # There are resources targeting API-versions lower than our minapi. For # various reasons it's easier to let aapt2 ignore these than for us to # remove them from our build (e.g. it's from a 3rd party library). build_utils.CheckOutput( compile_command, stderr_filter=lambda output: build_utils.FilterLines( output, r'ignoring configuration .* for (styleable|attribute)')) # Sorting the files in the partial ensures deterministic output from the # aapt2 link step which uses order of files in the partial. sorted_partial_path = os.path.join(partials_dir, dirname + '.sorted.zip') _SortZip(partial_path, sorted_partial_path) return sorted_partial_path
def filter_stderr(output): patterns = _IGNORE_WARNINGS # When using Bazel's Desugar tool to desugar lambdas and interface methods, # we do not provide D8 with a classpath, which causes a lot of warnings # from D8's default interface desugaring pass. # Not having a classpath makes incremental dexing much more effective. # D8 still does backported method desugaring. if not show_desugar_default_interface_warnings: patterns = list(patterns) + ['default or static interface methods'] combined_pattern = '|'.join(re.escape(p) for p in patterns) output = build_utils.FilterLines(output, combined_pattern) # Each warning has a prefix line of the file it's from. If we've filtered # out the warning, then also filter out the file header. # E.g.: # Warning in path/to/Foo.class: # Error message #1 indented here. # Error message #2 indented here. output = re.sub(r'^Warning in .*?:\n(?! )', '', output, flags=re.MULTILINE) return output
def filter_stderr(output): patterns = list(_IGNORE_WARNINGS) # When using Bazel's Desugar tool to desugar lambdas and interface methods, # we do not provide D8 with a classpath, which causes a lot of warnings from # D8's default interface desugaring pass. Not having a classpath makes # incremental dexing much more effective. D8 still does backported method # desugaring. # These warnings are also turned off when bytecode checks are turned off. if not show_desugar_default_interface_warnings: patterns += ['default or static interface methods'] combined_pattern = '|'.join(re.escape(p) for p in patterns) output = build_utils.FilterLines(output, combined_pattern) # Each warning has a prefix line of the file it's from. If we've filtered # out the warning, then also filter out the file header. # E.g.: # Warning in path/to/Foo.class: # Error message #1 indented here. # Error message #2 indented here. output = re.sub(r'^Warning in .*?:\n(?! )', '', output, flags=re.MULTILINE) # Caused by protobuf runtime using -identifiernamestring in a way that # doesn't work with R8. Looks like: # Rule matches ... (very long line) { # static java.lang.String CONTAINING_TYPE_*; # } output = re.sub( r'Rule matches the static final field `java\.lang\.String ' 'com\.google\.protobuf.*\{\n.*?\n\}\n?', '', output, flags=re.DOTALL) return output
def _RunLint(lint_binary_path, backported_methods_path, config_path, manifest_path, extra_manifest_paths, sources, classpath, cache_dir, android_sdk_version, aars, srcjars, min_sdk_version, resource_sources, resource_zips, android_sdk_root, lint_gen_dir, baseline, expected_warnings, testonly_target=False, warnings_as_errors=False): logging.info('Lint starting') cmd = [ lint_binary_path, '--quiet', # Silences lint's "." progress updates. '--disable', ','.join(_DISABLED_ALWAYS), ] if baseline: cmd.extend(['--baseline', baseline]) if testonly_target: cmd.extend(['--disable', ','.join(_DISABLED_FOR_TESTS)]) if not manifest_path: manifest_path = os.path.join(build_utils.DIR_SOURCE_ROOT, 'build', 'android', 'AndroidManifest.xml') logging.info('Generating config.xml') backported_methods = _RetrieveBackportedMethods(backported_methods_path) config_xml_node = _GenerateConfigXmlTree(config_path, backported_methods) generated_config_path = os.path.join(lint_gen_dir, 'config.xml') _WriteXmlFile(config_xml_node, generated_config_path) cmd.extend(['--config', generated_config_path]) logging.info('Generating Android manifest file') android_manifest_tree = _GenerateAndroidManifest(manifest_path, extra_manifest_paths, min_sdk_version, android_sdk_version) # Include the rebased manifest_path in the lint generated path so that it is # clear in error messages where the original AndroidManifest.xml came from. lint_android_manifest_path = os.path.join(lint_gen_dir, manifest_path) _WriteXmlFile(android_manifest_tree.getroot(), lint_android_manifest_path) resource_root_dir = os.path.join(lint_gen_dir, _RES_ZIP_DIR) # These are zip files with generated resources (e. g. strings from GRD). logging.info('Extracting resource zips') for resource_zip in resource_zips: # Use a consistent root and name rather than a temporary file so that # suppressions can be local to the lint target and the resource target. resource_dir = os.path.join(resource_root_dir, resource_zip) shutil.rmtree(resource_dir, True) os.makedirs(resource_dir) resource_sources.extend( build_utils.ExtractAll(resource_zip, path=resource_dir)) logging.info('Extracting aars') aar_root_dir = os.path.join(lint_gen_dir, _AAR_DIR) custom_lint_jars = [] custom_annotation_zips = [] if aars: for aar in aars: # Use relative source for aar files since they are not generated. aar_dir = os.path.join(aar_root_dir, os.path.splitext(_SrcRelative(aar))[0]) shutil.rmtree(aar_dir, True) os.makedirs(aar_dir) aar_files = build_utils.ExtractAll(aar, path=aar_dir) for f in aar_files: if f.endswith('lint.jar'): custom_lint_jars.append(f) elif f.endswith('annotations.zip'): custom_annotation_zips.append(f) logging.info('Extracting srcjars') srcjar_root_dir = os.path.join(lint_gen_dir, _SRCJAR_DIR) srcjar_sources = [] if srcjars: for srcjar in srcjars: # Use path without extensions since otherwise the file name includes # .srcjar and lint treats it as a srcjar. srcjar_dir = os.path.join(srcjar_root_dir, os.path.splitext(srcjar)[0]) shutil.rmtree(srcjar_dir, True) os.makedirs(srcjar_dir) # Sadly lint's srcjar support is broken since it only considers the first # srcjar. Until we roll a lint version with that fixed, we need to extract # it ourselves. srcjar_sources.extend(build_utils.ExtractAll(srcjar, path=srcjar_dir)) logging.info('Generating project file') project_file_root = _GenerateProjectFile(lint_android_manifest_path, android_sdk_root, cache_dir, sources, classpath, srcjar_sources, resource_sources, custom_lint_jars, custom_annotation_zips, android_sdk_version) project_xml_path = os.path.join(lint_gen_dir, 'project.xml') _WriteXmlFile(project_file_root, project_xml_path) cmd += ['--project', project_xml_path] logging.info('Preparing environment variables') env = os.environ.copy() # It is important that lint uses the checked-in JDK11 as it is almost 50% # faster than JDK8. env['JAVA_HOME'] = build_utils.JAVA_HOME # This is necessary so that lint errors print stack traces in stdout. env['LINT_PRINT_STACKTRACE'] = 'true' if baseline and not os.path.exists(baseline): # Generating new baselines is only done locally, and requires more memory to # avoid OOMs. env['LINT_OPTS'] = '-Xmx4g' else: # The default set in the wrapper script is 1g, but it seems not enough :( env['LINT_OPTS'] = '-Xmx2g' # This filter is necessary for JDK11. stderr_filter = build_utils.FilterReflectiveAccessJavaWarnings stdout_filter = lambda x: build_utils.FilterLines(x, 'No issues found') if expected_warnings: stdout_filter = functools.partial(_CheckLintWarning, expected_warnings) start = time.time() logging.debug('Lint command %s', ' '.join(cmd)) failed = True try: failed = bool( build_utils.CheckOutput(cmd, env=env, print_stdout=True, stdout_filter=stdout_filter, stderr_filter=stderr_filter, fail_on_output=warnings_as_errors)) finally: # When not treating warnings as errors, display the extra footer. is_debug = os.environ.get('LINT_DEBUG', '0') != '0' if failed: print('- For more help with lint in Chrome:', _LINT_MD_URL) if is_debug: print('- DEBUG MODE: Here is the project.xml: {}'.format( _SrcRelative(project_xml_path))) else: print('- Run with LINT_DEBUG=1 to enable lint configuration debugging') end = time.time() - start logging.info('Lint command took %ss', end) if not is_debug: shutil.rmtree(aar_root_dir, ignore_errors=True) shutil.rmtree(resource_root_dir, ignore_errors=True) shutil.rmtree(srcjar_root_dir, ignore_errors=True) os.unlink(project_xml_path) logging.info('Lint completed')
def stderr_filter(stderr): ignored_lines = [ # Summary contains warning count, which our filtering makes wrong. 'Warning: Tracereferences found', # TODO(agrieve): Create interface jars for these missing classes rather # than allowlisting here. 'dalvik.system', 'libcore.io', 'sun.misc.Unsafe', # Found in: com/facebook/fbui/textlayoutbuilder/StaticLayoutHelper 'android.text.StaticLayout.<init>', # Explicictly guarded by try (NoClassDefFoundError) in Flogger's # PlatformProvider. 'com.google.common.flogger.backend.google.GooglePlatform', 'com.google.common.flogger.backend.system.DefaultPlatform', # trichrome_webview_google_bundle contains this missing reference. # TODO(crbug.com/1142530): Fix this missing reference properly. 'org.chromium.build.NativeLibraries', # TODO(agrieve): Exclude these only when use_jacoco_coverage=true. 'java.lang.instrument.ClassFileTransformer', 'java.lang.instrument.IllegalClassFormatException', 'java.lang.instrument.Instrumentation', 'java.lang.management.ManagementFactory', 'javax.management.MBeanServer', 'javax.management.ObjectInstance', 'javax.management.ObjectName', 'javax.management.StandardMBean', # Explicitly guarded by try (NoClassDefFoundError) in Firebase's # KotlinDetector: com.google.firebase.platforminfo.KotlinDetector. 'kotlin.KotlinVersion', ] had_unfiltered_items = ' ' in stderr stderr = build_utils.FilterLines( stderr, '|'.join(re.escape(x) for x in ignored_lines)) if stderr: if ' ' in stderr: stderr = error_title + """ Tip: Build with: is_java_debug=false treat_warnings_as_errors=false enable_proguard_obfuscation=false and then use dexdump to see which class(s) reference them. E.g.: third_party/android_sdk/public/build-tools/*/dexdump -d \ out/Release/apks/YourApk.apk > dex.txt """ + stderr if 'FragmentActivity' in stderr: stderr += """ You may need to update build configs to run FragmentActivityReplacer for additional targets. See https://chromium.googlesource.com/chromium/src.git/+/master/docs/ui/android/bytecode_rewriting.md. """ elif had_unfiltered_items: # Left only with empty headings. All indented items filtered out. stderr = '' return stderr
def stderr_filter(stderr): ignored_lines = [ # Summary contains warning count, which our filtering makes wrong. 'Warning: Tracereferences found', # TODO(agrieve): Create interface jars for these missing classes rather # than allowlisting here. 'dalvik/system', 'libcore/io', 'sun/misc/Unsafe', # Found in: com/facebook/fbui/textlayoutbuilder/StaticLayoutHelper ('android/text/StaticLayout;<init>(Ljava/lang/CharSequence;IILandroid' '/text/TextPaint;ILandroid/text/Layout$Alignment;Landroid/text/' 'TextDirectionHeuristic;FFZLandroid/text/TextUtils$TruncateAt;II)V'), # Found in # com/google/android/gms/cast/framework/media/internal/ResourceProvider # Missing due to setting "strip_resources = true". 'com/google/android/gms/cast/framework/R', # Found in com/google/android/gms/common/GoogleApiAvailability # Missing due to setting "strip_drawables = true". 'com/google/android/gms/base/R$drawable', # Explicictly guarded by try (NoClassDefFoundError) in Flogger's # PlatformProvider. 'com/google/common/flogger/backend/google/GooglePlatform', 'com/google/common/flogger/backend/system/DefaultPlatform', # trichrome_webview_google_bundle contains this missing reference. # TODO(crbug.com/1142530): Fix this missing reference properly. 'org/chromium/base/library_loader/NativeLibraries', # Currently required when enable_chrome_android_internal=true. 'com/google/protos/research/ink/InkEventProto', 'ink_sdk/com/google/protobuf/Internal$EnumVerifier', 'ink_sdk/com/google/protobuf/MessageLite', 'com/google/protobuf/GeneratedMessageLite$GeneratedExtension', # Referenced from GeneratedExtensionRegistryLite. # Exists only for Chrome Modern (not Monochrome nor Trichrome). # TODO(agrieve): Figure out why. Perhaps related to Feed V2. ('com/google/wireless/android/play/playlog/proto/ClientAnalytics$' 'ClientInfo'), # TODO(agrieve): Exclude these only when use_jacoco_coverage=true. 'Ljava/lang/instrument/ClassFileTransformer', 'Ljava/lang/instrument/IllegalClassFormatException', 'Ljava/lang/instrument/Instrumentation', 'Ljava/lang/management/ManagementFactory', 'Ljavax/management/MBeanServer', 'Ljavax/management/ObjectInstance', 'Ljavax/management/ObjectName', 'Ljavax/management/StandardMBean', ] had_unfiltered_items = ' ' in stderr stderr = build_utils.FilterLines( stderr, '|'.join(re.escape(x) for x in ignored_lines)) if stderr: if ' ' in stderr: stderr = """ DEX contains references to non-existent symbols after R8 optimization. Tip: Build with: is_java_debug=false treat_warnings_as_errors=false enable_proguard_obfuscation=false and then use dexdump to see which class(s) reference them. E.g.: third_party/android_sdk/public/build-tools/*/dexdump -d \ out/Release/apks/YourApk.apk > dex.txt """ + stderr elif had_unfiltered_items: # Left only with empty headings. All indented items filtered out. stderr = '' return stderr
def stderr_filter(stderr): ignored_lines = [ # Summary contains warning count, which our filtering makes wrong. 'Warning: Tracereferences found', # TODO(agrieve): Create interface jars for these missing classes rather # than allowlisting here. 'dalvik/system', 'libcore/io', 'sun/misc/Unsafe', # Found in: com/facebook/fbui/textlayoutbuilder/StaticLayoutHelper ('android/text/StaticLayout;<init>(Ljava/lang/CharSequence;IILandroid' '/text/TextPaint;ILandroid/text/Layout$Alignment;Landroid/text/' 'TextDirectionHeuristic;FFZLandroid/text/TextUtils$TruncateAt;II)V' ), # Found in # com/google/android/gms/cast/framework/media/internal/ResourceProvider # Missing due to setting "strip_resources = true". 'com/google/android/gms/cast/framework/R', # Found in com/google/android/gms/common/GoogleApiAvailability # Missing due to setting "strip_drawables = true". 'com/google/android/gms/base/R$drawable', # Explicictly guarded by try (NoClassDefFoundError) in Flogger's # PlatformProvider. 'com/google/common/flogger/backend/google/GooglePlatform', 'com/google/common/flogger/backend/system/DefaultPlatform', # trichrome_webview_google_bundle contains this missing reference. # TODO(crbug.com/1142530): Fix this missing reference properly. 'org/chromium/base/library_loader/NativeLibraries', # TODO(agrieve): Exclude these only when use_jacoco_coverage=true. 'Ljava/lang/instrument/ClassFileTransformer', 'Ljava/lang/instrument/IllegalClassFormatException', 'Ljava/lang/instrument/Instrumentation', 'Ljava/lang/management/ManagementFactory', 'Ljavax/management/MBeanServer', 'Ljavax/management/ObjectInstance', 'Ljavax/management/ObjectName', 'Ljavax/management/StandardMBean', # Explicitly guarded by try (NoClassDefFoundError) in Firebase's # KotlinDetector: com.google.firebase.platforminfo.KotlinDetector. 'Lkotlin/KotlinVersion', ] had_unfiltered_items = ' ' in stderr stderr = build_utils.FilterLines( stderr, '|'.join(re.escape(x) for x in ignored_lines)) if stderr: if ' ' in stderr: stderr = """ DEX contains references to non-existent symbols after R8 optimization. Tip: Build with: is_java_debug=false treat_warnings_as_errors=false enable_proguard_obfuscation=false and then use dexdump to see which class(s) reference them. E.g.: third_party/android_sdk/public/build-tools/*/dexdump -d \ out/Release/apks/YourApk.apk > dex.txt """ + stderr if 'FragmentActivity' in stderr: stderr += """ You may need to update build configs to run FragmentActivityReplacer for additional targets. See https://chromium.googlesource.com/chromium/src.git/+/master/docs/ui/android/bytecode_rewriting.md. """ elif had_unfiltered_items: # Left only with empty headings. All indented items filtered out. stderr = '' return stderr
def main(argv): parser = optparse.OptionParser() build_utils.AddDepfileOption(parser) parser.add_option("--protoc", help="Path to protoc binary.") parser.add_option("--proto-path", help="Path to proto directory.") parser.add_option("--java-out-dir", help="Path to output directory for java files.") parser.add_option("--srcjar", help="Path to output srcjar.") parser.add_option("--stamp", help="File to touch on success.") parser.add_option("--nano", help="Use to generate nano protos.", action='store_true') parser.add_option( "--import-dir", action="append", default=[], help="Extra import directory for protos, can be repeated.") options, args = parser.parse_args(argv) build_utils.CheckOptions(options, parser, ['protoc', 'proto_path']) if not options.java_out_dir and not options.srcjar: print('One of --java-out-dir or --srcjar must be specified.') return 1 proto_path_args = ['--proto_path', options.proto_path] for path in options.import_dir: proto_path_args += ["--proto_path", path] with build_utils.TempDir() as temp_dir: if options.nano: # Specify arguments to the generator. generator_args = [ 'optional_field_style=reftypes', 'store_unknown_fields=true' ] out_arg = '--javanano_out=' + ','.join( generator_args) + ':' + temp_dir else: out_arg = '--java_out=lite:' + temp_dir # Generate Java files using protoc. build_utils.CheckOutput( [options.protoc] + proto_path_args + [out_arg] + args, # protoc generates superfluous warnings about LITE_RUNTIME deprecation # even though we are using the new non-deprecated method. stderr_filter=lambda output: build_utils.FilterLines( output, '|'.join( [r'optimize_for = LITE_RUNTIME', r'java/lite\.md']))) if options.java_out_dir: build_utils.DeleteDirectory(options.java_out_dir) shutil.copytree(temp_dir, options.java_out_dir) else: build_utils.ZipDir(options.srcjar, temp_dir) if options.depfile: assert options.srcjar deps = args + [options.protoc] build_utils.WriteDepfile(options.depfile, options.srcjar, deps, add_pydeps=False) if options.stamp: build_utils.Touch(options.stamp)