def _DexFilesFromPath(path): if zipfile.is_zipfile(path): with zipfile.ZipFile(path) as z: return [ dex_parser.DexFile(bytearray(z.read(name))) for name in z.namelist() if re.match(r'.*classes[0-9]*\.dex$', name) ] else: with open(path) as f: return dex_parser.DexFile(bytearray(f.read()))
def main(): parser = argparse.ArgumentParser() parser.add_argument('filename') args = parser.parse_args() if os.path.splitext(args.filename)[1] in ('.zip', '.apk', '.jar'): sizes, total_size, num_unique_methods = ExtractSizesFromZip( args.filename) else: with open(args.filename) as f: dexfile = dex_parser.DexFile(bytearray(f.read())) single_set_of_sizes, total_size = _ExtractSizesFromDexFile(dexfile) sizes = {"": single_set_of_sizes} num_unique_methods = single_set_of_sizes['methods'] file_basename = os.path.basename(args.filename) for classes_dex_file, classes_dex_sizes in sizes.iteritems(): for readable_name in _CONTRIBUTORS_TO_DEX_CACHE.itervalues(): if readable_name in classes_dex_sizes: perf_tests_results_helper.PrintPerfResult( '%s_%s_%s' % (file_basename, classes_dex_file, readable_name), 'total', [classes_dex_sizes[readable_name]], readable_name) perf_tests_results_helper.PrintPerfResult( '%s_unique_methods' % file_basename, 'total', [num_unique_methods], 'unique methods') perf_tests_results_helper.PrintPerfResult( '%s_DexCache_size' % (file_basename), 'total', [total_size], 'bytes of permanent dirty memory') return 0
def CollectFromZip(self, label, path): """Add dex stats from an .apk/.jar/.aab/.zip.""" with zipfile.ZipFile(path, 'r') as z: for subpath in z.namelist(): if not re.match(r'.*classes\d*\.dex$', subpath): continue dexfile = dex_parser.DexFile(bytearray(z.read(subpath))) self._CollectFromDexfile('{}!{}'.format(label, subpath), dexfile)
def ExtractSizesFromZip(path): dex_counts_by_file = {} dexcache_size = 0 dexfiles = {} with zipfile.ZipFile(path, 'r') as z: for subpath in z.namelist(): if not re.match(r'.*classes[0-9]*\.dex$', subpath): continue dexfile_name = os.path.basename(subpath) dexfiles[dexfile_name] = dex_parser.DexFile( bytearray(z.read(subpath))) for dexfile_name, dexfile in dexfiles.iteritems(): cur_dex_counts, cur_dexcache_size = _ExtractSizesFromDexFile(dexfile) dex_counts_by_file[dexfile_name] = cur_dex_counts dexcache_size += cur_dexcache_size num_unique_methods = dex_parser.CountUniqueDexMethods(dexfiles.values()) return dex_counts_by_file, dexcache_size, num_unique_methods
def _OptimizeWithR8(options, config_paths, libraries, dynamic_config_data, print_stdout=False): with build_utils.TempDir() as tmp_dir: if dynamic_config_data: dynamic_config_path = os.path.join(tmp_dir, 'dynamic_config.flags') with open(dynamic_config_path, 'w') as f: f.write(dynamic_config_data) config_paths = config_paths + [dynamic_config_path] tmp_mapping_path = os.path.join(tmp_dir, 'mapping.txt') # If there is no output (no classes are kept), this prevents this script # from failing. build_utils.Touch(tmp_mapping_path) tmp_output = os.path.join(tmp_dir, 'r8out') os.mkdir(tmp_output) split_contexts_by_name = {} if options.feature_names: for name, dest_dex, input_jars in zip(options.feature_names, options.dex_dests, options.feature_jars): parent_name = options.uses_split.get(name) if parent_name is None and name != 'base': parent_name = 'base' split_context = _SplitContext(name, dest_dex, input_jars, tmp_output, parent_name=parent_name) split_contexts_by_name[name] = split_context else: # Base context will get populated via "extra_jars" below. split_contexts_by_name['base'] = _SplitContext('base', options.output_path, [], tmp_output) base_context = split_contexts_by_name['base'] # R8 OOMs with the default xmx=1G. cmd = build_utils.JavaCmd(options.warnings_as_errors, xmx='2G') + [ '-Dcom.android.tools.r8.allowTestProguardOptions=1', '-Dcom.android.tools.r8.disableHorizontalClassMerging=1', ] if options.disable_outlining: cmd += ['-Dcom.android.tools.r8.disableOutlining=1'] if options.dump_inputs: cmd += ['-Dcom.android.tools.r8.dumpinputtofile=r8inputs.zip'] cmd += [ '-cp', options.r8_path, 'com.android.tools.r8.R8', '--no-data-resources', '--output', base_context.staging_dir, '--pg-map-output', tmp_mapping_path, ] if options.disable_checks: # Info level priority logs are not printed by default. cmd += ['--map-diagnostics:CheckDiscardDiagnostic', 'error', 'info'] if options.desugar_jdk_libs_json: cmd += [ '--desugared-lib', options.desugar_jdk_libs_json, '--desugared-lib-pg-conf-output', options.desugared_library_keep_rule_output, ] if options.min_api: cmd += ['--min-api', options.min_api] if options.force_enable_assertions: cmd += ['--force-enable-assertions'] for lib in libraries: cmd += ['--lib', lib] for config_file in config_paths: cmd += ['--pg-conf', config_file] if options.main_dex_rules_path: for main_dex_rule in options.main_dex_rules_path: cmd += ['--main-dex-rules', main_dex_rule] _DeDupeInputJars(split_contexts_by_name) # Add any extra inputs to the base context (e.g. desugar runtime). extra_jars = set(options.input_paths) for split_context in split_contexts_by_name.values(): extra_jars -= split_context.input_jars base_context.input_jars.update(extra_jars) for split_context in split_contexts_by_name.values(): if split_context is base_context: continue for in_jar in sorted(split_context.input_jars): cmd += ['--feature', in_jar, split_context.staging_dir] cmd += sorted(base_context.input_jars) try: stderr_filter = dex.CreateStderrFilter( options.show_desugar_default_interface_warnings) logging.debug('Running R8') build_utils.CheckOutput(cmd, print_stdout=print_stdout, stderr_filter=stderr_filter, fail_on_output=options.warnings_as_errors) except build_utils.CalledProcessError as err: debugging_link = ('\n\nR8 failed. Please see {}.'.format( 'https://chromium.googlesource.com/chromium/src/+/HEAD/build/' 'android/docs/java_optimization.md#Debugging-common-failures\n')) raise build_utils.CalledProcessError(err.cwd, err.args, err.output + debugging_link) base_has_imported_lib = False if options.desugar_jdk_libs_json: logging.debug('Running L8') existing_files = build_utils.FindInDirectory(base_context.staging_dir) jdk_dex_output = os.path.join(base_context.staging_dir, 'classes%d.dex' % (len(existing_files) + 1)) # Use -applymapping to avoid name collisions. l8_dynamic_config_path = os.path.join(tmp_dir, 'l8_dynamic_config.flags') with open(l8_dynamic_config_path, 'w') as f: f.write("-applymapping '{}'\n".format(tmp_mapping_path)) # Pass the dynamic config so that obfuscation options are picked up. l8_config_paths = [dynamic_config_path, l8_dynamic_config_path] if os.path.exists(options.desugared_library_keep_rule_output): l8_config_paths.append(options.desugared_library_keep_rule_output) base_has_imported_lib = dex_jdk_libs.DexJdkLibJar( options.r8_path, options.min_api, options.desugar_jdk_libs_json, options.desugar_jdk_libs_jar, options.desugar_jdk_libs_configuration_jar, jdk_dex_output, options.warnings_as_errors, l8_config_paths) if int(options.min_api) >= 24 and base_has_imported_lib: with open(jdk_dex_output, 'rb') as f: dexfile = dex_parser.DexFile(bytearray(f.read())) for m in dexfile.IterMethodSignatureParts(): print('{}#{}'.format(m[0], m[2])) assert False, ( 'Desugared JDK libs are disabled on Monochrome and newer - see ' 'crbug.com/1159984 for details, and see above list for desugared ' 'classes and methods.') logging.debug('Collecting ouputs') base_context.CreateOutput(base_has_imported_lib, options.desugared_library_keep_rule_output) for split_context in split_contexts_by_name.values(): if split_context is not base_context: split_context.CreateOutput() with open(options.mapping_output, 'w') as out_file, \ open(tmp_mapping_path) as in_file: # Mapping files generated by R8 include comments that may break # some of our tooling so remove those (specifically: apkanalyzer). out_file.writelines(l for l in in_file if not l.startswith('#')) return base_context
def CollectFromDex(self, label, path): """Add dex stats from a .dex file.""" with open(path, 'rb') as f: dexfile = dex_parser.DexFile(bytearray(f.read())) self._CollectFromDexfile(label, dexfile)
def _OptimizeWithR8(options, config_paths, libraries, dynamic_config_data, print_stdout=False): with build_utils.TempDir() as tmp_dir: if dynamic_config_data: tmp_config_path = os.path.join(tmp_dir, 'proguard_config.txt') with open(tmp_config_path, 'w') as f: f.write(dynamic_config_data) config_paths = config_paths + [tmp_config_path] tmp_mapping_path = os.path.join(tmp_dir, 'mapping.txt') # If there is no output (no classes are kept), this prevents this script # from failing. build_utils.Touch(tmp_mapping_path) tmp_output = os.path.join(tmp_dir, 'r8out') os.mkdir(tmp_output) feature_contexts = [] if options.feature_names: for name, dest_dex, input_paths in zip(options.feature_names, options.dex_dests, options.feature_jars): feature_context = _DexPathContext(name, dest_dex, input_paths, tmp_output) if name == 'base': base_dex_context = feature_context else: feature_contexts.append(feature_context) else: base_dex_context = _DexPathContext('base', options.output_path, options.input_paths, tmp_output) cmd = build_utils.JavaCmd(options.warnings_as_errors) + [ '-Dcom.android.tools.r8.allowTestProguardOptions=1', '-Dcom.android.tools.r8.verticalClassMerging=1', ] if options.disable_outlining: cmd += ['-Dcom.android.tools.r8.disableOutlining=1'] if options.dump_inputs: cmd += ['-Dcom.android.tools.r8.dumpinputtofile=r8inputs.zip'] cmd += [ '-cp', options.r8_path, 'com.android.tools.r8.R8', '--no-data-resources', '--output', base_dex_context.staging_dir, '--pg-map-output', tmp_mapping_path, ] if options.disable_checks: # Info level priority logs are not printed by default. cmd += [ '--map-diagnostics:CheckDiscardDiagnostic', 'error', 'info' ] if options.desugar_jdk_libs_json: cmd += [ '--desugared-lib', options.desugar_jdk_libs_json, '--desugared-lib-pg-conf-output', options.desugared_library_keep_rule_output, ] if options.min_api: cmd += ['--min-api', options.min_api] if options.force_enable_assertions: cmd += ['--force-enable-assertions'] for lib in libraries: cmd += ['--lib', lib] for config_file in config_paths: cmd += ['--pg-conf', config_file] if options.main_dex_rules_path: for main_dex_rule in options.main_dex_rules_path: cmd += ['--main-dex-rules', main_dex_rule] base_jars = set(base_dex_context.input_paths) input_path_map = defaultdict(set) for feature in feature_contexts: parent = options.uses_split.get(feature.name, feature.name) input_path_map[parent].update(feature.input_paths) # If a jar is present in multiple features, it should be moved to the base # module. all_feature_jars = set() for input_paths in input_path_map.values(): base_jars.update(all_feature_jars.intersection(input_paths)) all_feature_jars.update(input_paths) module_input_jars = base_jars.copy() for feature in feature_contexts: input_paths = input_path_map.get(feature.name) # Input paths can be missing for a child feature present in the uses_split # map. These features get their input paths added to the parent, and are # split out later with DexSplitter. if input_paths is None: continue feature_input_jars = [ p for p in input_paths if p not in module_input_jars ] module_input_jars.update(feature_input_jars) for in_jar in feature_input_jars: cmd += ['--feature', in_jar, feature.staging_dir] cmd += sorted(base_jars) # Add any extra input jars to the base module (e.g. desugar runtime). extra_jars = set(options.input_paths) - module_input_jars cmd += sorted(extra_jars) try: stderr_filter = dex.CreateStderrFilter( options.show_desugar_default_interface_warnings) logging.debug('Running R8') build_utils.CheckOutput(cmd, print_stdout=print_stdout, stderr_filter=stderr_filter, fail_on_output=options.warnings_as_errors) except build_utils.CalledProcessError as err: debugging_link = ('\n\nR8 failed. Please see {}.'.format( 'https://chromium.googlesource.com/chromium/src/+/HEAD/build/' 'android/docs/java_optimization.md#Debugging-common-failures\n' )) raise build_utils.CalledProcessError(err.cwd, err.args, err.output + debugging_link) base_has_imported_lib = False if options.desugar_jdk_libs_json: logging.debug('Running L8') existing_files = build_utils.FindInDirectory( base_dex_context.staging_dir) jdk_dex_output = os.path.join( base_dex_context.staging_dir, 'classes%d.dex' % (len(existing_files) + 1)) base_has_imported_lib = dex_jdk_libs.DexJdkLibJar( options.r8_path, options.min_api, options.desugar_jdk_libs_json, options.desugar_jdk_libs_jar, options.desugar_jdk_libs_configuration_jar, options.desugared_library_keep_rule_output, jdk_dex_output, options.warnings_as_errors) if int(options.min_api) >= 24 and base_has_imported_lib: with open(jdk_dex_output, 'rb') as f: dexfile = dex_parser.DexFile(bytearray(f.read())) for m in dexfile.IterMethodSignatureParts(): print('{}#{}'.format(m[0], m[2])) assert False, ( 'Desugared JDK libs are disabled on Monochrome and newer - see ' 'crbug.com/1159984 for details, and see above list for desugared ' 'classes and methods.') if options.uses_split: _SplitChildFeatures(options, feature_contexts, base_dex_context, tmp_dir, tmp_mapping_path, print_stdout) logging.debug('Collecting ouputs') base_dex_context.CreateOutput( base_has_imported_lib, options.desugared_library_keep_rule_output) for feature in feature_contexts: feature.CreateOutput() with open(options.mapping_output, 'w') as out_file, \ open(tmp_mapping_path) as in_file: # Mapping files generated by R8 include comments that may break # some of our tooling so remove those (specifically: apkanalyzer). out_file.writelines(l for l in in_file if not l.startswith('#'))