def Main(): utils.check_java_version() args = parse_arguments() output_dir = args.output if args.golem: golem.link_third_party() with utils.TempDir() as temp_dir: if not output_dir: output_dir = temp_dir xmx = None if args.tool == 'dx': tool_file = DX_JAR tool_args = [ '--dex', '--output=' + output_dir, '--multi-dex', '--min-sdk-version=' + MIN_SDK_VERSION ] xmx = '-Xmx1600m' else: tool_file = utils.D8_JAR tool_args = ['--output', output_dir, '--min-api', MIN_SDK_VERSION] if args.tool == 'd8-release': tool_args.append('--release') xmx = '-Xmx600m' cmd = [] track_memory_file = None if args.print_memoryuse: track_memory_file = os.path.join(output_dir, utils.MEMORY_USE_TMP_FILE) cmd.extend(['tools/track_memory.sh', track_memory_file]) if tool_file.endswith('.jar'): assert xmx is not None cmd.extend(['java', xmx, '-jar']) cmd.extend([tool_file] + tool_args + [FRAMEWORK_JAR]) utils.PrintCmd(cmd) t0 = time.time() subprocess.check_call(cmd) dt = time.time() - t0 if args.print_memoryuse: print('{}-Total(MemoryUse): {}'.format( args.name, utils.grep_memoryuse(track_memory_file))) dex_files = [f for f in glob(os.path.join(output_dir, '*.dex'))] code_size = 0 for dex_file in dex_files: code_size += os.path.getsize(dex_file) print('{}-Total(RunTimeRaw): {} ms'.format(args.name, 1000.0 * dt)) print('{}-Total(CodeSize): {}'.format(args.name, code_size)) utils.print_dexsegments(args.name, dex_files)
def Main(): args = parse_arguments() if args.golem: golem.link_third_party() utils.check_java_version() with utils.TempDir() as temp_dir: cmd_prefix = [] output_dir = args.output if args.output else temp_dir temp_dir = os.path.join(args.output, 'tmp') if args.output else temp_dir track_memory_file = None if args.print_memoryuse: track_memory_file = os.path.join(output_dir, utils.MEMORY_USE_TMP_FILE) cmd_prefix.extend(['tools/track_memory.sh', track_memory_file]) name = 'CompileHelloExample' tool = args.tool output_mode = args.output_mode lib = None if output_mode == 'dex': name += 'Dex' lib = utils.get_android_jar(28) else: lib = utils.RT_JAR extra = [] if args.large: name += 'Large' extra = EXTRA_INPUTS if args.noopt: name += 'NoOpt' cmds = Compile( tool, output_mode, lib, extra, output_dir, args.noopt, temp_dir, ) t0 = time.time() for cmd in cmds: fullcmd = cmd_prefix + cmd utils.PrintCmd(fullcmd) subprocess.check_output(fullcmd) dt = time.time() - t0 if args.print_memoryuse: print('{}(MemoryUse): {}'.format( name, utils.grep_memoryuse(track_memory_file))) print('{}(RunTimeRaw): {} ms'.format(name, 1000.0 * dt))
def main(argv): (options, args) = ParseOptions(argv) if not options.ignore_java_version: utils.check_java_version() if options.run_all: return run_all(options, args) return run_with_options(options, args)
def Main(): utils.check_java_version() args = parse_arguments() if not args.no_build: gradle.RunGradle(['CompatDx']) cmd = [REPLAY_SCRIPT, 'java', '-jar', utils.COMPATDX_JAR] utils.PrintCmd(cmd) subprocess.check_call(cmd) # collect dex files below OUT_DIR dex_files = (chain.from_iterable( glob(join(x[0], '*.dex')) for x in os.walk(OUT_DIR))) for dex_file in dex_files: utils.verify_with_dex2oat(dex_file)
def run_with_options(options, args, extra_args=None): if extra_args is None: extra_args = [] app_provided_pg_conf = False # todo(121018500): remove when memory is under control if not any('-Xmx' in arg for arg in extra_args): extra_args.append('-Xmx8G') if options.golem: golem.link_third_party() options.out = os.getcwd() if not options.ignore_java_version: utils.check_java_version() outdir = options.out data = None if options.app == 'gmscore': options.version = options.version or 'v9' data = gmscore_data elif options.app == 'nest': options.version = options.version or '20180926' data = nest_data elif options.app == 'youtube': options.version = options.version or '12.22' data = youtube_data elif options.app == 'chrome': options.version = options.version or '180917' data = chrome_data elif options.app == 'gmail': options.version = options.version or '170604.16' data = gmail_data else: raise Exception("You need to specify '--app={}'".format( '|'.join(APPS))) if options.compiler not in COMPILERS: raise Exception("You need to specify '--compiler={}'".format( '|'.join(COMPILERS))) if options.compiler_build not in COMPILER_BUILDS: raise Exception("You need to specify '--compiler-build={}'".format( '|'.join(COMPILER_BUILDS))) if not options.version in data.VERSIONS.keys(): print('No version {} for application {}'.format( options.version, options.app)) print('Valid versions are {}'.format(data.VERSIONS.keys())) return 1 version = data.VERSIONS[options.version] if not options.type: options.type = 'deploy' if options.compiler == 'r8' \ else 'proguarded' if options.type not in version: print('No type {} for version {}'.format(options.type, options.version)) print('Valid types are {}'.format(version.keys())) return 1 values = version[options.type] inputs = None # For R8 'deploy' the JAR is located using the Proguard configuration # -injars option. For chrome and nest we don't have the injars in the # proguard files. if 'inputs' in values and (options.compiler != 'r8' or options.type != 'deploy' or options.app == 'chrome' or options.app == 'nest'): inputs = values['inputs'] args.extend(['--output', outdir]) if 'min-api' in values: args.extend(['--min-api', values['min-api']]) if 'main-dex-list' in values: args.extend(['--main-dex-list', values['main-dex-list']]) if options.compiler == 'r8': if 'pgconf' in values and not options.k: sanitized_lib_path = os.path.join(os.path.abspath(outdir), 'sanitized_lib.jar') sanitized_pgconf_path = os.path.join(os.path.abspath(outdir), 'sanitized.config') SanitizeLibraries(sanitized_lib_path, sanitized_pgconf_path, values['pgconf']) args.extend(['--pg-conf', sanitized_pgconf_path]) app_provided_pg_conf = True if options.k: args.extend(['--pg-conf', options.k]) if 'maindexrules' in values: for rules in values['maindexrules']: args.extend(['--main-dex-rules', rules]) if 'allow-type-errors' in values: extra_args.append('-Dcom.android.tools.r8.allowTypeErrors=1') if 'proto-shrinking' in values: extra_args.append( '-Dcom.android.tools.r8.generatedExtensionRegistryShrinking=1') if not options.no_libraries and 'libraries' in values: for lib in values['libraries']: args.extend(['--lib', lib]) if not outdir.endswith('.zip') and not outdir.endswith('.jar') \ and not os.path.exists(outdir): os.makedirs(outdir) # Additional flags for the compiler from the configuration file. if 'flags' in values: args.extend(values['flags'].split(' ')) if options.compiler == 'r8': if 'r8-flags' in values: args.extend(values['r8-flags'].split(' ')) # Additional flags for the compiler from the command line. if options.compiler_flags: args.extend(options.compiler_flags.split(' ')) if options.r8_flags: args.extend(options.r8_flags.split(' ')) if inputs: args.extend(inputs) t0 = time.time() if options.dump_args_file: with open(options.dump_args_file, 'w') as args_file: args_file.writelines([arg + os.linesep for arg in args]) else: with utils.TempDir() as temp: if options.print_memoryuse and not options.track_memory_to_file: options.track_memory_to_file = os.path.join( temp, utils.MEMORY_USE_TMP_FILE) if options.compiler == 'r8' and app_provided_pg_conf: # Ensure that output of -printmapping and -printseeds go to the output # location and not where the app Proguard configuration places them. if outdir.endswith('.zip') or outdir.endswith('.jar'): pg_outdir = os.path.dirname(outdir) else: pg_outdir = outdir additional_pg_conf = GenerateAdditionalProguardConfiguration( temp, os.path.abspath(pg_outdir)) args.extend(['--pg-conf', additional_pg_conf]) build = not options.no_build and not options.golem stderr_path = os.path.join(temp, 'stderr') with open(stderr_path, 'w') as stderr: if options.compiler_build == 'full': tool = options.compiler else: assert (options.compiler_build == 'lib') tool = 'r8lib-' + options.compiler exit_code = toolhelper.run( tool, args, build=build, debug=not options.no_debug, profile=options.profile, track_memory_file=options.track_memory_to_file, extra_args=extra_args, stderr=stderr, timeout=options.timeout) if exit_code != 0: with open(stderr_path) as stderr: stderr_text = stderr.read() print(stderr_text) if 'java.lang.OutOfMemoryError' in stderr_text: print('Failure was OOM') return OOM_EXIT_CODE return exit_code if options.print_memoryuse: print('{}(MemoryUse): {}'.format( options.print_memoryuse, utils.grep_memoryuse(options.track_memory_to_file))) if options.print_runtimeraw: print('{}(RunTimeRaw): {} ms'.format(options.print_runtimeraw, 1000.0 * (time.time() - t0))) if options.print_dexsegments: dex_files = glob(os.path.join(outdir, '*.dex')) utils.print_dexsegments(options.print_dexsegments, dex_files) return 0
def Main(argv): utils.check_java_version() options = parse_arguments(argv) if options.golem: golem.link_third_party() outdir = options.out if options.app == 'gmscore': version = 'v10' data = gmscore_data base = data.V10_BASE elif options.app == 'youtube': version = '12.22' data = youtube_data base = data.V12_22_BASE else: raise Exception('Unexpected') args = ['-forceprocessing'] if not outdir.endswith('.zip') and not outdir.endswith('.jar') \ and not exists(outdir): makedirs(outdir) values_deploy = data.VERSIONS[version]['deploy'] values_proguarded = data.VERSIONS[version]['proguarded'] assert 'pgconf' in values_deploy for pgconf in values_deploy['pgconf']: args.extend(['@' + pgconf]) # find seeds file inputs = data.VERSIONS[version]['proguarded']['inputs'] assert len(inputs) == 1 basename_wo_ext = splitext(os.path.basename(inputs[0]))[0] seeds_filename = basename_wo_ext + '.seeds' seeds_files = [] for root, dirnames, filenames in os.walk(join(base, 'blaze-out')): for filename in fnmatch.filter(filenames, seeds_filename): seeds_files.append(os.path.join(root, filename)) assert len(seeds_files) == 1 seeds_path = seeds_files[0] proguarded_jar_path = splitext(seeds_path)[0] + '.jar' # Remove write-protection from seeds file. The seeds file is an output of # ProGuard so it aborts if this is not writeable. st = os.stat(seeds_path) os.chmod(seeds_path, st.st_mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) t0 = time.time() proguard_memoryuse = None with utils.TempDir() as temp: track_memory_file = None if options.print_memoryuse: track_memory_file = join(temp, utils.MEMORY_USE_TMP_FILE) proguard.run( args, track_memory_file = track_memory_file, stdout=open(os.devnull, 'w')) if options.print_memoryuse: proguard_memoryuse = utils.grep_memoryuse(track_memory_file) # run dex on the result if options.compatdx: jar = utils.COMPATDX_JAR else: jar = DX_JAR with utils.TempDir() as temp: track_memory_file = None cmd = [] if options.print_memoryuse: track_memory_file = join(temp, utils.MEMORY_USE_TMP_FILE) cmd.extend(['tools/track_memory.sh', track_memory_file]) cmd.extend(['java', '-jar', jar, '--multi-dex', '--output=' + outdir]) if 'min-api' in values_proguarded: cmd.append('--min-sdk-version=' + values_proguarded['min-api']) cmd.extend(['--dex', proguarded_jar_path]) utils.PrintCmd(cmd); check_call(cmd) if options.print_memoryuse: dx_memoryuse = utils.grep_memoryuse(track_memory_file) print('{}(MemoryUse): {}' .format(options.print_memoryuse, max(proguard_memoryuse, dx_memoryuse))) if options.print_runtimeraw: print('{}(RunTimeRaw): {} ms' .format(options.print_runtimeraw, 1000.0 * (time.time() - t0))) if options.print_dexsegments: dex_files = glob(os.path.join(outdir, '*.dex')) utils.print_dexsegments(options.print_dexsegments, dex_files)
def run_with_options(options, args, extra_args=None, stdout=None, quiet=False): if extra_args is None: extra_args = [] app_provided_pg_conf = False # todo(121018500): remove when memory is under control if not any('-Xmx' in arg for arg in extra_args): if options.max_memory: extra_args.append('-Xmx%sM' % options.max_memory) else: extra_args.append('-Xmx8G') if options.golem: golem.link_third_party() options.out = os.getcwd() if not options.ignore_java_version: utils.check_java_version() if options.print_times: extra_args.append('-Dcom.android.tools.r8.printtimes=1') outdir = options.out (version_id, data) = get_version_and_data(options) if options.compiler not in COMPILERS: raise Exception("You need to specify '--compiler={}'".format( '|'.join(COMPILERS))) if options.compiler_build not in COMPILER_BUILDS: raise Exception("You need to specify '--compiler-build={}'".format( '|'.join(COMPILER_BUILDS))) if not version_id in data.VERSIONS.keys(): print('No version {} for application {}'.format( version_id, options.app)) print('Valid versions are {}'.format(data.VERSIONS.keys())) return 1 version = data.VERSIONS[version_id] type = get_type(options) if type not in version: print('No type {} for version {}'.format(type, version)) print('Valid types are {}'.format(version.keys())) return 1 values = version[type] inputs = [] # For R8 'deploy' the JAR is located using the Proguard configuration # -injars option. For chrome and nest we don't have the injars in the # proguard files. if 'inputs' in values and (options.compiler != 'r8' or type != 'deploy' or options.app == 'chrome' or options.app == 'nest' or options.app == 'r8' or options.app == 'iosched' or options.app == 'tachiyomi'): inputs = values['inputs'] args.extend(['--output', outdir]) if 'min-api' in values: args.extend(['--min-api', values['min-api']]) if 'main-dex-list' in values: args.extend(['--main-dex-list', values['main-dex-list']]) libraries = values['libraries'] if 'libraries' in values else [] if options.compiler == 'r8': if 'pgconf' in values and not options.k: if has_injars_and_libraryjars(values['pgconf']): sanitized_lib_path = os.path.join(os.path.abspath(outdir), 'sanitized_lib.jar') sanitized_pgconf_path = os.path.join(os.path.abspath(outdir), 'sanitized.config') SanitizeLibrariesInPgconf(sanitized_lib_path, sanitized_pgconf_path, values['pgconf']) libraries = [sanitized_lib_path] args.extend(['--pg-conf', sanitized_pgconf_path]) else: # -injars without -libraryjars or vice versa is not supported. check_no_injars_and_no_libraryjars(values['pgconf']) for pgconf in values['pgconf']: args.extend(['--pg-conf', pgconf]) if 'sanitize_libraries' in values and values[ 'sanitize_libraries']: sanitized_lib_path = os.path.join(os.path.abspath(outdir), 'sanitized_lib.jar') SanitizeLibraries(sanitized_lib_path, values['libraries'], values['inputs']) libraries = [sanitized_lib_path] inputs = values['inputs'] app_provided_pg_conf = True if options.k: args.extend(['--pg-conf', options.k]) if 'maindexrules' in values: for rules in values['maindexrules']: args.extend(['--main-dex-rules', rules]) if 'allow-type-errors' in values: extra_args.append('-Dcom.android.tools.r8.allowTypeErrors=1') extra_args.append( '-Dcom.android.tools.r8.disallowClassInlinerGracefulExit=1') if options.debug_agent: if not options.compiler_build == 'full': print( 'WARNING: Running debugging agent on r8lib is questionable...') extra_args.append( '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' ) if not options.no_libraries: for lib in libraries: args.extend(['--lib', lib]) if not outdir.endswith('.zip') and not outdir.endswith('.jar') \ and not os.path.exists(outdir): os.makedirs(outdir) if options.hash: # Download r8-<hash>.jar from # https://storage.googleapis.com/r8-releases/raw/<hash>/. download_path = archive.GetUploadDestination(options.hash, 'r8.jar', True) assert utils.file_exists_on_cloud_storage(download_path), ( 'Could not find r8.jar file from provided hash: %s' % options.hash) destination = os.path.join(utils.LIBS, 'r8-' + options.hash + '.jar') utils.download_file_from_cloud_storage(download_path, destination, quiet=quiet) # Additional flags for the compiler from the configuration file. if 'flags' in values: args.extend(values['flags'].split(' ')) if options.compiler == 'r8': if 'r8-flags' in values: args.extend(values['r8-flags'].split(' ')) # Additional flags for the compiler from the command line. if options.compiler_flags: args.extend(options.compiler_flags.split(' ')) if options.r8_flags: args.extend(options.r8_flags.split(' ')) # Feature jars. features = values['features'] if 'features' in values else [] for i, feature in enumerate(features, start=1): feature_out = os.path.join(outdir, 'feature-%d.zip' % i) for feature_jar in feature['inputs']: args.extend(['--feature', feature_jar, feature_out]) args.extend(inputs) t0 = time.time() if options.dump_args_file: with open(options.dump_args_file, 'w') as args_file: args_file.writelines([arg + os.linesep for arg in args]) else: with utils.TempDir() as temp: if options.print_memoryuse and not options.track_memory_to_file: options.track_memory_to_file = os.path.join( temp, utils.MEMORY_USE_TMP_FILE) if options.compiler == 'r8' and app_provided_pg_conf: # Ensure that output of -printmapping and -printseeds go to the output # location and not where the app Proguard configuration places them. if outdir.endswith('.zip') or outdir.endswith('.jar'): pg_outdir = os.path.dirname(outdir) else: pg_outdir = outdir if not options.no_extra_pgconf: additional_pg_conf = GenerateAdditionalProguardConfiguration( temp, os.path.abspath(pg_outdir)) args.extend(['--pg-conf', additional_pg_conf]) build = not options.no_build and not options.golem stderr_path = os.path.join(temp, 'stderr') with open(stderr_path, 'w') as stderr: jar = None main = None if options.compiler_build == 'full': tool = options.compiler else: assert (options.compiler_build == 'lib') tool = 'r8lib-' + options.compiler if options.hash: jar = os.path.join(utils.LIBS, 'r8-' + options.hash + '.jar') main = 'com.android.tools.r8.' + options.compiler.upper() exit_code = toolhelper.run( tool, args, build=build, debug=not options.no_debug, profile=options.profile, track_memory_file=options.track_memory_to_file, extra_args=extra_args, stdout=stdout, stderr=stderr, timeout=options.timeout, quiet=quiet, cmd_prefix=['taskset', '-c', options.cpu_list] if options.cpu_list else [], jar=jar, main=main) if exit_code != 0: with open(stderr_path) as stderr: stderr_text = stderr.read() if not quiet: print(stderr_text) if 'java.lang.OutOfMemoryError' in stderr_text: if not quiet: print('Failure was OOM') return OOM_EXIT_CODE return exit_code if options.print_memoryuse: print('{}(MemoryUse): {}'.format( options.print_memoryuse, utils.grep_memoryuse(options.track_memory_to_file))) if options.print_runtimeraw: print('{}(RunTimeRaw): {} ms'.format(options.print_runtimeraw, 1000.0 * (time.time() - t0))) if options.print_dexsegments: dex_files = glob(os.path.join(outdir, '*.dex')) utils.print_dexsegments(options.print_dexsegments, dex_files) return 0
def main(argv): utils.check_java_version() app_provided_pg_conf = False (options, args) = ParseOptions(argv) outdir = options.out data = None if options.app == 'gmscore': options.version = options.version or 'v9' data = gmscore_data elif options.app == 'youtube': options.version = options.version or '12.22' data = youtube_data elif options.app == 'gmail': options.version = options.version or '170604.16' data = gmail_data else: raise Exception("You need to specify '--app={}'".format( '|'.join(APPS))) if options.compiler not in COMPILERS: raise Exception("You need to specify '--compiler={}'".format( '|'.join(COMPILERS))) if not options.version in data.VERSIONS.keys(): print('No version {} for application {}'.format( options.version, options.app)) print('Valid versions are {}'.format(data.VERSIONS.keys())) return 1 version = data.VERSIONS[options.version] if not options.type: options.type = 'deploy' if options.compiler == 'r8' \ else 'proguarded' if options.type not in version: print('No type {} for version {}'.format(options.type, options.version)) print('Valid types are {}'.format(version.keys())) return 1 values = version[options.type] inputs = None # For R8 'deploy' the JAR is located using the Proguard configuration # -injars option. if 'inputs' in values and (options.compiler != 'r8' or options.type != 'deploy'): inputs = values['inputs'] args.extend(['--output', outdir]) if 'min-api' in values: args.extend(['--min-api', values['min-api']]) if options.compiler == 'r8': if 'pgmap' in values: args.extend(['--pg-map', values['pgmap']]) if 'pgconf' in values and not options.k: for pgconf in values['pgconf']: args.extend(['--pg-conf', pgconf]) app_provided_pg_conf = True if options.k: args.extend(['--pg-conf', options.k]) if 'maindexrules' in values: for rules in values['maindexrules']: args.extend(['--main-dex-rules', rules]) if not options.no_libraries and 'libraries' in values: for lib in values['libraries']: args.extend(['--lib', lib]) if not outdir.endswith('.zip') and not outdir.endswith('.jar') \ and not os.path.exists(outdir): os.makedirs(outdir) if options.compiler == 'r8': if 'r8-flags' in values: args.extend(values['r8-flags'].split(' ')) if options.compiler_flags: args.extend(options.compiler_flags.split(' ')) if options.r8_flags: args.extend(options.r8_flags.split(' ')) if inputs: args.extend(inputs) t0 = time.time() if options.dump_args_file: with open(options.dump_args_file, 'w') as args_file: args_file.writelines([arg + os.linesep for arg in args]) else: with utils.TempDir() as temp: if options.print_memoryuse and not options.track_memory_to_file: options.track_memory_to_file = os.path.join( temp, utils.MEMORY_USE_TMP_FILE) if options.compiler == 'd8': d8.run(args, not options.no_build, not options.no_debug, options.profile, options.track_memory_to_file) else: if app_provided_pg_conf: # Ensure that output of -printmapping and -printseeds go to the output # location and not where the app Proguard configuration places them. if outdir.endswith('.zip') or outdir.endswith('.jar'): pg_outdir = os.path.dirname(outdir) else: pg_outdir = outdir additional_pg_conf = GenerateAdditionalProguardConfiguration( temp, os.path.abspath(pg_outdir)) args.extend(['--pg-conf', additional_pg_conf]) r8.run(args, not options.no_build, not options.no_debug, options.profile, options.track_memory_to_file) if options.print_memoryuse: print('{}(MemoryUse): {}'.format( options.print_memoryuse, utils.grep_memoryuse(options.track_memory_to_file))) if options.print_runtimeraw: print('{}(RunTimeRaw): {} ms'.format(options.print_runtimeraw, 1000.0 * (time.time() - t0))) if options.print_dexsegments: dex_files = glob(os.path.join(outdir, '*.dex')) utils.print_dexsegments(options.print_dexsegments, dex_files)
elif options.retracer == 'proguard': retracer_args = ['-jar', proguard.getRetraceJar()] elif options.retracer == 'remapper': retracer_args = [ '-jar', os.path.join(utils.THIRD_PARTY, 'remapper', 'remapper_deploy.jar') ] else: assert False, "Unexpected retracer " + options.retracer retrace_args = [jdk.GetJavaExecutable()] + retracer_args + [ os.path.join(utils.THIRD_PARTY, 'retrace_benchmark', 'r8lib.jar.map'), os.path.join(utils.THIRD_PARTY, 'retrace_benchmark', 'stacktrace.txt') ] utils.PrintCmd(retrace_args) t0 = time.time() subprocess.check_call(retrace_args) t1 = time.time() if options.print_runtimeraw: print('{}(RunTimeRaw): {} ms'.format(options.print_runtimeraw, 1000.0 * (t1 - t0))) if __name__ == '__main__': options = parse_arguments(sys.argv[1:]) if options.golem: golem.link_third_party() if not options.ignore_java_version: utils.check_java_version() with utils.TempDir() as temp: run_retrace(options, temp)