def Main():
utils.check_java_version()
args = parse_arguments()
output_dir = args.output
if args.golem:
golem.link_third_party()
with utils.TempDir() as temp_dir:
if not output_dir:
output_dir = temp_dir
xmx = None
if args.tool == 'dx':
tool_file = DX_JAR
tool_args = [
'--dex', '--output=' + output_dir, '--multi-dex',
'--min-sdk-version=' + MIN_SDK_VERSION
]
xmx = '-Xmx1600m'
else:
tool_file = utils.D8_JAR
tool_args = ['--output', output_dir, '--min-api', MIN_SDK_VERSION]
if args.tool == 'd8-release':
tool_args.append('--release')
xmx = '-Xmx600m'
cmd = []
track_memory_file = None
if args.print_memoryuse:
track_memory_file = os.path.join(output_dir,
utils.MEMORY_USE_TMP_FILE)
cmd.extend(['tools/track_memory.sh', track_memory_file])
if tool_file.endswith('.jar'):
assert xmx is not None
cmd.extend(['java', xmx, '-jar'])
cmd.extend([tool_file] + tool_args + [FRAMEWORK_JAR])
utils.PrintCmd(cmd)
t0 = time.time()
subprocess.check_call(cmd)
dt = time.time() - t0
if args.print_memoryuse:
print('{}-Total(MemoryUse): {}'.format(
args.name, utils.grep_memoryuse(track_memory_file)))
dex_files = [f for f in glob(os.path.join(output_dir, '*.dex'))]
code_size = 0
for dex_file in dex_files:
code_size += os.path.getsize(dex_file)
print('{}-Total(RunTimeRaw): {} ms'.format(args.name, 1000.0 * dt))
print('{}-Total(CodeSize): {}'.format(args.name, code_size))
utils.print_dexsegments(args.name, dex_files)
def Main():
(options, args) = parse_options()
if options.golem:
golem.link_third_party()
with utils.TempDir() as temp:
dex_path = os.path.join(temp, "classes.jar")
proguard_conf = os.path.join(temp, 'proguard.conf')
with open(proguard_conf, 'w') as f:
f.write(PROGUARD_CONF)
benchmark_jar = get_jar_for_benchmark(options.benchmark)
r8_args = [
'--lib',
utils.get_android_jar(26), # Only works with api 26
'--output',
dex_path,
'--pg-conf',
proguard_conf,
'--min-api',
str(options.api),
benchmark_jar
]
toolhelper.run('r8', r8_args, build=not options.golem)
if options.use_device:
result = run_art_device(dex_path)
else:
result = run_art(dex_path)
print('Kotlin_{}(RunTimeRaw): {} ms'.format(options.benchmark, result))
def Main():
args = parse_arguments()
if args.golem:
golem.link_third_party()
utils.check_java_version()
with utils.TempDir() as temp_dir:
cmd_prefix = []
output_dir = args.output if args.output else temp_dir
temp_dir = os.path.join(args.output,
'tmp') if args.output else temp_dir
track_memory_file = None
if args.print_memoryuse:
track_memory_file = os.path.join(output_dir,
utils.MEMORY_USE_TMP_FILE)
cmd_prefix.extend(['tools/track_memory.sh', track_memory_file])
name = 'CompileHelloExample'
tool = args.tool
output_mode = args.output_mode
lib = None
if output_mode == 'dex':
name += 'Dex'
lib = utils.get_android_jar(28)
else:
lib = utils.RT_JAR
extra = []
if args.large:
name += 'Large'
extra = EXTRA_INPUTS
if args.noopt:
name += 'NoOpt'
cmds = Compile(
tool,
output_mode,
lib,
extra,
output_dir,
args.noopt,
temp_dir,
)
t0 = time.time()
for cmd in cmds:
fullcmd = cmd_prefix + cmd
utils.PrintCmd(fullcmd)
subprocess.check_output(fullcmd)
dt = time.time() - t0
if args.print_memoryuse:
print('{}(MemoryUse): {}'.format(
name, utils.grep_memoryuse(track_memory_file)))
print('{}(RunTimeRaw): {} ms'.format(name, 1000.0 * dt))
def Main():
args = parse_arguments()
if args.golem:
# Ensure that we don't have a running daemon
exitcode = subprocess.call(['pkill', 'java'])
assert exitcode == 0 or exitcode == 1
golem.link_third_party()
if args.tool == 'd8':
tool = Benchmark.Tools.D8
desugarMode = Benchmark.DesugarMode.D8_DESUGARING
else:
tool = Benchmark.Tools.DX
desugarMode = Benchmark.DesugarMode.DESUGAR_TOOL
buildTimeBenchmarks = [
Benchmark('AntennaPod', os.path.join('antenna-pod', 'AntennaPod'),
':app', [':app:assembleDebug'], ['clean']),
Benchmark(
'Maps', 'gradle-java-1.6', ':maps',
[':maps:assembleDebug', '--settings-file', 'settings.gradle.maps'],
['clean']),
Benchmark('Music2', 'gradle-java-1.6', ':music2Old', [
':music2Old:assembleDebug', '--settings-file',
'settings.gradle.music2Old'
], ['clean']),
Benchmark('Velvet', 'gradle-java-1.6', ':velvet', [
':velvet:assembleDebug', '--settings-file',
'settings.gradle.velvet'
], ['clean']),
Benchmark('SantaTracker', 'santa-tracker', ':santa-tracker',
[':santa-tracker:assembleDebug'], ['clean']),
# disabled for now, apparently because of b/74227571
# Benchmark('Tachiyomi',
# 'tachiyomi',
# ':app',
# ['assembleStandardDebug'],
# ['clean']),
Benchmark('WordPress', 'wordpress', ':WordPress',
['assembleVanillaDebug'], ['clean']),
]
if not args.skip_download and not args.golem:
EnsurePresence(
os.path.join('third_party', 'benchmarks', 'android-sdk'),
'android SDK')
EnsurePresence(os.path.join('third_party', 'gradle-plugin'),
'Android Gradle plugin')
toRun = buildTimeBenchmarks
if args.benchmark:
toRun = [b for b in toRun if b.displayName == args.benchmark]
if len(toRun) != 1:
raise AssertionError("Unknown benchmark: " + args.benchmark)
for benchmark in toRun:
if not args.skip_download:
benchmark.EnsurePresence()
benchmark.Clean()
stdOut = benchmark.Build(tool, desugarMode)
PrintBuildTimeForGolem(benchmark, stdOut)
'--lib',
utils.RT_JAR,
'--min-api',
'10000',
'--no-desugaring',
],
debug=False,
build=False)
if return_code != 0:
sys.exit(return_code)
if __name__ == '__main__':
options = parse_arguments(sys.argv[1:])
if options.golem:
golem.link_third_party()
with utils.TempDir() as temp:
memory_file = os.path.join(temp, 'memory.dump')
r8_output = os.path.join(temp, 'r8.zip')
d8_r8_output = os.path.join(temp, 'd8r8.zip')
d8_pg_output = os.path.join(temp, 'd8pg.zip')
return_code = minify_tool.minify_tool(input_jar=PINNED_R8_JAR,
output_jar=r8_output,
debug=False,
build=False,
track_memory_file=memory_file,
benchmark_name="BootstrapR8")
if return_code != 0:
sys.exit(return_code)
def main(argv):
(options, args) = ParseOptions(argv)
if options.bot:
utils.DownloadFromGoogleCloudStorage(utils.OPENSOURCE_APPS_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.ANDROID_SDK + '.tar.gz.sha1',
bucket='r8-deps-internal',
auth=True)
if os.path.exists(WORKING_DIR):
shutil.rmtree(WORKING_DIR)
shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR)
os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join(
utils.ANDROID_SDK)
os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3'
# TODO(b/141081520): Set to True once fixed.
options.no_logging = False
# TODO(b/141081520): Remove logging filter once fixed.
options.app_logging_filter = ['sqldelight']
options.shrinker = ['r8', 'r8-full']
print(options.shrinker)
if options.golem:
golem.link_third_party()
if os.path.exists(WORKING_DIR):
shutil.rmtree(WORKING_DIR)
shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR)
os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join(
utils.ANDROID_SDK)
os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3'
options.disable_assertions = True
options.ignore_versions = True
options.no_build = True
options.r8_compilation_steps = 1
options.quiet = True
options.gradle_pre_runs = 2
options.use_daemon = True
options.no_logging = True
if not os.path.exists(WORKING_DIR):
os.makedirs(WORKING_DIR)
if options.download_only:
clone_repositories(options.quiet)
return
with utils.TempDir() as temp_dir:
if not (options.no_build or options.golem):
gradle.RunGradle(['r8', '-Pno_internal'])
build_r8lib = False
for shrinker in options.shrinker:
if IsMinifiedR8(shrinker):
build_r8lib = True
if build_r8lib:
gradle.RunGradle(['r8lib', '-Pno_internal'])
if options.hash:
# Download r8-<hash>.jar from
# https://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.hash)
update_prebuilds_in_android.download_hash(
temp_dir, 'com/android/tools/r8/' + options.hash, target)
as_utils.MoveFile(
os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
elif options.version:
# Download r8-<version>.jar from
# https://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.version)
update_prebuilds_in_android.download_version(
temp_dir, 'com/android/tools/r8/' + options.version, target)
as_utils.MoveFile(
os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
else:
# Make a copy of r8.jar and r8lib.jar such that they stay the same for
# the entire execution of this script.
if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker:
assert os.path.isfile(utils.R8_JAR), 'Cannot build without r8.jar'
shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
if 'r8' in options.shrinker or 'r8-full' in options.shrinker:
assert os.path.isfile(utils.R8LIB_JAR), 'Cannot build without r8lib.jar'
shutil.copyfile(utils.R8LIB_JAR, os.path.join(temp_dir, 'r8lib.jar'))
result_per_shrinker_per_app = []
# If we are running on golem we kill all java processes after the run
# to ensure no hanging gradle daemons.
with EnsureNoGradleAlive(options.golem):
for (app, repo) in options.apps:
if app.skip:
continue
result_per_shrinker_per_app.append(
(app, GetResultsForApp(app, repo, options, temp_dir)))
return LogResultsForApps(result_per_shrinker_per_app, options)
def run_with_options(options, args, extra_args=None):
if extra_args is None:
extra_args = []
app_provided_pg_conf = False
# todo(121018500): remove when memory is under control
if not any('-Xmx' in arg for arg in extra_args):
extra_args.append('-Xmx8G')
if options.golem:
golem.link_third_party()
options.out = os.getcwd()
if not options.ignore_java_version:
utils.check_java_version()
outdir = options.out
data = None
if options.app == 'gmscore':
options.version = options.version or 'v9'
data = gmscore_data
elif options.app == 'nest':
options.version = options.version or '20180926'
data = nest_data
elif options.app == 'youtube':
options.version = options.version or '12.22'
data = youtube_data
elif options.app == 'chrome':
options.version = options.version or '180917'
data = chrome_data
elif options.app == 'gmail':
options.version = options.version or '170604.16'
data = gmail_data
else:
raise Exception("You need to specify '--app={}'".format(
'|'.join(APPS)))
if options.compiler not in COMPILERS:
raise Exception("You need to specify '--compiler={}'".format(
'|'.join(COMPILERS)))
if options.compiler_build not in COMPILER_BUILDS:
raise Exception("You need to specify '--compiler-build={}'".format(
'|'.join(COMPILER_BUILDS)))
if not options.version in data.VERSIONS.keys():
print('No version {} for application {}'.format(
options.version, options.app))
print('Valid versions are {}'.format(data.VERSIONS.keys()))
return 1
version = data.VERSIONS[options.version]
if not options.type:
options.type = 'deploy' if options.compiler == 'r8' \
else 'proguarded'
if options.type not in version:
print('No type {} for version {}'.format(options.type,
options.version))
print('Valid types are {}'.format(version.keys()))
return 1
values = version[options.type]
inputs = None
# For R8 'deploy' the JAR is located using the Proguard configuration
# -injars option. For chrome and nest we don't have the injars in the
# proguard files.
if 'inputs' in values and (options.compiler != 'r8'
or options.type != 'deploy' or options.app
== 'chrome' or options.app == 'nest'):
inputs = values['inputs']
args.extend(['--output', outdir])
if 'min-api' in values:
args.extend(['--min-api', values['min-api']])
if 'main-dex-list' in values:
args.extend(['--main-dex-list', values['main-dex-list']])
if options.compiler == 'r8':
if 'pgconf' in values and not options.k:
sanitized_lib_path = os.path.join(os.path.abspath(outdir),
'sanitized_lib.jar')
sanitized_pgconf_path = os.path.join(os.path.abspath(outdir),
'sanitized.config')
SanitizeLibraries(sanitized_lib_path, sanitized_pgconf_path,
values['pgconf'])
args.extend(['--pg-conf', sanitized_pgconf_path])
app_provided_pg_conf = True
if options.k:
args.extend(['--pg-conf', options.k])
if 'maindexrules' in values:
for rules in values['maindexrules']:
args.extend(['--main-dex-rules', rules])
if 'allow-type-errors' in values:
extra_args.append('-Dcom.android.tools.r8.allowTypeErrors=1')
if 'proto-shrinking' in values:
extra_args.append(
'-Dcom.android.tools.r8.generatedExtensionRegistryShrinking=1')
if not options.no_libraries and 'libraries' in values:
for lib in values['libraries']:
args.extend(['--lib', lib])
if not outdir.endswith('.zip') and not outdir.endswith('.jar') \
and not os.path.exists(outdir):
os.makedirs(outdir)
# Additional flags for the compiler from the configuration file.
if 'flags' in values:
args.extend(values['flags'].split(' '))
if options.compiler == 'r8':
if 'r8-flags' in values:
args.extend(values['r8-flags'].split(' '))
# Additional flags for the compiler from the command line.
if options.compiler_flags:
args.extend(options.compiler_flags.split(' '))
if options.r8_flags:
args.extend(options.r8_flags.split(' '))
if inputs:
args.extend(inputs)
t0 = time.time()
if options.dump_args_file:
with open(options.dump_args_file, 'w') as args_file:
args_file.writelines([arg + os.linesep for arg in args])
else:
with utils.TempDir() as temp:
if options.print_memoryuse and not options.track_memory_to_file:
options.track_memory_to_file = os.path.join(
temp, utils.MEMORY_USE_TMP_FILE)
if options.compiler == 'r8' and app_provided_pg_conf:
# Ensure that output of -printmapping and -printseeds go to the output
# location and not where the app Proguard configuration places them.
if outdir.endswith('.zip') or outdir.endswith('.jar'):
pg_outdir = os.path.dirname(outdir)
else:
pg_outdir = outdir
additional_pg_conf = GenerateAdditionalProguardConfiguration(
temp, os.path.abspath(pg_outdir))
args.extend(['--pg-conf', additional_pg_conf])
build = not options.no_build and not options.golem
stderr_path = os.path.join(temp, 'stderr')
with open(stderr_path, 'w') as stderr:
if options.compiler_build == 'full':
tool = options.compiler
else:
assert (options.compiler_build == 'lib')
tool = 'r8lib-' + options.compiler
exit_code = toolhelper.run(
tool,
args,
build=build,
debug=not options.no_debug,
profile=options.profile,
track_memory_file=options.track_memory_to_file,
extra_args=extra_args,
stderr=stderr,
timeout=options.timeout)
if exit_code != 0:
with open(stderr_path) as stderr:
stderr_text = stderr.read()
print(stderr_text)
if 'java.lang.OutOfMemoryError' in stderr_text:
print('Failure was OOM')
return OOM_EXIT_CODE
return exit_code
if options.print_memoryuse:
print('{}(MemoryUse): {}'.format(
options.print_memoryuse,
utils.grep_memoryuse(options.track_memory_to_file)))
if options.print_runtimeraw:
print('{}(RunTimeRaw): {} ms'.format(options.print_runtimeraw,
1000.0 * (time.time() - t0)))
if options.print_dexsegments:
dex_files = glob(os.path.join(outdir, '*.dex'))
utils.print_dexsegments(options.print_dexsegments, dex_files)
return 0
def Main(argv):
utils.check_java_version()
options = parse_arguments(argv)
if options.golem:
golem.link_third_party()
outdir = options.out
if options.app == 'gmscore':
version = 'v10'
data = gmscore_data
base = data.V10_BASE
elif options.app == 'youtube':
version = '12.22'
data = youtube_data
base = data.V12_22_BASE
else:
raise Exception('Unexpected')
args = ['-forceprocessing']
if not outdir.endswith('.zip') and not outdir.endswith('.jar') \
and not exists(outdir):
makedirs(outdir)
values_deploy = data.VERSIONS[version]['deploy']
values_proguarded = data.VERSIONS[version]['proguarded']
assert 'pgconf' in values_deploy
for pgconf in values_deploy['pgconf']:
args.extend(['@' + pgconf])
# find seeds file
inputs = data.VERSIONS[version]['proguarded']['inputs']
assert len(inputs) == 1
basename_wo_ext = splitext(os.path.basename(inputs[0]))[0]
seeds_filename = basename_wo_ext + '.seeds'
seeds_files = []
for root, dirnames, filenames in os.walk(join(base, 'blaze-out')):
for filename in fnmatch.filter(filenames, seeds_filename):
seeds_files.append(os.path.join(root, filename))
assert len(seeds_files) == 1
seeds_path = seeds_files[0]
proguarded_jar_path = splitext(seeds_path)[0] + '.jar'
# Remove write-protection from seeds file. The seeds file is an output of
# ProGuard so it aborts if this is not writeable.
st = os.stat(seeds_path)
os.chmod(seeds_path,
st.st_mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
t0 = time.time()
proguard_memoryuse = None
with utils.TempDir() as temp:
track_memory_file = None
if options.print_memoryuse:
track_memory_file = join(temp, utils.MEMORY_USE_TMP_FILE)
proguard.run(
args,
track_memory_file = track_memory_file,
stdout=open(os.devnull, 'w'))
if options.print_memoryuse:
proguard_memoryuse = utils.grep_memoryuse(track_memory_file)
# run dex on the result
if options.compatdx:
jar = utils.COMPATDX_JAR
else:
jar = DX_JAR
with utils.TempDir() as temp:
track_memory_file = None
cmd = []
if options.print_memoryuse:
track_memory_file = join(temp, utils.MEMORY_USE_TMP_FILE)
cmd.extend(['tools/track_memory.sh', track_memory_file])
cmd.extend(['java', '-jar', jar, '--multi-dex',
'--output=' + outdir])
if 'min-api' in values_proguarded:
cmd.append('--min-sdk-version=' + values_proguarded['min-api'])
cmd.extend(['--dex', proguarded_jar_path])
utils.PrintCmd(cmd);
check_call(cmd)
if options.print_memoryuse:
dx_memoryuse = utils.grep_memoryuse(track_memory_file)
print('{}(MemoryUse): {}'
.format(options.print_memoryuse,
max(proguard_memoryuse, dx_memoryuse)))
if options.print_runtimeraw:
print('{}(RunTimeRaw): {} ms'
.format(options.print_runtimeraw, 1000.0 * (time.time() - t0)))
if options.print_dexsegments:
dex_files = glob(os.path.join(outdir, '*.dex'))
utils.print_dexsegments(options.print_dexsegments, dex_files)
def main(argv):
(options, args) = ParseOptions(argv)
if options.golem:
golem.link_third_party()
if os.path.exists(WORKING_DIR):
shutil.rmtree(WORKING_DIR)
shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR)
os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join(
utils.ANDROID_SDK)
os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3'
options.disable_assertions = True
options.ignore_versions = True
options.no_build = True
options.r8_compilation_steps = 1
options.quiet = True
options.gradle_pre_runs = 2
options.use_daemon = True
if not os.path.exists(WORKING_DIR):
os.makedirs(WORKING_DIR)
if options.download_only:
clone_repositories(options.quiet)
return
with utils.TempDir() as temp_dir:
if not (options.no_build or options.golem):
gradle.RunGradle(['r8', 'r8lib'])
if options.version:
# Download r8-<version>.jar from
# http://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.version)
update_prebuilds_in_android.download_version(
temp_dir, 'com/android/tools/r8/' + options.version, target)
as_utils.MoveFile(os.path.join(temp_dir, target),
os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
else:
# Make a copy of r8.jar and r8lib.jar such that they stay the same for
# the entire execution of this script.
if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker:
assert os.path.isfile(
utils.R8_JAR), 'Cannot build without r8.jar'
shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
if 'r8' in options.shrinker or 'r8-full' in options.shrinker:
assert os.path.isfile(
utils.R8LIB_JAR), 'Cannot build without r8lib.jar'
shutil.copyfile(utils.R8LIB_JAR,
os.path.join(temp_dir, 'r8lib.jar'))
result_per_shrinker_per_app = []
for (app, repo) in options.apps:
if app.skip:
continue
result_per_shrinker_per_app.append(
(app, GetResultsForApp(app, repo, options, temp_dir)))
LogResultsForApps(result_per_shrinker_per_app, options)
def run_with_options(options, args, extra_args=None, stdout=None, quiet=False):
if extra_args is None:
extra_args = []
app_provided_pg_conf = False
# todo(121018500): remove when memory is under control
if not any('-Xmx' in arg for arg in extra_args):
if options.max_memory:
extra_args.append('-Xmx%sM' % options.max_memory)
else:
extra_args.append('-Xmx8G')
if options.golem:
golem.link_third_party()
options.out = os.getcwd()
if not options.ignore_java_version:
utils.check_java_version()
if options.print_times:
extra_args.append('-Dcom.android.tools.r8.printtimes=1')
outdir = options.out
(version_id, data) = get_version_and_data(options)
if options.compiler not in COMPILERS:
raise Exception("You need to specify '--compiler={}'".format(
'|'.join(COMPILERS)))
if options.compiler_build not in COMPILER_BUILDS:
raise Exception("You need to specify '--compiler-build={}'".format(
'|'.join(COMPILER_BUILDS)))
if not version_id in data.VERSIONS.keys():
print('No version {} for application {}'.format(
version_id, options.app))
print('Valid versions are {}'.format(data.VERSIONS.keys()))
return 1
version = data.VERSIONS[version_id]
type = get_type(options)
if type not in version:
print('No type {} for version {}'.format(type, version))
print('Valid types are {}'.format(version.keys()))
return 1
values = version[type]
inputs = []
# For R8 'deploy' the JAR is located using the Proguard configuration
# -injars option. For chrome and nest we don't have the injars in the
# proguard files.
if 'inputs' in values and (options.compiler != 'r8' or type != 'deploy'
or options.app == 'chrome'
or options.app == 'nest' or options.app == 'r8'
or options.app == 'iosched'
or options.app == 'tachiyomi'):
inputs = values['inputs']
args.extend(['--output', outdir])
if 'min-api' in values:
args.extend(['--min-api', values['min-api']])
if 'main-dex-list' in values:
args.extend(['--main-dex-list', values['main-dex-list']])
libraries = values['libraries'] if 'libraries' in values else []
if options.compiler == 'r8':
if 'pgconf' in values and not options.k:
if has_injars_and_libraryjars(values['pgconf']):
sanitized_lib_path = os.path.join(os.path.abspath(outdir),
'sanitized_lib.jar')
sanitized_pgconf_path = os.path.join(os.path.abspath(outdir),
'sanitized.config')
SanitizeLibrariesInPgconf(sanitized_lib_path,
sanitized_pgconf_path,
values['pgconf'])
libraries = [sanitized_lib_path]
args.extend(['--pg-conf', sanitized_pgconf_path])
else:
# -injars without -libraryjars or vice versa is not supported.
check_no_injars_and_no_libraryjars(values['pgconf'])
for pgconf in values['pgconf']:
args.extend(['--pg-conf', pgconf])
if 'sanitize_libraries' in values and values[
'sanitize_libraries']:
sanitized_lib_path = os.path.join(os.path.abspath(outdir),
'sanitized_lib.jar')
SanitizeLibraries(sanitized_lib_path, values['libraries'],
values['inputs'])
libraries = [sanitized_lib_path]
inputs = values['inputs']
app_provided_pg_conf = True
if options.k:
args.extend(['--pg-conf', options.k])
if 'maindexrules' in values:
for rules in values['maindexrules']:
args.extend(['--main-dex-rules', rules])
if 'allow-type-errors' in values:
extra_args.append('-Dcom.android.tools.r8.allowTypeErrors=1')
extra_args.append(
'-Dcom.android.tools.r8.disallowClassInlinerGracefulExit=1')
if options.debug_agent:
if not options.compiler_build == 'full':
print(
'WARNING: Running debugging agent on r8lib is questionable...')
extra_args.append(
'-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
)
if not options.no_libraries:
for lib in libraries:
args.extend(['--lib', lib])
if not outdir.endswith('.zip') and not outdir.endswith('.jar') \
and not os.path.exists(outdir):
os.makedirs(outdir)
if options.hash:
# Download r8-<hash>.jar from
# https://storage.googleapis.com/r8-releases/raw/<hash>/.
download_path = archive.GetUploadDestination(options.hash, 'r8.jar',
True)
assert utils.file_exists_on_cloud_storage(download_path), (
'Could not find r8.jar file from provided hash: %s' % options.hash)
destination = os.path.join(utils.LIBS, 'r8-' + options.hash + '.jar')
utils.download_file_from_cloud_storage(download_path,
destination,
quiet=quiet)
# Additional flags for the compiler from the configuration file.
if 'flags' in values:
args.extend(values['flags'].split(' '))
if options.compiler == 'r8':
if 'r8-flags' in values:
args.extend(values['r8-flags'].split(' '))
# Additional flags for the compiler from the command line.
if options.compiler_flags:
args.extend(options.compiler_flags.split(' '))
if options.r8_flags:
args.extend(options.r8_flags.split(' '))
# Feature jars.
features = values['features'] if 'features' in values else []
for i, feature in enumerate(features, start=1):
feature_out = os.path.join(outdir, 'feature-%d.zip' % i)
for feature_jar in feature['inputs']:
args.extend(['--feature', feature_jar, feature_out])
args.extend(inputs)
t0 = time.time()
if options.dump_args_file:
with open(options.dump_args_file, 'w') as args_file:
args_file.writelines([arg + os.linesep for arg in args])
else:
with utils.TempDir() as temp:
if options.print_memoryuse and not options.track_memory_to_file:
options.track_memory_to_file = os.path.join(
temp, utils.MEMORY_USE_TMP_FILE)
if options.compiler == 'r8' and app_provided_pg_conf:
# Ensure that output of -printmapping and -printseeds go to the output
# location and not where the app Proguard configuration places them.
if outdir.endswith('.zip') or outdir.endswith('.jar'):
pg_outdir = os.path.dirname(outdir)
else:
pg_outdir = outdir
if not options.no_extra_pgconf:
additional_pg_conf = GenerateAdditionalProguardConfiguration(
temp, os.path.abspath(pg_outdir))
args.extend(['--pg-conf', additional_pg_conf])
build = not options.no_build and not options.golem
stderr_path = os.path.join(temp, 'stderr')
with open(stderr_path, 'w') as stderr:
jar = None
main = None
if options.compiler_build == 'full':
tool = options.compiler
else:
assert (options.compiler_build == 'lib')
tool = 'r8lib-' + options.compiler
if options.hash:
jar = os.path.join(utils.LIBS,
'r8-' + options.hash + '.jar')
main = 'com.android.tools.r8.' + options.compiler.upper()
exit_code = toolhelper.run(
tool,
args,
build=build,
debug=not options.no_debug,
profile=options.profile,
track_memory_file=options.track_memory_to_file,
extra_args=extra_args,
stdout=stdout,
stderr=stderr,
timeout=options.timeout,
quiet=quiet,
cmd_prefix=['taskset', '-c', options.cpu_list]
if options.cpu_list else [],
jar=jar,
main=main)
if exit_code != 0:
with open(stderr_path) as stderr:
stderr_text = stderr.read()
if not quiet:
print(stderr_text)
if 'java.lang.OutOfMemoryError' in stderr_text:
if not quiet:
print('Failure was OOM')
return OOM_EXIT_CODE
return exit_code
if options.print_memoryuse:
print('{}(MemoryUse): {}'.format(
options.print_memoryuse,
utils.grep_memoryuse(options.track_memory_to_file)))
if options.print_runtimeraw:
print('{}(RunTimeRaw): {} ms'.format(options.print_runtimeraw,
1000.0 * (time.time() - t0)))
if options.print_dexsegments:
dex_files = glob(os.path.join(outdir, '*.dex'))
utils.print_dexsegments(options.print_dexsegments, dex_files)
return 0
def run_with_options(options, args):
app_provided_pg_conf = False;
if options.golem:
golem.link_third_party()
options.out = os.getcwd()
outdir = options.out
data = None
if options.app == 'gmscore':
options.version = options.version or 'v9'
data = gmscore_data
elif options.app == 'nest':
options.version = options.version or '20180926'
data = nest_data
elif options.app == 'youtube':
options.version = options.version or '12.22'
data = youtube_data
elif options.app == 'chrome':
options.version = options.version or 'default'
data = chrome_data
elif options.app == 'gmail':
options.version = options.version or '170604.16'
data = gmail_data
else:
raise Exception("You need to specify '--app={}'".format('|'.join(APPS)))
if options.compiler not in COMPILERS:
raise Exception("You need to specify '--compiler={}'"
.format('|'.join(COMPILERS)))
if not options.version in data.VERSIONS.keys():
print('No version {} for application {}'
.format(options.version, options.app))
print('Valid versions are {}'.format(data.VERSIONS.keys()))
return 1
version = data.VERSIONS[options.version]
if not options.type:
options.type = 'deploy' if options.compiler == 'r8' \
else 'proguarded'
if options.type not in version:
print('No type {} for version {}'.format(options.type, options.version))
print('Valid types are {}'.format(version.keys()))
return 1
values = version[options.type]
inputs = None
# For R8 'deploy' the JAR is located using the Proguard configuration
# -injars option. For chrome and nest we don't have the injars in the
# proguard files.
if 'inputs' in values and (options.compiler != 'r8'
or options.type != 'deploy'
or options.app == 'chrome'
or options.app == 'nest'):
inputs = values['inputs']
args.extend(['--output', outdir])
if 'min-api' in values:
args.extend(['--min-api', values['min-api']])
if 'main-dex-list' in values:
args.extend(['--main-dex-list', values['main-dex-list']])
if options.compiler == 'r8':
if 'pgconf' in values and not options.k:
for pgconf in values['pgconf']:
args.extend(['--pg-conf', pgconf])
app_provided_pg_conf = True
if options.k:
args.extend(['--pg-conf', options.k])
if 'maindexrules' in values:
for rules in values['maindexrules']:
args.extend(['--main-dex-rules', rules])
if not options.no_libraries and 'libraries' in values:
for lib in values['libraries']:
args.extend(['--lib', lib])
if not outdir.endswith('.zip') and not outdir.endswith('.jar') \
and not os.path.exists(outdir):
os.makedirs(outdir)
# Additional flags for the compiler from the configuration file.
if 'flags' in values:
args.extend(values['flags'].split(' '))
if options.compiler == 'r8':
if 'r8-flags' in values:
args.extend(values['r8-flags'].split(' '))
# Additional flags for the compiler from the command line.
if options.compiler_flags:
args.extend(options.compiler_flags.split(' '))
if options.r8_flags:
args.extend(options.r8_flags.split(' '))
if inputs:
args.extend(inputs)
t0 = time.time()
if options.dump_args_file:
with open(options.dump_args_file, 'w') as args_file:
args_file.writelines([arg + os.linesep for arg in args])
else:
with utils.TempDir() as temp:
if options.print_memoryuse and not options.track_memory_to_file:
options.track_memory_to_file = os.path.join(temp,
utils.MEMORY_USE_TMP_FILE)
if options.compiler == 'r8' and app_provided_pg_conf:
# Ensure that output of -printmapping and -printseeds go to the output
# location and not where the app Proguard configuration places them.
if outdir.endswith('.zip') or outdir.endswith('.jar'):
pg_outdir = os.path.dirname(outdir)
else:
pg_outdir = outdir
additional_pg_conf = GenerateAdditionalProguardConfiguration(
temp, os.path.abspath(pg_outdir))
args.extend(['--pg-conf', additional_pg_conf])
build = not options.no_build and not options.golem
exit_code = toolhelper.run(options.compiler, args,
build=build,
debug=not options.no_debug,
profile=options.profile,
track_memory_file=options.track_memory_to_file)
if exit_code != 0:
return exit_code
if options.print_memoryuse:
print('{}(MemoryUse): {}'
.format(options.print_memoryuse,
utils.grep_memoryuse(options.track_memory_to_file)))
if options.print_runtimeraw:
print('{}(RunTimeRaw): {} ms'
.format(options.print_runtimeraw, 1000.0 * (time.time() - t0)))
if options.print_dexsegments:
dex_files = glob(os.path.join(outdir, '*.dex'))
utils.print_dexsegments(options.print_dexsegments, dex_files)
return 0