def BuildAppWithShrinker(
app, repo, shrinker, checkout_dir, out_dir, temp_dir, options,
keepRuleSynthesisForRecompilation=False):
print('[{}] Building {} with {}{}'.format(
datetime.now().strftime("%H:%M:%S"),
app.name,
shrinker,
' for recompilation' if keepRuleSynthesisForRecompilation else ''))
print('To compile locally: '
'tools/run_on_as_app.py --shrinker {} --r8-compilation-steps {} '
'--app {} {}'.format(
shrinker,
options.r8_compilation_steps,
app.name,
'--r8-compilation-steps-only'
if options.r8_compilation_steps_only else ''))
print('HINT: use --shrinker r8-nolib --no-build if you have a local R8.jar')
# Add settings.gradle file if it is not present to prevent gradle from finding
# the settings.gradle file in the r8 root when apps are placed under
# $R8/build.
as_utils.add_settings_gradle(checkout_dir, app.name)
# Add 'r8.jar' to top-level build.gradle.
as_utils.add_r8_dependency(checkout_dir, temp_dir, IsMinifiedR8(shrinker))
archives_base_name = app.archives_base_name
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# Set -printconfiguration in Proguard rules.
proguard_config_dest = os.path.abspath(
os.path.join(out_dir, 'proguard-rules.pro'))
as_utils.SetPrintConfigurationDirective(
app, checkout_dir, proguard_config_dest)
env_vars = {}
env_vars['JAVA_HOME'] = jdk.GetJdk8Home()
env_vars['ANDROID_HOME'] = utils.getAndroidHome()
if not options.disable_assertions:
env_vars['JAVA_OPTS'] = '-ea:com.android.tools.r8...'
release_target = app.releaseTarget
if not release_target:
app_module = app.module.replace('/', ':')
app_flavor = (app.flavor.capitalize() if app.flavor else '') + 'Release'
release_target = app_module + ':' + 'assemble' + app_flavor
# Build using gradle.
args = [release_target,
'-g=' + os.path.join(checkout_dir, GRADLE_USER_HOME),
'-Pandroid.enableR8=' + str(IsR8(shrinker)).lower(),
'-Pandroid.enableR8.fullMode=' + str(IsR8FullMode(shrinker)).lower()]
if app.has_lint_task:
args.extend(['-x', app_module + ':lintVital' + app_flavor])
if options.bot:
args.extend(['--console=plain', '--info'])
# Warm up gradle if pre_runs > 0. For posterity we generate the same sequence
# as the benchmarking at https://github.com/madsager/santa-tracker-android.
for i in range(0, options.gradle_pre_runs):
if i == 0:
utils.RunGradlew(
["--stop"],
env_vars=env_vars,
quiet=options.quiet,
logging=IsLoggingEnabledFor(app, options),
use_daemon=options.use_daemon)
utils.RunGradlew(
args,
env_vars=env_vars,
quiet=options.quiet,
clean=i > 0,
use_daemon=options.use_daemon,
logging=IsLoggingEnabledFor(app, options))
if keepRuleSynthesisForRecompilation:
args.append('-Dcom.android.tools.r8.keepRuleSynthesisForRecompilation=true')
if options.gradle_flags:
args.extend(options.gradle_flags.split(' '))
args.append('--profile')
stdout = utils.RunGradlew(
args,
env_vars=env_vars,
quiet=options.quiet,
use_daemon=options.use_daemon,
logging=IsLoggingEnabledFor(app, options))
apk_base_name = (archives_base_name
+ (('-' + app.flavor) if app.flavor else '') + '-release')
signed_apk_name = (
app.signed_apk_name
if app.signed_apk_name
else apk_base_name + '.apk')
unsigned_apk_name = apk_base_name + '-unsigned.apk'
build_dir = os.path.join(app.module, app.build_dir)
build_output_apks = os.path.join(build_dir, 'outputs', 'apk')
if app.flavor:
build_output_apks = os.path.join(build_output_apks, app.flavor, 'release')
else:
build_output_apks = os.path.join(build_output_apks, 'release')
signed_apk = os.path.join(build_output_apks, signed_apk_name)
unsigned_apk = os.path.join(build_output_apks, unsigned_apk_name)
assert os.path.isfile(signed_apk) or os.path.isfile(unsigned_apk), (
"Expected a file to be present at {} or {}, found: {}\n"
"Standard out from compilation: {}".format(
signed_apk,
unsigned_apk,
', '.join(
as_utils.ListFiles(build_dir, lambda x : x.endswith('.apk'))),
stdout))
if options.sign_apks and not os.path.isfile(signed_apk):
assert os.path.isfile(unsigned_apk)
if options.sign_apks:
apk_utils.sign_with_apksigner(
unsigned_apk,
signed_apk,
options.keystore,
options.keystore_password,
quiet=options.quiet,
logging=IsLoggingEnabledFor(app, options))
if os.path.isfile(signed_apk):
apk_dest = os.path.join(out_dir, signed_apk_name)
as_utils.MoveFile(signed_apk, apk_dest, quiet=options.quiet)
else:
apk_dest = os.path.join(out_dir, unsigned_apk_name)
as_utils.MoveFile(unsigned_apk, apk_dest, quiet=options.quiet)
assert ('r8' not in shrinker
or CheckIsBuiltWithExpectedR8(apk_dest, temp_dir, shrinker, options))
profile_dest_dir = os.path.join(out_dir, 'profile')
as_utils.MoveProfileReportTo(profile_dest_dir, stdout, quiet=options.quiet)
# Ensure that the gradle daemon is stopped if we are running with it.
if options.use_daemon:
utils.RunGradlew(['--stop', '-g=' + os.path.join(checkout_dir, GRADLE_USER_HOME)])
return (apk_dest, profile_dest_dir, proguard_config_dest)
def main(argv):
(options, args) = ParseOptions(argv)
if options.bot:
utils.DownloadFromGoogleCloudStorage(utils.OPENSOURCE_APPS_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.ANDROID_SDK + '.tar.gz.sha1',
bucket='r8-deps-internal',
auth=True)
if os.path.exists(WORKING_DIR):
shutil.rmtree(WORKING_DIR)
shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR)
os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join(
utils.ANDROID_SDK)
os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3'
# TODO(b/141081520): Set to True once fixed.
options.no_logging = False
# TODO(b/141081520): Remove logging filter once fixed.
options.app_logging_filter = ['sqldelight']
options.shrinker = ['r8', 'r8-full']
print(options.shrinker)
if options.golem:
golem.link_third_party()
if os.path.exists(WORKING_DIR):
shutil.rmtree(WORKING_DIR)
shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR)
os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join(
utils.ANDROID_SDK)
os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3'
options.disable_assertions = True
options.ignore_versions = True
options.no_build = True
options.r8_compilation_steps = 1
options.quiet = True
options.gradle_pre_runs = 2
options.use_daemon = True
options.no_logging = True
if not os.path.exists(WORKING_DIR):
os.makedirs(WORKING_DIR)
if options.download_only:
clone_repositories(options.quiet)
return
with utils.TempDir() as temp_dir:
if not (options.no_build or options.golem):
gradle.RunGradle(['r8', '-Pno_internal'])
build_r8lib = False
for shrinker in options.shrinker:
if IsMinifiedR8(shrinker):
build_r8lib = True
if build_r8lib:
gradle.RunGradle(['r8lib', '-Pno_internal'])
if options.hash:
# Download r8-<hash>.jar from
# https://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.hash)
update_prebuilds_in_android.download_hash(
temp_dir, 'com/android/tools/r8/' + options.hash, target)
as_utils.MoveFile(
os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
elif options.version:
# Download r8-<version>.jar from
# https://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.version)
update_prebuilds_in_android.download_version(
temp_dir, 'com/android/tools/r8/' + options.version, target)
as_utils.MoveFile(
os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
else:
# Make a copy of r8.jar and r8lib.jar such that they stay the same for
# the entire execution of this script.
if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker:
assert os.path.isfile(utils.R8_JAR), 'Cannot build without r8.jar'
shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
if 'r8' in options.shrinker or 'r8-full' in options.shrinker:
assert os.path.isfile(utils.R8LIB_JAR), 'Cannot build without r8lib.jar'
shutil.copyfile(utils.R8LIB_JAR, os.path.join(temp_dir, 'r8lib.jar'))
result_per_shrinker_per_app = []
# If we are running on golem we kill all java processes after the run
# to ensure no hanging gradle daemons.
with EnsureNoGradleAlive(options.golem):
for (app, repo) in options.apps:
if app.skip:
continue
result_per_shrinker_per_app.append(
(app, GetResultsForApp(app, repo, options, temp_dir)))
return LogResultsForApps(result_per_shrinker_per_app, options)
def BuildAppWithShrinker(app,
repo,
shrinker,
checkout_dir,
out_dir,
temp_dir,
options,
keepRuleSynthesisForRecompilation=False):
print('Building {} with {}{}'.format(
app.name, shrinker,
' for recompilation' if keepRuleSynthesisForRecompilation else ''))
# Add settings.gradle file if it is not present to prevent gradle from finding
# the settings.gradle file in the r8 root when apps are placed under
# $R8/build.
as_utils.add_settings_gradle(checkout_dir, app.name)
# Add 'r8.jar' to top-level build.gradle.
as_utils.add_r8_dependency(checkout_dir, temp_dir, IsMinifiedR8(shrinker))
archives_base_name = app.archives_base_name
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# Set -printconfiguration in Proguard rules.
proguard_config_dest = os.path.abspath(
os.path.join(out_dir, 'proguard-rules.pro'))
as_utils.SetPrintConfigurationDirective(app, checkout_dir,
proguard_config_dest)
env_vars = {}
env_vars['ANDROID_HOME'] = utils.getAndroidHome()
if not options.disable_assertions:
env_vars['JAVA_OPTS'] = '-ea:com.android.tools.r8...'
releaseTarget = app.releaseTarget
if not releaseTarget:
releaseTarget = app.module.replace('/', ':') + ':' + 'assemble' + (
app.flavor.capitalize() if app.flavor else '') + 'Release'
# Build using gradle.
args = [
releaseTarget, '-Pandroid.enableR8=' + str(IsR8(shrinker)).lower(),
'-Pandroid.enableR8.fullMode=' + str(IsR8FullMode(shrinker)).lower()
]
# Warm up gradle if pre_runs > 0. For posterity we generate the same sequence
# as the benchmarking at https://github.com/madsager/santa-tracker-android.
for i in range(0, options.gradle_pre_runs):
if i == 0:
utils.RunGradlew(["--stop"],
env_vars=env_vars,
quiet=options.quiet,
logging=IsLoggingEnabledFor(app, options),
use_daemon=options.use_daemon)
utils.RunGradlew(args,
env_vars=env_vars,
quiet=options.quiet,
clean=i > 0,
use_daemon=options.use_daemon,
logging=IsLoggingEnabledFor(app, options))
if keepRuleSynthesisForRecompilation:
args.append(
'-Dcom.android.tools.r8.keepRuleSynthesisForRecompilation=true')
if options.gradle_flags:
args.extend(options.gradle_flags.split(' '))
args.append('--profile')
stdout = utils.RunGradlew(args,
env_vars=env_vars,
quiet=options.quiet,
use_daemon=options.use_daemon,
logging=IsLoggingEnabledFor(app, options))
apk_base_name = (archives_base_name +
(('-' + app.flavor) if app.flavor else '') + '-release')
signed_apk_name = (app.signed_apk_name
if app.signed_apk_name else apk_base_name + '.apk')
unsigned_apk_name = apk_base_name + '-unsigned.apk'
build_dir = os.path.join(app.module, app.build_dir)
build_output_apks = os.path.join(build_dir, 'outputs', 'apk')
if app.flavor:
build_output_apks = os.path.join(build_output_apks, app.flavor,
'release')
else:
build_output_apks = os.path.join(build_output_apks, 'release')
signed_apk = os.path.join(build_output_apks, signed_apk_name)
unsigned_apk = os.path.join(build_output_apks, unsigned_apk_name)
assert os.path.isfile(signed_apk) or os.path.isfile(unsigned_apk), (
"Expected a file to be present at {} or {}, found: {}".format(
signed_apk, unsigned_apk, ', '.join(
as_utils.ListFiles(build_dir, lambda x: x.endswith('.apk')))))
if options.sign_apks and not os.path.isfile(signed_apk):
assert os.path.isfile(unsigned_apk)
if options.sign_apks:
apk_utils.sign_with_apksigner(unsigned_apk,
signed_apk,
options.keystore,
options.keystore_password,
quiet=options.quiet,
logging=IsLoggingEnabledFor(
app, options))
if os.path.isfile(signed_apk):
apk_dest = os.path.join(out_dir, signed_apk_name)
as_utils.MoveFile(signed_apk, apk_dest, quiet=options.quiet)
else:
apk_dest = os.path.join(out_dir, unsigned_apk_name)
as_utils.MoveFile(unsigned_apk, apk_dest, quiet=options.quiet)
assert ('r8' not in shrinker or CheckIsBuiltWithExpectedR8(
apk_dest, temp_dir, shrinker, options))
profile_dest_dir = os.path.join(out_dir, 'profile')
as_utils.MoveProfileReportTo(profile_dest_dir, stdout, quiet=options.quiet)
return (apk_dest, profile_dest_dir, proguard_config_dest)
def main(argv):
(options, args) = parse_options(argv)
if options.bot:
options.no_logging = True
options.shrinker = ['r8', 'r8-full']
print(options.shrinker)
if options.golem:
options.disable_assertions = True
options.no_build = True
options.r8_compilation_steps = 1
options.quiet = True
options.no_logging = True
if options.generate_golem_config:
print_golem_config(options)
return 0
with utils.TempDir() as temp_dir:
if options.hash:
# Download r8-<hash>.jar from
# https://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.hash)
update_prebuilds_in_android.download_hash(
temp_dir, 'com/android/tools/r8/' + options.hash, target)
as_utils.MoveFile(os.path.join(temp_dir, target),
os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
elif version_is_built_jar(options.version):
# Download r8-<version>.jar from
# https://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.version)
update_prebuilds_in_android.download_version(
temp_dir, 'com/android/tools/r8/' + options.version, target)
as_utils.MoveFile(os.path.join(temp_dir, target),
os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
elif options.version == 'master':
if not (options.no_build or options.golem):
gradle.RunGradle(['r8', '-Pno_internal'])
build_r8lib = False
for shrinker in options.shrinker:
if is_minified_r8(shrinker):
build_r8lib = True
if build_r8lib:
gradle.RunGradle(['r8lib', '-Pno_internal'])
# Make a copy of r8.jar and r8lib.jar such that they stay the same for
# the entire execution of this script.
if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker:
assert os.path.isfile(
utils.R8_JAR), 'Cannot build without r8.jar'
shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
if 'r8' in options.shrinker or 'r8-full' in options.shrinker:
assert os.path.isfile(
utils.R8LIB_JAR), 'Cannot build without r8lib.jar'
shutil.copyfile(utils.R8LIB_JAR,
os.path.join(temp_dir, 'r8lib.jar'))
result_per_shrinker_per_app = []
for app in options.apps:
if app.skip:
continue
result_per_shrinker_per_app.append(
(app, get_results_for_app(app, options, temp_dir)))
errors = log_results_for_apps(result_per_shrinker_per_app, options)
if errors > 0:
dest = 'gs://r8-test-results/r8-libs/' + str(int(time.time()))
utils.upload_file_to_cloud_storage(
os.path.join(temp_dir, 'r8lib.jar'), dest)
print('R8lib saved to %s' % dest)
return errors
def main(argv):
(options, args) = ParseOptions(argv)
if options.golem:
golem.link_third_party()
if os.path.exists(WORKING_DIR):
shutil.rmtree(WORKING_DIR)
shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR)
os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join(
utils.ANDROID_SDK)
os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3'
options.disable_assertions = True
options.ignore_versions = True
options.no_build = True
options.r8_compilation_steps = 1
options.quiet = True
options.gradle_pre_runs = 2
options.use_daemon = True
if not os.path.exists(WORKING_DIR):
os.makedirs(WORKING_DIR)
if options.download_only:
clone_repositories(options.quiet)
return
with utils.TempDir() as temp_dir:
if not (options.no_build or options.golem):
gradle.RunGradle(['r8', 'r8lib'])
if options.version:
# Download r8-<version>.jar from
# http://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.version)
update_prebuilds_in_android.download_version(
temp_dir, 'com/android/tools/r8/' + options.version, target)
as_utils.MoveFile(os.path.join(temp_dir, target),
os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
else:
# Make a copy of r8.jar and r8lib.jar such that they stay the same for
# the entire execution of this script.
if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker:
assert os.path.isfile(
utils.R8_JAR), 'Cannot build without r8.jar'
shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
if 'r8' in options.shrinker or 'r8-full' in options.shrinker:
assert os.path.isfile(
utils.R8LIB_JAR), 'Cannot build without r8lib.jar'
shutil.copyfile(utils.R8LIB_JAR,
os.path.join(temp_dir, 'r8lib.jar'))
result_per_shrinker_per_app = []
for (app, repo) in options.apps:
if app.skip:
continue
result_per_shrinker_per_app.append(
(app, GetResultsForApp(app, repo, options, temp_dir)))
LogResultsForApps(result_per_shrinker_per_app, options)
def BuildAppWithShrinker(app,
config,
shrinker,
checkout_dir,
out_dir,
temp_dir,
options,
keepRuleSynthesisForRecompilation=False):
print('Building {} with {}{}'.format(
app, shrinker,
' for recompilation' if keepRuleSynthesisForRecompilation else ''))
# Add/remove 'r8.jar' from top-level build.gradle.
if options.disable_tot:
as_utils.remove_r8_dependency(checkout_dir)
else:
as_utils.add_r8_dependency(checkout_dir, temp_dir,
IsMinifiedR8(shrinker))
app_module = config.get('app_module', 'app')
archives_base_name = config.get('archives_base_name', app_module)
flavor = config.get('flavor')
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# Set -printconfiguration in Proguard rules.
proguard_config_dest = os.path.abspath(
os.path.join(out_dir, 'proguard-rules.pro'))
as_utils.SetPrintConfigurationDirective(app, config, checkout_dir,
proguard_config_dest)
env = {}
env['ANDROID_HOME'] = utils.ANDROID_HOME
env['JAVA_OPTS'] = '-ea:com.android.tools.r8...'
releaseTarget = config.get('releaseTarget')
if not releaseTarget:
releaseTarget = app_module + ':' + 'assemble' + (
flavor.capitalize() if flavor else '') + 'Release'
# Value for property android.enableR8.
enableR8 = 'r8' in shrinker
# Value for property android.enableR8.fullMode.
enableR8FullMode = shrinker == 'r8full' or shrinker == 'r8full-minified'
# Build gradlew command line.
cmd = [
'./gradlew', '--no-daemon', 'clean', releaseTarget, '--profile',
'--stacktrace', '-Pandroid.enableR8=' + str(enableR8).lower(),
'-Pandroid.enableR8.fullMode=' + str(enableR8FullMode).lower()
]
if keepRuleSynthesisForRecompilation:
cmd.append(
'-Dcom.android.tools.r8.keepRuleSynthesisForRecompilation=true')
if options.gradle_flags:
cmd.extend(options.gradle_flags.split(' '))
stdout = utils.RunCmd(cmd, env, quiet=options.quiet)
apk_base_name = (archives_base_name + (('-' + flavor) if flavor else '') +
'-release')
signed_apk_name = config.get('signed-apk-name', apk_base_name + '.apk')
unsigned_apk_name = apk_base_name + '-unsigned.apk'
build_dir = config.get('build_dir', 'build')
build_output_apks = os.path.join(app_module, build_dir, 'outputs', 'apk')
if flavor:
build_output_apks = os.path.join(build_output_apks, flavor, 'release')
else:
build_output_apks = os.path.join(build_output_apks, 'release')
signed_apk = os.path.join(build_output_apks, signed_apk_name)
unsigned_apk = os.path.join(build_output_apks, unsigned_apk_name)
if options.sign_apks and not os.path.isfile(signed_apk):
assert os.path.isfile(unsigned_apk)
if options.sign_apks:
keystore = 'app.keystore'
keystore_password = '******'
apk_utils.sign_with_apksigner(utils.ANDROID_BUILD_TOOLS,
unsigned_apk, signed_apk, keystore,
keystore_password)
if os.path.isfile(signed_apk):
apk_dest = os.path.join(out_dir, signed_apk_name)
as_utils.MoveFile(signed_apk, apk_dest, quiet=options.quiet)
else:
apk_dest = os.path.join(out_dir, unsigned_apk_name)
as_utils.MoveFile(unsigned_apk, apk_dest, quiet=options.quiet)
assert IsBuiltWithR8(apk_dest, temp_dir, options) == ('r8' in shrinker), (
'Unexpected marker in generated APK for {}'.format(shrinker))
profile_dest_dir = os.path.join(out_dir, 'profile')
as_utils.MoveProfileReportTo(profile_dest_dir, stdout, quiet=options.quiet)
return (apk_dest, profile_dest_dir, proguard_config_dest)