def BuildDesugaredLibrary(checkout_dir, variant):
if (variant != 'jdk8' and variant != 'jdk11'):
raise Exception('Variant ' + variant + 'is not supported')
with utils.ChangedWorkingDirectory(checkout_dir):
bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin', 'bazel')
cmd = [
bazel, 'build',
'maven_release' + ('_jdk11' if variant == 'jdk11' else '')
]
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
cmd = [bazel, 'shutdown']
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
# Locate the library jar and the maven zip with the jar from the
# bazel build.
if variant == 'jdk8':
library_jar = os.path.join(checkout_dir, 'bazel-bin', 'src',
'share', 'classes', 'java',
'libjava.jar')
else:
library_jar = os.path.join(checkout_dir, 'bazel-bin', 'jdk11',
'src', 'java_base_selected.jar')
maven_zip = os.path.join(
checkout_dir, 'bazel-bin',
LIBRARY_NAME + ('_jdk11' if variant != 'jdk11' else '') + '.zip')
return (library_jar, maven_zip)
def sign(unsigned_apk, signed_apk, keystore):
print 'Signing (ignore the warnings)'
cmd = ['zip', '-d', unsigned_apk, 'META-INF/*']
utils.PrintCmd(cmd)
subprocess.call(cmd)
cmd = [
'jarsigner', '-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1',
'-keystore', keystore, '-storepass', 'android', '-signedjar',
signed_apk, unsigned_apk, 'androiddebugkey'
]
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
def main():
args = sys.argv[1:]
if len(args) == 0:
gradle.RunGradle(['r8lib'])
r8lib = utils.R8LIB_JAR
r8map = utils.R8LIB + '.map'
elif len(args) == 2:
r8lib = args[0]
r8map = args[1]
elif len(args) == 1 and args[0] == '--help':
print(
'Usage: test_self_retrace.py [<path-to-r8lib-jar> <path-to-r8lib-map]'
)
print('If the path is missing the script builds and uses ' +
utils.R8LIB_JAR)
return
else:
raise Exception("Only two argument allowed, see '--help'.")
# Run 'r8 --help' which throws an exception.
cmd = [
jdk.GetJavaExecutable(), '-cp', r8lib, 'com.android.tools.r8.R8',
'--help'
]
os.environ["R8_THROW_EXCEPTION_FOR_TESTING_RETRACE"] = "1"
utils.PrintCmd(cmd)
p = subprocess.Popen(cmd, stderr=subprocess.PIPE)
_, stacktrace = p.communicate()
assert (p.returncode != 0)
assert (EXCEPTION_LINE in stacktrace)
# r8lib must be minified, original class names must not be present.
assert ('SelfRetraceTest' not in stacktrace)
# Run the retrace tool.
cmd = [jdk.GetJavaExecutable(), '-jar', utils.RETRACE_JAR, r8map]
utils.PrintCmd(cmd)
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
retrace_stdout, _ = p.communicate(stacktrace)
assert p.returncode == 0
retrace_lines = retrace_stdout.splitlines()
line_index = -1
for line in retrace_lines:
if line_index < 0:
if 'java.lang.RuntimeException' in line:
assert (EXCEPTION_LINE in line)
line_index = 0
else:
assert EXPECTED_LINES[line_index] in line
line_index += 1
if line_index >= len(EXPECTED_LINES):
break
assert (line_index >= 0)
def RunGradleInGetOutput(gradleCmd, args, cwd, env=None):
EnsureDeps()
cmd = [gradleCmd]
cmd.extend(args)
utils.PrintCmd(cmd)
with utils.ChangedWorkingDirectory(cwd):
return subprocess.check_output(cmd, env=GetJavaEnv(env))
def ExtractMarker(apk, temp_dir, options):
r8_jar = os.path.join(temp_dir, 'r8.jar')
r8lib_jar = os.path.join(temp_dir, 'r8lib.jar')
# Use the copy of r8.jar or r8lib.jar if one is there.
if os.path.isfile(r8_jar):
cmd = [
jdk.GetJavaExecutable(), '-ea', '-jar', r8_jar, 'extractmarker',
apk
]
elif os.path.isfile(r8lib_jar):
cmd = [
jdk.GetJavaExecutable(), '-ea', '-cp', r8lib_jar,
'com.android.tools.r8.ExtractMarker', apk
]
else:
script = os.path.join(utils.TOOLS_DIR, 'extractmarker.py')
cmd = ['python', script, apk]
utils.PrintCmd(cmd, quiet=options.quiet)
stdout = subprocess.check_output(cmd)
# Return the last line.
lines = stdout.strip().splitlines()
assert len(lines) >= 1
return lines[-1]
def generate_jar_with_desugar_configuration(configuration, implementation,
conversions, destination):
with utils.TempDir() as tmp_dir:
# Add conversion classes.
with zipfile.ZipFile(conversions, 'r') as conversions_zip:
conversions_zip.extractall(tmp_dir)
# Add configuration
configuration_dir = join(tmp_dir, 'META-INF', 'desugar', 'd8')
makedirs(configuration_dir)
copyfile(configuration, join(configuration_dir, 'desugar.json'))
# Add lint configuartion.
lint_dir = join(configuration_dir, 'lint')
makedirs(lint_dir)
cmd = [
jdk.GetJavaExecutable(), '-cp', utils.R8_JAR,
'com.android.tools.r8.GenerateLintFiles', configuration,
implementation, lint_dir
]
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
make_archive(destination, 'zip', tmp_dir)
move(destination + '.zip', destination)
def run(dexfile, oatfile=None, version='default', verbose=[]):
# dex2oat accepts non-existent dex files, check here instead
if not os.path.exists(dexfile):
raise Exception('DEX file not found: "{}"'.format(dexfile))
with utils.TempDir() as temp:
if not oatfile:
oatfile = os.path.join(temp, "out.oat")
base = os.path.join(LINUX_DIR, DIRS[version])
product = PRODUCTS[version]
arch = ARCHS[version]
cmd = [
os.path.join(base, 'bin', 'dex2oat'),
'--android-root=' +
os.path.join(base, 'product', product, 'system'),
'--runtime-arg',
'-Xnorelocate',
'--dex-file=' + dexfile,
'--oat-file=' + oatfile,
'--instruction-set=' + arch,
]
for flag in verbose:
cmd += ['--runtime-arg', '-verbose:' + flag]
env = {"LD_LIBRARY_PATH": os.path.join(base, 'lib')}
utils.PrintCmd(cmd)
subprocess.check_call(cmd, env=env)
def run_retrace(options, temp):
if options.download_benchmarks:
download_benchmarks()
if options.retracer == 'r8':
retracer_args = [
'-cp', utils.R8LIB_JAR, 'com.android.tools.r8.retrace.Retrace'
]
elif options.retracer == 'proguard':
retracer_args = ['-jar', proguard.getRetraceJar()]
elif options.retracer == 'remapper':
retracer_args = [
'-jar',
os.path.join(utils.THIRD_PARTY, 'remapper', 'remapper_deploy.jar')
]
else:
assert False, "Unexpected retracer " + options.retracer
retrace_args = [jdk.GetJavaExecutable()] + retracer_args + [
os.path.join(utils.THIRD_PARTY, 'retrace_benchmark', 'r8lib.jar.map'),
os.path.join(utils.THIRD_PARTY, 'retrace_benchmark', 'stacktrace.txt')
]
utils.PrintCmd(retrace_args)
t0 = time.time()
subprocess.check_call(retrace_args)
t1 = time.time()
if options.print_runtimeraw:
print('{}(RunTimeRaw): {} ms'.format(options.print_runtimeraw,
1000.0 * (t1 - t0)))
def Main():
utils.check_java_version()
args = parse_arguments()
output_dir = args.output
if args.golem:
golem.link_third_party()
with utils.TempDir() as temp_dir:
if not output_dir:
output_dir = temp_dir
xmx = None
if args.tool == 'dx':
tool_file = DX_JAR
tool_args = [
'--dex', '--output=' + output_dir, '--multi-dex',
'--min-sdk-version=' + MIN_SDK_VERSION
]
xmx = '-Xmx1600m'
else:
tool_file = utils.D8_JAR
tool_args = ['--output', output_dir, '--min-api', MIN_SDK_VERSION]
if args.tool == 'd8-release':
tool_args.append('--release')
xmx = '-Xmx600m'
cmd = []
track_memory_file = None
if args.print_memoryuse:
track_memory_file = os.path.join(output_dir,
utils.MEMORY_USE_TMP_FILE)
cmd.extend(['tools/track_memory.sh', track_memory_file])
if tool_file.endswith('.jar'):
assert xmx is not None
cmd.extend(['java', xmx, '-jar'])
cmd.extend([tool_file] + tool_args + [FRAMEWORK_JAR])
utils.PrintCmd(cmd)
t0 = time.time()
subprocess.check_call(cmd)
dt = time.time() - t0
if args.print_memoryuse:
print('{}-Total(MemoryUse): {}'.format(
args.name, utils.grep_memoryuse(track_memory_file)))
dex_files = [f for f in glob(os.path.join(output_dir, '*.dex'))]
code_size = 0
for dex_file in dex_files:
code_size += os.path.getsize(dex_file)
print('{}-Total(RunTimeRaw): {} ms'.format(args.name, 1000.0 * dt))
print('{}-Total(CodeSize): {}'.format(args.name, code_size))
utils.print_dexsegments(args.name, dex_files)
def execute(cmd, archive, env=None):
utils.PrintCmd(cmd)
with utils.TempDir() as temp:
try:
stderr_fd = None
stdout_fd = None
exitcode = 0
stderr = os.path.join(temp, 'stderr')
stderr_fd = open(stderr, 'w')
stdout = os.path.join(temp, 'stdout')
stdout_fd = open(stdout, 'w')
popen = subprocess.Popen(cmd,
bufsize=1024 * 1024 * 10,
stdout=stdout_fd,
stderr=stderr_fd,
env=env)
begin = time.time()
timed_out = False
while popen.poll() == None:
if time.time() - begin > RUN_TIMEOUT:
popen.terminate()
timed_out = True
time.sleep(2)
exitcode = popen.returncode
finally:
if stderr_fd:
stderr_fd.close()
if stdout_fd:
stdout_fd.close()
if exitcode != 0:
handle_output(archive, stderr, stdout, popen.returncode,
timed_out, ' '.join(cmd))
return exitcode
def Main():
args = parse_arguments()
if args.golem:
golem.link_third_party()
utils.check_java_version()
with utils.TempDir() as temp_dir:
cmd_prefix = []
output_dir = args.output if args.output else temp_dir
temp_dir = os.path.join(args.output,
'tmp') if args.output else temp_dir
track_memory_file = None
if args.print_memoryuse:
track_memory_file = os.path.join(output_dir,
utils.MEMORY_USE_TMP_FILE)
cmd_prefix.extend(['tools/track_memory.sh', track_memory_file])
name = 'CompileHelloExample'
tool = args.tool
output_mode = args.output_mode
lib = None
if output_mode == 'dex':
name += 'Dex'
lib = utils.get_android_jar(28)
else:
lib = utils.RT_JAR
extra = []
if args.large:
name += 'Large'
extra = EXTRA_INPUTS
if args.noopt:
name += 'NoOpt'
cmds = Compile(
tool,
output_mode,
lib,
extra,
output_dir,
args.noopt,
temp_dir,
)
t0 = time.time()
for cmd in cmds:
fullcmd = cmd_prefix + cmd
utils.PrintCmd(fullcmd)
subprocess.check_output(fullcmd)
dt = time.time() - t0
if args.print_memoryuse:
print('{}(MemoryUse): {}'.format(
name, utils.grep_memoryuse(track_memory_file)))
print('{}(RunTimeRaw): {} ms'.format(name, 1000.0 * dt))
def run(args, track_memory_file=None, stdout=None, stderr=None):
cmd = []
if track_memory_file:
cmd.extend(['tools/track_memory.sh', track_memory_file])
cmd.extend(['java', '-jar', PROGUARD_JAR])
cmd.extend(args)
utils.PrintCmd(cmd)
subprocess.call(cmd, stdout=stdout, stderr=stderr)
def write_sorted_lines(cmd_args, output_path):
utils.PrintCmd(cmd_args)
output_lines = subprocess.check_output(cmd_args).splitlines(True)
print("Write output to %s" % output_path)
output_lines.sort()
with open(output_path, 'w') as fp:
for line in output_lines:
fp.write(line)
def run_adb(args, ignore_exit=False):
command = ['adb']
command.extend(args)
utils.PrintCmd(command)
# On M adb install-multiple exits 1 but succeed in installing.
if ignore_exit:
subprocess.call(command)
else:
subprocess.check_call(command)
def split(app):
split_spec = os.path.join(get_sample_dir(app), 'split.spec')
command = [
DEFAULT_DEXSPLITTER, '--input',
get_dex_path(app), '--output',
get_bin_path(app), '--feature-splits', split_spec
]
utils.PrintCmd(command)
subprocess.check_call(command)
def sign_with_apksigner(build_tools_dir, unsigned_apk, signed_apk, keystore,
password):
cmd = [
os.path.join(build_tools_dir, 'apksigner'), 'sign', '-v', '--ks',
keystore, '--ks-pass', 'pass:'******'--min-sdk-version', '19',
'--out', signed_apk, unsigned_apk
]
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
def run(tool,
args,
build=None,
debug=True,
profile=False,
track_memory_file=None,
extra_args=None,
stderr=None,
stdout=None,
return_stdout=False,
timeout=0,
quiet=False,
cmd_prefix=None):
cmd = []
if cmd_prefix:
cmd.extend(cmd_prefix)
if build is None:
build, args = extract_build_from_args(args)
if build:
gradle.RunGradle(['r8lib' if tool.startswith('r8lib') else 'r8'])
if track_memory_file:
cmd.extend(['tools/track_memory.sh', track_memory_file])
cmd.append(jdk.GetJavaExecutable())
if extra_args:
cmd.extend(extra_args)
if debug:
cmd.append('-ea')
if profile:
cmd.append('-agentlib:hprof=cpu=samples,interval=1,depth=8')
if tool == 'r8lib-d8':
cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.D8'])
elif tool == 'r8lib-r8':
cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.R8'])
else:
cmd.extend(['-jar', utils.R8_JAR, tool])
lib, args = extract_lib_from_args(args)
if lib:
cmd.extend(["--lib", lib])
cmd.extend(args)
utils.PrintCmd(cmd, quiet=quiet)
if timeout > 0:
kill = lambda process: process.kill()
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
timer = Timer(timeout, kill, [proc])
try:
timer.start()
stdout, stderr = proc.communicate()
finally:
timer.cancel()
return stdout if return_stdout else proc.returncode
else:
if return_stdout:
return subprocess.check_output(cmd)
return subprocess.call(cmd, stdout=stdout, stderr=stderr)
def RunGradle(args, throw_on_failure=True):
EnsureGradle()
cmd = [GRADLE]
cmd.extend(args)
utils.PrintCmd(cmd)
with utils.ChangedWorkingDirectory(utils.REPO_ROOT):
return_value = subprocess.call(cmd)
if throw_on_failure and return_value != 0:
raise
return return_value
def RunGradleIn(gradleCmd, args, cwd, throw_on_failure=True, env=None):
EnsureDeps()
cmd = [gradleCmd]
cmd.extend(args)
utils.PrintCmd(cmd)
with utils.ChangedWorkingDirectory(cwd):
return_value = subprocess.call(cmd, env=GetJavaEnv(env))
if throw_on_failure and return_value != 0:
raise Exception('Failed to execute gradle')
return return_value
def Main():
args = parse_arguments()
with utils.TempDir() as temp_dir:
if args.tool in ['dx', 'goyt']:
tool_args = ['--dex', '--output=' + temp_dir, '--multi-dex',
'--min-sdk-version=' + MIN_SDK_VERSION]
if args.tool == 'goyt':
tool_file = GOYT_EXE
tool_args = ['--num-threads=4'] + tool_args
elif args.tool == 'dx':
tool_file = DX_JAR
else:
tool_file = D8_JAR
tool_args = ['--output', temp_dir, '--min-api', MIN_SDK_VERSION]
if args.tool == 'd8-release':
tool_args.append('--release')
cmd = []
track_memory_file = None
if args.print_memoryuse:
track_memory_file = os.path.join(temp_dir, utils.MEMORY_USE_TMP_FILE)
cmd.extend(['tools/track_memory.sh', track_memory_file])
if tool_file.endswith('.jar'):
cmd.extend(['java', '-jar'])
cmd.extend([tool_file] + tool_args + [FRAMEWORK_JAR])
utils.PrintCmd(cmd)
t0 = time.time()
subprocess.check_call(cmd)
dt = time.time() - t0
if args.print_memoryuse:
print('{}(MemoryUse): {}'
.format(args.name, utils.grep_memoryuse(track_memory_file)))
dex_files = [f for f in glob(os.path.join(temp_dir, '*.dex'))]
code_size = 0
for dex_file in dex_files:
code_size += os.path.getsize(dex_file)
print('{}-Total(RunTimeRaw): {} ms'
.format(args.name, 1000.0 * dt))
print('{}-Total(CodeSize): {}'
.format(args.name, code_size))
utils.print_dexsegments(args.name, dex_files)
def run(args,
version=DEFAULT,
track_memory_file=None,
stdout=None,
stderr=None):
cmd = []
if track_memory_file:
cmd.extend(['tools/track_memory.sh', track_memory_file])
cmd.extend(getCmd(args, version))
utils.PrintCmd(cmd)
subprocess.call(cmd, stdout=stdout, stderr=stderr)
def IsBuiltWithR8(apk, temp_dir, options):
r8_jar = os.path.join(temp_dir, 'r8.jar')
# Use the copy of r8.jar if it is there.
if os.path.isfile(r8_jar):
cmd = ['java', '-ea', '-jar', r8_jar, 'extractmarker', apk]
else:
script = os.path.join(utils.TOOLS_DIR, 'extractmarker.py')
cmd = ['python', script, apk]
utils.PrintCmd(cmd, quiet=options.quiet)
return '~~R8' in subprocess.check_output(cmd).strip()
def generate_proto_apks(apks, options):
proto_apks = []
for apk in apks:
proto_apk = apk + '.proto'
cmd = [
options.aapt2, 'convert', '-o', proto_apk, '--output-format',
'proto', apk
]
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
proto_apks.append(proto_apk)
return proto_apks
def test_d8sample(paths):
with utils.TempDir() as path:
args = ['java', '-cp', '%s:%s' % (SAMPLE_JAR, ":".join(paths)),
'com.android.tools.apiusagesample.D8ApiUsageSample',
'--output', path,
'--min-api', str(API_LEVEL),
'--lib', utils.get_android_jar(API_LEVEL),
'--classpath', utils.R8_JAR,
'--main-dex-list', '/dev/null',
os.path.join(utils.BUILD, 'test/examples/hello.jar')]
utils.PrintCmd(args)
subprocess.check_call(args)
def compile_with_javac(api, app):
with utils.ChangedWorkingDirectory(get_sample_dir(app)):
files = glob.glob(SRC_LOCATION.format(app=app))
classpath = '%s:%s' % (utils.get_android_jar(api), get_guava_jar())
command = [
DEFAULT_JAVAC, '-classpath', classpath, '-sourcepath',
'%s:%s:%s' %
(get_src_path(app), get_gen_path(app), get_guava_jar()), '-d',
get_bin_path(app)
]
command.extend(files)
utils.PrintCmd(command)
subprocess.check_call(command)
def repack(processed_out, original_apk, temp):
processed_apk = os.path.join(temp, 'processed.apk')
shutil.copyfile(original_apk, processed_apk)
if not processed_out:
print 'Using original APK as is'
return processed_apk
print 'Repacking APK with dex files from', processed_apk
with utils.ChangedWorkingDirectory(temp):
cmd = ['zip', '-d', 'processed.apk', '*.dex']
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
if processed_out.endswith('.zip') or processed_out.endswith('.jar'):
cmd = ['unzip', processed_out, '-d', temp]
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
processed_out = temp
with utils.ChangedWorkingDirectory(processed_out):
dex = glob.glob('*.dex')
cmd = ['zip', '-u', '-9', processed_apk] + dex
utils.PrintCmd(cmd)
subprocess.check_call(cmd)
return processed_apk
def test_r8cfcommand(paths):
with utils.TempDir() as path:
# SAMPLE_JAR and LIB_JAR should not have any classes in common, since e.g.
# R8CommandParser should have been minified in LIB_JAR.
# Just in case R8CommandParser is also present in LIB_JAR, we put
# SAMPLE_JAR first on the classpath to use its version of R8CommandParser.
args = ['java', '-cp', '%s:%s' % (SAMPLE_JAR, ":".join(paths)),
'com.android.tools.r8.R8CommandParser',
'--classfile',
'--output', path + "/output.jar",
'--lib', utils.RT_JAR,
os.path.join(utils.BUILD, 'test/examples/hello.jar')]
utils.PrintCmd(args)
subprocess.check_call(args)
def run(args, build=True):
if build:
gradle.RunGradle(['copyMavenDeps'])
cmd = []
cmd.append('java')
cp = ":".join([os.path.join(utils.REPO_ROOT, 'build/deps/asm-6.2.1.jar'),
os.path.join(utils.REPO_ROOT, 'build/deps/asm-util-6.2.1.jar')])
cmd.extend(['-cp', cp])
cmd.append('org.objectweb.asm.util.ASMifier')
cmd.extend(args)
utils.PrintCmd(cmd)
result = subprocess.check_output(cmd)
print(result)
return result
def prepare_wrapper(dist, temp):
wrapper_file = os.path.join(
utils.REPO_ROOT,
'src/main/java/com/android/tools/r8/utils/CompileDumpCompatR8.java')
cmd = [
jdk.GetJavacExecutable(),
wrapper_file,
'-d',
temp,
'-cp',
dist,
]
utils.PrintCmd(cmd)
subprocess.check_output(cmd)
return temp
def run(args, build=True, debug=True, profile=False, track_memory_file=None):
if build:
gradle.RunGradle(['D8'])
cmd = []
if track_memory_file:
cmd.extend(['tools/track_memory.sh', track_memory_file])
cmd.append('java')
if debug:
cmd.append('-ea')
if profile:
cmd.append('-agentlib:hprof=cpu=samples,interval=1,depth=8')
cmd.extend(['-jar', D8_JAR])
cmd.extend(args)
utils.PrintCmd(cmd)
subprocess.check_call(cmd)