def Main():
args = parse_arguments()
# Ensure that everything is downloaded before generating the pom list
gradle.RunGradle(['-stop'])
gradle_deps = gradle.RunGradleGetOutput(
['printMavenDeps', '-Pupdatemavendeps']).splitlines()
gradle_poms = read_gradle_cache_pom_files()
# Example output lines:
# POM: /usr.../org.ow2.asm/asm/6.2.1/3bc91be104d9292ff1dcc3dbf1002b7c320e767d/asm-6.2.1.pom org.ow2.asm:asm:6.2.1
# JAR: /usr.../com.google.code.gson/gson/2.7/751f548c85fa49f330cecbb1875893f971b33c4e/gson-2.7.jar
poms = [l[5:] for l in gradle_deps if l.startswith('POM: ')]
jars = [l[5:] for l in gradle_deps if l.startswith('JAR: ')]
descriptor_to_entry = {}
parents = []
for pom in poms:
split = pom.split(' ')
filepath = split[0]
gradle_descriptor = split[1]
descriptor = get_descriptor_from_path(filepath)
assert descriptor == gradle_descriptor
(group, artifact, version) = parse_descriptor(gradle_descriptor)
descriptor_to_entry[descriptor] = Entry(group, artifact, version, filepath)
parent = get_parent(filepath, gradle_poms)
while parent:
descriptor = get_descriptor_from_path(parent.path)
descriptor_to_entry[descriptor] = parent
parent = get_parent(parent.path, gradle_poms)
for jar in jars:
if jar.startswith(utils.REPO_ROOT):
continue
descriptor = get_descriptor_from_path(jar)
assert descriptor in descriptor_to_entry
descriptor_to_entry[descriptor].set_jar(jar)
has_missing = False
for descriptor in descriptor_to_entry:
entry = descriptor_to_entry[descriptor]
if not utils.file_exists_on_cloud_storage(entry.get_cloud_pom_location()):
if args.check_mirror:
has_missing = True
print 'Missing dependency for: ' + descriptor
else:
print 'Uploading: %s' % entry.path
utils.upload_file_to_cloud_storage(entry.path, entry.get_cloud_pom_location())
if entry.jar:
if not utils.file_exists_on_cloud_storage(entry.get_cloud_jar_location()):
if args.check_mirror:
has_missing = True
print 'Missing dependency for: ' + descriptor
else:
print 'Uploading: %s' % entry.jar
utils.upload_file_to_cloud_storage(entry.jar, entry.get_cloud_jar_location())
if args.check_mirror:
if has_missing:
print('The maven mirror has missing dependencies, please run'
'tools/maven_mirror.py')
return 1
else:
print('Mirror is up to date with all dependencies')
def main():
args = parse_arguments()
r8lib_map_path = args.map
hashOrVersion = args.commit_hash or args.version
if hashOrVersion:
download_path = archive.GetUploadDestination(
hashOrVersion, 'r8lib.jar.map', args.commit_hash is not None)
if utils.file_exists_on_cloud_storage(download_path):
r8lib_map_path = tempfile.NamedTemporaryFile().name
utils.download_file_from_cloud_storage(download_path,
r8lib_map_path)
else:
print('Could not find map file from argument: %s.' % hashOrVersion)
return 1
retrace_args = ['java', '-jar', utils.RETRACE_JAR, r8lib_map_path]
if args.stacktrace:
retrace_args.append(args.stacktrace)
return subprocess.call(retrace_args)
def run(map_path,
hash_or_version,
stacktrace,
is_hash,
no_r8lib,
quiet=False,
debug=False):
if hash_or_version:
download_path = archive.GetUploadDestination(hash_or_version,
'r8lib.jar.map', is_hash)
if utils.file_exists_on_cloud_storage(download_path):
map_path = tempfile.NamedTemporaryFile().name
utils.download_file_from_cloud_storage(download_path, map_path)
else:
print('Could not find map file from argument: %s.' %
hash_or_version)
return 1
retrace_args = [jdk.GetJavaExecutable()]
if debug:
retrace_args.append(
'-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005'
)
retrace_args += [
'-cp', utils.R8_JAR if no_r8lib else utils.R8LIB_JAR,
'com.android.tools.r8.retrace.Retrace', map_path
]
if quiet:
retrace_args.append('--quiet')
if stacktrace:
retrace_args.append(stacktrace)
utils.PrintCmd(retrace_args, quiet=quiet)
return subprocess.call(retrace_args)
def run(map_path, hash_or_version, stacktrace, is_hash, no_r8lib):
if hash_or_version:
download_path = archive.GetUploadDestination(hash_or_version,
'r8lib.jar.map', is_hash)
if utils.file_exists_on_cloud_storage(download_path):
map_path = tempfile.NamedTemporaryFile().name
utils.download_file_from_cloud_storage(download_path, map_path)
else:
print('Could not find map file from argument: %s.' %
hash_or_version)
return 1
retrace_args = [
jdk.GetJavaExecutable(), '-cp',
utils.R8_JAR if no_r8lib else utils.R8LIB_JAR,
'com.android.tools.r8.retrace.Retrace', map_path
]
if stacktrace:
retrace_args.append(stacktrace)
utils.PrintCmd(retrace_args)
return subprocess.call(retrace_args)
def get_magic_file_exists(name):
return utils.file_exists_on_cloud_storage(get_magic_file_gs_path(name))
def run_with_options(options, args, extra_args=None, stdout=None, quiet=False):
if extra_args is None:
extra_args = []
app_provided_pg_conf = False
# todo(121018500): remove when memory is under control
if not any('-Xmx' in arg for arg in extra_args):
if options.max_memory:
extra_args.append('-Xmx%sM' % options.max_memory)
else:
extra_args.append('-Xmx8G')
if options.golem:
golem.link_third_party()
options.out = os.getcwd()
if not options.ignore_java_version:
utils.check_java_version()
if options.print_times:
extra_args.append('-Dcom.android.tools.r8.printtimes=1')
outdir = options.out
(version_id, data) = get_version_and_data(options)
if options.compiler not in COMPILERS:
raise Exception("You need to specify '--compiler={}'".format(
'|'.join(COMPILERS)))
if options.compiler_build not in COMPILER_BUILDS:
raise Exception("You need to specify '--compiler-build={}'".format(
'|'.join(COMPILER_BUILDS)))
if not version_id in data.VERSIONS.keys():
print('No version {} for application {}'.format(
version_id, options.app))
print('Valid versions are {}'.format(data.VERSIONS.keys()))
return 1
version = data.VERSIONS[version_id]
type = get_type(options)
if type not in version:
print('No type {} for version {}'.format(type, version))
print('Valid types are {}'.format(version.keys()))
return 1
values = version[type]
inputs = []
# For R8 'deploy' the JAR is located using the Proguard configuration
# -injars option. For chrome and nest we don't have the injars in the
# proguard files.
if 'inputs' in values and (options.compiler != 'r8' or type != 'deploy'
or options.app == 'chrome'
or options.app == 'nest' or options.app == 'r8'
or options.app == 'iosched'
or options.app == 'tachiyomi'):
inputs = values['inputs']
args.extend(['--output', outdir])
if 'min-api' in values:
args.extend(['--min-api', values['min-api']])
if 'main-dex-list' in values:
args.extend(['--main-dex-list', values['main-dex-list']])
libraries = values['libraries'] if 'libraries' in values else []
if options.compiler == 'r8':
if 'pgconf' in values and not options.k:
if has_injars_and_libraryjars(values['pgconf']):
sanitized_lib_path = os.path.join(os.path.abspath(outdir),
'sanitized_lib.jar')
sanitized_pgconf_path = os.path.join(os.path.abspath(outdir),
'sanitized.config')
SanitizeLibrariesInPgconf(sanitized_lib_path,
sanitized_pgconf_path,
values['pgconf'])
libraries = [sanitized_lib_path]
args.extend(['--pg-conf', sanitized_pgconf_path])
else:
# -injars without -libraryjars or vice versa is not supported.
check_no_injars_and_no_libraryjars(values['pgconf'])
for pgconf in values['pgconf']:
args.extend(['--pg-conf', pgconf])
if 'sanitize_libraries' in values and values[
'sanitize_libraries']:
sanitized_lib_path = os.path.join(os.path.abspath(outdir),
'sanitized_lib.jar')
SanitizeLibraries(sanitized_lib_path, values['libraries'],
values['inputs'])
libraries = [sanitized_lib_path]
inputs = values['inputs']
app_provided_pg_conf = True
if options.k:
args.extend(['--pg-conf', options.k])
if 'maindexrules' in values:
for rules in values['maindexrules']:
args.extend(['--main-dex-rules', rules])
if 'allow-type-errors' in values:
extra_args.append('-Dcom.android.tools.r8.allowTypeErrors=1')
extra_args.append(
'-Dcom.android.tools.r8.disallowClassInlinerGracefulExit=1')
if options.debug_agent:
if not options.compiler_build == 'full':
print(
'WARNING: Running debugging agent on r8lib is questionable...')
extra_args.append(
'-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
)
if not options.no_libraries:
for lib in libraries:
args.extend(['--lib', lib])
if not outdir.endswith('.zip') and not outdir.endswith('.jar') \
and not os.path.exists(outdir):
os.makedirs(outdir)
if options.hash:
# Download r8-<hash>.jar from
# https://storage.googleapis.com/r8-releases/raw/<hash>/.
download_path = archive.GetUploadDestination(options.hash, 'r8.jar',
True)
assert utils.file_exists_on_cloud_storage(download_path), (
'Could not find r8.jar file from provided hash: %s' % options.hash)
destination = os.path.join(utils.LIBS, 'r8-' + options.hash + '.jar')
utils.download_file_from_cloud_storage(download_path,
destination,
quiet=quiet)
# Additional flags for the compiler from the configuration file.
if 'flags' in values:
args.extend(values['flags'].split(' '))
if options.compiler == 'r8':
if 'r8-flags' in values:
args.extend(values['r8-flags'].split(' '))
# Additional flags for the compiler from the command line.
if options.compiler_flags:
args.extend(options.compiler_flags.split(' '))
if options.r8_flags:
args.extend(options.r8_flags.split(' '))
# Feature jars.
features = values['features'] if 'features' in values else []
for i, feature in enumerate(features, start=1):
feature_out = os.path.join(outdir, 'feature-%d.zip' % i)
for feature_jar in feature['inputs']:
args.extend(['--feature', feature_jar, feature_out])
args.extend(inputs)
t0 = time.time()
if options.dump_args_file:
with open(options.dump_args_file, 'w') as args_file:
args_file.writelines([arg + os.linesep for arg in args])
else:
with utils.TempDir() as temp:
if options.print_memoryuse and not options.track_memory_to_file:
options.track_memory_to_file = os.path.join(
temp, utils.MEMORY_USE_TMP_FILE)
if options.compiler == 'r8' and app_provided_pg_conf:
# Ensure that output of -printmapping and -printseeds go to the output
# location and not where the app Proguard configuration places them.
if outdir.endswith('.zip') or outdir.endswith('.jar'):
pg_outdir = os.path.dirname(outdir)
else:
pg_outdir = outdir
if not options.no_extra_pgconf:
additional_pg_conf = GenerateAdditionalProguardConfiguration(
temp, os.path.abspath(pg_outdir))
args.extend(['--pg-conf', additional_pg_conf])
build = not options.no_build and not options.golem
stderr_path = os.path.join(temp, 'stderr')
with open(stderr_path, 'w') as stderr:
jar = None
main = None
if options.compiler_build == 'full':
tool = options.compiler
else:
assert (options.compiler_build == 'lib')
tool = 'r8lib-' + options.compiler
if options.hash:
jar = os.path.join(utils.LIBS,
'r8-' + options.hash + '.jar')
main = 'com.android.tools.r8.' + options.compiler.upper()
exit_code = toolhelper.run(
tool,
args,
build=build,
debug=not options.no_debug,
profile=options.profile,
track_memory_file=options.track_memory_to_file,
extra_args=extra_args,
stdout=stdout,
stderr=stderr,
timeout=options.timeout,
quiet=quiet,
cmd_prefix=['taskset', '-c', options.cpu_list]
if options.cpu_list else [],
jar=jar,
main=main)
if exit_code != 0:
with open(stderr_path) as stderr:
stderr_text = stderr.read()
if not quiet:
print(stderr_text)
if 'java.lang.OutOfMemoryError' in stderr_text:
if not quiet:
print('Failure was OOM')
return OOM_EXIT_CODE
return exit_code
if options.print_memoryuse:
print('{}(MemoryUse): {}'.format(
options.print_memoryuse,
utils.grep_memoryuse(options.track_memory_to_file)))
if options.print_runtimeraw:
print('{}(RunTimeRaw): {} ms'.format(options.print_runtimeraw,
1000.0 * (time.time() - t0)))
if options.print_dexsegments:
dex_files = glob(os.path.join(outdir, '*.dex'))
utils.print_dexsegments(options.print_dexsegments, dex_files)
return 0
