def finalize_redex(state):
state.lib_manager.__exit__(*sys.exc_info())
repack_start_time = timer()
state.unpack_manager.__exit__(*sys.exc_info())
state.zip_manager.__exit__(*sys.exc_info())
align_and_sign_output_apk(
state.zip_manager.output_apk,
state.args.out,
# In dev mode, reset timestamps.
state.args.reset_zip_timestamps or state.args.dev,
state.args.sign,
state.args.keystore,
state.args.keyalias,
state.args.keypass,
state.args.ignore_zipalign,
state.args.page_align_libs,
)
log("Creating output APK finished in {:.2f} seconds".format(
timer() - repack_start_time))
meta_file_dir = join(state.dex_dir, "meta/")
assert os.path.isdir(
meta_file_dir), "meta dir %s does not exist" % meta_file_dir
copy_all_file_to_out_dir(meta_file_dir, state.args.out, "*",
"all redex generated artifacts")
copy_all_file_to_out_dir(state.dex_dir, state.args.out, "*.dot",
"approximate shape graphs")
def find_android_build_tools_by_buck():
def load_android_buckconfig_values():
cmd = ["buck", "audit", "config", "android", "--json"]
global root_dir_for_buck
cwd = root_dir_for_buck if root_dir_for_buck is not None else os.getcwd(
)
# Set NO_BUCKD to minimize disruption to any currently running buckd
env = dict(os.environ)
env["NO_BUCKD"] = "1"
raw = subprocess.check_output(cmd,
cwd=cwd,
stderr=subprocess.DEVNULL,
env=env)
return json.loads(raw)
log("Computing SDK path from buck")
try:
buckconfig = load_android_buckconfig_values()
except BaseException as e:
log("Failed loading buckconfig: %s" % e)
return None
if "android.sdk_path" not in buckconfig:
return None
sdk_path = buckconfig.get("android.sdk_path")
if "android.build_tools_version" in buckconfig:
version = buckconfig["android.build_tools_version"]
assert isinstance(sdk_path, str)
return join(sdk_path, "build-tools", version)
else:
return _find_biggest_build_tools_version(sdk_path)
def copy_file_to_out_dir(tmp, apk_output_path, name, human_name, out_name):
output_dir = os.path.dirname(apk_output_path)
output_path = os.path.join(output_dir, out_name)
tmp_path = tmp + "/" + name
if os.path.isfile(tmp_path):
subprocess.check_call(["cp", tmp_path, output_path])
log("Copying " + human_name + " map to output dir")
else:
log("Skipping " + human_name + " copy, since no file found to copy")
def copy_file_to_out_dir(tmp, apk_output_path, name, human_name, out_name):
output_dir = os.path.dirname(apk_output_path)
output_path = os.path.join(output_dir, out_name)
tmp_path = tmp + '/' + name
if os.path.isfile(tmp_path):
subprocess.check_call(['cp', tmp_path, output_path])
log('Copying ' + human_name + ' map to output dir')
else:
log('Skipping ' + human_name + ' copy, since no file found to copy')
def copy_file_to_out_dir(tmp, apk_output_path, name, human_name, out_name):
output_dir = os.path.dirname(apk_output_path)
output_path = os.path.join(output_dir, out_name)
tmp_path = tmp + '/' + name
if os.path.isfile(tmp_path):
subprocess.check_call(['cp', tmp_path, output_path])
log('Copying ' + human_name + ' map to output dir')
else:
log('Skipping ' + human_name + ' copy, since no file found to copy')
def finalize_redex(state):
state.lib_manager.__exit__(*sys.exc_info())
repack_start_time = timer()
state.unpack_manager.__exit__(*sys.exc_info())
state.zip_manager.__exit__(*sys.exc_info())
align_and_sign_output_apk(
state.zip_manager.output_apk,
state.args.out,
# In dev mode, reset timestamps.
state.args.reset_zip_timestamps or state.args.dev,
state.args.sign,
state.args.keystore,
state.args.keyalias,
state.args.keypass,
state.args.ignore_zipalign,
state.args.page_align_libs,
)
log("Creating output APK finished in {:.2f} seconds".format(
timer() - repack_start_time))
meta_file_dir = join(state.dex_dir, "meta/")
assert os.path.isdir(
meta_file_dir), "meta dir %s does not exist" % meta_file_dir
copy_all_file_to_out_dir(meta_file_dir, state.args.out, "*",
"all redex generated artifacts")
redex_stats_filename = state.config_dict.get("stats_output",
"redex-stats.txt")
redex_stats_file = join(dirname(meta_file_dir), redex_stats_filename)
if isfile(redex_stats_file):
with open(redex_stats_file, "r") as fr:
apk_input_size = getsize(state.args.input_apk)
apk_output_size = getsize(state.args.out)
redex_stats_json = json.load(fr)
redex_stats_json["input_stats"]["total_stats"][
"num_compressed_apk_bytes"] = apk_input_size
redex_stats_json["output_stats"]["total_stats"][
"num_compressed_apk_bytes"] = apk_output_size
update_redex_stats_file = join(dirname(state.args.out),
redex_stats_filename)
with open(update_redex_stats_file, "w") as fw:
json.dump(redex_stats_json, fw)
# Write invocation file
with open(join(dirname(state.args.out), "redex.py-invocation.txt"),
"w") as f:
print("%s" % " ".join(map(shlex.quote, sys.argv)), file=f)
copy_all_file_to_out_dir(state.dex_dir, state.args.out, "*.dot",
"approximate shape graphs")
def finalize_redex(state):
# This dir was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if state.temporary_libs_dir is not None:
shutil.rmtree(state.temporary_libs_dir)
repack_start_time = timer()
log("Repacking dex files")
have_locators = state.config_dict.get("emit_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
state.dex_mode.repackage(
get_dex_file_path(state.args, state.extracted_apk_dir),
state.dex_dir,
have_locators,
fast_repackage=state.args.dev,
)
locator_store_id = 1
for module in state.application_modules:
log("repacking module: " + module.get_name() + " with id " +
str(locator_store_id))
module.repackage(
state.extracted_apk_dir,
state.dex_dir,
have_locators,
locator_store_id,
fast_repackage=state.args.dev,
)
locator_store_id = locator_store_id + 1
log("Creating output apk")
create_output_apk(
state.extracted_apk_dir,
state.args.out,
state.args.sign,
state.args.keystore,
state.args.keyalias,
state.args.keypass,
state.args.ignore_zipalign,
state.args.page_align_libs,
)
log("Creating output APK finished in {:.2f} seconds".format(
timer() - repack_start_time))
meta_file_dir = join(state.dex_dir, "meta/")
assert os.path.isdir(
meta_file_dir), "meta dir %s does not exist" % meta_file_dir
copy_all_file_to_out_dir(meta_file_dir, state.args.out, "*",
"all redex generated artifacts")
copy_all_file_to_out_dir(state.dex_dir, state.args.out, "*.dot",
"approximate shape graphs")
def copy_file_to_out_dir(tmp, apk_output_path, name, human_name, out_name):
output_dir = os.path.dirname(apk_output_path)
output_path = os.path.join(output_dir, out_name)
tmp_path = tmp + "/" + name
if os.path.isfile(tmp_path):
shutil.copy2(tmp_path, output_path)
log("Copying " + human_name + " map to output dir")
logging.warning("Copying " + human_name + " map to output_dir: " + output_path)
else:
log("Skipping " + human_name + " copy, since no file found to copy")
logging.warning("Skipping " + human_name + " copy, since no file found to copy")
def __enter__(self):
dex_file_path = self.get_dex_file_path(self.input_apk,
self.extracted_apk_dir)
self.dex_mode = pyredex.unpacker.detect_secondary_dex_mode(
dex_file_path)
log("Detected dex mode " + str(type(self.dex_mode).__name__))
log("Unpacking dex files")
self.dex_mode.unpackage(dex_file_path, self.dex_dir)
log("Detecting Application Modules")
store_metadata_dir = make_temp_dir(".application_module_metadata",
self.debug_mode)
self.application_modules = pyredex.unpacker.ApplicationModule.detect(
self.extracted_apk_dir)
store_files = []
for module in self.application_modules:
canary_prefix = module.get_canary_prefix()
log("found module: " + module.get_name() + " " +
(canary_prefix
if canary_prefix is not None else "(no canary prefix)"))
store_path = os.path.join(self.dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(self.extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir,
module.get_name() + ".json")
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
return store_files
def overwrite_proguard_maps(redex_rename_map_path, apk_output_path, dex_dir,
pg_file, redex_pg_out_filename):
log('running overwrite proguard step')
redex_rename_map_path = join(dex_dir, redex_rename_map_path)
log('redex map is at ' + str(redex_rename_map_path))
log('pg map is at ' + str(pg_file))
assert os.path.isfile(redex_rename_map_path)
output_dir = os.path.dirname(apk_output_path)
output_file = join(output_dir, redex_pg_out_filename)
log('wrote redex pg format mapping file to ' + str(output_file))
shutil.move(redex_rename_map_path, output_file)
def __exit__(self, *args):
remove_signature_files(self.extracted_apk_dir)
if isfile(self.output_apk):
os.remove(self.output_apk)
log("Creating output apk")
with zipfile.ZipFile(self.output_apk, "w") as new_apk:
# Need sorted output for deterministic zip file. Sorting `dirnames` will
# ensure the tree walk order. Sorting `filenames` will ensure the files
# inside the tree.
# This scheme uses less memory than collecting all files first.
for dirpath, dirnames, filenames in os.walk(self.extracted_apk_dir):
dirnames.sort()
for filename in sorted(filenames):
filepath = join(dirpath, filename)
archivepath = filepath[len(self.extracted_apk_dir) + 1 :]
try:
compress = self.per_file_compression[archivepath]
except KeyError:
compress = zipfile.ZIP_DEFLATED
new_apk.write(filepath, archivepath, compress_type=compress)
def unpackage(self, extracted_apk_dir, dex_dir, unpackage_metadata=False):
self.dex_mode = XZSDexMode(
dex_asset_dir=self.path,
store_name=self.name,
dex_prefix=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies,
)
if self.dex_mode.detect(extracted_apk_dir):
log("module " + self.name + " is XZSDexMode")
self.dex_mode.unpackage(extracted_apk_dir, dex_dir, unpackage_metadata)
else:
self.dex_mode = SubdirDexMode(
dex_asset_dir=self.path,
store_name=self.name,
dex_prefix=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies,
)
if self.dex_mode.detect(extracted_apk_dir):
log("module " + self.name + " is SubdirDexMode")
self.dex_mode.unpackage(extracted_apk_dir, dex_dir, unpackage_metadata)
else:
self.dex_mode = Api21ModuleDexMode(
dex_asset_dir=self.path,
store_name=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies,
)
log("module " + self.name + " is Api21ModuleDexMode")
self.dex_mode.unpackage(extracted_apk_dir, dex_dir, unpackage_metadata)
def __exit__(self, *args: typing.Any) -> None:
log("Repacking dex files")
log("Emit Locator Strings: %s" % self.have_locators)
dex_mode = self.dex_mode
assert dex_mode is not None
dex_mode.repackage(
self.extracted_apk_dir,
self.dex_dir,
self.have_locators,
locator_store_id=0,
fast_repackage=self.fast_repackage,
reset_timestamps=self.reset_timestamps,
)
locator_store_id = 1
for module in self.application_modules:
log("repacking module: " + module.get_name() + " with id " +
str(locator_store_id))
module.repackage(
self.extracted_apk_dir,
self.dex_dir,
self.have_locators,
locator_store_id,
fast_repackage=self.fast_repackage,
reset_timestamps=self.reset_timestamps,
)
locator_store_id = locator_store_id + 1
def unpackage(self, extracted_apk_dir, dex_dir):
self.dex_mode = XZSDexMode(dex_asset_dir=self.path,
store_name=self.name,
dex_prefix=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies)
if (self.dex_mode.detect(extracted_apk_dir)):
log('module ' + self.name + ' is XZSDexMode')
self.dex_mode.unpackage(extracted_apk_dir, dex_dir)
else:
self.dex_mode = SubdirDexMode(dex_asset_dir=self.path,
store_name=self.name,
dex_prefix=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies)
if (self.dex_mode.detect(extracted_apk_dir)):
log('module ' + self.name + ' is SubdirDexMode')
self.dex_mode.unpackage(extracted_apk_dir, dex_dir)
else:
self.dex_mode = Api21ModuleDexMode(dex_asset_dir=self.path,
store_name=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies)
log('module ' + self.name + ' is Api21ModuleDexMode')
self.dex_mode.unpackage(extracted_apk_dir, dex_dir)
def __enter__(self) -> typing.List[str]:
dex_mode = detect_secondary_dex_mode(self.extracted_apk_dir,
self.is_bundle)
self.dex_mode = dex_mode
log("Unpacking dex files")
dex_mode.unpackage(self.extracted_apk_dir, self.dex_dir)
log("Detecting Application Modules")
store_metadata_dir = make_temp_dir(".application_module_metadata",
self.debug_mode)
self.application_modules = ApplicationModule.detect(
self.extracted_apk_dir, self.is_bundle)
store_files = []
for module in self.application_modules:
canary_prefix = module.get_canary_prefix()
log("found module: " + module.get_name() + " " +
(canary_prefix
if canary_prefix is not None else "(no canary prefix)"))
store_path = os.path.join(self.dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(self.extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir,
module.get_name() + ".json")
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
return store_files
def __exit__(self, *args):
log("Repacking dex files")
log("Emit Locator Strings: %s" % self.have_locators)
self.dex_mode.repackage(
self.get_dex_file_path(self.input_apk, self.extracted_apk_dir),
self.dex_dir,
self.have_locators,
fast_repackage=self.fast_repackage,
reset_timestamps=self.reset_timestamps,
)
locator_store_id = 1
for module in self.application_modules:
log("repacking module: " + module.get_name() + " with id " +
str(locator_store_id))
module.repackage(
self.extracted_apk_dir,
self.dex_dir,
self.have_locators,
locator_store_id,
fast_repackage=self.fast_repackage,
reset_timestamps=self.reset_timestamps,
)
locator_store_id = locator_store_id + 1
def overwrite_proguard_maps(
redex_rename_map_path,
apk_output_path,
dex_dir,
pg_file):
log('running overwrite proguard step')
redex_rename_map_path = join(dex_dir, redex_rename_map_path)
log('redex map is at ' + str(redex_rename_map_path))
log('pg map is at ' + str(pg_file))
assert os.path.isfile(redex_rename_map_path)
redex_pg_file = "redex-class-rename-map.txt"
output_dir = os.path.dirname(apk_output_path)
output_file = join(output_dir, redex_pg_file)
log('wrote redex pg format mapping file to ' + str(output_file))
shutil.move(redex_rename_map_path, output_file)
def unpackage(self,
extracted_apk_dir: str,
dex_dir: str,
unpackage_metadata: bool = False) -> None:
dex_mode = XZSDexMode(
secondary_dir=self.path,
store_name=self.name,
dex_prefix=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies,
)
if dex_mode.detect(extracted_apk_dir):
self.dex_mode = dex_mode
log("module " + self.name + " is XZSDexMode")
dex_mode.unpackage(extracted_apk_dir, dex_dir, unpackage_metadata)
return
dex_mode = SubdirDexMode(
secondary_dir=self.path,
store_name=self.name,
dex_prefix=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies,
)
if dex_mode.detect(extracted_apk_dir):
self.dex_mode = dex_mode
log("module " + self.name + " is SubdirDexMode")
dex_mode.unpackage(extracted_apk_dir, dex_dir, unpackage_metadata)
return
dex_mode = Api21ModuleDexMode(
secondary_dir=self.path,
store_name=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies,
)
self.dex_mode = dex_mode
log("module " + self.name + " is Api21ModuleDexMode")
dex_mode.unpackage(extracted_apk_dir, dex_dir, unpackage_metadata)
def run_redex(args):
debug_mode = args.unpack_only or args.debug
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
passes_list = config_dict['redex']['passes']
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
libs_dir = join(extracted_apk_dir, 'lib')
temporary_lib_file = join(libs_dir, 'concated_native_libs.so')
if os.path.exists(xz_compressed_libs) and os.path.exists(libs_dir):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.external_summaries="%s"' % args.side_effect_summaries
)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
run_pass(binary,
args,
config,
config_dict,
extracted_apk_dir,
dex_dir,
dexen,
debugger)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
dex_mode.repackage(
extracted_apk_dir, dex_dir, have_locators, fast_repackage=args.dev
)
locator_store_id = 1
for module in application_modules:
log('repacking module: ' + module.get_name() + ' with id ' + str(locator_store_id))
module.repackage(
extracted_apk_dir, dex_dir, have_locators, locator_store_id,
fast_repackage=args.dev
)
locator_store_id = locator_store_id + 1
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass, args.ignore_zipalign, args.page_align_libs)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map-v2', 'line number map v2', 'redex-line-number-map-v2')
copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'outliner-artifacts.bin', 'outliner artifacts', 'redex-outliner-artifacts.bin')
copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt', 'class id map', 'redex-class-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'bytecode_offset_map.txt', 'bytecode offset map', 'redex-bytecode-offset-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'coldstart_fields_in_R_classes.txt', 'resources accessed during coldstart', 'redex-tracked-coldstart-resources.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_dependencies.txt', 'stats', 'redex-class-dependencies.txt')
copy_file_to_out_dir(dex_dir, args.out, 'resid-optres-mapping.json', 'resid map after optres pass', 'redex-resid-optres-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'resid-dedup-mapping.json', 'resid map after dedup pass', 'redex-resid-dedup-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'resid-splitres-mapping.json', 'resid map after split pass', 'redex-resid-splitres-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'type-erasure-mappings.txt', 'class map after type erasure pass', 'redex-type-erasure-mappings.txt')
copy_file_to_out_dir(dex_dir, args.out, 'instrument-methods-idx.txt', 'instrumented methods id map', 'redex-instrument-methods-idx.txt')
copy_file_to_out_dir(dex_dir, args.out, 'cleanup-removed-classes.txt', 'cleanup removed classes', 'redex-cleanup-removed-classes.txt')
if config_dict.get('proguard_map_output', '') != '':
# if our map output strategy is overwrite, we don't merge at all
# if you enable ObfuscatePass, this needs to be overwrite
if config_dict.get('proguard_map_output_strategy', 'merge') == 'overwrite':
overwrite_proguard_maps(
config_dict['proguard_map_output'],
args.out,
dex_dir,
args.proguard_map)
else:
merge_proguard_maps(
config_dict['proguard_map_output'],
args.input_apk,
args.out,
dex_dir,
args.proguard_map)
else:
assert 'RenameClassesPass' not in passes_list and\
'RenameClassesPassV2' not in passes_list
remove_temp_dirs()
def run_redex_binary(state):
if state.args.redex_binary is None:
try:
state.args.redex_binary = subprocess.check_output(
['which', 'redex-all']).rstrip().decode('ascii')
except subprocess.CalledProcessError:
pass
if state.args.redex_binary is None:
# __file__ can be /path/fb-redex.pex/redex.pyc
dir_name = dirname(abspath(__file__))
while not isdir(dir_name):
dir_name = dirname(dir_name)
state.args.redex_binary = join(dir_name, 'redex-all')
if not isfile(state.args.redex_binary) or not os.access(
state.args.redex_binary, os.X_OK):
sys.exit('redex-all is not found or is not executable: ' +
state.args.redex_binary)
log('Running redex binary at ' + state.args.redex_binary)
args = [state.args.redex_binary] + [
'--apkdir', state.extracted_apk_dir, '--outdir', state.dex_dir
]
if state.args.config:
args += ['--config', state.args.config]
if state.args.verify_none_mode or state.config_dict.get(
"verify_none_mode"):
args += ['--verify-none-mode']
if state.args.is_art_build:
args += ['--is-art-build']
if state.args.enable_instrument_pass or state.config_dict.get(
"enable_instrument_pass"):
args += ['--enable-instrument-pass']
if state.args.warn:
args += ['--warn', state.args.warn]
args += ['--proguard-config=' + x for x in state.args.proguard_configs]
if state.args.proguard_map:
args += ['-Sproguard_map=' + state.args.proguard_map]
args += ['--jarpath=' + x for x in state.args.jarpaths]
if state.args.printseeds:
args += ['--printseeds=' + state.args.printseeds]
if state.args.used_js_assets:
args += ['--used-js-assets=' + x for x in state.args.used_js_assets]
if state.args.arch:
args += ['--arch=' + state.args.arch]
args += ['-S' + x for x in state.args.passthru]
args += ['-J' + x for x in state.args.passthru_json]
args += state.dexen
# Stop before a pass and output intermediate dex and IR meta data.
if state.stop_pass_idx != -1:
args += [
'--stop-pass',
str(state.stop_pass_idx), '--output-ir', state.args.output_ir
]
if state.debugger == 'lldb':
args = ['lldb', '--'] + args
elif state.debugger == 'gdb':
args = ['gdb', '--args'] + args
start = timer()
if state.args.debug:
print('cd %s && %s' % (os.getcwd(), ' '.join(map(quote, args))))
sys.exit()
env = logger.setup_trace_for_child(os.environ)
logger.flush()
add_extra_environment_args(env)
# Our CI system occasionally fails because it is trying to write the
# redex-all binary when this tries to run. This shouldn't happen, and
# might be caused by a JVM bug. Anyways, let's retry and hope it stops.
for i in range(5):
try:
subprocess.check_call(args, env=env)
except OSError as err:
if err.errno == errno.ETXTBSY:
if i < 4:
time.sleep(5)
continue
raise err
except subprocess.CalledProcessError as err:
script_filenames = write_debugger_commands(args)
raise RuntimeError(
('redex-all crashed with exit code %s! ' % err.returncode) +
('You can re-run it '
'under gdb by running %(gdb_script_name)s or under lldb '
'by running %(lldb_script_name)s') % script_filenames)
break
log('Dex processing finished in {:.2f} seconds'.format(timer() - start))
def prepare_redex(args):
debug_mode = args.unpack_only or args.debug
# avoid accidentally mixing up file formats since we now support
# both apk files and Android bundle files
if not args.unpack_only:
assert get_file_ext(args.input_apk) == get_file_ext(
args.out), ('Input file extension ("' +
get_file_ext(args.input_apk) +
'") should be the same as output file extension ("' +
get_file_ext(args.out) + '")')
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == ".":
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + ".redex_extracted_apk"
dex_dir = unpack_dir_basename + ".redex_dexen"
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print("Error: destination directory already exists!")
print("APK: " + extracted_apk_dir)
print("DEX: " + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log("Using config " + (config if config is not None else "(default)"))
log("Using binary " + (binary if binary is not None else "(default)"))
if args.unpack_only or config is None:
config_dict = {}
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
# stop_pass_idx >= 0 means need stop before a pass and dump intermediate result
stop_pass_idx = -1
if args.stop_pass:
passes_list = config_dict.get("redex", {}).get("passes", [])
stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass)
if not args.output_ir or isfile(args.output_ir):
print("Error: output_ir should be a directory")
sys.exit(1)
try:
os.makedirs(args.output_ir)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir(".redex_extracted_apk", debug_mode)
directory = make_temp_dir(".redex_unaligned", False)
unaligned_apk_path = join(directory, "redex-unaligned.apk")
zip_manager = ZipManager(args.input_apk, extracted_apk_dir,
unaligned_apk_path)
zip_manager.__enter__()
if not dex_dir:
dex_dir = make_temp_dir(".redex_dexen", debug_mode)
unpack_manager = UnpackManager(
args.input_apk,
extracted_apk_dir,
dex_dir,
have_locators=config_dict.get("emit_locator_strings"),
debug_mode=debug_mode,
fast_repackage=args.dev,
)
store_files = unpack_manager.__enter__()
lib_manager = LibraryManager(extracted_apk_dir)
lib_manager.__enter__()
if args.unpack_only:
print("APK: " + extracted_apk_dir)
print("DEX: " + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in sorted(store_files):
dexen.append(store)
log("Unpacking APK finished in {:.2f} seconds".format(timer() -
unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'ObjectSensitiveDcePass.side_effect_summaries="%s"' %
args.side_effect_summaries)
if args.escape_summaries is not None:
args.passthru_json.append(
'ObjectSensitiveDcePass.escape_summaries="%s"' %
args.escape_summaries)
for key_value_str in args.passthru_json:
key_value = key_value_str.split("=", 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" %
(key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" %
(key, value, key_value_str, prev_value))
config_dict[key] = json.loads(value)
log("Running redex-all on {} dex files ".format(len(dexen)))
if args.lldb:
debugger = "lldb"
elif args.gdb:
debugger = "gdb"
else:
debugger = None
return State(
args=args,
config_dict=config_dict,
debugger=debugger,
dex_dir=dex_dir,
dexen=dexen,
extracted_apk_dir=extracted_apk_dir,
stop_pass_idx=stop_pass_idx,
lib_manager=lib_manager,
unpack_manager=unpack_manager,
zip_manager=zip_manager,
)
def run_redex_binary(state, term_handler):
if state.args.redex_binary is None:
state.args.redex_binary = shutil.which("redex-all")
if state.args.redex_binary is None:
# __file__ can be /path/fb-redex.pex/redex.pyc
dir_name = dirname(abspath(__file__))
while not isdir(dir_name):
dir_name = dirname(dir_name)
state.args.redex_binary = join(dir_name, "redex-all")
if not isfile(state.args.redex_binary) or not os.access(
state.args.redex_binary, os.X_OK):
sys.exit("redex-all is not found or is not executable: " +
state.args.redex_binary)
log("Running redex binary at " + state.args.redex_binary)
args = [state.args.redex_binary] + [
"--apkdir",
state.extracted_apk_dir,
"--outdir",
state.dex_dir,
]
if state.args.cmd_prefix is not None:
args = state.args.cmd_prefix.split() + args
if state.args.config:
args += ["--config", state.args.config]
if state.args.verify_none_mode or state.config_dict.get(
"verify_none_mode"):
args += ["--verify-none-mode"]
if state.args.is_art_build:
args += ["--is-art-build"]
if state.args.redacted:
args += ["--redacted"]
if state.args.disable_dex_hasher:
args += ["--disable-dex-hasher"]
if state.args.enable_instrument_pass or state.config_dict.get(
"enable_instrument_pass"):
args += ["--enable-instrument-pass"]
if state.args.warn:
args += ["--warn", state.args.warn]
args += ["--proguard-config=" + x for x in state.args.proguard_configs]
if state.args.proguard_map:
args += ["-Sproguard_map=" + state.args.proguard_map]
args += ["--jarpath=" + x for x in state.args.jarpaths]
if state.args.printseeds:
args += ["--printseeds=" + state.args.printseeds]
if state.args.used_js_assets:
args += ["--used-js-assets=" + x for x in state.args.used_js_assets]
if state.args.arch:
args += ["--arch=" + state.args.arch]
args += ["-S" + x for x in state.args.passthru]
args += ["-J" + x for x in state.args.passthru_json]
args += state.dexen
# Stop before a pass and output intermediate dex and IR meta data.
if state.stop_pass_idx != -1:
args += [
"--stop-pass",
str(state.stop_pass_idx),
"--output-ir",
state.args.output_ir,
]
prefix = (dbg_prefix(state.debugger, state.args.debug_source_root)
if state.debugger is not None else [])
start = timer()
if state.args.debug:
print("cd %s && %s" %
(os.getcwd(), " ".join(prefix + list(map(quote, args)))))
sys.exit()
env = logger.setup_trace_for_child(os.environ)
logger.flush()
add_extra_environment_args(env)
def run():
sigint_handler = SigIntHandler()
sigint_handler.install()
try:
proc, handler = run_and_stream_stderr(prefix + args, env,
(logger.trace_fp.fileno(), ))
sigint_handler.set_proc(proc)
sigint_handler.set_state(RedexState.STARTED)
returncode, err_out = handler()
sigint_handler.set_state(RedexState.POSTPROCESSING)
if returncode != 0:
# Check for crash traces.
maybe_addr2line(err_out)
if returncode == -6: # SIGABRT
maybe_reprint_error(err_out, term_handler)
gdb_script_name = write_debugger_command(
"gdb", state.args.debug_source_root, args)
lldb_script_name = write_debugger_command(
"lldb", state.args.debug_source_root, args)
raise RuntimeError((
"redex-all crashed with exit code {}! You can re-run it " +
"under gdb by running {} or under lldb by running {}"
).format(returncode, gdb_script_name, lldb_script_name))
return True
except OSError as err:
if err.errno == errno.ETXTBSY:
return False
raise err
finally:
sigint_handler.set_state(RedexState.FINISHED)
sigint_handler.uninstall()
# Our CI system occasionally fails because it is trying to write the
# redex-all binary when this tries to run. This shouldn't happen, and
# might be caused by a JVM bug. Anyways, let's retry and hope it stops.
for _ in range(5):
if run():
break
log("Dex processing finished in {:.2f} seconds".format(timer() - start))
def finalize_redex(state):
# This dir was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if state.temporary_libs_dir is not None:
shutil.rmtree(state.temporary_libs_dir)
repack_start_time = timer()
log('Repacking dex files')
have_locators = state.config_dict.get("emit_locator_strings")
have_name_based_locators = state.config_dict.get("emit_name_based_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
log("Emit Name Based Locator Strings: %s" % have_name_based_locators)
state.dex_mode.repackage(
state.extracted_apk_dir, state.dex_dir, have_locators, have_name_based_locators, fast_repackage=state.args.dev
)
locator_store_id = 1
for module in state.application_modules:
log('repacking module: ' + module.get_name() +
' with id ' + str(locator_store_id))
module.repackage(
state.extracted_apk_dir, state.dex_dir, have_locators, have_name_based_locators, locator_store_id,
fast_repackage=state.args.dev
)
locator_store_id = locator_store_id + 1
log('Creating output apk')
create_output_apk(state.extracted_apk_dir, state.args.out, state.args.sign, state.args.keystore,
state.args.keyalias, state.args.keypass, state.args.ignore_zipalign, state.args.page_align_libs)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(state.dex_dir, state.args.out,
'redex-line-number-map', 'line number map', 'redex-line-number-map')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'redex-line-number-map-v2',
'line number map v2', 'redex-line-number-map-v2')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'stats.txt', 'stats', 'redex-stats.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'filename_mappings.txt',
'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'outliner-artifacts.bin',
'outliner artifacts', 'redex-outliner-artifacts.bin')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'class_mapping.txt', 'class id map', 'redex-class-id-map.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'bytecode_offset_map.txt',
'bytecode offset map', 'redex-bytecode-offset-map.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'coldstart_fields_in_R_classes.txt',
'resources accessed during coldstart', 'redex-tracked-coldstart-resources.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'class_dependencies.txt', 'stats', 'redex-class-dependencies.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'resid-optres-mapping.json',
'resid map after optres pass', 'redex-resid-optres-mapping.json')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'resid-dedup-mapping.json',
'resid map after dedup pass', 'redex-resid-dedup-mapping.json')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'resid-splitres-mapping.json',
'resid map after split pass', 'redex-resid-splitres-mapping.json')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'type-erasure-mappings.txt',
'class map after type erasure pass', 'redex-type-erasure-mappings.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'instrument-metadata.txt',
'metadata file for instrumentation', 'redex-instrument-metadata.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'cleanup-removed-classes.txt',
'cleanup removed classes', 'redex-cleanup-removed-classes.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'unreachable-removed-symbols.txt',
'unreachable removed symbols', 'redex-unreachable-removed-symbols.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'opt-decisions.json', 'opt info', 'redex-opt-decisions.json')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'redex-debug-line-map-v2',
'debug method id map', 'redex-debug-line-map-v2')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'class-method-info-map.txt',
'class method info map', 'redex-class-method-info-map.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'cfp-unsafe-references.txt',
'cfp unsafe references', 'redex-cfp-unsafe-references.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'merge-interface-mappings.txt',
'merged interface to merger interface', 'redex-merge-interface-mappings.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'reachability-graph',
'reachability graph', 'redex-reachability-graph')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'method-override-graph',
'method override graph', 'redex-method-override-graph')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'iodi-metadata',
'iodi metadata', 'iodi-metadata')
copy_all_file_to_out_dir(
state.dex_dir, state.args.out, '*.dot', 'approximate shape graphs')
if state.config_dict.get('proguard_map_output', '') != '':
# if our map output strategy is overwrite, we don't merge at all
# if you enable ObfuscatePass, this needs to be overwrite
if state.config_dict.get('proguard_map_output_strategy', 'merge') == 'overwrite':
overwrite_proguard_maps(
state.config_dict['proguard_map_output'],
state.args.out,
state.dex_dir,
state.args.proguard_map)
else:
merge_proguard_maps(
state.config_dict['proguard_map_output'],
state.args.input_apk,
state.args.out,
state.dex_dir,
state.args.proguard_map)
else:
passes_list = state.config_dict.get('redex', {}).get('passes', [])
assert 'RenameClassesPass' not in passes_list and\
'RenameClassesPassV2' not in passes_list
def finalize_redex(state):
state.lib_manager.__exit__(*sys.exc_info())
repack_start_time = timer()
state.unpack_manager.__exit__(*sys.exc_info())
state.zip_manager.__exit__(*sys.exc_info())
align_and_sign_output_apk(
state.zip_manager.output_apk,
state.args.out,
# In dev mode, reset timestamps.
state.args.reset_zip_timestamps or state.args.dev,
state.args.sign,
state.args.keystore,
state.args.keyalias,
state.args.keypass,
state.args.ignore_zipalign,
state.args.page_align_libs,
)
log(
"Creating output APK finished in {:.2f} seconds".format(
timer() - repack_start_time
)
)
meta_file_dir = join(state.dex_dir, "meta/")
assert os.path.isdir(meta_file_dir), "meta dir %s does not exist" % meta_file_dir
copy_all_file_to_out_dir(
meta_file_dir, state.args.out, "*", "all redex generated artifacts"
)
if state.args.enable_instrument_pass:
log("Creating redex-instrument-metadata.zip")
zipfile_path = join(dirname(state.args.out), "redex-instrument-metadata.zip")
FILES = [
join(dirname(state.args.out), f)
for f in (
"redex-instrument-metadata.txt",
"redex-source-block-method-dictionary.csv",
"redex-source-blocks.csv",
)
]
# Write a checksum file.
hash = hashlib.md5()
for f in FILES:
hash.update(open(f, "rb").read())
checksum_path = join(dirname(state.args.out), "redex-instrument-checksum.txt")
with open(checksum_path, "w") as f:
f.write(f"{hash.hexdigest()}\n")
with zipfile.ZipFile(zipfile_path, "w", compression=zipfile.ZIP_DEFLATED) as z:
for f in [*FILES, checksum_path]:
z.write(f, os.path.basename(f))
for f in [*FILES, checksum_path]:
os.remove(f)
redex_stats_filename = state.config_dict.get("stats_output", "redex-stats.txt")
redex_stats_file = join(dirname(meta_file_dir), redex_stats_filename)
if isfile(redex_stats_file):
with open(redex_stats_file, "r") as fr:
apk_input_size = getsize(state.args.input_apk)
apk_output_size = getsize(state.args.out)
redex_stats_json = json.load(fr)
redex_stats_json["input_stats"]["total_stats"][
"num_compressed_apk_bytes"
] = apk_input_size
redex_stats_json["output_stats"]["total_stats"][
"num_compressed_apk_bytes"
] = apk_output_size
update_redex_stats_file = join(
dirname(state.args.out), redex_stats_filename
)
with open(update_redex_stats_file, "w") as fw:
json.dump(redex_stats_json, fw)
# Write invocation file
with open(join(dirname(state.args.out), "redex.py-invocation.txt"), "w") as f:
print("%s" % " ".join(map(shlex.quote, sys.argv)), file=f)
copy_all_file_to_out_dir(
state.dex_dir, state.args.out, "*.dot", "approximate shape graphs"
)
def merge_proguard_maps(
redex_rename_map_path,
input_apk_path,
apk_output_path,
dex_dir,
pg_file):
log('running merge proguard step')
redex_rename_map_path = join(dex_dir, redex_rename_map_path)
log('redex map is at ' + str(redex_rename_map_path))
log('pg map is at ' + str(pg_file))
assert os.path.isfile(redex_rename_map_path)
redex_pg_file = "redex-class-rename-map.txt"
output_dir = os.path.dirname(apk_output_path)
output_file = join(output_dir, redex_pg_file)
# If -dontobfuscate is set, proguard won't produce a mapping file, but
# buck will create an empty mapping.txt. Check for this case.
if pg_file and os.path.getsize(pg_file) > 0:
update_proguard_mapping_file(
pg_file,
redex_rename_map_path,
output_file)
log('merging proguard map with redex class rename map')
log('pg mapping file input is ' + str(pg_file))
log('wrote redex pg format mapping file to ' + str(output_file))
else:
log('no proguard map file found')
shutil.move(redex_rename_map_path, output_file)
def run_redex(args):
debug_mode = args.unpack_only or args.debug
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
try:
config_dict = json.load(config_file)
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
passes_list = config_dict['redex']['passes']
unpack_start_time = timer()
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata',
debug_mode)
for module in application_modules:
log('found module: ' + module.get_name() + ' ' +
module.get_canary_prefix())
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir,
module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
temporary_lib_file = join(extracted_apk_dir, 'lib/concated_native_libs.so')
if os.path.exists(xz_compressed_libs):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs,
temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(timer() -
unpack_start_time))
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" %
(key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
log("Got Override %s = %s from %s. Previous %s" %
(key, value, key_value_str, config_dict[key]))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
run_pass(binary, args, config, config_dict, extracted_apk_dir, dex_dir,
dexen, debugger)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
dex_mode.repackage(extracted_apk_dir, dex_dir, have_locators)
locator_store_id = 1
for module in application_modules:
log('repacking module: ' + module.get_name() + ' with id ' +
str(locator_store_id))
module.repackage(extracted_apk_dir, dex_dir, have_locators,
locator_store_id)
locator_store_id = locator_store_id + 1
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map',
'line number map', 'redex-line-number-map')
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map-v2',
'line number map v2', 'redex-line-number-map-v2')
copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats',
'redex-stats.txt')
copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt',
'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'outliner-artifacts.bin',
'outliner artifacts', 'redex-outliner-artifacts.bin')
copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt',
'method id map', 'redex-method-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt',
'class id map', 'redex-class-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out,
'coldstart_fields_in_R_classes.txt',
'resources accessed during coldstart',
'redex-tracked-coldstart-resources.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_dependencies.txt', 'stats',
'redex-class-dependencies.txt')
if config_dict.get('proguard_map_output', '') != '':
merge_proguard_maps(config_dict['proguard_map_output'], args.input_apk,
args.out, dex_dir, args.proguard_map)
else:
assert 'RenameClassesPass' not in passes_list and\
'RenameClassesPassV2' not in passes_list
remove_temp_dirs()
def run_pass(
executable_path,
script_args,
config_path,
config_json,
apk_dir,
dex_dir,
dexfiles,
debugger,
):
if executable_path is None:
try:
executable_path = subprocess.check_output(['which', 'redex-all']
).rstrip().decode('ascii')
except subprocess.CalledProcessError:
pass
if executable_path is None:
# __file__ can be /path/fb-redex.pex/redex.pyc
dir_name = dirname(abspath(__file__))
while not isdir(dir_name):
dir_name = dirname(dir_name)
executable_path = join(dir_name, 'redex-all')
if not isfile(executable_path) or not os.access(executable_path, os.X_OK):
sys.exit('redex-all is not found or is not executable: ' + executable_path)
log('Running redex binary at ' + executable_path)
args = [executable_path] + [
'--apkdir', apk_dir,
'--outdir', dex_dir]
if config_path:
args += ['--config', config_path]
if script_args.verify_none_mode or config_json.get("verify_none_mode"):
args += ['--verify-none-mode']
if script_args.is_art_build:
args += ['--is-art-build']
if script_args.warn:
args += ['--warn', script_args.warn]
args += ['--proguard-config=' + x for x in script_args.proguard_configs]
if script_args.proguard_map:
args += ['-Sproguard_map=' + script_args.proguard_map]
args += ['--jarpath=' + x for x in script_args.jarpaths]
if script_args.printseeds:
args += ['--printseeds=' + script_args.printseeds]
args += ['-S' + x for x in script_args.passthru]
args += ['-J' + x for x in script_args.passthru_json]
args += dexfiles
if debugger == 'lldb':
args = ['lldb', '--'] + args
elif debugger == 'gdb':
args = ['gdb', '--args'] + args
start = timer()
if script_args.debug:
print(' '.join(map(quote, args)))
sys.exit()
env = logger.setup_trace_for_child(os.environ)
logger.flush()
add_extra_environment_args(env)
# Our CI system occasionally fails because it is trying to write the
# redex-all binary when this tries to run. This shouldn't happen, and
# might be caused by a JVM bug. Anyways, let's retry and hope it stops.
for i in range(5):
try:
subprocess.check_call(args, env=env)
except OSError as err:
if err.errno == errno.ETXTBSY:
if i < 4:
time.sleep(5)
continue
raise err
except subprocess.CalledProcessError as err:
script_filenames = write_debugger_commands(args)
raise RuntimeError(
('redex-all crashed with exit code %s! ' % err.returncode) +
('You can re-run it '
'under gdb by running %(gdb_script_name)s or under lldb '
'by running %(lldb_script_name)s') % script_filenames)
break
log('Dex processing finished in {:.2f} seconds'.format(timer() - start))
def prepare_redex(args):
debug_mode = args.unpack_only or args.debug
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
# stop_pass_idx >= 0 means need stop before a pass and dump intermediate result
stop_pass_idx = -1
if args.stop_pass:
passes_list = config_dict.get('redex', {}).get('passes', [])
stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass)
if not args.output_ir or isfile(args.output_ir):
print('Error: output_ir should be a directory')
sys.exit(1)
try:
os.makedirs(args.output_ir)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir(
'.application_module_metadata', debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' +
(canary_prefix if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(
store_metadata_dir, module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
libs_to_extract = []
temporary_libs_dir = None
xz_lib_name = 'libs.xzs'
zstd_lib_name = 'libs.zstd'
for root, _, filenames in os.walk(extracted_apk_dir):
for filename in fnmatch.filter(filenames, xz_lib_name):
libs_to_extract.append(join(root, filename))
for filename in fnmatch.filter(filenames, zstd_lib_name):
fullpath = join(root, filename)
# For voltron modules BUCK creates empty zstd files for each module
if os.path.getsize(fullpath) > 0:
libs_to_extract.append(fullpath)
if len(libs_to_extract) > 0:
libs_dir = join(extracted_apk_dir, 'lib')
extracted_dir = join(libs_dir, '__extracted_libs__')
# Ensure both directories exist.
temporary_libs_dir = ensure_libs_dir(libs_dir, extracted_dir)
lib_count = 0
for lib_to_extract in libs_to_extract:
extract_path = join(extracted_dir, "lib_{}.so".format(lib_count))
if lib_to_extract.endswith(xz_lib_name):
cmd = 'xz -d --stdout {} > {}'.format(
lib_to_extract, extract_path)
else:
cmd = 'zstd -d {} -o {}'.format(lib_to_extract, extract_path)
subprocess.check_call(cmd, shell=True)
lib_count += 1
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.side_effect_summaries="%s"' % args.side_effect_summaries
)
if args.escape_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.escape_summaries="%s"' % args.escape_summaries
)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" %
(key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" %
(key, value, key_value_str, prev_value))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
return State(
application_modules=application_modules,
args=args,
config_dict=config_dict,
debugger=debugger,
dex_dir=dex_dir,
dexen=dexen,
dex_mode=dex_mode,
extracted_apk_dir=extracted_apk_dir,
temporary_libs_dir=temporary_libs_dir,
stop_pass_idx=stop_pass_idx)
def run_pass(
executable_path,
script_args,
config_path,
config_json,
apk_dir,
dex_dir,
dexfiles,
debugger,
):
if executable_path is None:
try:
executable_path = subprocess.check_output(['which',
'redex-all']).rstrip()
except subprocess.CalledProcessError:
pass
if executable_path is None:
# __file__ can be /path/fb-redex.pex/redex.pyc
dir_name = dirname(abspath(__file__))
while not isdir(dir_name):
dir_name = dirname(dir_name)
executable_path = join(dir_name, 'redex-all')
if not isfile(executable_path) or not os.access(executable_path, os.X_OK):
sys.exit('redex-all is not found or is not executable')
log('Running redex binary at ' + executable_path)
args = [executable_path] + ['--apkdir', apk_dir, '--outdir', dex_dir]
if config_path:
args += ['--config', config_path]
if script_args.warn:
args += ['--warn', script_args.warn]
args += ['--proguard-config=' + x for x in script_args.proguard_configs]
if script_args.keep:
args += ['--seeds', script_args.keep]
if script_args.proguard_map:
args += ['-Sproguard_map=' + script_args.proguard_map]
args += ['--jarpath=' + x for x in script_args.jarpaths]
if script_args.printseeds:
args += ['--printseeds=' + script_args.printseeds]
args += ['-S' + x for x in script_args.passthru]
args += ['-J' + x for x in script_args.passthru_json]
args += dexfiles
if debugger == 'lldb':
args = ['lldb', '--'] + args
elif debugger == 'gdb':
args = ['gdb', '--args'] + args
start = timer()
if script_args.debug:
print(' '.join(args))
sys.exit()
env = logger.setup_trace_for_child(os.environ)
logger.flush()
# Our CI system occasionally fails because it is trying to write the
# redex-all binary when this tries to run. This shouldn't happen, and
# might be caused by a JVM bug. Anyways, let's retry and hope it stops.
for i in range(5):
try:
subprocess.check_call(args, env=env)
except OSError as err:
if err.errno == errno.ETXTBSY:
if i < 4:
time.sleep(5)
continue
raise err
except subprocess.CalledProcessError as err:
err.cmd = ' '.join(args)
raise err
break
log('Dex processing finished in {:.2f} seconds'.format(timer() - start))
def run_redex(args):
debug_mode = args.unpack_only or args.debug
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
passes_list = config_dict['redex']['passes']
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
libs_dir = join(extracted_apk_dir, 'lib')
temporary_lib_file = join(libs_dir, 'concated_native_libs.so')
if os.path.exists(xz_compressed_libs) and os.path.exists(libs_dir):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.external_summaries="%s"' % args.side_effect_summaries
)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
run_pass(binary,
args,
config,
config_dict,
extracted_apk_dir,
dex_dir,
dexen,
debugger)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
dex_mode.repackage(
extracted_apk_dir, dex_dir, have_locators, fast_repackage=args.dev
)
locator_store_id = 1
for module in application_modules:
log('repacking module: ' + module.get_name() + ' with id ' + str(locator_store_id))
module.repackage(
extracted_apk_dir, dex_dir, have_locators, locator_store_id,
fast_repackage=args.dev
)
locator_store_id = locator_store_id + 1
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass, args.ignore_zipalign, args.page_align_libs)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map-v2', 'line number map v2', 'redex-line-number-map-v2')
copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'outliner-artifacts.bin', 'outliner artifacts', 'redex-outliner-artifacts.bin')
copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt', 'class id map', 'redex-class-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'bytecode_offset_map.txt', 'bytecode offset map', 'redex-bytecode-offset-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'coldstart_fields_in_R_classes.txt', 'resources accessed during coldstart', 'redex-tracked-coldstart-resources.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_dependencies.txt', 'stats', 'redex-class-dependencies.txt')
copy_file_to_out_dir(dex_dir, args.out, 'resid-optres-mapping.json', 'resid map after optres pass', 'redex-resid-optres-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'resid-dedup-mapping.json', 'resid map after dedup pass', 'redex-resid-dedup-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'resid-splitres-mapping.json', 'resid map after split pass', 'redex-resid-splitres-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'type-erasure-mappings.txt', 'class map after type erasure pass', 'redex-type-erasure-mappings.txt')
copy_file_to_out_dir(dex_dir, args.out, 'instrument-methods-idx.txt', 'instrumented methods id map', 'redex-instrument-methods-idx.txt')
copy_file_to_out_dir(dex_dir, args.out, 'cleanup-removed-classes.txt', 'cleanup removed classes', 'redex-cleanup-removed-classes.txt')
if config_dict.get('proguard_map_output', '') != '':
# if our map output strategy is overwrite, we don't merge at all
# if you enable ObfuscatePass, this needs to be overwrite
if config_dict.get('proguard_map_output_strategy', 'merge') == 'overwrite':
overwrite_proguard_maps(
config_dict['proguard_map_output'],
args.out,
dex_dir,
args.proguard_map)
else:
merge_proguard_maps(
config_dict['proguard_map_output'],
args.input_apk,
args.out,
dex_dir,
args.proguard_map)
else:
assert 'RenameClassesPass' not in passes_list and\
'RenameClassesPassV2' not in passes_list
remove_temp_dirs()
def run_redex_binary(state):
if state.args.redex_binary is None:
try:
state.args.redex_binary = subprocess.check_output(['which', 'redex-all']
).rstrip().decode('ascii')
except subprocess.CalledProcessError:
pass
if state.args.redex_binary is None:
# __file__ can be /path/fb-redex.pex/redex.pyc
dir_name = dirname(abspath(__file__))
while not isdir(dir_name):
dir_name = dirname(dir_name)
state.args.redex_binary = join(dir_name, 'redex-all')
if not isfile(state.args.redex_binary) or not os.access(state.args.redex_binary, os.X_OK):
sys.exit('redex-all is not found or is not executable: ' +
state.args.redex_binary)
log('Running redex binary at ' + state.args.redex_binary)
args = [state.args.redex_binary] + [
'--apkdir', state.extracted_apk_dir,
'--outdir', state.dex_dir]
if state.args.config:
args += ['--config', state.args.config]
if state.args.verify_none_mode or state.config_dict.get("verify_none_mode"):
args += ['--verify-none-mode']
if state.args.is_art_build:
args += ['--is-art-build']
if state.args.enable_instrument_pass or state.config_dict.get("enable_instrument_pass"):
args += ['--enable-instrument-pass']
if state.args.warn:
args += ['--warn', state.args.warn]
args += ['--proguard-config=' + x for x in state.args.proguard_configs]
if state.args.proguard_map:
args += ['-Sproguard_map=' + state.args.proguard_map]
args += ['--jarpath=' + x for x in state.args.jarpaths]
if state.args.printseeds:
args += ['--printseeds=' + state.args.printseeds]
if state.args.used_js_assets:
args += ['--used-js-assets=' + x for x in state.args.used_js_assets]
args += ['-S' + x for x in state.args.passthru]
args += ['-J' + x for x in state.args.passthru_json]
args += state.dexen
# Stop before a pass and output intermediate dex and IR meta data.
if state.stop_pass_idx != -1:
args += ['--stop-pass', str(state.stop_pass_idx),
'--output-ir', state.args.output_ir]
if state.debugger == 'lldb':
args = ['lldb', '--'] + args
elif state.debugger == 'gdb':
args = ['gdb', '--args'] + args
start = timer()
if state.args.debug:
print('cd %s && %s' % (os.getcwd(), ' '.join(map(quote, args))))
sys.exit()
env = logger.setup_trace_for_child(os.environ)
logger.flush()
add_extra_environment_args(env)
# Our CI system occasionally fails because it is trying to write the
# redex-all binary when this tries to run. This shouldn't happen, and
# might be caused by a JVM bug. Anyways, let's retry and hope it stops.
for i in range(5):
try:
subprocess.check_call(args, env=env)
except OSError as err:
if err.errno == errno.ETXTBSY:
if i < 4:
time.sleep(5)
continue
raise err
except subprocess.CalledProcessError as err:
script_filenames = write_debugger_commands(args)
raise RuntimeError(
('redex-all crashed with exit code %s! ' % err.returncode) +
('You can re-run it '
'under gdb by running %(gdb_script_name)s or under lldb '
'by running %(lldb_script_name)s') % script_filenames)
break
log('Dex processing finished in {:.2f} seconds'.format(timer() - start))
def prepare_redex(args):
debug_mode = args.unpack_only or args.debug
# avoid accidentally mixing up file formats since we now support
# both apk files and Android bundle files
if not args.unpack_only:
assert get_file_ext(args.input_apk) == get_file_ext(args.out),\
"Input file extension (\"" +\
get_file_ext(args.input_apk) +\
"\") should be the same as output file extension (\"" +\
get_file_ext(args.out) + "\")"
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if args.unpack_only or config is None:
config_dict = {}
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
# stop_pass_idx >= 0 means need stop before a pass and dump intermediate result
stop_pass_idx = -1
if args.stop_pass:
passes_list = config_dict.get('redex', {}).get('passes', [])
stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass)
if not args.output_ir or isfile(args.output_ir):
print('Error: output_ir should be a directory')
sys.exit(1)
try:
os.makedirs(args.output_ir)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_file_path = get_dex_file_path(args, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(dex_file_path)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(dex_file_path, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata',
debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' +
(canary_prefix
if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir,
module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
libs_to_extract = []
temporary_libs_dir = None
xz_lib_name = 'libs.xzs'
zstd_lib_name = 'libs.zstd'
for root, _, filenames in os.walk(extracted_apk_dir):
for filename in fnmatch.filter(filenames, xz_lib_name):
libs_to_extract.append(join(root, filename))
for filename in fnmatch.filter(filenames, zstd_lib_name):
fullpath = join(root, filename)
# For voltron modules BUCK creates empty zstd files for each module
if os.path.getsize(fullpath) > 0:
libs_to_extract.append(fullpath)
if len(libs_to_extract) > 0:
libs_dir = join(extracted_apk_dir, 'lib')
extracted_dir = join(libs_dir, '__extracted_libs__')
# Ensure both directories exist.
temporary_libs_dir = ensure_libs_dir(libs_dir, extracted_dir)
lib_count = 0
for lib_to_extract in libs_to_extract:
extract_path = join(extracted_dir, "lib_{}.so".format(lib_count))
if lib_to_extract.endswith(xz_lib_name):
cmd = 'xz -d --stdout {} > {}'.format(lib_to_extract,
extract_path)
else:
cmd = 'zstd -d {} -o {}'.format(lib_to_extract, extract_path)
subprocess.check_call(cmd, shell=True)
lib_count += 1
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(timer() -
unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'ObjectSensitiveDcePass.side_effect_summaries="%s"' %
args.side_effect_summaries)
if args.escape_summaries is not None:
args.passthru_json.append(
'ObjectSensitiveDcePass.escape_summaries="%s"' %
args.escape_summaries)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" %
(key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" %
(key, value, key_value_str, prev_value))
config_dict[key] = json.loads(value)
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
return State(application_modules=application_modules,
args=args,
config_dict=config_dict,
debugger=debugger,
dex_dir=dex_dir,
dexen=dexen,
dex_mode=dex_mode,
extracted_apk_dir=extracted_apk_dir,
temporary_libs_dir=temporary_libs_dir,
stop_pass_idx=stop_pass_idx)
def unpackage(self, extracted_apk_dir, dex_dir):
src = join(extracted_apk_dir, self._xzs_dir,
self._xzs_filename)
dest = join(dex_dir, self._xzs_filename)
# Move secondary dexen
shutil.move(src, dest)
# concat_jar is a bunch of .dex.jar files concatenated together.
concat_jar = join(dex_dir, self._xzs_filename[:-4])
cmd = 'cat {} | xz -d --threads 6 > {}'.format(dest, concat_jar)
subprocess.check_call(cmd, shell=True)
dex_order = []
with open(join(extracted_apk_dir, self._xzs_dir, 'metadata.txt')) as dex_metadata:
for line in dex_metadata.read().splitlines():
if line[0] != '.':
tokens = line.split()
search_pattern = self._store_name + '-(\d+)\.dex\.jar\.xzs\.tmp~'
match = re.search(search_pattern, tokens[0])
if match is None:
raise Exception('unable to find match in ' + tokens[0] + ' for ' + search_pattern)
dex_order.append(int(match.group(1)))
# Sizes of the concatenated .dex.jar files are stored in .meta files.
# Read the sizes of each .dex.jar file and un-concatenate them.
jar_size_regex = 'jar:(\d+)'
secondary_dir = join(extracted_apk_dir, self._xzs_dir)
jar_sizes = {}
for i in dex_order:
filename = self._store_name + '-%d.dex.jar.xzs.tmp~.meta' % i
metadata_path = join(secondary_dir, filename)
if isfile(metadata_path):
with open(metadata_path) as f:
jar_sizes[i] = \
int(re.match(jar_size_regex, f.read()).group(1))
os.remove(metadata_path)
log('found jar ' + filename + ' of size ' + str(jar_sizes[i]))
else:
break
with open(concat_jar, 'rb') as cj:
for i in dex_order:
jarpath = join(dex_dir, self._store_name + '-%d.dex.jar' % i)
with open(jarpath, 'wb') as jar:
jar.write(cj.read(jar_sizes[i]))
for j in jar_sizes.keys():
jar_size = getsize(dex_dir + '/' + self._store_name + '-' + str(j) + '.dex.jar')
log('validating ' + self._store_name + '-' + str(j) + '.dex.jar size=' + str(jar_size) + ' expecting=' + str(jar_sizes[j]))
assert jar_sizes[j] == jar_size
assert sum(jar_sizes.values()) == getsize(concat_jar)
# Clean up everything other than dexen in the dex directory
os.remove(concat_jar)
os.remove(dest)
# Lastly, unzip all the jar files and delete them
for jarpath in abs_glob(dex_dir, '*.jar'):
extract_dex_from_jar(jarpath, jarpath[:-4])
os.remove(jarpath)
BaseDexMode.unpackage(self, extracted_apk_dir, dex_dir)
def merge_proguard_maps(redex_rename_map_path, input_apk_path, apk_output_path,
dex_dir, pg_file):
log('running merge proguard step')
redex_rename_map_path = join(dex_dir, redex_rename_map_path)
log('redex map is at ' + str(redex_rename_map_path))
log('pg map is at ' + str(pg_file))
assert os.path.isfile(redex_rename_map_path)
redex_pg_file = "redex-class-rename-map.txt"
output_dir = os.path.dirname(apk_output_path)
output_file = join(output_dir, redex_pg_file)
# If -dontobfuscate is set, proguard won't produce a mapping file, but
# buck will create an empty mapping.txt. Check for this case.
if pg_file and os.path.getsize(pg_file) > 0:
update_proguard_mapping_file(pg_file, redex_rename_map_path,
output_file)
log('merging proguard map with redex class rename map')
log('pg mapping file input is ' + str(pg_file))
log('wrote redex pg format mapping file to ' + str(output_file))
else:
log('no proguard map file found')
shutil.move(redex_rename_map_path, output_file)
def __enter__(self):
log("Extracting apk...")
with zipfile.ZipFile(self.input_apk) as z:
for info in z.infolist():
self.per_file_compression[info.filename] = info.compress_type
z.extractall(self.extracted_apk_dir)
def finalize_redex(state):
# This dir was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if state.temporary_libs_dir is not None:
shutil.rmtree(state.temporary_libs_dir)
repack_start_time = timer()
log('Repacking dex files')
have_locators = state.config_dict.get("emit_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
state.dex_mode.repackage(state.extracted_apk_dir,
state.dex_dir,
have_locators,
fast_repackage=state.args.dev)
locator_store_id = 1
for module in state.application_modules:
log('repacking module: ' + module.get_name() + ' with id ' +
str(locator_store_id))
module.repackage(state.extracted_apk_dir,
state.dex_dir,
have_locators,
locator_store_id,
fast_repackage=state.args.dev)
locator_store_id = locator_store_id + 1
log('Creating output apk')
create_output_apk(state.extracted_apk_dir, state.args.out, state.args.sign,
state.args.keystore, state.args.keyalias,
state.args.keypass, state.args.ignore_zipalign,
state.args.page_align_libs)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(state.dex_dir, state.args.out,
'redex-line-number-map', 'line number map',
'redex-line-number-map')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'redex-line-number-map-v2', 'line number map v2',
'redex-line-number-map-v2')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'stats.txt', 'stats',
'redex-stats.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'filename_mappings.txt', 'src strings map',
'redex-src-strings-map.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'outliner-artifacts.bin', 'outliner artifacts',
'redex-outliner-artifacts.bin')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'method_mapping.txt',
'method id map', 'redex-method-id-map.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'class_mapping.txt',
'class id map', 'redex-class-id-map.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'bytecode_offset_map.txt', 'bytecode offset map',
'redex-bytecode-offset-map.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'coldstart_fields_in_R_classes.txt',
'resources accessed during coldstart',
'redex-tracked-coldstart-resources.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'class_dependencies.txt', 'stats',
'redex-class-dependencies.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'resid-optres-mapping.json',
'resid map after optres pass',
'redex-resid-optres-mapping.json')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'resid-dedup-mapping.json',
'resid map after dedup pass',
'redex-resid-dedup-mapping.json')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'resid-splitres-mapping.json',
'resid map after split pass',
'redex-resid-splitres-mapping.json')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'type-erasure-mappings.txt',
'class map after type erasure pass',
'redex-type-erasure-mappings.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'instrument-metadata.txt',
'metadata file for instrumentation',
'redex-instrument-metadata.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'cleanup-removed-classes.txt',
'cleanup removed classes',
'redex-cleanup-removed-classes.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out, 'opt-decisions.json',
'opt info', 'redex-opt-decisions.json')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'redex-debug-line-map.txt', 'debug line map',
'redex-debug-line-map.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'redex-debug-line-map-v2', 'debug method id map',
'redex-debug-line-map-v2')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'class-method-info-map.txt', 'class method info map',
'redex-class-method-info-map.txt')
copy_file_to_out_dir(state.dex_dir, state.args.out,
'cfp-unsafe-references.txt', 'cfp unsafe references',
'redex-cfp-unsafe-references.txt')
copy_all_file_to_out_dir(state.dex_dir, state.args.out, '*.dot',
'approximate shape graphs')
if state.config_dict.get('proguard_map_output', '') != '':
# if our map output strategy is overwrite, we don't merge at all
# if you enable ObfuscatePass, this needs to be overwrite
if state.config_dict.get('proguard_map_output_strategy',
'merge') == 'overwrite':
overwrite_proguard_maps(state.config_dict['proguard_map_output'],
state.args.out, state.dex_dir,
state.args.proguard_map)
else:
merge_proguard_maps(state.config_dict['proguard_map_output'],
state.args.input_apk, state.args.out,
state.dex_dir, state.args.proguard_map)
else:
passes_list = state.config_dict.get('redex', {}).get('passes', [])
assert 'RenameClassesPass' not in passes_list and\
'RenameClassesPassV2' not in passes_list
def unpackage(self, extracted_apk_dir, dex_dir, unpackage_metadata=False):
src = join(extracted_apk_dir, self._xzs_dir, self._xzs_filename)
dest = join(dex_dir, self._xzs_filename)
# Move secondary dexen
shutil.move(src, dest)
# concat_jar is a bunch of .dex.jar files concatenated together.
concat_jar = join(dex_dir, self._xzs_filename[:-4])
cmd = "cat {} | xz -d --threads 6 > {}".format(dest, concat_jar)
subprocess.check_call(cmd, shell=True)
if unpackage_metadata:
shutil.copy(join(extracted_apk_dir, self._xzs_dir, "metadata.txt"),
dex_dir)
dex_order = []
with open(join(extracted_apk_dir, self._xzs_dir,
"metadata.txt")) as dex_metadata:
for line in dex_metadata.read().splitlines():
if line[0] != ".":
tokens = line.split()
search_pattern = self._store_name + r"-(\d+)\.dex\.jar\.xzs\.tmp~"
match = re.search(search_pattern, tokens[0])
if match is None:
raise Exception("unable to find match in " +
tokens[0] + " for " + search_pattern)
dex_order.append(int(match.group(1)))
# Sizes of the concatenated .dex.jar files are stored in .meta files.
# Read the sizes of each .dex.jar file and un-concatenate them.
jar_size_regex = r"jar:(\d+)"
secondary_dir = join(extracted_apk_dir, self._xzs_dir)
jar_sizes = {}
for i in dex_order:
filename = self._store_name + "-%d.dex.jar.xzs.tmp~.meta" % i
metadata_path = join(secondary_dir, filename)
if isfile(metadata_path):
with open(metadata_path) as f:
jar_sizes[i] = int(
re.match(jar_size_regex, f.read()).group(1))
os.remove(metadata_path)
log("found jar " + filename + " of size " + str(jar_sizes[i]))
else:
break
with open(concat_jar, "rb") as cj:
for i in dex_order:
jarpath = join(dex_dir, self._store_name + "-%d.dex.jar" % i)
with open(jarpath, "wb") as jar:
jar.write(cj.read(jar_sizes[i]))
for j in jar_sizes.keys():
jar_size = getsize(dex_dir + "/" + self._store_name + "-" +
str(j) + ".dex.jar")
log("validating " + self._store_name + "-" + str(j) +
".dex.jar size=" + str(jar_size) + " expecting=" +
str(jar_sizes[j]))
assert jar_sizes[j] == jar_size
assert sum(jar_sizes.values()) == getsize(concat_jar)
# Clean up everything other than dexen in the dex directory
os.remove(concat_jar)
os.remove(dest)
# Lastly, unzip all the jar files and delete them
for jarpath in abs_glob(dex_dir, "*.jar"):
extract_dex_from_jar(jarpath, jarpath[:-4])
os.remove(jarpath)
BaseDexMode.unpackage(self, extracted_apk_dir, dex_dir)
def run_pass(
executable_path,
script_args,
config_path,
config_json,
apk_dir,
dex_dir,
dexfiles,
debugger,
):
if executable_path is None:
try:
executable_path = subprocess.check_output(['which', 'redex-all']
).rstrip().decode('ascii')
except subprocess.CalledProcessError:
pass
if executable_path is None:
# __file__ can be /path/fb-redex.pex/redex.pyc
dir_name = dirname(abspath(__file__))
while not isdir(dir_name):
dir_name = dirname(dir_name)
executable_path = join(dir_name, 'redex-all')
if not isfile(executable_path) or not os.access(executable_path, os.X_OK):
sys.exit('redex-all is not found or is not executable: ' + executable_path)
log('Running redex binary at ' + executable_path)
args = [executable_path] + [
'--apkdir', apk_dir,
'--outdir', dex_dir]
if config_path:
args += ['--config', config_path]
if script_args.verify_none_mode or config_json.get("verify_none_mode"):
args += ['--verify-none-mode']
if script_args.is_art_build:
args += ['--is-art-build']
if script_args.warn:
args += ['--warn', script_args.warn]
args += ['--proguard-config=' + x for x in script_args.proguard_configs]
if script_args.proguard_map:
args += ['-Sproguard_map=' + script_args.proguard_map]
args += ['--jarpath=' + x for x in script_args.jarpaths]
if script_args.printseeds:
args += ['--printseeds=' + script_args.printseeds]
if script_args.rmethods:
args += ['--rmethods=' + script_args.rmethods]
if script_args.amethods:
args += ['--amethods=' + script_args.amethods]
args += ['-S' + x for x in script_args.passthru]
args += ['-J' + x for x in script_args.passthru_json]
args += dexfiles
if debugger == 'lldb':
args = ['lldb', '--'] + args
elif debugger == 'gdb':
args = ['gdb', '--args'] + args
start = timer()
if script_args.debug:
print(' '.join(args))
sys.exit()
env = logger.setup_trace_for_child(os.environ)
logger.flush()
add_extra_environment_args(env)
# Our CI system occasionally fails because it is trying to write the
# redex-all binary when this tries to run. This shouldn't happen, and
# might be caused by a JVM bug. Anyways, let's retry and hope it stops.
for i in range(5):
try:
subprocess.check_call(args, env=env)
except OSError as err:
if err.errno == errno.ETXTBSY:
if i < 4:
time.sleep(5)
continue
raise err
except subprocess.CalledProcessError as err:
script_filenames = write_debugger_commands(args)
raise RuntimeError(
('redex-all crashed with exit code %s! ' % err.returncode) +
('You can re-run it '
'under gdb by running %(gdb_script_name)s or under lldb '
'by running %(lldb_script_name)s') % script_filenames)
break
log('Dex processing finished in {:.2f} seconds'.format(timer() - start))
