def run_redex(args):
debug_mode = args.unpack_only or args.debug
unpack_start_time = timer()
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
for module in application_modules:
log('found module: ' + module.get_name() + ' ' + module.get_canary_prefix())
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_files.append(module.write_redex_metadata(store_path))
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
temporary_lib_file = join(extracted_apk_dir, 'lib/concated_native_libs.so')
if os.path.exists(xz_compressed_libs):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
config_dict = json.load(config_file)
passes_list = config_dict['redex']['passes']
newtmp = tempfile.mkdtemp()
log('Replacing /tmp in config with {}'.format(newtmp))
# Fix up the config dict to relocate all /tmp references
relocate_tmp(config_dict, newtmp)
# Rewrite the relocated config file to our tmp, for use by redex binary
if config is not None:
config = newtmp + "/rewritten.config"
with open(config, 'w') as fp:
json.dump(config_dict, fp)
log('Running redex-all on {} dex files '.format(len(dexen)))
run_pass(binary,
args,
config,
config_dict,
extracted_apk_dir,
dex_dir,
dexen)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
dex_mode.repackage(extracted_apk_dir, dex_dir, have_locators)
for module in application_modules:
log('repacking module: ' + module.get_name())
module.repackage(extracted_apk_dir, dex_dir, have_locators)
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(newtmp, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
copy_file_to_out_dir(newtmp, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
copy_file_to_out_dir(newtmp, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(newtmp, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
if 'RenameClassesPass' in passes_list:
merge_proguard_map_with_rename_output(
args.input_apk,
args.out,
config_dict,
args.proguard_map)
else:
log('Skipping rename map merging, because we didn\'t run the rename pass')
shutil.rmtree(newtmp)
remove_temp_dirs()
def prepare_redex(args):
debug_mode = args.unpack_only or args.debug
# avoid accidentally mixing up file formats since we now support
# both apk files and Android bundle files
if not args.unpack_only:
assert get_file_ext(args.input_apk) == get_file_ext(args.out),\
"Input file extension (\"" +\
get_file_ext(args.input_apk) +\
"\") should be the same as output file extension (\"" +\
get_file_ext(args.out) + "\")"
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if args.unpack_only or config is None:
config_dict = {}
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
# stop_pass_idx >= 0 means need stop before a pass and dump intermediate result
stop_pass_idx = -1
if args.stop_pass:
passes_list = config_dict.get('redex', {}).get('passes', [])
stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass)
if not args.output_ir or isfile(args.output_ir):
print('Error: output_ir should be a directory')
sys.exit(1)
try:
os.makedirs(args.output_ir)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_file_path = get_dex_file_path(args, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(dex_file_path)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(dex_file_path, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata',
debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' +
(canary_prefix
if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir,
module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
libs_to_extract = []
temporary_libs_dir = None
xz_lib_name = 'libs.xzs'
zstd_lib_name = 'libs.zstd'
for root, _, filenames in os.walk(extracted_apk_dir):
for filename in fnmatch.filter(filenames, xz_lib_name):
libs_to_extract.append(join(root, filename))
for filename in fnmatch.filter(filenames, zstd_lib_name):
fullpath = join(root, filename)
# For voltron modules BUCK creates empty zstd files for each module
if os.path.getsize(fullpath) > 0:
libs_to_extract.append(fullpath)
if len(libs_to_extract) > 0:
libs_dir = join(extracted_apk_dir, 'lib')
extracted_dir = join(libs_dir, '__extracted_libs__')
# Ensure both directories exist.
temporary_libs_dir = ensure_libs_dir(libs_dir, extracted_dir)
lib_count = 0
for lib_to_extract in libs_to_extract:
extract_path = join(extracted_dir, "lib_{}.so".format(lib_count))
if lib_to_extract.endswith(xz_lib_name):
cmd = 'xz -d --stdout {} > {}'.format(lib_to_extract,
extract_path)
else:
cmd = 'zstd -d {} -o {}'.format(lib_to_extract, extract_path)
subprocess.check_call(cmd, shell=True)
lib_count += 1
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(timer() -
unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'ObjectSensitiveDcePass.side_effect_summaries="%s"' %
args.side_effect_summaries)
if args.escape_summaries is not None:
args.passthru_json.append(
'ObjectSensitiveDcePass.escape_summaries="%s"' %
args.escape_summaries)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" %
(key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" %
(key, value, key_value_str, prev_value))
config_dict[key] = json.loads(value)
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
return State(application_modules=application_modules,
args=args,
config_dict=config_dict,
debugger=debugger,
dex_dir=dex_dir,
dexen=dexen,
dex_mode=dex_mode,
extracted_apk_dir=extracted_apk_dir,
temporary_libs_dir=temporary_libs_dir,
stop_pass_idx=stop_pass_idx)
def run_redex(args):
debug_mode = args.unpack_only or args.debug
unpack_start_time = timer()
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata',
debug_mode)
for module in application_modules:
log('found module: ' + module.get_name() + ' ' +
module.get_canary_prefix())
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir,
module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
temporary_lib_file = join(extracted_apk_dir, 'lib/concated_native_libs.so')
if os.path.exists(xz_compressed_libs):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs,
temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(timer() -
unpack_start_time))
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
config_dict = json.load(config_file)
passes_list = config_dict['redex']['passes']
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" %
(key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
log("Got Override %s = %s from %s. Previous %s" %
(key, value, key_value_str, config_dict[key]))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
run_pass(binary, args, config, config_dict, extracted_apk_dir, dex_dir,
dexen)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
dex_mode.repackage(extracted_apk_dir, dex_dir, have_locators)
for module in application_modules:
log('repacking module: ' + module.get_name())
module.repackage(extracted_apk_dir, dex_dir, have_locators)
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map',
'line number map', 'redex-line-number-map')
copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats',
'redex-stats.txt')
copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt',
'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt',
'method id map', 'redex-method-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt',
'class id map', 'redex-class-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out,
'coldstart_fields_in_R_classes.txt',
'resources accessed during coldstart',
'redex-tracked-coldstart-resources.txt')
if 'RenameClassesPass' in passes_list or 'RenameClassesPassV2' in passes_list:
merge_proguard_map_with_rename_output(passes_list, args.input_apk,
args.out, dex_dir, config_dict,
args.proguard_map)
else:
log('Skipping rename map merging, because we didn\'t run the rename pass'
)
remove_temp_dirs()
def run_redex(args):
debug_mode = args.unpack_only or args.debug
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
passes_list = config_dict['redex']['passes']
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
libs_dir = join(extracted_apk_dir, 'lib')
temporary_lib_file = join(libs_dir, 'concated_native_libs.so')
if os.path.exists(xz_compressed_libs) and os.path.exists(libs_dir):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.external_summaries="%s"' % args.side_effect_summaries
)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
run_pass(binary,
args,
config,
config_dict,
extracted_apk_dir,
dex_dir,
dexen,
debugger)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
dex_mode.repackage(
extracted_apk_dir, dex_dir, have_locators, fast_repackage=args.dev
)
locator_store_id = 1
for module in application_modules:
log('repacking module: ' + module.get_name() + ' with id ' + str(locator_store_id))
module.repackage(
extracted_apk_dir, dex_dir, have_locators, locator_store_id,
fast_repackage=args.dev
)
locator_store_id = locator_store_id + 1
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass, args.ignore_zipalign, args.page_align_libs)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map-v2', 'line number map v2', 'redex-line-number-map-v2')
copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'outliner-artifacts.bin', 'outliner artifacts', 'redex-outliner-artifacts.bin')
copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt', 'class id map', 'redex-class-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'bytecode_offset_map.txt', 'bytecode offset map', 'redex-bytecode-offset-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'coldstart_fields_in_R_classes.txt', 'resources accessed during coldstart', 'redex-tracked-coldstart-resources.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_dependencies.txt', 'stats', 'redex-class-dependencies.txt')
copy_file_to_out_dir(dex_dir, args.out, 'resid-optres-mapping.json', 'resid map after optres pass', 'redex-resid-optres-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'resid-dedup-mapping.json', 'resid map after dedup pass', 'redex-resid-dedup-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'resid-splitres-mapping.json', 'resid map after split pass', 'redex-resid-splitres-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'type-erasure-mappings.txt', 'class map after type erasure pass', 'redex-type-erasure-mappings.txt')
copy_file_to_out_dir(dex_dir, args.out, 'instrument-methods-idx.txt', 'instrumented methods id map', 'redex-instrument-methods-idx.txt')
copy_file_to_out_dir(dex_dir, args.out, 'cleanup-removed-classes.txt', 'cleanup removed classes', 'redex-cleanup-removed-classes.txt')
if config_dict.get('proguard_map_output', '') != '':
# if our map output strategy is overwrite, we don't merge at all
# if you enable ObfuscatePass, this needs to be overwrite
if config_dict.get('proguard_map_output_strategy', 'merge') == 'overwrite':
overwrite_proguard_maps(
config_dict['proguard_map_output'],
args.out,
dex_dir,
args.proguard_map)
else:
merge_proguard_maps(
config_dict['proguard_map_output'],
args.input_apk,
args.out,
dex_dir,
args.proguard_map)
else:
assert 'RenameClassesPass' not in passes_list and\
'RenameClassesPassV2' not in passes_list
remove_temp_dirs()
def prepare_redex(args):
debug_mode = args.unpack_only or args.debug
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
# stop_pass_idx >= 0 means need stop before a pass and dump intermediate result
stop_pass_idx = -1
if args.stop_pass:
passes_list = config_dict.get('redex', {}).get('passes', [])
stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass)
if not args.output_ir or isfile(args.output_ir):
print('Error: output_ir should be a directory')
sys.exit(1)
try:
os.makedirs(args.output_ir)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir(
'.application_module_metadata', debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' +
(canary_prefix if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(
store_metadata_dir, module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
libs_to_extract = []
temporary_libs_dir = None
xz_lib_name = 'libs.xzs'
zstd_lib_name = 'libs.zstd'
for root, _, filenames in os.walk(extracted_apk_dir):
for filename in fnmatch.filter(filenames, xz_lib_name):
libs_to_extract.append(join(root, filename))
for filename in fnmatch.filter(filenames, zstd_lib_name):
fullpath = join(root, filename)
# For voltron modules BUCK creates empty zstd files for each module
if os.path.getsize(fullpath) > 0:
libs_to_extract.append(fullpath)
if len(libs_to_extract) > 0:
libs_dir = join(extracted_apk_dir, 'lib')
extracted_dir = join(libs_dir, '__extracted_libs__')
# Ensure both directories exist.
temporary_libs_dir = ensure_libs_dir(libs_dir, extracted_dir)
lib_count = 0
for lib_to_extract in libs_to_extract:
extract_path = join(extracted_dir, "lib_{}.so".format(lib_count))
if lib_to_extract.endswith(xz_lib_name):
cmd = 'xz -d --stdout {} > {}'.format(
lib_to_extract, extract_path)
else:
cmd = 'zstd -d {} -o {}'.format(lib_to_extract, extract_path)
subprocess.check_call(cmd, shell=True)
lib_count += 1
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.side_effect_summaries="%s"' % args.side_effect_summaries
)
if args.escape_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.escape_summaries="%s"' % args.escape_summaries
)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" %
(key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" %
(key, value, key_value_str, prev_value))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
return State(
application_modules=application_modules,
args=args,
config_dict=config_dict,
debugger=debugger,
dex_dir=dex_dir,
dexen=dexen,
dex_mode=dex_mode,
extracted_apk_dir=extracted_apk_dir,
temporary_libs_dir=temporary_libs_dir,
stop_pass_idx=stop_pass_idx)
def run_redex(args):
debug_mode = args.unpack_only or args.debug
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
passes_list = config_dict['redex']['passes']
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
libs_dir = join(extracted_apk_dir, 'lib')
temporary_lib_file = join(libs_dir, 'concated_native_libs.so')
if os.path.exists(xz_compressed_libs) and os.path.exists(libs_dir):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.external_summaries="%s"' % args.side_effect_summaries
)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
run_pass(binary,
args,
config,
config_dict,
extracted_apk_dir,
dex_dir,
dexen,
debugger)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
dex_mode.repackage(
extracted_apk_dir, dex_dir, have_locators, fast_repackage=args.dev
)
locator_store_id = 1
for module in application_modules:
log('repacking module: ' + module.get_name() + ' with id ' + str(locator_store_id))
module.repackage(
extracted_apk_dir, dex_dir, have_locators, locator_store_id,
fast_repackage=args.dev
)
locator_store_id = locator_store_id + 1
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass, args.ignore_zipalign, args.page_align_libs)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map-v2', 'line number map v2', 'redex-line-number-map-v2')
copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'outliner-artifacts.bin', 'outliner artifacts', 'redex-outliner-artifacts.bin')
copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt', 'class id map', 'redex-class-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'bytecode_offset_map.txt', 'bytecode offset map', 'redex-bytecode-offset-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'coldstart_fields_in_R_classes.txt', 'resources accessed during coldstart', 'redex-tracked-coldstart-resources.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_dependencies.txt', 'stats', 'redex-class-dependencies.txt')
copy_file_to_out_dir(dex_dir, args.out, 'resid-optres-mapping.json', 'resid map after optres pass', 'redex-resid-optres-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'resid-dedup-mapping.json', 'resid map after dedup pass', 'redex-resid-dedup-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'resid-splitres-mapping.json', 'resid map after split pass', 'redex-resid-splitres-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'type-erasure-mappings.txt', 'class map after type erasure pass', 'redex-type-erasure-mappings.txt')
copy_file_to_out_dir(dex_dir, args.out, 'instrument-methods-idx.txt', 'instrumented methods id map', 'redex-instrument-methods-idx.txt')
copy_file_to_out_dir(dex_dir, args.out, 'cleanup-removed-classes.txt', 'cleanup removed classes', 'redex-cleanup-removed-classes.txt')
if config_dict.get('proguard_map_output', '') != '':
# if our map output strategy is overwrite, we don't merge at all
# if you enable ObfuscatePass, this needs to be overwrite
if config_dict.get('proguard_map_output_strategy', 'merge') == 'overwrite':
overwrite_proguard_maps(
config_dict['proguard_map_output'],
args.out,
dex_dir,
args.proguard_map)
else:
merge_proguard_maps(
config_dict['proguard_map_output'],
args.input_apk,
args.out,
dex_dir,
args.proguard_map)
else:
assert 'RenameClassesPass' not in passes_list and\
'RenameClassesPassV2' not in passes_list
remove_temp_dirs()
def prepare_redex(args):
debug_mode = args.unpack_only or args.debug
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
libs_dir = join(extracted_apk_dir, 'lib')
temporary_lib_file = join(libs_dir, 'concated_native_libs.so')
if os.path.exists(xz_compressed_libs) and os.path.exists(libs_dir):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.side_effect_summaries="%s"' % args.side_effect_summaries
)
if args.escape_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.escape_summaries="%s"' % args.escape_summaries
)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
return State(
application_modules=application_modules,
args=args,
config_dict=config_dict,
debugger=debugger,
dex_dir=dex_dir,
dexen=dexen,
dex_mode=dex_mode,
extracted_apk_dir=extracted_apk_dir,
temporary_lib_file=temporary_lib_file)