def create_output_apk(extracted_apk_dir, output_apk_path, sign, keystore,
key_alias, key_password):
# Remove old signature files
for f in abs_glob(extracted_apk_dir, 'META-INF/*'):
cert_path = join(extracted_apk_dir, f)
if isfile(cert_path):
os.remove(cert_path)
directory = make_temp_dir('.redex_unaligned', False)
unaligned_apk_path = join(directory, 'redex-unaligned.apk')
if isfile(unaligned_apk_path):
os.remove(unaligned_apk_path)
# Create new zip file
with zipfile.ZipFile(unaligned_apk_path, 'w') as unaligned_apk:
for dirpath, dirnames, filenames in os.walk(extracted_apk_dir):
for filename in filenames:
filepath = join(dirpath, filename)
archivepath = filepath[len(extracted_apk_dir) + 1:]
try:
compress = per_file_compression[archivepath]
except KeyError:
compress = zipfile.ZIP_DEFLATED
unaligned_apk.write(filepath, archivepath,
compress_type=compress)
# Add new signature
if sign:
subprocess.check_call([
'jarsigner',
'-sigalg', 'SHA1withRSA',
'-digestalg', 'SHA1',
'-keystore', keystore,
'-storepass', key_password,
unaligned_apk_path, key_alias],
stdout=sys.stderr)
if isfile(output_apk_path):
os.remove(output_apk_path)
zipalign(unaligned_apk_path, output_apk_path)
def create_output_apk(extracted_apk_dir, output_apk_path, sign, keystore,
key_alias, key_password, ignore_zipalign, page_align):
# Remove old signature files
for f in abs_glob(extracted_apk_dir, 'META-INF/*'):
cert_path = join(extracted_apk_dir, f)
if isfile(cert_path):
os.remove(cert_path)
directory = make_temp_dir('.redex_unaligned', False)
unaligned_apk_path = join(directory, 'redex-unaligned.apk')
if isfile(unaligned_apk_path):
os.remove(unaligned_apk_path)
# Create new zip file
with zipfile.ZipFile(unaligned_apk_path, 'w') as unaligned_apk:
for dirpath, _dirnames, filenames in os.walk(extracted_apk_dir):
for filename in filenames:
filepath = join(dirpath, filename)
archivepath = filepath[len(extracted_apk_dir) + 1:]
try:
compress = per_file_compression[archivepath]
except KeyError:
compress = zipfile.ZIP_DEFLATED
unaligned_apk.write(filepath, archivepath,
compress_type=compress)
# Add new signature
if sign:
sign_apk(keystore, key_password, key_alias, unaligned_apk_path)
if isfile(output_apk_path):
os.remove(output_apk_path)
try:
os.makedirs(dirname(output_apk_path))
except OSError as e:
if e.errno != errno.EEXIST:
raise
zipalign(unaligned_apk_path, output_apk_path, ignore_zipalign, page_align)
def run_redex(args):
debug_mode = args.unpack_only or args.debug
unpack_start_time = timer()
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
for module in application_modules:
log('found module: ' + module.get_name() + ' ' + module.get_canary_prefix())
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_files.append(module.write_redex_metadata(store_path))
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
temporary_lib_file = join(extracted_apk_dir, 'lib/concated_native_libs.so')
if os.path.exists(xz_compressed_libs):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
config_dict = json.load(config_file)
passes_list = config_dict['redex']['passes']
newtmp = tempfile.mkdtemp()
log('Replacing /tmp in config with {}'.format(newtmp))
# Fix up the config dict to relocate all /tmp references
relocate_tmp(config_dict, newtmp)
# Rewrite the relocated config file to our tmp, for use by redex binary
if config is not None:
config = newtmp + "/rewritten.config"
with open(config, 'w') as fp:
json.dump(config_dict, fp)
log('Running redex-all on {} dex files '.format(len(dexen)))
run_pass(binary,
args,
config,
config_dict,
extracted_apk_dir,
dex_dir,
dexen)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
dex_mode.repackage(extracted_apk_dir, dex_dir, have_locators)
for module in application_modules:
log('repacking module: ' + module.get_name())
module.repackage(extracted_apk_dir, dex_dir, have_locators)
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(newtmp, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
copy_file_to_out_dir(newtmp, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
copy_file_to_out_dir(newtmp, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(newtmp, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
if 'RenameClassesPass' in passes_list:
merge_proguard_map_with_rename_output(
args.input_apk,
args.out,
config_dict,
args.proguard_map)
else:
log('Skipping rename map merging, because we didn\'t run the rename pass')
shutil.rmtree(newtmp)
remove_temp_dirs()
def prepare_redex(args):
debug_mode = args.unpack_only or args.debug
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
# stop_pass_idx >= 0 means need stop before a pass and dump intermediate result
stop_pass_idx = -1
if args.stop_pass:
passes_list = config_dict.get('redex', {}).get('passes', [])
stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass)
if not args.output_ir or isfile(args.output_ir):
print('Error: output_ir should be a directory')
sys.exit(1)
try:
os.makedirs(args.output_ir)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir(
'.application_module_metadata', debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' +
(canary_prefix if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(
store_metadata_dir, module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
libs_to_extract = []
temporary_libs_dir = None
xz_lib_name = 'libs.xzs'
zstd_lib_name = 'libs.zstd'
for root, _, filenames in os.walk(extracted_apk_dir):
for filename in fnmatch.filter(filenames, xz_lib_name):
libs_to_extract.append(join(root, filename))
for filename in fnmatch.filter(filenames, zstd_lib_name):
fullpath = join(root, filename)
# For voltron modules BUCK creates empty zstd files for each module
if os.path.getsize(fullpath) > 0:
libs_to_extract.append(fullpath)
if len(libs_to_extract) > 0:
libs_dir = join(extracted_apk_dir, 'lib')
extracted_dir = join(libs_dir, '__extracted_libs__')
# Ensure both directories exist.
temporary_libs_dir = ensure_libs_dir(libs_dir, extracted_dir)
lib_count = 0
for lib_to_extract in libs_to_extract:
extract_path = join(extracted_dir, "lib_{}.so".format(lib_count))
if lib_to_extract.endswith(xz_lib_name):
cmd = 'xz -d --stdout {} > {}'.format(
lib_to_extract, extract_path)
else:
cmd = 'zstd -d {} -o {}'.format(lib_to_extract, extract_path)
subprocess.check_call(cmd, shell=True)
lib_count += 1
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.side_effect_summaries="%s"' % args.side_effect_summaries
)
if args.escape_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.escape_summaries="%s"' % args.escape_summaries
)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" %
(key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" %
(key, value, key_value_str, prev_value))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
return State(
application_modules=application_modules,
args=args,
config_dict=config_dict,
debugger=debugger,
dex_dir=dex_dir,
dexen=dexen,
dex_mode=dex_mode,
extracted_apk_dir=extracted_apk_dir,
temporary_libs_dir=temporary_libs_dir,
stop_pass_idx=stop_pass_idx)
def run_redex(args):
debug_mode = args.unpack_only or args.debug
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
passes_list = config_dict['redex']['passes']
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
libs_dir = join(extracted_apk_dir, 'lib')
temporary_lib_file = join(libs_dir, 'concated_native_libs.so')
if os.path.exists(xz_compressed_libs) and os.path.exists(libs_dir):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.external_summaries="%s"' % args.side_effect_summaries
)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
run_pass(binary,
args,
config,
config_dict,
extracted_apk_dir,
dex_dir,
dexen,
debugger)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
dex_mode.repackage(
extracted_apk_dir, dex_dir, have_locators, fast_repackage=args.dev
)
locator_store_id = 1
for module in application_modules:
log('repacking module: ' + module.get_name() + ' with id ' + str(locator_store_id))
module.repackage(
extracted_apk_dir, dex_dir, have_locators, locator_store_id,
fast_repackage=args.dev
)
locator_store_id = locator_store_id + 1
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass, args.ignore_zipalign, args.page_align_libs)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map-v2', 'line number map v2', 'redex-line-number-map-v2')
copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'outliner-artifacts.bin', 'outliner artifacts', 'redex-outliner-artifacts.bin')
copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt', 'class id map', 'redex-class-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'bytecode_offset_map.txt', 'bytecode offset map', 'redex-bytecode-offset-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'coldstart_fields_in_R_classes.txt', 'resources accessed during coldstart', 'redex-tracked-coldstart-resources.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_dependencies.txt', 'stats', 'redex-class-dependencies.txt')
copy_file_to_out_dir(dex_dir, args.out, 'resid-optres-mapping.json', 'resid map after optres pass', 'redex-resid-optres-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'resid-dedup-mapping.json', 'resid map after dedup pass', 'redex-resid-dedup-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'resid-splitres-mapping.json', 'resid map after split pass', 'redex-resid-splitres-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'type-erasure-mappings.txt', 'class map after type erasure pass', 'redex-type-erasure-mappings.txt')
copy_file_to_out_dir(dex_dir, args.out, 'instrument-methods-idx.txt', 'instrumented methods id map', 'redex-instrument-methods-idx.txt')
copy_file_to_out_dir(dex_dir, args.out, 'cleanup-removed-classes.txt', 'cleanup removed classes', 'redex-cleanup-removed-classes.txt')
if config_dict.get('proguard_map_output', '') != '':
# if our map output strategy is overwrite, we don't merge at all
# if you enable ObfuscatePass, this needs to be overwrite
if config_dict.get('proguard_map_output_strategy', 'merge') == 'overwrite':
overwrite_proguard_maps(
config_dict['proguard_map_output'],
args.out,
dex_dir,
args.proguard_map)
else:
merge_proguard_maps(
config_dict['proguard_map_output'],
args.input_apk,
args.out,
dex_dir,
args.proguard_map)
else:
assert 'RenameClassesPass' not in passes_list and\
'RenameClassesPassV2' not in passes_list
remove_temp_dirs()
def prepare_redex(args):
logging.debug("Preparing...")
debug_mode = args.unpack_only or args.debug
if args.android_sdk_path:
add_android_sdk_path(args.android_sdk_path)
# avoid accidentally mixing up file formats since we now support
# both apk files and Android bundle files
if not args.unpack_only:
assert get_file_ext(args.input_apk) == get_file_ext(
args.out), ('Input file extension ("' +
get_file_ext(args.input_apk) +
'") should be the same as output file extension ("' +
get_file_ext(args.out) + '")')
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == ".":
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + ".redex_extracted_apk"
dex_dir = unpack_dir_basename + ".redex_dexen"
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print("Error: destination directory already exists!")
print("APK: " + extracted_apk_dir)
print("DEX: " + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
logging.debug("Using config %s",
config if config is not None else "(default)")
logging.debug("Using binary %s",
binary if binary is not None else "(default)")
if args.unpack_only or config is None:
config_dict = {}
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
# stop_pass_idx >= 0 means need stop before a pass and dump intermediate result
stop_pass_idx = -1
if args.stop_pass:
passes_list = config_dict.get("redex", {}).get("passes", [])
stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass)
if not args.output_ir or isfile(args.output_ir):
print("Error: output_ir should be a directory")
sys.exit(1)
try:
os.makedirs(args.output_ir)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
logging.debug("Unpacking...")
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir(".redex_extracted_apk", debug_mode)
directory = make_temp_dir(".redex_unaligned", False)
unaligned_apk_path = join(directory, "redex-unaligned.apk")
zip_manager = ZipManager(args.input_apk, extracted_apk_dir,
unaligned_apk_path)
zip_manager.__enter__()
if not dex_dir:
dex_dir = make_temp_dir(".redex_dexen", debug_mode)
unpack_manager = UnpackManager(
args.input_apk,
extracted_apk_dir,
dex_dir,
have_locators=config_dict.get("emit_locator_strings"),
debug_mode=debug_mode,
fast_repackage=args.dev,
reset_timestamps=args.reset_zip_timestamps or args.dev,
)
store_files = unpack_manager.__enter__()
lib_manager = LibraryManager(extracted_apk_dir)
lib_manager.__enter__()
if args.unpack_only:
print("APK: " + extracted_apk_dir)
print("DEX: " + dex_dir)
sys.exit()
# Unpack profiles, if they exist.
_handle_profiles(args, debug_mode)
logging.debug("Moving contents to expected structure...")
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in sorted(store_files):
dexen.append(store)
logging.debug(
"Unpacking APK finished in {:.2f} seconds".format(timer() -
unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'ObjectSensitiveDcePass.side_effect_summaries="%s"' %
args.side_effect_summaries)
if args.escape_summaries is not None:
args.passthru_json.append(
'ObjectSensitiveDcePass.escape_summaries="%s"' %
args.escape_summaries)
for key_value_str in args.passthru_json:
key_value = key_value_str.split("=", 1)
if len(key_value) != 2:
logging.debug(
"Json Pass through %s is not valid. Split len: %s",
key_value_str,
len(key_value),
)
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
logging.debug(
"Got Override %s = %s from %s. Previous %s",
key,
value,
key_value_str,
prev_value,
)
config_dict[key] = json.loads(value)
# Scan for framework files. If not found, warn and add them if available.
_check_android_sdk_api(args)
# Check for shrinker heuristics.
_check_shrinker_heuristics(args)
# Scan for SDK jar. If not found, warn and add if available.
_check_android_sdk(args)
logging.debug("Running redex-all on %d dex files ", len(dexen))
if args.lldb:
debugger = "lldb"
elif args.gdb:
debugger = "gdb"
else:
debugger = None
return State(
args=args,
config_dict=config_dict,
debugger=debugger,
dex_dir=dex_dir,
dexen=dexen,
extracted_apk_dir=extracted_apk_dir,
stop_pass_idx=stop_pass_idx,
lib_manager=lib_manager,
unpack_manager=unpack_manager,
zip_manager=zip_manager,
)
def prepare_redex(args):
debug_mode = args.unpack_only or args.debug
# avoid accidentally mixing up file formats since we now support
# both apk files and Android bundle files
if not args.unpack_only:
assert get_file_ext(args.input_apk) == get_file_ext(
args.out), ('Input file extension ("' +
get_file_ext(args.input_apk) +
'") should be the same as output file extension ("' +
get_file_ext(args.out) + '")')
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == ".":
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + ".redex_extracted_apk"
dex_dir = unpack_dir_basename + ".redex_dexen"
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print("Error: destination directory already exists!")
print("APK: " + extracted_apk_dir)
print("DEX: " + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log("Using config " + (config if config is not None else "(default)"))
log("Using binary " + (binary if binary is not None else "(default)"))
if args.unpack_only or config is None:
config_dict = {}
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
# stop_pass_idx >= 0 means need stop before a pass and dump intermediate result
stop_pass_idx = -1
if args.stop_pass:
passes_list = config_dict.get("redex", {}).get("passes", [])
stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass)
if not args.output_ir or isfile(args.output_ir):
print("Error: output_ir should be a directory")
sys.exit(1)
try:
os.makedirs(args.output_ir)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir(".redex_extracted_apk", debug_mode)
log("Extracting apk...")
unzip_apk(args.input_apk, extracted_apk_dir)
dex_file_path = get_dex_file_path(args, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(dex_file_path)
log("Detected dex mode " + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir(".redex_dexen", debug_mode)
log("Unpacking dex files")
dex_mode.unpackage(dex_file_path, dex_dir)
log("Detecting Application Modules")
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir(".application_module_metadata",
debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log("found module: " + module.get_name() + " " +
(canary_prefix
if canary_prefix is not None else "(no canary prefix)"))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir,
module.get_name() + ".json")
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
libs_to_extract = []
temporary_libs_dir = None
xz_lib_name = "libs.xzs"
zstd_lib_name = "libs.zstd"
for root, _, filenames in os.walk(extracted_apk_dir):
for filename in fnmatch.filter(filenames, xz_lib_name):
libs_to_extract.append(join(root, filename))
for filename in fnmatch.filter(filenames, zstd_lib_name):
fullpath = join(root, filename)
# For voltron modules BUCK creates empty zstd files for each module
if os.path.getsize(fullpath) > 0:
libs_to_extract.append(fullpath)
if len(libs_to_extract) > 0:
libs_dir = join(extracted_apk_dir, "lib")
extracted_dir = join(libs_dir, "__extracted_libs__")
# Ensure both directories exist.
temporary_libs_dir = ensure_libs_dir(libs_dir, extracted_dir)
lib_count = 0
for lib_to_extract in libs_to_extract:
extract_path = join(extracted_dir, "lib_{}.so".format(lib_count))
if lib_to_extract.endswith(xz_lib_name):
cmd = "xz -d --stdout {} > {}".format(lib_to_extract,
extract_path)
else:
cmd = "zstd -d {} -o {}".format(lib_to_extract, extract_path)
subprocess.check_call(cmd, shell=True)
lib_count += 1
if args.unpack_only:
print("APK: " + extracted_apk_dir)
print("DEX: " + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log("Unpacking APK finished in {:.2f} seconds".format(timer() -
unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'ObjectSensitiveDcePass.side_effect_summaries="%s"' %
args.side_effect_summaries)
if args.escape_summaries is not None:
args.passthru_json.append(
'ObjectSensitiveDcePass.escape_summaries="%s"' %
args.escape_summaries)
for key_value_str in args.passthru_json:
key_value = key_value_str.split("=", 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" %
(key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" %
(key, value, key_value_str, prev_value))
config_dict[key] = json.loads(value)
log("Running redex-all on {} dex files ".format(len(dexen)))
if args.lldb:
debugger = "lldb"
elif args.gdb:
debugger = "gdb"
else:
debugger = None
return State(
application_modules=application_modules,
args=args,
config_dict=config_dict,
debugger=debugger,
dex_dir=dex_dir,
dexen=dexen,
dex_mode=dex_mode,
extracted_apk_dir=extracted_apk_dir,
temporary_libs_dir=temporary_libs_dir,
stop_pass_idx=stop_pass_idx,
)
def prepare_redex(args):
debug_mode = args.unpack_only or args.debug
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
libs_dir = join(extracted_apk_dir, 'lib')
temporary_lib_file = join(libs_dir, 'concated_native_libs.so')
if os.path.exists(xz_compressed_libs) and os.path.exists(libs_dir):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.side_effect_summaries="%s"' % args.side_effect_summaries
)
if args.escape_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.escape_summaries="%s"' % args.escape_summaries
)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
return State(
application_modules=application_modules,
args=args,
config_dict=config_dict,
debugger=debugger,
dex_dir=dex_dir,
dexen=dexen,
dex_mode=dex_mode,
extracted_apk_dir=extracted_apk_dir,
temporary_lib_file=temporary_lib_file)
def run_redex(args):
debug_mode = args.unpack_only or args.debug
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
try:
config_dict = json.load(config_file)
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
passes_list = config_dict['redex']['passes']
unpack_start_time = timer()
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata',
debug_mode)
for module in application_modules:
log('found module: ' + module.get_name() + ' ' +
module.get_canary_prefix())
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir,
module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
temporary_lib_file = join(extracted_apk_dir, 'lib/concated_native_libs.so')
if os.path.exists(xz_compressed_libs):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs,
temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(timer() -
unpack_start_time))
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" %
(key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
log("Got Override %s = %s from %s. Previous %s" %
(key, value, key_value_str, config_dict[key]))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
run_pass(binary, args, config, config_dict, extracted_apk_dir, dex_dir,
dexen, debugger)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
dex_mode.repackage(extracted_apk_dir, dex_dir, have_locators)
locator_store_id = 1
for module in application_modules:
log('repacking module: ' + module.get_name() + ' with id ' +
str(locator_store_id))
module.repackage(extracted_apk_dir, dex_dir, have_locators,
locator_store_id)
locator_store_id = locator_store_id + 1
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass, args.ignore_zipalign)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map',
'line number map', 'redex-line-number-map')
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map-v2',
'line number map v2', 'redex-line-number-map-v2')
copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats',
'redex-stats.txt')
copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt',
'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'outliner-artifacts.bin',
'outliner artifacts', 'redex-outliner-artifacts.bin')
copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt',
'method id map', 'redex-method-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt',
'class id map', 'redex-class-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'bytecode_offset_map.txt',
'bytecode offset map',
'redex-bytecode-offset-map.txt')
copy_file_to_out_dir(dex_dir, args.out,
'coldstart_fields_in_R_classes.txt',
'resources accessed during coldstart',
'redex-tracked-coldstart-resources.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_dependencies.txt', 'stats',
'redex-class-dependencies.txt')
if config_dict.get('proguard_map_output', '') != '':
merge_proguard_maps(config_dict['proguard_map_output'], args.input_apk,
args.out, dex_dir, args.proguard_map)
else:
assert 'RenameClassesPass' not in passes_list and\
'RenameClassesPassV2' not in passes_list
remove_temp_dirs()
def run_redex(args):
debug_mode = args.unpack_only or args.debug
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
passes_list = config_dict['redex']['passes']
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
libs_dir = join(extracted_apk_dir, 'lib')
temporary_lib_file = join(libs_dir, 'concated_native_libs.so')
if os.path.exists(xz_compressed_libs) and os.path.exists(libs_dir):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.external_summaries="%s"' % args.side_effect_summaries
)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
run_pass(binary,
args,
config,
config_dict,
extracted_apk_dir,
dex_dir,
dexen,
debugger)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
dex_mode.repackage(
extracted_apk_dir, dex_dir, have_locators, fast_repackage=args.dev
)
locator_store_id = 1
for module in application_modules:
log('repacking module: ' + module.get_name() + ' with id ' + str(locator_store_id))
module.repackage(
extracted_apk_dir, dex_dir, have_locators, locator_store_id,
fast_repackage=args.dev
)
locator_store_id = locator_store_id + 1
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass, args.ignore_zipalign, args.page_align_libs)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map-v2', 'line number map v2', 'redex-line-number-map-v2')
copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'outliner-artifacts.bin', 'outliner artifacts', 'redex-outliner-artifacts.bin')
copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt', 'class id map', 'redex-class-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'bytecode_offset_map.txt', 'bytecode offset map', 'redex-bytecode-offset-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'coldstart_fields_in_R_classes.txt', 'resources accessed during coldstart', 'redex-tracked-coldstart-resources.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_dependencies.txt', 'stats', 'redex-class-dependencies.txt')
copy_file_to_out_dir(dex_dir, args.out, 'resid-optres-mapping.json', 'resid map after optres pass', 'redex-resid-optres-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'resid-dedup-mapping.json', 'resid map after dedup pass', 'redex-resid-dedup-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'resid-splitres-mapping.json', 'resid map after split pass', 'redex-resid-splitres-mapping.json')
copy_file_to_out_dir(dex_dir, args.out, 'type-erasure-mappings.txt', 'class map after type erasure pass', 'redex-type-erasure-mappings.txt')
copy_file_to_out_dir(dex_dir, args.out, 'instrument-metadata.txt', 'metadata file for instrumentation', 'redex-instrument-metadata.txt')
copy_file_to_out_dir(dex_dir, args.out, 'cleanup-removed-classes.txt', 'cleanup removed classes', 'redex-cleanup-removed-classes.txt')
copy_file_to_out_dir(dex_dir, args.out, 'opt.txt', 'opt info', 'redex-opt.txt')
if config_dict.get('proguard_map_output', '') != '':
# if our map output strategy is overwrite, we don't merge at all
# if you enable ObfuscatePass, this needs to be overwrite
if config_dict.get('proguard_map_output_strategy', 'merge') == 'overwrite':
overwrite_proguard_maps(
config_dict['proguard_map_output'],
args.out,
dex_dir,
args.proguard_map)
else:
merge_proguard_maps(
config_dict['proguard_map_output'],
args.input_apk,
args.out,
dex_dir,
args.proguard_map)
else:
assert 'RenameClassesPass' not in passes_list and\
'RenameClassesPassV2' not in passes_list
remove_temp_dirs()
def prepare_redex(args):
debug_mode = args.unpack_only or args.debug
extracted_apk_dir = None
dex_dir = None
if args.unpack_only and args.unpack_dest:
if args.unpack_dest[0] == '.':
# Use APK's name
unpack_dir_basename = os.path.splitext(args.input_apk)[0]
else:
unpack_dir_basename = args.unpack_dest[0]
extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
dex_dir = unpack_dir_basename + '.redex_dexen'
try:
os.makedirs(extracted_apk_dir)
os.makedirs(dex_dir)
extracted_apk_dir = os.path.abspath(extracted_apk_dir)
dex_dir = os.path.abspath(dex_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Error: destination directory already exists!')
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit(1)
raise e
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
else:
with open(config) as config_file:
try:
lines = config_file.readlines()
config_dict = json.loads(remove_comments(lines))
except ValueError:
raise ValueError("Invalid JSON in ReDex config file: %s" %
config_file.name)
# stop_pass_idx >= 0 means need stop before a pass and dump intermediate result
stop_pass_idx = -1
if args.stop_pass:
passes_list = config_dict.get('redex', {}).get('passes', [])
stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass)
if not args.output_ir or isfile(args.output_ir):
print('Error: output_ir should be a directory')
sys.exit(1)
try:
os.makedirs(args.output_ir)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
unpack_start_time = timer()
if not extracted_apk_dir:
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
if not dex_dir:
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata',
debug_mode)
for module in application_modules:
canary_prefix = module.get_canary_prefix()
log('found module: ' + module.get_name() + ' ' +
(canary_prefix
if canary_prefix is not None else '(no canary prefix)'))
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir,
module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
libs_to_extract = []
temporary_libs_dir = None
xz_lib_name = 'libs.xzs'
zstd_lib_name = 'libs.zstd'
for root, _, filenames in os.walk(extracted_apk_dir):
for filename in fnmatch.filter(filenames, xz_lib_name):
libs_to_extract.append(join(root, filename))
for filename in fnmatch.filter(filenames, zstd_lib_name):
fullpath = join(root, filename)
# For voltron modules BUCK creates empty zstd files for each module
if os.path.getsize(fullpath) > 0:
libs_to_extract.append(fullpath)
if len(libs_to_extract) > 0:
libs_dir = join(extracted_apk_dir, 'lib')
extracted_dir = join(libs_dir, '__extracted_libs__')
# Ensure both directories exist.
temporary_libs_dir = ensure_libs_dir(libs_dir, extracted_dir)
lib_count = 0
for lib_to_extract in libs_to_extract:
extract_path = join(extracted_dir, "lib_{}.so".format(lib_count))
if lib_to_extract.endswith(xz_lib_name):
cmd = 'xz -d --stdout {} > {}'.format(lib_to_extract,
extract_path)
else:
cmd = 'zstd -d {} -o {}'.format(lib_to_extract, extract_path)
subprocess.check_call(cmd, shell=True)
lib_count += 1
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(timer() -
unpack_start_time))
if args.side_effect_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.side_effect_summaries="%s"' %
args.side_effect_summaries)
if args.escape_summaries is not None:
args.passthru_json.append(
'DeadCodeEliminationPass.escape_summaries="%s"' %
args.escape_summaries)
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" %
(key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
prev_value = config_dict.get(key, "(No previous value)")
log("Got Override %s = %s from %s. Previous %s" %
(key, value, key_value_str, prev_value))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
if args.lldb:
debugger = 'lldb'
elif args.gdb:
debugger = 'gdb'
else:
debugger = None
return State(application_modules=application_modules,
args=args,
config_dict=config_dict,
debugger=debugger,
dex_dir=dex_dir,
dexen=dexen,
dex_mode=dex_mode,
extracted_apk_dir=extracted_apk_dir,
temporary_libs_dir=temporary_libs_dir,
stop_pass_idx=stop_pass_idx)
def run_redex(args):
debug_mode = args.unpack_only or args.debug
unpack_start_time = timer()
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode)
for module in application_modules:
log('found module: ' + module.get_name() + ' ' + module.get_canary_prefix())
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
temporary_lib_file = join(extracted_apk_dir, 'lib/concated_native_libs.so')
if os.path.exists(xz_compressed_libs):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
config_dict = json.load(config_file)
passes_list = config_dict['redex']['passes']
newtmp = tempfile.mkdtemp()
log('Replacing /tmp in config with {}'.format(newtmp))
# Fix up the config dict to relocate all /tmp references
relocate_tmp(config_dict, newtmp)
# Rewrite the relocated config file to our tmp, for use by redex binary
if config is not None:
config = newtmp + "/rewritten.config"
with open(config, 'w') as fp:
json.dump(config_dict, fp)
log('Running redex-all on {} dex files '.format(len(dexen)))
run_pass(binary,
args,
config,
config_dict,
extracted_apk_dir,
dex_dir,
dexen)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
dex_mode.repackage(extracted_apk_dir, dex_dir, have_locators)
for module in application_modules:
log('repacking module: ' + module.get_name())
module.repackage(extracted_apk_dir, dex_dir, have_locators)
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(newtmp, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
copy_file_to_out_dir(newtmp, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
copy_file_to_out_dir(newtmp, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(newtmp, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
if 'RenameClassesPass' in passes_list or 'RenameClassesPassV2' in passes_list:
merge_proguard_map_with_rename_output(
passes_list,
args.input_apk,
args.out,
config_dict,
args.proguard_map)
else:
log('Skipping rename map merging, because we didn\'t run the rename pass')
shutil.rmtree(newtmp)
remove_temp_dirs()
