def update_proguard_mapping_file(pg_map, redex_map, output_file,
should_verify):
with open(pg_map,
'r') as pg_map, open(redex_map,
'r') as redex_map, open(output_file,
'w') as output:
redex_dict = {}
for line in redex_map:
pair = line.split(' -> ')
unmangled = pgize(pair[0])
mangled = pgize(pair[1])
redex_dict[unmangled] = mangled
for line in pg_map:
match_obj = re.match(r'^(.*) -> (.*):', line)
if match_obj:
unmangled = match_obj.group(1)
mangled = match_obj.group(2)
new_mapping = line.rstrip()
if unmangled in redex_dict:
out_mangled = redex_dict.pop(unmangled)
new_mapping = unmangled + ' -> ' + out_mangled + ':'
print(new_mapping, file=output)
else:
print(line.rstrip(), file=output)
else:
print(line.rstrip(), file=output)
if should_verify and len(redex_dict) != 0:
for unmangled in redex_dict.iterkeys():
log('Could not find %s in proguard map' % unmangled)
raise Exception('Error when updating proguard map')
def run_pass(
executable_path,
script_args,
config_path,
config_json,
apk_dir,
dex_dir,
dexfiles,
):
if executable_path is None:
executable_path = shutil.which('redex-all')
if executable_path is None:
executable_path = join(dirname(abspath(__file__)), 'redex-all')
if not isfile(executable_path) or not os.access(executable_path, os.X_OK):
sys.exit('redex-all is not found or is not executable')
log('Running redex binary at ' + executable_path)
args = [executable_path] + [
'--apkdir', apk_dir,
'--outdir', dex_dir]
if config_path:
args += ['--config', config_path]
if script_args.warn:
args += ['--warn', script_args.warn]
if script_args.proguard_config:
args += ['--proguard-config', script_args.proguard_config]
if script_args.keep:
args += ['--seeds', script_args.keep]
if script_args.proguard_map:
args += ['-Sproguard_map=' + script_args.proguard_map]
args += ['--jarpath=' + x for x in script_args.jarpaths]
args += ['-S' + x for x in script_args.passthru]
args += ['-J' + x for x in script_args.passthru_json]
args += dexfiles
start = timer()
if script_args.debug:
print(' '.join(args))
sys.exit()
# Our CI system occasionally fails because it is trying to write the
# redex-all binary when this tries to run. This shouldn't happen, and
# might be caused by a JVM bug. Anyways, let's retry and hope it stops.
for i in range(5):
try:
subprocess.check_call(args)
except OSError as err:
if err.errno == errno.ETXTBSY:
if i < 4:
time.sleep(5)
continue
raise err
break
log('Dex processing finished in {:.2f} seconds'.format(timer() - start))
def copy_file_to_out_dir(tmp, apk_output_path, name, human_name, out_name):
output_dir = os.path.dirname(apk_output_path)
output_path = os.path.join(output_dir, out_name)
tmp_path = tmp + '/' + name
if os.path.isfile(tmp_path):
subprocess.check_call(['cp', tmp_path, output_path])
log('Copying ' + human_name + ' map to output dir')
else:
log('Skipping ' + human_name + ' copy, since no file found to copy')
def relocate_tmp(d, newtmp):
"""
Walks through the config dict and changes and string value that begins with
"/tmp/" to our tmp dir for this run. This is to avoid collisions of
simultaneously running redexes.
"""
for k, v in d.items():
if isinstance(v, dict):
relocate_tmp(v, newtmp)
else:
if (isinstance(v, str) or isinstance(v, unicode)) and v.startswith("/tmp/"):
d[k] = newtmp + "/" + v[5:]
log("Replaced {0} in config with {1}".format(v, d[k]))
def run_pass(
executable_path,
script_args,
config_path,
config_json,
apk_dir,
dex_dir,
dexfiles,
):
if executable_path is None:
executable_path = shutil.which('redex-all')
if executable_path is None:
executable_path = join(dirname(abspath(__file__)), 'redex-all')
if not isfile(executable_path) or not os.access(executable_path, os.X_OK):
sys.exit('redex-all is not found or is not executable')
log('Running redex binary at ' + executable_path)
args = [executable_path] + [
'--apkdir', apk_dir,
'--outdir', dex_dir]
if config_path:
args += ['--config', config_path]
if script_args.warn:
args += ['--warn', script_args.warn]
if script_args.proguard_config:
args += ['--proguard-config', script_args.proguard_config]
if script_args.keep:
args += ['--seeds', script_args.keep]
if script_args.proguard_map:
args += ['-Sproguard_map=' + script_args.proguard_map]
args += ['--jarpath=' + x for x in script_args.jarpaths]
args += ['-S' + x for x in script_args.passthru]
args += ['-J' + x for x in script_args.passthru_json]
args += dexfiles
start = timer()
args = ' '.join(shlex.quote(x) for x in args)
if script_args.time:
args = 'time ' + args
if script_args.debug:
print(args)
sys.exit()
subprocess.check_call(args, shell=True)
log('Dex processing finished in {:.2f} seconds'.format(timer() - start))
def unpackage(self, extracted_apk_dir, dex_dir):
self.dex_mode = XZSDexMode(dex_asset_dir=self.path,
store_name=self.name,
dex_prefix=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies)
if (self.dex_mode.detect(extracted_apk_dir)):
log('module ' + self.name + ' is XZSDexMode')
self.dex_mode.unpackage(extracted_apk_dir, dex_dir)
else:
self.dex_mode = SubdirDexMode(dex_asset_dir=self.path,
store_name=self.name,
dex_prefix=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies)
if (self.dex_mode.detect(extracted_apk_dir)):
log('module ' + self.name + ' is SubdirDexMode')
self.dex_mode.unpackage(extracted_apk_dir, dex_dir)
else:
self.dex_mode = Api21ModuleDexMode(
dex_asset_dir=self.path,
store_name=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies)
log('module ' + self.name + ' is Api21ModuleDexMode')
self.dex_mode.unpackage(extracted_apk_dir, dex_dir)
def unpackage(self, extracted_apk_dir, dex_dir):
self.dex_mode = XZSDexMode(dex_asset_dir=self.path,
store_name=self.name,
dex_prefix=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies)
if (self.dex_mode.detect(extracted_apk_dir)):
log('module ' + self.name + ' is XZSDexMode')
self.dex_mode.unpackage(extracted_apk_dir, dex_dir)
else:
self.dex_mode = SubdirDexMode(dex_asset_dir=self.path,
store_name=self.name,
dex_prefix=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies)
if (self.dex_mode.detect(extracted_apk_dir)):
log('module ' + self.name + ' is SubdirDexMode')
self.dex_mode.unpackage(extracted_apk_dir, dex_dir)
else:
self.dex_mode = Api21ModuleDexMode(dex_asset_dir=self.path,
store_name=self.name,
canary_prefix=self.canary_prefix,
store_id=self.name,
dependencies=self.dependencies)
log('module ' + self.name + ' is Api21ModuleDexMode')
self.dex_mode.unpackage(extracted_apk_dir, dex_dir)
def unpackage(self, extracted_apk_dir, dex_dir):
src = join(extracted_apk_dir, self._xzs_dir, self._xzs_filename)
dest = join(dex_dir, self._xzs_filename)
# Move secondary dexen
shutil.move(src, dest)
# concat_jar is a bunch of .dex.jar files concatenated together.
concat_jar = join(dex_dir, self._xzs_filename[:-4])
cmd = 'cat {} | xz -d --threads 6 > {}'.format(dest, concat_jar)
subprocess.check_call(cmd, shell=True)
dex_order = []
with open(join(extracted_apk_dir, self._xzs_dir,
'metadata.txt')) as dex_metadata:
for line in dex_metadata.read().splitlines():
if line[0] != '.':
tokens = line.split()
search_pattern = self._store_name + '-(\d+)\.dex\.jar\.xzs\.tmp~'
match = re.search(search_pattern, tokens[0])
if match is None:
raise Exception('unable to find match in ' +
tokens[0] + ' for ' + search_pattern)
dex_order.append(int(match.group(1)))
# Sizes of the concatenated .dex.jar files are stored in .meta files.
# Read the sizes of each .dex.jar file and un-concatenate them.
jar_size_regex = 'jar:(\d+)'
secondary_dir = join(extracted_apk_dir, self._xzs_dir)
jar_sizes = {}
for i in dex_order:
filename = self._store_name + '-%d.dex.jar.xzs.tmp~.meta' % i
metadata_path = join(secondary_dir, filename)
if isfile(metadata_path):
with open(metadata_path) as f:
jar_sizes[i] = \
int(re.match(jar_size_regex, f.read()).group(1))
os.remove(metadata_path)
log('found jar ' + filename + ' of size ' + str(jar_sizes[i]))
else:
break
with open(concat_jar, 'rb') as cj:
for i in dex_order:
jarpath = join(dex_dir, self._store_name + '-%d.dex.jar' % i)
with open(jarpath, 'wb') as jar:
jar.write(cj.read(jar_sizes[i]))
for j in jar_sizes.keys():
jar_size = getsize(dex_dir + '/' + self._store_name + '-' +
str(j) + '.dex.jar')
log('validating ' + self._store_name + '-' + str(j) +
'.dex.jar size=' + str(jar_size) + ' expecting=' +
str(jar_sizes[j]))
assert jar_sizes[j] == jar_size
assert sum(jar_sizes.values()) == getsize(concat_jar)
# Clean up everything other than dexen in the dex directory
os.remove(concat_jar)
os.remove(dest)
# Lastly, unzip all the jar files and delete them
for jarpath in abs_glob(dex_dir, '*.jar'):
extract_dex_from_jar(jarpath, jarpath[:-4])
os.remove(jarpath)
BaseDexMode.unpackage(self, extracted_apk_dir, dex_dir)
def run_redex(args):
debug_mode = args.unpack_only or args.debug
unpack_start_time = timer()
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
for module in application_modules:
log('found module: ' + module.get_name() + ' ' + module.get_canary_prefix())
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_files.append(module.write_redex_metadata(store_path))
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
temporary_lib_file = join(extracted_apk_dir, 'lib/concated_native_libs.so')
if os.path.exists(xz_compressed_libs):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(
timer() - unpack_start_time))
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
config_dict = json.load(config_file)
passes_list = config_dict['redex']['passes']
newtmp = tempfile.mkdtemp()
log('Replacing /tmp in config with {}'.format(newtmp))
# Fix up the config dict to relocate all /tmp references
relocate_tmp(config_dict, newtmp)
# Rewrite the relocated config file to our tmp, for use by redex binary
if config is not None:
config = newtmp + "/rewritten.config"
with open(config, 'w') as fp:
json.dump(config_dict, fp)
log('Running redex-all on {} dex files '.format(len(dexen)))
run_pass(binary,
args,
config,
config_dict,
extracted_apk_dir,
dex_dir,
dexen)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
dex_mode.repackage(extracted_apk_dir, dex_dir, have_locators)
for module in application_modules:
log('repacking module: ' + module.get_name())
module.repackage(extracted_apk_dir, dex_dir, have_locators)
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(newtmp, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
copy_file_to_out_dir(newtmp, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
copy_file_to_out_dir(newtmp, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(newtmp, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
if 'RenameClassesPass' in passes_list:
merge_proguard_map_with_rename_output(
args.input_apk,
args.out,
config_dict,
args.proguard_map)
else:
log('Skipping rename map merging, because we didn\'t run the rename pass')
shutil.rmtree(newtmp)
remove_temp_dirs()
def merge_proguard_map_with_rename_output(
input_apk_path,
apk_output_path,
config_dict,
pg_file):
log('running merge proguard step')
redex_rename_map_path = config_dict['RenameClassesPass']['class_rename']
log('redex map is at ' + str(redex_rename_map_path))
if os.path.isfile(redex_rename_map_path):
redex_pg_file = "redex-class-rename-map.txt"
# find proguard file
if pg_file:
output_dir = os.path.dirname(apk_output_path)
output_file = output_file = join(output_dir, redex_pg_file)
update_proguard_mapping_file(pg_file, redex_rename_map_path, output_file)
log('merging proguard map with redex class rename map')
log('pg mapping file input is ' + str(pg_file))
log('wrote redex pg format mapping file to ' + str(output_file))
else:
log('no proguard map file found')
else:
log('Skipping merging of rename maps, since redex rename map file not found')
def run_redex(args):
debug_mode = args.unpack_only or args.debug
unpack_start_time = timer()
extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)
log('Extracting apk...')
unzip_apk(args.input_apk, extracted_apk_dir)
dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
log('Detected dex mode ' + str(type(dex_mode).__name__))
dex_dir = make_temp_dir('.redex_dexen', debug_mode)
log('Unpacking dex files')
dex_mode.unpackage(extracted_apk_dir, dex_dir)
log('Detecting Application Modules')
application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
store_files = []
store_metadata_dir = make_temp_dir('.application_module_metadata',
debug_mode)
for module in application_modules:
log('found module: ' + module.get_name() + ' ' +
module.get_canary_prefix())
store_path = os.path.join(dex_dir, module.get_name())
os.mkdir(store_path)
module.unpackage(extracted_apk_dir, store_path)
store_metadata = os.path.join(store_metadata_dir,
module.get_name() + '.json')
module.write_redex_metadata(store_path, store_metadata)
store_files.append(store_metadata)
# Some of the native libraries can be concatenated together into one
# xz-compressed file. We need to decompress that file so that we can scan
# through it looking for classnames.
xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
temporary_lib_file = join(extracted_apk_dir, 'lib/concated_native_libs.so')
if os.path.exists(xz_compressed_libs):
cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs,
temporary_lib_file)
subprocess.check_call(cmd, shell=True)
if args.unpack_only:
print('APK: ' + extracted_apk_dir)
print('DEX: ' + dex_dir)
sys.exit()
# Move each dex to a separate temporary directory to be operated by
# redex.
dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
for store in store_files:
dexen.append(store)
log('Unpacking APK finished in {:.2f} seconds'.format(timer() -
unpack_start_time))
config = args.config
binary = args.redex_binary
log('Using config ' + (config if config is not None else '(default)'))
log('Using binary ' + (binary if binary is not None else '(default)'))
if config is None:
config_dict = {}
passes_list = []
else:
with open(config) as config_file:
config_dict = json.load(config_file)
passes_list = config_dict['redex']['passes']
for key_value_str in args.passthru_json:
key_value = key_value_str.split('=', 1)
if len(key_value) != 2:
log("Json Pass through %s is not valid. Split len: %s" %
(key_value_str, len(key_value)))
continue
key = key_value[0]
value = key_value[1]
log("Got Override %s = %s from %s. Previous %s" %
(key, value, key_value_str, config_dict[key]))
config_dict[key] = value
log('Running redex-all on {} dex files '.format(len(dexen)))
run_pass(binary, args, config, config_dict, extracted_apk_dir, dex_dir,
dexen)
# This file was just here so we could scan it for classnames, but we don't
# want to pack it back up into the apk
if os.path.exists(temporary_lib_file):
os.remove(temporary_lib_file)
repack_start_time = timer()
log('Repacking dex files')
have_locators = config_dict.get("emit_locator_strings")
log("Emit Locator Strings: %s" % have_locators)
dex_mode.repackage(extracted_apk_dir, dex_dir, have_locators)
for module in application_modules:
log('repacking module: ' + module.get_name())
module.repackage(extracted_apk_dir, dex_dir, have_locators)
log('Creating output apk')
create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
args.keyalias, args.keypass)
log('Creating output APK finished in {:.2f} seconds'.format(
timer() - repack_start_time))
copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map',
'line number map', 'redex-line-number-map')
copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats',
'redex-stats.txt')
copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt',
'src strings map', 'redex-src-strings-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt',
'method id map', 'redex-method-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt',
'class id map', 'redex-class-id-map.txt')
copy_file_to_out_dir(dex_dir, args.out,
'coldstart_fields_in_R_classes.txt',
'resources accessed during coldstart',
'redex-tracked-coldstart-resources.txt')
if 'RenameClassesPass' in passes_list or 'RenameClassesPassV2' in passes_list:
merge_proguard_map_with_rename_output(passes_list, args.input_apk,
args.out, dex_dir, config_dict,
args.proguard_map)
else:
log('Skipping rename map merging, because we didn\'t run the rename pass'
)
remove_temp_dirs()
def merge_proguard_map_with_rename_output(passes_list, input_apk_path,
apk_output_path, dex_dir,
config_dict, pg_file):
log('running merge proguard step')
if 'RenameClassesPass' in passes_list:
redex_rename_map_path = config_dict['RenameClassesPass'][
'class_rename']
elif 'RenameClassesPassV2' in passes_list:
redex_rename_map_path = config_dict['RenameClassesPassV2'][
'class_rename']
else:
raise ValueError(
"merge_proguard_map_with_rename_output called without a rename classes pass"
)
redex_rename_map_path = join(dex_dir, redex_rename_map_path)
log('redex map is at ' + str(redex_rename_map_path))
log('pg map is at ' + str(pg_file))
if os.path.isfile(redex_rename_map_path):
redex_pg_file = "redex-class-rename-map.txt"
# If -dontobfuscate is set, proguard won't produce a mapping file, but
# buck will create an empty mapping.txt. Check for this case.
if pg_file and os.path.getsize(pg_file) > 0:
output_dir = os.path.dirname(apk_output_path)
output_file = output_file = join(output_dir, redex_pg_file)
update_proguard_mapping_file(pg_file,
redex_rename_map_path,
output_file,
should_verify='RenameClassesPassV2'
in passes_list)
log('merging proguard map with redex class rename map')
log('pg mapping file input is ' + str(pg_file))
log('wrote redex pg format mapping file to ' + str(output_file))
else:
log('no proguard map file found')
else:
log('Skipping merging of rename maps, since redex rename map file not found'
)
def merge_proguard_maps(
redex_rename_map_path,
input_apk_path,
apk_output_path,
dex_dir,
pg_file):
log('running merge proguard step')
redex_rename_map_path = join(dex_dir, redex_rename_map_path)
log('redex map is at ' + str(redex_rename_map_path))
log('pg map is at ' + str(pg_file))
assert os.path.isfile(redex_rename_map_path)
redex_pg_file = "redex-class-rename-map.txt"
output_dir = os.path.dirname(apk_output_path)
output_file = join(output_dir, redex_pg_file)
# If -dontobfuscate is set, proguard won't produce a mapping file, but
# buck will create an empty mapping.txt. Check for this case.
if pg_file and os.path.getsize(pg_file) > 0:
update_proguard_mapping_file(
pg_file,
redex_rename_map_path,
output_file)
log('merging proguard map with redex class rename map')
log('pg mapping file input is ' + str(pg_file))
log('wrote redex pg format mapping file to ' + str(output_file))
else:
log('no proguard map file found')
shutil.move(redex_rename_map_path, output_file)
def run_pass(
executable_path,
script_args,
config_path,
config_json,
apk_dir,
dex_dir,
dexfiles,
debugger,
):
if executable_path is None:
try:
executable_path = subprocess.check_output(['which',
'redex-all']).rstrip()
except subprocess.CalledProcessError:
pass
if executable_path is None:
# __file__ can be /path/fb-redex.pex/redex.pyc
dir_name = dirname(abspath(__file__))
while not isdir(dir_name):
dir_name = dirname(dir_name)
executable_path = join(dir_name, 'redex-all')
if not isfile(executable_path) or not os.access(executable_path, os.X_OK):
sys.exit('redex-all is not found or is not executable')
log('Running redex binary at ' + executable_path)
args = [executable_path] + ['--apkdir', apk_dir, '--outdir', dex_dir]
if config_path:
args += ['--config', config_path]
if script_args.warn:
args += ['--warn', script_args.warn]
args += ['--proguard-config=' + x for x in script_args.proguard_configs]
if script_args.keep:
args += ['--seeds', script_args.keep]
if script_args.proguard_map:
args += ['-Sproguard_map=' + script_args.proguard_map]
args += ['--jarpath=' + x for x in script_args.jarpaths]
if script_args.printseeds:
args += ['--printseeds=' + script_args.printseeds]
args += ['-S' + x for x in script_args.passthru]
args += ['-J' + x for x in script_args.passthru_json]
args += dexfiles
if debugger == 'lldb':
args = ['lldb', '--'] + args
elif debugger == 'gdb':
args = ['gdb', '--args'] + args
start = timer()
if script_args.debug:
print(' '.join(args))
sys.exit()
# Our CI system occasionally fails because it is trying to write the
# redex-all binary when this tries to run. This shouldn't happen, and
# might be caused by a JVM bug. Anyways, let's retry and hope it stops.
for i in range(5):
try:
subprocess.check_call(args)
except OSError as err:
if err.errno == errno.ETXTBSY:
if i < 4:
time.sleep(5)
continue
raise err
except subprocess.CalledProcessError as err:
err.cmd = ' '.join(args)
raise err
break
log('Dex processing finished in {:.2f} seconds'.format(timer() - start))
