def update_proguard_mapping_file(pg_map, redex_map, output_file, should_verify): with open(pg_map, 'r') as pg_map, open(redex_map, 'r') as redex_map, open(output_file, 'w') as output: redex_dict = {} for line in redex_map: pair = line.split(' -> ') unmangled = pgize(pair[0]) mangled = pgize(pair[1]) redex_dict[unmangled] = mangled for line in pg_map: match_obj = re.match(r'^(.*) -> (.*):', line) if match_obj: unmangled = match_obj.group(1) mangled = match_obj.group(2) new_mapping = line.rstrip() if unmangled in redex_dict: out_mangled = redex_dict.pop(unmangled) new_mapping = unmangled + ' -> ' + out_mangled + ':' print(new_mapping, file=output) else: print(line.rstrip(), file=output) else: print(line.rstrip(), file=output) if should_verify and len(redex_dict) != 0: for unmangled in redex_dict.iterkeys(): log('Could not find %s in proguard map' % unmangled) raise Exception('Error when updating proguard map')
def run_pass( executable_path, script_args, config_path, config_json, apk_dir, dex_dir, dexfiles, ): if executable_path is None: executable_path = shutil.which('redex-all') if executable_path is None: executable_path = join(dirname(abspath(__file__)), 'redex-all') if not isfile(executable_path) or not os.access(executable_path, os.X_OK): sys.exit('redex-all is not found or is not executable') log('Running redex binary at ' + executable_path) args = [executable_path] + [ '--apkdir', apk_dir, '--outdir', dex_dir] if config_path: args += ['--config', config_path] if script_args.warn: args += ['--warn', script_args.warn] if script_args.proguard_config: args += ['--proguard-config', script_args.proguard_config] if script_args.keep: args += ['--seeds', script_args.keep] if script_args.proguard_map: args += ['-Sproguard_map=' + script_args.proguard_map] args += ['--jarpath=' + x for x in script_args.jarpaths] args += ['-S' + x for x in script_args.passthru] args += ['-J' + x for x in script_args.passthru_json] args += dexfiles start = timer() if script_args.debug: print(' '.join(args)) sys.exit() # Our CI system occasionally fails because it is trying to write the # redex-all binary when this tries to run. This shouldn't happen, and # might be caused by a JVM bug. Anyways, let's retry and hope it stops. for i in range(5): try: subprocess.check_call(args) except OSError as err: if err.errno == errno.ETXTBSY: if i < 4: time.sleep(5) continue raise err break log('Dex processing finished in {:.2f} seconds'.format(timer() - start))
def copy_file_to_out_dir(tmp, apk_output_path, name, human_name, out_name): output_dir = os.path.dirname(apk_output_path) output_path = os.path.join(output_dir, out_name) tmp_path = tmp + '/' + name if os.path.isfile(tmp_path): subprocess.check_call(['cp', tmp_path, output_path]) log('Copying ' + human_name + ' map to output dir') else: log('Skipping ' + human_name + ' copy, since no file found to copy')
def relocate_tmp(d, newtmp): """ Walks through the config dict and changes and string value that begins with "/tmp/" to our tmp dir for this run. This is to avoid collisions of simultaneously running redexes. """ for k, v in d.items(): if isinstance(v, dict): relocate_tmp(v, newtmp) else: if (isinstance(v, str) or isinstance(v, unicode)) and v.startswith("/tmp/"): d[k] = newtmp + "/" + v[5:] log("Replaced {0} in config with {1}".format(v, d[k]))
def run_pass( executable_path, script_args, config_path, config_json, apk_dir, dex_dir, dexfiles, ): if executable_path is None: executable_path = shutil.which('redex-all') if executable_path is None: executable_path = join(dirname(abspath(__file__)), 'redex-all') if not isfile(executable_path) or not os.access(executable_path, os.X_OK): sys.exit('redex-all is not found or is not executable') log('Running redex binary at ' + executable_path) args = [executable_path] + [ '--apkdir', apk_dir, '--outdir', dex_dir] if config_path: args += ['--config', config_path] if script_args.warn: args += ['--warn', script_args.warn] if script_args.proguard_config: args += ['--proguard-config', script_args.proguard_config] if script_args.keep: args += ['--seeds', script_args.keep] if script_args.proguard_map: args += ['-Sproguard_map=' + script_args.proguard_map] args += ['--jarpath=' + x for x in script_args.jarpaths] args += ['-S' + x for x in script_args.passthru] args += ['-J' + x for x in script_args.passthru_json] args += dexfiles start = timer() args = ' '.join(shlex.quote(x) for x in args) if script_args.time: args = 'time ' + args if script_args.debug: print(args) sys.exit() subprocess.check_call(args, shell=True) log('Dex processing finished in {:.2f} seconds'.format(timer() - start))
def unpackage(self, extracted_apk_dir, dex_dir): self.dex_mode = XZSDexMode(dex_asset_dir=self.path, store_name=self.name, dex_prefix=self.name, canary_prefix=self.canary_prefix, store_id=self.name, dependencies=self.dependencies) if (self.dex_mode.detect(extracted_apk_dir)): log('module ' + self.name + ' is XZSDexMode') self.dex_mode.unpackage(extracted_apk_dir, dex_dir) else: self.dex_mode = SubdirDexMode(dex_asset_dir=self.path, store_name=self.name, dex_prefix=self.name, canary_prefix=self.canary_prefix, store_id=self.name, dependencies=self.dependencies) if (self.dex_mode.detect(extracted_apk_dir)): log('module ' + self.name + ' is SubdirDexMode') self.dex_mode.unpackage(extracted_apk_dir, dex_dir) else: self.dex_mode = Api21ModuleDexMode( dex_asset_dir=self.path, store_name=self.name, canary_prefix=self.canary_prefix, store_id=self.name, dependencies=self.dependencies) log('module ' + self.name + ' is Api21ModuleDexMode') self.dex_mode.unpackage(extracted_apk_dir, dex_dir)
def unpackage(self, extracted_apk_dir, dex_dir): self.dex_mode = XZSDexMode(dex_asset_dir=self.path, store_name=self.name, dex_prefix=self.name, canary_prefix=self.canary_prefix, store_id=self.name, dependencies=self.dependencies) if (self.dex_mode.detect(extracted_apk_dir)): log('module ' + self.name + ' is XZSDexMode') self.dex_mode.unpackage(extracted_apk_dir, dex_dir) else: self.dex_mode = SubdirDexMode(dex_asset_dir=self.path, store_name=self.name, dex_prefix=self.name, canary_prefix=self.canary_prefix, store_id=self.name, dependencies=self.dependencies) if (self.dex_mode.detect(extracted_apk_dir)): log('module ' + self.name + ' is SubdirDexMode') self.dex_mode.unpackage(extracted_apk_dir, dex_dir) else: self.dex_mode = Api21ModuleDexMode(dex_asset_dir=self.path, store_name=self.name, canary_prefix=self.canary_prefix, store_id=self.name, dependencies=self.dependencies) log('module ' + self.name + ' is Api21ModuleDexMode') self.dex_mode.unpackage(extracted_apk_dir, dex_dir)
def unpackage(self, extracted_apk_dir, dex_dir): src = join(extracted_apk_dir, self._xzs_dir, self._xzs_filename) dest = join(dex_dir, self._xzs_filename) # Move secondary dexen shutil.move(src, dest) # concat_jar is a bunch of .dex.jar files concatenated together. concat_jar = join(dex_dir, self._xzs_filename[:-4]) cmd = 'cat {} | xz -d --threads 6 > {}'.format(dest, concat_jar) subprocess.check_call(cmd, shell=True) dex_order = [] with open(join(extracted_apk_dir, self._xzs_dir, 'metadata.txt')) as dex_metadata: for line in dex_metadata.read().splitlines(): if line[0] != '.': tokens = line.split() search_pattern = self._store_name + '-(\d+)\.dex\.jar\.xzs\.tmp~' match = re.search(search_pattern, tokens[0]) if match is None: raise Exception('unable to find match in ' + tokens[0] + ' for ' + search_pattern) dex_order.append(int(match.group(1))) # Sizes of the concatenated .dex.jar files are stored in .meta files. # Read the sizes of each .dex.jar file and un-concatenate them. jar_size_regex = 'jar:(\d+)' secondary_dir = join(extracted_apk_dir, self._xzs_dir) jar_sizes = {} for i in dex_order: filename = self._store_name + '-%d.dex.jar.xzs.tmp~.meta' % i metadata_path = join(secondary_dir, filename) if isfile(metadata_path): with open(metadata_path) as f: jar_sizes[i] = \ int(re.match(jar_size_regex, f.read()).group(1)) os.remove(metadata_path) log('found jar ' + filename + ' of size ' + str(jar_sizes[i])) else: break with open(concat_jar, 'rb') as cj: for i in dex_order: jarpath = join(dex_dir, self._store_name + '-%d.dex.jar' % i) with open(jarpath, 'wb') as jar: jar.write(cj.read(jar_sizes[i])) for j in jar_sizes.keys(): jar_size = getsize(dex_dir + '/' + self._store_name + '-' + str(j) + '.dex.jar') log('validating ' + self._store_name + '-' + str(j) + '.dex.jar size=' + str(jar_size) + ' expecting=' + str(jar_sizes[j])) assert jar_sizes[j] == jar_size assert sum(jar_sizes.values()) == getsize(concat_jar) # Clean up everything other than dexen in the dex directory os.remove(concat_jar) os.remove(dest) # Lastly, unzip all the jar files and delete them for jarpath in abs_glob(dex_dir, '*.jar'): extract_dex_from_jar(jarpath, jarpath[:-4]) os.remove(jarpath) BaseDexMode.unpackage(self, extracted_apk_dir, dex_dir)
def run_redex(args): debug_mode = args.unpack_only or args.debug unpack_start_time = timer() extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode) log('Extracting apk...') unzip_apk(args.input_apk, extracted_apk_dir) dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir) log('Detected dex mode ' + str(type(dex_mode).__name__)) dex_dir = make_temp_dir('.redex_dexen', debug_mode) log('Unpacking dex files') dex_mode.unpackage(extracted_apk_dir, dex_dir) log('Detecting Application Modules') application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir) store_files = [] for module in application_modules: log('found module: ' + module.get_name() + ' ' + module.get_canary_prefix()) store_path = os.path.join(dex_dir, module.get_name()) os.mkdir(store_path) module.unpackage(extracted_apk_dir, store_path) store_files.append(module.write_redex_metadata(store_path)) # Some of the native libraries can be concatenated together into one # xz-compressed file. We need to decompress that file so that we can scan # through it looking for classnames. xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs') temporary_lib_file = join(extracted_apk_dir, 'lib/concated_native_libs.so') if os.path.exists(xz_compressed_libs): cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file) subprocess.check_call(cmd, shell=True) if args.unpack_only: print('APK: ' + extracted_apk_dir) print('DEX: ' + dex_dir) sys.exit() # Move each dex to a separate temporary directory to be operated by # redex. dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir)) for store in store_files: dexen.append(store) log('Unpacking APK finished in {:.2f} seconds'.format( timer() - unpack_start_time)) config = args.config binary = args.redex_binary log('Using config ' + (config if config is not None else '(default)')) log('Using binary ' + (binary if binary is not None else '(default)')) if config is None: config_dict = {} passes_list = [] else: with open(config) as config_file: config_dict = json.load(config_file) passes_list = config_dict['redex']['passes'] newtmp = tempfile.mkdtemp() log('Replacing /tmp in config with {}'.format(newtmp)) # Fix up the config dict to relocate all /tmp references relocate_tmp(config_dict, newtmp) # Rewrite the relocated config file to our tmp, for use by redex binary if config is not None: config = newtmp + "/rewritten.config" with open(config, 'w') as fp: json.dump(config_dict, fp) log('Running redex-all on {} dex files '.format(len(dexen))) run_pass(binary, args, config, config_dict, extracted_apk_dir, dex_dir, dexen) # This file was just here so we could scan it for classnames, but we don't # want to pack it back up into the apk if os.path.exists(temporary_lib_file): os.remove(temporary_lib_file) repack_start_time = timer() log('Repacking dex files') have_locators = config_dict.get("emit_locator_strings") dex_mode.repackage(extracted_apk_dir, dex_dir, have_locators) for module in application_modules: log('repacking module: ' + module.get_name()) module.repackage(extracted_apk_dir, dex_dir, have_locators) log('Creating output apk') create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore, args.keyalias, args.keypass) log('Creating output APK finished in {:.2f} seconds'.format( timer() - repack_start_time)) copy_file_to_out_dir(newtmp, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map') copy_file_to_out_dir(newtmp, args.out, 'stats.txt', 'stats', 'redex-stats.txt') copy_file_to_out_dir(newtmp, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt') copy_file_to_out_dir(newtmp, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt') if 'RenameClassesPass' in passes_list: merge_proguard_map_with_rename_output( args.input_apk, args.out, config_dict, args.proguard_map) else: log('Skipping rename map merging, because we didn\'t run the rename pass') shutil.rmtree(newtmp) remove_temp_dirs()
def merge_proguard_map_with_rename_output( input_apk_path, apk_output_path, config_dict, pg_file): log('running merge proguard step') redex_rename_map_path = config_dict['RenameClassesPass']['class_rename'] log('redex map is at ' + str(redex_rename_map_path)) if os.path.isfile(redex_rename_map_path): redex_pg_file = "redex-class-rename-map.txt" # find proguard file if pg_file: output_dir = os.path.dirname(apk_output_path) output_file = output_file = join(output_dir, redex_pg_file) update_proguard_mapping_file(pg_file, redex_rename_map_path, output_file) log('merging proguard map with redex class rename map') log('pg mapping file input is ' + str(pg_file)) log('wrote redex pg format mapping file to ' + str(output_file)) else: log('no proguard map file found') else: log('Skipping merging of rename maps, since redex rename map file not found')
def run_redex(args): debug_mode = args.unpack_only or args.debug unpack_start_time = timer() extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode) log('Extracting apk...') unzip_apk(args.input_apk, extracted_apk_dir) dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir) log('Detected dex mode ' + str(type(dex_mode).__name__)) dex_dir = make_temp_dir('.redex_dexen', debug_mode) log('Unpacking dex files') dex_mode.unpackage(extracted_apk_dir, dex_dir) log('Detecting Application Modules') application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir) store_files = [] store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode) for module in application_modules: log('found module: ' + module.get_name() + ' ' + module.get_canary_prefix()) store_path = os.path.join(dex_dir, module.get_name()) os.mkdir(store_path) module.unpackage(extracted_apk_dir, store_path) store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json') module.write_redex_metadata(store_path, store_metadata) store_files.append(store_metadata) # Some of the native libraries can be concatenated together into one # xz-compressed file. We need to decompress that file so that we can scan # through it looking for classnames. xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs') temporary_lib_file = join(extracted_apk_dir, 'lib/concated_native_libs.so') if os.path.exists(xz_compressed_libs): cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file) subprocess.check_call(cmd, shell=True) if args.unpack_only: print('APK: ' + extracted_apk_dir) print('DEX: ' + dex_dir) sys.exit() # Move each dex to a separate temporary directory to be operated by # redex. dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir)) for store in store_files: dexen.append(store) log('Unpacking APK finished in {:.2f} seconds'.format(timer() - unpack_start_time)) config = args.config binary = args.redex_binary log('Using config ' + (config if config is not None else '(default)')) log('Using binary ' + (binary if binary is not None else '(default)')) if config is None: config_dict = {} passes_list = [] else: with open(config) as config_file: config_dict = json.load(config_file) passes_list = config_dict['redex']['passes'] for key_value_str in args.passthru_json: key_value = key_value_str.split('=', 1) if len(key_value) != 2: log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value))) continue key = key_value[0] value = key_value[1] log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, config_dict[key])) config_dict[key] = value log('Running redex-all on {} dex files '.format(len(dexen))) run_pass(binary, args, config, config_dict, extracted_apk_dir, dex_dir, dexen) # This file was just here so we could scan it for classnames, but we don't # want to pack it back up into the apk if os.path.exists(temporary_lib_file): os.remove(temporary_lib_file) repack_start_time = timer() log('Repacking dex files') have_locators = config_dict.get("emit_locator_strings") log("Emit Locator Strings: %s" % have_locators) dex_mode.repackage(extracted_apk_dir, dex_dir, have_locators) for module in application_modules: log('repacking module: ' + module.get_name()) module.repackage(extracted_apk_dir, dex_dir, have_locators) log('Creating output apk') create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore, args.keyalias, args.keypass) log('Creating output APK finished in {:.2f} seconds'.format( timer() - repack_start_time)) copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map') copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats', 'redex-stats.txt') copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt') copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt') copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt', 'class id map', 'redex-class-id-map.txt') copy_file_to_out_dir(dex_dir, args.out, 'coldstart_fields_in_R_classes.txt', 'resources accessed during coldstart', 'redex-tracked-coldstart-resources.txt') if 'RenameClassesPass' in passes_list or 'RenameClassesPassV2' in passes_list: merge_proguard_map_with_rename_output(passes_list, args.input_apk, args.out, dex_dir, config_dict, args.proguard_map) else: log('Skipping rename map merging, because we didn\'t run the rename pass' ) remove_temp_dirs()
def merge_proguard_map_with_rename_output(passes_list, input_apk_path, apk_output_path, dex_dir, config_dict, pg_file): log('running merge proguard step') if 'RenameClassesPass' in passes_list: redex_rename_map_path = config_dict['RenameClassesPass'][ 'class_rename'] elif 'RenameClassesPassV2' in passes_list: redex_rename_map_path = config_dict['RenameClassesPassV2'][ 'class_rename'] else: raise ValueError( "merge_proguard_map_with_rename_output called without a rename classes pass" ) redex_rename_map_path = join(dex_dir, redex_rename_map_path) log('redex map is at ' + str(redex_rename_map_path)) log('pg map is at ' + str(pg_file)) if os.path.isfile(redex_rename_map_path): redex_pg_file = "redex-class-rename-map.txt" # If -dontobfuscate is set, proguard won't produce a mapping file, but # buck will create an empty mapping.txt. Check for this case. if pg_file and os.path.getsize(pg_file) > 0: output_dir = os.path.dirname(apk_output_path) output_file = output_file = join(output_dir, redex_pg_file) update_proguard_mapping_file(pg_file, redex_rename_map_path, output_file, should_verify='RenameClassesPassV2' in passes_list) log('merging proguard map with redex class rename map') log('pg mapping file input is ' + str(pg_file)) log('wrote redex pg format mapping file to ' + str(output_file)) else: log('no proguard map file found') else: log('Skipping merging of rename maps, since redex rename map file not found' )
def merge_proguard_maps( redex_rename_map_path, input_apk_path, apk_output_path, dex_dir, pg_file): log('running merge proguard step') redex_rename_map_path = join(dex_dir, redex_rename_map_path) log('redex map is at ' + str(redex_rename_map_path)) log('pg map is at ' + str(pg_file)) assert os.path.isfile(redex_rename_map_path) redex_pg_file = "redex-class-rename-map.txt" output_dir = os.path.dirname(apk_output_path) output_file = join(output_dir, redex_pg_file) # If -dontobfuscate is set, proguard won't produce a mapping file, but # buck will create an empty mapping.txt. Check for this case. if pg_file and os.path.getsize(pg_file) > 0: update_proguard_mapping_file( pg_file, redex_rename_map_path, output_file) log('merging proguard map with redex class rename map') log('pg mapping file input is ' + str(pg_file)) log('wrote redex pg format mapping file to ' + str(output_file)) else: log('no proguard map file found') shutil.move(redex_rename_map_path, output_file)
def run_pass( executable_path, script_args, config_path, config_json, apk_dir, dex_dir, dexfiles, debugger, ): if executable_path is None: try: executable_path = subprocess.check_output(['which', 'redex-all']).rstrip() except subprocess.CalledProcessError: pass if executable_path is None: # __file__ can be /path/fb-redex.pex/redex.pyc dir_name = dirname(abspath(__file__)) while not isdir(dir_name): dir_name = dirname(dir_name) executable_path = join(dir_name, 'redex-all') if not isfile(executable_path) or not os.access(executable_path, os.X_OK): sys.exit('redex-all is not found or is not executable') log('Running redex binary at ' + executable_path) args = [executable_path] + ['--apkdir', apk_dir, '--outdir', dex_dir] if config_path: args += ['--config', config_path] if script_args.warn: args += ['--warn', script_args.warn] args += ['--proguard-config=' + x for x in script_args.proguard_configs] if script_args.keep: args += ['--seeds', script_args.keep] if script_args.proguard_map: args += ['-Sproguard_map=' + script_args.proguard_map] args += ['--jarpath=' + x for x in script_args.jarpaths] if script_args.printseeds: args += ['--printseeds=' + script_args.printseeds] args += ['-S' + x for x in script_args.passthru] args += ['-J' + x for x in script_args.passthru_json] args += dexfiles if debugger == 'lldb': args = ['lldb', '--'] + args elif debugger == 'gdb': args = ['gdb', '--args'] + args start = timer() if script_args.debug: print(' '.join(args)) sys.exit() # Our CI system occasionally fails because it is trying to write the # redex-all binary when this tries to run. This shouldn't happen, and # might be caused by a JVM bug. Anyways, let's retry and hope it stops. for i in range(5): try: subprocess.check_call(args) except OSError as err: if err.errno == errno.ETXTBSY: if i < 4: time.sleep(5) continue raise err except subprocess.CalledProcessError as err: err.cmd = ' '.join(args) raise err break log('Dex processing finished in {:.2f} seconds'.format(timer() - start))