def prepare_redex(args): debug_mode = args.unpack_only or args.debug # avoid accidentally mixing up file formats since we now support # both apk files and Android bundle files if not args.unpack_only: assert get_file_ext(args.input_apk) == get_file_ext(args.out),\ "Input file extension (\"" +\ get_file_ext(args.input_apk) +\ "\") should be the same as output file extension (\"" +\ get_file_ext(args.out) + "\")" extracted_apk_dir = None dex_dir = None if args.unpack_only and args.unpack_dest: if args.unpack_dest[0] == '.': # Use APK's name unpack_dir_basename = os.path.splitext(args.input_apk)[0] else: unpack_dir_basename = args.unpack_dest[0] extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk' dex_dir = unpack_dir_basename + '.redex_dexen' try: os.makedirs(extracted_apk_dir) os.makedirs(dex_dir) extracted_apk_dir = os.path.abspath(extracted_apk_dir) dex_dir = os.path.abspath(dex_dir) except OSError as e: if e.errno == errno.EEXIST: print('Error: destination directory already exists!') print('APK: ' + extracted_apk_dir) print('DEX: ' + dex_dir) sys.exit(1) raise e config = args.config binary = args.redex_binary log('Using config ' + (config if config is not None else '(default)')) log('Using binary ' + (binary if binary is not None else '(default)')) if args.unpack_only or config is None: config_dict = {} else: with open(config) as config_file: try: lines = config_file.readlines() config_dict = json.loads(remove_comments(lines)) except ValueError: raise ValueError("Invalid JSON in ReDex config file: %s" % config_file.name) # stop_pass_idx >= 0 means need stop before a pass and dump intermediate result stop_pass_idx = -1 if args.stop_pass: passes_list = config_dict.get('redex', {}).get('passes', []) stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass) if not args.output_ir or isfile(args.output_ir): print('Error: output_ir should be a directory') sys.exit(1) try: os.makedirs(args.output_ir) except OSError as e: if e.errno != errno.EEXIST: raise e unpack_start_time = timer() if not extracted_apk_dir: extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode) log('Extracting apk...') unzip_apk(args.input_apk, extracted_apk_dir) dex_file_path = get_dex_file_path(args, extracted_apk_dir) dex_mode = unpacker.detect_secondary_dex_mode(dex_file_path) log('Detected dex mode ' + str(type(dex_mode).__name__)) if not dex_dir: dex_dir = make_temp_dir('.redex_dexen', debug_mode) log('Unpacking dex files') dex_mode.unpackage(dex_file_path, dex_dir) log('Detecting Application Modules') application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir) store_files = [] store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode) for module in application_modules: canary_prefix = module.get_canary_prefix() log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)')) store_path = os.path.join(dex_dir, module.get_name()) os.mkdir(store_path) module.unpackage(extracted_apk_dir, store_path) store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json') module.write_redex_metadata(store_path, store_metadata) store_files.append(store_metadata) # Some of the native libraries can be concatenated together into one # xz-compressed file. We need to decompress that file so that we can scan # through it looking for classnames. libs_to_extract = [] temporary_libs_dir = None xz_lib_name = 'libs.xzs' zstd_lib_name = 'libs.zstd' for root, _, filenames in os.walk(extracted_apk_dir): for filename in fnmatch.filter(filenames, xz_lib_name): libs_to_extract.append(join(root, filename)) for filename in fnmatch.filter(filenames, zstd_lib_name): fullpath = join(root, filename) # For voltron modules BUCK creates empty zstd files for each module if os.path.getsize(fullpath) > 0: libs_to_extract.append(fullpath) if len(libs_to_extract) > 0: libs_dir = join(extracted_apk_dir, 'lib') extracted_dir = join(libs_dir, '__extracted_libs__') # Ensure both directories exist. temporary_libs_dir = ensure_libs_dir(libs_dir, extracted_dir) lib_count = 0 for lib_to_extract in libs_to_extract: extract_path = join(extracted_dir, "lib_{}.so".format(lib_count)) if lib_to_extract.endswith(xz_lib_name): cmd = 'xz -d --stdout {} > {}'.format(lib_to_extract, extract_path) else: cmd = 'zstd -d {} -o {}'.format(lib_to_extract, extract_path) subprocess.check_call(cmd, shell=True) lib_count += 1 if args.unpack_only: print('APK: ' + extracted_apk_dir) print('DEX: ' + dex_dir) sys.exit() # Move each dex to a separate temporary directory to be operated by # redex. dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir)) for store in store_files: dexen.append(store) log('Unpacking APK finished in {:.2f} seconds'.format(timer() - unpack_start_time)) if args.side_effect_summaries is not None: args.passthru_json.append( 'ObjectSensitiveDcePass.side_effect_summaries="%s"' % args.side_effect_summaries) if args.escape_summaries is not None: args.passthru_json.append( 'ObjectSensitiveDcePass.escape_summaries="%s"' % args.escape_summaries) for key_value_str in args.passthru_json: key_value = key_value_str.split('=', 1) if len(key_value) != 2: log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value))) continue key = key_value[0] value = key_value[1] prev_value = config_dict.get(key, "(No previous value)") log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value)) config_dict[key] = json.loads(value) log('Running redex-all on {} dex files '.format(len(dexen))) if args.lldb: debugger = 'lldb' elif args.gdb: debugger = 'gdb' else: debugger = None return State(application_modules=application_modules, args=args, config_dict=config_dict, debugger=debugger, dex_dir=dex_dir, dexen=dexen, dex_mode=dex_mode, extracted_apk_dir=extracted_apk_dir, temporary_libs_dir=temporary_libs_dir, stop_pass_idx=stop_pass_idx)
def prepare_redex(args): logging.debug("Preparing...") debug_mode = args.unpack_only or args.debug if args.android_sdk_path: add_android_sdk_path(args.android_sdk_path) # avoid accidentally mixing up file formats since we now support # both apk files and Android bundle files file_ext = get_file_ext(args.input_apk) if not args.unpack_only: assert file_ext == get_file_ext( args.out), ('Input file extension ("' + file_ext + '") should be the same as output file extension ("' + get_file_ext(args.out) + '")') extracted_apk_dir = None dex_dir = None if args.unpack_only and args.unpack_dest: if args.unpack_dest[0] == ".": # Use APK's name unpack_dir_basename = os.path.splitext(args.input_apk)[0] else: unpack_dir_basename = args.unpack_dest[0] extracted_apk_dir = unpack_dir_basename + ".redex_extracted_apk" dex_dir = unpack_dir_basename + ".redex_dexen" try: os.makedirs(extracted_apk_dir) os.makedirs(dex_dir) extracted_apk_dir = os.path.abspath(extracted_apk_dir) dex_dir = os.path.abspath(dex_dir) except OSError as e: if e.errno == errno.EEXIST: print("Error: destination directory already exists!") print("APK: " + extracted_apk_dir) print("DEX: " + dex_dir) sys.exit(1) raise e config = args.config binary = args.redex_binary logging.debug("Using config %s", config if config is not None else "(default)") logging.debug("Using binary %s", binary if binary is not None else "(default)") if args.unpack_only or config is None: config_dict = {} else: with open(config) as config_file: try: lines = config_file.readlines() config_dict = json.loads(remove_comments(lines)) except ValueError: raise ValueError("Invalid JSON in ReDex config file: %s" % config_file.name) # stop_pass_idx >= 0 means need stop before a pass and dump intermediate result stop_pass_idx = -1 if args.stop_pass: passes_list = config_dict.get("redex", {}).get("passes", []) stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass) if not args.output_ir or isfile(args.output_ir): print("Error: output_ir should be a directory") sys.exit(1) try: os.makedirs(args.output_ir) except OSError as e: if e.errno != errno.EEXIST: raise e logging.debug("Unpacking...") unpack_start_time = timer() if not extracted_apk_dir: extracted_apk_dir = make_temp_dir(".redex_extracted_apk", debug_mode) directory = make_temp_dir(".redex_unaligned", False) unaligned_apk_path = join(directory, "redex-unaligned." + file_ext) zip_manager = ZipManager(args.input_apk, extracted_apk_dir, unaligned_apk_path) zip_manager.__enter__() if not dex_dir: dex_dir = make_temp_dir(".redex_dexen", debug_mode) is_bundle = isfile(join(extracted_apk_dir, "BundleConfig.pb")) unpack_manager = UnpackManager( args.input_apk, extracted_apk_dir, dex_dir, have_locators=config_dict.get("emit_locator_strings"), debug_mode=debug_mode, fast_repackage=args.dev, reset_timestamps=args.reset_zip_timestamps or args.dev, is_bundle=is_bundle, ) store_files = unpack_manager.__enter__() lib_manager = LibraryManager(extracted_apk_dir, is_bundle=is_bundle) lib_manager.__enter__() if args.unpack_only: print("APK: " + extracted_apk_dir) print("DEX: " + dex_dir) sys.exit() # Unpack profiles, if they exist. _handle_profiles(args, debug_mode) logging.debug("Moving contents to expected structure...") # Move each dex to a separate temporary directory to be operated by # redex. dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir)) for store in sorted(store_files): dexen.append(store) logging.debug( "Unpacking APK finished in {:.2f} seconds".format(timer() - unpack_start_time)) if args.side_effect_summaries is not None: args.passthru_json.append( 'ObjectSensitiveDcePass.side_effect_summaries="%s"' % args.side_effect_summaries) if args.escape_summaries is not None: args.passthru_json.append( 'ObjectSensitiveDcePass.escape_summaries="%s"' % args.escape_summaries) for key_value_str in args.passthru_json: key_value = key_value_str.split("=", 1) if len(key_value) != 2: logging.debug( "Json Pass through %s is not valid. Split len: %s", key_value_str, len(key_value), ) continue key = key_value[0] value = key_value[1] prev_value = config_dict.get(key, "(No previous value)") logging.debug( "Got Override %s = %s from %s. Previous %s", key, value, key_value_str, prev_value, ) config_dict[key] = json.loads(value) # Scan for framework files. If not found, warn and add them if available. _check_android_sdk_api(args) # Check for shrinker heuristics. _check_shrinker_heuristics(args) # Scan for SDK jar. If not found, warn and add if available. _check_android_sdk(args) logging.debug("Running redex-all on %d dex files ", len(dexen)) if args.lldb: debugger = "lldb" elif args.gdb: debugger = "gdb" else: debugger = None return State( args=args, config_dict=config_dict, debugger=debugger, dex_dir=dex_dir, dexen=dexen, extracted_apk_dir=extracted_apk_dir, stop_pass_idx=stop_pass_idx, lib_manager=lib_manager, unpack_manager=unpack_manager, zip_manager=zip_manager, )
def prepare_redex(args): debug_mode = args.unpack_only or args.debug extracted_apk_dir = None dex_dir = None if args.unpack_only and args.unpack_dest: if args.unpack_dest[0] == '.': # Use APK's name unpack_dir_basename = os.path.splitext(args.input_apk)[0] else: unpack_dir_basename = args.unpack_dest[0] extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk' dex_dir = unpack_dir_basename + '.redex_dexen' try: os.makedirs(extracted_apk_dir) os.makedirs(dex_dir) extracted_apk_dir = os.path.abspath(extracted_apk_dir) dex_dir = os.path.abspath(dex_dir) except OSError as e: if e.errno == errno.EEXIST: print('Error: destination directory already exists!') print('APK: ' + extracted_apk_dir) print('DEX: ' + dex_dir) sys.exit(1) raise e config = args.config binary = args.redex_binary log('Using config ' + (config if config is not None else '(default)')) log('Using binary ' + (binary if binary is not None else '(default)')) if config is None: config_dict = {} else: with open(config) as config_file: try: lines = config_file.readlines() config_dict = json.loads(remove_comments(lines)) except ValueError: raise ValueError("Invalid JSON in ReDex config file: %s" % config_file.name) # stop_pass_idx >= 0 means need stop before a pass and dump intermediate result stop_pass_idx = -1 if args.stop_pass: passes_list = config_dict.get('redex', {}).get('passes', []) stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass) if not args.output_ir or isfile(args.output_ir): print('Error: output_ir should be a directory') sys.exit(1) try: os.makedirs(args.output_ir) except OSError as e: if e.errno != errno.EEXIST: raise e unpack_start_time = timer() if not extracted_apk_dir: extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode) log('Extracting apk...') unzip_apk(args.input_apk, extracted_apk_dir) dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir) log('Detected dex mode ' + str(type(dex_mode).__name__)) if not dex_dir: dex_dir = make_temp_dir('.redex_dexen', debug_mode) log('Unpacking dex files') dex_mode.unpackage(extracted_apk_dir, dex_dir) log('Detecting Application Modules') application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir) store_files = [] store_metadata_dir = make_temp_dir( '.application_module_metadata', debug_mode) for module in application_modules: canary_prefix = module.get_canary_prefix() log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)')) store_path = os.path.join(dex_dir, module.get_name()) os.mkdir(store_path) module.unpackage(extracted_apk_dir, store_path) store_metadata = os.path.join( store_metadata_dir, module.get_name() + '.json') module.write_redex_metadata(store_path, store_metadata) store_files.append(store_metadata) # Some of the native libraries can be concatenated together into one # xz-compressed file. We need to decompress that file so that we can scan # through it looking for classnames. libs_to_extract = [] temporary_libs_dir = None xz_lib_name = 'libs.xzs' zstd_lib_name = 'libs.zstd' for root, _, filenames in os.walk(extracted_apk_dir): for filename in fnmatch.filter(filenames, xz_lib_name): libs_to_extract.append(join(root, filename)) for filename in fnmatch.filter(filenames, zstd_lib_name): fullpath = join(root, filename) # For voltron modules BUCK creates empty zstd files for each module if os.path.getsize(fullpath) > 0: libs_to_extract.append(fullpath) if len(libs_to_extract) > 0: libs_dir = join(extracted_apk_dir, 'lib') extracted_dir = join(libs_dir, '__extracted_libs__') # Ensure both directories exist. temporary_libs_dir = ensure_libs_dir(libs_dir, extracted_dir) lib_count = 0 for lib_to_extract in libs_to_extract: extract_path = join(extracted_dir, "lib_{}.so".format(lib_count)) if lib_to_extract.endswith(xz_lib_name): cmd = 'xz -d --stdout {} > {}'.format( lib_to_extract, extract_path) else: cmd = 'zstd -d {} -o {}'.format(lib_to_extract, extract_path) subprocess.check_call(cmd, shell=True) lib_count += 1 if args.unpack_only: print('APK: ' + extracted_apk_dir) print('DEX: ' + dex_dir) sys.exit() # Move each dex to a separate temporary directory to be operated by # redex. dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir)) for store in store_files: dexen.append(store) log('Unpacking APK finished in {:.2f} seconds'.format( timer() - unpack_start_time)) if args.side_effect_summaries is not None: args.passthru_json.append( 'DeadCodeEliminationPass.side_effect_summaries="%s"' % args.side_effect_summaries ) if args.escape_summaries is not None: args.passthru_json.append( 'DeadCodeEliminationPass.escape_summaries="%s"' % args.escape_summaries ) for key_value_str in args.passthru_json: key_value = key_value_str.split('=', 1) if len(key_value) != 2: log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value))) continue key = key_value[0] value = key_value[1] prev_value = config_dict.get(key, "(No previous value)") log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value)) config_dict[key] = value log('Running redex-all on {} dex files '.format(len(dexen))) if args.lldb: debugger = 'lldb' elif args.gdb: debugger = 'gdb' else: debugger = None return State( application_modules=application_modules, args=args, config_dict=config_dict, debugger=debugger, dex_dir=dex_dir, dexen=dexen, dex_mode=dex_mode, extracted_apk_dir=extracted_apk_dir, temporary_libs_dir=temporary_libs_dir, stop_pass_idx=stop_pass_idx)