Ejemplo n.º 1
0
def run_redex(args):
    debug_mode = args.unpack_only or args.debug

    unpack_start_time = timer()
    extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)

    log('Extracting apk...')
    unzip_apk(args.input_apk, extracted_apk_dir)

    dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
    log('Detected dex mode ' + str(type(dex_mode).__name__))
    dex_dir = make_temp_dir('.redex_dexen', debug_mode)

    log('Unpacking dex files')
    dex_mode.unpackage(extracted_apk_dir, dex_dir)

    log('Detecting Application Modules')
    application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
    store_files = []
    for module in application_modules:
        log('found module: ' + module.get_name() + ' ' + module.get_canary_prefix())
        store_path = os.path.join(dex_dir, module.get_name())
        os.mkdir(store_path)
        module.unpackage(extracted_apk_dir, store_path)
        store_files.append(module.write_redex_metadata(store_path))

    # Some of the native libraries can be concatenated together into one
    # xz-compressed file. We need to decompress that file so that we can scan
    # through it looking for classnames.
    xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
    temporary_lib_file = join(extracted_apk_dir, 'lib/concated_native_libs.so')
    if os.path.exists(xz_compressed_libs):
        cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
        subprocess.check_call(cmd, shell=True)

    if args.unpack_only:
        print('APK: ' + extracted_apk_dir)
        print('DEX: ' + dex_dir)
        sys.exit()

    # Move each dex to a separate temporary directory to be operated by
    # redex.
    dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
    for store in store_files:
        dexen.append(store)
    log('Unpacking APK finished in {:.2f} seconds'.format(
            timer() - unpack_start_time))

    config = args.config
    binary = args.redex_binary
    log('Using config ' + (config if config is not None else '(default)'))
    log('Using binary ' + (binary if binary is not None else '(default)'))

    if config is None:
        config_dict = {}
        passes_list = []
    else:
        with open(config) as config_file:
            config_dict = json.load(config_file)
            passes_list = config_dict['redex']['passes']

    newtmp = tempfile.mkdtemp()
    log('Replacing /tmp in config with {}'.format(newtmp))

    # Fix up the config dict to relocate all /tmp references
    relocate_tmp(config_dict, newtmp)

    # Rewrite the relocated config file to our tmp, for use by redex binary
    if config is not None:
        config = newtmp + "/rewritten.config"
        with open(config, 'w') as fp:
            json.dump(config_dict, fp)

    log('Running redex-all on {} dex files '.format(len(dexen)))
    run_pass(binary,
             args,
             config,
             config_dict,
             extracted_apk_dir,
             dex_dir,
             dexen)

    # This file was just here so we could scan it for classnames, but we don't
    # want to pack it back up into the apk
    if os.path.exists(temporary_lib_file):
        os.remove(temporary_lib_file)

    repack_start_time = timer()

    log('Repacking dex files')
    have_locators = config_dict.get("emit_locator_strings")
    dex_mode.repackage(extracted_apk_dir, dex_dir, have_locators)

    for module in application_modules:
        log('repacking module: ' + module.get_name())
        module.repackage(extracted_apk_dir, dex_dir, have_locators)


    log('Creating output apk')
    create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
            args.keyalias, args.keypass)
    log('Creating output APK finished in {:.2f} seconds'.format(
            timer() - repack_start_time))
    copy_file_to_out_dir(newtmp, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
    copy_file_to_out_dir(newtmp, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
    copy_file_to_out_dir(newtmp, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
    copy_file_to_out_dir(newtmp, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')

    if 'RenameClassesPass' in passes_list:
        merge_proguard_map_with_rename_output(
            args.input_apk,
            args.out,
            config_dict,
            args.proguard_map)
    else:
        log('Skipping rename map merging, because we didn\'t run the rename pass')

    shutil.rmtree(newtmp)
    remove_temp_dirs()
Ejemplo n.º 2
0
def prepare_redex(args):
    debug_mode = args.unpack_only or args.debug

    # avoid accidentally mixing up file formats since we now support
    # both apk files and Android bundle files
    if not args.unpack_only:
        assert get_file_ext(args.input_apk) == get_file_ext(args.out),\
            "Input file extension (\"" +\
            get_file_ext(args.input_apk) +\
            "\") should be the same as output file extension (\"" +\
            get_file_ext(args.out) + "\")"

    extracted_apk_dir = None
    dex_dir = None
    if args.unpack_only and args.unpack_dest:
        if args.unpack_dest[0] == '.':
            # Use APK's name
            unpack_dir_basename = os.path.splitext(args.input_apk)[0]
        else:
            unpack_dir_basename = args.unpack_dest[0]
        extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
        dex_dir = unpack_dir_basename + '.redex_dexen'
        try:
            os.makedirs(extracted_apk_dir)
            os.makedirs(dex_dir)
            extracted_apk_dir = os.path.abspath(extracted_apk_dir)
            dex_dir = os.path.abspath(dex_dir)
        except OSError as e:
            if e.errno == errno.EEXIST:
                print('Error: destination directory already exists!')
                print('APK: ' + extracted_apk_dir)
                print('DEX: ' + dex_dir)
                sys.exit(1)
            raise e

    config = args.config
    binary = args.redex_binary
    log('Using config ' + (config if config is not None else '(default)'))
    log('Using binary ' + (binary if binary is not None else '(default)'))

    if args.unpack_only or config is None:
        config_dict = {}
    else:
        with open(config) as config_file:
            try:
                lines = config_file.readlines()
                config_dict = json.loads(remove_comments(lines))
            except ValueError:
                raise ValueError("Invalid JSON in ReDex config file: %s" %
                                 config_file.name)

    # stop_pass_idx >= 0 means need stop before a pass and dump intermediate result
    stop_pass_idx = -1
    if args.stop_pass:
        passes_list = config_dict.get('redex', {}).get('passes', [])
        stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass)
        if not args.output_ir or isfile(args.output_ir):
            print('Error: output_ir should be a directory')
            sys.exit(1)
        try:
            os.makedirs(args.output_ir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise e

    unpack_start_time = timer()
    if not extracted_apk_dir:
        extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)

    log('Extracting apk...')
    unzip_apk(args.input_apk, extracted_apk_dir)

    dex_file_path = get_dex_file_path(args, extracted_apk_dir)

    dex_mode = unpacker.detect_secondary_dex_mode(dex_file_path)
    log('Detected dex mode ' + str(type(dex_mode).__name__))
    if not dex_dir:
        dex_dir = make_temp_dir('.redex_dexen', debug_mode)

    log('Unpacking dex files')
    dex_mode.unpackage(dex_file_path, dex_dir)

    log('Detecting Application Modules')
    application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
    store_files = []
    store_metadata_dir = make_temp_dir('.application_module_metadata',
                                       debug_mode)
    for module in application_modules:
        canary_prefix = module.get_canary_prefix()
        log('found module: ' + module.get_name() + ' ' +
            (canary_prefix
             if canary_prefix is not None else '(no canary prefix)'))
        store_path = os.path.join(dex_dir, module.get_name())
        os.mkdir(store_path)
        module.unpackage(extracted_apk_dir, store_path)
        store_metadata = os.path.join(store_metadata_dir,
                                      module.get_name() + '.json')
        module.write_redex_metadata(store_path, store_metadata)
        store_files.append(store_metadata)

    # Some of the native libraries can be concatenated together into one
    # xz-compressed file. We need to decompress that file so that we can scan
    # through it looking for classnames.
    libs_to_extract = []
    temporary_libs_dir = None
    xz_lib_name = 'libs.xzs'
    zstd_lib_name = 'libs.zstd'
    for root, _, filenames in os.walk(extracted_apk_dir):
        for filename in fnmatch.filter(filenames, xz_lib_name):
            libs_to_extract.append(join(root, filename))
        for filename in fnmatch.filter(filenames, zstd_lib_name):
            fullpath = join(root, filename)
            # For voltron modules BUCK creates empty zstd files for each module
            if os.path.getsize(fullpath) > 0:
                libs_to_extract.append(fullpath)
    if len(libs_to_extract) > 0:
        libs_dir = join(extracted_apk_dir, 'lib')
        extracted_dir = join(libs_dir, '__extracted_libs__')
        # Ensure both directories exist.
        temporary_libs_dir = ensure_libs_dir(libs_dir, extracted_dir)
        lib_count = 0
        for lib_to_extract in libs_to_extract:
            extract_path = join(extracted_dir, "lib_{}.so".format(lib_count))
            if lib_to_extract.endswith(xz_lib_name):
                cmd = 'xz -d --stdout {} > {}'.format(lib_to_extract,
                                                      extract_path)
            else:
                cmd = 'zstd -d {} -o {}'.format(lib_to_extract, extract_path)
            subprocess.check_call(cmd, shell=True)
            lib_count += 1

    if args.unpack_only:
        print('APK: ' + extracted_apk_dir)
        print('DEX: ' + dex_dir)
        sys.exit()

    # Move each dex to a separate temporary directory to be operated by
    # redex.
    dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
    for store in store_files:
        dexen.append(store)
    log('Unpacking APK finished in {:.2f} seconds'.format(timer() -
                                                          unpack_start_time))

    if args.side_effect_summaries is not None:
        args.passthru_json.append(
            'ObjectSensitiveDcePass.side_effect_summaries="%s"' %
            args.side_effect_summaries)

    if args.escape_summaries is not None:
        args.passthru_json.append(
            'ObjectSensitiveDcePass.escape_summaries="%s"' %
            args.escape_summaries)

    for key_value_str in args.passthru_json:
        key_value = key_value_str.split('=', 1)
        if len(key_value) != 2:
            log("Json Pass through %s is not valid. Split len: %s" %
                (key_value_str, len(key_value)))
            continue
        key = key_value[0]
        value = key_value[1]
        prev_value = config_dict.get(key, "(No previous value)")
        log("Got Override %s = %s from %s. Previous %s" %
            (key, value, key_value_str, prev_value))
        config_dict[key] = json.loads(value)

    log('Running redex-all on {} dex files '.format(len(dexen)))
    if args.lldb:
        debugger = 'lldb'
    elif args.gdb:
        debugger = 'gdb'
    else:
        debugger = None

    return State(application_modules=application_modules,
                 args=args,
                 config_dict=config_dict,
                 debugger=debugger,
                 dex_dir=dex_dir,
                 dexen=dexen,
                 dex_mode=dex_mode,
                 extracted_apk_dir=extracted_apk_dir,
                 temporary_libs_dir=temporary_libs_dir,
                 stop_pass_idx=stop_pass_idx)
Ejemplo n.º 3
0
def run_redex(args):
    debug_mode = args.unpack_only or args.debug

    unpack_start_time = timer()
    extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)

    log('Extracting apk...')
    unzip_apk(args.input_apk, extracted_apk_dir)

    dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
    log('Detected dex mode ' + str(type(dex_mode).__name__))
    dex_dir = make_temp_dir('.redex_dexen', debug_mode)

    log('Unpacking dex files')
    dex_mode.unpackage(extracted_apk_dir, dex_dir)

    log('Detecting Application Modules')
    application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
    store_files = []
    store_metadata_dir = make_temp_dir('.application_module_metadata',
                                       debug_mode)
    for module in application_modules:
        log('found module: ' + module.get_name() + ' ' +
            module.get_canary_prefix())
        store_path = os.path.join(dex_dir, module.get_name())
        os.mkdir(store_path)
        module.unpackage(extracted_apk_dir, store_path)
        store_metadata = os.path.join(store_metadata_dir,
                                      module.get_name() + '.json')
        module.write_redex_metadata(store_path, store_metadata)
        store_files.append(store_metadata)

    # Some of the native libraries can be concatenated together into one
    # xz-compressed file. We need to decompress that file so that we can scan
    # through it looking for classnames.
    xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
    temporary_lib_file = join(extracted_apk_dir, 'lib/concated_native_libs.so')
    if os.path.exists(xz_compressed_libs):
        cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs,
                                              temporary_lib_file)
        subprocess.check_call(cmd, shell=True)

    if args.unpack_only:
        print('APK: ' + extracted_apk_dir)
        print('DEX: ' + dex_dir)
        sys.exit()

    # Move each dex to a separate temporary directory to be operated by
    # redex.
    dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
    for store in store_files:
        dexen.append(store)
    log('Unpacking APK finished in {:.2f} seconds'.format(timer() -
                                                          unpack_start_time))

    config = args.config
    binary = args.redex_binary
    log('Using config ' + (config if config is not None else '(default)'))
    log('Using binary ' + (binary if binary is not None else '(default)'))

    if config is None:
        config_dict = {}
        passes_list = []
    else:
        with open(config) as config_file:
            config_dict = json.load(config_file)
            passes_list = config_dict['redex']['passes']

    for key_value_str in args.passthru_json:
        key_value = key_value_str.split('=', 1)
        if len(key_value) != 2:
            log("Json Pass through %s is not valid. Split len: %s" %
                (key_value_str, len(key_value)))
            continue
        key = key_value[0]
        value = key_value[1]
        log("Got Override %s = %s from %s. Previous %s" %
            (key, value, key_value_str, config_dict[key]))
        config_dict[key] = value

    log('Running redex-all on {} dex files '.format(len(dexen)))
    run_pass(binary, args, config, config_dict, extracted_apk_dir, dex_dir,
             dexen)

    # This file was just here so we could scan it for classnames, but we don't
    # want to pack it back up into the apk
    if os.path.exists(temporary_lib_file):
        os.remove(temporary_lib_file)

    repack_start_time = timer()

    log('Repacking dex files')
    have_locators = config_dict.get("emit_locator_strings")
    log("Emit Locator Strings: %s" % have_locators)

    dex_mode.repackage(extracted_apk_dir, dex_dir, have_locators)

    for module in application_modules:
        log('repacking module: ' + module.get_name())
        module.repackage(extracted_apk_dir, dex_dir, have_locators)

    log('Creating output apk')
    create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
                      args.keyalias, args.keypass)
    log('Creating output APK finished in {:.2f} seconds'.format(
        timer() - repack_start_time))
    copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map',
                         'line number map', 'redex-line-number-map')
    copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats',
                         'redex-stats.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt',
                         'src strings map', 'redex-src-strings-map.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt',
                         'method id map', 'redex-method-id-map.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt',
                         'class id map', 'redex-class-id-map.txt')
    copy_file_to_out_dir(dex_dir, args.out,
                         'coldstart_fields_in_R_classes.txt',
                         'resources accessed during coldstart',
                         'redex-tracked-coldstart-resources.txt')

    if 'RenameClassesPass' in passes_list or 'RenameClassesPassV2' in passes_list:
        merge_proguard_map_with_rename_output(passes_list, args.input_apk,
                                              args.out, dex_dir, config_dict,
                                              args.proguard_map)
    else:
        log('Skipping rename map merging, because we didn\'t run the rename pass'
            )

    remove_temp_dirs()
Ejemplo n.º 4
0
def run_redex(args):
    debug_mode = args.unpack_only or args.debug

    extracted_apk_dir = None
    dex_dir = None
    if args.unpack_only and args.unpack_dest:
        if args.unpack_dest[0] == '.':
            # Use APK's name
            unpack_dir_basename = os.path.splitext(args.input_apk)[0]
        else:
            unpack_dir_basename = args.unpack_dest[0]
        extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
        dex_dir = unpack_dir_basename + '.redex_dexen'
        try:
            os.makedirs(extracted_apk_dir)
            os.makedirs(dex_dir)
            extracted_apk_dir = os.path.abspath(extracted_apk_dir)
            dex_dir = os.path.abspath(dex_dir)
        except OSError as e:
            if e.errno == errno.EEXIST:
                print('Error: destination directory already exists!')
                print('APK: ' + extracted_apk_dir)
                print('DEX: ' + dex_dir)
                sys.exit(1)
            raise e

    config = args.config
    binary = args.redex_binary
    log('Using config ' + (config if config is not None else '(default)'))
    log('Using binary ' + (binary if binary is not None else '(default)'))

    if config is None:
        config_dict = {}
        passes_list = []
    else:
        with open(config) as config_file:
            try:
                lines = config_file.readlines()
                config_dict = json.loads(remove_comments(lines))
            except ValueError:
                raise ValueError("Invalid JSON in ReDex config file: %s" %
                                 config_file.name)
            passes_list = config_dict['redex']['passes']

    unpack_start_time = timer()
    if not extracted_apk_dir:
        extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)

    log('Extracting apk...')
    unzip_apk(args.input_apk, extracted_apk_dir)

    dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
    log('Detected dex mode ' + str(type(dex_mode).__name__))
    if not dex_dir:
        dex_dir = make_temp_dir('.redex_dexen', debug_mode)

    log('Unpacking dex files')
    dex_mode.unpackage(extracted_apk_dir, dex_dir)

    log('Detecting Application Modules')
    application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
    store_files = []
    store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode)
    for module in application_modules:
        canary_prefix = module.get_canary_prefix()
        log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)'))
        store_path = os.path.join(dex_dir, module.get_name())
        os.mkdir(store_path)
        module.unpackage(extracted_apk_dir, store_path)
        store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json')
        module.write_redex_metadata(store_path, store_metadata)
        store_files.append(store_metadata)

    # Some of the native libraries can be concatenated together into one
    # xz-compressed file. We need to decompress that file so that we can scan
    # through it looking for classnames.
    xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
    libs_dir = join(extracted_apk_dir, 'lib')
    temporary_lib_file = join(libs_dir, 'concated_native_libs.so')
    if os.path.exists(xz_compressed_libs) and os.path.exists(libs_dir):
        cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
        subprocess.check_call(cmd, shell=True)

    if args.unpack_only:
        print('APK: ' + extracted_apk_dir)
        print('DEX: ' + dex_dir)
        sys.exit()

    # Move each dex to a separate temporary directory to be operated by
    # redex.
    dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
    for store in store_files:
        dexen.append(store)
    log('Unpacking APK finished in {:.2f} seconds'.format(
        timer() - unpack_start_time))

    if args.side_effect_summaries is not None:
        args.passthru_json.append(
            'DeadCodeEliminationPass.external_summaries="%s"' % args.side_effect_summaries
        )

    for key_value_str in args.passthru_json:
        key_value = key_value_str.split('=', 1)
        if len(key_value) != 2:
            log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value)))
            continue
        key = key_value[0]
        value = key_value[1]
        prev_value = config_dict.get(key, "(No previous value)")
        log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value))
        config_dict[key] = value

    log('Running redex-all on {} dex files '.format(len(dexen)))
    if args.lldb:
        debugger = 'lldb'
    elif args.gdb:
        debugger = 'gdb'
    else:
        debugger = None

    run_pass(binary,
             args,
             config,
             config_dict,
             extracted_apk_dir,
             dex_dir,
             dexen,
             debugger)

    # This file was just here so we could scan it for classnames, but we don't
    # want to pack it back up into the apk
    if os.path.exists(temporary_lib_file):
        os.remove(temporary_lib_file)

    repack_start_time = timer()

    log('Repacking dex files')
    have_locators = config_dict.get("emit_locator_strings")
    log("Emit Locator Strings: %s" % have_locators)

    dex_mode.repackage(
        extracted_apk_dir, dex_dir, have_locators, fast_repackage=args.dev
    )

    locator_store_id = 1
    for module in application_modules:
        log('repacking module: ' + module.get_name() + ' with id ' + str(locator_store_id))
        module.repackage(
            extracted_apk_dir, dex_dir, have_locators, locator_store_id,
            fast_repackage=args.dev
        )
        locator_store_id = locator_store_id + 1

    log('Creating output apk')
    create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
            args.keyalias, args.keypass, args.ignore_zipalign, args.page_align_libs)
    log('Creating output APK finished in {:.2f} seconds'.format(
            timer() - repack_start_time))
    copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
    copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map-v2', 'line number map v2', 'redex-line-number-map-v2')
    copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'outliner-artifacts.bin', 'outliner artifacts', 'redex-outliner-artifacts.bin')
    copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt', 'class id map', 'redex-class-id-map.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'bytecode_offset_map.txt', 'bytecode offset map', 'redex-bytecode-offset-map.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'coldstart_fields_in_R_classes.txt', 'resources accessed during coldstart', 'redex-tracked-coldstart-resources.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'class_dependencies.txt', 'stats', 'redex-class-dependencies.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'resid-optres-mapping.json', 'resid map after optres pass', 'redex-resid-optres-mapping.json')
    copy_file_to_out_dir(dex_dir, args.out, 'resid-dedup-mapping.json', 'resid map after dedup pass', 'redex-resid-dedup-mapping.json')
    copy_file_to_out_dir(dex_dir, args.out, 'resid-splitres-mapping.json', 'resid map after split pass', 'redex-resid-splitres-mapping.json')
    copy_file_to_out_dir(dex_dir, args.out, 'type-erasure-mappings.txt', 'class map after type erasure pass', 'redex-type-erasure-mappings.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'instrument-methods-idx.txt', 'instrumented methods id map', 'redex-instrument-methods-idx.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'cleanup-removed-classes.txt', 'cleanup removed classes', 'redex-cleanup-removed-classes.txt')

    if config_dict.get('proguard_map_output', '') != '':
        # if our map output strategy is overwrite, we don't merge at all
        # if you enable ObfuscatePass, this needs to be overwrite
        if config_dict.get('proguard_map_output_strategy', 'merge') == 'overwrite':
            overwrite_proguard_maps(
                config_dict['proguard_map_output'],
                args.out,
                dex_dir,
                args.proguard_map)
        else:
            merge_proguard_maps(
                config_dict['proguard_map_output'],
                args.input_apk,
                args.out,
                dex_dir,
                args.proguard_map)
    else:
        assert 'RenameClassesPass' not in passes_list and\
                'RenameClassesPassV2' not in passes_list

    remove_temp_dirs()
Ejemplo n.º 5
0
def prepare_redex(args):
    debug_mode = args.unpack_only or args.debug

    extracted_apk_dir = None
    dex_dir = None
    if args.unpack_only and args.unpack_dest:
        if args.unpack_dest[0] == '.':
            # Use APK's name
            unpack_dir_basename = os.path.splitext(args.input_apk)[0]
        else:
            unpack_dir_basename = args.unpack_dest[0]
        extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
        dex_dir = unpack_dir_basename + '.redex_dexen'
        try:
            os.makedirs(extracted_apk_dir)
            os.makedirs(dex_dir)
            extracted_apk_dir = os.path.abspath(extracted_apk_dir)
            dex_dir = os.path.abspath(dex_dir)
        except OSError as e:
            if e.errno == errno.EEXIST:
                print('Error: destination directory already exists!')
                print('APK: ' + extracted_apk_dir)
                print('DEX: ' + dex_dir)
                sys.exit(1)
            raise e

    config = args.config
    binary = args.redex_binary
    log('Using config ' + (config if config is not None else '(default)'))
    log('Using binary ' + (binary if binary is not None else '(default)'))

    if config is None:
        config_dict = {}
    else:
        with open(config) as config_file:
            try:
                lines = config_file.readlines()
                config_dict = json.loads(remove_comments(lines))
            except ValueError:
                raise ValueError("Invalid JSON in ReDex config file: %s" %
                                 config_file.name)

    # stop_pass_idx >= 0 means need stop before a pass and dump intermediate result
    stop_pass_idx = -1
    if args.stop_pass:
        passes_list = config_dict.get('redex', {}).get('passes', [])
        stop_pass_idx = get_stop_pass_idx(passes_list, args.stop_pass)
        if not args.output_ir or isfile(args.output_ir):
            print('Error: output_ir should be a directory')
            sys.exit(1)
        try:
            os.makedirs(args.output_ir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise e

    unpack_start_time = timer()
    if not extracted_apk_dir:
        extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)

    log('Extracting apk...')
    unzip_apk(args.input_apk, extracted_apk_dir)

    dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
    log('Detected dex mode ' + str(type(dex_mode).__name__))
    if not dex_dir:
        dex_dir = make_temp_dir('.redex_dexen', debug_mode)

    log('Unpacking dex files')
    dex_mode.unpackage(extracted_apk_dir, dex_dir)

    log('Detecting Application Modules')
    application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
    store_files = []
    store_metadata_dir = make_temp_dir(
        '.application_module_metadata', debug_mode)
    for module in application_modules:
        canary_prefix = module.get_canary_prefix()
        log('found module: ' + module.get_name() + ' ' +
            (canary_prefix if canary_prefix is not None else '(no canary prefix)'))
        store_path = os.path.join(dex_dir, module.get_name())
        os.mkdir(store_path)
        module.unpackage(extracted_apk_dir, store_path)
        store_metadata = os.path.join(
            store_metadata_dir, module.get_name() + '.json')
        module.write_redex_metadata(store_path, store_metadata)
        store_files.append(store_metadata)

    # Some of the native libraries can be concatenated together into one
    # xz-compressed file. We need to decompress that file so that we can scan
    # through it looking for classnames.
    libs_to_extract = []
    temporary_libs_dir = None
    xz_lib_name = 'libs.xzs'
    zstd_lib_name = 'libs.zstd'
    for root, _, filenames in os.walk(extracted_apk_dir):
        for filename in fnmatch.filter(filenames, xz_lib_name):
            libs_to_extract.append(join(root, filename))
        for filename in fnmatch.filter(filenames, zstd_lib_name):
            fullpath = join(root, filename)
            # For voltron modules BUCK creates empty zstd files for each module
            if os.path.getsize(fullpath) > 0:
                libs_to_extract.append(fullpath)
    if len(libs_to_extract) > 0:
        libs_dir = join(extracted_apk_dir, 'lib')
        extracted_dir = join(libs_dir, '__extracted_libs__')
        # Ensure both directories exist.
        temporary_libs_dir = ensure_libs_dir(libs_dir, extracted_dir)
        lib_count = 0
        for lib_to_extract in libs_to_extract:
            extract_path = join(extracted_dir, "lib_{}.so".format(lib_count))
            if lib_to_extract.endswith(xz_lib_name):
                cmd = 'xz -d --stdout {} > {}'.format(
                    lib_to_extract, extract_path)
            else:
                cmd = 'zstd -d {} -o {}'.format(lib_to_extract, extract_path)
            subprocess.check_call(cmd, shell=True)
            lib_count += 1

    if args.unpack_only:
        print('APK: ' + extracted_apk_dir)
        print('DEX: ' + dex_dir)
        sys.exit()

    # Move each dex to a separate temporary directory to be operated by
    # redex.
    dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
    for store in store_files:
        dexen.append(store)
    log('Unpacking APK finished in {:.2f} seconds'.format(
        timer() - unpack_start_time))

    if args.side_effect_summaries is not None:
        args.passthru_json.append(
            'DeadCodeEliminationPass.side_effect_summaries="%s"' % args.side_effect_summaries
        )

    if args.escape_summaries is not None:
        args.passthru_json.append(
            'DeadCodeEliminationPass.escape_summaries="%s"' % args.escape_summaries
        )

    for key_value_str in args.passthru_json:
        key_value = key_value_str.split('=', 1)
        if len(key_value) != 2:
            log("Json Pass through %s is not valid. Split len: %s" %
                (key_value_str, len(key_value)))
            continue
        key = key_value[0]
        value = key_value[1]
        prev_value = config_dict.get(key, "(No previous value)")
        log("Got Override %s = %s from %s. Previous %s" %
            (key, value, key_value_str, prev_value))
        config_dict[key] = value

    log('Running redex-all on {} dex files '.format(len(dexen)))
    if args.lldb:
        debugger = 'lldb'
    elif args.gdb:
        debugger = 'gdb'
    else:
        debugger = None

    return State(
        application_modules=application_modules,
        args=args,
        config_dict=config_dict,
        debugger=debugger,
        dex_dir=dex_dir,
        dexen=dexen,
        dex_mode=dex_mode,
        extracted_apk_dir=extracted_apk_dir,
        temporary_libs_dir=temporary_libs_dir,
        stop_pass_idx=stop_pass_idx)
Ejemplo n.º 6
0
def run_redex(args):
    debug_mode = args.unpack_only or args.debug

    extracted_apk_dir = None
    dex_dir = None
    if args.unpack_only and args.unpack_dest:
        if args.unpack_dest[0] == '.':
            # Use APK's name
            unpack_dir_basename = os.path.splitext(args.input_apk)[0]
        else:
            unpack_dir_basename = args.unpack_dest[0]
        extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
        dex_dir = unpack_dir_basename + '.redex_dexen'
        try:
            os.makedirs(extracted_apk_dir)
            os.makedirs(dex_dir)
            extracted_apk_dir = os.path.abspath(extracted_apk_dir)
            dex_dir = os.path.abspath(dex_dir)
        except OSError as e:
            if e.errno == errno.EEXIST:
                print('Error: destination directory already exists!')
                print('APK: ' + extracted_apk_dir)
                print('DEX: ' + dex_dir)
                sys.exit(1)
            raise e

    config = args.config
    binary = args.redex_binary
    log('Using config ' + (config if config is not None else '(default)'))
    log('Using binary ' + (binary if binary is not None else '(default)'))

    if config is None:
        config_dict = {}
        passes_list = []
    else:
        with open(config) as config_file:
            try:
                lines = config_file.readlines()
                config_dict = json.loads(remove_comments(lines))
            except ValueError:
                raise ValueError("Invalid JSON in ReDex config file: %s" %
                                 config_file.name)
            passes_list = config_dict['redex']['passes']

    unpack_start_time = timer()
    if not extracted_apk_dir:
        extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)

    log('Extracting apk...')
    unzip_apk(args.input_apk, extracted_apk_dir)

    dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
    log('Detected dex mode ' + str(type(dex_mode).__name__))
    if not dex_dir:
        dex_dir = make_temp_dir('.redex_dexen', debug_mode)

    log('Unpacking dex files')
    dex_mode.unpackage(extracted_apk_dir, dex_dir)

    log('Detecting Application Modules')
    application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
    store_files = []
    store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode)
    for module in application_modules:
        canary_prefix = module.get_canary_prefix()
        log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)'))
        store_path = os.path.join(dex_dir, module.get_name())
        os.mkdir(store_path)
        module.unpackage(extracted_apk_dir, store_path)
        store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json')
        module.write_redex_metadata(store_path, store_metadata)
        store_files.append(store_metadata)

    # Some of the native libraries can be concatenated together into one
    # xz-compressed file. We need to decompress that file so that we can scan
    # through it looking for classnames.
    xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
    libs_dir = join(extracted_apk_dir, 'lib')
    temporary_lib_file = join(libs_dir, 'concated_native_libs.so')
    if os.path.exists(xz_compressed_libs) and os.path.exists(libs_dir):
        cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
        subprocess.check_call(cmd, shell=True)

    if args.unpack_only:
        print('APK: ' + extracted_apk_dir)
        print('DEX: ' + dex_dir)
        sys.exit()

    # Move each dex to a separate temporary directory to be operated by
    # redex.
    dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
    for store in store_files:
        dexen.append(store)
    log('Unpacking APK finished in {:.2f} seconds'.format(
        timer() - unpack_start_time))

    if args.side_effect_summaries is not None:
        args.passthru_json.append(
            'DeadCodeEliminationPass.external_summaries="%s"' % args.side_effect_summaries
        )

    for key_value_str in args.passthru_json:
        key_value = key_value_str.split('=', 1)
        if len(key_value) != 2:
            log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value)))
            continue
        key = key_value[0]
        value = key_value[1]
        prev_value = config_dict.get(key, "(No previous value)")
        log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value))
        config_dict[key] = value

    log('Running redex-all on {} dex files '.format(len(dexen)))
    if args.lldb:
        debugger = 'lldb'
    elif args.gdb:
        debugger = 'gdb'
    else:
        debugger = None

    run_pass(binary,
             args,
             config,
             config_dict,
             extracted_apk_dir,
             dex_dir,
             dexen,
             debugger)

    # This file was just here so we could scan it for classnames, but we don't
    # want to pack it back up into the apk
    if os.path.exists(temporary_lib_file):
        os.remove(temporary_lib_file)

    repack_start_time = timer()

    log('Repacking dex files')
    have_locators = config_dict.get("emit_locator_strings")
    log("Emit Locator Strings: %s" % have_locators)

    dex_mode.repackage(
        extracted_apk_dir, dex_dir, have_locators, fast_repackage=args.dev
    )

    locator_store_id = 1
    for module in application_modules:
        log('repacking module: ' + module.get_name() + ' with id ' + str(locator_store_id))
        module.repackage(
            extracted_apk_dir, dex_dir, have_locators, locator_store_id,
            fast_repackage=args.dev
        )
        locator_store_id = locator_store_id + 1

    log('Creating output apk')
    create_output_apk(extracted_apk_dir, args.out, args.sign, args.keystore,
            args.keyalias, args.keypass, args.ignore_zipalign, args.page_align_libs)
    log('Creating output APK finished in {:.2f} seconds'.format(
            timer() - repack_start_time))
    copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map', 'line number map', 'redex-line-number-map')
    copy_file_to_out_dir(dex_dir, args.out, 'redex-line-number-map-v2', 'line number map v2', 'redex-line-number-map-v2')
    copy_file_to_out_dir(dex_dir, args.out, 'stats.txt', 'stats', 'redex-stats.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'filename_mappings.txt', 'src strings map', 'redex-src-strings-map.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'outliner-artifacts.bin', 'outliner artifacts', 'redex-outliner-artifacts.bin')
    copy_file_to_out_dir(dex_dir, args.out, 'method_mapping.txt', 'method id map', 'redex-method-id-map.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'class_mapping.txt', 'class id map', 'redex-class-id-map.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'bytecode_offset_map.txt', 'bytecode offset map', 'redex-bytecode-offset-map.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'coldstart_fields_in_R_classes.txt', 'resources accessed during coldstart', 'redex-tracked-coldstart-resources.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'class_dependencies.txt', 'stats', 'redex-class-dependencies.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'resid-optres-mapping.json', 'resid map after optres pass', 'redex-resid-optres-mapping.json')
    copy_file_to_out_dir(dex_dir, args.out, 'resid-dedup-mapping.json', 'resid map after dedup pass', 'redex-resid-dedup-mapping.json')
    copy_file_to_out_dir(dex_dir, args.out, 'resid-splitres-mapping.json', 'resid map after split pass', 'redex-resid-splitres-mapping.json')
    copy_file_to_out_dir(dex_dir, args.out, 'type-erasure-mappings.txt', 'class map after type erasure pass', 'redex-type-erasure-mappings.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'instrument-methods-idx.txt', 'instrumented methods id map', 'redex-instrument-methods-idx.txt')
    copy_file_to_out_dir(dex_dir, args.out, 'cleanup-removed-classes.txt', 'cleanup removed classes', 'redex-cleanup-removed-classes.txt')

    if config_dict.get('proguard_map_output', '') != '':
        # if our map output strategy is overwrite, we don't merge at all
        # if you enable ObfuscatePass, this needs to be overwrite
        if config_dict.get('proguard_map_output_strategy', 'merge') == 'overwrite':
            overwrite_proguard_maps(
                config_dict['proguard_map_output'],
                args.out,
                dex_dir,
                args.proguard_map)
        else:
            merge_proguard_maps(
                config_dict['proguard_map_output'],
                args.input_apk,
                args.out,
                dex_dir,
                args.proguard_map)
    else:
        assert 'RenameClassesPass' not in passes_list and\
                'RenameClassesPassV2' not in passes_list

    remove_temp_dirs()
Ejemplo n.º 7
0
def prepare_redex(args):
    debug_mode = args.unpack_only or args.debug

    extracted_apk_dir = None
    dex_dir = None
    if args.unpack_only and args.unpack_dest:
        if args.unpack_dest[0] == '.':
            # Use APK's name
            unpack_dir_basename = os.path.splitext(args.input_apk)[0]
        else:
            unpack_dir_basename = args.unpack_dest[0]
        extracted_apk_dir = unpack_dir_basename + '.redex_extracted_apk'
        dex_dir = unpack_dir_basename + '.redex_dexen'
        try:
            os.makedirs(extracted_apk_dir)
            os.makedirs(dex_dir)
            extracted_apk_dir = os.path.abspath(extracted_apk_dir)
            dex_dir = os.path.abspath(dex_dir)
        except OSError as e:
            if e.errno == errno.EEXIST:
                print('Error: destination directory already exists!')
                print('APK: ' + extracted_apk_dir)
                print('DEX: ' + dex_dir)
                sys.exit(1)
            raise e

    config = args.config
    binary = args.redex_binary
    log('Using config ' + (config if config is not None else '(default)'))
    log('Using binary ' + (binary if binary is not None else '(default)'))

    if config is None:
        config_dict = {}
    else:
        with open(config) as config_file:
            try:
                lines = config_file.readlines()
                config_dict = json.loads(remove_comments(lines))
            except ValueError:
                raise ValueError("Invalid JSON in ReDex config file: %s" %
                                 config_file.name)

    unpack_start_time = timer()
    if not extracted_apk_dir:
        extracted_apk_dir = make_temp_dir('.redex_extracted_apk', debug_mode)

    log('Extracting apk...')
    unzip_apk(args.input_apk, extracted_apk_dir)

    dex_mode = unpacker.detect_secondary_dex_mode(extracted_apk_dir)
    log('Detected dex mode ' + str(type(dex_mode).__name__))
    if not dex_dir:
        dex_dir = make_temp_dir('.redex_dexen', debug_mode)

    log('Unpacking dex files')
    dex_mode.unpackage(extracted_apk_dir, dex_dir)

    log('Detecting Application Modules')
    application_modules = unpacker.ApplicationModule.detect(extracted_apk_dir)
    store_files = []
    store_metadata_dir = make_temp_dir('.application_module_metadata', debug_mode)
    for module in application_modules:
        canary_prefix = module.get_canary_prefix()
        log('found module: ' + module.get_name() + ' ' + (canary_prefix if canary_prefix is not None else '(no canary prefix)'))
        store_path = os.path.join(dex_dir, module.get_name())
        os.mkdir(store_path)
        module.unpackage(extracted_apk_dir, store_path)
        store_metadata = os.path.join(store_metadata_dir, module.get_name() + '.json')
        module.write_redex_metadata(store_path, store_metadata)
        store_files.append(store_metadata)

    # Some of the native libraries can be concatenated together into one
    # xz-compressed file. We need to decompress that file so that we can scan
    # through it looking for classnames.
    xz_compressed_libs = join(extracted_apk_dir, 'assets/lib/libs.xzs')
    libs_dir = join(extracted_apk_dir, 'lib')
    temporary_lib_file = join(libs_dir, 'concated_native_libs.so')
    if os.path.exists(xz_compressed_libs) and os.path.exists(libs_dir):
        cmd = 'xz -d --stdout {} > {}'.format(xz_compressed_libs, temporary_lib_file)
        subprocess.check_call(cmd, shell=True)

    if args.unpack_only:
        print('APK: ' + extracted_apk_dir)
        print('DEX: ' + dex_dir)
        sys.exit()

    # Move each dex to a separate temporary directory to be operated by
    # redex.
    dexen = move_dexen_to_directories(dex_dir, dex_glob(dex_dir))
    for store in store_files:
        dexen.append(store)
    log('Unpacking APK finished in {:.2f} seconds'.format(
        timer() - unpack_start_time))

    if args.side_effect_summaries is not None:
        args.passthru_json.append(
            'DeadCodeEliminationPass.side_effect_summaries="%s"' % args.side_effect_summaries
        )

    if args.escape_summaries is not None:
        args.passthru_json.append(
            'DeadCodeEliminationPass.escape_summaries="%s"' % args.escape_summaries
        )

    for key_value_str in args.passthru_json:
        key_value = key_value_str.split('=', 1)
        if len(key_value) != 2:
            log("Json Pass through %s is not valid. Split len: %s" % (key_value_str, len(key_value)))
            continue
        key = key_value[0]
        value = key_value[1]
        prev_value = config_dict.get(key, "(No previous value)")
        log("Got Override %s = %s from %s. Previous %s" % (key, value, key_value_str, prev_value))
        config_dict[key] = value

    log('Running redex-all on {} dex files '.format(len(dexen)))
    if args.lldb:
        debugger = 'lldb'
    elif args.gdb:
        debugger = 'gdb'
    else:
        debugger = None

    return State(
        application_modules=application_modules,
        args=args,
        config_dict=config_dict,
        debugger=debugger,
        dex_dir=dex_dir,
        dexen=dexen,
        dex_mode=dex_mode,
        extracted_apk_dir=extracted_apk_dir,
        temporary_lib_file=temporary_lib_file)