Ejemplo n.º 1
0
class BinaryCacheBuilder(object):
    """Collect all binaries needed by perf.data in binary_cache."""
    def __init__(self, ndk_path, disable_adb_root):
        self.adb = AdbHelper(enable_switch_to_root=not disable_adb_root)
        self.readelf = ReadElf(ndk_path)
        self.binary_cache_dir = 'binary_cache'
        if not os.path.isdir(self.binary_cache_dir):
            os.makedirs(self.binary_cache_dir)
        self.binaries = {}


    def build_binary_cache(self, perf_data_path, symfs_dirs):
        self._collect_used_binaries(perf_data_path)
        self.copy_binaries_from_symfs_dirs(symfs_dirs)
        self._pull_binaries_from_device()
        self._pull_kernel_symbols()


    def _collect_used_binaries(self, perf_data_path):
        """read perf.data, collect all used binaries and their build id (if available)."""
        # A dict mapping from binary name to build_id
        binaries = {}
        lib = ReportLib()
        lib.SetRecordFile(perf_data_path)
        lib.SetLogSeverity('error')
        while True:
            sample = lib.GetNextSample()
            if sample is None:
                lib.Close()
                break
            symbols = [lib.GetSymbolOfCurrentSample()]
            callchain = lib.GetCallChainOfCurrentSample()
            for i in range(callchain.nr):
                symbols.append(callchain.entries[i].symbol)

            for symbol in symbols:
                dso_name = symbol.dso_name
                if dso_name not in binaries:
                    if is_jit_symfile(dso_name):
                        continue
                    binaries[dso_name] = lib.GetBuildIdForPath(dso_name)
        self.binaries = binaries


    def copy_binaries_from_symfs_dirs(self, symfs_dirs):
        """collect all files in symfs_dirs."""
        if not symfs_dirs:
            return

        # It is possible that the path of the binary in symfs_dirs doesn't match
        # the one recorded in perf.data. For example, a file in symfs_dirs might
        # be "debug/arm/obj/armeabi-v7a/libsudo-game-jni.so", but the path in
        # perf.data is "/data/app/xxxx/lib/arm/libsudo-game-jni.so". So we match
        # binaries if they have the same filename (like libsudo-game-jni.so)
        # and same build_id.

        # Map from filename to binary paths.
        filename_dict = {}
        for binary in self.binaries:
            index = binary.rfind('/')
            filename = binary[index+1:]
            paths = filename_dict.get(filename)
            if paths is None:
                filename_dict[filename] = paths = []
            paths.append(binary)

        # Walk through all files in symfs_dirs, and copy matching files to build_cache.
        for symfs_dir in symfs_dirs:
            for root, _, files in os.walk(symfs_dir):
                for filename in files:
                    paths = filename_dict.get(filename)
                    if not paths:
                        continue
                    build_id = self._read_build_id(os.path.join(root, filename))
                    if not build_id:
                        continue
                    for binary in paths:
                        expected_build_id = self.binaries.get(binary)
                        if expected_build_id == build_id:
                            self._copy_to_binary_cache(os.path.join(root, filename),
                                                       expected_build_id, binary)
                            break


    def _copy_to_binary_cache(self, from_path, expected_build_id, target_file):
        if target_file[0] == '/':
            target_file = target_file[1:]
        target_file = target_file.replace('/', os.sep)
        target_file = os.path.join(self.binary_cache_dir, target_file)
        if not self._need_to_copy(from_path, target_file, expected_build_id):
            # The existing file in binary_cache can provide more information, so no need to copy.
            return
        target_dir = os.path.dirname(target_file)
        if not os.path.isdir(target_dir):
            os.makedirs(target_dir)
        log_info('copy to binary_cache: %s to %s' % (from_path, target_file))
        shutil.copy(from_path, target_file)


    def _need_to_copy(self, source_file, target_file, expected_build_id):
        if not os.path.isfile(target_file):
            return True
        if self._read_build_id(target_file) != expected_build_id:
            return True
        return self._get_file_stripped_level(source_file) < self._get_file_stripped_level(
            target_file)


    def _get_file_stripped_level(self, file_path):
        """Return stripped level of an ELF file. Larger value means more stripped."""
        sections = self.readelf.get_sections(file_path)
        if '.debug_line' in sections:
            return 0
        if '.symtab' in sections:
            return 1
        return 2


    def _pull_binaries_from_device(self):
        """pull binaries needed in perf.data to binary_cache."""
        for binary in self.binaries:
            build_id = self.binaries[binary]
            if not binary.startswith('/') or binary == "//anon" or binary.startswith("/dev/"):
                # [kernel.kallsyms] or unknown, or something we can't find binary.
                continue
            binary_cache_file = binary[1:].replace('/', os.sep)
            binary_cache_file = os.path.join(self.binary_cache_dir, binary_cache_file)
            self._check_and_pull_binary(binary, build_id, binary_cache_file)


    def _check_and_pull_binary(self, binary, expected_build_id, binary_cache_file):
        """If the binary_cache_file exists and has the expected_build_id, there
           is no need to pull the binary from device. Otherwise, pull it.
        """
        need_pull = True
        if os.path.isfile(binary_cache_file):
            need_pull = False
            if expected_build_id:
                build_id = self._read_build_id(binary_cache_file)
                if expected_build_id != build_id:
                    need_pull = True
        if need_pull:
            target_dir = os.path.dirname(binary_cache_file)
            if not os.path.isdir(target_dir):
                os.makedirs(target_dir)
            if os.path.isfile(binary_cache_file):
                os.remove(binary_cache_file)
            log_info('pull file to binary_cache: %s to %s' % (binary, binary_cache_file))
            self._pull_file_from_device(binary, binary_cache_file)
        else:
            log_info('use current file in binary_cache: %s' % binary_cache_file)


    def _read_build_id(self, file_path):
        """read build id of a binary on host."""
        return self.readelf.get_build_id(file_path)


    def _pull_file_from_device(self, device_path, host_path):
        if self.adb.run(['pull', device_path, host_path]):
            return True
        # In non-root device, we can't pull /data/app/XXX/base.odex directly.
        # Instead, we can first copy the file to /data/local/tmp, then pull it.
        filename = device_path[device_path.rfind('/')+1:]
        if (self.adb.run(['shell', 'cp', device_path, '/data/local/tmp']) and
                self.adb.run(['pull', '/data/local/tmp/' + filename, host_path])):
            self.adb.run(['shell', 'rm', '/data/local/tmp/' + filename])
            return True
        log_warning('failed to pull %s from device' % device_path)
        return False


    def _pull_kernel_symbols(self):
        file_path = os.path.join(self.binary_cache_dir, 'kallsyms')
        if os.path.isfile(file_path):
            os.remove(file_path)
        if self.adb.switch_to_root():
            self.adb.run(['shell', '"echo 0 >/proc/sys/kernel/kptr_restrict"'])
            self.adb.run(['pull', '/proc/kallsyms', file_path])
Ejemplo n.º 2
0
class NativeLibDownloader(object):
    """Download native libs on device.

    1. Collect info of all native libs in the native_lib_dir on host.
    2. Check the available native libs in /data/local/tmp/native_libs on device.
    3. Sync native libs on device.
    """
    def __init__(self, ndk_path, device_arch, adb):
        self.adb = adb
        self.readelf = ReadElf(ndk_path)
        self.device_arch = device_arch
        self.need_archs = self._get_need_archs()
        self.host_build_id_map = {}  # Map from build_id to HostElfEntry.
        self.device_build_id_map = {
        }  # Map from build_id to relative_path on device.
        # Map from filename to HostElfEntry for elf files without build id.
        self.no_build_id_file_map = {}
        self.name_count_map = {
        }  # Used to give a unique name for each library.
        self.dir_on_device = NATIVE_LIBS_DIR_ON_DEVICE
        self.build_id_list_file = 'build_id_list'

    def _get_need_archs(self):
        """Return the archs of binaries needed on device."""
        if self.device_arch == 'arm64':
            return ['arm', 'arm64']
        if self.device_arch == 'arm':
            return ['arm']
        if self.device_arch == 'x86_64':
            return ['x86', 'x86_64']
        if self.device_arch == 'x86':
            return ['x86']
        return []

    def collect_native_libs_on_host(self, native_lib_dir):
        self.host_build_id_map.clear()
        for root, _, files in os.walk(native_lib_dir):
            for name in files:
                if not name.endswith('.so'):
                    continue
                self.add_native_lib_on_host(os.path.join(root, name), name)

    def add_native_lib_on_host(self, path, name):
        arch = self.readelf.get_arch(path)
        if arch not in self.need_archs:
            return
        sections = self.readelf.get_sections(path)
        score = 0
        if '.debug_info' in sections:
            score = 3
        elif '.gnu_debugdata' in sections:
            score = 2
        elif '.symtab' in sections:
            score = 1
        build_id = self.readelf.get_build_id(path)
        if build_id:
            entry = self.host_build_id_map.get(build_id)
            if entry:
                if entry.score < score:
                    entry.path = path
                    entry.score = score
            else:
                repeat_count = self.name_count_map.get(name, 0)
                self.name_count_map[name] = repeat_count + 1
                unique_name = name if repeat_count == 0 else name + '_' + str(
                    repeat_count)
                self.host_build_id_map[build_id] = HostElfEntry(
                    path, unique_name, score)
        else:
            entry = self.no_build_id_file_map.get(name)
            if entry:
                if entry.score < score:
                    entry.path = path
                    entry.score = score
            else:
                self.no_build_id_file_map[name] = HostElfEntry(
                    path, name, score)

    def collect_native_libs_on_device(self):
        self.device_build_id_map.clear()
        self.adb.check_run(['shell', 'mkdir', '-p', self.dir_on_device])
        if os.path.exists(self.build_id_list_file):
            os.remove(self.build_id_list_file)
        result, output = self.adb.run_and_return_output(
            ['shell', 'ls', self.dir_on_device])
        if not result:
            return
        file_set = set(output.strip().split())
        if self.build_id_list_file not in file_set:
            return
        self.adb.run(['pull', self.dir_on_device + self.build_id_list_file])
        if os.path.exists(self.build_id_list_file):
            with open(self.build_id_list_file, 'rb') as fh:
                for line in fh.readlines():
                    line = bytes_to_str(line).strip()
                    items = line.split('=')
                    if len(items) == 2:
                        build_id, filename = items
                        if filename in file_set:
                            self.device_build_id_map[build_id] = filename
            remove(self.build_id_list_file)

    def sync_native_libs_on_device(self):
        # Push missing native libs on device.
        for build_id in self.host_build_id_map:
            if build_id not in self.device_build_id_map:
                entry = self.host_build_id_map[build_id]
                self.adb.check_run(
                    ['push', entry.path, self.dir_on_device + entry.name])
        # Remove native libs not exist on host.
        for build_id in self.device_build_id_map:
            if build_id not in self.host_build_id_map:
                name = self.device_build_id_map[build_id]
                self.adb.run(['shell', 'rm', self.dir_on_device + name])
        # Push new build_id_list on device.
        with open(self.build_id_list_file, 'wb') as fh:
            for build_id in self.host_build_id_map:
                s = str_to_bytes(
                    '%s=%s\n' %
                    (build_id, self.host_build_id_map[build_id].name))
                fh.write(s)
        self.adb.check_run([
            'push', self.build_id_list_file,
            self.dir_on_device + self.build_id_list_file
        ])
        os.remove(self.build_id_list_file)

        # Push elf files without build id on device.
        for entry in self.no_build_id_file_map.values():
            target = self.dir_on_device + entry.name

            # Skip download if we have a file with the same name and size on device.
            result, output = self.adb.run_and_return_output(
                ['shell', 'ls', '-l', target],
                log_output=False,
                log_stderr=False)
            if result:
                items = output.split()
                if len(items) > 5:
                    try:
                        file_size = int(items[4])
                    except ValueError:
                        file_size = 0
                    if file_size == os.path.getsize(entry.path):
                        continue
            self.adb.check_run(['push', entry.path, target])
Ejemplo n.º 3
0
class PprofProfileGenerator(object):
    def __init__(self, config):
        self.config = config
        self.lib = ReportLib()

        config['binary_cache_dir'] = 'binary_cache'
        if not os.path.isdir(config['binary_cache_dir']):
            config['binary_cache_dir'] = None
        else:
            self.lib.SetSymfs(config['binary_cache_dir'])
        if config.get('perf_data_path'):
            self.lib.SetRecordFile(config['perf_data_path'])
        kallsyms = 'binary_cache/kallsyms'
        if os.path.isfile(kallsyms):
            self.lib.SetKallsymsFile(kallsyms)
        if config.get('show_art_frames'):
            self.lib.ShowArtFrames()
        self.comm_filter = set(
            config['comm_filters']) if config.get('comm_filters') else None
        if config.get('pid_filters'):
            self.pid_filter = {int(x) for x in config['pid_filters']}
        else:
            self.pid_filter = None
        if config.get('tid_filters'):
            self.tid_filter = {int(x) for x in config['tid_filters']}
        else:
            self.tid_filter = None
        self.dso_filter = set(
            config['dso_filters']) if config.get('dso_filters') else None
        self.max_chain_length = config['max_chain_length']
        self.profile = profile_pb2.Profile()
        self.profile.string_table.append('')
        self.string_table = {}
        self.sample_types = {}
        self.sample_map = {}
        self.sample_list = []
        self.location_map = {}
        self.location_list = []
        self.mapping_map = {}
        self.mapping_list = []
        self.function_map = {}
        self.function_list = []

        # Map from dso_name in perf.data to (binary path, build_id).
        self.binary_map = {}
        self.read_elf = ReadElf(self.config['ndk_path'])

    def gen(self):
        # 1. Process all samples in perf.data, aggregate samples.
        while True:
            report_sample = self.lib.GetNextSample()
            if report_sample is None:
                self.lib.Close()
                break
            event = self.lib.GetEventOfCurrentSample()
            symbol = self.lib.GetSymbolOfCurrentSample()
            callchain = self.lib.GetCallChainOfCurrentSample()

            if not self._filter_report_sample(report_sample):
                continue

            sample_type_id = self.get_sample_type_id(event.name)
            sample = Sample()
            sample.add_value(sample_type_id, 1)
            sample.add_value(sample_type_id + 1, report_sample.period)
            if self._filter_symbol(symbol):
                location_id = self.get_location_id(report_sample.ip, symbol)
                sample.add_location_id(location_id)
            for i in range(max(0, callchain.nr - self.max_chain_length),
                           callchain.nr):
                entry = callchain.entries[i]
                if self._filter_symbol(symbol):
                    location_id = self.get_location_id(entry.ip, entry.symbol)
                    sample.add_location_id(location_id)
            if sample.location_ids:
                self.add_sample(sample)

        # 2. Generate line info for locations and functions.
        self.gen_source_lines()

        # 3. Produce samples/locations/functions in profile
        for sample in self.sample_list:
            self.gen_profile_sample(sample)
        for mapping in self.mapping_list:
            self.gen_profile_mapping(mapping)
        for location in self.location_list:
            self.gen_profile_location(location)
        for function in self.function_list:
            self.gen_profile_function(function)

        return self.profile

    def _filter_report_sample(self, sample):
        """Return true if the sample can be used."""
        if self.comm_filter:
            if sample.thread_comm not in self.comm_filter:
                return False
        if self.pid_filter:
            if sample.pid not in self.pid_filter:
                return False
        if self.tid_filter:
            if sample.tid not in self.tid_filter:
                return False
        return True

    def _filter_symbol(self, symbol):
        if not self.dso_filter or symbol.dso_name in self.dso_filter:
            return True
        return False

    def get_string_id(self, str_value):
        if not str_value:
            return 0
        str_id = self.string_table.get(str_value)
        if str_id is not None:
            return str_id
        str_id = len(self.string_table) + 1
        self.string_table[str_value] = str_id
        self.profile.string_table.append(str_value)
        return str_id

    def get_string(self, str_id):
        return self.profile.string_table[str_id]

    def get_sample_type_id(self, name):
        sample_type_id = self.sample_types.get(name)
        if sample_type_id is not None:
            return sample_type_id
        sample_type_id = len(self.profile.sample_type)
        sample_type = self.profile.sample_type.add()
        sample_type.type = self.get_string_id('event_' + name + '_samples')
        sample_type.unit = self.get_string_id('count')
        sample_type = self.profile.sample_type.add()
        sample_type.type = self.get_string_id('event_' + name + '_count')
        sample_type.unit = self.get_string_id('count')
        self.sample_types[name] = sample_type_id
        return sample_type_id

    def get_location_id(self, ip, symbol):
        binary_path, build_id = self.get_binary(symbol.dso_name)
        mapping_id = self.get_mapping_id(symbol.mapping[0], binary_path,
                                         build_id)
        location = Location(mapping_id, ip, symbol.vaddr_in_file)
        function_id = self.get_function_id(symbol.symbol_name, binary_path,
                                           symbol.symbol_addr)
        if function_id:
            # Add Line only when it has a valid function id, see http://b/36988814.
            # Default line info only contains the function name
            line = Line()
            line.function_id = function_id
            location.lines.append(line)

        exist_location = self.location_map.get(location.key)
        if exist_location:
            return exist_location.id
        # location_id starts from 1
        location.id = len(self.location_list) + 1
        self.location_list.append(location)
        self.location_map[location.key] = location
        return location.id

    def get_mapping_id(self, report_mapping, filename, build_id):
        filename_id = self.get_string_id(filename)
        build_id_id = self.get_string_id(build_id)
        mapping = Mapping(report_mapping.start, report_mapping.end,
                          report_mapping.pgoff, filename_id, build_id_id)
        exist_mapping = self.mapping_map.get(mapping.key)
        if exist_mapping:
            return exist_mapping.id
        # mapping_id starts from 1
        mapping.id = len(self.mapping_list) + 1
        self.mapping_list.append(mapping)
        self.mapping_map[mapping.key] = mapping
        return mapping.id

    def get_binary(self, dso_name):
        """ Return (binary_path, build_id) for a given dso_name. """
        value = self.binary_map.get(dso_name)
        if value:
            return value

        binary_path = dso_name
        build_id = ''

        # The build ids in perf.data are padded to 20 bytes, but pprof needs without padding.
        # So read build id from the binary in binary_cache, and check it with build id in
        # perf.data.
        build_id_in_perf_data = self.lib.GetBuildIdForPath(dso_name)
        # Try elf_path in binary cache.
        elf_path = find_real_dso_path(dso_name,
                                      self.config['binary_cache_dir'])
        if elf_path:
            elf_build_id = self.read_elf.get_build_id(elf_path, False)
            if build_id_in_perf_data:
                match = build_id_in_perf_data == self.read_elf.pad_build_id(
                    elf_build_id)
            else:
                # odex files generated by ART on Android O don't contain build id.
                match = not elf_build_id
            if match:
                build_id = elf_build_id
                binary_path = elf_path

        # When there is no matching elf_path, try converting build_id in perf.data.
        if not build_id and build_id_in_perf_data.startswith('0x'):
            # Fallback to the way used by TrimZeroesFromBuildIDString() in quipper.
            build_id = build_id_in_perf_data[2:]  # remove '0x'
            padding = '0' * 8
            while build_id.endswith(padding):
                build_id = build_id[:-len(padding)]

        self.binary_map[dso_name] = (binary_path, build_id)
        return (binary_path, build_id)

    def get_mapping(self, mapping_id):
        return self.mapping_list[mapping_id - 1] if mapping_id > 0 else None

    def get_function_id(self, name, dso_name, vaddr_in_file):
        if name == 'unknown':
            return 0
        function = Function(self.get_string_id(name),
                            self.get_string_id(dso_name), vaddr_in_file)
        exist_function = self.function_map.get(function.key)
        if exist_function:
            return exist_function.id
        # function_id starts from 1
        function.id = len(self.function_list) + 1
        self.function_list.append(function)
        self.function_map[function.key] = function
        return function.id

    def get_function(self, function_id):
        return self.function_list[function_id - 1] if function_id > 0 else None

    def add_sample(self, sample):
        exist_sample = self.sample_map.get(sample.key)
        if exist_sample:
            exist_sample.add_values(sample.values)
        else:
            self.sample_list.append(sample)
            self.sample_map[sample.key] = sample

    def gen_source_lines(self):
        # 1. Create Addr2line instance
        if not self.config.get('binary_cache_dir'):
            log_info(
                "Can't generate line information because binary_cache is missing."
            )
            return
        if not find_tool_path('llvm-symbolizer', self.config['ndk_path']):
            log_info(
                "Can't generate line information because can't find llvm-symbolizer."
            )
            return
        # We have changed dso names to paths in binary_cache in self.get_binary(). So no need to
        # pass binary_cache_dir to addr2line.
        addr2line = Addr2Nearestline(self.config['ndk_path'], None, True)

        # 2. Put all needed addresses to it.
        for location in self.location_list:
            mapping = self.get_mapping(location.mapping_id)
            dso_name = self.get_string(mapping.filename_id)
            if location.lines:
                function = self.get_function(location.lines[0].function_id)
                addr2line.add_addr(dso_name, function.vaddr_in_dso,
                                   location.vaddr_in_dso)
        for function in self.function_list:
            dso_name = self.get_string(function.dso_name_id)
            addr2line.add_addr(dso_name, function.vaddr_in_dso,
                               function.vaddr_in_dso)

        # 3. Generate source lines.
        addr2line.convert_addrs_to_lines()

        # 4. Annotate locations and functions.
        for location in self.location_list:
            if not location.lines:
                continue
            mapping = self.get_mapping(location.mapping_id)
            dso_name = self.get_string(mapping.filename_id)
            dso = addr2line.get_dso(dso_name)
            if not dso:
                continue
            sources = addr2line.get_addr_source(dso, location.vaddr_in_dso)
            if not sources:
                continue
            for (source_id, source) in enumerate(sources):
                source_file, source_line, function_name = source
                function_id = self.get_function_id(function_name, dso_name, 0)
                if function_id == 0:
                    continue
                if source_id == 0:
                    # Clear default line info
                    location.lines = []
                location.lines.append(
                    self.add_line(source_file, source_line, function_id))

        for function in self.function_list:
            dso_name = self.get_string(function.dso_name_id)
            if function.vaddr_in_dso:
                dso = addr2line.get_dso(dso_name)
                if not dso:
                    continue
                sources = addr2line.get_addr_source(dso, function.vaddr_in_dso)
                if sources:
                    source_file, source_line, _ = sources[0]
                    function.source_filename_id = self.get_string_id(
                        source_file)
                    function.start_line = source_line

    def add_line(self, source_file, source_line, function_id):
        line = Line()
        function = self.get_function(function_id)
        function.source_filename_id = self.get_string_id(source_file)
        line.function_id = function_id
        line.line = source_line
        return line

    def gen_profile_sample(self, sample):
        profile_sample = self.profile.sample.add()
        profile_sample.location_id.extend(sample.location_ids)
        sample_type_count = len(self.sample_types) * 2
        values = [0] * sample_type_count
        for sample_type_id in sample.values:
            values[sample_type_id] = sample.values[sample_type_id]
        profile_sample.value.extend(values)

    def gen_profile_mapping(self, mapping):
        profile_mapping = self.profile.mapping.add()
        profile_mapping.id = mapping.id
        profile_mapping.memory_start = mapping.memory_start
        profile_mapping.memory_limit = mapping.memory_limit
        profile_mapping.file_offset = mapping.file_offset
        profile_mapping.filename = mapping.filename_id
        profile_mapping.build_id = mapping.build_id_id
        profile_mapping.has_filenames = True
        profile_mapping.has_functions = True
        if self.config.get('binary_cache_dir'):
            profile_mapping.has_line_numbers = True
            profile_mapping.has_inline_frames = True
        else:
            profile_mapping.has_line_numbers = False
            profile_mapping.has_inline_frames = False

    def gen_profile_location(self, location):
        profile_location = self.profile.location.add()
        profile_location.id = location.id
        profile_location.mapping_id = location.mapping_id
        profile_location.address = location.address
        for i in range(len(location.lines)):
            line = profile_location.line.add()
            line.function_id = location.lines[i].function_id
            line.line = location.lines[i].line

    def gen_profile_function(self, function):
        profile_function = self.profile.function.add()
        profile_function.id = function.id
        profile_function.name = function.name_id
        profile_function.system_name = function.name_id
        profile_function.filename = function.source_filename_id
        profile_function.start_line = function.start_line
Ejemplo n.º 4
0
class NativeLibDownloader(object):
    """Download native libs on device.

    1. Collect info of all native libs in the native_lib_dir on host.
    2. Check the available native libs in /data/local/tmp/native_libs on device.
    3. Sync native libs on device.
    """
    def __init__(self, ndk_path, device_arch, adb):
        self.adb = adb
        self.readelf = ReadElf(ndk_path)
        self.device_arch = device_arch
        self.need_archs = self._get_need_archs()
        self.host_build_id_map = {}  # Map from build_id to HostElfEntry.
        self.device_build_id_map = {}  # Map from build_id to relative_path on device.
        self.name_count_map = {}  # Used to give a unique name for each library.
        self.dir_on_device = NATIVE_LIBS_DIR_ON_DEVICE
        self.build_id_list_file = 'build_id_list'

    def _get_need_archs(self):
        """Return the archs of binaries needed on device."""
        if self.device_arch == 'arm64':
            return ['arm', 'arm64']
        if self.device_arch == 'arm':
            return ['arm']
        if self.device_arch == 'x86_64':
            return ['x86', 'x86_64']
        if self.device_arch == 'x86':
            return ['x86']
        return []

    def collect_native_libs_on_host(self, native_lib_dir):
        self.host_build_id_map.clear()
        for root, _, files in os.walk(native_lib_dir):
            for name in files:
                if not name.endswith('.so'):
                    continue
                self.add_native_lib_on_host(os.path.join(root, name), name)

    def add_native_lib_on_host(self, path, name):
        build_id = self.readelf.get_build_id(path)
        if not build_id:
            return
        arch = self.readelf.get_arch(path)
        if arch not in self.need_archs:
            return
        sections = self.readelf.get_sections(path)
        score = 0
        if '.debug_info' in sections:
            score = 3
        elif '.gnu_debugdata' in sections:
            score = 2
        elif '.symtab' in sections:
            score = 1
        entry = self.host_build_id_map.get(build_id)
        if entry:
            if entry.score < score:
                entry.path = path
                entry.score = score
        else:
            repeat_count = self.name_count_map.get(name, 0)
            self.name_count_map[name] = repeat_count + 1
            unique_name = name if repeat_count == 0 else name + '_' + str(repeat_count)
            self.host_build_id_map[build_id] = HostElfEntry(path, unique_name, score)

    def collect_native_libs_on_device(self):
        self.device_build_id_map.clear()
        self.adb.check_run(['shell', 'mkdir', '-p', self.dir_on_device])
        if os.path.exists(self.build_id_list_file):
            os.remove(self.build_id_list_file)
        self.adb.run(['pull', self.dir_on_device + self.build_id_list_file])
        if os.path.exists(self.build_id_list_file):
            with open(self.build_id_list_file, 'rb') as fh:
                for line in fh.readlines():
                    line = bytes_to_str(line).strip()
                    items = line.split('=')
                    if len(items) == 2:
                        self.device_build_id_map[items[0]] = items[1]
            remove(self.build_id_list_file)

    def sync_natives_libs_on_device(self):
        # Push missing native libs on device.
        for build_id in self.host_build_id_map:
            if build_id not in self.device_build_id_map:
                entry = self.host_build_id_map[build_id]
                self.adb.check_run(['push', entry.path, self.dir_on_device + entry.name])
        # Remove native libs not exist on host.
        for build_id in self.device_build_id_map:
            if build_id not in self.host_build_id_map:
                name = self.device_build_id_map[build_id]
                self.adb.run(['shell', 'rm', self.dir_on_device + name])
        # Push new build_id_list on device.
        with open(self.build_id_list_file, 'wb') as fh:
            for build_id in self.host_build_id_map:
                s = str_to_bytes('%s=%s\n' % (build_id, self.host_build_id_map[build_id].name))
                fh.write(s)
        self.adb.check_run(['push', self.build_id_list_file,
                            self.dir_on_device + self.build_id_list_file])
        os.remove(self.build_id_list_file)
Ejemplo n.º 5
0
class NativeLibDownloader(object):
    """Download native libs on device.

    1. Collect info of all native libs in the native_lib_dir on host.
    2. Check the available native libs in /data/local/tmp/native_libs on device.
    3. Sync native libs on device.
    """
    def __init__(self, ndk_path, device_arch, adb):
        self.adb = adb
        self.readelf = ReadElf(ndk_path)
        self.device_arch = device_arch
        self.need_archs = self._get_need_archs()
        self.host_build_id_map = {}  # Map from build_id to HostElfEntry.
        self.device_build_id_map = {}  # Map from build_id to relative_path on device.
        self.name_count_map = {}  # Used to give a unique name for each library.
        self.dir_on_device = NATIVE_LIBS_DIR_ON_DEVICE
        self.build_id_list_file = 'build_id_list'

    def _get_need_archs(self):
        """Return the archs of binaries needed on device."""
        if self.device_arch == 'arm64':
            return ['arm', 'arm64']
        if self.device_arch == 'arm':
            return ['arm']
        if self.device_arch == 'x86_64':
            return ['x86', 'x86_64']
        if self.device_arch == 'x86':
            return ['x86']
        return []

    def collect_native_libs_on_host(self, native_lib_dir):
        self.host_build_id_map.clear()
        for root, _, files in os.walk(native_lib_dir):
            for name in files:
                if not name.endswith('.so'):
                    continue
                self.add_native_lib_on_host(os.path.join(root, name), name)

    def add_native_lib_on_host(self, path, name):
        build_id = self.readelf.get_build_id(path)
        if not build_id:
            return
        arch = self.readelf.get_arch(path)
        if arch not in self.need_archs:
            return
        sections = self.readelf.get_sections(path)
        score = 0
        if '.debug_info' in sections:
            score = 3
        elif '.gnu_debugdata' in sections:
            score = 2
        elif '.symtab' in sections:
            score = 1
        entry = self.host_build_id_map.get(build_id)
        if entry:
            if entry.score < score:
                entry.path = path
                entry.score = score
        else:
            repeat_count = self.name_count_map.get(name, 0)
            self.name_count_map[name] = repeat_count + 1
            unique_name = name if repeat_count == 0 else name + '_' + str(repeat_count)
            self.host_build_id_map[build_id] = HostElfEntry(path, unique_name, score)

    def collect_native_libs_on_device(self):
        self.device_build_id_map.clear()
        self.adb.check_run(['shell', 'mkdir', '-p', self.dir_on_device])
        if os.path.exists(self.build_id_list_file):
            os.remove(self.build_id_list_file)
        self.adb.run(['pull', self.dir_on_device + self.build_id_list_file])
        if os.path.exists(self.build_id_list_file):
            with open(self.build_id_list_file, 'rb') as fh:
                for line in fh.readlines():
                    line = line.strip()
                    items = line.split('=')
                    if len(items) == 2:
                        self.device_build_id_map[items[0]] = items[1]
            remove(self.build_id_list_file)

    def sync_natives_libs_on_device(self):
        # Push missing native libs on device.
        for build_id in self.host_build_id_map:
            if build_id not in self.device_build_id_map:
                entry = self.host_build_id_map[build_id]
                self.adb.check_run(['push', entry.path, self.dir_on_device + entry.name])
        # Remove native libs not exist on host.
        for build_id in self.device_build_id_map:
            if build_id not in self.host_build_id_map:
                name = self.device_build_id_map[build_id]
                self.adb.run(['shell', 'rm', self.dir_on_device + name])
        # Push new build_id_list on device.
        with open(self.build_id_list_file, 'wb') as fh:
            for build_id in self.host_build_id_map:
                fh.write('%s=%s\n' % (build_id, self.host_build_id_map[build_id].name))
        self.adb.check_run(['push', self.build_id_list_file,
                            self.dir_on_device + self.build_id_list_file])
        os.remove(self.build_id_list_file)