class BinaryCacheBuilder(object):
"""Collect all binaries needed by perf.data in binary_cache."""
def __init__(self, ndk_path, disable_adb_root):
self.adb = AdbHelper(enable_switch_to_root=not disable_adb_root)
self.readelf = ReadElf(ndk_path)
self.binary_cache_dir = 'binary_cache'
if not os.path.isdir(self.binary_cache_dir):
os.makedirs(self.binary_cache_dir)
self.binaries = {}
def build_binary_cache(self, perf_data_path, symfs_dirs):
self._collect_used_binaries(perf_data_path)
self.copy_binaries_from_symfs_dirs(symfs_dirs)
self._pull_binaries_from_device()
self._pull_kernel_symbols()
def _collect_used_binaries(self, perf_data_path):
"""read perf.data, collect all used binaries and their build id (if available)."""
# A dict mapping from binary name to build_id
binaries = {}
lib = ReportLib()
lib.SetRecordFile(perf_data_path)
lib.SetLogSeverity('error')
while True:
sample = lib.GetNextSample()
if sample is None:
lib.Close()
break
symbols = [lib.GetSymbolOfCurrentSample()]
callchain = lib.GetCallChainOfCurrentSample()
for i in range(callchain.nr):
symbols.append(callchain.entries[i].symbol)
for symbol in symbols:
dso_name = symbol.dso_name
if dso_name not in binaries:
if is_jit_symfile(dso_name):
continue
binaries[dso_name] = lib.GetBuildIdForPath(dso_name)
self.binaries = binaries
def copy_binaries_from_symfs_dirs(self, symfs_dirs):
"""collect all files in symfs_dirs."""
if not symfs_dirs:
return
# It is possible that the path of the binary in symfs_dirs doesn't match
# the one recorded in perf.data. For example, a file in symfs_dirs might
# be "debug/arm/obj/armeabi-v7a/libsudo-game-jni.so", but the path in
# perf.data is "/data/app/xxxx/lib/arm/libsudo-game-jni.so". So we match
# binaries if they have the same filename (like libsudo-game-jni.so)
# and same build_id.
# Map from filename to binary paths.
filename_dict = {}
for binary in self.binaries:
index = binary.rfind('/')
filename = binary[index+1:]
paths = filename_dict.get(filename)
if paths is None:
filename_dict[filename] = paths = []
paths.append(binary)
# Walk through all files in symfs_dirs, and copy matching files to build_cache.
for symfs_dir in symfs_dirs:
for root, _, files in os.walk(symfs_dir):
for filename in files:
paths = filename_dict.get(filename)
if not paths:
continue
build_id = self._read_build_id(os.path.join(root, filename))
if not build_id:
continue
for binary in paths:
expected_build_id = self.binaries.get(binary)
if expected_build_id == build_id:
self._copy_to_binary_cache(os.path.join(root, filename),
expected_build_id, binary)
break
def _copy_to_binary_cache(self, from_path, expected_build_id, target_file):
if target_file[0] == '/':
target_file = target_file[1:]
target_file = target_file.replace('/', os.sep)
target_file = os.path.join(self.binary_cache_dir, target_file)
if not self._need_to_copy(from_path, target_file, expected_build_id):
# The existing file in binary_cache can provide more information, so no need to copy.
return
target_dir = os.path.dirname(target_file)
if not os.path.isdir(target_dir):
os.makedirs(target_dir)
log_info('copy to binary_cache: %s to %s' % (from_path, target_file))
shutil.copy(from_path, target_file)
def _need_to_copy(self, source_file, target_file, expected_build_id):
if not os.path.isfile(target_file):
return True
if self._read_build_id(target_file) != expected_build_id:
return True
return self._get_file_stripped_level(source_file) < self._get_file_stripped_level(
target_file)
def _get_file_stripped_level(self, file_path):
"""Return stripped level of an ELF file. Larger value means more stripped."""
sections = self.readelf.get_sections(file_path)
if '.debug_line' in sections:
return 0
if '.symtab' in sections:
return 1
return 2
def _pull_binaries_from_device(self):
"""pull binaries needed in perf.data to binary_cache."""
for binary in self.binaries:
build_id = self.binaries[binary]
if not binary.startswith('/') or binary == "//anon" or binary.startswith("/dev/"):
# [kernel.kallsyms] or unknown, or something we can't find binary.
continue
binary_cache_file = binary[1:].replace('/', os.sep)
binary_cache_file = os.path.join(self.binary_cache_dir, binary_cache_file)
self._check_and_pull_binary(binary, build_id, binary_cache_file)
def _check_and_pull_binary(self, binary, expected_build_id, binary_cache_file):
"""If the binary_cache_file exists and has the expected_build_id, there
is no need to pull the binary from device. Otherwise, pull it.
"""
need_pull = True
if os.path.isfile(binary_cache_file):
need_pull = False
if expected_build_id:
build_id = self._read_build_id(binary_cache_file)
if expected_build_id != build_id:
need_pull = True
if need_pull:
target_dir = os.path.dirname(binary_cache_file)
if not os.path.isdir(target_dir):
os.makedirs(target_dir)
if os.path.isfile(binary_cache_file):
os.remove(binary_cache_file)
log_info('pull file to binary_cache: %s to %s' % (binary, binary_cache_file))
self._pull_file_from_device(binary, binary_cache_file)
else:
log_info('use current file in binary_cache: %s' % binary_cache_file)
def _read_build_id(self, file_path):
"""read build id of a binary on host."""
return self.readelf.get_build_id(file_path)
def _pull_file_from_device(self, device_path, host_path):
if self.adb.run(['pull', device_path, host_path]):
return True
# In non-root device, we can't pull /data/app/XXX/base.odex directly.
# Instead, we can first copy the file to /data/local/tmp, then pull it.
filename = device_path[device_path.rfind('/')+1:]
if (self.adb.run(['shell', 'cp', device_path, '/data/local/tmp']) and
self.adb.run(['pull', '/data/local/tmp/' + filename, host_path])):
self.adb.run(['shell', 'rm', '/data/local/tmp/' + filename])
return True
log_warning('failed to pull %s from device' % device_path)
return False
def _pull_kernel_symbols(self):
file_path = os.path.join(self.binary_cache_dir, 'kallsyms')
if os.path.isfile(file_path):
os.remove(file_path)
if self.adb.switch_to_root():
self.adb.run(['shell', '"echo 0 >/proc/sys/kernel/kptr_restrict"'])
self.adb.run(['pull', '/proc/kallsyms', file_path])
class NativeLibDownloader(object):
"""Download native libs on device.
1. Collect info of all native libs in the native_lib_dir on host.
2. Check the available native libs in /data/local/tmp/native_libs on device.
3. Sync native libs on device.
"""
def __init__(self, ndk_path, device_arch, adb):
self.adb = adb
self.readelf = ReadElf(ndk_path)
self.device_arch = device_arch
self.need_archs = self._get_need_archs()
self.host_build_id_map = {} # Map from build_id to HostElfEntry.
self.device_build_id_map = {
} # Map from build_id to relative_path on device.
# Map from filename to HostElfEntry for elf files without build id.
self.no_build_id_file_map = {}
self.name_count_map = {
} # Used to give a unique name for each library.
self.dir_on_device = NATIVE_LIBS_DIR_ON_DEVICE
self.build_id_list_file = 'build_id_list'
def _get_need_archs(self):
"""Return the archs of binaries needed on device."""
if self.device_arch == 'arm64':
return ['arm', 'arm64']
if self.device_arch == 'arm':
return ['arm']
if self.device_arch == 'x86_64':
return ['x86', 'x86_64']
if self.device_arch == 'x86':
return ['x86']
return []
def collect_native_libs_on_host(self, native_lib_dir):
self.host_build_id_map.clear()
for root, _, files in os.walk(native_lib_dir):
for name in files:
if not name.endswith('.so'):
continue
self.add_native_lib_on_host(os.path.join(root, name), name)
def add_native_lib_on_host(self, path, name):
arch = self.readelf.get_arch(path)
if arch not in self.need_archs:
return
sections = self.readelf.get_sections(path)
score = 0
if '.debug_info' in sections:
score = 3
elif '.gnu_debugdata' in sections:
score = 2
elif '.symtab' in sections:
score = 1
build_id = self.readelf.get_build_id(path)
if build_id:
entry = self.host_build_id_map.get(build_id)
if entry:
if entry.score < score:
entry.path = path
entry.score = score
else:
repeat_count = self.name_count_map.get(name, 0)
self.name_count_map[name] = repeat_count + 1
unique_name = name if repeat_count == 0 else name + '_' + str(
repeat_count)
self.host_build_id_map[build_id] = HostElfEntry(
path, unique_name, score)
else:
entry = self.no_build_id_file_map.get(name)
if entry:
if entry.score < score:
entry.path = path
entry.score = score
else:
self.no_build_id_file_map[name] = HostElfEntry(
path, name, score)
def collect_native_libs_on_device(self):
self.device_build_id_map.clear()
self.adb.check_run(['shell', 'mkdir', '-p', self.dir_on_device])
if os.path.exists(self.build_id_list_file):
os.remove(self.build_id_list_file)
result, output = self.adb.run_and_return_output(
['shell', 'ls', self.dir_on_device])
if not result:
return
file_set = set(output.strip().split())
if self.build_id_list_file not in file_set:
return
self.adb.run(['pull', self.dir_on_device + self.build_id_list_file])
if os.path.exists(self.build_id_list_file):
with open(self.build_id_list_file, 'rb') as fh:
for line in fh.readlines():
line = bytes_to_str(line).strip()
items = line.split('=')
if len(items) == 2:
build_id, filename = items
if filename in file_set:
self.device_build_id_map[build_id] = filename
remove(self.build_id_list_file)
def sync_native_libs_on_device(self):
# Push missing native libs on device.
for build_id in self.host_build_id_map:
if build_id not in self.device_build_id_map:
entry = self.host_build_id_map[build_id]
self.adb.check_run(
['push', entry.path, self.dir_on_device + entry.name])
# Remove native libs not exist on host.
for build_id in self.device_build_id_map:
if build_id not in self.host_build_id_map:
name = self.device_build_id_map[build_id]
self.adb.run(['shell', 'rm', self.dir_on_device + name])
# Push new build_id_list on device.
with open(self.build_id_list_file, 'wb') as fh:
for build_id in self.host_build_id_map:
s = str_to_bytes(
'%s=%s\n' %
(build_id, self.host_build_id_map[build_id].name))
fh.write(s)
self.adb.check_run([
'push', self.build_id_list_file,
self.dir_on_device + self.build_id_list_file
])
os.remove(self.build_id_list_file)
# Push elf files without build id on device.
for entry in self.no_build_id_file_map.values():
target = self.dir_on_device + entry.name
# Skip download if we have a file with the same name and size on device.
result, output = self.adb.run_and_return_output(
['shell', 'ls', '-l', target],
log_output=False,
log_stderr=False)
if result:
items = output.split()
if len(items) > 5:
try:
file_size = int(items[4])
except ValueError:
file_size = 0
if file_size == os.path.getsize(entry.path):
continue
self.adb.check_run(['push', entry.path, target])
class PprofProfileGenerator(object):
def __init__(self, config):
self.config = config
self.lib = ReportLib()
config['binary_cache_dir'] = 'binary_cache'
if not os.path.isdir(config['binary_cache_dir']):
config['binary_cache_dir'] = None
else:
self.lib.SetSymfs(config['binary_cache_dir'])
if config.get('perf_data_path'):
self.lib.SetRecordFile(config['perf_data_path'])
kallsyms = 'binary_cache/kallsyms'
if os.path.isfile(kallsyms):
self.lib.SetKallsymsFile(kallsyms)
if config.get('show_art_frames'):
self.lib.ShowArtFrames()
self.comm_filter = set(
config['comm_filters']) if config.get('comm_filters') else None
if config.get('pid_filters'):
self.pid_filter = {int(x) for x in config['pid_filters']}
else:
self.pid_filter = None
if config.get('tid_filters'):
self.tid_filter = {int(x) for x in config['tid_filters']}
else:
self.tid_filter = None
self.dso_filter = set(
config['dso_filters']) if config.get('dso_filters') else None
self.max_chain_length = config['max_chain_length']
self.profile = profile_pb2.Profile()
self.profile.string_table.append('')
self.string_table = {}
self.sample_types = {}
self.sample_map = {}
self.sample_list = []
self.location_map = {}
self.location_list = []
self.mapping_map = {}
self.mapping_list = []
self.function_map = {}
self.function_list = []
# Map from dso_name in perf.data to (binary path, build_id).
self.binary_map = {}
self.read_elf = ReadElf(self.config['ndk_path'])
def gen(self):
# 1. Process all samples in perf.data, aggregate samples.
while True:
report_sample = self.lib.GetNextSample()
if report_sample is None:
self.lib.Close()
break
event = self.lib.GetEventOfCurrentSample()
symbol = self.lib.GetSymbolOfCurrentSample()
callchain = self.lib.GetCallChainOfCurrentSample()
if not self._filter_report_sample(report_sample):
continue
sample_type_id = self.get_sample_type_id(event.name)
sample = Sample()
sample.add_value(sample_type_id, 1)
sample.add_value(sample_type_id + 1, report_sample.period)
if self._filter_symbol(symbol):
location_id = self.get_location_id(report_sample.ip, symbol)
sample.add_location_id(location_id)
for i in range(max(0, callchain.nr - self.max_chain_length),
callchain.nr):
entry = callchain.entries[i]
if self._filter_symbol(symbol):
location_id = self.get_location_id(entry.ip, entry.symbol)
sample.add_location_id(location_id)
if sample.location_ids:
self.add_sample(sample)
# 2. Generate line info for locations and functions.
self.gen_source_lines()
# 3. Produce samples/locations/functions in profile
for sample in self.sample_list:
self.gen_profile_sample(sample)
for mapping in self.mapping_list:
self.gen_profile_mapping(mapping)
for location in self.location_list:
self.gen_profile_location(location)
for function in self.function_list:
self.gen_profile_function(function)
return self.profile
def _filter_report_sample(self, sample):
"""Return true if the sample can be used."""
if self.comm_filter:
if sample.thread_comm not in self.comm_filter:
return False
if self.pid_filter:
if sample.pid not in self.pid_filter:
return False
if self.tid_filter:
if sample.tid not in self.tid_filter:
return False
return True
def _filter_symbol(self, symbol):
if not self.dso_filter or symbol.dso_name in self.dso_filter:
return True
return False
def get_string_id(self, str_value):
if not str_value:
return 0
str_id = self.string_table.get(str_value)
if str_id is not None:
return str_id
str_id = len(self.string_table) + 1
self.string_table[str_value] = str_id
self.profile.string_table.append(str_value)
return str_id
def get_string(self, str_id):
return self.profile.string_table[str_id]
def get_sample_type_id(self, name):
sample_type_id = self.sample_types.get(name)
if sample_type_id is not None:
return sample_type_id
sample_type_id = len(self.profile.sample_type)
sample_type = self.profile.sample_type.add()
sample_type.type = self.get_string_id('event_' + name + '_samples')
sample_type.unit = self.get_string_id('count')
sample_type = self.profile.sample_type.add()
sample_type.type = self.get_string_id('event_' + name + '_count')
sample_type.unit = self.get_string_id('count')
self.sample_types[name] = sample_type_id
return sample_type_id
def get_location_id(self, ip, symbol):
binary_path, build_id = self.get_binary(symbol.dso_name)
mapping_id = self.get_mapping_id(symbol.mapping[0], binary_path,
build_id)
location = Location(mapping_id, ip, symbol.vaddr_in_file)
function_id = self.get_function_id(symbol.symbol_name, binary_path,
symbol.symbol_addr)
if function_id:
# Add Line only when it has a valid function id, see http://b/36988814.
# Default line info only contains the function name
line = Line()
line.function_id = function_id
location.lines.append(line)
exist_location = self.location_map.get(location.key)
if exist_location:
return exist_location.id
# location_id starts from 1
location.id = len(self.location_list) + 1
self.location_list.append(location)
self.location_map[location.key] = location
return location.id
def get_mapping_id(self, report_mapping, filename, build_id):
filename_id = self.get_string_id(filename)
build_id_id = self.get_string_id(build_id)
mapping = Mapping(report_mapping.start, report_mapping.end,
report_mapping.pgoff, filename_id, build_id_id)
exist_mapping = self.mapping_map.get(mapping.key)
if exist_mapping:
return exist_mapping.id
# mapping_id starts from 1
mapping.id = len(self.mapping_list) + 1
self.mapping_list.append(mapping)
self.mapping_map[mapping.key] = mapping
return mapping.id
def get_binary(self, dso_name):
""" Return (binary_path, build_id) for a given dso_name. """
value = self.binary_map.get(dso_name)
if value:
return value
binary_path = dso_name
build_id = ''
# The build ids in perf.data are padded to 20 bytes, but pprof needs without padding.
# So read build id from the binary in binary_cache, and check it with build id in
# perf.data.
build_id_in_perf_data = self.lib.GetBuildIdForPath(dso_name)
# Try elf_path in binary cache.
elf_path = find_real_dso_path(dso_name,
self.config['binary_cache_dir'])
if elf_path:
elf_build_id = self.read_elf.get_build_id(elf_path, False)
if build_id_in_perf_data:
match = build_id_in_perf_data == self.read_elf.pad_build_id(
elf_build_id)
else:
# odex files generated by ART on Android O don't contain build id.
match = not elf_build_id
if match:
build_id = elf_build_id
binary_path = elf_path
# When there is no matching elf_path, try converting build_id in perf.data.
if not build_id and build_id_in_perf_data.startswith('0x'):
# Fallback to the way used by TrimZeroesFromBuildIDString() in quipper.
build_id = build_id_in_perf_data[2:] # remove '0x'
padding = '0' * 8
while build_id.endswith(padding):
build_id = build_id[:-len(padding)]
self.binary_map[dso_name] = (binary_path, build_id)
return (binary_path, build_id)
def get_mapping(self, mapping_id):
return self.mapping_list[mapping_id - 1] if mapping_id > 0 else None
def get_function_id(self, name, dso_name, vaddr_in_file):
if name == 'unknown':
return 0
function = Function(self.get_string_id(name),
self.get_string_id(dso_name), vaddr_in_file)
exist_function = self.function_map.get(function.key)
if exist_function:
return exist_function.id
# function_id starts from 1
function.id = len(self.function_list) + 1
self.function_list.append(function)
self.function_map[function.key] = function
return function.id
def get_function(self, function_id):
return self.function_list[function_id - 1] if function_id > 0 else None
def add_sample(self, sample):
exist_sample = self.sample_map.get(sample.key)
if exist_sample:
exist_sample.add_values(sample.values)
else:
self.sample_list.append(sample)
self.sample_map[sample.key] = sample
def gen_source_lines(self):
# 1. Create Addr2line instance
if not self.config.get('binary_cache_dir'):
log_info(
"Can't generate line information because binary_cache is missing."
)
return
if not find_tool_path('llvm-symbolizer', self.config['ndk_path']):
log_info(
"Can't generate line information because can't find llvm-symbolizer."
)
return
# We have changed dso names to paths in binary_cache in self.get_binary(). So no need to
# pass binary_cache_dir to addr2line.
addr2line = Addr2Nearestline(self.config['ndk_path'], None, True)
# 2. Put all needed addresses to it.
for location in self.location_list:
mapping = self.get_mapping(location.mapping_id)
dso_name = self.get_string(mapping.filename_id)
if location.lines:
function = self.get_function(location.lines[0].function_id)
addr2line.add_addr(dso_name, function.vaddr_in_dso,
location.vaddr_in_dso)
for function in self.function_list:
dso_name = self.get_string(function.dso_name_id)
addr2line.add_addr(dso_name, function.vaddr_in_dso,
function.vaddr_in_dso)
# 3. Generate source lines.
addr2line.convert_addrs_to_lines()
# 4. Annotate locations and functions.
for location in self.location_list:
if not location.lines:
continue
mapping = self.get_mapping(location.mapping_id)
dso_name = self.get_string(mapping.filename_id)
dso = addr2line.get_dso(dso_name)
if not dso:
continue
sources = addr2line.get_addr_source(dso, location.vaddr_in_dso)
if not sources:
continue
for (source_id, source) in enumerate(sources):
source_file, source_line, function_name = source
function_id = self.get_function_id(function_name, dso_name, 0)
if function_id == 0:
continue
if source_id == 0:
# Clear default line info
location.lines = []
location.lines.append(
self.add_line(source_file, source_line, function_id))
for function in self.function_list:
dso_name = self.get_string(function.dso_name_id)
if function.vaddr_in_dso:
dso = addr2line.get_dso(dso_name)
if not dso:
continue
sources = addr2line.get_addr_source(dso, function.vaddr_in_dso)
if sources:
source_file, source_line, _ = sources[0]
function.source_filename_id = self.get_string_id(
source_file)
function.start_line = source_line
def add_line(self, source_file, source_line, function_id):
line = Line()
function = self.get_function(function_id)
function.source_filename_id = self.get_string_id(source_file)
line.function_id = function_id
line.line = source_line
return line
def gen_profile_sample(self, sample):
profile_sample = self.profile.sample.add()
profile_sample.location_id.extend(sample.location_ids)
sample_type_count = len(self.sample_types) * 2
values = [0] * sample_type_count
for sample_type_id in sample.values:
values[sample_type_id] = sample.values[sample_type_id]
profile_sample.value.extend(values)
def gen_profile_mapping(self, mapping):
profile_mapping = self.profile.mapping.add()
profile_mapping.id = mapping.id
profile_mapping.memory_start = mapping.memory_start
profile_mapping.memory_limit = mapping.memory_limit
profile_mapping.file_offset = mapping.file_offset
profile_mapping.filename = mapping.filename_id
profile_mapping.build_id = mapping.build_id_id
profile_mapping.has_filenames = True
profile_mapping.has_functions = True
if self.config.get('binary_cache_dir'):
profile_mapping.has_line_numbers = True
profile_mapping.has_inline_frames = True
else:
profile_mapping.has_line_numbers = False
profile_mapping.has_inline_frames = False
def gen_profile_location(self, location):
profile_location = self.profile.location.add()
profile_location.id = location.id
profile_location.mapping_id = location.mapping_id
profile_location.address = location.address
for i in range(len(location.lines)):
line = profile_location.line.add()
line.function_id = location.lines[i].function_id
line.line = location.lines[i].line
def gen_profile_function(self, function):
profile_function = self.profile.function.add()
profile_function.id = function.id
profile_function.name = function.name_id
profile_function.system_name = function.name_id
profile_function.filename = function.source_filename_id
profile_function.start_line = function.start_line
class NativeLibDownloader(object):
"""Download native libs on device.
1. Collect info of all native libs in the native_lib_dir on host.
2. Check the available native libs in /data/local/tmp/native_libs on device.
3. Sync native libs on device.
"""
def __init__(self, ndk_path, device_arch, adb):
self.adb = adb
self.readelf = ReadElf(ndk_path)
self.device_arch = device_arch
self.need_archs = self._get_need_archs()
self.host_build_id_map = {} # Map from build_id to HostElfEntry.
self.device_build_id_map = {} # Map from build_id to relative_path on device.
self.name_count_map = {} # Used to give a unique name for each library.
self.dir_on_device = NATIVE_LIBS_DIR_ON_DEVICE
self.build_id_list_file = 'build_id_list'
def _get_need_archs(self):
"""Return the archs of binaries needed on device."""
if self.device_arch == 'arm64':
return ['arm', 'arm64']
if self.device_arch == 'arm':
return ['arm']
if self.device_arch == 'x86_64':
return ['x86', 'x86_64']
if self.device_arch == 'x86':
return ['x86']
return []
def collect_native_libs_on_host(self, native_lib_dir):
self.host_build_id_map.clear()
for root, _, files in os.walk(native_lib_dir):
for name in files:
if not name.endswith('.so'):
continue
self.add_native_lib_on_host(os.path.join(root, name), name)
def add_native_lib_on_host(self, path, name):
build_id = self.readelf.get_build_id(path)
if not build_id:
return
arch = self.readelf.get_arch(path)
if arch not in self.need_archs:
return
sections = self.readelf.get_sections(path)
score = 0
if '.debug_info' in sections:
score = 3
elif '.gnu_debugdata' in sections:
score = 2
elif '.symtab' in sections:
score = 1
entry = self.host_build_id_map.get(build_id)
if entry:
if entry.score < score:
entry.path = path
entry.score = score
else:
repeat_count = self.name_count_map.get(name, 0)
self.name_count_map[name] = repeat_count + 1
unique_name = name if repeat_count == 0 else name + '_' + str(repeat_count)
self.host_build_id_map[build_id] = HostElfEntry(path, unique_name, score)
def collect_native_libs_on_device(self):
self.device_build_id_map.clear()
self.adb.check_run(['shell', 'mkdir', '-p', self.dir_on_device])
if os.path.exists(self.build_id_list_file):
os.remove(self.build_id_list_file)
self.adb.run(['pull', self.dir_on_device + self.build_id_list_file])
if os.path.exists(self.build_id_list_file):
with open(self.build_id_list_file, 'rb') as fh:
for line in fh.readlines():
line = bytes_to_str(line).strip()
items = line.split('=')
if len(items) == 2:
self.device_build_id_map[items[0]] = items[1]
remove(self.build_id_list_file)
def sync_natives_libs_on_device(self):
# Push missing native libs on device.
for build_id in self.host_build_id_map:
if build_id not in self.device_build_id_map:
entry = self.host_build_id_map[build_id]
self.adb.check_run(['push', entry.path, self.dir_on_device + entry.name])
# Remove native libs not exist on host.
for build_id in self.device_build_id_map:
if build_id not in self.host_build_id_map:
name = self.device_build_id_map[build_id]
self.adb.run(['shell', 'rm', self.dir_on_device + name])
# Push new build_id_list on device.
with open(self.build_id_list_file, 'wb') as fh:
for build_id in self.host_build_id_map:
s = str_to_bytes('%s=%s\n' % (build_id, self.host_build_id_map[build_id].name))
fh.write(s)
self.adb.check_run(['push', self.build_id_list_file,
self.dir_on_device + self.build_id_list_file])
os.remove(self.build_id_list_file)
class NativeLibDownloader(object):
"""Download native libs on device.
1. Collect info of all native libs in the native_lib_dir on host.
2. Check the available native libs in /data/local/tmp/native_libs on device.
3. Sync native libs on device.
"""
def __init__(self, ndk_path, device_arch, adb):
self.adb = adb
self.readelf = ReadElf(ndk_path)
self.device_arch = device_arch
self.need_archs = self._get_need_archs()
self.host_build_id_map = {} # Map from build_id to HostElfEntry.
self.device_build_id_map = {} # Map from build_id to relative_path on device.
self.name_count_map = {} # Used to give a unique name for each library.
self.dir_on_device = NATIVE_LIBS_DIR_ON_DEVICE
self.build_id_list_file = 'build_id_list'
def _get_need_archs(self):
"""Return the archs of binaries needed on device."""
if self.device_arch == 'arm64':
return ['arm', 'arm64']
if self.device_arch == 'arm':
return ['arm']
if self.device_arch == 'x86_64':
return ['x86', 'x86_64']
if self.device_arch == 'x86':
return ['x86']
return []
def collect_native_libs_on_host(self, native_lib_dir):
self.host_build_id_map.clear()
for root, _, files in os.walk(native_lib_dir):
for name in files:
if not name.endswith('.so'):
continue
self.add_native_lib_on_host(os.path.join(root, name), name)
def add_native_lib_on_host(self, path, name):
build_id = self.readelf.get_build_id(path)
if not build_id:
return
arch = self.readelf.get_arch(path)
if arch not in self.need_archs:
return
sections = self.readelf.get_sections(path)
score = 0
if '.debug_info' in sections:
score = 3
elif '.gnu_debugdata' in sections:
score = 2
elif '.symtab' in sections:
score = 1
entry = self.host_build_id_map.get(build_id)
if entry:
if entry.score < score:
entry.path = path
entry.score = score
else:
repeat_count = self.name_count_map.get(name, 0)
self.name_count_map[name] = repeat_count + 1
unique_name = name if repeat_count == 0 else name + '_' + str(repeat_count)
self.host_build_id_map[build_id] = HostElfEntry(path, unique_name, score)
def collect_native_libs_on_device(self):
self.device_build_id_map.clear()
self.adb.check_run(['shell', 'mkdir', '-p', self.dir_on_device])
if os.path.exists(self.build_id_list_file):
os.remove(self.build_id_list_file)
self.adb.run(['pull', self.dir_on_device + self.build_id_list_file])
if os.path.exists(self.build_id_list_file):
with open(self.build_id_list_file, 'rb') as fh:
for line in fh.readlines():
line = line.strip()
items = line.split('=')
if len(items) == 2:
self.device_build_id_map[items[0]] = items[1]
remove(self.build_id_list_file)
def sync_natives_libs_on_device(self):
# Push missing native libs on device.
for build_id in self.host_build_id_map:
if build_id not in self.device_build_id_map:
entry = self.host_build_id_map[build_id]
self.adb.check_run(['push', entry.path, self.dir_on_device + entry.name])
# Remove native libs not exist on host.
for build_id in self.device_build_id_map:
if build_id not in self.host_build_id_map:
name = self.device_build_id_map[build_id]
self.adb.run(['shell', 'rm', self.dir_on_device + name])
# Push new build_id_list on device.
with open(self.build_id_list_file, 'wb') as fh:
for build_id in self.host_build_id_map:
fh.write('%s=%s\n' % (build_id, self.host_build_id_map[build_id].name))
self.adb.check_run(['push', self.build_id_list_file,
self.dir_on_device + self.build_id_list_file])
os.remove(self.build_id_list_file)