def testEncodeDictOfLists_Join_Empty(self):
test_dict1 = {}
test_dict2 = {}
expected = {}
encoded1 = parallel.EncodeDictOfLists(test_dict1)
encoded2 = parallel.EncodeDictOfLists(test_dict2)
encoded = parallel.JoinEncodedDictOfLists([encoded1, encoded2])
decoded = parallel.DecodeDictOfLists(encoded)
self.assertEqual(expected, decoded)
def RunNmOnIntermediates(target, tool_prefix, output_directory):
"""Returns encoded_symbol_names_by_path, encoded_string_addresses_by_path.
Args:
target: Either a single path to a .a (as a string), or a list of .o paths.
"""
is_archive = isinstance(target, str)
args = [path_util.GetNmPath(tool_prefix), '--no-sort', '--defined-only']
if is_archive:
args.append(target)
else:
args.extend(target)
# pylint: disable=unexpected-keyword-arg
proc = subprocess.Popen(args,
cwd=output_directory,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding='utf-8')
# llvm-nm can print 'no symbols' to stderr. Capture and count the number of
# lines, to be returned to the caller.
stdout, stderr = proc.communicate()
assert proc.returncode == 0, 'NM failed: ' + ' '.join(args)
num_no_symbols = len(stderr.splitlines())
lines = stdout.splitlines()
# Empty .a file has no output.
if not lines:
return parallel.EMPTY_ENCODED_DICT, parallel.EMPTY_ENCODED_DICT
is_multi_file = not lines[0]
lines = iter(lines)
if is_multi_file:
next(lines)
path = next(lines)[:-1] # Path ends with a colon.
else:
assert not is_archive
path = target[0]
symbol_names_by_path = {}
string_addresses_by_path = {}
while path:
if is_archive:
# E.g. foo/bar.a(baz.o)
path = '%s(%s)' % (target, path)
mangled_symbol_names, string_addresses = _ParseOneObjectFileNmOutput(
lines)
symbol_names_by_path[path] = mangled_symbol_names
if string_addresses:
string_addresses_by_path[path] = string_addresses
path = next(lines, ':')[:-1]
# The multiprocess API uses pickle, which is ridiculously slow. More than 2x
# faster to use join & split.
# TODO(agrieve): We could use path indices as keys rather than paths to cut
# down on marshalling overhead.
return (parallel.EncodeDictOfLists(symbol_names_by_path),
parallel.EncodeDictOfLists(string_addresses_by_path),
num_no_symbols)
def testEncodeDictOfLists_JoinMultiple(self):
test_dict1 = {'key1': ['a']}
test_dict2 = {'key2': ['b']}
expected = {'key1': ['a'], 'key2': ['b']}
encoded1 = parallel.EncodeDictOfLists(test_dict1)
encoded2 = parallel.EncodeDictOfLists({})
encoded3 = parallel.EncodeDictOfLists(test_dict2)
encoded = parallel.JoinEncodedDictOfLists([encoded1, encoded2, encoded3])
decoded = parallel.DecodeDictOfLists(encoded)
self.assertEquals(expected, decoded)
def _HandleMessage(self, message):
if message[0] == _MSG_ANALYZE_PATHS:
assert self._allow_analyze_paths, (
'Cannot call AnalyzePaths() after AnalyzeStringLiterals()s.')
# Invert '\x01'.join(paths), favoring paths = [] over paths = [''] since
# the latter is less likely to happen.
paths = message[1].split('\x01') if message[1] else []
self._job_queue.put(
lambda: self._worker_analyzer.AnalyzePaths(paths))
elif message[0] == _MSG_SORT_PATHS:
assert self._allow_analyze_paths, (
'Cannot call SortPaths() after AnalyzeStringLiterals()s.')
self._job_queue.put(self._worker_analyzer.SortPaths)
elif message[0] == _MSG_ANALYZE_STRINGS:
self._WaitForAnalyzePathJobs()
elf_path, string_positions = message[1:]
self._job_queue.put(
lambda: self._worker_analyzer.AnalyzeStringLiterals(
elf_path, string_positions))
elif message[0] == _MSG_GET_SYMBOL_NAMES:
self._WaitForAnalyzePathJobs()
self._pipe.send(None)
paths_by_name = self._worker_analyzer.GetSymbolNames()
self._pipe.send(parallel.EncodeDictOfLists(paths_by_name))
elif message[0] == _MSG_GET_STRINGS:
self._job_queue.join()
# Send a None packet so that other side can measure IPC transfer time.
self._pipe.send(None)
self._pipe.send(self._worker_analyzer.GetEncodedStringPositions())
def ResolveStringPiecesIndirect(encoded_string_addresses_by_path, string_data,
tool_prefix, output_directory):
string_addresses_by_path = parallel.DecodeDictOfLists(
encoded_string_addresses_by_path)
# Assign |target| as archive path, or a list of object paths.
any_path = next(iter(string_addresses_by_path.keys()))
target = _ExtractArchivePath(any_path)
if not target:
target = list(string_addresses_by_path.keys())
# Run readelf to find location of .rodata within the .o files.
section_positions_by_path = _LookupStringSectionPositions(
target, tool_prefix, output_directory)
# Load the .rodata sections (from object files) as strings.
string_sections_by_path = _ReadStringSections(target, output_directory,
section_positions_by_path)
def GeneratePathAndValues():
for path, object_addresses in string_addresses_by_path.items():
for value in _IterStringLiterals(
path, object_addresses, string_sections_by_path.get(path)):
yield path, value
ret = _AnnotateStringData(string_data, GeneratePathAndValues())
return [parallel.EncodeDictOfLists(x) for x in ret]
def ResolveStringPieces(encoded_strings_by_path, string_data):
# ast.literal_eval() undoes repr() applied to strings.
strings_by_path = parallel.DecodeDictOfLists(
encoded_strings_by_path, value_transform=ast.literal_eval)
def GeneratePathAndValues():
for path, strings in strings_by_path.items():
for value in strings:
yield path, value
ret = _AnnotateStringData(string_data, GeneratePathAndValues())
return [parallel.EncodeDictOfLists(x) for x in ret]
def RunBcAnalyzerOnIntermediates(target, tool_prefix, output_directory):
"""Calls bcanalyzer and returns encoded map from path to strings.
Args:
target: A list of BC file paths.
"""
assert isinstance(target, list)
runner = _BcAnalyzerRunner(tool_prefix, output_directory)
strings_by_path = {}
for t in target:
strings_by_path[t] = [s for _, s in _ParseBcAnalyzer(runner.RunOnFile(t))]
# Escape strings by repr() so there will be no special characters to interfere
# parallel.EncodeDictOfLists() and decoding.
return parallel.EncodeDictOfLists(strings_by_path, value_transform=repr)
def _CollectAliasesByAddressAsyncHelper(elf_path, tool_prefix): result = CollectAliasesByAddress(elf_path, tool_prefix) return parallel.EncodeDictOfLists(result, key_transform=str)
def testEncodeDictOfLists_Join_Singl(self):
test_dict1 = {'key1': ['a']}
encoded1 = parallel.EncodeDictOfLists(test_dict1)
encoded = parallel.JoinEncodedDictOfLists([encoded1])
decoded = parallel.DecodeDictOfLists(encoded)
self.assertEqual(test_dict1, decoded)
def testEncodeDictOfLists_ValueTransform(self):
test_dict = {'a': ['0', '1', '2'], 'b': ['3', '4']}
expected = {'a': [0, 1, 2], 'b': [3, 4]}
encoded = parallel.EncodeDictOfLists(test_dict)
decoded = parallel.DecodeDictOfLists(encoded, value_transform=int)
self.assertEqual(expected, decoded)
def testEncodeDictOfLists_KeyTransform(self):
test_dict = {0: ['a', 'b', 'c'], 9: ['a', 'b']}
encoded = parallel.EncodeDictOfLists(test_dict, key_transform=str)
decoded = parallel.DecodeDictOfLists(encoded, key_transform=int)
self.assertEqual(test_dict, decoded)
def testEncodeDictOfLists_AllStrings(self):
test_dict = {'foo': ['a', 'b', 'c'], 'foo2': ['a', 'b']}
encoded = parallel.EncodeDictOfLists(test_dict)
decoded = parallel.DecodeDictOfLists(encoded)
self.assertEqual(test_dict, decoded)
def testEncodeDictOfLists_EmptyValue(self):
test_dict = {'foo': []}
encoded = parallel.EncodeDictOfLists(test_dict)
decoded = parallel.DecodeDictOfLists(encoded)
self.assertEqual(test_dict, decoded)
def testEncodeDictOfLists_Empty(self):
test_dict = {}
encoded = parallel.EncodeDictOfLists(test_dict)
decoded = parallel.DecodeDictOfLists(encoded)
self.assertEquals(test_dict, decoded)
