def testEncodeDictOfLists_Join_Empty(self): test_dict1 = {} test_dict2 = {} expected = {} encoded1 = parallel.EncodeDictOfLists(test_dict1) encoded2 = parallel.EncodeDictOfLists(test_dict2) encoded = parallel.JoinEncodedDictOfLists([encoded1, encoded2]) decoded = parallel.DecodeDictOfLists(encoded) self.assertEqual(expected, decoded)
def RunNmOnIntermediates(target, tool_prefix, output_directory): """Returns encoded_symbol_names_by_path, encoded_string_addresses_by_path. Args: target: Either a single path to a .a (as a string), or a list of .o paths. """ is_archive = isinstance(target, str) args = [path_util.GetNmPath(tool_prefix), '--no-sort', '--defined-only'] if is_archive: args.append(target) else: args.extend(target) # pylint: disable=unexpected-keyword-arg proc = subprocess.Popen(args, cwd=output_directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8') # llvm-nm can print 'no symbols' to stderr. Capture and count the number of # lines, to be returned to the caller. stdout, stderr = proc.communicate() assert proc.returncode == 0, 'NM failed: ' + ' '.join(args) num_no_symbols = len(stderr.splitlines()) lines = stdout.splitlines() # Empty .a file has no output. if not lines: return parallel.EMPTY_ENCODED_DICT, parallel.EMPTY_ENCODED_DICT is_multi_file = not lines[0] lines = iter(lines) if is_multi_file: next(lines) path = next(lines)[:-1] # Path ends with a colon. else: assert not is_archive path = target[0] symbol_names_by_path = {} string_addresses_by_path = {} while path: if is_archive: # E.g. foo/bar.a(baz.o) path = '%s(%s)' % (target, path) mangled_symbol_names, string_addresses = _ParseOneObjectFileNmOutput( lines) symbol_names_by_path[path] = mangled_symbol_names if string_addresses: string_addresses_by_path[path] = string_addresses path = next(lines, ':')[:-1] # The multiprocess API uses pickle, which is ridiculously slow. More than 2x # faster to use join & split. # TODO(agrieve): We could use path indices as keys rather than paths to cut # down on marshalling overhead. return (parallel.EncodeDictOfLists(symbol_names_by_path), parallel.EncodeDictOfLists(string_addresses_by_path), num_no_symbols)
def testEncodeDictOfLists_JoinMultiple(self): test_dict1 = {'key1': ['a']} test_dict2 = {'key2': ['b']} expected = {'key1': ['a'], 'key2': ['b']} encoded1 = parallel.EncodeDictOfLists(test_dict1) encoded2 = parallel.EncodeDictOfLists({}) encoded3 = parallel.EncodeDictOfLists(test_dict2) encoded = parallel.JoinEncodedDictOfLists([encoded1, encoded2, encoded3]) decoded = parallel.DecodeDictOfLists(encoded) self.assertEquals(expected, decoded)
def _HandleMessage(self, message): if message[0] == _MSG_ANALYZE_PATHS: assert self._allow_analyze_paths, ( 'Cannot call AnalyzePaths() after AnalyzeStringLiterals()s.') # Invert '\x01'.join(paths), favoring paths = [] over paths = [''] since # the latter is less likely to happen. paths = message[1].split('\x01') if message[1] else [] self._job_queue.put( lambda: self._worker_analyzer.AnalyzePaths(paths)) elif message[0] == _MSG_SORT_PATHS: assert self._allow_analyze_paths, ( 'Cannot call SortPaths() after AnalyzeStringLiterals()s.') self._job_queue.put(self._worker_analyzer.SortPaths) elif message[0] == _MSG_ANALYZE_STRINGS: self._WaitForAnalyzePathJobs() elf_path, string_positions = message[1:] self._job_queue.put( lambda: self._worker_analyzer.AnalyzeStringLiterals( elf_path, string_positions)) elif message[0] == _MSG_GET_SYMBOL_NAMES: self._WaitForAnalyzePathJobs() self._pipe.send(None) paths_by_name = self._worker_analyzer.GetSymbolNames() self._pipe.send(parallel.EncodeDictOfLists(paths_by_name)) elif message[0] == _MSG_GET_STRINGS: self._job_queue.join() # Send a None packet so that other side can measure IPC transfer time. self._pipe.send(None) self._pipe.send(self._worker_analyzer.GetEncodedStringPositions())
def ResolveStringPiecesIndirect(encoded_string_addresses_by_path, string_data, tool_prefix, output_directory): string_addresses_by_path = parallel.DecodeDictOfLists( encoded_string_addresses_by_path) # Assign |target| as archive path, or a list of object paths. any_path = next(iter(string_addresses_by_path.keys())) target = _ExtractArchivePath(any_path) if not target: target = list(string_addresses_by_path.keys()) # Run readelf to find location of .rodata within the .o files. section_positions_by_path = _LookupStringSectionPositions( target, tool_prefix, output_directory) # Load the .rodata sections (from object files) as strings. string_sections_by_path = _ReadStringSections(target, output_directory, section_positions_by_path) def GeneratePathAndValues(): for path, object_addresses in string_addresses_by_path.items(): for value in _IterStringLiterals( path, object_addresses, string_sections_by_path.get(path)): yield path, value ret = _AnnotateStringData(string_data, GeneratePathAndValues()) return [parallel.EncodeDictOfLists(x) for x in ret]
def ResolveStringPieces(encoded_strings_by_path, string_data): # ast.literal_eval() undoes repr() applied to strings. strings_by_path = parallel.DecodeDictOfLists( encoded_strings_by_path, value_transform=ast.literal_eval) def GeneratePathAndValues(): for path, strings in strings_by_path.items(): for value in strings: yield path, value ret = _AnnotateStringData(string_data, GeneratePathAndValues()) return [parallel.EncodeDictOfLists(x) for x in ret]
def RunBcAnalyzerOnIntermediates(target, tool_prefix, output_directory): """Calls bcanalyzer and returns encoded map from path to strings. Args: target: A list of BC file paths. """ assert isinstance(target, list) runner = _BcAnalyzerRunner(tool_prefix, output_directory) strings_by_path = {} for t in target: strings_by_path[t] = [s for _, s in _ParseBcAnalyzer(runner.RunOnFile(t))] # Escape strings by repr() so there will be no special characters to interfere # parallel.EncodeDictOfLists() and decoding. return parallel.EncodeDictOfLists(strings_by_path, value_transform=repr)
def _CollectAliasesByAddressAsyncHelper(elf_path, tool_prefix): result = CollectAliasesByAddress(elf_path, tool_prefix) return parallel.EncodeDictOfLists(result, key_transform=str)
def testEncodeDictOfLists_Join_Singl(self): test_dict1 = {'key1': ['a']} encoded1 = parallel.EncodeDictOfLists(test_dict1) encoded = parallel.JoinEncodedDictOfLists([encoded1]) decoded = parallel.DecodeDictOfLists(encoded) self.assertEqual(test_dict1, decoded)
def testEncodeDictOfLists_ValueTransform(self): test_dict = {'a': ['0', '1', '2'], 'b': ['3', '4']} expected = {'a': [0, 1, 2], 'b': [3, 4]} encoded = parallel.EncodeDictOfLists(test_dict) decoded = parallel.DecodeDictOfLists(encoded, value_transform=int) self.assertEqual(expected, decoded)
def testEncodeDictOfLists_KeyTransform(self): test_dict = {0: ['a', 'b', 'c'], 9: ['a', 'b']} encoded = parallel.EncodeDictOfLists(test_dict, key_transform=str) decoded = parallel.DecodeDictOfLists(encoded, key_transform=int) self.assertEqual(test_dict, decoded)
def testEncodeDictOfLists_AllStrings(self): test_dict = {'foo': ['a', 'b', 'c'], 'foo2': ['a', 'b']} encoded = parallel.EncodeDictOfLists(test_dict) decoded = parallel.DecodeDictOfLists(encoded) self.assertEqual(test_dict, decoded)
def testEncodeDictOfLists_EmptyValue(self): test_dict = {'foo': []} encoded = parallel.EncodeDictOfLists(test_dict) decoded = parallel.DecodeDictOfLists(encoded) self.assertEqual(test_dict, decoded)
def testEncodeDictOfLists_Empty(self): test_dict = {} encoded = parallel.EncodeDictOfLists(test_dict) decoded = parallel.DecodeDictOfLists(encoded) self.assertEquals(test_dict, decoded)