def _RunNm(self, paths_by_type): """Calls nm to get symbols and (for non-BC files) string addresses.""" # Downstream functions rely upon .a not being grouped. batches = self._MakeBatches(paths_by_type.arch, None) # Combine object files and Bitcode files for nm. BATCH_SIZE = 50 # Arbitrarily chosen. batches.extend( self._MakeBatches(paths_by_type.obj + paths_by_type.bc, BATCH_SIZE)) results = self._DoBulkFork(nm.RunNmOnIntermediates, batches) # Names are still mangled. all_paths_by_name = self._paths_by_name total_no_symbols = 0 for encoded_syms, encoded_strs, num_no_symbols in results: total_no_symbols += num_no_symbols symbol_names_by_path = parallel.DecodeDictOfLists(encoded_syms) for path, names in symbol_names_by_path.items(): for name in names: all_paths_by_name[name].append(path) if encoded_strs != parallel.EMPTY_ENCODED_DICT: self._encoded_string_addresses_by_path_chunks.append( encoded_strs) if total_no_symbols: logging.warn('nm found no symbols in %d objects.', total_no_symbols)
def ResolveStringPiecesIndirect(encoded_string_addresses_by_path, string_data, tool_prefix, output_directory): string_addresses_by_path = parallel.DecodeDictOfLists( encoded_string_addresses_by_path) # Assign |target| as archive path, or a list of object paths. any_path = next(iter(string_addresses_by_path.keys())) target = _ExtractArchivePath(any_path) if not target: target = list(string_addresses_by_path.keys()) # Run readelf to find location of .rodata within the .o files. section_positions_by_path = _LookupStringSectionPositions( target, tool_prefix, output_directory) # Load the .rodata sections (from object files) as strings. string_sections_by_path = _ReadStringSections(target, output_directory, section_positions_by_path) def GeneratePathAndValues(): for path, object_addresses in string_addresses_by_path.items(): for value in _IterStringLiterals( path, object_addresses, string_sections_by_path.get(path)): yield path, value ret = _AnnotateStringData(string_data, GeneratePathAndValues()) return [parallel.EncodeDictOfLists(x) for x in ret]
def testEncodeDictOfLists_Join_Empty(self): test_dict1 = {} test_dict2 = {} expected = {} encoded1 = parallel.EncodeDictOfLists(test_dict1) encoded2 = parallel.EncodeDictOfLists(test_dict2) encoded = parallel.JoinEncodedDictOfLists([encoded1, encoded2]) decoded = parallel.DecodeDictOfLists(encoded) self.assertEqual(expected, decoded)
def GetStringPositions(self): self._pipe.send((_MSG_GET_STRINGS, )) self._pipe.recv() # None logging.debug('Decoding string symbol results from forked process') result = self._pipe.recv() return [ parallel.DecodeDictOfLists(x, value_transform=_DecodePosition) for x in result ]
def testEncodeDictOfLists_JoinMultiple(self): test_dict1 = {'key1': ['a']} test_dict2 = {'key2': ['b']} expected = {'key1': ['a'], 'key2': ['b']} encoded1 = parallel.EncodeDictOfLists(test_dict1) encoded2 = parallel.EncodeDictOfLists({}) encoded3 = parallel.EncodeDictOfLists(test_dict2) encoded = parallel.JoinEncodedDictOfLists([encoded1, encoded2, encoded3]) decoded = parallel.DecodeDictOfLists(encoded) self.assertEquals(expected, decoded)
def ResolveStringPieces(encoded_strings_by_path, string_data): # ast.literal_eval() undoes repr() applied to strings. strings_by_path = parallel.DecodeDictOfLists( encoded_strings_by_path, value_transform=ast.literal_eval) def GeneratePathAndValues(): for path, strings in strings_by_path.items(): for value in strings: yield path, value ret = _AnnotateStringData(string_data, GeneratePathAndValues()) return [parallel.EncodeDictOfLists(x) for x in ret]
def testAnalyzer(self): # Save global param in bcanalyzer. saved_char_width_limit = bcanalyzer._CHAR_WIDTH_LIMIT for width_limit, include_4byte_strings in [(2, False), (4, True)]: # Tweak global param in bcanalyzer. bcanalyzer._CHAR_WIDTH_LIMIT = width_limit encoded_results = bcanalyzer.RunBcAnalyzerOnIntermediates( ['test.o'], _TEST_TOOL_PREFIX, _TEST_OUTPUT_DIR) results = parallel.DecodeDictOfLists( encoded_results, value_transform=ast.literal_eval) self.assertEquals(['test.o'], results.keys()) str_list = results['test.o'] # See mock_bcanalyzer.py for details on the C++ test file. expected = [] expected.append(_MakeString(8, ['Test1a', 0])) expected.append(_MakeString(8, ['Test1b', 0])) expected.append(_MakeString(8, ['Test2a', 0])) expected.append(_MakeString(8, ['Test2b', 0])) expected.append(_MakeString(16, ['Test3a', 0])) expected.append(_MakeString(16, ['Test3b', 0])) if include_4byte_strings: expected.append(_MakeString(32, ['Test4a', 0])) expected.append(_MakeString(32, ['Test4b', 0])) expected.append(_MakeString(8, [1, 0, 0, 1, 1, 0])) expected.append(_MakeString(8, [1, 0, 0, 1, 1, 1])) expected.append(_MakeString(8, ['Test5a', 0])) expected.append(_MakeString(8, ['Test5b', 1])) expected.append(_MakeString(16, ['Test6a', 0])) expected.append(_MakeString(16, ['Test6b', 1])) if include_4byte_strings: expected.append(_MakeString(32, ['Test7a', 0])) expected.append(_MakeString(32, ['Test7b', 1])) expected.append(_MakeString(8, ['Test8a', 0])) expected.append(_MakeString(8, ['Test8b', 0])) # Exclude |{u8a, u8b, u16a, u16b, u32a, u32b, u64a, u64b}|. # Exclude |{s8empty, s16empty, s32empty}|. expected.append(_MakeString(8, ['1a', 0])) # Exclude |zeros|, which should be in .bss section. self.assertEquals(expected, str_list) # Restore globa param in bcanalyzer. bcanalyzer._CHAR_WIDTH_LIMIT = saved_char_width_limit
def decode(encoded): return parallel.DecodeDictOfLists(encoded, key_transform=int)
def testEncodeDictOfLists_Join_Singl(self): test_dict1 = {'key1': ['a']} encoded1 = parallel.EncodeDictOfLists(test_dict1) encoded = parallel.JoinEncodedDictOfLists([encoded1]) decoded = parallel.DecodeDictOfLists(encoded) self.assertEqual(test_dict1, decoded)
def testEncodeDictOfLists_ValueTransform(self): test_dict = {'a': ['0', '1', '2'], 'b': ['3', '4']} expected = {'a': [0, 1, 2], 'b': [3, 4]} encoded = parallel.EncodeDictOfLists(test_dict) decoded = parallel.DecodeDictOfLists(encoded, value_transform=int) self.assertEqual(expected, decoded)
def testEncodeDictOfLists_KeyTransform(self): test_dict = {0: ['a', 'b', 'c'], 9: ['a', 'b']} encoded = parallel.EncodeDictOfLists(test_dict, key_transform=str) decoded = parallel.DecodeDictOfLists(encoded, key_transform=int) self.assertEqual(test_dict, decoded)
def testEncodeDictOfLists_AllStrings(self): test_dict = {'foo': ['a', 'b', 'c'], 'foo2': ['a', 'b']} encoded = parallel.EncodeDictOfLists(test_dict) decoded = parallel.DecodeDictOfLists(encoded) self.assertEqual(test_dict, decoded)
def testEncodeDictOfLists_EmptyValue(self): test_dict = {'foo': []} encoded = parallel.EncodeDictOfLists(test_dict) decoded = parallel.DecodeDictOfLists(encoded) self.assertEqual(test_dict, decoded)
def GetSymbolNames(self): self._pipe.send((_MSG_GET_SYMBOL_NAMES, )) self._pipe.recv() # None logging.debug('Decoding nm results from forked process') encoded_paths_by_name = self._pipe.recv() return parallel.DecodeDictOfLists(encoded_paths_by_name)
def GetStringPositions(self): return [ parallel.DecodeDictOfLists(x, value_transform=_DecodePosition) for x in self._list_of_encoded_elf_string_ranges_by_path ]
def testEncodeDictOfLists_Empty(self): test_dict = {} encoded = parallel.EncodeDictOfLists(test_dict) decoded = parallel.DecodeDictOfLists(encoded) self.assertEquals(test_dict, decoded)