def _DoArchiveTest(self, use_output_directory=True, use_elf=True,
use_pak=False, debug_measures=False):
with tempfile.NamedTemporaryFile(suffix='.size') as temp_file:
self._DoArchive(
temp_file.name, use_output_directory=use_output_directory,
use_elf=use_elf, use_pak=use_pak, debug_measures=debug_measures)
size_info = archive.LoadAndPostProcessSizeInfo(temp_file.name)
# Check that saving & loading is the same as directly parsing.
expected_size_info = self._CloneSizeInfo(
use_output_directory=use_output_directory, use_elf=use_elf,
use_pak=use_pak)
self.assertEquals(expected_size_info.metadata, size_info.metadata)
# Don't cluster.
expected_size_info.symbols = expected_size_info.raw_symbols
size_info.symbols = size_info.raw_symbols
expected = list(describe.GenerateLines(expected_size_info, verbose=True))
actual = list(describe.GenerateLines(size_info, verbose=True))
self.assertEquals(expected, actual)
sym_strs = (repr(sym) for sym in size_info.symbols)
stats = describe.DescribeSizeInfoCoverage(size_info)
if size_info.metadata:
metadata = describe.DescribeMetadata(size_info.metadata)
else:
metadata = []
return itertools.chain(metadata, stats, sym_strs)
def _DoArchiveTest(self,
use_output_directory=True,
use_elf=True,
debug_measures=False):
with tempfile.NamedTemporaryFile(suffix='.size') as temp_file:
args = [temp_file.name, '--map-file', _TEST_MAP_PATH]
if use_output_directory:
# Let autodetection find output_directory when --elf-file is used.
if not use_elf:
args += ['--output-directory', _TEST_OUTPUT_DIR]
else:
args += ['--no-source-paths']
if use_elf:
args += ['--elf-file', _TEST_ELF_PATH]
_RunApp('archive', args, debug_measures=debug_measures)
size_info = archive.LoadAndPostProcessSizeInfo(temp_file.name)
# Check that saving & loading is the same as directly parsing the .map.
expected_size_info = self._CloneSizeInfo(
use_output_directory=use_output_directory, use_elf=use_elf)
self.assertEquals(expected_size_info.metadata, size_info.metadata)
# Don't cluster.
expected_size_info.symbols = expected_size_info.raw_symbols
size_info.symbols = size_info.raw_symbols
expected = list(describe.GenerateLines(expected_size_info))
actual = list(describe.GenerateLines(size_info))
self.assertEquals(expected, actual)
sym_strs = (repr(sym) for sym in size_info.symbols)
stats = describe.DescribeSizeInfoCoverage(size_info)
if size_info.metadata:
metadata = describe.DescribeMetadata(size_info.metadata)
else:
metadata = []
return itertools.chain(metadata, stats, sym_strs)
def test_FullDescription(self):
size_info = self._CloneSizeInfo(use_elf=True)
# Show both clustered and non-clustered so that they can be compared.
size_info.symbols = size_info.raw_symbols
return itertools.chain(
describe.GenerateLines(size_info, verbose=True),
describe.GenerateLines(size_info.symbols._Clustered(), recursive=True,
verbose=True),
)
def _DoArchiveTest(self,
use_output_directory=True,
use_elf=False,
use_apk=False,
use_minimal_apks=False,
use_pak=False,
use_aux_elf=False,
ignore_linker_map=False,
debug_measures=False,
include_padding=False):
with tempfile.NamedTemporaryFile(suffix='.size') as temp_file:
self._DoArchive(temp_file.name,
use_output_directory=use_output_directory,
use_elf=use_elf,
use_apk=use_apk,
use_minimal_apks=use_minimal_apks,
use_pak=use_pak,
use_aux_elf=use_aux_elf,
ignore_linker_map=ignore_linker_map,
debug_measures=debug_measures,
include_padding=include_padding)
size_info = archive.LoadAndPostProcessSizeInfo(temp_file.name)
# Check that saving & loading is the same as directly parsing.
expected_size_info = self._CloneSizeInfo(
use_output_directory=use_output_directory,
use_elf=use_elf,
use_apk=use_apk,
use_minimal_apks=use_minimal_apks,
use_pak=use_pak,
use_aux_elf=use_aux_elf,
ignore_linker_map=ignore_linker_map)
self.assertEqual(_AllMetadata(expected_size_info),
_AllMetadata(size_info))
# Don't cluster.
expected_size_info.symbols = expected_size_info.raw_symbols
size_info.symbols = size_info.raw_symbols
expected = list(
describe.GenerateLines(expected_size_info, verbose=True))
actual = list(describe.GenerateLines(size_info, verbose=True))
self.assertEqual(expected, actual)
sym_strs = (repr(sym) for sym in size_info.symbols)
stats = data_quality.DescribeSizeInfoCoverage(size_info)
if len(size_info.containers) == 1:
# If there's only one container, merge the its metadata into build_config.
merged_data_desc = describe.DescribeDict(size_info.metadata_legacy)
else:
merged_data_desc = describe.DescribeDict(size_info.build_config)
for m in _AllMetadata(size_info):
merged_data_desc.extend(describe.DescribeDict(m))
return itertools.chain(merged_data_desc, stats, sym_strs)
def test_Diff_Basic(self):
size_info1 = self._CloneSizeInfo(use_elf=False)
size_info2 = self._CloneSizeInfo(use_elf=False)
size_info1.metadata = {"foo": 1, "bar": [1, 2, 3], "baz": "yes"}
size_info2.metadata = {"foo": 1, "bar": [1, 3], "baz": "yes"}
size_info1.raw_symbols -= size_info1.raw_symbols[:2]
size_info2.raw_symbols -= size_info2.raw_symbols[-3:]
changed_sym = size_info1.raw_symbols.WhereNameMatches(
'Patcher::Name_')[0]
changed_sym.size -= 10
padding_sym = size_info2.raw_symbols.WhereNameMatches(
'symbol gap 0')[0]
padding_sym.padding += 20
padding_sym.size += 20
d = diff.Diff(size_info1, size_info2)
d.raw_symbols = d.raw_symbols.Sorted()
self.assertEquals(d.raw_symbols.CountsByDiffStatus()[1:], [2, 2, 3])
changed_sym = d.raw_symbols.WhereNameMatches('Patcher::Name_')[0]
padding_sym = d.raw_symbols.WhereNameMatches('symbol gap 0')[0]
# Padding-only deltas should sort after all non-padding changes.
padding_idx = d.raw_symbols.index(padding_sym)
self.assertLess(d.raw_symbols.index(changed_sym), padding_idx)
# And before bss.
self.assertTrue(d.raw_symbols[padding_idx + 1].IsBss())
return describe.GenerateLines(d, verbose=True)
def _PrintFunc(self,
obj=None,
verbose=False,
recursive=False,
use_pager=None,
to_file=None):
"""Prints out the given Symbol / SymbolGroup / SymbolDiff / SizeInfo.
For convenience, |obj| will be appended to the global "printed" list.
Args:
obj: The object to be printed. Defaults to size_infos[-1]. Also accepts an
index into the |printed| array for showing previous results.
verbose: Show more detailed output.
recursive: Print children of nested SymbolGroups.
use_pager: Pipe output through `less`. Ignored when |obj| is a Symbol.
default is to automatically pipe when output is long.
to_file: Rather than print to stdio, write to the given file.
"""
if isinstance(obj, int):
obj = self._printed_variables[obj]
elif not self._printed_variables or self._printed_variables[-1] != obj:
if not isinstance(obj, models.SymbolGroup) or len(obj) > 0:
self._printed_variables.append(obj)
obj = obj if obj is not None else self._size_infos[-1]
lines = describe.GenerateLines(obj,
verbose=verbose,
recursive=recursive)
_WriteToStream(lines, use_pager=use_pager, to_file=to_file)
def _PrintFunc(self,
obj=None,
verbose=False,
summarize=True,
recursive=False,
use_pager=None,
to_file=None):
"""Prints out the given Symbol / SymbolGroup / SizeInfo.
For convenience, |obj| will be appended to the global "printed" list.
Args:
obj: The object to be printed.
verbose: Show more detailed output.
summarize: If False, show symbols only (no headers / summaries).
recursive: Print children of nested SymbolGroups.
use_pager: Pipe output through `less`. Ignored when |obj| is a Symbol.
default is to automatically pipe when output is long.
to_file: Rather than print to stdio, write to the given file.
"""
if obj is not None:
self._printed_variables.append(obj)
lines = describe.GenerateLines(obj,
verbose=verbose,
recursive=recursive,
summarize=summarize,
format_name='text')
_WriteToStream(lines, use_pager=use_pager, to_file=to_file)
def _CreateTestingSymbolsDeltas(symbols):
testing_symbols = symbols.WhereIsDex().WhereNameMatches(
'ForTest').WhereDiffStatusIs(models.DIFF_STATUS_ADDED)
lines = None
if len(testing_symbols):
lines = list(describe.GenerateLines(testing_symbols, summarize=False))
return lines, _SizeDelta('Added symbols named "ForTest"', 'symbols', 0,
len(testing_symbols))
def test_ActualDiff(self):
map1 = self._GetParsedMap()
map2 = self._GetParsedMap()
map1.symbols.symbols.pop(-1)
map2.symbols.symbols.pop(0)
map1.symbols[1].size -= 10
diff = models.Diff(map1, map2)
return describe.GenerateLines(diff)
def _CreateSupersizeDiff(apk_name, before_dir, after_dir):
before_size_path = os.path.join(before_dir, apk_name + '.size')
after_size_path = os.path.join(after_dir, apk_name + '.size')
before = archive.LoadAndPostProcessSizeInfo(before_size_path)
after = archive.LoadAndPostProcessSizeInfo(after_size_path)
size_info_delta = diff.Diff(before, after, sort=True)
lines = list(describe.GenerateLines(size_info_delta))
return lines, size_info_delta
def test_SaveDeltaSizeInfo(self):
# Check that saving & loading is the same as directly parsing.
orig_info1 = self._CloneSizeInfo(use_apk=True, use_aux_elf=True)
orig_info2 = self._CloneSizeInfo(use_elf=True)
orig_delta = diff.Diff(orig_info1, orig_info2)
with tempfile.NamedTemporaryFile(suffix='.sizediff') as sizediff_file:
file_format.SaveDeltaSizeInfo(orig_delta, sizediff_file.name)
new_info1, new_info2 = archive.LoadAndPostProcessDeltaSizeInfo(
sizediff_file.name)
new_delta = diff.Diff(new_info1, new_info2)
# File format discards unchanged symbols.
orig_delta.raw_symbols = orig_delta.raw_symbols.WhereDiffStatusIs(
models.DIFF_STATUS_UNCHANGED).Inverted()
self.assertEqual(
'\n'.join(describe.GenerateLines(orig_delta, verbose=True)),
'\n'.join(describe.GenerateLines(new_delta, verbose=True)))
def test_ActualDiff(self):
size_info1 = self._CloneSizeInfo()
size_info2 = self._CloneSizeInfo()
size_info1.metadata = {"foo": 1, "bar": [1, 2, 3], "baz": "yes"}
size_info2.metadata = {"foo": 1, "bar": [1, 3], "baz": "yes"}
size_info1.symbols -= size_info1.symbols[:2]
size_info2.symbols -= size_info2.symbols[-3:]
size_info1.symbols[1].size -= 10
diff = models.Diff(size_info1, size_info2)
return describe.GenerateLines(diff, verbose=True)
def test_Diff_Basic(self):
size_info1 = self._CloneSizeInfo(use_elf=False)
size_info2 = self._CloneSizeInfo(use_elf=False)
size_info1.metadata = {"foo": 1, "bar": [1,2,3], "baz": "yes"}
size_info2.metadata = {"foo": 1, "bar": [1,3], "baz": "yes"}
size_info1.symbols -= size_info1.symbols[:2]
size_info2.symbols -= size_info2.symbols[-3:]
size_info1.symbols[1].size -= 10
d = diff.Diff(size_info1, size_info2)
return describe.GenerateLines(d, verbose=True)
def test_SymbolGroupMethods(self):
all_syms = self._CloneSizeInfo(use_elf=True).symbols
global_syms = all_syms.WhereNameMatches('GLOBAL')
# Tests Filter(), Inverted(), and __sub__().
non_global_syms = global_syms.Inverted()
self.assertEqual(non_global_syms, (all_syms - global_syms))
# Tests Sorted() and __add__().
self.assertEqual(all_syms.Sorted(),
(global_syms + non_global_syms).Sorted())
# Tests GroupedByName() and __len__().
return itertools.chain(
['GroupedByName()'],
describe.GenerateLines(all_syms.GroupedByName()),
['GroupedByName(depth=1)'],
describe.GenerateLines(all_syms.GroupedByName(depth=1)),
['GroupedByName(depth=-1)'],
describe.GenerateLines(all_syms.GroupedByName(depth=-1)),
['GroupedByName(depth=1, min_count=2)'],
describe.GenerateLines(all_syms.GroupedByName(depth=1, min_count=2)),
)
def _WriteFunc(self, obj, path, verbose=False):
"""Same as Print(), but writes to a file.
Example: Write(Diff(size_info2, size_info1), 'output.txt')
"""
parent_dir = os.path.dirname(path)
if parent_dir and not os.path.exists(parent_dir):
os.makedirs(parent_dir)
with file_format.OpenMaybeGz(path, 'w') as file_obj:
lines = describe.GenerateLines(obj, verbose=verbose)
describe.WriteLines(lines, file_obj.write)
def _CreateAndWriteSupersizeDiff(apk_name, before_dir, after_dir, output_path):
before_size_path = os.path.join(before_dir, apk_name + '.size')
after_size_path = os.path.join(after_dir, apk_name + '.size')
before = archive.LoadAndPostProcessSizeInfo(before_size_path)
after = archive.LoadAndPostProcessSizeInfo(after_size_path)
size_info_delta = diff.Diff(before, after, sort=True)
with open(output_path, 'w') as f:
f.writelines(l + '\n' for l in describe.GenerateLines(size_info_delta))
return size_info_delta
def _PrintFunc(self, obj, verbose=False, use_pager=None, to_file=None):
"""Prints out the given Symbol / SymbolGroup / SymbolDiff / SizeInfo.
Args:
obj: The object to be printed.
verbose: Show more detailed output.
use_pager: Pipe output through `less`. Ignored when |obj| is a Symbol.
default is to automatically pipe when output is long.
to_file: Rather than print to stdio, write to the given file.
"""
lines = describe.GenerateLines(obj, verbose=verbose)
_WriteToStream(lines, use_pager=use_pager, to_file=to_file)
def _SymbolDiffHelper(symbols):
added = symbols.WhereDiffStatusIs(models.DIFF_STATUS_ADDED)
removed = symbols.WhereDiffStatusIs(models.DIFF_STATUS_REMOVED)
both = (added + removed).SortedByName()
lines = None
if len(both) > 0:
lines = [
'Added: {}'.format(len(added)),
'Removed: {}'.format(len(removed)),
]
lines.extend(describe.GenerateLines(both, summarize=False))
return lines, len(added) - len(removed)
def test_SymbolGroupMethods(self):
all_syms = self._GetParsedMap().symbols
global_syms = all_syms.WhereNameMatches('GLOBAL')
# Tests Filter(), Inverted(), and __sub__().
non_global_syms = global_syms.Inverted()
self.assertEqual(non_global_syms.symbols,
(all_syms - global_syms).symbols)
# Tests Sorted() and __add__().
self.assertEqual(all_syms.Sorted().symbols,
(global_syms + non_global_syms).Sorted().symbols)
# Tests GroupByNamespace() and __len__().
return itertools.chain(
['GroupByNamespace()'],
describe.GenerateLines(all_syms.GroupByNamespace()),
['GroupByNamespace(depth=1)'],
describe.GenerateLines(all_syms.GroupByNamespace(depth=1)),
['GroupByNamespace(depth=1, fallback=None)'],
describe.GenerateLines(
all_syms.GroupByNamespace(depth=1, fallback=None)),
['GroupByNamespace(depth=1, min_count=2)'],
describe.GenerateLines(
all_syms.GroupByNamespace(depth=1, min_count=2)),
)
def test_Diff_Basic(self):
size_info1 = self._CloneSizeInfo(use_pak=True)
size_info2 = self._CloneSizeInfo(use_pak=True)
size_info2.build_config['git_revision'] = 'xyz789'
container1 = size_info1.containers[0]
container2 = size_info2.containers[0]
container1.metadata = {"foo": 1, "bar": [1, 2, 3], "baz": "yes"}
container2.metadata = {"foo": 1, "bar": [1, 3], "baz": "yes"}
size_info1.raw_symbols -= size_info1.raw_symbols.WhereNameMatches(
r'pLinuxKernelCmpxchg|pLinuxKernelMemoryBarrier')
size_info2.raw_symbols -= size_info2.raw_symbols.WhereNameMatches(
r'IDS_AW_WEBPAGE_PARENTAL_|IDS_WEB_FONT_FAMILY|IDS_WEB_FONT_SIZE')
changed_sym = size_info1.raw_symbols.WhereNameMatches(
'Patcher::Name_')[0]
changed_sym.size -= 10
padding_sym = size_info2.raw_symbols.WhereNameMatches(
'symbol gap 0')[0]
padding_sym.padding += 20
padding_sym.size += 20
# Test pak symbols changing .grd files. They should not show as changed.
pak_sym = size_info2.raw_symbols.WhereNameMatches(
r'IDR_PDF_COMPOSITOR_MANIFEST')[0]
pak_sym.full_name = pak_sym.full_name.replace('.grd', '2.grd')
# Serialize & de-serialize so that name normalization runs again for the pak
# symbol.
bytesio = io.BytesIO()
file_format.SaveSizeInfo(size_info2, 'path', file_obj=bytesio)
bytesio.seek(0)
size_info2 = archive.LoadAndPostProcessSizeInfo('path',
file_obj=bytesio)
d = diff.Diff(size_info1, size_info2)
d.raw_symbols = d.raw_symbols.Sorted()
self.assertEqual((1, 2, 3), d.raw_symbols.CountsByDiffStatus()[1:])
changed_sym = d.raw_symbols.WhereNameMatches('Patcher::Name_')[0]
padding_sym = d.raw_symbols.WhereNameMatches('symbol gap 0')[0]
bss_sym = d.raw_symbols.WhereInSection(models.SECTION_BSS)[0]
# Padding-only deltas should sort after all non-padding changes.
padding_idx = d.raw_symbols.index(padding_sym)
changed_idx = d.raw_symbols.index(changed_sym)
bss_idx = d.raw_symbols.index(bss_sym)
self.assertLess(changed_idx, padding_idx)
# And before bss.
self.assertLess(padding_idx, bss_idx)
return describe.GenerateLines(d, verbose=True)
def _CsvFunc(self, obj=None, verbose=False, use_pager=None, to_file=None):
"""Prints out the given Symbol / SymbolGroup / SizeInfo in CSV format.
For convenience, |obj| will be appended to the global "printed" list.
Args:
obj: The object to be printed as CSV.
use_pager: Pipe output through `less`. Ignored when |obj| is a Symbol.
default is to automatically pipe when output is long.
to_file: Rather than print to stdio, write to the given file.
"""
if obj is not None:
self._printed_variables.append(obj)
lines = describe.GenerateLines(obj, verbose=verbose, recursive=False,
format_name='csv')
_WriteToStream(lines, use_pager=use_pager, to_file=to_file)
def test_Diff_Basic(self):
size_info1 = self._CloneSizeInfo(use_pak=True)
size_info2 = self._CloneSizeInfo(use_pak=True)
size_info2.build_config['git_revision'] = 'xyz789'
container1 = size_info1.containers[0]
container2 = size_info2.containers[0]
container1.metadata = {"foo": 1, "bar": [1, 2, 3], "baz": "yes"}
container2.metadata = {"foo": 1, "bar": [1, 3], "baz": "yes"}
size_info1.raw_symbols -= size_info1.raw_symbols[:2]
size_info2.raw_symbols -= size_info2.raw_symbols[-3:]
changed_sym = size_info1.raw_symbols.WhereNameMatches(
'Patcher::Name_')[0]
changed_sym.size -= 10
padding_sym = size_info2.raw_symbols.WhereNameMatches(
'symbol gap 0')[0]
padding_sym.padding += 20
padding_sym.size += 20
pak_sym = size_info2.raw_symbols.WhereInSection(
models.SECTION_PAK_NONTRANSLATED)[0]
pak_sym.full_name = 'foo: ' + pak_sym.full_name.split()[-1]
# Serialize & de-serialize so that name normalization runs again for the pak
# symbol.
bytesio = io.BytesIO()
file_format.SaveSizeInfo(size_info2, 'path', file_obj=bytesio)
bytesio.seek(0)
size_info2 = archive.LoadAndPostProcessSizeInfo('path',
file_obj=bytesio)
d = diff.Diff(size_info1, size_info2)
d.raw_symbols = d.raw_symbols.Sorted()
self.assertEqual(d.raw_symbols.CountsByDiffStatus()[1:], (2, 2, 3))
changed_sym = d.raw_symbols.WhereNameMatches('Patcher::Name_')[0]
padding_sym = d.raw_symbols.WhereNameMatches('symbol gap 0')[0]
bss_sym = d.raw_symbols.WhereInSection(models.SECTION_BSS)[0]
# Padding-only deltas should sort after all non-padding changes.
padding_idx = d.raw_symbols.index(padding_sym)
changed_idx = d.raw_symbols.index(changed_sym)
bss_idx = d.raw_symbols.index(bss_sym)
self.assertLess(changed_idx, padding_idx)
# And before bss.
self.assertLess(padding_idx, bss_idx)
return describe.GenerateLines(d, verbose=True)
def _SymbolDiffHelper(title_fragment, symbols):
added = symbols.WhereDiffStatusIs(models.DIFF_STATUS_ADDED)
removed = symbols.WhereDiffStatusIs(models.DIFF_STATUS_REMOVED)
both = (added + removed).SortedByName()
lines = []
if len(both) > 0:
for group in both.GroupedByContainer():
counts = group.CountsByDiffStatus()
lines += [
'===== {} Added & Removed ({}) ====='.format(
title_fragment, group.full_name),
'Added: {}'.format(counts[models.DIFF_STATUS_ADDED]),
'Removed: {}'.format(counts[models.DIFF_STATUS_REMOVED]), ''
]
lines.extend(describe.GenerateLines(group, summarize=False))
lines += ['']
return lines, len(added) - len(removed)
def _CsvFunc(self, obj=None, verbose=False, use_pager=None, to_file=None):
"""Prints out the given Symbol / SymbolGroup / SizeInfo in CSV format.
For convenience, |obj| will be appended to the global "printed" list.
Args:
obj: The object to be printed as CSV. Defaults to |size_infos[-1]|. Also
accepts an index into the |_printed_variables| array for showing
previous results.
use_pager: Pipe output through `less`. Ignored when |obj| is a Symbol.
default is to automatically pipe when output is long.
to_file: Rather than print to stdio, write to the given file.
"""
obj = self._GetObjToPrint(obj)
lines = describe.GenerateLines(obj,
verbose=verbose,
recursive=False,
format_name='csv')
_WriteToStream(lines, use_pager=use_pager, to_file=to_file)
def _PrintFunc(self, obj, verbose=False, use_pager=None):
"""Prints out the given Symbol / SymbolGroup / SymbolDiff / SizeInfo.
Args:
obj: The object to be printed.
use_pager: Whether to pipe output through `less`. Ignored when |obj| is a
Symbol.
"""
lines = describe.GenerateLines(obj, verbose=verbose)
if use_pager is None and sys.stdout.isatty():
# Does not take into account line-wrapping... Oh well.
first_lines = list(itertools.islice(lines, _THRESHOLD_FOR_PAGER))
if len(first_lines) == _THRESHOLD_FOR_PAGER:
use_pager = True
lines = itertools.chain(first_lines, lines)
if use_pager:
with _LessPipe() as stdin:
describe.WriteLines(lines, stdin.write)
else:
describe.WriteLines(lines, sys.stdout.write)
def test_FullDescription(self):
return describe.GenerateLines(self._CloneSizeInfo().Cluster(),
recursive=True, verbose=True)
