def _fill_snapshot(dump, bucket_set, sorters):
root = OrderedDict()
root['time'] = dump.time
root['worlds'] = OrderedDict()
root['worlds']['vm'] = CatCommand._fill_world(
dump, bucket_set, sorters, 'vm')
root['worlds']['malloc'] = CatCommand._fill_world(
dump, bucket_set, sorters, 'malloc')
return root
def do(self, sys_argv):
options, args = self._parse_args(sys_argv, 1)
dump_path = args[1]
# TODO(dmikurube): Support shared memory.
alternative_dirs_dict = {}
if options.alternative_dirs:
for alternative_dir_pair in options.alternative_dirs.split(':'):
target_path, host_path = alternative_dir_pair.split('@', 1)
alternative_dirs_dict[target_path] = host_path
(bucket_set, dumps) = SubCommand.load_basic_files(
dump_path, True, alternative_dirs=alternative_dirs_dict)
# Load all sorters.
sorters = SorterSet()
json_root = OrderedDict()
json_root['version'] = 1
json_root['run_id'] = None
for dump in dumps:
if json_root['run_id'] and json_root['run_id'] != dump.run_id:
LOGGER.error('Inconsistent heap profile dumps.')
json_root['run_id'] = ''
break
json_root['run_id'] = dump.run_id
json_root['roots'] = []
for sorter in sorters:
if sorter.root:
json_root['roots'].append([sorter.world, sorter.name])
json_root['default_template'] = 'l2'
json_root['templates'] = sorters.templates.as_dict()
orders = OrderedDict()
orders['worlds'] = OrderedDict()
for world in ['vm', 'malloc']:
orders['worlds'][world] = OrderedDict()
orders['worlds'][world]['breakdown'] = OrderedDict()
for sorter in sorters.iter_world(world):
order = []
for rule in sorter.iter_rule():
if rule.name not in order:
order.append(rule.name)
orders['worlds'][world]['breakdown'][sorter.name] = order
json_root['orders'] = orders
json_root['snapshots'] = []
for dump in dumps:
LOGGER.info('Sorting a dump %s...' % dump.path)
json_root['snapshots'].append(
self._fill_snapshot(dump, bucket_set, sorters))
if options.indent:
json.dump(json_root, sys.stdout, indent=2)
else:
json.dump(json_root, sys.stdout)
print ''
def _fill_world(dump, bucket_set, sorters, world):
root = OrderedDict()
root['name'] = world
if world == 'vm':
root['unit_fields'] = ['size', 'reserved']
elif world == 'malloc':
root['unit_fields'] = ['size', 'alloc_count', 'free_count']
# Make { vm | malloc } units with their sizes.
root['units'] = OrderedDict()
unit_set = UnitSet(world)
if world == 'vm':
for unit in CatCommand._iterate_vm_unit(dump, None, bucket_set):
unit_set.append(unit)
for unit in unit_set:
root['units'][unit.unit_id] = [unit.committed, unit.reserved]
elif world == 'malloc':
for unit in CatCommand._iterate_malloc_unit(dump, bucket_set):
unit_set.append(unit)
for unit in unit_set:
root['units'][unit.unit_id] = [
unit.size, unit.alloc_count, unit.free_count]
# Iterate for { vm | malloc } sorters.
root['breakdown'] = OrderedDict()
for sorter in sorters.iter_world(world):
LOGGER.info(' Sorting with %s:%s.' % (sorter.world, sorter.name))
breakdown = OrderedDict()
for rule in sorter.iter_rule():
category = OrderedDict()
category['name'] = rule.name
subs = []
for sub_world, sub_breakdown in rule.iter_subs():
subs.append([sub_world, sub_breakdown])
if subs:
category['subs'] = subs
if rule.hidden:
category['hidden'] = True
category['units'] = []
breakdown[rule.name] = category
for unit in unit_set:
found = sorter.find(unit)
if found:
# Note that a bucket which doesn't match any rule is just dropped.
breakdown[found.name]['units'].append(unit.unit_id)
root['breakdown'][sorter.name] = breakdown
return root
def _fill_world(dump, bucket_set, sorters, world):
root = OrderedDict()
root['name'] = world
if world == 'vm':
root['unit_fields'] = ['size', 'reserved']
elif world == 'malloc':
root['unit_fields'] = ['size', 'alloc_count', 'free_count']
# Make { vm | malloc } units with their sizes.
root['units'] = OrderedDict()
unit_set = UnitSet(world)
if world == 'vm':
for unit in CatCommand._iterate_vm_unit(dump, None, bucket_set):
unit_set.append(unit)
for unit in unit_set:
root['units'][unit.unit_id] = [unit.committed, unit.reserved]
elif world == 'malloc':
for unit in CatCommand._iterate_malloc_unit(dump, bucket_set):
unit_set.append(unit)
for unit in unit_set:
root['units'][unit.unit_id] = [
unit.size, unit.alloc_count, unit.free_count
]
# Iterate for { vm | malloc } sorters.
root['breakdown'] = OrderedDict()
for sorter in sorters.iter_world(world):
breakdown = OrderedDict()
for unit in unit_set:
found = sorter.find(unit)
if found.name not in breakdown:
category = OrderedDict()
category['name'] = found.name
category['color'] = 'random'
subs = []
for sub_world, sub_breakdown in found.iter_subs():
subs.append([sub_world, sub_breakdown])
if subs:
category['subs'] = subs
if found.hidden:
category['hidden'] = True
category['units'] = []
breakdown[found.name] = category
breakdown[found.name]['units'].append(unit.unit_id)
root['breakdown'][sorter.name] = breakdown
return root
def flatten_all_category_trees(category_trees):
flattened_labels = set()
flattened_table = []
for category_tree in category_trees:
flattened = OrderedDict()
for label, subtotal in flatten(category_tree):
flattened_labels.add(label)
flattened[label] = subtotal
flattened_table.append(flattened)
return flattened_labels, flattened_table
def accumulate(template, snapshot, units_dict, target_units):
"""Accumulates units in a JSON |snapshot| with applying a given |template|.
Args:
template: A template tree included in a dmprof cat JSON file.
snapshot: A snapshot in a dmprof cat JSON file.
units_dict: A dict of units in worlds.
target_units: A list of unit ids which are a target of this accumulation.
"""
world = template[0]
breakdown = template[1]
rules = template[2]
remainder_units = target_units.copy()
category_tree = OrderedDict()
total = 0
for rule, match in snapshot[world]['breakdown'][breakdown].iteritems():
if 'hidden' in match and match['hidden']:
continue
matched_units = set(match['units']).intersection(target_units)
subtotal = 0
for unit_id in matched_units:
subtotal += units_dict[world][unit_id]
total += subtotal
remainder_units = remainder_units.difference(matched_units)
if rule not in rules:
# A category matched with |rule| is a leaf of the breakdown tree.
# It is NOT broken down more.
category_tree[rule] = subtotal
continue
# A category matched with |rule| is broken down more.
subtemplate = rules[rule]
subworld = subtemplate[0]
subbreakdown = subtemplate[1]
if subworld == world:
# Break down in the same world: consider units.
category_tree[
rule], accounted_total, subremainder_units = accumulate(
subtemplate, snapshot, units_dict, matched_units)
subremainder_total = 0
if subremainder_units:
for unit_id in subremainder_units:
subremainder_total += units_dict[world][unit_id]
category_tree[rule][None] = subremainder_total
if subtotal != accounted_total + subremainder_total:
print >> sys.stderr, (
'WARNING: Sum of %s:%s is different from %s by %d bytes.' %
(subworld, subbreakdown, rule, subtotal -
(accounted_total + subremainder_total)))
else:
# Break down in a different world: consider only the total size.
category_tree[rule], accounted_total, _ = accumulate(
subtemplate, snapshot, units_dict,
set(units_dict[subworld].keys()))
if subtotal >= accounted_total:
category_tree[rule][None] = subtotal - accounted_total
else:
print >> sys.stderr, (
'WARNING: Sum of %s:%s is larger than %s by %d bytes.' %
(subworld, subbreakdown, rule, accounted_total - subtotal))
print >> sys.stderr, (
'WARNING: Assuming remainder of %s is 0.' % rule)
category_tree[rule][None] = 0
return category_tree, total, remainder_units
def find(self, address_list):
result = OrderedDict()
for address in address_list:
result[address] = self._mapping[address]
return result
