def massif_result(self) -> int:
"""Opens the massif file and calculates the peak memory consumption.
Returns:
Peak memory consuption as integer.
"""
data = msparser.parse_file(self.path + "/benchmarks/massif.out")
peak = data["peak_snapshot_index"]
peak_data = data["snapshots"][peak]
peak_mem = int(peak_data["mem_heap"]) + \
int(peak_data["mem_heap_extra"]) + \
int(peak_data["mem_stack"])
return peak_mem
def main():
(options, args) = parse_args()
for path in args[0:]:
try:
mdata = msparser.parse_file(path)
if options.output == "json":
print_as_json(mdata, options.indent)
elif options.output == "gnuplot":
print_gnuplot_script(mdata, os.path.basename(path), options.format, options.xsize, options.ysize)
elif options.output == "table":
print_gnuplot_dtable(mdata)
except ParseError as perr:
print(perr, file=sys.stderr)
def massif_to_csv(massif_filename):
'''Convert a Massif file to a CSV string.'''
data = msparser.parse_file(massif_filename)
header = 'time,stack,heap,heap_extra'
lines = [header]
for snapshot in data['snapshots']:
time = snapshot['time']
stack = snapshot['mem_stack']
heap = snapshot['mem_heap']
heap_extra = snapshot['mem_heap_extra']
lines.append(f'{time},{stack},{heap},{heap_extra}')
return '\n'.join(lines)
def main():
(options, args) = parse_args()
for path in args[0:]:
try:
mdata = msparser.parse_file(path)
if options.output == "json":
print_as_json(mdata, options.indent)
elif options.output == "gnuplot":
print_gnuplot_script(mdata, os.path.basename(path),
options.format, options.xsize,
options.ysize)
elif options.output == "table":
print_gnuplot_dtable(mdata)
except ParseError as perr:
print(perr, file=sys.stderr)
def test_parse(self):
actual = msparser.parse_file(path_to_actual)
with open(path_to_expected) as fd_to_expected:
expected = json.load(fd_to_expected)
self.assertEqual(expected, actual)
sub_sp = subprocess.Popen(["./install/bin/subscriber-profiling 1 topic_name"],
shell=True)
pub_sp = subprocess.Popen((
"valgrind --tool=massif --stacks=yes --detailed-freq=1 --max-snapshots=300 --threshold=1.0 --massif-out-file=./massif-publisher.out ./install/bin/publisher-profiling 2 topic_name 8"
).split(),
shell=False)
time.sleep(5)
pub_sp.send_signal(signal.SIGINT)
sub_sp.terminate()
agent_sp.terminate()
time.sleep(1)
std_heap_usage = 0
data = msparser.parse_file('massif-publisher.out')
peak_index = data['peak_snapshot_index']
peak_snapshot = data['snapshots'][peak_index]
for c in peak_snapshot['heap_tree']['children']:
if c['details'] and c['details']['function'] == '???':
std_heap_usage = c['nbytes']
stack_usage = round((peak_snapshot['mem_stack'] / 1000), 2)
heap_usage = round((peak_snapshot['mem_heap'] / 1000), 2)
total_usage = round(
((peak_snapshot['mem_stack'] + peak_snapshot['mem_heap'] +
peak_snapshot['mem_heap_extra'] - std_heap_usage) / 1000), 2)
print("stack usage: ", stack_usage)
print("heap usage: ", heap_usage)
print("total usage: ", total_usage)
import sys
import msparser
import csv
stack = []
heap = []
msparser_data = msparser.parse_file(sys.argv[1])
for snapshot in msparser_data['snapshots']:
if snapshot['mem_heap'] != 0:
stack.append(snapshot['mem_stack'])
heap.append(snapshot['mem_heap'])
with open(sys.argv[2], 'w+') as csv_file:
csv_writer = csv.writer(csv_file, delimiter=',', quoting=csv.QUOTE_ALL)
csv_writer.writerow(['stack', 'heap'])
csv_writer.writerow([max(stack), max(heap)])
for i in range(n):
sub_sps.append(subprocess.Popen(["./install/bin/subscriber-profiling {} topic_name_{}".format(client_key, i)], shell=True))
client_key += 1
pub_sps.append(subprocess.Popen(["./install/bin/publisher-profiling {} topic_name_{} {}".format(client_key, i, t)], shell=True))
client_key += 1
time.sleep(5)
for i in range(n):
pub_sps[i].terminate()
sub_sps[i].terminate()
agent_sp.send_signal(signal.SIGINT)
time.sleep(1.0)
data = msparser.parse_file('massif-agent.out')
peak_index = data['peak_snapshot_index']
peak_snapshot = data['snapshots'][peak_index]
for c in peak_snapshot['heap_tree']['children']:
if c['details'] and c['details']['function'] == '???':
std_heap = c['nbytes']
stack.append(round((peak_snapshot['mem_stack'] / 1000), 2))
heap.append(round((peak_snapshot['mem_heap'] / 1000), 2))
total.append(round(((peak_snapshot['mem_stack'] + peak_snapshot['mem_heap'] + peak_snapshot['mem_heap_extra'] - std_heap) / 1000), 2))
stack_usage.append(stack)
heap_usage.append(heap)
total_usage.append(total)
fig, ax = plt.subplots()
def plotMem(
massifFile,
filter, # filter by timer name
minTimeDiff, # filter by difference in beginning and end
minMemDiff, # filter by change in memory usage
shortTimers, # exclude short timers
memUnit='MB', # unit for memory
displayTimers=True):
# first parse the log file valgrind created for us
data = msparser.parse_file(massifFile)
massifFile = Path(massifFile)
cmd = data['cmd']
timeUnit = data['time_unit']
snapshots = data['snapshots']
times = []
memHeap = []
for s in snapshots:
try:
times.append(s['time'])
memHeap.append(formatMem(s['mem_heap'], memUnit))
except:
pass
# now parse all the snapshot pairs we took in the timers
# (We compile MueLu with MueLu_TIMEMONITOR_MASSIF_SNAPSHOTS=1 )
snapshotPairs = []
for f in Path('.').glob(massifFile + "*start.out"):
fEnd = f.replace('start.out', 'stop.out')
label = Path(f).basename().stripext().replace('.start', '')
label = label.replace(massifFile.basename() + '.', '')
try:
label, counter = label.rsplit('.', 1)
except:
pass
try:
data = msparser.parse_file(f)
dataEnd = msparser.parse_file(fEnd)
assert data['time_unit'] == timeUnit
assert dataEnd['time_unit'] == timeUnit
data = data['snapshots']
dataEnd = dataEnd['snapshots']
assert (len(data)) == 1
assert (len(dataEnd)) == 1
assert data[0]['time'] <= dataEnd[0]['time'], f
data[0]['label'] = label
data[0]['counter'] = counter
data[0]['mem_heap'] = formatMem(data[0]['mem_heap'], memUnit)
dataEnd[0]['mem_heap'] = formatMem(dataEnd[0]['mem_heap'], memUnit)
times.append(data[0]['time'])
times.append(dataEnd[0]['time'])
memHeap.append(data[0]['mem_heap'])
memHeap.append(dataEnd[0]['mem_heap'])
snapshotPairs += [(data[0], dataEnd[0])]
except FileNotFoundError:
print(f)
# sort the snapshots
times = np.array(times)
memHeap = np.array(memHeap)
idx = np.argsort(times)
print('maximum heap memory usage: {}'.format(memHeap.max()))
times = times[idx]
memHeap = memHeap[idx]
times = times[memHeap > minMemDiff]
memHeap = memHeap[memHeap > minMemDiff]
assert (len(times) > 0)
# plot the curve of memory usage
plt.plot(times, memHeap)
if displayTimers:
# now, filter through the snapshot pairs
# otherwise, the plot becomes very messy
filter = re.compile(filter)
told = (-2 * minTimeDiff, -2 * minTimeDiff)
snapshotPairsNew = []
for i, pair in enumerate(
sorted(snapshotPairs, key=lambda x: x[0]['time'])):
if (filter.search(pair[0]['label'])
and abs(pair[0]['mem_heap'] - pair[1]['mem_heap']) >
minMemDiff):
t = [pair[0]['time'], pair[1]['time']]
if (abs(t[0] - told[0]) < minTimeDiff
and abs(t[1] - told[1]) < minTimeDiff):
print('Timers "{}" and "{}" seems to coincide'.format(
nameold, pair[0]['label']))
continue
if (t[1] - t[0] < shortTimers):
continue
told = t
nameold = pair[0]['label']
snapshotPairsNew.append(pair)
snapshotPairs = snapshotPairsNew
# stack the snapshot pairs
height = max(memHeap) / len(snapshotPairs)
for i, pair in enumerate(
sorted(snapshotPairs, key=lambda x: x[0]['time'])):
plt.gca().add_patch(
patches.Rectangle((pair[0]['time'], i * height),
pair[1]['time'] - pair[0]['time'],
height,
alpha=0.5,
facecolor='red'))
plt.text(pair[0]['time'], (i + 0.5) * height,
tex_escape(pair[0]['label']))
# add vertical lines at start and end for each timer
plt.plot([pair[0]['time'], pair[0]['time']], [0, max(memHeap)],
'-',
c='grey',
alpha=0.5)
plt.plot([pair[1]['time'], pair[1]['time']], [0, max(memHeap)],
'-',
c='grey',
alpha=0.5)
# add circles on these lines for memory usage at beginning and end
plt.scatter([pair[0]['time'], pair[1]['time']],
[pair[0]['mem_heap'], pair[1]['mem_heap']],
c='r')
plt.xlabel(timeUnit)
plt.ylabel(memUnit)
plt.title(tex_escape(cmd))
'--shortTimers',
help='filter out timers that have fewer instructions than this',
type=int,
default=-1)
parser.add_argument('--filter',
help='regexp to filter for timers to include',
type=str,
default='\(level|\(total,_level|TpetraExt')
parser.add_argument('--memUnit',
help='memory unit (KB, MB)',
type=str,
default='MB')
args = parser.parse_args()
massifFile = args.massifFile[0]
data = msparser.parse_file(massifFile)
timeUnit = data['time_unit']
if timeUnit == 'i':
if args.shortTimers is -1:
args.shortTimers = 5e7
if args.minTimeDiff is -1:
args.minTimeDiff = 1e7
elif timeUnit == 'ms':
if args.shortTimers is -1:
args.shortTimers = 90
if args.minTimeDiff is -1:
args.minTimeDiff = 20
else:
raise NotImplementedError()
def test_parse(self):
actual = msparser.parse_file(path_to_actual)
with open(path_to_expected) as fd_to_expected:
expected = json.load(fd_to_expected)
self.assertEqual(expected, actual)
sub_sp = subprocess.Popen((
"valgrind --tool=massif --stacks=yes --detailed-freq=1 --max-snapshots=300 --threshold=1.0 --massif-out-file=./massif-subscriber.out ./install/bin/subscriber-profiling 1 topic_name"
).split(),
shell=False)
pub_sp = subprocess.Popen(["./install/bin/publisher-profiling 2 topic_name 8"],
shell=True)
time.sleep(5)
pub_sp.terminate()
sub_sp.send_signal(signal.SIGINT)
agent_sp.terminate()
time.sleep(1)
std_heap_usage = 0
data = msparser.parse_file('massif-subscriber.out')
peak_index = data['peak_snapshot_index']
peak_snapshot = data['snapshots'][peak_index]
for c in peak_snapshot['heap_tree']['children']:
if c['details'] and c['details']['function'] == '???':
std_heap_usage = c['nbytes']
stack_usage = round((peak_snapshot['mem_stack'] / 1000), 2)
heap_usage = round((peak_snapshot['mem_heap'] / 1000), 2)
total_usage = round(
((peak_snapshot['mem_stack'] + peak_snapshot['mem_heap'] +
peak_snapshot['mem_heap_extra'] - std_heap_usage) / 1000), 2)
print("stack usage: ", stack_usage)
print("heap usage: ", heap_usage)
print("total usage: ", total_usage)
