def massif_result(self) -> int: """Opens the massif file and calculates the peak memory consumption. Returns: Peak memory consuption as integer. """ data = msparser.parse_file(self.path + "/benchmarks/massif.out") peak = data["peak_snapshot_index"] peak_data = data["snapshots"][peak] peak_mem = int(peak_data["mem_heap"]) + \ int(peak_data["mem_heap_extra"]) + \ int(peak_data["mem_stack"]) return peak_mem
def main(): (options, args) = parse_args() for path in args[0:]: try: mdata = msparser.parse_file(path) if options.output == "json": print_as_json(mdata, options.indent) elif options.output == "gnuplot": print_gnuplot_script(mdata, os.path.basename(path), options.format, options.xsize, options.ysize) elif options.output == "table": print_gnuplot_dtable(mdata) except ParseError as perr: print(perr, file=sys.stderr)
def massif_to_csv(massif_filename): '''Convert a Massif file to a CSV string.''' data = msparser.parse_file(massif_filename) header = 'time,stack,heap,heap_extra' lines = [header] for snapshot in data['snapshots']: time = snapshot['time'] stack = snapshot['mem_stack'] heap = snapshot['mem_heap'] heap_extra = snapshot['mem_heap_extra'] lines.append(f'{time},{stack},{heap},{heap_extra}') return '\n'.join(lines)
def test_parse(self): actual = msparser.parse_file(path_to_actual) with open(path_to_expected) as fd_to_expected: expected = json.load(fd_to_expected) self.assertEqual(expected, actual)
sub_sp = subprocess.Popen(["./install/bin/subscriber-profiling 1 topic_name"], shell=True) pub_sp = subprocess.Popen(( "valgrind --tool=massif --stacks=yes --detailed-freq=1 --max-snapshots=300 --threshold=1.0 --massif-out-file=./massif-publisher.out ./install/bin/publisher-profiling 2 topic_name 8" ).split(), shell=False) time.sleep(5) pub_sp.send_signal(signal.SIGINT) sub_sp.terminate() agent_sp.terminate() time.sleep(1) std_heap_usage = 0 data = msparser.parse_file('massif-publisher.out') peak_index = data['peak_snapshot_index'] peak_snapshot = data['snapshots'][peak_index] for c in peak_snapshot['heap_tree']['children']: if c['details'] and c['details']['function'] == '???': std_heap_usage = c['nbytes'] stack_usage = round((peak_snapshot['mem_stack'] / 1000), 2) heap_usage = round((peak_snapshot['mem_heap'] / 1000), 2) total_usage = round( ((peak_snapshot['mem_stack'] + peak_snapshot['mem_heap'] + peak_snapshot['mem_heap_extra'] - std_heap_usage) / 1000), 2) print("stack usage: ", stack_usage) print("heap usage: ", heap_usage) print("total usage: ", total_usage)
import sys import msparser import csv stack = [] heap = [] msparser_data = msparser.parse_file(sys.argv[1]) for snapshot in msparser_data['snapshots']: if snapshot['mem_heap'] != 0: stack.append(snapshot['mem_stack']) heap.append(snapshot['mem_heap']) with open(sys.argv[2], 'w+') as csv_file: csv_writer = csv.writer(csv_file, delimiter=',', quoting=csv.QUOTE_ALL) csv_writer.writerow(['stack', 'heap']) csv_writer.writerow([max(stack), max(heap)])
for i in range(n): sub_sps.append(subprocess.Popen(["./install/bin/subscriber-profiling {} topic_name_{}".format(client_key, i)], shell=True)) client_key += 1 pub_sps.append(subprocess.Popen(["./install/bin/publisher-profiling {} topic_name_{} {}".format(client_key, i, t)], shell=True)) client_key += 1 time.sleep(5) for i in range(n): pub_sps[i].terminate() sub_sps[i].terminate() agent_sp.send_signal(signal.SIGINT) time.sleep(1.0) data = msparser.parse_file('massif-agent.out') peak_index = data['peak_snapshot_index'] peak_snapshot = data['snapshots'][peak_index] for c in peak_snapshot['heap_tree']['children']: if c['details'] and c['details']['function'] == '???': std_heap = c['nbytes'] stack.append(round((peak_snapshot['mem_stack'] / 1000), 2)) heap.append(round((peak_snapshot['mem_heap'] / 1000), 2)) total.append(round(((peak_snapshot['mem_stack'] + peak_snapshot['mem_heap'] + peak_snapshot['mem_heap_extra'] - std_heap) / 1000), 2)) stack_usage.append(stack) heap_usage.append(heap) total_usage.append(total) fig, ax = plt.subplots()
def plotMem( massifFile, filter, # filter by timer name minTimeDiff, # filter by difference in beginning and end minMemDiff, # filter by change in memory usage shortTimers, # exclude short timers memUnit='MB', # unit for memory displayTimers=True): # first parse the log file valgrind created for us data = msparser.parse_file(massifFile) massifFile = Path(massifFile) cmd = data['cmd'] timeUnit = data['time_unit'] snapshots = data['snapshots'] times = [] memHeap = [] for s in snapshots: try: times.append(s['time']) memHeap.append(formatMem(s['mem_heap'], memUnit)) except: pass # now parse all the snapshot pairs we took in the timers # (We compile MueLu with MueLu_TIMEMONITOR_MASSIF_SNAPSHOTS=1 ) snapshotPairs = [] for f in Path('.').glob(massifFile + "*start.out"): fEnd = f.replace('start.out', 'stop.out') label = Path(f).basename().stripext().replace('.start', '') label = label.replace(massifFile.basename() + '.', '') try: label, counter = label.rsplit('.', 1) except: pass try: data = msparser.parse_file(f) dataEnd = msparser.parse_file(fEnd) assert data['time_unit'] == timeUnit assert dataEnd['time_unit'] == timeUnit data = data['snapshots'] dataEnd = dataEnd['snapshots'] assert (len(data)) == 1 assert (len(dataEnd)) == 1 assert data[0]['time'] <= dataEnd[0]['time'], f data[0]['label'] = label data[0]['counter'] = counter data[0]['mem_heap'] = formatMem(data[0]['mem_heap'], memUnit) dataEnd[0]['mem_heap'] = formatMem(dataEnd[0]['mem_heap'], memUnit) times.append(data[0]['time']) times.append(dataEnd[0]['time']) memHeap.append(data[0]['mem_heap']) memHeap.append(dataEnd[0]['mem_heap']) snapshotPairs += [(data[0], dataEnd[0])] except FileNotFoundError: print(f) # sort the snapshots times = np.array(times) memHeap = np.array(memHeap) idx = np.argsort(times) print('maximum heap memory usage: {}'.format(memHeap.max())) times = times[idx] memHeap = memHeap[idx] times = times[memHeap > minMemDiff] memHeap = memHeap[memHeap > minMemDiff] assert (len(times) > 0) # plot the curve of memory usage plt.plot(times, memHeap) if displayTimers: # now, filter through the snapshot pairs # otherwise, the plot becomes very messy filter = re.compile(filter) told = (-2 * minTimeDiff, -2 * minTimeDiff) snapshotPairsNew = [] for i, pair in enumerate( sorted(snapshotPairs, key=lambda x: x[0]['time'])): if (filter.search(pair[0]['label']) and abs(pair[0]['mem_heap'] - pair[1]['mem_heap']) > minMemDiff): t = [pair[0]['time'], pair[1]['time']] if (abs(t[0] - told[0]) < minTimeDiff and abs(t[1] - told[1]) < minTimeDiff): print('Timers "{}" and "{}" seems to coincide'.format( nameold, pair[0]['label'])) continue if (t[1] - t[0] < shortTimers): continue told = t nameold = pair[0]['label'] snapshotPairsNew.append(pair) snapshotPairs = snapshotPairsNew # stack the snapshot pairs height = max(memHeap) / len(snapshotPairs) for i, pair in enumerate( sorted(snapshotPairs, key=lambda x: x[0]['time'])): plt.gca().add_patch( patches.Rectangle((pair[0]['time'], i * height), pair[1]['time'] - pair[0]['time'], height, alpha=0.5, facecolor='red')) plt.text(pair[0]['time'], (i + 0.5) * height, tex_escape(pair[0]['label'])) # add vertical lines at start and end for each timer plt.plot([pair[0]['time'], pair[0]['time']], [0, max(memHeap)], '-', c='grey', alpha=0.5) plt.plot([pair[1]['time'], pair[1]['time']], [0, max(memHeap)], '-', c='grey', alpha=0.5) # add circles on these lines for memory usage at beginning and end plt.scatter([pair[0]['time'], pair[1]['time']], [pair[0]['mem_heap'], pair[1]['mem_heap']], c='r') plt.xlabel(timeUnit) plt.ylabel(memUnit) plt.title(tex_escape(cmd))
'--shortTimers', help='filter out timers that have fewer instructions than this', type=int, default=-1) parser.add_argument('--filter', help='regexp to filter for timers to include', type=str, default='\(level|\(total,_level|TpetraExt') parser.add_argument('--memUnit', help='memory unit (KB, MB)', type=str, default='MB') args = parser.parse_args() massifFile = args.massifFile[0] data = msparser.parse_file(massifFile) timeUnit = data['time_unit'] if timeUnit == 'i': if args.shortTimers is -1: args.shortTimers = 5e7 if args.minTimeDiff is -1: args.minTimeDiff = 1e7 elif timeUnit == 'ms': if args.shortTimers is -1: args.shortTimers = 90 if args.minTimeDiff is -1: args.minTimeDiff = 20 else: raise NotImplementedError()
sub_sp = subprocess.Popen(( "valgrind --tool=massif --stacks=yes --detailed-freq=1 --max-snapshots=300 --threshold=1.0 --massif-out-file=./massif-subscriber.out ./install/bin/subscriber-profiling 1 topic_name" ).split(), shell=False) pub_sp = subprocess.Popen(["./install/bin/publisher-profiling 2 topic_name 8"], shell=True) time.sleep(5) pub_sp.terminate() sub_sp.send_signal(signal.SIGINT) agent_sp.terminate() time.sleep(1) std_heap_usage = 0 data = msparser.parse_file('massif-subscriber.out') peak_index = data['peak_snapshot_index'] peak_snapshot = data['snapshots'][peak_index] for c in peak_snapshot['heap_tree']['children']: if c['details'] and c['details']['function'] == '???': std_heap_usage = c['nbytes'] stack_usage = round((peak_snapshot['mem_stack'] / 1000), 2) heap_usage = round((peak_snapshot['mem_heap'] / 1000), 2) total_usage = round( ((peak_snapshot['mem_stack'] + peak_snapshot['mem_heap'] + peak_snapshot['mem_heap_extra'] - std_heap_usage) / 1000), 2) print("stack usage: ", stack_usage) print("heap usage: ", heap_usage) print("total usage: ", total_usage)