def createJSONData(native_interval_,
nativenum_intervals_,
interval_,
num_intervals_,
resultsdir_,
outputdir_,
title_,
mcpat,
verbose=False,
requested_cores_list=[]):
if verbose:
print 'Generate JSON data for Level 2'
global native_interval, nativenum_intervals, interval, num_intervals, resultsdir, outputdir, title, use_mcpat, stats, config
native_interval = native_interval_
nativenum_intervals = nativenum_intervals_
interval = interval_
num_intervals = num_intervals_
resultsdir = resultsdir_
outputdir = outputdir_
title = title_
use_mcpat = mcpat
stats = sniper_stats.SniperStats(resultsdir_)
config = sniper_lib.get_config(resultsdir=resultsdir_)
initialize()
collectCPIStackDataFIC(verbose=verbose,
requested_cores_list=requested_cores_list)
collectCPIStackDataFCC(verbose=verbose,
requested_cores_list=requested_cores_list)
writetojson(outputdir, "cpipercentage", "cpi", 1, verbose=verbose)
writetojson(outputdir,
"cpipercentagesimplified",
"cpisimplified",
1,
verbose=verbose)
writeinfo(outputdir, verbose)
writemarkers(outputdir, verbose)
writelabels(outputdir, "cpipercentage", "cpi")
writelabels(outputdir, "cpipercentagesimplified", "cpisimplified")
writelabels(outputdir, "cpific", "cpific")
writelabels(outputdir, "simple", "cpisimplified")
writeIPCvaluestoJSON(outputdir)
if (use_mcpat):
collectMcPATData(verbose)
writetojson(outputdir, "power", "mcpat", 1, verbose)
writetojson(outputdir, "energy", "mcpat", 2, verbose)
writetojson(outputdir, "energypercentage", "mcpat", 3, verbose)
writelabels(outputdir, "power", "mcpat")
writelabels(outputdir, "energy", "mcpat")
writelabels(outputdir, "energypercentage", "mcpat")
def __init__(self, resultsdir='.'):
filename = os.path.join(resultsdir, 'sim.rtntracefull')
if not os.path.exists(filename):
raise IOError('Cannot find trace file %s' % filename)
config = sniper_lib.get_config(resultsdir=resultsdir)
freq = 1e9 * float(
sniper_config.get_config(config, 'perf_model/core/frequency'))
self.fs_to_cycles = freq / 1e15
self.functions = {}
self.calls = {}
self.children = collections.defaultdict(set)
self.roots = set()
self.totals = {}
fp = open(filename)
self.headers = fp.readline().strip().split('\t')
for line in fp:
if line.startswith(':'):
eip, name, location = line.strip().split('\t')
eip = eip[1:]
self.functions[eip] = Function(eip, name, location)
else:
line = line.strip().split('\t')
stack = line[0].split(':')
eip = stack[-1]
stack = ':'.join(map(self.translateEip, stack))
data = dict(zip(self.headers[1:], map(long, line[1:])))
if stack in self.calls:
self.calls[stack].add(data)
else:
self.calls[stack] = Call(str(self.functions[eip]), eip,
stack, data)
parent = stack.rpartition(':')[0]
self.children[parent].add(stack)
self.roots = set(self.calls.keys())
for parent in self.calls:
for child in self.children[parent]:
self.roots.remove(child)
# Construct a list of calls where each child is ordered before its parent.
calls_ordered = collections.deque()
calls_tovisit = collections.deque(self.roots)
while calls_tovisit:
stack = calls_tovisit.pop()
calls_ordered.appendleft(stack)
calls_tovisit.extend(self.children[stack])
# Now implement a non-recursive version of buildTotal, which requires that each
# function's children have been visited before processing the parent,
# by visiting calls_ordered in left-to-right order.
for stack in calls_ordered:
self.calls[stack].buildTotal(self)
def __init__(self, resultsdir = '.'):
filename = os.path.join(resultsdir, 'sim.rtntracefull')
if not os.path.exists(filename):
raise IOError('Cannot find trace file %s' % filename)
config = sniper_lib.get_config(resultsdir = resultsdir)
freq = 1e9 * float(sniper_config.get_config(config, 'perf_model/core/frequency'))
self.fs_to_cycles = freq / 1e15
self.functions = {}
self.calls = {}
self.children = collections.defaultdict(set)
self.roots = set()
self.totals = {}
fp = open(filename)
self.headers = fp.readline().strip().split('\t')
for line in fp:
if line.startswith(':'):
eip, name, location = line.strip().split('\t')
eip = eip[1:]
self.functions[eip] = Function(eip, name, location)
else:
line = line.strip().split('\t')
stack = line[0].split(':')
eip = stack[-1]
stack = ':'.join(map(self.translateEip, stack))
data = dict(zip(self.headers[1:], map(long, line[1:])))
if stack in self.calls:
self.calls[stack].add(data)
else:
self.calls[stack] = Call(str(self.functions[eip]), eip, stack, data)
parent = stack.rpartition(':')[0]
self.children[parent].add(stack)
self.roots = set(self.calls.keys())
for parent in self.calls:
for child in self.children[parent]:
self.roots.remove(child)
# Construct a list of calls where each child is ordered before its parent.
calls_ordered = collections.deque()
calls_tovisit = collections.deque(self.roots)
while calls_tovisit:
stack = calls_tovisit.pop()
calls_ordered.appendleft(stack)
calls_tovisit.extend(self.children[stack])
# Now implement a non-recursive version of buildTotal, which requires that each
# function's children have been visited before processing the parent,
# by visiting calls_ordered in left-to-right order.
for stack in calls_ordered:
self.calls[stack].buildTotal(self)
def SniperStats(resultsdir = '.', jobid = None):
if jobid:
import sniper_stats_jobid
stats = sniper_stats_jobid.SniperStatsJobid(jobid)
elif os.path.exists(os.path.join(resultsdir, 'sim.stats.sqlite3')):
import sniper_stats_sqlite
stats = sniper_stats_sqlite.SniperStatsSqlite(os.path.join(resultsdir, 'sim.stats.sqlite3'))
elif os.path.exists(os.path.join(resultsdir, 'sim.stats.db')):
import sniper_stats_db
stats = sniper_stats_db.SniperStatsDb(os.path.join(resultsdir, 'sim.stats.db'))
else:
import sniper_stats_compat
stats = sniper_stats_compat.SniperStatsCompat(resultsdir)
stats.config = sniper_lib.get_config(jobid, resultsdir)
return stats
def SniperStats(resultsdir = '.', jobid = None):
if jobid:
import sniper_stats_jobid
stats = sniper_stats_jobid.SniperStatsJobid(jobid)
elif os.path.exists(os.path.join(resultsdir, 'sim.stats.sqlite3')):
import sniper_stats_sqlite
stats = sniper_stats_sqlite.SniperStatsSqlite(os.path.join(resultsdir, 'sim.stats.sqlite3'))
elif os.path.exists(os.path.join(resultsdir, 'sim.stats.db')):
import sniper_stats_db
stats = sniper_stats_db.SniperStatsDb(os.path.join(resultsdir, 'sim.stats.db'))
else:
import sniper_stats_compat
stats = sniper_stats_compat.SniperStatsCompat(resultsdir)
stats.config = sniper_lib.get_config(jobid, resultsdir)
return stats
def createJSONData(native_interval_, nativenum_intervals_, interval_, num_intervals_, resultsdir_, outputdir_, title_, mcpat, verbose = False, requested_cores_list = []):
if verbose:
print 'Generate JSON data for Level 2'
global native_interval, nativenum_intervals, interval, num_intervals, resultsdir, outputdir, title, use_mcpat, stats, config
native_interval = native_interval_
nativenum_intervals = nativenum_intervals_
interval = interval_
num_intervals = num_intervals_
resultsdir = resultsdir_
outputdir = outputdir_
title = title_
use_mcpat = mcpat
stats = sniper_stats.SniperStats(resultsdir_)
config = sniper_lib.get_config(resultsdir = resultsdir_)
initialize()
collectCPIStackDataFIC(verbose = verbose, requested_cores_list = requested_cores_list)
collectCPIStackDataFCC(verbose = verbose, requested_cores_list = requested_cores_list)
writetojson(outputdir,"cpipercentage","cpi",1,verbose=verbose)
writetojson(outputdir,"cpipercentagesimplified","cpisimplified",1,verbose=verbose)
writeinfo(outputdir,verbose)
writemarkers(outputdir,verbose)
writelabels(outputdir,"cpipercentage","cpi")
writelabels(outputdir,"cpipercentagesimplified","cpisimplified")
writelabels(outputdir, "cpific","cpific")
writelabels(outputdir, "simple","cpisimplified")
writeIPCvaluestoJSON(outputdir)
if(use_mcpat):
collectMcPATData(verbose)
writetojson(outputdir,"power","mcpat",1,verbose)
writetojson(outputdir,"energy","mcpat",2,verbose)
writetojson(outputdir,"energypercentage","mcpat",3,verbose)
writelabels(outputdir,"power","mcpat")
writelabels(outputdir,"energy","mcpat")
writelabels(outputdir,"energypercentage","mcpat")
for o, a in opts:
if o in ('-h', '--help'):
usage()
if o == '-d':
resultsdir = a
if o == '-n':
numberofresults = int(a)
if o == '-v' or o == '--verbose':
verbose = True
if o == '-j':
jobid = int(a)
if verbose:
print 'This script generates automatic suggestions for optimization'
config = sniper_lib.get_config(jobid=jobid, resultsdir=resultsdir)
if verbose:
print "parsing functions from sim.rtntrace"
for fn in ("sim.rtntrace", "rtntrace.out"):
rtntrace = sniper_lib.get_results_file(filename=fn,
jobid=jobid,
resultsdir=resultsdir)
if rtntrace:
break
functions, total = functionparser.parseFunctions(inputdata=rtntrace)
optimizationlist = runModules(functions, config)
totaltime = total["nonidle_elapsed_time"]
print
print "Overall best optimizations"
printResult(optimizationlist, totaltime, numberofresults, functions)
def gen_topology(resultsdir = '.', jobid = None, outputobj = sys.stdout, format = 'svg', embedded = False):
names = ('hwcontext', 'smt', 'L1-I', 'L1-D', 'L2', 'L3', 'L4', 'tag-dir', 'nuca-cache', 'dram-cache', 'dram-cntlr')
ids = dict([ (name, collections.defaultdict(lambda: None)) for name in names ])
stats = sniper_stats.SniperStats(resultsdir, jobid)
config = sniper_lib.get_config(resultsdir = resultsdir, jobid = jobid)
try:
topology = stats.get_topology()
except:
print >> sys.stderr, "Failed getting topology information"
topology = None
max_id = 0
if topology:
for name, lid, mid in stats.get_topology():
if name not in names:
print >> sys.stderr, 'Unknown component', name
continue
ids[name][int(lid)] = int(mid)
max_id = max(max_id, int(lid))
def format_config(name, lid):
caches = {'L1-I': 'l1_icache', 'L1-D': 'l1_dcache', 'L2': 'l2_cache', 'L3': 'l3_cache', 'L4': 'l4_cache'}
if name in caches:
value = sniper_config.get_config(config, 'perf_model/%s/cache_size' % caches[name], lid)
return sniper_lib.format_size(1024 * long(value), digits = 0)
elif name == 'dram-cache':
value = sniper_config.get_config(config, 'perf_model/dram/cache/cache_size', lid)
return sniper_lib.format_size(1024 * long(value), digits = 0)
else:
return ''
if format == 'text':
print >> outputobj, ' '*20,
for lid in range(max_id+1):
print >> outputobj, '%3d' % lid,
print >> outputobj
for name in names:
if ids[name].keys():
print >> outputobj, '%-20s' % name,
for lid in range(max_id+1):
mid = ids[name][lid]
if mid is None:
value = ' '
elif mid == lid:
value = 'X'
else:
value = '<'
print >> outputobj, '%3s' % value,
print >> outputobj
elif format == 'svg':
class Svg:
def __init__(self):
self.margin_x = 50; self.step_x = 110
self.margin_y = 50; self.step_y = 50
self.size_x = 0; self.size_y = 0
self.items = []
def paint_box(self, (x, y), (w, h), name = '', label = 0, color = '#ffffff', zorder = 0, margin = (.2, .2), root = (0, 0)):
x += root[0]; y += root[1]
self.size_x = max(self.size_x, (x+w) * self.step_x); self.size_y = max(self.size_y, (y+h) * self.step_y)
svg = '''\
<rect id="%s" x="%d" y="%d" width="%d" height="%d" rx="0"
style="stroke:#000000;stroke-width:1;stroke-linejoin:miter; stroke-linecap:butt;fill:%s;"/>
''' % (name, self.margin_x + x * self.step_x, self.margin_y + y * self.step_y,
(w - margin[0]) * self.step_x, (h - margin[1]) * self.step_y, color)
if label:
svg += '''\
<text xml:space="preserve" x="%d" y="%d" fill="#000000" font-family="Times" font-style="normal" font-weight="normal" font-size="12" text-anchor="start">%s</text>
''' % (self.margin_x + (x + .1) * self.step_x, self.margin_y + (y + .3) * self.step_y, label)
self.items.append((zorder, svg))
print format_event(timestamp, core, thread, 'Marker: %s' % format_marker(value0, value1, description))
elif event == sniper_stats.EVENT_THREAD_NAME:
print format_event(timestamp, core, thread, 'Thread name: %s' % description)
elif event == sniper_stats.EVENT_APP_START:
print format_event(timestamp, core, thread, 'Application %d start' % value0)
elif event == sniper_stats.EVENT_APP_EXIT:
print format_event(timestamp, core, thread, 'Application %d exit' % value0)
elif event == sniper_stats.EVENT_THREAD_CREATE:
print format_event(timestamp, core, thread, 'Thread created: application %d by thread %d' % (value0, value1))
elif event == sniper_stats.EVENT_THREAD_EXIT:
print format_event(timestamp, core, thread, 'Thread exit')
else:
print format_event(timestamp, core, thread, 'Unknown event %d (%d, %d, %s)' % (event, value0, value1, description))
if do_config:
config = sniper_lib.get_config(resultsdir = resultsdir, jobid = jobid)
for k, v in sorted(config.items()):
print '%s=%s' % (k, v)
if do_stats:
def print_result(key, value):
if type(value) is dict:
for _key, _value in sorted(value.items()):
print_result(key+'.'+_key, _value)
else:
print key, '=',
if type(value) is list:
print ', '.join(map(str, value))
else:
for o, a in opts:
if o in ("-h", "--help"):
usage()
if o == "-d":
resultsdir = a
if o == "-n":
numberofresults = int(a)
if o == "-v" or o == "--verbose":
verbose = True
if o == "-j":
jobid = int(a)
if verbose:
print "This script generates automatic suggestions for optimization"
config = sniper_lib.get_config(jobid=jobid, resultsdir=resultsdir)
if verbose:
print "parsing functions from sim.rtntrace"
for fn in ("sim.rtntrace", "rtntrace.out"):
rtntrace = sniper_lib.get_results_file(filename=fn, jobid=jobid, resultsdir=resultsdir)
if rtntrace:
break
functions, total = functionparser.parseFunctions(inputdata=rtntrace)
optimizationlist = runModules(functions, config)
totaltime = total["core_elapsed_time"]
print
print "Overall best optimizations"
printResult(optimizationlist, totaltime, numberofresults, functions)
print
print "Overall best combined optimizations"
def gen_topology(resultsdir='.',
jobid=None,
outputobj=sys.stdout,
format='svg',
embedded=False):
names = ('hwcontext', 'smt', 'L1-I', 'L1-D', 'L2', 'L3', 'L4', 'tag-dir',
'nuca-cache', 'dram-cache', 'dram-cntlr')
ids = dict([(name, collections.defaultdict(lambda: None))
for name in names])
stats = sniper_stats.SniperStats(resultsdir, jobid)
config = sniper_lib.get_config(resultsdir=resultsdir, jobid=jobid)
try:
topology = stats.get_topology()
except:
print >> sys.stderr, "Failed getting topology information"
topology = None
max_id = 0
if topology:
for name, lid, mid in stats.get_topology():
if name not in names:
print >> sys.stderr, 'Unknown component', name
continue
ids[name][int(lid)] = int(mid)
max_id = max(max_id, int(lid))
def format_config(name, lid):
caches = {
'L1-I': 'l1_icache',
'L1-D': 'l1_dcache',
'L2': 'l2_cache',
'L3': 'l3_cache',
'L4': 'l4_cache'
}
if name in caches:
value = sniper_config.get_config(
config, 'perf_model/%s/cache_size' % caches[name], lid)
return sniper_lib.format_size(1024 * long(value), digits=0)
elif name == 'dram-cache':
value = sniper_config.get_config(
config, 'perf_model/dram/cache/cache_size', lid)
return sniper_lib.format_size(1024 * long(value), digits=0)
else:
return ''
if format == 'text':
print >> outputobj, ' ' * 20,
for lid in range(max_id + 1):
print >> outputobj, '%3d' % lid,
print >> outputobj
for name in names:
if ids[name].keys():
print >> outputobj, '%-20s' % name,
for lid in range(max_id + 1):
mid = ids[name][lid]
if mid is None:
value = ' '
elif mid == lid:
value = 'X'
else:
value = '<'
print >> outputobj, '%3s' % value,
print >> outputobj
elif format == 'svg':
class Svg:
def __init__(self):
self.margin_x = 50
self.step_x = 110
self.margin_y = 50
self.step_y = 50
self.size_x = 0
self.size_y = 0
self.items = []
def paint_box(self,
(x, y),
(w, h),
name='',
label=0,
color='#ffffff',
zorder=0,
margin=(.2, .2),
root=(0, 0)):
x += root[0]
y += root[1]
self.size_x = max(self.size_x, (x + w) * self.step_x)
self.size_y = max(self.size_y, (y + h) * self.step_y)
svg = '''\
<rect id="%s" x="%d" y="%d" width="%d" height="%d" rx="0"
style="stroke:#000000;stroke-width:1;stroke-linejoin:miter; stroke-linecap:butt;fill:%s;"/>
''' % (name, self.margin_x + x * self.step_x,
self.margin_y + y * self.step_y, (w - margin[0]) * self.step_x,
(h - margin[1]) * self.step_y, color)
if label:
svg += '''\
<text xml:space="preserve" x="%d" y="%d" fill="#000000" font-family="Times" font-style="normal" font-weight="normal" font-size="12" text-anchor="start">%s</text>
''' % (self.margin_x + (x + .1) * self.step_x, self.margin_y +
(y + .3) * self.step_y, label)
self.items.append((zorder, svg))