def heat_map(request, pk):
data = get_data(pk)
k1 = {'intel': ['intel_pmc3']}
k2 = {'intel': ['MEM_LOAD_RETIRED_L1D_HIT']}
#k2 = {'intel': ['INSTRUCTIONS_RETIRED']}
ts0 = tspl.TSPLBase(None, k1, k2, job_stats=data)
k2 = {'intel': ['CLOCKS_UNHALTED_CORE']}
ts1 = tspl.TSPLBase(None, k1, k2, job_stats=data)
cpi = np.array([])
hosts = []
for v in ts0.data[0]:
hosts.append(v)
ncores = len(ts0.data[0][v])
for k in range(ncores):
i = np.array(ts0.data[0][v][k], dtype=np.float)
c = np.array(ts1.data[0][v][k], dtype=np.float)
ratio = np.divide(np.diff(i), np.diff(c))
if not cpi.size: cpi = np.array([ratio])
else: cpi = np.vstack((cpi, ratio))
cpi_min, cpi_max = cpi.min(), cpi.max()
fig, ax = plt.subplots(1, 1, figsize=(8, 12), dpi=110)
ycore = np.arange(cpi.shape[0] + 1)
time = ts0.t / 3600.
yhost = np.arange(len(hosts) + 1) * ncores + ncores
fontsize = 10
if len(yhost) > 80:
fontsize /= 0.5 * np.log(len(yhost))
plt.yticks(yhost - ncores / 2., hosts, size=fontsize)
plt.pcolormesh(time, ycore, cpi, vmin=cpi_min, vmax=cpi_max)
plt.axis([time.min(), time.max(), ycore.min(), ycore.max()])
plt.title('L1D Load Hits per Core Clock Cycle')
plt.colorbar()
ax.set_xlabel('Time (hrs)')
plt.close()
return figure_to_response(fig)
def __init__(self,jobid,k1,k2,aggregate=True,stats=None):
## Build ts and ld object for a job
self.k1=k1
self.k2=k2
self.jobid=jobid
self.aggregate=aggregate
try:
if self.aggregate:
self.ts=tspl.TSPLSum(jobid,self.k1,self.k2,job_data=stats)
else:
self.ts=tspl.TSPLBase(jobid,self.k1,self.k2,job_data=stats)
if not self.ld:
self.ld=lariat_utils.LariatData()
self.ld.get_job(self.ts.j.id,
end_epoch=self.ts.j.end_time,
daysback=3,
directory=lariat_path)
return
except tspl.TSPLException as e:
return
except EOFError as e:
print 'End of file found reading: ' + jobid
return
def setup(self, job_data):
self.aggregate = True
self.min_time = 3600
self.min_hosts = 1
self.waynesses=[x+1 for x in range(32)]
self.ignore_qs = []
self.metric = float("nan")
try:
if self.aggregate:
self.ts=tspl.TSPLSum("",self.k1,self.k2,job_data=job_data)
else:
self.ts=tspl.TSPLBase("",self.k1,self.k2,job_data=job_data)
except tspl.TSPLException as e:
return False
except EOFError as e:
print('End of file found reading: ' + job_path)
return False
if not tspl_utils.checkjob(self.ts,self.min_time,
self.waynesses,skip_queues=self.ignore_qs):
return False
elif self.ts.numhosts < self.min_hosts:
return False
else:
return True
def main():
parser = argparse.ArgumentParser(
description='Plot a key pair for some jobs')
parser.add_argument('-t', help='Threshold', metavar='thresh')
parser.add_argument('key1',
help='First key',
nargs='?',
default='amd64_core')
parser.add_argument('key2',
help='Second key',
nargs='?',
default='SSE_FLOPS')
parser.add_argument('filearg',
help='File, directory, or quoted'
' glob pattern',
nargs='?',
default='jobs')
parser.add_argument('-f', help='Set full mode', action='store_true')
parser.add_argument('-m', help='Set heatmap mode', action='store_true')
parser.add_argument('--max',
help='Use max instead of mean',
action='store_true')
n = parser.parse_args()
filelist = tspl_utils.getfilelist(n.filearg)
if n.max:
func = max
else:
func = scipy.stats.tmean
for file in filelist:
try:
if n.f:
full = '_full'
ts = tspl.TSPLBase(file, [n.key1], [n.key2])
else:
full = ''
ts = tspl.TSPLSum(file, [n.key1], [n.key2])
except tspl.TSPLException as e:
continue
if not tspl_utils.checkjob(ts, 3600, 16):
continue
reduction = [] # place to store reductions via func
for v in ts:
rate = numpy.divide(numpy.diff(v), numpy.diff(ts.t))
reduction.append(func(rate))
m = func(reduction)
if not n.t or m > float(n.t):
print ts.j.id + ': ' + str(m)
if n.m:
heatmap(ts, n, m, full)
else:
lineplot(ts, n, m, full)
else:
print ts.j.id + ': under threshold, ' + str(m) + ' < ' + n.t
def getuser(file, user, output_dir):
try:
ts = tspl.TSPLBase(file, ['lnet'], ['rx_bytes'])
except tspl.TSPLException as e:
return
if ts.owner == user:
masterplot.mp_wrapper(file, output_dir=output_dir, wide=True)
def compute_imbalance(file, k1, k2, threshold, plot_flag, full_flag, ratios):
try:
if full_flag:
full = '_full'
ts = tspl.TSPLBase(file, k1, k2)
else:
full = ''
ts = tspl.TSPLSum(file, k1, k2)
except tspl.TSPLException as e:
return
except EOFError as e:
print 'End of file found reading: ' + file
return
ignore_qs = ['gpu', 'gpudev', 'vis', 'visdev']
if not tspl_utils.checkjob(ts, 3600, 16, ignore_qs): # 1 hour, 16way only
return
elif ts.numhosts < 2: # At least 2 hosts
print ts.j.id + ': 1 host'
return
tmid = (ts.t[:-1] + ts.t[1:]) / 2.0
rng = range(1, len(tmid)) # Throw out first and last
tmid = tmid[rng]
maxval = numpy.zeros(len(rng))
minval = numpy.ones(len(rng)) * 1e100
rate = []
for v in ts:
rate.append(numpy.divide(numpy.diff(v)[rng], numpy.diff(ts.t)[rng]))
maxval = numpy.maximum(maxval, rate[-1])
minval = numpy.minimum(minval, rate[-1])
vals = []
mean = []
std = []
for j in range(len(rng)):
vals.append([])
for v in rate:
vals[j].append(v[j])
mean.append(scipy.stats.tmean(vals[j]))
std.append(scipy.stats.tstd(vals[j]))
imbl = maxval - minval
ratio = numpy.divide(std, mean)
ratio2 = numpy.divide(imbl, maxval)
var = scipy.stats.tmean(ratio) # mean of ratios is the threshold statistic
# Save away a list of ratios per user
ratios[ts.j.id] = [var, ts.owner]
print ts.j.id + ': ' + str(var)
# If over the threshold, plot this job (This should be factored out)
if plot_flag and abs(var) > threshold:
fig, ax = plt.subplots(2, 1, figsize=(8, 8), dpi=80)
plot_ratios(ts, tmid, ratio, ratio2, rate, var, fig, ax, full)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-f', help='Set full mode', action='store_true')
parser.add_argument('filearg',
help='File, directory, or quoted'
' glob pattern',
nargs='?',
default='jobs')
n = parser.parse_args()
filelist = tspl_utils.getfilelist(n.filearg)
for file in filelist:
try:
full = ''
ts = tspl.TSPLBase(file,
['amd64_sock', 'amd64_sock', 'amd64_sock'],
['HT0', 'HT1', 'HT2'])
except tspl.TSPLException as e:
continue
if not tspl_utils.checkjob(ts, 3600, 16): # 1 hour, 16way only
continue
elif ts.numhosts < 2: # At least 2 hosts
print ts.j.id + ': 1 host'
continue
print ts.j.id
tmid = (ts.t[:-1] + ts.t[1:]) / 2.0
dt = numpy.diff(ts.t)
fig, ax = plt.subplots(1, 1, figsize=(8, 6), dpi=80)
ax.hold = True
xmin, xmax = [0., 0.]
c = Colors()
for k in ts.j.hosts.keys():
h = ts.j.hosts[k]
col = c.next()
for i in range(3):
for j in range(4):
rate = numpy.divide(numpy.diff(ts.data[i][k][j]), dt)
xmin, xmax = [min(xmin, min(rate)), max(xmax, max(rate))]
ax.plot(tmid / 3600, rate, '-' + col)
if xmax > 2.0e9:
print ts.j.id + ' over limit: %(v)8.3f' % {'v': xmax}
else:
plt.close()
continue
plt.suptitle(ts.title)
xmin, xmax = tspl_utils.expand_range(xmin, xmax, .1)
ax.set_ylim(bottom=xmin, top=xmax)
fname = '_'.join(['graph', ts.j.id, 'HT_rates'])
fig.savefig(fname)
plt.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-f', help='Set full mode', action='store_true')
parser.add_argument('key1',
help='First key',
nargs='?',
default='amd64_core')
parser.add_argument('key2',
help='Second key',
nargs='?',
default='SSE_FLOPS')
parser.add_argument('filearg',
help='File, directory, or quoted'
' glob pattern',
nargs='?',
default='jobs')
n = parser.parse_args()
filelist = tspl_utils.getfilelist(n.filearg)
for file in filelist:
try:
if n.f:
full = '_full'
ts = tspl.TSPLBase(file, [n.key1], [n.key2])
else:
full = ''
ts = tspl.TSPLSum(file, [n.key1], [n.key2])
except tspl.TSPLException as e:
continue
if not tspl_utils.checkjob(ts, 3600, 16): # 1 hour, 16way only
continue
elif ts.numhosts < 2: # At least 2 hosts
print ts.j.id + ': 1 host'
continue
print ts.j.id
fig, ax = plt.subplots(1, 1, figsize=(8, 6), dpi=80)
xmin, xmax = [0., 0.]
for v in ts:
rate = numpy.divide(numpy.diff(v), numpy.diff(ts.t))
xmin, xmax = [min(xmin, min(rate)), max(xmax, max(rate))]
ax.hold = True
ax.plot(rate[1:], rate[:-1], '.')
ax.set_ylim(bottom=xmin, top=xmax)
ax.set_xlim(left=xmin, right=xmax)
fname = '_'.join(
['graph', ts.j.id, ts.k1[0], ts.k2[0], 'phase' + full])
fig.savefig(fname)
plt.close()
def main():
parser = argparse.ArgumentParser(description='Look for lack of correlation'
' between two key pairs/')
parser.add_argument('threshold',
help='Treshold Pearson R',
nargs='?',
default=0.8)
parser.add_argument('keya1',
help='Key A1',
nargs='?',
default='amd64_core')
parser.add_argument('keya2', help='Key A2', nargs='?', default='DCSF')
parser.add_argument('keyb1',
help='Key B1',
nargs='?',
default='amd64_core')
parser.add_argument('keyb2', help='Key B2', nargs='?', default='SSE_FLOPS')
parser.add_argument('filearg',
help='File, directory, or quoted'
' glob pattern',
nargs='?',
default='jobs')
parser.add_argument('-f', help='Set full mode', action='store_true')
n = parser.parse_args()
filelist = tspl_utils.getfilelist(n.filearg)
threshold = n.threshold
k1 = [n.keya1, n.keyb1]
k2 = [n.keya2, n.keyb2]
for file in filelist:
try:
if n.f:
full = '_full'
ts = tspl.TSPLBase(file, k1, k2)
else:
full = ''
ts = tspl.TSPLSum(file, k1, k2)
except tspl.TSPLException as e:
continue
if not tspl_utils.checkjob(ts, 3600, 16):
continue
r = pearson(ts)
print ts.j.id + ': ' + str(r)
if abs(r) < float(threshold):
plot_correlation(ts, r, full)
def getcode(file, code, output_dir):
try:
ts = tspl.TSPLBase(file, ['lnet'], ['rx_bytes'])
except tspl.TSPLException as e:
return
ld = lariat_utils.LariatData(ts.j.id, ts.j.end_time,
analyze_conf.lariat_path)
ename = ld.exc.split('/')[-1]
ename = ld.comp_name(ename, ld.equiv_patterns)
if ename == code:
print ts.j.id, ename, ts.wayness
masterplot.master_plot(file,
output_dir=output_dir,
mintime=1,
wayness=ts.wayness)
def compute_imbalance(file, k1, k2, thresh, lariat_dict):
try:
ts = tspl.TSPLBase(file, k1, k2)
except tspl.TSPLException as e:
return
except EOFError as e:
print 'End of file found reading: ' + file
return
ignore_qs = ['gpu', 'gpudev', 'vis', 'visdev']
if not tspl_utils.checkjob(ts, 3600, 16, ignore_qs): # 1 hour, 16way only
return
elif ts.numhosts < 2: # At least 2 hosts
print ts.j.id + ': 1 host'
return
if lariat_dict == None:
ld = lariat_utils.LariatData(ts.j.id,
end_epoch=ts.j.end_time,
daysback=3,
directory=analyze_conf.lariat_path)
else:
ld = lariat_utils.LariatData(ts.j.id, olddata=lariat_dict)
if ld.wayness == -1:
print 'Unknown wayness: ', ts.j.id
return
elif ld.wayness != ts.wayness:
print 'Lariat and TACC Stats disagree about wayness. Skipping: ', ts.j.id
return
tmid = (ts.t[:-1] + ts.t[1:]) / 2.0
rng = range(1, len(tmid)) # Throw out first and last
tmid = tmid[rng]
for h in ts.data[0].keys():
host_data = ts.data[0][h]
maxval = numpy.zeros(len(rng))
minval = numpy.ones(len(rng)) * 1e100
rate = []
for v in host_data:
rate.append(numpy.diff(v)[rng] / numpy.diff(ts.t)[rng])
maxval = numpy.maximum(maxval, rate[-1])
minval = numpy.minimum(minval, rate[-1])
vals = []
mean = []
std = []
for j in range(len(rng)):
vals.append([])
for v in rate:
vals[j].append(v[j])
mean.append(scipy.stats.tmean(vals[j]))
std.append(scipy.stats.tstd(vals[j]))
ratio = numpy.divide(std, mean)
var = scipy.stats.tmean(ratio)
if abs(var) > thresh:
print ts.j.id + ': ' + str(var)
return file
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-f', help='Set full mode', action='store_true')
parser.add_argument('key1', help='First key', nargs='?',
default='amd64_core')
parser.add_argument('key2', help='Second key', nargs='?',
default='SSE_FLOPS')
parser.add_argument('filearg', help='File, directory, or quoted'
' glob pattern', nargs='?',default='jobs')
n=parser.parse_args()
filelist=tspl_utils.getfilelist(n.filearg)
for file in filelist:
try:
if n.f:
full='_full'
ts=tspl.TSPLBase(file,[n.key1],[n.key2])
else:
full=''
ts=tspl.TSPLSum(file,[n.key1],[n.key2])
except tspl.TSPLException as e:
continue
if not tspl_utils.checkjob(ts,3600,16): # 1 hour, 16way only
continue
elif ts.numhosts < 2: # At least 2 hosts
print ts.j.id + ': 1 host'
continue
print ts.j.id
tmid=(ts.t[:-1]+ts.t[1:])/2.0
s=[]
for v in ts:
s=v
break
fig,ax=plt.subplots(2,1,figsize=(8,6),dpi=80)
ax[0].hold=True
ax[1].hold=True
xmin,xmax=[0.,0.]
xmin1,xmax1=[0.,0.]
dt=numpy.diff(ts.t)
for v in ts:
rate=numpy.array(numpy.divide(numpy.diff(v),dt),dtype=numpy.int64)
d=numpy.linalg.norm(rate,ord=1)/float(len(rate))
xmin,xmax=[min(xmin,min(rate)),max(xmax,max(rate))]
xmin1,xmax1=[min(xmin1,min(rate-d)),max(xmax1,max(rate-d))]
ax[0].plot(tmid,rate)
ax[1].plot(tmid,rate-d)
xmin,xmax=tspl_utils.expand_range(xmin,xmax,.1)
xmin1,xmax1=tspl_utils.expand_range(xmin1,xmax1,.1)
ax[0].set_ylim(bottom=xmin,top=xmax)
ax[1].set_ylim(bottom=xmin1,top=xmax1)
fname='_'.join(['graph',ts.j.id,ts.k1[0],ts.k2[0],'adjust'+full])
fig.savefig(fname)
plt.close()