def plot_lines(ax, ts, index, xscale=1.0, yscale=1.0, xlabel='', ylabel=''):
tmid=(ts.t[:-1]+ts.t[1:])/2.0
ax.hold=True
for k in ts.j.hosts.keys():
v=assemble(ts.data,index,k,0)
rate=numpy.divide(numpy.diff(v),numpy.diff(ts.t))
ax.plot(tmid/xscale,rate/yscale)
tspl_utils.adjust_yaxis_range(ax,0.1)
ax.yaxis.set_major_locator( matplotlib.ticker.MaxNLocator(nbins=6))
setlabels(ax,ts,index,xlabel,ylabel,yscale)
def plot_mmm(ax,
ts,
index,
xscale=1.0,
yscale=1.0,
xlabel='',
ylabel='',
do_rate=False):
tmid = (ts.t[:-1] + ts.t[1:]) / 2.0
d = []
for k in ts.j.hosts.keys():
v = ts.assemble(index, k, 0)
if do_rate:
d.append(numpy.divide(numpy.diff(v), numpy.diff(ts.t)))
else:
d.append((v[:-1] + v[1:]) / 2.0)
a = numpy.array(d)
mn = []
p25 = []
p50 = []
p75 = []
mx = []
for i in range(len(ts.t) - 1):
mn.append(min(a[:, i]))
p25.append(scipy.stats.scoreatpercentile(a[:, i], 25))
p50.append(scipy.stats.scoreatpercentile(a[:, i], 50))
p75.append(scipy.stats.scoreatpercentile(a[:, i], 75))
mx.append(max(a[:, i]))
mn = numpy.array(mn)
p25 = numpy.array(p25)
p50 = numpy.array(p50)
p75 = numpy.array(p75)
mx = numpy.array(mx)
ax.hold = True
ax.plot(tmid / xscale, mn / yscale, '--')
ax.plot(tmid / xscale, p25 / yscale)
ax.plot(tmid / xscale, p50 / yscale)
ax.plot(tmid / xscale, p75 / yscale)
ax.plot(tmid / xscale, mx / yscale, '--')
setlabels(ax, ts, index, xlabel, ylabel, yscale)
ax.yaxis.set_major_locator(matplotlib.ticker.MaxNLocator(nbins=4))
tspl_utils.adjust_yaxis_range(ax, 0.1)
def plot_mmm(ax, ts, index, xscale=1.0, yscale=1.0, xlabel='', ylabel='',
do_rate=False):
tmid=(ts.t[:-1]+ts.t[1:])/2.0
d=[]
for k in ts.j.hosts.keys():
v=ts.assemble(index,k,0)
if do_rate:
d.append(numpy.divide(numpy.diff(v),numpy.diff(ts.t)))
else:
d.append((v[:-1]+v[1:])/2.0)
a=numpy.array(d)
mn=[]
p25=[]
p50=[]
p75=[]
mx=[]
for i in range(len(ts.t)-1):
mn.append(min(a[:,i]))
p25.append(scipy.stats.scoreatpercentile(a[:,i],25))
p50.append(scipy.stats.scoreatpercentile(a[:,i],50))
p75.append(scipy.stats.scoreatpercentile(a[:,i],75))
mx.append(max(a[:,i]))
mn=numpy.array(mn)
p25=numpy.array(p25)
p50=numpy.array(p50)
p75=numpy.array(p75)
mx=numpy.array(mx)
ax.hold=True
ax.plot(tmid/xscale,mn/yscale,'--')
ax.plot(tmid/xscale,p25/yscale)
ax.plot(tmid/xscale,p50/yscale)
ax.plot(tmid/xscale,p75/yscale)
ax.plot(tmid/xscale,mx/yscale,'--')
setlabels(ax,ts,index,xlabel,ylabel,yscale)
ax.yaxis.set_major_locator( matplotlib.ticker.MaxNLocator(nbins=4))
tspl_utils.adjust_yaxis_range(ax,0.1)
def plot_lines(ax,
ts,
index,
xscale=1.0,
yscale=1.0,
xlabel='',
ylabel='',
do_rate=True):
tmid = (ts.t[:-1] + ts.t[1:]) / 2.0
ax.hold = True
for k in ts.j.hosts.keys():
v = ts.assemble(index, k, 0)
if do_rate:
rate = numpy.divide(numpy.diff(v), numpy.diff(ts.t))
ax.plot(tmid / xscale, rate / yscale)
else:
val = (v[:-1] + v[1:]) / 2.0
ax.plot(tmid / xscale, val / yscale)
tspl_utils.adjust_yaxis_range(ax, 0.1)
ax.yaxis.set_major_locator(matplotlib.ticker.MaxNLocator(nbins=6))
setlabels(ax, ts, index, xlabel, ylabel, yscale)
def main():
parser = argparse.ArgumentParser(
description='Deal with a directory of pickle'
' files nightly')
parser.add_argument('-p',
help='Set number of processes',
nargs=1,
type=int,
default=[1])
parser.add_argument('-o',
help='Output directory',
nargs=1,
type=str,
default=['.'],
metavar='output_dir')
parser.add_argument('-q',
help='Queue',
nargs=1,
type=str,
default=['normal'],
metavar='queue')
parser.add_argument('filearg',
help='File, directory, or quoted'
' glob pattern',
nargs='?',
default='jobs')
n = parser.parse_args()
filelist = tspl_utils.getfilelist(n.filearg)
target_queue = n.q[0]
pool = multiprocessing.Pool(processes=n.p[0])
m = multiprocessing.Manager()
files = m.list()
partial_getqueue = functools.partial(getqueue, queue=target_queue)
res = pool.map(partial_getqueue, filelist)
pool.close()
pool.join()
res = filter(lambda x: x != None, res)
if len(res) == 0:
print 'no jobs found'
return
res_sorted = sorted(res, key=lambda x: x[0])
res2 = zip(*res_sorted)
t = res2[0]
min_dram_rate = res2[1]
max_dram_rate = res2[2]
mean_dram_rate = res2[3]
min_l1_rate = res2[4]
max_l1_rate = res2[5]
mean_l1_rate = res2[6]
min_lnet_rate = res2[7]
max_lnet_rate = res2[8]
mean_lnet_rate = res2[9]
min_ib_rate = res2[10]
max_ib_rate = res2[11]
mean_ib_rate = res2[12]
min_user_rate = res2[13]
max_user_rate = res2[14]
mean_user_rate = res2[15]
min_flops_rate = res2[16]
max_flops_rate = res2[17]
mean_flops_rate = res2[18]
min_mem_usage = res2[19]
max_mem_usage = res2[20]
mean_mem_usage = res2[21]
ids = res2[22]
start_date = datetime.datetime.fromtimestamp(
t[0]).strftime('%Y-%m-%d %H:%M:%S')
end_date = datetime.datetime.fromtimestamp(
t[-1]).strftime('%Y-%m-%d %H:%M:%S')
fig, ax = plt.subplots(6, 1, figsize=(8, 12), dpi=80)
plt.subplots_adjust(hspace=0.35)
ax[0].plot(t, min_flops_rate, 'x', t, mean_flops_rate, '+', t,
max_flops_rate, '*')
ax[0].set_ylabel('GFlops/s')
ax[0].set_xticklabels(labels=[])
ax[1].plot(t, min_dram_rate, 'x', t, mean_dram_rate, '+', t, max_dram_rate,
'*')
ax[1].set_ylabel('DRAM BW MB/s')
ax[1].set_xticklabels(labels=[])
ax[2].plot(t, min_mem_usage, 'x', t, mean_mem_usage, '+', t, max_mem_usage,
'*')
ax[2].set_ylabel('DRAM Usage GB')
ax[2].set_xticklabels(labels=[])
ax[3].plot(t, min_lnet_rate, 'x', t, mean_lnet_rate, '+', t, max_lnet_rate,
'*')
ax[3].set_ylabel('Lnet Rate MB/s')
ax[3].set_xticklabels(labels=[])
ax[4].plot(t, min_ib_rate, 'x', t, mean_ib_rate, '+', t, max_ib_rate, '*')
ax[4].set_ylabel('IB - Lnet Rate MB/s')
ax[4].set_xticklabels(labels=[])
ax[5].plot(t, min_user_rate, 'x', t, mean_user_rate, '+', t, max_user_rate,
'*')
ax[5].set_ylabel('CPU User Fraction')
ax[5].set_xticklabels(labels=[])
for i in range(6):
tspl_utils.adjust_yaxis_range(ax[i], 0.1)
ax[5].set_xlabel('t')
plt.suptitle(target_queue + ' ' + start_date + ' -- ' + end_date)
fname = target_queue
fig.savefig(fname)
plt.close()
print 'Found', len(res_sorted), 'jobs for', target_queue, ids
def master_plot(file, threshold=False):
k1 = [
'amd64_core', 'amd64_core', 'amd64_sock', 'lnet', 'lnet', 'ib_sw',
'ib_sw', 'cpu'
]
k2 = [
'SSE_FLOPS', 'DCSF', 'DRAM', 'rx_bytes', 'tx_bytes', 'rx_bytes',
'tx_bytes', 'user'
]
try:
print file
ts = tspl.TSPLSum(file, k1, k2)
except tspl.TSPLException as e:
return
if not tspl_utils.checkjob(ts, 3600, 16):
return
elif ts.numhosts < 2:
print ts.j.id + ': 1 host'
return
tmid = (ts.t[:-1] + ts.t[1:]) / 2.0
fig, ax = plt.subplots(6, 1, figsize=(8, 12), dpi=80)
# Plot flop rate
ax[0].hold = True
for k in ts.j.hosts.keys():
h = ts.j.hosts[k]
rate = numpy.divide(numpy.diff(ts.data[0][k][0]), numpy.diff(ts.t))
ax[0].plot(tmid / 3600, rate)
ax[0].set_ylabel('Total ' + ts.k1[0] + '\n' + ts.k2[0] + '/s')
# Plot DCSF rate
ax[1].hold = True
for k in ts.j.hosts.keys():
h = ts.j.hosts[k]
rate = numpy.divide(numpy.diff(ts.data[1][k][0]), numpy.diff(ts.t))
ax[1].plot(tmid / 3600, rate)
ax[1].set_ylabel('Total ' + ts.k1[1] + '\n' + ts.k2[1] + '/s')
#Plot DRAM rate
ax[2].hold = True
for k in ts.j.hosts.keys():
h = ts.j.hosts[k]
rate = numpy.divide(numpy.diff(ts.data[2][k][0]), numpy.diff(ts.t))
ax[2].plot(tmid / 3600, rate)
ax[2].set_ylabel('Total ' + ts.k1[2] + '\n' + ts.k2[2] + '/s')
# Plot lnet sum rate
ax[3].hold = True
for k in ts.j.hosts.keys():
h = ts.j.hosts[k]
rate = numpy.divide(numpy.diff(ts.data[3][k][0] + ts.data[4][k][0]),
numpy.diff(ts.t))
ax[3].plot(tmid / 3600, rate / (1024. * 1024.))
ax[3].set_ylabel('Total lnet MB/s')
# Plot remaining IB sum rate
ax[4].hold = True
for k in ts.j.hosts.keys():
h = ts.j.hosts[k]
v = ts.data[5][k][0] + ts.data[6][k][0] - (ts.data[3][k][0] +
ts.data[4][k][0])
rate = numpy.divide(numpy.diff(v), numpy.diff(ts.t))
ax[4].plot(tmid / 3600, rate / (1024 * 1024.))
ax[4].set_ylabel('Total (ib_sw-lnet) MB/s')
#Plot CPU user time
ax[5].hold = True
for k in ts.j.hosts.keys():
h = ts.j.hosts[k]
rate = numpy.divide(numpy.diff(ts.data[7][k][0] / 100 / ts.wayness),
numpy.diff(ts.t))
ax[5].plot(tmid / 3600, rate)
ax[5].set_ylabel('Total ' + ts.k1[7] + '\n' + ts.k2[7] + '/s')
ax[5].set_xlabel('Time (hr)')
print ts.j.id + ': '
title = ts.title
if threshold:
title += ', V: %(v)-8.3f' % {'v': threshold}
plt.suptitle(title)
plt.subplots_adjust(hspace=0.35)
for a in ax:
tspl_utils.adjust_yaxis_range(a, 0.1)
fname = '_'.join(['graph', ts.j.id, 'master'])
fig.savefig(fname)
plt.close()
def master_plot(file,threshold=False):
k1=['amd64_core','amd64_core','amd64_sock','lnet','lnet','ib_sw','ib_sw',
'cpu']
k2=['SSE_FLOPS','DCSF','DRAM','rx_bytes','tx_bytes','rx_bytes','tx_bytes',
'user']
try:
print file
ts=tspl.TSPLSum(file,k1,k2)
except tspl.TSPLException as e:
return
if not tspl_utils.checkjob(ts,3600,16):
return
elif ts.numhosts < 2:
print ts.j.id + ': 1 host'
return
tmid=(ts.t[:-1]+ts.t[1:])/2.0
fig,ax=plt.subplots(6,1,figsize=(8,12),dpi=80)
# Plot flop rate
ax[0].hold=True
for k in ts.j.hosts.keys():
h=ts.j.hosts[k]
rate=numpy.divide(numpy.diff(ts.data[0][k][0]),numpy.diff(ts.t))
ax[0].plot(tmid/3600,rate)
ax[0].set_ylabel('Total ' + ts.k1[0] + '\n' + ts.k2[0] + '/s')
# Plot DCSF rate
ax[1].hold=True
for k in ts.j.hosts.keys():
h=ts.j.hosts[k]
rate=numpy.divide(numpy.diff(ts.data[1][k][0]),numpy.diff(ts.t))
ax[1].plot(tmid/3600,rate)
ax[1].set_ylabel('Total ' + ts.k1[1] + '\n' + ts.k2[1] + '/s')
#Plot DRAM rate
ax[2].hold=True
for k in ts.j.hosts.keys():
h=ts.j.hosts[k]
rate=numpy.divide(numpy.diff(ts.data[2][k][0]),numpy.diff(ts.t))
ax[2].plot(tmid/3600,rate)
ax[2].set_ylabel('Total ' + ts.k1[2] + '\n' + ts.k2[2] + '/s')
# Plot lnet sum rate
ax[3].hold=True
for k in ts.j.hosts.keys():
h=ts.j.hosts[k]
rate=numpy.divide(numpy.diff(ts.data[3][k][0]+ts.data[4][k][0]),
numpy.diff(ts.t))
ax[3].plot(tmid/3600,rate/(1024.*1024.))
ax[3].set_ylabel('Total lnet MB/s')
# Plot remaining IB sum rate
ax[4].hold=True
for k in ts.j.hosts.keys():
h=ts.j.hosts[k]
v=ts.data[5][k][0]+ts.data[6][k][0]-(ts.data[3][k][0]+ts.data[4][k][0])
rate=numpy.divide(numpy.diff(v),numpy.diff(ts.t))
ax[4].plot(tmid/3600,rate/(1024*1024.))
ax[4].set_ylabel('Total (ib_sw-lnet) MB/s')
#Plot CPU user time
ax[5].hold=True
for k in ts.j.hosts.keys():
h=ts.j.hosts[k]
rate=numpy.divide(numpy.diff(ts.data[7][k][0]/100/ts.wayness),
numpy.diff(ts.t))
ax[5].plot(tmid/3600,rate)
ax[5].set_ylabel('Total ' + ts.k1[7] + '\n' + ts.k2[7] + '/s')
ax[5].set_xlabel('Time (hr)')
print ts.j.id + ': '
title=ts.title
if threshold:
title+=', V: %(v)-8.3f' % {'v': threshold}
plt.suptitle(title)
plt.subplots_adjust(hspace=0.35)
for a in ax:
tspl_utils.adjust_yaxis_range(a,0.1)
fname='_'.join(['graph',ts.j.id,'master'])
fig.savefig(fname)
plt.close()
def main():
parser = argparse.ArgumentParser(description='Look for high meta data rate'\
' to Lustre')
parser.add_argument('-t',
metavar='thresh',
help='Treshold metadata rate',
nargs=1,
default=[100000.])
parser.add_argument('filearg',
help='File, directory, or quoted'
' glob pattern',
nargs='?',
default='jobs')
n = parser.parse_args()
thresh = float(n.t[0])
print thresh
filelist = tspl_utils.getfilelist(n.filearg)
# k1=['llite', 'llite', 'llite', 'llite', 'llite',
# 'llite', 'llite', 'llite', 'llite', 'llite',
# 'llite', 'llite', 'llite', 'llite', 'llite',
# 'llite', 'llite', 'llite', 'llite', 'llite',
# 'llite', 'llite', 'llite', 'llite', 'llite',
# 'llite']
# k2=['open','close','mmap','seek','fsync','setattr',
# 'truncate','flock','getattr','statfs','alloc_inode',
# 'setxattr','getxattr',' listxattr',
# 'removexattr', 'inode_permission', 'readdir',
# 'create','lookup','link','unlink','symlink','mkdir',
# 'rmdir','mknod','rename',]
k1 = [
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
'llite',
]
k2 = [
'open',
'close',
'mmap',
'fsync',
'setattr',
'truncate',
'flock',
'getattr',
'statfs',
'alloc_inode',
'setxattr',
' listxattr',
'removexattr',
'readdir',
'create',
'lookup',
'link',
'unlink',
'symlink',
'mkdir',
'rmdir',
'mknod',
'rename',
]
for file in filelist:
try:
ts = tspl.TSPLSum(file, k1, k2)
except tspl.TSPLException as e:
continue
if not tspl_utils.checkjob(ts, 3600., range(1, 33)):
continue
tmid = (ts.t[:-1] + ts.t[1:]) / 2.0
ld = lariat_utils.LariatData(ts.j.id, ts.j.end_time, 'lariatData')
meta_rate = numpy.zeros_like(tmid)
for k in ts.j.hosts.keys():
meta_rate += numpy.diff(ts.assemble(range(0, len(k1)), k,
0)) / numpy.diff(ts.t)
meta_rate /= float(ts.numhosts)
if numpy.max(meta_rate) > thresh:
title = ts.title
if ld.exc != 'unknown':
title += ', E: ' + ld.exc.split('/')[-1]
fig, ax = plt.subplots(1, 1, figsize=(10, 8), dpi=80)
plt.subplots_adjust(hspace=0.35)
plt.suptitle(title)
markers = ('o', 'x', '+', '^', 's', '8', 'p', 'h', '*', 'D', '<',
'>', 'v', 'd', '.')
colors = ('b', 'g', 'r', 'c', 'm', 'k', 'y')
cnt = 0
for v in ts.data:
for host in v:
for vals in v[host]:
rate = numpy.diff(vals) / numpy.diff(ts.t)
c = colors[cnt % len(colors)]
m = markers[cnt % len(markers)]
# print cnt,(cnt % len(colors)), (cnt % len(markers)), k2[cnt], c, m
ax.plot(tmid / 3600.,
rate,
marker=m,
markeredgecolor=c,
linestyle='-',
color=c,
markerfacecolor='None',
label=k2[cnt])
ax.hold = True
cnt = cnt + 1
ax.set_ylabel('Meta Data Rate (op/s)')
tspl_utils.adjust_yaxis_range(ax, 0.1)
handles, labels = ax.get_legend_handles_labels()
new_handles = {}
for h, l in zip(handles, labels):
new_handles[l] = h
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.9, box.height])
ax.legend(new_handles.values(),
new_handles.keys(),
prop={'size': 8},
bbox_to_anchor=(1.05, 1),
borderaxespad=0.,
loc=2)
fname = '_'.join(['metadata', ts.j.id, ts.owner])
fig.savefig(fname)
plt.close()
def main():
parser=argparse.ArgumentParser(description='Deal with a directory of pickle'
' files nightly')
parser.add_argument('-p', help='Set number of processes',
nargs=1, type=int, default=[1])
parser.add_argument('-o', help='Output directory',
nargs=1, type=str, default=['.'], metavar='output_dir')
parser.add_argument('-u', help='User',
nargs=1, type=str, default=['bbarth'], metavar='username')
parser.add_argument('filearg', help='File, directory, or quoted'
' glob pattern', nargs='?',default='jobs')
n=parser.parse_args()
filelist=tspl_utils.getfilelist(n.filearg)
target_user=n.u[0]
pool = multiprocessing.Pool(processes=n.p[0])
m = multiprocessing.Manager()
files = m.list()
partial_getuser=functools.partial(getuser,user=target_user)
res = pool.map(partial_getuser,filelist)
pool.close()
pool.join()
res = filter(lambda x: x != None, res)
if len(res) == 0:
print 'no jobs found'
return
res_sorted = sorted(res, key=lambda x:x[0])
res2 = zip(*res_sorted)
t=res2[0]
min_dram_rate=res2[1]
max_dram_rate=res2[2]
mean_dram_rate=res2[3]
min_l1_rate=res2[4]
max_l1_rate=res2[5]
mean_l1_rate=res2[6]
min_lnet_rate=res2[7]
max_lnet_rate=res2[8]
mean_lnet_rate=res2[9]
min_ib_rate=res2[10]
max_ib_rate=res2[11]
mean_ib_rate=res2[12]
min_user_rate=res2[13]
max_user_rate=res2[14]
mean_user_rate=res2[15]
min_flops_rate=res2[16]
max_flops_rate=res2[17]
mean_flops_rate=res2[18]
min_mem_usage=res2[19]
max_mem_usage=res2[20]
mean_mem_usage=res2[21]
ids=res2[22]
start_date = datetime.datetime.fromtimestamp(t[0]).strftime('%Y-%m-%d %H:%M:%S')
end_date = datetime.datetime.fromtimestamp(t[-1]).strftime('%Y-%m-%d %H:%M:%S')
fig,ax=plt.subplots(6,1,figsize=(8,12),dpi=80)
plt.subplots_adjust(hspace=0.35)
ax[0].plot(t,min_flops_rate,'x',t,mean_flops_rate,'+',t,max_flops_rate,'*')
ax[0].set_ylabel('GFlops/s')
ax[0].set_xticklabels(labels=[])
ax[1].plot(t,min_dram_rate,'x',t,mean_dram_rate,'+',t,max_dram_rate,'*')
ax[1].set_ylabel('DRAM BW MB/s')
ax[1].set_xticklabels(labels=[])
ax[2].plot(t,min_mem_usage,'x',t,mean_mem_usage,'+',t,max_mem_usage,'*')
ax[2].set_ylabel('DRAM Usage GB')
ax[2].set_xticklabels(labels=[])
ax[3].plot(t,min_lnet_rate,'x',t,mean_lnet_rate,'+',t,max_lnet_rate,'*')
ax[3].set_ylabel('Lnet Rate MB/s')
ax[3].set_xticklabels(labels=[])
ax[4].plot(t,min_ib_rate,'x',t,mean_ib_rate,'+',t,max_ib_rate,'*')
ax[4].set_ylabel('IB - Lnet Rate MB/s')
ax[4].set_xticklabels(labels=[])
ax[5].plot(t,min_user_rate,'x',t,mean_user_rate,'+',t,max_user_rate,'*')
ax[5].set_ylabel('CPU User Fraction')
ax[5].set_xticklabels(labels=[])
for i in range(6):
tspl_utils.adjust_yaxis_range(ax[i],0.1)
ax[5].set_xlabel('t')
plt.suptitle(target_user+' '+start_date+' -- '+end_date)
fname=target_user
fig.savefig(fname)
plt.close()
print 'Found', len(res_sorted), 'jobs for', target_user, ids
def main():
parser = argparse.ArgumentParser(description="Look for imbalance between" "hosts for a pair of keys")
parser.add_argument("filearg", help="File, directory, or quoted" " glob pattern", nargs="?", default="jobs")
n = parser.parse_args()
filelist = tspl_utils.getfilelist(n.filearg)
for file in filelist:
res = get_data(file)
if res is None:
continue
(
ts,
ld,
tmid,
read_rate,
write_rate,
stall_rate,
clock_rate,
avx_rate,
sse_rate,
meta_rate,
l1_rate,
l2_rate,
l3_rate,
load_rate,
read_frac,
stall_frac,
) = res
title = ts.title
if ld.exc != "unknown":
title += ", E: " + ld.exc.split("/")[-1]
fig, ax = plt.subplots(5, 1, figsize=(8, 8), dpi=80)
plt.subplots_adjust(hspace=0.35)
plt.suptitle(title)
ax[0].plot(tmid / 3600.0, read_frac)
ax[0].set_ylabel("DRAM Read Fraction")
# ax[0].set_ylim(getlimits(read_frac))
tspl_utils.adjust_yaxis_range(ax[0], 0.1)
# ax[1].plot(tmid/3600., stall_frac)
# ax[1].set_ylabel('Stall Fraction')
# tspl_utils.adjust_yaxis_range(ax[1],0.1)
ax[1].plot(tmid / 3600.0, avx_rate / 1e9)
ax[1].hold = True
ax[1].plot(tmid / 3600.0, sse_rate / 1e9, "r")
ax[1].set_ylabel("AVX Rate")
tspl_utils.adjust_yaxis_range(ax[1], 0.1)
ax[2].plot(tmid / 3600.0, clock_rate)
ax[2].set_ylabel("Observed Clock Rate")
tspl_utils.adjust_yaxis_range(ax[2], 0.1)
ax[3].plot(tmid / 3600.0, meta_rate)
ax[3].set_ylabel("Meta Data Rate")
tspl_utils.adjust_yaxis_range(ax[3], 0.1)
ax[4].plot(tmid / 3600.0, load_rate - (l1_rate + l2_rate + l3_rate))
ax[4].set_ylabel("Cache Miss Rate?")
tspl_utils.adjust_yaxis_range(ax[3], 0.1)
fname = "_".join(["plot", ts.j.id, ts.owner])
fig.savefig(fname)
plt.close()
def main():
parser = argparse.ArgumentParser(description='Look for high meta data rate'\
' to Lustre')
parser.add_argument('-t', metavar='thresh',
help='Treshold metadata rate',
nargs=1, default=[100000.])
parser.add_argument('filearg', help='File, directory, or quoted'
' glob pattern', nargs='?',default='jobs')
n=parser.parse_args()
thresh=float(n.t[0])
print thresh
filelist=tspl_utils.getfilelist(n.filearg)
# k1=['llite', 'llite', 'llite', 'llite', 'llite',
# 'llite', 'llite', 'llite', 'llite', 'llite',
# 'llite', 'llite', 'llite', 'llite', 'llite',
# 'llite', 'llite', 'llite', 'llite', 'llite',
# 'llite', 'llite', 'llite', 'llite', 'llite',
# 'llite']
# k2=['open','close','mmap','seek','fsync','setattr',
# 'truncate','flock','getattr','statfs','alloc_inode',
# 'setxattr','getxattr',' listxattr',
# 'removexattr', 'inode_permission', 'readdir',
# 'create','lookup','link','unlink','symlink','mkdir',
# 'rmdir','mknod','rename',]
k1=['llite', 'llite', 'llite', 'llite', 'llite',
'llite', 'llite', 'llite', 'llite', 'llite',
'llite', 'llite', 'llite', 'llite', 'llite',
'llite', 'llite', 'llite', 'llite', 'llite',
'llite', 'llite', 'llite', ]
k2=['open','close','mmap','fsync','setattr',
'truncate','flock','getattr','statfs','alloc_inode',
'setxattr',' listxattr',
'removexattr', 'readdir',
'create','lookup','link','unlink','symlink','mkdir',
'rmdir','mknod','rename',]
for file in filelist:
try:
ts=tspl.TSPLSum(file,k1,k2)
except tspl.TSPLException as e:
continue
if not tspl_utils.checkjob(ts,3600.,range(1,33)):
continue
tmid=(ts.t[:-1]+ts.t[1:])/2.0
ld=lariat_utils.LariatData(ts.j.id,ts.j.end_time,'lariatData')
meta_rate = numpy.zeros_like(tmid)
for k in ts.j.hosts.keys():
meta_rate +=numpy.diff(ts.assemble(range(0,len(k1)),k,0))/numpy.diff(ts.t)
meta_rate /= float(ts.numhosts)
if numpy.max(meta_rate) > thresh:
title=ts.title
if ld.exc != 'unknown':
title += ', E: ' + ld.exc.split('/')[-1]
fig,ax=plt.subplots(1,1,figsize=(10,8),dpi=80)
plt.subplots_adjust(hspace=0.35)
plt.suptitle(title)
markers = ('o','x','+','^','s','8','p',
'h','*','D','<','>','v','d','.')
colors = ('b','g','r','c','m','k','y')
cnt=0
for v in ts.data:
for host in v:
for vals in v[host]:
rate=numpy.diff(vals)/numpy.diff(ts.t)
c=colors[cnt % len(colors)]
m=markers[cnt % len(markers)]
# print cnt,(cnt % len(colors)), (cnt % len(markers)), k2[cnt], c, m
ax.plot(tmid/3600., rate, marker=m,
markeredgecolor=c, linestyle='-', color=c,
markerfacecolor='None', label=k2[cnt])
ax.hold=True
cnt=cnt+1
ax.set_ylabel('Meta Data Rate (op/s)')
tspl_utils.adjust_yaxis_range(ax,0.1)
handles,labels=ax.get_legend_handles_labels()
new_handles={}
for h,l in zip(handles,labels):
new_handles[l]=h
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.9, box.height])
ax.legend(new_handles.values(),new_handles.keys(),prop={'size':8},
bbox_to_anchor=(1.05,1), borderaxespad=0., loc=2)
fname='_'.join(['metadata',ts.j.id,ts.owner])
fig.savefig(fname)
plt.close()
