def master_plot(file,mode='lines',threshold=False,
output_dir='.',prefix='graph',mintime=3600,wayness=16,
header='Master',lariat_dict=None,wide=False,job_stats=None):
k1={'amd64' :
['amd64_core','amd64_core','amd64_sock','lnet','lnet',
'ib_sw','ib_sw','cpu'],
'intel' : ['intel_pmc3', 'intel_pmc3', 'intel_pmc3',
'lnet', 'lnet', 'ib_ext','ib_ext','cpu','mem','mem'],
'intel_snb' : ['intel_snb_imc', 'intel_snb_imc', 'intel_snb',
'lnet', 'lnet', 'ib_sw','ib_sw','cpu',
'intel_snb', 'intel_snb', 'mem', 'mem'],
}
k2={'amd64':
['SSE_FLOPS','DCSF','DRAM','rx_bytes','tx_bytes',
'rx_bytes','tx_bytes','user'],
'intel' : ['MEM_LOAD_RETIRED_L1D_HIT', 'FP_COMP_OPS_EXE_X87',
'INSTRUCTIONS_RETIRED', 'rx_bytes','tx_bytes',
'port_recv_data','port_xmit_data','user', 'MemUsed', 'AnonPages'],
'intel_snb' : ['CAS_READS', 'CAS_WRITES', 'LOAD_L1D_ALL',
'rx_bytes','tx_bytes', 'rx_bytes','tx_bytes','user',
'SSE_D_ALL', 'SIMD_D_256', 'MemUsed', 'AnonPages'],
}
try:
print file
ts=tspl.TSPLSum(file,k1,k2,job_stats)
except tspl.TSPLException as e:
return
ignore_qs=[]#'gpu','gpudev','vis','visdev']
if not tspl_utils.checkjob(ts,mintime,wayness,ignore_qs):
return
if lariat_dict == None:
ld=lariat_utils.LariatData(ts.j.id,end_epoch=ts.j.end_time,daysback=3,directory=analyze_conf.lariat_path)
elif lariat_dict == "pass": ld = lariat_utils.LariatData(ts.j.id)
else:
ld=lariat_utils.LariatData(ts.j.id,olddata=lariat_dict)
wayness=ts.wayness
if ld.wayness != -1 and ld.wayness < ts.wayness:
wayness=ld.wayness
if wide:
fig,ax=plt.subplots(6,2,figsize=(15.5,12),dpi=110)
# Make 2-d array into 1-d, and reorder so that the left side is blank
ax=my_utils.flatten(ax)
ax_even=ax[0:12:2]
ax_odd =ax[1:12:2]
ax=ax_odd + ax_even
for a in ax_even:
a.axis('off')
else:
fig,ax=plt.subplots(6,1,figsize=(8,12),dpi=110)
if mode == 'hist':
plot=plot_thist
elif mode == 'percentile':
plot=plot_mmm
else:
plot=plot_lines
if ts.pmc_type == 'intel_snb' :
# Plot key 1
plot(ax[0],ts,[8,9],3600.,1e9,
ylabel='Total AVX +\nSSE Ginst/s')
# Plot key 2
plot(ax[1],ts,[0,1],3600.,1.0/64.0*1024.*1024.*1024.,
ylabel='Total Mem BW GB/s')
#Plot key 3
#plot(ax[2],ts,[2],3600.,1.0/64.0*1e9, ylabel='L1 BW GB/s')
plot(ax[2],ts,[10,-11],3600.,1024.0*1024.0*1024.0, ylabel='Memory Usage GB',
do_rate=False)
elif ts.pmc_type == 'intel':
plot(ax[0],ts,[1],3600.,1e9,ylabel='FP Ginst/s')
plot(ax[2],ts,[8,-9],3600.,1024.0*1024.0*1024.0, ylabel='Memory Usage GB',do_rate=False)
else:
#Fix this to support the old amd plots
print ts.pmc_type + ' not supported'
return
# Plot lnet sum rate
plot(ax[3],ts,[3,4],3600.,1024.**2,ylabel='Total lnet MB/s')
# Plot remaining IB sum rate
if ts.pmc_type == 'intel_snb' :
plot(ax[4],ts,[5,6,-3,-4],3600.,1024.**2,ylabel='Total (ib_sw-lnet) MB/s')
elif ts.pmc_type == 'intel' :
plot(ax[4],ts,[5,6,-3,-4],3600.,1024.**2,ylabel='Total (ib_ext-lnet) MB/s')
#Plot CPU user time
plot(ax[5],ts,[7],3600.,wayness*100.,
xlabel='Time (hr)',
ylabel='Total cpu user\nfraction')
print ts.j.id + ': '
plt.subplots_adjust(hspace=0.35)
if wide:
left_text=header+'\n'+my_utils.summary_text(ld,ts)
text_len=len(left_text.split('\n'))
fontsize=ax[0].yaxis.label.get_size()
linespacing=1.2
fontrate=float(fontsize*linespacing)/72./15.5
yloc=.8-fontrate*(text_len-1) # this doesn't quite work. fontrate is too
# small by a small amount
plt.figtext(.05,yloc,left_text,linespacing=linespacing)
fname='_'.join([prefix,ts.j.id,ts.owner,'wide_master'])
elif header != None:
title=header+'\n'+ts.title
if threshold:
title+=', V: %(v)-6.1f' % {'v': threshold}
title += '\n' + ld.title()
plt.suptitle(title)
fname='_'.join([prefix,ts.j.id,ts.owner,'master'])
else:
fname='_'.join([prefix,ts.j.id,ts.owner,'master'])
if mode == 'hist':
fname+='_hist'
elif mode == 'percentile':
fname+='_perc'
plt.close()
return fig, fname
def test_net(self, writer=None):
total_metrics = [0 for _ in range(len(self.metrics_names))]
total_loss = 0
predictions = None
labels = None
for test_i in range(0, self.test_iter):
batch = self.test_batch_iter.next()
if predictions is None:
if self.mode == 'regression':
predictions = np.zeros(([
0,
] + self.prediction_op.shape.as_list()[1:]))
else: # segmentation
predictions = np.zeros(((0, ) + batch[1].shape[1:]))
if labels is None:
if self.mode == 'regression':
labels = np.zeros(([
0,
] + self.prediction_op.shape.as_list()[1:]))
else: # segmentation
labels = np.zeros(((0, ) + batch[1].shape[1:]))
res = self.sess.run(
self.metrics_ops + [self.loss_op, self.prediction_op],
feed_dict=self.create_feed_dict(batch, phase=0))
for i in range(len(total_metrics)):
total_metrics[i] += res[i]
total_loss += res[len(total_metrics)]
predictions = np.concatenate(
[predictions, res[len(total_metrics) + 1]], axis=0)
if self.mode == 'regression':
labels = np.concatenate(
[labels, [flatten(b) for b in batch[1]]], axis=0)
else:
labels = np.concatenate([labels, batch[1]], axis=0)
for i in range(len(total_metrics)):
total_metrics[i] = total_metrics[i] * 1. / self.test_iter
total_loss = total_loss * 1. / self.test_iter
# add loss and metrics to tensorboard
if self.mode == 'regression':
img = distance_plot_for_tensorboard(labels[:, 0].flatten(),
predictions[:, 0].flatten(),
self.eval_image_shape)
if 'coord' in self.labels:
img_coords = coords_xyz_plot_for_tensorboard(
np.concatenate([labels[:, 1:4], labels[:, 4:7]], axis=0),
np.concatenate([predictions[:, 1:4], predictions[:, 4:7]],
axis=0), self.eval_coords_image_shape)
elif self.mode == 'segmentation':
img = confusion_matrix_for_tensorboard(labels[:, 0].flatten(),
predictions[:, 0].flatten(),
self.labels,
self.eval_image_shape)
feed_dict = {
var: value
for var, value in zip(self.test_metrics_vars, total_metrics)
}
feed_dict.update({
self.test_loss_var: total_loss,
self.eval_image_var: img
})
if 'coord' in self.labels:
feed_dict.update({self.eval_coords_image_var: img_coords})
feed_dict[K.learning_phase()] = 0
if self.show_activation is not False:
# feed activation images four times to deal with multi-gpu (up to 4)
# if net has multiple inputs, need to feed multiple activation images
if not isinstance(self.show_activation, tuple):
images = (self.show_activation, )
else:
images = self.show_activation
for i, img in enumerate(images):
feed_dict.update(
{self.net.inputs[i]: np.concatenate((img, img, img, img))})
summary = self.sess.run(self.test_summary_op, feed_dict=feed_dict)
writer.add_summary(summary, self.global_step_var.eval())
del img
return total_metrics, total_loss
def master_plot(file,mode='lines',threshold=False,
output_dir='.',prefix='graph',mintime=3600,wayness=16,
header='Master'):
k1={'amd64' :
['amd64_core','amd64_core','amd64_sock','lnet','lnet',
'ib_sw','ib_sw','cpu'],
'intel' : ['intel_pmc3', 'intel_pmc3', 'intel_pmc3',
'lnet', 'lnet', 'ib_sw','ib_sw','cpu']
}
k2={'amd64':
['SSE_FLOPS','DCSF','DRAM','rx_bytes','tx_bytes',
'rx_bytes','tx_bytes','user'],
'intel' : ['PMC3', 'PMC2', 'FIXED_CTR0',
'rx_bytes','tx_bytes', 'rx_bytes','tx_bytes','user']
}
try:
print file
ts=tspl.TSPLSum(file,k1,k2)
except tspl.TSPLException as e:
return
if not tspl_utils.checkjob(ts,mintime,wayness):
return
fig,ax=plt.subplots(6,1,figsize=(8,12),dpi=80)
ax=my_utils.flatten(ax)
if mode == 'hist':
plot=plot_thist
elif mode == 'percentile':
plot=plot_mmm
else:
plot=plot_lines
# Plot SSE FLOPS
plot(ax[0],ts,[0],3600.)
# Plot DCSF rate
plot(ax[1],ts,[1],3600.,1e9)
#Plot DRAM rate
plot(ax[2],ts,[2],3600.,1e9)
# Plot lnet sum rate
plot(ax[3],ts,[3,4],3600.,1024.**2,ylabel='Total lnet MB/s')
# Plot remaining IB sum rate
plot(ax[4],ts,[5,6,-3,-4],3600.,1024.**2,ylabel='Total (ib_sw-lnet) MB/s')
#Plot CPU user time
plot(ax[5],ts,[7],3600.,ts.wayness*100.,
xlabel='Time (hr)',
ylabel='Total cpu user\nfraction')
print ts.j.id + ': '
print 'cc'
title=header+'\n'+ts.title
if threshold:
title+=', V: %(v)-8.3f' % {'v': threshold}
ld=lariat_utils.LariatData(ts.j.id,ts.j.end_time,'/scratch/projects/lariatData')
title += '\n' + ld.title()
print 'dd'
plt.suptitle(title)
plt.subplots_adjust(hspace=0.35)
fname='_'.join([prefix,ts.j.id,ts.owner,'master'])
if mode == 'hist':
fname+='_hist'
elif mode == 'percentile':
fname+='_perc'
fig.savefig(output_dir+'/'+fname)
plt.close()
def master_plot(file,
mode='lines',
threshold=False,
output_dir='.',
prefix='graph',
mintime=3600,
wayness=16,
header='Master',
lariat_dict=None,
wide=False,
job_stats=None):
k1 = {
'amd64': [
'amd64_core', 'amd64_core', 'amd64_sock', 'lnet', 'lnet', 'ib_sw',
'ib_sw', 'cpu'
],
'intel': [
'intel_pmc3', 'intel_pmc3', 'intel_pmc3', 'lnet', 'lnet', 'ib_ext',
'ib_ext', 'cpu', 'mem', 'mem'
],
'intel_snb': [
'intel_snb_imc', 'intel_snb_imc', 'intel_snb', 'lnet', 'lnet',
'ib_sw', 'ib_sw', 'cpu', 'intel_snb', 'intel_snb', 'mem', 'mem'
],
}
k2 = {
'amd64': [
'SSE_FLOPS', 'DCSF', 'DRAM', 'rx_bytes', 'tx_bytes', 'rx_bytes',
'tx_bytes', 'user'
],
'intel': [
'MEM_LOAD_RETIRED_L1D_HIT', 'FP_COMP_OPS_EXE_X87',
'INSTRUCTIONS_RETIRED', 'rx_bytes', 'tx_bytes', 'port_recv_data',
'port_xmit_data', 'user', 'MemUsed', 'AnonPages'
],
'intel_snb': [
'CAS_READS', 'CAS_WRITES', 'LOAD_L1D_ALL', 'rx_bytes', 'tx_bytes',
'rx_bytes', 'tx_bytes', 'user', 'SSE_D_ALL', 'SIMD_D_256',
'MemUsed', 'AnonPages'
],
}
try:
print file
ts = tspl.TSPLSum(file, k1, k2, job_stats)
except tspl.TSPLException as e:
return
ignore_qs = [] #'gpu','gpudev','vis','visdev']
if not tspl_utils.checkjob(ts, mintime, wayness, ignore_qs):
return
if lariat_dict == None:
ld = lariat_utils.LariatData(ts.j.id,
end_epoch=ts.j.end_time,
daysback=3,
directory=analyze_conf.lariat_path)
elif lariat_dict == "pass":
ld = lariat_utils.LariatData(ts.j.id)
else:
ld = lariat_utils.LariatData(ts.j.id, olddata=lariat_dict)
wayness = ts.wayness
if ld.wayness != -1 and ld.wayness < ts.wayness:
wayness = ld.wayness
if wide:
fig, ax = plt.subplots(6, 2, figsize=(15.5, 12), dpi=110)
# Make 2-d array into 1-d, and reorder so that the left side is blank
ax = my_utils.flatten(ax)
ax_even = ax[0:12:2]
ax_odd = ax[1:12:2]
ax = ax_odd + ax_even
for a in ax_even:
a.axis('off')
else:
fig, ax = plt.subplots(6, 1, figsize=(8, 12), dpi=110)
if mode == 'hist':
plot = plot_thist
elif mode == 'percentile':
plot = plot_mmm
else:
plot = plot_lines
if ts.pmc_type == 'intel_snb':
# Plot key 1
plot(ax[0], ts, [8, 9], 3600., 1e9, ylabel='Total AVX +\nSSE Ginst/s')
# Plot key 2
plot(ax[1],
ts, [0, 1],
3600.,
1.0 / 64.0 * 1024. * 1024. * 1024.,
ylabel='Total Mem BW GB/s')
#Plot key 3
#plot(ax[2],ts,[2],3600.,1.0/64.0*1e9, ylabel='L1 BW GB/s')
plot(ax[2],
ts, [10, -11],
3600.,
1024.0 * 1024.0 * 1024.0,
ylabel='Memory Usage GB',
do_rate=False)
elif ts.pmc_type == 'intel':
plot(ax[0], ts, [1], 3600., 1e9, ylabel='FP Ginst/s')
plot(ax[2],
ts, [8, -9],
3600.,
1024.0 * 1024.0 * 1024.0,
ylabel='Memory Usage GB',
do_rate=False)
else:
#Fix this to support the old amd plots
print ts.pmc_type + ' not supported'
return
# Plot lnet sum rate
plot(ax[3], ts, [3, 4], 3600., 1024.**2, ylabel='Total lnet MB/s')
# Plot remaining IB sum rate
if ts.pmc_type == 'intel_snb':
plot(ax[4],
ts, [5, 6, -3, -4],
3600.,
1024.**2,
ylabel='Total (ib_sw-lnet) MB/s')
elif ts.pmc_type == 'intel':
plot(ax[4],
ts, [5, 6, -3, -4],
3600.,
1024.**2,
ylabel='Total (ib_ext-lnet) MB/s')
#Plot CPU user time
plot(ax[5],
ts, [7],
3600.,
wayness * 100.,
xlabel='Time (hr)',
ylabel='Total cpu user\nfraction')
print ts.j.id + ': '
plt.subplots_adjust(hspace=0.35)
if wide:
left_text = header + '\n' + my_utils.summary_text(ld, ts)
text_len = len(left_text.split('\n'))
fontsize = ax[0].yaxis.label.get_size()
linespacing = 1.2
fontrate = float(fontsize * linespacing) / 72. / 15.5
yloc = .8 - fontrate * (text_len - 1
) # this doesn't quite work. fontrate is too
# small by a small amount
plt.figtext(.05, yloc, left_text, linespacing=linespacing)
fname = '_'.join([prefix, ts.j.id, ts.owner, 'wide_master'])
elif header != None:
title = header + '\n' + ts.title
if threshold:
title += ', V: %(v)-6.1f' % {'v': threshold}
title += '\n' + ld.title()
plt.suptitle(title)
fname = '_'.join([prefix, ts.j.id, ts.owner, 'master'])
else:
fname = '_'.join([prefix, ts.j.id, ts.owner, 'master'])
if mode == 'hist':
fname += '_hist'
elif mode == 'percentile':
fname += '_perc'
plt.close()
return fig, fname