def plot_animation_each_neuron(name_s, save_name, print_loss=False):
"""Plot the movie for all the networks in the information plane"""
# If we want to print the loss function also
#The bins that we extened the x axis of the accuracy each time
epochs_bins = [0, 500, 1500, 3000, 6000, 10000, 20000]
data_array = utils.get_data(name_s[0][0])
data = np.squeeze(data_array['information'])
f, (axes) = plt.subplots(1, 1)
axes = [axes]
f.subplots_adjust(left=0.14, bottom=0.1, right=.928, top=0.94, wspace=0.13, hspace=0.55)
colors = LAYERS_COLORS
#new/old version
Ix = np.squeeze(data[0,:, :, :])
Iy = np.squeeze(data[1,:, :, :])
#Interploation of the samplings (because we don't cauclaute the infomration in each epoch)
#interp_data_x = interp1d(epochsInds, Ix, axis=1)
#interp_data_y = interp1d(epochsInds, Iy, axis=1)
#new_x = np.arange(0,epochsInds[-1])
#new_data = np.array([interp_data_x(new_x), interp_data_y(new_x)])
""""
train_data = interp1d(epochsInds, np.squeeze(train_data), axis=1)(new_x)
test_data = interp1d(epochsInds, np.squeeze(test_data), axis=1)(new_x)
if print_loss:
loss_train_data = interp1d(epochsInds, np.squeeze(loss_train_data), axis=1)(new_x)
loss_test_data=interp1d(epochsInds, np.squeeze(loss_test_data), axis=1)(new_x)
"""
line_ani = animation.FuncAnimation(f, update_line_each_neuron, Ix.shape[1], repeat=False,
interval=1, blit=False, fargs=(print_loss, Ix, axes,Iy,train_data,test_data,epochs_bins, loss_train_data,loss_test_data, colors,epochsInds))
Writer = animation.writers['ffmpeg']
writer = Writer(fps=100)
#Save the movie
line_ani.save(save_name+'_movie.mp4',writer=writer,dpi=250)
plt.show()
def write_maxMI_from_fig(str_names, mode, save_name):
"""Plot the data in the given names with the given mode"""
args = netp.get_default_parser(None)
[
font_size, axis_font, bar_font, colorbar_axis, sizes, yticks, xticks,
title_strs, f, axes
] = load_figures(mode, str_names)
#Go over all the files
file_name = "Max_MI_%s.txt" % (version) #("VERSION_NOT_IMPLEMENTED")
file = open(file_name, "a")
# file.write("trstepF lambdaF nF trstepR lambdaR nR maxI(T;Y) correspondingI(X;T) [for each layer]")
file.write(
"\n%d %s %d %d %s %d " %
(trstepF, args.lambdaF, args.nF, trstepR, args.lambdaR, args.nR))
for i in range(len(str_names)):
for j in range(len(str_names[i])):
name_s = str_names[i][j]
#Load data for the given file
data_array = utils.get_data(name_s)
data = np.squeeze(np.array(data_array['information']))
I_XT_array = np.array(extract_array(data, 'local_IXT'))
I_TY_array = np.array(extract_array(data, 'local_ITY'))
maxI_TY = np.amax(I_TY_array, 0)
correspI_XT = np.argmax(I_TY_array, 0)
for ii in range(0, I_TY_array.shape[1]):
file = open(file_name, "a")
file.write("%.5f %.5f " %
(maxI_TY[ii], I_XT_array[correspI_XT[ii], ii]))
print("Maximum I(T;Y) saved in %s" % file_name)
file = open(file_name, "a")
file.close
def plot_hist(str_name, save_name='dist'):
data_array = utils.get_data(str_name)
params = np.squeeze(np.array(data_array['information']))
ind_array = data_array['params']['epochsInds']
DKL_YgX_YgT = utils.extract_array(params, 'DKL_YgX_YgT')
p_ts = utils.extract_array(params, 'pts')
H_Xgt = utils.extract_array(params, 'H_Xgt')
f, (axes) = plt.subplots(3, 1)
#axes = [axes]
f.subplots_adjust(left=0.14,
bottom=0.1,
right=.928,
top=0.94,
wspace=0.13,
hspace=0.55)
colors = LAYERS_COLORS
line_ani = animation.FuncAnimation(
f,
update_bars_num_of_ts,
len(p_ts),
repeat=False,
interval=1,
blit=False,
fargs=[p_ts, H_Xgt, DKL_YgX_YgT, axes, ind_array])
Writer = animation.writers['ffmpeg']
writer = Writer(fps=50)
#Save the movie
line_ani.save(save_name + '_movie.mp4', writer=writer, dpi=250)
plt.show()
def plot_pearson(name):
"""Plot the pearsin coeff of the neurons for each layer"""
data_array = utils.get_data(name)
ws = data_array['weights']
f = plt.figure(figsize=(12, 8))
axes = f.add_subplot(111)
#The number of neurons in each layer -
#TODO need to change it to be auto
sizes =[10,7, 5, 4,3,2 ]
#The mean of pearson coeffs of all the layers
pearson_mean =[]
#Go over all the layers
for layer in range(len(sizes)):
inner_pearson_mean =[]
#Go over all the weights in the layer
for k in range(len(ws)):
ws_current = np.squeeze(ws[k][0][0][-1])
#Go over the neurons
for neuron in range(len(ws_current[layer])):
person_t = []
#Go over the rest of the neurons
for neuron_second in range(neuron+1, len(ws_current[layer])):
# pearson correlation between 2 layers
pearson_c, p_val =sis.pearsonr(ws_current[layer][neuron], ws_current[layer][neuron_second])
person_t.append(pearson_c)
inner_pearson_mean.append(np.mean(person_t))
pearson_mean.append(np.mean(inner_pearson_mean))
#Plot the coeff
axes.bar(np.arange(1,7), np.abs(np.array(pearson_mean))*np.sqrt(sizes), align='center')
axes.set_xlabel('Layer')
axes.set_ylabel('Abs(Pearson)*sqrt(N_i)')
rects = axes.patches
# Now make some labels
labels = ["L%d (%d nuerons)" % (i, j) for i, j in zip(range(len(rects)), sizes)]
plt.xticks(np.arange(1,7), labels)
def plot_figures(str_names, mode, save_name):
"""Plot the data in the given names with the given mode"""
[font_size, axis_font, bar_font, colorbar_axis,
sizes, yticks, xticks,title_strs, f, axes] = load_figures(mode, str_names)
#Go over all the files
for i in range(len(str_names)):
for j in range(len(str_names[i])):
name_s = str_names[i][j]
data_array = utils.get_data(name_s)
data = np.squeeze(np.array(data_array['information']))
I_XT_array = np.array(extract_array(data, 'IXT'))
I_TY_array = np.array(extract_array(data, 'ITY'))
snapepochs = data_array['params']['snapepochs']
if mode == 3:
plot_by_training_samples(I_XT_array, I_TY_array, axes,
snapepochs, f, i, j, sizes[i][j],
font_size, yticks, xticks,
colorbar_axis, title_strs[i][j],
axis_font, bar_font, save_name)
elif mode == 6:
plot_norms(axes, snapepochs,data_array['norms1'],
data_array['norms2'])
else:
plot_all_epochs(data_array, I_XT_array, I_TY_array, axes,
snapepochs, f, i, j, sizes[i][j], font_size,
yticks, xticks, colorbar_axis,
title_strs[i][j], axis_font,
bar_font, save_name)
plt.savefig('{}{}.png'.format(save_name, mode))
def plot_snapshots(name_s,
save_name,
i,
time_stemps=[13, 180, 963],
font_size=36,
axis_font=28,
fig_size=(14, 6)):
"""Plot snapshots of the given network"""
f, (axes) = plt.subplots(1,
len(time_stemps),
sharey=True,
figsize=fig_size)
f.subplots_adjust(left=0.095,
bottom=0.18,
right=.99,
top=0.97,
wspace=0.03,
hspace=0.03)
#Adding axes labels
to_do = [[True, True], [True, False], [True, False]]
data_array = utils.get_data(name_s)
data = np.squeeze(data_array['information'])
update_line_specipic_points(time_stemps, data, axes, to_do, font_size,
axis_font)
f.savefig(save_name + '.jpg', dpi=200, format='jpg')
def plot_figures(str_names, mode, save_name):
"""Plot the data in the given names with the given mode"""
[font_size, axis_font, bar_font, colorbar_axis, sizes, yticks, xticks,title_strs, f, axes] = load_figures(mode, str_names)
#Go over all the files
for i in range(len(str_names)):
for j in range(len(str_names[i])):
name_s = str_names[i][j]
#Load data for the given file
data_array = utils.get_data(name_s)
data = np.squeeze(np.array(data_array['information']))
I_XT_array = np.array(extract_array(data, 'local_IXT'))
I_TY_array = np.array(extract_array(data, 'local_ITY'))
#I_XT_array = np.array(extract_array(data, 'IXT_vartional'))
#I_TY_array = np.array(extract_array(data, 'ITY_vartional'))
epochsInds = data_array['params']['epochsInds']
#I_XT_array = np.squeeze(np.array(data))[:, :, 0]
#I_TY_array = np.squeeze(np.array(data))[:, :, 1]
#Plot it
if mode ==3:
plot_by_training_samples(I_XT_array, I_TY_array, axes, epochsInds, f, i, j, sizes[i][j], font_size, yticks, xticks, colorbar_axis, title_strs[i][j], axis_font, bar_font, save_name)
elif mode ==6:
plot_norms(axes, epochsInds,data_array['norms1'],data_array['norms2'])
else:
plot_all_epochs(data_array, I_XT_array, I_TY_array, axes, epochsInds, f, i, j, sizes[i][j], font_size, yticks, xticks,
colorbar_axis, title_strs[i][j], axis_font, bar_font, save_name)
plt.show()
def plot_pearson(name):
"""Plot the pearsin coeff of the neurons for each layer"""
data_array = utils.get_data(name)
ws = data_array['weights']
f = plt.figure(figsize=(12, 8))
axes = f.add_subplot(111)
#The number of neurons in each layer -
#TODO need to change it to be auto
sizes =[10,7, 5, 4,3,2 ]
#The mean of pearson coeffs of all the layers
pearson_mean =[]
#Go over all the layers
for layer in range(len(sizes)):
inner_pearson_mean =[]
#Go over all the weights in the layer
for k in range(len(ws)):
ws_current = np.squeeze(ws[k][0][0][-1])
#Go over the neurons
for neuron in range(len(ws_current[layer])):
person_t = []
#Go over the rest of the neurons
for neuron_second in range(neuron+1, len(ws_current[layer])):
pearson_c, p_val =sis.pearsonr(ws_current[layer][neuron], ws_current[layer][neuron_second])
person_t.append(pearson_c)
inner_pearson_mean.append(np.mean(person_t))
pearson_mean.append(np.mean(inner_pearson_mean))
#Plot the coeff
axes.bar(np.arange(1,7), np.abs(np.array(pearson_mean))*np.sqrt(sizes), align='center')
axes.set_xlabel('Layer')
axes.set_ylabel('Abs(Pearson)*sqrt(N_i)')
rects = axes.patches
# Now make some labels
labels = ["L%d (%d nuerons)" % (i, j) for i, j in zip(range(len(rects)), sizes)]
plt.xticks(np.arange(1,7), labels)
def plot_animation_each_neuron(name_s, save_name, print_loss=False):
"""Plot the movie for all the networks in the information plane"""
# If we want to print the loss function also
#The bins that we extened the x axis of the accuracy each time
epochs_bins = [0, 500, 1500, 3000, 6000, 10000, 20000]
data_array = utils.get_data(name_s[0][0])
data = np.squeeze(data_array['information'])
f, (axes) = plt.subplots(1, 1)
axes = [axes]
f.subplots_adjust(left=0.14, bottom=0.1, right=.928, top=0.94, wspace=0.13, hspace=0.55)
colors = LAYERS_COLORS
#new/old version
Ix = np.squeeze(data[0,:, :, :])
Iy = np.squeeze(data[1,:, :, :])
#Interploation of the samplings (because we don't cauclaute the infomration in each epoch)
#interp_data_x = interp1d(epochsInds, Ix, axis=1)
#interp_data_y = interp1d(epochsInds, Iy, axis=1)
#new_x = np.arange(0,epochsInds[-1])
#new_data = np.array([interp_data_x(new_x), interp_data_y(new_x)])
""""
train_data = interp1d(epochsInds, np.squeeze(train_data), axis=1)(new_x)
test_data = interp1d(epochsInds, np.squeeze(test_data), axis=1)(new_x)
if print_loss:
loss_train_data = interp1d(epochsInds, np.squeeze(loss_train_data), axis=1)(new_x)
loss_test_data=interp1d(epochsInds, np.squeeze(loss_test_data), axis=1)(new_x)
"""
line_ani = animation.FuncAnimation(f, update_line_each_neuron, Ix.shape[1], repeat=False,
interval=1, blit=False, fargs=(print_loss, Ix, axes,Iy,train_data,test_data,epochs_bins, loss_train_data,loss_test_data, colors,epochsInds))
Writer = animation.writers['ffmpeg']
writer = Writer(fps=100)
#Save the movie
line_ani.save(save_name+'_movie.mp4',writer=writer,dpi=250)
plt.show()
def plot_figures(str_names, mode, save_name):
"""Plot the data in the given names with the given mode"""
[font_size, axis_font, bar_font, colorbar_axis, sizes, yticks, xticks,title_strs, f, axes] = load_figures(mode, str_names)
#Go over all the files
for i in range(len(str_names)):
for j in range(len(str_names[i])):
name_s = str_names[i][j]
#Load data for the given file
data_array = utils.get_data(name_s)
# np.squeeze here, already not supporting multiple repeats
data = np.squeeze(np.array(data_array['information']))
repeat = False
if len(data.shape) == 3:
repeat = True
I_XT_array = np.array(extract_array(data, 'local_IXT', repeat))
I_TY_array = np.array(extract_array(data, 'local_ITY', repeat)) # use extract_array
# I_XT_array = np.array(extract_array(data, 'IXT_vartional'))
#I_TY_array = np.array(extract_array(data, 'ITY_vartional'))
epochsInds = data_array['params']['epochsInds']
#I_XT_array = np.squeeze(np.array(data))[:, :, 0]
#I_TY_array = np.squeeze(np.array(data))[:, :, 1]
#Plot it
if mode ==3:
plot_by_training_samples(I_XT_array, I_TY_array, axes, epochsInds, f, i, j, sizes[i][j], font_size, yticks, xticks, colorbar_axis, title_strs[i][j], axis_font, bar_font, save_name)
elif mode == 6:
plot_norms(axes, epochsInds, data_array['l1_norms'],data_array['l2_norms']) # changed here is important
else:
# this is the normal version...
# Note that we turn off plot_error (because it's just 0), possibly info not saved
plot_all_epochs(data_array, I_XT_array, I_TY_array, axes, epochsInds, f, i, j, sizes[i][j], font_size, yticks, xticks,
colorbar_axis, title_strs[i][j], axis_font, bar_font, save_name, plot_error=False)
plt.show()
def plot_snapshots(name_s, save_name, i, time_stemps=[13, 180, 963],font_size = 36,axis_font = 28,fig_size = (14, 6)):
"""Plot snapshots of the given network"""
f, (axes) = plt.subplots(1, len(time_stemps), sharey=True, figsize=fig_size)
f.subplots_adjust(left=0.095, bottom=0.18, right=.99, top=0.97, wspace=0.03, hspace=0.03)
#Adding axes labels
to_do = [[True, True], [True, False], [True, False]]
data_array = utils.get_data(name_s)
data = np.squeeze(data_array['information'])
update_line_specipic_points(time_stemps, data, axes, to_do, font_size, axis_font)
f.savefig(save_name + '.jpg', dpi=200, format='jpg')
def plot_figures(str_names, mode, save_name):
"""Plot the data in the given names with the given mode"""
[
font_size, axis_font, bar_font, colorbar_axis, sizes, yticks, xticks,
title_strs, f, axes
] = load_figures(mode, str_names)
#Go over all the files
for i in range(len(str_names)):
for j in range(len(str_names[i])):
name_s = str_names[i][j]
#Load data for the given file
data_array = utils.get_data(name_s)
data = np.squeeze(np.array(data_array['information']))
I_XT_array = np.array(extract_array(data, 'local_IXT'))
I_TY_array = np.array(extract_array(data, 'local_ITY'))
#I_XT_array = np.array(extract_array(data, 'IXT_vartional'))
#I_TY_array = np.array(extract_array(data, 'ITY_vartional'))
epochsInds = data_array['params']['epochsInds']
#I_XT_array = np.squeeze(np.array(data))[:, :, 0]
#I_TY_array = np.squeeze(np.array(data))[:, :, 1]
#Plot it
if mode == 3:
plot_by_training_samples(I_XT_array, I_TY_array, axes,
epochsInds, f, i, j, sizes[i][j],
font_size, yticks, xticks,
colorbar_axis, title_strs[i][j],
axis_font, bar_font, save_name)
elif mode == 6:
plot_norms(axes, epochsInds, data_array['norms1'],
data_array['norms2'])
else:
#plot_all_epochs(data_array, I_XT_array, I_TY_array, None, epochsInds, None, i, j, None, None, None, None,
# None, None, None, None, save_name, plot_error=False)
pass
plot_all_epochs(data_array,
I_XT_array,
I_TY_array,
axes,
epochsInds,
f,
i,
j,
sizes[i][j],
font_size,
yticks,
xticks,
colorbar_axis,
title_strs[i][j],
axis_font,
bar_font,
save_name,
plot_error=False)
def plot_alphas(str_name, save_name='dist'):
data_array = utils.get_data(str_name)
params = np.squeeze(np.array(data_array['information']))
I_XT_array = np.squeeze(np.array(extract_array(params, 'IXT')))
I_XT_array_var = np.squeeze(np.array(extract_array(params, 'IXT_vartional')))
I_TY_array_var = np.squeeze(np.array(extract_array(params, 'ITY_vartional')))
I_TY_array = np.squeeze(np.array(extract_array(params, 'ITY')))
sigmas = np.linspace(0, 0.3, 20)
for i in range(0,20):
print (i, sigmas[i])
f1, axes1 = plt.subplots(1, 1)
axes1.plot(I_XT_array, I_XT_array_var[:,:,i], linewidth=5)
axes1.plot([0, 15.1], [0, 15.1], transform=axes1.transAxes)
axes1.set_title('Sigmma=' +str(sigmas[i]))
axes1.set_ylim([0,15.1])
axes1.set_xlim([0,15.1])
return
def plot_hist(str_name, save_name='dist'):
data_array = utils.get_data(str_name)
params = np.squeeze(np.array(data_array['information']))
ind_array = data_array['params']['epochsInds']
DKL_YgX_YgT = utils.extract_array(params, 'DKL_YgX_YgT')
p_ts = utils.extract_array(params, 'pts')
H_Xgt = utils.extract_array(params, 'H_Xgt')
f, (axes) = plt.subplots(3, 1)
#axes = [axes]
f.subplots_adjust(left=0.14, bottom=0.1, right=.928, top=0.94, wspace=0.13, hspace=0.55)
colors = LAYERS_COLORS
line_ani = animation.FuncAnimation(f, update_bars_num_of_ts, len(p_ts), repeat=False,
interval=1, blit=False, fargs=[p_ts,H_Xgt,DKL_YgX_YgT, axes,ind_array])
Writer = animation.writers['ffmpeg']
writer = Writer(fps=50)
#Save the movie
line_ani.save(save_name+'_movie.mp4',writer=writer,dpi=250)
plt.show()
def plot_animation_each_neuron(name_s, save_name, print_loss=False):
"""Plot the movie for all the networks in the information plane"""
# If we want to print the loss function also
#The bins that we extened the x axis of the accuracy each time
epochs_bins = [0, 500, 1500, 3000, 6000, 10000, 20000]
data_array = utils.get_data(name_s[0][0])
data = np.squeeze(data_array['information'])
f, (axes) = plt.subplots(1, 1)
axes = [axes]
f.subplots_adjust(left=0.14, bottom=0.1, right=.928, top=0.94, wspace=0.13, hspace=0.55)
colors = LAYERS_COLORS
#new/old version
Ix = np.squeeze(data[0,:, :, :])
Iy = np.squeeze(data[1,:, :, :])
line_ani = animation.FuncAnimation(f, update_line_each_neuron, Ix.shape[1], repeat=False,
interval=1, blit=False, fargs=(print_loss, Ix, axes,Iy,train_data,test_data,epochs_bins, loss_train_data,loss_test_data, colors,snapepochs))
Writer = animation.writers['ffmpeg']
writer = Writer(fps=100)
#Save the movie
line_ani.save(save_name+'_movie.mp4',writer=writer,dpi=250)
def save_plot_data(str_names, mode, save_name):
"""Save the data in the given names with the given mode"""
args = netp.get_default_parser(None)
[
font_size, axis_font, bar_font, colorbar_axis, sizes, yticks, xticks,
title_strs, f, axes
] = load_figures(mode, str_names)
#Go over all the files
for i in range(len(str_names)):
for j in range(len(str_names[i])):
name_s = str_names[i][j]
#Load data for the given file
data_array = utils.get_data(name_s)
data = np.squeeze(np.array(data_array['information']))
I_XT_array = np.array(extract_array(data, 'local_IXT'))
I_TY_array = np.array(extract_array(data, 'local_ITY'))
maxI_TY = np.amax(I_TY_array, 0)
correspI_XT = np.argmax(I_TY_array, 0)
text_x = 'I_XT_%s.out' % version
text_y = 'I_YT_%s.out' % version
np.savetxt(text_x, I_XT_array, delimiter=' ')
np.savetxt(text_y, I_TY_array, delimiter=' ')
def plot_animation(name_s, save_name):
"""Plot the movie for all the networks in the information plane"""
# If we want to print the loss function also
print_loss = False
#The bins that we extened the x axis of the accuracy each time
epochs_bins = [0, 500, 1500, 3000, 6000, 10000, 20000]
data_array = utils.get_data(name_s[0][0])
data = data_array['infomration']
snapepochs = data_array['snapepochs']
loss_train_data = data_array['loss_train']
loss_test_data = data_array['loss_test_data']
f, (axes) = plt.subplots(2, 1)
f.subplots_adjust(left=0.14, bottom=0.1, right=.928, top=0.94, wspace=0.13, hspace=0.55)
colors = LAYERS_COLORS
#new/old version
if False:
Ix = np.squeeze(data[0,:,-1,-1, :, :])
Iy = np.squeeze(data[1,:,-1,-1, :, :])
else:
Ix = np.squeeze(data[0, :, -1, -1, :, :])[np.newaxis,:,:]
Iy = np.squeeze(data[1, :, -1, -1, :, :])[np.newaxis,:,:]
#Interploation of the samplings (because we don't cauclaute the infomration in each epoch)
interp_data_x = interp1d(snapepochs, Ix, axis=1)
interp_data_y = interp1d(snapepochs, Iy, axis=1)
new_x = np.arange(0,snapepochs[-1])
new_data = np.array([interp_data_x(new_x), interp_data_y(new_x)])
if print_loss:
loss_train_data = interp1d(snapepochs, np.squeeze(loss_train_data), axis=1)(new_x)
loss_test_data=interp1d(snapepochs, np.squeeze(loss_test_data), axis=1)(new_x)
line_ani = animation.FuncAnimation(f, update_line, len(new_x), repeat=False,
interval=1, blit=False, fargs=(print_loss, new_data, axes,new_x,train_data,test_data,epochs_bins, loss_train_data,loss_test_data, colors))
Writer = animation.writers['ffmpeg']
writer = Writer(fps=100)
#Save the movie
line_ani.save(save_name+'_movie2.mp4',writer=writer,dpi=250)
def plot_alphas(str_name, save_name='dist'):
data_array = utils.get_data(str_name)
params = np.squeeze(np.array(data_array['information']))
I_XT_array = np.squeeze(np.array(extract_array(params, 'local_IXT')))
""""
for i in range(I_XT_array.shape[2]):
f1, axes1 = plt.subplots(1, 1)
axes1.plot(I_XT_array[:,:,i])
plt.show()
return
"""
I_XT_array_var = np.squeeze(np.array(extract_array(params, 'IXT_vartional')))
I_TY_array_var = np.squeeze(np.array(extract_array(params, 'ITY_vartional')))
I_TY_array = np.squeeze(np.array(extract_array(params, 'local_ITY')))
"""
f1, axes1 = plt.subplots(1, 1)
#axes1.plot(I_XT_array,I_TY_array)
f1, axes2 = plt.subplots(1, 1)
axes1.plot(I_XT_array ,I_TY_array_var)
axes2.plot(I_XT_array ,I_TY_array)
f1, axes1 = plt.subplots(1, 1)
axes1.plot(I_TY_array, I_TY_array_var)
axes1.plot([0, 1.1], [0, 1.1], transform=axes1.transAxes)
#axes1.set_title('Sigmma=' + str(sigmas[i]))
axes1.set_ylim([0, 1.1])
axes1.set_xlim([0, 1.1])
plt.show()
return
"""
#for i in range()
sigmas = np.linspace(0, 0.3, 20)
for i in range(0,20):
print (i, sigmas[i])
f1, axes1 = plt.subplots(1, 1)
axes1.plot(I_XT_array, I_XT_array_var[:,:,i], linewidth=5)
axes1.plot([0, 15.1], [0, 15.1], transform=axes1.transAxes)
axes1.set_title('Sigmma=' +str(sigmas[i]))
axes1.set_ylim([0,15.1])
axes1.set_xlim([0,15.1])
plt.show()
return
epochs_s = data_array['params']['epochsInds']
f, axes = plt.subplots(1, 1)
#epochs_s = []
colors = LAYERS_COLORS
linestyles = [ '--', '-.', '-','', ' ',':', '']
epochs_s =[0, -1]
for j in epochs_s:
for i in range(0, I_XT_array.shape[1]):
axes.plot(sigmas, I_XT_array_var[j,i,:],color = colors[i], linestyle = linestyles[j], label='Layer-'+str(i) +' Epoch - ' +str(epochs_s[j]))
title_str = 'I(X;T) for different layers as function of $\sigma$ (The width of the gaussian)'
x_label = '$\sigma$'
y_label = '$I(X;T)$'
x_lim = [0, 3]
utils.adjustAxes(axes, axis_font=20, title_str=title_str, x_ticks=[], y_ticks=[], x_lim=x_lim, y_lim=None,
set_xlabel=True, set_ylabel=True, x_label=x_label, y_label=y_label, set_xlim=True, set_ylim=False,
set_ticks=False,
label_size=20, set_yscale=False,
set_xscale=False, yscale=None, xscale=None, ytick_labels='', genreal_scaling=False)
axes.legend()
plt.show()
def plot_alphas(str_name, save_name='dist'):
data_array = utils.get_data(str_name)
params = np.squeeze(np.array(data_array['information']))
I_XT_array = np.squeeze(np.array(extract_array(params, 'local_IXT')))
""""
for i in range(I_XT_array.shape[2]):
f1, axes1 = plt.subplots(1, 1)
axes1.plot(I_XT_array[:,:,i])
plt.show()
return
"""
I_XT_array_var = np.squeeze(np.array(extract_array(params, 'IXT_vartional')))
I_TY_array_var = np.squeeze(np.array(extract_array(params, 'ITY_vartional')))
I_TY_array = np.squeeze(np.array(extract_array(params, 'local_ITY')))
"""
f1, axes1 = plt.subplots(1, 1)
#axes1.plot(I_XT_array,I_TY_array)
f1, axes2 = plt.subplots(1, 1)
axes1.plot(I_XT_array ,I_TY_array_var)
axes2.plot(I_XT_array ,I_TY_array)
f1, axes1 = plt.subplots(1, 1)
axes1.plot(I_TY_array, I_TY_array_var)
axes1.plot([0, 1.1], [0, 1.1], transform=axes1.transAxes)
#axes1.set_title('Sigmma=' + str(sigmas[i]))
axes1.set_ylim([0, 1.1])
axes1.set_xlim([0, 1.1])
plt.show()
return
"""
#for i in range()
sigmas = np.linspace(0, 0.3, 20)
for i in range(0,20):
print (i, sigmas[i])
f1, axes1 = plt.subplots(1, 1)
axes1.plot(I_XT_array, I_XT_array_var[:,:,i], linewidth=5)
axes1.plot([0, 15.1], [0, 15.1], transform=axes1.transAxes)
axes1.set_title('Sigmma=' +str(sigmas[i]))
axes1.set_ylim([0,15.1])
axes1.set_xlim([0,15.1])
plt.show()
return
epochs_s = data_array['params']['epochsInds']
f, axes = plt.subplots(1, 1)
#epochs_s = []
colors = LAYERS_COLORS
linestyles = [ '--', '-.', '-','', ' ',':', '']
epochs_s =[0, -1]
for j in epochs_s:
for i in range(0, I_XT_array.shape[1]):
axes.plot(sigmas, I_XT_array_var[j,i,:],color = colors[i], linestyle = linestyles[j], label='Layer-'+str(i) +' Epoch - ' +str(epochs_s[j]))
title_str = 'I(X;T) for different layers as function of $\sigma$ (The width of the gaussian)'
x_label = '$\sigma$'
y_label = '$I(X;T)$'
x_lim = [0, 3]
utils.adjustAxes(axes, axis_font=20, title_str=title_str, x_ticks=[], y_ticks=[], x_lim=x_lim, y_lim=None,
set_xlabel=True, set_ylabel=True, x_label=x_label, y_label=y_label, set_xlim=True, set_ylim=False,
set_ticks=False,
label_size=20, set_yscale=False,
set_xscale=False, yscale=None, xscale=None, ytick_labels='', genreal_scaling=False)
axes.legend()
plt.show()
def plot_gradients(name_s=None, data_array=None, figures_dir=''):
"""Plot the gradients and the means of the networks over the batches"""
if data_array == None:
data_array = plt_ut.get_data(name_s[0][0])
#plot_loss_figures(data_array, xlim = [0, 7000] )
#The gradients - the diemnstions are #epochs X #Batchs # Layers
conv_net = False
if conv_net:
gradients = data_array['var_grad_val'][0][0][0]
num_of_epochs = len(gradients)
num_of_batchs = len(gradients[0])
num_of_layers = len(gradients[0][0]) / 2
else:
gradients = np.squeeze(data_array['var_grad_val'])[:, :, :]
num_of_epochs, num_of_batchs, num_of_layers = gradients.shape
num_of_layers = int(num_of_layers / 2)
#The indxes where we sampled the network
print(np.squeeze(data_array['var_grad_val'])[0, 0].shape)
snapepochs = (data_array['params']['snapepochs']).astype(np.int)
#The norms of the layers
#l2_norm = calc_weights_norms(data_array['ws_all'])
f_log, axes_log, f_norms, axes_norms, f_snr, axes_snr, axes_gaus, f_gaus = create_figs(
)
p_1, p_0, sum_y, p_3, p_4 = [], [], [], [], []
# Go over the layers
cov_traces_all, means_all = [], []
all_gradients = np.empty(num_of_layers, dtype=np.object)
#print np.squeeze(data_array['var_grad_val']).shape
for layer in range(0, num_of_layers):
# The traces of the covarince and the means of the gradients for the current layer
# Go over all the epochs
cov_traces, means = [], []
gradinets_layer = []
for epoch_index in range(num_of_epochs):
# the gradients are dimensions of #batchs X # output weights - when output weights is the number of wieghts that go out from the layer
gradients_current_epoch_and_layer = flatted_graidnet(
gradients, epoch_index, 2 * layer)
gradinets_layer.append(gradients_current_epoch_and_layer)
num_of_output_weights = gradients_current_epoch_and_layer.shape[1]
# the average vector over the batchs - this is vector in the size of #output weights
# We averged over the batchs - It's mean vector of the batchs!
average_vec = np.mean(gradients_current_epoch_and_layer, axis=0)
# The sqrt of the sum over all the weights of the squares of the gradinets - Sqrt of AA^T - This is a number
gradients_mean = LA.norm(average_vec)
# The covarince matrix is in the size of #output weights X #output weights
sum_covs_mat = np.zeros(
(average_vec.shape[0], average_vec.shape[0]))
# Go over all the vectors of batchs (each vector is the size of # output weights, reduce the mean (over the batchs)
# and calculate the covariance matrix
for batch_index in range(num_of_batchs):
# This is in the size of the #output weights
current_vec = gradients_current_epoch_and_layer[
batch_index, :] - average_vec
# The outer product of the current gradinet of the weights (in this specipic batch) with the transpose of it -
# give a matrix in the size of # output weights X # output weights
current_cov_mat = np.einsum('i,j', current_vec, current_vec)
#current_cov_mat = np.dot(current_vec[:,None], current_vec[None,:])
# Sum the covarince matrixes over the batchs
sum_covs_mat += current_cov_mat
#Take the mean of the cov matrix over the batchs - The size is #output weights X # output weights
mean_cov_mat = sum_covs_mat / num_of_batchs
#The trace of the mean of the cov matrix - a number
trac_cov = np.sqrt(np.trace(mean_cov_mat))
means.append(gradients_mean)
cov_traces.append(trac_cov)
"""
#cov_traces.append(np.mean(grad_norms))
#means.append(norm_mean)
c_var,c_mean,total_w = [], [],[]
for neuron in range(len(grad[epoch_number][0][layer])/10):
gradients_list = np.array([grad[epoch_number][i][layer][neuron] for i in range(len(grad[epoch_number]))])
total_w.extend(gradients_list.T)
grad_norms1 = np.std(gradients_list, axis=0)
mean_la = np.abs(np.mean(np.array(gradients_list), axis=0))
#mean_la = LA.norm(gradients_list, axis=0)
c_var.append(np.mean(grad_norms1))
c_mean.append(np.mean(mean_la))
#total_w is in size [num_of_total_weights, num of epochs]
total_w = np.array(total_w)
#c_var.append(np.sqrt(np.trace(np.cov(np.array(total_w).T)))/np.cov(np.array(total_w).T).shape[0])
#print np.mean(c_mean).shape
means.append(np.mean(c_mean))
cov_traces.append(np.mean(c_var))
"""
gradinets_layer = np.array(gradinets_layer)
all_gradients[layer] = gradinets_layer
cov_traces_all.append(np.array(cov_traces))
means_all.append(np.array(means))
#The cov_traces and the means are vectors with the dimension of # epochs
#y_var = np.array(cov_traces)
#y_mean = np.array(means)
y_var = np.sum(cov_traces_all, axis=0)
y_mean = np.sum(means_all, axis=0)
snr = y_mean**2 / y_var
#Plot the gradients and the means
c_p1, = axes_log.plot(snapepochs[:],
np.sqrt(y_var),
markersize=4,
linewidth=4,
color=colors[layer],
linestyle=':',
markeredgewidth=0.2,
dashes=[4, 4])
c_p0, = axes_log.plot(snapepochs[:],
y_mean,
linewidth=2,
color=colors[layer])
c_p3, = axes_snr.plot(snapepochs[:],
snr,
linewidth=2,
color=colors[layer])
c_p4, = axes_gaus.plot(snapepochs[:],
np.log(1 + snr),
linewidth=2,
color=colors[layer])
#For the legend
p_0.append(c_p0), p_1.append(c_p1), sum_y.append(y_mean), p_3.append(
c_p3), p_4.append(c_p4)
plt_ut.adjust_axes(axes_log,
axes_norms,
p_0,
p_1,
f_log,
f_norms,
axes_snr,
f_snr,
p_3,
axes_gaus,
f_gaus,
p_4,
directory_name=figures_dir)
plt.show()
def plot_gradients(name_s=None, data_array=None, figures_dir=''):
"""Plot the gradients and the means of the networks over the batches"""
if data_array == None:
data_array= plt_ut.get_data(name_s[0][0])
#plot_loss_figures(data_array, xlim = [0, 7000] )
#The gradients - the diemnstions are #epochs X #Batchs # Layers
conv_net = False
if conv_net:
gradients =data_array['var_grad_val'][0][0][0]
num_of_epochs = len(gradients)
num_of_batchs = len(gradients[0])
num_of_layers = len(gradients[0][0]) / 2
else:
gradients = np.squeeze(data_array['var_grad_val'])[:, :, :]
num_of_epochs,num_of_batchs, num_of_layers = gradients.shape
num_of_layers = int(num_of_layers / 2)
#The indxes where we sampled the network
print (np.squeeze(data_array['var_grad_val'])[0,0].shape)
epochsInds = (data_array['params']['epochsInds']).astype(np.int)
#The norms of the layers
#l2_norm = calc_weights_norms(data_array['ws_all'])
f_log, axes_log, f_norms, axes_norms, f_snr, axes_snr,axes_gaus, f_gaus = create_figs()
p_1, p_0, sum_y ,p_3, p_4= [], [], [], [], []
# Go over the layers
cov_traces_all,means_all = [],[]
all_gradients = np.empty(num_of_layers, dtype=np.object)
#print np.squeeze(data_array['var_grad_val']).shape
for layer in range(0,num_of_layers):
# The traces of the covarince and the means of the gradients for the current layer
# Go over all the epochs
cov_traces, means = [], []
gradinets_layer = []
for epoch_index in range(num_of_epochs):
# the gradients are dimensions of #batchs X # output weights - when output weights is the number of wieghts that go out from the layer
gradients_current_epoch_and_layer = flatted_graidnet(gradients, epoch_index, 2 * layer)
gradinets_layer.append(gradients_current_epoch_and_layer)
num_of_output_weights = gradients_current_epoch_and_layer.shape[1]
# the average vector over the batchs - this is vector in the size of #output weights
# We averged over the batchs - It's mean vector of the batchs!
average_vec = np.mean(gradients_current_epoch_and_layer, axis=0)
# The sqrt of the sum over all the weights of the squares of the gradinets - Sqrt of AA^T - This is a number
gradients_mean = LA.norm(average_vec)
# The covarince matrix is in the size of #output weights X #output weights
sum_covs_mat = np.zeros((average_vec.shape[0], average_vec.shape[0]))
# Go over all the vectors of batchs (each vector is the size of # output weights, reduce the mean (over the batchs)
# and calculate the covariance matrix
for batch_index in range(num_of_batchs):
# This is in the size of the #output weights
current_vec = gradients_current_epoch_and_layer[batch_index, :] - average_vec
# The outer product of the current gradinet of the weights (in this specipic batch) with the transpose of it -
# give a matrix in the size of # output weights X # output weights
current_cov_mat = np.einsum('i,j', current_vec, current_vec)
#current_cov_mat = np.dot(current_vec[:,None], current_vec[None,:])
# Sum the covarince matrixes over the batchs
sum_covs_mat+=current_cov_mat
#Take the mean of the cov matrix over the batchs - The size is #output weights X # output weights
mean_cov_mat = sum_covs_mat / num_of_batchs
#The trace of the mean of the cov matrix - a number
trac_cov = np.sqrt(np.trace(mean_cov_mat))
means.append(gradients_mean)
cov_traces.append(trac_cov)
"""
#cov_traces.append(np.mean(grad_norms))
#means.append(norm_mean)
c_var,c_mean,total_w = [], [],[]
for neuron in range(len(grad[epoch_number][0][layer])/10):
gradients_list = np.array([grad[epoch_number][i][layer][neuron] for i in range(len(grad[epoch_number]))])
total_w.extend(gradients_list.T)
grad_norms1 = np.std(gradients_list, axis=0)
mean_la = np.abs(np.mean(np.array(gradients_list), axis=0))
#mean_la = LA.norm(gradients_list, axis=0)
c_var.append(np.mean(grad_norms1))
c_mean.append(np.mean(mean_la))
#total_w is in size [num_of_total_weights, num of epochs]
total_w = np.array(total_w)
#c_var.append(np.sqrt(np.trace(np.cov(np.array(total_w).T)))/np.cov(np.array(total_w).T).shape[0])
#print np.mean(c_mean).shape
means.append(np.mean(c_mean))
cov_traces.append(np.mean(c_var))
"""
gradinets_layer = np.array(gradinets_layer)
all_gradients[layer]= gradinets_layer
cov_traces_all.append(np.array(cov_traces))
means_all.append(np.array(means))
#The cov_traces and the means are vectors with the dimension of # epochs
#y_var = np.array(cov_traces)
#y_mean = np.array(means)
y_var = np.sum(cov_traces_all, axis=0)
y_mean = np.sum(means_all, axis=0)
snr = y_mean**2 / y_var
#Plot the gradients and the means
c_p1, = axes_log.plot(epochsInds[:], np.sqrt(y_var),markersize = 4, linewidth = 4,color = colors[layer], linestyle=':', markeredgewidth=0.2, dashes = [4,4])
c_p0,= axes_log.plot(epochsInds[:], y_mean, linewidth = 2,color = colors[layer])
c_p3,= axes_snr.plot(epochsInds[:],snr, linewidth = 2,color = colors[layer])
c_p4,= axes_gaus.plot(epochsInds[:],np.log(1+snr), linewidth = 2,color = colors[layer])
#For the legend
p_0.append(c_p0), p_1.append(c_p1),sum_y.append(y_mean) , p_3.append(c_p3), p_4.append(c_p4)
plt_ut.adjust_axes(axes_log, axes_norms, p_0, p_1, f_log, f_norms, axes_snr, f_snr, p_3, axes_gaus, f_gaus, p_4,
directory_name=figures_dir)
plt.show()