def all_model_analysis(folder):
files_list = os.walk(folder)
analyser = BaseAnalyser()
#results_dict = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
#print(files_list)
for stuff in files_list:
print(stuff)
if '/' in stuff[0]:
name = stuff[0]
model = name + '/model.h5'
loaded_model = model_load(model)
X, tensor_X = dataset_generator(-1, 3, -1, 3, 0.25)
output = analyser.get_output(tensor_X, loaded_model, -2)
z1_arr = [z_value[0] for z_value in output]
z2_arr = [z_value[1] for z_value in output]
plt.plot(z1_arr, label='z1')
plt.plot(z2_arr, label='z2')
plt.legend()
plt.xticks([])
plt.title("Z1 and Z2 values Square from -1 to 3")
plt.savefig(name + '/z1_z2_experiment.png')
plt.clf()
def square_info_two_scores(name):
model = name + '/model.h5'
loaded_model = model_load(model)
analyser = BaseAnalyser()
X, tensor_X = dataset_generator(-2.5, 3.76, -2.5, 3.76, 0.01)
output = analyser.get_output(tensor_X, loaded_model, -2)
z1_arr = [z_value[0] for z_value in output]
z2_arr = [z_value[1] for z_value in output]
plt.plot(z1_arr, color='red', label='Class 1')
plt.plot(z2_arr, color='green', label='Class 2')
plt.legend()
plt.xlabel('x')
plt.ylabel('Class Scores')
x_ticks = [[-2.5, 3.75], [3.75, 3.75], [3.75, -2.5], [-2.5, -2.5],
[-2.5, 3.75]]
x_tick_pos = [X.index(tick) for tick in x_ticks]
x_tick_pos[-1] = X[1:].index([-2.5, 3.75])
plt.xticks(x_tick_pos, [str((x[0], x[1])) for x in x_ticks])
plt.savefig(name + '/z_values_square_experiment.png')
plt.clf()
def softmax_problems_small_probs(name):
model = name + '/model.h5'
loaded_model = model_load(model)
analyser = BaseAnalyser()
X = []
x_coords = []
# (0.5, 3.25, 0.25)
for x_coord in np.arange(0.5, 3.76, 0.01):
x_coord = round(x_coord, 2)
X.append([x_coord, 1])
x_coords.append(x_coord)
tensor_X = tf.convert_to_tensor(X)
output = analyser.get_output(tensor_X, loaded_model, -1)
z1_arr = [z_value[0] for z_value in output]
z2_arr = [z_value[1] for z_value in output]
x_ticks, x_ticks_pos = get_ticks(X, 5)
plt.plot(z1_arr, color='red', label='Class 1')
plt.plot(z2_arr, color='green', label='Class 2')
plt.legend()
#plt.xticks([0, 3, 7, 11], ['0.5', '1.25', '2.25', '3.25'])
plt.xticks(x_ticks_pos, x_ticks)
#plt.xticks([0, 3, 7, 11, 14], ['1.0', '1.75', '2.75', '3.75', '4.75'])
plt.xlabel('x')
plt.ylabel('Class Probabilities')
#plt.show()
plt.savefig(name + '/probabilities_experiment_outward.png')
plt.clf()
def softmax_layer_play(model_folder, pos=True):
loaded_model = model_load(model_folder + '/model.h5')
analyser = BaseAnalyser()
moons = MoonsDataset()
if pos:
fixed_b = [1, 1]
else:
fixed_b = [-1, -1]
X, y, X_test, y_test = moons.four_set_two_moons()
#print(X)
if pos:
output_folder = model_folder + '/softmax_modding_pos'
else:
output_folder = model_folder + '/softmax_modding_minus'
folder_creater(output_folder)
c_values = []
for c_1 in range(1, 11, 2):
for c_2 in range(1, 11, 2):
c_values.append([c_1, c_2])
#print(model(X))
#print(radial_softmax(model(X), [5, 5], [1, 1]))
for c in c_values:
softmax_play_plot(loaded_model, -2.0, 3.0, True, output_folder,
'c={}_b={}'.format(c, fixed_b), X, y, c, fixed_b)
softmax_play_plot(loaded_model, 0.0, 2.0, True, output_folder,
'c={}_b={}'.format(c, fixed_b), X, y, c, fixed_b)
def temperature_scaling(model_folder):
loaded_model = model_load(model_folder + '/model.h5')
analyser = BaseAnalyser()
moons = MoonsDataset()
fixed_b = [1, 1]
X, y, X_test, y_test = moons.four_set_two_moons()
# print(X)
output_folder = model_folder + '/temperature_scaling'
folder_creater(output_folder)
#print(loaded_model.layers[-1].weights)
c = loaded_model.layers[-1].get_weights()[0]
b = loaded_model.layers[-1].get_weights()[1]
#print(loaded_model.layers[-1].get_weights())
for temp in np.arange(0.25, 2.25, 0.25):
softmax_play_plot(loaded_model,
-2.0,
3.0,
True,
output_folder,
'c={}_b={}_temp={}'.format(c, b, temp),
X,
y,
c,
b,
temp=temp)
softmax_play_plot(loaded_model,
0.0,
2.0,
True,
output_folder,
'c={}_b={}_temp={}'.format(c, b, temp),
X,
y,
c,
b,
temp=temp)
softmax_play_plot(loaded_model,
-5.0,
6.0,
True,
output_folder,
'c={}_b={}_temp={}'.format(c, b, temp),
X,
y,
c,
b,
temp=temp)
def softmax_problems_eigth_scores(name):
model = name + '/model.h5'
loaded_model = model_load(model)
analyser = BaseAnalyser()
X = []
x_coords = []
for x_coord in np.arange(1, 4.01, 0.01):
x_coord = round(x_coord, 2)
X.append([x_coord, 1])
x_coords.append(x_coord)
tensor_X = tf.convert_to_tensor(X)
output = analyser.get_output(tensor_X, loaded_model, -2)
z1_arr = [z_value[0] for z_value in output]
z2_arr = [z_value[1] for z_value in output]
z3_arr = [z_value[2] for z_value in output]
z4_arr = [z_value[3] for z_value in output]
z5_arr = [z_value[4] for z_value in output]
z6_arr = [z_value[5] for z_value in output]
z7_arr = [z_value[6] for z_value in output]
z8_arr = [z_value[7] for z_value in output]
x_ticks, x_ticks_pos = get_ticks(X, 5)
plt.plot(z1_arr, color='red', label='Class 1')
plt.plot(z2_arr, color='green', label='Class 2')
plt.plot(z3_arr, color='black', label='Class 3')
plt.plot(z4_arr, color='gray', label='Class 4')
plt.plot(z5_arr, color='m', label='Class 5')
plt.plot(z6_arr, color='darksalmon', label='Class 6')
plt.plot(z7_arr, color='tan', label='Class 7')
plt.plot(z8_arr, color='olivedrab', label='Class 8')
plt.legend()
#plt.xticks([0, 3, 7, 11, 14], ['1.0', '1.75', '2.75', '3.75', '4.75'])
plt.xticks(x_ticks_pos, x_ticks)
plt.xlabel('x')
plt.ylabel('Class Scores')
#plt.show()
plt.savefig(name + '/z_values_experiment_outward.png')
plt.clf()
def square_info_eight_probs(name, classes=8):
model = name + '/model.h5'
loaded_model = model_load(model)
analyser = BaseAnalyser()
X, tensor_X = dataset_generator(-2, 4.01, -2, 4.01, 0.01)
output = analyser.get_output(tensor_X, loaded_model, -1)
z1_arr = [z_value[0] for z_value in output]
z2_arr = [z_value[1] for z_value in output]
if classes == 8:
z3_arr = [z_value[2] for z_value in output]
z4_arr = [z_value[3] for z_value in output]
z5_arr = [z_value[4] for z_value in output]
z6_arr = [z_value[5] for z_value in output]
z7_arr = [z_value[6] for z_value in output]
z8_arr = [z_value[7] for z_value in output]
plt.plot(z1_arr, color='red', label='Class 1')
plt.plot(z2_arr, color='green', label='Class 2')
if classes == 8:
plt.plot(z3_arr, color='black', label='Class 3')
plt.plot(z4_arr, color='gray', label='Class 4')
plt.plot(z5_arr, color='m', label='Class 5')
plt.plot(z6_arr, color='darksalmon', label='Class 6')
plt.plot(z7_arr, color='tan', label='Class 7')
plt.plot(z8_arr, color='olivedrab', label='Class 8')
plt.legend()
x_ticks = [[-2.0, 4.0], [4.0, 4.0], [4.0, -2.0], [-2.0, -2.0], [-2.0, 4.0]]
x_tick_pos = [X.index(tick) for tick in x_ticks]
x_tick_pos[-1] = X[1:].index([-2.0, 4.0])
plt.xticks(x_tick_pos, [str((x[0], x[1])) for x in x_ticks])
#plt.xticks([0, 25, 50, 75, 99], ['(-2.0, 4.0)', '(4.0, 4.0)', '(4.0, -2.0)', '(-2.0, -2.0)', '(-2.0, 4.0)'])
plt.xlabel('Coordinates')
plt.ylabel('Class Probabilities')
plt.savefig(name + '/probabilities_square_experiment.png')
plt.clf()
def softmax_play_plot(model,
plot_min,
plot_max,
max_prob,
file_name,
layers,
X,
y,
c,
b,
classes=2,
temp=None):
"""
Based on Hein, M. et al. :https://github.com/max-andr/relu_networks_overconfident/blob/master/analysis.py
"""
n_grid = 200
x_plot = np.linspace(plot_min, plot_max, n_grid)
y_plot = np.linspace(plot_min, plot_max, n_grid)
points = []
for xx in x_plot:
for yy in y_plot:
points.append((yy, xx))
points = np.array(points)
analyser = BaseAnalyser()
probs = analyser.get_output(points, model, -2)
x_probs = analyser.get_output(X, model, -2)
if temp is None:
probs = radial_softmax(probs, c, b)
x_probs = radial_softmax(x_probs, c, b)
else:
probs = radial_softmax_temp(probs, c, b, temp)
x_probs = radial_softmax_temp(x_probs, c, b, temp)
z_plot = probs.max(1)
z_plot = z_plot.reshape(len(x_plot), len(y_plot)) * 100
loss = np.round(np.asarray(cross_ent(x_probs, y)), 4)
mmc = np.round(np.asarray(np.mean(analyser.max_conf(x_probs))), 4)
layers = layers + '_mmc={}_loss={}'.format(mmc, loss)
vmax = 100
vmin = 50 if max_prob else 0
if classes == 8:
vmin = 10
plt.contourf(x_plot, y_plot, z_plot, levels=np.linspace(10, 100, 90))
cbar = plt.colorbar(ticks=np.linspace(vmin, vmax, 10))
cbar.ax.set_title('confidence', fontsize=12, pad=12)
cbar.set_ticklabels([
'10%', '20%', '30%', '40%', '50%', '60%', '70%', '80%', '90%',
'100%'
])
else:
plt.contourf(x_plot, y_plot, z_plot, levels=np.linspace(50, 100, 50))
cbar = plt.colorbar(ticks=np.linspace(vmin, vmax, 6))
cbar.ax.set_title('confidence', fontsize=12, pad=12)
cbar.set_ticklabels(['50%', '60%', '70%', '80%', '90%', '100%'])
y_np = np.array(y)
X0 = X[y_np.argmax(1) == 0]
X1 = X[y_np.argmax(1) == 1]
plt.scatter(X0[:, 0],
X0[:, 1],
s=20,
edgecolors='red',
facecolor='None',
marker='o',
linewidths=0.2)
plt.scatter(X1[:, 0],
X1[:, 1],
s=20,
edgecolors='green',
facecolor='None',
marker='s',
linewidths=0.2)
if classes == 8:
X2 = X[y_np.argmax(1) == 2]
X3 = X[y_np.argmax(1) == 3]
X4 = X[y_np.argmax(1) == 4]
X5 = X[y_np.argmax(1) == 5]
X6 = X[y_np.argmax(1) == 6]
X7 = X[y_np.argmax(1) == 7]
plt.scatter(X2[:, 0],
X2[:, 1],
s=20,
edgecolors='black',
facecolor='None',
marker='s',
linewidths=0.2)
plt.scatter(X3[:, 0],
X3[:, 1],
s=20,
edgecolors='gray',
facecolor='None',
marker='s',
linewidths=0.2)
plt.scatter(X4[:, 0],
X4[:, 1],
s=20,
edgecolors='m',
facecolor='None',
marker='s',
linewidths=0.2)
plt.scatter(X5[:, 0],
X5[:, 1],
s=20,
edgecolors='darksalmon',
facecolor='None',
marker='s',
linewidths=0.2)
plt.scatter(X6[:, 0],
X6[:, 1],
s=20,
edgecolors='tan',
facecolor='None',
marker='s',
linewidths=0.2)
plt.scatter(X7[:, 0],
X7[:, 1],
s=20,
edgecolors='olivedrab',
facecolor='None',
marker='s',
linewidths=0.2)
plt.xlim([plot_min, plot_max])
plt.ylim([plot_min, plot_max])
plt.gca().set_aspect('equal', adjustable='box')
plt.savefig('{}/{}_{:.1f}_{:.1f}_max_prob={}.pdf'.format(
file_name, layers, plot_min, plot_max, max_prob),
transparent=True)
plt.clf()