#--------------------------------------------------------------------
#solver.training_animation(y_exact, y_predict, epoch, loss)
#--------------------------------------------------------------------
#------------------LOSS-SURFACE--------------------------------
#--------------------------------------------------------------------
solver.neural_net.save_weights('./data/minimum_0')
nnmodel = nn_model.Tensorflow_NNModel(solver.neural_net,
solver.neural_net.loss, solver.x,
'./data/minimum_0')
# proz uses to scale from [-5,5] domain for loss surface plotting, find a spacing n such that it gets as close to 0 as possible
#filenames needs to be an array of files
vis.visualize(nnmodel,
solver.neural_net.loss,
solver.x, ['./data/minimum_0'],
200,
'./data/example',
random_dir=True,
proz=0.5,
verbose=True)
tplot.plot_loss_2D('./data/example.npz', './data/plot2d', is_log=False)
tplot.plot_loss_3D('./data/example.npz',
'./data/plot3d',
is_log=False,
degrees=120)
outs = np.load('./data/example.npz', allow_pickle=True)
outs = outs["a"]
tplot.plot3D(outs[0][0], outs[0][1], outs[0][2])
np.min(outs[0][2])
orig_acc = []
torch.save(net.state_dict(), "data/resnet_minima_point_"+str(0))
for ep in range(1,50+1):
print("Epoch %i"%(ep))
tloss = train_net(net,ep)
orig_loss.append(tloss)
accu = test(net)
orig_acc.append(accu)
torch.save(net.state_dict(), "data/resnet_minima_point_"+str(ep))
np.save("stat_data/resnet_orig_loss", orig_loss)
np.save("stat_data/resnet_orig_acc", orig_acc)
###################################################
######### Visualize Loss and Trajectory ###########
###################################################
num_files = 50
fileindices=np.linspace(0,num_files,num_files + 1)
filesname = ["./data/resnet_minima_point_"+str(int(i)) for i in fileindices]
nnmodel = nn_model.PyTorch_NNModel(net,train_vis,filesname[-1])
vis.visualize(nnmodel,filesname,50,"minima_vis_pca",proz=.4,verbose=True)
tplot.plot_loss_2D("minima_vis_pca.npz",filename="resnet_minima_2D_plot_pca")
tplot.plot_loss_3D("minima_vis_pca.npz",filename="resnet_minima_3D_plot_pca",degrees=50)
flat_minimum2 = flat_weights_dict(minimum2)
"Subtract to make two vectors"
v_dir = flat_minimum1 - flat_minimum0
w_dir = flat_minimum2 - flat_minimum0
###################################################
######### Visualize Loss and Trajectory ###########
###################################################
num_files = 2
fileindices = np.linspace(0, num_files, num_files + 1)
filesname = ["./data/minima_point_" + str(int(i)) for i in fileindices]
nnmodel = nn_model.PyTorch_NNModel(net, train_vis, filesname[-1])
vis.visualize(nnmodel,
filesname,
30,
"three_minima_example",
v_vec=v_dir.cpu().numpy(),
w_vec=w_dir.cpu().numpy(),
proz=.4,
verbose=True)
tplot.plot_loss_2D("three_minima_example.npz",
filename="three_minima_contour",
is_log=True)
tplot.plot_loss_3D("three_minima_example.npz",
filename="three_minima_3D",
height=40,
degrees=100,
is_log=True)
np.save("stat_data/Lenet_orig_loss", orig_loss)
np.save("stat_data/Lenet_orig_acc", orig_acc)
###################################################
######### Visualize Loss and Trajectory ###########
###################################################
num_files = 10
fileindices=np.linspace(0,num_files,num_files + 1)
filesname = ["./data/Lenet_minima_point_"+str(int(i)) for i in fileindices]
nnmodel = nn_model.PyTorch_NNModel(net,train_vis,filesname[-1])
vis.visualize_eigendirs(nnmodel,filesname,40,"minima_vis_eigen",train_data,nn.CrossEntropyLoss(),proz=.4,percentage=0.01, verbose=True)
tplot.plot_loss_2D("minima_vis_eigen.npz",filename="Lenet_minima_2D_plot_eigen",is_log=True)
tplot.plot_loss_3D("minima_vis_eigen.npz",filename="Lenet_minima_3D_plot_eigen",degrees=50)
###########################################################
######### Calculate Eigenvalue Density Spectrum ###########
###########################################################
hf.stochastic_lanczos(net,train_data,nn.CrossEntropyLoss(),"Lenet_eigenvals",percentage=0.05,num_iters=5,verbose=True)
x1,x2 = hf.get_xlim(["Lenet_eigenvals.npz"])
x = np.linspace(x1-2,x2+2,1000)
func = hf.get_spectrum(x,np.sqrt(1e-2),"Lenet_eigenvals.npz")
f = plt.figure()
plt.semilogy(x,func)
plt.ylim(1e-9,1e2)
plt.grid(True)