def test_batchnorm(self):
print("\n======== TestFCNet.test_batchnorm :")
dataset = load_data(self.data_dir, self.verbose)
num_train = 10
small_data = {
'X_train': dataset['X_train'][:num_train],
'y_train': dataset['y_train'][:num_train],
'X_val': dataset['X_val'][:num_train],
'y_val': dataset['y_val'][:num_train]
}
input_dim = 32 * 32 * 3
hidden_dims = [100, 100, 100, 100, 100, 100]
weight_scale = 2e-2
learning_rate = 1e-3
update_rule = 'adam'
bn_model = fcnet.FCNet(input_dim=input_dim,
hidden_dims=hidden_dims,
weight_scale=weight_scale,
use_batchnorm=True)
bn_solver = solver.Solver(
bn_model,
small_data,
num_epochs=self.num_epochs,
update_rule=update_rule,
optim_config={'learning_rate': learning_rate},
verbose=True,
print_every=self.print_every)
print("Training with batchnorm")
bn_solver.train()
fc_model = fcnet.FCNet(input_dim=input_dim,
hidden_dims=hidden_dims,
weight_scale=weight_scale,
use_batchnorm=False)
fc_solver = solver.Solver(
fc_model,
small_data,
num_epochs=self.num_epochs,
update_rule=update_rule,
optim_config={'learning_rate': learning_rate},
verbose=True,
print_every=self.print_every)
print("Training without batchnorm")
fc_solver.train()
print("======== TestFCNet.test_batchnorm: <END> ")
def test_weight_init(self):
print("======== TestFCNet.test_weight_init:")
dataset = load_data(self.data_dir, self.verbose)
num_train = 1500
small_data = {
'X_train': dataset['X_train'][:num_train],
'y_train': dataset['y_train'][:num_train],
'X_val': dataset['X_val'][:num_train],
'y_val': dataset['y_val'][:num_train]
}
input_dim = 32 * 32 * 3
hidden_dims = [100, 100, 100, 100, 100, 100, 100, 100, 100]
weight_scale = 2e-2
reg = 2e-2
learning_rate = 1e-3
batch_size = 50
update_rule = 'adam'
weight_init = ['gauss', 'gauss_sqrt2', 'xavier']
model_dict = {}
for w in weight_init:
model = fcnet.FCNet(input_dim=input_dim,
hidden_dims=hidden_dims,
weight_scale=weight_scale,
reg=reg,
weight_init=w)
model_dict[w] = model
solver_dict = {}
for k, m in model_dict.items():
if self.verbose:
print(m)
solv = solver.Solver(
m,
small_data,
print_every=self.print_every,
num_epochs=self.num_epochs,
batch_size=batch_size, # previously 25
update_rule=update_rule,
optim_config={'learning_rate': learning_rate})
solv.train()
#skey = '%s-%s' % (m.__repr__(), k)
skey = '%s' % k
solver_dict[skey] = solv
if self.draw_plots:
fig, ax = vis_solver.get_train_fig()
vis_solver.plot_solver_compare(ax, solver_dict)
#vis_solver.plot_solver(ax, solv)
plt.show()
print("======== TestFCNet.test_weight_init: <END> ")
def test_all_optim_fcnet_5layer(self):
print("\n======== TestSolverFCNet.test_all_optim_fcnet_5layer:")
dataset = load_data(self.data_dir, self.verbose)
num_train = 50
small_data = {
'X_train': dataset['X_train'][:num_train],
'y_train': dataset['y_train'][:num_train],
'X_val': dataset['X_val'][:num_train],
'y_val': dataset['y_val'][:num_train]
}
#input_dim = small_data['X_train'].shape[0]
input_dim = 3 * 32 * 32
hidden_dims = [100, 100, 100, 100, 100]
#hidden_dims = [100, 50, 10] # just some random dims
weight_scale = 5e-2
reg = 1e-1
batch_size = 50
solvers = {}
# Solver params
optim_list = ['rmsprop', 'sgd_momentum', 'adam', 'sgd']
lr = {'rmsprop': 1e-4, 'adam': 1e-3, 'sgd': 1e-3, 'sgd_momentum': 1e-3}
for update_rule in optim_list:
print("Using update rule %s" % update_rule)
model = fcnet.FCNet(input_dim=input_dim,
hidden_dims=hidden_dims,
weight_scale=weight_scale,
reg=reg,
dtype=np.float64)
if self.verbose:
print(model)
model_solver = solver.Solver(
model,
small_data,
print_every=self.print_every,
num_epochs=self.num_epochs,
batch_size=batch_size, # previously 25
update_rule=update_rule,
optim_config={'learning_rate': lr[update_rule]})
solvers[update_rule] = model_solver
model_solver.train()
# get some figure handles and plot the data
if self.draw_plots:
fig, ax = get_figure_handles()
plot_test_result(ax, solvers, self.num_epochs)
fig.set_size_inches(8, 8)
fig.tight_layout()
plt.show()
print("======== TestSolverFCNet.test_all_optim_fcnet_5layer: <END> ")
def test_model_restore(self):
print("\n======== TestSolverCheckpoint.test_model_restore:")
dataset = load_data(self.data_dir, self.verbose)
num_train = 50
small_data = {
'X_train': dataset['X_train'][:num_train],
'y_train': dataset['y_train'][:num_train],
'X_val': dataset['X_val'][:num_train],
'y_val': dataset['y_val'][:num_train]
}
#input_dim = small_data['X_train'].shape[0]
input_dim = 3 * 32 * 32
#hidden_dims = [100, 100, 100, 100, 100]
hidden_dims = [100, 50, 10] # just some random dims
weight_scale = 5e-2
learning_rate = 1e-2
batch_size = 50
update_rule = 'adam'
model = fcnet.FCNet(input_dim=input_dim,
hidden_dims=hidden_dims,
weight_scale=weight_scale,
dtype=np.float64)
if self.verbose:
print(model)
ref_solver = solver.Solver(
model,
small_data,
print_every=self.print_every,
num_epochs=self.num_epochs,
batch_size=batch_size, # previously 25
update_rule=update_rule,
optim_config={'learning_rate': learning_rate})
ref_solver.train()
ref_solver_file = 'tests/ref_solver.pkl'
ref_solver.save(ref_solver_file)
test_solver = solver.Solver(model, small_data)
test_solver.load(ref_solver_file)
# Compare the two solvers
print("Checking parameters")
self.assertEqual(ref_solver.update_rule, test_solver.update_rule)
self.assertEqual(ref_solver.num_epochs, test_solver.num_epochs)
self.assertEqual(ref_solver.batch_size, test_solver.batch_size)
self.assertEqual(ref_solver.print_every, test_solver.print_every)
self.assertEqual(ref_solver.lr_decay, test_solver.lr_decay)
#self.assertDictEqual(ref_solver.optim_config, test_solver.optim_config)
print("======== TestSolverCheckpoint.test_model_restore: <END> ")
def test_adam_vs_rmsprop_fcnet(self):
print("\n======== TestSolverFCNet.test_adam_vs_rmsprop:")
dataset = load_data(self.data_dir, self.verbose)
num_train = 50
small_data = {
'X_train': dataset['X_train'][:num_train],
'y_train': dataset['y_train'][:num_train],
'X_val': dataset['X_val'][:num_train],
'y_val': dataset['y_val'][:num_train]
}
#input_dim = small_data['X_train'].shape[0]
input_dim = 3 * 32 * 32
#hidden_dims = [100, 100, 100, 100, 100]
hidden_dims = [100, 100, 100, 100, 100]
weight_scale = 5e-2
batch_size = 50
reg = 1e-1
lr = {'rmsprop': 1e-4, 'adam': 1e-3}
update_rule = ['rmsprop', 'adam']
solvers = {}
for u in update_rule:
model = fcnet.FCNet(input_dim=input_dim,
hidden_dims=hidden_dims,
weight_scale=weight_scale,
reg=reg,
dtype=np.float64)
if self.verbose:
print(model)
model_solver = solver.Solver(
model,
small_data,
print_every=self.print_every,
num_epochs=self.num_epochs,
batch_size=batch_size, # previously 25
update_rule=u,
optim_config={'learning_rate': lr[u]})
solvers[u] = model_solver
model_solver.train()
if self.draw_plots is True:
fig, ax = get_figure_handles()
plot_test_result(ax, solvers, self.num_epochs)
fig.set_size_inches(8, 8)
fig.tight_layout()
plt.show()
print("======== TestSolverFCNet.test_adam_vs_rmsprop: <END> ")
def test_fcnet_2layer_dropout(self):
print("\n======== TestFCNetDropout.test_fcnet_2layer_dropout :")
dataset = load_data(self.data_dir, self.verbose)
num_train = 10
small_data = {
'X_train': dataset['X_train'][:num_train],
'y_train': dataset['y_train'][:num_train],
'X_val': dataset['X_val'][:num_train],
'y_val': dataset['y_val'][:num_train]
}
#input_dim = small_data['X_train'].shape[0]
input_dim = 3 * 32 * 32
#hidden_dims = [100, 100, 100, 100]
batch_size = 100
solvers = {}
dropout_probs = [0.0, 0.3, 0.5, 0.7]
for d in dropout_probs:
model = fcnet.FCNet(hidden_dims=[500],
input_dim=input_dim,
num_classes=10,
dropout=d,
weight_scale=2e-2)
s = solver.Solver(model,
small_data,
num_epochs=self.num_epochs,
batch_size=batch_size,
update_rule='adam',
optim_config={'learning_rate': 5e-4},
verbose=True,
print_every=self.print_every)
print("Training with dropout %f" % d)
s.train()
solvers['p=' + str(d)] = s
if self.draw_plots:
fig, ax = get_figure_handles()
plot_test_result(ax, solvers, self.num_epochs)
fig.set_size_inches(8, 8)
fig.tight_layout()
plt.show()
print("======== TestFCNetDropout.test_fcnet_2layer_dropout: <END> ")
def test_fcnet_3layer_overfit(self):
print("\n======== TestFCNet.test_fcnet_3layer_overfit:")
dataset = load_data(self.data_dir, self.verbose)
num_train = 50
small_data = {
'X_train': dataset['X_train'][:num_train],
'y_train': dataset['y_train'][:num_train],
'X_val': dataset['X_val'][:num_train],
'y_val': dataset['y_val'][:num_train]
}
#input_dim = small_data['X_train'].shape[0]
input_dim = 3 * 32 * 32
hidden_dims = [100, 100]
weight_scale = 0.079564
learning_rate = 0.003775
# Get model and solver
model = fcnet.FCNet(input_dim=input_dim,
hidden_dims=hidden_dims,
weight_scale=weight_scale,
dtype=np.float64,
verbose=True)
print(model)
model_solver = solver.Solver(
model,
small_data,
print_every=self.print_every,
num_epochs=self.num_epochs,
batch_size=50, # previously 25
update_rule='sgd',
optim_config={'learning_rate': learning_rate})
model_solver.train()
# Plot results
if self.draw_plots is True:
plt.plot(model_solver.loss_history, 'o')
plt.title('Training loss history (3 layers)')
plt.xlabel('Iteration')
plt.ylabel('Training loss')
plt.show()
print("======== TestFCNet.test_fcnet_3layer_overfit: <END> ")
def test_rmsprop_fcnet(self):
print("\n======== TestSolverFCNet.test_rmsprop_fcnet:")
dataset = load_data(self.data_dir, self.verbose)
num_train = 50
small_data = {
'X_train': dataset['X_train'][:num_train],
'y_train': dataset['y_train'][:num_train],
'X_val': dataset['X_val'][:num_train],
'y_val': dataset['y_val'][:num_train]
}
#input_dim = small_data['X_train'].shape[0]
input_dim = 3 * 32 * 32
#hidden_dims = [100, 100, 100, 100, 100]
hidden_dims = [100, 50, 10] # just some random dims
weight_scale = 5e-2
learning_rate = 1e-2
batch_size = 50
update_rule = 'rmsprop'
model = fcnet.FCNet(input_dim=input_dim,
hidden_dims=hidden_dims,
weight_scale=weight_scale,
dtype=np.float64)
if self.verbose:
print(model)
model_solver = solver.Solver(
model,
small_data,
print_every=self.print_every,
num_epochs=self.num_epochs,
batch_size=batch_size, # previously 25
update_rule=update_rule,
optim_config={'learning_rate': learning_rate})
model_solver.train()
if self.draw_plots is True:
solvers = {'rmsprop': model_solver}
fig, ax = get_figure_handles()
plot_test_result(ax, solvers, self.num_epochs)
fig.set_size_inches(8, 8)
fig.tight_layout()
plt.show()
print("======== TestSolverFCNet.test_rmsprop_fcnet: <END> ")
def test_dropout_backward(self):
print("\n======== TestLayersDropout.test_dropout_backward:")
N = 2
D = 15
H1 = 20
H2 = 30
C = 10
X = np.random.randn(N, D)
y = np.random.randint(C, size=(N, ))
dropout_probs = [0.3, 0.6, 0.1]
import pymllib.classifiers.fcnet as fcnet
# Network params
hidden_dims = [H1, H2]
weight_scale = 5e-2
for p in dropout_probs:
print("Running check with dropout p = %f" % p)
model = fcnet.FCNet(hidden_dims=hidden_dims,
input_dim=D,
num_classes=C,
dropout=p,
weight_scale=weight_scale,
seed=123,
dtype=np.float64)
loss, grads = model.loss(X, y)
print("Initial loss : %f" % loss)
for n in sorted(grads):
f = lambda _: model.loss(X, y)[0]
grad_num = check_gradient.eval_numerical_gradient(
f, model.params[n])
grad_error = error.rel_error(grad_num, grads[n])
print("%s relative error : %.2e" % (n, grad_error))
print("======== TestLayersDropout.test_dropout_backward: <END> ")
def test_fcnet_5layer_loss(self):
print("\n======== TestFCNet.test_fcnet_5layer_loss:")
dataset = load_data(self.data_dir, self.verbose)
num_train = 50
small_data = {
'X_train': dataset['X_train'][:num_train],
'y_train': dataset['y_train'][:num_train],
'X_val': dataset['X_val'][:num_train],
'y_val': dataset['y_val'][:num_train]
}
#input_dim = small_data['X_train'].shape[0]
input_dim = 3 * 32 * 32
hidden_dims = [100, 100, 100, 100]
weight_scale = 1e-2
learning_rate = 1e-2
# Get model and solver
model = fcnet.FCNet(input_dim=input_dim,
hidden_dims=hidden_dims,
weight_scale=weight_scale,
reg=0.0,
dtype=np.float64)
print(model)
model_solver = solver.Solver(
model,
small_data,
print_every=self.print_every,
num_epochs=self.num_epochs,
batch_size=50, # previously 25
update_rule='sgd',
optim_config={'learning_rate': learning_rate})
model_solver.train()
print("======== TestFCNet.test_fcnet_5layer_loss: <END> ")
def test_fcnet_loss(self):
print("\n======== TestFCNet.test_fcnet_loss:")
# Some model parameters
np.random.seed(231)
N = 3
D = 5
H = 50
C = 7
std = 1e-2
# Get model
model = fcnet.FCNet(hidden_dims=[H],
input_dim=D,
num_classes=C,
weight_scale=std,
dtype=np.float64,
verbose=True)
W1_std = abs(model.params['W1'].std() - std)
W2_std = abs(model.params['W2'].std() - std)
b1 = model.params['b1']
b2 = model.params['b2']
# Check that the weights are sensible
self.assertLess(W1_std,
std / 10.0,
msg="Problem in first layer weights")
self.assertLess(W2_std,
std / 10.0,
msg="Problem in second layer weights")
self.assertTrue(np.all(b1 == 0), msg="Problem in first layer biases")
self.assertTrue(np.all(b2 == 0), msg="Problem in second layer biases")
print("\tTest time forward pass")
model.params['W1'] = np.linspace(-0.7, 0.3, num=D * H).reshape(D, H)
model.params['W2'] = np.linspace(-0.3, 0.4, num=H * C).reshape(H, C)
model.params['b1'] = np.linspace(-0.1, 0.9, num=H)
model.params['b2'] = np.linspace(-0.9, 0.1, num=C)
# Get data
X = np.linspace(-5.5, 4.5, num=N * D).reshape(D, N).T
y = np.random.randint(C, size=N)
scores = model.loss(X)
correct_scores = np.asarray([[
11.53165108, 12.2917344, 13.05181771, 13.81190102, 14.57198434,
15.33206765, 16.09215096
],
[
12.05769098, 12.74614105, 13.43459113,
14.1230412, 14.81149128, 15.49994135,
16.18839143
],
[
12.58373087, 13.20054771, 13.8173455,
14.43418138, 15.05099822, 15.66781506,
16.2846319
]])
scores_diff = np.abs(scores - correct_scores).sum()
# Cheating constant
if self.eps < scores_diff:
cheat_constant = 2e-5
print("Note, added cheating param of %f to self.eps (%f)" %
(cheat_constant, self.eps))
else:
cheat_constant = 0.0
self.assertLess(scores_diff, self.eps + cheat_constant)
print("======== TestFCNet.test_fcnet_loss: <END> ")
