def get_vals(lr,hidden_dims1,hidden_dims2,lr_decay,reg,drop):
global json_log2,json_log
lr = 10 ** lr
model = FullyConnectedNet(hidden_dims=[int(hidden_dims1),int(hidden_dims2)], input_dim=48 * 48 * 1, reg=reg, num_classes=7, dtype=np.float64,dropout=drop)
solver = Solver(model, data,
update_rule='sgd_momentum',
optim_config={'learning_rate': lr,}, lr_decay = lr_decay,
num_epochs=50, batch_size=70,
print_every=1000000)
solver.train()
solver._save_checkpoint()
#SAVE THE VALUES TO A FILE
val_acc = solver.best_val_acc
acc = max(solver.train_acc_history)
loss = min(solver.loss_history)
json_log2.write(json.dumps({'Learning Rate': lr,
'accuracy': acc, 'val_acc': val_acc ,
'loss': loss,"lr_decay":lr_decay,
'dropout':drop,'reg':reg,
'layer_1': hidden_dims1,'layer_2': hidden_dims2}) + ',\n')
json_log.write(json.dumps({'Learning Rate': lr,
'accuracy': solver.train_acc_history,
'val_acc': solver.val_acc_history,
'loss': solver.loss_history,"lr_decay":lr_decay,
'dropout':drop,'reg':reg,
'layer_1': hidden_dims1,'layer_2': hidden_dims2}) + ',\n')
return solver.best_val_acc
def train_FER2013():
###########################################################################
# BEGIN OF YOUR CODE #
###########################################################################
#pickle_FER() #used to load and store the FER2013 dataset for faster performance
optim = False # use for fine tuning learning rate
out = get_FER(num_training=10000)
data = {
'X_train': out['X_train'], # training data
'y_train': out['y_train'], # training labels
'X_val': out['X_val'], # validation data
'y_val': out['y_val'] # validation labels
}
model = FullyConnectedNet(input_dim=48 * 48 * 1,
hidden_dims=[40],
num_classes=7,
dropout=0,
reg=0,
seed=int(time.time()))
if optim:
count = 1
reached = False
threshold = 0.35 #set threshold here
while not reached:
np.random.seed(int(time.time()))
print("iteration number{}".format(count))
lr = np.random.uniform(0.001, 0.003)
print("testing lr: {}".format(lr))
solver = Solver(model,
data,
update_rule='sgd_momentum',
optim_config={
'learning_rate': lr,
'momentum': 0.5
},
lr_decay=0.8,
num_epochs=100,
batch_size=100,
print_every=100)
solver.train()
if max(solver.val_acc_history) >= threshold:
reached = True
count += 1
print("Final lr: {}".format(lr))
else:
solver = Solver(model,
data,
update_rule='sgd_momentum',
optim_config={
'learning_rate': 0.0018742807840127864,
'momentum': 0.5
},
lr_decay=0.8,
num_epochs=100,
batch_size=100,
print_every=100)
solver.train()
def stage_optim_layers(fcnn, data):
# 1st stage: search in coarse range
lower = COARSE_RANGE["lower"]
upper = COARSE_RANGE["upper"]
layers = []
accs = []
for it in range(MAX_ITER):
layer = randint(lower, upper, 2)
print("Experiment {} for layer: {}".format(it + 1, layer))
layers.append(layer)
fcnn.set_hidden_dims(layer)
solver = Solver(
fcnn,
data,
update_rule='sgd_momentum',
optim_config={
"learning_rate": LEARNING_RATE,
"momentum": MOMENTUM
},
lr_decay=LEARNING_DECAY,
print_every=100,
batch_size=BATCH_SIZE,
checkpoint_name="checkpoints/test" if CHECKOUT else None,
num_epochs=EPOCH_NUM)
solver.train()
accs.append(solver.best_val_acc)
loss = solver.loss_history
t_acc = solver.train_acc_history
v_acc = solver.val_acc_history
draw_loss_acc(
loss, t_acc, v_acc,
"layer_{}_{}/{}-{:4}-{:4}".format(lower, upper, it + 1, layer[0],
layer[1]))
path = "params/"
filename = "layer_{}_{}.txt".format(lower, upper)
if not os.path.exists(path):
os.makedirs(path)
with open(path + filename, 'w') as file:
for line, layer in enumerate(layers):
stat = "{}. layer: [{:4}, {:4}] - accuracy: {:.4f}".format(
line + 1, layer[0], layer[1], accs[line])
file.write(stat + "\n")
best_layer_idx = np.argmax(np.asarray(accs), axis=0)
best_layer = layers[best_layer_idx]
best_layer_acc = accs[best_layer_idx]
file.write(
"Best accur layer - {}: [{:4}, {:4}] with accuracy {:.4f}\n".
format(best_layer_idx + 1, best_layer[0], best_layer[1],
best_layer_acc))
def train_overfit_net(save_net = False):
# get CIFAR10 data
data = get_CIFAR10_data()
# subsample the data. 50 draws
indices_to_select = np.random.randint(0, len(data["X_train"]), 50)
# extract the samples
data["X_train"] = data["X_train"][indices_to_select]
data["y_train"] = data["y_train"][indices_to_select]
# intialize net
model = FullyConnectedNet([50],
input_dim = 32*32*3,
num_classes = 10,
dropout = 0,
reg = 0.0,
weight_scale = 1e-2)
# initialize solver
solver = Solver(model,data,
update_rule = 'sgd',
optim_config = {'learning_rate': 5e-4},
lr_decay = 0.85,
num_epochs = 20,
batch_size = 5,
print_every = 100)
# train the net
solver.train()
if save_net:
# test the net and save its training, validation and testing accuracies.
# get training accuracy
train_acc = str.format("{0:.2f}", solver.check_accuracy(data["X_train"], data["y_train"]) * 100) + "\%"
# get validation accuracy
val_acc = str.format("{0:.2f}", solver.best_val_acc * 100) + "\%"
# get testing accuracy
test_acc = str.format("{0:.2f}", solver.check_accuracy(data["X_test"], data["y_test"]) * 100) + "\%"
text = "Accuracies: " + train_acc + " training, " + val_acc + " validation \& " + test_acc + " testing."
# write to file
append_to_file("nets/overfit_net/info.tex", text, mode = "w")
# save net info
save_net_info("nets/overfit_net", solver)
def stage_optim_drop(fcnn, data):
# 1st stage: search in coarse range
lower = COARSE_RANGE["lower"]
upper = COARSE_RANGE["upper"]
drops = []
accs = []
for it in range(MAX_ITER):
drop = uniform(lower, upper)
print("Experiment {} for drop: {}".format(it + 1, drop))
drops.append(drop)
fcnn.dropout_params['p'] = drop
solver = Solver(
fcnn,
data,
update_rule='sgd_momentum',
optim_config={
"learning_rate": LEARNING_RATE,
"momentum": MOMENTUM
},
lr_decay=LEARNING_DECAY,
print_every=100,
batch_size=BATCH_SIZE,
checkpoint_name="checkpoints/test" if CHECKOUT else None,
num_epochs=EPOCH_NUM)
solver.train()
accs.append(solver.best_val_acc)
loss = solver.loss_history
t_acc = solver.train_acc_history
v_acc = solver.val_acc_history
draw_loss_acc(
loss, t_acc, v_acc,
"drop_{}_{}/{}-{:.5f}".format(lower, upper, it + 1, drop))
fcnn.reset()
path = "params/"
filename = "drop_{}_{}.txt".format(lower, upper)
if not os.path.exists(path):
os.makedirs(path)
with open(path + filename, 'w') as file:
for line, drop in enumerate(drops):
stat = "{}. drop: {:.5f} - accuracy: {:.4f}".format(
line + 1, drop, accs[line])
file.write(stat + "\n")
best_drop_idx = np.argmax(np.asarray(accs), axis=0)
best_drop = drops[best_drop_idx]
best_drop_acc = accs[best_drop_idx]
file.write(
"Best accur drop - {}: {:.5f} with accuracy {:.4f}\n".format(
best_drop_idx + 1, best_drop, best_drop_acc))
def train_cifar10_net(save_net=False):
"""
Uses a Solver instance to train a TwoLayerNet that achieves at least 50%
accuracy on the validation set.
"""
# get CIFAR10 data
data = get_CIFAR10_data()
# intialize net
model = FullyConnectedNet([100],
input_dim=32 * 32 * 3,
num_classes=10,
dropout=0,
reg=0.0)
# initialize solver
solver = Solver(model,
data,
update_rule='sgd',
optim_config={'learning_rate': 1e-3},
lr_decay=0.95,
num_epochs=20,
batch_size=100,
print_every=100)
# train the net
solver.train()
if save_net:
# test the net and save its training, validation and testing accuracies.
# get training accuracy
train_acc = str.format(
"{0:.2f}",
solver.check_accuracy(data["X_train"], data["y_train"]) *
100) + "\%"
# get validation accuracy
val_acc = str.format("{0:.2f}", solver.best_val_acc * 100) + "\%"
# get testing accuracy
test_acc = str.format(
"{0:.2f}",
solver.check_accuracy(data["X_test"], data["y_test"]) * 100) + "\%"
text = "Accuracies: " + train_acc + " training, " + val_acc + " validation \& " + test_acc + " testing."
# write to file
append_to_file("nets/train_net/info.tex", text, mode="w")
# save net info
save_net_info("nets/train_net", solver)
def train_net(data,
hidden_dims,
input_dim,
num_classes,
dropout = 0,
reg = 0,
learning_rate = 5e-4,
momentum = 0,
num_epochs = 20,
batch_size = 100,
lr_decay = 0.95,
update_rule = "sdg",
verbose = True):
"""
Uses a solver instance to train a neural net on the given dataset.
args:
- data: dictionary with X_train, y_train, X_test, and y_test
- plot: if True, generates a plot and saves in the given folder
- pickle: if True, pickles the model in the given folder
"""
# initialize net
model = FullyConnectedNet(hidden_dims,
input_dim,
num_classes,
dropout,
reg)
# initialize solver
solver = Solver(model,
data,
update_rule = update_rule,
optim_config = {'learning_rate': learning_rate,
'momentum': momentum},
lr_decay = lr_decay,
num_epochs = num_epochs,
batch_size = batch_size,
print_every = 100,
verbose = verbose)
# train the network
solver.train()
# return the solver
return solver
#f = open('model.pickle', 'rb')
#model = pickle.load(f)
#f.close()
data = get_FER2013_data(num_test=3589)
solver = Solver(model,
data,
update_rule='sgd_momentum',
optim_config={
'learning_rate': 5e-3,
},
lr_decay=0.95,
num_epochs=35,
batch_size=100,
print_every=200)
solver.train()
save = input("Save model? ")
if (save is 'y'):
f = open('model.pickle', 'wb')
pickle.dump(model, f)
f.close()
print("Model saved!")
plt.subplot(2, 1, 1)
plt.title("Training loss")
plt.plot(solver.loss_history, "o")
plt.xlabel("Iteratition")
plt.subplot(2, 1, 2)
plt.title('Accuracy')
def stage_optim_lr(fcnn, data):
# 1st stage: search in coarse range
lower = COARSE_RANGE["lower"]
upper = COARSE_RANGE["upper"]
lrs = []
match_values = []
accs = []
for it in range(MAX_ITER):
lr = 10**uniform(lower, upper)
lrs.append(lr)
solver = Solver(
fcnn,
data,
update_rule='sgd_momentum',
optim_config={
"learning_rate": lr,
"momentum": MOMENTUM
},
lr_decay=LEARNING_DECAY,
print_every=100,
batch_size=BATCH_SIZE,
checkpoint_name="checkpoints/test" if CHECKOUT else None,
num_epochs=EPOCH_NUM,
tune_lr=True)
solver.train()
match_values.append(lr_update_match(solver.updates))
accs.append(solver.best_val_acc)
loss = [x for x in solver.loss_history if not x > 2.0]
t_acc = solver.train_acc_history
v_acc = solver.val_acc_history
draw_loss_acc(loss, t_acc, v_acc,
"lr_{}_{}/{}-{:.5f}".format(lower, upper, it + 1, lr))
fcnn.reset()
match_values = np.asarray(match_values)
path = "params/"
filename = "lr_{}_{}.txt".format(lower, upper)
if not os.path.exists(path):
os.makedirs(path)
with open(path + filename, 'w') as file:
for line, lr in enumerate(lrs):
stat = "{}. lr: {:.5f} - match ratio: {:.4f} - accuracy: {:.4f}".format(
line + 1, lr, match_values[line], accs[line])
file.write(stat + "\n")
best_lr_idx = np.argmax(match_values, axis=0)
best_lr = lrs[best_lr_idx]
best_lr_acc = accs[best_lr_idx]
file.write("Best match lr - {}: {:.5f} with accuracy {:.4f}\n".format(
best_lr_idx, best_lr, best_lr_acc))
best_lr_idx = np.argmax(np.asarray(accs), axis=0)
best_lr = lrs[best_lr_idx]
best_lr_acc = accs[best_lr_idx]
file.write("Best accur lr - {}: {:.5f} with accuracy {:.4f}\n".format(
best_lr_idx, best_lr, best_lr_acc))
