def get_valid_score(train_loader, valid_loader, n_epoch, step_size, weight_decay):
print("weight_decay: ", weight_decay)
net = KLayerNeuralNetwork(n_hidden_nodes=[50, 50], p_dropout=0.0, batch_norm=True, batch_norm_momentum=0.7)
scheduler = CyclicLR(eta_min=1e-5, eta_max=1e-1, step_size=step_size)
# =========================================================================
best_valid_acc = -np.inf
for epoch in range(n_epoch):
train(train_loader, net, weight_decay, scheduler)
valid_acc = evaluate(valid_loader, net)
if valid_acc > best_valid_acc:
best_valid_acc = valid_acc
print("[Evaluate] Valid. Acc.: %.3f%%" % (
valid_acc*100))
print("[Result] Valid. Acc.: %.3f%%" % (
best_valid_acc*100))
return best_valid_acc
4. with / without batchnorm should behave the same
"""
import numpy as np
from utils.load_batch import cifar10_DataLoader
from utils.load_batch import load_batch
from utils.handle_data import data_split
from utils import train
from utils.lrate import StepLR
from utils.lrate import CyclicLR
from clsr.nn_kl import KLayerNeuralNetwork
from utils.preprocess import StandardScaler
if __name__ == "__main__":
train_data = load_batch("cifar-10-batches-py/data_batch_1")
net = KLayerNeuralNetwork(n_hidden_nodes=[50, 50],
p_dropout=0.0,
batch_norm=False,
batch_norm_momentum=.9)
scaler = StandardScaler()
scaler.fit(train_data['pixel_data'])
train_data['pixel_data'] = scaler.transform(train_data['pixel_data'])
batch_size = 100
train_loader = cifar10_DataLoader(train_data,
batch_size=batch_size,
shuffle=False)
ntrain = train_data['labels'].shape[0]
n_epoch = 200
scheduler = StepLR(0.05, 20, gamma=.5)
# scheduler = CyclicLR(eta_min=1e-5, eta_max=1e-1, step_size=1000)
scaler = StandardScaler()
scaler.fit(train_data['pixel_data'])
train_data['pixel_data'] = scaler.transform(train_data['pixel_data'])
valid_data['pixel_data'] = scaler.transform(valid_data['pixel_data'])
test_data['pixel_data'] = scaler.transform(test_data['pixel_data'])
print("Done preprocessing!")
# ==================================================================
# make dataloader
batch_size = 100
train_loader = cifar10_DataLoader(train_data, batch_size=batch_size)
valid_loader = cifar10_DataLoader(valid_data, batch_size=batch_size)
test_loader = cifar10_DataLoader(test_data, batch_size=batch_size)
# ==================================================================
net = KLayerNeuralNetwork(batch_norm=True,
batch_norm_momentum=0.7,
n_hidden_nodes=[50, 30, 20, 20, 10, 10, 10, 10],
p_dropout=0.0)
ntrain = train_data['labels'].shape[0]
n_step_per_cycle = 5
ncycle = 2
n_epoch = ncycle * n_step_per_cycle * 2
iter_per_epoch = int(np.ceil(ntrain / batch_size))
step_size = n_step_per_cycle * iter_per_epoch
weight_decay = 0.005
scheduler = CyclicLR(eta_min=1e-5, eta_max=1e-1, step_size=step_size)
# =========================================================================
print("--------- Train Schedule ---------")
print("ncycle: ", ncycle)
print("n_epoch: ", n_epoch)
print("step_size: ", step_size)
print("iter_per_epoch: ", iter_per_epoch)
# make dataloader
batch_size = 100
train_loader = cifar10_DataLoader(train_data, batch_size=batch_size)
valid_loader = cifar10_DataLoader(valid_data, batch_size=batch_size)
test_loader = cifar10_DataLoader(test_data, batch_size=batch_size)
# ==================================================================
# net = KLayerNeuralNetwork(n_hidden_nodes=[50, 50], p_dropout=0.0)
# net = KLayerNeuralNetwork(n_hidden_nodes=[70, 50], p_dropout=0.0)
# net = KLayerNeuralNetwork(p_dropout=0.0,
# n_hidden_nodes= [50, 30, 20, 20, 10, 10, 10, 10], batch_norm=False
# )
# net = KLayerNeuralNetwork(p_dropout=0.0,
# n_hidden_nodes= [50, 30, 20, 20, 10, 10, 10, 10], batch_norm=True
# )
net = KLayerNeuralNetwork(p_dropout=0.2,
n_hidden_nodes=[500, 250, 100, 50],
batch_norm=True,
batch_norm_momentum=0.9)
ntrain = train_data['labels'].shape[0]
n_step_per_cycle = 5
ncycle = 10
n_epoch = ncycle * n_step_per_cycle * 2
# n_epoch = 1
iter_per_epoch = int(np.ceil(ntrain / batch_size))
step_size = n_step_per_cycle * iter_per_epoch
# weight_decay = 0.005
weight_decay = 0.00
scheduler = CyclicLR(eta_min=1e-5, eta_max=1e-1, step_size=step_size)
# =========================================================================
print("--------- Train Schedule ---------")
print("ncycle: ", ncycle)
print("n_epoch: ", n_epoch)
scaler.fit(train_data['pixel_data'])
train_data['pixel_data'] = scaler.transform(train_data['pixel_data'])
valid_data['pixel_data'] = scaler.transform(valid_data['pixel_data'])
test_data['pixel_data'] = scaler.transform(test_data['pixel_data'])
print("Done preprocessing!")
# ==================================================================
# make dataloader
batch_size = 100
train_loader = cifar10_DataLoader(train_data, batch_size=batch_size)
valid_loader = cifar10_DataLoader(valid_data, batch_size=batch_size)
test_loader = cifar10_DataLoader(test_data, batch_size=batch_size)
# ==================================================================
# net = KLayerNeuralNetwork(n_hidden_nodes=[50, 50], p_dropout=0.0, batch_norm=True)
net = KLayerNeuralNetwork(p_dropout=0.0,
n_hidden_nodes=[50, 30, 20, 20, 10, 10, 10, 10],
batch_norm=is_bn,
wgt_init="random",
init_sig=init_sig)
ntrain = train_data['labels'].shape[0]
n_step_per_cycle = 5
ncycle = 2
n_epoch = ncycle * n_step_per_cycle * 2
iter_per_epoch = int(np.ceil(ntrain / batch_size))
step_size = n_step_per_cycle * iter_per_epoch
weight_decay = 0.005
scheduler = CyclicLR(eta_min=1e-5, eta_max=1e-1, step_size=step_size)
# =========================================================================
print("--------- Train Schedule ---------")
print("ncycle: ", ncycle)
print("n_epoch: ", n_epoch)
print("step_size: ", step_size)
if __name__ == "__main__":
test_data = load_batch("cifar-10-batches-py/test_batch")
batch_size = 10
n_features = 40
test_loader = cifar10_DataLoader(test_data, batch_size=batch_size)
test_inputs = None
test_labels = None
for inputs, labels in test_loader:
test_inputs = inputs[:n_features, :]
test_labels = labels
break
net = KLayerNeuralNetwork(p_dropout=0.0,
n_features=n_features,
n_hidden_nodes=[50],
batch_norm=True,
dtype='float64')
check_num_grad(net, test_inputs, test_labels)
net = KLayerNeuralNetwork(p_dropout=0.0,
n_features=n_features,
n_hidden_nodes=[50, 50],
batch_norm=True,
dtype='float64')
check_num_grad(net, test_inputs, test_labels)
net = KLayerNeuralNetwork(p_dropout=0.0,
n_features=n_features,
n_hidden_nodes=[50, 50, 50],
batch_norm=False,