def runSoftmax():
# Load the dataset
f = gzip.open('../mnist.pkl.gz', 'rb')
train_set, valid_set, test_set = pickle.load(f, encoding='latin1')
f.close()
if len(sys.argv) > 1:
exp_name = sys.argv[1]
else:
exp_name = 'exp'
os.makedirs(exp_name, exist_ok=True)
logger = open("./" + exp_name + "/log", "w")
data = {
'X_train': train_set[0], # training data
'y_train': train_set[1], # training labels
'X_val': valid_set[0], # validation data
'y_val': valid_set[1] # validation labels
}
model = SoftmaxClassifier(input_dim=28 * 28, hidden_dim=1000, reg=0.0)
solver = Solver(model,
data,
logger,
update_rule='sgd',
optim_config={
'learning_rate': 0.02,
},
lr_decay=1,
num_epochs=5,
batch_size=32,
print_every=100)
solver.train()
test_acc = solver.check_accuracy(test_set[0], test_set[1])
toprint = "test_acc: " + str(test_acc)
print(toprint)
def create_best_model(data, best_parmas):
ws, bs, ipa, ld, dp, lr = best_parmas
input_dims = 32 * 32 * 3
hidden_dims = [100, 100, 100, 100, 100]
num_classes = 10
num_epochs = 10
update_rule = 'adam'
model = FullyConnectedNet(input_dims,
hidden_dims,
num_classes,
weight_scale=ws,
use_batchnorm=True,
dropout=dp)
solver = Solver(model,
data,
optim_config={'learning_rate': lr},
batch_size=bs,
iters_per_ann=ipa,
num_epochs=num_epochs,
update_rule=update_rule,
print_every=200,
verbose=True,
lr_decay=ld)
solver.train()
solver.visualization_model()
test_acc = solver.check_accuracy(data['X_test'], data['y_test'])
print('Test accuracy: {}'.format(test_acc))
def runSVM():
with open('../data.pkl', "rb") as f:
X, Y = pickle.load(f, encoding='latin1')
if len(sys.argv) > 1:
exp_name = sys.argv[1]
else:
exp_name = 'exp'
os.makedirs(exp_name, exist_ok=True)
logger = open("./" + exp_name + "/log", "w")
# Y[Y==0] = -1
data = {
'X_train': X[:500], # training data
'y_train': Y[:500], # training labels
'X_val': X[500:750], # validation data
'y_val': Y[500:750] # validation labels
}
model = SVM(input_dim=20, hidden_dim=1000, reg=0.0)
solver = Solver(model,
data,
logger,
update_rule='sgd',
optim_config={
'learning_rate': 0.1,
},
lr_decay=1,
num_epochs=120,
batch_size=50,
print_every=100)
solver.train()
test_acc = solver.check_accuracy(X[750:], Y[750:])
toprint = "test_acc: " + str(test_acc)
print(toprint)
def main():
from keras.datasets import mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
#data preprocessing for neural network with fully-connected layers
# data = {
# 'X_train': np.array(x_train[:55000], np.float32).reshape((55000, -1)), # training data
# 'y_train': np.array(y_train[:55000], np.int32), # training labels
# 'X_val': np.array(x_train[55000:], np.float32).reshape((5000, -1)), # validation data
# 'y_val': np.array(y_train[55000:], np.int32), # validation labels
# }
# model = SoftmaxClassifier(hidden_dim=10000)
# data preprocessing for neural network with convolutional layers
data = {
'X_train':
np.array(x_train[:55000], np.float32).reshape(
(55000, 1, 28, 28)), # training data
'y_train':
np.array(y_train[:55000], np.int32), # training labels
'X_val':
np.array(x_train[55000:], np.float32).reshape(
(5000, 1, 28, 28)), # validation data
'y_val':
np.array(y_train[55000:], np.int32), # validation labels
}
model = ConvNet(hidden_dim=100, filter_size=9)
# running experiments with convolutional neural network could be time-consuming
# you may use a small number of training samples for debugging
# and then take use of all training data to report your experimental results.
solver = Solver(model,
data,
update_rule='sgd',
optim_config={
'learning_rate': 1e-3,
},
lr_decay=0.95,
num_epochs=10,
batch_size=100,
print_every=10)
solver.train()
# Plot the training losses
plt.plot(solver.loss_history)
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.title('Training loss history')
plt.show()
plt.savefig('loss.png')
plt.close()
#test_acc = solver.check_accuracy(X=np.array(x_test, np.float32).reshape((10000, -1)), y=y_test)
test_acc = solver.check_accuracy(X=np.array(x_test, np.float32).reshape(
(10000, 1, 28, 28)),
y=y_test)
print('Test accuracy', test_acc)
def runCNN_multiclass():
f = gzip.open('../mnist.pkl.gz', 'rb')
train_set, valid_set, test_set = pickle.load(f, encoding='latin1')
f.close()
exp_name = sys.argv[1]
if len(sys.argv) > 2:
cont_exp = sys.argv[2]
else:
cont_exp = None
os.makedirs(exp_name, exist_ok=True)
logger = open("./" + exp_name + "/log", "w")
datapack = (np.concatenate((train_set[0], valid_set[0]), axis=0),
np.concatenate((train_set[1], valid_set[1]), axis=0))
data = {
'X_train': datapack[0][:55000], # training data
'y_train': datapack[1][:55000], # training labels
'X_val': datapack[0][55000:], # validation data
'y_val': datapack[1][55000:] # validation labels
}
ConvConfig = {
'input_dim': (1, 28, 28),
'num_filters': 32,
'filter_size': 7,
'hidden_dim': 100,
'num_classes': 10,
'weight_scale': 1e-3,
'reg': 0.,
'bn': True,
'dropout': True,
'cont_exp': cont_exp
}
logger.write(str(ConvConfig) + '\n')
model = ConvNet(**ConvConfig)
solver = Solver(model,
data,
logger,
update_rule='adam',
optim_config={
'learning_rate': 0.001,
},
lr_decay=0.95,
num_epochs=5,
batch_size=100,
print_every=10,
exp_name=exp_name)
solver.train()
test_acc = solver.check_accuracy(test_set[0], test_set[1])
toprint = "test_acc: " + str(test_acc)
print(toprint)
logger.write(toprint)
logger.flush()
def auto_get_params(data):
N = data['X_train'].shape[0]
num_train = np.random.choice(N, 2000, replace=True)
small_data = {
'X_train': data['X_train'][num_train, :],
'y_train': data['y_train'][num_train],
'X_val': data['X_val'],
'y_val': data['y_val']
}
best_acc = -1
best_params = None
results = {}
learning_rate = [0.001, 0.002]
lr_decay = [0.99, 0.97]
weight_scales = [0.001, 0.002]
reg = [0.001, 0.005]
for lr in learning_rate:
for ld in lr_decay:
for ws in weight_scales:
for rs in reg:
print(lr, ld, ws, rs)
model = CNN(weight_scales=ws, reg=rs, use_batchnorm=True)
solver = Solver(model,
small_data,
optim_config={'learning_rate': lr},
batch_size=30,
iters_per_ann=400,
num_epochs=1,
update_rule='adam',
print_every=20,
verbose=True,
lr_decay=ld)
solver.train()
val_acc = solver.check_accuracy(data['X_test'],
data['y_test'])
results[(lr, ld, ws, rs)] = val_acc
if best_acc < val_acc:
best_acc = val_acc
best_params = (lr, ld, ws, rs)
for lr, ld, ws, rs in sorted(results):
val_accuracy = results[(lr, ld, ws, rs)]
print('lr %s, ld %s, ws %s, rs %s, val accuracy: %f' %
(lr, ld, ws, rs, val_accuracy))
print(
'best validation accuracy achieved during cross-validation: {}, which parameters is {}'
.format(best_acc, best_params))
return best_params
def main():
from keras.datasets import mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
# data preprocessing for neural network with fully-connected layers
data = {
'X_train': np.array(x_train[:55000], np.float32).reshape(
(55000, -1)), # training data
'y_train': np.array(y_train[:55000], np.int32), # training labels
'X_val': np.array(x_train[55000:], np.float32).reshape(
(5000, -1)), # validation data
'y_val': np.array(y_train[55000:], np.int32), # validation labels
}
model = softmax.SoftmaxClassifier(hidden_dim=100)
#breakpoint()
# data preprocessing for neural network with convolutional layers
#data = {
# 'X_train': np.array(x_train[:55000], np.float32).reshape((55000, 1, 28, 28)), # training data
# 'y_train': np.array(y_train[:55000], np.int32), # training labels
# 'X_val': np.array(x_train[55000:], np.float32).reshape((5000, 1, 28, 28)), # validation data
# 'y_val': np.array(y_train[55000:], np.int32), # validation labels
#}
#model = cnn.ConvNet(hidden_dim=100)
# the update rule of 'adam' can be used to replace 'sgd' if it is helpful.
solver = Solver(model,
data,
update_rule='sgd',
optim_config={
'learning_rate': 1e-3,
},
lr_decay=0.95,
num_epochs=10,
batch_size=100,
print_every=10)
solver.train()
# Plot the training losses
plt.plot(solver.loss_history)
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.title('Training loss history')
plt.show()
plt.savefig('loss.png')
plt.close()
test_acc = solver.check_accuracy(X=np.array(x_test, np.float32).reshape(
(10000, -1)),
y=y_test)
#test_acc = solver.check_accuracy(X=np.array(x_test, np.float32).reshape((10000, 1, 28, 28)), y=y_test)
print('Test accuracy', test_acc)
def train_model(data):
print('Train model ---------------------------------------------')
model = CNN(weight_scales=0.002, reg=0.001, use_batchnorm=True)
solver = Solver(model,
data,
optim_config={'learning_rate': 1e-3},
batch_size=80,
iters_per_ann=400,
num_epochs=10,
update_rule='adam',
print_every=20,
verbose=True,
lr_decay=0.97)
solver.train()
solver.visualization_model()
test_acc = solver.check_accuracy(data['X_test'], data['y_test'])
print('Test acc: {}'.format(test_acc))
weight_scale=ws,
dtype=np.float32,
seed=None)
solver = Solver(
model,
data,
print_every=400,
num_epochs=50,
batch_size=100,
update_rule='sgd_momentum', # sgd, sgd_momentum, rmsprop, adam
optim_config={
'learning_rate': learnrate,
})
solver.train()
_, y_train_err = solver.check_accuracy(data['X_train'],
data['y_train'])
scores, y_val_err = solver.check_accuracy(data['X_val'],
data['y_val'])
results[(learnrate, reg, ws)] = (y_train_err, y_val_err)
if y_val_err < minErr:
minErr = y_val_err
best_net = solver
best_scores = scores
solver = best_net
for learnrate, reg, ws in sorted(results):
train_err, val_err = results[(learnrate, reg, ws)]
print('lr %e reg %e ws %e train err: %f val err: %f' %
(learnrate, reg, ws, train_err, val_err))
print('minErr achieved during cross-validation: %f' % minErr)
'X_train': x_train,
'y_train': y_train,
'X_val': x_val,
'y_val': y_val,
'X_test': x_test,
'y_test': y_test
}
return data
data = load_data(choice='multi')
print(data['X_train'].shape)
model = SoftmaxClassifier(hidden_dim=50, reg=0)
solver = Solver(model,
data,
update_rule='sgd',
optim_config={
'learning_rate': 2e-3,
},
lr_decay=1,
num_epochs=1,
batch_size=50,
print_every=2)
solver.train()
acc = solver.check_accuracy(X=data['X_test'], y=data['y_test'])
print(acc)
np.save('mnist_data.npy',X)
np.save('mnist_labels.npy',y)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1)
X_train = X_train.reshape(-1, 1, 28, 28)
X_val = X_val.reshape(-1, 1, 28, 28)
X_test = X_test.reshape(-1, 1, 28, 28)
# N, D = X_train.shape
print (X_train.shape)
data = {'X_train': X_train, 'y_train': y_train,
'X_val': X_val, 'y_val': y_val,
'X_test': X_test, 'y_test': y_test
}
model = ConvNet()
solver = Solver(model, data,
update_rule='sgd',
optim_config={
'learning_rate': 2e-3,
},
lr_decay=1,
num_epochs=1, batch_size=50,
print_every=2)
solver.train()
acc = solver.check_accuracy(X=X_test, y=y_test)
print(acc)
x_train, y_train = X[:500], y[:500]
x_val, y_val = X[500:750], y[500:750]
x_test, y_test = X[750:], y[750:]
data = {
'X_train': x_train,
'y_train': y_train,
'X_val': x_val,
'y_val': y_val,
'X_test': x_test,
'y_test': y_test
}
N, D = x_train.shape
model = SVM(input_dim=D, hidden_dim=100, reg=0.05)
solver = Solver(model,
data,
update_rule='sgd',
optim_config={
'learning_rate': 5e-2,
},
lr_decay=1,
num_epochs=1000,
batch_size=20,
print_every=200)
solver.train()
acc = solver.check_accuracy(x_test, y_test)
print(acc)
def auto_get_params(data):
input_dims = 32 * 32 * 3
hidden_dims = [100, 100, 100, 100, 100]
num_classes = 10
N = data['X_train'].shape[0]
num_train = np.random.choice(N, 5000, replace=True)
small_data = {
'X_train': data['X_train'][num_train, :],
'y_train': data['y_train'][num_train],
'X_val': data['X_val'],
'y_val': data['y_val']
}
weight_scale = [2e-2]
batch_size = [250]
iters_per_ann = [400]
num_epochs = 10
update_rule = 'adam'
lr_decay = [0.95]
dropout = [0.9]
learning_rate = [9e-4]
best_params = None
best_acc = -1
results = {}
for ws in weight_scale:
for bs in batch_size:
for ipa in iters_per_ann:
for ld in lr_decay:
for dp in dropout:
for lr in learning_rate:
print(ws, bs, ipa, ld, dp, lr)
model = FullyConnectedNet(input_dims,
hidden_dims,
num_classes,
weight_scale=ws,
use_batchnorm=True,
dropout=dp)
solver = Solver(model,
small_data,
optim_config={'learning_rate': lr},
batch_size=bs,
iters_per_ann=ipa,
num_epochs=num_epochs,
update_rule=update_rule,
print_every=200,
verbose=True,
lr_decay=ld)
solver.train()
val_acc = solver.check_accuracy(
small_data['X_val'], small_data['y_val'])
results[(ws, bs, ipa, ld, dp, lr)] = val_acc
if val_acc > best_acc:
best_acc = val_acc
best_params = (ws, bs, ipa, ld, dp, lr)
best_model = model
for ws, bs, ipa, ld, dp, lr in sorted(results):
val_accuracy = results[(ws, bs, ipa, ld, dp, lr)]
print('ws %s, bs %s, ipa %s, id %s, dp %s, lr %s val accuracy: %f' %
(ws, bs, ipa, ld, dp, lr, val_accuracy))
print(
'best validation accuracy achieved during cross-validation: {}, which parameters is {}'
.format(best_acc, best_params))
return best_params
input, labels = None, None
with open('data.pkl', 'rb') as f:
input, labels = pickle.load(f, encoding='latin1')
data = {}
data['X_train'] = input[:500][:]
data['y_train'] = labels[:500]
data['X_val'] = input[500:750][:]
data['y_val'] = labels[500:750]
data['X_test'] = input[750:][:]
data['y_test'] = labels[750:]
logistic_model = LogisticClassifier(input_dim=data['X_train'].shape[1],
hidden_dim=None,
weight_scale=1e-3,
reg=0.05)
s = Solver(logistic_model,
data,
update_rule='sgd',
optim_config={
'learning_rate': 1000e-3,
},
lr_decay=0.98,
num_epochs=50,
batch_size=20,
print_every=100)
s.train()
acc = s.check_accuracy(data['X_test'], data['y_test'], batch_size=20)
print(acc)
