def runSoftmax(): # Load the dataset f = gzip.open('../mnist.pkl.gz', 'rb') train_set, valid_set, test_set = pickle.load(f, encoding='latin1') f.close() if len(sys.argv) > 1: exp_name = sys.argv[1] else: exp_name = 'exp' os.makedirs(exp_name, exist_ok=True) logger = open("./" + exp_name + "/log", "w") data = { 'X_train': train_set[0], # training data 'y_train': train_set[1], # training labels 'X_val': valid_set[0], # validation data 'y_val': valid_set[1] # validation labels } model = SoftmaxClassifier(input_dim=28 * 28, hidden_dim=1000, reg=0.0) solver = Solver(model, data, logger, update_rule='sgd', optim_config={ 'learning_rate': 0.02, }, lr_decay=1, num_epochs=5, batch_size=32, print_every=100) solver.train() test_acc = solver.check_accuracy(test_set[0], test_set[1]) toprint = "test_acc: " + str(test_acc) print(toprint)
def create_best_model(data, best_parmas): ws, bs, ipa, ld, dp, lr = best_parmas input_dims = 32 * 32 * 3 hidden_dims = [100, 100, 100, 100, 100] num_classes = 10 num_epochs = 10 update_rule = 'adam' model = FullyConnectedNet(input_dims, hidden_dims, num_classes, weight_scale=ws, use_batchnorm=True, dropout=dp) solver = Solver(model, data, optim_config={'learning_rate': lr}, batch_size=bs, iters_per_ann=ipa, num_epochs=num_epochs, update_rule=update_rule, print_every=200, verbose=True, lr_decay=ld) solver.train() solver.visualization_model() test_acc = solver.check_accuracy(data['X_test'], data['y_test']) print('Test accuracy: {}'.format(test_acc))
def runSVM(): with open('../data.pkl', "rb") as f: X, Y = pickle.load(f, encoding='latin1') if len(sys.argv) > 1: exp_name = sys.argv[1] else: exp_name = 'exp' os.makedirs(exp_name, exist_ok=True) logger = open("./" + exp_name + "/log", "w") # Y[Y==0] = -1 data = { 'X_train': X[:500], # training data 'y_train': Y[:500], # training labels 'X_val': X[500:750], # validation data 'y_val': Y[500:750] # validation labels } model = SVM(input_dim=20, hidden_dim=1000, reg=0.0) solver = Solver(model, data, logger, update_rule='sgd', optim_config={ 'learning_rate': 0.1, }, lr_decay=1, num_epochs=120, batch_size=50, print_every=100) solver.train() test_acc = solver.check_accuracy(X[750:], Y[750:]) toprint = "test_acc: " + str(test_acc) print(toprint)
def main(): from keras.datasets import mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() #data preprocessing for neural network with fully-connected layers # data = { # 'X_train': np.array(x_train[:55000], np.float32).reshape((55000, -1)), # training data # 'y_train': np.array(y_train[:55000], np.int32), # training labels # 'X_val': np.array(x_train[55000:], np.float32).reshape((5000, -1)), # validation data # 'y_val': np.array(y_train[55000:], np.int32), # validation labels # } # model = SoftmaxClassifier(hidden_dim=10000) # data preprocessing for neural network with convolutional layers data = { 'X_train': np.array(x_train[:55000], np.float32).reshape( (55000, 1, 28, 28)), # training data 'y_train': np.array(y_train[:55000], np.int32), # training labels 'X_val': np.array(x_train[55000:], np.float32).reshape( (5000, 1, 28, 28)), # validation data 'y_val': np.array(y_train[55000:], np.int32), # validation labels } model = ConvNet(hidden_dim=100, filter_size=9) # running experiments with convolutional neural network could be time-consuming # you may use a small number of training samples for debugging # and then take use of all training data to report your experimental results. solver = Solver(model, data, update_rule='sgd', optim_config={ 'learning_rate': 1e-3, }, lr_decay=0.95, num_epochs=10, batch_size=100, print_every=10) solver.train() # Plot the training losses plt.plot(solver.loss_history) plt.xlabel('Iteration') plt.ylabel('Loss') plt.title('Training loss history') plt.show() plt.savefig('loss.png') plt.close() #test_acc = solver.check_accuracy(X=np.array(x_test, np.float32).reshape((10000, -1)), y=y_test) test_acc = solver.check_accuracy(X=np.array(x_test, np.float32).reshape( (10000, 1, 28, 28)), y=y_test) print('Test accuracy', test_acc)
def runCNN_multiclass(): f = gzip.open('../mnist.pkl.gz', 'rb') train_set, valid_set, test_set = pickle.load(f, encoding='latin1') f.close() exp_name = sys.argv[1] if len(sys.argv) > 2: cont_exp = sys.argv[2] else: cont_exp = None os.makedirs(exp_name, exist_ok=True) logger = open("./" + exp_name + "/log", "w") datapack = (np.concatenate((train_set[0], valid_set[0]), axis=0), np.concatenate((train_set[1], valid_set[1]), axis=0)) data = { 'X_train': datapack[0][:55000], # training data 'y_train': datapack[1][:55000], # training labels 'X_val': datapack[0][55000:], # validation data 'y_val': datapack[1][55000:] # validation labels } ConvConfig = { 'input_dim': (1, 28, 28), 'num_filters': 32, 'filter_size': 7, 'hidden_dim': 100, 'num_classes': 10, 'weight_scale': 1e-3, 'reg': 0., 'bn': True, 'dropout': True, 'cont_exp': cont_exp } logger.write(str(ConvConfig) + '\n') model = ConvNet(**ConvConfig) solver = Solver(model, data, logger, update_rule='adam', optim_config={ 'learning_rate': 0.001, }, lr_decay=0.95, num_epochs=5, batch_size=100, print_every=10, exp_name=exp_name) solver.train() test_acc = solver.check_accuracy(test_set[0], test_set[1]) toprint = "test_acc: " + str(test_acc) print(toprint) logger.write(toprint) logger.flush()
def auto_get_params(data): N = data['X_train'].shape[0] num_train = np.random.choice(N, 2000, replace=True) small_data = { 'X_train': data['X_train'][num_train, :], 'y_train': data['y_train'][num_train], 'X_val': data['X_val'], 'y_val': data['y_val'] } best_acc = -1 best_params = None results = {} learning_rate = [0.001, 0.002] lr_decay = [0.99, 0.97] weight_scales = [0.001, 0.002] reg = [0.001, 0.005] for lr in learning_rate: for ld in lr_decay: for ws in weight_scales: for rs in reg: print(lr, ld, ws, rs) model = CNN(weight_scales=ws, reg=rs, use_batchnorm=True) solver = Solver(model, small_data, optim_config={'learning_rate': lr}, batch_size=30, iters_per_ann=400, num_epochs=1, update_rule='adam', print_every=20, verbose=True, lr_decay=ld) solver.train() val_acc = solver.check_accuracy(data['X_test'], data['y_test']) results[(lr, ld, ws, rs)] = val_acc if best_acc < val_acc: best_acc = val_acc best_params = (lr, ld, ws, rs) for lr, ld, ws, rs in sorted(results): val_accuracy = results[(lr, ld, ws, rs)] print('lr %s, ld %s, ws %s, rs %s, val accuracy: %f' % (lr, ld, ws, rs, val_accuracy)) print( 'best validation accuracy achieved during cross-validation: {}, which parameters is {}' .format(best_acc, best_params)) return best_params
def main(): from keras.datasets import mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() # data preprocessing for neural network with fully-connected layers data = { 'X_train': np.array(x_train[:55000], np.float32).reshape( (55000, -1)), # training data 'y_train': np.array(y_train[:55000], np.int32), # training labels 'X_val': np.array(x_train[55000:], np.float32).reshape( (5000, -1)), # validation data 'y_val': np.array(y_train[55000:], np.int32), # validation labels } model = softmax.SoftmaxClassifier(hidden_dim=100) #breakpoint() # data preprocessing for neural network with convolutional layers #data = { # 'X_train': np.array(x_train[:55000], np.float32).reshape((55000, 1, 28, 28)), # training data # 'y_train': np.array(y_train[:55000], np.int32), # training labels # 'X_val': np.array(x_train[55000:], np.float32).reshape((5000, 1, 28, 28)), # validation data # 'y_val': np.array(y_train[55000:], np.int32), # validation labels #} #model = cnn.ConvNet(hidden_dim=100) # the update rule of 'adam' can be used to replace 'sgd' if it is helpful. solver = Solver(model, data, update_rule='sgd', optim_config={ 'learning_rate': 1e-3, }, lr_decay=0.95, num_epochs=10, batch_size=100, print_every=10) solver.train() # Plot the training losses plt.plot(solver.loss_history) plt.xlabel('Iteration') plt.ylabel('Loss') plt.title('Training loss history') plt.show() plt.savefig('loss.png') plt.close() test_acc = solver.check_accuracy(X=np.array(x_test, np.float32).reshape( (10000, -1)), y=y_test) #test_acc = solver.check_accuracy(X=np.array(x_test, np.float32).reshape((10000, 1, 28, 28)), y=y_test) print('Test accuracy', test_acc)
def train_model(data): print('Train model ---------------------------------------------') model = CNN(weight_scales=0.002, reg=0.001, use_batchnorm=True) solver = Solver(model, data, optim_config={'learning_rate': 1e-3}, batch_size=80, iters_per_ann=400, num_epochs=10, update_rule='adam', print_every=20, verbose=True, lr_decay=0.97) solver.train() solver.visualization_model() test_acc = solver.check_accuracy(data['X_test'], data['y_test']) print('Test acc: {}'.format(test_acc))
weight_scale=ws, dtype=np.float32, seed=None) solver = Solver( model, data, print_every=400, num_epochs=50, batch_size=100, update_rule='sgd_momentum', # sgd, sgd_momentum, rmsprop, adam optim_config={ 'learning_rate': learnrate, }) solver.train() _, y_train_err = solver.check_accuracy(data['X_train'], data['y_train']) scores, y_val_err = solver.check_accuracy(data['X_val'], data['y_val']) results[(learnrate, reg, ws)] = (y_train_err, y_val_err) if y_val_err < minErr: minErr = y_val_err best_net = solver best_scores = scores solver = best_net for learnrate, reg, ws in sorted(results): train_err, val_err = results[(learnrate, reg, ws)] print('lr %e reg %e ws %e train err: %f val err: %f' % (learnrate, reg, ws, train_err, val_err)) print('minErr achieved during cross-validation: %f' % minErr)
'X_train': x_train, 'y_train': y_train, 'X_val': x_val, 'y_val': y_val, 'X_test': x_test, 'y_test': y_test } return data data = load_data(choice='multi') print(data['X_train'].shape) model = SoftmaxClassifier(hidden_dim=50, reg=0) solver = Solver(model, data, update_rule='sgd', optim_config={ 'learning_rate': 2e-3, }, lr_decay=1, num_epochs=1, batch_size=50, print_every=2) solver.train() acc = solver.check_accuracy(X=data['X_test'], y=data['y_test']) print(acc)
np.save('mnist_data.npy',X) np.save('mnist_labels.npy',y) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1) X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1) X_train = X_train.reshape(-1, 1, 28, 28) X_val = X_val.reshape(-1, 1, 28, 28) X_test = X_test.reshape(-1, 1, 28, 28) # N, D = X_train.shape print (X_train.shape) data = {'X_train': X_train, 'y_train': y_train, 'X_val': X_val, 'y_val': y_val, 'X_test': X_test, 'y_test': y_test } model = ConvNet() solver = Solver(model, data, update_rule='sgd', optim_config={ 'learning_rate': 2e-3, }, lr_decay=1, num_epochs=1, batch_size=50, print_every=2) solver.train() acc = solver.check_accuracy(X=X_test, y=y_test) print(acc)
x_train, y_train = X[:500], y[:500] x_val, y_val = X[500:750], y[500:750] x_test, y_test = X[750:], y[750:] data = { 'X_train': x_train, 'y_train': y_train, 'X_val': x_val, 'y_val': y_val, 'X_test': x_test, 'y_test': y_test } N, D = x_train.shape model = SVM(input_dim=D, hidden_dim=100, reg=0.05) solver = Solver(model, data, update_rule='sgd', optim_config={ 'learning_rate': 5e-2, }, lr_decay=1, num_epochs=1000, batch_size=20, print_every=200) solver.train() acc = solver.check_accuracy(x_test, y_test) print(acc)
def auto_get_params(data): input_dims = 32 * 32 * 3 hidden_dims = [100, 100, 100, 100, 100] num_classes = 10 N = data['X_train'].shape[0] num_train = np.random.choice(N, 5000, replace=True) small_data = { 'X_train': data['X_train'][num_train, :], 'y_train': data['y_train'][num_train], 'X_val': data['X_val'], 'y_val': data['y_val'] } weight_scale = [2e-2] batch_size = [250] iters_per_ann = [400] num_epochs = 10 update_rule = 'adam' lr_decay = [0.95] dropout = [0.9] learning_rate = [9e-4] best_params = None best_acc = -1 results = {} for ws in weight_scale: for bs in batch_size: for ipa in iters_per_ann: for ld in lr_decay: for dp in dropout: for lr in learning_rate: print(ws, bs, ipa, ld, dp, lr) model = FullyConnectedNet(input_dims, hidden_dims, num_classes, weight_scale=ws, use_batchnorm=True, dropout=dp) solver = Solver(model, small_data, optim_config={'learning_rate': lr}, batch_size=bs, iters_per_ann=ipa, num_epochs=num_epochs, update_rule=update_rule, print_every=200, verbose=True, lr_decay=ld) solver.train() val_acc = solver.check_accuracy( small_data['X_val'], small_data['y_val']) results[(ws, bs, ipa, ld, dp, lr)] = val_acc if val_acc > best_acc: best_acc = val_acc best_params = (ws, bs, ipa, ld, dp, lr) best_model = model for ws, bs, ipa, ld, dp, lr in sorted(results): val_accuracy = results[(ws, bs, ipa, ld, dp, lr)] print('ws %s, bs %s, ipa %s, id %s, dp %s, lr %s val accuracy: %f' % (ws, bs, ipa, ld, dp, lr, val_accuracy)) print( 'best validation accuracy achieved during cross-validation: {}, which parameters is {}' .format(best_acc, best_params)) return best_params
input, labels = None, None with open('data.pkl', 'rb') as f: input, labels = pickle.load(f, encoding='latin1') data = {} data['X_train'] = input[:500][:] data['y_train'] = labels[:500] data['X_val'] = input[500:750][:] data['y_val'] = labels[500:750] data['X_test'] = input[750:][:] data['y_test'] = labels[750:] logistic_model = LogisticClassifier(input_dim=data['X_train'].shape[1], hidden_dim=None, weight_scale=1e-3, reg=0.05) s = Solver(logistic_model, data, update_rule='sgd', optim_config={ 'learning_rate': 1000e-3, }, lr_decay=0.98, num_epochs=50, batch_size=20, print_every=100) s.train() acc = s.check_accuracy(data['X_test'], data['y_test'], batch_size=20) print(acc)