def wine_test(epochs=200, batch_size=16, lr=0.1): print( 'wine test: params - epochs {0}, batch_size: {1}, learning rate: {2}'. format(epochs, batch_size, lr)) features, target = load_wine(return_X_y=True) # Make a train/test split using 30% test size RANDOM_STATE = 42 X_train, X_test, y_train, y_test = train_test_split( features, target, test_size=0.9, random_state=RANDOM_STATE) sc = StandardScaler( ) # makes every features zero mean standard deviation 1 - makes learning problem much easier (massages the error function so sgd works better) sc.fit(X_train) s = SoftmaxClassifier(num_classes=3) X_train = sc.transform(X_train) X_train = np.c_[np.ones(X_train.shape[0]), X_train] # adds bias var X_test = sc.transform(X_test) X_test = np.c_[np.ones(X_test.shape[0]), X_test] # adds bias var s.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, lr=lr) print('Softmax Wine Classifier') print_score(s, X_train, X_test, y_train, y_test) hist = s.history fig, ax = plt.subplots() ax.plot(np.array(range(1, 1 + len(hist))), hist, 'b-x') ax.set_title('Cost as a function of epoch for wine data') ax.set_xlabel('epoch') ax.set_ylabel('Ein (1/n NLL)') export_fig(fig, 'softmax_wine_cost_per_epoch.png') plt.show()
def runSoftmax(): # Load the dataset f = gzip.open('../mnist.pkl.gz', 'rb') train_set, valid_set, test_set = pickle.load(f, encoding='latin1') f.close() if len(sys.argv) > 1: exp_name = sys.argv[1] else: exp_name = 'exp' os.makedirs(exp_name, exist_ok=True) logger = open("./" + exp_name + "/log", "w") data = { 'X_train': train_set[0], # training data 'y_train': train_set[1], # training labels 'X_val': valid_set[0], # validation data 'y_val': valid_set[1] # validation labels } model = SoftmaxClassifier(input_dim=28 * 28, hidden_dim=1000, reg=0.0) solver = Solver(model, data, logger, update_rule='sgd', optim_config={ 'learning_rate': 0.02, }, lr_decay=1, num_epochs=5, batch_size=32, print_every=100) solver.train() test_acc = solver.check_accuracy(test_set[0], test_set[1]) toprint = "test_acc: " + str(test_acc) print(toprint)
def get_classifier(classifier): if classifier == 'svc': return LinearSVC() elif classifier == 'softmax': return SoftmaxClassifier() elif classifier == 'xgb': return xgb.XGBClassifier() else: return None
def digits_visualize(epochs=1, batch_size=64, lr=0.01): sc = SoftmaxClassifier(num_classes=10) X_train, y_train = load_digits_train_data() sc.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, lr=lr) w = sc.W rs = w.reshape(28, 28, 10, order='F') rs2 = np.transpose(rs, axes=[1, 0, 2]) fig, ax = plt.subplots() ax.imshow(rs2.reshape(28, -1, order='F'), cmap='bone') ax.set_title('digits weight vector visualized') export_fig(fig, 'softmax_weight_vector.png') plt.show()
def main(): from keras.datasets import mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() # pdb.set_trace() # data preprocessing for neural network with fully-connected layers2222 data = { 'X_train': np.array(x_train[:55000], np.float32).reshape( (55000, -1)), # training data 'y_train': np.array(y_train[:55000], np.int32), # training labels 'X_val': np.array(x_train[55000:], np.float32).reshape( (5000, -1)), # validation data 'y_val': np.array(y_train[55000:], np.int32), # validation labels } model = SoftmaxClassifier(hidden_dim=100) # data preprocessing for neural network with convolutional layers333333 # data = { # 'X_train': np.array(x_train[:5000], np.float32).reshape((5000, 1, 28, 28)), # training data # 'y_train': np.array(y_train[:5000], np.int32), # training labels # 'X_val': np.array(x_train[55000:], np.float32).reshape((5000), 1, 28, 28), # validation data # 'y_val': np.array(y_train[55000:], np.int32), # validation labels # } # model = ConvNet(hidden_dim=100) # running experiments with convolutional neural network could be time-consuming # you may use a small number of training samples for debugging # and then take use of all training data to report your experimental results. solver = Solver(model, data, update_rule='sgd', optim_config={ 'learning_rate': 1e-3, }, lr_decay=0.95, num_epochs=10, batch_size=100, print_every=10) solver.train() # Plot the training losses plt.plot(solver.loss_history) plt.xlabel('Iteration') plt.ylabel('Loss') plt.title('Training loss history') plt.show() plt.savefig('loss.png') plt.close() test_acc = solver.check_accuracy(X=np.array(x_test, np.float32).reshape( (10000, -1)), y=y_test) # test_acc = solver.check_accuracy(X=np.array(x_test, np.float32).reshape((10000, 1, 28, 28)), y=y_test) print('Test accuracy', test_acc)
def digits_test(epochs=10, batch_size=32, lr=0.05): print( 'digits test: params - epochs {0}, batch_size: {1}, learning rate: {2}' .format(epochs, batch_size, lr)) sc = SoftmaxClassifier(num_classes=10) X_train, y_train = load_digits_train_data() X_test, y_test = load_digits_test_data() sc.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, lr=lr) print_score(sc, X_train, X_test, y_train, y_test) fig, ax = plt.subplots() hist = sc.history ax.plot(np.array(range(1, 1 + len(hist))), hist, 'b-x') ax.set_xlabel('epoch') ax.set_ylabel('Ein (1/n NLL)') ax.set_title('softmax cost on digits as function of epoch') export_fig(fig, 'softmax_cost_per_epoch.png') plt.show()
def train(): # load data data = load_cifar10() train_data = {k: data[k] for k in ['X_train', 'y_train', 'X_val', 'y_val']} # initialize model model = SoftmaxClassifier(hidden_dim=300) # start training ####################################################################### # TODO: Set up model hyperparameters # ####################################################################### model, train_acc_history, val_acc_history = trainNetwork( model, train_data, learning_rate=5e-3, lr_decay=0.9, num_epochs=40, batch_size=128, print_every=1000) ####################################################################### # END OF YOUR CODE # ####################################################################### # report test accuracy acc = testNetwork(model, data['X_test'], data['y_test']) print("Test accuracy: {}".format(acc)) ####################################################################### # Save your model with model.save(filepath) once you finish training # ####################################################################### model.save('wh') import matplotlib.pyplot as plt plt.plot(train_acc_history, label='train') plt.plot(val_acc_history, label='validation') plt.legend(loc='upper left') plt.savefig('pltwohidden.jpg')
import numpy as np from data import make_spiral_data, plot_2d_data from softmax import SoftmaxClassifier if __name__ == '__main__': # generate spiral data n_classes = 3 data, y = make_spiral_data(100, n_classes, 2) fig = plot_2d_data(data, y) fig.show() # train model model = SoftmaxClassifier(n_classes) model.fit(data, y) print("Training accuracy is {:0.2f}".format(model.training_accuracy)) print("Training loss is {:0.3f}".format(model.training_loss)) boundaries = model.plot_boundaries(data, y) boundaries.show() loss_vs_epoch = model.plot_training_loss() loss_vs_epoch.show() # test model against 'new' data new_data, new_y = make_spiral_data(50, n_classes, 2) predictions = model.predict(new_data) print("Test accuracy is {:0.2f}".format(model.accuracy(new_data, new_y)))
data = { 'X_train': x_train, 'y_train': y_train, 'X_val': x_val, 'y_val': y_val, 'X_test': x_test, 'y_test': y_test } return data data = load_data(choice='multi') print(data['X_train'].shape) model = SoftmaxClassifier(hidden_dim=50, reg=0) solver = Solver(model, data, update_rule='sgd', optim_config={ 'learning_rate': 2e-3, }, lr_decay=1, num_epochs=1, batch_size=50, print_every=2) solver.train() acc = solver.check_accuracy(X=data['X_test'], y=data['y_test'])
# We use the Frobenius norm to compare the two versions # of the gradient. grad_difference = np.linalg.norm(grad_naive - grad_vectorized, ord='fro') print 'Loss difference: %f' % np.abs(loss_naive - loss_vectorized) print 'Gradient difference: %f' % grad_difference results = {} best_val = -1 best_softmax = None #learning_rates = [1e-7, 5e-7, 1e-6, 5e-6] #regularization_strengths = [ 5e4, 1e5, 5e5, 1e8] learning_rates = [5e-6] regularization_strengths = [1e5] cl = SoftmaxClassifier() loss_hist = [] for lr, rs in itertools.product(learning_rates, regularization_strengths): _ = cl.train(X_train, y_train, lr, rs, num_iters=4000, batch_size=400, verbose=True) loss, _ = cl.loss(X_val, y_val, rs) pred_t = cl.predict(X_train) pred_v = cl.predict(X_val) #embed() train_match = np.where(pred_t == y_train) train_accuracy = float(len(train_match[0])) / len(y_train)
print("Fetching") X , y = fetch_openml('mnist_784',version=1,return_X_y=True) y = y.astype(np.int32) np.save('mnist_data.npy',X) np.save('mnist_labels.npy',y) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1) X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1) print (X_train.shape) data = {'X_train': X_train, 'y_train': y_train, 'X_val': X_val, 'y_val': y_val, 'X_test': X_test, 'y_test': y_test } model = SoftmaxClassifier() solver = Solver(model, data, update_rule='sgd', optim_config={ 'learning_rate': 2e-3, }, lr_decay=1, num_epochs=1, batch_size=50, print_every=2) solver.train() acc = solver.check_accuracy(X=X_test, y=y_test) print(acc)