def wine_test(epochs=200, batch_size=16, lr=0.1):
print(
'wine test: params - epochs {0}, batch_size: {1}, learning rate: {2}'.
format(epochs, batch_size, lr))
features, target = load_wine(return_X_y=True)
# Make a train/test split using 30% test size
RANDOM_STATE = 42
X_train, X_test, y_train, y_test = train_test_split(
features, target, test_size=0.9, random_state=RANDOM_STATE)
sc = StandardScaler(
) # makes every features zero mean standard deviation 1 - makes learning problem much easier (massages the error function so sgd works better)
sc.fit(X_train)
s = SoftmaxClassifier(num_classes=3)
X_train = sc.transform(X_train)
X_train = np.c_[np.ones(X_train.shape[0]), X_train] # adds bias var
X_test = sc.transform(X_test)
X_test = np.c_[np.ones(X_test.shape[0]), X_test] # adds bias var
s.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, lr=lr)
print('Softmax Wine Classifier')
print_score(s, X_train, X_test, y_train, y_test)
hist = s.history
fig, ax = plt.subplots()
ax.plot(np.array(range(1, 1 + len(hist))), hist, 'b-x')
ax.set_title('Cost as a function of epoch for wine data')
ax.set_xlabel('epoch')
ax.set_ylabel('Ein (1/n NLL)')
export_fig(fig, 'softmax_wine_cost_per_epoch.png')
plt.show()
def pick_hyperparams(X_train, y_train, X_val, y_val, learning_rates, regularization_strengths, iterations=4000, batches=400):
results = {}
best_val = -1
best_softmax = None
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# Save the best trained softmax classifer in best_softmax. #
# Hint: about 10 lines of code expected
################################################################################
for lr in learning_rates:
for reg in regularization_strengths:
print lr, reg
softmax = SoftmaxClassifier()
softmax.train(X_train, y_train, lr, reg, iterations, batches)
predy = softmax.predict(X_val)
val = np.mean(predy == y_val)
predtrain = softmax.predict(X_train)
valtrain = np.mean(predtrain == y_val)
results[(lr, reg)] = (valtrain, val)
if val > best_val:
best_val = val
best_softmax = softmax
return best_softmax, results, best_val
def digits_visualize(epochs=1, batch_size=64, lr=0.01):
sc = SoftmaxClassifier(num_classes=10)
X_train, y_train = load_digits_train_data()
sc.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, lr=lr)
w = sc.W
rs = w.reshape(28, 28, 10, order='F')
rs2 = np.transpose(rs, axes=[1, 0, 2])
fig, ax = plt.subplots()
ax.imshow(rs2.reshape(28, -1, order='F'), cmap='bone')
ax.set_title('digits weight vector visualized')
export_fig(fig, 'softmax_weight_vector.png')
plt.show()
def runSoftmax():
# Load the dataset
f = gzip.open('../mnist.pkl.gz', 'rb')
train_set, valid_set, test_set = pickle.load(f, encoding='latin1')
f.close()
if len(sys.argv) > 1:
exp_name = sys.argv[1]
else:
exp_name = 'exp'
os.makedirs(exp_name, exist_ok=True)
logger = open("./" + exp_name + "/log", "w")
data = {
'X_train': train_set[0], # training data
'y_train': train_set[1], # training labels
'X_val': valid_set[0], # validation data
'y_val': valid_set[1] # validation labels
}
model = SoftmaxClassifier(input_dim=28 * 28, hidden_dim=1000, reg=0.0)
solver = Solver(model,
data,
logger,
update_rule='sgd',
optim_config={
'learning_rate': 0.02,
},
lr_decay=1,
num_epochs=5,
batch_size=32,
print_every=100)
solver.train()
test_acc = solver.check_accuracy(test_set[0], test_set[1])
toprint = "test_acc: " + str(test_acc)
print(toprint)
def digits_test(epochs=10, batch_size=32, lr=0.05):
print(
'digits test: params - epochs {0}, batch_size: {1}, learning rate: {2}'
.format(epochs, batch_size, lr))
sc = SoftmaxClassifier(num_classes=10)
X_train, y_train = load_digits_train_data()
X_test, y_test = load_digits_test_data()
sc.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, lr=lr)
print_score(sc, X_train, X_test, y_train, y_test)
fig, ax = plt.subplots()
hist = sc.history
ax.plot(np.array(range(1, 1 + len(hist))), hist, 'b-x')
ax.set_xlabel('epoch')
ax.set_ylabel('Ein (1/n NLL)')
ax.set_title('softmax cost on digits as function of epoch')
export_fig(fig, 'softmax_cost_per_epoch.png')
plt.show()
def get_classifier(classifier):
if classifier == 'svc':
return LinearSVC()
elif classifier == 'softmax':
return SoftmaxClassifier()
elif classifier == 'xgb':
return xgb.XGBClassifier()
else:
return None
def main():
from keras.datasets import mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
# pdb.set_trace()
# data preprocessing for neural network with fully-connected layers2222
data = {
'X_train': np.array(x_train[:55000], np.float32).reshape(
(55000, -1)), # training data
'y_train': np.array(y_train[:55000], np.int32), # training labels
'X_val': np.array(x_train[55000:], np.float32).reshape(
(5000, -1)), # validation data
'y_val': np.array(y_train[55000:], np.int32), # validation labels
}
model = SoftmaxClassifier(hidden_dim=100)
# data preprocessing for neural network with convolutional layers333333
# data = {
# 'X_train': np.array(x_train[:5000], np.float32).reshape((5000, 1, 28, 28)), # training data
# 'y_train': np.array(y_train[:5000], np.int32), # training labels
# 'X_val': np.array(x_train[55000:], np.float32).reshape((5000), 1, 28, 28), # validation data
# 'y_val': np.array(y_train[55000:], np.int32), # validation labels
# }
# model = ConvNet(hidden_dim=100)
# running experiments with convolutional neural network could be time-consuming
# you may use a small number of training samples for debugging
# and then take use of all training data to report your experimental results.
solver = Solver(model,
data,
update_rule='sgd',
optim_config={
'learning_rate': 1e-3,
},
lr_decay=0.95,
num_epochs=10,
batch_size=100,
print_every=10)
solver.train()
# Plot the training losses
plt.plot(solver.loss_history)
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.title('Training loss history')
plt.show()
plt.savefig('loss.png')
plt.close()
test_acc = solver.check_accuracy(X=np.array(x_test, np.float32).reshape(
(10000, -1)),
y=y_test)
# test_acc = solver.check_accuracy(X=np.array(x_test, np.float32).reshape((10000, 1, 28, 28)), y=y_test)
print('Test accuracy', test_acc)
def train():
# load data
data = load_cifar10()
train_data = {k: data[k] for k in ['X_train', 'y_train', 'X_val', 'y_val']}
# initialize model
model = SoftmaxClassifier(hidden_dim=300)
# start training
#######################################################################
# TODO: Set up model hyperparameters #
#######################################################################
model, train_acc_history, val_acc_history = trainNetwork(
model,
train_data,
learning_rate=5e-3,
lr_decay=0.9,
num_epochs=40,
batch_size=128,
print_every=1000)
#######################################################################
# END OF YOUR CODE #
#######################################################################
# report test accuracy
acc = testNetwork(model, data['X_test'], data['y_test'])
print("Test accuracy: {}".format(acc))
#######################################################################
# Save your model with model.save(filepath) once you finish training #
#######################################################################
model.save('wh')
import matplotlib.pyplot as plt
plt.plot(train_acc_history, label='train')
plt.plot(val_acc_history, label='validation')
plt.legend(loc='upper left')
plt.savefig('pltwohidden.jpg')
>>>>>>> 89dd6a53aa0ff700b713b57c5d8d001424557b1d
# TODO: Use the validation set to tune hyperparameters for softmax classifier
# choose learning rate and regularization strength (use the code from softmax_hw.py)
<<<<<<< HEAD
results = {}
best_val = -1
best_softmax = None
learning_rates = [1e-4,1e-3,1e-2,1e-1]
regularization_strengths = [0.01,0.05,0.1,0.5,1]
for lr in learning_rates:
for rs in regularization_strengths:
print("calculating: lr=%e,reg=%e"%(lr,rs))
ns=SoftmaxClassifier()
ns.train(X_train,y_train,lr,rs,batch_size=400,num_iters=2000)
ta=np.mean(y_train == ns.predict(X_train))
va=np.mean(y_val == ns.predict(X_val))
results[lr,rs]=(ta,va)
if va>best_val:
best_val=va
best_softmax=ns
# TODO: Evaluate best softmax classifier on set aside test set (use the code from softmax_hw.py)
# Print out results.
for lr, reg in sorted(results):
train_accuracy, val_accuracy = results[(lr, reg)]
print 'lr %e reg %e train accuracy: %f val accuracy: %f' % (
print("Fetching")
X , y = fetch_openml('mnist_784',version=1,return_X_y=True)
y = y.astype(np.int32)
np.save('mnist_data.npy',X)
np.save('mnist_labels.npy',y)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1)
print (X_train.shape)
data = {'X_train': X_train, 'y_train': y_train,
'X_val': X_val, 'y_val': y_val,
'X_test': X_test, 'y_test': y_test
}
model = SoftmaxClassifier()
solver = Solver(model, data,
update_rule='sgd',
optim_config={
'learning_rate': 2e-3,
},
lr_decay=1,
num_epochs=1, batch_size=50,
print_every=2)
solver.train()
acc = solver.check_accuracy(X=X_test, y=y_test)
print(acc)
# We use the Frobenius norm to compare the two versions
# of the gradient.
grad_difference = np.linalg.norm(grad_naive - grad_vectorized, ord='fro')
print 'Loss difference: %f' % np.abs(loss_naive - loss_vectorized)
print 'Gradient difference: %f' % grad_difference
# Use the validation set to tune hyperparameters (regularization strength and
# learning rate). You should experiment with different ranges for the learning
# rates and regularization strengths; if you are careful you should be able to
# get a classification accuracy of over 0.35 on the validation set and the test set.
results = {}
best_val = -1
best_softmax = SoftmaxClassifier()
learning_rates = [2e-6]#[1e-5, 2e-6, 12-7]
regularization_strengths = [0.01]#[.1, .01, .05, .005]
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# Save the best trained softmax classifer in best_softmax. #
# Hint: about 10 lines of code expected
################################################################################
classifier = SoftmaxClassifier()
accuracy = lambda x,y: np.mean(classifier.predict(x) == y)
for lr in learning_rates:
for reg in regularization_strengths:
classifier.train(X_train, y_train, learning_rate=lr, reg=reg)
import numpy as np
from data import make_spiral_data, plot_2d_data
from softmax import SoftmaxClassifier
if __name__ == '__main__':
# generate spiral data
n_classes = 3
data, y = make_spiral_data(100, n_classes, 2)
fig = plot_2d_data(data, y)
fig.show()
# train model
model = SoftmaxClassifier(n_classes)
model.fit(data, y)
print("Training accuracy is {:0.2f}".format(model.training_accuracy))
print("Training loss is {:0.3f}".format(model.training_loss))
boundaries = model.plot_boundaries(data, y)
boundaries.show()
loss_vs_epoch = model.plot_training_loss()
loss_vs_epoch.show()
# test model against 'new' data
new_data, new_y = make_spiral_data(50, n_classes, 2)
predictions = model.predict(new_data)
print("Test accuracy is {:0.2f}".format(model.accuracy(new_data, new_y)))
# work with large amount of files (50 000 froze my laptop completely until reboot) #classify = one_vs_allLogisticRegressor(np.arange(10)) #best_params, best_classifier = utils.getBestRegOVA(classify, X_train, y_train, X_val, y_val, regularization_strengths, pen='l1') """ #best LR = [15] learning_rates = [15] # [11,12,13,14,15,16,17,18,19,20] #Best RS = [1e-3] regularization_strengths = [2e-3] #[1e-3,2e-3,3e-3,4e-3,5e-3,6e-3,7e-3] classify = SoftmaxClassifier() # takes classifier, X_train, y_train, X_val, y_val, learning_rates, regularization_strengths and optionally print_train and print_val best_params, best_classifier = utils.getBestRegAndLearnSoftMax(classify, X_train, y_train, X_val, y_val, learning_rates, regularization_strengths) # end """ print "\nTraining the classifier..." best_classifier = SoftmaxClassifier() best_classifier.train(X_train, y_train, reg=1e1, learning_rate=15) print np.mean(best_classifier.predict(X_val) == y_val) print theta.shape hogimg = best_classifier.theta[1025:1537,:].reshape(8,8,8,10) utils.visualizeHOGTheta(hogimg) print "\nMaking the final prediction..." sys.stdout.flush() labels = [] ids = [] batch = 50000 for j in range(3, 6): print "\nPart ", j + 1, " of 3" sys.stdout.flush()
np.savetxt("accuracies.txt",accuracies,delimiter=',')
print "best batch size is ", best_batch_size
print "best iteration is ", best_iteration
'''
best_iteration = 3000
best_batch_size = 500
# select lambda using fmin train
best_softmax_fmin = None
best_val_fmin=-1
for reg in regularization_strengths:
softmax=SoftmaxClassifier()
softmax.train(X_train,y_train,best_learning_rate,reg=best_reg, num_iters=best_iteration,
batch_size=best_batch_size, verbose=True)
y_pred_val=softmax.predict(X_val)
current_val = np.mean(y_pred_val==y_val)
if(current_val>best_val_fmin):
best_val_fmin = current_val
best_softmax_fmin = softmax
best_reg_fmin = reg
print "best reg for fmin is ", best_reg_fmin
if best_softmax_fmin:
y_test_pred = best_softmax_fmin.predict(X_test)
test_accuracy = np.mean(y_test == y_test_pred)
print 'softmax_fmin on raw pixels final test set accuracy: %f' % (test_accuracy, )
MUSIC_DIR = "../music/"
genres = ["blues", "classical", "country", "disco", "hiphop", "jazz", "metal", "pop", "reggae", "rock"]
X, y = music_utils.read_ceps(genres, MUSIC_DIR)
# X,y = music_utils.read_fft(genres,MUSIC_DIR)
# TODO: Split into train, validation and test sets
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.2)
X_train, X_val, y_train, y_val = cross_validation.train_test_split(X_train, y_train, test_size=0.1)
# TODO: Use the validation set to tune hyperparameters for softmax classifier
# choose learning rate and regularization strength (use the code from softmax_hw.py)
results = {}
best_val = -1
best_params = None
best_softmax = SoftmaxClassifier()
learning_rates = [0.005]
regularization_strengths = [1]
classifier = SoftmaxClassifier()
accuracy = lambda x, y: np.mean(classifier.predict(x) == y)
for lr in learning_rates:
for reg in regularization_strengths:
classifier.train(X_train, y_train, learning_rate=lr, reg=reg, num_iters=10000, batch_size=y_train.size / 4)
val_accuracy = accuracy(X_val, y_val)
train_accuracy = accuracy(X_train, y_train)
results[(lr, reg)] = (train_accuracy, val_accuracy)
print "lr %e reg %e train accuracy: %f val accuracy: %f" % (lr, reg, train_accuracy, val_accuracy)
if val_accuracy > best_val:
# We use the Frobenius norm to compare the two versions
# of the gradient.
grad_difference = np.linalg.norm(grad_naive - grad_vectorized, ord='fro')
print 'Loss difference: %f' % np.abs(loss_naive - loss_vectorized)
print 'Gradient difference: %f' % grad_difference
results = {}
best_val = -1
best_softmax = None
#learning_rates = [1e-7, 5e-7, 1e-6, 5e-6]
#regularization_strengths = [ 5e4, 1e5, 5e5, 1e8]
learning_rates = [5e-6]
regularization_strengths = [1e5]
cl = SoftmaxClassifier()
loss_hist = []
for lr, rs in itertools.product(learning_rates, regularization_strengths):
_ = cl.train(X_train,
y_train,
lr,
rs,
num_iters=4000,
batch_size=400,
verbose=True)
loss, _ = cl.loss(X_val, y_val, rs)
pred_t = cl.predict(X_train)
pred_v = cl.predict(X_val)
#embed()
train_match = np.where(pred_t == y_train)
train_accuracy = float(len(train_match[0])) / len(y_train)
data = {
'X_train': x_train,
'y_train': y_train,
'X_val': x_val,
'y_val': y_val,
'X_test': x_test,
'y_test': y_test
}
return data
data = load_data(choice='multi')
print(data['X_train'].shape)
model = SoftmaxClassifier(hidden_dim=50, reg=0)
solver = Solver(model,
data,
update_rule='sgd',
optim_config={
'learning_rate': 2e-3,
},
lr_decay=1,
num_epochs=1,
batch_size=50,
print_every=2)
solver.train()
acc = solver.check_accuracy(X=data['X_test'], y=data['y_test'])
# We use the Frobenius norm to compare the two versions
# of the gradient.
grad_difference = np.linalg.norm(grad_naive - grad_vectorized, ord='fro')
print 'Loss difference: %f' % np.abs(loss_naive - loss_vectorized)
print 'Gradient difference: %f' % grad_difference
results = {}
best_val = -1
best_softmax = None
#learning_rates = [1e-7, 5e-7, 1e-6, 5e-6]
#regularization_strengths = [ 5e4, 1e5, 5e5, 1e8]
learning_rates = [ 5e-6]
regularization_strengths = [1e5]
cl = SoftmaxClassifier()
loss_hist = []
for lr,rs in itertools.product(learning_rates,regularization_strengths):
_ = cl.train(X_train, y_train, lr, rs, num_iters = 4000, batch_size = 400, verbose = True)
loss,_ = cl.loss(X_val,y_val, rs)
pred_t = cl.predict(X_train)
pred_v = cl.predict(X_val)
#embed()
train_match = np.where(pred_t == y_train)
train_accuracy = float(len(train_match[0]))/len(y_train)
val_match = np.where(pred_v == y_val)
val_accuracy = float(len(val_match[0]))/len(y_val)
results[(lr,rs)] = (train_accuracy,val_accuracy)
loss_hist.append(loss)
# print("For lr,rs = ",lr,rs, "Loss value = ",loss)
X_train, X_val, y_train, y_val = cross_validation.train_test_split(X_train, y_train, test_size=0.1)
# TODO: Use the validation set to tune hyperparameters for softmax classifier
# choose learning rate and regularization strength (use the code from softmax_hw.py)
batch_sizes = [200, 300, 400]
iterations = [1000,2000,3000]
learning_rates = [5e-7, 1e-6, 5e-6]
regularization_strengths = [1e2, 1e3,1e4, 1e5]
best_val = -1
best_softmax = None
for batch_idx, batch_size in enumerate(batch_sizes):
for it_idx, iteration in enumerate(iterations):
for learningrate in learning_rates:
for regularization in regularization_strengths:
softmax=SoftmaxClassifier()
softmax.train(X_train,y_train,learningrate,reg=regularization, num_iters=iteration,batch_size=batch_size, verbose=True)
y_pred_val=softmax.predict(X_val)
current_val = np.mean(y_pred_val==y_val)
if(current_val>best_val):
best_val = current_val
best_softmax = softmax
best_learning_rate = learningrate
best_reg = regularization
best_iteration = iteration
best_batch_size = batch_size
print "best batch size is ", best_batch_size
print "best iteration is ", best_iteration
print "best reg is ", best_reg
