k = 10000
nn_cache = "../Data/hw2-data/MNIST_nn_cache.npz"
if not os.path.exists(nn_cache):
RuntimeError("Transformation matrix v doesn't exist!")
else:
print("Reading from cache...")
res = np.load(nn_cache)
v = res["v"]
# Load in MNIST training data then transform it
print("Loading MNIST Training data...")
X_train, y_train = mu.load_mnist(dataset='training')
X_train = ru.naive_nn_layer(X_train, k=k, v=v)
y_train_true = np.asarray(y_train[:, None] == np.arange(max(y_train)+1),dtype=int).squeeze()
print("Estimated best lambda: %.3lf" % val.estimate_lambda(X_train, scale=1.0e-2))
# Load in MNIST training data then transform it
print("Loading MNIST Testing data...")
X_test, y_test = mu.load_mnist(dataset='testing')
X_test = ru.naive_nn_layer(X_test, k=k, v=v)
y_test_true = np.asarray(y_test[:, None] == np.arange(max(y_test)+1),dtype=int).squeeze()
#######################################################
#
# Run minibatch with bestfit?
#
#######################################################
if run_minibatch_sgd:
# Performance
X_train, y_train = mu.load_mnist(dataset='training')
# Transform X
print("Performing naive neural network layer transformation on training set...")
X_train = ru.naive_nn_layer(X_train, k=k, v=v)
# Load in MNIST data
print("Loading MNIST Testing data...")
X_test, y_test = mu.load_mnist(dataset='testing')
# Transform X
print("Performing naive neural network layer transformation on testing set...")
X_test = ru.naive_nn_layer(X_test, k=k, v=v)
# Fit for the class prediction regression coefficients on transformed training set
best_lambda = val.estimate_lambda(X_train, scale=1.0)
print("Best lambda: %.3lf." % best_lambda)
y_train_true = np.asarray(y_train[:, None] == np.arange(max(y_train)+1),dtype=int).squeeze()
y_test_true = np.asarray(y_test[:, None] == np.arange(max(y_test)+1),dtype=int).squeeze()
print("Fitting with ridge regression...")
w0, w = ri.fit_ridge(X_train, y_train_true, lam=best_lambda)
# Using fit on training set, predict labels for train, test data by selecting whichever
# prediction is the largest (one vs all classification)
y_hat_train = cu.multi_linear_classifier(X_train, w, w0)
y_hat_test = cu.multi_linear_classifier(X_test, w, w0)
# Compute 01 Loss!
print("Training 01 Loss:",val.loss_01(y_train,y_hat_train))
print("Testing 01 Loss:",val.loss_01(y_test,y_hat_test))
"Performing naive neural network layer transformation on training set..."
)
X_train = ru.naive_nn_layer(X_train, k=k, v=v)
# Load in MNIST data
print("Loading MNIST Testing data...")
X_test, y_test = mu.load_mnist(dataset='testing')
# Transform X
print(
"Performing naive neural network layer transformation on testing set..."
)
X_test = ru.naive_nn_layer(X_test, k=k, v=v)
# Fit for the class prediction regression coefficients on transformed training set
best_lambda = val.estimate_lambda(X_train, scale=1.0)
print("Best lambda: %.3lf." % best_lambda)
y_train_true = np.asarray(y_train[:, None] == np.arange(max(y_train) + 1),
dtype=int).squeeze()
y_test_true = np.asarray(y_test[:, None] == np.arange(max(y_test) + 1),
dtype=int).squeeze()
print("Fitting with ridge regression...")
w0, w = ri.fit_ridge(X_train, y_train_true, lam=best_lambda)
# Using fit on training set, predict labels for train, test data by selecting whichever
# prediction is the largest (one vs all classification)
y_hat_train = cu.multi_linear_classifier(X_train, w, w0)
y_hat_test = cu.multi_linear_classifier(X_test, w, w0)
# Compute 01 Loss!
print("Training 01 Loss:", val.loss_01(y_train, y_hat_train))
# Load in MNIST training data
print("Loading MNIST Testing data...")
X_test, y_test = mu.load_mnist(dataset='testing')
y_test_true = np.asarray(y_test[:, None] == np.arange(max(y_test) + 1),
dtype=int).squeeze()
#######################################################
#
# Run reg path over lambda, eta?
#
#######################################################
if find_best_lam:
print("Running regularization path to optmize lambda, eta...")
print("Estimated best lambda: %.3lf" %
val.estimate_lambda(X_train, scale=1.0e-1))
# Split training data into subtraining set, validation set
X_tr, y_tr, X_val, y_val = val.split_data(X_train,
y_train,
frac=frac,
seed=seed)
# Filter y values to 0, 1 labels
y_tr_true = np.asarray(y_tr[:, None] == np.arange(max(y_tr) + 1),
dtype=int).squeeze()
y_val_true = np.asarray(y_val[:, None] == np.arange(max(y_val) + 1),
dtype=int).squeeze()
# Define arrays
eta_arr = np.logspace(-7, -3, num)
save_plots = True
run_sgd = True
run_minibatch_sgd = True
# Classifier parameters
best_lambda = 0.1 # Found via reg path
eps = 1.0e-3
seed = 42
sparse = False
Nclass = 10
# Load in MNIST training data
print("Loading MNIST Training data...")
X_train, y_train = mu.load_mnist(dataset='training')
y_train_true = np.asarray(y_train[:, None] == np.arange(max(y_train)+1),dtype=int).squeeze()
print("Estimated best lambda: %.3lf" % val.estimate_lambda(X_train))
# Load in MNIST training data
print("Loading MNIST Testing data...")
X_test, y_test = mu.load_mnist(dataset='testing')
y_test_true = np.asarray(y_test[:, None] == np.arange(max(y_test)+1),dtype=int).squeeze()
#######################################################
#
# Run minibatch with bestfit?
#
#######################################################
# Search for the best regularization constant using a regularization path?
# Use minibatch SGD to find it for speed/convergence reasons
if find_best_lambda:
