def sgdm(m, degrees, n_epochs, b, eta, noise=0, gamma = 0): #stocastic gradient decent with momentum
np.random.seed(1337)
x = np.random.rand(m,degrees) #+1?
y = np.random.rand(m,degrees) #+1?
X_mesh, Y_mesh = np.meshgrid(x, y)
z = f.FrankeFunction(X_mesh, Y_mesh) + noise*np.random.randn(X_mesh.shape[0], Y_mesh.shape[0])
z= np.ravel(z)
X = f.X_make(X_mesh,Y_mesh, degrees)
#SPLIT AND SCALE
X_tr, X_te, z_tr, z_te = train_test_split(X,z, test_size=0.3)
scaler = StandardScaler()
# X_tr = scaler.fit(X_tr).transform(X_tr)
# z_tr = scaler.transform(z_tr.reshape(-1,1))
# z_te = scaler.fit(z_te).transform(z_te)
# removes the mean and scales each feature/variable to unit variance
scaler.fit(X_tr) # compute the mean and std to be used for later scaling
X_tr= scaler.transform(X_tr) # perform standardization by centering and scaling
X_te = scaler.transform(X_te) # fit to data, then transform it
z_tr = z_tr.reshape(-1,1)
z_te = z_te.reshape(-1,1)
scaler.fit(z_tr)
z_tr = scaler.transform(z_tr)
z_te = scaler.transform(z_te)
l = int((degrees+1)*(degrees+2)/2) #length of design matrix row
beta = np.random.randn(l,1) #length of a design matrix row
#b = int(m/batch_num) #batch size
batch_num = int(m/b)
if m%batch_num:
print('warning; batch number and dataset not compatible')
v = 0
mse_eval = np.zeros(n_epochs)
index_array = np.arange(m) #indexes of rows
batch_array= np.arange(batch_num)
batch_array *=b
for epoch in range(n_epochs):
np.random.shuffle(index_array)
for i in range(batch_num): #m is number of batches
xi = X_tr[index_array[batch_array[i]]: (index_array[(batch_array[i]+1)])]
zi = z_tr[index_array[batch_array[i]]: (index_array[(batch_array[i]+1)])]
gradients = 2/b * xi.T @ (xi @ beta - zi.reshape(-1,1)) #derived from cost function
#eta = 0.001#learning_rate(epoch*m+i)
v = gamma*v + eta*gradients
beta = beta - v
z_eval = X_te.dot(beta)
mse_eval[epoch] = f.MSE(z_te, z_eval)
beta_ols = f.OLS(X_tr, z_tr)
z_ols = X_te.dot(beta_ols)
mse_beta = f.MSE(z_te, z_ols)
return beta, mse_eval, mse_beta
def bootstrap(x, y, max_deg, boots_num):
np.random.seed(130)
"""
applies the bootstrap algorithm
args:
x,y (np.array): initial datapoints
max_deg (int):
boots_num (int): number of bootstraps
"""
x, y = np.meshgrid(x, y)
z = np.ravel(
f.FrankeFunction(x, y) +
0.5 * np.random.randn(np.shape(x)[0],
np.shape(y)[1]))
MSE_degree_values = np.zeros(max_deg)
MSE_test_degree_values = np.zeros(max_deg)
MSE_train_values = np.zeros(boots_num)
MSE_test_values = np.zeros(boots_num)
for k, deg in enumerate(range(1, max_deg)):
#Degrees loop that contains K-fold
X_design = f.X_make(x, y, deg)
scaler = StandardScaler()
X_tr, X_te, z_tr, z_te = train_test_split(X_design, z, test_size=0.2)
scaler.fit(X_tr)
X_train = scaler.transform(X_tr)
X_test = scaler.transform(X_te)
#doing this AFTER train test split. otherwise the test data
#gets affected by the train data
z_bootstrap = np.empty(int(len(z_tr)))
index_array = np.arange(0, len(z_tr), 1)
for i in range(boots_num):
indx = resample(index_array, random_state=0)
z_bootstrap = z_tr[indx]
z_test = X_test.dot(f.OLS(X_train[indx, :], z_bootstrap))
z_train = X_train.dot(f.OLS(X_train[indx, :], z_bootstrap))
MSE_train_values[i] = f.MSE(z_tr, z_train)
MSE_test_values[i] = f.MSE(z_te, z_test)
MSE_degree_values[k] = np.sum(MSE_train_values) / boots_num
MSE_test_degree_values[k] = np.sum(MSE_test_values) / boots_num
return MSE_degree_values, MSE_test_degree_values
n_epochs = 300 n_batches = 100 eta = 0.5 lmbda = 0 neurons = [10, 10] n_outputs = 1 hidden_act = act.Sigmoid() output_act = act.Identity() # create data using franke function seed = 2034 np.random.seed(seed) x = np.sort(np.random.uniform(0, 1, n)) y = np.sort(np.random.uniform(0, 1, n)) x, y = np.meshgrid(x, y) z = np.ravel(f.FrankeFunction(x, y) + 0.1*np.random.randn(x.shape[0], x.shape[1])) z = z.reshape(-1, 1) # set up the design matrix data = DataPrep() X = data.design_matrix(x, y, degree=1)[:, 1:] # split data in train and test and scale it X_train, X_test, z_train, z_test = data.train_test_scale(X, z) # set up the neural network network = NeuralNetwork(X_train.shape[1], neurons, n_outputs, cost.MSE()) network.create_layers(hidden_act, output_act, seed) # train the network batch_size = len(X_train)//n_batches
def d_relu(x):
return 1. * (x > 0)
np.random.seed(5)
n = 5 # Project 1
N = 200 # Project 1
k = 20 # Project 1
x = np.sort(np.random.uniform(0, 1, N))
y = np.sort(np.random.uniform(0, 1, N))
X = func.create_X(x, y, n)
XX, YY = np.meshgrid(x, y)
ZZ = func.FrankeFunction(XX, YY)
z = func.Create_data(XX, YY, ZZ, noise=True)
X = func.create_X(XX, YY, n)
X_train, X_test, y_train, y_test = train_test_split(X,
z,
test_size=1. / k)
epochs = 500 # 1000
batch_size = 100
X_train, X_test = X_train.T, X_test.T
eta_vals = np.logspace(-6, -4, 4)
lmbd_vals = np.logspace(-6, -4, 4)
MSE_array = np.zeros(
def no_resampling(n, maxdegree, noise, method=f.OLS, lmbda=0, seed=7053):
# arrays for plotting of error
polydegree = np.zeros(maxdegree)
MSE_OLS = np.zeros(maxdegree)
R2Score_OLS = np.zeros(maxdegree)
MSE_test = np.zeros(maxdegree)
MSE_train = np.zeros(maxdegree)
MSE_train_scaled = np.zeros(maxdegree)
MSE_test_scaled = np.zeros(maxdegree)
R2Score_scaled = np.zeros(maxdegree)
# Make data
np.random.seed(seed)
x = np.sort(np.random.uniform(0, 1, n))
y = np.sort(np.random.uniform(0, 1, n))
x, y = np.meshgrid(x, y)
# Franke Function
z = np.ravel(f.FrankeFunction(x, y) + noise * np.random.randn(n, n))
for degree in range(0, maxdegree):
polydegree[degree] = degree
#Create design matrix
X = f.design_matrix(x, y, degree)
# Split in training and test data
X_train, X_test, z_train, z_test = train_test_split(X,
z.reshape(-1, 1),
test_size=0.3)
# OLS estimate train/test without scaled
beta_OLS_train = f.OLS(X_train, z_train)
ztilde_test = X_test @ beta_OLS_train
ztilde_train = X_train @ beta_OLS_train
MSE_train[degree] = f.MSE(z_train, ztilde_train)
MSE_test[degree] = f.MSE(z_test, ztilde_test)
# Scale data
scaler = StandardScaler(
) # removes the mean and scales each feature/variable to unit variance
scaler.fit(
X_train) # compute the mean and std to be used for later scaling
X_train_scaled = scaler.transform(
X_train) # perform standardization by centering and scaling
X_test_scaled = scaler.transform(
X_test) # fit to data, then transform it
scaler.fit(z_train)
# z_train_scaled = scaler.transform(z_train)
# z_test_scaled = scaler.transform(z_test)
z_train_scaled = z_train
z_test_scaled = z_test
# Set the first column to 1 since StandardScaler sets it to 0
X_train_scaled[:, 0] = 1
X_test_scaled[:, 0] = 1
if method == f.OLS:
beta_train_scaled = method(X_train_scaled, z_train_scaled)
z_tilde_test_scaled = X_test_scaled @ beta_train_scaled
z_tilde_train_scaled = X_train_scaled @ beta_train_scaled
elif method == f.Ridge:
beta_train_scaled = method(X_train_scaled, z_train_scaled, lmbda,
degree)
z_tilde_test_scaled = X_test_scaled @ beta_train_scaled
z_tilde_train_scaled = X_train_scaled @ beta_train_scaled
elif method == 'Lasso':
clf_lasso = skl.Lasso(alpha=lmbda, fit_intercept=False).fit(
X_train_scaled, z_train_scaled)
z_tilde_test_scaled = clf_lasso.predict(X_test_scaled)
z_tilde_train_scaled = clf_lasso.predict(X_train_scaled)
MSE_train_scaled[degree] = f.MSE(z_train_scaled, z_tilde_train_scaled)
MSE_test_scaled[degree] = f.MSE(z_test_scaled, z_tilde_test_scaled)
R2Score_scaled[degree] = f.R2Score(z_test_scaled, z_tilde_test_scaled)
return polydegree, MSE_train, MSE_test, MSE_train_scaled, MSE_test_scaled, R2Score_scaled
# Start figure
fig = plt.figure()
ax = fig.gca(projection='3d')
# Plot the surface
# ztilde_plot = np.reshape(ztilde, (n, n))
# surf = ax.plot_surface(x, y, ztilde_plot, cmap=cm.coolwarm, linewidth=0, antialiased=False)
# Customize the z axis
ax.set_zlim(-0.10, 1.40)
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))
# Add a colar bar which maps values to colors
fig.colorbar(surf, shrink=0.5, aspect=5)
plt.show()
np.random.seed(1488)
max_degree = 14
dp = 20
deg = np.arange(0, max_degree, 1)
MSEtest = np.zeros(max_degree)
MSEtrain = np.zeros(max_degree)
for i, degree in enumerate(deg):
# Make data.
x = np.linspace(0, 1, dp)
y = np.linspace(0, 1, dp)
x, y = np.meshgrid(x, y)
X = f.X_make(x, y, degree)
z = f.FrankeFunction(x, y) + 0.1 * np.random.randn(dp, dp)
z = np.ravel(z)
#seperate and scale
X_train, X_test, z_train, z_test = train_test_split(X, z, test_size=0.2)
print(f"z: {np.shape(z)}")
scaler = StandardScaler()
scaler.fit(X)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
print(f" z_train: {np.shape(z_train)}")
print(f" X: {np.shape(X)}")
print(f" X_train: {np.shape(X_train)}")
import matplotlib.pyplot as plt
from p1_a import no_resampling
n = 100
maxdegree = 5
noise = 0.1
seed = 4018
# Make data
np.random.seed(seed)
x = np.sort(np.random.uniform(0, 1, n))
y = np.sort(np.random.uniform(0, 1, n))
x, y = np.meshgrid(x, y)
# Franke Function
z = np.ravel(f.FrankeFunction(x, y) + noise*np.random.randn(n, n))
for degree in range(maxdegree):
#Create design matrix
X = f.design_matrix(x, y, degree)
X_train, X_test, z_train, z_test = train_test_split(X, z)
# Ordinary least squares
beta_OLS = f.OLS(X, z)
ztilde = X @ beta_OLS
# Confidence interval as function of beta
var_beta = f.variance_beta(X, noise)
err_beta = 1.96*np.sqrt(var_beta) # 95% confidence interval
beta_idx = np.linspace(0, X.shape[1] - 1, X.shape[1])
