def compareLight(learn, epochs, itrs, alpha, batchSize, o=10):
np.random.seed(2)
N = 1000
x = np.sort(np.random.uniform(0, 1, N))
y = np.sort(np.random.uniform(0, 1, N))
z = FrankeFunction(x, y)
MSE1, MSE2, MSE3, MSE4 = [], [], [], []
for i in range(o):
X = X_Mat(x, y, o)
B3 = SGD(X, z, learn, epochs, itrs, alpha,
batchSize) #for some reason, OLS and RIDGE act weird
zpred3 = np.dot(X, B3) #if this happens below them. Why I do not know.
B = OLS(X, z)
zpred = np.dot(X, B)
B2 = RIDGE(X, z)
zpred2 = np.dot(X, B2)
sgdreg = SGDRegressor(max_iter=3000,
penalty='l2',
eta0=0.05,
learning_rate='adaptive',
alpha=0.00001,
loss='epsilon_insensitive',
fit_intercept=False)
sgdreg.fit(X, z)
B4 = sgdreg.coef_
zpred4 = np.dot(X, B4)
MSE1.append(MSE(z, zpred))
MSE2.append(MSE(z, zpred2))
MSE3.append(MSE(z, zpred3))
MSE4.append(MSE(z, zpred4))
plt.plot(range(o), MSE1, label='SciKit OLS')
plt.plot(range(o), MSE2, label='SciKit Ridge')
plt.plot(range(o), MSE3, label='SGD')
plt.plot(range(o), MSE4, label='SciKit SGD')
plt.legend()
plt.title(
'Mean-Squared Errors given polynomial order for different methods')
plt.xlabel('Polynomial Order')
plt.ylabel('MSE')
plt.show()
from sklearn.model_selection import train_test_split
# Load the terrain
terrain = imread('SRTM_data_Norway_1.tif')
# just fixing a set of points
N = 1000
o = 5 # polynomial order
k = 5
terrain = terrain[:N,:N]
z = np.ravel(terrain)
# Creates mesh of image pixels
x = np.linspace(0,1, np.shape(terrain)[0])
y = np.linspace(0,1, np.shape(terrain)[1])
x_mesh, y_mesh = np.meshgrid(x,y)
# Note the use of meshgrid
X = X_Mat(x_mesh, y_mesh,o)
X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)
MSE, ldas = crossval(x,y,z,5,5,'Ridge')
MSE = np.mean(MSE, axis=1)
plt.plot(np.log10(ldas),MSE)
plt.xlabel('Log10(lambdas)')
plt.ylabel('MSE')
plt.title('MSE for Ridge using Cross Validation')
plt.show()
plt.figure()
plt.title('Terrain over Norway 1')
plt.imshow(terrain, cmap='gray')
plt.xlabel('X')
plt.ylabel('Y')
plt.show()
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, PolynomialFeatures
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LinearRegression
from sklearn.utils import resample
np.random.seed(80085)
N = 30
nboots = 100
o = 5
amp = 0.2
x = np.sort(np.random.uniform(0, 1, N))
y = np.sort(np.random.uniform(0, 1, N))
z = FrankeFunction(x, y)
z += np.random.normal(0, 1, x.shape) * amp
X = X_Mat(x, y, o)
#b1 = sciOLS(X,z)
#b2 = OLS(X,z)
#zpred = X_test @ b2
#ztilde = X_train @ b2
"""
print(zpred,zmesh)
fig = plt.figure()
heat = plt.imshow(zmesh)
fig.colorbar(heat, shrink=0.5, aspect=5)
plt.show()
"""
vari, bias, error, Type = variancebias('lasso', z, N, nboots=nboots, order=o)
plt.plot(range(o), vari, label='Variance')
def TF(inputs=inputs,
labels=labels,
Type='sgd',
nlayers=[70, 50, 30],
alayers=['sig', 'sig', 'sig'],
outa='sm',
pens=['l2', 'l2', 'l2'],
o=5,
out=10,
epochs=100,
batchSize=100,
learns=np.logspace(-5, 1, 7),
lmbds=np.logspace(-5, 1, 7),
one_return=False,
bestFound=False,
plotting=False,
reg=False):
np.random.seed(2)
if outa.lower() == 'sm':
outa = 'softmax'
elif outa.lower() == 'sig':
outa = 'sigmoid'
if bestFound == True:
Type = 'sgd'
nlayers = [80, 50, 30, 20]
alayers = ['sig', 'sig', 'sig', 'sig']
outa = 'softmax'
pens = ['l2', 'l2', 'l2', 'l2']
out = 10
epochs = 100
batchSize = 100
learns = 1
lmbds = 1e-4
one_return = True
if reg == True:
N = 1000
x = np.sort(np.random.uniform(0, 1, N))
y = np.sort(np.random.uniform(0, 1, N))
labels = FrankeFunction(x, y)
inputs = X_Mat(x, y, o)
model = TensorFlow(Type, nlayers, alayers, outa, pens, out, reg)
scores = model.fitter(inputs, labels, epochs, batchSize, learns, lmbds,
one_return)
if one_return == False and plotting == True and reg == False:
plt.figure()
for i in range(scores.shape[0]):
plt.loglog(lmbds,
scores[i][:],
label=r'$\eta = {}$'.format(learns[i]))
plt.legend()
plt.title(r"""Accuracy for all different $\eta$ with varying $\lambda$
using the {} method""".format(model.Func))
plt.xlabel(r'$\lambda$')
plt.ylabel('Accuracy [%]')
plt.show()
elif one_return == False and plotting == True and reg == True:
plt.figure()
plt.semilogx(learns, scores)
plt.title(
'The R2 Score of a FFNN Regression case with a variable Learning Rate'
)
plt.xlabel('Learning rate')
plt.ylabel('R2 Score')
plt.show()
def soloSGD(learn, epochs, itrs, batchSize, o=10):
np.random.seed(2)
N = 1000
x = np.sort(np.random.uniform(0, 1, N))
y = np.sort(np.random.uniform(0, 1, N))
z = FrankeFunction(x, y)
k = 0
l = 0
m = 0
alpha = 0.9
lst = [learn, epochs, itrs, batchSize]
for i in lst:
if hasattr(i, "__len__") == True:
l = k
array = i
m += 1
k += 1
if m > 1:
raise ValueError('Only one input can be an array-like')
if m != 0:
if l == 0:
error = np.zeros((o, len(learn)))
if l == 1:
error = np.zeros((o, len(epochs)))
if l == 2:
error = np.zeros((o, len(itrs)))
if l == 3:
error = np.zeros((o, len(batchSize)))
for i in range(o):
X = X_Mat(x, y, o)
j = 0
for k in array:
if l == 0:
variable = 'Learning Rate'
B = SGD(X, z, k, epochs, itrs, alpha, batchSize)
zpred = np.dot(X, B)
error[i][j] = MSE(z, zpred)
elif l == 1:
variable = 'Epochs'
B = SGD(X, z, learn, k, itrs, alpha, batchSize)
zpred = np.dot(X, B)
error[i][j] = MSE(z, zpred)
elif l == 2:
variable = 'Iterations'
B = SGD(X, z, learn, epochs, k, alpha, batchSize)
zpred = np.dot(X, B)
error[i][j] = MSE(z, zpred)
elif l == 3:
variable = 'Batch Size'
B = SGD(X, z, learn, epochs, itrs, alpha, k)
zpred = np.dot(X, B)
error[i][j] = MSE(z, zpred)
j += 1
plt.title(
'Mean-Squared-Error as a function of Polynomial Order\n with %s from %g to %g'
% (variable, array[0], array[-1]))
for i in range(len(error[1])):
plt.plot(range(o),
error[:, i],
label='%s = %g' % (variable, array[i]))
plt.xlabel('Polynomial Order')
plt.ylabel('MSE')
plt.legend(loc='upper right')
plt.show()
for i in range(len(error[1])):
print('MSE:', array[i], np.mean(error[:][i]))
else:
error = []
for i in range(o):
X = X_Mat(x, y, o)
B = SGD(X, z, learn, epochs, itrs, alpha, batchSize)
zpred = np.dot(X, B)
error.append(MSE(z, zpred))
plt.plot(range(o), error)
plt.xlabel('Polynomial Order')
plt.ylabel('MSE')
plt.title(
'MSE of our Custom Stochastic Gradient Descent model on its own')
plt.show()
def LogisticRegression(Type,
aFunc='sm',
penalty='l2',
designOrder=5,
iters=600,
epochs=200,
alpha=0.9,
kappa=1,
batchSize=50,
learn=0.001,
plotting=False,
bestFound=False):
np.random.seed(2)
k = 0
l = 0
m = 0
vari = False
lst = [designOrder, iters, batchSize, learn, epochs]
for i in lst:
if hasattr(i, "__len__") == True:
vari = True
l = k
array = i
m += 1
k += 1
if m > 1:
raise ValueError('Only one input can be an array-like')
if bestFound == True: #will use the best parameters found by me whilst coding and testing
print(
'Fitting with best possible parameters\n'
) #note that these can probably be optimized further, these are just some examples of good results
if Type.lower() == 'gd': #gives an MSE of about ~ 0.02
designOrder = 5
aFunc = 'relu'
penalty = 'l2'
iters = 800
epochs = 500
alpha = 0.6
learn = 1
bestArray = [iters, epochs, alpha, learn]
bestFunc = 'Rectified Linear Unit'
elif Type.lower() == 'sgdmb': #gives an MSE of about ~ 0.07
designOrder = 5
aFunc = 'elu'
penalty = 'l2'
iters = 400
alpha = 0.8
epochs = 200
batchSize = 50
learn = 0.0005
bestArray = [iters, epochs, alpha, batchSize, learn]
bestFunc = 'Exponential Linear Unit'
else: #assumes normal SGD w/o mini batches, MSE of about ~ 0.11
designOrder = 5
aFunc = 'sp'
penalty = 'l2'
iters = 600
epochs = 200
alpha = 0.7
learn = 0.006
kappa = 2.3
bestArray = [iters, epochs, alpha, kappa, learn]
bestFunc = 'Softplus'
N = 1000
x = np.sort(np.random.uniform(0, 1, N))
y = np.sort(np.random.uniform(0, 1, N))
z = FrankeFunction(x, y)
ploter = plt.plot
if vari == True:
Blst = []
for i in array:
if l == 0:
variable = 'Polynomial Order'
X = X_Mat(x, y, i)
X_train, X_test, z_train, z_test = train_test_split(
X, z, test_size=0.2)
model = LR(X, z, Type, aFunc, iters, epochs, penalty, alpha, k,
batchSize)
model.fitter(X_train, z_train, learn)
Blst.append(model.B)
elif l == 1:
variable = 'Iterations'
X = X_Mat(x, y, designOrder)
X_train, X_test, z_train, z_test = train_test_split(
X, z, test_size=0.2)
model = LR(X, z, Type, aFunc, i, epochs, penalty, alpha, k,
batchSize)
model.fitter(X_train, z_train, learn)
Blst.append(model.B)
elif l == 2:
variable = 'Batch Size'
X = X_Mat(x, y, designOrder)
X_train, X_test, z_train, z_test = train_test_split(
X, z, test_size=0.2)
model = LR(X, z, Type, aFunc, iters, epochs, penalty, k, alpha,
i)
model.fitter(X_train, z_train, learn)
Blst.append(model.B)
elif l == 3:
variable = 'Learning Rate'
X = X_Mat(x, y, designOrder)
X_train, X_test, z_train, z_test = train_test_split(
X, z, test_size=0.2)
model = LR(X, z, Type, aFunc, iters, epochs, penalty, alpha, k,
batchSize)
model.fitter(X_train, z_train, i)
Blst.append(model.B)
ploter = plt.semilogx
elif l == 4:
variable = 'Epochs'
X = X_Mat(x, y, designOrder)
X_train, X_test, z_train, z_test = train_test_split(
X, z, test_size=0.2)
model = LR(X, z, Type, aFunc, iters, i, penalty, alpha, k,
batchSize)
model.fitter(X_train, z_train, learn)
Blst.append(model.B)
msestore = []
for i in range(len(array)):
zpred4 = np.dot(X, Blst[i])
print('\n%s: %g\nMSE: %g' % (variable, array[i], MSE(z, zpred4)))
msestore.append(MSE(z, zpred4))
if plotting == True:
plt.title('Mean-Squared-Error as a function of %s\nfrom %g to %g' %
(variable, array[0], array[-1]))
ploter(array, msestore)
plt.xlabel('%s' % (variable))
plt.ylabel('MSE')
plt.show()
else:
X = X_Mat(x, y, designOrder)
X_train, X_test, z_train, z_test = train_test_split(X,
z,
test_size=0.2)
model = LR(X, z, Type, aFunc, iters, epochs, penalty, alpha, kappa,
batchSize)
model.fitter(X_train, z_train, learn)
B4 = model.B
zpred4 = np.dot(X, B4)
print('MSE: %g || R2: %g' % (MSE(z, zpred4), R2(z, zpred4)))
if bestFound == True:
if Type.lower() == 'sgdmb':
print(
'\nActivation Function: %s\n%s\nUsing the variables:\nIterations: %g\nEpochs: %g\nAlpha: %g\nBatch Size: %g\nLearning Rate: %g'
% (bestFunc, '=' * 45, bestArray[0], bestArray[1],
bestArray[2], bestArray[3], bestArray[4]))
elif Type.lower() == 'sgd':
print(
'Activation Function: %s\n%s\nUsing the variables:\nIterations: %g\nEpochs: %g\nAlpha Parameter: %g\nSharpness Parameter: %g\nLearning Rate: %g'
% (bestFunc, '=' * 30, bestArray[0], bestArray[1],
bestArray[2], bestArray[3], bestArray[4]))
else:
print(
'Activation Function: %s\n%s\nUsing the variables:\nIterations: %g\nEpochs: %g\nAlpha: %g\nLearning Rate: %g'
% (bestFunc, '=' * 45, bestArray[0], bestArray[1],
bestArray[2], bestArray[3]))
ax = fig.gca(projection='3d')
np.random.seed(80085)
N = 500
o = 7
x = np.sort(np.random.uniform(0,1,N))
y = np.sort(np.random.uniform(0,1,N))
xm, ym = np.meshgrid(x,y)
z = FrankeFunction(xm,ym)
#z =+ np.random.normal(size = (N,N))
xr = np.linspace(0,1,N)
yr = np.linspace(0,1,N)
xm, ym = np.meshgrid(x,y)
def y_t(Xr,B):
return Xr @ B
X = X_Mat(xm,ym,o)
Xr = X_Mat(xm,ym,o)
B, ztilde, zpred = OLS(X,z)
surf = ax.plot_surface(xm, ym, (y_t(Xr,B)).reshape((N,N)),cmap=cm.coolwarm,linewidth=0, antialiased=False)
ax.set_zlim(-0.10, 1.40)
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))
fig.colorbar(surf, shrink=0.5, aspect=5)
for angle in range(0, 360):
ax.view_init(30, angle)
plt.draw()
plt.pause(.001)
#Franke3D(x,y)
z = np.ravel(z)
X_train,X_test,z_Train,z_Test = train_test_split(X,z,test_size=0.2)
from main import TF, LogisticRegression, MNISTsolver, compareLight, soloSGD import numpy as np from sklearn import datasets from functions import X_Mat, FrankeFunction #usually the function doesn't need an input, but just in case. digits = datasets.load_digits() inputs = digits.images labels = digits.target n_inputs = len(inputs) inputs = inputs.reshape(n_inputs, -1) N = 1000 x = np.sort(np.random.uniform(0, 1, N)) y = np.sort(np.random.uniform(0, 1, N)) z = FrankeFunction(x, y) X = X_Mat(x, y, 5) #An example of a decent result of our custom SGD in relation to SciKit methods #compareLight(0.6, 300, 600, 0.9, 100) #An example of our custom SGD alone with a variable learning rate (and polynomial order) #soloSGD(np.logspace(-5,1,7),200,200,50,o=10) #An example of our custom SGD alone with a variable epoch number (and polynomial order) #soloSGD(0.6, [20,50,100,200],600 ,50 , o=10) #An example of our custom SGD alone with a variable iteration count (and polynomial order) #soloSGD(0.6,200,[100,200,300,500],50,o=10) #An example of our custom SGD alone with a variable batch count (and polynomial order) #soloSGD(0.6,200,600,[10,30,50,70,100],o=10)