def makeImageLabel(self,
minibatchSize=None,
epochN=None,
outputFunction=None):
"""Imports dataset from sklearn.datasets.load_digits().
Args:
minibatchSize (int, optional): Number of datapoints calculated at a time. Defaults to 64.
epochN (int, optional): Number of times dataset is iterated through. Defaults to 500.
outputFunction (function, optional) : Function input data is put through
"""
self.outputFunction = defaultingFunc(self.outputFunction,
outputFunction, softMax)
self.minibatchSize = defaultingFunc(self.minibatchSize, minibatchSize,
64)
self.epochN = defaultingFunc(self.epochN, epochN, 500)
from sklearn import datasets
digits = datasets.load_digits()
digits.images = digits.images
digits.target = digits.target
images = digits.images
labels = digits.target
images = images.reshape(len(images), -1)
labels = labels.reshape(len(labels), -1)
labels = to_categorical_numpy(labels)
self.X = images
self.z = labels
self.N, self.nfeatures = self.X.shape
self.minibatchN = int(self.N / self.minibatchSize)
def giveInput(self, X=None, z=None, scaling=None):
"""Takes input and output values
Args:
X (numpy.nparray, optional) : Input data. Defaults to previously given self.X.
z (numpy.ndarray, optional) : Expected data, used for training. Defaults to previously given self.z.
scaling (bool, optional) : Determines whether or not to scale the data. Defaults to False.
"""
self.X = defaultingFunc(self.X, X, self.X)
self.z = defaultingFunc(self.z, z, self.z)
scaling = defaultingFunc(None, scaling, False)
if scaling:
if self.X[:, 0].any() != 1:
X = np.empty(self.X.shape[0], self.X.shape[1] + 1)
X[:, 1:] = self.X
X[:, 0] = 1
self.X = X
self.X[:, 1:] -= np.mean(self.X[:, 1:], axis=0)
self.Ninputs, self.nfeatures = self.X.shape
self.Ncategories = self.z.shape[1]
self.activationValues = np.zeros(
(self.hiddenLN, self.Ninputs, self.hiddenNN))
def classify(self):
"""The main Stochastic Gradient Descent method
Generates a random theta using numpy.random.rand(self.nfeatures,1) and
adjusts it using the SGD method
Will call makeImageLabel() if self.X is not yet defined and defineStepLength()
if self.stepLength is not yet defined, using default values for each
"""
if self.X is None:
print(
"in LogisticClassifier.classify(): using defaulting self.makeImageLabel() due to self.X not being defined"
)
regressor.makeImageLabel()
if self.stepLength is None:
print(
"in LogisticClassifier.classify(): using defaulting self.defineStepLength() due to self.stepLength not being defined"
)
self.defineStepLength()
self.theta = np.random.rand(self.nfeatures, self.z.shape[1])
indexSelectionPool = range(self.N)
inertia = np.zeros_like(self.theta)
drag = defaultingFunc(self.drag, self.drag, 0)
l2 = defaultingFunc(self.l2, self.l2, 0)
for epoch in range(self.epochN):
batch = np.random.choice(indexSelectionPool,
self.minibatchSize,
replace=False)
for batchIndex in range(self.minibatchN):
xi = self.X[batch].reshape(self.minibatchSize, -1)
zi = self.z[batch].reshape(self.minibatchSize, -1)
# .reshape gets correct shape if self.minibatchSize = 1, which
# might result in floats instead of arrays when using the @ operator
error = self.outputFunction(
xi @ self.theta / self.minibatchSize) - zi
gradient = xi.T @ error
step = self.stepLength(epoch * self.minibatchN + batchIndex)
l2_term = (self.theta**2).sum() * l2
inertia = drag * inertia
gradient += inertia + l2_term
self.theta -= gradient * step
# self.theta = np.linalg.pinv(self.X.T @ self.X) @ self.X.T @ self.z
# Analytical solution that can be used to compare results
predict = self.X @ self.theta
self.ACCscore = accuracyScore(self.outputFunction(predict / self.N),
self.z)
def craftDataset(self,
N=None,
minibatchSize=None,
epochN=None,
noisefactor=None,
modellingFunction=None):
"""Crafts dataset. Can be used to overwrite values for noisefactor, dataN, p, minibatchSize,
and epochN if these values are given as arguments.
Creates two linearly spaced datasets between 0 and 1 of size dataN/2,
then creates two uniformly random datasets between 0 and 1 of size
dataN/2. These two sets of datasets are concatenated together to form
the x and y datasets, which are put in a meshgrid and flattened to get
a good spread of data.
The response values are then generated using the modellingFunction,
which is FrankeFunction by default.
Args:
N (int, optional): square root of the number of datapoints used in total. Defaults to 100.
minibatchSize (int, optional): amount of datapoints used in each iteration. Defaults to 64.
epochN (int, optional): number of times dataset is iterated through. Defaults to 500.
noisefactor (float, optional): noise as a fraction of mean data added to data. Defaults to 0.1.
modellingFunction (function, optional): function used for modelling. Defaults to self.Frankefunction.
"""
self.noisefactor = defaultingFunc(self.noisefactor, noisefactor, 0.0)
self.dataN = defaultingFunc(self.dataN, N, 100)
self.minibatchSize = defaultingFunc(self.minibatchSize, minibatchSize,
64)
self.epochN = defaultingFunc(self.epochN, epochN, 500)
self.N = self.dataN**2
self.minibatchN = int(self.N / self.minibatchSize)
# Generates linearly and randomly spaced datasets
x1 = y1 = np.linspace(0, 1, mt.ceil(self.dataN / 2))
x2 = np.random.uniform(0, 1, mt.floor(self.dataN / 2))
y2 = np.random.uniform(0, 1, mt.floor(self.dataN / 2))
# Contatenates datasets, makes a meshgrid, and flattens them
x = np.concatenate((x1, x2))
y = np.concatenate((y1, y2))
x, y = np.meshgrid(x, y)
self.x = x.flatten()
self.y = y.flatten()
# Applies modelling function and generates expected values
self.modellingFunction = defaultingFunc(self.modellingFunction,
modellingFunction,
self.FrankeFunction)
self.zTrue = self.modellingFunction(self.x, self.y)
self.z = self.zTrue + self.noisefactor * np.random.randn(
self.N) * self.zTrue.mean()
self.z = self.z.reshape(-1, 1)
def giveInput(self, X=None, z=None):
"""Takes input and output values
Args:
X (numpy.nparray, optional): Input data . Defaults to previously given self.X.
z (numpy.ndarray, optional): Expected data, used for training. Defaults to previously given self.z.
"""
self.X = defaultingFunc(self.X, X, self.X)
self.z = defaultingFunc(self.z, z, self.z)
self.Ninputs, self.nfeatures = self.X.shape
self.Ncategories = self.z.shape[1]
self.activationValues = np.zeros(
(self.hiddenLN, self.Ninputs, self.hiddenNN))
def predict(self, X=None, z=None):
"""Feeds data through the network and predicts values for self.X.
If X is given self.z will temporarily be set to an array of correct
shape for the prediction with all elements equal to 0
Args:
X (numpy.ndarray, optional): If given, will be used to predict by the network. Defaults to self.X.
z (numpy.ndarray, optional): If given, will be used to judge the prediction done. Defaults to self.z
If X is given as none, z-argument will be discarded
"""
if X is not None:
X_storage = self.X
z_storage = self.z
self.X = X
self.z = defaultingFunc(None, z,
np.zeros((X.shape[0], self.Ncategories)))
self.giveInput()
if self.hiddenLN == 0:
self.logisticSpecialCasePredict()
else:
self.feedIn()
for i in range(1, self.hiddenLN):
self.feedForward(i)
self.feedOut()
if X is not None:
self.X = X_storage
self.z = z_storage
def makePolynomial(self, p=None, scaling=None):
"""Crafts the design matrix X as a scaled two-dimensional polynomial of
x and y, as defined by craftDataset()
Will call craftDataset if self.x, self.y, or self.z are not yet defined
and use default values
Args:
p (int, optional): polynomial degree used. Defaults to 8.
scaling (bool, optional): If true scales design matrix by subtracting average. Defaults to True.
"""
self.p = defaultingFunc(self.p, p, 8)
if scaling is None: scaling = True
if self.x is None or self.y is None or self.z is None:
print(
"in dataHandler.makePolynomial(): using defaulting self.craftDataset() due to self.x, self.y, or self.z not being defined"
)
self.craftDataset()
self.nfeatures = int(((self.p + 1) * (self.p + 2)) / 2)
self.X = np.zeros((len(self.x), self.nfeatures))
ind = 0
for i in range(self.p + 1):
for j in range(self.p + 1 - i):
self.X[:, ind] = self.x**i * self.y**j
ind += 1
if scaling:
self.X[:, 1:] -= np.mean(self.X[:, 1:], axis=0)
def train(self, splitData=None):
"""Trains network, ignores numpy warnings to simplify training with
multiple parameters where some weights will explode
Splits dataset into training and testing data (80%20%, respectively) if splitData is True
Only trains on batches of size self.minibatchSize at a time
After training, if splitData is True, the network is fed one last time
with test data to generate a score.
Args:
splitData (boolean, optional): Will split data into training and testing sets if True, won't otherwise. Defaults to True.
"""
splitData = defaultingFunc(None, splitData, True)
if self.inputWeights is None and self.logisticWeights is None:
self.getBiasesWeights()
X_full, z_full = self.X, self.z
if splitData:
X_train, X_test, z_train, z_test = train_test_split(self.X, self.z)
else:
X_train = X_full
z_train = z_full
batchIndexSelectionPool = range(X_train.shape[0])
batchSelection = np.random.choice(batchIndexSelectionPool,
self.minibatchSize,
replace=False)
self.X, self.z = X_train[batchSelection, :], z_train[batchSelection, :]
self.giveInput()
self.scoreHistory = np.zeros((self.epochN, 2))
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
for epoch in range(self.epochN):
for _ in range(self.minibatchN):
batchSelection = np.random.choice(batchIndexSelectionPool,
self.minibatchSize,
replace=False)
self.X, self.z = X_train[batchSelection, :], z_train[
batchSelection, :]
self.predict()
if self.hiddenLN != 0:
self.backPropagate()
else:
self.logisticSpecialCasePropagateBack()
self.scoreHistory[epoch] = self.R2, self.score
else:
if splitData:
self.X, self.z = X_test, z_test
self.giveInput()
self.predict()
self.X, self.z = X_full, z_full
def feedIn(self, X=None):
"""Feeds data into first hidden layer
Args:
X (numpy.ndarray, optional): input data. Defaults to previously given data.
"""
X = defaultingFunc(self.X, X, None)
self.activationValues[0] = X @ self.inputWeights + self.inputBias
self.activationValues[0] = self.activationFunction(
self.activationValues[0], self.alpha)
def giveParameters(self,
epochN=None,
minibatchSize=None,
bias=None,
eta=None,
lmbd=None,
alpha=None,
activationFunction=None,
outputFunction=None):
"""Takes parameters for the network
Args:
epochN (int, optional): number of times dataset is iterated through. Defaults to 500.
minibatchSize (int, optional): amount of datapoints used in each iteration. Defaults to 64.
bias (float, optional): Is added to nodes before applying activation function. Defaults to 1e-02.
eta (function or float, optional): learning rate, either given as a function that produces a
generator or a float. Defaults to 1e-06.
lmbd (float, optional): momentum rate. Defaults to 1e-06.
alpha (float, optional): parameter used in activation/output function. Defaults to 1e-01.
activationFunction (function, optional): function used for activating hidden nodes. Defaults to sigmoid.
modellingFunoutputFunctionction (function, optional): function used for activeting outgoing nodes. Defaults to softMax.
"""
self.epochN = defaultingFunc(epochN, self.epochN, 500)
self.minibatchSize = defaultingFunc(minibatchSize, self.minibatchSize,
64)
self.bias = defaultingFunc(bias, self.bias, 1e-02)
self.eta = defaultingFunc(self.defineEta(eta), self.eta,
self.defineEta(1e-06))
self.lmbd = defaultingFunc(lmbd, self.lmbd, 1e-06)
self.alpha = defaultingFunc(alpha, self.alpha, 1e-01)
self.activationFunction = defaultingFunc(activationFunction,
self.activationFunction,
sigmoid)
self.outputFunction = defaultingFunc(outputFunction,
self.outputFunction, softMax)
self.minibatchN = int(self.Ninputs / self.minibatchSize)
def benchmark(X,
z,
parameters,
NeuralNetwork=None,
mode=None,
randomSearch=None,
writingPermissions=None,
N=None):
"""Benchmarks Neural Networks using all permutations of given parameters
and finds the optimal values. Functions for either classification or
regression, will write all iterations to file if given writing permissions.
Args:
X (numpy.ndarray): Input values
z (numpy.ndarray): True values for inputs
parameters (dict): dictionary of all parameters
NeuralNetwork (NN, optional): Neural Network class to be benchmarked, Defaults to neural network in main script named NN.
mode (string, optional): Determines whether network is benchmarked for "classification" or "regression". Defaults to "classification".
writingPermissions (bool, optional): Determines whether script has permission to write to file. Defaults to False.
Returns:
(float, list, string): A tuple of best score, a list of parameters
used, and a string containing those parameters
that can be printed
"""
writingPermissions = defaultingFunc(None, writingPermissions, False)
mode = defaultingFunc(None, mode, "classification")
NeuralNetwork = defaultingFunc(None, NeuralNetwork, NN)
randomSearch = defaultingFunc(None, randomSearch, False)
N = defaultingFunc(None, N, 10000)
dataSelection = np.random.choice(range(X.shape[0]),
int(parameters["datafraction"] *
X.shape[0]),
replace=False)
X = X[dataSelection]
z = z[dataSelection]
tasksToDo = 1
tasksDone = 0
minimum = -np.inf
params = "n/a"
if writingPermissions:
outfile = open(
"parameterValues/NeuralNetworkParameters_" + mode + ".txt", "w")
print("Starting benchmarking with parameters: ")
for parameter in parameters:
try:
tasksToDo *= len(parameters[parameter])
except:
pass
print(parameter, ":", parameters[parameter])
if writingPermissions:
outfile.write(
str(parameter) + " : " + str(parameters[parameter]) + "\n")
if randomSearch:
for i in range(N):
score = 0
hiddenLN = np.random.choice(parameters["hiddenLN"])
hiddenNN = np.random.choice(parameters["hiddenNN"])
epochN = np.random.choice(parameters["epochN"])
minibatchSize = np.random.choice(parameters["minibatchSize"])
# print((parameters["eta"]))
eta = parameters["eta"][np.random.choice(
range(len(parameters["eta"])))]
# exit()
lmbd = np.random.choice(parameters["lmbd"])
alpha = np.random.choice(parameters["alpha"])
activationFunction = np.random.choice(
parameters["activationFunction"])
outputFunction = np.random.choice(parameters["outputFunction"])
for _ in range(parameters["#repetitions"]):
network = NeuralNetwork(hiddenNN=32, hiddenLN=hiddenLN)
network.giveInput(X, z)
network.giveParameters(epochN=epochN,
minibatchSize=minibatchSize,
eta=etaDefinerDefiner(eta[0], eta[1]),
lmbd=lmbd,
alpha=alpha,
activationFunction=activationFunction,
outputFunction=outputFunction)
network.getBiasesWeights()
network.train()
if mode.lower() == "classification":
score += network.score / parameters["#repetitions"]
elif mode.lower() == "regression":
score += network.R2 / parameters["#repetitions"]
paramSTR = "epochN:{}, minibatchSize:{}, eta:{}, lmbd:{}, alpha:{}, activFunct:{}, outFunc:{}, layers:{}, nodes:{}"\
.format(epochN, minibatchSize, eta, lmbd, alpha, activationFunction.__name__, outputFunction.__name__, hiddenLN, hiddenNN)
if score > minimum:
minimum = score
paramSTROUT = paramSTR
params = [
hiddenLN, hiddenNN, epochN, minibatchSize, eta, lmbd,
alpha, activationFunction, outputFunction
]
print(i, "/", N, "| score: {:.3f}".format(score), "| params:",
paramSTR)
return minimum, params, paramSTROUT
for hiddenLN in parameters["hiddenLN"]:
for hiddenNN in parameters["hiddenNN"]:
for epochN in parameters["epochN"]:
for minibatchSize in parameters["minibatchSize"]:
for eta in parameters["eta"]:
for lmbd in parameters["lmbd"]:
for alpha in parameters["alpha"]:
for activationFunction in parameters[
"activationFunction"]:
for outputFunction in parameters[
"outputFunction"]:
score = 0
tasksDone += 1
for _ in range(
parameters["#repetitions"]):
network = NeuralNetwork(
hiddenNN=32, hiddenLN=hiddenLN)
network.giveInput(X, z)
network.giveParameters(
epochN=epochN,
minibatchSize=minibatchSize,
eta=etaDefinerDefiner(
eta[0], eta[1]),
lmbd=lmbd,
alpha=alpha,
activationFunction=
activationFunction,
outputFunction=outputFunction)
network.getBiasesWeights()
network.train()
if mode.lower(
) == "classification":
score += network.score / parameters[
"#repetitions"]
elif mode.lower() == "regression":
score += network.R2 / parameters[
"#repetitions"]
paramSTR = "epochN:{}, minibatchSize:{}, eta:{}, lmbd:{}, alpha:{}, activFunct:{}, outFunc:{}, layers:{}, nodes:{}"\
.format(epochN, minibatchSize, eta, lmbd, alpha, activationFunction.__name__, outputFunction.__name__, hiddenLN, hiddenNN)
if score > minimum:
minimum = score
paramSTROUT = paramSTR
params = [
hiddenLN, hiddenNN, epochN,
minibatchSize, eta, lmbd,
alpha, activationFunction,
outputFunction
]
if writingPermissions:
outfile.write(
str(score) + " | " + paramSTR +
"\n")
print("Task done:", tasksDone, "/",
tasksToDo,
"| score: {:.3f}".format(score),
"| params:", paramSTR)
if writingPermissions:
outfile.write("Optimal: " + str(minimum) + " | " + paramSTROUT)
outfile.close()
return minimum, params, paramSTROUT