def __init__(self, Para):
Linear.__init__(self, Para)
try:
self.outChannel = self.hPara['outChannel']
self.bias = self.hPara['bias']
except:
self.printError(
"FullyConnected parameters does not exit or not complete.")
def readCommand(argv):
"Processes the command used to run from the command line."
from optparse import OptionParser
parser = OptionParser(USAGE_STRING)
parser.add_option('-m', '--modelName', help=default('modelConfig'))
parser.add_option('-d',
'--data',
help=default('input'),
default='data.bin',
type="string")
parser.add_option('-t',
'--target',
help=default('gradoutput'),
default='labels.bin',
type="string")
options, otherjunk = parser.parse_args(argv)
if len(otherjunk) != 0:
raise Exception('Command line input not understood: ' + str(otherjunk))
args = {}
model_name = options.modelName
training_data_path = options.data
target_labels_path = options.target
Data = torchfile.load(training_data_path)
Labels = torchfile.load(target_labels_path)
Data = torch.tensor(normalize(Data)).double()
Data = Data.reshape(Data.shape[0], 108 * 108)
Labels = torch.tensor(Labels).long()
# trainingData = Data[0:int(Data.shape[0]*0.9),:]
# trainingLabels = Labels[0:int(Data.shape[0]*0.9)]
my_model = Model.Model()
my_model.addLayer(Linear(108 * 108, 1024))
my_model.addLayer(ReLu())
my_model.addLayer(Linear(1024, 256))
my_model.addLayer(ReLu())
my_model.addLayer(Linear(256, 6))
my_model.addLayer(ReLu())
train_and_test(my_model, Data, Labels, 1, 432, 0.01, 0.001)
try:
os.mkdir(model_name)
except:
pass
weights = []
weights.append(my_model.layers[0].W)
weights.append(my_model.layers[0].B)
weights.append(my_model.layers[2].W)
weights.append(my_model.layers[2].B)
weights.append(my_model.layers[4].W)
weights.append(my_model.layers[4].B)
torch.save(weights, model_name + "/model.bin")
def __init__(self, hidden_layers):
Items = []
linear = Linear(2, 25)
Items.append(linear)
Items.append(ReLu())
for i in range(hidden_layers - 1):
Items.append(Linear(25, 25))
Items.append(ReLu())
Items.append(tanh())
Items.append(Linear(25, 2))
self.model = Sequential(Items)
def setUp(self):
self.feature_count = 50
self.number_docs = 1000
self.docs = range(self.number_docs)
self.features = np.random.rand(self.number_docs, self.feature_count)
self.linear_model = Linear(self.feature_count)
self.linear_w = self.linear_model.initialize_weights("random")
self.hidden_model = OneHiddenLayer(self.feature_count)
self.hidden_w = self.hidden_model.initialize_weights("random")
def testSGD(X, Y):
#construct
seq = Sequential()
seq.add_module(Linear(2, 5))
seq.add_module(Non_linear.Tanh())
seq.add_module(Linear(5, 1))
seq.add_module(Non_linear.Sigmoide())
def fctSig(res):
return np.array([[1 if res[i] > 0.5 else 0] for i in range(len(res))])
#evolute
maxIter = 300
rn = SGD(seq, X, Y, 50, MSE, fctSig, maxIter)
return rn.moduleList.histLoss, "SGD", maxIter
def testAutoEncoder():
#pepre data
uspsdatatrain = "data/USPS_train.txt"
uspsdatatest = "data/USPS_test.txt"
alltrainx, alltrainy = load_usps(uspsdatatrain)
alltestx, alltesty = load_usps(uspsdatatest)
neg = 9
pos = 6
datax, datay = get_usps([neg, pos], alltrainx, alltrainy)
datay = np.array([1 if datay[i] == 6 else 0 for i in range(len(datay))])
testx, testy = get_usps([neg, pos], alltestx, alltesty)
maxIter = 100
#rn encodage
encodage = Sequential()
encodage.add_module(Linear(256, 100))
encodage.add_module(Non_linear.Tanh())
encodage.add_module(Linear(100, 10))
encodage.add_module(Non_linear.Tanh())
# rn decodage
encodage.add_module(Linear(10, 100))
encodage.add_module(Non_linear.Tanh())
encodage.add_module(Linear(100, 256))
encodage.add_module(Non_linear.Sigmoide())
#rn decodage
# decodage = Sequential()
# decodage.add_module(Linear(10, 100))
# decodage.add_module(Non_linear.Tanh())
# decodage.add_module(Linear(100, 256))
# decodage.add_module(Non_linear.Sigmoide())
for i in range(maxIter):
#forward
# print(datax[0])
print(i)
encodage.forward(datax)
# print(encodage.forwards[-1][0])
encodage.backward(datax, datax, loss=BCE, gradient_step=0.1)
if i % 10 == 0:
# plt.figure()
# plt.imshow(datax[0].reshape(16, 16), cmap="gray")
# plt.title("Image original de 9: {}".format(datay[0]))
# plt.savefig("plot/num/origine9.png")
# plt.close()
plt.figure()
plt.imshow(encodage.forwards[-1][-10].reshape(16, 16), cmap="gray")
plt.title("Image apres autoEncoder de 6".format(datay[0]))
plt.savefig("plot/num/6_iter" + i.__str__() + ".png")
plt.close()
return encodage.histLoss, "AutoEncoder", maxIter
def __init__(self, Para):
Linear.__init__(self, Para)
try:
self.padding = Para['padding']
if self.padding:
self.padShape = Para[
'padShape'] # 2D tuple (pH,pW) padding on each side
else:
self.padShape = (0, 0)
self.outChannel = Para['outChannel']
self.kernelShape = Para[
'kernelShape'] # should be a 2D tuple (kH,kW)
self.stride = Para['stride'] # same for height and width dimension
self.bias = Para['bias']
except:
self.printError("Conv2D parameters does not exit or not complete.")
def main():
#while True:
#try:
filename = str(raw_input("Enter filename (Test data used if blank): "))
if (filename == ""):
filename = "testData.txt"
#break
#except:
#ValueError("File not found!")
input_data = np.loadtxt(filename)
# load data as vectors by taking transpose
xValues = np.matrix(input_data[:,0]).transpose()
yValues = np.matrix(input_data[:,1]).transpose()
errorValues = np.matrix(input_data[:,2]).transpose()
chi_sq = ChiSq(xValues, yValues, errorValues, function=Linear())
parameters = [0.0, 1.0]
chi_sq.setParameters(parameters)
chi_sq_value = chi_sq.evaluateChiSq()
print('Chi squared is: '+ str(chi_sq_value))
def testOptim(X, Y):
#construct
seq = Sequential()
seq.add_module(Linear(2, 5))
seq.add_module(Non_linear.Tanh())
seq.add_module(Linear(5, 1))
seq.add_module(Non_linear.Sigmoide())
def fctSig(res):
return np.array([[1 if res[i] > 0.5 else 0] for i in range(len(res))])
#evolute
maxIter = 100
optim = Optim(seq, fctsort=fctSig)
for _ in range(maxIter):
optim.step(X, Y)
return optim.moduleList.histLoss, "Optim", maxIter
def testSequential(X, Y):
#construct
seq = Sequential()
seq.add_module(Linear(2, 5))
seq.add_module(Non_linear.Tanh())
seq.add_module(Linear(5, 1))
seq.add_module(Non_linear.Sigmoide())
def fctSig(res):
return np.array([[1 if res[i] > 0.5 else 0] for i in range(len(res))])
#evolute
maxIter = 100
for _ in range(maxIter):
seq.forward(X)
seq.backward(X, Y, fctsort=fctSig)
return seq.histLoss, "Sequential", maxIter
def train(x_train, y_train, alpha=0.001, weight_decay=0.0, epochs=40):
for i in range(epochs):
train_loss = 0
count = 0
for j in range(img_num):
input = x_train[j].copy().reshape(1, 784) / 255.0
target = y_train[j].copy()
#Extracting hidden features using RBM
input, _ = model_test.sample_h_given_v(input)
#Forward
linear = Linear.forward(input)
loss = cross_entropy(linear, target)
train_loss += loss
#Backward
prob = np.exp(linear) / np.sum(np.exp(linear), axis=1)
prob[0, target] -= 1
Linear.backward(input, prob, alpha, weight_decay)
#Training Accuracy
train_prob = np.exp(linear)
if np.argmax(train_prob) == target:
count += 1
#Testing Accuracy
num_test = 10000
test_count = 0
for k in range(num_test):
test_input = x_test[k].copy().reshape(1, 784) / 255.0
test_input, _ = model_test.sample_h_given_v(test_input)
test_target = y_test[k].copy()
test_prob = Linear.forward(test_input)
test_prob = np.exp(test_prob)
if np.argmax(test_prob) == test_target:
test_count += 1
print("Epoch", i, " ", "Loss", " ", train_loss / img_num, "train_Acc",
" ", count / img_num, "test_Acc", " ", test_count / num_test)
classifier_weight_rbm = open(b"model/classifier_weight_rbm.npy", "wb")
pickle.dump(Linear.weight, classifier_weight_rbm)
classifier_bias_rbm = open(b"model/classifier_bias_rbm.npy", "wb")
pickle.dump(Linear.bias, classifier_bias_rbm)
def __init__(self, vac_size: int, hidden_size: int, seq_size: int):
self.vac_size = vac_size
self.hidden_size = hidden_size
self.seq_size = seq_size
#Encode
self.lstm1 = LSTMCell(vac_size=self.vac_size,
hidden_size=self.hidden_size,
return_seq=False)
self.repeat = RepeatVector(self.seq_size)
#Decode
self.lstm2 = LSTMCell(self.hidden_size, self.vac_size, return_seq=True)
self.linear = Linear(self.vac_size, self.vac_size)
def setUp(self):
self.feature_count = 50
self.number_docs = 1000
self.docs = range(self.number_docs)
self.features = np.random.rand(self.number_docs, self.feature_count)
self.linear_model = Linear(self.feature_count)
self.linear_w = self.linear_model.initialize_weights("random")
self.hidden_model = OneHiddenLayer(self.feature_count)
self.hidden_w = self.hidden_model.initialize_weights("random")
class TestRankers(unittest.TestCase):
def setUp(self):
self.feature_count = 50
self.number_docs = 1000
self.docs = range(self.number_docs)
self.features = np.random.rand(self.number_docs, self.feature_count)
self.linear_model = Linear(self.feature_count)
self.linear_w = self.linear_model.initialize_weights("random")
self.hidden_model = OneHiddenLayer(self.feature_count)
self.hidden_w = self.hidden_model.initialize_weights("random")
def testLinear(self):
scores = self.linear_model.score(self.features, self.linear_w)
docs1 = [d for _, d in sorted(zip(scores, self.docs))]
scores = self.linear_model.score(self.features, self.linear_w * 200)
docs2 = [d for _, d in sorted(zip(scores, self.docs))]
self.assertListEqual(docs1, docs2, "Linear Ranker should be magnitude"
"independent")
def testOneHiddenLayer(self):
scores = self.hidden_model.score(self.features, self.hidden_w)
docs1 = [d for _, d in sorted(zip(scores, self.docs))]
scores = self.hidden_model.score(self.features, self.hidden_w * 10)
docs2 = [d for _, d in sorted(zip(scores, self.docs))]
self.assertNotEqual(
docs1, docs2, "Hidden Layer Ranker should be "
"magnitude dependent")
def testInitOneHiddenLayer(self):
orderings = set()
reps = 1000
for _ in range(reps):
w = self.hidden_model.initialize_weights("random")
scores = self.hidden_model.score(self.features, w)
ordering = tuple([d
for _, d in sorted(zip(scores, self.docs))][:10])
orderings.add(ordering)
self.assertEqual(reps, len(orderings))
class TestRankers(unittest.TestCase):
def setUp(self):
self.feature_count = 50
self.number_docs = 1000
self.docs = range(self.number_docs)
self.features = np.random.rand(self.number_docs, self.feature_count)
self.linear_model = Linear(self.feature_count)
self.linear_w = self.linear_model.initialize_weights("random")
self.hidden_model = OneHiddenLayer(self.feature_count)
self.hidden_w = self.hidden_model.initialize_weights("random")
def testLinear(self):
scores = self.linear_model.score(self.features, self.linear_w)
docs1 = [d for _, d in sorted(zip(scores, self.docs))]
scores = self.linear_model.score(self.features, self.linear_w * 200)
docs2 = [d for _, d in sorted(zip(scores, self.docs))]
self.assertListEqual(docs1, docs2, "Linear Ranker should be magnitude"
"independent")
def testOneHiddenLayer(self):
scores = self.hidden_model.score(self.features, self.hidden_w)
docs1 = [d for _, d in sorted(zip(scores, self.docs))]
scores = self.hidden_model.score(self.features, self.hidden_w * 10)
docs2 = [d for _, d in sorted(zip(scores, self.docs))]
self.assertNotEqual(docs1, docs2, "Hidden Layer Ranker should be "
"magnitude dependent")
def testInitOneHiddenLayer(self):
orderings = set()
reps = 1000
for _ in range(reps):
w = self.hidden_model.initialize_weights("random")
scores = self.hidden_model.score(self.features, w)
ordering = tuple([d for _, d in
sorted(zip(scores, self.docs))][:10])
orderings.add(ordering)
self.assertEqual(reps, len(orderings))
def Solve(self):
if self.method is 'Constant':
from Constant import Constant
self.yn = Constant(self.x0, self.y0, self.xn)
elif self.method is 'Linear':
from Linear import Linear
self.yn = Linear(self.x0, self.y0, self.xn)
elif self.method is 'Polynomial':
from Polynomial import Polynomial
self.yn = Polynomial(self.x0, self.y0, self.xn)
elif self.method is 'Splines':
from Splines import Splines
self.yn = Splines(self.x0, self.y0, self.xn)
def testSoftmax():
uspsdatatrain = "data/USPS_train.txt"
uspsdatatest = "data/USPS_test.txt"
X, Y = load_usps(uspsdatatrain)
Xtest, Ytest = load_usps(uspsdatatest)
onehot = np.zeros((Y.size, 10), dtype=np.int)
onehot[np.arange(Y.size), Y] = 1
# print(X.shape,Y.shape,onehot.shape)
seq = Sequential()
seq.add_module(Linear(256, 50))
seq.add_module(Non_linear.Tanh())
seq.add_module(Linear(50, 10))
seq.add_module(Softmax())
# print(X[0])
# return 0
# evolute
maxIter = 100
optim = Optim(seq, loss=CrossEntropy, eps=0.01)
# print(onehot[0])
for _ in range(maxIter):
optim.step(X, onehot)
return optim.moduleList.histLoss, "Softmax", maxIter
def __init__(self, vac_size: int, hidden_sizes: Tuple[int, int],
seq_size: int):
"""
Class implements RNNAutoencoder.
Architecture of RNNAutoencoder have 2 lstm layers in encoder and
2 lstm layers with linear layer in decoder
:param vac_size: int
:param hidden_sizes: Tuple[int, int]
:param seq_size: int
"""
self.vac_size = vac_size
self.hidden_size_1 = hidden_sizes[0]
self.hidden_size_2 = hidden_sizes[1]
self.seq_size = seq_size
#Encode
self.lstm1 = LSTMCell(vac_size=self.vac_size,
hidden_size=self.hidden_size_1,
return_seq=True)
self.lstm2 = LSTMCell(vac_size=self.hidden_size_1,
hidden_size=self.hidden_size_2,
return_seq=False)
self.repeat = RepeatVector(self.seq_size)
#Decode
self.lstm3 = LSTMCell(self.hidden_size_2,
self.hidden_size_1,
return_seq=True)
self.lstm4 = LSTMCell(self.hidden_size_1,
self.vac_size,
return_seq=True)
self.linear = Linear(self.vac_size, self.vac_size)
def test_interchange(self):
print 'Single Scan Interchange algorithm'
for N in map(int, [10, 1e2, 1e3, 1e4, 1e5, 1e6]):
pr = PPReader('population.csv', 'dummy.csv', N=N)
# Read dummy files into buffer
start_time = time.time()
#pr.init()
elapsed_time = time.time() - start_time
sys.stdout.write('init done: %f seconds taken.\t' % elapsed_time)
sys.stdout.flush()
# Actual test
start_time = time.time()
lin = Linear(dist_func, K=5)
lin.update(pr)
elapsed_time = time.time() - start_time
sampled = lin.get_sampled()
sys.stdout.write('%d: %f\n' % (N, elapsed_time))
sys.stdout.flush()
def buildModel(config):
f = open(config, "r")
n = int(f.readline())
testModel = Model()
for i in range(n):
tokens = f.readline().split()
if tokens[0] == "linear":
inpNodes = int(tokens[1])
outNodes = int(tokens[2])
linLayer = Linear(inpNodes, outNodes)
testModel.addLayer(linLayer)
if tokens[0] == "relu":
reluLayer = ReLU()
testModel.addLayer(reluLayer)
tokens = f.readline().split()
weightsPath = tokens[0]
weights = torchfile.load(weightsPath)
tokens = f.readline().split()
biasPath = tokens[0]
biases = torchfile.load(biasPath)
cnt = 0
for i in range(n):
if testModel.Layers[i].type == 0:
testModel.Layers[i].W = torch.from_numpy(weights[cnt].T).double()
testModel.Layers[i].B = torch.from_numpy(
biases[cnt].T).double().unsqueeze(1)
cnt += 1
f.close()
return testModel
Data = torchfile.load("data.bin")
Labels = torchfile.load("labels.bin")
Data = torch.tensor(normalize(Data)).double()
Data = Data.reshape(Data.shape[0], 108 * 108)
Labels = torch.tensor(Labels).long()
trainingData = Data[0:int(Data.shape[0] * 0.9), :]
trainingLabels = Labels[0:int(Data.shape[0] * 0.9)]
validationData = Data[int(Data.shape[0] * 0.9):Data.shape[0], :]
validationLabels = Labels[int(Data.shape[0] * 0.9):Data.shape[0]]
my_model = Model.Model()
my_model.addLayer(Linear(108 * 108, 1024))
my_model.addLayer(ReLu())
my_model.addLayer(Linear(1024, 256))
my_model.addLayer(ReLu())
my_model.addLayer(Linear(256, 6))
my_model.addLayer(ReLu())
my_criterion = Criterion.Criterion()
def train_and_test(trainingData, trainingLabels, validationData,
validationLabels, noIters, batchSize, alpha,
lr): # can add lambda
global my_model
noBatches = int(trainingLabels.shape[0] / batchSize)
def model_tanh():
return Sequential(Linear(2,25),Tanh(),Linear(25,25),
Tanh(), Linear(25,25), Tanh(),Linear(25,2))
def model_relu():
return Sequential(Linear(2,25),Relu(),Linear(25,25),
Relu(),Linear(25,25), Relu(),Linear(25,2))
def readCommand(argv):
"Processes the command used to run from the command line."
from optparse import OptionParser
parser = OptionParser(USAGE_STRING)
parser.add_option('-c',
'--config',
help=default('modelConfig'),
default='CS 763 Deep Learning HW/modelConfig_1.txt')
parser.add_option('-i',
'--i',
help=default('input'),
default='CS 763 Deep Learning HW/input_sample_1.bin',
type="string")
parser.add_option(
'-g',
'--og',
help=default('gradoutput'),
default='CS 763 Deep Learning HW/gradOutput_sample_1.bin',
type="string")
parser.add_option('-o', '--o', help=default('output'), type="string")
parser.add_option('-w', '--ow', help=default('gradweights'), type="string")
parser.add_option('-b', '--ob', help=default('gradb'), type="string")
parser.add_option('-d', '--ig', help=default('gradinput'), type="string")
options, otherjunk = parser.parse_args(argv)
if len(otherjunk) != 0:
raise Exception('Command line input not understood: ' + str(otherjunk))
args = {}
model_config_path = options.config
input_path = options.i
gradoutput_path = options.og
output_path = options.o
gradweights_path = options.ow
gradb_path = options.ob
gradinput_path = options.ig
modelConfig_file = open(model_config_path, "r")
data = modelConfig_file.readlines()
my_model = Model.Model()
my_criterion = Criterion.Criterion()
input_weight = 0
Bias_weight = 0
Number_layer = int(data[0])
for i in range(Number_layer):
layer = data[1 + i].split()
if (len(layer) > 1):
my_model.addLayer(Linear(int(layer[1]), int(layer[2])))
else:
my_model.addLayer(ReLu())
Path_sample_weight = data[Number_layer + 1][:-1]
Path_sample_bias = data[Number_layer + 2][:-1]
input = torchfile.load(input_path)
input = torch.tensor(input).double().reshape((input.shape[0], -1))
input_weight = torchfile.load(Path_sample_weight)
input_bias = torchfile.load(Path_sample_bias)
input_weight = [torch.tensor(weight).double() for weight in input_weight]
input_bias = [
torch.tensor(bias).double().reshape((-1, 1)) for bias in input_bias
]
Outputs = my_model.forward2(input, input_weight, input_bias, True)
dl_do = my_criterion.backward(Outputs, trLabels)
# gradoutput = my_model.backward(input, dl_do, 0, 0)
[gradInput, gradWeights, gradBias] = my_model.backward2(input, dl_do, 0, 0)
torch.save(Outputs, output_path)
torch.save(gradWeights, gradweights_path)
torch.save(gradBias, gradb_path)
torch.save(gradInput, gradinput_path)
Ytrain = torchfile.load(args.ytrain)
Ytrain = torch.from_numpy(Ytrain).long().unsqueeze(1)
Ytest = Ytrain[test[0:5000], :]
Ytrain = Ytrain[test[5000:], :]
noTrain = Xtrain.shape[0]
batchSize = args.b
epochs = args.e
alpha = args.a
moment = 0.9
myModel = Model(moment)
myModel.addLayer(Flatten())
myModel.addLayer(Linear(108 * 108, 80))
myModel.addLayer(batchNorm())
myModel.addLayer(sigactiv())
myModel.addLayer(Dropout(0.7))
myModel.addLayer(Linear(80, 20))
myModel.addLayer(batchNorm())
myModel.addLayer(sigactiv())
myModel.addLayer(Linear(20, 10))
myModel.addLayer(batchNorm())
myModel.addLayer(sigactiv())
myModel.addLayer(Linear(10, 6))
criterion = Criterion()
if args.loadModel:
model = torch.load("modelParams.txt")
k = 3
fout.write('x,y,label\n')
# Generate population
pp = population1()
for x, y in pp:
fout.write('%f,%f,a\n' % (x, y))
x, y = zip(*pp)
plt.scatter(x, y, s=1, c='b')
# Generate sampled data
def dist_func(a, b):
d = pow(np.linalg.norm(a - b), 2)
return d
lin = Linear(dist_func, K = 50, r = 0.3)
lin.update(pp)
sampled = lin.get_sampled()
for x, y in sampled:
fout.write('%f,%f,b\n' % (x, y))
fout.close()
x, y = zip(*sampled)
plt.scatter(x, y, s=30, c='r')
plt.show()
x, y = pair
loss += 1.0 / dist_func(x, y)
return loss
np.random.seed(1)
for psize in xrange(10, 110, 10):
#for psize in xrange(100, 1100, 100):
# Generate population
population = np.random.rand(psize, 2)
# Generate sampled data
K = 2
# First: we get samples using linear algorithm
lin = Linear(dist_func, K = K, r = 1)
lin.update(population)
lin_sample = lin.get_sampled()
# Second: we enumerate all the samples of size K, and find the one with the
# minimal loss.
min_loss = float('Inf')
bf_sample = None
for c in combinations(population, K):
loss = loss_of(c)
if loss < min_loss:
min_loss = loss
bf_sample = c
print 'Population size: %d' % psize
print 'Linear\tsize: %d, loss: %f' % (len(lin_sample), loss_of(lin_sample))
return Other
# ----------------------------------------------------------------------
# Module Tests
# ----------------------------------------------------------------------
if __name__ == '__main__':
import numpy as np
from Linear import Linear
S = ScalingBunch()
S.X = Linear(10.0, 0.0)
S.Y = Linear(2.0, 1.0)
data = OrderedBunch()
data.X = 10.0
data.Y = np.array([1, 2, 3.])
print data
data = data / S
print data
data = data * S
print data
class RNNAutoencoder:
def __init__(self, vac_size: int, hidden_sizes: Tuple[int, int],
seq_size: int):
"""
Class implements RNNAutoencoder.
Architecture of RNNAutoencoder have 2 lstm layers in encoder and
2 lstm layers with linear layer in decoder
:param vac_size: int
:param hidden_sizes: Tuple[int, int]
:param seq_size: int
"""
self.vac_size = vac_size
self.hidden_size_1 = hidden_sizes[0]
self.hidden_size_2 = hidden_sizes[1]
self.seq_size = seq_size
#Encode
self.lstm1 = LSTMCell(vac_size=self.vac_size,
hidden_size=self.hidden_size_1,
return_seq=True)
self.lstm2 = LSTMCell(vac_size=self.hidden_size_1,
hidden_size=self.hidden_size_2,
return_seq=False)
self.repeat = RepeatVector(self.seq_size)
#Decode
self.lstm3 = LSTMCell(self.hidden_size_2,
self.hidden_size_1,
return_seq=True)
self.lstm4 = LSTMCell(self.hidden_size_1,
self.vac_size,
return_seq=True)
self.linear = Linear(self.vac_size, self.vac_size)
def params(self):
"""
returns parameters of all model
:return: dict
"""
return {
'lstm1': self.lstm1.params(),
'lstm2': self.lstm2.params(),
'lstm3': self.lstm3.params(),
'lstm4': self.lstm4.params(),
'linear': self.linear.params()
}
def clear_gradients(self):
"""
function which clears gradients
:return:
"""
self.lstm1.clear_gradients()
self.lstm2.clear_gradients()
self.lstm3.clear_gradients()
self.lstm4.clear_gradients()
self.linear.clear_gradients()
def forward(self, X: np.ndarray):
"""
forward pass through the model
:param X: np.ndarray
:return: predictions of model
"""
self.clear_gradients()
encode = self.lstm2.forward(self.lstm1.forward(X))
bridge = self.repeat.forward(encode)
decode = self.lstm4.forward(self.lstm3.forward(bridge))
decode = decode.reshape(decode.shape[0], decode.shape[1])
pred = self.linear.forward(decode)
return pred
def compute_loss_and_gradient(self, X: np.ndarray, y: np.ndarray):
"""
function which implement forward pass and calculation of loss and its derivative
:param X: not-sorted one-hot array (seq_size, vac_size, 1)
:param y: sorted sequence (seq_size, )
:return: loss and its derivative
"""
pred = self.forward(X)
loss, dpredication = softmax_cross_entropy(pred, y)
return loss, dpredication
def repeat_backward(self, x: np.ndarray):
"""
function which repeat vector for backward pass
:param x: np.ndarray size (vac_size, 1)
:return: d_out :np.ndarray size(seq_size, vac_size, 1)
"""
d_out = np.zeros((self.seq_size, *x.shape))
d_out[-1] = x
return d_out
def backward(self, d_out: np.ndarray):
"""
backward pass through model
:param d_out: derivative of loss
:return:
"""
d_l = self.linear.backward(d_out)
d_l = d_l.reshape(*d_l.shape, 1)
d_l = self.lstm3.backward(self.lstm4.backward(d_l))
bridge = self.repeat_backward(self.repeat.backward(d_l))
d_x = self.lstm1.backward(self.lstm2.backward(bridge))
return d_x
def predict(self, X: np.ndarray):
"""
predict answer of the model
:param X:
:return:
"""
pred = self.forward(X)
probs = softmax(pred)
return np.argmax(probs, axis=1)
def check_all_gradients(num_checks: int = 5):
print('Checking Layers Only')
print('Checking Linear Layer')
for _ in range(num_checks):
seq_size = np.random.randint(low=1, high=128)
n_in = np.random.randint(low=1, high=128)
n_out = np.random.randint(low=1, high=128)
assert check_layer_gradient(Linear(n_in=n_in, n_out=n_out),
np.random.randn(seq_size, n_in))
print('Checking Linear Layer Paramter W')
for _ in range(num_checks):
seq_size = np.random.randint(low=1, high=128)
n_in = np.random.randint(low=1, high=128)
n_out = np.random.randint(low=1, high=128)
assert check_layer_param_gradient(Linear(n_in=n_in, n_out=n_out),
np.random.randn(seq_size, n_in), 'W')
print('Checking Linear Layer Paramter b')
for _ in range(num_checks):
seq_size = np.random.randint(low=1, high=128)
n_in = np.random.randint(low=1, high=128)
n_out = np.random.randint(low=1, high=128)
assert check_layer_param_gradient(Linear(n_in=n_in, n_out=n_out),
np.random.randn(seq_size, n_in), 'b')
print('Checking RepeatVector Layer')
for _ in range(num_checks):
seq_size = np.random.randint(low=1, high=128)
n_in = np.random.randint(low=1, high=128)
assert check_layer_gradient(RepeatVector(seq_size=seq_size),
np.random.randn(n_in, 1))
print('Checking LSTM Layer')
for _ in range(num_checks):
vac_size = np.random.randint(low=10, high=32)
hidden_size = np.random.randint(low=1, high=128)
seq_size = np.random.randint(low=1, high=32)
assert check_layer_gradient(
LSTMCell.LSTMCell(vac_size=vac_size,
hidden_size=hidden_size,
return_seq=True),
np.random.randn(seq_size, vac_size, 1))
print('Checking LSTM Parameter W_forget')
for _ in range(num_checks):
vac_size = np.random.randint(low=10, high=32)
hidden_size = np.random.randint(low=1, high=32)
seq_size = np.random.randint(low=1, high=32)
assert check_layer_param_gradient(
LSTMCell.LSTMCell(vac_size=vac_size,
hidden_size=hidden_size,
return_seq=True),
np.random.randn(seq_size, vac_size, 1), 'W_forget')
print('Checking LSTM Parameter W_input')
for _ in range(num_checks):
vac_size = np.random.randint(low=10, high=32)
hidden_size = np.random.randint(low=1, high=32)
seq_size = np.random.randint(low=1, high=32)
assert check_layer_param_gradient(
LSTMCell.LSTMCell(vac_size=vac_size,
hidden_size=hidden_size,
return_seq=True),
np.random.randn(seq_size, vac_size, 1), 'W_input')
print('Checking LSTM Parameter W_cell_state')
for _ in range(num_checks):
vac_size = np.random.randint(low=10, high=32)
hidden_size = np.random.randint(low=1, high=32)
seq_size = np.random.randint(low=1, high=32)
assert check_layer_param_gradient(
LSTMCell.LSTMCell(vac_size=vac_size,
hidden_size=hidden_size,
return_seq=True),
np.random.randn(seq_size, vac_size, 1), 'W_cell_state')
print('Checking LSTM Parameter W_output')
for _ in range(num_checks):
vac_size = np.random.randint(low=10, high=32)
hidden_size = np.random.randint(low=1, high=32)
seq_size = np.random.randint(low=1, high=32)
assert check_layer_param_gradient(
LSTMCell.LSTMCell(vac_size=vac_size,
hidden_size=hidden_size,
return_seq=True),
np.random.randn(seq_size, vac_size, 1), 'W_output')
print('Checking LSTM Parameter b_forget')
for _ in range(num_checks):
vac_size = np.random.randint(low=10, high=32)
hidden_size = np.random.randint(low=1, high=32)
seq_size = np.random.randint(low=1, high=32)
assert check_layer_param_gradient(
LSTMCell.LSTMCell(vac_size=vac_size,
hidden_size=hidden_size,
return_seq=True),
np.random.randn(seq_size, vac_size, 1), 'b_forget')
print('Checking LSTM Parameter b_input')
for _ in range(num_checks):
vac_size = np.random.randint(low=10, high=32)
hidden_size = np.random.randint(low=1, high=128)
seq_size = np.random.randint(low=1, high=32)
assert check_layer_param_gradient(
LSTMCell.LSTMCell(vac_size=vac_size,
hidden_size=hidden_size,
return_seq=True),
np.random.randn(seq_size, vac_size, 1), 'b_input')
print('Checking LSTM Parameter b_cell_state')
for _ in range(num_checks):
vac_size = np.random.randint(low=10, high=32)
hidden_size = np.random.randint(low=1, high=32)
seq_size = np.random.randint(low=1, high=32)
assert check_layer_param_gradient(
LSTMCell.LSTMCell(vac_size=vac_size,
hidden_size=hidden_size,
return_seq=True),
np.random.randn(seq_size, vac_size, 1), 'b_cell_state')
print('Checking LSTM Parameter b_output')
for _ in range(num_checks):
vac_size = np.random.randint(low=10, high=32)
hidden_size = np.random.randint(low=1, high=32)
seq_size = np.random.randint(low=1, high=32)
assert check_layer_param_gradient(
LSTMCell.LSTMCell(vac_size=vac_size,
hidden_size=hidden_size,
return_seq=True),
np.random.randn(seq_size, vac_size, 1), 'b_output')
print('Checking All Two Layer Model Paramters')
for _ in range(num_checks):
vac_size = np.random.randint(low=10, high=32)
seq_size = np.random.randint(low=1, high=32)
ds = Dataset(vac_size=vac_size, seq_size=seq_size)
X, y = ds.generate_seq()
assert check_model_gradient(model=RNNAutoencoder(vac_size=vac_size,
hidden_sizes=(12, 12),
seq_size=seq_size),
X=X,
y=y)
print('Checking All One Layer Model Paramters')
for _ in range(num_checks):
vac_size = np.random.randint(low=10, high=32)
seq_size = np.random.randint(low=1, high=32)
ds = Dataset(vac_size=vac_size, seq_size=seq_size)
X, y = ds.generate_seq()
assert check_model_gradient(model=RNNAutoencoderOneLayer(
vac_size=vac_size, hidden_size=12, seq_size=seq_size),
X=X,
y=y)
print('All Gradients Are Fine! Lets Train Model!')
sys.path.append('dl/')
from Sequential import Sequential
from Linear import Linear
from Functionnals import Relu
import Optimizer
import Criterion
from helpers import train, generate_disc_data, compute_accuracy
#setting the type of tensor
torch.set_default_dtype(torch.float32)
#disable autograd
torch.set_grad_enabled(False)
#create model
model = Sequential(Linear(2, 25), Relu(), Linear(25, 25), Relu(),
Linear(25, 25), Relu(), Linear(25, 2))
#create data_sets with one hot encoding for MSE
train_input, train_target = generate_disc_data(one_hot_labels=True)
test_input, test_target = generate_disc_data(one_hot_labels=True)
#normalize the data
mean, std = train_input.mean(), train_input.std()
train_input.sub_(mean).div_(std)
test_input.sub_(mean).div_(std)
#define loss
criterion = Criterion.MSE()
#define optimizer
def train_model():
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
trainset = torchvision.datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=transform)
testset = torchvision.datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform)
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=1,
shuffle=True)
testloader = torch.utils.data.DataLoader(testset, batch_size=1)
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck')
# def imshow(img):
# img = img / 2 + 0.5 # unnormalize
# npimg = img.numpy()
# plt.imshow(np.transpose(npimg, (1, 2, 0)))
# plt.show()
#
# dataiter = iter(trainloader)
# images, labels = dataiter.next()
my_model = Model()
my_model.addLayer(Linear(3072, 1024))
my_model.addLayer(ReLu())
# my_model.addLayer(Linear(2048, 1024))
# my_model.addLayer(ReLu())
my_model.addLayer(Linear(1024, 512))
my_model.addLayer(ReLu())
#
my_model.addLayer(Linear(512, 5))
# my_model.addLayer(Softmax())
# my_model.addLayer(CrossEntropy())
running_loss = 0
epochs = 7
train_count = 0
train_losses, test_losses = [], []
i = 0
for epoch in range(epochs):
for images, labels in trainloader:
if train_on_gpu:
images, labels = images.to(device), labels.to(device)
# print(labels)
if labels <= 4:
train_count += 1
images = images.view(images.size(0), -1)
final_prob = my_model.forward(images)
backward_grad = CrossEntropy().backward(final_prob, labels)
# print(backward_grad)
my_model.backward(images, backward_grad, alpha=0.001)
running_loss += (CrossEntropy().forward(final_prob, labels))
if (train_count + 1) % 500 == 0:
i = i + 1
test_loss = 0
accuracy = 0
correct_class = 0
test_count = 0
for images, labels in testloader:
if train_on_gpu:
images, labels = images.to(device), labels.to(device)
if labels <= 4:
test_count += 1
images = images.view(images.size(0), -1)
score = my_model.forward(images)
test_loss += CrossEntropy().forward(score, labels)
ps = torch.exp(score)
top_p, top_class = ps.topk(1, dim=1)
if top_class == labels:
correct_class += 1
train_losses.append(running_loss / (train_count + 1))
test_losses.append(test_loss / (test_count + 1))
# plt.plot(train_losses, label='Training loss')
# plt.plot(test_losses, label='Validation loss')
# plt.savefig('myfilename.png', dpi=100)
print(f"Epoch {i}.. "
f"Train loss: {running_loss/(train_count):.3f} .."
f"Test loss: {test_loss/(test_count + 1):.3f} .."
f"Test accuracy: {correct_class/(test_count + 1):.3f}")
test_count = 0
train_count = 0
running_loss = 0
plt.plot(train_losses, label='Training loss')
plt.plot(test_losses, label='Validation loss')
plt.legend(frameon=False)
plt.savefig('final.png', dpi=100)
return my_model
target_loc = sys.argv[i + 1]
if not os.path.exists(model_name):
os.makedirs(model_name)
batch_size = 12
criterion = Criterion()
dataset = Dataset(batch_size)
model = Model(2, 128, 153, 153, True)
dataset.read_data(data_loc, 'X_train')
dataset.read_data(target_loc, 'Y_train')
train_data_len = len(dataset.X_train)
model.add_layer(RNN(153, 128, 20))
model.add_layer(Linear(128, 2))
train(8, 1)
train(3, 1e-1)
accuracy(0, train_data_len)
train(6, 1e-2)
accuracy(0, train_data_len)
train(3, 1e-3)
accuracy(0, train_data_len)
train(8, 1e-6)
accuracy(0, train_data_len)
train(3, 1e-7)
accuracy(0, train_data_len)
file = open(model_name + '/model.bin', 'wb')
torch.save(model, file)
def run_all_model(train_input,
train_target,
test_input,
test_target,
Sample_number,
save_plot=False):
# Define constants along the test
hidden_nb = 25
std = 0.1
eta = 3e-1
batch_size = 200
epochs_number = 1000
# Model 1. No dropout; constant learning rate (SGD)
print('\nModel 1: Optimizer: SGD; No dropout; ReLU; CrossEntropy')
# Define model name for plots
mname = 'Model1'
# Define structure of the network
linear_1 = Linear(2, hidden_nb)
relu_1 = Relu()
linear_2 = Linear(hidden_nb, hidden_nb)
relu_2 = Relu()
linear_3 = Linear(hidden_nb, hidden_nb)
relu_3 = Relu()
linear_4 = Linear(hidden_nb, 2)
loss = CrossEntropy()
model_1 = Sequential(linear_1,
relu_1,
linear_2,
relu_2,
linear_3,
relu_3,
linear_4,
loss=CrossEntropy())
# Initialize weights
model_1.normalize_parameters(mean=0, std=std)
# Define optimizer
optimizer = Sgd(eta)
# Train model
my_loss_1 = train_model(model_1, train_input, train_target, optimizer,
epochs_number, Sample_number, batch_size)
# Evalute model and produce plots
model_1_perf = evaluate_model(model_1,
train_input,
train_target,
test_input,
test_target,
my_loss_1,
save_plot,
mname=mname)
# Model 2. No dropout; decreasing learning rate (DecreaseSGD)
print('\nModel 2: Optimizer: DecreaseSGD; No dropout; ReLU; CrossEntropy')
# Define model name for plots
mname = 'Model2'
# Define structure of the network
linear_1 = Linear(2, hidden_nb)
relu_1 = Relu()
linear_2 = Linear(hidden_nb, hidden_nb)
relu_2 = Relu()
linear_3 = Linear(hidden_nb, hidden_nb)
relu_3 = Relu()
linear_4 = Linear(hidden_nb, 2)
model_2 = Sequential(linear_1,
relu_1,
linear_2,
relu_2,
linear_3,
relu_3,
linear_4,
loss=CrossEntropy())
# Initialize weights
model_2.normalize_parameters(mean=0, std=std)
# Define optimizer
optimizer = DecreaseSGD(eta)
# Train model
my_loss_2 = train_model(model_2, train_input, train_target, optimizer,
epochs_number, Sample_number, batch_size)
# Evalute model and produce plots
model_2_perf = evaluate_model(model_2,
train_input,
train_target,
test_input,
test_target,
my_loss_2,
save_plot,
mname=mname)
# Model 3. No dropout; Adam Optimizer
print('\nModel 3: Optimizer: Adam; No dropout; ReLU; CrossEntropy')
# Define model name for plots
mname = 'Model3'
# Custom hyperparameters
eta_adam = 1e-3
epochs_number_adam = 500
# Define structure of the network
linear_1 = Linear(2, hidden_nb)
relu_1 = Relu()
linear_2 = Linear(hidden_nb, hidden_nb)
relu_2 = Relu()
linear_3 = Linear(hidden_nb, hidden_nb)
relu_3 = Relu()
linear_4 = Linear(hidden_nb, 2)
loss = CrossEntropy()
model_3 = Sequential(linear_1,
relu_1,
linear_2,
relu_2,
linear_3,
relu_3,
linear_4,
loss=CrossEntropy())
# Initialize weights
model_3.normalize_parameters(mean=0, std=std)
# Define optimizer
optimizer = Adam(eta_adam, 0.9, 0.99, 1e-8)
# Train model
my_loss_3 = train_model(model_3, train_input, train_target, optimizer,
epochs_number_adam, Sample_number, batch_size)
# Evalute model and produce plots
model_3_perf = evaluate_model(model_3,
train_input,
train_target,
test_input,
test_target,
my_loss_3,
save_plot,
mname=mname)
# PLOT TO COMPARE OPTIMIZERS
if save_plot:
fig = plt.figure(figsize=(10, 4))
plt.plot(range(0, epochs_number), my_loss_1, linewidth=1)
plt.plot(range(0, epochs_number), my_loss_2, linewidth=1)
plt.plot(range(0, epochs_number_adam), my_loss_3, linewidth=1)
plt.legend(["SGD", "Decreasing SGD", "Adam"])
plt.title("Loss")
plt.xlabel("Epochs")
plt.savefig('output/compare_optimizers.pdf', bbox_inches='tight')
plt.close(fig)
# Model 4. Dropout; SGD
print('\nModel 4: Optimizer: SGD; Dropout; ReLU; CrossEntropy')
# Define model name for plots
mname = 'Model4'
# Define structure of the network
dropout = 0.15
linear_1 = Linear(2, hidden_nb)
relu_1 = Relu()
linear_2 = Linear(hidden_nb, hidden_nb, dropout=dropout)
relu_2 = Relu()
linear_3 = Linear(hidden_nb, hidden_nb, dropout=dropout)
relu_3 = Relu()
linear_4 = Linear(hidden_nb, 2)
model_4 = Sequential(linear_1,
relu_1,
linear_2,
relu_2,
linear_3,
relu_3,
linear_4,
loss=CrossEntropy())
# Initialize weights
model_4.normalize_parameters(mean=0, std=std)
# Define optimizer
optimizer = Sgd(eta)
# Train model
my_loss_4 = train_model(model_4, train_input, train_target, optimizer,
epochs_number, Sample_number, batch_size)
# Evalute model and produce plots
model_4_perf = evaluate_model(model_4,
train_input,
train_target,
test_input,
test_target,
my_loss_4,
save_plot,
mname=mname)
# PLOT TO COMPARE DROPOUT AND NO DROPOUT
if save_plot:
fig = plt.figure(figsize=(10, 4))
plt.plot(range(0, epochs_number), my_loss_1, linewidth=1)
plt.plot(range(0, epochs_number), my_loss_4, linewidth=1)
plt.legend(["Without Dropout", "With Dropout"])
plt.title("Loss")
plt.xlabel("Epochs")
plt.savefig('output/compare_dropout.pdf', bbox_inches='tight')
plt.close(fig)
print('\nEvaluation of different activation functions\n')
# Model 5. No Dropout; SGD; Tanh
print('\nModel 5: Optimizer: SGD; No dropout; Tanh; CrossEntropy')
# Define model name for plots
mname = 'Model5'
# Define structure of the network
linear_1 = Linear(2, hidden_nb)
relu_1 = Tanh()
linear_2 = Linear(hidden_nb, hidden_nb)
relu_2 = Tanh()
linear_3 = Linear(hidden_nb, hidden_nb)
relu_3 = Tanh()
linear_4 = Linear(hidden_nb, 2)
model_5 = Sequential(linear_1,
relu_1,
linear_2,
relu_2,
linear_3,
relu_3,
linear_4,
loss=CrossEntropy())
# Initialize weights
model_5.normalize_parameters(mean=0, std=std)
# Define optimizer
optimizer = Sgd(eta)
# Train model
my_loss_5 = train_model(model_5, train_input, train_target, optimizer,
epochs_number, Sample_number, batch_size)
# Evalute model and produce plots
model_5_perf = evaluate_model(model_5,
train_input,
train_target,
test_input,
test_target,
my_loss_5,
save_plot,
mname=mname)
# Model 6. Xavier Initialization
print(
'\nModel 6: Optimizer: SGD; No dropout; Tanh; Xavier initialization; CrossEntropy'
)
# Define model name for plots
mname = 'Model6'
# Define network structure
linear_1 = Linear(2, hidden_nb)
relu_1 = Tanh()
linear_2 = Linear(hidden_nb, hidden_nb)
relu_2 = Tanh()
linear_3 = Linear(hidden_nb, hidden_nb)
relu_3 = Tanh()
linear_4 = Linear(hidden_nb, 2)
model_6 = Sequential(linear_1,
relu_1,
linear_2,
relu_2,
linear_3,
relu_3,
linear_4,
loss=CrossEntropy())
model_6.xavier_parameters()
optimizer = Sgd()
# Train model
my_loss_6 = train_model(model_6, train_input, train_target, optimizer,
epochs_number, Sample_number, batch_size)
# Evalute model and produce plots
model_6_perf = evaluate_model(model_6,
train_input,
train_target,
test_input,
test_target,
my_loss_6,
save_plot,
mname=mname)
# Model 7. Sigmoid
print('\nModel 7: Optimizer: SGD; No dropout; Sigmoid; CrossEntropy')
# Define model name for plots
mname = 'Model7'
# Define parameter for sigmoid activation
p_lambda = 0.1
# Define network structure
linear_1 = Linear(2, hidden_nb)
relu_1 = Sigmoid(p_lambda)
linear_2 = Linear(hidden_nb, hidden_nb)
relu_2 = Sigmoid(p_lambda)
linear_3 = Linear(hidden_nb, hidden_nb)
relu_3 = Sigmoid(p_lambda)
linear_4 = Linear(hidden_nb, 2)
model_7 = Sequential(linear_1,
relu_1,
linear_2,
relu_2,
linear_3,
relu_3,
linear_4,
loss=CrossEntropy())
model_7.normalize_parameters(mean=0.5, std=1)
optimizer = Sgd(eta=0.5)
# Train model
my_loss_7 = train_model(model_7, train_input, train_target, optimizer,
epochs_number, Sample_number, batch_size)
# Evalute model and produce plots
model_7_perf = evaluate_model(model_7,
train_input,
train_target,
test_input,
test_target,
my_loss_7,
save_plot,
mname=mname)
# PLOT TO COMPARE EFFECT OF DIFFERENT ACTIVATIONS
if save_plot:
fig = plt.figure(figsize=(10, 4))
plt.plot(range(0, epochs_number), my_loss_1, linewidth=0.5)
plt.plot(range(0, epochs_number), my_loss_5, linewidth=0.5, alpha=0.8)
plt.plot(range(0, epochs_number), my_loss_6, linewidth=0.5, alpha=0.8)
plt.plot(range(0, epochs_number), my_loss_7, linewidth=0.5)
plt.legend(["Relu", "Tanh", "Tanh (Xavier)", "Sigmoid"])
plt.title("Loss")
plt.xlabel("Epochs")
plt.savefig('output/compare_activations.pdf', bbox_inches='tight')
plt.close(fig)
print('\nEvaluation of base model with MSE loss\n')
# Model 8. MSE loss
print('\nModel 8: Optimizer: SGD; No dropout; Relu; MSE')
# Define model name for plots
mname = 'Model8'
linear_1 = Linear(2, hidden_nb)
relu_1 = Relu()
linear_2 = Linear(hidden_nb, hidden_nb)
relu_2 = Relu()
linear_3 = Linear(hidden_nb, hidden_nb)
relu_3 = Relu()
linear_4 = Linear(hidden_nb, 2)
loss = LossMSE()
model_8 = Sequential(linear_1,
relu_1,
linear_2,
relu_2,
linear_3,
relu_3,
linear_4,
loss=loss)
model_8.normalize_parameters(mean=0, std=std)
optimizer = Sgd(eta)
# Train model
my_loss_8 = train_model(model_8, train_input, train_target, optimizer,
epochs_number, Sample_number, batch_size)
# Evalute model and produce plots
model_8_perf = evaluate_model(model_8,
train_input,
train_target,
test_input,
test_target,
my_loss_8,
save_plot,
mname=mname)
print('Evaluation done! ')
train_loss = torch.tensor([
model_1_perf[0], model_2_perf[0], model_3_perf[0], model_4_perf[0],
model_5_perf[0], model_6_perf[0], model_7_perf[0], model_8_perf[0]
])
train_error = torch.tensor([
model_1_perf[1], model_2_perf[1], model_3_perf[1], model_4_perf[1],
model_5_perf[1], model_6_perf[1], model_7_perf[1], model_8_perf[1]
])
test_loss = torch.tensor([
model_1_perf[2], model_2_perf[2], model_3_perf[2], model_4_perf[2],
model_5_perf[2], model_6_perf[2], model_7_perf[2], model_8_perf[2]
])
test_error = torch.tensor([
model_1_perf[3], model_2_perf[3], model_3_perf[3], model_4_perf[3],
model_5_perf[3], model_6_perf[3], model_7_perf[3], model_8_perf[3]
])
return train_loss, train_error, test_loss, test_error
from Linear import Linear
from L import calcular_L
from sympy import *
# Define global parameters
A_1, A_c, A_2, K_1, K_c, K_2, b_c = var("A_1 A_c A_2 K_1 K_c K_2 b_c")
# Create a linear object
sys = Linear()
# Define the sizes of the state and the input
x = sys.state(3)
u = sys.input(1)
# Define functions that can't be defined as functions above, i.e.
# f(y,x_1,x_2,x_3,...,x_n) = g(y,x_1,x_2,x_3,...,x_n)
# The method receive the name of the function, the f(y,x_1,x_2,...,x_n)
# function, the g(y,x_1,x_2,...,x_n) function, and a list with tuples
# representing the args that call the function (this is a hack,
# because the actual matching system is not as good as i want).
# State function
sys.f(Matrix(
[u[0]/A_1 -K_1 / A_1 * pow(x[0], 2.475),
(K_1 / A_c) * pow(x[0], 2.475) - (K_c * b_c / A_c) * pow(x[1], 1.8),
(K_c * b_c / A_2) * pow(x[1], 1.8) - (K_2 / A_2) * x[2]
])
)
# Output function
def train_model():
# Transform the image by normalizing it
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
# Download Training data
trainset = torchvision.datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=transform)
# Download Test data
testset = torchvision.datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform)
# Make trainloader
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=1,
shuffle=True)
# Make testloader
testloader = torch.utils.data.DataLoader(testset, batch_size=1)
# Class present in training and test data
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck')
"""
# Function to display Image
# def imshow(img):
# img = img / 2 + 0.5 # unnormalize
# npimg = img.numpy()
# plt.imshow(np.transpose(npimg, (1, 2, 0)))
# plt.show()
#
# dataiter = iter(trainloader)
# images, labels = dataiter.next()
"""
# My model
my_model = Model()
my_model.addLayer(Linear(3072, 1024))
my_model.addLayer(ReLu())
# my_model.addLayer(Linear(2048, 1024))
# my_model.addLayer(ReLu())
my_model.addLayer(Linear(1024, 512))
my_model.addLayer(ReLu())
my_model.addLayer(Linear(512, 2))
# my_model.addLayer(Softmax())
# my_model.addLayer(CrossEntropy())
# Loop to train the Model
running_loss = 0
# Number of epochs
epochs = 7
train_count = 0
train_losses, test_losses = [], []
train_correct = 0
i = 0
for epoch in range(epochs):
for images, labels in trainloader:
# Transfer it to GPU
if train_on_gpu:
images, labels = images.to(device), labels.to(device)
if labels == 0 or labels == 1:
# To count number to training image in each epoch
train_count += 1
# Flatteing of image to bring it to size batch_sizex(32*32*3)
images = images.view(images.size(0), -1)
# forward the image through the model
final_prob = my_model.forward(images)
# Calculate the backward gradient of CrossEntropy
backward_grad = CrossEntropy().backward(final_prob, labels)
# changing in to exp score
ps = torch.exp(final_prob)
# getting the top class
top_p, top_class = ps.topk(1, dim=1)
if top_class == labels:
train_correct += 1
# Backpropagate the model
my_model.backward(images, backward_grad, alpha=0.001)
# calculate the running loss
running_loss += (CrossEntropy().forward(final_prob, labels))
# Function to Calculate Validation loss and accuracy on Validation data
if (train_count + 1) % 500 == 0:
i = i + 1
test_loss = 0
correct_class = 0
test_count = 0
for images, labels in testloader:
if train_on_gpu:
images, labels = images.to(device), labels.to(device)
if labels == 0 or labels == 1:
test_count += 1
# Flatteing of image
images = images.view(images.size(0), -1)
# forward the image in trained model
score = my_model.forward(images)
# calculate loss
test_loss += CrossEntropy().forward(score, labels)
# selct the top class with max score
ps = torch.exp(score)
top_p, top_class = ps.topk(1, dim=1)
# if top_class is same as the target label than increse correct count by 1
if top_class == labels:
correct_class += 1
# Append to plot graph
train_losses.append(running_loss / (train_count + 1))
test_losses.append(test_loss / (test_count + 1))
print(f"Epoch {i}.. "
f"Train loss: {running_loss/(train_count):.3f} .."
f"Test loss: {test_loss/(test_count):.3f} .."
f"Train accuracy: {train_correct/(train_count):.3f}.."
f"Test accuracy: {correct_class/(test_count):.3f}")
train_correct = 0
train_count = 0
running_loss = 0
plt.plot(train_losses, label='Training loss')
plt.plot(test_losses, label='Validation loss')
plt.legend(frameon=False)
#plt.savefig('mlp2.png', dpi=100)
return my_model
rho_e = 714
# Define global functions that can be defined in an explicit way, i.e.
# y = f(x_1,x_2,...x_n) \forall n \in N and x_i \notequal y
def L(beta):
return L_k + 4 * r * ( beta - tan(beta)) + l / cos(beta)
def T(rho):
return Y * A * (rho_o /rho - 1)
def rho(L,M):
return M /(A * L)
# Create a linear object
sys = Linear()
# Define the sizes of the state and the input
x = sys.state(3)
u = sys.input(2)
# Define functions that can't be defined as functions above, i.e.
# f(y,x_1,x_2,x_3,...,x_n) = g(y,x_1,x_2,x_3,...,x_n)
# The method receive the name of the function, the f(y,x_1,x_2,...,x_n)
# function, the g(y,x_1,x_2,...,x_n) function, and a list with tuples
# representing the args that call the function (this is a hack,
# because the actual matching system is not as good as i want).
beta = sys.function(
"beta",
lambda y,x: l * sin(y) - 4*r, # f(y, x_1, x_2, x_3. ... , x_n)