def compile(self, loss):
if loss == "binary_crossentropy":
self.loss = BinaryCrossentropy(self)
elif loss == "mse":
self.loss = MSE(self)
else:
logger.warning("Invalid loss function.")
self.loss = None
def main():
train_data = np.array([[0, 0], [0, 1], [0, 2], [0, 3], [0, 4], [0, 5],
[0, 6], [0, 7], [1, 0], [1, 1], [1, 2], [1, 3],
[1, 4], [1, 5], [1, 6], [1, 7], [2, 0], [2, 1],
[2, 2], [2, 3], [2, 4], [2, 5], [2, 6], [2, 7],
[3, 0], [3, 1], [3, 2], [3, 3], [3, 4], [3, 5],
[3, 6], [3, 7], [4, 0], [4, 1], [4, 2], [4, 3],
[4, 4], [4, 5], [4, 6], [4, 7], [5, 0], [5, 1],
[5, 2], [5, 3], [5, 4], [5, 5], [5, 6], [5, 7],
[6, 0], [6, 1], [6, 2], [6, 3], [6, 4], [6, 5],
[6, 6], [6, 7], [7, 0], [7, 1], [7, 2], [7, 3],
[7, 4], [7, 5], [7, 6], [7, 7]])
label_data = np.array(
[[0], [1], [2], [3], [4], [5], [6], [7], [1], [0], [3], [2], [5], [4],
[7], [6], [2], [3], [0], [1], [6], [7], [4], [5], [3], [2], [1], [0],
[7], [6], [5], [4], [4], [5], [6], [7], [0], [1], [2], [3], [5], [4],
[7], [6], [1], [0], [3], [2], [6], [7], [4], [5], [2], [3], [0], [1],
[7], [6], [5], [4], [3], [2], [1], [0]])
learn_rate = 0.15
epoch = 600000
loss_list = []
w1, b1, w2, b2 = init(input_size=2, hidden_size=16, output_size=1)
#print(w1, w2, b1, b2)
for i in range(epoch):
affine1 = Affine(w1, b1)
affine2 = Affine(w2, b2)
sigmoid = Sigmoid()
loss = MSE()
x1 = affine1.forward(train_data)
y1 = sigmoid.forward(x1)
x2 = affine2.forward(y1)
ls = loss.mean_square_error(x2, label_data)
print(ls)
loss_list.append(ls)
dout = loss.backward(x2, label_data)
dx = affine2.backward(dout)
w2 = w2 - learn_rate * affine2.dw
b2 = b2 - learn_rate * affine2.db
dy1 = sigmoid.backward(dx)
dx = affine1.backward(dy1)
b1 = b1 - learn_rate * affine1.db
w1 = w1 - learn_rate * affine1.dw
#print(w1,w2,b1,b2)
plt.plot(loss_list)
plt.show()
acc(w1, b1, w2, b2, train_data, label_data)
def train_nn(
net: NeuralNet,
inputs: Tensor,
targets: Tensor,
epochs: int = 1,
loss: Loss = MSE(),
batch_iter: DataIterator = BatchIterator(),
optimizer: Optimizer = SGD(),
) -> None:
for epoch in range(epochs):
epoch_loss = 0.
for batch in batch_iter(inputs, targets):
pred = net.forward(batch.input)
batch_loss = loss.loss(pred, batch.target)
epoch_loss += batch_loss
loss_grad = loss.grad(pred, batch.target)
net_grad = net.backward(loss_grad)
optimizer.step(net)
print(f'epoch:{epoch}, loss:{epoch_loss}')
def __init__(self, encoder=None, sampler=None, decoder=None):
super().__init__()
if encoder != None and sampler != None and decoder != None:
self.layers = encoder.layers + [sampler.mean, sampler.logVar
] + decoder.layers
self.encoder = encoder
self.sampler = sampler
self.decoder = decoder
self.decoder.loss = MSE()
def train(self,
dataset,
loss=MSE(),
epochs=1,
metrics=["train_loss", "test_loss"],
tensorboard=False,
callbacks={}):
super().train(dataset,
loss=loss,
epochs=epochs,
metrics=metrics,
tensorboard=tensorboard,
callbacks=callbacks,
autoencoder=True,
noise=None)
def train(net: NeuralNet,
inputs: Tensor,
targets: Tensor,
num_epochs: int = 5000,
iterator: DataIterator = BatchIterator(),
loss: Loss = MSE(),
optimizer: Optimizer = SGD()
) -> None:
for epoch in range(num_epochs):
epoch_loss = 0.0
for batch in iterator(inputs, targets):
predicted = net.forward(batch.inputs)
epoch_loss += loss.loss(predicted, batch.targets)
grad = loss.grad(predicted, batch.targets)
net.backward(grad)
optimizer.step(net)
print(epoch, epoch_loss)
def train(net: NetWork,
inputs: Tensor,
targets: Tensor,
epochs: int = 500,
loss: Loss = MSE(),
optimizer: Optimizer = SGD(),
iterator: DataIterator = BatchIterator(),
show_info: bool = False):
for epoch in range(epochs):
epoch_loss = .0
for batch_inputs, batch_targets in iterator(inputs, targets):
predictions = net.forward(batch_inputs)
epoch_loss += loss.loss(predictions, batch_targets)
grad = loss.grad(predictions, batch_targets)
net.backward(grad)
optimizer.step(net)
if show_info:
print('epoch:{}, loss:{}'.format(epoch, epoch_loss))
def train(self, X, label, loss=MSE()):
if self.configured == False:
prevLayer = None
for l in self.layers:
l.configure(X.shape, "TRAIN", prevLayer)
prevLayer = l
self.configured = True
self.Y = X
for layer in self.layers:
self.Y = layer.forward(self.Y)
err = loss.calc(self.Y, label, self.batch)
err_delta = loss.partial_derivative(self.Y, label)
for layer in self.layers[::-1]:
err_delta = layer.backward(err_delta)
return err
def train(self,
dataset,
loss=MSE(),
epochs=1,
metrics=["train_loss", "test_loss"],
tensorboard=False,
callbacks={},
autoencoder=False,
noise=None):
metricsWriter = Writer(metrics, callbacks, tensorboard)
self.loss = loss
ind = 0 # number of samples processed
for i in range(epochs):
logging.debug(f" *** EPOCH {i+1}/{epochs} ***")
for (train, test,
batchSize) in dataset.batches(onehot_encoded=True,
autoencoder=autoencoder,
noise=noise):
# set batch size before training
for layer in self.layers:
layer.setBatchSize(batchSize)
self.feedforward(train[0])
self.backpropagate(train[1])
if ind % 1000 < batchSize:
if "train_loss" in metrics:
metricsWriter.add(metric="train_loss",
index=ind,
value=self.getLoss(train[1]))
if "train_accuracy" in metrics:
metricsWriter.add(metric="train_accuracy",
index=ind,
value=self.getAccuracy(train[1]))
self.validate(test,
ind,
callbacks,
writer=metricsWriter,
metrics=metrics)
ind += batchSize
metricsWriter.close()
def load(self, name):
modelDir = f"./models/{name}"
self.encoder = MLP()
self.decoder = MLP()
self.decoder.loss = MSE()
# load encoder and decoder
for name, model in [("encoder", self.encoder),
("decoder", self.decoder)]:
layerDir = [
dir for dir in os.listdir(modelDir)
if os.path.isdir(os.path.join(modelDir, dir)) and name in dir
]
layerDir.sort(key=lambda x: int(x.strip(f"{name}_layer")))
for dir in layerDir:
layerFolder = os.path.join(modelDir, dir)
if "dense.json" in os.listdir(layerFolder):
# this is a dense layer
newLayer = Dense()
newLayer.load(layerFolder)
model.layers.append(newLayer)
# load aditional information about sampler
with open(f"{modelDir}/sampler.json", "r") as file:
data = json.load(file)
inputDim = data["inputDim"]
outputDim = data["outputDim"]
self.sampler = Sampler(inputDim, outputDim)
# load mean and logvar layer
self.sampler.mean = Dense()
self.sampler.mean.load(os.path.join(modelDir, f"sampler_mean"))
self.sampler.logVar = Dense()
self.sampler.logVar.load(os.path.join(modelDir, f"sampler_logvar"))
self.layers = self.encoder.layers + [
self.sampler.mean, self.sampler.logVar
] + self.decoder.layers
outputDim=1024,
activation=LeakyReLU(0.2),
optimizer=optimizer))
discriminator.addLayer(
Dense(inputDim=1024,
outputDim=512,
activation=LeakyReLU(0.2),
optimizer=optimizer))
discriminator.addLayer(
Dense(inputDim=512,
outputDim=256,
activation=LeakyReLU(0.2),
optimizer=optimizer))
discriminator.addLayer(
Dense(inputDim=256,
outputDim=1,
activation=Sigmoid(),
optimizer=optimizer))
gan = GAN(generator, discriminator)
print(gan)
gan.train(dataset,
loss=MSE(),
epochs=50,
metrics=["generator_loss", "discriminator_loss"],
tensorboard=True,
callbacks=[])
gan.save("tryout_gan")
class Model(object):
def __init__(self):
self.layers = []
def add(self, layer):
if (self.layers == []):
if layer.input_dim is None:
logger.warning("First layer has no input dimension.")
self.layers.append(layer)
else:
last_layer = self.layers[-1]
layer.prev = last_layer
if isinstance(layer, Layer):
layer.input_dim = last_layer.output_dim
elif isinstance(layer, Activation):
layer.input_dim = last_layer.output_dim
layer.output_dim = layer.input_dim
self.layers.append(layer)
assert (layer.output_dim is not None)
assert (layer.input_dim is not None)
if isinstance(layer, Layer):
layer.build()
def compile(self, loss):
if loss == "binary_crossentropy":
self.loss = BinaryCrossentropy(self)
elif loss == "mse":
self.loss = MSE(self)
else:
logger.warning("Invalid loss function.")
self.loss = None
def evaluate(self, X, Y):
if isinstance(self.loss, MSE):
return self.loss.mse(X, Y)
N, _ = X.shape
pred_val = self.predict(X)
y_answer = np.argmax(Y, axis=1)
pred_answer = np.argmax(pred_val, axis=1)
accuracy = 1 - float(np.sum(abs(y_answer - pred_answer))) / N
return accuracy
def fit(self, X, Y, batch_size=32, iterations=500, validation_data=None):
N, _ = X.shape
history = []
batches = range(N / batch_size)
batches = [(i * batch_size, (i + 1) * batch_size) for i in batches]
for epoch in xrange(1, iterations + 1):
np.random.shuffle(batches)
total_loss = 0
for start, end in batches:
batchX = X[start:end]
batchY = Y[start:end]
loss = self.loss.apply_grad(batchX, batchY)
total_loss += loss
train_accuracy = self.evaluate(X, Y)
if validation_data is not None:
Xval = validation_data[0]
yval = validation_data[1]
val_accuracy = self.evaluate(Xval, yval)
val_loss = self.loss(Xval, yval)
else:
val_accuracy = "NO EVALUATION DATA"
val_loss = "NO EVALUATION DATA"
history.append(
[epoch, total_loss, val_loss, train_accuracy, val_accuracy])
print '-----------------------'
print 'Epoch', epoch
print 'Total Training Loss:', total_loss
print 'Total Validation Loss', val_loss
print 'Training Accuracy:', train_accuracy
print 'Validation Accuracy:', val_accuracy
print '-----------------------'
return history
def predict(self, X):
n, input_dim = X.shape
if len(self.layers) == 0:
logger.warning("Model has no layers.")
return None
if input_dim != self.layers[0].input_dim:
logger.warning("Input dimension does not match.")
return None
return self.layers[-1].call(X)
inputs = np.array([
[0, 0],
[1, 0],
[0, 1],
[1, 1]
])
targets = np.array([
[1, 0],
[0, 1],
[0, 1],
[1, 0]
])
net = NeuralNetwork([
LinearLayer(inputSize=2, outputSize=2),
LeakyRelu(),
LinearLayer(inputSize=2, outputSize=2),
Tanh()
])
train(net, inputs, targets, loss= MSE(), num_epochs=5000, optimizer=MBGD(learningRate=0.01), showGraph=True)
# net.loadParamsFromFile("/home/ayush/scratch/Net/aknet/serialized.json")
for x, y in zip(inputs, targets):
predicted = net.forward(x)
print(x, predicted, y)
net.serialize("serialized.json")
prefix = np.random.rand()*2
dataX[i,:] = np.linspace(prefix, prefix+np.pi*4, dataPoints)
labelY = np.sin(dataX)
epoch = 60
batchSize = 20
input_shape = 2048
units = 256
last_units = 64
lstm = Network([LSTM(units, input_shape),
FullyConnect(units=last_units, input_shape=units)
],
learning_rate=learning_rate,
optimizer=Momentum(0.9),
batch=batchSize,
)
err_prev = 0
for e in range(epoch):
err = 0
for batchIdx in range(0, dataNum, batchSize):
batchData = labelY[batchIdx:batchIdx + batchSize, :]
for timeIdx in range(0, dataPoints-input_shape, input_shape):
err += lstm.train(batchData[:, timeIdx:timeIdx+input_shape], batchData[:, timeIdx+input_shape:timeIdx+input_shape+last_units], loss=MSE())
print "epoch", e
print "\t", err/batchSize
err_prev = err
print lstm.predict(labelY[:batchSize, :input_shape]), labelY[:batchSize, input_shape:input_shape+units]
nxt = lstm.predict(labelY[:batchSize,:input_shape])
plt.plot(dataX[0,input_shape:input_shape+last_units], nxt[0, :], dataX[0, input_shape:input_shape+last_units], labelY[0, input_shape:input_shape+last_units])
plt.show()
def train(self,
dataset,
loss=MSE(),
epochs=1,
metrics=["generator_loss", "discriminator_loss"],
tensorboard=False,
callbacks={},
autoencoder=False,
noise=None):
metricsWriter = Writer(metrics, callbacks, tensorboard)
imgNoise = np.random.standard_normal(
size=(self.generator.layers[0].inputDim, 10))
ind = 0 # number of samples processed
for i in range(epochs):
logging.debug(f" *** EPOCH {i+1}/{epochs} ***")
for (train, test,
batchSize) in dataset.batches(onehot_encoded=True,
autoencoder=autoencoder,
noise=noise):
# set batch size before training
for layer in self.layers:
layer.setBatchSize(batchSize)
# 1. Train discriminator
fake_img = self.generator.feedforward(self.sample(batchSize))
real_img = np.asarray(train[0])
input = np.concatenate((fake_img, real_img), axis=1)
label = np.concatenate((np.zeros(
(1, batchSize)), 0.9 * np.ones((1, batchSize))),
axis=1)
self.discriminator.feedforward(np.asarray(input))
self.discriminator.backpropagate(np.asarray(label))
discriminatorLoss = self.discriminator.getLoss(
np.asarray(label))
# 2. Train generator
#fake_img = self.generator.feedforward(self.sample(batchSize))
self.discriminator.feedforward(fake_img)
self.discriminator.backpropagate(np.ones((1, batchSize)),
updateParameters=False)
discriminatorGradient = self.discriminator.layers[0].gradient
self.generator.backpropagate(discriminatorGradient,
useLoss=False)
generatorLoss = self.discriminator.getLoss(
np.ones((1, batchSize)))
if ind % 1000 < batchSize:
if "generator_loss" in metrics:
metricsWriter.add(metric="generator_loss",
index=ind,
value=generatorLoss)
if "discriminator_loss" in metrics:
metricsWriter.add(metric="discriminator_loss",
index=ind,
value=discriminatorLoss)
#self.validate(test, ind, callbacks, writer = metricsWriter, metrics = metrics)
ind += batchSize
self.generateImages(imgNoise, i)
metricsWriter.close()
],
#FullyConnect(units=last_units)],
# FullyConnect(units=last_units, input_shape=144)],
learning_rate = learning_rate,
optimizer=Momentum(0.9),
batch=batchSize,
dtype=np.float32
)
err_prev = 0
for e in range(epoch):
err = 0
for batchIdx in range(0, dataNum, batchSize):
batchData = dataset[batchIdx:batchIdx + batchSize,:,:]
batchLabel = label[batchIdx:batchIdx + batchSize,:]
err += net.train(batchData, batchLabel, loss=MSE())
print "epoch", e
print "\t", err
if abs(err_prev - err) < 10e-15:
print "early_stop"
break
err_prev = err
print net.predict(dataset[0, :, :].reshape(1,1,input_shape)), label[0,:]
print net.predict(dataset[dataNum-1, :].reshape(1,1,input_shape)), label[dataNum-1,:]
"""
# 100 epoch 10 batch 8*8 image
epoch = 100
batchSize = 10
def main(args):
total_size = 2000
train_size = 1000
test_size = 1000
data, target = generate_disc_set(total_size, random_state=1)
train_data, train_target = data[:train_size], target[:train_size]
test_data, test_target = data[test_size:], target[test_size:]
colours = ['blue', 'green', 'red']
def colour_labels(labels):
return list(map(lambda x: colours[x], labels))
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.scatter(train_data[:, 0],
train_data[:, 1],
c=colour_labels(train_target.argmax(1)),
edgecolors='none')
plt.title('Train Data')
plt.xlabel(r'$x_{1}$')
plt.ylabel(r'$x_{2}$')
plt.subplot(1, 2, 2)
plt.scatter(test_data[:, 0],
test_data[:, 1],
c=colour_labels(test_target.argmax(1)),
edgecolors='none')
plt.title('Test Data')
plt.xlabel(r'$x_{1}$')
plt.ylabel(r'$x_{2}$')
plt.pause(1)
plt.show(block=False)
if args.loss == 'mse':
net_loss = MSE()
net = Sequential(DenseLayer(2, 25), ReLU(), DenseLayer(25, 25), ReLU(),
DenseLayer(25, 25), ReLU(), DenseLayer(25, 2))
elif args.loss == 'softmax_loss':
net_loss = CrossEntropy()
net = Sequential(DenseLayer(2, 25), ReLU(), DenseLayer(25, 25), ReLU(),
DenseLayer(25, 25), ReLU(), DenseLayer(25, 2),
SoftMax())
else:
raise ValueError(
args.loss +
' is invalid loss. Please use either \'mse\' or \'softmax_loss\'.')
def sgd(x, dx, config):
for cur_layer_x, cur_layer_dx in zip(x, dx):
for cur_x, cur_dx in zip(cur_layer_x, cur_layer_dx):
cur_old_grad = config['learning_rate'] * cur_dx
if cur_old_grad.shape[0] == 1:
cur_x = cur_x.reshape(cur_old_grad.shape)
cur_x.add_(-cur_old_grad)
def train_model(model,
model_loss,
train_data,
train_target,
lr=0.005,
batch_size=1,
n_epoch=50):
optimizer_config = {'learning_rate': lr}
train_loss_history = []
test_loss_history = []
for i in range(n_epoch):
loss = 0
k = 0
for x_batch, y_batch in get_batches(train_data, train_target,
batch_size):
model.zero_grad_params()
# Forward
pred = model.forward(x_batch)
loss += model_loss.forward(pred, y_batch)
# Backward
lg = model_loss.backward(pred, y_batch)
model.backward(lg)
# Update weights
sgd(net.get_params(), net.get_grad_params(), optimizer_config)
k += 1
train_loss_history.append(loss / k)
test_pred = model.forward(test_data)
test_loss = model_loss.forward(test_pred, test_target)
test_loss_history.append(test_loss)
print('#Epoch {}: current train loss = {:.4f}'.format(
i + 1,
loss.item() / k))
return train_loss_history, test_loss_history
print('Training started...')
train_loss_history, test_loss_history = train_model(net,
net_loss,
train_data,
train_target,
n_epoch=50)
print('Final train loss: {:.4f}'.format(train_loss_history[-1]))
print('Final test loss: {:.4f}'.format(test_loss_history[-1]))
plt.figure(2, figsize=(8, 6))
plt.title("Train and Test Loss")
plt.xlabel("#Epochs")
plt.ylabel("loss")
plt.plot(train_loss_history, 'b')
plt.plot(test_loss_history, 'r')
plt.legend(['train loss', 'test loss'])
plt.pause(1)
plt.show(block=False)
train_res = net.forward(train_data)
errors_train = compute_nb_errors(train_res, train_target)
print("Number of errors on the train set: " + str(errors_train))
train_res = train_res.argmax(1)
train_res[train_res != train_target.argmax(1)] = 2
test_res = net.forward(test_data)
errors_test = compute_nb_errors(test_res, test_target)
print("Number of errors on the test set: " + str(errors_test))
test_res = test_res.argmax(1)
test_res[test_res != test_target.argmax(1)] = 2
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.scatter(train_data[:, 0],
train_data[:, 1],
c=colour_labels(train_res),
edgecolors='none')
plt.xlabel(r'$x_{1}$')
plt.ylabel(r'$x_{2}$')
plt.title(f'Train Data, {errors_train} errors')
plt.subplot(1, 2, 2)
plt.scatter(test_data[:, 0],
test_data[:, 1],
c=colour_labels(test_res),
edgecolors='none')
plt.xlabel(r'$x_{1}$')
plt.ylabel(r'$x_{2}$')
plt.title(f'Test Data, {errors_test} errors')
plt.show()
