def linear_regression(a=1.0, b=0.0):
X = np.linspace(-100, 100, 200)
X = X.reshape((-1, 1))
[train_x, test_x] = split_data(X, ratio=0.8, random=True)
train_y = a * train_x + b
test_y = a * test_x + b
i = Input(1)
x = Dense(1)(i)
# define trainer
trainer = Trainer(loss='mse',
optimizer=Adam(learning_rate=0.2),
batch_size=50,
epochs=50)
# create model
model = Sequential(i, x, trainer)
model.summary()
# training process
model.fit(train_x, train_y)
# predict
y_hat = model.predict(test_x)
plt.plot(test_x, test_y, 'b')
plt.plot(test_x, y_hat, 'r')
plt.show()
def run():
file_path = os.path.dirname(
os.path.realpath(__file__)) + "/dlmb_mnist_example.json"
# If a file of the neural-net model's architexture already exists,
# then there is no need to build a new model.
if os.path.isfile(file_path):
# load the model and get its predictions based on x_test
nn_model = Sequential()
nn_model.load(file_path)
predictions = nn_model.predict(x_test)
# compare the predictions to the correct labels
print(
f"This model got a {validate_model(predictions, y_test)/100}% accuracy"
)
# If the file doesn't exist then we need to build a neural-net model and train it.
else:
# Build the neural-net model
nn_model = Sequential([
Dense(
128, 784, activation="ReLU"
), # for the layer_dim we want 128 outputs and 784 inputs (each pixel on the image)
Batchnorm(128),
Dense(128, 128, activation="ReLU"),
Batchnorm(128),
Dense(32, 128, activation="ReLU"),
Batchnorm(32),
Dense(10, 32, activation="Softmax"
) # We have 10 nodes in the layer for each number from 0 - 9
])
nn_model.build(loss="crossentropy", optimizer="adam")
# Crossentropy is a good loss function when you are doing logistic regression (classification)
# Adam is one of the most popular optimizers
nn_model.train(x_train, y_train, epochs=10, batch_size=1000)
# Train the model
# We go through the data 10 times and split the data of 60000 samples into 1000 sized batches leaving 60 samples
# Now we save the model so we can use it again without re-training
nn_model.save(file_path) # When saving, files must end in .json
def linear_classification(a=1.0, b=0.0, graph=False):
# prepare data
x = np.linspace(-100, 100, 200)
y = a * x + b
X = np.array(list(zip(x, y))) + np.random.randn(200, 2) * 100
Y = to_one_hot(np.where(a * X[:, 0] + b > X[:, 1], 1, 0))
(train_x, train_y), (test_x, test_y) = split_data(X,
Y,
ratio=0.8,
random=True)
# build simple FNN
i = Input(2)
x = Dense(2, activation='softmax')(i)
# define trainer
trainer = Trainer(loss='cross_entropy',
optimizer=Adam(learning_rate=0.05),
batch_size=50,
epochs=50,
metrics=['accuracy'])
# create model
model = Sequential(i, x, trainer)
model.summary()
# training process
model.fit(train_x, train_y)
print(model.evaluate(test_x, test_y))
if graph:
plt.plot(model.history['loss'])
plt.show()
# predict
y_hat = model.predict(test_x)
y_hat = np.argmax(y_hat, axis=1)
simple_plot(test_x, y_hat, a, b)
def binary_classification():
def separate_label(data):
X = normalize(data[:, :2].astype('float32'))
Y = np.where(data[:, 2] == b'black', 0, 1)
return X, Y
# prepare train data
data_dir = "data/examples/binary_classification"
train_data_path = os.path.join(data_dir, 'training.arff')
train_data = load_arff(train_data_path)
train_x, train_y = separate_label(train_data)
train_y = to_one_hot(train_y)
# build simple FNN
i = Input(2)
x = Dense(30, activation='relu')(i)
x = Dense(30, activation='relu')(x)
x = Dense(2, activation='softmax')(x)
# define trainer
trainer = Trainer(loss='cross_entropy', optimizer=Adam(clipvalue=1.0), batch_size=256, epochs=500, metrics=['accuracy'])
# create model
model = Sequential(i, x, trainer)
model.summary()
# training process
model.fit(train_x, train_y)
plt.plot(range(len(model.history['loss'])), model.history['loss'])
plt.show()
# predict
test_data_path = os.path.join(data_dir, 'test.arff')
test_data = load_arff(test_data_path)
test_x, _ = separate_label(test_data)
y_hat = model.predict(test_x)
simple_plot(test_x, y_hat)
def universal_approximation(f, x):
[train_x, test_x] = split_data(x, ratio=0.8, random=True)
train_y = f(train_x)
test_x = np.sort(test_x, axis=0)
test_y = f(test_x)
# build simple FNN
i = Input(1)
x = Dense(50, activation='relu')(i)
x = Dense(1)(x)
# define trainer
schedule = ExponentialDecay(initial_learning_rate=0.01, decay_rate=0.75)
trainer = Trainer(loss='mse',
optimizer=Adam(learning_rate=schedule),
batch_size=50,
epochs=750)
# create model
model = Sequential(i, x, trainer)
model.summary()
# training process
start = time.time()
model.fit(train_x, train_y)
print(time.time() - start)
plt.plot(range(len(model.history['loss'])), model.history['loss'])
plt.show()
# predict
y_hat = model.predict(test_x)
plt.plot(test_x, test_y, 'b-', label='original')
plt.plot(test_x, y_hat, 'r-', label='predicted')
plt.legend()
plt.show()
model.add(Dense(2, 2, kernel_initializer=saved_kernel_0, bias_initializer=saved_bias_0, alpha=50.0))
model.add(Sigmoid())
model.add(Dense(1, 2, kernel_initializer=saved_kernel_1, bias_initializer=saved_bias_1, alpha=50.0))
model.add(Sigmoid())
X = np.array([[0, 0],
[0, 1],
[1, 0],
[1, 1]])
y = np.array([[1],
[0],
[0],
[1]])
print("Prediction")
p = model.predict(X)
print(p)
print("Error")
print(p-y)
loss_function = SquaredError()
custom_loss = CustomLoss()
print("Training")
loss_history = model.fit(X, y, epochs=100, batch_size=4, steps_per_epoch=1000, halt=False, loss=custom_loss)
print("Prediction")
p = model.predict(X)
print(p)
print("Error")
print(p-y)
print("Weights in first dense layer")
Dense(16, 784, kernel_initializer=truncated_normal,
bias_initializer=zeros))
model.add(Sigmoid())
model.add(
Dense(10, 16, kernel_initializer=truncated_normal, bias_initializer=zeros))
model.add(Sigmoid())
loss = SquaredError()
loss_history = model.fit(train_imgs,
train_labels_one_hot,
batch_size=32,
epochs=10,
loss=loss,
halt=False)
pred = model.predict(test_imgs)
pred_labels = pred.argmax(1)
print("MSE", loss.evaluate(pred, test_labels_one_hot).mean(0))
print("Percentage correct", np.mean(pred_labels == test_labels) * 100)
print("Prediction for first 5 images")
print(pred[0:5, :].argmax(1))
print("True labels")
print(test_labels[0:5])
plt.plot(np.arange(0, 10), loss_history.mean(1))
plt.title("Graph of mean loss over all one-hot outputs")
plt.xlabel("Epoch")
plt.ylabel("Mean loss")
plt.show()
print(model.save("mnist_model.pkl"))
with open("pickled_mnist.pkl", "br") as fh:
data = pickle.load(fh)
train_imgs = data[0]
test_imgs = data[1]
train_labels = data[2]
test_labels = data[3]
train_labels_one_hot = data[4]
test_labels_one_hot = data[5]
model = Sequential()
model.restore("mnist_model.pkl")
loss = SquaredError()
pred = model.predict(test_imgs)
pred_labels = pred.argmax(1)
print("MSE", loss.evaluate(pred, test_labels_one_hot).mean(0))
print("Percentage correct", np.mean(pred_labels==test_labels)*100)
print("Prediction for first 5 images")
print(pred[0:5, :].argmax(1))
print("True labels")
print(test_labels[0:5])
fig, ax = plt.subplots(2, 5)
for i, ax in enumerate(ax.flatten()):
im_idx = np.argwhere(test_labels == i)[0, 0]
print(test_imgs[im_idx, :].shape)
img = np.reshape(test_imgs[im_idx, :], (28, 28))
ax.imshow(img, cmap="gray_r")