def get_initial_weights(self, model_type):
tf.reset_default_graph()
if model_type == "perceptron":
m = Perceptron()
inputs = tf.placeholder(tf.float32, shape=(None, 28 * 28))
_ = m.get_model(features={"x": inputs},
labels=None,
mode='predict',
params=None)
elif model_type == 'cnn-mnist':
m = CNN()
inputs = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
_ = m.get_model(features={"x": inputs},
labels=None,
mode='predict',
params=None)
elif model_type == 'cnn-cifar10':
m = CNN()
inputs = tf.placeholder(tf.float32, shape=(None, 32, 32, 3))
_ = m.get_model(features={"x": inputs},
labels=None,
mode='predict',
params=None)
else:
raise ValueError(
"Model {model_type} not supported.".format(model_type))
with tf.Session().as_default() as sess:
sess.run(tf.global_variables_initializer())
collection = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
weights = {tensor.name: sess.run(tensor) for tensor in collection}
tf.reset_default_graph()
return weights
def setup_model(self, model_type):
self.model_type = model_type
if model_type == "perceptron":
self.model = Perceptron()
else:
raise ValueError("Model {model_type} not supported." \
.format(model_type))
def run_pam(args):
set_global_seeds(args['seed'])
dataset = DataLoader(args['dataset'])
X_train, X_test, X_val, y_train, y_test, y_val = dataset.prepare_train_test_val(
args)
model = Perceptron(feature_dim=X_train.shape[-1], margin=args['margin'])
model.fit(X_train, y_train)
return model.score(X_test, y_test)
def setup_model(self, model_type):
self.model_type = model_type
if model_type == "perceptron":
self.model = Perceptron()
elif model_type == "cnn-mnist":
self.model = CNN()
elif model_type == "cnn-cifar10":
self.model = CNN()
else:
raise ValueError("Model {0} not supported.".format(model_type))
def find_best_margin(args):
""" return `best_margin / 0.1` """
set_global_seeds(args['seed'])
dataset = DataLoader(args['dataset'])
X_train, X_test, X_val, y_train, y_test, y_val = dataset.prepare_train_test_val(
args)
results = []
for margin in MARGINS:
model = Perceptron(feature_dim=X_train.shape[-1], margin=margin)
model.fit(X_train, y_train)
results.append(model.score(X_val, y_val))
return results
def setup_model(self, model_type):
self.model_type = model_type
if model_type == "perceptron":
self.model = Perceptron()
self.weights_metadata = self.model.get_weights_shape()
elif model_type == "cnn":
#TODO: Support CNN
self.model = CNN()
elif model_type == "lstm":
#TODO: Support LSTM
self.model = LSTM()
elif model_type == "gan":
self.model = ConversationalNetwork()
self.model.build_model(is_retraining=True)
else:
raise ValueError("Model {0} not supported.".format(model_type))
def test_perceptron():
from models.perceptron import Perceptron
x, y = np.random.randn(2, 500, 2), np.zeros([2, 500], dtype=int)
x[0] += np.array([1, -1])
x[1] += np.array([-1, 1])
y[0] = -1
y[1] = 1
plot_scatter(x[0], x[1], 'Real')
x = x.reshape(-1, 2)
y = y.flatten()
perceptron = Perceptron(input_dim=2, lr=1e-4)
train_perceptron(perceptron, x, y, epochs=100)
pred = perceptron.predict(x)
plot_scatter_with_line(x[pred == -1], x[pred == 1], perceptron.weights, 'Pred')
acc = np.sum(pred == y) / len(pred)
print(f'Acc = {100 * acc:.2f}%')
def main():
# load datasets
train_dataset = load_dataset("data/truecased_reviews_train.jsonl")
dev_dataset = load_dataset("data/truecased_reviews_dev.jsonl")
test_dataset = load_dataset("data/truecased_reviews_test.jsonl")
# Part I: Feature Engineering
# Step 1. create feature vectors by calling compute_features() with all three datasets as parameters
train_vecs, dev_vecs, test_vecs = compute_features(train_dataset,
dev_dataset,
test_dataset)
print("Proportion of +1 label: ", np.mean(train_dataset.y == 1))
print("Proportion of -1 label: ", np.mean(train_dataset.y == -1))
# Step 2. train a Naive Bayes Classifier (scikit MultinomialNB() )
# TODO complete implementation
mnb = MultinomialNB()
mnb.fit(train_vecs, train_dataset.y) # fit model to train set
# Step 3. Check performance
prediction = mnb.predict(test_vecs) # test model
test_acc = compute_average_accuracy(
prediction, test_dataset.y) # compare actual and predicted labels
print("Test Accuracy = ", test_acc)
# Question 1(e)
# calculate remaining vocabulary size
vectorizere = CountVectorizer(tokenizer=no_tokenizer,
lowercase=False,
binary=True,
min_df=2)
train_X_features_e = vectorizere.fit_transform(train_dataset.X)
print("Remaining vocabulary size = ", train_X_features_e.shape[1])
# Part II: Perceptron Algorithm
# TODO: Implement the body of Perceptron.train() and Perceptron.predict()
# parameters for the perceptron model
num_epochs = 20
num_features = train_vecs.shape[1]
averaged = False # only MSc students should need to touch this!
# Step 1. Initialise model with hyperparameters
perceptron = Perceptron(num_epochs, num_features, averaged, shuf=False)
# Step 2. Train model
print("Training model for {} epochs".format(num_epochs))
perceptron.train(train_vecs, train_dataset.y, dev_vecs,
dev_dataset.y) #train model (original)
plt.xlabel("Epochs")
plt.ylabel("Accuracy") # plot graph
plt.title("Perceptron Train & Dev Accuracy (original)")
plt.legend()
plt.savefig('Perceptron (original).jpg')
plt.show(block=False)
# Repeat for shuffled datasets
perceptron_shuf = Perceptron(num_epochs, num_features, averaged, shuf=True)
print("Training model for {} epochs".format(num_epochs))
perceptron_shuf.train(train_vecs, train_dataset.y, dev_vecs,
dev_dataset.y) # train model (shuffled)
plt.xlabel("Epochs")
plt.ylabel("Accuracy") # plot graph
plt.title("Perceptron Train & Dev Accuracy (shuffled)")
plt.legend()
plt.savefig('Perceptron (shuffled).jpg')
plt.show()
# Step 3. Compute performance on test set
test_preds = perceptron.predict(
test_vecs) # predict test set using unshuffled trained model
test_accuracy = compute_average_accuracy(test_preds, test_dataset.y)
print("\nTest accuracy is: ", test_accuracy)
# Part III: Gradient Descent
# TODO: Implement the body of GD.train() and GD.predict()
# parameters for the gradient descent algorithm
max_iter = 20
num_features = train_vecs.shape[1]
# eta step (Change default value=0 and choose wisely! Double-check CW instructions)
# lambda term for regularisation (also choose wisely!)
# Step 1. Initialise model with hyperparameters
# three sets of combinations of eta and lambda
linear_model = GD(max_iter=max_iter,
num_features=num_features,
eta=0.000015,
lam=10)
linear_model2 = GD(max_iter=max_iter,
num_features=num_features,
eta=0.000009,
lam=10)
linear_model3 = GD(max_iter=max_iter,
num_features=num_features,
eta=0.000003,
lam=100)
# Step 2. Train model on a subset of the training set (first 10k examples)
# train model with first set
print("\nTraining model for {} max_iter".format(max_iter))
linear_model.train(train_vecs[:10000], train_dataset.y[:10000], dev_vecs,
dev_dataset.y)
plt.plot(range(max_iter),
linear_model.train_acc_list,
label="Train, eta=0.000015, lam=10")
plt.plot(range(max_iter),
linear_model.dev_acc_list,
label="Dev, eta=0.000015, lam=10")
# train model with second set
linear_model2.train(train_vecs[:10000], train_dataset.y[:10000], dev_vecs,
dev_dataset.y)
plt.plot(range(max_iter),
linear_model2.train_acc_list,
label="Train, eta=0.000009, lam=10")
plt.plot(range(max_iter),
linear_model2.dev_acc_list,
label="Dev, eta=0.000009, lam=10")
# train model with third set
linear_model3.train(train_vecs[:10000], train_dataset.y[:10000], dev_vecs,
dev_dataset.y)
plt.plot(range(max_iter),
linear_model3.train_acc_list,
label="Train, eta=0.000003, lam=100")
plt.plot(range(max_iter),
linear_model3.dev_acc_list,
label="Dev, eta=0.000003, lam=100")
plt.xlabel("Iterations")
plt.ylabel("Accuracy") # plot graph
plt.title("Gradient Descent Train & Dev Accuracy")
plt.legend()
plt.savefig("GD Accuracy Curve.png") # plot graph
plt.show()
# plot loss curves
plt.plot(range(max_iter),
linear_model.train_loss_list,
label="Train, eta=0.000015, lam=10")
plt.plot(range(max_iter),
linear_model.dev_loss_list,
label="Dev, eta=0.000015, lam=10")
plt.plot(range(max_iter),
linear_model2.train_loss_list,
label="Train, eta=0.000009, lam=10")
plt.plot(range(max_iter),
linear_model2.dev_loss_list,
label="Dev, eta=0.000009, lam=10")
plt.plot(range(max_iter),
linear_model3.train_loss_list,
label="Train, eta=0.000003, lam=100")
plt.plot(range(max_iter),
linear_model3.dev_loss_list,
label="Dev, eta=0.000003, lam=100")
plt.xlabel("Iterations")
plt.ylabel("Loss") # plot graph
plt.title("Gradient Descent Loss Curve")
plt.legend()
plt.savefig("GD Loss Curve.png")
plt.show()
# Step 4. Compute performance on test set
test_preds, pred_avg_loss = linear_model.predict(
test_vecs,
test_dataset.y) # use the model with the best eta and lambda
test_acc = compute_average_accuracy(test_preds, test_dataset.y)
print("Test accuracy = ", test_acc)
print("Predicted Average Loss = ", pred_avg_loss)
Used to test instance of Perceptron class.
"""
import json
import pprint
from pathlib import Path
from models.perceptron import Perceptron
pp = pprint.PrettyPrinter()
model = Perceptron(train='pa2_train_clean.csv',
validation='pa2_valid_clean.csv',
test='pa2_test_no_label_clean.csv',
label='label',
mod_type='kernel',
max_iter=15,
p=2)
# learned_model = model.train(max_iter=15)
learned_model = model.train_model()
# Save output for learned model to .json file.
file_name = Path('model_output', 'test_kern.json')
file_path = Path(__file__).parent.resolve().joinpath(file_name)
# Create output directory if doesn't exist.
output_dir = file_path.parent.resolve()
if not Path(output_dir).exists():
Path(output_dir).mkdir()
"""."""
from models.perceptron import Perceptron
from models.point import Point
INPUTS = [-1, 0.5]
PERCEPTRON = Perceptron()
POINTS = [Point() for _ in range(200)]
training_index = 0
for point in POINTS:
inputs = [point.x, point.y]
PERCEPTRON.train(inputs, point.label)
guess = PERCEPTRON.guess(inputs)
if guess == point.label:
print('found')
else:
print(guess, point.label)
plt.axis('off')
lr = 0.01
input_dim = 2
batch_size = 1000
n_epochs = 12
n_batches = 5
seed = 1337
torch.manual_seed(seed)
np.random.seed(seed)
perceptron = Perceptron(input_dim=input_dim)
optimizer = optim.Adam(params=perceptron.parameters(), lr=lr)
bce_loss = nn.BCELoss()
losses = []
x_data_static, y_truth_static = get_toy_data(batch_size)
fig, ax = plt.subplots(1, 1, figsize=(10, 5))
visualize_results(perceptron,
x_data_static,
y_truth_static,
ax=ax,
title='Initial Model State')
plt.axis('off')
change = 1.0
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from models.perceptron import Perceptron
# Use custom styling from file
matplotlib.rc_file('../plotstyle')
# Data for AND gate
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], dtype='float32')
y = np.array([[0], [0], [0], [1]], dtype='float32')
# Define and train model
model = Perceptron(data=X, labels=y, num_input=2)
model.fit(alpha=0.1, epochs=5000)
# Print results
print('x1\tx2\tlabel\tprediction')
for i in range(X.shape[0]):
print('{x1}\t{x2}\t{label}\t{prediction}'.format(x1=X[i, 0],
x2=X[i, 1],
label=y[i, 0],
prediction=model.predict(
X[i, :])[0][0]))
# Plot results
weights = model.w
bias = model.b
x_fit, y_fit = np.linspace(-1, 2, 100), []