def setup_model(self, model_type):
self.model_type = model_type
if model_type == "perceptron":
self.model = Perceptron()
else:
raise ValueError("Model {model_type} not supported." \
.format(model_type))
def run_pam(args):
set_global_seeds(args['seed'])
dataset = DataLoader(args['dataset'])
X_train, X_test, X_val, y_train, y_test, y_val = dataset.prepare_train_test_val(
args)
model = Perceptron(feature_dim=X_train.shape[-1], margin=args['margin'])
model.fit(X_train, y_train)
return model.score(X_test, y_test)
def find_best_margin(args):
""" return `best_margin / 0.1` """
set_global_seeds(args['seed'])
dataset = DataLoader(args['dataset'])
X_train, X_test, X_val, y_train, y_test, y_val = dataset.prepare_train_test_val(
args)
results = []
for margin in MARGINS:
model = Perceptron(feature_dim=X_train.shape[-1], margin=margin)
model.fit(X_train, y_train)
results.append(model.score(X_val, y_val))
return results
def get_initial_weights(self, model_type):
tf.reset_default_graph()
if model_type == "perceptron":
m = Perceptron()
inputs = tf.placeholder(tf.float32, shape=(None, 28*28))
_ = m.get_model(features={"x": inputs}, labels=None, mode='predict', params=None)
else:
raise ValueError("Model {model_type} not supported.".format(model_type))
with tf.Session().as_default() as sess:
sess.run(tf.global_variables_initializer())
collection = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
weights = {tensor.name:sess.run(tensor) for tensor in collection}
tf.reset_default_graph()
return weights
def setup_model(self, model_type):
self.model_type = model_type
if model_type == "perceptron":
self.model = Perceptron()
elif model_type == "cnn-mnist":
self.model = CNN()
elif model_type == "cnn-cifar10":
self.model = CNN()
else:
raise ValueError("Model {0} not supported.".format(model_type))
def test_perceptron():
from models.perceptron import Perceptron
x, y = np.random.randn(2, 500, 2), np.zeros([2, 500], dtype=int)
x[0] += np.array([1, -1])
x[1] += np.array([-1, 1])
y[0] = -1
y[1] = 1
plot_scatter(x[0], x[1], 'Real')
x = x.reshape(-1, 2)
y = y.flatten()
perceptron = Perceptron(input_dim=2, lr=1e-4)
train_perceptron(perceptron, x, y, epochs=100)
pred = perceptron.predict(x)
plot_scatter_with_line(x[pred == -1], x[pred == 1], perceptron.weights, 'Pred')
acc = np.sum(pred == y) / len(pred)
print(f'Acc = {100 * acc:.2f}%')
def setup_model(self, model_type):
self.model_type = model_type
if model_type == "perceptron":
self.model = Perceptron()
self.weights_metadata = self.model.get_weights_shape()
elif model_type == "cnn":
#TODO: Support CNN
self.model = CNN()
elif model_type == "lstm":
#TODO: Support LSTM
self.model = LSTM()
elif model_type == "gan":
self.model = ConversationalNetwork()
self.model.build_model(is_retraining=True)
else:
raise ValueError("Model {0} not supported.".format(model_type))
def main():
# load datasets
train_dataset = load_dataset("data/truecased_reviews_train.jsonl")
dev_dataset = load_dataset("data/truecased_reviews_dev.jsonl")
test_dataset = load_dataset("data/truecased_reviews_test.jsonl")
# Part I: Feature Engineering
# Step 1. create feature vectors by calling compute_features() with all three datasets as parameters
train_vecs, dev_vecs, test_vecs = compute_features(train_dataset,
dev_dataset,
test_dataset)
print("Proportion of +1 label: ", np.mean(train_dataset.y == 1))
print("Proportion of -1 label: ", np.mean(train_dataset.y == -1))
# Step 2. train a Naive Bayes Classifier (scikit MultinomialNB() )
# TODO complete implementation
mnb = MultinomialNB()
mnb.fit(train_vecs, train_dataset.y) # fit model to train set
# Step 3. Check performance
prediction = mnb.predict(test_vecs) # test model
test_acc = compute_average_accuracy(
prediction, test_dataset.y) # compare actual and predicted labels
print("Test Accuracy = ", test_acc)
# Question 1(e)
# calculate remaining vocabulary size
vectorizere = CountVectorizer(tokenizer=no_tokenizer,
lowercase=False,
binary=True,
min_df=2)
train_X_features_e = vectorizere.fit_transform(train_dataset.X)
print("Remaining vocabulary size = ", train_X_features_e.shape[1])
# Part II: Perceptron Algorithm
# TODO: Implement the body of Perceptron.train() and Perceptron.predict()
# parameters for the perceptron model
num_epochs = 20
num_features = train_vecs.shape[1]
averaged = False # only MSc students should need to touch this!
# Step 1. Initialise model with hyperparameters
perceptron = Perceptron(num_epochs, num_features, averaged, shuf=False)
# Step 2. Train model
print("Training model for {} epochs".format(num_epochs))
perceptron.train(train_vecs, train_dataset.y, dev_vecs,
dev_dataset.y) #train model (original)
plt.xlabel("Epochs")
plt.ylabel("Accuracy") # plot graph
plt.title("Perceptron Train & Dev Accuracy (original)")
plt.legend()
plt.savefig('Perceptron (original).jpg')
plt.show(block=False)
# Repeat for shuffled datasets
perceptron_shuf = Perceptron(num_epochs, num_features, averaged, shuf=True)
print("Training model for {} epochs".format(num_epochs))
perceptron_shuf.train(train_vecs, train_dataset.y, dev_vecs,
dev_dataset.y) # train model (shuffled)
plt.xlabel("Epochs")
plt.ylabel("Accuracy") # plot graph
plt.title("Perceptron Train & Dev Accuracy (shuffled)")
plt.legend()
plt.savefig('Perceptron (shuffled).jpg')
plt.show()
# Step 3. Compute performance on test set
test_preds = perceptron.predict(
test_vecs) # predict test set using unshuffled trained model
test_accuracy = compute_average_accuracy(test_preds, test_dataset.y)
print("\nTest accuracy is: ", test_accuracy)
# Part III: Gradient Descent
# TODO: Implement the body of GD.train() and GD.predict()
# parameters for the gradient descent algorithm
max_iter = 20
num_features = train_vecs.shape[1]
# eta step (Change default value=0 and choose wisely! Double-check CW instructions)
# lambda term for regularisation (also choose wisely!)
# Step 1. Initialise model with hyperparameters
# three sets of combinations of eta and lambda
linear_model = GD(max_iter=max_iter,
num_features=num_features,
eta=0.000015,
lam=10)
linear_model2 = GD(max_iter=max_iter,
num_features=num_features,
eta=0.000009,
lam=10)
linear_model3 = GD(max_iter=max_iter,
num_features=num_features,
eta=0.000003,
lam=100)
# Step 2. Train model on a subset of the training set (first 10k examples)
# train model with first set
print("\nTraining model for {} max_iter".format(max_iter))
linear_model.train(train_vecs[:10000], train_dataset.y[:10000], dev_vecs,
dev_dataset.y)
plt.plot(range(max_iter),
linear_model.train_acc_list,
label="Train, eta=0.000015, lam=10")
plt.plot(range(max_iter),
linear_model.dev_acc_list,
label="Dev, eta=0.000015, lam=10")
# train model with second set
linear_model2.train(train_vecs[:10000], train_dataset.y[:10000], dev_vecs,
dev_dataset.y)
plt.plot(range(max_iter),
linear_model2.train_acc_list,
label="Train, eta=0.000009, lam=10")
plt.plot(range(max_iter),
linear_model2.dev_acc_list,
label="Dev, eta=0.000009, lam=10")
# train model with third set
linear_model3.train(train_vecs[:10000], train_dataset.y[:10000], dev_vecs,
dev_dataset.y)
plt.plot(range(max_iter),
linear_model3.train_acc_list,
label="Train, eta=0.000003, lam=100")
plt.plot(range(max_iter),
linear_model3.dev_acc_list,
label="Dev, eta=0.000003, lam=100")
plt.xlabel("Iterations")
plt.ylabel("Accuracy") # plot graph
plt.title("Gradient Descent Train & Dev Accuracy")
plt.legend()
plt.savefig("GD Accuracy Curve.png") # plot graph
plt.show()
# plot loss curves
plt.plot(range(max_iter),
linear_model.train_loss_list,
label="Train, eta=0.000015, lam=10")
plt.plot(range(max_iter),
linear_model.dev_loss_list,
label="Dev, eta=0.000015, lam=10")
plt.plot(range(max_iter),
linear_model2.train_loss_list,
label="Train, eta=0.000009, lam=10")
plt.plot(range(max_iter),
linear_model2.dev_loss_list,
label="Dev, eta=0.000009, lam=10")
plt.plot(range(max_iter),
linear_model3.train_loss_list,
label="Train, eta=0.000003, lam=100")
plt.plot(range(max_iter),
linear_model3.dev_loss_list,
label="Dev, eta=0.000003, lam=100")
plt.xlabel("Iterations")
plt.ylabel("Loss") # plot graph
plt.title("Gradient Descent Loss Curve")
plt.legend()
plt.savefig("GD Loss Curve.png")
plt.show()
# Step 4. Compute performance on test set
test_preds, pred_avg_loss = linear_model.predict(
test_vecs,
test_dataset.y) # use the model with the best eta and lambda
test_acc = compute_average_accuracy(test_preds, test_dataset.y)
print("Test accuracy = ", test_acc)
print("Predicted Average Loss = ", pred_avg_loss)
class Client(object):
def __init__(self, iden):
self.iden = iden
def setup_data(self, X_train, y_train):
self.X_train = X_train
self.y_train = y_train
def setup_model(self, model_type):
self.model_type = model_type
if model_type == "perceptron":
self.model = Perceptron()
else:
raise ValueError("Model {model_type} not supported." \
.format(model_type))
def setup_training(self, batch_size, epochs, learning_rate):
self.batch_size = self.X_train.shape[0] if batch_size == -1 else batch_size
self.epochs = epochs
self.params = {'learning_rate': learning_rate}
def train(self, weights):
logging.info('Training just started.')
if weights:
metagraph = self.create_fresh_metagraph(fresh_weights=(weights == None))
self.model.load_weights(weights, metagraph, self.get_checkpoints_folder())
tf.reset_default_graph()
logging.info('Fresh metagraph created.')
classifier = tf.estimator.Estimator(
model_fn=self.model.get_model,
model_dir=self.get_checkpoints_folder(),
params = self.params
)
# tensors_to_log = {"probabilities": "softmax_tensor"}
# logging_hook = tf.train.LoggingTensorHook(
# tensors=tensors_to_log, every_n_iter=50)
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": self.X_train},
y=self.y_train,
batch_size=self.batch_size,
num_epochs=self.epochs,
shuffle=True
)
classifier.train(
input_fn=train_input_fn,
#hooks=[logging_hook]
)
logging.info('Training complete.')
weights = self.model.get_weights(self.get_latest_checkpoint())
metagraph_dir = self.get_latest_checkpoint() + '.meta'
with open(metagraph_dir, 'rb') as f:
metagraph_contents = f.read()
f.close()
return weights, metagraph_contents, self.X_train[0].size
def create_fresh_metagraph(self, fresh_weights):
tf.reset_default_graph()
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": self.X_train},
y=self.y_train,
batch_size=1,
num_epochs=None,
shuffle=False
)
#tensors_to_log = {"probabilities": "softmax_tensor"}
#logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log, every_n_iter=1)
classifier = tf.estimator.Estimator(
model_fn=self.model.get_model,
model_dir=self.get_checkpoints_folder(),
params = self.params
)
classifier.train(input_fn=train_input_fn, steps=1)
tf.reset_default_graph()
with tf.Session().as_default() as sess:
new_saver = tf.train.import_meta_graph(
self.get_latest_checkpoint() + '.meta'
)
meta_graph_def = tf.train.export_meta_graph()
if fresh_weights:
sess.run(tf.global_variables_initializer())
new_saver.save(sess, self.get_latest_checkpoint())
tf.reset_default_graph()
return meta_graph_def
def get_checkpoints_folder(self):
return "./checkpoints-{0}/{1}/".format(self.iden, self.model_type)
def get_latest_checkpoint(self):
return tf.train.latest_checkpoint(self.get_checkpoints_folder())
Used to test instance of Perceptron class.
"""
import json
import pprint
from pathlib import Path
from models.perceptron import Perceptron
pp = pprint.PrettyPrinter()
model = Perceptron(train='pa2_train_clean.csv',
validation='pa2_valid_clean.csv',
test='pa2_test_no_label_clean.csv',
label='label',
mod_type='kernel',
max_iter=15,
p=2)
# learned_model = model.train(max_iter=15)
learned_model = model.train_model()
# Save output for learned model to .json file.
file_name = Path('model_output', 'test_kern.json')
file_path = Path(__file__).parent.resolve().joinpath(file_name)
# Create output directory if doesn't exist.
output_dir = file_path.parent.resolve()
if not Path(output_dir).exists():
Path(output_dir).mkdir()
"""."""
from models.perceptron import Perceptron
from models.point import Point
INPUTS = [-1, 0.5]
PERCEPTRON = Perceptron()
POINTS = [Point() for _ in range(200)]
training_index = 0
for point in POINTS:
inputs = [point.x, point.y]
PERCEPTRON.train(inputs, point.label)
guess = PERCEPTRON.guess(inputs)
if guess == point.label:
print('found')
else:
print(guess, point.label)
plt.axis('off')
lr = 0.01
input_dim = 2
batch_size = 1000
n_epochs = 12
n_batches = 5
seed = 1337
torch.manual_seed(seed)
np.random.seed(seed)
perceptron = Perceptron(input_dim=input_dim)
optimizer = optim.Adam(params=perceptron.parameters(), lr=lr)
bce_loss = nn.BCELoss()
losses = []
x_data_static, y_truth_static = get_toy_data(batch_size)
fig, ax = plt.subplots(1, 1, figsize=(10, 5))
visualize_results(perceptron,
x_data_static,
y_truth_static,
ax=ax,
title='Initial Model State')
plt.axis('off')
change = 1.0
Date last modified: 10/26/2019
Python Version: 3.7
To run average perceptron.
Outputs model results to /model_output folder.
"""
import csv
import json
from pathlib import Path
from models.perceptron import Perceptron
model = Perceptron(train='pa2_train_clean.csv',
validation='pa2_valid_clean.csv',
test='pa2_test_no_label_clean.csv',
label='label',
mod_type='average',
max_iter=15)
learned_model = model.train_model()
# Save output for learned model to .json file.
output_folder = Path('model_output')
output_path = Path(__file__).parent.resolve().joinpath(output_folder)
training_file = output_path.joinpath(Path('ap_training.json'))
# Create output directory if doesn't exist.
if not Path(output_path).exists():
Path(output_path).mkdir()
with open(training_file, 'w') as f:
json.dump(learned_model, f, indent=4)
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from models.perceptron import Perceptron
# Use custom styling from file
matplotlib.rc_file('../plotstyle')
# Data for AND gate
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], dtype='float32')
y = np.array([[0], [0], [0], [1]], dtype='float32')
# Define and train model
model = Perceptron(data=X, labels=y, num_input=2)
model.fit(alpha=0.1, epochs=5000)
# Print results
print('x1\tx2\tlabel\tprediction')
for i in range(X.shape[0]):
print('{x1}\t{x2}\t{label}\t{prediction}'.format(x1=X[i, 0],
x2=X[i, 1],
label=y[i, 0],
prediction=model.predict(
X[i, :])[0][0]))
# Plot results
weights = model.w
bias = model.b
x_fit, y_fit = np.linspace(-1, 2, 100), []