def test(self):
# Training error
print "Training error:"
evaluator = Evaluator(self.X_train, self.Y_train, self.W)
evaluator.MSE()
evaluator.accuracy()
# Testing error
print "Testing error:"
evaluator = Evaluator(self.X_test, self.Y_test, self.W)
evaluator.MSE()
evaluator.accuracy()
def test(self, label=None):
# Training error
print "Training error:"
evaluator = Evaluator(self.X_train, self.Y_train, self.W)
#evaluator.MSE()
evaluator.accuracy()
#evaluator.confusion()
# Testing error
print "Testing error:"
evaluator = Evaluator(self.X_test, self.Y_test, self.W)
#evaluator.MSE()
evaluator.accuracy()
evaluator.confusion()
FPR, TPR = evaluator.roc()
plt.plot(FPR, TPR, label=label)
plt.axis([0.0,0.5,0.5,1.0])
decay_threshold=decthr,
stop_threshold=stopthr
)
architecture = [784, 400, 400, 10]
net = network.Network(architecture, lr)
validation_errors, training_costs = trainer.sgd(
net,
tr_d,
mb,
momentum=mom,
evaluator=evaluator,
scheduler=scheduler
)
best_net = scheduler.highest_accuracy_network
val_err = Utils.error_fraction(
evaluator.accuracy(va_d, best_net), len(va_d[0])
)*100
eva_err = Utils.error_fraction(
evaluator.accuracy(te_d, best_net), len(te_d[0])
)*100
curr_time = time.strftime("%Y%m%d-%H%M%S")
Io.save(
best_net,
"networks/" +
curr_time +
"_" + str(architecture).replace(' ', '') +
"_valerr" + str(val_err) +
"_evaerr" + str(eva_err) +
"_lr" + str(lr) +
"_mom" + str(mom) +
"_dec" + str(dec) +
# Create a dataset of the non-test datasets.
ds = SemEvalData(blank=True)
for j in range(num_folds):
if j != i:
ds = ds + datasets[j]
input, tags, order = NeuralNet.format_dataset(ds, embedder)
# Train the network.
network = NeuralNet(hidden=128,
layers=2,
input=len(input[0]),
output=2)
# Train on non-test datasets.
loss, accuracy = network.train(input, tags, iterations=10)
accuracies += [accuracy]
# Test on test dataset.
test, _, order = NeuralNet.format_dataset(datasets[i], embedder)
predictions.update(network.predict(test, order))
from evaluator import Evaluator
neural_cm = Evaluator.compare(dataset.tags, predictions)
print("Neural Network Accuracy: " +
str(round(Evaluator.accuracy(neural_cm) * 100, 2)) + "%")
# Save the networks results.
with open('output.csv', 'w') as writefile:
writefile.write(NeuralNet.pred_to_csv(predictions))
def run(self, options):
# sess = tf.Session()
evaluator = Evaluator()
data = Data()
model = Model()
statPlot = StatPlot()
best_accuracy = 0
print('Getting/Transforming Data.')
# Initialize the data pipeline
images, targets = data.batches(options, train_logical=True)
# Get batch test images and targets from pipline
test_images, test_targets = data.batches(options, train_logical=False)
# Initialize step
global_step = tf.Variable(0, name='global_step', trainable=False)
# Declare Model
print('Creating the CIFAR10 Model.')
with tf.Session() as sess:
with tf.variable_scope('model_definition') as scope:
# Declare the training network model
model_output = model.run(images, options)
#model_output = cifar_cnn_model(images, batch_size)
# This is very important!!! We must set the scope to REUSE the variables,
# otherwise, when we set the test network model, it will create new random
# variables. Otherwise we get random evaluations on the test batches.
scope.reuse_variables()
test_output = model.run(test_images, options)
# test_output = cifar_cnn_model(test_images, batch_size)
# Declare loss function
print('Declare Loss Function.')
loss = model.loss(model_output, targets)
# Create accuracy function
accuracy = evaluator.accuracy(test_output, test_targets)
# Create training operations
print('Creating the Training Operation.')
#generation_num = tf.Variable(0, trainable=False)
train_op = self.train(loss, options, global_step)
# Initialize Variables
print('Initializing the Variables.')
init = tf.global_variables_initializer()
sess.run(init)
saver = tf.train.Saver()
if options.getRunOptions('restore_boolean'):
path_to_restore_checkpoint_file = os.path.join(
options.getRunOptions('log_dir'),
options.getRunOptions('restor_file'))
assert tf.train.checkpoint_exists(path_to_restore_checkpoint_file), \
'%s not found' % path_to_restore_checkpoint_file
saver.restore(sess, path_to_restore_checkpoint_file)
print('Model restored from file: %s' %
path_to_restore_checkpoint_file)
# Initialize queue (This queue will feed into the model, so no placeholders necessary)
tf.train.start_queue_runners(sess=sess)
print('Starting Training')
train_loss = []
test_accuracy = []
start = sess.run(global_step)
for i in range(start, options.getRunOptions('generations')):
_, loss_value, global_step_val = sess.run(
[train_op, loss, global_step])
if (i + 1) % options.getRunOptions('output_every') == 0:
train_loss.append(loss_value)
output = 'Generation {}: Loss = {:.5f}'.format((i + 1),
loss_value)
print(output)
# if (i+1) % options.getRunOptions('eval_every') == 0:
[temp_accuracy] = sess.run([accuracy])
if temp_accuracy > best_accuracy:
test_accuracy.append(temp_accuracy)
acc_output = ' --- Test Accuracy = {:.2f}%.'.format(
100. * temp_accuracy)
print(acc_output)
path_to_checkpoint_file = saver.save(
sess,
os.path.join(options.getRunOptions('log_dir'),
'model.ckpt'))
print('=> Model saved to file: %s' %
path_to_checkpoint_file)
best_accuracy = temp_accuracy
if (i + 1) % options.getRunOptions('eval_every') == 0:
path_to_checkpoint_file = saver.save(
sess,
os.path.join(options.getRunOptions('log_dir'),
'model_eval_every.ckpt'))
print('=> Model saved to file: %s' %
path_to_checkpoint_file)
# statPlot.plot(train_loss, test_accuracy, options)
tf.summary.FileWriter(options.getRunOptions('log_dir'),
sess.graph)
return (train_loss, test_accuracy)
from polluted import PollutedSpambase
from evaluator import Evaluator
from descent import GradientDescent
if __name__=="__main__":
# Get data
dataset = PollutedSpambase()
train_data, train_labels = dataset.training()
test_data, test_labels = dataset.testing()
# Do Logistic Regression
gd = GradientDescent(train_data, train_labels)
# 200,000 iterations gives ~85% acc
W = gd.logreg_stoch(it=200001)
# Evaluate solution
evaluator = Evaluator([test_data], [test_labels], [W])
evaluator.accuracy()
