def test(Xtest, ytest, Weights, Biases):
testNum = len(ytest)
correct = 0
J = 0
avg = 0
for i in xrange(testNum):
if type(Xtest[i]) == type('a'):
fin = open(Xtest[i])
X = p.load(fin)
fin.close()
else:
X = Xtest[i]
#print len(X)
FFNN.cleanActivation(X[0])
h = FFNN.forwardpropagation(X[0], None, Weights, Biases)
#print h.T
t = ytest[i]
J += -np.log(h[t])
avg += h[t]
predict = h.argmax()
#print 'test case (', i, '), h = ', predict, '(predicted ', h[predict], ')', t, '(actural',\
# h[t], ')'
if predict == t:
correct += 1
print ' average target output', (avg + .0) / testNum
# if bestAccuracy < .2:
# print 'oops, too bad'
print J, (correct + .0) / testNum
return J / testNum, (correct + .0) / testNum
def train_evaluate_ffnn(
train_exs: List[SentimentExample], dev_exs: List[SentimentExample],
test_exs: List[SentimentExample],
word_vectors: WordEmbeddings) -> List[SentimentExample]:
"""
Train a feedforward neural network on the given training examples, using dev_exs for development, and returns
predictions on the *blind* test examples passed in. Returned predictions should be SentimentExample objects with
predicted labels and the same sentences as input (but these won't be read by the external code). The code is set
up to go all the way to test predictions so you have more freedom to handle example processing as you see fit.
:param train_exs:
:param dev_exs:
:param test_exs:
:param word_vectors:
:return:
"""
# 59 is the max sentence length in the corpus, so let's set this to 60
seq_max_len = 60
# To get you started off, we'll pad the training input to 60 words to make it a square matrix.
train_mat = np.asarray([
pad_to_length(np.array(ex.indexed_words), seq_max_len)
for ex in train_exs
])
# Also store the sequence lengths -- this could be useful for training LSTMs
train_seq_lens = np.array([len(ex.indexed_words) for ex in train_exs])
# Labels
train_labels_arr = np.array([ex.label for ex in train_exs])
input_size = word_vectors.get_embedding_length()
hid_size = word_vectors.get_embedding_length()
num_classes = 2
ffnn = FFNN(input_size, hid_size, num_classes) # TODO: Add embed layer?
lr = 0.001
epochs = 5
learn_weights(lr, epochs, ffnn, train_labels_arr, train_mat, word_vectors,
num_classes, train_seq_lens)
# Begin Prediction
dev_mat = np.asarray([
pad_to_length(np.array(ex.indexed_words), seq_max_len)
for ex in dev_exs
])
dev_seq_lens = np.array([len(ex.indexed_words) for ex in dev_exs])
guesses = []
correct = 0
for idx in range(0, len(dev_exs)):
# Note that we only feed in the x, not the y, since we're not training. We're also extracting different
# quantities from the running of the computation graph, namely the probabilities, prediction, and z
x = form_input(dev_mat[idx], word_vectors, dev_seq_lens[idx])
probs = ffnn.forward(x)
prediction = int(torch.argmax(probs))
guess = SentimentExample(dev_exs[idx].indexed_words, prediction)
guesses.append(guess)
if prediction == dev_exs[idx].label:
correct += 1
accuracy = 100 * correct / len(dev_exs)
print('accuracy: ')
print(accuracy)
return guesses
def nn_reachable_sets(filemat, p, ii, jj, lb, ub, unsafe_mat, unsafe_vec):
W = filemat['W'][0]
b = filemat['b'][0]
range_for_scaling = filemat['range_for_scaling'][0]
means_for_scaling = filemat['means_for_scaling'][0]
for i in range(5):
lb[i] = (lb[i] - means_for_scaling[i]) / range_for_scaling[i]
ub[i] = (ub[i] - means_for_scaling[i]) / range_for_scaling[i]
norm_mat = range_for_scaling[5] * np.eye(5)
norm_vec = means_for_scaling[5] * np.ones((5, 1))
nnet0 = nnet.nnetwork(W, b)
cube_lattice = cl.CubeLattice(lb, ub)
initial_input = cube_lattice.to_poly_lattice()
cores = multiprocessing.cpu_count()
start_time = time.time()
outputSets = ffnn.nnet_output(nnet0, initial_input, ("parallel", cores))
resl = verify_parallel(outputSets, norm_mat, norm_vec, unsafe_mat,
unsafe_vec)
elapsed_time = time.time() - start_time
filename = "logs/output_info_" + str(p) + "_" + str(ii) + "_" + str(
jj) + ".txt"
file = open(filename, 'w')
file.write('time elapsed: %f seconds \n' % elapsed_time)
file.write('number of polytopes: %d \n' % len(outputSets))
file.write('verification result: ' + resl + '\n')
file.close()
outputSets = []
class CorefEngine:
if __name__ == "__main__":
# handles passed-in args
args = params.setCorefEngineParams()
stoppingPoints = [0.45]
hddcrp_parsed = None
# parses the real, actual corpus (ECB's XML files)
corpus = ECBParser(args)
helper = ECBHelper(args, corpus, hddcrp_parsed)
# loads stanford's parsed version of our corpus and aligns it w/
# our representation -- so we can use their features
stan = StanParser(args, corpus)
helper.addStanfordAnnotations(stan)
# runs CCNN -> FFNN
if args.testMentions == "hddcrp":
hddcrp_parsed = HDDCRPParser(args.hddcrpFullFile) # loads HDDCRP's pred or gold mentions file
elif not args.testMentions.startsWith("ecb"):
print("ERROR: args.testMentions should be ecbdev, ecbtest, or hddcrp")
exit(1)
ccnnEngine = CCNN(args, corpus, helper, hddcrp_parsed)
(dev_pairs, dev_preds, testing_pairs, testing_preds) = ccnnEngine.run()
ffnnEngine = FFNN(args, corpus, helper, hddcrp_parsed, dev_pairs, dev_preds, testing_pairs, testing_preds)
ffnnEngine.train()
for sp in stoppingPoints:
(predictedClusters, goldenClusters) = ffnnEngine.cluster(sp)
print("# goldencluster:",str(len(goldenClusters)))
print("# predicted:",str(len(predictedClusters)))
if args.testMentions.startsWith("ecb"): # use corpus' gold test set
(bcub_p, bcub_r, bcub_f1, muc_p, muc_r, muc_f1, ceafe_p, ceafe_r, ceafe_f1, conll_f1) = get_conll_scores(goldenClusters, predictedClusters)
print("FFNN F1 sp:",str(sp),"=",str(conll_f1),"OTHERS:",str(muc_f1),str(bcub_f1),str(ceafe_f1))
elif args.testMentions == "hddcrp":
print("FFNN on HDDCRP")
helper.writeCoNLLFile(predictedClusters, sp)
else:
print("ERROR: args.testMentions should be ecbdev, ecbtest, or hddcrp")
exit(1)
def nn_cross_validation(X, y, folds, hidden_layer_size, learning_rate, ratio = 0.2, epochs = 30, callbacks = None, validation_split = None, save_loss = True):
# Cross validation
cv = StratifiedKFold(n_splits = folds)
results = {'accuracy' : [], 'f1-score' : [], 'recall' : [], 'precision' : [], 'space' : []}
history = []
for count, (train_index, test_index) in enumerate(cv.split(X, y), 1):
# Inizializzo modello
model = ff.create_sequential(input_size = (82, ), hidden_layer_size = hidden_layer_size, learning_rate = learning_rate, activation = 'relu')
print("Indici Train: ", train_index, " Indici Test", test_index)
print("Totale elementi: ", len(X))
# Seleziono i fold su cui lavorare
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
# Bilancio solamente il dataset di train (Modifico anche quello originale?)
X_train, y_train = init.undersample(X_train, y_train, ratio = ratio)
# Training
if validation_split is not None:
X_train, y_train = helpers.shuffle(X_train, y_train)
history.append(ff.train(model, X_train, y_train, cbs = callbacks, validation_split = validation_split, epochs = epochs))
# Da modificare
model = load_model('best_model.h5')
# Valuto size modello (Metto location come param)
size = ff.model_size(model, location = f"local_exec/classifier_testing/esperimenti_ffnn/risultati/pesi_modelli_addestrati/ff_{count}.p")
# Score del classificatore sul testing
scores = ff.evaluate(model, X_test, y_test, verbose = False)
# Aggiorno risultati delle metriche (Manca dev. st e size)
results['accuracy'].append(scores['accuracy'])
results['space'].append(size)
results['f1-score'].append(scores['1']['f1-score'])
results['precision'].append(scores['1']['precision'])
results['recall'].append(scores['1']['recall'])
# Salvo grafico loss
if save_loss: ff.save_losses_plot(history[:3], f"test_plot_neuron{hidden_layer_size}_lr{learning_rate}", colors = ['r', 'b', 'g'])
# Media sui 5 risultati
mean_results = {key : sum(value) / folds for key, value in results.items()}
std_results = {key : (sum([((x - mean_results[key]) ** 2) for x in value]) / folds) ** 0.5 for key, value in results.items()}
return mean_results, std_results
def build_and_train(optimizing_function, numOfOut):
print('building the model ... ')
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
rng = np.random.RandomState(1234)
# construct the MLP class
classifier = FFNN(rng=rng,
input=x,
n_in=n_in,
n_hidden=n_hidden,
n_out=n_out)
# the cost we minimize during training is the negative log likelihood of
# the model plus the regularization terms (L1 and L2); cost is expressed
# here symbolically
cost = (classifier.negative_log_likelihood(y) + L1_reg * classifier.L1 +
L2_reg * classifier.L2_sqr)
# compiling a Theano function that computes the mistakes that are made
# by the model on a minibatch
## test_model_accuracy = theano.function(
## inputs=[index],
## outputs=classifier.errors(y),
## givens={
## x: test_set_x[index * batch_size: (index + 1) * batch_size],
## y: test_set_y[index * batch_size: (index + 1) * batch_size]
## }
## )
##
## test_model = theano.function(
## inputs=[index],
## outputs=[y, classifier.y_pred],
## givens={
## x: test_set_x[index * batch_size: (index + 1) * batch_size],
## y: test_set_y[index * batch_size: (index + 1) * batch_size]
## }
##)
validate_model_accuracy = theano.function(
inputs=[index],
outputs=classifier.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
validate_model = theano.function(
inputs=[index],
outputs=[y, classifier.y_pred],
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
# compute the gradient of cost with respect to theta (sotred in params)
# the resulting gradients will be stored in a list gparams
gparams = [T.grad(cost, param) for param in classifier.params]
# specify how to update the parameters of the model as a list of
# (variable, update expression) pairs
# given two lists of the same length, A = [a1, a2, a3, a4] and
# B = [b1, b2, b3, b4], zip generates a list C of same size, where each
# element is a pair formed from the two lists :
# C = [(a1, b1), (a2, b2), (a3, b3), (a4, b4)]
updates = [(param, param - learning_rate * gparam)
for param, gparam in zip(classifier.params, gparams)]
# compiling a Theano function `train_model` that returns the cost, but
# in the same time updates the parameter of the model based on the rules
# defined in `updates`
train_model = theano.function(
inputs=[index],
outputs=cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
})
print('training ... ')
# early-stopping parameters
patience = 25000 # look at this many examples regardless
patience_increase = 2 # wait this much longer when a new best is found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience // 2)
# go through this many
# minibatches before checking the network
# on the validation set; in this case we
# check every epoch
best_classifier = classifier
best_validation_loss = np.inf
best_iter = 0
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
minibatch_avg_cost = train_model(minibatch_index)
# iteration number
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
# compute zero-one loss on validation set
# Otimizing for accuracy
validation_loss = [
validate_model_accuracy(i) for i in range(n_valid_batches)
]
this_validation_loss = np.mean(validation_loss)
# Optimizing for precision
validation_precision = [
np.mean(
precision_score(*validate_model(i),
pos_label=None,
average=None)[0:numOfOut])
for i in range(n_valid_batches)
]
this_validation_precision = np.mean(validation_precision)
# Optimizing for recall
validation_recall = [
np.mean(
recall_score(*validate_model(i),
pos_label=None,
average=None)[0:numOfOut])
for i in range(n_valid_batches)
]
this_validation_recall = np.mean(validation_recall)
# Optimizing for f-score
validation_fscore = [
np.mean(
f1_score(*validate_model(i),
pos_label=None,
average=None)[0:numOfOut])
for i in range(n_valid_batches)
]
this_validation_fscore = np.mean(validation_fscore)
print(
'epoch %i, minibatch %i/%i, average validation precision %f, validation recall %f, validation fscore %f, and loss %f %%'
% (epoch, minibatch_index + 1, n_train_batches,
this_validation_precision * 100,
this_validation_recall * 100, this_validation_fscore *
100, this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (this_validation_loss <
best_validation_loss * improvement_threshold):
patience = max(patience, iter * patience_increase)
best_validation_loss = this_validation_loss
best_iter = iter
best_classifier = classifier
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print("the total iterations are: " + str(iter))
print(('Optimization complete. Best validation score of %f %% '
'obtained at iteration %i, with test performance %f %%') %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print(('The code for file ran for %.2fm' %
((end_time - start_time) / 60.)))
return best_classifier
def build_and_train():
print('... building the model')
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
rng = np.random.RandomState(1234)
# construct the MLP class
classifier = FFNN(
rng=rng,
input=x,
n_in=n_in,
n_hidden=n_hidden,
n_out=n_out
)
# the cost we minimize during training is the negative log likelihood of
# the model plus the regularization terms (L1 and L2); cost is expressed
# here symbolically
cost = (
classifier.negative_log_likelihood(y)
+ L1_reg * classifier.L1
+ L2_reg * classifier.L2_sqr
)
# compiling a Theano function that computes the mistakes that are made
# by the model on a minibatch
test_model = theano.function(
inputs=[index],
outputs=classifier.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
}
)
validate_model = theano.function(
inputs=[index],
outputs=classifier.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
}
)
# compute the gradient of cost with respect to theta (sotred in params)
# the resulting gradients will be stored in a list gparams
gparams = [T.grad(cost, param) for param in classifier.params]
# specify how to update the parameters of the model as a list of
# (variable, update expression) pairs
# given two lists of the same length, A = [a1, a2, a3, a4] and
# B = [b1, b2, b3, b4], zip generates a list C of same size, where each
# element is a pair formed from the two lists :
# C = [(a1, b1), (a2, b2), (a3, b3), (a4, b4)]
updates = [
(param, param - learning_rate * gparam)
for param, gparam in zip(classifier.params, gparams)
]
# compiling a Theano function `train_model` that returns the cost, but
# in the same time updates the parameter of the model based on the rules
# defined in `updates`
train_model = theano.function(
inputs=[index],
outputs=cost,
updates=updates,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size],
y: train_set_y[index * batch_size: (index + 1) * batch_size]
}
)
print('... training')
# early-stopping parameters
patience = 10000 # look at this many examples regardless
patience_increase = 2 # wait this much longer when a new best is found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience // 2)
# go through this many
# minibatches before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = np.inf
best_iter = 0
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
minibatch_avg_cost = train_model(minibatch_index)
# iteration number
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
# compute zero-one loss on validation set
validation_losses = [validate_model(i) for i
in range(n_valid_batches)]
this_validation_loss = np.mean(validation_losses)
print(
'epoch %i, minibatch %i/%i, validation error %f %%' %
(
epoch,
minibatch_index + 1,
n_train_batches,
this_validation_loss * 100.
)
)
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = [test_model(i) for i
in range(n_test_batches)]
test_score = np.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
# save the best model
#with open('best_model.pkl', 'wb') as f:
# pickle.dump(classifier, f)
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print(('Optimization complete. Best validation score of %f %% '
'obtained at iteration %i, with test performance %f %%') %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print(('The code for file ran for %.2fm' % ((end_time - start_time) / 60.)))
# -*- coding: utf-8 -*-
"""
Created on Sat May 31 15:07:05 2014
@author: mou
"""
import FFNN
inlayer = FFNN.layer('input', range(0, 2), 2)
hidlayer = FFNN.layer('hid', range(2, 4), 2)
outlayer = FFNN.layer('out', range(4, 6), 2)
con1 = FFNN.connection(inlayer, hidlayer, 0, 2, 0, 2, range(0, 4), Wcoef=.3)
con2 = FFNN.connection(hidlayer, outlayer, 0, 2, 0, 2, range(4, 8), Wcoef=1)
inlayer.connectUp.append(con1)
hidlayer.connectDown.append(con1)
hidlayer.connectUp.append(con2)
outlayer.connectDown.append(con2)
inlayer.successiveUpper = hidlayer
hidlayer.successiveUpper = outlayer
outlayer.successiveLower = hidlayer
hidlayer.successiveLower = inlayer
import numpy as np
Weights = np.random.uniform(-0.02, .02, 8).reshape(-1)
Biases = np.random.uniform(-0.02, .02, 6).reshape(-1)
gradWeights = np.zeros_like(Weights)
default=False)
parser.add_argument(
'--sequence_size',
type=int,
help='size of the lstm sequence only works for LSTM and Merge',
default=5)
args = parser.parse_args()
if args.FFNN == True:
if args.run_test == True:
import FFNNtest
FFNNtest.run(args)
else:
import FFNN
FFNN.main(args)
elif args.LSTM == True:
if args.run_test == True:
import LSTMtest
LSTMtest.run(args)
else:
import LSTM
LSTM.main(args)
elif args.MERGE == True:
if args.run_test == True:
import mergetest
mergetest.run(args)
else:
import merge
merge.main(args)
else:
def Cost(tokenMap, positive, negative, \
Weights, Biases, gradWeights=None, gradBiases=None, debug=False, noGrad=False):
FFNN.cleanActivation(positive[0])
FFNN.cleanActivation(negative[0])
# forward propagation
FFNN.forwardpropagation(positive[0], None, Weights, Biases)
FFNN.forwardpropagation(negative[0], None, Weights, Biases)
posOut = positive[-1].y
negOut = negative[-1].y
numFea = len(posOut)
posTarIdx = tokenMap[positive[-1].name]
negTarIdx = tokenMap[negative[-1].name]
posTar = Biases[posTarIdx * numFea: numFea * (posTarIdx + 1)]
negTar = Biases[negTarIdx * numFea: numFea * (negTarIdx + 1)]
posDiff = posOut - posTar
negDiff = negOut - negTar
MSpos = .5 * np.sum(posDiff * posDiff)
MSneg = .5 * np.sum(negDiff * negDiff)
costPara = gl.C_2 * (np.sum(Weights * Weights) + np.sum(Biases * Biases))
# Error = max{0, MSpos, }
Error = gl.margin + MSpos - MSneg
# want MSpos < MSneg - margin
if (not debug and Error < 0):
return 0, costPara
elif noGrad:
return Error, Error + costPara
# compute gradient
FFNN.cleanDerivatives(positive[-1])
FFNN.cleanDerivatives(negative[-1])
positive[-1].dE_by_dy = posOut - posTar
negative[-1].dE_by_dy = negTar - negOut
FFNN.backpropagation(positive[-1], Weights, Biases, gradWeights, gradBiases)
FFNN.backpropagation(negative[-1], Weights, Biases, gradWeights, gradBiases)
gradBiases[posTarIdx * numFea: numFea * (posTarIdx + 1)] += posTar - posOut
gradBiases[negTarIdx * numFea: numFea * (negTarIdx + 1)] += negOut - negTar
gradWeights += gl.C * Weights
gradBiases += gl.C * Biases
return Error, Error + costPara
# -*- coding: utf-8 -*-
"""
Created on Sat May 31 15:07:05 2014
@author: mou
"""
import FFNN
inlayer = FFNN.layer('input', range(0,2), 2)
hidlayer = FFNN.layer('hid', range(2,4), 2)
outlayer = FFNN.layer('out', range(4,6), 2)
con1 = FFNN.connection(inlayer, hidlayer, 0, 2, 0, 2, range(0,4), Wcoef = .3)
con2 = FFNN.connection(hidlayer, outlayer, 0, 2, 0, 2, range(4,8), Wcoef = 1)
inlayer.connectUp.append(con1)
hidlayer.connectDown.append(con1)
hidlayer.connectUp.append(con2)
outlayer.connectDown.append(con2)
inlayer.successiveUpper = hidlayer
hidlayer.successiveUpper = outlayer
outlayer.successiveLower = hidlayer
hidlayer.successiveLower = inlayer
import numpy as np
Weights = np.random.uniform(-0.02, .02, 8).reshape(-1)
Biases = np.random.uniform(-0.02, .02, 6).reshape(-1)
def train(self):
print('building the model ... ')
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector(
'y') # the labels are presented as 1D vector of [int] labels
classifier = FFNN(rng=self.rng,
input=x,
n_in=self.n_in,
n_hidden=self.n_hidden,
n_out=self.n_out)
cost = (classifier.negative_log_likelihood(y) +
self.L1_reg * classifier.L1 + self.L2_reg * classifier.L2_sqr)
validate_model_accuracy = theano.function(
inputs=[index],
outputs=classifier.errors(y),
givens={
x:
self.valid_set_x[index * self.batch_size:(index + 1) *
self.batch_size],
y:
self.valid_set_y[index * self.batch_size:(index + 1) *
self.batch_size]
})
validate_model = theano.function(
inputs=[index],
outputs=[y, classifier.y_pred],
givens={
x:
self.valid_set_x[index * self.batch_size:(index + 1) *
self.batch_size],
y:
self.valid_set_y[index * self.batch_size:(index + 1) *
self.batch_size]
})
gparams = [T.grad(cost, param) for param in classifier.params]
updates = [(param, param - self.learning_rate * gparam)
for param, gparam in zip(classifier.params, gparams)]
train_model = theano.function(
inputs=[index],
outputs=cost,
updates=updates,
givens={
x:
self.train_set_x[index * self.batch_size:(index + 1) *
self.batch_size],
y:
self.train_set_y[index * self.batch_size:(index + 1) *
self.batch_size]
})
print('training ... ')
patience = 10000 # look at this many examples regardless
patience_increase = self.patience_increase # wait this much longer when a new best is found
improvement_threshold = 0.995 # a relative improvement of this much is
validation_frequency = min(self.n_train_batches, patience // 2)
best_validation_loss = np.inf
best_iter = 0
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < self.n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(self.n_train_batches):
minibatch_avg_cost = train_model(minibatch_index)
# iteration number
iter = (epoch - 1) * self.n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_loss = [
validate_model_accuracy(i)
for i in range(self.n_valid_batches)
]
this_validation_loss = np.mean(validation_loss)
validation_precision = [
np.mean(
precision_score(*validate_model(i),
pos_label=None,
average=None))
for i in range(self.n_valid_batches)
]
this_validation_precision = np.mean(validation_precision)
validation_recall = [
np.mean(
recall_score(*validate_model(i),
pos_label=None,
average=None))
for i in range(self.n_valid_batches)
]
this_validation_recall = np.mean(validation_recall)
validation_fscore = [
np.mean(
f1_score(*validate_model(i),
pos_label=None,
average=None))
for i in range(self.n_valid_batches)
]
this_validation_fscore = np.mean(validation_fscore)
print(
'epoch %i, minibatch %i/%i, average validation precision %f, validation recall %f, validation fscore %f, and loss %f %%'
%
(epoch, minibatch_index + 1, self.n_train_batches,
this_validation_precision * 100,
this_validation_recall * 100, this_validation_fscore *
100, this_validation_loss * 100.))
if this_validation_loss < best_validation_loss:
if (this_validation_loss <
best_validation_loss * improvement_threshold):
patience = max(patience, iter * patience_increase)
best_validation_loss = this_validation_loss
best_validation_precision = this_validation_precision
best_validation_recall = this_validation_recall
best_validation_fscore = this_validation_fscore
best_iter = iter
best_states = classifier.getstate()
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print((
'Optimization complete. Best validation score of %f %% obtained at iteration %i, with best vaildation recall : %f, precision: %f, fscore: %f %%'
) % (best_validation_loss * 100., best_iter + 1,
this_validation_recall * 100., this_validation_precision * 100.,
this_validation_fscore * 100.))
print(('The code for file ran for %.2fm' %
((end_time - start_time) / 60.)))
self.best_classifier = classifier
self.best_weights = best_states
def run(args):
for mt in testModels3:
import FFNN
[model, test] = mt
args.test = test
FFNN.main(args, model)
def build_and_train(optimizing_function):
print('building the model ... ')
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of [int] labels
rng = np.random.RandomState(1234)
classifier = FFNN(rng=rng,input=x,n_in=n_in,n_hidden=n_hidden,n_out=n_out)
cost = (classifier.negative_log_likelihood(y) + L1_reg * classifier.L1 + L2_reg * classifier.L2_sqr)
test_model_accuracy = theano.function(inputs=[index],outputs=classifier.errors(y),
givens={
x: test_set_x[index * batch_size: (index + 1) * batch_size],
y: test_set_y[index * batch_size: (index + 1) * batch_size]
}
)
test_model = theano.function(
inputs=[index],
outputs=[y, classifier.y_pred],
givens={
x: test_set_x[index * batch_size: (index + 1) * batch_size],
y: test_set_y[index * batch_size: (index + 1) * batch_size]
}
)
validate_model_accuracy = theano.function(
inputs=[index],
outputs=classifier.errors(y),
givens={
x: valid_set_x[index * batch_size: (index + 1) * batch_size],
y: valid_set_y[index * batch_size: (index + 1) * batch_size]
}
)
validate_model = theano.function(
inputs=[index],
outputs=[y, classifier.y_pred],
givens={
x: valid_set_x[index * batch_size: (index + 1) * batch_size],
y: valid_set_y[index * batch_size: (index + 1) * batch_size]
}
)
gparams = [T.grad(cost, param) for param in classifier.params]
updates = [(param, param - learning_rate * gparam) for param, gparam in zip(classifier.params, gparams)]
train_model = theano.function(
inputs = [index],
outputs = cost,
updates = updates,
givens = {
x: train_set_x[index*batch_size: (index + 1) * batch_size],
y: train_set_y[index*batch_size: (index + 1) * batch_size]
}
)
print('training ... ')
patience = 10000 # look at this many examples regardless
patience_increase = PATIENCE_INCREASE # wait this much longer when a new best is found
improvement_threshold = 0.995 # a relative improvement of this much is
validation_frequency = min(n_train_batches, patience // 2)
best_validation_loss = np.inf
best_iter = 0
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
minibatch_avg_cost = train_model(minibatch_index)
# iteration number
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_loss = [validate_model_accuracy(i) for i in range(n_valid_batches)]
this_validation_loss = np.mean(validation_loss)
validation_precision = [np.mean(precision_score(*validate_model(i),pos_label=None,average=None)) for i in range(n_valid_batches)]
this_validation_precision = np.mean(validation_precision)
validation_recall = [np.mean(recall_score(*validate_model(i),pos_label=None,average=None)) for i in range(n_valid_batches)]
this_validation_recall = np.mean(validation_recall)
validation_fscore = [np.mean(f1_score(*validate_model(i),pos_label=None,average=None)) for i in range(n_valid_batches)]
this_validation_fscore = np.mean(validation_fscore)
print('epoch %i, minibatch %i/%i, average validation precision %f, validation recall %f, validation fscore %f, and loss %f %%' %(epoch,minibatch_index + 1,n_train_batches,this_validation_precision * 100,this_validation_recall * 100,this_validation_fscore * 100,this_validation_loss * 100.))
if this_validation_loss < best_validation_loss:
if (this_validation_loss < best_validation_loss * improvement_threshold):
patience = max(patience, iter * patience_increase)
best_validation_loss = this_validation_loss
best_iter = iter
test_losses = [np.mean(optimizing_function(*test_model(i),pos_label=None,average=None)[1]) for i in range(n_test_batches)]
test_score = np.mean(test_losses)
best_states = classifier.getstate()
print(('epoch %i, minibatch %i/%i, test performance for the optimizing function for positive labels %f %%') %(epoch, minibatch_index + 1, n_train_batches,test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print(('Optimization complete. Best validation score of %f %% obtained at iteration %i, with test performance %f %%') %(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print(('The code for file ran for %.2fm' % ((end_time - start_time) / 60.)))
return classifier, best_states
}
)
y_predict = np.zeros([n_out,len(test_x)])
y_predict[i] = [test_model(i) for i in range(test_x)]
return y_predict
# function to get the
def dummifier(vector):
return list(vector).index(1)
if __name__ == '__main__':
n_pos=45 # dimension of POS embeddings
n_in=551 + n_pos
n_out = 27
# assuming that the input data is already vectorized word embeddings
Xt = pickle.load(open('../data/test_data.pkl','rb'))
Yt = pickle.load(open('../data/test_label.pkl','rb'))
params = pickle.load(open('../data/optim_params.pkl','rb'))
classifier = FFNN(
input=params,
n_in=n_in,
n_hidden=n_hidden,
n_out=n_out
)
Yt_hat = predict(classifier, Xt, paramas)