class Model:
"""
Simple predictive model for forecasting words from
sequence using LSTMs. Choose how many LSTMs to stack
what size their memory should be, and how many
words can be predicted.
"""
def __init__(self, hidden_size, input_size, vocab_size, stack_size=1, celltype=LSTM):
# core layer in RNN/LSTM
self.model = StackedCells(input_size, celltype=celltype, layers =[hidden_size] * stack_size)
# add an embedding
self.model.layers.insert(0, Embedding(vocab_size, input_size))
# add a classifier:
self.model.layers.append(Layer(hidden_size, vocab_size, activation = softmax))
self.turing_params = Parameters()
#init turing machine model
self.turing_updates , self.turing_predict = turing_model.build(self.turing_params , hidden_size , vocab_size)
# inputs are matrices of indices,
# each row is a sentence, each column a timestep
self._stop_word = theano.shared(np.int32(999999999), name="stop word")
self.for_how_long = T.ivector()
self.input_mat = T.imatrix()
self.priming_word = T.iscalar()
self.srng = T.shared_randomstreams.RandomStreams(np.random.randint(0, 1024))
# create symbolic variables for prediction:
#change by darong #issue : what is greedy
self.lstm_predictions = self.create_lstm_prediction()
self.final_predictions = self.create_final_prediction()
# create symbolic variable for greedy search:
self.greedy_predictions = self.create_lstm_prediction(greedy=True)
# create gradient training functions:
self.create_cost_fun()#create 2 cost func(lstm final)
self.lstm_lr = 0.01
self.turing_lr = 0.01
self.all_lr = 0.01
self.create_training_function()#create 3 functions(lstm turing all)
self.create_predict_function()#create 2 predictions(lstm final)
# create ppl
self.lstm_ppl = self.create_lstm_ppl()
self.final_ppl = self.create_final_ppl()
self.create_ppl_function()
def save(self, save_file, vocab):
pickle.dump(self.model, open(save_file, "wb")) # pickle is for lambda function, cPickle cannot
pickle.dump(vocab, open(save_file+'.vocab', "wb")) # pickle is for lambda function, cPickle cannot
def save_turing(self, save_file):
self.turing_params.save(save_file + '.turing')
def load(self, load_file, lr):
self.model = pickle.load(open(load_file, "rb"))
if os.path.isfile(load_file + '.turing') :
self.turing_params.load(load_file + '.turing')
else :
print "no turing model!!!! pretrain with lstm param"
self.turing_params['W_input_hidden'] = self.model.layers[-1].params[0].get_value().T #not sure
self.turing_params['W_read_hidden'] = self.model.layers[-1].params[0].get_value().T
self.turing_params['b_hidden_0'] = self.model.layers[-1].params[1].get_value()
temp = self.model.layers[1].initial_hidden_state.get_value()[self.hidden_size:]
self.turing_params['memory_init'] = temp.reshape((1,)+temp.shape)
# need to compile again for calculating predictions after loading lstm
self.srng = T.shared_randomstreams.RandomStreams(np.random.randint(0, 1024))
self.lstm_predictions = self.create_lstm_prediction()
self.final_predictions = self.create_final_prediction()
self.greedy_predictions = self.create_lstm_prediction(greedy=True)#can change to final
self.create_cost_fun()#create 2 cost func(lstm final)
self.lstm_lr = lr
self.turing_lr = lr#change this
self.all_lr = lr
self.create_training_function()#create 3 functions(lstm turing all)
self.create_predict_function()#create 2 predictions(lstm final)
self.lstm_ppl = self.create_lstm_ppl()
self.final_ppl = self.create_final_ppl()
self.create_ppl_function()
print "done loading model"
# print "done compile"
def stop_on(self, idx):
self._stop_word.set_value(idx)
@property
def params(self):
return self.model.params
def create_lstm_prediction(self, greedy=False):
def step(idx, *states):
# new hiddens are the states we need to pass to LSTMs
# from past. Because the StackedCells also include
# the embeddings, and those have no state, we pass
# a "None" instead:
new_hiddens = [None] + list(states)
new_states = self.model.forward(idx, prev_hiddens = new_hiddens)
if greedy:
new_idxes = new_states[-1]
new_idx = new_idxes.argmax()
# provide a stopping condition for greedy search:
return ([new_idx.astype(self.priming_word.dtype)] + new_states[1:-1]), theano.scan_module.until(T.eq(new_idx,self._stop_word))
else:
return new_states[1:]
# in sequence forecasting scenario we take everything
# up to the before last step, and predict subsequent
# steps ergo, 0 ... n - 1, hence:
inputs = self.input_mat[:, 0:-1]
num_examples = inputs.shape[0]
# pass this to Theano's recurrence relation function:
# choose what gets outputted at each timestep:
if greedy:
outputs_info = [dict(initial=self.priming_word, taps=[-1])] + [initial_state_with_taps(layer) for layer in self.model.layers[1:-1]]
result, _ = theano.scan(fn=step,
n_steps=200,
outputs_info=outputs_info)
else:
outputs_info = [initial_state_with_taps(layer, num_examples) for layer in self.model.layers[1:]]
result, _ = theano.scan(fn=step,
sequences=[inputs.T],
outputs_info=outputs_info)
if greedy:
return result[0]
# softmaxes are the last layer of our network,
# and are at the end of our results list:
return result[-1].transpose((2,0,1))
# we reorder the predictions to be:
# 1. what row / example
# 2. what timestep
# 3. softmax dimension
def create_final_prediction(self, greedy=False):
def step(idx, *states):
# new hiddens are the states we need to pass to LSTMs
# from past. Because the StackedCells also include
# the embeddings, and those have no state, we pass
# a "None" instead:
new_hiddens = [None] + list(states)
new_states = self.model.forward(idx, prev_hiddens = new_hiddens)
if greedy:
new_idxes = new_states[-1]
new_idx = new_idxes.argmax()
# provide a stopping condition for greedy search:
return ([new_idx.astype(self.priming_word.dtype)] + new_states[1:-1]), theano.scan_module.until(T.eq(new_idx,self._stop_word))
else:
return new_states[1:]
# in sequence forecasting scenario we take everything
# up to the before last step, and predict subsequent
# steps ergo, 0 ... n - 1, hence:
inputs = self.input_mat[:, 0:-1]
num_examples = inputs.shape[0]
# pass this to Theano's recurrence relation function:
# choose what gets outputted at each timestep:
if greedy:
outputs_info = [dict(initial=self.priming_word, taps=[-1])] + [initial_state_with_taps(layer) for layer in self.model.layers[1:-1]]
result, _ = theano.scan(fn=step,
n_steps=200,
outputs_info=outputs_info)
else:
outputs_info = [initial_state_with_taps(layer, num_examples) for layer in self.model.layers[1:]]
result, _ = theano.scan(fn=step,
sequences=[inputs.T],
outputs_info=outputs_info)
if greedy:
return result[0]
# softmaxes are the last layer of our network,
# and are at the end of our results list:
hidden_size = result[-2].shape[2]/2
turing_result = self.turing_predict(result[-2][:,:,hidden_size:])
#the last layer do transpose before compute
return turing_result.transpose((1,0,2))
# we reorder the predictions to be:
# 1. what row / example
# 2. what timestep
# 3. softmax dimension
def create_cost_fun (self):
# create a cost function that
# takes each prediction at every timestep
# and guesses next timestep's value:
what_to_predict = self.input_mat[:, 1:]
# because some sentences are shorter, we
# place masks where the sentences end:
# (for how long is zero indexed, e.g. an example going from `[2,3)`)
# has this value set 0 (here we substract by 1):
for_how_long = self.for_how_long - 1
# all sentences start at T=0:
starting_when = T.zeros_like(self.for_how_long)
self.lstm_cost = masked_loss(self.lstm_predictions,
what_to_predict,
for_how_long,
starting_when).sum()
self.final_cost = masked_loss(self.final_predictions,
what_to_predict,
for_how_long,
starting_when).sum()
def create_predict_function(self):
self.lstm_pred_fun = theano.function(
inputs=[self.input_mat],
outputs=self.lstm_predictions,
allow_input_downcast=True
)
self.final_pred_fun = theano.function(
inputs=[self.input_mat],
outputs=self.final_predictions,
allow_input_downcast=True
)
self.greedy_fun = theano.function(
inputs=[self.priming_word],
outputs=T.concatenate([T.shape_padleft(self.priming_word), self.greedy_predictions]),
allow_input_downcast=True
)
def create_training_function(self):
updates, _, _, _, _ = create_optimization_updates(self.lstm_cost, self.params, method="SGD", lr=self.lstm_lr)
# updates, _, _, _, _ = create_optimization_updates(self.cost, self.params, method="adadelta", lr=self.lr)
self.lstm_update_fun = theano.function(
inputs=[self.input_mat, self.for_how_long],
outputs=self.lstm_cost,
updates=updates,
allow_input_downcast=True)
updates_turing = self.turing_updates(self.final_cost , lr=self.turing_lr)
# updates, _, _, _, _ = create_optimization_updates(self.cost, self.params, method="adadelta", lr=self.lr)
self.turing_update_fun = theano.function(
inputs=[self.input_mat, self.for_how_long],
outputs=self.final_cost,
updates=updates_turing,
mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True),
allow_input_downcast=True)
all_updates_lstm, _, _, _, _ = create_optimization_updates(self.final_cost, self.params, method="SGD", lr=self.all_lr,part=True)
all_updates_turing_temp = self.turing_updates(self.final_cost , lr=self.all_lr)
updates_all = all_updates_lstm
for pair in all_updates_turing_temp :
updates_all[pair[0]] = pair[1]
self.all_update_fun = theano.function(
inputs=[self.input_mat, self.for_how_long],
outputs=self.final_cost,
updates=updates_all,
allow_input_downcast=True)
def create_lstm_ppl(self):
def timestep(predictions, label, len_example, total_len_example):
label_binary = T.gt(label[0:len_example-1], 0)
oov_count = T.shape(label_binary)[0] - T.sum(label_binary)
a = total_len_example
return T.sum(T.log( 1./ predictions[T.arange(len_example-1), label[0:len_example-1]]) * label_binary ), oov_count
result, _ = theano.scan(fn=timestep,
sequences=[ self.lstm_predictions, self.input_mat[:, 1:], self.for_how_long ],
non_sequences=T.sum(self.for_how_long))
oov_count_total = T.sum(result[1])
return T.exp(T.sum(result[0]).astype(theano.config.floatX)/(T.sum(self.for_how_long) - oov_count_total).astype(theano.config.floatX)).astype(theano.config.floatX)
def create_final_ppl(self):
def timestep(predictions, label, len_example, total_len_example):
label_binary = T.gt(label[0:len_example-1], 0)
oov_count = T.shape(label_binary)[0] - T.sum(label_binary)
a = total_len_example
return T.sum(T.log( 1./ predictions[T.arange(len_example-1), label[0:len_example-1]]) * label_binary ), oov_count
result, _ = theano.scan(fn=timestep,
sequences=[ self.final_predictions, self.input_mat[:, 1:], self.for_how_long ],
non_sequences=T.sum(self.for_how_long))
oov_count_total = T.sum(result[1])
return T.exp(T.sum(result[0]).astype(theano.config.floatX)/(T.sum(self.for_how_long) - oov_count_total).astype(theano.config.floatX)).astype(theano.config.floatX)
def create_ppl_function(self):
self.lstm_ppl_fun = theano.function(
inputs=[self.input_mat, self.for_how_long],
outputs=self.lstm_ppl,
allow_input_downcast=True)
self.final_ppl_fun = theano.function(
inputs=[self.input_mat, self.for_how_long],
outputs=self.final_ppl,
allow_input_downcast=True)
def __call__(self, x):
return self.pred_fun(x)#any problem??
gradients = T.grad(cost, wrt=params)
gradient_acc = [theano.shared(0 * p.get_value()) for p in params]
counter = theano.shared(np.float32(0.))
acc = theano.function(inputs=[X, Y],
outputs=cost,
updates=[(a, a + g)
for a, g in zip(gradient_acc, gradients)] +
[(counter, counter + np.float32(1.))])
update = theano.function(
inputs=[],outputs=[],
updates = updates.momentum(params,[ g / counter for g in gradient_acc ]) \
+ [ (a, np.float32(0) * a) for a in gradient_acc ] \
+ [ (counter,np.float32(0.)) ]
)
test = theano.function(inputs=[X, Y], outputs=probs[:, Y])
training_examples = [word.strip() for word in open('dictionary.txt')]
import random
for _ in xrange(1500):
random.shuffle(training_examples)
for i, string in enumerate(training_examples):
print acc(font.imagify(string), label_seq(string))
if i % 20 == 0: update()
if i % 100 == 0:
hinton.plot(test(font.imagify("test"), label_seq("test")).T,
max_arr=1.)
hinton.plot(font.imagify("test").T[::-1].astype('float32'))
P.save('model.pkl')
acc = theano.function(
inputs=[X, Y],
outputs=cost,
updates = [
(a,a + g) for a,g in zip(gradient_acc,gradients)
] + [(counter,counter + np.float32(1.))]
)
update = theano.function(
inputs=[],outputs=[],
updates = updates.momentum(params,[ g / counter for g in gradient_acc ]) \
+ [ (a, np.float32(0) * a) for a in gradient_acc ] \
+ [ (counter,np.float32(0.)) ]
)
test = theano.function(
inputs=[X,Y],
outputs=probs[:,Y]
)
training_examples = [ word.strip() for word in open('dictionary.txt') ]
import random
for _ in xrange(1500):
random.shuffle(training_examples)
for i,string in enumerate(training_examples):
print acc(font.imagify(string),label_seq(string))
if i % 20 == 0: update()
if i % 100 == 0:
hinton.plot(test(font.imagify("test"),label_seq("test")).T,max_arr=1.)
hinton.plot(font.imagify("test").T[::-1].astype('float32'))
P.save('model.pkl')
test_group_answers = islice(group_answers,test_instance_count)
test_data = data_io.story_question_answer_idx(
test_group_answers,
vocab_in
)
test_data = ( x for x in test_data if x[1].shape[0] <= length_limit )
tests = [ np.array(
test(input_data,idxs,question_data,ans_w,ans_evds),
dtype=np.float32
)
for input_data,idxs,question_data,ans_w,ans_evds in test_data ]
errors = sum(tests)/len(tests)
print "Error rate:",errors
print "Starting epoch ",epoch
if errors < best_error * 0.9 :
P.save('model.pkl')
print "Wrote model."
best_error = errors
length_limit += 2
else:
# learning_rate = learning_rate / 2
# batch_size = max(1,batch_size//2)
# print "Learning rate:",learning_rate
P.save('tmp.model.pkl')
buffer_size = 256 / batch_size
train_group_answers = data_io.randomise(group_answers)
training_data = data_io.story_question_answer_idx(train_group_answers,vocab_in)
training_data = ( x for x in training_data if x[1].shape[0] <= length_limit )
training_data = data_io.sortify(training_data,key=lambda x:x[1].shape[0])
batched_training_data = data_io.batch(
best_cost = np.inf
increase_count = 0
seen = 0
for epoch in xrange(max_epochs):
print "Epoch:", epoch + 1
print "Batch size:", batch_size
# Run test on validation set
data_stream = data_io.stream(data_file, char2id)
test_stream = islice(data_stream, validation_count)
test_cost = test(test_stream)
print "Perplexity:", test_cost
if test_cost < improvement_threshold * best_cost:
best_cost = test_cost
P.save(output_file)
increase_count = 0
else:
increase_count += 1
if increase_count > patience:
break
# Run training
data_stream = data_io.randomise(data_stream, buffer_size=1024)
data_stream = data_io.sortify(data_stream, key=lambda x: len(x), buffer_size=512)
batch_data_stream = data_io.batch(data_stream, batch_size=batch_size)
batch_data_stream = data_io.randomise(batch_data_stream)
for batch in batch_data_stream:
avg_cost = train(batch)
if np.isnan(avg_cost):
_, hidden = lstm_layer(X_rep)
output = T.nnet.softmax(T.dot(hidden, P.W_output))
delay = 5
label = X[:-delay]
predicted = output[delay:]
cost = -T.sum(T.log(predicted[T.arange(predicted.shape[0]), label]))
params = P.values()
gradients = T.grad(cost, wrt=params)
update_methods = {
'standard': [(p, p - 0.001 * g) for p, g in zip(params, gradients)],
# 'rmsprop' : updates.rmsprop(params,gradients),
# 'adadelta': updates.rmsprop(params,gradients),
}
P.save('init.pkl')
for update_method in update_methods:
print "Using update method:", update_method
with open('train.%s.smart_init.log' % update_method, 'w') as log:
train = theano.function(
inputs=[X],
outputs=cost,
updates=update_methods[update_method],
)
P.load('init.pkl')
while True:
cost_val = train(
np.random.randint(0, 8, size=20).astype(np.int32))
total = 0
count = 0
for chunk in stream:
vals = test(chunk)
loss = vals[0]
if chunk.shape > 1:
total += chunk.shape[0] * loss
count += chunk.shape[0]
print ' '.join(str(v) for v in vals),
return total / count
best_cost = np.inf
for epoch in xrange(50):
print "Epoch %d" % epoch,
cost = validation()
print cost,
if cost < best_cost:
print "Saving...."
P.save('model.pkl')
best_cost = cost
else:
print
for chunk in data_stream():
chunk_X.set_value(chunk)
batches = int(math.floor(chunk.shape[0] / float(batch_size)))
for i in xrange(batches):
loss = train(i)
print ' '.join(str(v) for v in loss) # , show_betas()
# pprint({p.name: g for p, g in zip(parameters, grad_norms)})
P.save('unval_model.pkl')
output = T.nnet.softmax(T.dot(hidden,P.W_output))
delay = 5
label = X[:-delay]
predicted = output[delay:]
cost = -T.sum(T.log(predicted[T.arange(predicted.shape[0]),label]))
params = P.values()
gradients = T.grad(cost,wrt=params)
update_methods = {
'standard': [ (p, p - 0.001 * g) for p,g in zip(params,gradients) ],
# 'rmsprop' : updates.rmsprop(params,gradients),
# 'adadelta': updates.rmsprop(params,gradients),
}
P.save('init.pkl')
for update_method in update_methods:
print "Using update method:",update_method
with open('train.%s.smart_init.log'%update_method,'w') as log:
train = theano.function(
inputs = [X],
outputs = cost,
updates = update_methods[update_method],
)
P.load('init.pkl')
while True:
cost_val = train(np.random.randint(0,8,size=20).astype(np.int32))
log.write("%0.5f\n"%cost_val)
while True:
group_answers = data_io.group_answers(training_file)
test_group_answers = islice(group_answers, test_instance_count)
test_data = data_io.story_question_answer_idx(test_group_answers,
vocab_in)
test_data = (x for x in test_data if x[1].shape[0] <= length_limit)
tests = [
np.array(test(input_data, idxs, question_data, ans_w, ans_evds),
dtype=np.float32)
for input_data, idxs, question_data, ans_w, ans_evds in test_data
]
errors = sum(tests) / len(tests)
print "Error rate:", errors
print "Starting epoch ", epoch
if errors < best_error * 0.9:
P.save('model.pkl')
print "Wrote model."
best_error = errors
length_limit += 2
else:
# learning_rate = learning_rate / 2
# batch_size = max(1,batch_size//2)
# print "Learning rate:",learning_rate
P.save('tmp.model.pkl')
buffer_size = 256 / batch_size
train_group_answers = data_io.randomise(group_answers)
training_data = data_io.story_question_answer_idx(
train_group_answers, vocab_in)
training_data = (x for x in training_data
if x[1].shape[0] <= length_limit)
batched_stream = data_io.buffered_random(batched_stream, buffer_items=4)
return batched_stream
def validate():
stream = data_io.stream_file('data/train.%02d.pklgz' % 0)
stream = data_io.buffered_sort(stream, key=lambda x: x[1].shape[0], buffer_items=128)
batched_stream = reader.batch_and_pad(stream, batch_size=32, mean=mean, std=std)
total_cost = 0
total_frames = 0
for data, lengths in batched_stream:
batch_avg_cost = test(data,lengths)
batch_frames = np.sum(lengths)
total_cost += batch_avg_cost * batch_frames
total_frames += batch_frames
return total_cost / total_frames
import train_loop
train_loop.run(
data_iterator=stream,
train_fun=lambda batch:train(batch[0],batch[1]),
validation_score=validate,
save_best_params=lambda:P.save('model.pkl'),
load_best_params=lambda:P.load('model.pkl'),
max_epochs=1000,
patience=5000,
patience_increase=2,
improvement_threshold=0.999,
)
stream = data_io.stream_file('data/train.%02d.pklgz' % 0)
stream = data_io.buffered_sort(stream,
key=lambda x: x[1].shape[0],
buffer_items=128)
batched_stream = reader.batch_and_pad(stream,
batch_size=32,
mean=mean,
std=std)
total_cost = 0
total_frames = 0
for data, lengths in batched_stream:
batch_avg_cost = test(data, lengths)
batch_frames = np.sum(lengths)
total_cost += batch_avg_cost * batch_frames
total_frames += batch_frames
return total_cost / total_frames
import train_loop
train_loop.run(
data_iterator=stream,
train_fun=lambda batch: train(batch[0], batch[1]),
validation_score=validate,
save_best_params=lambda: P.save('model.pkl'),
load_best_params=lambda: P.load('model.pkl'),
max_epochs=1000,
patience=5000,
patience_increase=2,
improvement_threshold=0.999,
)
