def main(num_epochs=NEPOCH):
print("Loading data ...")
snli = SNLI(batch_size=BSIZE)
train_batches = list(snli.train_minibatch_generator())
dev_batches = list(snli.dev_minibatch_generator())
test_batches = list(snli.test_minibatch_generator())
W_word_embedding = snli.weight # W shape: (# vocab size, WE_DIM)
W_word_embedding =
snli.weight / \
(numpy.linalg.norm(snli.weight, axis=1).reshape(
snli.weight.shape[0], 1) + 0.00001)
def main(num_epochs=NEPOCH):
print("Loading data ...")
snli = SNLI(batch_size=BSIZE)
train_batches = list(snli.train_minibatch_generator())
dev_batches = list(snli.dev_minibatch_generator())
test_batches = list(snli.test_minibatch_generator())
W_word_embedding = snli.weight # W shape: (# vocab size, WE_DIM)
del snli
print("Building network ...")
########### sentence embedding encoder ###########
"""
# sentence vector, with each number standing for a word number
input_var = T.TensorType('int32', [False, False])('sentence_vector')
input_var.tag.test_value = numpy.hstack((numpy.random.randint(1, 10000, (BSIZE, 20), 'int32'),
numpy.zeros((BSIZE, 5)).astype('int32')))
input_var.tag.test_value[1, 20:22] = (413, 45)
l_in = lasagne.layers.InputLayer(shape=(BSIZE, None), input_var=input_var)
input_mask = T.TensorType('int32', [False, False])('sentence_mask')
input_mask.tag.test_value = numpy.hstack((numpy.ones((BSIZE, 20), dtype='int32'),
numpy.zeros((BSIZE, 5), dtype='int32')))
input_mask.tag.test_value[1, 20:22] = 1
l_mask = lasagne.layers.InputLayer(shape=(BSIZE, None), input_var=input_mask)
# output shape (BSIZE, None, WEDIM)
l_word_embed = lasagne.layers.EmbeddingLayer(
l_in,
input_size=W_word_embedding.shape[0],
output_size=W_word_embedding.shape[1],
W=W_word_embedding)
"""
########### input layers ###########
# hypothesis
input_var_h = T.TensorType('int32', [False, False])('hypothesis_vector')
input_var_h.tag.test_value = numpy.hstack((numpy.random.randint(1, 10000, (BSIZE, 18), 'int32'),
numpy.zeros((BSIZE, 6)).astype('int32')))
l_in_h = lasagne.layers.InputLayer(shape=(BSIZE, None), input_var=input_var_h)
input_mask_h = T.TensorType('int32', [False, False])('hypo_mask')
input_mask_h.tag.test_value = numpy.hstack((numpy.ones((BSIZE, 18), dtype='int32'),
numpy.zeros((BSIZE, 6), dtype='int32')))
input_mask_h.tag.test_value[1, 18:22] = 1
l_mask_h = lasagne.layers.InputLayer(shape=(BSIZE, None), input_var=input_mask_h)
# premise
input_var_p = T.TensorType('int32', [False, False])('premise_vector')
input_var_p.tag.test_value = numpy.hstack((numpy.random.randint(1, 10000, (BSIZE, 16), 'int32'),
numpy.zeros((BSIZE, 3)).astype('int32')))
l_in_p = lasagne.layers.InputLayer(shape=(BSIZE, None), input_var=input_var_p)
input_mask_p = T.TensorType('int32', [False, False])('premise_mask')
input_mask_p.tag.test_value = numpy.hstack((numpy.ones((BSIZE, 16), dtype='int32'),
numpy.zeros((BSIZE, 3), dtype='int32')))
input_mask_p.tag.test_value[1, 16:18] = 1
l_mask_p = lasagne.layers.InputLayer(shape=(BSIZE, None), input_var=input_mask_p)
###################################
# output shape (BSIZE, None, WEDIM)
l_hypo_embed = lasagne.layers.EmbeddingLayer(
l_in_h,
input_size=W_word_embedding.shape[0],
output_size=W_word_embedding.shape[1],
W=W_word_embedding)
l_prem_embed = lasagne.layers.EmbeddingLayer(
l_in_p,
input_size=W_word_embedding.shape[0],
output_size=W_word_embedding.shape[1],
W=l_hypo_embed.W)
# ATTEND
l_hypo_embed_dpout = lasagne.layers.DropoutLayer(l_hypo_embed, p=DPOUT, rescale=True)
l_hypo_embed_hid1 = DenseLayer3DInput(
l_hypo_embed_dpout, num_units=EMBDHIDA, nonlinearity=lasagne.nonlinearities.rectify)
l_hypo_embed_hid1_dpout = lasagne.layers.DropoutLayer(l_hypo_embed_hid1, p=DPOUT, rescale=True)
l_hypo_embed_hid2 = DenseLayer3DInput(
l_hypo_embed_hid1_dpout, num_units=EMBDHIDB, nonlinearity=lasagne.nonlinearities.rectify)
l_prem_embed_dpout = lasagne.layers.DropoutLayer(l_prem_embed, p=DPOUT, rescale=True)
l_prem_embed_hid1 = DenseLayer3DInput(
l_prem_embed_dpout, num_units=EMBDHIDA, nonlinearity=lasagne.nonlinearities.rectify)
l_prem_embed_hid1_dpout = lasagne.layers.DropoutLayer(l_prem_embed_hid1, p=DPOUT, rescale=True)
l_prem_embed_hid2 = DenseLayer3DInput(
l_prem_embed_hid1_dpout, num_units=EMBDHIDB, nonlinearity=lasagne.nonlinearities.rectify)
# output dim: (BSIZE, NROWx, NROWy)
l_e = ComputeEmbeddingPool([l_hypo_embed_hid2, l_prem_embed_hid2])
# output dim: (BSIZE, NROWy, DIM)
l_hypo_weighted = AttendOnEmbedding([l_hypo_embed, l_e], masks=[l_mask_h, l_mask_p], direction='col')
# output dim: (BSIZE, NROWx, DIM)
l_prem_weighted = AttendOnEmbedding([l_prem_embed, l_e], masks=[l_mask_h, l_mask_p], direction='row')
# COMPARE
# output dim: (BSIZE, NROW, 4*LSTMHID)
l_hypo_premwtd = lasagne.layers.ConcatLayer([l_hypo_embed, l_prem_weighted], axis=2)
l_prem_hypowtd = lasagne.layers.ConcatLayer([l_prem_embed, l_hypo_weighted], axis=2)
l_hypo_premwtd_dpout = lasagne.layers.DropoutLayer(l_hypo_premwtd, p=DPOUT, rescale=True)
l_hypo_comphid1 = DenseLayer3DInput(
l_hypo_premwtd_dpout, num_units=COMPHIDA, nonlinearity=lasagne.nonlinearities.rectify)
l_hypo_comphid1_dpout = lasagne.layers.DropoutLayer(l_hypo_comphid1, p=DPOUT, rescale=True)
l_hypo_comphid2 = DenseLayer3DInput(
l_hypo_comphid1_dpout, num_units=COMPHIDB, nonlinearity=lasagne.nonlinearities.rectify)
l_prem_hypowtd_dpout = lasagne.layers.DropoutLayer(l_prem_hypowtd, p=DPOUT, rescale=True)
l_prem_comphid1 = DenseLayer3DInput(
l_prem_hypowtd_dpout, num_units=COMPHIDA,
W=l_hypo_comphid1.W, b=l_hypo_comphid1.b, nonlinearity=lasagne.nonlinearities.rectify)
l_prem_comphid1_dpout = lasagne.layers.DropoutLayer(l_prem_comphid1, p=DPOUT, rescale=True)
l_prem_comphid2 = DenseLayer3DInput(
l_prem_comphid1_dpout, num_units=COMPHIDB,
W=l_hypo_comphid2.W, b=l_hypo_comphid2.b, nonlinearity=lasagne.nonlinearities.rectify)
# AGGREGATE
# output dim: (BSIZE, 4*LSTMHID)
l_hypo_mean = MeanOverDim(l_hypo_comphid2, mask=l_mask_h, dim=1)
l_prem_mean = MeanOverDim(l_prem_comphid2, mask=l_mask_p, dim=1)
l_v1v2 = lasagne.layers.ConcatLayer([l_hypo_mean, l_prem_mean], axis=1)
l_v1v2_dpout = lasagne.layers.DropoutLayer(l_v1v2, p=DPOUT, rescale=True)
l_outhid = lasagne.layers.DenseLayer(
l_v1v2_dpout, num_units=OUTHID, nonlinearity=lasagne.nonlinearities.rectify)
l_outhid_dpout = lasagne.layers.DropoutLayer(l_outhid, p=DPOUT, rescale=True)
l_output = lasagne.layers.DenseLayer(
l_outhid_dpout, num_units=3, nonlinearity=lasagne.nonlinearities.softmax)
########### target, cost, validation, etc. ##########
target_values = T.ivector('target_output')
target_values.tag.test_value = numpy.asarray([1,] * BSIZE, dtype='int32')
network_output = lasagne.layers.get_output(l_output)
network_prediction = T.argmax(network_output, axis=1)
error_rate = T.mean(T.neq(network_prediction, target_values))
network_output_clean = lasagne.layers.get_output(l_output, deterministic=True)
network_prediction_clean = T.argmax(network_output_clean, axis=1)
error_rate_clean = T.mean(T.neq(network_prediction_clean, target_values))
cost = T.mean(T.nnet.categorical_crossentropy(network_output, target_values))
cost_clean = T.mean(T.nnet.categorical_crossentropy(network_output_clean, target_values))
# Retrieve all parameters from the network
all_params = lasagne.layers.get_all_params(l_output)
if not UPDATEWE:
all_params.remove(l_hypo_embed.W)
numparams = sum([numpy.prod(i) for i in [i.shape.eval() for i in all_params]])
print("Number of params: {}\nName\t\t\tShape\t\t\tSize".format(numparams))
print("-----------------------------------------------------------------")
for item in all_params:
print("{0:24}{1:24}{2}".format(item, item.shape.eval(), numpy.prod(item.shape.eval())))
# if exist param file then load params
look_for = 'params' + os.sep + 'params_' + filename + '.pkl'
if os.path.isfile(look_for):
print("Resuming from file: " + look_for)
all_param_values = cPickle.load(open(look_for, 'rb'))
for p, v in zip(all_params, all_param_values):
p.set_value(v)
# Compute SGD updates for training
print("Computing updates ...")
updates = lasagne.updates.adagrad(cost, all_params, LR)
# Theano functions for training and computing cost
print("Compiling functions ...")
train = theano.function(
[l_in_h.input_var, l_mask_h.input_var,
l_in_p.input_var, l_mask_p.input_var, target_values],
[cost, error_rate], updates=updates)
# mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=False))
compute_cost = theano.function(
[l_in_h.input_var, l_mask_h.input_var,
l_in_p.input_var, l_mask_p.input_var, target_values],
[cost_clean, error_rate_clean])
# mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=False))
def evaluate(mode):
if mode == 'dev':
data = dev_batches
if mode == 'test':
data = test_batches
set_cost = 0.
set_error_rate = 0.
for batches_seen, (hypo, hm, premise, pm, truth) in enumerate(data, 1):
_cost, _error = compute_cost(hypo, hm, premise, pm, truth)
set_cost = (1.0 - 1.0 / batches_seen) * set_cost + \
1.0 / batches_seen * _cost
set_error_rate = (1.0 - 1.0 / batches_seen) * set_error_rate + \
1.0 / batches_seen * _error
return set_cost, set_error_rate
print("Done. Evaluating scratch model ...")
dev_set_cost, dev_set_error = evaluate('dev')
print("BEFORE TRAINING: dev cost %f, error %f" % (dev_set_cost, dev_set_error))
print("Training ...")
try:
for epoch in range(num_epochs):
train_set_cost = 0.
train_set_error = 0.
start = time.time()
for batches_seen, (hypo, hm, premise, pm, truth) in enumerate(train_batches, 1):
_cost, _error = train(hypo, hm, premise, pm, truth)
train_set_cost = (1.0 - 1.0 / batches_seen) * train_set_cost + \
1.0 / batches_seen * _cost
train_set_error = (1.0 - 1.0 / batches_seen) * train_set_error + \
1.0 / batches_seen * _error
if batches_seen % 100 == 0:
end = time.time()
print("Sample %d %.2fs, lr %.4f, train cost %f, error %f" % (
batches_seen * BSIZE,
end - start,
LR,
train_set_cost,
train_set_error))
start = end
if batches_seen % 2000 == 0:
dev_set_cost, dev_set_error = evaluate('dev')
print("***dev cost %f, error %f" % (dev_set_cost, dev_set_error))
# save parameters
all_param_values = [p.get_value() for p in all_params]
cPickle.dump(all_param_values,
open('params' + os.sep + 'params_' + filename + '.pkl', 'wb'))
dev_set_cost, dev_set_error = evaluate('dev')
test_set_cost, test_set_error = evaluate('test')
print("epoch %d, cost: train %f dev %f test %f;\n"
" error train %f dev %f test %f" % (
epoch,
train_set_cost, dev_set_cost, test_set_cost,
train_set_error, dev_set_error, test_set_error))
except KeyboardInterrupt:
pdb.set_trace()
pass
def main(num_epochs=NUM_EPOCHS):
print("Loading data ...")
snli = SNLI(batch_size=BATCH_SIZE)
train_batches = list(snli.train_minibatch_generator())
dev_batches = list(snli.dev_minibatch_generator())
test_batches = list(snli.test_minibatch_generator())
W_word_embedding = snli.weight # W shape: (# vocab size, WE_DIM)
del snli
print("Building network ...")
########### sentence embedding encoder ###########
# sentence vector, with each number standing for a word number
input_var = T.TensorType('int32', [False, False])('sentence_vector')
input_var.tag.test_value = numpy.hstack(
(numpy.random.randint(1, 10000, (50, 20), 'int32'), numpy.zeros(
(50, 5)).astype('int32')))
input_var.tag.test_value[1, 20:22] = (413, 45)
l_in = lasagne.layers.InputLayer(shape=(BATCH_SIZE, None),
input_var=input_var)
input_mask = T.TensorType('int32', [False, False])('sentence_mask')
input_mask.tag.test_value = numpy.hstack((numpy.ones(
(50, 20), dtype='int32'), numpy.zeros((50, 5), dtype='int32')))
input_mask.tag.test_value[1, 20:22] = 1
l_mask = lasagne.layers.InputLayer(shape=(BATCH_SIZE, None),
input_var=input_mask)
# output shape (BATCH_SIZE, None, WE_DIM)
l_word_embed = lasagne.layers.EmbeddingLayer(
l_in,
input_size=W_word_embedding.shape[0],
output_size=W_word_embedding.shape[1],
W=W_word_embedding) # how to set it to be non-trainable?
# bidirectional LSTM
l_forward = lasagne.layers.LSTMLayer(
l_word_embed,
mask_input=l_mask,
num_units=LSTM_HIDDEN,
ingate=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=init.Normal(STD)),
forgetgate=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=init.Normal(STD)),
cell=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=None,
nonlinearity=nonlinearities.tanh),
outgate=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=init.Normal(STD)),
nonlinearity=lasagne.nonlinearities.tanh,
peepholes=False,
grad_clipping=GRAD_CLIP)
l_backward = lasagne.layers.LSTMLayer(
l_word_embed,
mask_input=l_mask,
num_units=LSTM_HIDDEN,
ingate=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=init.Normal(STD)),
forgetgate=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=init.Normal(STD)),
cell=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=None,
nonlinearity=nonlinearities.tanh),
outgate=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=init.Normal(STD)),
nonlinearity=lasagne.nonlinearities.tanh,
peepholes=False,
grad_clipping=GRAD_CLIP,
backwards=True)
# output dim: (BATCH_SIZE, None, 2*LSTM_HIDDEN)
l_concat = lasagne.layers.ConcatLayer([l_forward, l_backward], axis=2)
# Attention mechanism to get sentence embedding
# output dim: (BATCH_SIZE, None, ATTENTION_HIDDEN)
l_ws1 = DenseLayer3DInput(l_concat, num_units=ATTENTION_HIDDEN)
# output dim: (BATCH_SIZE, None, N_ROWS)
l_ws2 = DenseLayer3DInput(l_ws1, num_units=N_ROWS, nonlinearity=None)
l_annotations = Softmax3D(l_ws2, mask=l_mask)
# output dim: (BATCH_SIZE, 2*LSTM_HIDDEN, N_ROWS)
l_sentence_embedding = ApplyAttention([l_annotations, l_concat])
# beam search? Bi lstm in the sentence embedding layer? etc.
########### get embeddings for hypothesis and premise ###########
# hypothesis
input_var_h = T.TensorType('int32', [False, False])('hypothesis_vector')
input_var_h.tag.test_value = numpy.hstack(
(numpy.random.randint(1, 10000, (50, 18), 'int32'), numpy.zeros(
(50, 6)).astype('int32')))
l_in_h = lasagne.layers.InputLayer(shape=(BATCH_SIZE, None),
input_var=input_var_h)
input_mask_h = T.TensorType('int32', [False, False])('hypo_mask')
input_mask_h.tag.test_value = numpy.hstack((numpy.ones(
(50, 18), dtype='int32'), numpy.zeros((50, 6), dtype='int32')))
input_mask_h.tag.test_value[1, 18:22] = 1
l_mask_h = lasagne.layers.InputLayer(shape=(BATCH_SIZE, None),
input_var=input_mask_h)
# premise
input_var_p = T.TensorType('int32', [False, False])('premise_vector')
input_var_p.tag.test_value = numpy.hstack(
(numpy.random.randint(1, 10000, (50, 16), 'int32'), numpy.zeros(
(50, 3)).astype('int32')))
l_in_p = lasagne.layers.InputLayer(shape=(BATCH_SIZE, None),
input_var=input_var_p)
input_mask_p = T.TensorType('int32', [False, False])('premise_mask')
input_mask_p.tag.test_value = numpy.hstack((numpy.ones(
(50, 16), dtype='int32'), numpy.zeros((50, 3), dtype='int32')))
input_mask_p.tag.test_value[1, 16:18] = 1
l_mask_p = lasagne.layers.InputLayer(shape=(BATCH_SIZE, None),
input_var=input_mask_p)
hypothesis_embedding, hypothesis_annotation = lasagne.layers.get_output(
[l_sentence_embedding, l_annotations], {
l_in: l_in_h.input_var,
l_mask: l_mask_h.input_var
})
premise_embedding, premise_annotation = lasagne.layers.get_output(
[l_sentence_embedding, l_annotations], {
l_in: l_in_p.input_var,
l_mask: l_mask_p.input_var
})
########### gated encoder and output MLP ##########
l_hypo_embed = lasagne.layers.InputLayer(shape=(BATCH_SIZE, N_ROWS,
2 * LSTM_HIDDEN),
input_var=hypothesis_embedding)
l_pre_embed = lasagne.layers.InputLayer(shape=(BATCH_SIZE, N_ROWS,
2 * LSTM_HIDDEN),
input_var=premise_embedding)
# output dim: (BATCH_SIZE, 2*LSTM_HIDDEN, N_ROWS)
l_factors = GatedEncoder3D([l_hypo_embed, l_pre_embed],
num_hfactors=2 * LSTM_HIDDEN)
# Dropout:
l_factors_noise = lasagne.layers.DropoutLayer(l_factors,
p=GAEREG,
rescale=True)
# l_hids = DenseLayer3DWeight()
l_outhid = lasagne.layers.DenseLayer(
l_factors_noise,
num_units=OUT_HIDDEN,
nonlinearity=lasagne.nonlinearities.rectify)
# Dropout:
l_outhid_noise = lasagne.layers.DropoutLayer(l_outhid,
p=GAEREG,
rescale=True)
l_output = lasagne.layers.DenseLayer(
l_outhid_noise,
num_units=3,
nonlinearity=lasagne.nonlinearities.softmax)
########### target, cost, validation, etc. ##########
target_values = T.ivector('target_output')
target_values.tag.test_value = numpy.asarray([
1,
] * 50, dtype='int32')
network_output = lasagne.layers.get_output(l_output)
network_output_clean = lasagne.layers.get_output(l_output,
deterministic=True)
# penalty term and cost
attention_penalty = T.mean(
(
T.batched_dot(
hypothesis_annotation,
# pay attention to this line:
# T.extra_ops.cpu_contiguous(hypothesis_annotation.dimshuffle(0, 2, 1))
hypothesis_annotation.dimshuffle(0, 2, 1)) -
T.eye(hypothesis_annotation.shape[1]).dimshuffle('x', 0, 1))**2,
axis=(0, 1, 2)
) + T.mean(
(
T.batched_dot(
premise_annotation,
# T.extra_ops.cpu_contiguous(premise_annotation.dimshuffle(0, 2, 1)) # ditto.
premise_annotation.dimshuffle(0, 2, 1) # ditto.
) - T.eye(premise_annotation.shape[1]).dimshuffle('x', 0, 1))**2,
axis=(0, 1, 2))
cost = T.mean(T.nnet.categorical_crossentropy(network_output, target_values) + \
ATTENTION_PENALTY * attention_penalty)
cost_clean = T.mean(T.nnet.categorical_crossentropy(network_output_clean, target_values) + \
ATTENTION_PENALTY * attention_penalty)
# Retrieve all parameters from the network
all_params = lasagne.layers.get_all_params(l_output) + \
lasagne.layers.get_all_params(l_sentence_embedding)
numparams = sum(
[numpy.prod(i) for i in [i.shape.eval() for i in all_params]])
print("Number of params: {}".format(numparams))
# if exist param file then load params
look_for = 'params' + os.sep + 'params_' + filename + '.pkl'
if os.path.isfile(look_for):
print("Resuming from file: " + look_for)
all_param_values = cPickle.load(open(look_for, 'rb'))
for p, v in zip(all_params, all_param_values):
p.set_value(v)
# withoutwe_params = all_params + [l_word_embed.W]
# Compute updates for training
print("Computing updates ...")
updates = lasagne.updates.adagrad(cost, all_params, LEARNING_RATE)
# Theano functions for training and computing cost
print("Compiling functions ...")
network_prediction = T.argmax(network_output, axis=1)
error_rate = T.mean(T.neq(network_prediction, target_values))
network_prediction_clean = T.argmax(network_output_clean, axis=1)
error_rate_clean = T.mean(T.neq(network_prediction_clean, target_values))
train = theano.function([
l_in_h.input_var, l_mask_h.input_var, l_in_p.input_var,
l_mask_p.input_var, target_values
], [cost, error_rate],
updates=updates)
compute_cost = theano.function([
l_in_h.input_var, l_mask_h.input_var, l_in_p.input_var,
l_mask_p.input_var, target_values
], [cost_clean, error_rate_clean])
def evaluate(mode):
if mode == 'dev':
data = dev_batches
if mode == 'test':
data = test_batches
set_cost = 0.
set_error_rate = 0.
for batches_seen, (hypo, hm, premise, pm, truth) in enumerate(data, 1):
_cost, _error = compute_cost(hypo, hm, premise, pm, truth)
set_cost = (1.0 - 1.0 / batches_seen) * set_cost + \
1.0 / batches_seen * _cost
set_error_rate = (1.0 - 1.0 / batches_seen) * set_error_rate + \
1.0 / batches_seen * _error
return set_cost, set_error_rate
dev_set_cost, dev_set_error = evaluate('dev')
print("BEFORE TRAINING: dev cost %f, error %f" %
(dev_set_cost, dev_set_error))
print("Training ...")
try:
for epoch in range(num_epochs):
train_set_cost = 0.
train_set_error = 0.
start = time.time()
for batches_seen, (hypo, hm, premise, pm,
truth) in enumerate(train_batches, 1):
_cost, _error = train(hypo, hm, premise, pm, truth)
train_set_cost = (1.0 - 1.0 / batches_seen) * train_set_cost + \
1.0 / batches_seen * _cost
train_set_error = (1.0 - 1.0 / batches_seen) * train_set_error + \
1.0 / batches_seen * _error
if batches_seen % 100 == 0:
end = time.time()
print("Sample %d %.2fs, lr %.4f, train cost %f, error %f" %
(batches_seen * BATCH_SIZE, LEARNING_RATE,
end - start, train_set_cost, train_set_error))
start = end
if batches_seen % 2000 == 0:
dev_set_cost, dev_set_error = evaluate('dev')
test_set_cost, test_set_error = evaluate('test')
print("***dev cost %f, error %f" %
(dev_set_cost, dev_set_error))
print("***test cost %f, error %f" %
(test_set_cost, test_set_error))
# save parameters
all_param_values = [p.get_value() for p in all_params]
cPickle.dump(
all_param_values,
open('params' + os.sep + 'params_' + filename + '.pkl', 'wb'))
# load params
# all_param_values = cPickle.load(open('params' + os.sep + 'params_' + filename, 'rb'))
# for p, v in zip(all_params, all_param_values):
# p.set_value(v)
dev_set_cost, dev_set_error = evaluate('dev')
test_set_cost, test_set_error = evaluate('test')
print("epoch %d, cost: train %f dev %f test %f;\n"
" error train %f dev %f test %f" %
(epoch, train_set_cost, dev_set_cost, test_set_cost,
train_set_error, dev_set_error, test_set_error))
except KeyboardInterrupt:
pdb.set_trace()
pass
def main(num_epochs=NEPOCH):
print("Loading data ...")
snli = SNLI(batch_size=BSIZE)
train_batches = list(snli.train_minibatch_generator())
dev_batches = list(snli.dev_minibatch_generator())
test_batches = list(snli.test_minibatch_generator())
W_word_embedding = snli.weight # W shape: (# vocab size, WE_DIM)
W_word_embedding = snli.weight / \
(numpy.linalg.norm(snli.weight, axis=1).reshape(snli.weight.shape[0], 1) + \
0.00001)
del snli
print("Building network ...")
########### input layers ###########
# hypothesis
input_var_h = T.TensorType('int32', [False, False])('hypothesis_vector')
input_var_h.tag.test_value = numpy.hstack(
(numpy.random.randint(1, 10000, (BSIZE, 18),
'int32'), numpy.zeros(
(BSIZE, 6)).astype('int32')))
l_in_h = lasagne.layers.InputLayer(shape=(BSIZE, None),
input_var=input_var_h)
input_mask_h = T.TensorType('int32', [False, False])('hypo_mask')
input_mask_h.tag.test_value = numpy.hstack((numpy.ones(
(BSIZE, 18), dtype='int32'), numpy.zeros((BSIZE, 6), dtype='int32')))
input_mask_h.tag.test_value[1, 18:22] = 1
l_mask_h = lasagne.layers.InputLayer(shape=(BSIZE, None),
input_var=input_mask_h)
# premise
input_var_p = T.TensorType('int32', [False, False])('premise_vector')
input_var_p.tag.test_value = numpy.hstack(
(numpy.random.randint(1, 10000, (BSIZE, 16),
'int32'), numpy.zeros(
(BSIZE, 3)).astype('int32')))
l_in_p = lasagne.layers.InputLayer(shape=(BSIZE, None),
input_var=input_var_p)
input_mask_p = T.TensorType('int32', [False, False])('premise_mask')
input_mask_p.tag.test_value = numpy.hstack((numpy.ones(
(BSIZE, 16), dtype='int32'), numpy.zeros((BSIZE, 3), dtype='int32')))
input_mask_p.tag.test_value[1, 16:18] = 1
l_mask_p = lasagne.layers.InputLayer(shape=(BSIZE, None),
input_var=input_mask_p)
###################################
# output shape (BSIZE, None, WEDIM)
l_hypo_embed = lasagne.layers.EmbeddingLayer(
l_in_h,
input_size=W_word_embedding.shape[0],
output_size=W_word_embedding.shape[1],
W=W_word_embedding)
l_prem_embed = lasagne.layers.EmbeddingLayer(
l_in_p,
input_size=W_word_embedding.shape[0],
output_size=W_word_embedding.shape[1],
W=l_hypo_embed.W)
# EMBEDING MAPPING: output shape (BSIZE, None, WEMAP)
l_hypo_reduced_embed = DenseLayer3DInput(l_hypo_embed,
num_units=WEMAP,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=None)
l_hypo_embed_dpout = lasagne.layers.DropoutLayer(l_hypo_reduced_embed,
p=DPOUT,
rescale=True)
l_prem_reduced_embed = DenseLayer3DInput(l_prem_embed,
num_units=WEMAP,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=None)
l_prem_embed_dpout = lasagne.layers.DropoutLayer(l_prem_reduced_embed,
p=DPOUT,
rescale=True)
# ATTEND
l_hypo_embed_hid1 = DenseLayer3DInput(
l_hypo_embed_dpout,
num_units=EMBDHIDA,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=lasagne.nonlinearities.rectify)
l_hypo_embed_hid1_dpout = lasagne.layers.DropoutLayer(l_hypo_embed_hid1,
p=DPOUT,
rescale=True)
l_hypo_embed_hid2 = DenseLayer3DInput(
l_hypo_embed_hid1_dpout,
num_units=EMBDHIDB,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=lasagne.nonlinearities.rectify)
l_prem_embed_hid1 = DenseLayer3DInput(
l_prem_embed_dpout,
num_units=EMBDHIDA,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=lasagne.nonlinearities.rectify)
l_prem_embed_hid1_dpout = lasagne.layers.DropoutLayer(l_prem_embed_hid1,
p=DPOUT,
rescale=True)
l_prem_embed_hid2 = DenseLayer3DInput(
l_prem_embed_hid1_dpout,
num_units=EMBDHIDB,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=lasagne.nonlinearities.rectify)
# output dim: (BSIZE, NROWx, NROWy)
l_e = ComputeEmbeddingPool([l_hypo_embed_hid1, l_prem_embed_hid2])
# output dim: (BSIZE, NROWy, DIM)
l_hypo_weighted = AttendOnEmbedding([l_hypo_reduced_embed, l_e],
masks=[l_mask_h, l_mask_p],
direction='col')
# output dim: (BSIZE, NROWx, DIM)
l_prem_weighted = AttendOnEmbedding([l_prem_reduced_embed, l_e],
masks=[l_mask_h, l_mask_p],
direction='row')
# COMPARE
# output dim: (BSIZE, NROW, 4*LSTMHID)
l_hypo_premwtd = lasagne.layers.ConcatLayer(
[l_hypo_reduced_embed, l_prem_weighted], axis=2)
l_prem_hypowtd = lasagne.layers.ConcatLayer(
[l_prem_reduced_embed, l_hypo_weighted], axis=2)
l_hypo_premwtd_dpout = lasagne.layers.DropoutLayer(l_hypo_premwtd,
p=DPOUT,
rescale=True)
l_hypo_comphid1 = DenseLayer3DInput(
l_hypo_premwtd_dpout,
num_units=COMPHIDA,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=lasagne.nonlinearities.rectify)
l_hypo_comphid1_dpout = lasagne.layers.DropoutLayer(l_hypo_comphid1,
p=DPOUT,
rescale=True)
l_hypo_comphid2 = DenseLayer3DInput(
l_hypo_comphid1_dpout,
num_units=COMPHIDB,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=lasagne.nonlinearities.rectify)
l_prem_hypowtd_dpout = lasagne.layers.DropoutLayer(l_prem_hypowtd,
p=DPOUT,
rescale=True)
l_prem_comphid1 = DenseLayer3DInput(
l_prem_hypowtd_dpout,
num_units=COMPHIDA,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=lasagne.nonlinearities.rectify)
l_prem_comphid1_dpout = lasagne.layers.DropoutLayer(l_prem_comphid1,
p=DPOUT,
rescale=True)
l_prem_comphid2 = DenseLayer3DInput(
l_prem_comphid1_dpout,
num_units=COMPHIDB,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=lasagne.nonlinearities.rectify)
# AGGREGATE
# output dim: (BSIZE, 4*LSTMHID)
l_hypo_mean = MeanOverDim(l_hypo_comphid2, mask=l_mask_h, dim=1)
l_prem_mean = MeanOverDim(l_prem_comphid2, mask=l_mask_p, dim=1)
l_v1v2 = lasagne.layers.ConcatLayer([l_hypo_mean, l_prem_mean], axis=1)
l_v1v2_dpout = lasagne.layers.DropoutLayer(l_v1v2, p=DPOUT, rescale=True)
l_outhid1 = lasagne.layers.DenseLayer(
l_v1v2_dpout,
num_units=OUTHID,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=lasagne.nonlinearities.rectify)
l_outhid1_dpout = lasagne.layers.DropoutLayer(l_outhid1,
p=DPOUT,
rescale=True)
l_outhid2 = lasagne.layers.DenseLayer(
l_outhid1_dpout,
num_units=OUTHID,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=lasagne.nonlinearities.rectify)
# l_outhid2_dpout = lasagne.layers.DropoutLayer(l_outhid2, p=DPOUT, rescale=True)
l_output = lasagne.layers.DenseLayer(
l_outhid2,
num_units=3,
W=init.Normal(),
b=init.Constant(0.),
nonlinearity=lasagne.nonlinearities.softmax)
########### target, cost, validation, etc. ##########
target_values = T.ivector('target_output')
target_values.tag.test_value = numpy.asarray([
1,
] * BSIZE, dtype='int32')
network_output = lasagne.layers.get_output(l_output)
network_prediction = T.argmax(network_output, axis=1)
error_rate = T.mean(T.neq(network_prediction, target_values))
network_output_clean = lasagne.layers.get_output(l_output,
deterministic=True)
network_prediction_clean = T.argmax(network_output_clean, axis=1)
error_rate_clean = T.mean(T.neq(network_prediction_clean, target_values))
cost = T.mean(
T.nnet.categorical_crossentropy(network_output, target_values))
cost_clean = T.mean(
T.nnet.categorical_crossentropy(network_output_clean, target_values))
# Retrieve all parameters from the network
all_params = lasagne.layers.get_all_params(l_output)
if not UPDATEWE:
all_params.remove(l_hypo_embed.W)
numparams = sum(
[numpy.prod(i) for i in [i.shape.eval() for i in all_params]])
print("Number of params: {}\nName\t\t\tShape\t\t\tSize".format(numparams))
print("-----------------------------------------------------------------")
for item in all_params:
print("{0:24}{1:24}{2}".format(item, item.shape.eval(),
numpy.prod(item.shape.eval())))
# if exist param file then load params
look_for = 'params' + os.sep + 'params_' + filename + '.pkl'
if os.path.isfile(look_for):
print("Resuming from file: " + look_for)
all_param_values = cPickle.load(open(look_for, 'rb'))
for p, v in zip(all_params, all_param_values):
p.set_value(v)
# Compute SGD updates for training
print("Computing updates ...")
updates = lasagne.updates.adagrad(cost, all_params, LR)
# Theano functions for training and computing cost
print("Compiling functions ...")
train = theano.function([
l_in_h.input_var, l_mask_h.input_var, l_in_p.input_var,
l_mask_p.input_var, target_values
], [cost, error_rate],
updates=updates)
# mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=False))
compute_cost = theano.function([
l_in_h.input_var, l_mask_h.input_var, l_in_p.input_var,
l_mask_p.input_var, target_values
], [cost_clean, error_rate_clean])
# mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=False))
def evaluate(mode):
if mode == 'dev':
data = dev_batches
if mode == 'test':
data = test_batches
set_cost = 0.
set_error_rate = 0.
for batches_seen, (hypo, hm, premise, pm, truth) in enumerate(data, 1):
_cost, _error = compute_cost(hypo, hm, premise, pm, truth)
set_cost = (1.0 - 1.0 / batches_seen) * set_cost + \
1.0 / batches_seen * _cost
set_error_rate = (1.0 - 1.0 / batches_seen) * set_error_rate + \
1.0 / batches_seen * _error
return set_cost, set_error_rate
print("Done. Evaluating scratch model ...")
dev_set_cost, dev_set_error = evaluate('dev')
print("BEFORE TRAINING: dev cost %f, error %f" %
(dev_set_cost, dev_set_error))
print("Training ...")
try:
for epoch in range(num_epochs):
train_set_cost = 0.
train_set_error = 0.
start = time.time()
for batches_seen, (hypo, hm, premise, pm,
truth) in enumerate(train_batches, 1):
_cost, _error = train(hypo, hm, premise, pm, truth)
train_set_cost = (1.0 - 1.0 / batches_seen) * train_set_cost + \
1.0 / batches_seen * _cost
train_set_error = (1.0 - 1.0 / batches_seen) * train_set_error + \
1.0 / batches_seen * _error
if (batches_seen * BSIZE) % 5000 == 0:
end = time.time()
print("Sample %d %.2fs, lr %.4f, train cost %f, error %f" %
(batches_seen * BSIZE, end - start, LR,
train_set_cost, train_set_error))
start = end
if (batches_seen * BSIZE) % 100000 == 0:
dev_set_cost, dev_set_error = evaluate('dev')
print("***dev cost %f, error %f" %
(dev_set_cost, dev_set_error))
# save parameters
all_param_values = [p.get_value() for p in all_params]
cPickle.dump(
all_param_values,
open('params' + os.sep + 'params_' + filename + '.pkl', 'wb'))
dev_set_cost, dev_set_error = evaluate('dev')
test_set_cost, test_set_error = evaluate('test')
print("epoch %d, cost: train %f dev %f test %f;\n"
" error train %f dev %f test %f" %
(epoch, train_set_cost, dev_set_cost, test_set_cost,
train_set_error, dev_set_error, test_set_error))
except KeyboardInterrupt:
pdb.set_trace()
pass
def main(num_epochs=NEPOCH):
print("Loading data ...")
snli = SNLI(batch_size=BSIZE)
train_batches = list(snli.train_minibatch_generator())
dev_batches = list(snli.dev_minibatch_generator())
test_batches = list(snli.test_minibatch_generator())
W_word_embedding = snli.weight # W shape: (# vocab size, WE_DIM)
del snli
print("Building network ...")
########### sentence embedding encoder ###########
# sentence vector, with each number standing for a word number
input_var = T.TensorType('int32', [False, False])('sentence_vector')
input_var.tag.test_value = numpy.hstack(
(numpy.random.randint(1, 10000, (BSIZE, 20),
'int32'), numpy.zeros(
(BSIZE, 5)).astype('int32')))
input_var.tag.test_value[1, 20:22] = (413, 45)
l_in = lasagne.layers.InputLayer(shape=(BSIZE, None), input_var=input_var)
input_mask = T.TensorType('int32', [False, False])('sentence_mask')
input_mask.tag.test_value = numpy.hstack((numpy.ones(
(BSIZE, 20), dtype='int32'), numpy.zeros((BSIZE, 5), dtype='int32')))
input_mask.tag.test_value[1, 20:22] = 1
l_mask = lasagne.layers.InputLayer(shape=(BSIZE, None),
input_var=input_mask)
# output shape (BSIZE, None, WEDIM)
l_word_embed = lasagne.layers.EmbeddingLayer(
l_in,
input_size=W_word_embedding.shape[0],
output_size=W_word_embedding.shape[1],
W=W_word_embedding)
# bidirectional LSTM
l_forward = lasagne.layers.LSTMLayer(
l_word_embed,
mask_input=l_mask,
num_units=LSTMHID,
ingate=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=init.Normal(STD)),
forgetgate=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=init.Normal(STD)),
cell=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=None,
nonlinearity=nonlinearities.tanh),
outgate=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=init.Normal(STD)),
nonlinearity=lasagne.nonlinearities.tanh,
peepholes=False,
grad_clipping=GCLIP)
l_backward = lasagne.layers.LSTMLayer(
l_word_embed,
mask_input=l_mask,
num_units=LSTMHID,
ingate=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=init.Normal(STD)),
forgetgate=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=init.Normal(STD)),
cell=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=None,
nonlinearity=nonlinearities.tanh),
outgate=Gate(W_in=init.Normal(STD),
W_hid=init.Normal(STD),
W_cell=init.Normal(STD)),
nonlinearity=lasagne.nonlinearities.tanh,
peepholes=False,
grad_clipping=GCLIP,
backwards=True)
# output dim: (BSIZE, None, 2*LSTMHID)
l_concat = lasagne.layers.ConcatLayer([l_forward, l_backward], axis=2)
l_concat_dpout = lasagne.layers.DropoutLayer(
l_concat, p=DPOUT, rescale=True) # might not need this line
# Attention mechanism to get sentence embedding
# output dim: (BSIZE, None, ATTHID)
l_ws1 = DenseLayer3DInput(l_concat_dpout, num_units=ATTHID)
l_ws1_dpout = lasagne.layers.DropoutLayer(l_ws1, p=DPOUT, rescale=True)
# output dim: (BSIZE, None, NROW)
l_ws2 = DenseLayer3DInput(l_ws1_dpout, num_units=NROW, nonlinearity=None)
l_annotations = Softmax3D(l_ws2, mask=l_mask)
# output dim: (BSIZE, 2*LSTMHID, NROW)
l_sentence_embedding = ApplyAttention([l_annotations, l_concat])
# beam search? Bi lstm in the sentence embedding layer? etc.
########### get embeddings for hypothesis and premise ###########
# hypothesis
input_var_h = T.TensorType('int32', [False, False])('hypothesis_vector')
input_var_h.tag.test_value = numpy.hstack(
(numpy.random.randint(1, 10000, (BSIZE, 18),
'int32'), numpy.zeros(
(BSIZE, 6)).astype('int32')))
l_in_h = lasagne.layers.InputLayer(shape=(BSIZE, None),
input_var=input_var_h)
input_mask_h = T.TensorType('int32', [False, False])('hypo_mask')
input_mask_h.tag.test_value = numpy.hstack((numpy.ones(
(BSIZE, 18), dtype='int32'), numpy.zeros((BSIZE, 6), dtype='int32')))
input_mask_h.tag.test_value[1, 18:22] = 1
l_mask_h = lasagne.layers.InputLayer(shape=(BSIZE, None),
input_var=input_mask_h)
# premise
input_var_p = T.TensorType('int32', [False, False])('premise_vector')
input_var_p.tag.test_value = numpy.hstack(
(numpy.random.randint(1, 10000, (BSIZE, 16),
'int32'), numpy.zeros(
(BSIZE, 3)).astype('int32')))
l_in_p = lasagne.layers.InputLayer(shape=(BSIZE, None),
input_var=input_var_p)
input_mask_p = T.TensorType('int32', [False, False])('premise_mask')
input_mask_p.tag.test_value = numpy.hstack((numpy.ones(
(BSIZE, 16), dtype='int32'), numpy.zeros((BSIZE, 3), dtype='int32')))
input_mask_p.tag.test_value[1, 16:18] = 1
l_mask_p = lasagne.layers.InputLayer(shape=(BSIZE, None),
input_var=input_mask_p)
hypothesis_embedding, hypothesis_annotation = lasagne.layers.get_output(
[l_sentence_embedding, l_annotations], {
l_in: l_in_h.input_var,
l_mask: l_mask_h.input_var
})
premise_embedding, premise_annotation = lasagne.layers.get_output(
[l_sentence_embedding, l_annotations], {
l_in: l_in_p.input_var,
l_mask: l_mask_p.input_var
})
hypothesis_embedding_clean, hypothesis_annotation_clean = lasagne.layers.get_output(
[l_sentence_embedding, l_annotations], {
l_in: l_in_h.input_var,
l_mask: l_mask_h.input_var
},
deterministic=True)
premise_embedding_clean, premise_annotation_clean = lasagne.layers.get_output(
[l_sentence_embedding, l_annotations], {
l_in: l_in_p.input_var,
l_mask: l_mask_p.input_var
},
deterministic=True)
########### gated encoder and output MLP ##########
l_hypo_embed = lasagne.layers.InputLayer(shape=(BSIZE, NROW, 2 * LSTMHID),
input_var=hypothesis_embedding)
l_hypo_embed_dpout = lasagne.layers.DropoutLayer(l_hypo_embed,
p=DPOUT,
rescale=True)
l_pre_embed = lasagne.layers.InputLayer(shape=(BSIZE, NROW, 2 * LSTMHID),
input_var=premise_embedding)
l_pre_embed_dpout = lasagne.layers.DropoutLayer(l_pre_embed,
p=DPOUT,
rescale=True)
# output dim: (BSIZE, NROW, 2*LSTMHID)
l_factors = GatedEncoder3D([l_hypo_embed_dpout, l_pre_embed_dpout],
num_hfactors=2 * LSTMHID)
l_factors_dpout = lasagne.layers.DropoutLayer(l_factors,
p=DPOUT,
rescale=True)
# l_hids = DenseLayer3DWeight()
l_outhid = lasagne.layers.DenseLayer(
l_factors_dpout,
num_units=OUTHID,
nonlinearity=lasagne.nonlinearities.rectify)
l_outhid_dpout = lasagne.layers.DropoutLayer(l_outhid,
p=DPOUT,
rescale=True)
l_output = lasagne.layers.DenseLayer(
l_outhid_dpout,
num_units=3,
nonlinearity=lasagne.nonlinearities.softmax)
########### target, cost, validation, etc. ##########
target_values = T.ivector('target_output')
target_values.tag.test_value = numpy.asarray([
1,
] * BSIZE, dtype='int32')
network_output = lasagne.layers.get_output(l_output)
network_prediction = T.argmax(network_output, axis=1)
accuracy = T.mean(T.eq(network_prediction, target_values))
network_output_clean = lasagne.layers.get_output(
l_output, {
l_hypo_embed: hypothesis_embedding_clean,
l_pre_embed: premise_embedding_clean
},
deterministic=True)
network_prediction_clean = T.argmax(network_output_clean, axis=1)
accuracy_clean = T.mean(T.eq(network_prediction_clean, target_values))
# penalty term and cost
attention_penalty = T.mean(
(T.batched_dot(hypothesis_annotation,
hypothesis_annotation.dimshuffle(0, 2, 1)) -
T.eye(hypothesis_annotation.shape[1]).dimshuffle('x', 0, 1))**2,
axis=(0, 1, 2)) + T.mean(
(T.batched_dot(premise_annotation,
premise_annotation.dimshuffle(0, 2, 1)) -
T.eye(premise_annotation.shape[1]).dimshuffle('x', 0, 1))**2,
axis=(0, 1, 2))
L2_lstm = ((l_forward.W_in_to_ingate ** 2).sum() + \
(l_forward.W_hid_to_ingate ** 2).sum() + \
(l_forward.W_in_to_forgetgate ** 2).sum() + \
(l_forward.W_hid_to_forgetgate ** 2).sum() + \
(l_forward.W_in_to_cell ** 2).sum() + \
(l_forward.W_hid_to_cell ** 2).sum() + \
(l_forward.W_in_to_outgate ** 2).sum() + \
(l_forward.W_hid_to_outgate ** 2).sum() + \
(l_backward.W_in_to_ingate ** 2).sum() + \
(l_backward.W_hid_to_ingate ** 2).sum() + \
(l_backward.W_in_to_forgetgate ** 2).sum() + \
(l_backward.W_hid_to_forgetgate ** 2).sum() + \
(l_backward.W_in_to_cell ** 2).sum() + \
(l_backward.W_hid_to_cell ** 2).sum() + \
(l_backward.W_in_to_outgate ** 2).sum() + \
(l_backward.W_hid_to_outgate ** 2).sum())
L2_attention = (l_ws1.W**2).sum() + (l_ws2.W**2).sum()
L2_gae = (l_factors.Wxf**2).sum() + (l_factors.Wyf**2).sum()
L2_outputhid = (l_outhid.W**2).sum()
L2_softmax = (l_output.W**2).sum()
L2 = L2_lstm + L2_attention + L2_gae + L2_outputhid + L2_softmax
cost = T.mean(T.nnet.categorical_crossentropy(network_output, target_values)) + \
L2REG * L2
cost_clean = T.mean(T.nnet.categorical_crossentropy(network_output_clean, target_values)) + \
L2REG * L2
if ATTPENALTY != 0.:
cost = cost + ATTPENALTY * attention_penalty
cost_clean = cost_clean + ATTPENALTY * attention_penalty
# Retrieve all parameters from the network
all_params = lasagne.layers.get_all_params(l_output) + \
lasagne.layers.get_all_params(l_sentence_embedding)
if not UPDATEWE:
all_params.remove(l_word_embed.W)
numparams = sum(
[numpy.prod(i) for i in [i.shape.eval() for i in all_params]])
print("Number of params: {}\nName\t\t\tShape\t\t\tSize".format(numparams))
print("-----------------------------------------------------------------")
for item in all_params:
print("{0:24}{1:24}{2}".format(item, item.shape.eval(),
numpy.prod(item.shape.eval())))
# if exist param file then load params
look_for = 'params' + os.sep + 'params_' + filename + '.pkl'
if os.path.isfile(look_for):
print("Resuming from file: " + look_for)
all_param_values = cPickle.load(open(look_for, 'rb'))
for p, v in zip(all_params, all_param_values):
p.set_value(v)
# Compute SGD updates for training
print("Computing updates ...")
updates = lasagne.updates.adagrad(cost, all_params, LR)
# Theano functions for training and computing cost
print("Compiling functions ...")
train = theano.function([
l_in_h.input_var, l_mask_h.input_var, l_in_p.input_var,
l_mask_p.input_var, target_values
], [cost, accuracy],
updates=updates)
compute_cost = theano.function([
l_in_h.input_var, l_mask_h.input_var, l_in_p.input_var,
l_mask_p.input_var, target_values
], [cost_clean, accuracy_clean])
predict = theano.function([
l_in_h.input_var, l_mask_h.input_var, l_in_p.input_var,
l_mask_p.input_var
], network_prediction_clean)
def evaluate(mode, verbose=False):
if mode == 'dev':
data = dev_batches
if mode == 'test':
data = test_batches
set_cost = 0.
set_accuracy = 0.
for batches_seen, (hypo, hm, premise, pm, truth) in enumerate(data, 1):
_cost, _accuracy = compute_cost(hypo, hm, premise, pm, truth)
set_cost = (1.0 - 1.0 / batches_seen) * set_cost + \
1.0 / batches_seen * _cost
set_accuracy = (1.0 - 1.0 / batches_seen) * set_accuracy + \
1.0 / batches_seen * _accuracy
if verbose == True:
predicted = []
truth = []
for batches_seen, (hypo, hm, premise, pm,
th) in enumerate(data, 1):
predicted.append(predict(hypo, hm, premise, pm))
truth.append(th)
truth = numpy.concatenate(truth)
predicted = numpy.concatenate(predicted)
cm = confusion_matrix(truth, predicted)
pr_a = cm.trace() * 1.0 / truth.size
pr_e = ((cm.sum(axis=0)*1.0/truth.size) * \
(cm.sum(axis=1)*1.0/truth.size)).sum()
k = (pr_a - pr_e) / (1 - pr_e)
print(mode + " set statistics:")
print("kappa index of agreement: %f" % k)
print("confusion matrix:")
print(cm)
return set_cost, set_accuracy
print("Done. Evaluating scratch model ...")
test_set_cost, test_set_accuracy = evaluate('test', verbose=True)
print("BEFORE TRAINING: dev cost %f, accuracy %f" %
(test_set_cost, test_set_accuracy))
print("Training ...")
try:
for epoch in range(num_epochs):
train_set_cost = 0.
train_set_accuracy = 0.
start = time.time()
for batches_seen, (hypo, hm, premise, pm,
truth) in enumerate(train_batches, 1):
_cost, _accuracy = train(hypo, hm, premise, pm, truth)
train_set_cost = (1.0 - 1.0 / batches_seen) * train_set_cost + \
1.0 / batches_seen * _cost
train_set_accuracy = (1.0 - 1.0 / batches_seen) * train_set_accuracy + \
1.0 / batches_seen * _accuracy
if batches_seen % 100 == 0:
end = time.time()
print(
"Sample %d %.2fs, lr %.4f, train cost %f, accuracy %f"
% (batches_seen * BSIZE, end - start, LR,
train_set_cost, train_set_accuracy))
start = end
if batches_seen % 2000 == 0:
dev_set_cost, dev_set_accuracy = evaluate('dev')
print("***dev cost %f, accuracy %f" %
(dev_set_cost, dev_set_accuracy))
# save parameters
all_param_values = [p.get_value() for p in all_params]
cPickle.dump(
all_param_values,
open('params' + os.sep + 'params_' + filename + '.pkl', 'wb'))
dev_set_cost, dev_set_accuracy = evaluate('dev')
test_set_cost, test_set_accuracy = evaluate('test', verbose=True)
print("epoch %d, cost: train %f dev %f test %f;\n"
" accu: train %f dev %f test %f" %
(epoch, train_set_cost, dev_set_cost, test_set_cost,
train_set_accuracy, dev_set_accuracy, test_set_accuracy))
except KeyboardInterrupt:
pdb.set_trace()
pass