def build_model(args):
x = tensor.tensor3('features', dtype=floatX)
y = tensor.tensor3('targets', dtype=floatX)
linear = Linear(input_dim=1, output_dim=4 * args.units)
rnn = LSTM(dim=args.units, activation=Tanh())
linear2 = Linear(input_dim=args.units, output_dim=1)
prediction = Tanh().apply(linear2.apply(rnn.apply(linear.apply(x))))
prediction = prediction[:-1, :, :]
# SquaredError does not work on 3D tensor
y = y.reshape((y.shape[0] * y.shape[1], y.shape[2]))
prediction = prediction.reshape((prediction.shape[0] * prediction.shape[1],
prediction.shape[2]))
cost = SquaredError().apply(y, prediction)
# Initialization
linear.weights_init = IsotropicGaussian(0.1)
linear2.weights_init = IsotropicGaussian(0.1)
linear.biases_init = Constant(0)
linear2.biases_init = Constant(0)
rnn.weights_init = Orthogonal()
return cost
def __init__(self, config, vocab_size, id_to_vocab, logger):
question = tensor.imatrix('question')
question_mask = tensor.imatrix('question_mask')
context = tensor.imatrix('context')
context_mask = tensor.imatrix('context_mask')
answer = tensor.ivector('answer')
candidates = tensor.imatrix('candidates')
candidates_mask = tensor.imatrix('candidates_mask')
# question_actual = tensor.imatrix('question_actual')
# context_actual = tensor.imatrix('context_actual')
# answer_actual = tensor.imatrix('answer_actual')
bricks = []
question = question.dimshuffle(1, 0)
question_mask = question_mask.dimshuffle(1, 0)
context = context.dimshuffle(1, 0)
context_mask = context_mask.dimshuffle(1, 0)
# Embed questions and cntext
embed = LookupTable(vocab_size, config.embed_size, name='question_embed')
bricks.append(embed)
qembed = embed.apply(question)
cembed = embed.apply(context)
qlstms, qhidden_list = make_bidir_lstm_stack(qembed, config.embed_size, question_mask.astype(theano.config.floatX),
config.question_lstm_size, config.question_skip_connections, 'q')
clstms, chidden_list = make_bidir_lstm_stack(cembed, config.embed_size, context_mask.astype(theano.config.floatX),
config.ctx_lstm_size, config.ctx_skip_connections, 'ctx')
bricks = bricks + qlstms + clstms
# Calculate question encoding (concatenate layer1)
if config.question_skip_connections:
qenc_dim = 2*sum(config.question_lstm_size)
qenc = tensor.concatenate([h[-1,:,:] for h in qhidden_list], axis=1)
else:
qenc_dim = 2*config.question_lstm_size[-1]
#u
qenc = tensor.concatenate([h[-1,:,:] for h in qhidden_list[-2:]], axis=1)
qenc.name = 'qenc'
# Calculate context encoding (concatenate layer1)
if config.ctx_skip_connections:
cenc_dim = 2*sum(config.ctx_lstm_size)
cenc = tensor.concatenate(chidden_list, axis=2)
else:
cenc_dim = 2*config.ctx_lstm_size[-1]
cenc = tensor.concatenate(chidden_list[-2:], axis=2)
cenc.name = 'cenc'
# Attention mechanism MLP
attention_mlp = MLP(dims=config.attention_mlp_hidden + [1],
activations=config.attention_mlp_activations[1:] + [Identity()],
name='attention_mlp')
attention_qlinear = Linear(input_dim=qenc_dim, output_dim=config.attention_mlp_hidden[0], name='attq')
attention_clinear = Linear(input_dim=cenc_dim, output_dim=config.attention_mlp_hidden[0], use_bias=False, name='attc')
bricks += [attention_mlp, attention_qlinear, attention_clinear]
layer1 = Tanh().apply(attention_clinear.apply(cenc.reshape((cenc.shape[0]*cenc.shape[1], cenc.shape[2])))
.reshape((cenc.shape[0],cenc.shape[1],config.attention_mlp_hidden[0]))
+ attention_qlinear.apply(qenc)[None, :, :])
layer1.name = 'layer1'
att_weights = attention_mlp.apply(layer1.reshape((layer1.shape[0]*layer1.shape[1], layer1.shape[2])))
att_weights.name = 'att_weights_0'
att_weights = att_weights.reshape((layer1.shape[0], layer1.shape[1]))
att_weights.name = 'att_weights'
#r
attended = tensor.sum(cenc * tensor.nnet.softmax(att_weights.T).T[:, :, None], axis=0)
attended.name = 'attended'
# Now we can calculate our output
out_mlp = MLP(dims=[cenc_dim + qenc_dim] + config.out_mlp_hidden + [config.n_entities],
activations=config.out_mlp_activations + [Identity()],
name='out_mlp')
bricks += [out_mlp]
# g^AR
probs = out_mlp.apply(tensor.concatenate([attended, qenc], axis=1))
probs.name = 'probs'
is_candidate = tensor.eq(tensor.arange(config.n_entities, dtype='int32')[None, None, :],
tensor.switch(candidates_mask, candidates, -tensor.ones_like(candidates))[:, :, None]).sum(axis=1)
probs = tensor.switch(is_candidate, probs, -1000 * tensor.ones_like(probs))
# Calculate prediction, cost and error rate
pred = probs.argmax(axis=1)
cost = Softmax().categorical_cross_entropy(answer, probs).mean()
error_rate = tensor.neq(answer, pred).mean()
# Apply dropout
cg = ComputationGraph([cost, error_rate])
if config.w_noise > 0:
noise_vars = VariableFilter(roles=[WEIGHT])(cg)
cg = apply_noise(cg, noise_vars, config.w_noise)
if config.dropout > 0:
cg = apply_dropout(cg, qhidden_list + chidden_list, config.dropout)
[cost_reg, error_rate_reg] = cg.outputs
# Other stuff
cost_reg.name = cost.name = 'cost'
error_rate_reg.name = error_rate.name = 'error_rate'
self.sgd_cost = cost_reg
self.monitor_vars = [[cost_reg], [error_rate_reg]]
self.monitor_vars_valid = [[cost], [error_rate]]
# Initialize bricks
for brick in bricks:
brick.weights_init = config.weights_init
brick.biases_init = config.biases_init
brick.initialize()
def __init__(self, config, vocab_size):
question = tensor.imatrix('question')
question_mask = tensor.imatrix('question_mask')
context = tensor.imatrix('context')
context_mask = tensor.imatrix('context_mask')
answer = tensor.imatrix('answer')
answer_mask = tensor.imatrix('answer_mask')
bricks = []
question = question.dimshuffle(1, 0)
question_mask = question_mask.dimshuffle(1, 0)
context = context.dimshuffle(1, 0)
context_mask = context_mask.dimshuffle(1, 0)
answer = answer.dimshuffle(1, 0)
answer_mask = answer_mask.dimshuffle(1, 0)
# Embed questions and context
embed = LookupTable(vocab_size,
config.embed_size,
name='question_embed')
embed.weights_init = IsotropicGaussian(0.01)
# Calculate question encoding (concatenate layer1)
qembed = embed.apply(question)
qlstms, qhidden_list = make_bidir_lstm_stack(
qembed, config.embed_size,
question_mask.astype(theano.config.floatX),
config.question_lstm_size, config.question_skip_connections, 'q')
bricks = bricks + qlstms
if config.question_skip_connections:
qenc_dim = 2 * sum(config.question_lstm_size)
qenc = tensor.concatenate([h[-1, :, :] for h in qhidden_list],
axis=1)
else:
qenc_dim = 2 * config.question_lstm_size[-1]
qenc = tensor.concatenate([h[-1, :, :] for h in qhidden_list[-2:]],
axis=1)
qenc.name = 'qenc'
# Calculate context encoding (concatenate layer1)
cembed = embed.apply(context)
clstms, chidden_list = make_bidir_lstm_stack(
cembed, config.embed_size,
context_mask.astype(theano.config.floatX), config.ctx_lstm_size,
config.ctx_skip_connections, 'ctx')
bricks = bricks + clstms
if config.ctx_skip_connections:
cenc_dim = 2 * sum(config.ctx_lstm_size) #2 : fw & bw
cenc = tensor.concatenate(chidden_list, axis=2)
else:
cenc_dim = 2 * config.question_lstm_size[-1]
cenc = tensor.concatenate(chidden_list[-2:], axis=2)
cenc.name = 'cenc'
# Attention mechanism MLP
attention_mlp = MLP(dims=config.attention_mlp_hidden + [1],
activations=config.attention_mlp_activations[1:] +
[Identity()],
name='attention_mlp')
attention_qlinear = Linear(input_dim=qenc_dim,
output_dim=config.attention_mlp_hidden[0],
name='attq')
attention_clinear = Linear(input_dim=cenc_dim,
output_dim=config.attention_mlp_hidden[0],
use_bias=False,
name='attc')
bricks += [attention_mlp, attention_qlinear, attention_clinear]
layer1 = Tanh().apply(
attention_clinear.apply(
cenc.reshape((cenc.shape[0] * cenc.shape[1], cenc.shape[2]
))).reshape((cenc.shape[0], cenc.shape[1],
config.attention_mlp_hidden[0])) +
attention_qlinear.apply(qenc)[None, :, :])
layer1.name = 'layer1'
att_weights = attention_mlp.apply(
layer1.reshape(
(layer1.shape[0] * layer1.shape[1], layer1.shape[2])))
att_weights = att_weights.reshape((layer1.shape[0], layer1.shape[1]))
att_weights = tensor.nnet.sigmoid(att_weights.T).T
att_weights.name = 'att_weights'
att_target = tensor.eq(
tensor.tile(answer[None, :, :], (context.shape[0], 1, 1)),
tensor.tile(context[:, None, :],
(1, answer.shape[0], 1))).sum(axis=1).clip(0, 1)
cost = (tensor.nnet.binary_crossentropy(att_weights, att_target) *
context_mask).sum() / context_mask.sum()
self.predictions = tensor.gt(att_weights, 0.1) * context
# Apply dropout
cg = ComputationGraph([cost])
if config.w_noise > 0:
noise_vars = VariableFilter(roles=[WEIGHT])(cg)
cg = apply_noise(cg, noise_vars, config.w_noise)
if config.dropout > 0:
cg = apply_dropout(cg, qhidden_list + chidden_list, config.dropout)
[cost_reg] = cg.outputs
# Other stuff
cost.name = 'cost'
cost_reg.name = 'cost_reg'
self.sgd_cost = cost_reg
self.monitor_vars = [[cost_reg]]
self.monitor_vars_valid = [[cost_reg]]
# Initialize bricks
embed.initialize()
for brick in bricks:
brick.weights_init = config.weights_init
brick.biases_init = config.biases_init
brick.initialize()
name="conv2")
maxpool2 = MaxPooling(pooling_size=(2, 1), name='maxpool2')
data2 = conv2.apply(data1)
# cut borders
d1 = (data2.shape[2] - data1.shape[2])/2
data2 = data2[:, :, d1:d1+data1.shape[2], :]
# max pool
data2 = maxpool2.apply(data2)
# activation
data2 = Tanh(name='act_data2').apply(data2)
# fully connected layers
fc = MLP(dims=[25*50, 100, 100, num_output_classes],
activations=[Rectifier(name='r1'), Rectifier(name='r2'), Identity()])
output = fc.apply(data2.reshape((data2.shape[0], 25*50)))
# COST AND ERROR MEASURE
cost = Softmax().categorical_cross_entropy(label, output).mean()
cost.name = 'cost'
error_rate = tensor.neq(tensor.argmax(output, axis=1), label).mean()
error_rate.name = 'error_rate'
# REGULARIZATION
cg = ComputationGraph([cost, error_rate])
if weight_noise > 0:
noise_vars = VariableFilter(roles=[WEIGHT])(cg)
cg = apply_noise(cg, noise_vars, weight_noise)
def __init__(self, config, vocab_size):
question = tensor.imatrix('question')
question_mask = tensor.imatrix('question_mask')
context = tensor.imatrix('context')
context_mask = tensor.imatrix('context_mask')
answer = tensor.imatrix('answer')
answer_mask = tensor.imatrix('answer_mask')
bricks = []
question = question.dimshuffle(1, 0)
question_mask = question_mask.dimshuffle(1, 0)
context = context.dimshuffle(1, 0)
context_mask = context_mask.dimshuffle(1, 0)
answer = answer.dimshuffle(1, 0)
answer_mask = answer_mask.dimshuffle(1, 0)
# Embed questions and context
embed = LookupTable(vocab_size,
config.embed_size,
name='question_embed')
embed.weights_init = IsotropicGaussian(0.01)
# Calculate question encoding (concatenate layer1)
qembed = embed.apply(question)
qlstms, qhidden_list = make_bidir_lstm_stack(
qembed, config.embed_size,
question_mask.astype(theano.config.floatX),
config.question_lstm_size, config.question_skip_connections, 'q')
bricks = bricks + qlstms
if config.question_skip_connections:
qenc_dim = 2 * sum(config.question_lstm_size)
qenc = tensor.concatenate([h[-1, :, :] for h in qhidden_list],
axis=1)
else:
qenc_dim = 2 * config.question_lstm_size[-1]
qenc = tensor.concatenate([h[-1, :, :] for h in qhidden_list[-2:]],
axis=1)
qenc.name = 'qenc'
# Calculate context encoding (concatenate layer1)
cembed = embed.apply(context)
clstms, chidden_list = make_bidir_lstm_stack(
cembed, config.embed_size,
context_mask.astype(theano.config.floatX), config.ctx_lstm_size,
config.ctx_skip_connections, 'ctx')
bricks = bricks + clstms
if config.ctx_skip_connections:
cenc_dim = 2 * sum(config.ctx_lstm_size) #2 : fw & bw
cenc = tensor.concatenate(chidden_list, axis=2)
else:
cenc_dim = 2 * config.question_lstm_size[-1]
cenc = tensor.concatenate(chidden_list[-2:], axis=2)
cenc.name = 'cenc'
# Attention mechanism MLP fwd
attention_mlp_fwd = MLP(
dims=config.attention_mlp_hidden + [1],
activations=config.attention_mlp_activations[1:] + [Identity()],
name='attention_mlp_fwd')
attention_qlinear_fwd = Linear(
input_dim=qenc_dim,
output_dim=config.attention_mlp_hidden[0],
name='attq_fwd')
attention_clinear_fwd = Linear(
input_dim=cenc_dim / 2,
output_dim=config.attention_mlp_hidden[0],
use_bias=False,
name='attc_fwd')
bricks += [
attention_mlp_fwd, attention_qlinear_fwd, attention_clinear_fwd
]
layer1_fwd = Tanh(name='tanh_fwd')
layer1_fwd = layer1_fwd.apply(
attention_clinear_fwd.apply(cenc[:, :, :cenc_dim / 2].reshape(
(cenc.shape[0] * cenc.shape[1], cenc.shape[2] /
2))).reshape((cenc.shape[0], cenc.shape[1],
config.attention_mlp_hidden[0])) +
attention_qlinear_fwd.apply(qenc)[None, :, :])
att_weights_fwd = attention_mlp_fwd.apply(
layer1_fwd.reshape((layer1_fwd.shape[0] * layer1_fwd.shape[1],
layer1_fwd.shape[2])))
att_weights_fwd = att_weights_fwd.reshape(
(layer1_fwd.shape[0], layer1_fwd.shape[1]))
att_weights_fwd = tensor.nnet.softmax(att_weights_fwd.T)
att_weights_fwd.name = 'att_weights_fwd'
attended_fwd = tensor.sum(cenc[:, :, :cenc_dim / 2] *
att_weights_fwd.T[:, :, None],
axis=0)
attended_fwd.name = 'attended_fwd'
# Attention mechanism MLP bwd
attention_mlp_bwd = MLP(
dims=config.attention_mlp_hidden + [1],
activations=config.attention_mlp_activations[1:] + [Identity()],
name='attention_mlp_bwd')
attention_qlinear_bwd = Linear(
input_dim=qenc_dim,
output_dim=config.attention_mlp_hidden[0],
name='attq_bwd')
attention_clinear_bwd = Linear(
input_dim=cenc_dim / 2,
output_dim=config.attention_mlp_hidden[0],
use_bias=False,
name='attc_bwd')
bricks += [
attention_mlp_bwd, attention_qlinear_bwd, attention_clinear_bwd
]
layer1_bwd = Tanh(name='tanh_bwd')
layer1_bwd = layer1_bwd.apply(
attention_clinear_bwd.apply(cenc[:, :, cenc_dim / 2:].reshape(
(cenc.shape[0] * cenc.shape[1], cenc.shape[2] /
2))).reshape((cenc.shape[0], cenc.shape[1],
config.attention_mlp_hidden[0])) +
attention_qlinear_bwd.apply(qenc)[None, :, :])
att_weights_bwd = attention_mlp_bwd.apply(
layer1_bwd.reshape((layer1_bwd.shape[0] * layer1_bwd.shape[1],
layer1_bwd.shape[2])))
att_weights_bwd = att_weights_bwd.reshape(
(layer1_bwd.shape[0], layer1_bwd.shape[1]))
att_weights_bwd = tensor.nnet.softmax(att_weights_bwd.T)
att_weights_bwd.name = 'att_weights_bwd'
attended_bwd = tensor.sum(cenc[:, :, cenc_dim / 2:] *
att_weights_bwd.T[:, :, None],
axis=0)
attended_bwd.name = 'attended_bwd'
ctx_question = tensor.concatenate([attended_fwd, attended_bwd, qenc],
axis=1)
ctx_question.name = 'ctx_question'
answer_bag = to_bag(answer, vocab_size)
answer_bag = tensor.set_subtensor(answer_bag[:, 0:3], 0)
relevant_items = answer_bag.sum(axis=1, dtype=theano.config.floatX)
def createSequences(j, index, c_enc, c_enc_dim, c_context,
c_window_size):
sequence = tensor.concatenate([
c_context[j:j + index, :],
tensor.zeros((c_window_size - index, c_context.shape[1]))
],
axis=0)
enc = tensor.concatenate([
c_enc[j + index - 1, :, :], c_enc[j, :, :-1],
tensor.tile(c_window_size[None, None], (c_enc.shape[1], 1))
],
axis=1)
return enc, sequence
def createTargetValues(j, index, c_context, c_vocab_size):
sequence_bag = to_bag(c_context[j:j + index, :], c_vocab_size)
sequence_bag = tensor.set_subtensor(sequence_bag[:, 0:3], 0)
selected_items = sequence_bag.sum(axis=1,
dtype=theano.config.floatX)
tp = (sequence_bag * answer_bag).sum(axis=1,
dtype=theano.config.floatX)
precision = tp / (selected_items + 0.00001)
recall = tp / (relevant_items + 0.00001)
#precision = tensor.set_subtensor(precision[tensor.isnan(precision)], 0.0)
#recall = tensor.set_subtensor(recall[tensor.isnan(recall)], 1.0)
macroF1 = (2 *
(precision * recall)) / (precision + recall + 0.00001)
#macroF1 = tensor.set_subtensor(macroF1[tensor.isnan(macroF1)], 0.0)
return macroF1
window_size = 3
senc = []
sequences = []
pred_targets = []
for i in range(1, window_size + 1):
(all_enc, all_sequence), _ = theano.scan(
fn=createSequences,
sequences=tensor.arange(cenc.shape[0] - i + 1),
non_sequences=[i, cenc, cenc_dim, context, window_size])
(all_macroF1), _ = theano.scan(
fn=createTargetValues,
sequences=tensor.arange(cenc.shape[0] - i + 1),
non_sequences=[i, context, vocab_size])
senc.append(all_enc)
sequences.append(all_sequence)
pred_targets.append(all_macroF1)
senc = tensor.concatenate(senc, axis=0)
sequences = tensor.concatenate(sequences, axis=0)
pred_targets = tensor.concatenate(pred_targets, axis=0)
# F1 prediction Bilinear
prediction_linear = Linear(input_dim=2 * cenc_dim,
output_dim=cenc_dim + qenc_dim,
name='pred_linear')
bricks += [prediction_linear]
pred_weights = ctx_question[None, :, :] * prediction_linear.apply(
senc.reshape(
(senc.shape[0] * senc.shape[1], senc.shape[2]))).reshape(
(senc.shape[0], senc.shape[1], senc.shape[2]))
pred_weights = pred_weights.sum(axis=2)
pred_weights = tensor.nnet.sigmoid(pred_weights.T).T
pred_weights.name = 'pred_weights'
pred_targets = pred_targets / (pred_targets.sum(axis=0) + 0.00001)
pred_weights = pred_weights / (pred_weights.sum(axis=0) + 0.00001)
#numpy.set_printoptions(edgeitems=500)
#pred_targets = theano.printing.Print('pred_targets')(pred_targets)
#pred_weights = theano.printing.Print('pred_weights')(pred_weights)
cost = tensor.nnet.binary_crossentropy(pred_weights,
pred_targets).mean()
self.predictions = sequences[pred_weights.argmax(axis=0), :,
tensor.arange(sequences.shape[2])].T
# Apply dropout
cg = ComputationGraph([cost])
if config.w_noise > 0:
noise_vars = VariableFilter(roles=[WEIGHT])(cg)
cg = apply_noise(cg, noise_vars, config.w_noise)
if config.dropout > 0:
cg = apply_dropout(cg, qhidden_list + chidden_list, config.dropout)
[cost_reg] = cg.outputs
# Other stuff
cost.name = 'cost'
cost_reg.name = 'cost_reg'
self.sgd_cost = cost_reg
self.monitor_vars = [[cost_reg]]
self.monitor_vars_valid = [[cost_reg]]
# Initialize bricks
embed.initialize()
for brick in bricks:
brick.weights_init = config.weights_init
brick.biases_init = config.biases_init
brick.initialize()
def __init__(self, config, vocab_size):
question = tensor.imatrix('question')
# set up 32-bit integer matrices
question_mask = tensor.imatrix('question_mask')
context = tensor.imatrix('context')
context_mask = tensor.imatrix('context_mask')
answer = tensor.ivector('answer')
candidates = tensor.imatrix('candidates')
candidates_mask = tensor.imatrix('candidates_mask')
# and the multple choice answers:
ans1 = tensor.ivector('ans1')
ans1_mask = tensor.ivector('ans1_mask')
ans2 = tensor.ivector('ans2')
ans2_mask = tensor.ivector('ans2_mask')
ans3 = tensor.ivector('ans3')
ans3_mask = tensor.ivector('ans3_mask')
ans4 = tensor.ivector('ans4')
ans4_mask = tensor.ivector('ans4_mask')
bricks = []
# inverts 1st and 2nd dimensions of matrix
question = question.dimshuffle(1, 0)
question_mask = question_mask.dimshuffle(1, 0)
context = context.dimshuffle(1, 0)
context_mask = context_mask.dimshuffle(1, 0)
# Embed questions and cntext
embed = LookupTable(vocab_size, config.embed_size, name='question_embed')
bricks.append(embed)
qembed = embed.apply(question)
cembed = embed.apply(context)
a1embed = embed.apply(ans1)
a2embed = embed.apply(ans2)
a3embed = embed.apply(ans3)
a4embed = embed.apply(ans4)
qlstms, qhidden_list = make_bidir_lstm_stack(qembed, config.embed_size, question_mask.astype(theano.config.floatX),
config.question_lstm_size, config.question_skip_connections, 'q')
clstms, chidden_list = make_bidir_lstm_stack(cembed, config.embed_size, context_mask.astype(theano.config.floatX),
config.ctx_lstm_size, config.ctx_skip_connections, 'ctx')
bricks = bricks + qlstms + clstms
# Calculate question encoding (concatenate layer1)
if config.question_skip_connections:
qenc_dim = 2*sum(config.question_lstm_size)
qenc = tensor.concatenate([h[-1,:,:] for h in qhidden_list], axis=1)
else:
qenc_dim = 2*config.question_lstm_size[-1]
qenc = tensor.concatenate([h[-1,:,:] for h in qhidden_list[-2:]], axis=1)
qenc.name = 'qenc'
# Calculate context encoding (concatenate layer1)
if config.ctx_skip_connections:
cenc_dim = 2*sum(config.ctx_lstm_size)
cenc = tensor.concatenate(chidden_list, axis=2)
else:
cenc_dim = 2*config.ctx_lstm_size[-1]
cenc = tensor.concatenate(chidden_list[-2:], axis=2)
cenc.name = 'cenc'
# Attention mechanism MLP
attention_mlp = MLP(dims=config.attention_mlp_hidden + [1],
activations=config.attention_mlp_activations[1:] + [Identity()],
name='attention_mlp')
attention_qlinear = Linear(input_dim=qenc_dim, output_dim=config.attention_mlp_hidden[0], name='attq')
attention_clinear = Linear(input_dim=cenc_dim, output_dim=config.attention_mlp_hidden[0], use_bias=False, name='attc')
bricks += [attention_mlp, attention_qlinear, attention_clinear]
layer1 = Tanh().apply(attention_clinear.apply(cenc.reshape((cenc.shape[0]*cenc.shape[1], cenc.shape[2])))
.reshape((cenc.shape[0],cenc.shape[1],config.attention_mlp_hidden[0]))
+ attention_qlinear.apply(qenc)[None, :, :])
layer1.name = 'layer1'
att_weights = attention_mlp.apply(layer1.reshape((layer1.shape[0]*layer1.shape[1], layer1.shape[2])))
att_weights.name = 'att_weights_0'
att_weights = att_weights.reshape((layer1.shape[0], layer1.shape[1]))
att_weights.name = 'att_weights'
attended = tensor.sum(cenc * tensor.nnet.softmax(att_weights.T).T[:, :, None], axis=0)
attended.name = 'attended'
# Now we can calculate our output
out_mlp = MLP(dims=[cenc_dim + qenc_dim] + config.out_mlp_hidden + [config.n_entities],
activations=config.out_mlp_activations + [Identity()],
name='out_mlp')
bricks += [out_mlp]
probs = out_mlp.apply(tensor.concatenate([attended, qenc], axis=1))
probs.name = 'probs'
# not needed anymore, since we're not only looking at entities
# is_candidate = tensor.eq(tensor.arange(config.n_entities, dtype='int32')[None, None, :],
# tensor.switch(candidates_mask, candidates, -tensor.ones_like(candidates))[:, :, None]).sum(axis=1)
# probs = tensor.switch(is_candidate, probs, -1000 * tensor.ones_like(probs))
# Calculate prediction, cost and error rate
# vocab = tensor.arange(10)
# probs = numpy.asarray([0, 0.8, 0, 0.2], dtype=numpy.float32)
# context = numpy.asarray([3, 2, 8, 1], dtype=numpy.int32)
# ans3 = numpy.asarray([2, 8, 1], dtype=numpy.int32)
# ans1 = numpy.asarray([1, 3, 4], dtype=numpy.int32)
# ans2 = numpy.asarray([1, 1, 4], dtype=numpy.int32)
# convert probs vector to one that's the same size as vocab, with all zeros except probs:
# probs = tensor.switch(is_candidate, probs, -1000 * tensor.ones_like(probs))
probsPadded = tensor.zeros_like(vocab_size, dtype=numpy.float32)
probsSubset = probsPadded[cembed] #TODO this should be masked
b = tensor.set_subtensor(probsSubset, probs)
# get the similarity score of each (masked) answer with the context probs:
ans1probs = b[a1enc]
ans1score = tensor.switch(ans1_mask, ans1probs, tensor.zeros_like(ans1probs)).sum()
ans2probs = b[a2enc]
ans2score = ans2probs.sum()
ans3probs = b[a3enc]
ans3score = ans3probs.sum()
ans4probs = b[a4enc]
ans4score = ans4probs.sum()
# and pick the best one:
allans = tensor.stacklists([ans1score, ans2score, ans3score, ans4score])
pred = tensor.argmax(allans)
cg = ComputationGraph([ans1probs, ans1score, ans2probs, ans2score, ans3probs, ans3score, ans4probs, ans4score, allans, pred])
f = cg.get_theano_function()
out = f()
#pred = probs.argmax(axis=1)
#print "pred"
#print pred TODO CHANGE THIS!
cost = Softmax().categorical_cross_entropy(answer, probs).mean()
error_rate = tensor.neq(answer, pred).mean()
# Apply dropout
cg = ComputationGraph([cost, error_rate])
if config.w_noise > 0:
noise_vars = VariableFilter(roles=[WEIGHT])(cg)
cg = apply_noise(cg, noise_vars, config.w_noise)
if config.dropout > 0:
cg = apply_dropout(cg, qhidden_list + chidden_list, config.dropout)
[cost_reg, error_rate_reg] = cg.outputs
# Other stuff
cost_reg.name = cost.name = 'cost'
error_rate_reg.name = error_rate.name = 'error_rate'
self.probs = probs
self.probs.name = "probs"
self.cost = cost
self.cost.name = "cost"
#
self.sgd_cost = cost_reg
self.monitor_vars = [[cost_reg], [error_rate_reg]]
self.monitor_vars_valid = [[cost], [error_rate]]
# Initialize bricks
for brick in bricks:
brick.weights_init = config.weights_init
brick.biases_init = config.biases_init
brick.initialize()
def get_prediction_function():
question = tensor.imatrix('question')
question_mask = tensor.imatrix('question_mask')
context = tensor.imatrix('context')
context_mask = tensor.imatrix('context_mask')
answer = tensor.ivector('answer')
candidates = tensor.imatrix('candidates')
candidates_mask = tensor.imatrix('candidates_mask')
"""
question = question.dimshuffle(1, 0)
question_mask = question_mask.dimshuffle(1, 0)
context = context.dimshuffle(1, 0)
context_mask = context_mask.dimshuffle(1, 0)
"""
# Embed questions and cntext
embed = bricks[-5]
qembed = embed.apply(question.dimshuffle(1, 0))
cembed = embed.apply(context.dimshuffle(1, 0))
global _qembed,_cembed
_qembed = theano.function([question], qembed)
_cembed = theano.function([context], cembed)
qhidden_list = make_bidir_lstm_stack(qembed, config.embed_size, question_mask.dimshuffle(1, 0).astype(theano.config.floatX),
config.question_lstm_size, config.question_skip_connections, 'q')
chidden_list = make_bidir_lstm_stack(cembed, config.embed_size, context_mask.dimshuffle(1, 0).astype(theano.config.floatX),
config.ctx_lstm_size, config.ctx_skip_connections, 'ctx')
global _qhidden, _chidden
_qhidden = theano.function([question, question_mask], qhidden_list)
_chidden = theano.function([context, context_mask], chidden_list)
# Calculate question encoding (concatenate layer1)
if config.question_skip_connections:
qenc_dim = 2*sum(config.question_lstm_size)
qenc = tensor.concatenate([h[-1,:,:] for h in qhidden_list], axis=1)
else:
qenc_dim = 2*config.question_lstm_size[-1]
qenc = tensor.concatenate([h[-1,:,:] for h in qhidden_list[-2:]], axis=1)
qenc.name = 'qenc'
# Calculate context encoding (concatenate layer1)
if config.ctx_skip_connections:
cenc_dim = 2*sum(config.ctx_lstm_size)
cenc = tensor.concatenate(chidden_list, axis=2)
else:
cenc_dim = 2*config.ctx_lstm_size[-1]
cenc = tensor.concatenate(chidden_list[-2:], axis=2)
cenc.name = 'cenc'
global _qenc, _cenc
_qenc = theano.function([question, question_mask], qenc)
_cenc = theano.function([context, context_mask], cenc)
# Attention mechanism MLP
attention_mlp = bricks[-2] #attention_mlp
attention_qlinear = bricks[4] #attq
attention_clinear = bricks[11] # attc
layer1 = Tanh().apply(attention_clinear.apply(cenc.reshape((cenc.shape[0]*cenc.shape[1], cenc.shape[2])))
.reshape((cenc.shape[0],cenc.shape[1],config.attention_mlp_hidden[0]))
+ attention_qlinear.apply(qenc)[None, :, :])
global _attention_clinear, _attention_qlinear
_attention_clinear = theano.function([context, context_mask], attention_clinear.apply(cenc.reshape((cenc.shape[0]*cenc.shape[1], cenc.shape[2]))).reshape((cenc.shape[0],cenc.shape[1],config.attention_mlp_hidden[0])))
_attention_qlinear = theano.function([question, question_mask], attention_qlinear.apply(qenc)[None, :, :])
layer1.name = 'layer1'
att_weights = attention_mlp.apply(layer1.reshape((layer1.shape[0]*layer1.shape[1], layer1.shape[2])))
att_weights.name = 'att_weights_0'
att_weights = att_weights.reshape((layer1.shape[0], layer1.shape[1]))
att_weights.name = 'att_weights'
attended = tensor.sum(cenc * tensor.nnet.softmax(att_weights.T).T[:, :, None], axis=0)
attended.name = 'attended'
global _attended
_attended = theano.function([question, question_mask, context, context_mask], attended)
# Now we can calculate our output
out_mlp = bricks[-1] #out_mlp
probs = out_mlp.apply(tensor.concatenate([attended, qenc], axis=1))
probs.name = 'probs'
f = theano.function([question, question_mask, context, context_mask], probs)
return f
def __init__(self, config, vocab_size):
question = tensor.imatrix('question')
question_mask = tensor.imatrix('question_mask')
context = tensor.imatrix('context')
context_mask = tensor.imatrix('context_mask')
answer = tensor.imatrix('answer')
answer_mask = tensor.imatrix('answer_mask')
bricks = []
question = question.dimshuffle(1, 0)
question_mask = question_mask.dimshuffle(1, 0)
context = context.dimshuffle(1, 0)
context_mask = context_mask.dimshuffle(1, 0)
answer = answer.dimshuffle(1, 0)
answer_mask = answer_mask.dimshuffle(1, 0)
# Embed questions and context
embed = LookupTable(vocab_size, config.embed_size, name='question_embed')
embed.weights_init = IsotropicGaussian(0.01)
# Calculate question encoding (concatenate layer1)
qembed = embed.apply(question)
qlstms, qhidden_list = make_bidir_lstm_stack(qembed, config.embed_size, question_mask.astype(theano.config.floatX),
config.question_lstm_size, config.question_skip_connections, 'q')
bricks = bricks + qlstms
if config.question_skip_connections:
qenc_dim = 2*sum(config.question_lstm_size)
qenc = tensor.concatenate([h[-1,:,:] for h in qhidden_list], axis=1)
else:
qenc_dim = 2*config.question_lstm_size[-1]
qenc = tensor.concatenate([h[-1,:,:] for h in qhidden_list[-2:]], axis=1)
qenc.name = 'qenc'
# Calculate context encoding (concatenate layer1)
cembed = embed.apply(context)
cqembed = tensor.concatenate([cembed, tensor.extra_ops.repeat(qenc[None, :, :], cembed.shape[0], axis=0)], axis=2)
clstms, chidden_list = make_bidir_lstm_stack(cqembed, config.embed_size + qenc_dim, context_mask.astype(theano.config.floatX),
config.ctx_lstm_size, config.ctx_skip_connections, 'ctx')
bricks = bricks + clstms
if config.ctx_skip_connections:
cenc_dim = 2*sum(config.ctx_lstm_size) #2 : fw & bw
cenc = tensor.concatenate(chidden_list, axis=2)
else:
cenc_dim = 2*config.question_lstm_size[-1]
cenc = tensor.concatenate(chidden_list[-2:], axis=2)
cenc.name = 'cenc'
# Attention mechanism MLP start
attention_mlp_start = MLP(dims=config.attention_mlp_hidden + [1],
activations=config.attention_mlp_activations[1:] + [Identity()],
name='attention_mlp_start')
attention_qlinear_start = Linear(input_dim=qenc_dim, output_dim=config.attention_mlp_hidden[0], name='attq_start') #Wum
attention_clinear_start = Linear(input_dim=cenc_dim, output_dim=config.attention_mlp_hidden[0], use_bias=False, name='attc_start') # Wym
bricks += [attention_mlp_start, attention_qlinear_start, attention_clinear_start]
layer1_start = Tanh(name='layer1_start')
layer1_start = layer1_start.apply(attention_clinear_start.apply(cenc.reshape((cenc.shape[0]*cenc.shape[1], cenc.shape[2])))
.reshape((cenc.shape[0],cenc.shape[1],config.attention_mlp_hidden[0]))
+ attention_qlinear_start.apply(qenc)[None, :, :])
att_weights_start = attention_mlp_start.apply(layer1_start.reshape((layer1_start.shape[0]*layer1_start.shape[1], layer1_start.shape[2])))
att_weights_start = att_weights_start.reshape((layer1_start.shape[0], layer1_start.shape[1]))
att_weights_start = tensor.nnet.softmax(att_weights_start.T).T
attended = tensor.sum(cenc * att_weights_start[:, :, None], axis=0)
attended.name = 'attended'
# Attention mechanism MLP end
attention_mlp_end = MLP(dims=config.attention_mlp_hidden + [1],
activations=config.attention_mlp_activations[1:] + [Identity()],
name='attention_mlp_end')
attention_qlinear_end = Linear(input_dim=qenc_dim, output_dim=config.attention_mlp_hidden[0], name='attq_end') #Wum
attention_clinear_end = Linear(input_dim=cenc_dim, output_dim=config.attention_mlp_hidden[0], use_bias=False, name='attc_end') # Wym
bricks += [attention_mlp_end, attention_qlinear_end, attention_clinear_end]
layer1_end = Tanh(name='layer1_end')
layer1_end = layer1_end.apply(attention_clinear_end.apply(cenc.reshape((cenc.shape[0]*cenc.shape[1], cenc.shape[2])))
.reshape((cenc.shape[0],cenc.shape[1],config.attention_mlp_hidden[0]))
+ attention_qlinear_end.apply(attended)[None, :, :])
att_weights_end = attention_mlp_end.apply(layer1_end.reshape((layer1_end.shape[0]*layer1_end.shape[1], layer1_end.shape[2])))
att_weights_end = att_weights_end.reshape((layer1_end.shape[0], layer1_end.shape[1]))
att_weights_end = tensor.nnet.softmax(att_weights_end.T).T
att_weights_start = tensor.dot(tensor.le(tensor.tile(theano.tensor.arange(context.shape[0])[None,:], (context.shape[0], 1)),
tensor.tile(theano.tensor.arange(context.shape[0])[:,None], (1, context.shape[0]))), att_weights_start)
att_weights_end = tensor.dot(tensor.ge(tensor.tile(theano.tensor.arange(context.shape[0])[None,:], (context.shape[0], 1)),
tensor.tile(theano.tensor.arange(context.shape[0])[:,None], (1, context.shape[0]))), att_weights_end)
# add attention from left and right
#att_weights = att_weights_start * att_weights_end
att_weights = tensor.minimum(att_weights_start, att_weights_end)
att_target = tensor.eq(tensor.tile(answer[None,:,:], (context.shape[0], 1, 1)),
tensor.tile(context[:,None,:], (1, answer.shape[0], 1))).sum(axis=1).clip(0,1)
self.predictions = tensor.gt(att_weights, 0.5) * context
att_target = att_target / (att_target.sum(axis=0) + 0.00001)
att_weights = att_weights / (att_weights.sum(axis=0) + 0.00001)
cost = (tensor.nnet.binary_crossentropy(att_weights, att_target) * context_mask).sum() / context_mask.sum()
# Apply dropout
cg = ComputationGraph([cost])
if config.w_noise > 0:
noise_vars = VariableFilter(roles=[WEIGHT])(cg)
cg = apply_noise(cg, noise_vars, config.w_noise)
if config.dropout > 0:
cg = apply_dropout(cg, qhidden_list + chidden_list, config.dropout)
[cost_reg] = cg.outputs
# Other stuff
cost.name = 'cost'
cost_reg.name = 'cost_reg'
att_weights_start.name = 'att_weights_start'
att_weights_end.name = 'att_weights_end'
att_target.name = 'att_target'
att_weights.name = 'att_weights'
self.predictions.name = 'pred'
self.sgd_cost = cost_reg
self.monitor_vars = [[cost_reg]]
self.monitor_vars_valid = [[cost_reg]]
self.analyse_vars= [cost, self.predictions, att_weights_start, att_weights_end, att_weights, att_target]
# Initialize bricks
embed.initialize()
for brick in bricks:
brick.weights_init = config.weights_init
brick.biases_init = config.biases_init
brick.initialize()
def __init__(self, config, vocab_size):
question = tensor.imatrix('question')
question_mask = tensor.imatrix('question_mask')
context = tensor.imatrix('context')
context_mask = tensor.imatrix('context_mask')
answer = tensor.ivector('answer')
candidates = tensor.imatrix('candidates')
candidates_mask = tensor.imatrix('candidates_mask')
bricks = []
question = question.dimshuffle(1, 0)
question_mask = question_mask.dimshuffle(1, 0)
context = context.dimshuffle(1, 0)
context_mask = context_mask.dimshuffle(1, 0)
# Embed questions and cntext
embed = LookupTable(vocab_size, config.embed_size, name='question_embed')
bricks.append(embed)
qembed = embed.apply(question)
cembed = embed.apply(context)
qlstms, qhidden_list = make_bidir_lstm_stack(qembed, config.embed_size, question_mask.astype(theano.config.floatX),
config.question_lstm_size, config.question_skip_connections, 'q')
clstms, chidden_list = make_bidir_lstm_stack(cembed, config.embed_size, context_mask.astype(theano.config.floatX),
config.ctx_lstm_size, config.ctx_skip_connections, 'ctx')
bricks = bricks + qlstms + clstms
# Calculate question encoding (concatenate layer1)
if config.question_skip_connections:
qenc_dim = 2*sum(config.question_lstm_size)
qenc = tensor.concatenate([h[-1,:,:] for h in qhidden_list], axis=1)
else:
qenc_dim = 2*config.question_lstm_size[-1]
qenc = tensor.concatenate([h[-1,:,:] for h in qhidden_list[-2:]], axis=1)
qenc.name = 'qenc'
# Calculate context encoding (concatenate layer1)
if config.ctx_skip_connections:
cenc_dim = 2*sum(config.ctx_lstm_size)
cenc = tensor.concatenate(chidden_list, axis=2)
else:
cenc_dim = 2*config.ctx_lstm_size[-1]
cenc = tensor.concatenate(chidden_list[-2:], axis=2)
cenc.name = 'cenc'
# Attention mechanism MLP
attention_mlp = MLP(dims=config.attention_mlp_hidden + [1],
activations=config.attention_mlp_activations[1:] + [Identity()],
name='attention_mlp')
attention_qlinear = Linear(input_dim=qenc_dim, output_dim=config.attention_mlp_hidden[0], name='attq')
attention_clinear = Linear(input_dim=cenc_dim, output_dim=config.attention_mlp_hidden[0], use_bias=False, name='attc')
bricks += [attention_mlp, attention_qlinear, attention_clinear]
layer1 = Tanh().apply(attention_clinear.apply(cenc.reshape((cenc.shape[0]*cenc.shape[1], cenc.shape[2])))
.reshape((cenc.shape[0],cenc.shape[1],config.attention_mlp_hidden[0]))
+ attention_qlinear.apply(qenc)[None, :, :])
layer1.name = 'layer1'
att_weights = attention_mlp.apply(layer1.reshape((layer1.shape[0]*layer1.shape[1], layer1.shape[2])))
att_weights.name = 'att_weights_0'
att_weights = att_weights.reshape((layer1.shape[0], layer1.shape[1]))
att_weights.name = 'att_weights'
attended = tensor.sum(cenc * tensor.nnet.softmax(att_weights.T).T[:, :, None], axis=0)
attended.name = 'attended'
# Now we can calculate our output
out_mlp = MLP(dims=[cenc_dim + qenc_dim] + config.out_mlp_hidden + [config.n_entities],
activations=config.out_mlp_activations + [Identity()],
name='out_mlp')
bricks += [out_mlp]
probs = out_mlp.apply(tensor.concatenate([attended, qenc], axis=1))
probs.name = 'probs'
is_candidate = tensor.eq(tensor.arange(config.n_entities, dtype='int32')[None, None, :],
tensor.switch(candidates_mask, candidates, -tensor.ones_like(candidates))[:, :, None]).sum(axis=1)
probs = tensor.switch(is_candidate, probs, -1000 * tensor.ones_like(probs))
# Calculate prediction, cost and error rate
pred = probs.argmax(axis=1)
cost = Softmax().categorical_cross_entropy(answer, probs).mean()
error_rate = tensor.neq(answer, pred).mean()
# Apply dropout
cg = ComputationGraph([cost, error_rate])
if config.w_noise > 0:
noise_vars = VariableFilter(roles=[WEIGHT])(cg)
cg = apply_noise(cg, noise_vars, config.w_noise)
if config.dropout > 0:
cg = apply_dropout(cg, qhidden_list + chidden_list, config.dropout)
[cost_reg, error_rate_reg] = cg.outputs
# Other stuff
cost_reg.name = cost.name = 'cost'
error_rate_reg.name = error_rate.name = 'error_rate'
self.sgd_cost = cost_reg
self.monitor_vars = [[cost_reg], [error_rate_reg]]
self.monitor_vars_valid = [[cost], [error_rate]]
# Initialize bricks
for brick in bricks:
brick.weights_init = config.weights_init
brick.biases_init = config.biases_init
brick.initialize()
def __init__(self, config, vocab_size):
question = tensor.imatrix('question')
question_mask = tensor.imatrix('question_mask')
context = tensor.imatrix('context')
context_mask = tensor.imatrix('context_mask')
answer = tensor.imatrix('answer')
answer_mask = tensor.imatrix('answer_mask')
ans_indices = tensor.imatrix('ans_indices') # n_steps * n_samples
ans_indices_mask = tensor.imatrix('ans_indices_mask')
bricks = []
question = question.dimshuffle(1, 0)
question_mask = question_mask.dimshuffle(1, 0)
context = context.dimshuffle(1, 0)
context_mask = context_mask.dimshuffle(1, 0)
answer = answer.dimshuffle(1, 0)
answer_mask = answer_mask.dimshuffle(1, 0)
ans_indices = ans_indices.dimshuffle(1, 0)
ans_indices_mask = ans_indices_mask.dimshuffle(1, 0)
# Embed questions and context
embed = LookupTable(vocab_size, config.embed_size, name='embed')
#embed.weights_init = IsotropicGaussian(0.01)
embed.weights_init = Constant(
init_embedding_table(filename='embeddings/vocab_embeddings.txt'))
# one directional LSTM encoding
q_lstm_ins = Linear(input_dim=config.embed_size,
output_dim=4 * config.pre_lstm_size,
name='q_lstm_in')
q_lstm = LSTM(dim=config.pre_lstm_size,
activation=Tanh(),
name='q_lstm')
c_lstm_ins = Linear(input_dim=config.embed_size,
output_dim=4 * config.pre_lstm_size,
name='c_lstm_in')
c_lstm = LSTM(dim=config.pre_lstm_size,
activation=Tanh(),
name='c_lstm')
bricks += [q_lstm, c_lstm, q_lstm_ins, c_lstm_ins]
q_tmp = q_lstm_ins.apply(embed.apply(question))
c_tmp = c_lstm_ins.apply(embed.apply(context))
q_hidden, _ = q_lstm.apply(q_tmp,
mask=question_mask.astype(
theano.config.floatX)) # lq, bs, dim
c_hidden, _ = c_lstm.apply(c_tmp,
mask=context_mask.astype(
theano.config.floatX)) # lc, bs, dim
# Attention mechanism Bilinear question
attention_question = Linear(input_dim=config.pre_lstm_size,
output_dim=config.pre_lstm_size,
name='att_question')
bricks += [attention_question]
att_weights_question = q_hidden[
None, :, :, :] * attention_question.apply(
c_hidden.reshape(
(c_hidden.shape[0] * c_hidden.shape[1],
c_hidden.shape[2]))).reshape(
(c_hidden.shape[0], c_hidden.shape[1],
c_hidden.shape[2]))[:,
None, :, :] # --> lc,lq,bs,dim
att_weights_question = att_weights_question.sum(
axis=3) # sum over axis 3 -> dimensions --> lc,lq,bs
att_weights_question = att_weights_question.dimshuffle(
0, 2, 1) # --> lc,bs,lq
att_weights_question = att_weights_question.reshape(
(att_weights_question.shape[0] * att_weights_question.shape[1],
att_weights_question.shape[2])) # --> lc*bs,lq
att_weights_question = tensor.nnet.softmax(
att_weights_question
) # softmax over axis 1 -> length of question # --> lc*bs,lq
att_weights_question = att_weights_question.reshape(
(c_hidden.shape[0], q_hidden.shape[1],
q_hidden.shape[0])) # --> lc,bs,lq
att_weights_question = att_weights_question.dimshuffle(
0, 2, 1) # --> lc,lq,bs
attended_question = tensor.sum(
q_hidden[None, :, :, :] * att_weights_question[:, :, :, None],
axis=1) # sum over axis 1 -> length of question --> lc,bs,dim
attended_question.name = 'attended_question'
# Match LSTM
cqembed = tensor.concatenate([c_hidden, attended_question], axis=2)
mlstms, mhidden_list = make_bidir_lstm_stack(
cqembed, 2 * config.pre_lstm_size,
context_mask.astype(theano.config.floatX), config.match_lstm_size,
config.match_skip_connections, 'match')
bricks = bricks + mlstms
if config.match_skip_connections:
menc_dim = 2 * sum(config.match_lstm_size)
menc = tensor.concatenate(mhidden_list, axis=2)
else:
menc_dim = 2 * config.match_lstm_size[-1]
menc = tensor.concatenate(mhidden_list[-2:], axis=2)
menc.name = 'menc'
# Attention mechanism MLP start
attention_mlp_start = MLP(
dims=config.attention_mlp_hidden + [1],
activations=config.attention_mlp_activations[1:] + [Identity()],
name='attention_mlp_start')
attention_clinear_start = Linear(
input_dim=menc_dim,
output_dim=config.attention_mlp_hidden[0],
name='attm_start') # Wym
bricks += [attention_mlp_start, attention_clinear_start]
layer1_start = Tanh(name='layer1_start')
layer1_start = layer1_start.apply(
attention_clinear_start.apply(
menc.reshape(
(menc.shape[0] * menc.shape[1], menc.shape[2]))).reshape(
(menc.shape[0], menc.shape[1],
config.attention_mlp_hidden[0])))
att_weights_start = attention_mlp_start.apply(
layer1_start.reshape(
(layer1_start.shape[0] * layer1_start.shape[1],
layer1_start.shape[2])))
att_weights_start = att_weights_start.reshape(
(layer1_start.shape[0], layer1_start.shape[1]))
att_weights_start = tensor.nnet.softmax(att_weights_start.T).T
attended = tensor.sum(menc * att_weights_start[:, :, None], axis=0)
attended.name = 'attended'
# Attention mechanism MLP end
attention_mlp_end = MLP(
dims=config.attention_mlp_hidden + [1],
activations=config.attention_mlp_activations[1:] + [Identity()],
name='attention_mlp_end')
attention_qlinear_end = Linear(
input_dim=menc_dim,
output_dim=config.attention_mlp_hidden[0],
name='atts_end') #Wum
attention_clinear_end = Linear(
input_dim=menc_dim,
output_dim=config.attention_mlp_hidden[0],
use_bias=False,
name='attm_end') # Wym
bricks += [
attention_mlp_end, attention_qlinear_end, attention_clinear_end
]
layer1_end = Tanh(name='layer1_end')
layer1_end = layer1_end.apply(
attention_clinear_end.apply(
menc.reshape((menc.shape[0] * menc.shape[1], menc.shape[2]
))).reshape((menc.shape[0], menc.shape[1],
config.attention_mlp_hidden[0])) +
attention_qlinear_end.apply(attended)[None, :, :])
att_weights_end = attention_mlp_end.apply(
layer1_end.reshape((layer1_end.shape[0] * layer1_end.shape[1],
layer1_end.shape[2])))
att_weights_end = att_weights_end.reshape(
(layer1_end.shape[0], layer1_end.shape[1]))
att_weights_end = tensor.nnet.softmax(att_weights_end.T).T
att_weights_start = tensor.dot(
tensor.le(
tensor.tile(
theano.tensor.arange(context.shape[0])[None, :],
(context.shape[0], 1)),
tensor.tile(
theano.tensor.arange(context.shape[0])[:, None],
(1, context.shape[0]))), att_weights_start)
att_weights_end = tensor.dot(
tensor.ge(
tensor.tile(
theano.tensor.arange(context.shape[0])[None, :],
(context.shape[0], 1)),
tensor.tile(
theano.tensor.arange(context.shape[0])[:, None],
(1, context.shape[0]))), att_weights_end)
# add attention from left and right
att_weights = att_weights_start * att_weights_end
#att_weights = tensor.minimum(att_weights_start, att_weights_end)
att_target = tensor.zeros((ans_indices.shape[1], context.shape[0]),
dtype=theano.config.floatX)
att_target = tensor.set_subtensor(
att_target[tensor.arange(ans_indices.shape[1]), ans_indices], 1)
att_target = att_target.dimshuffle(1, 0)
#att_target = tensor.eq(tensor.tile(answer[None,:,:], (context.shape[0], 1, 1)),
# tensor.tile(context[:,None,:], (1, answer.shape[0], 1))).sum(axis=1).clip(0,1)
self.predictions = tensor.gt(att_weights, 0.25) * context
att_target = att_target / (att_target.sum(axis=0) + 0.00001)
#att_weights = att_weights / (att_weights.sum(axis=0) + 0.00001)
cost = (tensor.nnet.binary_crossentropy(att_weights, att_target) *
context_mask).sum() / context_mask.sum()
# Apply dropout
cg = ComputationGraph([cost])
if config.w_noise > 0:
noise_vars = VariableFilter(roles=[WEIGHT])(cg)
cg = apply_noise(cg, noise_vars, config.w_noise)
if config.dropout > 0:
cg = apply_dropout(cg, mhidden_list, config.dropout)
[cost_reg] = cg.outputs
# Other stuff
cost.name = 'cost'
cost_reg.name = 'cost_reg'
att_weights_start.name = 'att_weights_start'
att_weights_end.name = 'att_weights_end'
att_weights.name = 'att_weights'
att_target.name = 'att_target'
self.predictions.name = 'pred'
self.sgd_cost = cost_reg
self.monitor_vars = [[cost_reg]]
self.monitor_vars_valid = [[cost_reg]]
self.analyse_vars = [
cost, self.predictions, att_weights_start, att_weights_end,
att_weights, att_target
]
# Initialize bricks
embed.initialize()
for brick in bricks:
brick.weights_init = config.weights_init
brick.biases_init = config.biases_init
brick.initialize()
