def construct_model(vocab_size, embedding_dim, ngram_order, hidden_dims,
activations):
# Construct the model
x = tensor.lmatrix('features')
y = tensor.lvector('targets')
lookup = LookupTable(length=vocab_size, dim=embedding_dim, name='lookup')
hidden = MLP(activations=activations + [None],
dims=[ngram_order * embedding_dim] + hidden_dims +
[vocab_size])
embeddings = lookup.apply(x)
embeddings = embeddings.flatten(ndim=2) # Concatenate embeddings
activations = hidden.apply(embeddings)
cost = Softmax().categorical_cross_entropy(y, activations)
# Initialize parameters
lookup.weights_init = IsotropicGaussian(0.001)
hidden.weights_init = IsotropicGaussian(0.01)
hidden.biases_init = Constant(0.001)
lookup.initialize()
hidden.initialize()
return cost
def construct_model(vocab_size, embedding_dim, ngram_order, hidden_dims,
activations):
# Construct the model
x = tensor.lmatrix('features')
y = tensor.lvector('targets')
lookup = LookupTable(length=vocab_size, dim=embedding_dim, name='lookup')
hidden = MLP(activations=activations + [None],
dims=[ngram_order * embedding_dim] + hidden_dims +
[vocab_size])
embeddings = lookup.apply(x)
embeddings = embeddings.flatten(ndim=2) # Concatenate embeddings
activations = hidden.apply(embeddings)
cost = Softmax().categorical_cross_entropy(y, activations)
# Initialize parameters
lookup.weights_init = IsotropicGaussian(0.001)
hidden.weights_init = IsotropicGaussian(0.01)
hidden.biases_init = Constant(0.001)
lookup.initialize()
hidden.initialize()
return cost
attention = SimpleSequenceAttention(state_names=source_names,
state_dims=[hidden_size_recurrent],
attended_dim=context_size)
generator = SequenceGenerator(readout=readout,
transition=transition,
attention=attention,
name="generator")
generator.weights_init = IsotropicGaussian(0.01)
generator.biases_init = Constant(0.)
generator.initialize()
mlp_context.weights_init = IsotropicGaussian(0.01)
mlp_context.biases_init = Constant(0.)
mlp_context.initialize()
#ipdb.set_trace()
cost_matrix = generator.cost_matrix(x,
x_mask,
attended=mlp_context.apply(context))
cost = cost_matrix.sum() / x_mask.sum()
cost.name = "sequence_log_likelihood"
cg = ComputationGraph(cost)
model = Model(cost)
#################
# Algorithm
#################
attention = SimpleSequenceAttention(
state_names = source_names,
state_dims = [hidden_size_recurrent],
attended_dim = context_size)
generator = SequenceGenerator(readout=readout,
transition=transition,
attention = attention,
name = "generator")
generator.weights_init = IsotropicGaussian(0.01)
generator.biases_init = Constant(0.)
generator.initialize()
mlp_context.weights_init = IsotropicGaussian(0.01)
mlp_context.biases_init = Constant(0.)
mlp_context.initialize()
#ipdb.set_trace()
cost_matrix = generator.cost_matrix(x, x_mask,
attended = mlp_context.apply(context))
cost = cost_matrix.sum()/x_mask.sum()
cost.name = "sequence_log_likelihood"
cg = ComputationGraph(cost)
model = Model(cost)
#################
# Algorithm
#################
