def __init__(self, n_tags, name='CRF'):
"""
Initialize neural network.
"""
self.name = name
self.transitions = shared((n_tags, n_tags), name + '__transitions')
self.params = [self.transitions]
def __init__(self, input_dim, hidden_dim, name):
"""
Initialize neural network.
"""
self.input_dim = input_dim
self.hidden_dim = hidden_dim
self.name = name
self.W = shared((input_dim, hidden_dim * 4), name + 'W')
self.U = shared((hidden_dim, hidden_dim * 4), name + 'U')
#self.W = block_orth_normal_initializer((input_dim, hidden_dim * 4), [input_dim,], [hidden_dim] * 4, name + 'W')
#self.U = block_orth_normal_initializer((hidden_dim, hidden_dim * 4), [hidden_dim,], [hidden_dim] * 4, name + 'U')
self.b = shared((hidden_dim * 4, ), name + 'b')
self.c_0 = shared((hidden_dim, ), name + '__c_0')
self.h_0 = shared((hidden_dim, ), name + '__h_0')
self.params = [self.W, self.U, self.b]
def __init__(self,
input_dim,
output_dim,
bias=True,
activation='sigmoid',
name='hidden_layer'):
self.input_dim = input_dim
self.output_dim = output_dim
self.bias = bias
self.name = name
if activation is None:
self.activation = None
elif activation == 'tanh':
self.activation = T.tanh
elif activation == 'sigmoid':
self.activation = T.nnet.sigmoid
elif activation == 'softmax':
self.activation = T.nnet.softmax
elif activation == 'relu':
self.activation = T.nnet.relu
else:
raise Exception("Unknown activation function: " % activation)
# Initialize weights and bias
self.weights = shared((input_dim, output_dim), name + '__weights')
# Define parameters
if self.bias:
self.bias_weights = shared((output_dim, ), name + '__bias')
self.params = [self.weights, self.bias_weights]
self.L2_sqr = ((self.weights**2).sum() +
(self.bias_weights**2).sum())
else:
self.params = [self.weights]
self.L2_sqr = ((self.weights**2).sum())
def __init__(self, input_dim, output_dim, name='embedding_layer'):
"""
Typically, input_dim is the vocabulary size,
and output_dim the embedding dimension.
"""
self.input_dim = input_dim
self.output_dim = output_dim
self.name = name
# Randomly generate weights
self.embeddings = shared((input_dim, output_dim),
self.name + '__embeddings')
# Define parameters
self.params = [self.embeddings]
self.updates = OrderedDict({
self.embeddings:
self.embeddings /
T.sqrt(T.sum(self.embeddings**2, axis=1, keepdims=True))
})
self.normalize = theano.function(inputs=[],
outputs=[],
updates=self.updates)
self.L2_sqr = ((self.embeddings**2).sum())
self.zero_vec_tensor = T.vector()
self.zero_vec = np.zeros(self.output_dim)
self.set_word_zero = theano.function(
[self.zero_vec_tensor],
updates=[(self.embeddings,
T.set_subtensor(self.embeddings[0, :],
self.zero_vec_tensor))],
allow_input_downcast=True)
self.updates = OrderedDict({
self.embeddings:
self.embeddings /
T.sqrt(T.sum(self.embeddings**2, axis=1)).dimshuffle(0, 'x')
})
self.normalize = theano.function([], [], updates=self.updates)