def _init_nn(self):
"""Initialize neural network.
"""
self.intm_dim = max(MIN_DIM, self.ndim - (self.ndim - self.n_y) / 2)
# indices of word embeddings
self.W_INDICES_ARG1 = TT.ivector(name="W_INDICES_ARG1")
self.W_INDICES_ARG2 = TT.ivector(name="W_INDICES_ARG2")
# connective's index
self.CONN_INDEX = TT.iscalar(name="CONN_INDEX")
# initialize the matrix of word embeddings
self.init_w_emb()
# word embeddings of the arguments
self.EMB_ARG1 = self.W_EMB[self.W_INDICES_ARG1]
self.EMB_ARG2 = self.W_EMB[self.W_INDICES_ARG2]
# connective's embedding
self._init_conn_emb()
self.EMB_CONN = self.CONN_EMB[self.CONN_INDEX]
# perform matrix decomposition
_, _, self.ARG1 = TT.nlinalg.svd(self.EMB_ARG1,
full_matrices=True)
_, _, self.ARG2 = TT.nlinalg.svd(self.EMB_ARG2,
full_matrices=True)
self.ARG_DIFF = self.ARG1 - self.ARG2
# map decomposed matrices to the intermediate level
self.ARG_DIFF2I = theano.shared(value=HE_UNIFORM((self.ndim, 1)),
name="ARG_DIFF2I")
self.arg_diff_bias = theano.shared(value=HE_UNIFORM((1, self.ndim)),
name="arg_diff_bias")
self._params.extend([self.ARG_DIFF2I, self.arg_diff_bias])
self.ARGS = (TT.dot(self.ARG_DIFF, self.ARG_DIFF2I).T +
self.arg_diff_bias).flatten()
# define final units
self.I = TT.concatenate((self.ARGS, self.EMB_CONN))
self.I2Y = theano.shared(value=HE_UNIFORM((self.n_y,
self.ndim + self.intm_dim)),
name="I2Y")
self.y_bias = theano.shared(value=HE_UNIFORM((1, self.n_y)),
name="y_bias")
self._params.extend([self.I2Y, self.y_bias])
self.Y_pred = TT.nnet.softmax(TT.dot(self.I2Y, self.I).T + self.y_bias)
# initialize cost and optimization functions
self.Y_gold = TT.vector(name="Y_gold")
self._cost = TT.sum((self.Y_pred - self.Y_gold) ** 2)
self._dev_cost = TT.sum((self.Y_pred - self.Y_gold) ** 2)
self._pred_class = TT.argmax(self.Y_pred)
grads = TT.grad(self._cost, wrt=self._params)
self._init_funcs(grads)
def _init_w_emb(self):
"""Initialize task-specific word embeddings.
"""
self.W_EMB = theano.shared(
value=HE_UNIFORM((self.w_i, self.ndim)), name="W_EMB")
self._params.append(self.W_EMB)
def _init_w_emb(self):
"""Initialize task-specific word embeddings.
"""
self.W_EMB = theano.shared(value=HE_UNIFORM((self.w_i, self.ndim)),
name="W_EMB")
self._params.append(self.W_EMB)
def _init_nn(self):
"""Initialize neural network.
"""
self.intm_dim = max(MIN_DIM, self.ndim - (self.ndim - self.n_y) / 2)
# indices of word embeddings
self.W_INDICES_ARG1 = TT.ivector(name="W_INDICES_ARG1")
self.W_INDICES_ARG2 = TT.ivector(name="W_INDICES_ARG2")
# connective's index
self.CONN_INDEX = TT.iscalar(name="CONN_INDEX")
# initialize the matrix of word embeddings
self.init_w_emb()
# word embeddings of the arguments
self.EMB_ARG1 = self.W_EMB[self.W_INDICES_ARG1]
self.EMB_ARG2 = self.W_EMB[self.W_INDICES_ARG2]
# connective's embedding
self._init_conn_emb()
self.EMB_CONN = self.CONN_EMB[self.CONN_INDEX]
# perform matrix decomposition
_, _, self.ARG1 = TT.nlinalg.svd(self.EMB_ARG1, full_matrices=True)
_, _, self.ARG2 = TT.nlinalg.svd(self.EMB_ARG2, full_matrices=True)
self.ARG_DIFF = self.ARG1 - self.ARG2
# map decomposed matrices to the intermediate level
self.ARG_DIFF2I = theano.shared(value=HE_UNIFORM((self.ndim, 1)),
name="ARG_DIFF2I")
self.arg_diff_bias = theano.shared(value=HE_UNIFORM((1, self.ndim)),
name="arg_diff_bias")
self._params.extend([self.ARG_DIFF2I, self.arg_diff_bias])
self.ARGS = (TT.dot(self.ARG_DIFF, self.ARG_DIFF2I).T +
self.arg_diff_bias).flatten()
# define final units
self.I = TT.concatenate((self.ARGS, self.EMB_CONN))
self.I2Y = theano.shared(value=HE_UNIFORM(
(self.n_y, self.ndim + self.intm_dim)),
name="I2Y")
self.y_bias = theano.shared(value=HE_UNIFORM((1, self.n_y)),
name="y_bias")
self._params.extend([self.I2Y, self.y_bias])
self.Y_pred = TT.nnet.softmax(TT.dot(self.I2Y, self.I).T + self.y_bias)
# initialize cost and optimization functions
self.Y_gold = TT.vector(name="Y_gold")
self._cost = TT.sum((self.Y_pred - self.Y_gold)**2)
self._dev_cost = TT.sum((self.Y_pred - self.Y_gold)**2)
self._pred_class = TT.argmax(self.Y_pred)
grads = TT.grad(self._cost, wrt=self._params)
self._init_funcs(grads)
def _init_conn_emb(self):
"""Initialize task-specific connective embeddings.
"""
self.CONN_EMB = theano.shared(value=HE_UNIFORM(
(self.c_i, self.intm_dim)),
name="CONN_EMB")
self._params.append(self.CONN_EMB)
def _init_conn_emb(self):
"""Initialize task-specific connective embeddings.
"""
self.CONN_EMB = theano.shared(
value=HE_UNIFORM((self.c_i, self.intm_dim)),
name="CONN_EMB")
self._params.append(self.CONN_EMB)
def _init_nn(self):
"""Initialize neural network.
"""
self.intm_dim = max(100, self.ndim - (self.ndim - self.n_y) / 2)
# indices of word embeddings
self.W_INDICES_ARG1 = TT.ivector(name="W_INDICES_ARG1")
self.W_INDICES_ARG2 = TT.ivector(name="W_INDICES_ARG2")
# connective's index
self.CONN_INDEX = TT.iscalar(name="CONN_INDEX")
# initialize the matrix of word embeddings
self.init_w_emb()
# word embeddings of the arguments
self.EMB_ARG1 = self.W_EMB[self.W_INDICES_ARG1]
self.EMB_ARG2 = self.W_EMB[self.W_INDICES_ARG2]
# connective's embedding
self._init_conn_emb()
self.EMB_CONN = self.CONN_EMB[self.CONN_INDEX]
# initialize forward LSTM unit
invars = ((self.EMB_ARG1, False), (self.EMB_ARG2, False))
params, outvars = self._init_lstm(invars)
self._params.extend(params)
self.F_OUT_ARG1, self.F_OUT_ARG2 = outvars
self.F_ARG1 = TT.mean(self.F_OUT_ARG1, axis=0)
self.F_ARG2 = TT.mean(self.F_OUT_ARG2, axis=0)
# define final units
self.I = TT.concatenate((self.F_ARG1, self.F_ARG2, self.EMB_CONN))
self.I2Y = theano.shared(value=HE_UNIFORM(
(self.n_y, self.intm_dim * 3)),
name="I2Y")
self.y_bias = theano.shared(value=HE_UNIFORM((1, self.n_y)),
name="y_bias")
self._params.extend([self.I2Y, self.y_bias])
self.Y_pred = TT.nnet.softmax(TT.dot(self.I2Y, self.I).T + self.y_bias)
# initialize cost and optimization functions
self.Y_gold = TT.vector(name="Y_gold")
self._cost = TT.sum((self.Y_pred - self.Y_gold)**2)
self._dev_cost = TT.sum((self.Y_pred - self.Y_gold)**2)
self._pred_class = TT.argmax(self.Y_pred)
grads = TT.grad(self._cost, wrt=self._params)
self._init_funcs(grads)
def _init_w2v_emb(self):
"""Initialize word2vec embedding matrix.
"""
w_emb = np.empty((self.w_i, self.ndim))
w_emb[self.unk_w_i, :] = 1e-2 # prevent zeros in this row
for w, i in self.w2emb_i.iteritems():
if i == self.unk_w_i:
continue
w_emb[i] = self.w2v[w]
self.W_EMB = theano.shared(value=floatX(w_emb), name="W_EMB")
# We unload embeddings every time before the training to free more
# memory. Feel free to comment the line below, if you have plenty of
# RAM.
self.w2v.unload()
def __init__(self, a_w2v=False, a_lstsq=False, a_max_iters=MAX_ITERS):
"""Class constructor.
Args:
a_w2v (bool):
use pre-trained word2vec instance
a_lstsq (bool):
pre-train task-specific word embeddings, but use least-square
method to generate embeddings for unknown words from generic
word2vec vectors
a_max_iters (int):
maximum number of iterations
"""
# access to the original word2vec resource
if a_lstsq:
a_w2v = True
if a_w2v:
self.w2v = Word2Vec # singleton object
else:
self.w2v = None
self.lstsq = a_lstsq
self._plain_w2v = self.w2v and not self.lstsq
# matrix mapping word2vec to task-specific embeddings
self.max_iters = a_max_iters
self.w2emb = None
self.ndim = -1 # vector dimensionality will be initialized later
self.intm_dim = -1
# mapping from word to its embedding index
self.unk_w_i = 0
self._aux_keys = set((0, ))
self.w_i = 1
self.w2emb_i = dict()
# mapping from connective to its embedding index
self.unk_c_i = 0
self.c_i = 1
self.c2emb_i = dict()
# variables needed for training
self._trained = False
self._params = []
self._w_stat = self._pred_class = None
self.use_dropout = theano.shared(floatX(0.))
self.W_EMB = self.CONN_EMB = self._cost = self._dev_cost = None
# initialize theano functions to None
self._reset_funcs()
# set up functions for obtaining word embeddings at train and test
# times
self._init_wemb_funcs()
def __init__(self, a_w2v=False, a_lstsq=False, a_max_iters=MAX_ITERS):
"""Class constructor.
Args:
a_w2v (bool):
use pre-trained word2vec instance
a_lstsq (bool):
pre-train task-specific word embeddings, but use least-square
method to generate embeddings for unknown words from generic
word2vec vectors
a_max_iters (int):
maximum number of iterations
"""
# access to the original word2vec resource
if a_lstsq:
a_w2v = True
if a_w2v:
self.w2v = Word2Vec # singleton object
else:
self.w2v = None
self.lstsq = a_lstsq
self._plain_w2v = self.w2v and not self.lstsq
# matrix mapping word2vec to task-specific embeddings
self.max_iters = a_max_iters
self.w2emb = None
self.ndim = -1 # vector dimensionality will be initialized later
self.intm_dim = -1
# mapping from word to its embedding index
self.unk_w_i = 0
self._aux_keys = set((0, ))
self.w_i = 1
self.w2emb_i = dict()
# mapping from connective to its embedding index
self.unk_c_i = 0
self.c_i = 1
self.c2emb_i = dict()
# variables needed for training
self._trained = False
self._params = []
self._w_stat = self._pred_class = None
self.use_dropout = theano.shared(floatX(0.))
self.W_EMB = self.CONN_EMB = self._cost = self._dev_cost = None
# initialize theano functions to None
self._reset_funcs()
# set up functions for obtaining word embeddings at train and test
# times
self._init_wemb_funcs()
def _init_w2v_emb(self):
"""Initialize word2vec embedding matrix.
"""
w_emb = np.empty((self.w_i, self.ndim))
w_emb[self.unk_w_i, :] = 1e-2 # prevent zeros in this row
for w, i in self.w2emb_i.iteritems():
if i == self.unk_w_i:
continue
w_emb[i] = self.w2v[w]
self.W_EMB = theano.shared(value=floatX(w_emb),
name="W_EMB")
# We unload embeddings every time before the training to free more
# memory. Feel free to comment the line below, if you have plenty of
# RAM.
self.w2v.unload()
def _init_lstm(self, a_invars, a_sfx="-forward"):
"""Initialize LSTM layer.
Args:
a_invars (list(theano.shared)):
list of input parameters as symbolic theano variable
a_sfx (str):
suffix to use for function and parameter names
Returns:
(2-tuple):
parameters to be optimized and list of symbolic outputs from the
function
"""
intm_dim = self.intm_dim
# initialize transformation matrices and bias term
W_dim = (intm_dim, self.ndim)
W = np.concatenate([
ORTHOGONAL(W_dim),
ORTHOGONAL(W_dim),
ORTHOGONAL(W_dim),
ORTHOGONAL(W_dim)
],
axis=0)
W = theano.shared(value=W, name="W" + a_sfx)
U_dim = (intm_dim, intm_dim)
U = np.concatenate([
ORTHOGONAL(U_dim),
ORTHOGONAL(U_dim),
ORTHOGONAL(U_dim),
ORTHOGONAL(U_dim)
],
axis=0)
U = theano.shared(value=U, name="U" + a_sfx)
V = ORTHOGONAL(U_dim) # V for vendetta
V = theano.shared(value=V, name="V" + a_sfx)
b_dim = (1, intm_dim * 4)
b = theano.shared(value=HE_UNIFORM(b_dim), name="b" + a_sfx)
params = [W, U, V, b]
# initialize dropout units
w_do = theano.shared(value=floatX(np.ones((4 * intm_dim, ))),
name="w_do")
w_do = self._init_dropout(w_do)
u_do = theano.shared(value=floatX(np.ones((4 * intm_dim, ))),
name="u_do")
u_do = self._init_dropout(u_do)
# custom function for splitting up matrix parts
def _slice(_x, n, dim):
if _x.ndim == 3:
return _x[:, :, n * dim:(n + 1) * dim]
return _x[:, n * dim:(n + 1) * dim]
# define recurrent LSTM unit
def _step(x_, h_, c_, W, U, V, b, w_do, u_do):
"""Recurrent LSTM unit.
Note:
The general order of function parameters to fn is:
sequences (if any), prior result(s) (if needed),
non-sequences (if any)
Args:
x_ (theano.shared): input vector
h_ (theano.shared): output vector
c_ (theano.shared): memory state
W (theano.shared): input transform matrix
U (theano.shared): inner-state transform matrix
V (theano.shared): output transform matrix
b (theano.shared): bias vector
w_do (TT.col): dropout unit for the W matrix
u_do (TT.col): dropout unit for the U matrix
Returns:
(2-tuple(h, c))
new hidden and memory states
"""
# pre-compute common terms:
# W \in R^{236 x 100}
# x \in R^{1 x 100}
# U \in R^{236 x 59}
# h \in R^{1 x 59}
# b \in R^{1 x 236}
# w_do \in R^{236 x 1}
# u_do \in R^{236 x 1}
# xhb \in R^{1 x 236}
xhb = (TT.dot(W * w_do.dimshuffle(
(0, 'x')), x_.T) + TT.dot(U * u_do.dimshuffle(
(0, 'x')), h_.T)).T + b
# i \in R^{1 x 59}
i = TT.nnet.sigmoid(_slice(xhb, 0, intm_dim))
# f \in R^{1 x 59}
f = TT.nnet.sigmoid(_slice(xhb, 1, intm_dim))
# c \in R^{1 x 59}
c = TT.tanh(_slice(xhb, 2, intm_dim))
c = i * c + f * c_
# V \in R^{59 x 59}
# o \in R^{1 x 59}
o = TT.nnet.sigmoid(_slice(xhb, 3, intm_dim) + TT.dot(V, c.T).T)
# h \in R^{1 x 59}
h = o * TT.tanh(c)
# return current output and memory state
return h.flatten(), c.flatten()
m = 0
n = intm_dim
ov = None
outvars = []
for iv, igbw in a_invars:
m = iv.shape[0]
ret, _ = theano.scan(_step,
sequences=[iv],
outputs_info=[
floatX(np.zeros((n, ))),
floatX(np.zeros((n, )))
],
non_sequences=[W, U, V, b, w_do, u_do],
name="LSTM" + str(iv) + a_sfx,
n_steps=m,
truncate_gradient=TRUNCATE_GRADIENT,
go_backwards=igbw)
ov = ret[0]
outvars.append(ov)
return params, outvars
