def forward(
self,
input,
units,
W=initializers.he,
b=numpy.zeros,
trainable_W=True,
trainable_b=True,
):
self.create_variable(
"W",
W,
(numpy.prod(input.shape[1:]), units),
trainable=trainable_W,
)
self.create_variable("b", b, (units, ), trainable=trainable_b)
if numpy.prod(input.shape[1:]) != self.W.shape[0]:
raise RuntimeError(
"input to Dense layer {} has different dim".format(self))
if self.b is not None:
return T.dot(T.flatten2d(input), self.W) + self.b
else:
return T.dot(T.flatten2d(input), self.W)
def gate(
carry,
x,
Wf,
Uf,
bf,
Wi,
Ui,
bi,
Wo,
Uo,
bo,
Wc,
Uc,
bc,
sigma_g,
sigma_c,
sigma_h,
):
h, c = carry[0], carry[1]
f = sigma_g(T.dot(x, Wf) + bf + T.dot(h, Uf))
i = sigma_g(T.dot(x, Wi) + bi + T.dot(h, Ui))
o = sigma_g(T.dot(x, Wo) + bo + T.dot(h, Uo))
ctilde = sigma_c(T.dot(x, Wc) + bc + T.dot(h, Uc))
cnew = f * c + i * ctilde
hnew = o * sigma_h(cnew)
return T.stack([hnew, cnew]), h
def __init__(
self,
input,
units,
W=initializers.glorot_uniform,
b=numpy.zeros,
trainable_W=True,
trainable_b=True,
W_preprocessor=None,
b_preprocessor=None,
inplace_W=False,
inplace_b=False,
flatten=True,
):
if flatten:
width_in = numpy.prod(input.shape[1:])
else:
width_in = input.shape[-1]
W = create_variable(
"W",
W,
(units, width_in),
trainable=trainable_W,
preprocessor=W_preprocessor,
inplace=inplace_W,
)
b = create_variable(
"b",
b,
(units, ),
trainable=trainable_b,
preprocessor=b_preprocessor,
inplace=inplace_b,
)
if flatten:
flat_input = T.flatten2d(input)
else:
flat_input = input
if b is not None and W is None:
return flat_input + b
elif b is None and W is not None:
return T.dot(flat_input, W.T)
elif b is not None and W is not None:
return T.dot(flat_input, W.T) + b
else:
return flat_input
def minimal_gate(h, x, Wh, Uh, bh, Wz, Uz, bz, sigma, phi):
ft = sigma(T.dot(x, Wz) + bz + T.dot(h, Uz))
h_hat = phi(T.dot(x, Wh) + bh + T.dot(h * ft, Uh))
ht = (1 - ft) * h + ft * h_hat
return ht, ht
def full_gate(h, x, Wh, Uh, bh, Wz, Uz, bz, Wr, Ur, br, sigma, phi):
zt = sigma(T.dot(x, Wz) + bz + T.dot(h, Uz))
rt = sigma(T.dot(x, Wr) + br + T.dot(h, Ur))
h_hat = phi(T.dot(x, Wh) + bh + T.dot(h * rt, Uh))
ht = (1 - zt) * h + zt * h_hat
return ht, ht
def gate(h, x, W, H, b, sigma):
ht = sigma(T.dot(x, W) + b + T.dot(h, H))
return ht, ht
def forward(self, input):
if numpy.prod(input.shape[1:]) != self.W.shape[0]:
raise RuntimeError(
'input to Dense layer {} has different dim'.format(self))
return T.dot(T.flatten2d(input), self.W) + self.b