def compute_loss(self, input_vocab, output_vocab, window_words, hidden_states): g, rnn_distribution, a = self.decode_one_step(input_vocab, window_words, hidden_states) # define p_vocab as 0 if output word is not in vocab p_vocab = F.select_item( rnn_distribution, xp.array( [self.vocab[output_vocab]], dtype=xp.int32)) if output_vocab in self.vocab else Variable( xp.array([0.0], dtype=xp.float32)) # compute cross entropy indexes = [i for i, x in enumerate(window_words) if x == output_vocab] exist_var = Variable(xp.array([0], dtype=xp.float32)) for idx in indexes: exist_var += F.select_item(a, xp.array([idx], dtype=xp.int32)) p_ptr = F.cast(exist_var, xp.float32) if indexes else Variable( xp.array([0.0], dtype=xp.float32)) cross_entropy = -F.log( F.linear_interpolate(g, p_vocab, p_ptr) + Variable(xp.array([0.01], dtype=xp.float32))) # compute attention loss attention_loss = F.cast(-F.log(g + exist_var), xp.float32) if indexes else Variable( xp.array([0.0], dtype=xp.float32)) return cross_entropy + attention_loss
def _call_1step(net: NStepRNNBase, hidden: ArrayLike, input: ArrayLike): if hidden is None: hidden = net.init_hx(input)[0] x = input h = hidden w = net.ws[0] b = net.bs[0] xw = F.concat([w[0], w[1], w[2]], axis=0) hw = F.concat([w[3], w[4], w[5]], axis=0) xb = F.concat([b[0], b[1], b[2]], axis=0) hb = F.concat([b[3], b[4], b[5]], axis=0) gru_x = F.linear(x, xw, xb) gru_h = F.linear(h, hw, hb) W_r_x, W_z_x, W_x = F.split_axis(gru_x, 3, axis=1) U_r_h, U_z_h, U_x = F.split_axis(gru_h, 3, axis=1) r = F.sigmoid(W_r_x + U_r_h) z = F.sigmoid(W_z_x + U_z_h) h_bar = F.tanh(W_x + r * U_x) h = F.linear_interpolate(z, hidden, h_bar) return h
def check_forward(self, p_data, x_data, y_data): p = chainer.Variable(p_data) x = chainer.Variable(x_data) y = chainer.Variable(y_data) z = functions.linear_interpolate(p, x, y) self.assertEqual(z.data.dtype.type, self.dtype) expect = self.p * self.x + (1 - self.p) * self.y testing.assert_allclose(z.data, expect, **self.check_forward_options)
def check_forward(self, p_data, x_data, y_data): p = chainer.Variable(p_data) x = chainer.Variable(x_data) y = chainer.Variable(y_data) z = functions.linear_interpolate(p, x, y) self.assertEqual(z.data.dtype.type, self.dtype) expect = self.p * self.x + (1 - self.p) * self.y testing.assert_allclose( z.data, expect, **self.check_forward_options)
def __call__(self, x, h): z = self.W_z(x) h_bar = self.W(x) if h is not None: r = F.sigmoid(self.W_r(x) + self.U_r(h)) z += self.U_z(h) h_bar += self.U(r * h) z = F.sigmoid(z) h_bar = F.tanh(h_bar) if h is not None: h_new = F.linear_interpolate(z, h_bar, h) else: h_new = z * h_bar return h_new
def __call__(self, x): z = self.W_z(x) h_bar = self.W(x) if self.h is not None: r = F.sigmoid(self.W_r(x) + self.U_r(self.h)) z += self.U_z(self.h) h_bar += self.U(r * self.h) z = F.sigmoid(z) h_bar = F.tanh(h_bar) if self.h is not None: h_new = F.linear_interpolate(z, h_bar, self.h) else: h_new = z * h_bar self.h = h_new # save the state return h_new
def __call__(self, word, context): #like a statefulGRU code z = self.W_z(word) h_bar = self.W(word) if self.h is not None: r = chainFunc.sigmoid( self.W_r(word) + self.U_r(self.h) + self.C_r(context)) z += self.U_z(self.h) + self.C_z(context) h_bar += self.U(r * self.h) + self.C(context) z = chainFunc.sigmoid(z) h_bar = chainFunc.tanh(h_bar) if self.h is not None: h_new = chainFunc.linear_interpolate(z, h_bar, self.h) else: h_new = z * h_bar self.h = h_new return self.h
def linear_interpolate(): x0 = rand((1, 2, 3, 4)) x1 = rand((1, 2, 3, 4)) x2 = rand((1, 2, 3, 4)) y = F.linear_interpolate(x0, x1, x2) return {'input-0': x0, 'input-1': x1, 'input-2': x2}, {'out': y}
def forward(self, inputs, device): p, x, y = inputs ret = functions.linear_interpolate(p, x, y) ret = functions.cast(ret, numpy.float64) return ret,
def forward(self, inputs, device): p, x, y = inputs ret = functions.linear_interpolate(p, x, y) ret = functions.cast(ret, numpy.float64) return ret,