def restart(self, init_c = Node(), init_h = Node()): """Initializes internal states.""" out_size = self.pwhh.shape()[1] self.wxh = F.parameter(self.pwxh) self.whh = F.parameter(self.pwhh) self.bh = F.parameter(self.pbh) self.c = init_c if init_c.valid() else F.zeros([out_size]) self.h = init_h if init_h.valid() else F.zeros([out_size])
def make_graph(inputs, train): x = F.input(inputs) w1 = F.parameter(pw1) b1 = F.parameter(pb1) h = F.relu(w1 @ x + b1) h = F.dropout(h, .5, train) w2 = F.parameter(pw2) b2 = F.parameter(pb2) return w2 @ h + b2
def forward(self, inputs): batch_size = len(inputs[0]) wlookup = F.parameter(self.pwlookup) wxs = F.parameter(self.pwxs) wsy = F.parameter(self.pwsy) s = F.zeros(Shape([NUM_HIDDEN_UNITS], batch_size)) outputs = [] for i in range(len(inputs) - 1): w = F.pick(wlookup, inputs[i], 1) x = w + s s = F.sigmoid(wxs @ x) outputs.append(wsy @ s) return outputs
def make_graph(inputs): # We first store input values explicitly on GPU 0. x = F.input(inputs, device=dev0) w1 = F.parameter(pw1) b1 = F.parameter(pb1) w2 = F.parameter(pw2) b2 = F.parameter(pb2) # The hidden layer is calculated and implicitly stored on GPU 0. h_on_gpu0 = F.relu(w1 @ x + b1) # `copy()` transfers the hiddne layer to GPU 1. h_on_gpu1 = F.copy(h_on_gpu0, dev1) # The output layer is calculated and implicitly stored on GPU 1. return w2 @ h_on_gpu1 + b2
def encode(self, src_batch, train): # Reversed encoding. src_lookup = F.parameter(self.psrc_lookup_) self.src_lstm_.init() for it in src_batch: x = F.pick(src_lookup, it, 1) x = F.dropout(x, self.dropout_rate_, train) self.src_lstm_.forward(x) # Initializes decoder states. self.trg_lookup_ = F.parameter(self.ptrg_lookup_) self.why_ = F.parameter(self.pwhy_) self.by_ = F.parameter(self.pby_) self.trg_lstm_.init(self.src_lstm_.get_c(), self.src_lstm_.get_h())
def train_func(trainer): dev = D.Naive(12345) Device.set_default(dev) g = Graph() Graph.set_default(g) pw1 = Parameter([8, 2], I.XavierUniform()) pb1 = Parameter([8], I.Constant(0)) pw2 = Parameter([1, 8], I.XavierUniform()) pb2 = Parameter([1], I.Constant(0)) trainer.add_parameter(pw1) trainer.add_parameter(pb1) trainer.add_parameter(pw2) trainer.add_parameter(pb2) input_data = [1, 1, 1, -1, -1, 1, -1, -1] output_data = [1, -1, -1, 1] for i in range(10): g.clear() x = F.input(input_data, Shape([2], 4)) w1 = F.parameter(pw1) b1 = F.parameter(pb1) w2 = F.parameter(pw2) b2 = F.parameter(pb2) h = F.tanh(w1 @ x + b1) y = w2 @ h + b2 t = F.input(output_data, Shape([], 4)) diff = t - y loss = F.batch.mean(diff * diff) trainer.reset_gradients() loss.backward() trainer.update() return [ pw1.value.to_list(), pb1.value.to_list(), pw2.value.to_list(), pb2.value.to_list() ]
def forward(self, inputs, train): batch_size = len(inputs[0]) lookup = F.parameter(self.plookup) self.rnn1.restart() self.rnn2.restart() self.hy.reset() outputs = [] for i in range(len(inputs) - 1): x = F.pick(lookup, inputs[i], 1) x = F.dropout(x, DROPOUT_RATE, train) h1 = self.rnn1.forward(x) h1 = F.dropout(h1, DROPOUT_RATE, train) h2 = self.rnn2.forward(h1) h2 = F.dropout(h2, DROPOUT_RATE, train) outputs.append(self.hy.forward(h2)) return outputs
def forward(self, inputs, train): batch_size = len(inputs[0]) lookup = F.parameter(self.plookup_) self.rnn1_.init() self.rnn2_.init() self.hy_.init() xs = [ F.dropout(F.pick(lookup, inputs[i], 1), DROPOUT_RATE, train) for i in range(len(inputs) - 1) ] hs1 = self.rnn1_.forward(xs) for i in range(len(inputs) - 1): hs1[i] = F.dropout(hs1[i], DROPOUT_RATE, train) hs2 = self.rnn2_.forward(hs1) outputs = [ self.hy_.forward(F.dropout(hs2[i], DROPOUT_RATE, train)) for i in range(len(inputs) - 1) ] return outputs
def encode(self, src_batch, train): """Encodes source sentences and prepares internal states.""" # Embedding lookup. src_lookup = F.parameter(self.psrc_lookup) e_list = [] for x in src_batch: e = F.pick(src_lookup, x, 1) e = F.dropout(e, self.dropout_rate, train) e_list.append(e) # Forward encoding self.src_fw_lstm.restart() f_list = [] for e in e_list: f = self.src_fw_lstm.forward(e) f = F.dropout(f, self.dropout_rate, train) f_list.append(f) # Backward encoding self.src_bw_lstm.restart() b_list = [] for e in reversed(e_list): b = self.src_bw_lstm.forward(e) b = F.dropout(b, self.dropout_rate, train) b_list.append(b) b_list.reverse() # Concatenates RNN states. fb_list = [f_list[i] + b_list[i] for i in range(len(src_batch))] self.concat_fb = F.concat(fb_list, 1) self.t_concat_fb = F.transpose(self.concat_fb) # Initializes decode states. embed_size = self.psrc_lookup.shape()[0] self.trg_lookup = F.parameter(self.ptrg_lookup) self.whj = F.parameter(self.pwhj) self.bj = F.parameter(self.pbj) self.wjy = F.parameter(self.pwjy) self.by = F.parameter(self.pby) self.feed = F.zeros([embed_size]) self.trg_lstm.restart( self.src_fw_lstm.get_c() + self.src_bw_lstm.get_c(), self.src_fw_lstm.get_h() + self.src_bw_lstm.get_h())
def encode(self, src_batch, train): # Embedding lookup. src_lookup = F.parameter(self.psrc_lookup_) e_list = [] for x in src_batch: e = F.pick(src_lookup, x, 1) e = F.dropout(e, self.dropout_rate_, train) e_list.append(e) # Forward encoding self.src_fw_lstm_.init() f_list = [] for e in e_list: f = self.src_fw_lstm_.forward(e) f = F.dropout(f, self.dropout_rate_, train) f_list.append(f) # Backward encoding self.src_bw_lstm_.init() b_list = [] for e in reversed(e_list): b = self.src_bw_lstm_.forward(e) b = F.dropout(b, self.dropout_rate_, train) b_list.append(b) b_list.reverse() # Concatenates RNN states. fb_list = [f_list[i] + b_list[i] for i in range(len(src_batch))] self.concat_fb_ = F.concat(fb_list, 1) self.t_concat_fb_ = F.transpose(self.concat_fb_) # Initializes decode states. self.trg_lookup_ = F.parameter(self.ptrg_lookup_) self.whj_ = F.parameter(self.pwhj_) self.bj_ = F.parameter(self.pbj_) self.wjy_ = F.parameter(self.pwjy_) self.by_ = F.parameter(self.pby_) self.feed_ = F.zeros([self.embed_size_]) self.trg_lstm_.init( self.src_fw_lstm_.get_c() + self.src_bw_lstm_.get_c(), self.src_fw_lstm_.get_h() + self.src_bw_lstm_.get_h())
def init(self): self.w_ = F.parameter(self.pw_) self.b_ = F.parameter(self.pb_)
def init(self): self.w_ = F.parameter(self.pw_) self.bf_ = F.parameter(self.pbf_) self.br_ = F.parameter(self.pbr_)
def main(): dev = D.Naive() # or D.CUDA(gpuid) Device.set_default(dev) # Parameters pw1 = Parameter([8, 2], I.XavierUniform()) pb1 = Parameter([8], I.Constant(0)) pw2 = Parameter([1, 8], I.XavierUniform()) pb2 = Parameter([], I.Constant(0)) # Optimizer optimizer = O.SGD(0.1) # Registers parameters. optimizer.add_parameter(pw1) optimizer.add_parameter(pb1) optimizer.add_parameter(pw2) optimizer.add_parameter(pb2) # Training data input_data = [ np.array([1, 1], dtype=np.float32), # Sample 1 np.array([1, -1], dtype=np.float32), # Sample 2 np.array([-1, 1], dtype=np.float32), # Sample 3 np.array([-1, -1], dtype=np.float32), # Sample 4 ] output_data = [ np.array([1], dtype=np.float32), # Label 1 np.array([-1], dtype=np.float32), # Label 2 np.array([-1], dtype=np.float32), # Label 3 np.array([1], dtype=np.float32), # Label 4 ] g = Graph() Graph.set_default(g) for i in range(10): g.clear() # Builds a computation graph. x = F.input(input_data) w1 = F.parameter(pw1) b1 = F.parameter(pb1) w2 = F.parameter(pw2) b2 = F.parameter(pb2) h = F.tanh(w1 @ x + b1) y = w2 @ h + b2 # Obtains values. y_val = y.to_list() print("epoch ", i, ":") for j in range(4): print(" [", j, "]: ", y_val[j]) # Extends the computation graph to calculate loss values. t = F.input(output_data) diff = t - y loss = F.batch.mean(diff * diff) # Obtains the loss. loss_val = loss.to_float() print(" loss: ", loss_val) # Updates parameters. optimizer.reset_gradients() loss.backward() optimizer.update()
def reset(self): self.w = F.parameter(self.pw) self.b = F.parameter(self.pb)
def restart(self): self.wxh = F.parameter(self.pwxh) self.whh = F.parameter(self.pwhh) self.bh = F.parameter(self.pbh) self.h = self.c = F.zeros([self.out_size])
def init(self): self.wxh_ = F.parameter(self.pwxh_) self.whh_ = F.parameter(self.pwhh_) self.bh_ = F.parameter(self.pbh_) self.h_ = self.c_ = F.zeros([self.out_size_])
def init(self, init_c=Node(), init_h=Node()): self.wxh_ = F.parameter(self.pwxh_) self.whh_ = F.parameter(self.pwhh_) self.bh_ = F.parameter(self.pbh_) self.c_ = init_c if init_c.valid() else F.zeros([self.out_size_]) self.h_ = init_h if init_h.valid() else F.zeros([self.out_size_])
def restart(self): self.w = F.parameter(self.pw) self.bf = F.parameter(self.pbf) self.br = F.parameter(self.pbr)