def create_rnn_op(self): x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', append_batch_size=False) x.stop_gradient = False rnn_0 = layers.StaticRNN() with rnn_0.step(): x_t = rnn_0.step_input(x) mem_pre = rnn_0.memory(shape=[-1, self.input_dim], batch_ref=x) mem = layers.elementwise_add(x=mem_pre, y=x_t) rnn_0.update_memory(mem_pre, mem) rnn_0.output(mem) rnn_1 = layers.StaticRNN() with rnn_1.step(): mem_pre = rnn_1.memory(shape=[-1, self.input_dim], batch_ref=x) x_t = rnn_1.step_input(x) last_rnn_output = rnn_0() last_rnn_sum = fluid.layers.reduce_sum(last_rnn_output) mem = layers.elementwise_add(x=x_t, y=last_rnn_sum) y = layers.elementwise_add(x=mem_pre, y=mem) rnn_1.update_memory(mem_pre, mem) rnn_1.output(y) return rnn_1()
def setUp(self): self.setup_program() self.data_field = {"x", "h_boot"} self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = PySimpleRNN1(self.input_shape, self.output_shape) with fluid.program_guard(self.main_program, self.startup_program): x = layers.data( shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', append_batch_size=False) x.stop_gradient = False h_boot = layers.data( shape=[self.input_dim], dtype='float32', name='h_boot') h_boot.stop_gradient = False forward_only_rnn = layers.StaticRNN() with forward_only_rnn.step(): h_pre = forward_only_rnn.memory(init=h_boot) x_t = forward_only_rnn.step_input(x) h = layers.scale( x=layers.elementwise_add( x=h_pre, y=x_t), scale=self.py_rnn.scale) forward_only_rnn.update_memory(h_pre, h) forward_only_rnn.output(h) forward_only_output = forward_only_rnn() forward_only_output.stop_gradient = True self.forward_only_output = layers.mean(forward_only_output) rnn = layers.StaticRNN() with rnn.step(): h_pre = rnn.memory(init=h_boot) x_t = rnn.step_input(x) h = layers.scale( x=layers.elementwise_add( x=h_pre, y=x_t), scale=self.py_rnn.scale) rnn.update_memory(h_pre, h) rnn.output(h) self.output = layers.mean(rnn())
def create_rnn_op(self): x = layers.data( shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', append_batch_size=False, **self.p_info) x.stop_gradient = False h_boot = layers.data( shape=[self.input_dim], dtype='float32', name='h_boot', **self.p_info) h_boot.stop_gradient = False rnn = layers.StaticRNN(main_program=self.main_program) with rnn.step(): h_pre = rnn.memory(init=h_boot) x_t = rnn.step_input(x) h = layers.scale( x=layers.elementwise_add( x=h_pre, y=x_t, **self.p_info), scale=self.py_rnn.scale, **self.p_info) rnn.update_memory(h_pre, h) rnn.output(h) return rnn()
def create_rnn_op(self): x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', append_batch_size=False) x.stop_gradient = False h_boot1 = layers.data(shape=[self.batch_size, self.input_dim], dtype='float32', name='h_boot1', append_batch_size=False) h_boot1.stop_gradient = False h_boot2 = layers.data(shape=[self.batch_size, self.input_dim], dtype='float32', name='h_boot2', append_batch_size=False) h_boot2.stop_gradient = False rnn = layers.StaticRNN() with rnn.step(): h_pre1 = rnn.memory(init=h_boot1) h_pre2 = rnn.memory(init=h_boot2) x_t = rnn.step_input(x) mem1 = layers.scale(x=h_pre1, scale=1.0) mem2 = layers.scale(x=h_pre2, scale=1.0) out = layers.sums(input=[mem1, x_t, mem2]) rnn.update_memory(h_pre1, mem1) rnn.update_memory(h_pre2, mem2) rnn.output(out) return rnn()
def create_rnn_op(self): x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', append_batch_size=False) x.stop_gradient = False h_boot = layers.data(shape=[self.input_dim], dtype='float32', name='h_boot') h_boot.stop_gradient = False rnn = layers.StaticRNN() with rnn.step(): h_pre = rnn.memory(init=h_boot) x_t = rnn.step_input(x) temp_l = layers.fc(input=x_t, size=self.input_dim, param_attr='W', bias_attr=False) temp_r = layers.fc(input=h_pre, size=self.input_dim, param_attr='U', bias_attr=False) h = layers.sigmoid(x=layers.elementwise_add(x=temp_l, y=temp_r)) rnn.update_memory(h_pre, h) rnn.output(h) return rnn()
def create_rnn_op(self): x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', append_batch_size=False) x.stop_gradient = False emb = layers.data( name='emb', shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', append_batch_size=False) emb.stop_gradient = False w1 = layers.data(shape=[self.input_dim, self.input_dim], dtype='float32', name='w1', append_batch_size=False) w1.stop_gradient = False w2 = layers.data(shape=[self.input_dim * 2, self.input_dim], dtype='float32', name='w2', append_batch_size=False) w2.stop_gradient = False rnn = layers.StaticRNN() def dot_attention(query, memory): attn = layers.matmul(query, memory, transpose_y=True) weight = layers.softmax(attn) weight_memory = layers.matmul(weight, memory) return weight_memory, weight y = layers.matmul(emb, w1) with rnn.step(): pre_h = rnn.memory(shape=(self.sent_len, self.input_dim), batch_ref=x, init_value=0.0) step_in = rnn.step_input(x) concat_in = layers.concat([step_in, pre_h], 1) new_h = layers.matmul(concat_in, w2) new_h = layers.unsqueeze(new_h, [1]) new_h, _ = dot_attention(new_h, y) new_h = layers.squeeze(new_h, [1]) rnn.update_memory(pre_h, new_h) rnn.step_output(new_h) return rnn()
def static_rnn_net(self): x = layers.data(shape=[SEQ_LEN, BATCH_SIZE, INPUT_DIM], dtype="float32", name="x", append_batch_size=False) x.stop_gradient = False rnn = layers.StaticRNN() with rnn.step(): x_t = rnn.step_input(x) h_pre = rnn.memory(shape=[-1, INPUT_DIM], batch_ref=x_t) h = layers.scale(x=layers.elementwise_add(x=h_pre, y=x_t), scale=self.scale) rnn.update_memory(h_pre, h) rnn.output(h) return layers.mean(rnn())
def create_rnn_op(self): x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', append_batch_size=False) x.stop_gradient = False rnn = layers.StaticRNN() with rnn.step(): mem_pre = rnn.memory(shape=[-1, self.input_dim], batch_ref=x) x_t = rnn.step_input(x) mem = layers.elementwise_add(x=mem_pre, y=x_t) rnn.update_memory(mem_pre, mem) rnn.output(mem) return rnn()
def create_rnn_op(self): x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], dtype="float32", name="x", append_batch_size=False) x.stop_gradient = False h_boot = layers.data(shape=[self.input_dim], dtype="float32", name="h_boot") h_boot.stop_gradient = True rnn = layers.StaticRNN() with rnn.step(): h_pre = rnn.memory(init=h_boot) # init doesn't have gradient x_t = rnn.step_input(x) temp_l = layers.fc( input=x_t, size=self.input_dim, param_attr=ParamAttr( name="W", initializer=fluid.initializer.ConstantInitializer(1.0)), bias_attr=False) temp_r = layers.fc( input=h_pre, size=self.input_dim, param_attr=ParamAttr( name="U", initializer=fluid.initializer.ConstantInitializer(0.0)), bias_attr=False) h = layers.sigmoid(x=layers.elementwise_add(temp_l, temp_r)) rnn.update_memory(h_pre, h) rnn.output(h) return rnn()