def _build(self):
self._cell_forward = ph.GRUCell('cell_forward',
input_size=self._input_size,
state_size=self._state_size)
self._cell_backward = ph.GRUCell('cell_backward',
input_size=self._input_size,
state_size=self._state_size)
def _build(self):
# 网络模块定义 --- build
self._emb = photinia.Linear('EMB', self._voc_size, self._emb_size)
self._cell = photinia.GRUCell('CELL', self._emb_size, self._state_size)
self._lin = photinia.Linear('LIN', self._state_size, self._voc_size)
# 输入定义
seq = tf.placeholder(
shape=(None, None, self._voc_size),
dtype=photinia.dtype
)
seq_0 = seq[:, :-1, :]
seq_1 = seq[:, 1:, :]
batch_size = tf.shape(seq)[0]
# RNN结构
init_state = tf.zeros(
shape=(batch_size, self._state_size),
dtype=photinia.dtype
)
states = tf.scan(
fn=self._rnn_step,
elems=tf.transpose(seq_0, (1, 0, 2)),
initializer=init_state
)
probs = tf.map_fn(
fn=self._state_to_prob,
elems=states
)
outputs = tf.map_fn(
fn=self._prob_to_output,
elems=probs
)
probs = tf.transpose(probs, (1, 0, 2))
outputs = tf.transpose(outputs, (1, 0, 2))
outputs = tf.concat((seq[:, 0:1, :], outputs), 1)
loss = tf.reduce_mean(-tf.log(1e-5 + tf.reduce_sum(seq_1 * probs, 2)), 1)
loss = tf.reduce_mean(loss)
self._add_slot(
'train',
outputs=loss,
inputs=seq,
updates=tf.train.AdamOptimizer(1e-3).minimize(loss)
)
self._add_slot(
'evaluate',
outputs=outputs,
inputs=seq
)
#
word = tf.placeholder(
shape=(None, self._voc_size),
dtype=photinia.dtype
)
emb = self._emb.setup(word)
emb = photinia.lrelu(emb)
self._add_slot(
'embedding',
outputs=emb,
inputs=word
)
def _build(self):
ph.GRUCell('cell',
self._wemb_size,
self._state_size,
with_bias=False,
activation=self._activation,
w_init=ph.TruncatedNormal(0, 1e-3),
u_init=ph.TruncatedNormal(0, 1e-3))
def _build(self):
if self._emb_layer is None:
self._emb_layer = ph.Linear('emb_layer', self._voc_size,
self._emb_size)
else:
self._emb_size = self._emb_layer.output_size
self._cell = ph.GRUCell('cell', self._emb_size, self._state_size)
self._out_layer = ph.Linear('out_layer', self._state_size,
self._voc_size)
def _build(self):
input_x = tf.tile(self._input_x, [self._num_mc_samples] + [1] * (len(self._input_x.shape) - 1))
g_net = self._glimpse_network
l_net = self._location_network
input_stddev = tf.placeholder(
shape=(),
dtype=ph.dtype,
name='input_stddev'
)
cell = self._cell = ph.GRUCell(
'cell',
g_net.output_size,
self._state_size,
w_init=ph.init.GlorotUniform()
)
batch_size = tf.shape(input_x)[0]
init_state = tf.zeros(shape=(batch_size, self._state_size), dtype=ph.dtype)
init_loc = tf.random_uniform((batch_size, 2), minval=-1, maxval=1)
def _loop(acc, _):
prev_state, loc, _ = acc
g = g_net.setup(input_x, loc)
state = cell.setup(g, prev_state)
next_loc, next_mean = l_net.setup(state, input_stddev)
return state, next_loc, next_mean
states, locs, means = tf.scan(
fn=_loop,
elems=tf.zeros(shape=(self._num_steps,), dtype=tf.int8),
initializer=(init_state, init_loc, init_loc)
) # (num_steps, batch_size, *)
baseline_layer = self._baseline_layer = ph.Linear('baseline_layer', self._state_size, 1)
def _make_baseline(state):
baseline = baseline_layer.setup(state) # (batch_size, 1)
baseline = tf.reshape(baseline, (-1,)) # (batch_size,)
return baseline
baselines = tf.map_fn(_make_baseline, states) # (num_steps, batch_size)
baselines = tf.transpose(baselines) # (batch_size, num_steps)
predict_layer = self._predict_layer = ph.Linear('predict_layer', self._state_size, self._num_classes)
last_state = states[-1] # (batch_size, state_size)
prob = predict_layer.setup(last_state)
prob = tf.nn.softmax(prob) # (batch_size, num_classes)
label = tf.argmax(prob, 1) # (batch_size,)
self._step_predict = ph.Step(
inputs=input_x,
outputs=label,
givens={input_stddev: 1e-3}
)
self._input_label = ph.placeholder('input_label', (None,), tf.int64)
input_label = tf.tile(self._input_label, (self._num_mc_samples,))
prob_ = tf.one_hot(input_label, self._num_classes) # (batch_size, num_classes)
predict_loss = self._predict_loss = -tf.reduce_mean(ph.ops.log_likelihood(prob_, prob))
reward = tf.cast(tf.equal(label, input_label), tf.float32) # (batch_size,)
rewards = tf.reshape(reward, (-1, 1)) # (batch_size, 1)
rewards = tf.tile(rewards, (1, self._num_steps)) # (batch_size, num_steps)
rewards = tf.stop_gradient(rewards)
baseline_loss = self._baseline_loss = tf.reduce_mean(ph.ops.mean_square_error(rewards, baselines))
advantages = rewards - tf.stop_gradient(baselines)
logll = self._log_gaussian(locs, means, input_stddev)
logll = tf.reduce_sum(logll, 2) # (num_steps, batch_size)
logll = tf.transpose(logll) # (batch_size, num_steps)
logll_ratio = self._logll_ratio = tf.reduce_mean(logll * advantages)
loss = self._loss = predict_loss - logll_ratio + baseline_loss
if self._reg is not None:
self._reg.setup(self.get_trainable_variables())
update = self._optimizer.minimize(loss + self._reg.get_loss())
else:
update = self._optimizer.minimize(loss)
self._step_train = ph.Step(
inputs=(self._input_x, self._input_label),
outputs=(loss, tf.reduce_mean(rewards)),
updates=update,
givens={input_stddev: self._stddev}
)
def _build(self):
self._emb_layer = ph.Linear(
'emb_layer', self._voc_size,
self._emb_size) if self._emb_size is not None else None
self._cell = ph.GRUCell('cell', self._emb_size, self._state_size)
def _build(self):
self._cell = ph.GRUCell('cell',
input_size=self._input_size,
state_size=self._state_size)
def _build(self):
self._emb_layer = ph.Linear('emb_layer', self._voc_size,
self._emb_size)
self._cell = ph.GRUCell('cell', self._emb_size, self._state_size)
