def inference(x, n_in=None, n_time=None, n_hidden=None, n_out=None):
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.01)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.zeros(shape, dtype=tf.float32)
return tf.Variable(initial)
# 時系列データの形式をAPIの仕様に合わせるため、最終的に
# (ミニバッチサイズ, 入力次元数) が時間長分ある形に変形
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, n_in])
x = tf.split(x, n_time, 0)
cell_forward = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
cell_backward = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, _, _ = \
tf.nn.static_bidirectional_rnn(cell_forward, cell_backward, x,
dtype=tf.float32)
W = weight_variable([n_hidden * 2, n_out])
b = bias_variable([n_out])
y = tf.nn.softmax(tf.matmul(outputs[-1], W) + b)
return y
def lstm_model3(inputs, reuse=False, layers_number=2, num_units=256, scope="l"):
shape = inputs[0].shape
observation_n = []
for i in range(len(inputs)):
obs = inputs[i]
if not reuse and i == 0:
reuse = False
else:
reuse = True
x = []
with tf.variable_scope(scope, reuse=reuse):
for j in range(shape[2]):
dr_reuse = True
if j == 0 and not reuse:
dr_reuse = False
out = layers.fully_connected(obs[:, :, j], num_outputs=num_units * 4, activation_fn=tf.nn.relu,
scope="first", reuse=dr_reuse)
out = layers.fully_connected(out, num_outputs=num_units, activation_fn=tf.nn.relu,
scope="second", reuse=dr_reuse)
x.append(tf.expand_dims(out, 2))
x = tf.concat(x, 2)
lstm_size = x.shape[1]
# dimension reduction 3096->1024->256
with tf.variable_scope(scope, reuse=reuse):
x = tf.transpose(x, (2, 0, 1)) # (time_steps, batch_size, state_size)
lstm_cell = rnn.BasicLSTMCell(lstm_size, forget_bias=1, state_is_tuple=True)
cell = rnn.MultiRNNCell([lstm_cell] * layers_number, state_is_tuple=True)
with tf.variable_scope("Multi_Layer_RNN"):
cell_outputs, states = tf.nn.dynamic_rnn(cell, x, dtype=tf.float32)
outputs = cell_outputs[-1:, :, :]
outputs = tf.squeeze(outputs, 0)
observation_n.append(outputs)
return observation_n
def RNN(x, weights, biases):
x = tf.transpose(x, [1, 0, 2]) #64,13,21 ->13,64,21
x = tf.reshape(x, [-1, n_input]) #13*64,21
#将x 切成n_steps=28个子张量
x = tf.split(axis=0, num_or_size_splits=n_window, value=x)
lstm_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def __init__(self, args, data, infer=False):
if infer:
args.batch_size = 1
args.seq_length = 1
with tf.name_scope('inputs'):
self.input_data = tf.placeholder(
tf.int32, [args.batch_size, args.seq_length])
self.target_data = tf.placeholder(
tf.int32, [args.batch_size, args.seq_length])
with tf.name_scope('model'):
self.cell = rnn_cell.BasicLSTMCell(args.state_size)
self.cell = rnn_cell.MultiRNNCell([self.cell] * args.num_layers)
self.initial_state = self.cell.zero_state(args.batch_size,
tf.float32)
with tf.variable_scope('rnnlm'):
w = tf.get_variable('softmax_w',
[args.state_size, data.vocab_size])
b = tf.get_variable('softmax_b', [data.vocab_size])
with tf.device("/cpu:0"):
embedding = tf.get_variable(
'embedding', [data.vocab_size, args.state_size])
inputs = tf.nn.embedding_lookup(embedding, self.input_data)
outputs, last_state = tf.nn.dynamic_rnn(
self.cell, inputs, initial_state=self.initial_state)
with tf.name_scope('loss'):
output = tf.reshape(outputs, [-1, args.state_size])
self.logits = tf.matmul(output, w) + b
self.probs = tf.nn.softmax(self.logits)
self.last_state = last_state
targets = tf.reshape(self.target_data, [-1])
loss = seq2seq.sequence_loss_by_example(
[self.logits], [targets],
[tf.ones_like(targets, dtype=tf.float32)])
self.cost = tf.reduce_sum(loss) / args.batch_size
tf.summary.scalar('loss', self.cost)
with tf.name_scope('optimize'):
self.lr = tf.placeholder(tf.float32, [])
tf.summary.scalar('learning_rate', self.lr)
optimizer = tf.train.AdamOptimizer(self.lr)
tvars = tf.trainable_variables()
grads = tf.gradients(self.cost, tvars)
for g in grads:
tf.summary.histogram(g.name, g)
grads, _ = tf.clip_by_global_norm(grads, args.grad_clip)
self.train_op = optimizer.apply_gradients(zip(grads, tvars))
self.merged_op = tf.summary.merge_all()
def reccurent_neural_network(x):
layer = {
'weights': tf.Variable(tf.random_normal([rnn_size, n_classes])),
'biases': tf.Variable(tf.random_normal([n_classes]))
}
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, chunk_size])
x = tf.split(x, n_chunks, 0)
lstm_cell = rnn_cell.BasicLSTMCell(rnn_size)
outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)
output = tf.matmul(outputs[-1], layer['weights']) + layer['biases']
return output
def inference(x, y, n_batch, is_training,
input_digits=None, output_digits=None,
n_hidden=None, n_out=None):
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.01)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.zeros(shape, dtype=tf.float32)
return tf.Variable(initial)
# Encoder
encoder = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
state = encoder.zero_state(n_batch, tf.float32)
encoder_outputs = []
encoder_states = []
with tf.variable_scope('Encoder'):
for t in range(input_digits):
if t > 0:
tf.get_variable_scope().reuse_variables()
(output, state) = encoder(x[:, t, :], state)
encoder_outputs.append(output)
encoder_states.append(state)
# Decoder
decoder = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
state = encoder_states[-1]
decoder_outputs = [encoder_outputs[-1]]
# 出力層の重みとバイアスを事前に定義
V = weight_variable([n_hidden, n_out])
c = bias_variable([n_out])
outputs = []
with tf.variable_scope('Decoder'):
for t in range(1, output_digits):
if t > 1:
tf.get_variable_scope().reuse_variables()
if is_training is True:
(output, state) = decoder(y[:, t-1, :], state)
else:
# 直前の出力を入力に用いる
linear = tf.matmul(decoder_outputs[-1], V) + c
out = tf.nn.softmax(linear)
outputs.append(out)
out = tf.one_hot(tf.argmax(out, -1), depth=output_digits)
(output, state) = decoder(out, state)
decoder_outputs.append(output)
if is_training is True:
output = tf.reshape(tf.concat(decoder_outputs, axis=1),
[-1, output_digits, n_hidden])
linear = tf.einsum('ijk,kl->ijl', output, V) + c
# linear = tf.matmul(output, V) + c
return tf.nn.softmax(linear)
else:
# 最後の出力を求める
linear = tf.matmul(decoder_outputs[-1], V) + c
out = tf.nn.softmax(linear)
outputs.append(out)
output = tf.reshape(tf.concat(outputs, axis=1),
[-1, output_digits, n_out])
return output