def BiRNN(x, weights, biases):
# Prepare data shape to match `bidirectional_rnn` function requirements
# Current data input shape: (batch_size, n_steps, n_input)
# Required shape: 'n_steps' tensors list of shape (batch_size, n_input)
# Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input)
x = tf.unstack(x, n_steps, 1)
# Define lstm cells with tensorflow
# Forward direction cell
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
# Backward direction cell
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
# Get lstm cell output
try:
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,
dtype=tf.float32)
except Exception: # Old TensorFlow version only returns outputs not states
outputs = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,
dtype=tf.float32)
# Linear activation, using rnn inner loop last output
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def inference(x, n_in=None, n_time=None, n_hidden=None, n_out=None):
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.01)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.zeros(shape, dtype=tf.float32)
return tf.Variable(initial)
# 시계열 데이터의 형식을 API의 사양에 맞추기 위해 최종적으로
# (미니배치 크기, 입력 차원수)가 시간 길이 만큼 있는 형태로 변형한다
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, n_in])
x = tf.split(x, n_time, 0)
cell_forward = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
cell_backward = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, _, _ = \
rnn.static_bidirectional_rnn(cell_forward, cell_backward, x,
dtype=tf.float32)
W = weight_variable([n_hidden * 2, n_out])
b = bias_variable([n_out])
y = tf.nn.softmax(tf.matmul(outputs[-1], W) + b)
return y
def summery_BiRNN(self, x, weights, biases, Lstmcell, sen_len):
x = tf.unstack(x, self.sentence_length, 1)
with tf.variable_scope('summery_nn'):
outputs, _, _ = rnn.static_bidirectional_rnn(Lstmcell['fw_lstm'],
Lstmcell['bw_lstm'],
x,
dtype=tf.float32)
sum_len = tf.reduce_sum(sen_len, 0)
return tf.matmul(outputs[-1], weights['summery']) + biases['summery']
def BiRnn(x,n_hidden):
# Unstack to get a list of 'n_steps' tensors of shape (batch_size, embedding_size)
x = tf.unstack(x, n_steps, 1)
lstm_fw_cells = rnn.MultiRNNCell([attn_cell(n_hidden) for _ in range(num_layers)] , state_is_tuple=True)
lstm_bw_cells = rnn.MultiRNNCell([attn_cell(n_hidden) for _ in range(num_layers)] , state_is_tuple=True)
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cells, lstm_bw_cells, x,dtype=tf.float32)
outputs = tf.transpose(tf.stack(outputs), perm=[1, 0, 2])
outputs = tf.reshape(outputs,[-1,2*n_hidden])
return outputs
def intent_BiRNN(x, weights, biases):
x = tf.unstack(x, n_words, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
return tf.matmul(outputs[-1], weights['intent_out']) + biases['intent_out']
def BiRNN(x, weights, biases):
x = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=0.5)
# Backward direction cell
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=0.5)
# Get lstm cell output
try:
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,
dtype=tf.float32)
except Exception: # Old TensorFlow version only returns outputs not states
outputs = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,
dtype=tf.float32)
# Linear activation, using rnn inner loop last output
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def RNN(x, weights, biases):
# Prepare data shape to match `rnn` function requirements
# Current data input shape: (batch_size, n_steps, n_input)
# Required shape: 'n_steps' tensors list of shape (batch_size, n_input)
# Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input)
x = tf.unstack(x, n_steps, 1)
# Define a lstm cell with tensorflow
#lstm_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#lstm_cell = rnn.GRUCell(n_hidden)
lstm_cell = rnn.BasicLSTMCell(n_hidden,
forget_bias=1.0,
state_is_tuple=True,
reuse=tf.get_variable_scope().reuse)
lstm_cell_bk = rnn.BasicLSTMCell(n_hidden,
forget_bias=1.0,
state_is_tuple=True,
reuse=tf.get_variable_scope().reuse)
# make the deep rnn
no_of_layers = 8 # layer number of drnn
stacked_lstm = rnn.MultiRNNCell([
#rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple = True, reuse = tf.get_variable_scope().reuse)
rnn.LSTMCell(n_hidden,
use_peepholes=True,
forget_bias=1.0,
state_is_tuple=True,
reuse=tf.get_variable_scope().reuse)
for _ in range(no_of_layers)
])
stacked_lstm_bk = rnn.MultiRNNCell([
#rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple = True, reuse = tf.get_variable_scope().reuse)
rnn.LSTMCell(n_hidden,
use_peepholes=True,
forget_bias=1.0,
state_is_tuple=True,
reuse=tf.get_variable_scope().reuse)
for _ in range(no_of_layers)
])
# providing the dropout for rnn
#lstm_cell = rnn.DropoutWrapper(lstm_cell, output_keep_prob=0.5) # for rnn
stacked_lstm = rnn.DropoutWrapper(stacked_lstm,
output_keep_prob=0.5) # for deep rnn
# Get lstm cell output
#outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32) # single layer rnn
#outputs, states = rnn.static_rnn(stacked_lstm,, x, dtype=tf.float32) # deep rnn
#outputs, states, states_bk = rnn.static_bidirectional_rnn(lstm_cell, lstm_cell_bk, x, dtype=tf.float32) # single layer dirnn
outputs, states, states_bk = rnn.static_bidirectional_rnn(
stacked_lstm, stacked_lstm_bk, x, dtype=tf.float32) # deep dirnn
# Linear activation, using rnn inner loop last output
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def _build_net(self):
self.x = tf.placeholder(tf.float32, [None, time_step, unit_num])
self.y = tf.placeholder(tf.int32, [None, time_step])
seq_x = tf.reshape(self.x, [-1, time_step * unit_num])
seq_x = tf.split(seq_x, time_step, axis=1)
cell_forward = tf.contrib.rnn.BasicLSTMCell(unit_num)
cell_backward = tf.contrib.rnn.BasicLSTMCell(unit_num)
cell_forward = DropoutWrapper(cell_forward,
input_keep_prob=1.0,
output_keep_prob=DROPOUT_RATE)
cell_backward = DropoutWrapper(cell_backward,
input_keep_prob=1.0,
output_keep_prob=DROPOUT_RATE)
# outputs:(time_step, batch_size, 2 * num_units)
outputs, output_state_fw, output_state_bw = \
static_bidirectional_rnn(cell_forward, cell_backward, seq_x, dtype=tf.float32)
# Convert (time_step, batch_size, 2 * num_units) to (batch_size, time_step, 2 * num_units)
rnn_features = tf.transpose(outputs, [1, 0, 2])
rnn_features = tf.reshape(rnn_features, [-1, 2 * unit_num])
# CNN
# You could use more advanced kernel, which is introduced in
# https://github.com/tensorflow/models/blob/master/tutorials/image/cifar10/cifar10.py
filter_deep = 2 # 推荐2 或 1 # 值越大，CNN特征占的比例就越多。 如果是2表示CNN的特征和bi-rnn的特征数量一样。
cnn_W = tf.get_variable("cnn_w",
shape=[EMBEDDING_SIZE, 3, 1, filter_deep])
cnn_b = tf.get_variable("cnn_b", shape=[filter_deep])
# it is better to make the units number equal to the RNN unit number
# cnn_input : (batch_size, time_step, unit_num, 1)
cnn_input = tf.expand_dims(self.x, axis=3)
# conv_features : (batch_size, time_step, unit_num, 2)
conv_features = tf.nn.conv2d(
cnn_input, cnn_W, strides=[1, 1, 1, 1], padding='SAME') + cnn_b
conv_features = tf.nn.dropout(conv_features, keep_prob=DROPOUT_RATE)
conv_features = tf.reshape(conv_features, [-1, unit_num * filter_deep])
all_feature = tf.concat([rnn_features, conv_features], axis=1)
# projection:
W = tf.get_variable(
"projection_w",
[(filter_deep + 2) * unit_num, TAGS_NUM]) #这里的2是指bi-rnn，所以是个常量
b = tf.get_variable("projection_b", [TAGS_NUM])
projection = tf.matmul(all_feature, W) + b
self.outputs = tf.reshape(projection, [-1, time_step, TAGS_NUM])
# self.outputs = tf.transpose(output, [1,0,2]) #BATCH_SIZE * time_step * TAGS_NUM
self.log_likelihood, self.transition_params = tf.contrib.crf.crf_log_likelihood(
self.outputs, self.y, sequence_lengths)
# Add a training op to tune the parameters.
self.loss = tf.reduce_mean(-self.log_likelihood)
self.train_op = tf.train.AdamOptimizer().minimize(self.loss)
def BiRNN(x, weights, biases):
x = tf.unstack(x, n_steps, 1)
#lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_fw_cell = rnn.GRUCell(FLAGS.hidden_size)
#lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.GRUCell(FLAGS.hidden_size)
try:
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
except Exception:
outputs = rnn.static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
return tf.nn.softmax(
tf.matmul(outputs[-1], weights['out']) + biases['out'])
def bilstm_model(inputs, targets, config):
"""
use BasicLSTMCell,MultiRNNCell method to build lstm model
retun logits,cost and others
"""
batch_size = config.batch_size
num_steps = config.num_steps
num_layers = config.num_layers
hidden_size = config.hidden_size
vocab_size = config.vocab_size
target_num = config.target_num # target output number
lstm_fw_cell = rnn.BasicLSTMCell(hidden_size,
forget_bias=0.0,
state_is_tuple=True)
lstm_bw_cell = rnn.BasicLSTMCell(hidden_size,
forget_bias=0.0,
state_is_tuple=True)
cell_fw = rnn.MultiRNNCell([lstm_fw_cell] * num_layers,
state_is_tuple=True)
cell_bw = rnn.MultiRNNCell([lstm_bw_cell] * num_layers,
state_is_tuple=True)
initial_state_fw = cell_fw.zero_state(batch_size, tf.float32)
initial_state_bw = cell_bw.zero_state(batch_size, tf.float32)
#split to get a list of n_steps tensors of shape (batch_size,n_input)
inputs_list = [
tf.squeeze(s, axis=1)
for s in tf.split(value=inputs, num_or_size_splits=num_steps, axis=1)
]
with tf.variable_scope('pos_bilstm'):
outputs, state_fw, state_bw = rnn.static_bidirectional_rnn(
cell_fw,
cell_bw,
inputs_list,
initial_state_fw=initial_state_fw,
initial_state_bw=initial_state_bw)
# outputs is a length T list of output vectors, which is [batch_size, 2*hidden_size]
output = tf.reshape(tf.concat(outputs, 1), [-1, hidden_size * 2])
softmax_w = tf.get_variable('softmax_w', [hidden_size * 2, target_num],
dtype=tf.float32)
softmax_b = tf.get_variable('softmax_b', [target_num], dtype=tf.float32)
logits = tf.matmul(output, softmax_w) + softmax_b
# add extra statistics to monitor
correct_prediction = tf.equal(tf.cast(tf.argmax(logits, 1), tf.int32),
tf.reshape(targets, [-1]))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=tf.reshape(
targets, [-1]),
logits=logits)
cost = tf.reduce_sum(loss) / batch_size
return cost, logits
def BdRNN(x, weight, bias, n_steps, n_hidden, n_input, name="output", reshape = False):
# Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input)
x = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
# Backward direction cell
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
try:
outputs, SL, SR = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,
dtype=tf.float32)
except Exception: #The older versions of TF do not produce the two output states
outputs = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,
dtype=tf.float32)
y = tf.add(tf.matmul(outputs, weight), bias, name="rnn_output")
return y, SL, SR
def BiRNN(x, num_hidden, keep_prob):
# Define weights
weights = {
# Hidden layer weights => 2*n_hidden because of forward + backward cells
'out': tf.Variable(tf.random_normal([2 * num_hidden, num_classes]))
}
biases = {'out': tf.Variable(tf.random_normal([num_classes]))}
# Prepare data shape to match `rnn` function requirements current data input shape: (batch_size, timesteps, n_input)
#batch size = number of sample points, time step = number of sample points to be considered at each instance of time
#num_input = number of features
# Required shape: 'timesteps' tensors list of shape (batch_size, num_input)
x = tf.unstack(
x, timesteps, 1
) # Unstack to get a list of 'timesteps' tensors of shape (batch_size, num_input)
# Define lstm cells with tensorflow
lstm_fw_cell = tf.nn.rnn_cell.DropoutWrapper(
rnn.BasicLSTMCell(num_hidden, forget_bias=0.4, reuse=tf.AUTO_REUSE),
input_keep_prob=keep_prob,
output_keep_prob=keep_prob,
variational_recurrent=False) # Forward direction cell
lstm_bw_cell = tf.nn.rnn_cell.DropoutWrapper(
rnn.BasicLSTMCell(num_hidden, forget_bias=0.4, reuse=tf.AUTO_REUSE),
input_keep_prob=keep_prob,
output_keep_prob=keep_prob,
variational_recurrent=False) # Backward direction cell
# Get lstm cell output
try:
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
except Exception: # Old TensorFlow version only returns outputs not states
outputs = rnn.static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
return (tf.matmul(outputs[-1], weights['out']) + biases['out']
) # Linear activation, using rnn inner loop last output
def rnn_unit(self, input, name='LSTM'):
# check, then set the right name
assert self.rnn_type in ['LSTM', 'GRU'], \
'rnn_type has to be either LSTM or GRU'
name = 'LSTM' if self.rnn_type == 'LSTM' else 'GRU'
if self.bidirection: name += '_bidir'
with tf.name_scope(name):
input = tf.nn.embedding_lookup(self.embedding_matrix, input)
# reshaping
# Permuting batch_size and n_steps
input = tf.transpose(input, [1, 0, 2])
# Reshaping to (n_steps*batch_size, n_input)
input = tf.reshape(input, [-1, self.char_embed_dim])
# Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)
rnn_inputs = tf.split(input, self.seq_len, 0)
# setting the correct RNN cell type (LSTM of GRU)
rnn_cell = rnn.BasicLSTMCell if self.rnn_type == 'LSTM' \
else rnn.GRUCell
# setting the args (forget_bias applies only to LSTM)
rnn_cell_args = {
'num_units': self.hidden_state_size,
'activation': self.activation_function
}
if 'LSTMCell' in str(rnn_cell.__call__):
rnn_cell_args['forget_bias'] = 1.0
rnn_cell(**rnn_cell_args)
cell_fw = rnn_cell(**rnn_cell_args)
cell_fw = rnn.DropoutWrapper(cell_fw,
output_keep_prob=self.keep_prob,
seed=self.seed)
if self.bidirection:
# add another cell for backwards direction and a dropout wrapper
cell_bw = rnn_cell(**rnn_cell_args)
cell_bw = rnn.DropoutWrapper(cell_bw,
output_keep_prob=self.keep_prob,
seed=self.seed)
outputs, _, _ = rnn.static_bidirectional_rnn(cell_fw,
cell_bw,
rnn_inputs,
dtype=tf.float32,
scope=name)
else:
outputs, _ = rnn.static_rnn(cell_fw,
rnn_inputs,
dtype=tf.float32,
scope=name)
if not self.target_rep: # take only last output (list for structure consistency)
outputs = [outputs[-1]]
if self.verbose_summary:
tf.summary.histogram('outputs', outputs, collections=['train'])
return outputs
def _init_model(self):
# Create multiple forward lstm cell
cell_fw = rnn.MultiRNNCell([
rnn.BasicLSTMCell(self._config['hidden_size'])
for _ in range(self._config['num_layers'])
])
# Create multiple backward lstm cell
cell_bw = rnn.MultiRNNCell([
rnn.BasicLSTMCell(self._config['hidden_size'])
for _ in range(self._config['num_layers'])
])
inputs = self._input.input_data
# Add dropout layer to the input data
if self._is_training and self._config['keep_prob'] < 1:
intpus = [
tf.nn.dropout(single_input, self._config['keep_prob'])
for single_input in inputs
]
self._outputs, _, _ = rnn.static_bidirectional_rnn(cell_fw,
cell_bw,
inputs,
dtype=tf.float32)
# Hidden layer weights => 2*hidden_size because of forward + backward cells
softmax_w = tf.get_variable(
"softmax_w",
[2 * self._config['hidden_size'], self._config['num_classes']])
softmax_b = tf.get_variable("softmax_b", [self._config['num_classes']])
# Linear activation, using rnn inner loop last output
# logit shape: [batch_size, num_classes]
self._logits = tf.matmul(self._outputs[-1], softmax_w) + softmax_b
# Define loss
# Required targets shape: [batch_size, num_classes] (one hot vector)
self._cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(
logits=self._logits, labels=self._input.targets))
# Evaluate model
self._correct_pred = tf.equal(tf.argmax(self._logits, 1),
tf.argmax(self._input.targets, 1))
self.accuracy = tf.reduce_mean(tf.cast(self._correct_pred, tf.float32))
# Define optimizer
self._lr = tf.Variable(0.0, trainable=False)
self._train_op = tf.train.AdamOptimizer(
learning_rate=self._lr).minimize(self._cost)
self._new_lr = tf.placeholder(tf.float32,
shape=[],
name="new_learning_rate")
self._lr_update = tf.assign(self._lr, self._new_lr)
def model(inputs, variables):
# Define lstm cells with tensorflow
inputs = tf.unstack(inputs, FRAME_WINDOW, 1)
with tf.variable_scope('layer1') as scope:
# Forward direction cell
lstm_fw_cell = rnn.BasicLSTMCell(HIDDEN_DIMENSION, forget_bias=1.0)
# Backward direction cell
lstm_bw_cell = rnn.BasicLSTMCell(HIDDEN_DIMENSION, forget_bias=1.0)
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
inputs,
dtype=tf.float32,
scope=scope)
with tf.variable_scope('layer2') as scope:
# Forward direction cell
lstm_fw_cell_1 = rnn.BasicLSTMCell(HIDDEN_DIMENSION, forget_bias=1.0)
# Backward direction cell
lstm_bw_cell_1 = rnn.BasicLSTMCell(HIDDEN_DIMENSION, forget_bias=1.0)
outputs_1, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell_1,
lstm_bw_cell_1,
outputs,
dtype=tf.float32,
scope=scope)
with tf.variable_scope('layer3') as scope:
# Forward direction cell
lstm_fw_cell_2 = rnn.BasicLSTMCell(HIDDEN_DIMENSION, forget_bias=1.0)
# Backward direction cell
lstm_bw_cell_2 = rnn.BasicLSTMCell(HIDDEN_DIMENSION, forget_bias=1.0)
outputs_2, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell_2,
lstm_bw_cell_2,
outputs_1,
dtype=tf.float32,
scope=scope)
return tf.nn.sigmoid(
tf.matmul(outputs_2[-1], variables['w1_lstm']) + variables['b1_lstm'])
def BiRNN(x, weights, biases):
x = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
# Backward direction cell
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
# Get lstm cell output
try:
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
except Exception:
outputs = rnn.static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def forward(self, user_metapath, item_metapath, is_training, rnn_cell=rnn.LSTMCell):
user_item_embedding_table = \
tf.get_variable(name="user_embedding", shape=[self.user_num+self.item_num, self.embedding_size],
initializer=tf.contrib.layers.xavier_initializer())
user_metapath_input = tf.nn.embedding_lookup(user_item_embedding_table, user_metapath)
user_forward_rnn_cell_list = [rnn_cell(num_units=self.lstm_hidden_state_size, name="user_fw_%d" % idx)
for idx in range(self.lstm_layer_num)]
user_backward_rnn_cell_list = [rnn_cell(num_units=self.lstm_hidden_state_size, name="user_bw_%d" % idx)
for idx in range(self.lstm_layer_num)]
user_multilayer_forward_rnn_cell = rnn.MultiRNNCell(user_forward_rnn_cell_list)
user_multilayer_backward_rnn_cell = rnn.MultiRNNCell(user_backward_rnn_cell_list)
user_output, _, _ = rnn.static_bidirectional_rnn(cell_fw=user_multilayer_forward_rnn_cell,
cell_bw=user_multilayer_backward_rnn_cell,
inputs=tf.unstack(user_metapath_input, num=3, axis=1),
dtype=tf.float32)
item_metapath_input = tf.nn.embedding_lookup(user_item_embedding_table, item_metapath)
item_forward_rnn_cell_list = [rnn_cell(num_units=self.lstm_hidden_state_size, name="item_fw_%d" % idx)
for idx in range(self.lstm_layer_num)]
item_backward_rnn_cell_list = [rnn_cell(num_units=self.lstm_hidden_state_size, name="item_bw_%d" % idx)
for idx in range(self.lstm_layer_num)]
item_multilayer_forward_rnn_cell = rnn.MultiRNNCell(item_forward_rnn_cell_list)
item_multilayer_backward_rnn_cell = rnn.MultiRNNCell(item_backward_rnn_cell_list)
item_output, _, _ = rnn.static_bidirectional_rnn(cell_fw=item_multilayer_forward_rnn_cell,
cell_bw=item_multilayer_backward_rnn_cell,
inputs=tf.unstack(item_metapath_input, num=3, axis=1),
dtype=tf.float32)
output = tf.concat([user_output[0], item_output[0]], axis=1)
for mlp_num_unit in self.mlp_hidden_size_list:
output = tf.keras.layers.Dense(units=mlp_num_unit, activation=tf.nn.relu,
kernel_initializer=tf.contrib.layers.xavier_initializer())(output)
output = tf.layers.dropout(output, training=is_training)
self.saver = tf.train.Saver(max_to_keep=100)
return output
def prediction(self):
data_unstacked = tf.unstack(self.data, n_steps, 1)
lstm_forward_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_backward_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_forward_cell,
lstm_backward_cell,
data_unstacked,
dtype=tf.float32)
return tf.matmul(outputs[-1], self.weights['out']) + self.biases['out']
def bi_rnn_model(x, weight, bias):
x = tf.unstack(x, timestep, axis=1)
#print(tf.shape(x))
lstm_cell_forward = rnn.BasicLSTMCell(rnn_hidden, forget_bias=1.0)
lstm_cell_backward = rnn.BasicLSTMCell(rnn_hidden, forget_bias=1.0)
output, _, _ = rnn.static_bidirectional_rnn(lstm_cell_forward,
lstm_cell_backward,
x,
dtype=tf.float32)
return tf.matmul(output[-1], weight['out']) + bias['out']
def discriminator(self, inputs, reuse=False):
with tf.variable_scope('discriminator', reuse=reuse) as scope:
# build RNN here
lstm_ = rnn.LSTMCell(self.Config.D_HIDDEN_NEURONS)
lstm_ = rnn.DropoutWrapper(
lstm_,
input_keep_prob=self.Config.DROPOUT_IN,
output_keep_prob=self.Config.DROPOUT_OUT)
stacked_lstm = rnn.MultiRNNCell(
[lstm_ for _ in range(self.Config.D_NUM_LAYERS)])
# initialize initial state to 0
init_state = stacked_lstm.zero_state(self.batchsize, tf.float32)
# bidirectional RNN
outputs, _, _ = rnn.static_bidirectional_rnn(
stacked_lstm,
stacked_lstm,
tf.unstack(tf.transpose(inputs, perm=[1, 0, 2])),
initial_state_fw=init_state,
initial_state_bw=init_state,
scope=scope)
outputs = tf.contrib.layers.fully_connected(
outputs,
self.Config.AE_DENSE_NEURONS,
reuse=reuse,
scope=scope,
activation_fn=None)
# specify activation function here
outputs = self.leakyrelu(outputs)
# dropout
outputs = tf.nn.dropout(outputs, keep_prob=self.Config.DROPOUT_OUT)
outputs = tf.transpose(outputs, perm=[1, 0, 2])
# use only the last hidden state as input to dense layer
outputs = tf.slice(
outputs, [0, self.Config.TIMESTEPS - 1, 0],
[self.batchsize, 1, self.Config.AE_DENSE_NEURONS])
# use last hidden state as input to a dense layer
outputs = tf.layers.dense(outputs,
int(self.Config.AE_DENSE_NEURONS / 2),
name='discriminator/pre_output')
# specify activation function here
outputs = self.leakyrelu(outputs)
# dropout
outputs = tf.nn.dropout(outputs, keep_prob=self.Config.DROPOUT_OUT)
# final output of discriminator
outputs = tf.layers.dense(outputs, 1, name='discriminator/output')
return outputs
def biRNN(x,weights,biase):
x = tf.transpose(x,[1,0,2])
x = tf.reshape(x,[-1,n_input])
x = tf.split(x,n_step,0)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden)
outs,_,_ = rnn.static_bidirectional_rnn(lstm_fw_cell,lstm_bw_cell,x,dtype = tf.float32)
return tf.matmul(outs[-1],weights['out'])+biase['out']
def bi_lstm_layer(self,inputs):
if self.hidden_layer_num >1:
lstm_fw = rnn.MultiRNNCell([self.lstm_cell() for _ in range(self.hidden_layer_num)])
lstm_bw = rnn.MultiRNNCell([self.lstm_cell() for _ in range(self.hidden_layer_num)])
else:
lstm_fw = self.lstm_cell()
lstm_bw = self.lstm_cell()
outpus,_,_ = rnn.static_bidirectional_rnn(lstm_fw,lstm_bw,inputs,sequence_length=self.lengths,dtype=tf.float32)
features = tf.reshape(outpus,[-1,self.num_hidden *2])
return features
def BiRNN(x, weights, biases):
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, n_input])
x = tf.split(x, n_steps)
lstm_fw_cell = BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, _, _ = static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
return tf.matmul(outputs[-1], weights) + biases
def brnn(x, config, is_training):
""" input args
a. num_layers
b. timesteps
c. cell_fn
d. activation_fn
e. batch_size
f. num_units
"""
# params
cell_fn = config.cell_fn
activation_fn = config.activation_fn
initializer_fn = config.initializer_fn
dropout_keep = config.dropout_keep
num_units = config.num_units
num_layers = config.num_layers
# get shape and reshape
batchsize, n_dim, n_step = get_shape(x)
x = tf.reshape(x, [-1, n_step, n_dim])
# transform to list
x = tf.unstack(x, n_step, axis=1)
# sequence_length
sequence_length = [n_dim for _ in range(batchsize)]
fw_cells = []
bw_cells = []
for idx in range(num_layers):
# define
fw_cell = cell_fn(num_units[idx], activation=activation_fn)
bw_cell = cell_fn(num_units[idx], activation=activation_fn)
# define dropout
if dropout_keep is not None and is_training:
fw_cell = rnn.DropoutWrapper(fw_cell,
output_keep_prob=dropout_keep)
bw_cell = rnn.DropoutWrapper(bw_cell,
output_keep_prob=dropout_keep)
fw_cells.append(fw_cell)
bw_cells.append(bw_cell)
# stacks multi-layer
fw_cells = rnn.MultiRNNCell(fw_cells, state_is_tuple=True)
bw_cells = rnn.MultiRNNCell(bw_cells, state_is_tuple=True)
# output
outputs, _, _ = rnn.static_bidirectional_rnn(
fw_cells,
bw_cells,
x,
dtype=tf.float32,
sequence_length=sequence_length)
logit = inner_product(outputs[-1], 1)
return logit, outputs
def generate_source_target(self, x, scope=None):
with tf.variable_scope(scope):
weights_st = weight_variable([2 * self.hidden_rnn * time_steps, 2 * self.hidden_rnn])
biases_st = bias_variable([2 * self.hidden_rnn])
if cell_type == 'lstm':
if num_layers > 1:
# define rnn-cell with tensor_flow
# forward direction cell
fw_cell_st = tf.contrib.rnn.MultiRNNCell([rnn.LSTMCell(self.hidden_rnn)
for _ in range(num_layers)])
# backward direction cell
bw_cell_st = tf.contrib.rnn.MultiRNNCell([rnn.LSTMCell(self.hidden_rnn)
for _ in range(num_layers)])
else:
fw_cell_st = rnn.LSTMCell(self.hidden_rnn)
# backward direction cell
bw_cell_st = rnn.LSTMCell(self.hidden_rnn)
elif cell_type == 'gru':
if num_layers > 1:
fw_cell_st = tf.contrib.rnn.MultiRNNCell([rnn.GRUCell(self.hidden_rnn)
for _ in range(num_layers)])
# backward direction cell
bw_cell_st = tf.contrib.rnn.MultiRNNCell([rnn.GRUCell(self.hidden_rnn)
for _ in range(num_layers)])
else:
fw_cell_st = rnn.GRUCell(self.hidden_rnn)
# backward direction cell
bw_cell_st = rnn.GRUCell(self.hidden_rnn)
else:
if num_layers > 1:
fw_cell_st = tf.contrib.rnn.MultiRNNCell([rnn.BasicRNNCell(self.hidden_rnn)
for _ in range(num_layers)])
# backward direction cell
bw_cell_st = tf.contrib.rnn.MultiRNNCell([rnn.BasicRNNCell(self.hidden_rnn)
for _ in range(num_layers)])
else:
fw_cell_st = rnn.BasicRNNCell(self.hidden_rnn)
# backward direction cell
bw_cell_st = rnn.BasicRNNCell(self.hidden_rnn)
# get rnn-cell outputs
l_outputs_st, a_st, b_st = rnn.static_bidirectional_rnn(fw_cell_st, bw_cell_st,
x, dtype=tf.float32)
l_outputs_st = tf.transpose(tf.stack(l_outputs_st, axis=0), perm=[1, 0, 2])
l_outputs_st = tf.reshape(l_outputs_st, [-1, 2 * self.hidden_rnn * time_steps])
outputs_st = tf.nn.tanh(tf.matmul(l_outputs_st, weights_st) + biases_st)
lo_gits_st = tf.reshape(outputs_st, [-1, 2 * self.hidden_rnn])
ab_st = tf.concat((a_st[1], b_st[1]), axis=1)
return lo_gits_st, ab_st
def Bi_RNN(X, W, B, nsteps, name):
X = tf.transpose(X, [1, 0, 2])
X = tf.reshape(X, [-1, diminput])
H_1 = tf.matmul(X, W["w1"]) + b["b1"]
H_1 = tf.split(H_1, nsteps, 0)
lstm_fw_cell = tf.nn.rnn_cell.BasicLSTMCell(dimhidden, forget_bias=1.0)
lstm_bw_cell = tf.nn.rnn_cell.BasicLSTMCell(dimhidden, forget_bias=1.0)
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
H_1,
dtype=tf.float32)
return tf.matmul(outputs[-1], W["w2"]) + b["b2"]
def BiRNN(x, weights, biases):
x = tf.reshape(x, [-1, N_INPUT])
x = tf.split(x, N_INPUT, 1)
rnn_cell_fw = rnn.MultiRNNCell([getCell(N_HIDDEN) for _ in range(2)])
# rnn_cell_fw = getCell(N_HIDDEN)
rnn_cell_bw = rnn.MultiRNNCell([getCell(N_HIDDEN) for _ in range(2)])
# rnn_cell_bw = getCell(N_HIDDEN)
outputs, _, _ = rnn.static_bidirectional_rnn(rnn_cell_fw,
rnn_cell_bw,
x,
dtype=tf.float32)
return tf.matmul(outputs[-2], weights['out']) + biases['out']
def generate_rnn_output(self):
"""
Generate RNN state outputs with word embeddings as inputs
"""
with tf.variable_scope("generate_seq_output"):
if self.bidirectional_rnn:
embedding = tf.get_variable(
"embedding",
[self.source_vocab_size, self.word_embedding_size],
initializer=initializers.xavier_initializer())
encoder_emb_inputs = [tf.nn.embedding_lookup(embedding, encoder_input) \
for encoder_input in self.encoder_inputs]
rnn_outputs = static_bidirectional_rnn(
self.cell_fw,
self.cell_bw,
encoder_emb_inputs,
sequence_length=self.sequence_length,
dtype=tf.float32)
# tf.keras.layers.Bidirectional(tf.keras.layers.RNN(self.cell_fw, unroll=True))
encoder_outputs, encoder_state_fw, encoder_state_bw = rnn_outputs
# with state_is_tuple = True, if num_layers > 1,
# here we simply use the state from last layer as the encoder state
state_fw = encoder_state_fw[-1]
state_bw = encoder_state_bw[-1]
encoder_state = tf.concat(
[tf.concat(state_fw, 1),
tf.concat(state_bw, 1)], 1)
top_states = [tf.reshape(e, [-1, 1, self.cell_fw.output_size \
+ self.cell_bw.output_size])
for e in encoder_outputs]
attention_states = tf.concat(top_states, 1)
else:
embedding = tf.get_variable(
"embedding",
[self.source_vocab_size, self.word_embedding_size])
encoder_emb_inputs = list()
encoder_emb_inputs = [tf.nn.embedding_lookup(embedding, encoder_input) \
for encoder_input in self.encoder_inputs]
rnn_outputs = static_rnn(self.cell_fw,
encoder_emb_inputs,
sequence_length=self.sequence_length,
dtype=tf.float32)
encoder_outputs, encoder_state = rnn_outputs
# with state_is_tuple = True, if num_layers > 1,
# here we use the state from last layer as the encoder state
state = encoder_state[-1]
encoder_state = tf.concat(state, 1)
top_states = [
tf.reshape(e, [-1, 1, self.cell_fw.output_size])
for e in encoder_outputs
]
attention_states = tf.concat(top_states, 1)
return encoder_outputs, encoder_state, attention_states
def BiRNN(x, weights, biases):
x = tf.unstack(x, num=timesteps, axis=1)
lstm_fw_cell = rnn.BasicLSTMCell(feature_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(feature_hidden, forget_bias=1.0)
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def BiRNN(x, seqlen):
inputs = tf.unstack(x, num=dataset.max_seqlen, axis=1)
lstm_fw_cell = rnn.BasicLSTMCell(num_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(num_hidden, forget_bias=1.0)
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
inputs,
dtype=tf.float32,
sequence_length=seqlen)
outputs = tf.transpose(tf.stack(outputs), perm=[1, 0, 2])
outputs = tf.reduce_max(outputs, axis=1)
return outputs
def BiRnn(x):
# Unstack to get a list of 'n_steps' tensors of shape (batch_size, embedding_size)
x = tf.unstack(x, n_steps, 1)
lstm_fw_cells = rnn.MultiRNNCell([attn_cell() for _ in range(num_layers)],
state_is_tuple=True)
lstm_bw_cells = rnn.MultiRNNCell([attn_cell() for _ in range(num_layers)],
state_is_tuple=True)
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cells,
lstm_bw_cells,
x,
dtype=tf.float32)
return outputs
def BiRNN(x, weights, biases):
x = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
return tf.add(tf.matmul(outputs[-1], weights["out"]), biases["out"])
def generate_rnn_output(self):
"""
Generate RNN state outputs with word embeddings as inputs
"""
with tf.variable_scope("generate_seq_output"):
if self.bidirectional_rnn:
embedding = tf.get_variable("embedding",
[self.source_vocab_size,
self.word_embedding_size])
encoder_emb_inputs = list()
encoder_emb_inputs = [tf.nn.embedding_lookup(embedding, encoder_input)\
for encoder_input in self.encoder_inputs]
rnn_outputs = static_bidirectional_rnn(self.cell_fw,
self.cell_bw,
encoder_emb_inputs,
sequence_length=self.sequence_length,
dtype=tf.float32)
encoder_outputs, encoder_state_fw, encoder_state_bw = rnn_outputs
# with state_is_tuple = True, if num_layers > 1,
# here we simply use the state from last layer as the encoder state
state_fw = encoder_state_fw[-1]
state_bw = encoder_state_bw[-1]
encoder_state = tf.concat([tf.concat(state_fw, 1),
tf.concat(state_bw, 1)], 1)
top_states = [tf.reshape(e, [-1, 1, self.cell_fw.output_size \
+ self.cell_bw.output_size])
for e in encoder_outputs]
attention_states = tf.concat(top_states, 1)
else:
embedding = tf.get_variable("embedding",
[self.source_vocab_size,
self.word_embedding_size])
encoder_emb_inputs = list()
encoder_emb_inputs = [tf.nn.embedding_lookup(embedding, encoder_input)\
for encoder_input in self.encoder_inputs]
rnn_outputs = static_rnn(self.cell_fw,
encoder_emb_inputs,
sequence_length=self.sequence_length,
dtype=tf.float32)
encoder_outputs, encoder_state = rnn_outputs
# with state_is_tuple = True, if num_layers > 1,
# here we use the state from last layer as the encoder state
state = encoder_state[-1]
encoder_state = tf.concat(state, 1)
top_states = [tf.reshape(e, [-1, 1, self.cell_fw.output_size])
for e in encoder_outputs]
attention_states = tf.concat(top_states, 1)
return encoder_outputs, encoder_state, attention_states
def generate_embedding_RNN_output(encoder_inputs,
cell,
num_encoder_symbols,
word_embedding_size,
num_heads=1,
dtype=dtypes.float32,
scope=None,
initial_state_attention=False,
sequence_length=None,
bidirectional_rnn=False):
"""
Generate RNN state outputs with word embeddings as inputs
- Note that this example code does not include output label dependency modeling.
One may add a loop function as in the rnn_decoder function in tf seq2seq.py
example to feed emitted label embedding back to RNN state.
"""
with variable_scope.variable_scope(scope or "generate_embedding_RNN_output"):
if bidirectional_rnn:
encoder_cell_fw = cell
encoder_cell_bw = cell
embedding = variable_scope.get_variable("embedding", [num_encoder_symbols, word_embedding_size])
encoder_embedded_inputs = list()
encoder_embedded_inputs = [embedding_ops.embedding_lookup(embedding, encoder_input) for encoder_input in encoder_inputs]
encoder_outputs, encoder_state_fw, encoder_state_bw = rnn.static_bidirectional_rnn(
encoder_cell_fw, encoder_cell_bw, encoder_embedded_inputs, sequence_length=sequence_length, dtype=dtype)
encoder_state = array_ops.concat(axis=1, values=[array_ops.concat(axis=1, values=encoder_state_fw), array_ops.concat(axis=1, values=encoder_state_bw)])
top_states = [array_ops.reshape(e, [-1, 1, cell.output_size*2])
for e in encoder_outputs]
attention_states = array_ops.concat(axis=1, values=top_states)
else:
encoder_cell = cell
embedding = variable_scope.get_variable("embedding", [num_encoder_symbols, word_embedding_size])
encoder_embedded_inputs = list()
encoder_embedded_inputs = [embedding_ops.embedding_lookup(embedding, encoder_input) for encoder_input in encoder_inputs]
encoder_outputs, encoder_state = rnn.rnn(
encoder_cell, encoder_embedded_inputs, sequence_length=sequence_length, dtype=dtype)
encoder_state = array_ops.concat(1, encoder_state)
top_states = [array_ops.reshape(e, [-1, 1, cell.output_size])
for e in encoder_outputs]
attention_states = array_ops.concat(1, top_states)
return encoder_outputs, encoder_state, attention_states
