def diagonal_lstm(inputs, scope='diagonal_lstm'):
with tf.compat.v1.variable_scope(scope):
skewed_inputs = skew(inputs, scope="skewed_i")
input_to_state = conv2d(skewed_inputs,
64, [1, 1],
mask_type="b",
scope="i_to_s")
column_wise_inputs = tf.transpose(input_to_state, perm=[0, 2, 1, 3])
batch, width, height, channel = column_wise_inputs.get_shape().as_list(
)
rnn_inputs = tf.reshape(column_wise_inputs,
[-1, width, height * channel])
cell = DiagonalLSTMCell(16, height, channel)
outputs, states = tf.compat.v1.nn.dynamic_rnn(cell,
inputs=rnn_inputs,
dtype=tf.float32)
width_first_outputs = tf.reshape(outputs, [-1, width, height, 16])
skewed_outputs = tf.transpose(width_first_outputs, perm=[0, 2, 1, 3])
outputs = unskew(skewed_outputs)
return outputs
def get_history_din_embedded(self):
# TODO: add mask info for this operation
his_days = [
'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine',
'ten', 'eleven', 'twelve', 'thirteen', 'fourteen'
]
for fir in his_days:
key = "history_" + fir + "_chap_ph"
embed_key = "history_" + fir + "_chap_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.chapters_embeddings_var,
getattr(self, key)))
for fir in his_days:
key = "history_" + fir + "_sec_ph"
embed_key = "history_" + fir + "_sec_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.sections_embeddings_var,
getattr(self, key)))
history_cha_embedded = []
history_sec_embedded = []
for fir in his_days[::-1]:
key_c = "history_" + fir + "_chap_embedded"
key_s = "history_" + fir + "_sec_embedded"
history_cha_embedded.append(
tf.reduce_mean(getattr(self, key_c), axis=-2))
history_sec_embedded.append(
tf.reduce_mean(getattr(self, key_s), axis=-2))
#self.history_all_embedded = tf.reshape(, [None,len(history_all_embedded),EMBEDDING_DIM])
# T*B*N -。 B*T*N
history_cha_emb = tf.transpose(history_cha_embedded, [1, 0, 2])
attention_cha_output = din_attention(tf.reduce_mean(
self.today_chapters_embedded, axis=-2),
history_cha_emb,
ATTENTION_SIZE,
self.history_mask_cha_ph,
stag="cha")
history_sec_emb = tf.transpose(history_sec_embedded, [1, 0, 2])
attention_sec_output = din_attention(tf.reduce_mean(
self.today_sections_embedded, axis=-2),
history_sec_emb,
ATTENTION_SIZE,
self.history_mask_sec_ph,
stag="sec")
att_fea1 = tf.reduce_sum(attention_cha_output, -2)
att_fea2 = tf.reduce_sum(attention_sec_output, -2)
#atte_out = tf.concat([att_fea1,att_fea2],axis=-1)
return att_fea1, att_fea2
def __init__(self, dim, samples
, init_w= tf.random_uniform_initializer(minval= -0.01, maxval= 0.01)
, ftype= tf.float32, scope= 'dbn'):
self.dim, self.ftype = dim, ftype
with tf.variable_scope(scope):
self.rbm = tuple(
Rbm(scope= "rbm{}".format(i)
, dim_v= dim_v
, dim_h= dim_h
, samples= samples
, init_w= init_w
, ftype= self.ftype)
for i, (dim_v, dim_h) in enumerate(zip(dim, dim[1:]), 1))
self.w = tuple(rbm.w for rbm in self.rbm[::-1])
self.wg = tuple(tf.transpose(w) for w in self.w)
self.wr = tuple(
tf.get_variable(name= "wr{}".format(i), shape= (dim_d, dim_a), initializer= init_w)
for i, (dim_d, dim_a) in enumerate(zip(self.dim, self.dim[1:]), 1))
self.lr_ = tf.placeholder(name= 'lr_', dtype= self.ftype, shape= ())
# wake
self.v_ = self.rbm[0].v_
with tf.name_scope('wake'):
recogn = [self.v_]
for w in self.wr: recogn.append(binary(tf.matmul(recogn[-1], w)))
self.recogn = tuple(recogn)
recogn = recogn[::-1]
eps = self.lr_ / tf.cast(tf.shape(self.v_)[0], dtype= self.ftype)
self.wake = tuple(
w.assign_add(tf.matmul((sj - pj), sk, transpose_a= True) * eps).op
for w, sk, sj, pj in zip(
self.w, recogn, recogn[1:]
, (tf.sigmoid(tf.matmul(s, w))
for w, s in zip(self.wg, recogn))))
# sleep
top = self.rbm[-1]
self.k_, (self.v, self.a) = top.k_, top.gibbs
with tf.name_scope('sleep'):
recons = [self.a, self.v]
for w in self.wg[1::]: recons.append(binary(tf.matmul(recons[-1], w)))
self.recons = tuple(recons)
recons = recons[::-1]
eps = self.lr_ / tf.cast(tf.shape(self.a)[0], dtype= self.ftype)
self.sleep = tuple(
w.assign_add(tf.matmul(sj, (sk - qk), transpose_a= True) * eps).op
for w, sj, sk, qk in zip(
self.wr, recons, recons[1:]
, (tf.sigmoid(tf.matmul(s, w))
for w, s in zip(self.wr, recons))))
# the waking world is the amnesia of dream.
self.v = self.recons[-1]
with tf.name_scope('ances'):
self.a = self.rbm[-1].h
ances = [self.a]
for w in self.wg: ances.append(binary(tf.matmul(ances[-1], w)))
self.ances = ances[-1]
self.step = 0
def skew(inputs, scope="skew"):
with tf.compat.v1.name_scope(scope):
batch, height, width, channel = inputs.get_shape().as_list()
rows = tf.split(inputs, height, 1)
new_width = width + height - 1
new_rows = []
for idx, row in enumerate(rows):
transposed_row = tf.transpose(tf.squeeze(row, [1]), perm=[0, 2, 1])
squeezed_row = tf.reshape(transposed_row, [-1, width])
padded_row = tf.pad(squeezed_row,
paddings=((0, 0), (idx, height - 1 - idx)))
unsqueezed_row = tf.reshape(padded_row, [-1, channel, new_width])
untransposed_row = tf.transpose(unsqueezed_row, perm=[0, 2, 1])
new_rows.append(untransposed_row)
outputs = tf.stack(new_rows, axis=1, name="output")
return outputs
def LSTM_Sentiment(input_tensor):
# Reference Paper: https://www.bioinf.jku.at/publications/older/2604.pdf
lstmCell = tf.compat.v1.nn.rnn_cell.BasicLSTMCell(1024)
output_rnn, _ = tf.compat.v1.nn.dynamic_rnn(lstmCell,
input_tensor,
dtype=tf.float32)
W_fc = tf.Variable(tf.random.truncated_normal([1024, 2]))
b_fc = tf.Variable(tf.constant(0.1, shape=[2]))
output_transposed = tf.transpose(output_rnn, perm=[1, 0, 2])
output = tf.gather(output_transposed,
int(output_transposed.get_shape()[0]) - 1)
return tf.identity(tf.matmul(output, W_fc) + b_fc, name="output")
def get_history_bgru_embedded(self):
# TODO: add mask info for this operation
his_days = [
'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine',
'ten', 'eleven', 'twelve', 'thirteen', 'fourteen'
]
for fir in his_days:
key = "history_" + fir + "_chap_ph"
embed_key = "history_" + fir + "_chap_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.chapters_embeddings_var,
getattr(self, key)))
for fir in his_days:
key = "history_" + fir + "_sec_ph"
embed_key = "history_" + fir + "_sec_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.sections_embeddings_var,
getattr(self, key)))
history_cha_embedded = []
history_sec_embedded = []
for fir in his_days[::-1]:
key_c = "history_" + fir + "_chap_embedded"
key_s = "history_" + fir + "_sec_embedded"
# B 3 128 B 128 14 B 128 B 14 128
history_cha_embedded.append(get_rnn_sum(getattr(self, key_c),
"rnncha"))
history_sec_embedded.append(get_rnn_sum(getattr(self, key_s),
"rnnsec"))
history_cha_emb = tf.transpose(history_cha_embedded, [1, 0, 2])
history_sec_emb = tf.transpose(history_sec_embedded, [1, 0, 2])
with tf.name_scope("GRU"):
num_layers = 2
HIDDEN_DIM = 128
KEEP_PROB = 0.8
with tf.name_scope('cell'):
def build_cell(n, m):
cell = tf.nn.rnn_cell.GRUCell(n)
cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=m)
return cell
num_units = [HIDDEN_DIM // 2, HIDDEN_DIM]
cell_fw = [build_cell(n, KEEP_PROB) for n in num_units]
cell_bw = [build_cell(n, KEEP_PROB) for n in num_units]
with tf.name_scope('gru'):
biout, output_fw, output_bw = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(
cell_fw,
cell_bw,
inputs=tf.concat([history_cha_emb, history_sec_emb], axis=-1),
dtype=tf.float32,
scope='cha')
rnnoutput = tf.reduce_sum(biout, axis=-2)
return rnnoutput
def get_history_gru_embedded(self):
# TODO: add mask info for this operation
his_days = [
'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine',
'ten', 'eleven', 'twelve', 'thirteen', 'fourteen'
]
for fir in his_days:
key = "history_" + fir + "_chap_ph"
embed_key = "history_" + fir + "_chap_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.chapters_embeddings_var,
getattr(self, key)))
for fir in his_days:
key = "history_" + fir + "_sec_ph"
embed_key = "history_" + fir + "_sec_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.sections_embeddings_var,
getattr(self, key)))
# [1,2][3][4,3,5][0,0]... chap 14天
# [][][] sec
# cha_sec 拼接后rnn cha rnn sec rnn 拼劲 或者不拼接
#
history_cha_embedded = []
history_sec_embedded = []
for fir in his_days[::-1]:
key_c = "history_" + fir + "_chap_embedded"
key_s = "history_" + fir + "_sec_embedded"
# B 3 128 B 128 14 B 128 B 14 128
history_cha_embedded.append(
tf.reduce_mean(getattr(self, key_c), axis=-2))
history_sec_embedded.append(
tf.reduce_mean(getattr(self, key_s), axis=-2))
history_cha_emb = tf.transpose(history_cha_embedded, [1, 0, 2])
history_sec_emb = tf.transpose(history_sec_embedded, [1, 0, 2])
with tf.name_scope("GRU"):
num_layers = 2
HIDDEN_DIM = 128
KEEP_PROB = 0.8
with tf.name_scope('gru1'):
def get_cell():
cell2 = tf.nn.rnn_cell.GRUCell(HIDDEN_DIM)
cell2_ = tf.nn.rnn_cell.DropoutWrapper(
cell2, output_keep_prob=KEEP_PROB)
return cell2_
cells = [get_cell() for _ in range(num_layers)]
Cell = tf.nn.rnn_cell.MultiRNNCell(cells, state_is_tuple=True)
rnnoutput, _ = tf.nn.dynamic_rnn(cell=Cell,
inputs=history_cha_emb,
dtype=tf.float32,
scope='cha')
rnnoutput1 = tf.reduce_sum(rnnoutput, axis=-2)
with tf.name_scope("GRU2"):
num_layers = 2
HIDDEN_DIM = 128
KEEP_PROB = 0.8
with tf.name_scope('gru2'):
cells2 = []
for _ in range(num_layers):
cell = tf.nn.rnn_cell.GRUCell(HIDDEN_DIM)
cell_ = tf.nn.rnn_cell.DropoutWrapper(
cell, output_keep_prob=KEEP_PROB)
cells2.append(cell_)
#cells2 = [get_cell2() for _ in range(num_layers)]
Cell2 = tf.nn.rnn_cell.MultiRNNCell(cells2, state_is_tuple=True)
rnnoutput2, _ = tf.nn.dynamic_rnn(cell=Cell2,
inputs=history_sec_emb,
dtype=tf.float32,
scope='SEC')
rnnoutput3 = tf.reduce_sum(rnnoutput2, axis=-2)
#gru_out = tf.concat([rnnoutput1,rnnoutput3],axis=-1)
return rnnoutput1, rnnoutput3
def get_history_din_embedded(self):
# TODO: add mask info for this operation
his_days = [
'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine',
'ten', 'eleven', 'twelve', 'thirteen', 'fourteen'
]
for fir in his_days:
key = "history_" + fir + "_chap_ph"
embed_key = "history_" + fir + "_chap_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.chapters_embeddings_var,
getattr(self, key)))
for fir in his_days:
key = "history_" + fir + "_sec_ph"
embed_key = "history_" + fir + "_sec_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.sections_embeddings_var,
getattr(self, key)))
history_cha_embedded = []
history_sec_embedded = []
for fir in his_days[::-1]:
key_c = "history_" + fir + "_chap_embedded"
key_s = "history_" + fir + "_sec_embedded"
history_cha_embedded.append(get_rnn_sum(getattr(self, key_c),
"rnncha"))
history_sec_embedded.append(get_rnn_sum(getattr(self, key_s),
"rnnsec"))
#self.history_all_embedded = tf.reshape(, [None,len(history_all_embedded),EMBEDDING_DIM])
# T*B*N -。 B*T*N
history_cha_emb = tf.transpose(history_cha_embedded, [1, 0, 2])
history_sec_emb = tf.transpose(history_sec_embedded, [1, 0, 2])
#dien
with tf.name_scope('rnn_1'):
rnn_outputs, _ = dynamic_rnn(GRUCell(HIDDEN_SIZE * 2),
inputs=tf.concat(
[history_cha_emb, history_sec_emb],
axis=-1),
sequence_length=self.history_len_ph,
dtype=tf.float32,
scope="gru1")
with tf.name_scope('Attention_layer_1'):
att_outputs, alphas = din_fcn_attention(tf.concat([
get_rnn_sum(self.today_chapters_embedded, "rnncha"),
get_rnn_sum(self.today_sections_embedded, "rnnsec")
],
axis=-1),
rnn_outputs,
ATTENTION_SIZE,
self.history_mask_cha_ph,
scope="1_1",
softmax_stag=1,
stag='1_1',
mode='LIST',
return_alphas=True)
with tf.name_scope('rnn_2'):
rnn_outputs2, final_state2 = dynamic_rnn(
VecAttGRUCell(HIDDEN_SIZE * 2),
inputs=rnn_outputs,
att_scores=tf.expand_dims(alphas, -1),
sequence_length=self.history_len_ph,
dtype=tf.float32,
scope="gru2")
return final_state2
