def baseline_encoder(inputs,
input_lengths,
hidden_cells=128,
layers=1,
code_dim=128):
"""
The baseline encoder for our variational model is is a unidirectional LSTM
whose final output parameterises the mean and std of a normal over our
latent space.
Outputs an edward Normal.
"""
with tf.variable_scope('encoder'):
# turn the inputs from [batch, max_len] into [batch, max_len, 16]
with tf.variable_scope('inputs'):
binary_input = _integer_to_binary(inputs, 16)
# project the inputs
projected_inputs = project_sequence(binary_input, 128)
# run an RNN over the lot
cells = [rnn_cell.LSTMCell(hidden_cells) for _ in range(layers)]
if layers > 1:
cell = rnn_cell.MultiRNNCell(cells)
else:
cell = cells[0]
outputs, _ = tf.nn.dynamic_rnn(cell, projected_inputs, input_lengths)
final_output = outputs[:, -1, :]
loc = tf.layers.dense(outputs, code_dim, name='code_mean')
scale = tf.layers.dense(
outputs, code_dim, activation=tf.nn.softplus, 'code_std')
return ed.models.Normal(loc=loc, scale=scale)
def lstm_model3(inputs, reuse=False, layers_number=2, num_units=256, scope="l"):
shape = inputs[0].shape
observation_n = []
for i in range(len(inputs)):
obs = inputs[i]
if not reuse and i == 0:
reuse = False
else:
reuse = True
x = []
with tf.variable_scope(scope, reuse=reuse):
for j in range(shape[2]):
dr_reuse = True
if j == 0 and not reuse:
dr_reuse = False
out = layers.fully_connected(obs[:, :, j], num_outputs=num_units * 4, activation_fn=tf.nn.relu,
scope="first", reuse=dr_reuse)
out = layers.fully_connected(out, num_outputs=num_units, activation_fn=tf.nn.relu,
scope="second", reuse=dr_reuse)
x.append(tf.expand_dims(out, 2))
x = tf.concat(x, 2)
lstm_size = x.shape[1]
# dimension reduction 3096->1024->256
with tf.variable_scope(scope, reuse=reuse):
x = tf.transpose(x, (2, 0, 1)) # (time_steps, batch_size, state_size)
lstm_cell = rnn.BasicLSTMCell(lstm_size, forget_bias=1, state_is_tuple=True)
cell = rnn.MultiRNNCell([lstm_cell] * layers_number, state_is_tuple=True)
with tf.variable_scope("Multi_Layer_RNN"):
cell_outputs, states = tf.nn.dynamic_rnn(cell, x, dtype=tf.float32)
outputs = cell_outputs[-1:, :, :]
outputs = tf.squeeze(outputs, 0)
observation_n.append(outputs)
return observation_n
def lstm_model(inputs, reuse=False, num_units=(64, 32), scope="l"):
debug = False
if debug:
import time
t = time.time()
observation_n = []
for i in range(len(inputs)):
x = inputs[i]
if not reuse and i == 0:
reuse = False
else:
reuse = True
with tf.variable_scope(scope, reuse=reuse):
x = tf.transpose(x,
(2, 0, 1)) # (time_steps, batch_size, state_size)
cells = [
rnn.LSTMCell(lstm_size, forget_bias=1, state_is_tuple=True)
for lstm_size in num_units
]
cell = rnn.MultiRNNCell(cells, state_is_tuple=True)
with tf.variable_scope("Multi_Layer_RNN"):
cell_outputs, states = tf.nn.dynamic_rnn(cell,
x,
dtype=tf.float32)
outputs = cell_outputs[-1:, :, :]
outputs = tf.squeeze(outputs, 0)
observation_n.append(outputs)
if debug:
print("lstm time: ", time.time() - t)
return observation_n
def __init__(self, args, data, infer=False):
if infer:
args.batch_size = 1
args.seq_length = 1
with tf.name_scope('inputs'):
self.input_data = tf.placeholder(
tf.int32, [args.batch_size, args.seq_length])
self.target_data = tf.placeholder(
tf.int32, [args.batch_size, args.seq_length])
with tf.name_scope('model'):
self.cell = rnn_cell.BasicLSTMCell(args.state_size)
self.cell = rnn_cell.MultiRNNCell([self.cell] * args.num_layers)
self.initial_state = self.cell.zero_state(args.batch_size,
tf.float32)
with tf.variable_scope('rnnlm'):
w = tf.get_variable('softmax_w',
[args.state_size, data.vocab_size])
b = tf.get_variable('softmax_b', [data.vocab_size])
with tf.device("/cpu:0"):
embedding = tf.get_variable(
'embedding', [data.vocab_size, args.state_size])
inputs = tf.nn.embedding_lookup(embedding, self.input_data)
outputs, last_state = tf.nn.dynamic_rnn(
self.cell, inputs, initial_state=self.initial_state)
with tf.name_scope('loss'):
output = tf.reshape(outputs, [-1, args.state_size])
self.logits = tf.matmul(output, w) + b
self.probs = tf.nn.softmax(self.logits)
self.last_state = last_state
targets = tf.reshape(self.target_data, [-1])
loss = seq2seq.sequence_loss_by_example(
[self.logits], [targets],
[tf.ones_like(targets, dtype=tf.float32)])
self.cost = tf.reduce_sum(loss) / args.batch_size
tf.summary.scalar('loss', self.cost)
with tf.name_scope('optimize'):
self.lr = tf.placeholder(tf.float32, [])
tf.summary.scalar('learning_rate', self.lr)
optimizer = tf.train.AdamOptimizer(self.lr)
tvars = tf.trainable_variables()
grads = tf.gradients(self.cost, tvars)
for g in grads:
tf.summary.histogram(g.name, g)
grads, _ = tf.clip_by_global_norm(grads, args.grad_clip)
self.train_op = optimizer.apply_gradients(zip(grads, tvars))
self.merged_op = tf.summary.merge_all()
def lstm_model(common_obs, inputs, reuse=tf.AUTO_REUSE, num_units=(64, 32), scope="l"):
# inputs.shape: [batch_size, time_steps, agents_number, shape]
observation_n = []
for i in range(len(inputs)):
x = tf.transpose(inputs[i], (1, 0, 2))
x = tf.concat((common_obs, x), 2)
with tf.variable_scope(scope, reuse=reuse):
cells = [rnn.LSTMCell(lstm_size, forget_bias=1, state_is_tuple=True) for lstm_size in num_units]
cell = rnn.MultiRNNCell(cells, state_is_tuple=True)
with tf.variable_scope("Multi_Layer_RNN"):
cell_outputs, states = tf.nn.dynamic_rnn(cell, x, dtype=tf.float32)
outputs = cell_outputs[-1:, :, :]
outputs = tf.squeeze(outputs, 0)
observation_n.append(outputs)
return observation_n
def __init__(self, is_training, config):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
self.size = size = config.hidden_size
vocab_size = config.vocab_size
self._input_data = tf.placeholder(tf.int32, [batch_size, num_steps])
self._mask = tf.placeholder(tf.float32,
[batch_size, None]) # 注意类型是float
self._input_word = tf.placeholder(tf.int32,
[batch_size, config.num_keywords])
self._init_output = tf.placeholder(tf.float32, [batch_size, size])
def single_cell_fn(unit_type,
num_units,
dropout,
mode,
forget_bias=1.0):
"""Create an instance of a single RNN cell."""
dropout = dropout if mode is True else 0.0
if unit_type == "lstm":
c = rnn_cell.LSTMCell(num_units,
forget_bias=forget_bias,
state_is_tuple=False)
elif unit_type == "gru":
c = rnn_cell.GRUCell(num_units)
else:
raise ValueError("Unknown unit type %s!" % unit_type)
if dropout > 0.0:
c = rnn_cell.DropoutWrapper(cell=c,
input_keep_prob=(1.0 - dropout))
return c
cell_list = []
for i in range(config.num_layers):
single_cell = single_cell_fn(unit_type="lstm",
num_units=size,
dropout=1 - config.keep_prob,
mode=is_training)
cell_list.append(single_cell)
cell = rnn_cell.MultiRNNCell(cell_list, state_is_tuple=False)
self._initial_state = cell.zero_state(batch_size, tf.float32)
with tf.device("/cpu:0"):
embedding_keyword = tf.get_variable(
'keyword_embedding',
[config.movie + config.score, config.word_embedding_size],
trainable=True,
initializer=tf.random_uniform_initializer(
-config.init_scale, config.init_scale))
embedding = tf.get_variable(
'word_embedding', [vocab_size, config.word_embedding_size],
trainable=True,
initializer=tf.constant_initializer(word_vec))
inputs = tf.nn.embedding_lookup(embedding, self._input_data)
keyword_inputs = tf.nn.embedding_lookup(embedding_keyword,
self._input_word)
if is_training and config.keep_prob < 1:
inputs = tf.nn.dropout(inputs, config.keep_prob)
# keyword_inputs = tf.nn.dropout(keyword_inputs, config.keep_prob)
self.initial_gate = tf.ones([batch_size, config.num_keywords])
atten_sum = tf.zeros([batch_size, config.num_keywords])
with tf.variable_scope("coverage"):
"""
u_f 是一个变量参数，他负责与topic相乘，得到的结果再通过sigmoid归一化到0～1之间，目的是为每一个控制信息分配一个初始比例
sen_len 是想计算每个样本的有效字数
假设每个样本，如果有两个控制条件的话，每一个控制条件的重要程度用一个0～1之间的数表示，（其实这里应该是 softmax更加合理）
有多少有效字，那么这句话中该控制条件就有多少的初始总分值
"""
u_f = tf.get_variable("u_f", [
config.num_keywords * config.word_embedding_size,
config.num_keywords
])
res1 = tf.sigmoid(
tf.matmul(tf.reshape(keyword_inputs, [batch_size, -1]),
u_f)) # todo
sen_len = tf.reduce_sum(self._mask, -1, keepdims=True)
phi_res = sen_len * res1
self.output1 = phi_res
outputs = []
output_state = self._init_output
state = self._initial_state
with tf.variable_scope("RNN"):
for time_step in range(num_steps):
# vs 里面放的是当前这个time step，上一个时刻的隐含层状态跟每一个主题的关系一个被gate消弱后的得分
vs = []
for kw_i in range(config.num_keywords):
with tf.variable_scope("RNN_attention"):
if time_step > 0 or kw_i > 0:
tf.get_variable_scope().reuse_variables()
u = tf.get_variable("u", [size, 1])
w1 = tf.get_variable("w1", [size, size])
w2 = tf.get_variable(
"w2", [config.word_embedding_size, size])
b = tf.get_variable("b1", [size])
# 加工上一次隐含层状态 线性变换一下
temp2 = tf.matmul(output_state, w1)
# 取到某一个主题的向量
temp3 = keyword_inputs[:, kw_i, :]
# 对主题的向量,线性变换一下
temp4 = tf.matmul(temp3, w2)
# 线性变换后的隐状态和主题add起来
temp5 = tf.add(temp2, temp4)
# 加上一个偏置项
temp6 = tf.add(temp5, b)
# 加上一个非线性
temp7 = tf.tanh(temp6)
# 在线性变换一下
vi = tf.matmul(temp7, u)
temp8 = self.initial_gate[:, kw_i:kw_i +
1] # 把kw_i主题对应的gate控制变量取出来，这个gate初始值都是1
temp9 = vi * temp8
vs.append(temp9)
self.attention_vs = tf.concat(vs, axis=1)
prob_p = tf.nn.softmax(self.attention_vs)
# 此处prob_p表示的是上一步的隐含层状态对每一个主题的注意力得分
self.initial_gate = self.initial_gate - (prob_p / phi_res)
temp10 = self._mask[:, time_step:time_step + 1]
atten_sum += prob_p * temp10
# （batchsize，2） * （batchsize，1）
# 如果某一个样本的这个time step的mask是0，那么对应这个样本的所有的主题的权重都为0
# 全部被mask掉了
# 全部主题的词向量的加权和
mt = tf.add_n([
prob_p[:, i:i + 1] * keyword_inputs[:, i, :]
for i in range(config.num_keywords)
])
with tf.variable_scope("RNN_sentence"):
if time_step > 0:
tf.get_variable_scope().reuse_variables()
temp11 = inputs[:, time_step, :]
# mt 是根据 time_step上一个时刻的 隐含层状态 和 主题 信息一起得到的
temp12 = tf.concat([temp11, mt], axis=1)
# 必须要保证 cell input 的 dims = hidden units
temp13 = tf.layers.dense(inputs=temp12, units=size)
(cell_output, state) = cell(temp13, state)
outputs.append(cell_output)
output_state = cell_output
self._last_output = cell_output
output = tf.reshape(tf.concat(outputs, axis=1), [-1, size])
softmax_w = tf.get_variable("softmax_w", [size, vocab_size])
softmax_b = tf.get_variable("softmax_b", [vocab_size])
logits = tf.matmul(output, softmax_w) + softmax_b
# loss = sequence_loss_by_example([logits], [tf.reshape(self._targets, [-1])], [tf.reshape(self._mask, [-1])])
# # 得到的是一个batch里面 所有字的 loss shape ： batch_size*seq_len
# self.cost1 = tf.reduce_sum(loss)
# self.cost2 = tf.reduce_sum((phi_res - atten_sum) ** 2)
# mask_sum = tf.reduce_sum(self._mask)
# self._cost = cost = (self.cost1 + 0.1 * self.cost2) / mask_sum
# # self._cost = cost = (self.cost1 + 0.1 * self.cost2)
self._final_state = state
self._prob = tf.nn.softmax(logits)
def __init__(self, is_training, filename):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
self.size = size = config.hidden_size
vocab_size = config.vocab_size
filename_queue = tf.train.string_input_producer([filename],
num_epochs=None)
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
# We know the length of both fields.
# If not the tf.VarLenFeature could be used
'input_data':
tf.FixedLenFeature([batch_size * num_steps], tf.int64),
'target':
tf.FixedLenFeature([batch_size * num_steps], tf.int64),
'mask':
tf.FixedLenFeature([batch_size * num_steps], tf.float32),
'key_words':
tf.FixedLenFeature([batch_size * config.num_keywords],
tf.int64)
})
self._input_data = tf.cast(features['input_data'], tf.int32)
self._targets = tf.cast(features['target'], tf.int32)
self._input_word = tf.cast(features['key_words'], tf.int32)
self._mask = tf.cast(features['mask'], tf.float32)
self._init_output = tf.placeholder(tf.float32, [batch_size, size])
self._input_data = tf.reshape(self._input_data, [batch_size, -1])
self._targets = tf.reshape(self._targets, [batch_size, -1])
self._input_word = tf.reshape(self._input_word, [batch_size, -1])
self._mask = tf.reshape(self._mask, [batch_size, -1])
def single_cell_fn(unit_type,
num_units,
dropout,
mode,
forget_bias=1.0):
"""Create an instance of a single RNN cell."""
dropout = dropout if mode == True else 0.0
if unit_type == "lstm":
single_cell = rnn_cell.LSTMCell(num_units,
forget_bias=forget_bias,
state_is_tuple=False)
else:
raise ValueError("Unknown unit type %s!" % unit_type)
if dropout > 0.0:
single_cell = rnn_cell.DropoutWrapper(
cell=single_cell, input_keep_prob=(1.0 - dropout))
return single_cell
cell_list = []
for i in range(config.num_layers):
single_cell = single_cell_fn(unit_type="lstm",
num_units=size,
dropout=1 - config.keep_prob,
mode=is_training)
cell_list.append(single_cell)
cell = rnn_cell.MultiRNNCell(cell_list, state_is_tuple=False)
self._initial_state = cell.zero_state(batch_size, tf.float32)
with tf.device("/cpu:0"):
embedding = tf.get_variable(
'word_embedding', [vocab_size, config.word_embedding_size],
trainable=True,
initializer=tf.constant_initializer(word_vec))
inputs = tf.nn.embedding_lookup(
embedding, self._input_data
) # 返回一个tensor，shape是(batch_size, num_steps, size)
keyword_inputs = tf.nn.embedding_lookup(embedding,
self._input_word)
if is_training and config.keep_prob < 1:
inputs = tf.nn.dropout(inputs, config.keep_prob)
gate = tf.ones([batch_size, config.num_keywords])
atten_sum = tf.zeros([batch_size, config.num_keywords])
with tf.variable_scope("coverage"):
u_f = tf.get_variable("u_f", [
config.num_keywords * config.word_embedding_size,
config.num_keywords
])
res1 = tf.sigmoid(
tf.matmul(tf.reshape(keyword_inputs, [batch_size, -1]), u_f))
temp1 = tf.reduce_sum(self._mask, 1, keepdims=True)
phi_res = temp1 * res1
self.output1 = phi_res
outputs = []
output_state = self._init_output
state = self._initial_state
with tf.variable_scope("RNN"):
for time_step in range(num_steps):
# vs 里面放的是 当前这个 time step， 上一个时刻的隐含层状态跟每一个主题的关系 一个 被 gate 消弱后的得分
vs = []
for s2 in range(config.num_keywords):
with tf.variable_scope("RNN_attention"):
if time_step > 0 or s2 > 0:
tf.get_variable_scope().reuse_variables()
u = tf.get_variable("u", [size, 1])
w1 = tf.get_variable("w1", [size, size])
w2 = tf.get_variable(
"w2", [config.word_embedding_size, size])
b = tf.get_variable("b1", [size])
# 加工上一次隐含层状态 线性变换一下
temp2 = tf.matmul(output_state, w1)
# 取到某一个主题的向量
temp3 = keyword_inputs[:, s2, :]
# 对主题的向量 线性变换一下
temp4 = tf.matmul(temp3, w2)
# 线性变换后的 隐状态 和 主题 add起来
temp5 = tf.add(temp2, temp4)
# 加上一个偏置项
temp6 = tf.add(temp5, b)
# 加上一个非线性
temp7 = tf.tanh(temp6)
# 在线性变换一下
vi = tf.matmul(temp7, u)
temp8 = gate[:, s2:s2 +
1] # 把 s2 主题对应的 gate 控制变量取出来，这个gate初始值都是1
temp9 = vi * temp8
vs.append(temp9)
self.attention_vs = tf.concat(vs, axis=1)
prob_p = tf.nn.softmax(self.attention_vs)
# 此处 prob_p 表示的是 上一步的隐含层状态 对每一个主题的 注意力得分
gate = gate - (prob_p / phi_res)
temp10 = self._mask[:, time_step:time_step + 1]
atten_sum += prob_p * temp10
# （32，5） * （32，1）
# 如果某一个样本的这个time step的mask是0，那么对应这个样本的所有的主题的权重都为0
# 全部被mask掉了
# 全部主题的词向量的加权和
mt = tf.add_n([
prob_p[:, i:i + 1] * keyword_inputs[:, i, :]
for i in range(config.num_keywords)
])
with tf.variable_scope("RNN_sentence"):
if time_step > 0:
tf.get_variable_scope().reuse_variables()
temp11 = inputs[:, time_step, :]
# mt 是根据 time_step上一个时刻的 隐含层状态 和 主题 信息一起得到的
temp12 = tf.concat([temp11, mt], axis=1)
# 必须要保证 cell input 的 dims = hidden units
temp13 = tf.layers.dense(inputs=temp12, units=size)
(cell_output, state) = cell(temp13, state)
outputs.append(cell_output)
output_state = cell_output
self._end_output = cell_output
self.output2 = atten_sum
output = tf.reshape(tf.concat(outputs, axis=1), [-1, size])
softmax_w = tf.get_variable("softmax_w", [size, vocab_size])
softmax_b = tf.get_variable("softmax_b", [vocab_size])
logits = tf.matmul(output, softmax_w) + softmax_b
try:
loss = tf.nn.seq2seq.sequence_loss_by_example(
[logits], [tf.reshape(self._targets, [-1])],
[tf.reshape(self._mask, [-1])],
average_across_timesteps=False)
except:
loss = sequence_loss_by_example([logits],
[tf.reshape(self._targets, [-1])],
[tf.reshape(self._mask, [-1])],
average_across_timesteps=False)
self.cost1 = tf.reduce_sum(loss)
self.cost2 = tf.reduce_sum((phi_res - atten_sum)**2)
self._cost = cost = (self.cost1 + 0.1 * self.cost2) / batch_size
self._final_state = state
self._prob = tf.nn.softmax(logits)
if not is_training:
prob = tf.nn.softmax(logits)
self._sample = tf.argmax(prob, 1)
return
self._lr = tf.Variable(0.0, trainable=False)
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars),
config.max_grad_norm)
optimizer = tf.train.AdamOptimizer(self.lr)
self._train_op = optimizer.apply_gradients(zip(grads, tvars))
def stacked_lstm_cell(self, is_training, num_units):
dropout_keep_prob = 0.5 if is_training else 1.0
cell = rnn.MultiRNNCell([self.lstm_cell(dropout_keep_probs, num_units)
for i in range(self.num_lstm_layers)])
return cell
def build_model(self, max_length, vocabulary_size, embedding_size,
num_hidden, num_layers, num_classes, learn_rate):
self.__graph = tf.Graph()
with self.__graph.as_default():
# to track progress
self.__global_step = tf.Variable(0, trainable=False)
# input parameters
self.__dropout = tf.placeholder(tf.float32)
self.__data = tf.placeholder(tf.int32,
shape=[self.__batch_size, max_length],
name="data")
self.__labels = tf.placeholder(
tf.float32,
shape=[self.__batch_size, num_classes],
name="labels")
self.__seq_length = tf.placeholder(tf.int32,
shape=[self.__batch_size],
name="seqlength")
# LSTM definition
network = rnn_cell.LSTMCell(num_hidden, embedding_size)
network = rnn_cell.DropoutWrapper(network,
output_keep_prob=self.__dropout)
network = rnn_cell.MultiRNNCell([network] * num_layers)
# loaded value from word2vec
self.__embeddings_lstm = tf.Variable(tf.random_uniform(
[vocabulary_size, embedding_size], -1.0, 1.0),
name="embeddings_lstm")
# get the word vectors learned previously
embed = tf.nn.embedding_lookup(self.__embeddings_lstm, self.__data)
#try:
outputs, states = tf.contrib.rnn.static_rnn(
network,
self.__unpack_sequence(embed),
dtype=tf.float32,
sequence_length=self.__seq_length)
#except AttributeError:
#outputs, states = rnn(network, unpack_sequence(embed), dtype=tf.float32, sequence_length=seq_length)
# Compute an average of all the outputs
# FOR VARIABLE SEQUENCE LENGTHS
# place the entire sequence into one big tensor using tf_pack.
packed_op = tf.stack(outputs)
# reduce sum the 0th dimension, which is the number of timesteps.
summed_op = tf.reduce_sum(packed_op, reduction_indices=0)
# Then, divide by the seq_length input - this is an np array of size
# batch_size that stores the length of each sequence.
# With any luck, this gives the output results.
averaged_op = tf.div(summed_op,
tf.cast(self.__seq_length, tf.float32))
# output classifier
# TODO perhaps put this within a context manager
softmax_weight = tf.Variable(
tf.truncated_normal([num_hidden, num_classes], stddev=0.1))
softmax_bias = tf.Variable(tf.constant(0.1, shape=[num_classes]))
temp = tf.matmul(averaged_op, softmax_weight) + softmax_bias
prediction = tf.nn.softmax(temp)
predict_output = tf.argmax(prediction, 1)
tf.summary.histogram("prediction", prediction)
self.__prin = tf.Print(prediction, [prediction],
message="pred is ")
# standard cross entropy loss
self.__loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=temp,
labels=self.__labels))
tf.summary.scalar("loss-xentropy", self.__loss)
self.__optimizer = tf.train.AdamOptimizer(learn_rate).minimize(
self.__loss, global_step=self.__global_step)
# examine performance
with tf.variable_scope("accuracy"):
correct_predictions = tf.equal(tf.argmax(prediction, 1),
tf.argmax(self.__labels, 1))
self.__accuracy = tf.reduce_mean(tf.cast(
correct_predictions, tf.float32),
name="accuracy")
tf.summary.scalar("accuracy", self.__accuracy)
self.__merged = tf.summary.merge_all()
self.__train_writer = tf.summary.FileWriter(
os.getcwd() + '/train', self.__graph)
self.__test_writer = tf.summary.FileWriter(os.getcwd() + '/test')
def __init__(self, rnn_size, rnn_layer, batch_size, input_embedding_size,
dim_image, dim_hidden, max_words_q, vocabulary_size,
max_words_d, vocabulary_size_d, drop_out_rate, num_answers,
variation, offline_text):
self.rnn_size = rnn_size
self.rnn_layer = rnn_layer
self.batch_size = batch_size
self.input_embedding_size = input_embedding_size
self.dim_image = dim_image
self.dim_hidden = dim_hidden
self.max_words_q = max_words_q
self.vocabulary_size = vocabulary_size
self.max_words_d = max_words_d
self.vocabulary_size_d = vocabulary_size_d
self.drop_out_rate = drop_out_rate
self.num_answers = num_answers
self.variation = variation
self.offline_text = offline_text
# question-embedding
self.embed_ques_W = tf.Variable(tf.random.uniform(
[self.vocabulary_size, self.input_embedding_size], -0.08, 0.08),
name='embed_ques_W')
# encoder: RNN body
self.lstm_1 = rnn_cell.LSTMCell(rnn_size,
input_embedding_size,
state_is_tuple=False)
self.lstm_dropout_1 = rnn_cell.DropoutWrapper(self.lstm_1,
output_keep_prob=1 -
self.drop_out_rate)
self.lstm_2 = rnn_cell.LSTMCell(rnn_size,
rnn_size,
state_is_tuple=False)
self.lstm_dropout_2 = rnn_cell.DropoutWrapper(self.lstm_2,
output_keep_prob=1 -
self.drop_out_rate)
self.stacked_lstm = rnn_cell.MultiRNNCell(
[self.lstm_dropout_1, self.lstm_dropout_2])
# state-embedding
self.embed_state_W = tf.Variable(tf.random.uniform(
[2 * rnn_size * rnn_layer, self.dim_hidden], -0.08, 0.08),
name='embed_state_W')
self.embed_state_b = tf.Variable(tf.random.uniform([self.dim_hidden],
-0.08, 0.08),
name='embed_state_b')
if self.variation in ['isq', 'sq']:
if self.offline_text == "False":
# print("\n\n\noffline_text false\n\n\n")
# description-embedding
self.embed_desc_W = tf.Variable(tf.random.uniform(
[self.vocabulary_size_d, self.input_embedding_size], -0.08,
0.08),
name='embed_desc_W')
# encoder: RNN body
self.lstm_1_d = rnn_cell.LSTMCell(rnn_size,
input_embedding_size,
state_is_tuple=False)
self.lstm_dropout_1_d = rnn_cell.DropoutWrapper(
self.lstm_1_d, output_keep_prob=1 - self.drop_out_rate)
self.lstm_2_d = rnn_cell.LSTMCell(rnn_size,
rnn_size,
state_is_tuple=False)
self.lstm_dropout_2_d = rnn_cell.DropoutWrapper(
self.lstm_2_d, output_keep_prob=1 - self.drop_out_rate)
self.stacked_lstm_d = rnn_cell.MultiRNNCell(
[self.lstm_dropout_1_d, self.lstm_dropout_2_d])
# description state-embedding
self.embed_state_desc_W = tf.Variable(
tf.random.uniform(
[2 * rnn_size * rnn_layer, self.dim_hidden], -0.08,
0.08),
name='embed_state_desc_W')
elif self.offline_text == "True":
# print("\n\n\noffline_text true\n\n\n")
self.embed_state_desc_W = tf.Variable(
tf.random.uniform([self.dim_hidden, self.dim_hidden],
-0.08, 0.08),
name='embed_state_desc_W')
self.embed_state_desc_b = tf.Variable(tf.random.uniform(
[self.dim_hidden], -0.08, 0.08),
name='embed_state_desc_b')
if self.variation in ['isq', 'iq']:
# image-embedding 1
self.embed_image_W = tf.Variable(tf.random.uniform(
[dim_image, self.dim_hidden], -0.08, 0.08),
name='embed_image_W')
self.embed_image_b = tf.Variable(tf.random.uniform([dim_hidden],
-0.08, 0.08),
name='embed_image_b')
# my code
# image-embedding 2
self.embed_image2_W = tf.Variable(tf.random.uniform(
[dim_image, self.dim_hidden], -0.08, 0.08),
name='embed_image2_W')
self.embed_image2_b = tf.Variable(tf.random.uniform([dim_hidden],
-0.08, 0.08),
name='embed_image2_b')
# image-embedding 3
self.embed_image3_W = tf.Variable(tf.random.uniform(
[dim_image, self.dim_hidden], -0.08, 0.08),
name='embed_image3_W')
self.embed_image3_b = tf.Variable(tf.random.uniform([dim_hidden],
-0.08, 0.08),
name='embed_image3_b')
# image-embedding 4
self.embed_image4_W = tf.Variable(tf.random.uniform(
[dim_image, self.dim_hidden], -0.08, 0.08),
name='embed_image4_W')
self.embed_image4_b = tf.Variable(tf.random.uniform([dim_hidden],
-0.08, 0.08),
name='embed_image4_b')
# image-embedding 5
self.embed_image5_W = tf.Variable(tf.random.uniform(
[dim_image, self.dim_hidden], -0.08, 0.08),
name='embed_image5_W')
self.embed_image5_b = tf.Variable(tf.random.uniform([dim_hidden],
-0.08, 0.08),
name='embed_image5_b')
# options-embedding
self.embed_options_W = tf.Variable(tf.random.uniform(
[self.num_answers, options_embedding_size], -0.1, 0.1),
name='embed_options_W')
# print("\n\nself.embed_options_W: {}\n\n".format(self.embed_options_W))
# self.lstm_o = rnn_cell.LSTMCell(64, state_is_tuple=False, reuse=tf.AUTO_REUSE)
# self.lstm_1_o = rnn_cell.LSTMCell(rnn_size, input_embedding_size, state_is_tuple=False)
# self.lstm_dropout_1_o = rnn_cell.DropoutWrapper(self.lstm_1_o, output_keep_prob = 1 - self.drop_out_rate)
# self.lstm_2_o = rnn_cell.LSTMCell(rnn_size, rnn_size, state_is_tuple=False)
# self.lstm_dropout_2_o = rnn_cell.DropoutWrapper(self.lstm_2_o, output_keep_prob = 1 - self.drop_out_rate)
# self.stacked_lstm_o = rnn_cell.MultiRNNCell([self.lstm_dropout_1_o, self.lstm_dropout_2_o])
# options state-embedding
# self.embed_options_state_W = tf.Variable(tf.random.uniform([2*rnn_size*rnn_layer, self.dim_hidden], -0.08,0.08),name='embed_options_state_W')
# self.embed_options_b = tf.Variable(tf.random.uniform([self.dim_hidden], -0.08, 0.08), name='embed_options_b')
# end my code
# score-embedding for 5-way
self.embed_score_W = tf.Variable(tf.random.uniform(
[dim_hidden, options_embedding_size], -0.08, 0.08),
name='embed_score_W')
self.embed_h_b = tf.Variable(tf.random.uniform(
[options_embedding_size], -0.08, 0.08),
name='embed_h_b')
self.embed_score_b = tf.Variable(tf.random.uniform([num_output], -0.08,
0.08),
name='embed_score_b')