def __init__(self, brain, h_size=128, lr=1e-4, n_layers=2, m_size=128,
normalize=False, use_recurrent=False):
LearningModel.__init__(self, m_size, normalize, use_recurrent, brain)
num_streams = 1
hidden_streams = self.create_new_obs(num_streams, h_size, n_layers)
hidden = hidden_streams[0]
self.dropout_rate = tf.placeholder(dtype=tf.float32, shape=[], name="dropout_rate")
hidden_reg = tf.layers.dropout(hidden, self.dropout_rate)
if self.use_recurrent:
self.memory_in = tf.placeholder(shape=[None, self.m_size], dtype=tf.float32, name='recurrent_in')
hidden_reg, self.memory_out = self.create_recurrent_encoder(hidden_reg, self.memory_in)
self.memory_out = tf.identity(self.memory_out, name='recurrent_out')
self.policy = tf.layers.dense(hidden_reg, self.a_size, activation=None, use_bias=False,
kernel_initializer=c_layers.variance_scaling_initializer(factor=0.01))
if brain.vector_action_space_type == "discrete":
self.action_probs = tf.nn.softmax(self.policy)
self.sample_action_float = tf.multinomial(self.policy, 1)
self.sample_action_float = tf.identity(self.sample_action_float, name="action")
self.sample_action = tf.cast(self.sample_action_float, tf.int32)
self.true_action = tf.placeholder(shape=[None], dtype=tf.int32, name="teacher_action")
self.action_oh = tf.one_hot(self.true_action, self.a_size)
self.loss = tf.reduce_sum(-tf.log(self.action_probs + 1e-10) * self.action_oh)
self.action_percent = tf.reduce_mean(tf.cast(
tf.equal(tf.cast(tf.argmax(self.action_probs, axis=1), tf.int32), self.sample_action), tf.float32))
else:
self.sample_action = tf.identity(self.policy, name="action")
self.true_action = tf.placeholder(shape=[None, self.a_size], dtype=tf.float32, name="teacher_action")
self.loss = tf.reduce_sum(tf.squared_difference(self.true_action, self.sample_action))
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
self.update = optimizer.minimize(self.loss)
def _g_recurrence_2(i, x_t, gen_x, h_tm1, h_tm1_manager, last_goal, real_goal):
# with tf.device('/cpu:0'):
cur_sen = tf.cond(i > 0, lambda:
tf.split(tf.concat([tf.transpose(gen_x.stack(), perm=[1, 0]), self.padding_array], 1),
[self.sequence_length, i - 1], 1)[0], lambda: self.padding_array)
with tf.variable_scope(self.scope):
feature = self.FeatureExtractor_unit(cur_sen, self.drop_out)
h_t_Worker = self.g_worker_recurrent_unit(x_t, h_tm1) # hidden_memory_tuple
o_t_Worker = self.g_worker_output_unit(h_t_Worker) # batch x vocab , logits not prob
o_t_Worker = tf.reshape(o_t_Worker, [self.batch_size, self.num_vocabulary, self.goal_size])
h_t_manager = self.g_manager_recurrent_unit(feature, h_tm1_manager)
sub_goal = self.g_manager_output_unit(h_t_manager)
sub_goal = tf.nn.l2_normalize(sub_goal, 1)
real_sub_goal = tf.add(last_goal, sub_goal)
w_g = tf.matmul(real_goal, self.g_change) # batch x goal_size
w_g = tf.nn.l2_normalize(w_g, 1)
w_g = tf.expand_dims(w_g, 2) # batch x goal_size x 1
x_logits = tf.matmul(o_t_Worker, w_g)
x_logits = tf.squeeze(x_logits)
log_prob = tf.log(tf.nn.softmax(x_logits))
next_token = tf.cast(tf.reshape(tf.multinomial(log_prob, 1), [self.batch_size]), tf.int32)
x_tp1 = tf.nn.embedding_lookup(self.g_embeddings, next_token) # batch x emb_dim
with tf.control_dependencies([cur_sen]):
gen_x = gen_x.write(i - 1, next_token) # indices, batch_size
return i + 1, x_tp1, gen_x, h_t_Worker, h_t_manager, \
tf.cond(((i) % self.step_size) > 0, lambda: real_sub_goal,
lambda: tf.constant(0.0, shape=[self.batch_size, self.goal_out_size])), \
tf.cond(((i) % self.step_size) > 0, lambda: real_goal, lambda: real_sub_goal)
def testSmallEntropy(self):
with self.test_session(use_gpu=self.use_gpu):
# A logit value of -10 corresponds to a probability of ~5e-5.
logits = tf.constant([[-10., 10., -10.], [-10., -10., 10.]])
num_samples = 1000
samples = tf.multinomial(logits, num_samples).eval()
self.assertAllEqual([[1] * num_samples, [2] * num_samples], samples)
def body(i, prev_base_state, prev_high_states, prev_y, prev_emb,
y_array):
state1 = decoder.grustep1.forward(prev_base_state, prev_emb)
att_ctx = decoder.attstep.forward(state1)
base_state = decoder.grustep2.forward(state1, att_ctx)
if decoder.high_gru_stack == None:
output = base_state
high_states = []
else:
if decoder.high_gru_stack.context_state_size == 0:
output, high_states = decoder.high_gru_stack.forward_single(
prev_high_states, base_state)
else:
output, high_states = decoder.high_gru_stack.forward_single(
prev_high_states, base_state, context=att_ctx)
logits = decoder.predictor.get_logits(prev_emb, output, att_ctx,
multi_step=False)
new_y = tf.multinomial(logits, num_samples=1)
new_y = tf.cast(new_y, dtype=tf.int32)
new_y = tf.squeeze(new_y, axis=1)
new_y = tf.where(tf.equal(prev_y, tf.constant(0, dtype=tf.int32)),
tf.zeros_like(new_y), new_y)
y_array = y_array.write(index=i, value=new_y)
new_emb = decoder.y_emb_layer.forward(new_y, factor=0)
return i+1, base_state, high_states, new_y, new_emb, y_array
def testEmpty(self):
classes = 5
with self.test_session(use_gpu=self.use_gpu):
for batch in 0, 3:
for samples in 0, 7:
x = tf.multinomial(tf.zeros([batch, classes]), samples).eval()
self.assertEqual(x.shape, (batch, samples))
def loop_function(prev,_):
prev = tf.nn.xw_plus_b(
prev, output_projection[0], output_projection[1])
prev_symbol = tf.cast(tf.reshape(tf.multinomial(prev, 1), [FLAGS.batch_size*FLAGS.max_dec_sen_num]), tf.int32)
emb_prev = tf.nn.embedding_lookup(embedding, prev_symbol)
return emb_prev
def UpdateProbs(self, inp):
"""Update probabilities of each particle based on 2D matrix inp which is a 2D perspectiuve projection of the scene"""
projection, onscreen = self.project()
filtered_projection = tf.to_int64(tf.select(onscreen, projection, tf.zeros_like(projection)))
per_state_probabilities = tf.gather_nd(inp, filtered_projection)
filtered_probabilities = tf.select(onscreen, per_state_probabilities, tf.zeros_like(per_state_probabilities))
new_state_indicies = tf.squeeze(tf.multinomial(tf.expand_dims(tf.log(filtered_probabilities),0), self.particles/10*9))
new_state = tf.gather(self.state, new_state_indicies)
# Add momentum
new_state = tf.concat(1, [new_state[:, 0:3] + new_state[:, 3:6], new_state[:, 3:10]])
# Add in particles for the "just come onscreen" case.
new_state = tf.concat(0, [new_state, tf.random_normal([self.particles/10, 10]) * self.initial_std + self.initial_bias])
new_state = new_state + tf.random_normal([self.particles, 10]) * self.update_std
# Todo: permute state by adding noise.
return self.state.assign(new_state)
def call(self, inputs):
"""Calculates logits and action.
Args:
inputs: Observations from a step in the cart-pole environment, of shape
`(batch_size, input_size)`
Returns:
logits: the logits output by the output layer. This can be viewed as the
likelihood vales of choosing the left (0) action. Shape:
`(batch_size, 1)`.
actions: randomly selected actions ({0, 1}) based on the logits. Shape:
`(batch_size, 1)`.
"""
hidden = self._hidden_layer(inputs)
logits = self._output_layer(hidden)
left_prob = tf.nn.sigmoid(logits)
action_probs = tf.concat([left_prob, 1.0 - left_prob], 1)
self._grad_fn = eager.implicit_gradients(
self._get_cross_entropy_and_save_actions)
actions = tf.multinomial(tf.log(action_probs), 1)
return logits, actions
def multinomial_squeeze(logits, temperature=1.0):
logits_shape = common_layers.shape_list(logits)
reshaped_logits = (
tf.reshape(logits, [-1, logits_shape[-1]]) / temperature)
choices = tf.multinomial(reshaped_logits, 1)
choices = tf.reshape(choices, logits_shape[:-1])
return choices
def generate_string(self, initial_logits, initial_state, sequence_length):
"""Builds sub-graph to generate a string, sampled from the model.
Args:
initial_logits: Starting logits to sampling from.
initial_state: Starting state for the RNN core.
sequence_length: Number of characters to sample.
Returns:
A Tensor of characters, with dimensions `[sequence_length, batch_size,
output_size]`.
"""
current_logits = initial_logits
current_state = initial_state
generated_letters = []
for _ in range(sequence_length):
# Sample a character index from distribution.
char_index = tf.squeeze(tf.multinomial(current_logits, 1))
char_one_hot = tf.one_hot(char_index, self._output_size, 1.0, 0.0)
generated_letters.append(char_one_hot)
# Feed character back into the deep_lstm.
gen_out_seq, current_state = self._core(
tf.nn.relu(self._embed_module(char_one_hot)),
current_state)
current_logits = self._output_module(gen_out_seq)
generated_string = tf.stack(generated_letters)
return generated_string
def create_dc_actor_critic(self, h_size, num_layers):
num_streams = 1
hidden_streams = self.create_new_obs(num_streams, h_size, num_layers)
hidden = hidden_streams[0]
if self.use_recurrent:
tf.Variable(self.m_size, name="memory_size", trainable=False, dtype=tf.int32)
self.prev_action = tf.placeholder(shape=[None], dtype=tf.int32, name='prev_action')
self.prev_action_oh = c_layers.one_hot_encoding(self.prev_action, self.a_size)
hidden = tf.concat([hidden, self.prev_action_oh], axis=1)
self.memory_in = tf.placeholder(shape=[None, self.m_size], dtype=tf.float32, name='recurrent_in')
hidden, self.memory_out = self.create_recurrent_encoder(hidden, self.memory_in)
self.memory_out = tf.identity(self.memory_out, name='recurrent_out')
self.policy = tf.layers.dense(hidden, self.a_size, activation=None, use_bias=False,
kernel_initializer=c_layers.variance_scaling_initializer(factor=0.01))
self.all_probs = tf.nn.softmax(self.policy, name="action_probs")
self.output = tf.multinomial(self.policy, 1)
self.output = tf.identity(self.output, name="action")
self.value = tf.layers.dense(hidden, 1, activation=None)
self.value = tf.identity(self.value, name="value_estimate")
self.entropy = -tf.reduce_sum(self.all_probs * tf.log(self.all_probs + 1e-10), axis=1)
self.action_holder = tf.placeholder(shape=[None], dtype=tf.int32)
self.selected_actions = c_layers.one_hot_encoding(self.action_holder, self.a_size)
self.all_old_probs = tf.placeholder(shape=[None, self.a_size], dtype=tf.float32, name='old_probabilities')
# We reshape these tensors to [batch x 1] in order to be of the same rank as continuous control probabilities.
self.probs = tf.expand_dims(tf.reduce_sum(self.all_probs * self.selected_actions, axis=1), 1)
self.old_probs = tf.expand_dims(tf.reduce_sum(self.all_old_probs * self.selected_actions, axis=1), 1)
def decoder_fn(time, cell_state, cell_input, cell_output, context_state):
with tf.name_scope(name, "simple_decoder_fn_inference",
[time, cell_state, cell_input, cell_output,
context_state]):
if cell_input is not None:
raise ValueError("Expected cell_input to be None, but saw: %s" %
cell_input)
if cell_output is None:
# invariant that this is time == 0
next_input_id = tf.ones([batch_size], dtype=dtype) * (
start_of_sequence_id)
done = tf.zeros([batch_size], dtype=tf.bool)
cell_state = encoder_state
cell_output = tf.zeros([cell_size],
dtype=tf.float32)
else:
softmax_output = output_fn(cell_output)
if sample:
next_input_id = tf.squeeze(tf.multinomial(softmax_output, 1), 1)
else:
next_input_id = tf.argmax(softmax_output, 1)
next_input_id = tf.cast(next_input_id, dtype=dtype)
done = tf.equal(next_input_id, end_of_sequence_id)
next_input = tf.gather(embeddings, next_input_id)
# if time > maxlen, return all true vector
done = tf.cond(
tf.greater(time, maximum_length),
lambda: tf.ones([batch_size], dtype=tf.bool),
lambda: done)
return (done, cell_state, next_input, next_input_id, context_state)
def __init__(self, q_values, observations, num_actions, stochastic, eps,
softmax, softmax_temp):
if softmax:
action_dist = Categorical(q_values / softmax_temp)
self.action = action_dist.sample()
self.action_prob = action_dist.sampled_action_prob()
return
deterministic_actions = tf.argmax(q_values, axis=1)
batch_size = tf.shape(observations)[0]
# Special case masked out actions (q_value ~= -inf) so that we don't
# even consider them for exploration.
random_valid_action_logits = tf.where(
tf.equal(q_values, tf.float32.min),
tf.ones_like(q_values) * tf.float32.min, tf.ones_like(q_values))
random_actions = tf.squeeze(
tf.multinomial(random_valid_action_logits, 1), axis=1)
chose_random = tf.random_uniform(
tf.stack([batch_size]), minval=0, maxval=1, dtype=tf.float32) < eps
stochastic_actions = tf.where(chose_random, random_actions,
deterministic_actions)
self.action = tf.cond(stochastic, lambda: stochastic_actions,
lambda: deterministic_actions)
self.action_prob = None
def _sample_single(args):
logits, n_draw = args[0], args[1] # [K], []
x = tf.multinomial(logits[tf.newaxis, ...], n_draw,
seed) # [1, n*n_draw]
x = tf.reshape(x, shape=[n, -1]) # [n, n_draw]
x = tf.reduce_sum(tf.one_hot(x, depth=k), axis=-2) # [n, k]
return x
def sample(self, projected_output):
"""Return integer ID tensor representing the sampled word.
Args:
projected_output: Tensor [1, 1, state_size], representing a single
decoder timestep output.
"""
# TODO: We really need a tf.control_dependencies check here (for rank).
with tf.name_scope('decoder_sampler', values=[projected_output]):
# Protect against extra size-1 dimensions; grab the 1D tensor
# of size state_size.
logits = tf.squeeze(projected_output)
if self.temperature < 0.02:
return tf.argmax(logits, axis=0)
# Convert logits to probability distribution.
probabilities = tf.div(logits, self.temperature)
projected_output = tf.div(
tf.exp(probabilities),
tf.reduce_sum(tf.exp(probabilities), axis=-1))
# Sample 1 time from the probability distribution.
sample_ID = tf.squeeze(
tf.multinomial(tf.expand_dims(probabilities, 0), 1))
return sample_ID
def testNegativeMinLogits(self):
tf.set_random_seed(78844)
with self.test_session(use_gpu=self.use_gpu):
logits = tf.constant([[np.finfo(np.float32).min] * 1023 + [0]])
num_samples = 1000
samples = tf.multinomial(logits, num_samples).eval()
self.assertAllEqual([[1023] * num_samples], samples)
def build_generator(self):
"""
Generator for generating captions
Support sample max or sample from distribution
No Beam search here; beam search is in decoder
"""
# Variables for the sample setting
self.sample_max = tf.Variable(True, trainable = False, name = "sample_max")
self.sample_temperature = tf.Variable(1.0, trainable = False, name = "temperature")
self.generator = []
with tf.variable_scope("rnnlm"):
flattened_ctx = tf.reshape(self.context, [self.batch_size, 196, 512])
ctx_mean = tf.reduce_mean(flattened_ctx, 1)
tf.get_variable_scope().reuse_variables()
initial_state = utils.get_initial_state(ctx_mean, self.cell.state_size)
#projected context
# This is used in attention module; do this outside the loop to reduce redundant computations
# with tf.variable_scope("attention"):
if self.att_hid_size == 0:
pctx = slim.fully_connected(flattened_ctx, 1, activation_fn = None, scope = 'ctx_att') # (batch) * 196 * 1
else:
pctx = slim.fully_connected(flattened_ctx, self.att_hid_size, activation_fn = None, scope = 'ctx_att') # (batch) * 196 * att_hid_size
rnn_input = tf.nn.embedding_lookup(self.Wemb, tf.zeros([self.batch_size], tf.int32))
prev_h = utils.last_hidden_vec(initial_state)
self.g_alphas = []
outputs = []
state = initial_state
for ind in range(MAX_STEPS):
with tf.variable_scope("attention"):
alpha = self.get_alpha(prev_h, pctx)
self.g_alphas.append(alpha)
weighted_context = tf.reduce_sum(flattened_ctx * tf.expand_dims(alpha, 2), 1)
output, state = self.cell(tf.concat(axis=1, values=[weighted_context, rnn_input]), state)
outputs.append(output)
prev_h = output
# Get the input of next timestep
prev_logit = slim.fully_connected(prev_h, self.vocab_size + 1, activation_fn = None, scope = 'logit')
prev_symbol = tf.stop_gradient(tf.cond(self.sample_max,
lambda: tf.argmax(prev_logit, 1), # pick the word with largest probability as the input of next time step
lambda: tf.squeeze(
tf.multinomial(tf.nn.log_softmax(prev_logit) / self.sample_temperature, 1), 1))) # Sample from the distribution
self.generator.append(prev_symbol)
rnn_input = tf.nn.embedding_lookup(self.Wemb, prev_symbol)
self.g_output = output = tf.reshape(tf.concat(axis=1, values=outputs), [-1, self.rnn_size]) # outputs[1:], because we don't calculate loss on time 0.
self.g_logits = logits = slim.fully_connected(output, self.vocab_size + 1, activation_fn = None, scope = 'logit')
self.g_probs = probs = tf.reshape(tf.nn.softmax(logits), [self.batch_size, MAX_STEPS, self.vocab_size + 1])
self.generator = tf.transpose(tf.reshape(tf.concat(axis=0, values=self.generator), [MAX_STEPS, -1]))
def st_sampler(logits):
"""straight-through stochastic sampler"""
flat_samples = tf.reshape(tf.multinomial(tf.reshape(logits, [-1, len(charmap)]), 1), [-1])
onehot = tf.reshape(tf.one_hot(flat_samples, len(charmap)), tf.shape(logits))
residual = onehot - logits
onehot = logits + tf.stop_gradient(residual)
return onehot
def multinomial_sample(x, vocab_size, temperature):
"""Multinomial sampling from a n-dimensional tensor."""
if temperature > 0:
samples = tf.multinomial(tf.reshape(x, [-1, vocab_size]) / temperature, 1)
else:
samples = tf.argmax(x, axis=-1)
reshaped_samples = tf.reshape(samples, common_layers.shape_list(x)[:-1])
return tf.to_int32(reshaped_samples)
def _g_recurrence_2(i, x_t, h_tm1, given_num, gen_x):
h_t = self.g_recurrent_unit(x_t, h_tm1) # hidden_memory_tuple
o_t = self.g_output_unit(h_t) # batch x vocab , logits not prob
log_prob = tf.log(tf.nn.softmax(o_t))
next_token = tf.cast(tf.reshape(tf.multinomial(log_prob, 1), [self.batch_size]), tf.int32)
x_tp1 = tf.nn.embedding_lookup(self.g_embeddings, next_token) # batch x emb_dim
gen_x = gen_x.write(i, next_token) # indices, batch_size
return i + 1, x_tp1, h_t, given_num, gen_x
def sample_from_logits(logits):
with tf.control_dependencies([tf.assert_greater(temperature, 0.0)]):
logits = tf.identity(logits)
reshaped_logits = (
tf.reshape(logits, [-1, tf.shape(logits)[-1]]) / temperature)
choices = tf.multinomial(reshaped_logits, 1)
choices = tf.reshape(choices,
tf.shape(logits)[:logits.get_shape().ndims - 1])
return choices
def provide_one_hot_labels(self, batch_size):
"""Provides one hot labels."""
pitch_counts = self.get_pitch_counts()
pitches = sorted(pitch_counts.keys())
counts = [pitch_counts[p] for p in pitches]
indices = tf.reshape(
tf.multinomial(tf.log([tf.to_float(counts)]), batch_size), [batch_size])
one_hot_labels = tf.one_hot(indices, depth=len(pitches))
return one_hot_labels
def loop(prev, i):
with tf.variable_scope(rnnlm_scope):
prev = slim.fully_connected(prev, self.vocab_size + 1, activation_fn = None, scope = 'logit')
prev_symbol = tf.stop_gradient(tf.cond(self.sample_max,
lambda: tf.argmax(prev, 1), # pick the word with largest probability as the input of next time step
lambda: tf.squeeze(
tf.multinomial(tf.nn.log_softmax(prev) / self.sample_temperature, 1), 1))) # Sample from the distribution
self.generator.append(prev_symbol)
return tf.nn.embedding_lookup(self.Wemb, prev_symbol)
def _pretrain_recurrence(i, x_t, h_tm1, g_predictions):
h_t = self.g_recurrent_unit(x_t, h_tm1)
o_t = self.g_output_unit(h_t)
log_prob = tf.log(tf.nn.softmax(o_t))
next_token = tf.cast(tf.reshape(tf.multinomial(log_prob, 1), [self.batch_size]), tf.int32)
g_predictions = g_predictions.write(i, tf.nn.softmax(o_t)) # batch x vocab_size
x_tp1 = tf.cond(tf.less(tf.constant(random.random()), self.curriculum_rate), lambda: ta_emb_x.read(i),
lambda: tf.nn.embedding_lookup(self.g_embeddings, next_token))
return i + 1, x_tp1, h_t, g_predictions
def testLargeLogits(self):
for neg in [True, False]:
with self.test_session(use_gpu=self.use_gpu):
logits = np.array([[1000.] * 5])
if neg:
logits *= -1
samples = tf.multinomial(logits, 10).eval()
# Sampled classes should be in-range.
self.assertTrue((samples >= 0).all())
self.assertTrue((samples < 5).all())
def create_model(seq, temp, vocab, hidden=HIDDEN_SIZE):
seq = tf.one_hot(seq, len(vocab))
output, in_state, out_state = create_rnn(seq, hidden)
# fully_connected is syntactic sugar for tf.matmul(w, output) + b
# it will create w and b for us
logits = tf.contrib.layers.fully_connected(output, len(vocab), None)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits[:, :-1], seq[:, 1:]))
# sample the next word from Maxwell-Boltzmann Distribution with temperature temp
sample = tf.multinomial(tf.exp(logits[:, -1] / temp), 1)[:, 0]
return loss, sample, in_state, out_state
def _g_recurrence_temperature(i, x_t, h_tm1, gen_o_temp, gen_x_temp, alpha):
h_t = self.g_recurrent_unit(x_t, h_tm1) # hidden_memory_tuple
o_t = self.g_output_unit(h_t)/alpha # batch x vocab , logits not prob
log_prob = tf.log(tf.nn.softmax(o_t))
next_token = tf.cast(tf.reshape(tf.multinomial(log_prob, 1), [self.batch_size]), tf.int32)
x_tp1 = tf.nn.embedding_lookup(self.g_embeddings, next_token) # batch x emb_dim
gen_o_temp = gen_o_temp.write(i, tf.reduce_sum(tf.multiply(tf.one_hot(next_token, self.num_vocabulary, 1.0, 0.0),
tf.nn.softmax(o_t)), 1)) # [batch_size] , prob
gen_x_temp = gen_x_temp.write(i, next_token) # indices, batch_size
return i + 1, x_tp1, h_t, gen_o_temp, gen_x_temp, alpha
def body(past, prev, output):
next_outputs = step(hparams, prev[:, tf.newaxis], past=past)
logits = next_outputs['logits'][:, -1, :] / tf.to_float(temperature)
logits = top_k_logits(logits, k=top_k)
samples = tf.multinomial(logits, num_samples=1, output_dtype=tf.int32)
return [
tf.concat([past, next_outputs['presents']], axis=-2),
tf.squeeze(samples, axis=[1]),
tf.concat([output, samples], axis=1),
]
def create_model(self):
seq = tf.one_hot(self.seq, len(self.vocab))
self.create_rnn(seq)
self.logits = tf.layers.dense(self.output, len(self.vocab), None)
loss = tf.nn.softmax_cross_entropy_with_logits(logits=self.logits[:, :-1],
labels=seq[:, 1:])
self.loss = tf.reduce_sum(loss)
# sample the next character from Maxwell-Boltzmann Distribution
# with temperature temp. It works equally well without tf.exp
self.sample = tf.multinomial(tf.exp(self.logits[:, -1] / self.temp), 1)[:, 0]
self.opt = tf.train.AdamOptimizer(self.lr).minimize(self.loss, global_step=self.gstep)
def __init__(self, dim):
self._dim = dim
weights_var = tf.placeholder(
dtype=tf.float32,
shape=(None, dim),
name="weights"
)
self._f_sample = tensor_utils.compile_function(
inputs=[weights_var],
outputs=tf.multinomial(weights_var, num_samples=1)[:, 0],
)
def create_variables(self):
with tf.name_scope("model_inputs"):
self.states = tf.placeholder(dtype=tf.float32,
shape=(None, self.state_dim),
name="states")
# rollout action based on current policy
with tf.name_scope("predict_actions"):
# initialize actor-critic network
with tf.variable_scope("actor_network"):
self.policy_outputs = self.actor_network(self.states)
with tf.variable_scope("critic_network"):
self.value_outputs = self.critic_network(self.states)
# predict actions from policy network
self.action_scores = tf.identity(self.policy_outputs,
name="action_scores")
self.predicted_actions = tf.multinomial(self.action_scores, 1)
# get variable list
actor_network_variables = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope="actor_network")
critic_network_variables = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope="critic_network")
# compute loss and gradients
with tf.name_scope("compute_pg_gradients"):
# gradients for selecting action from policy network
self.taken_actions = tf.placeholder(dtype=tf.int32,
shape=(None, ),
name="taken_actions")
self.discounted_rewards = tf.placeholder(dtype=tf.float32,
shape=(None, ),
name="discounted_rewards")
with tf.variable_scope("actor_network", reuse=True):
self.logprobs = self.actor_network(self.states)
with tf.variable_scope("critic_network", reuse=True):
self.estimated_values = self.critic_network(self.states)
# compute policy loss and regularization loss
self.cross_entropy_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.logprobs, labels=self.taken_actions)
self.pg_loss = tf.reduce_mean(self.cross_entropy_loss)
self.actor_reg_loss = tf.reduce_sum(
[tf.reduce_sum(tf.square(x)) for x in actor_network_variables])
self.actor_loss = self.pg_loss + self.reg_param * self.actor_reg_loss
# compute actor gradients
self.actor_gradients = self.optimizer.compute_gradients(
self.actor_loss, actor_network_variables)
# compute advantages A(s) = R - V(s)
self.advantages = tf.reduce_sum(self.discounted_rewards -
self.estimated_values)
# compute policy gradients
for i, (grad, var) in enumerate(self.actor_gradients):
if grad is not None:
self.actor_gradients[i] = (grad * self.advantages, var)
# compute critic gradients
self.mean_square_loss = tf.reduce_mean(
tf.square(self.discounted_rewards - self.estimated_values))
self.critic_reg_loss = tf.reduce_sum([
tf.reduce_sum(tf.square(x)) for x in critic_network_variables
])
self.critic_loss = self.mean_square_loss + self.reg_param * self.critic_reg_loss
self.critic_gradients = self.optimizer.compute_gradients(
self.critic_loss, critic_network_variables)
# collect all gradients
self.gradients = self.actor_gradients + self.critic_gradients
# gradients clipping by norm
for i, (grad, var) in enumerate(self.gradients):
if grad is not None:
self.gradients[i] = (tf.clip_by_norm(
grad, self.max_gradient), var)
# summarize gradients
for grad, var in self.gradients:
tf.summary.histogram(var.name, var)
if grad is not None:
tf.summary.histogram(var.name + '/gradients', grad)
# emit summaries
tf.summary.histogram("estimated_values", self.estimated_values)
tf.summary.scalar("actor_loss", self.actor_loss)
tf.summary.scalar("critic_loss", self.critic_loss)
tf.summary.scalar("reg_loss",
self.actor_reg_loss + self.critic_reg_loss)
# training update
with tf.name_scope("train_actor_critic"):
# apply gradients to update actor network
self.train_op = self.optimizer.apply_gradients(self.gradients)
self.summarize = tf.summary.merge_all()
self.no_op = tf.no_op()
def build_policy_network_op(self, scope="policy_network"):
"""
Build the policy network, construct the tensorflow operation to sample
actions from the policy network outputs, and compute the log probabilities
of the actions taken (for computing the loss later). These operations are
stored in self.sampled_action and self.logprob. Must handle both settings
of self.discrete.
Args:
scope: the scope of the neural network
TODO:
Discrete case:
action_logits: the logits for each action
HINT: use build_mlp, check self.config for layer_size and
n_layers
self.sampled_action: sample from these logits
HINT: use tf.multinomial + tf.squeeze
self.logprob: compute the log probabilities of the taken actions
HINT: 1. tf.nn.sparse_softmax_cross_entropy_with_logits computes
the *negative* log probabilities of labels, given logits.
2. taken actions are different than sampled actions!
Continuous case:
To build a policy in a continuous action space domain, we will have the
model output the means of each action dimension, and then sample from
a multivariate normal distribution with these means and trainable standard
deviation.
That is, the action a_t ~ N( mu(o_t), sigma)
where mu(o_t) is the network that outputs the means for each action
dimension, and sigma is a trainable variable for the standard deviations.
N here is a multivariate gaussian distribution with the given parameters.
action_means: the predicted means for each action dimension.
HINT: use build_mlp, check self.config for layer_size and
n_layers
log_std: a trainable variable for the log standard deviations.
HINT: think about why we use log std as the trainable variable instead of std
HINT: use tf.get_variable
HINT: The shape of this should match the shape of action dimension
self.sampled_action: sample from the gaussian distribution as described above
HINT: use tf.random_normal
HINT: use re-parametrization to obtain N(mu, sigma) from N(0, 1)
self.lobprob: the log probabilities of the taken actions
HINT: use tf.contrib.distributions.MultivariateNormalDiag
"""
#######################################################
######### YOUR CODE HERE - 8-12 lines. ############
if self.discrete:
mlp_out = build_mlp(self.observation_placeholder, self.action_dim,
scope, self.config.n_layers,
self.config.layer_size, self.config.activation)
self.sampled_action = tf.squeeze(
tf.multinomial(mlp_out, num_samples=1), 1)
self.logprob = -tf.nn.sparse_softmax_cross_entropy_with_logits(
self.action_placeholder, mlp_out)
else:
action_means = build_mlp(self.observation_placeholder,
self.action_dim, scope,
self.config.n_layers,
self.config.layer_size)
log_std = tf.get_variable("log_std",
shape=[1, self.action_dim],
trainable=True)
self.sampled_action = tf.random_normal((1, ),
mean=action_means,
stddev=tf.math.exp(log_std))
mvn = tf.contrib.distributions.MultivariateNormalDiag(
action_means, tf.math.exp(log_std))
self.lobprob = mvn.log_prob(self.action_placeholder).eval()
def _body(layer_id, inputs, prev_c, prev_h, anchors, anchors_w_1,
arc_seq, entropy, log_prob):
indices = tf.range(0, layer_id, dtype=tf.int32)
start_id = 4 * (layer_id - 2)
prev_layers = []
for i in range(2): # index_1, index_2
next_c, next_h = stack_lstm(inputs, prev_c, prev_h,
self.w_lstm)
prev_c, prev_h = next_c, next_h
query = anchors_w_1.gather(indices)
query = tf.reshape(query, [layer_id, self.lstm_size])
query = tf.tanh(query + tf.matmul(next_h[-1], self.w_attn_2))
query = tf.matmul(query, self.v_attn)
logits = tf.reshape(query, [1, layer_id])
if self.temperature is not None:
logits /= self.temperature
if self.tanh_constant is not None:
logits = self.tanh_constant * tf.tanh(logits)
index = tf.multinomial(logits, 1)
index = tf.to_int32(index)
index = tf.reshape(index, [1])
arc_seq = arc_seq.write(start_id + 2 * i, index)
curr_log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=index)
log_prob += curr_log_prob
curr_ent = tf.stop_gradient(
tf.nn.softmax_cross_entropy_with_logits(
logits=logits, labels=tf.nn.softmax(logits)))
entropy += curr_ent
prev_layers.append(anchors.read(tf.reduce_sum(index)))
inputs = prev_layers[-1]
for i in range(2): # op_1, op_2
next_c, next_h = stack_lstm(inputs, prev_c, prev_h,
self.w_lstm)
prev_c, prev_h = next_c, next_h
logits = tf.matmul(next_h[-1], self.w_soft) + self.b_soft
if self.temperature is not None:
logits /= self.temperature
if self.tanh_constant is not None:
op_tanh = self.tanh_constant / self.op_tanh_reduce
logits = op_tanh * tf.tanh(logits)
if use_bias:
logits += self.b_soft_no_learn
op_id = tf.multinomial(logits, 1)
op_id = tf.to_int32(op_id)
op_id = tf.reshape(op_id, [1])
arc_seq = arc_seq.write(start_id + 2 * i + 1, op_id)
curr_log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=op_id)
log_prob += curr_log_prob
curr_ent = tf.stop_gradient(
tf.nn.softmax_cross_entropy_with_logits(
logits=logits, labels=tf.nn.softmax(logits)))
entropy += curr_ent
inputs = tf.nn.embedding_lookup(self.w_emb, op_id)
next_c, next_h = stack_lstm(inputs, prev_c, prev_h, self.w_lstm)
anchors = anchors.write(layer_id, next_h[-1])
anchors_w_1 = anchors_w_1.write(
layer_id, tf.matmul(next_h[-1], self.w_attn_1))
inputs = self.g_emb
return (layer_id + 1, inputs, next_c, next_h, anchors, anchors_w_1,
arc_seq, entropy, log_prob)
def __init__(self, state_shape, n_actions, n_hidden, summary=True):
super(ActorCriticNetworkDiscreteCNNRNN, self).__init__()
self.state_shape = state_shape
self.n_actions = n_actions
self.n_hidden = n_hidden
self.summary = summary
self.states = tf.placeholder(tf.float32, [None] + state_shape,
name="states")
self.adv = tf.placeholder(tf.float32, name="advantage")
self.actions_taken = tf.placeholder(tf.float32, name="actions_taken")
self.r = tf.placeholder(tf.float32, [None], name="r")
x = self.states
# Convolution layers
for i in range(4):
x = tf.nn.elu(conv2d(x, 32, "l{}".format(i + 1), [3, 3], [2, 2]))
# Flatten
reshape = tf.expand_dims(flatten(x), [0])
lstm_size = 256
self.enc_cell = tf.contrib.rnn.BasicLSTMCell(lstm_size)
lstm_state_size = self.enc_cell.state_size
c_init = np.zeros((1, lstm_state_size.c), np.float32)
h_init = np.zeros((1, lstm_state_size.h), np.float32)
self.state_init = [c_init, h_init]
self.rnn_state_in = self.enc_cell.zero_state(1, tf.float32)
tf.add_to_collection("rnn_state_in_c", self.rnn_state_in.c)
tf.add_to_collection("rnn_state_in_h", self.rnn_state_in.h)
L3, self.rnn_state_out = tf.nn.dynamic_rnn(
cell=self.enc_cell,
inputs=reshape,
initial_state=self.rnn_state_in,
dtype=tf.float32)
tf.add_to_collection("rnn_state_out_c", self.rnn_state_out.c)
tf.add_to_collection("rnn_state_out_h", self.rnn_state_out.h)
L3 = tf.reshape(L3, [-1, lstm_size])
# Fully connected for Actor
self.logits = linear(L3, n_actions, "actionlogits",
normalized_columns_initializer(0.01))
self.value = tf.reshape(
linear(L3, 1, "value", normalized_columns_initializer(1.0)), [-1])
self.probs = tf.nn.softmax(self.logits)
self.action = tf.squeeze(tf.multinomial(
self.logits - tf.reduce_max(self.logits, [1], keep_dims=True), 1),
[1],
name="action")
self.action = tf.one_hot(self.action, n_actions)[0, :]
log_probs = tf.nn.log_softmax(self.logits)
self.actor_loss = -tf.reduce_sum(
tf.reduce_sum(log_probs * self.actions_taken, [1]) * self.adv)
self.critic_loss = 0.5 * tf.reduce_sum(tf.square(self.value - self.r))
self.entropy = -tf.reduce_sum(self.probs * log_probs)
self.loss = self.actor_loss + 0.5 * self.critic_loss - self.entropy * 0.01
self.vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
tf.get_variable_scope().name)
def sampling_typeless_SNIS_rs(p, parten, nodes_nbrs, nbr_segment,
edge_features, num_sample, n_node_type,
edge_type_array):
unique_nbrs = tf.unique_with_counts(nbr_segment)
num_nbrs = tf.size(unique_nbrs.y)
q = tf.gather(
tf.ones(num_nbrs) / tf.cast(num_nbrs, dtype=tf.float32),
unique_nbrs.idx)
samples = tf.unique(
tf.cast(tf.multinomial(tf.log([q]), num_sample)[0], tf.int32)).y
infos = tf.sparse_to_dense(tf.reshape(tf.contrib.framework.sort(samples),
[-1, 1]),
output_shape=tf.shape(unique_nbrs.idx),
sparse_values=tf.ones_like(samples,
dtype=tf.int32))
partitions = tf.gather(infos, unique_nbrs.idx)
samples_to_gather = tf.dynamic_partition(
tf.range(tf.size(partitions), dtype=tf.int32), partitions, 2)[1],
sampled_p = tf.gather(p, samples_to_gather)
sampled_q = tf.gather(tf.gather(q, unique_nbrs.idx), samples_to_gather)
sampled_parten = tf.gather(parten, samples_to_gather)
sampled_nbrs = tf.gather(nodes_nbrs, samples_to_gather)
nbrset = tf.dynamic_partition(sampled_nbrs, sampled_parten, n_node_type)
segset = tf.dynamic_partition(tf.gather(nbr_segment, samples_to_gather),
sampled_parten, n_node_type)
edge_f_set = []
feature_ids = tf.dynamic_partition(
tf.gather(tf.range(tf.size(nbr_segment)), samples_to_gather),
sampled_parten, n_node_type)
for i in range(n_node_type):
edge_f_set.append(tf.gather(edge_features, feature_ids[i]))
sampled_ps = tf.dynamic_partition(sampled_p, sampled_parten, n_node_type)
sampled_qs = tf.dynamic_partition(sampled_q, sampled_parten, n_node_type)
condition2 = [
tf.reduce_all(tf.math.greater(tf.size(nbrset[i]), 0))
for i in range(n_node_type)
]
all_weight = []
for i in range(n_node_type):
weights = tf.cond(
condition2[i],
false_fn=lambda: [tf.zeros(0)],
true_fn=lambda: calculate_pq_SNIS(segset[i], nbrset[i], edge_f_set[
i], sampled_ps[i], sampled_qs[i]))
all_weight.append(weights)
print(edge_type_array[i])
edge_type_array[i] = tf.cond(
condition2[i],
true_fn=lambda: edge_type_array[i],
false_fn=lambda: -tf.ones(tf.size(edge_type_array[i]), tf.int32))
num_sampled_edges = tf.size(samples_to_gather)
num_sampled_nbrs = tf.size(samples)
return [
all_weight, num_sampled_edges, num_sampled_nbrs, nbrset, segset,
edge_f_set, edge_type_array
]
def __init__(self, env):
self.env = env
num_actions = self.env.action_space.n
# we have three place holders we'll use...
# observations; used either during rollout to sample some actions, or
# during training when combined with actions_taken and advantages.
shape_with_batch = [None] + list(self.env.observation_space.shape)
self.observations = tf.placeholder(shape=shape_with_batch,
dtype=tf.float32)
# the actions we took during rollout
self.actions = tf.placeholder(tf.int32, name='actions')
# the advantages we got from taken 'action_taken' in 'observation'
self.advantages = tf.placeholder(tf.float32, name='advantages')
# our model is a very simple MLP
with tf.variable_scope("model"):
# stack of hidden layers on flattened input; (batch,2,2,7) -> (batch,28)
flat_input_state = slim.flatten(self.observations, scope='flat')
final_hidden = self.hidden_layers_starting_at(
flat_input_state, opts.hidden_layers)
logits = slim.fully_connected(inputs=final_hidden,
num_outputs=num_actions,
activation_fn=None)
# in the eval case just pick arg max
self.action_argmax = tf.argmax(logits, 1)
# for rollouts we need an op that samples actions from this
# model to give a stochastic action.
sample_action = tf.multinomial(logits, num_samples=1)
self.sampled_action_op = tf.reshape(sample_action, shape=[])
# we are trying to maximise the product of two components...
# 1) the log_p of "good" actions.
# 2) the advantage term based on the rewards from actions.
# first we need the log_p values for each observation for the actions we specifically
# took by sampling... we first run a log_softmax over the action logits to get
# probabilities.
log_softmax = tf.nn.log_softmax(logits)
self.debug_softmax = tf.exp(log_softmax)
# we then use a mask to only select the elements of the softmaxs that correspond
# to the actions we actually took. we could also do this by complex indexing and a
# gather but i always think this is more natural. the "cost" of dealing with the
# mostly zero one hot, as opposed to doing a gather on sparse indexes, isn't a big
# deal when the number of observations is >> number of actions.
action_mask = tf.one_hot(indices=self.actions, depth=num_actions)
action_log_prob = tf.reduce_sum(log_softmax * action_mask,
reduction_indices=1)
# the (element wise) product of these action log_p's with the total reward of the
# episode represents the quantity we want to maximise. we standardise the advantage
# values so roughly 1/2 +ve / -ve as a variance control.
action_mul_advantages = tf.mul(action_log_prob,
util.standardise(self.advantages))
self.loss = -tf.reduce_sum(
action_mul_advantages) # recall: we are maximising.
with tf.variable_scope("optimiser"):
# dynamically create optimiser based on opts
optimiser = util.construct_optimiser(opts)
# calc gradients
gradients = optimiser.compute_gradients(self.loss)
# potentially clip and wrap with debugging tf.Print
gradients = util.clip_and_debug_gradients(gradients, opts)
# apply
self.train_op = optimiser.apply_gradients(gradients)
def __init__(self, lr, brain, h_size, epsilon, beta, max_step, normalize,
num_layers):
"""
Creates Discrete Control Actor-Critic model.
:param brain: State-space size
:param h_size: Hidden layer size
"""
super(DiscreteControlModel, self).__init__()
self._create_global_steps()
self._create_reward_encoder()
self.normalize = normalize
hidden_state, hidden_visual, hidden = None, None, None
if brain.number_observations > 0:
height_size, width_size = brain.camera_resolutions[0][
'height'], brain.camera_resolutions[0]['width']
bw = brain.camera_resolutions[0]['blackAndWhite']
hidden_visual = self._create_visual_encoder(
height_size, width_size, bw, h_size, 1, tf.nn.elu,
num_layers)[0]
if brain.state_space_size > 0:
s_size = brain.state_space_size
if brain.state_space_type == "continuous":
hidden_state = self._create_continuous_state_encoder(
s_size, h_size, 1, tf.nn.elu, num_layers)[0]
else:
hidden_state = self._create_discrete_state_encoder(
s_size, h_size, 1, tf.nn.elu, num_layers)[0]
if hidden_visual is None and hidden_state is None:
raise Exception(
"No valid network configuration possible. "
"There are no states or observations in this brain")
elif hidden_visual is not None and hidden_state is None:
hidden = hidden_visual
elif hidden_visual is None and hidden_state is not None:
hidden = hidden_state
elif hidden_visual is not None and hidden_state is not None:
hidden = tf.concat([hidden_visual, hidden_state], axis=1)
a_size = brain.action_space_size
self.batch_size = tf.placeholder(shape=None,
dtype=tf.int32,
name='batch_size')
self.policy = tf.layers.dense(
hidden,
a_size,
activation=None,
use_bias=False,
kernel_initializer=c_layers.variance_scaling_initializer(
factor=0.01))
self.probs = tf.nn.softmax(self.policy, name="action_probs")
self.output = tf.multinomial(self.policy, 1)
self.output_max = tf.argmax(self.probs, name='action_max', axis=1)
self.output = tf.identity(self.output, name="action")
self.value = tf.layers.dense(
hidden,
1,
activation=None,
use_bias=False,
kernel_initializer=c_layers.variance_scaling_initializer(
factor=1.0))
self.value = tf.identity(self.value, name="value_estimate")
self.entropy = -tf.reduce_sum(self.probs * tf.log(self.probs + 1e-10),
axis=1)
self.action_holder = tf.placeholder(shape=[None], dtype=tf.int32)
self.selected_actions = c_layers.one_hot_encoding(
self.action_holder, a_size)
self.old_probs = tf.placeholder(shape=[None, a_size],
dtype=tf.float32,
name='old_probabilities')
self.responsible_probs = tf.reduce_sum(self.probs *
self.selected_actions,
axis=1)
self.old_responsible_probs = tf.reduce_sum(self.old_probs *
self.selected_actions,
axis=1)
self._create_ppo_optimizer(self.responsible_probs,
self.old_responsible_probs, self.value,
self.entropy, beta, epsilon, lr, max_step)
def body(*args) -> LoopState:
loop_state = LoopState(*args)
histories = loop_state.histories
feedables = loop_state.feedables
step = feedables.step
decoded_symbols_ta = histories.decoded_symbols.write(
step, feedables.input_symbol)
# shape (time, batch)
decoded_symbols = decoded_symbols_ta.stack()
decoded_symbols.set_shape([None, None])
decoded_symbols_in_batch = tf.transpose(decoded_symbols)
# mask (time, batch)
mask = histories.input_mask.stack()
mask.set_shape([None, None])
with tf.variable_scope(self._variable_scope, reuse=tf.AUTO_REUSE):
# shape (batch, time, dimension)
embedded_inputs = self.embed_inputs(decoded_symbols_in_batch)
last_layer = self.layer(self.depth, embedded_inputs,
tf.transpose(mask))
# (batch, state_size)
output_state = last_layer.temporal_states[:, -1, :]
# See train_logits definition
logits = tf.matmul(output_state, self.decoding_w)
logits += self.decoding_b
if sample:
next_symbols = tf.multinomial(logits, num_samples=1)
else:
next_symbols = tf.to_int32(tf.argmax(logits, axis=1))
int_unfinished_mask = tf.to_int32(
tf.logical_not(loop_state.feedables.finished))
# Note this works only when PAD_TOKEN_INDEX is 0. Otherwise
# this have to be rewritten
assert PAD_TOKEN_INDEX == 0
next_symbols = next_symbols * int_unfinished_mask
has_just_finished = tf.equal(next_symbols, END_TOKEN_INDEX)
has_finished = tf.logical_or(feedables.finished,
has_just_finished)
not_finished = tf.logical_not(has_finished)
new_feedables = DecoderFeedables(step=step + 1,
finished=has_finished,
input_symbol=next_symbols,
prev_logits=logits)
# TransformerHistories is a type and should be callable
# pylint: disable=not-callable
new_histories = TransformerHistories(
logits=histories.logits.write(step, logits),
decoder_outputs=histories.decoder_outputs.write(
step, output_state),
mask=histories.mask.write(step, not_finished),
outputs=histories.outputs.write(step, next_symbols),
# transformer-specific:
# TODO handle attention histories correctly
decoded_symbols=decoded_symbols_ta,
self_attention_histories=histories.self_attention_histories,
inter_attention_histories=histories.inter_attention_histories,
input_mask=histories.input_mask.write(
step + 1, tf.to_float(not_finished)))
# pylint: enable=not-callable
new_loop_state = LoopState(histories=new_histories,
constants=[],
feedables=new_feedables)
return new_loop_state
def generator(self, name="generator", reuse=False):
'''
Caption sampler, sample words follow the probability distribution.
'''
random_uniform_init = tf.random_uniform_initializer(minval=-0.1, maxval=0.1)
with tf.variable_scope(name):
if reuse:
tf.get_variable_scope().reuse_variables()
with tf.device("/cpu:0"), tf.variable_scope("word"):
# name: "gnerator/word"
word_emb_W = tf.get_variable("word_emb_W", [self.vocab_size, self.G_hidden_size], tf.float32, random_uniform_init)
with tf.variable_scope("image_feat"):
# name: "generator/image_feat"
image_feat_W = tf.get_variable("image_feat_W", [self.image_feat_dim, self.G_hidden_size], tf.float32, random_uniform_init)
image_feat_b = tf.get_variable("image_feat_b", [self.G_hidden_size], tf.float32, random_uniform_init)
with tf.variable_scope("output"):
# name: "generator/output"
output_W = tf.get_variable("output_W", [self.G_hidden_size, self.vocab_size], tf.float32, random_uniform_init)
output_b = tf.get_variable("output_b", [self.vocab_size], tf.float32, random_uniform_init)
with tf.variable_scope("lstm_encoder"):
if self.rnn_cell == 'lstm':
encoder = tf.nn.rnn_cell.LSTMCell(self.G_hidden_size, state_is_tuple=True)
elif self.rnn_cell == 'gru':
encoder = tf.nn.rnn_cell.GRUCell(self.G_hidden_size)
with tf.variable_scope("lstm_decoder"):
# WONT BE CREATED HERE
if self.rnn_cell == 'lstm':
decoder = tf.nn.rnn_cell.LSTMCell(self.G_hidden_size, state_is_tuple=True)
elif self.rnn_cell == 'gru':
decoder = tf.nn.rnn_cell.GRUCell(self.G_hidden_size)
#============================= encoder ===================================================================
state = encoder.zero_state(self.batch_size, tf.float32)
with tf.variable_scope("image_feat") as scope:
image_feat = self.batch_norm(self.image_feat[:,:], mode='train', name='')
image_feat_emb = tf.matmul(image_feat, image_feat_W) + image_feat_b # B,H
lstm_input = image_feat_emb
with tf.variable_scope("lstm_encoder") as scope:
_, state = encoder(lstm_input, state)
encoder_state = state
#============================= decoder ===================================================================
start_token = tf.constant(self.START, tf.int32, [self.batch_size])
mask = tf.constant(True, "bool", [self.batch_size])
sample_words = []
state = encoder_state
for j in range(self.lstm_steps):
with tf.device("/cpu:0"):
if j == 0:
decoder_input = tf.nn.embedding_lookup(word_emb_W, start_token)
else:
decoder_input = tf.nn.embedding_lookup(word_emb_W, sample_word)
with tf.variable_scope("lstm"):
if not j == 0:
tf.get_variable_scope().reuse_variables()
output, state = decoder(decoder_input, state)
logits = tf.matmul(output, output_W) + output_b
log_probs = tf.log(tf.clip_by_value(tf.nn.softmax(logits), 1e-20, 1.0)) # B,Vocab_size # add 1e-8 to prevent log(0)
# sample once from the multinomial distribution
# Montecarlo sampling
sample_word = tf.reshape(tf.multinomial(log_probs, 1), [self.batch_size]) # 1 means sample once
sample_words.append(sample_word)
return sample_words
def categorical_sample(logits, d):
value = tf.squeeze(
tf.multinomial(logits - tf.reduce_max(logits, [1], keep_dims=True), 1),
[1])
return tf.one_hot(value, d)
def __init__(self, n_inputs, n_outputs, **params):
super(TDM_agent, self).__init__(**params)
self.n_inputs = n_inputs
self.n_outputs = n_outputs
self.lr = params['agent_params'].pop('lr', 1e-3)
self.discount = params['agent_params'].pop('discount', 1e-3)
self.tau = params['agent_params'].pop('tau', 1e-3)
self.max_td = params['agent_params'].pop('max_td', 0)
self.soft_learning = params['agent_params'].pop('soft_learning', False)
self.reward_scale = params['agent_params'].pop('reward_scale', 1.0)
self.double = params['agent_params'].pop('double', False)
self.reward_scale = params['agent_params'].pop('reward_scale', 1.0)
self.huber_loss = params['agent_params'].pop('huber_loss', True)
self.clip_gradients = params['agent_params'].pop(
'clip_gradients', False)
self.train_steps_per_t = params['agent_params'].pop(
'train_steps_per_t', 1)
self.q_train_steps_per_t = params['agent_params'].pop(
'q_train_steps_per_t', 1)
self.extra_q_train_steps_per_t = self.q_train_steps_per_t - self.train_steps_per_t
assert self.extra_q_train_steps_per_t >= 0
self.multi_step = params['agent_params'].pop('multi_step', False)
if self.multi_step:
self.discount = self.discount**self.multi_step
assert not (self.soft_learning and self.double)
self._init_placeholders()
### QNET
self.qnet = Qnet(self.obs,
self.n_outputs,
params['network_spec'],
scope='qnet')
self.model_Q_params = self.qnet.get_params_internal()
self.model_Q_outputs = self.qnet.outputs
### FNET
self.fnet = Qnet([self.obs, self.actions, self.scaled_tds],
self.n_inputs,
params['network_spec'],
scope='fnet')
self.model_F_params = self.fnet.get_params_internal()
self.model_F_outputs = self.fnet.outputs
### RNET
self.rnet = Qnet([self.obs, self.actions, self.scaled_tds],
1,
params['network_spec'],
scope='rnet')
self.model_R_params = self.rnet.get_params_internal()
self.model_R_outputs = self.rnet.outputs
### ENET
if self.soft_learning:
self.model_Q_predict_action_from_next_obs = tf.stop_gradient(
tf.one_hot(
tf.multinomial(
self.qnet.make_network(inputs=self.next_obs), 1)[:, 0],
self.qnet.output_size))
else:
self.model_Q_predict_action_from_next_obs = tf.stop_gradient(
tf.one_hot(
tf.argmax(self.qnet.make_network(inputs=self.next_obs),
axis=1), self.qnet.output_size))
# Duplicate the Fnet with different variables for the target network
self.tfnet = Qnet([
self.next_obs, self.model_Q_predict_action_from_next_obs,
self.scaled_next_tds
],
self.n_inputs,
params['network_spec'],
scope='tfnet')
self.target_F_outputs = self.tfnet.outputs
self.target_F_params = self.tfnet.get_params_internal()
self.target_F_from_obs = self.tfnet.make_network(
inputs=[self.obs, self.actions, self.scaled_tds])
# Duplicate the Rnet with different variables for the target network
self.trnet = Qnet([
self.next_obs, self.model_Q_predict_action_from_next_obs,
self.scaled_next_tds
],
1,
params['network_spec'],
scope='trnet')
self.target_R_outputs = self.trnet.outputs
self.target_R_params = self.trnet.get_params_internal()
self.target_R_from_obs = self.trnet.make_network(
inputs=[self.obs, self.actions, self.scaled_tds])
# Duplicate the Qnet with different variables for the target network
self.tqnet = Qnet(tf.add(self.next_obs,
self.td_is_not_zero * self.target_F_outputs),
self.n_outputs,
params['network_spec'],
scope='tqnet')
self.target_Q_outputs = self.tqnet.outputs
self.target_Q_params = self.tqnet.get_params_internal()
if self.soft_learning:
# For soft learning
# V = sum(p(s,a) * (q(s,a) - log(p(s,a)))
# = sum(exp(q)/z * (q - log(exp(q)/z)))
# = sum(p* (log(z)))
# = log(z)
self.partition_function = tf.reduce_mean(
self.target_Q_outputs, axis=1) + tf.log(
tf.reduce_sum(
tf.exp(self.target_Q_outputs - tf.reduce_mean(
self.target_Q_outputs, axis=1, keepdims=True)),
axis=1))
self.target_V = self.partition_function
params['policy_params']['action_choice'] = params[
'policy_params'].get('action_choice', 'Boltzmann')
assert params['policy_params'][
'action_choice'] == 'Boltzmann' # Softmax on outputs
self.policy = Policy_Discrete_for_Qnet(self.qnet,
**params['policy_params'])
self.rb = Replay_Buffer(self.n_inputs,
self.n_outputs,
discrete_action=True,
multi_step=self.multi_step,
**params['replay_buffer_params'])
self.optimizer = tf.train.AdamOptimizer(learning_rate=self.lr)
self.train_ops = []
self._init_training_ops()
self.target_Q_update = uf.update_target_network(
self.model_Q_params,
self.target_Q_params,
tau=self.tau,
update_op_control_dependencies=self.q_train_op)
self.target_R_update = uf.update_target_network(
self.model_R_params,
self.target_R_params,
tau=self.tau,
update_op_control_dependencies=self.train_ops)
self.target_F_update = uf.update_target_network(
self.model_F_params,
self.target_F_params,
tau=self.tau,
update_op_control_dependencies=self.train_ops)
self.train_ops.append([self.target_R_update, self.target_F_update])
self.q_train_ops = tf.group(self.q_train_op, self.target_Q_update)
self.train_ops = tf.group(self.train_ops, self.q_train_ops)
self.loss_ops = [self.R_Loss, self.F_Loss, self.Q_Loss]
self._finish_agent_setup()
hidden2 = tf.layers.dense(hidden1,
n_hidden,
activation=tf.nn.relu,
kernel_initializer=intializer)
output_layer = tf.layers.dense(hidden2,
out,
activation=tf.nn.sigmoid,
kernel_initializer=intializer)
# Left, Right Probb
# Concatenate tensors along one direction
probb = tf.concat(values=[output_layer, 1 - output_layer], axis=1)
# Final Action "0" or "1"
action = tf.multinomial(probb, num_samples=1)
init = tf.global_variables_initializer()
# For 50 episodes of game, take 500 time steps and declare the game as Done.
n_steps = 500
episodes = 50
avg_steps = []
env = gym.make('CartPole-v1')
with tf.Session() as sess:
sess.run(init)
for i in range(episodes):
# Reset Environment
obs = env.reset()
def build_policy_network_op(self, scope="policy_network"):
"""
Build the policy network, construct the tensorflow operation to sample
actions from the policy network outputs, and compute the log probabilities
of the taken actions (for computing the loss later). These operations are
stored in self.sampled_action and self.logprob. Must handle both settings
of self.discrete.
TODO:
Discrete case:
logits: the logits for each action
HINT: use build_mlp
self.sampled_action: sample from these logits
HINT: use tf.multinomial + tf.squeeze
self.logprob: compute the log probabilities of the taken actions
HINT: 1. tf.nn.sparse_softmax_cross_entropy_with_logits computes
the *negative* log probabilities of labels, given logits.
2. taken actions are different than sampled actions!
Continuous case:
To build a policy in a continuous action space domain, we will have the
model output the means of each action dimension, and then sample from
a multivariate normal distribution with these means and trainable standard
deviation.
That is, the action a_t ~ N( mu(o_t), sigma)
where mu(o_t) is the network that outputs the means for each action
dimension, and sigma is a trainable variable for the standard deviations.
N here is a multivariate gaussian distribution with the given parameters.
action_means: the predicted means for each action dimension.
HINT: use build_mlp
log_std: a trainable variable for the log standard deviations.
--> think about why we use log std as the trainable variable instead of std
self.sampled_actions: sample from the gaussian distribution as described above
HINT: use tf.random_normal
self.lobprob: the log probabilities of the taken actions
HINT: use tf.contrib.distributions.MultivariateNormalDiag
"""
#######################################################
######### YOUR CODE HERE - 5-10 lines. ############
if self.discrete:
action_logits = build_mlp(
self.observation_placeholder,
self.action_dim,
"discrete",
) # TODO
self.sampled_action = tf.reshape(tf.multinomial(action_logits, 1),
[-1]) # TODO
self.logprob = -tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=self.action_placeholder, logits=action_logits) # TODO
else:
action_means = build_mlp(
self.observation_placeholder,
self.action_dim,
"continuous",
) # TODO
log_std = tf.get_variable("std", [self.action_dim],
dtype=tf.float32) # TODO
self.sampled_action = tf.random_normal(
shape=tf.shape(action_means),
mean=action_means,
stddev=tf.exp(log_std)) # TODO
self.logprob = tf.contrib.distributions.MultivariateNormalDiag(
loc=action_means, scale_diag=tf.exp(log_std)).log_prob(
self.action_placeholder) # TODO
def ssd_random_sample_patch(image,
labels,
bboxes,
ratio_list=[0.1, 0.3, 0.5, 0.7, 0.9, 1.],
name=None):
'''ssd_random_sample_patch.
select one min_iou
sample _width and _height from [0-width] and [0-height]
check if the aspect ratio between 0.5-2.
select left_top point from (width - _width, height - _height)
check if this bbox has a min_iou with all ground_truth bboxes
keep ground_truth those center is in this sampled patch, if none then try again
'''
def sample_width_height(width, height):
with tf.name_scope('sample_width_height'):
index = 0
max_attempt = 10
sampled_width, sampled_height = width, height
def condition(index, sampled_width, sampled_height, width, height):
return tf.logical_or(
tf.logical_and(
tf.logical_or(
tf.greater(sampled_width, sampled_height * 2),
tf.greater(sampled_height, sampled_width * 2)),
tf.less(index, max_attempt)), tf.less(index, 1))
def body(index, sampled_width, sampled_height, width, height):
sampled_width = tf.random_uniform(
[1], minval=0.3, maxval=0.999, dtype=tf.float32)[0] * width
sampled_height = tf.random_uniform(
[1], minval=0.3, maxval=0.999,
dtype=tf.float32)[0] * height
return index + 1, sampled_width, sampled_height, width, height
[index, sampled_width, sampled_height, _, _] = tf.while_loop(
condition,
body, [index, sampled_width, sampled_height, width, height],
parallel_iterations=4,
back_prop=False,
swap_memory=True)
return tf.cast(sampled_width,
tf.int32), tf.cast(sampled_height, tf.int32)
def jaccard_with_anchors(roi, bboxes):
with tf.name_scope('jaccard_with_anchors'):
int_ymin = tf.maximum(roi[0], bboxes[:, 0])
int_xmin = tf.maximum(roi[1], bboxes[:, 1])
int_ymax = tf.minimum(roi[2], bboxes[:, 2])
int_xmax = tf.minimum(roi[3], bboxes[:, 3])
h = tf.maximum(int_ymax - int_ymin + 1., 0.)
w = tf.maximum(int_xmax - int_xmin + 1., 0.)
inter_vol = h * w
union_vol = (roi[3] - roi[1] + 1.) * (roi[2] - roi[0] + 1.) + (
(bboxes[:, 2] - bboxes[:, 0] + 1.) *
(bboxes[:, 3] - bboxes[:, 1] + 1.) - inter_vol)
jaccard = tf.div(inter_vol, union_vol)
return jaccard
def areas(bboxes):
with tf.name_scope('bboxes_areas'):
vol = (bboxes[:, 3] - bboxes[:, 1] + 1.) * (bboxes[:, 2] -
bboxes[:, 0] + 1.)
return vol
def check_roi_center(width, height, labels, bboxes):
with tf.name_scope('check_roi_center'):
index = 0
max_attempt = 20
float_width = tf.to_float(width)
float_height = tf.to_float(height)
roi = [0., 0., float_height - 1., float_width - 1.]
mask = tf.cast(tf.zeros_like(labels, dtype=tf.uint8), tf.bool)
center_x, center_y = (bboxes[:, 1] + bboxes[:, 3]) / 2, (
bboxes[:, 0] + bboxes[:, 2]) / 2
def condition(index, roi, mask):
return tf.logical_or(
tf.logical_and(
tf.reduce_sum(tf.to_int32(mask)) < 1,
tf.less(index, max_attempt)), tf.less(index, 1))
def body(index, roi, mask):
sampled_width, sampled_height = sample_width_height(
float_width, float_height)
x = tf.random_uniform([],
minval=0,
maxval=width - sampled_width,
dtype=tf.int32)
y = tf.random_uniform([],
minval=0,
maxval=height - sampled_height,
dtype=tf.int32)
roi = [
tf.to_float(y),
tf.to_float(x),
tf.to_float(y + sampled_height),
tf.to_float(x + sampled_width)
]
mask_min = tf.logical_and(tf.greater(center_y, roi[0]),
tf.greater(center_x, roi[1]))
mask_max = tf.logical_and(tf.less(center_y, roi[2]),
tf.less(center_x, roi[3]))
mask = tf.logical_and(mask_min, mask_max)
return index + 1, roi, mask
[index, roi, mask] = tf.while_loop(condition,
body, [index, roi, mask],
parallel_iterations=10,
back_prop=False,
swap_memory=True)
mask_labels = tf.boolean_mask(labels, mask)
mask_bboxes = tf.boolean_mask(bboxes, mask)
return roi, mask_labels, mask_bboxes
def check_roi_overlap(width, height, labels, bboxes, min_iou):
with tf.name_scope('check_roi_overlap'):
index = 0
max_attempt = 50
float_width = tf.to_float(width)
float_height = tf.to_float(height)
roi = [0., 0., float_height - 1., float_width - 1.]
mask_labels = labels
mask_bboxes = bboxes
def condition(index, roi, mask_labels, mask_bboxes):
return tf.logical_or(
tf.logical_or(
tf.logical_and(
tf.reduce_sum(
tf.to_int32(
jaccard_with_anchors(
roi, mask_bboxes) < min_iou)) > 0,
tf.less(index, max_attempt)), tf.less(index, 1)),
tf.less(tf.shape(mask_labels)[0], 1))
def body(index, roi, mask_labels, mask_bboxes):
roi, mask_labels, mask_bboxes = check_roi_center(
width, height, labels, bboxes)
return index + 1, roi, mask_labels, mask_bboxes
[index, roi, mask_labels, mask_bboxes
] = tf.while_loop(condition,
body, [index, roi, mask_labels, mask_bboxes],
parallel_iterations=16,
back_prop=False,
swap_memory=True)
return tf.cond(
tf.greater(tf.shape(mask_labels)[0], 0), lambda: (tf.to_int32([
roi[0], roi[1], roi[2] - roi[0] + 1., roi[3] - roi[1] + 1.
]), mask_labels, mask_bboxes), lambda: (tf.to_int32(
[0., 0., float_height, float_width]), labels, bboxes))
def sample_patch(image, labels, bboxes, min_iou):
with tf.name_scope('sample_patch'):
height, width, depth = _ImageDimensions(image, rank=3)
roi_slice_range, mask_labels, mask_bboxes = check_roi_overlap(
width, height, labels, bboxes, min_iou)
# Add offset.
offset = tf.cast(
tf.stack([
roi_slice_range[0], roi_slice_range[1], roi_slice_range[0],
roi_slice_range[1]
]), mask_bboxes.dtype)
mask_bboxes = mask_bboxes - offset
cliped_ymin = tf.maximum(0., mask_bboxes[:, 0])
cliped_xmin = tf.maximum(0., mask_bboxes[:, 1])
cliped_ymax = tf.minimum(
tf.to_float(roi_slice_range[2]) - 1., mask_bboxes[:, 2])
cliped_xmax = tf.minimum(
tf.to_float(roi_slice_range[3]) - 1., mask_bboxes[:, 3])
mask_bboxes = tf.stack(
[cliped_ymin, cliped_xmin, cliped_ymax, cliped_xmax], axis=-1)
return tf.cond(
tf.logical_or(tf.less(roi_slice_range[2], 1),
tf.less(roi_slice_range[3], 1)), lambda:
(image, labels, bboxes), lambda:
(tf.slice(image, [roi_slice_range[0], roi_slice_range[1], 0], [
roi_slice_range[2], roi_slice_range[3], -1
]), mask_labels, mask_bboxes))
with tf.name_scope('ssd_random_sample_patch'):
image = tf.convert_to_tensor(image, name='image')
min_iou_list = tf.convert_to_tensor(ratio_list)
samples_min_iou = tf.multinomial(
tf.log([[1. / len(ratio_list)] * len(ratio_list)]), 1)
sampled_min_iou = min_iou_list[tf.cast(samples_min_iou[0][0],
tf.int32)]
return tf.cond(
tf.less(sampled_min_iou, 1.),
lambda: sample_patch(image, labels, bboxes, sampled_min_iou),
lambda: (image, labels, bboxes))
def train_PG(exp_name='',
env_name='CartPole-v0',
n_iter=100,
gamma=1.0,
min_timesteps_per_batch=1000,
max_path_length=None,
learning_rate=5e-3,
reward_to_go=True,
animate=True,
logdir=None,
normalize_advantages=True,
nn_baseline=False,
seed=0,
# network arguments
n_layers=1,
size=32
):
start = time.time()
# Configure output directory for logging
logz.configure_output_dir(logdir)
# Log experimental parameters
args = inspect.getargspec(train_PG)[0]
locals_ = locals()
params = {k: locals_[k] if k in locals_ else None for k in args}
logz.save_params(params)
# Set random seeds
tf.set_random_seed(seed)
np.random.seed(seed)
# Make the gym environment
env = gym.make(env_name)
# Is this env continuous, or discrete?
discrete = isinstance(env.action_space, gym.spaces.Discrete)
# Maximum length for episodes
max_path_length = max_path_length or env.spec.max_episode_steps
#========================================================================================#
# Notes on notation:
#
# Symbolic variables have the prefix sy_, to distinguish them from the numerical values
# that are computed later in the function
#
# Prefixes and suffixes:
# ob - observation
# ac - action
# _no - this tensor should have shape (batch size /n/, observation dim)
# _na - this tensor should have shape (batch size /n/, action dim)
# _n - this tensor should have shape (batch size /n/)
#
# Note: batch size /n/ is defined at runtime, and until then, the shape for that axis
# is None
#========================================================================================#
# Observation and action sizes
ob_dim = env.observation_space.shape[0]
ac_dim = env.action_space.n if discrete else env.action_space.shape[0]
#========================================================================================#
# ----------SECTION 4----------
# Placeholders
#
# Need these for batch observations / actions / advantages in policy gradient loss function.
#========================================================================================#
sy_ob_no = tf.placeholder(shape=[None, ob_dim], name="ob", dtype=tf.float32)
if discrete:
sy_ac_na = tf.placeholder(shape=[None], name="ac", dtype=tf.int32)
else:
sy_ac_na = tf.placeholder(shape=[None, ac_dim], name="ac", dtype=tf.float32)
# Define a placeholder for advantages
sy_adv_n = tf.placeholder(shape=[None], name='adv', dtype=tf.float32)
#========================================================================================#
# ----------SECTION 4----------
# Networks
#
# Make symbolic operations for
# 1. Policy network outputs which describe the policy distribution.
# a. For the discrete case, just logits for each action.
#
# b. For the continuous case, the mean / log std of a Gaussian distribution over
# actions.
#
# Hint: use the 'build_mlp' function you defined in utilities.
#
# Note: these ops should be functions of the placeholder 'sy_ob_no'
#
# 2. Producing samples stochastically from the policy distribution.
# a. For the discrete case, an op that takes in logits and produces actions.
#
# Should have shape [None]
#
# b. For the continuous case, use the reparameterization trick:
# The output from a Gaussian distribution with mean 'mu' and std 'sigma' is
#
# mu + sigma * z, z ~ N(0, I)
#
# This reduces the problem to just sampling z. (Hint: use tf.random_normal!)
#
# Should have shape [None, ac_dim]
#
# Note: these ops should be functions of the policy network output ops.
#
# 3. Computing the log probability of a set of actions that were actually taken,
# according to the policy.
#
# Note: these ops should be functions of the placeholder 'sy_ac_na', and the
# policy network output ops.
#
#========================================================================================#
if discrete:
# YOUR_CODE_HERE
sy_logits_na = build_mlp(sy_ob_no, ac_dim, 'policy', n_layers=n_layers, size=size)
sy_sampled_ac = tf.multinomial(sy_logits_na, 1)
sy_logprob_n = tf.nn.softmax_cross_entropy_with_logits_v2(labels=sy_ac_na, logits=sy_sampled_ac)
else:
# YOUR_CODE_HERE
sy_mean = TODO
sy_logstd = TODO # logstd should just be a trainable variable, not a network output.
sy_sampled_ac = TODO
sy_logprob_n = TODO # Hint: Use the log probability under a multivariate gaussian.
#========================================================================================#
# ----------SECTION 4----------
# Loss Function and Training Operation
#========================================================================================#
loss = tf.reduce_mean(sy_logprob_n) # Loss function that we'll differentiate to get the policy gradient.
update_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)
#========================================================================================#
# ----------SECTION 5----------
# Optional Baseline
#========================================================================================#
if nn_baseline:
baseline_prediction = tf.squeeze(build_mlp(
sy_ob_no,
1,
"nn_baseline",
n_layers=n_layers,
size=size))
# Define placeholders for targets, a loss function and an update op for fitting a
# neural network baseline. These will be used to fit the neural network baseline.
# YOUR_CODE_HERE
# This is some actor-critic stuff, not prepared for this
sy_b_n = tf.placeholder(shape=[None], name='b', dtype=tf.float32)
b_loss = tf.reduce_mean(tf.losses.mean_squared_error(labels=sy_b_n, predictions=baseline_prediction))
_update_op = tf.train.AdamOptimizer(learning_rate).minimize(b_loss)
#========================================================================================#
# Tensorflow Engineering: Config, Session, Variable initialization
#========================================================================================#
tf_config = tf.ConfigProto(inter_op_parallelism_threads=1, intra_op_parallelism_threads=1)
sess = tf.Session(config=tf_config)
sess.__enter__() # equivalent to `with sess:`
tf.global_variables_initializer().run() #pylint: disable=E1101
#========================================================================================#
# Training Loop
#========================================================================================#
total_timesteps = 0
for itr in range(n_iter):
print("********** Iteration %i ************"%itr)
# Collect paths until we have enough timesteps
timesteps_this_batch = 0
paths = []
while True:
ob = env.reset()
obs, acs, rewards = [], [], []
animate_this_episode=(len(paths)==0 and (itr % 10 == 0) and animate)
steps = 0
while True:
if animate_this_episode:
env.render()
time.sleep(0.05)
obs.append(ob)
ac = sess.run(sy_sampled_ac, feed_dict={sy_ob_no : ob[None]})
ac = ac[0]
acs.append(ac)
ob, rew, done, _ = env.step(ac)
rewards.append(rew)
steps += 1
if done or steps > max_path_length:
break
path = {"observation" : np.array(obs),
"reward" : np.array(rewards),
"action" : np.array(acs)}
paths.append(path)
timesteps_this_batch += pathlength(path)
if timesteps_this_batch > min_timesteps_per_batch:
break
total_timesteps += timesteps_this_batch
# Build arrays for observation, action for the policy gradient update by concatenating
# across paths
ob_no = np.concatenate([path["observation"] for path in paths])
ac_na = np.concatenate([path["action"] for path in paths])
#====================================================================================#
# ----------SECTION 4----------
# Computing Q-values
#
# Your code should construct numpy arrays for Q-values which will be used to compute
# advantages (which will in turn be fed to the placeholder you defined above).
#
# Recall that the expression for the policy gradient PG is
#
# PG = E_{tau} [sum_{t=0}^T grad log pi(a_t|s_t) * (Q_t - b_t )]
#
# where
#
# tau=(s_0, a_0, ...) is a trajectory,
# Q_t is the Q-value at time t, Q^{pi}(s_t, a_t),
# and b_t is a baseline which may depend on s_t.
#
# You will write code for two cases, controlled by the flag 'reward_to_go':
#
# Case 1: trajectory-based PG
#
# (reward_to_go = False)
#
# Instead of Q^{pi}(s_t, a_t), we use the total discounted reward summed over
# entire trajectory (regardless of which time step the Q-value should be for).
#
# For this case, the policy gradient estimator is
#
# E_{tau} [sum_{t=0}^T grad log pi(a_t|s_t) * Ret(tau)]
#
# where
#
# Ret(tau) = sum_{t'=0}^T gamma^t' r_{t'}.
#
# Thus, you should compute
#
# Q_t = Ret(tau)
#
# Case 2: reward-to-go PG
#
# (reward_to_go = True)
#
# Here, you estimate Q^{pi}(s_t, a_t) by the discounted sum of rewards starting
# from time step t. Thus, you should compute
#
# Q_t = sum_{t'=t}^T gamma^(t'-t) * r_{t'}
#
#
# Store the Q-values for all timesteps and all trajectories in a variable 'q_n',
# like the 'ob_no' and 'ac_na' above.
#
#====================================================================================#
# YOUR_CODE_HERE
if reward_to_go == False:
q_n = np.concatenate([ np.sum(path['reward'])*np.ones_like(path['actions']) for path in paths ])
else:
# the same as np.flip(np.cumsum(np.flip(b, axis=0)), axis=0)
# the [::-1] means use a stride of -1(backwards), one can use it instead of flip
q_n = np.concatenate([ np.cumsum(path['reward'][::-1])[::-1] for path in paths ])
#====================================================================================#
# ----------SECTION 5----------
# Computing Baselines
#====================================================================================#
if nn_baseline:
# If nn_baseline is True, use your neural network to predict reward-to-go
# at each timestep for each trajectory, and save the result in a variable 'b_n'
# like 'ob_no', 'ac_na', and 'q_n'.
#
# Hint #bl1: rescale the output from the nn_baseline to match the statistics
# (mean and std) of the current or previous batch of Q-values. (Goes with Hint
# #bl2 below.)
b_n = sess.run(baseline_prediction, feed_dict={sy_ob_no: ob_no})
adv_n = q_n - b_n
else:
adv_n = q_n.copy()
#====================================================================================#
# ----------SECTION 4----------
# Advantage Normalization
#====================================================================================#
if normalize_advantages:
# On the next line, implement a trick which is known empirically to reduce variance
# in policy gradient methods: normalize adv_n to have mean zero and std=1.
# YOUR_CODE_HERE
pass
#====================================================================================#
# ----------SECTION 5----------
# Optimizing Neural Network Baseline
#====================================================================================#
if nn_baseline:
# ----------SECTION 5----------
# If a neural network baseline is used, set up the targets and the inputs for the
# baseline.
#
# Fit it to the current batch in order to use for the next iteration. Use the
# baseline_update_op you defined earlier.
#
# Hint #bl2: Instead of trying to target raw Q-values directly, rescale the
# targets to have mean zero and std=1. (Goes with Hint #bl1 above.)
# YOUR_CODE_HERE
pass
#====================================================================================#
# ----------SECTION 4----------
# Performing the Policy Update
#====================================================================================#
# Call the update operation necessary to perform the policy gradient update based on
# the current batch of rollouts.
#
# For debug purposes, you may wish to save the value of the loss function before
# and after an update, and then log them below.
# YOUR_CODE_HERE
# Log diagnostics
returns = [path["reward"].sum() for path in paths]
ep_lengths = [pathlength(path) for path in paths]
logz.log_tabular("Time", time.time() - start)
logz.log_tabular("Iteration", itr)
logz.log_tabular("AverageReturn", np.mean(returns))
logz.log_tabular("StdReturn", np.std(returns))
logz.log_tabular("MaxReturn", np.max(returns))
logz.log_tabular("MinReturn", np.min(returns))
logz.log_tabular("EpLenMean", np.mean(ep_lengths))
logz.log_tabular("EpLenStd", np.std(ep_lengths))
logz.log_tabular("TimestepsThisBatch", timesteps_this_batch)
logz.log_tabular("TimestepsSoFar", total_timesteps)
logz.dump_tabular()
logz.pickle_tf_vars()
def generate_corruptions_for_fit(X,
entities_list=None,
eta=1,
corrupt_side='s+o',
entities_size=0,
rnd=None):
"""Generate corruptions for training.
Creates corrupted triples for each statement in an array of statements,
as described by :cite:`trouillon2016complex`.
.. note::
Collisions are not checked, as this will be computationally expensive :cite:`trouillon2016complex`.
That means that some corruptions *may* result in being positive statements (i.e. *unfiltered* settings).
.. note::
When processing large knowledge graphs, it may be useful to generate corruptions only using entities from
a single batch.
This also brings the benefit of creating more meaningful negatives, as entities used to corrupt are
sourced locally.
The function can be configured to generate corruptions *only* using the entities from the current batch.
You can enable such behaviour be setting ``entities_size==-1``. In such case, if ``entities_list=None``
all entities from the *current batch* will be used to generate corruptions.
Parameters
----------
X : Tensor, shape [n, 3]
An array of positive triples that will be used to create corruptions.
entities_list : list
List of entities to be used for generating corruptions. (default:None).
if ``entities_list=None``, all entities will be used to generate corruptions (default behaviour).
eta : int
The number of corruptions per triple that must be generated.
corrupt_side: string
Specifies which side of the triple to corrupt:
- 's': corrupt only subject.
- 'o': corrupt only object
- 's+o': corrupt both subject and object
entities_size: int
Size of entities to be used while generating corruptions. It assumes entity id's start from 0 and are
continuous. (default: 0).
When processing large knowledge graphs, it may be useful to generate corruptions only using entities from
a single batch.
This also brings the benefit of creating more meaningful negatives, as entities used to corrupt are
sourced locally.
The function can be configured to generate corruptions *only* using the entities from the current batch.
You can enable such behaviour be setting ``entities_size==-1``. In such case, if ``entities_list=None``
all entities from the *current batch* will be used to generate corruptions.
rnd: numpy.random.RandomState
A random number generator.
Returns
-------
out : Tensor, shape [n * eta, 3]
An array of corruptions for a list of positive triples x. For each row in X the corresponding corruption
indexes can be found at [index+i*n for i in range(eta)]
"""
logger.debug('Generating corruptions for fit.')
if corrupt_side not in ['s+o', 's', 'o']:
msg = 'Invalid argument value {} for corruption side passed for evaluation.'.format(
corrupt_side)
logger.error(msg)
raise ValueError(msg)
dataset = tf.reshape(tf.tile(tf.reshape(X, [-1]), [eta]),
[tf.shape(X)[0] * eta, 3])
if corrupt_side == 's+o':
keep_subj_mask = tf.tile(
tf.cast(
tf.random_uniform([tf.shape(X)[0]],
0,
2,
dtype=tf.int32,
seed=rnd), tf.bool), [eta])
else:
keep_subj_mask = tf.cast(tf.ones(tf.shape(X)[0] * eta, tf.int32),
tf.bool)
if corrupt_side == 's':
keep_subj_mask = tf.logical_not(keep_subj_mask)
keep_obj_mask = tf.logical_not(keep_subj_mask)
keep_subj_mask = tf.cast(keep_subj_mask, tf.int32)
keep_obj_mask = tf.cast(keep_obj_mask, tf.int32)
logger.debug('Created corruption masks.')
if entities_size != 0:
replacements = tf.random_uniform([tf.shape(dataset)[0]],
0,
entities_size,
dtype=tf.int32,
seed=rnd)
else:
if entities_list is None:
# use entities in the batch
entities_list, _ = tf.unique(
tf.squeeze(
tf.concat([
tf.slice(X, [0, 0], [tf.shape(X)[0], 1]),
tf.slice(X, [0, 2], [tf.shape(X)[0], 1])
], 0)))
random_indices = tf.squeeze(
tf.multinomial(tf.expand_dims(tf.zeros(tf.shape(entities_list)[0]),
0),
num_samples=tf.shape(dataset)[0],
seed=rnd))
replacements = tf.gather(entities_list, random_indices)
subjects = tf.math.add(tf.math.multiply(keep_subj_mask, dataset[:, 0]),
tf.math.multiply(keep_obj_mask, replacements))
logger.debug('Created corrupted subjects.')
relationships = dataset[:, 1]
logger.debug('Retained relationships.')
objects = tf.math.add(tf.math.multiply(keep_obj_mask, dataset[:, 2]),
tf.math.multiply(keep_subj_mask, replacements))
logger.debug('Created corrupted objects.')
out = tf.transpose(tf.stack([subjects, relationships, objects]))
logger.debug('Returning corruptions for fit.')
return out
def train_mode(self,
vocab,
encoder_dim,
encoder_states,
encoder_features,
passage_word_idx,
passage_mask,
init_state,
decoder_inputs,
answer_batch,
loss_weights,
mode_gen='ce_train'):
'''
encoder_dim: int-valued
encoder_states: [batch_size, passage_len, encoder_dim].
passage_word_idx: [batch_size, passage_len] int32
passage_mask: [batch_size, passage_len] 0/1
init_state: Tuple of [batch_size, gen_hidden_size]
decoder_inputs: [batch_size, max_dec_steps].
answer_batch: [batch_size, max_dec_steps]
'''
options = self.options
input_shape = tf.shape(encoder_states)
batch_size = input_shape[0]
passage_len = input_shape[1]
# map decoder inputs to word embeddings
decoder_inputs = tf.unstack(decoder_inputs,
axis=1) # max_enc_steps * [batch_size]
answer_batch_unstack = tf.unstack(answer_batch, axis=1)
# initialize all the variables
state_t_1 = init_state
context_t_1 = tf.zeros([batch_size, encoder_dim])
coverage_t_1 = None
# store variables from each time-step
coverages = []
attn_dists = []
p_gens = []
vocab_scores = []
sampled_words = []
self.encoder_features = encoder_features
with variable_scope.variable_scope("attention_decoder"):
# Get the weight vectors v and W_c (W_c is for coverage)
v = variable_scope.get_variable("v", [options.attention_vec_size])
v = tf.expand_dims(tf.expand_dims(v, axis=0), axis=0)
w_c = None
if options.use_coverage:
with variable_scope.variable_scope("coverage"):
w_c = variable_scope.get_variable(
"w_c", [options.attention_vec_size])
w_c = tf.expand_dims(tf.expand_dims(w_c, axis=0), axis=0)
# For each step, dec_input => lstm_output => vocab_score
wordidx_t = decoder_inputs[0] # [batch_size] int32
for i in range(options.max_answer_len):
if mode_gen in (
'ce_train',
'loss',
):
wordidx_t = decoder_inputs[
i] # the wordidx_t must from decoder_inputs for phrase model
word_t = self.embedding_lookup(wordidx_t)
if i > 0:
variable_scope.get_variable_scope().reuse_variables()
(state_t, context_t, coverage_t, attn_dist_t,
p_gen_t, output_t) = self.one_step_decoder(
state_t_1, context_t_1, coverage_t_1, word_t,
encoder_states, self.encoder_features, passage_word_idx,
passage_mask, v, w_c, vocab)
coverages.append(coverage_t)
attn_dists.append(attn_dist_t)
p_gens.append(p_gen_t)
vocab_scores.append(output_t) # The vocabulary distributions.
state_t_1 = state_t
context_t_1 = context_t
coverage_t_1 = coverage_t
if mode_gen == 'greedy':
wordidx_t = tf.argmax(output_t, 1) # [batch_size]
wordidx_t = tf.reshape(wordidx_t, [-1]) # [batch_size]
elif mode_gen == 'sample':
log_score_t = tf.log(output_t) # [batch_size, vsize]
wordidx_t = tf.multinomial(log_score_t,
1) # [batch_size, 1]
wordidx_t = tf.reshape(wordidx_t, [-1]) # [batch_size]
elif mode_gen in (
'ce_train',
'loss',
):
wordidx_t = answer_batch_unstack[i]
else:
assert False, 'unknown generating mode %s' % mode_gen
sampled_words.append(wordidx_t)
if len(sampled_words) != 0:
sampled_words = tf.stack(sampled_words,
axis=1) # [batch_size, max_dec_steps]
vocab_scores = tf.stack(vocab_scores,
axis=1) # [batch_size, max_dec_steps, vocab]
# calculating loss
self._loss = None
if mode_gen in (
'ce_train',
'loss',
):
xent = CE_loss(vocab_scores, answer_batch,
loss_weights) # [batch_size]
if mode_gen == 'loss':
xent *= self.placeholders.reward # multiply with rewards
self._loss = tf.reduce_mean(xent)
# Calculate coverage loss from the attention distributions
if options.use_coverage:
with tf.variable_scope('coverage_loss'):
self._coverage_loss = _coverage_loss(
attn_dists, loss_weights)
self._loss = self._loss + options.cov_loss_wt * self._coverage_loss
# accuracy is calculated only under 'ce_train', where true answer is given
if mode_gen == 'ce_train':
accuracy = _mask_and_accuracy(vocab_scores, answer_batch,
loss_weights)
return accuracy, self._loss, sampled_words
else:
return None, self._loss, sampled_words
def create_selection_weights(name,
type_,
shape,
inv_t=1,
initializer=tf.zeros_initializer(),
regularizer=None,
names=None):
"""Create a SelectionWeights tuple.
Args:
name: Name for the underlying variable containing the unnormalized weights.
type_: "softmax" or "sigmoid" or ("softmax_topk", k) where k is an int.
shape: Shape for the variable.
inv_t: Inverse of the temperature to use in normalization.
initializer: Initializer for the variable, passed to `tf.get_variable`.
regularizer: Regularizer for the variable. A callable which accepts
`tempered_var` and `normalized`.
names: Name of each selection.
Returns:
The created SelectionWeights tuple.
Raises:
ValueError: if type_ is not in the supported range.
"""
var = tf.get_variable(name, shape, initializer=initializer)
if callable(inv_t):
inv_t = inv_t(var)
if inv_t == 1:
tempered_var = var
else:
tempered_var = var * inv_t
if type_ == "softmax":
weights = tf.nn.softmax(tempered_var)
elif type_ == "sigmoid":
weights = tf.nn.sigmoid(tempered_var)
elif isinstance(type_, (list, tuple)) and type_[0] == "softmax_topk":
assert len(shape) == 1
# TODO(rshin): Change this to select without replacement?
selection = tf.multinomial(tf.expand_dims(var, axis=0), 4)
selection = tf.squeeze(selection, axis=0) # [k] selected classes.
to_run = tf.one_hot(selection, shape[0]) # [k x nmodules] one-hot.
# [nmodules], 0=not run, 1=run.
to_run = tf.minimum(tf.reduce_sum(to_run, axis=0), 1)
weights = tf.nn.softmax(tempered_var - 1e9 * (1.0 - to_run))
else:
raise ValueError("Unknown type: %s" % type_)
if regularizer is not None:
loss = regularizer(tempered_var, weights)
if loss is not None:
tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, loss)
if names is not None:
tf.get_collection_ref("selection_weight_names/" + var.name).extend(
names.flatten() if isinstance(names, np.ndarray) else names)
tf.add_to_collection("selection_weight_names_tensor/" + var.name,
tf.constant(names))
return SelectionWeights(var=var,
tempered_var=tempered_var,
inv_t=inv_t,
normalized=weights)
def __init__(self, lr, o_size_h, o_size_w, a_size, h_size, epsilon, beta,
max_step):
"""
Creates Discrete Control Actor-Critic model for use with visual observations (images).
:param o_size_h: Observation height.
:param o_size_w: Observation width.
:param a_size: Action-space size.
:param h_size: Hidden layer size.
"""
self.observation_in = tf.placeholder(
shape=[None, o_size_h, o_size_w, 1],
dtype=tf.float32,
name='observation_0')
self.conv1 = tf.layers.conv2d(self.observation_in,
32,
kernel_size=[3, 3],
strides=[2, 2],
use_bias=False,
activation=tf.nn.elu)
self.conv2 = tf.layers.conv2d(self.conv1,
64,
kernel_size=[3, 3],
strides=[2, 2],
use_bias=False,
activation=tf.nn.elu)
self.batch_size = tf.placeholder(shape=None, dtype=tf.int32)
hidden = tf.layers.dense(c_layers.flatten(self.conv2),
h_size,
use_bias=False,
activation=tf.nn.elu)
self.policy = tf.layers.dense(
hidden,
a_size,
activation=None,
use_bias=False,
kernel_initializer=c_layers.variance_scaling_initializer(
factor=0.1))
self.probs = tf.nn.softmax(self.policy)
self.action = tf.multinomial(self.policy, 1)
self.output = tf.identity(self.action, name='action')
self.value = tf.layers.dense(hidden,
1,
activation=None,
use_bias=False)
self.entropy = -tf.reduce_sum(self.probs * tf.log(self.probs + 1e-10),
axis=1)
self.action_holder = tf.placeholder(shape=[None], dtype=tf.int32)
self.selected_actions = c_layers.one_hot_encoding(
self.action_holder, a_size)
self.old_probs = tf.placeholder(shape=[None, a_size],
dtype=tf.float32,
name='old_probabilities')
self.responsible_probs = tf.reduce_sum(self.probs *
self.selected_actions,
axis=1)
self.old_responsible_probs = tf.reduce_sum(self.old_probs *
self.selected_actions,
axis=1)
PPOModel.__init__(self, self.responsible_probs,
self.old_responsible_probs, self.value, self.entropy,
beta, epsilon, lr, max_step)
def select_sample_input():
current_logits = logits.read(ts)
decoder_input = tf.to_int32(tf.multinomial(current_logits, 1))
decoder_input = tf.stop_gradient(decoder_input)
return tf.squeeze(decoder_input, [1])
def make_data_tensor(self, train=True):
if train:
folders = self.metatrain_character_folders
# number of tasks, not number of meta-iterations. (divide by metabatch size to measure)
num_total_batches = 200000
else:
folders = self.metaval_character_folders
num_total_batches = 600
# make list of files
print('Generating filenames')
all_filenames = []
for _ in range(num_total_batches):
sampled_character_folders = random.sample(folders,
self.num_classes)
random.shuffle(sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
# make queue for tensorflow to read from
filename_queue = tf.train.string_input_producer(
tf.convert_to_tensor(all_filenames), shuffle=False)
print('Generating image processing ops')
image_reader = tf.WholeFileReader()
_, image_file = image_reader.read(filename_queue)
if FLAGS.datasource == 'miniimagenet':
image = tf.image.decode_jpeg(image_file, channels=3)
image.set_shape((self.img_size[0], self.img_size[1], 3))
image = tf.reshape(image, [self.dim_input])
image = tf.cast(image, tf.float32) / 255.0
else:
image = tf.image.decode_png(image_file)
image.set_shape((self.img_size[0], self.img_size[1], 1))
image = tf.reshape(image, [self.dim_input])
image = tf.cast(image, tf.float32) / 255.0
image = 1.0 - image # invert
num_preprocess_threads = 1 # TODO - enable this to be set to >1
min_queue_examples = 256
examples_per_batch = self.num_classes * self.num_samples_per_class
batch_image_size = self.batch_size * examples_per_batch
print('Batching images')
images = tf.train.batch(
[image],
batch_size=batch_image_size,
num_threads=num_preprocess_threads,
capacity=min_queue_examples + 3 * batch_image_size,
)
all_image_batches, all_label_batches = [], []
print('Manipulating image data to be right shape')
for i in range(self.batch_size):
image_batch = images[i * examples_per_batch:(i + 1) *
examples_per_batch]
if FLAGS.datasource == 'omniglot':
# omniglot augments the dataset by rotating digits to create new classes
# get rotation per class (e.g. 0,1,2,0,0 if there are 5 classes)
rotations = tf.multinomial(tf.log([[1., 1., 1., 1.]]),
self.num_classes)
label_batch = tf.convert_to_tensor(labels)
new_list, new_label_list = [], []
for k in range(self.num_samples_per_class):
class_idxs = tf.range(0, self.num_classes)
class_idxs = tf.random_shuffle(class_idxs)
true_idxs = class_idxs * self.num_samples_per_class + k
new_list.append(tf.gather(image_batch, true_idxs))
if FLAGS.datasource == 'omniglot': # and FLAGS.train:
new_list[-1] = tf.stack([
tf.reshape(
tf.image.rot90(tf.reshape(
new_list[-1][ind],
[self.img_size[0], self.img_size[1], 1]),
k=tf.cast(
rotations[0, class_idxs[ind]],
tf.int32)), (self.dim_input, ))
for ind in range(self.num_classes)
])
new_label_list.append(tf.gather(label_batch, true_idxs))
new_list = tf.concat(
new_list, 0
) # has shape [self.num_classes*self.num_samples_per_class, self.dim_input]
new_label_list = tf.concat(new_label_list, 0)
all_image_batches.append(new_list)
all_label_batches.append(new_label_list)
all_image_batches = tf.stack(all_image_batches)
all_label_batches = tf.stack(all_label_batches)
all_label_batches = tf.one_hot(all_label_batches, self.num_classes)
return all_image_batches, all_label_batches
def __init__(
self
):
self.num_layers = 4
self.num_branches = 6
self.lstm_size = 32
self.num_blocks_per_branch = 6
self.l2_reg = 1e-4
self.lstm_weight = []
for layer_id in range(self.num_layers):
with tf.variable_scope("layer_{}".format(layer_id)):
w = tf.get_variable("w", [2 * self.lstm_size, 4 * self.lstm_size])
self.lstem_weight.append(w)
self.num_configs = (2 ** self.num_blocks_per_branch) - 1
with tf.variable_scope("embedding"):
self.embed_graph = tf.get_variable("embed_graph", [1, self.lstm_size])
self.embed_weight = tf.get_variable("weight", [
self.num_blocks_per_branch,
self.lstm_size
])
with tf.variable_scope("softmax"):
self.softmax_weight = tf.get_variable("weight", [
self.lstm_size,
self.num_blocks_per_branch
])
with tf.variable_scope("critic"):
self.critic_weight = tf.get_variable("weight", [self.lstm_size, 1])
arc_seq = []
sample_log_probs = []
all_h = []
inputs = self.embed_graph
prev_channel = [
tf.zeros([1, self.lstm_size], dtype=tf.float32)
for _ in range(self.lstm_num_layers)
]
prev_height = [
tf.zeros([1, self.lstm_size], dtype=tf.float32)
for _ in range(self.lstm_num_layers)
]
for layer_id in range(self.num_layers):
for branch_id in range(self.num_branches):
next_channel, next_height = stack_lstm(
inputs,
prev_channel,
prev_height,
self.lstm_weight
)
all_h.append(tf.stop_gradient(next_height[-1]))
logits = tf.matmul(next_height[-1], self.softmax_weight)
logits = 1.10 * tf.tanh(logits)
config_id = tf.multinomial(logits, 1)
config_id = tf.to_int32(config_id)
config_id = tf.reshape(config_id, [1])
arc_seq.append(config_id)
log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits,
labels=config_id
)
inputs = tf.nn.embedding_lookup(self.embed_weight, config_id)
self.sample_arc = tf.concat(arc_seq, axis=0)
self.sample_log_probs = tf.concat(sample_log_probs, axis=0)
self.ppl = tf.exp(
tf.reduce_sum(self.sample_log_probs) /
tf.to_float(self.num_layers, self.num_branches)
)
self.all_h = all_h
def multinomial_sample(x, vocab_size, temperature):
"""Multinomial sampling from a n-dimensional tensor."""
samples = tf.multinomial(tf.reshape(x, [-1, vocab_size]) / temperature, 1)
reshaped_samples = tf.reshape(samples, tf.shape(x)[:-1])
return tf.to_int32(reshaped_samples)
def loop_fn(time, cell_output, cell_state, loop_state):
if cell_output is None: # time == 0
next_cell_state = encoder_states
next_input = tf.tile(go_embedding, to_T([N, 1]))
else: # time > 0
next_cell_state = cell_state
# compute the attention map over the input sequence
# a_raw has shape [T, N, 1]
att_raw = tf.reduce_sum(
tf.tanh(tf.nn.xw_plus_b(cell_output, W_a, b_a) +
self.encoder_h_transformed) * v,
axis=2, keep_dims=True)
# softmax along the first dimension (T) over not finished examples
# att has shape [T, N, 1]
att = tf.nn.softmax(att_raw, dim=0)*self.seq_not_finished
att = att / tf.reduce_sum(att + 1e-10, axis=0, keep_dims=True)
# d has shape [N, lstm_dim]
d2 = tf.reduce_sum(att*self.encoder_outputs, axis=0)
# token_scores has shape [N, num_vocab]
token_scores = tf.nn.xw_plus_b(
tf.concat([cell_output, d2], axis=1),
W_y, b_y)
decoding_state = loop_state[2]
# token_validity has shape [N, num_vocab]
token_validity = _get_valid_tokens(decoding_state, self.W, self.b)
token_validity.set_shape([None, self.decoder_num_vocab])
if use_gt_layout is not None:
# when there's ground-truth layout, do not re-normalize prob
# and treat all tokens as valid
token_validity = tf.logical_or(token_validity, use_gt_layout)
validity_mult = tf.cast(token_validity, tf.float32)
# predict the next token (behavior depending on parameters)
if sampling:
token_scores_valid = token_scores - (1-validity_mult) * 50
# TODO:debug
sampled_token = tf.cast(tf.reshape(
tf.multinomial(token_scores_valid/self.temperature, 1), [-1]), tf.int32)
# make sure that the predictions are ALWAYS valid
# (it can be invalid with very small prob)
# If not, just fall back to min cases
# pred_mask has shape [N, num_vocab]
sampled_mask = tf.equal(mask_range, tf.reshape(sampled_token, [-1, 1]))
is_sampled_valid = tf.reduce_any(
tf.logical_and(sampled_mask, token_validity),
axis=1)
# Fall back to max score (no sampling)
min_score = tf.reduce_min(token_scores)
token_scores_valid = tf.where(token_validity, token_scores,
tf.ones_like(token_scores)*(min_score-1))
max_score_token = tf.cast(tf.argmax(token_scores_valid, 1), tf.int32)
predicted_token = tf.where(is_sampled_valid, sampled_token, max_score_token)
else:
min_score = tf.reduce_min(token_scores)
token_scores_valid = tf.where(token_validity, token_scores,
tf.ones_like(token_scores)*(min_score-1))
# predicted_token has shape [N]
predicted_token = tf.cast(tf.argmax(token_scores_valid, 1), tf.int32)
if use_gt_layout is not None:
predicted_token = (gt_layout_batch[time-1] * gt_layout_mult
+ predicted_token * pred_layout_mult)
# a robust version of softmax
# all_token_probs has shape [N, num_vocab]
all_token_probs = tf.nn.softmax(token_scores) * validity_mult
# tf.check_numerics(all_token_probs, 'NaN/Inf before div')
all_token_probs = all_token_probs / tf.reduce_sum(all_token_probs + 1e-10, axis=1, keep_dims=True)
# tf.check_numerics(all_token_probs, 'NaN/Inf after div')
# mask has shape [N, num_vocab]
mask = tf.equal(mask_range, tf.reshape(predicted_token, [-1, 1]))
# token_prob has shape [N], the probability of the predicted token
# although token_prob is not needed for predicting the next token
# it is needed in output (for policy gradient training)
# [N, num_vocab]
token_prob = tf.reduce_sum(all_token_probs * tf.cast(mask, tf.float32), axis=1)
# tf.assert_positive(token_prob)
neg_entropy = tf.reduce_sum(
all_token_probs * tf.log(all_token_probs + (1-validity_mult) + 1e-10),
axis=1)
# update states
updated_decoding_state = _update_decoding_state(
decoding_state, predicted_token, self.P)
# the prediction is from the cell output of the last step
# timestep (t-1), feed it as input into timestep t
next_input = tf.nn.embedding_lookup(embedding_mat, predicted_token)
elements_finished = tf.greater_equal(time, T_max)
# loop_state is a 5-tuple, representing
# 1) the predicted_tokens
# 2) the prob of predicted_tokens
# 3) the decoding state (used for validity)
# 4) the negative entropy of policy (accumulated across timesteps)
# 5) the attention
if loop_state is None: # time == 0
# Write the predicted token into the output
predicted_token_array = tf.TensorArray(dtype=tf.int32, size=T_max,
infer_shape=False)
token_prob_array = tf.TensorArray(dtype=tf.float32, size=T_max,
infer_shape=False)
init_decoding_state = tf.tile(to_T([[0, 0, T_max]], dtype=tf.int32), to_T([N, 1]))
att_array = tf.TensorArray(dtype=tf.float32, size=T_max,
infer_shape=False)
next_loop_state = (predicted_token_array,
token_prob_array,
init_decoding_state,
tf.zeros(to_T([N]), dtype=tf.float32),
att_array)
else: # time > 0
t_write = time-1
next_loop_state = (loop_state[0].write(t_write, predicted_token),
loop_state[1].write(t_write, token_prob),
updated_decoding_state,
loop_state[3] + neg_entropy,
loop_state[4].write(t_write, att))
return (elements_finished, next_input, next_cell_state, cell_output,
next_loop_state)
def random_category(self, size, dtype):
prior = tf.ones(tf.stack((tf.shape(self.gan.inputs.x)[0], size)))*1./size
dist = tf.log(prior + TINY)
sample=tf.multinomial(dist, num_samples=1)[:, 0]
return tf.one_hot(sample, size, dtype=dtype)
def inner_loop(i, alive_seq):
logit = symbols_to_logits_fn(alive_seq)[0]
new_samples = tf.multinomial(logit, 1)
new_samples = tf.to_int32(new_samples)
alive_seq = tf.concat([alive_seq, new_samples], 1)
return (i + 1, alive_seq)
def _build_sampler(self):
"""Build the sampler ops and the log_prob ops."""
arc_seq = []
sample_log_probs = []
sample_entropy = []
all_h = []
all_h_w = []
# sampler ops
inputs = self.g_emb
prev_c, prev_h = [], []
for _ in range(self.lstm_num_layers):
prev_c.append(tf.zeros([1, self.lstm_size], dtype=tf.float32))
prev_h.append(tf.zeros([1, self.lstm_size], dtype=tf.float32))
# used = tf.zeros([self.rhn_depth, 2], dtype=tf.int32)
for layer_id in range(self.rhn_depth):
next_c, next_h = stack_lstm(inputs, prev_c, prev_h, self.w_lstm)
prev_c, prev_h = next_c, next_h
all_h.append(next_h[-1])
all_h_w.append(tf.matmul(next_h[-1], self.attn_w_1))
if layer_id > 0:
query = tf.matmul(next_h[-1], self.attn_w_2)
query = query + tf.concat(all_h_w[:-1], axis=0)
query = tf.tanh(query)
logits = tf.matmul(query, self.attn_v)
logits = tf.reshape(logits, [1, layer_id])
if self.temperature is not None:
logits /= self.temperature
if self.tanh_constant is not None:
logits = self.tanh_constant * tf.tanh(logits)
diff = tf.to_float(layer_id - tf.range(0, layer_id)) ** 2
logits -= tf.reshape(diff, [1, layer_id]) / 6.0
skip_index = tf.multinomial(logits, 1)
skip_index = tf.to_int32(skip_index)
skip_index = tf.reshape(skip_index, [1])
arc_seq.append(skip_index)
log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=skip_index)
sample_log_probs.append(log_prob)
entropy = log_prob * tf.exp(-log_prob)
sample_entropy.append(tf.stop_gradient(entropy))
inputs = tf.nn.embedding_lookup(
tf.concat(all_h[:-1], axis=0), skip_index)
inputs /= (0.1 + tf.to_float(layer_id - skip_index))
else:
inputs = self.g_emb
next_c, next_h = stack_lstm(inputs, prev_c, prev_h, self.w_lstm)
prev_c, prev_h = next_c, next_h
logits = tf.matmul(next_h[-1], self.w_soft)
if self.temperature is not None:
logits /= self.temperature
if self.tanh_constant is not None:
logits = self.tanh_constant * tf.tanh(logits)
func = tf.multinomial(logits, 1)
func = tf.to_int32(func)
func = tf.reshape(func, [1])
arc_seq.append(func)
log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=func)
sample_log_probs.append(log_prob)
entropy = log_prob * tf.exp(-log_prob)
sample_entropy.append(tf.stop_gradient(entropy))
inputs = tf.nn.embedding_lookup(self.w_emb, func)
arc_seq = tf.concat(arc_seq, axis=0)
self.sample_arc = arc_seq
self.sample_log_probs = tf.concat(sample_log_probs, axis=0)
self.ppl = tf.exp(tf.reduce_mean(self.sample_log_probs))
sample_entropy = tf.concat(sample_entropy, axis=0)
self.sample_entropy = tf.reduce_sum(sample_entropy)
self.all_h = all_h
def image_augmentations(image,
data_augmentations,
model_input_image_size,
label=None):
"""Coordinating image augmentations for both image and heatmap."""
im_size = [int(x) for x in image.get_shape()]
im_size_check = np.any(
np.less_equal(model_input_image_size[:2], im_size[:2]))
if data_augmentations is not None:
# Pixel/image-level augmentations
if 'singleton' in data_augmentations:
image = tf.expand_dims(image, axis=-1)
print 'Adding singleton dimension to image.'
if 'singleton_label' in data_augmentations:
label = tf.expand_dims(label, axis=-1)
print 'Adding singleton dimension to label.'
if 'bsds_crop' in data_augmentations and im_size_check:
assert len(image.get_shape()) == 3, '4D not implemented yet.'
# intermediate_size = [171, 256, 3]
# intermediate_size = [256, 384, 3]
intermediate_size = [324, 484, 3]
image = tf.image.resize_image_with_crop_or_pad(
image, intermediate_size[0], intermediate_size[1])
label = tf.image.resize_image_with_crop_or_pad(
label, intermediate_size[0], intermediate_size[1])
print 'Applying BSDS crop.'
if 'uint8_rescale' in data_augmentations:
image = tf.cast(image, tf.float32) / 255.
print 'Applying uint8 rescale to the image.'
if 'uint8_rescale_label' in data_augmentations:
label = tf.cast(label, tf.float32) / 255.
print 'Applying uint8 rescale to the label.'
if 'uint8_rescale_-1_1' in data_augmentations:
image = 2 * (tf.cast(image, tf.float32) / 255.) - 1
print 'Applying uint8 rescale.'
if 'image_to_bgr' in data_augmentations:
image = tf.stack([image[:, :, 2], image[:, :, 1], image[:, :, 0]],
axis=-1)
if 'pascal_normalize' in data_augmentations:
image = image - [123.68, 116.78, 103.94]
if 'random_contrast' in data_augmentations:
assert len(image.get_shape()) == 3, '4D not implemented yet.'
image = tf.image.random_contrast(image, lower=0.2, upper=1.8)
print 'Applying random contrast.'
if 'random_brightness' in data_augmentations:
assert len(image.get_shape()) == 3, '4D not implemented yet.'
image = tf.image.random_brightness(image, max_delta=63.)
print 'Applying random brightness.'
if 'grayscale' in data_augmentations and im_size_check:
# image = tf.image.rgb_to_grayscale(image)
image = tf.expand_dims(image[:, :, 0], axis=-1) # ABOVE INSTEAD?
print 'Converting to grayscale.'
# Affine augmentations
if 'rotate' in data_augmentations and im_size_check:
max_theta = 22.
angle_rad = (max_theta / 180.) * math.pi
angles = tf.random_uniform([], -angle_rad, angle_rad)
transform = tf.contrib.image.angles_to_projective_transforms(
angles, im_size[0], im_size[1])
image = tf.contrib.image.transform(
image,
tf.contrib.image.compose_transforms(transform),
interpolation='BILINEAR') # or 'NEAREST'
print 'Applying random rotate.'
if 'rotate_image_label' in data_augmentations and im_size_check:
max_theta = 30.
angle_rad = (max_theta / 180.) * math.pi
angles = tf.random_uniform([], -angle_rad, angle_rad)
transform = tf.contrib.image.angles_to_projective_transforms(
angles, im_size[0], im_size[1])
image = tf.contrib.image.transform(
image,
tf.contrib.image.compose_transforms(transform),
interpolation='BILINEAR') # or 'NEAREST'
label = tf.contrib.image.transform(
label,
tf.contrib.image.compose_transforms(transform),
interpolation='BILINEAR') # or 'NEAREST'
print 'Applying random rotate.'
if 'random_scale_crop_image_label' in data_augmentations\
and im_size_check:
scale_choices = tf.convert_to_tensor([1., 1.02, 1.04, 1.06, 1.08])
samples = tf.multinomial(tf.log([tf.ones_like(scale_choices)]), 1)
image_shape = image.get_shape().as_list()
scale = scale_choices[tf.cast(samples[0][0], tf.int32)]
scale_tf = tf.cast(
tf.round(
np.asarray(model_input_image_size[:2]).astype(np.float32) *
scale), tf.int32)
combined = tf.concat([image, label], axis=-1)
combo_shape = combined.get_shape().as_list()
combined_crop = tf.random_crop(
combined, tf.concat([scale_tf, [combo_shape[-1]]], 0))
combined_resize = tf.squeeze(tf.image.resize_bicubic(
tf.expand_dims(combined_crop, axis=0),
model_input_image_size[:2],
align_corners=True),
axis=0)
image = combined_resize[:, :, :image_shape[-1]]
label = combined_resize[:, :, image_shape[-1]:]
image.set_shape(model_input_image_size)
label.set_shape(model_input_image_size[:2] +
[combo_shape[-1] - model_input_image_size[-1]])
if 'rc_res' in data_augmentations and im_size_check:
image = random_crop(image, model_input_image_size)
if len(model_input_image_size) > 2:
model_input_image_size = model_input_image_size[:2]
ms = [x // 2 for x in model_input_image_size]
image = resize_image_label(im=image,
model_input_image_size=ms,
f='bicubic')
print 'Applying random crop and resize.'
if 'cc_res' in data_augmentations and im_size_check:
image = center_crop(image, model_input_image_size)
if len(model_input_image_size) > 2:
model_input_image_size = model_input_image_size[:2]
ms = [x // 2 for x in model_input_image_size]
image = resize_image_label(im=image,
model_input_image_size=ms,
f='bicubic')
print 'Applying center crop and resize.'
if 'random_crop' in data_augmentations and im_size_check:
image = random_crop(image, model_input_image_size)
print 'Applying random crop.'
if 'center_crop' in data_augmentations and im_size_check:
image = center_crop(image, model_input_image_size)
print 'Applying center crop.'
if 'random_crop_image_label' in data_augmentations and im_size_check:
assert len(image.get_shape()) == 3, '4D not implemented yet.'
image, label = crop_image_label(image=image,
label=label,
size=model_input_image_size,
crop='random')
if 'center_crop_image_label' in data_augmentations and im_size_check:
assert len(image.get_shape()) == 3, '4D not implemented yet.'
image, label = crop_image_label(image=image,
label=label,
size=model_input_image_size,
crop='center')
if 'resize' in data_augmentations and im_size_check:
if len(model_input_image_size) > 2:
model_input_image_size = model_input_image_size[:2]
image = resize_image_label(
im=image,
model_input_image_size=model_input_image_size,
f='bicubic')
print 'Applying area resize.'
if 'jk_resize' in data_augmentations and im_size_check:
if len(model_input_image_size) > 2:
model_input_image_size = model_input_image_size[:2]
image = tf.image.resize_image_with_crop_or_pad(
image, model_input_image_size[0], model_input_image_size[1])
print 'Applying area resize.'
if 'resize_and_crop' in data_augmentations and im_size_check:
model_input_image_size_1 = np.asarray(
model_input_image_size[:2]) + 28
image = resize_image_label(
im=image,
model_input_image_size=model_input_image_size_1,
f='area')
image = center_crop(image, model_input_image_size)
print 'Applying area resize.'
if 'resize_nn' in data_augmentations and im_size_check:
assert len(image.get_shape()) == 3, '4D not implemented yet.'
if len(model_input_image_size) > 2:
model_input_image_size = model_input_image_size[:2]
image = resize_image_label(
im=image,
model_input_image_size=model_input_image_size,
f='nearest')
print 'Applying nearest resize.'
if 'resize_image_label' in data_augmentations and im_size_check:
assert len(image.get_shape()) == 3, '4D not implemented yet.'
if len(model_input_image_size) > 2:
model_input_image_size = model_input_image_size[:2]
image = resize_image_label(
im=image,
model_input_image_size=model_input_image_size,
f='bicubic')
label = resize_image_label(
im=label,
model_input_image_size=model_input_image_size,
f='bicubic')
print 'Applying bilinear resize.'
elif 'resize_nn_image_label' in data_augmentations and im_size_check:
assert len(image.get_shape()) == 3, '4D not implemented yet.'
if len(model_input_image_size) > 2:
model_input_image_size = model_input_image_size[:2]
image = resize_image_label(
im=image,
model_input_image_size=model_input_image_size,
f='nearest')
label = resize_image_label(
im=label,
model_input_image_size=model_input_image_size,
f='nearest')
print 'Applying nearest resize.'
else:
pass
if 'left_right' in data_augmentations:
image = image_flip(image, direction='left_right')
print 'Applying random flip left-right.'
if 'up_down' in data_augmentations:
image = image_flip(image, direction='up_down')
print 'Applying random flip up-down.'
if 'lr_flip_image_label' in data_augmentations:
assert len(image.get_shape()) == 3, '4D not implemented yet.'
image, label = lr_flip_image_label(image, label)
if 'ud_flip_image_label' in data_augmentations:
assert len(image.get_shape()) == 3, '4D not implemented yet.'
image, label = ud_flip_image_label(image, label)
if 'gaussian_noise' in data_augmentations:
im_shape = image.get_shape().as_list()
assert len(im_shape) == 3, '4D not implemented yet.'
sigma = 1. / 10.
mu = 0.
image = image + tf.random_normal(im_shape, mean=mu, stddev=sigma)
print 'Applying gaussian noise.'
if 'gaussian_noise_small' in data_augmentations:
im_shape = image.get_shape().as_list()
assert len(im_shape) == 3, '4D not implemented yet.'
sigma = 1. / 20.
mu = 0.
image = image + tf.random_normal(im_shape, mean=mu, stddev=sigma)
print 'Applying gaussian noise.'
if 'calculate_rate_time_crop' in data_augmentations:
im_shape = image.get_shape().as_list()
minval = im_shape[0] // 3
time_crop = tf.random_uniform([],
minval=minval,
maxval=im_shape[0],
dtype=tf.int32)
# For now always pull from the beginning
indices = tf.range(0, time_crop, dtype=tf.int32)
selected_image = tf.gather(image, indices)
padded_image = tf.zeros([im_shape[0] - time_crop] + im_shape[1:],
dtype=selected_image.dtype)
# Randomly concatenate pad to front or back
image = tf.cond(pred=tf.greater(
tf.random_uniform([], minval=0, maxval=1, dtype=tf.float32),
0.5),
true_fn=lambda: tf.concat(
[selected_image, padded_image], axis=0),
false_fn=lambda: tf.concat(
[padded_image, selected_image], axis=0))
image.set_shape(im_shape)
# Convert label to rate
label = label / im_shape[0]
if 'calculate_rate' in data_augmentations:
label = label / image.get_shape().as_list()[0]
print 'Applying rate transformation.'
if 'threshold' in data_augmentations:
image = tf.cast(tf.greater(image, 0.1), tf.float32)
print 'Applying threshold.'
if 'nonzero_label' in data_augmentations:
label = tf.cast(tf.greater(label, 0.2), tf.float32)
print 'Applying threshold.'
if 'zero_one' in data_augmentations:
image = tf.minimum(tf.maximum(image, 0.), 1.)
print 'Applying threshold.'
if 'timestep_duplication' in data_augmentations:
image = tf.stack([image for iid in range(7)])
print 'Applying timestep duplication.'
if 'per_image_standardization' in data_augmentations:
image = tf.image.per_image_standardization(image)
print 'Applying per-image zscore.'
if 'flip_polarity' in data_augmentations:
image = tf.abs(image - 1.)
if 'NCHW' in data_augmentations:
image = tf.transpose(image, (2, 0, 1))
else:
assert len(image.get_shape()) == 3, '4D not implemented yet.'
image = tf.image.resize_image_with_crop_or_pad(
image, model_input_image_size[0], model_input_image_size[1])
return image, label
n_outputs = 1
learning_rate = 0.01
initializer = tf.variance_scaling_initializer()
X = tf.placeholder(tf.float32, shape=[None, n_inputs])
hidden = tf.layers.dense(X,
n_hidden,
activation=tf.nn.elu,
kernel_initializer=initializer)
logits = tf.layers.dense(hidden, n_outputs)
outputs = tf.nn.sigmoid(logits) # probability of action 0 (left)
p_left_and_right = tf.concat(axis=1, values=[outputs, 1 - outputs])
action = tf.multinomial(tf.log(p_left_and_right), num_samples=1)
y = 1. - tf.to_float(action)
cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=y,
logits=logits)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads_and_vars = optimizer.compute_gradients(cross_entropy)
gradients = [grad for grad, variable in grads_and_vars]
gradient_placeholders = []
grads_and_vars_feed = []
for grad, variable in grads_and_vars:
gradient_placeholder = tf.placeholder(tf.float32, shape=grad.get_shape())
gradient_placeholders.append(gradient_placeholder)
grads_and_vars_feed.append((gradient_placeholder, variable))
training_op = optimizer.apply_gradients(grads_and_vars_feed)
def ptb_producer(doc, que, ans, batch_size, vocab=100, name=None, config=None):
"""Iterate on the raw PTB data.
This chunks up raw_data into batches of examples and returns Tensors that
are drawn from these batches.
Args:
raw_data: one of the raw data outputs from ptb_raw_data.
batch_size: int, the batch size.
num_steps: int, the number of unrolls.
name: the name of this operation (optional).
Returns:
A pair of Tensors, each shaped [batch_size, num_steps]. The second element
of the tuple is the same data time-shifted to the right by one.
Raises:
tf.errors.InvalidArgumentError: if batch_size or num_steps are too high.
"""
#print(ans)
with tf.name_scope(name, "PTBProducer", [doc, que, ans]):
doc_len = len(doc)
vocab = config.vocab_size
vans = []
for e in ans:
van = [0] * vocab
van[e] = 1
vans.append(van)
#print(doc)
d = len(doc[0])
q = len(que[0])
#print(d)
epoch_size = doc_len // batch_size
#print(epoch_size)
#print(ans)
doc = tf.convert_to_tensor(doc, name="documents", dtype=tf.int32)
que = tf.convert_to_tensor(que, name="questions", dtype=tf.int32)
vans = tf.convert_to_tensor(vans, name="vanswers", dtype=tf.int32)
ans = tf.convert_to_tensor(ans, name="answers", dtype=tf.int32)
#data_len = len(documents)
# batch_len = len(documents[0])
assertion = tf.assert_positive(
epoch_size,
message="epoch_size == 0, decrease batch_size or num_steps")
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name="epoch_size")
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
#elems = tf.convert_to_tensor([1,2,3,5])
batch_prob = []
#for batch_number in range(batch_size):
batch_prob.append([10.] * doc_len)
samples = tf.multinomial(tf.log(batch_prob),
batch_size) # note log-prob
print(tf.get_variable_scope().reuse == False)
x = []
y = []
z = []
zz = []
for batch_number in range(batch_size):
x.append(doc[tf.cast(samples[0][batch_number], tf.int32)])
y.append(que[tf.cast(samples[0][batch_number], tf.int32)])
z.append(vans[tf.cast(samples[0][batch_number], tf.int32)])
zz.append(ans[tf.cast(samples[0][batch_number], tf.int32)])
x = tf.convert_to_tensor(x, name="documents", dtype=tf.int32)
y = tf.convert_to_tensor(y, name="questions", dtype=tf.int32)
z = tf.convert_to_tensor(z, name="vanswers", dtype=tf.int32)
zz = tf.convert_to_tensor(zz, name="answers", dtype=tf.int32)
'''
x= tf.slice(doc,[i*batch_size,0],[batch_size,d])
y= tf.slice(que,[i*batch_size,0],[batch_size,q])
z= tf.slice(vans,[i*batch_size,0],[batch_size,vocab])
zz=tf.slice(ans,[i*batch_size],[batch_size])
'''
#print(i)
#print(epoch_size)
return x, y, z, zz, epoch_size