def loop_fn(time, prev_output, prev_state, array_targets: tf.TensorArray, array_outputs: tf.TensorArray):
"""
Main decoder loop
:param time: Day number
:param prev_output: Output(prediction) from previous step
:param prev_state: RNN state tensor from previous step
:param array_targets: Predictions, each step will append new value to this array
:param array_outputs: Raw RNN outputs (for regularization losses)
:return:
"""
# RNN inputs for current step
features = inputs_by_time[time]
# [batch, predict_window, readout_depth * n_heads] -> [batch, readout_depth * n_heads]
if attn_features is not None:
# [batch_size, 1] + [batch_size, input_depth]
attn = attn_features[:, time, :]
# Append previous predicted value + attention vector to input features
next_input = tf.concat([prev_output, features, attn], axis=1)
else:
# Append previous predicted value to input features
next_input = tf.concat([prev_output, features], axis=1)
# Run RNN cell
output, state = cell(next_input, prev_state)
# Make prediction from RNN outputs
projected_output = project_output(output)
# Append step results to the buffer arrays
if return_raw_outputs:
array_outputs = array_outputs.write(time, output)
array_targets = array_targets.write(time, projected_output)
# Increment time and return
return time + 1, projected_output, state, array_targets, array_outputs
def body(step: int, current_input: tf.Tensor, previous_states: tf.Tensor,
outputs: tf.TensorArray):
current_states_list = []
for i, cell in enumerate(cells):
with tf.variable_scope("rnn_cell_%d" % i):
cell_previous_hidden_vector = tf.squeeze(tf.slice(
previous_states, [0, i, 0], [-1, 1, -1]),
axis=[1])
output, state = cell(current_input,
cell_previous_hidden_vector)
# Set current_input to output for next cell iteration
current_input = output
current_states_list.append(state)
current_states = tf.concat(
[tf.expand_dims(state, axis=1) for state in current_states_list],
axis=1)
with tf.variable_scope("fully_connected_output"):
final_output = tf.contrib.layers.fully_connected(
current_input, num_outputs=1, activation_fn=tf.nn.sigmoid)
outputs.write(step, final_output)
return step + 1, current_input, current_states, outputs
def loop_fn(time, prev_output, prev_state, array_targets: tf.TensorArray,
array_outputs: tf.TensorArray):
"""
Main decoder loop
:param time: Step number
:param prev_output: Output(prediction) from previous step
:param prev_state: RNN state tensor from previous step
:param array_targets: Predictions, each step will append new value to this array
:param array_outputs: Raw RNN outputs (for regularization losses)
:return:
"""
# Append previous predicted value to input features
next_input = prev_output
# Run RNN cell
output, state = cell(next_input, prev_state)
# Make prediction from RNN outputs
projected_output = project_output(output)
# Append step results to the buffer arrays
if return_raw_outputs:
array_outputs = array_outputs.write(time, output)
array_targets = array_targets.write(time, projected_output)
# Increment time and return
return time + 1, projected_output, state, array_targets, array_outputs
def body(t, output_ta_t: tf.TensorArray, penalty_ta_t: tf.TensorArray):
proj_a_t = proj_a_ta.read(t)
proj_b_t = proj_b_ta.read(t)
sequence_lengths_t = sequence_lengths_ta.read(t)
proj_a_t_slice = proj_a_t[:sequence_lengths_t, :]
proj_b_t_slice = proj_b_t[:sequence_lengths_t, :]
energy = tf.tensordot(proj_a_t_slice,
proj_b_t_slice,
axes=[(1, ), (1, )])
# energy = energy*(1-tf.eye(sequence_lengths_t)) # mask diagonal
# edges = tf.nn.sigmoid(energy)
# edges = edges * (1 - tf.eye(sequence_lengths_t)) # mask diagonal
edges = tf.nn.softmax(energy, axis=-1)
# exp_edges = edges
# for _ in range(params.series_depth):
# exp_edges = tf.matmul(exp_edges, edges)
exp_edges = tf.linalg.expm(input=edges)
# penalty_t = tf.maximum(tf.trace(exp_edges) - tf.cast(sequence_lengths_t, tf.float32), 0)
penalty_t = tf.trace(exp_edges) - tf.cast(sequence_lengths_t,
tf.float32)
# penalty_t = tf.reduce_sum(tf.maximum(tf.trace(exp_edges) - tf.cast(sequence_lengths_t, tf.float32), 0))
length_diff = L - sequence_lengths_t
edges_padded = tf.pad(tensor=edges,
paddings=[(0, length_diff), (0, length_diff)])
output_ta_t1 = output_ta_t.write(value=edges_padded, index=t)
penalty_ta_t1 = penalty_ta_t.write(value=penalty_t, index=t)
return t + 1, output_ta_t1, penalty_ta_t1
def loop_fn(time, prev_state, array_targets: tf.TensorArray,
array_outputs: tf.TensorArray):
"""
Main rnn loop
:param time: Day number
:param prev_state: RNN state tensor from previous step
:param array_targets: Predictions, each step will append new value to this array
:param array_outputs: Raw RNN outputs (for regularization losses)
:return:
"""
# RNN inputs for current step
features = inputs_by_time[time]
# [batch, train_window, readout_depth * n_heads] -> [batch, readout_depth * n_heads]
# Append previous predicted value to input features
next_input = tf.concat([features, self.vm_id], axis=1)
# Run RNN cell
output, state = cell(next_input, prev_state)
# Make prediction from RNN outputs
projected_output = project_output(output)
# Append step results to the buffer arrays
if return_raw_outputs:
array_outputs = array_outputs.write(time, output)
array_targets = array_targets.write(time, projected_output)
# Increment time and return
return time + 1, state, array_targets, array_outputs
def loop_fn(time, prev_output, prev_state,
array_targets: tf.TensorArray,
array_outputs: tf.TensorArray):
"""
Main decoder loop
:param time: Day number
:param prev_output: Output(prediction) from previous step
:param prev_state: RNN state tensor from previous step
:param array_targets: Predictions, each step will append new value to this array
:param array_outputs: Raw RNN outputs (for regularization losses)
:return:
"""
# RNN inputs for current step
features = inputs_by_time[time]
# [batch, predict_window, readout_depth * n_heads] -> [batch, readout_depth * n_heads]
if attn_features is not None:
# [batch_size, 1] + [batch_size, input_depth]
attn = attn_features[:, time, :]
# Append previous predicted value + attention vector to input features
next_input = tf.concat([prev_output, features, attn], axis=1)
else:
# Append previous predicted value to input features
next_input = tf.concat([prev_output, features], axis=1)
# Run RNN cell
output, state = cell(next_input, prev_state)
# Make prediction from RNN outputs
projected_output = project_output(output)
# Append step results to the buffer arrays
if return_raw_outputs:
array_outputs = array_outputs.write(time, output)
array_targets = array_targets.write(time, projected_output)
# Increment time and return
return time + 1, projected_output, state, array_targets, array_outputs
def loop_fn(time, prev_output, prev_state, array_targets: tf.TensorArray, array_outputs: tf.TensorArray):
next_input = prev_output
output, state = cell(next_input, prev_state)
projected_output = project_fn(output)
array_outputs = array_outputs.write(time, output)
array_targets = array_targets.write(time, projected_output)
return time + 1, projected_output, state, array_targets, array_outputs
def loop_fn_train(time, prev_output, prev_state,
array_targets: tf.TensorArray,
array_outputs: tf.TensorArray):
next_input = tf.reshape(previous_y[:, time], (-1, 3))
output, state = decoder_cell(next_input, prev_state)
projected_output = project_fn(output)
array_outputs = array_outputs.write(time, output)
array_targets = array_targets.write(time, projected_output)
return time + 1, projected_output, state, array_targets, array_outputs
def loop_fn_inference(time, prev_output, prev_state,
array_targets: tf.TensorArray,
array_outputs: tf.TensorArray):
next_input = prev_output
output, state = decoder_cell(next_input, prev_state)
projected_output = project_fn(output)
array_outputs = array_outputs.write(time, output)
array_targets = array_targets.write(time, projected_output)
return time + 1, tf.nn.softmax(
projected_output), state, array_targets, array_outputs
def loop_fn(time, prev_output, prev_state, array_targets: tf.TensorArray, array_outputs: tf.TensorArray):
"""
Main decoder loop.
Args:
time: time series step number.
prev_output: Output(prediction) from previous step.
prev_state: RNN state tensor from previous step.
array_targets: Predictions, each step will append new value to
this array.
array_outputs: Raw RNN outputs (for regularization losses)
Returns:
(time + 1, projected_output, state, array_targets, array_outputs)
projected_output: the prediction for this step.
state: the updated state for this step.
array_targets: the updated targets array.
array_outputs: the updated hidden states array.
"""
# RNN inputs for current step
features = inputs_by_time[time]
# [batch, predict_window, readout_depth * n_heads] -> [batch, readout_depth * n_heads]
# Append previous predicted value to input features
next_input = tf.concat([prev_output, features], axis=1)
# Run RNN cell
output, state = cell(next_input, prev_state)
# Make prediction from RNN outputs
projected_output = project_output(output)
# Append step results to the buffer arrays
array_targets = array_targets.write(time, projected_output)
# Increment time and return
return time + 1, projected_output, state, array_targets, array_outputs
def body_fn(time, all_outputs: tf.TensorArray, inputs, state: LSTMStateTuple):
with tf.variable_scope("body_fn"):
if not use_generated_inputs:
next_inputs = inputs
inputs = inputs[:, time, :]
# context: (batch, feature_size)
# alpha: (batch, position_num)
context, alpha = self._attention_layer(features, state.h)
# todo: alpha regularization
if self.selector:
with tf.variable_scope("selector"):
beta = fully_connected(state.h,
num_outputs=1,
activation_fn=tf.nn.sigmoid,
weights_initializer=self.weight_initializer,
biases_initializer=self.const_initializer)
context = tf.multiply(beta, context, name="selected_context")
# decoder_input: (batch, embedding_size + feature_size)
decoder_input = tf.concat(values=[inputs, context], axis=1, name="decoder_input")
output, nxt_state = rnn_cell(decoder_input, state=state)
logits = self._decode_rnn_outputs(output, context, inputs, dropout=dropout)
all_outputs = all_outputs.write(time, logits)
if use_generated_inputs:
# todo: if hard attention is used, the policy gradient should be the distribution log likelihood:
# which means, we need to cache the sampled logit and then calc gradient of it with respect to weights.
next_inputs = self._word_embedding(self._sampler(logits), reuse=True)
return time + 1, all_outputs, next_inputs, nxt_state
def _make_pairs(index: int, x: tf.TensorArray) -> Tuple[int, tf.TensorArray]:
"""Make (target, context) pairs for a given target word.
Returns the next iteration index (variable), and a TensorArray
containing the output values.
Args:
index: The index of the target word in the sequence tensor.
x: Collection holding the output values.
"""
if randomly_offset:
shift = tf.random.uniform((), maxval=window_size, dtype=tf.int32)
else:
shift = 0
# Calculate indices of context words to the left and right of the target.
left = tf.range(tf.maximum(0, index - window_size + shift), index)
right = tf.range(index + 1, tf.minimum(n, index + 1 + window_size - shift))
# Concatenate left and right tensors
contexts = tf.concat([left, right], axis=0)
contexts = tf.gather(sequence, contexts)
# Create (target, context) pairs
targets = tf.fill(tf.shape(contexts), sequence[index])
pairs = tf.stack([targets, contexts], axis=1)
# Output values
return index + 1, x.write(index, pairs)
def _peel_element_from_iblt(
self, iblt: tf.Tensor, iblt_values: tf.Tensor,
out_strings: tf.TensorArray, out_counts: tf.TensorArray,
out_tensor_values: tf.TensorArray
) -> Tuple[tf.Tensor, tf.Tensor, tf.TensorArray, tf.TensorArray,
tf.TensorArray]:
"""Peels an element from IBLT and adds new peelable elements to queue."""
repetition, index = self.q.dequeue()
iblt, hash_indices, data_string, count = self._decode_and_remove(
iblt, repetition, index)
tensor_value = self._decode_value(iblt_values, repetition, index)
iblt_values = self._remove_value(iblt_values, hash_indices, tensor_value)
if tf.strings.length(data_string) > 0:
index = out_counts.size()
out_counts = out_counts.write(index, count)
out_strings = out_strings.write(index, data_string)
out_tensor_values = out_tensor_values.write(index, tensor_value)
for r in tf.range(self.repetitions, dtype=self._dtype):
if self._is_peelable(iblt, r, hash_indices[r]):
self.q.enqueue((r, hash_indices[r]))
return iblt, iblt_values, out_strings, out_counts, out_tensor_values
def body(i, arr: tf.TensorArray):
a_t = a_ta.read(i)
#a_n = tf.norm(a_t, axis=-1, ord=2)
b_t = b_ta.read(i)
b_lengths_t = b_lengths_ta.read(i)
b_part = b_t[:b_lengths_t, :] # (bl, d)
b_n = tf.norm(b_part, axis=-1, ord=2)
energy = tf.tensordot(a_t, b_part, axes=[(1, ), (1, )]) # (al, bl)
energy = energy / tf.expand_dims(b_n, 0)
attn = tf.nn.softmax(energy, axis=-1)
pattn = tf.pad(attn, [[0, 0], [0, bl - b_lengths_t]])
return i + tf.constant(1, dtype=tf.int32), arr.write(i, pattn)
def body(step, batch_states_ta: tf.TensorArray,
batch_outputs_ta: tf.TensorArray,
batch_outputs_counts_ta: tf.TensorArray,
batch_step_counts_ta: tf.TensorArray):
with tf.variable_scope("initial_hidden_vector"):
current_initial_hidden_vector_input = tf.gather(
initial_hidden_vector_input,
step,
name="current_initial_hidden_vector_input")
current_hidden_vector = self.__create_fully_connected_layers(
current_initial_hidden_vector_input,
[self.hidden_vector_size])
with tf.variable_scope("step_while_loop"):
current_step_count = tf.gather(truth_padded_data.step_counts,
step,
name="current_step_count")
current_outputs_padded = tf.gather(
truth_padded_data.outputs_padded,
step,
name="current_outputs_padded")
current_outputs_counts = tf.gather(
truth_padded_data.outputs_counts,
step,
name="current_outputs_counts")
current_states, current_outputs, current_outputs_counts, current_step_count = \
self.__step_while_loop(
current_step_count,
current_outputs_padded,
current_outputs_counts,
current_hidden_vector)
return \
step + 1, \
batch_states_ta.write(step, current_states, "write_batch_states"), \
batch_outputs_ta.write(step, current_outputs, "write_batch_outputs"), \
batch_outputs_counts_ta.write(step, current_outputs_counts, "write_batch_outputs_counts"), \
batch_step_counts_ta.write(step, current_step_count, "write_step_counts")
def loop_fn(time, prev_output, prev_state,
array_targets: tf.TensorArray,
array_outputs: tf.TensorArray):
"""
Main decoder loop
:param time: Day number
:param prev_output: Output(prediction) from previous step (?,1)
:param prev_state: RNN state tensor from previous step (1,?,267)
:param array_targets: Predictions, each step will append new value to this array 预测结果
:param array_outputs: Raw RNN outputs (for regularization losses) 原始解码输出
:return:
"""
# RNN inputs for current step (?,23)
features = inputs_by_time[time]
# [batch, predict_window, readout_depth * n_heads] -> [batch, readout_depth * n_heads]
# 根据上次预测结果加工成输入特征
if attn_features is not None:
# (?,63,64)--(?,64)
attn = attn_features[:, time, :]
# Append previous predicted value + attention vector to input features (?,1)+(?,23)+(?,64)==(?,88)
next_input = tf.concat([prev_output, features, attn], axis=1)
else:
# Append previous predicted value to input features (?,24)
next_input = tf.concat([prev_output, features], axis=1)
# Run RNN cell (?,88)-(?,267)
with tf.variable_scope('decoder_12'):
output, state = cell(next_input, prev_state)
# Make prediction from RNN outputs (?,1)
projected_output = project_output(output)
# Append step results to the buffer arrays
if return_raw_outputs:
array_outputs = array_outputs.write(time, output)
array_targets = array_targets.write(time, projected_output)
# Increment time and return
# output-转成预测值，state,[预测值]，[output]
return time + 1, projected_output, state, array_targets, array_outputs
def skip_gram_pairs_from_word(
word_index: int,
skip_grams_array: tf.TensorArray) -> Tuple[int, tf.TensorArray]:
"""
Helper method for generating skip-gram target/context pairs from a single word integer.
Parameters
----------
word_index : int
Word integer representation of word.
skip_grams_array : tf.TensorArray
TensorArray containing generated skip-gram target/context pairs.
Returns
-------
next_word_index : int
Next word_index to generate from.
next_skip_grams_array : tf.TensorArray
TensorArray containing newly generated skip-gram target/context pairs.
"""
# Get word integer
word_int = word_indices[word_index]
# Randomly sample window size
window_size = tf.random.uniform([],
minval=1,
maxval=max_window_size + 1,
dtype=tf.int32)
# Generate positive samples
left = tf.range(tf.maximum(word_index - window_size, 0), word_index)
right = tf.range(
word_index + 1,
tf.minimum(word_index + 1 + window_size, tf.size(word_indices)),
)
context_indices = tf.concat([left, right], axis=0)
context_word_indices = tf.gather(word_indices, context_indices)
positive_samples = tf.stack(
[
tf.fill(tf.shape(context_word_indices), word_int),
context_word_indices
],
axis=1,
)
return word_index + 1, skip_grams_array.write(word_index,
positive_samples)
def body(i, arr: tf.TensorArray):
energy_t = energy_ta.read(i)
b_lengths_t = b_lengths_ta.read(i)
attn = tf.nn.softmax(energy_t[:, :b_lengths_t], axis=-1)
pattn = tf.pad(attn, [[0, 0], [0, bl - b_lengths_t]])
return i + tf.constant(1, dtype=tf.int32), arr.write(i, pattn)
def loop_body(i, array: tf.TensorArray):
step_output = self.single_convolution(inputs, i)
array = array.write(i, step_output)
i += 1
return i, array
def body(step: int, stack_1, stack_2, states_ta: tf.TensorArray,
outputs_ta: tf.TensorArray, outputs_counts_ta: tf.TensorArray,
return_value: tf.Tensor):
# stack_1 = tf.Print(stack_1, [step, tf.slice(stack_1, [0, 1], [-1, 2])], "stack: ", summarize=100)
# Rebuild `stack` tuple
# TODO: Find way to avoid this by putting tuples into `tf.while_loop` arguments
stack = stack_1, stack_2
# Get the state and hidden vector we have to deal with for this iteration
state, hidden_vector, stack = state_stack.pop(stack)
# Get the summary for all hidden vectors excluding this one
hidden_vector_summary = state_stack.get_hidden_vector_summary(
stack)
# Get the number of outputs for padding
# TODO: Doing this twice, once here, and once when calling create_guess_layers. Fix this
num_outputs = tf.case(pred_fn_pairs=[
(tf.equal(state, i), lambda i=i: tf.constant(
self.object_type.get_all_states()[i].num_outputs))
for i in range(len(self.object_type.get_all_states()))
],
default=lambda: tf.constant(0))
# Call `create_guess_layers(...)` depending on what state we're in
next_hidden_vector, current_choice = tf.case(
pred_fn_pairs=[
(tf.equal(state, i), lambda i=i: create_guess_layers(
hidden_vector_summary, return_value, hidden_vector, i))
for i in range(len(self.object_type.get_all_states()))
],
default=lambda: (hidden_vector, tf.constant(
0, dtype=tf.float32) / tf.constant(0, tf.float32)))
# Zero pad the current choice
current_choice = tf.concat([
current_choice,
tf.zeros([self.max_outputs - tf.shape(current_choice)[0]])
],
axis=0,
name="current_choice_zero_padded")
# Reshape the hidden vector so we know what size it is
next_hidden_vector = tf.reshape(next_hidden_vector,
[self.hidden_vector_size],
name="next_hidden_vector_reshaped")
if self.training:
# If we're training, the choice we send to the update_state_stack_fn should be determined by the truth
stack_update_choice = tf.gather(truth_outputs_padded,
step,
name="choice_from_input")
else:
# Otherwise, the choice should be what we outputted
stack_update_choice = current_choice
# stack = (tf.Print(stack[0], [step, stack[0][:stack_2]], "estack: ", summarize=100), stack[1])
# Update the state stack
stack, return_value = self.__update_state_stack(
state, stack, next_hidden_vector, stack_update_choice)
return \
step + 1, \
(*stack), \
states_ta.write(step, state, "write_state"), \
outputs_ta.write(step, current_choice, "write_outputs"), \
outputs_counts_ta.write(step, num_outputs, "write_outputs_count"), \
return_value