def loop_fn(time, prev_output, prev_state, array_targets: tf.TensorArray, array_outputs: tf.TensorArray): """ Main decoder loop :param time: Day number :param prev_output: Output(prediction) from previous step :param prev_state: RNN state tensor from previous step :param array_targets: Predictions, each step will append new value to this array :param array_outputs: Raw RNN outputs (for regularization losses) :return: """ # RNN inputs for current step features = inputs_by_time[time] # [batch, predict_window, readout_depth * n_heads] -> [batch, readout_depth * n_heads] if attn_features is not None: # [batch_size, 1] + [batch_size, input_depth] attn = attn_features[:, time, :] # Append previous predicted value + attention vector to input features next_input = tf.concat([prev_output, features, attn], axis=1) else: # Append previous predicted value to input features next_input = tf.concat([prev_output, features], axis=1) # Run RNN cell output, state = cell(next_input, prev_state) # Make prediction from RNN outputs projected_output = project_output(output) # Append step results to the buffer arrays if return_raw_outputs: array_outputs = array_outputs.write(time, output) array_targets = array_targets.write(time, projected_output) # Increment time and return return time + 1, projected_output, state, array_targets, array_outputs
def body(step: int, current_input: tf.Tensor, previous_states: tf.Tensor, outputs: tf.TensorArray): current_states_list = [] for i, cell in enumerate(cells): with tf.variable_scope("rnn_cell_%d" % i): cell_previous_hidden_vector = tf.squeeze(tf.slice( previous_states, [0, i, 0], [-1, 1, -1]), axis=[1]) output, state = cell(current_input, cell_previous_hidden_vector) # Set current_input to output for next cell iteration current_input = output current_states_list.append(state) current_states = tf.concat( [tf.expand_dims(state, axis=1) for state in current_states_list], axis=1) with tf.variable_scope("fully_connected_output"): final_output = tf.contrib.layers.fully_connected( current_input, num_outputs=1, activation_fn=tf.nn.sigmoid) outputs.write(step, final_output) return step + 1, current_input, current_states, outputs
def loop_fn(time, prev_output, prev_state, array_targets: tf.TensorArray, array_outputs: tf.TensorArray): """ Main decoder loop :param time: Step number :param prev_output: Output(prediction) from previous step :param prev_state: RNN state tensor from previous step :param array_targets: Predictions, each step will append new value to this array :param array_outputs: Raw RNN outputs (for regularization losses) :return: """ # Append previous predicted value to input features next_input = prev_output # Run RNN cell output, state = cell(next_input, prev_state) # Make prediction from RNN outputs projected_output = project_output(output) # Append step results to the buffer arrays if return_raw_outputs: array_outputs = array_outputs.write(time, output) array_targets = array_targets.write(time, projected_output) # Increment time and return return time + 1, projected_output, state, array_targets, array_outputs
def body(t, output_ta_t: tf.TensorArray, penalty_ta_t: tf.TensorArray): proj_a_t = proj_a_ta.read(t) proj_b_t = proj_b_ta.read(t) sequence_lengths_t = sequence_lengths_ta.read(t) proj_a_t_slice = proj_a_t[:sequence_lengths_t, :] proj_b_t_slice = proj_b_t[:sequence_lengths_t, :] energy = tf.tensordot(proj_a_t_slice, proj_b_t_slice, axes=[(1, ), (1, )]) # energy = energy*(1-tf.eye(sequence_lengths_t)) # mask diagonal # edges = tf.nn.sigmoid(energy) # edges = edges * (1 - tf.eye(sequence_lengths_t)) # mask diagonal edges = tf.nn.softmax(energy, axis=-1) # exp_edges = edges # for _ in range(params.series_depth): # exp_edges = tf.matmul(exp_edges, edges) exp_edges = tf.linalg.expm(input=edges) # penalty_t = tf.maximum(tf.trace(exp_edges) - tf.cast(sequence_lengths_t, tf.float32), 0) penalty_t = tf.trace(exp_edges) - tf.cast(sequence_lengths_t, tf.float32) # penalty_t = tf.reduce_sum(tf.maximum(tf.trace(exp_edges) - tf.cast(sequence_lengths_t, tf.float32), 0)) length_diff = L - sequence_lengths_t edges_padded = tf.pad(tensor=edges, paddings=[(0, length_diff), (0, length_diff)]) output_ta_t1 = output_ta_t.write(value=edges_padded, index=t) penalty_ta_t1 = penalty_ta_t.write(value=penalty_t, index=t) return t + 1, output_ta_t1, penalty_ta_t1
def loop_fn(time, prev_state, array_targets: tf.TensorArray, array_outputs: tf.TensorArray): """ Main rnn loop :param time: Day number :param prev_state: RNN state tensor from previous step :param array_targets: Predictions, each step will append new value to this array :param array_outputs: Raw RNN outputs (for regularization losses) :return: """ # RNN inputs for current step features = inputs_by_time[time] # [batch, train_window, readout_depth * n_heads] -> [batch, readout_depth * n_heads] # Append previous predicted value to input features next_input = tf.concat([features, self.vm_id], axis=1) # Run RNN cell output, state = cell(next_input, prev_state) # Make prediction from RNN outputs projected_output = project_output(output) # Append step results to the buffer arrays if return_raw_outputs: array_outputs = array_outputs.write(time, output) array_targets = array_targets.write(time, projected_output) # Increment time and return return time + 1, state, array_targets, array_outputs
def loop_fn(time, prev_output, prev_state, array_targets: tf.TensorArray, array_outputs: tf.TensorArray): """ Main decoder loop :param time: Day number :param prev_output: Output(prediction) from previous step :param prev_state: RNN state tensor from previous step :param array_targets: Predictions, each step will append new value to this array :param array_outputs: Raw RNN outputs (for regularization losses) :return: """ # RNN inputs for current step features = inputs_by_time[time] # [batch, predict_window, readout_depth * n_heads] -> [batch, readout_depth * n_heads] if attn_features is not None: # [batch_size, 1] + [batch_size, input_depth] attn = attn_features[:, time, :] # Append previous predicted value + attention vector to input features next_input = tf.concat([prev_output, features, attn], axis=1) else: # Append previous predicted value to input features next_input = tf.concat([prev_output, features], axis=1) # Run RNN cell output, state = cell(next_input, prev_state) # Make prediction from RNN outputs projected_output = project_output(output) # Append step results to the buffer arrays if return_raw_outputs: array_outputs = array_outputs.write(time, output) array_targets = array_targets.write(time, projected_output) # Increment time and return return time + 1, projected_output, state, array_targets, array_outputs
def loop_fn(time, prev_output, prev_state, array_targets: tf.TensorArray, array_outputs: tf.TensorArray): next_input = prev_output output, state = cell(next_input, prev_state) projected_output = project_fn(output) array_outputs = array_outputs.write(time, output) array_targets = array_targets.write(time, projected_output) return time + 1, projected_output, state, array_targets, array_outputs
def loop_fn_train(time, prev_output, prev_state, array_targets: tf.TensorArray, array_outputs: tf.TensorArray): next_input = tf.reshape(previous_y[:, time], (-1, 3)) output, state = decoder_cell(next_input, prev_state) projected_output = project_fn(output) array_outputs = array_outputs.write(time, output) array_targets = array_targets.write(time, projected_output) return time + 1, projected_output, state, array_targets, array_outputs
def loop_fn_inference(time, prev_output, prev_state, array_targets: tf.TensorArray, array_outputs: tf.TensorArray): next_input = prev_output output, state = decoder_cell(next_input, prev_state) projected_output = project_fn(output) array_outputs = array_outputs.write(time, output) array_targets = array_targets.write(time, projected_output) return time + 1, tf.nn.softmax( projected_output), state, array_targets, array_outputs
def loop_fn(time, prev_output, prev_state, array_targets: tf.TensorArray, array_outputs: tf.TensorArray): """ Main decoder loop. Args: time: time series step number. prev_output: Output(prediction) from previous step. prev_state: RNN state tensor from previous step. array_targets: Predictions, each step will append new value to this array. array_outputs: Raw RNN outputs (for regularization losses) Returns: (time + 1, projected_output, state, array_targets, array_outputs) projected_output: the prediction for this step. state: the updated state for this step. array_targets: the updated targets array. array_outputs: the updated hidden states array. """ # RNN inputs for current step features = inputs_by_time[time] # [batch, predict_window, readout_depth * n_heads] -> [batch, readout_depth * n_heads] # Append previous predicted value to input features next_input = tf.concat([prev_output, features], axis=1) # Run RNN cell output, state = cell(next_input, prev_state) # Make prediction from RNN outputs projected_output = project_output(output) # Append step results to the buffer arrays array_targets = array_targets.write(time, projected_output) # Increment time and return return time + 1, projected_output, state, array_targets, array_outputs
def body_fn(time, all_outputs: tf.TensorArray, inputs, state: LSTMStateTuple): with tf.variable_scope("body_fn"): if not use_generated_inputs: next_inputs = inputs inputs = inputs[:, time, :] # context: (batch, feature_size) # alpha: (batch, position_num) context, alpha = self._attention_layer(features, state.h) # todo: alpha regularization if self.selector: with tf.variable_scope("selector"): beta = fully_connected(state.h, num_outputs=1, activation_fn=tf.nn.sigmoid, weights_initializer=self.weight_initializer, biases_initializer=self.const_initializer) context = tf.multiply(beta, context, name="selected_context") # decoder_input: (batch, embedding_size + feature_size) decoder_input = tf.concat(values=[inputs, context], axis=1, name="decoder_input") output, nxt_state = rnn_cell(decoder_input, state=state) logits = self._decode_rnn_outputs(output, context, inputs, dropout=dropout) all_outputs = all_outputs.write(time, logits) if use_generated_inputs: # todo: if hard attention is used, the policy gradient should be the distribution log likelihood: # which means, we need to cache the sampled logit and then calc gradient of it with respect to weights. next_inputs = self._word_embedding(self._sampler(logits), reuse=True) return time + 1, all_outputs, next_inputs, nxt_state
def _make_pairs(index: int, x: tf.TensorArray) -> Tuple[int, tf.TensorArray]: """Make (target, context) pairs for a given target word. Returns the next iteration index (variable), and a TensorArray containing the output values. Args: index: The index of the target word in the sequence tensor. x: Collection holding the output values. """ if randomly_offset: shift = tf.random.uniform((), maxval=window_size, dtype=tf.int32) else: shift = 0 # Calculate indices of context words to the left and right of the target. left = tf.range(tf.maximum(0, index - window_size + shift), index) right = tf.range(index + 1, tf.minimum(n, index + 1 + window_size - shift)) # Concatenate left and right tensors contexts = tf.concat([left, right], axis=0) contexts = tf.gather(sequence, contexts) # Create (target, context) pairs targets = tf.fill(tf.shape(contexts), sequence[index]) pairs = tf.stack([targets, contexts], axis=1) # Output values return index + 1, x.write(index, pairs)
def _peel_element_from_iblt( self, iblt: tf.Tensor, iblt_values: tf.Tensor, out_strings: tf.TensorArray, out_counts: tf.TensorArray, out_tensor_values: tf.TensorArray ) -> Tuple[tf.Tensor, tf.Tensor, tf.TensorArray, tf.TensorArray, tf.TensorArray]: """Peels an element from IBLT and adds new peelable elements to queue.""" repetition, index = self.q.dequeue() iblt, hash_indices, data_string, count = self._decode_and_remove( iblt, repetition, index) tensor_value = self._decode_value(iblt_values, repetition, index) iblt_values = self._remove_value(iblt_values, hash_indices, tensor_value) if tf.strings.length(data_string) > 0: index = out_counts.size() out_counts = out_counts.write(index, count) out_strings = out_strings.write(index, data_string) out_tensor_values = out_tensor_values.write(index, tensor_value) for r in tf.range(self.repetitions, dtype=self._dtype): if self._is_peelable(iblt, r, hash_indices[r]): self.q.enqueue((r, hash_indices[r])) return iblt, iblt_values, out_strings, out_counts, out_tensor_values
def body(i, arr: tf.TensorArray): a_t = a_ta.read(i) #a_n = tf.norm(a_t, axis=-1, ord=2) b_t = b_ta.read(i) b_lengths_t = b_lengths_ta.read(i) b_part = b_t[:b_lengths_t, :] # (bl, d) b_n = tf.norm(b_part, axis=-1, ord=2) energy = tf.tensordot(a_t, b_part, axes=[(1, ), (1, )]) # (al, bl) energy = energy / tf.expand_dims(b_n, 0) attn = tf.nn.softmax(energy, axis=-1) pattn = tf.pad(attn, [[0, 0], [0, bl - b_lengths_t]]) return i + tf.constant(1, dtype=tf.int32), arr.write(i, pattn)
def body(step, batch_states_ta: tf.TensorArray, batch_outputs_ta: tf.TensorArray, batch_outputs_counts_ta: tf.TensorArray, batch_step_counts_ta: tf.TensorArray): with tf.variable_scope("initial_hidden_vector"): current_initial_hidden_vector_input = tf.gather( initial_hidden_vector_input, step, name="current_initial_hidden_vector_input") current_hidden_vector = self.__create_fully_connected_layers( current_initial_hidden_vector_input, [self.hidden_vector_size]) with tf.variable_scope("step_while_loop"): current_step_count = tf.gather(truth_padded_data.step_counts, step, name="current_step_count") current_outputs_padded = tf.gather( truth_padded_data.outputs_padded, step, name="current_outputs_padded") current_outputs_counts = tf.gather( truth_padded_data.outputs_counts, step, name="current_outputs_counts") current_states, current_outputs, current_outputs_counts, current_step_count = \ self.__step_while_loop( current_step_count, current_outputs_padded, current_outputs_counts, current_hidden_vector) return \ step + 1, \ batch_states_ta.write(step, current_states, "write_batch_states"), \ batch_outputs_ta.write(step, current_outputs, "write_batch_outputs"), \ batch_outputs_counts_ta.write(step, current_outputs_counts, "write_batch_outputs_counts"), \ batch_step_counts_ta.write(step, current_step_count, "write_step_counts")
def loop_fn(time, prev_output, prev_state, array_targets: tf.TensorArray, array_outputs: tf.TensorArray): """ Main decoder loop :param time: Day number :param prev_output: Output(prediction) from previous step (?,1) :param prev_state: RNN state tensor from previous step (1,?,267) :param array_targets: Predictions, each step will append new value to this array 预测结果 :param array_outputs: Raw RNN outputs (for regularization losses) 原始解码输出 :return: """ # RNN inputs for current step (?,23) features = inputs_by_time[time] # [batch, predict_window, readout_depth * n_heads] -> [batch, readout_depth * n_heads] # 根据上次预测结果加工成输入特征 if attn_features is not None: # (?,63,64)--(?,64) attn = attn_features[:, time, :] # Append previous predicted value + attention vector to input features (?,1)+(?,23)+(?,64)==(?,88) next_input = tf.concat([prev_output, features, attn], axis=1) else: # Append previous predicted value to input features (?,24) next_input = tf.concat([prev_output, features], axis=1) # Run RNN cell (?,88)-(?,267) with tf.variable_scope('decoder_12'): output, state = cell(next_input, prev_state) # Make prediction from RNN outputs (?,1) projected_output = project_output(output) # Append step results to the buffer arrays if return_raw_outputs: array_outputs = array_outputs.write(time, output) array_targets = array_targets.write(time, projected_output) # Increment time and return # output-转成预测值,state,[预测值],[output] return time + 1, projected_output, state, array_targets, array_outputs
def skip_gram_pairs_from_word( word_index: int, skip_grams_array: tf.TensorArray) -> Tuple[int, tf.TensorArray]: """ Helper method for generating skip-gram target/context pairs from a single word integer. Parameters ---------- word_index : int Word integer representation of word. skip_grams_array : tf.TensorArray TensorArray containing generated skip-gram target/context pairs. Returns ------- next_word_index : int Next word_index to generate from. next_skip_grams_array : tf.TensorArray TensorArray containing newly generated skip-gram target/context pairs. """ # Get word integer word_int = word_indices[word_index] # Randomly sample window size window_size = tf.random.uniform([], minval=1, maxval=max_window_size + 1, dtype=tf.int32) # Generate positive samples left = tf.range(tf.maximum(word_index - window_size, 0), word_index) right = tf.range( word_index + 1, tf.minimum(word_index + 1 + window_size, tf.size(word_indices)), ) context_indices = tf.concat([left, right], axis=0) context_word_indices = tf.gather(word_indices, context_indices) positive_samples = tf.stack( [ tf.fill(tf.shape(context_word_indices), word_int), context_word_indices ], axis=1, ) return word_index + 1, skip_grams_array.write(word_index, positive_samples)
def body(i, arr: tf.TensorArray): energy_t = energy_ta.read(i) b_lengths_t = b_lengths_ta.read(i) attn = tf.nn.softmax(energy_t[:, :b_lengths_t], axis=-1) pattn = tf.pad(attn, [[0, 0], [0, bl - b_lengths_t]]) return i + tf.constant(1, dtype=tf.int32), arr.write(i, pattn)
def loop_body(i, array: tf.TensorArray): step_output = self.single_convolution(inputs, i) array = array.write(i, step_output) i += 1 return i, array
def body(step: int, stack_1, stack_2, states_ta: tf.TensorArray, outputs_ta: tf.TensorArray, outputs_counts_ta: tf.TensorArray, return_value: tf.Tensor): # stack_1 = tf.Print(stack_1, [step, tf.slice(stack_1, [0, 1], [-1, 2])], "stack: ", summarize=100) # Rebuild `stack` tuple # TODO: Find way to avoid this by putting tuples into `tf.while_loop` arguments stack = stack_1, stack_2 # Get the state and hidden vector we have to deal with for this iteration state, hidden_vector, stack = state_stack.pop(stack) # Get the summary for all hidden vectors excluding this one hidden_vector_summary = state_stack.get_hidden_vector_summary( stack) # Get the number of outputs for padding # TODO: Doing this twice, once here, and once when calling create_guess_layers. Fix this num_outputs = tf.case(pred_fn_pairs=[ (tf.equal(state, i), lambda i=i: tf.constant( self.object_type.get_all_states()[i].num_outputs)) for i in range(len(self.object_type.get_all_states())) ], default=lambda: tf.constant(0)) # Call `create_guess_layers(...)` depending on what state we're in next_hidden_vector, current_choice = tf.case( pred_fn_pairs=[ (tf.equal(state, i), lambda i=i: create_guess_layers( hidden_vector_summary, return_value, hidden_vector, i)) for i in range(len(self.object_type.get_all_states())) ], default=lambda: (hidden_vector, tf.constant( 0, dtype=tf.float32) / tf.constant(0, tf.float32))) # Zero pad the current choice current_choice = tf.concat([ current_choice, tf.zeros([self.max_outputs - tf.shape(current_choice)[0]]) ], axis=0, name="current_choice_zero_padded") # Reshape the hidden vector so we know what size it is next_hidden_vector = tf.reshape(next_hidden_vector, [self.hidden_vector_size], name="next_hidden_vector_reshaped") if self.training: # If we're training, the choice we send to the update_state_stack_fn should be determined by the truth stack_update_choice = tf.gather(truth_outputs_padded, step, name="choice_from_input") else: # Otherwise, the choice should be what we outputted stack_update_choice = current_choice # stack = (tf.Print(stack[0], [step, stack[0][:stack_2]], "estack: ", summarize=100), stack[1]) # Update the state stack stack, return_value = self.__update_state_stack( state, stack, next_hidden_vector, stack_update_choice) return \ step + 1, \ (*stack), \ states_ta.write(step, state, "write_state"), \ outputs_ta.write(step, current_choice, "write_outputs"), \ outputs_counts_ta.write(step, num_outputs, "write_outputs_count"), \ return_value