def testFallbackErrorNotVisibleWhenFallbackMethodRaises(self): ctx = context.context() ctx.ensure_initialized() try: math_ops.mat_mul([[1., 1.] * 2], [[1., 1.] * 3]) except errors.InvalidArgumentError: etype, value, tb = sys.exc_info() full_exception_text = " ".join( traceback.format_exception(etype, value, tb)) self.assertNotRegex(full_exception_text, "_FallbackException")
def __call__(self, hidden_output, scope=None): # if self.prevcontext is None: # self.prevcontext = prevctx with tf.variable_scope(scope or type(self).__name__): hidden_output = tf.reshape(hidden_output, [1, self.input_size]) e = math_ops.mat_mul(array_ops.concat( [hidden_output, self.prevcontext], 1), self.We2, transpose_b=True) e = math_ops.mat_mul(self._activation(e), self.We1, transpose_b=True) return e
def _test_fully_connected(tensor_in_sizes, filter_in_sizes, bias_in_size=None): """ One iteration of fully connected """ total_size_1 = 1 total_size_2 = 1 for s in tensor_in_sizes: total_size_1 *= s for s in filter_in_sizes: total_size_2 *= s # Initializes the input tensor with array containing incrementing # numbers from 1. data_array = [f * 1.0 for f in range(1, total_size_1 + 1)] filter_array = [f * 1.0 for f in range(1, total_size_2 + 1)] assert int(total_size_1 / tensor_in_sizes[0]) == filter_in_sizes[0], \ "input size and filter size are mismatched" with tf.Graph().as_default(): in_data = array_ops.placeholder(shape=tensor_in_sizes, dtype='float32') in_filter = constant_op.constant(filter_array, shape=filter_in_sizes, dtype='float32') # reshape N H W C into N H*W*C in_data_reshape = array_ops.reshape(in_data, [tensor_in_sizes[0], -1]) out = math_ops.mat_mul(in_data_reshape, in_filter) # if we have bias if bias_in_size: assert bias_in_size[0] == filter_in_sizes[1], "bias and filter size are mismatched" bias_array = [f * 1.0 for f in range(1, bias_in_size[0] + 1)] in_bias = constant_op.constant(bias_array, shape=bias_in_size, dtype='float32') out = nn_ops.bias_add(out, in_bias) data_array = np.reshape(data_array, tensor_in_sizes).astype('float32') compare_tflite_with_tvm(data_array, 'Placeholder:0', [in_data], [out])
def call(self, inputs): @custom_gradient def lr_multiplier(x): y = array_ops.identity(x) def grad(dy): return dy * self.lr_mul return y, grad kernel = lr_multiplier(self.coeff * self.kernel) rank = len(inputs.shape) if rank > 2: # Broadcasting is required for the inputs. outputs = standard_ops.tensordot(inputs, kernel, [[rank - 1], [0]]) # Reshape the output back to the original ndim of the input. if not context.executing_eagerly(): shape = inputs.shape.as_list() output_shape = shape[:-1] + [self.units] outputs.set_shape(output_shape) else: inputs = math_ops.cast(inputs, self._compute_dtype) if K.is_sparse(inputs): outputs = sparse_ops.sparse_tensor_dense_matmul(inputs, kernel) else: outputs = math_ops.mat_mul(inputs, kernel) if self.use_bias: outputs = nn.bias_add(outputs, self.bias) if self.activation is not None: return self.activation(outputs) # pylint: disable=not-callable return outputs
def update_context(self, last_output, useDropout_=False): ctx = math_ops.mat_mul(array_ops.concat( [last_output, self.prevcontext], 1), self._kernel, transpose_b=True) self.context = tf.nn.relu(ctx) if useDropout_: self.context = tf.nn.dropout(self.context, rate=0.5) return self.context
def call(self, inputs, training=None): if self.lr_mul == 1.0: W = self.coeff * self.kernel else: @custom_gradient def lr_multiplier(x): y = array_ops.identity(x) def grad(dy): return dy * self.lr_mul return y, grad W = lr_multiplier(self.coeff * self.kernel) training = self._get_training_value(training) # Update singular vector by power iteration W_T = array_ops.transpose(W) u = array_ops.identity(self.u) for i in range(self.power_iter): v = nn_impl.l2_normalize(math_ops.matmul(u, W)) # 1 x filters u = nn_impl.l2_normalize(math_ops.matmul(v, W_T)) # Spectral Normalization sigma_W = math_ops.matmul(math_ops.matmul(u, W), array_ops.transpose(v)) # Backprop doesn't need in power iteration sigma_W = array_ops.stop_gradient(sigma_W) W_bar = W / array_ops.squeeze(sigma_W) # Assign new singular vector training_value = tf_utils.constant_value(training) if training_value is not False: def u_update(): def true_branch(): return self._assign_singular_vector(self.u, u) def false_branch(): return self.u return tf_utils.smart_cond(training, true_branch, false_branch) self.add_update(u_update) # normal Dense using W_bar inputs = ops.convert_to_tensor(inputs) rank = common_shapes.rank(inputs) if rank > 2: # Broadcasting is required for the inputs. outputs = standard_ops.tensordot(inputs, W_bar, [[rank - 1], [0]]) # Reshape the output back to the original ndim of the input. if not context.executing_eagerly(): shape = inputs.shape.as_list() output_shape = shape[:-1] + [self.units] outputs.set_shape(output_shape) else: inputs = math_ops.cast(inputs, self._compute_dtype) outputs = math_ops.mat_mul(inputs, W_bar) if self.use_bias: outputs = nn.bias_add(outputs, self.bias) if self.activation is not None: return self.activation(outputs) # pylint: disable=not-callable return outputs
def build(self, inputs_shape): if inputs_shape[1].value is None: raise ValueError( "Expected inputs.shape[-1] to be known, saw shape: %s" % inputs_shape) input_depth = inputs_shape[1].value h_depth = self._num_units self._left_matrix = self.add_variable( "left_matrix", shape=[input_depth + h_depth, 64]) self._right_matrix = self.add_variable("right_matrix", shape=[64, 4 * self._num_units]) self._kernel = math_ops.mat_mul(self._left_matrix, self._right_matrix) self._bias = self.add_variable( "bias", shape=[4 * self._num_units], initializer=init_ops.zeros_initializer(dtype=self.dtype))
def dot(x, y): x_dim = ndim(x) y_dim = ndim(y) if x_dim is not None and (x_dim > 2 or y_dim > 2): x_shape = get_shape(x) y_shape = get_shape(y) y_perm = list(range(y_dim)) y_perm = [y_perm.pop(-2)] + y_perm xt = array_ops.reshape(x, [-1, x_shape[-1]]) yt = array_ops.reshape(array_ops.transpose(y, perm=y_perm), [y_shape[-2], -1]) return array_ops.reshape(math_ops.matmul(xt, yt), x_shape[:-1] + y_shape[:-2] + y_shape[-1:]) if is_sparse(x): outputs = sparse_ops.sparse_tensor_dense_matmul(x, y) else: outputs = math_ops.mat_mul(x, y) return outputs
def _test_fully_connected(tensor_in_sizes, filter_in_sizes, bias_in_size=None): """ One iteration of fully connected """ total_size_1 = 1 total_size_2 = 1 for s in tensor_in_sizes: total_size_1 *= s for s in filter_in_sizes: total_size_2 *= s # Initializes the input tensor with array containing incrementing # numbers from 1. data_array = [f * 1.0 for f in range(1, total_size_1 + 1)] filter_array = [f * 1.0 for f in range(1, total_size_2 + 1)] assert int(total_size_1 / tensor_in_sizes[0]) == filter_in_sizes[0], \ "input size and filter size are mismatched" with tf.Graph().as_default(): in_data = array_ops.placeholder(shape=tensor_in_sizes, dtype='float32') in_filter = constant_op.constant(filter_array, shape=filter_in_sizes, dtype='float32') # reshape N H W C into N H*W*C in_data_reshape = array_ops.reshape(in_data, [tensor_in_sizes[0], -1]) out = math_ops.mat_mul(in_data_reshape, in_filter) # if we have bias if bias_in_size: assert bias_in_size[0] == filter_in_sizes[1], "bias and filter size are mismatched" bias_array = [f * 1.0 for f in range(1, bias_in_size[0] + 1)] in_bias = constant_op.constant(bias_array, shape=bias_in_size, dtype='float32') out = nn_ops.bias_add(out, in_bias) tflite_data_array = np.reshape(data_array, tensor_in_sizes).astype('float32') tvm_data_array = np.transpose(tflite_data_array, axes=(0, 3, 1, 2)) compare_tflite_with_tvm(tflite_data_array, tvm_data_array, 'Placeholder:0', [in_data], [out])
def call(self, inputs, state): # User gating # batch_size x num_units item_embedding = inputs[:, :self._num_units] user_embedding = inputs[:, self._num_units:2 * self._num_units] time_embedding = inputs[:, 2 * self._num_units:] # batch_size x num_units user_gate_h = math_ops.matmul(state, self._user_gate_h_weight, name="u_gate_h_matmul") user_gate_i = math_ops.matmul(item_embedding, self._user_gate_i_weight, name="u_gate_i_matmul") user_gate_u = math_ops.matmul(user_embedding, self._user_gate_u_weight, name="u_gate_u_matmul") # batch_size x num_units user_gate_h = nn_ops.bias_add(user_gate_h, self._user_gate_h_bias, name="u_gate_h_bias_add") user_gate_i = nn_ops.bias_add(user_gate_i, self._user_gate_i_bias, name="u_gate_i_bias_add") user_gate_u = nn_ops.bias_add(user_gate_u, self._user_gate_u_bias, name="u_gate_u_bias_add") to_sum_eq1 = [user_gate_h, user_gate_i, user_gate_u] if self._modifying_eq1: user_gate_t = math_ops.matmul(time_embedding, self._user_gate_t_weight, name="u_gate_t_matmul") user_gate_t = nn_ops.bias_add(user_gate_t, self._user_gate_t_bias, name="u_gate_t_bias_add") to_sum_eq1.append(user_gate_t) # batch_size x num_units xi = math_ops.sigmoid(math_ops.add_n(to_sum_eq1), name="xi") # batch_size x num_units one_minus_xi = math_ops.subtract(1., xi, name="one_minus_xi") # Hidden state gating gated_i = math_ops.multiply(one_minus_xi, item_embedding, name="gated_i") gated_u = math_ops.multiply(xi, user_embedding, name="gated_u") # batch_size x 3*num_units to_concat_eq2 = [state, gated_i, gated_u] if self._modifying_eq2: to_concat_eq2.append(time_embedding) concat = tf.concat(to_concat_eq2, 1, name="concat_for_eq_two") # batch_size x 2*num_units u_r = math_ops.mat_mul(concat, self._h_gate_weight, name="matmul_u_r") u_r = math_ops.sigmoid(nn_ops.bias_add(u_r, self._h_gate_bias), name="sigmoid_u_r") # batch_size x num_units u, r = array_ops.split(u_r, num_or_size_splits=2, axis=1, name="split_u_r") one_minus_u = math_ops.subtract(1., u, name="one_minus_u") # Update vector # batch_size x num_units gated_h = math_ops.multiply(r, state, name="gated_h") update_vector_h = math_ops.mat_mul(gated_h, self._update_vector_h_weight, name="update_vector_h_matmul") update_vector_i = math_ops.mat_mul(gated_i, self._update_vector_i_weight, name="update_vector_i_matmul") update_vector_u = math_ops.mat_mul(gated_u, self._update_vector_u_weight, name="update_vector_u_matmul") update_vector_h = nn_ops.bias_add(update_vector_h, self._update_vector_h_bias, name="update_vector_h_bias_add") update_vector_i = nn_ops.bias_add(update_vector_i, self._update_vector_i_bias, name="update_vector_i_bias_add") update_vector_u = nn_ops.bias_add(update_vector_u, self._update_vector_u_bias, name="update_vector_u_bias_add") to_sum_eq3 = [update_vector_h, update_vector_i, update_vector_u] if self._modifying_eq3: update_vector_t = math_ops.mat_mul(time_embedding, self._update_vector_t_weight, name="update_vector_t_matmul") update_vector_t = nn_ops.bias_add(update_vector_t, self._update_vector_t_bias, name="update_vector_t_bias_add") to_sum_eq3.append(update_vector_t) k = math_ops.tanh(math_ops.add_n(to_sum_eq3), name="k") # Update hidden state new_h = math_ops.add(math_ops.multiply(one_minus_u, state), math_ops.multiply(u, k), name="new_h") return new_h, new_h
def loss(x): y = array_ops.reshape(math_ops.mat_mul(x, kernel), []) - array_ops.identity(1.) return y * y
def stlstm_loop(lstm_size, input_data, nb_classes, usePrevGCA=False, previousGCA=None, iters=2, do_norm=False, useDropout=False): """https://github.com/philipperemy/tensorflow-multi-dimensional-lstm/blob/master/md_lstm.py Implements multi dimension LSTM @param lstm_size: the hidden units @param input_data: the data to process of shape [batch,frames,joints,channels] @param scope_n : the scope returns (y,states) - y=[batch,frames,joints,lstm_size[1]] the output of the lstm """ with tf.variable_scope("ST-LSTM", reuse=tf.AUTO_REUSE): # Results list results = [] # Create ST-LTSM cells cell = STLSTMCell(lstm_size[0], input_shape=input_data.get_shape(), initializer=tf.truncated_normal_initializer, name="layer1", do_norm=do_norm) cell2 = STLSTMCell(lstm_size[1], input_shape=tf.TensorShape([lstm_size[0]]), initializer=tf.truncated_normal_initializer, name="layer2", do_norm=do_norm) # Create the GCA cells (one per iteration) gca_cells = [GCACell(lstm_size, ite) for ite in range(1, iters + 1)] # Get the shape of the input (batch_size, x, y, channels) # shape = input_data.get_shape().as_list() shape = tf.shape(input_data) batch_size = shape[0] T_dim = shape[1] S_dim = shape[2] channels = shape[3] # Get the number of features (total number of input values per step) # features = S_dim * channels # The batch size is inferred from the tensor size x = tf.reshape(input_data, [batch_size, T_dim, S_dim, channels]) # Reorder inputs to (t, s, batch_size, features) - t=T_dim, s=S_dim x = tf.transpose(x, [1, 2, 0, 3]) # Reshape to a one dimensional tensor of (t*s*batch_size , features) x = tf.reshape(x, [-1, batch_size, channels]) # Split tensor into t*s tensors of size (batch_size , features) # x = tf.split(axis=0, num_or_size_splits=T_dim*S_dim, value=x) # Create an input tensor array (literally an array of tensors) to use inside the loop inputs_ta = tf.TensorArray(dtype=tf.float32, size=T_dim * S_dim, name='input_array', dynamic_size=True, infer_shape=False) inputs_ta = inputs_ta.unstack(x) # Create a TensorArray for the order of the joints jointsorder_ta = tf.TensorArray(tf.int32, OUT_DIM1, clear_after_read=False) jointsorder_ta = jointsorder_ta.unstack(tf.constant(JOINTS_ORDER)) # Function to get the previous joints id (cs_prev,hs_prev) def get_prevS(t_, w_=1): # return S_dim + tf.mod(t_, S_dim) - tf.constant(w_) # return t_ - tf.constant(w_) return jointsorder_ta.read(tf.mod(t_, OUT_DIM1) - 1) # Function to get the previous time id (ct_prev,ht_prev) def get_prevT(t_, w_=S_dim): # return tf.mod(t_, w_) # - tf.constant(w_) return t_ - w_ def init_context(output_layer1): return tf.reduce_mean(output_layer1.stack(), axis=0) def process_information(id_, e_ta_): gca_ = gca_cells[it - 1] e_ta_ = e_ta_.write(id_, gca_(outputs_ta.read(id_))) return id_ + 1, e_ta_ # Controls the initial index zero = tf.constant(0) e_sum = tf.constant(0) # Body of the while loop operation that applies the MD LSTM def body1(id_, outputs_ta_, states_ta_): # If the current position is less or equal than the width, we are in the first row # so we read the zero state we added before. # If not, get the sample located at a width distance. prevstate_T = tf.cond( tf.less_equal(id_, OUT_DIM1), lambda: states_ta_.read(T_dim * OUT_DIM1 ), # first row = zero state lambda: states_ta_.read(get_prevT(id_, OUT_DIM1)) ) # other rows = previous time id (t-1) # If it is the first step we read the zero state if not we read the inmediate last prevstate_S = tf.cond( tf.less(zero, tf.mod(id_, OUT_DIM1)), lambda: states_ta_. read(get_prevS(id_) ), # get previous joint state id (j-1) from JOINT ORDER lambda: states_ta_.read(T_dim * OUT_DIM1 )) # first joint - get zero state # We build the input state in both dimensions current_state = prevstate_S[0], prevstate_T[0], prevstate_S[ 1], prevstate_T[1] # Now we calculate the hidden state and the new cell state out, state = cell(inputs_ta.read(id_), current_state) # We write the output to the output tensor array outputs_ta_ = outputs_ta_.write(id_, out) # And save the output state to the state tensor array states_ta_ = states_ta_.write(id_, state) # Return outputs and incremented time step return id_ + 1, outputs_ta_, states_ta_ # , outputs_ta2_, states_ta2_ # Body of the while loop operation that applies the MD LSTM def body2(id_, outputs_ta2_, states_ta2_, e_ta_): # Informativeness r = e_ta_.read(id_) / e_sum prevstate_T2 = tf.cond( tf.less_equal(id_, OUT_DIM1), lambda: states_ta2_.read(T_dim * OUT_DIM1 ), # first row = zero state lambda: states_ta2_.read(get_prevT(id_, OUT_DIM1)) ) # other rows = previous time id (t-1) # If it is the first step we read the zero state if not we read the inmediate last prevstate_S2 = tf.cond( tf.less(zero, tf.mod(id_, OUT_DIM1)), lambda: states_ta2_.read(get_prevS( id_)), # get previous joint state id (j-1) lambda: states_ta2_.read(T_dim * OUT_DIM1) ) # first joint - get zero state ! # Process cureent state and then the new state current_state2 = prevstate_S2[0], prevstate_T2[0], prevstate_S2[ 1], prevstate_T2[1] out2, state2 = cell2(outputs_ta.read(id_), current_state2, r) outputs_ta2_ = outputs_ta2_.write(id_, out2) states_ta2_ = states_ta2_.write(id_, state2) # Return outputs and incremented time step return id_ + 1, outputs_ta2_, states_ta2_, e_ta_ # Loop output condition. The index, given by the time, should be less than the # total number of steps defined within the image def condition1(id_, outputs_ta_, states_ta_): # , outputs_ta2_, states_ta2_ return tf.less(id_, T_dim * S_dim) # T_dim * S_dim def condition(id_, e_ta_): # , outputs_ta2_, states_ta2_ return tf.less(id_, T_dim * S_dim) def condition2(id_, outputs_ta2_, states_ta2_, e_ta_): # , outputs_ta2_, states_ta2_ return tf.less(id_, T_dim * S_dim) # Init ST-LSTM1 states and output arrays states_ta = tf.TensorArray(dtype=tf.float32, size=T_dim * OUT_DIM1 + 1, name='state_array_1', clear_after_read=False) outputs_ta = tf.TensorArray(dtype=tf.float32, size=T_dim * OUT_DIM1, name='output_array_1', clear_after_read=False) # initial cell hidden states: last position of the array = LSTMStateTuple filled with zeros states_ta = states_ta.write( T_dim * OUT_DIM1, LSTMStateTuple(tf.zeros([batch_size, lstm_size[0]], tf.float32), tf.zeros([batch_size, lstm_size[0]], tf.float32))) # Loop 1: First ST-LSTM layer index = tf.constant(0) _, outputs_ta, states_ta = tf.while_loop( condition1, body1, [index, outputs_ta, states_ta], parallel_iterations=1) for it in range(1, iters + 1): states_ta2 = tf.TensorArray(dtype=tf.float32, size=T_dim * OUT_DIM1 + 1, name='state_array2', clear_after_read=False) outputs_ta2 = tf.TensorArray(dtype=tf.float32, size=T_dim * OUT_DIM1, name='output_array2') # initial cell hidden states: last position of the array = LSTMStateTuple filled with zeros states_ta2 = states_ta2.write( T_dim * OUT_DIM1, LSTMStateTuple( tf.zeros([batch_size, lstm_size[1]], tf.float32), tf.zeros([batch_size, lstm_size[1]], tf.float32))) # Informativeness tensors # e_ta = tf.TensorArray(tf.float32, T_dim * OUT_DIM1, name='e_it{}'.format(it), clear_after_read=False) e_ta = tf.TensorArray(tf.float32, T_dim * OUT_DIM1, name='e_array', clear_after_read=False) # # Loop 1: First ST-LSTM layer # index = tf.constant(0) # _, outputs_ta, states_ta = tf.while_loop(condition1, body1, [index, outputs_ta, states_ta], # parallel_iterations=1) # Initialize context 0 if it == 1: if usePrevGCA: initial_context = tf.cond( tf.less(tf.constant(0, dtype=tf.int64), tf.count_nonzero(previousGCA)), lambda: previousGCA, lambda: init_context(outputs_ta)) gca_cells[0].set_prevcontext(initial_context) else: gca_cells[0].set_prevcontext(init_context(outputs_ta)) # Process e index = tf.constant(0) _, e_ta = tf.while_loop(condition, process_information, [index, e_ta], parallel_iterations=1) e_sum = tf.reduce_sum(e_ta.stack(), axis=0) # Loop 2: Second ST-LSTM layer index = tf.constant(0) _, outputs_ta2, states_ta2, _ = tf.while_loop( condition2, body2, [index, outputs_ta2, states_ta2, e_ta], parallel_iterations=1) # Update context ctx = gca_cells[it - 1].update_context( outputs_ta2.read(S_dim * T_dim - 1), useDropout) # it += 1 if it < iters: gca_cells[it].prevcontext = ctx # Compute Softmax from context Wc = tf.get_variable("Wc", [nb_classes, lstm_size[0]], tf.float32, tf.truncated_normal_initializer(), trainable=True) y = math_ops.mat_mul(Wc, gca_cells[it - 1].context, transpose_b=True) y = tf.nn.softmax(tf.transpose(y)) results.append(y) # Extract the output tensors from the processesed tensor array # outputs = outputs_ta2.stack() # states = states_ta2.stack() # Reshape outputs to match the shape of the input # y = tf.reshape(outputs, [T_dim, S_dim, batch_size, lstm_size[0]]) # For outputs_ta # states = tf.reshape(states, [T_dim,S_dim,batch_size,2,lstm_size[0]]) # y = tf.reshape(outputs, [T_dim, S_dim, batch_size, lstm_size[1]]) # Reorder te dimensions to match the input # y = tf.transpose(y, [2, 0, 1, 3]) # Global Context gca = gca_cells[-1].context # Return the output and the inner states return results, gca