def testMismatchedShapesExpandNonconcatDim(self): with self.test_session(use_gpu=False) as sess: sp_a = self._SparseTensor_3x3() sp_b = self._SparseTensor_3x5() sp_c = self._SparseTensor_3x2() sp_d = self._SparseTensor_2x3() sp_concat_dim0 = tf.sparse_concat(0, [sp_a, sp_b, sp_c, sp_d], expand_nonconcat_dim=True) sp_concat_dim1 = tf.sparse_concat(1, [sp_a, sp_b, sp_c, sp_d], expand_nonconcat_dim=True) sp_concat_dim0_out = sess.run(sp_concat_dim0) sp_concat_dim1_out = sess.run(sp_concat_dim1) self.assertAllEqual( sp_concat_dim0_out.indices, [[0, 2], [1, 0], [2, 0], [2, 2], [4, 1], [5, 0], [5, 3], [5, 4], [7, 0], [8, 0], [9, 1], [10, 0], [10, 2]]) self.assertAllEqual( sp_concat_dim0_out.values, [1, 2, 3, 4, 1, 2, 1, 0, 1, 2, 1, 1, 2]) self.assertAllEqual( sp_concat_dim0_out.shape, [11, 5]) self.assertAllEqual( sp_concat_dim1_out.indices, [[0, 2], [0, 11], [1, 0], [1, 4], [1, 8], [1, 10], [1, 12], [2, 0], [2, 2], [2, 3], [2, 6], [2, 7], [2, 8]]) self.assertAllEqual( sp_concat_dim1_out.values, [1, 1, 2, 1, 1, 1, 2, 3, 4, 2, 1, 0, 2]) self.assertAllEqual( sp_concat_dim1_out.shape, [3, 13])
def testMismatchedRank(self): with self.test_session(use_gpu=False): sp_a = self._SparseTensor_3x3() sp_e = self._SparseTensor_2x3x4() # Rank mismatches can be caught at shape-inference time with self.assertRaises(ValueError): tf.sparse_concat(1, [sp_a, sp_e])
def testMismatchedRankExpandNonconcatDim(self): with self.test_session(use_gpu=False): sp_a = self._SparseTensor_3x3() sp_e = self._SparseTensor_2x3x4() # Rank mismatches should be caught at shape-inference time, even for # expand_nonconcat_dim=True. with self.assertRaises(ValueError): tf.sparse_concat(1, [sp_a, sp_e], expand_nonconcat_dim=True)
def testSliceConcat(self): with self.test_session(use_gpu=False): sparse_tensors = tf.sparse_split(1, 2, self._SparseTensor_3x4x2()) concat_tensor = tf.sparse_concat(1, sparse_tensors) expected_output = self._SparseTensor_3x4x2() self.assertAllEqual(concat_tensor.indices.eval(), expected_output.indices.eval())
def testSliceConcat(self): for sp_input in (self._SparseTensorValue_3x4x2(), self._SparseTensor_3x4x2()): with self.test_session(use_gpu=False): sparse_tensors = tf.sparse_split(sp_input=sp_input, num_split=2, axis=1) concat_tensor = tf.sparse_concat(1, sparse_tensors) expected_output = self._SparseTensor_3x4x2() self.assertAllEqual(concat_tensor.indices.eval(), expected_output.indices.eval())
def testConcatDim0(self): with self.test_session(use_gpu=False) as sess: # concat(A, D): # [ 1] # [2 ] # [3 4] # [ 1 ] # [1 2] sp_a = self._SparseTensor_3x3() sp_d = self._SparseTensor_2x3() sp_concat = tf.sparse_concat(0, [sp_a, sp_d]) self.assertEqual(sp_concat.indices.get_shape(), [7, 2]) self.assertEqual(sp_concat.values.get_shape(), [7]) self.assertEqual(sp_concat.shape.get_shape(), [2]) concat_out = sess.run(sp_concat) self.assertAllEqual( concat_out.indices, [[0, 2], [1, 0], [2, 0], [2, 2], [3, 1], [4, 0], [4, 2]]) self.assertAllEqual( concat_out.values, np.array([1, 2, 3, 4, 1, 1, 2])) self.assertAllEqual( concat_out.shape, np.array([5, 3]))
def tensors_to_item(self, keys_to_tensors): """Maps the given dictionary of tensors to a concatenated list of bboxes. Args: keys_to_tensors: a mapping of TF-Example keys to parsed tensors. Returns: [time, num_boxes, 4] tensor of bounding box coordinates, in order [y_min, x_min, y_max, x_max]. Whether the tensor is a SparseTensor or a dense Tensor is determined by the return_dense parameter. Empty positions in the sparse tensor are filled with -1.0 values. """ sides = [] for key in self._full_keys: value = keys_to_tensors[key] expanded_dims = tf.concat( [tf.to_int64(tf.shape(value)), tf.constant([1], dtype=tf.int64)], 0) side = tf.sparse_reshape(value, expanded_dims) sides.append(side) bounding_boxes = tf.sparse_concat(2, sides) if self._return_dense: bounding_boxes = tf.sparse_tensor_to_dense( bounding_boxes, default_value=self._default_value) return bounding_boxes
def testMismatchedShapes(self): with self.test_session(use_gpu=False) as sess: sp_a = self._SparseTensor_3x3() sp_b = self._SparseTensor_3x5() sp_c = self._SparseTensor_3x2() sp_d = self._SparseTensor_2x3() sp_concat = tf.sparse_concat(1, [sp_a, sp_b, sp_c, sp_d]) # Shape mismatches can only be caught when the op is run with self.assertRaisesOpError("Input shapes must match"): sess.run(sp_concat)
def testShapeInferenceUnknownShapes(self): with self.test_session(use_gpu=False): sp_inputs = [ self._SparseTensor_UnknownShape(), self._SparseTensor_UnknownShape(val_shape=[3]), self._SparseTensor_UnknownShape(ind_shape=[1, 3]), self._SparseTensor_UnknownShape(shape_shape=[3])] sp_concat = tf.sparse_concat(0, sp_inputs) self.assertEqual(sp_concat.indices.get_shape().as_list(), [None, 3]) self.assertEqual(sp_concat.values.get_shape().as_list(), [None]) self.assertEqual(sp_concat.shape.get_shape(), [3])
def testConcat1(self): with self.test_session(use_gpu=False) as sess: # concat(A): # [ 1] # [2 ] # [3 4] for sp_a in (self._SparseTensorValue_3x3(), self._SparseTensor_3x3()): sp_concat = tf.sparse_concat(1, [sp_a]) self.assertEqual(sp_concat.indices.get_shape(), [4, 2]) self.assertEqual(sp_concat.values.get_shape(), [4]) self.assertEqual(sp_concat.shape.get_shape(), [2]) concat_out = sess.run(sp_concat) self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [2, 0], [2, 2]]) self.assertAllEqual(concat_out.values, [1, 2, 3, 4]) self.assertAllEqual(concat_out.shape, [3, 3])
def testConcatNonNumeric(self): with self.test_session(use_gpu=False) as sess: # concat(A, B): # [ a ] # [b e ] # [c d f g h] sp_a = self._SparseTensor_String3x3() sp_b = self._SparseTensor_String3x5() sp_concat = tf.sparse_concat(1, [sp_a, sp_b]) self.assertEqual(sp_concat.indices.get_shape(), [8, 2]) self.assertEqual(sp_concat.values.get_shape(), [8]) self.assertEqual(sp_concat.shape.get_shape(), [2]) concat_out = sess.run(sp_concat) self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [1, 4], [2, 0], [2, 2], [2, 3], [2, 6], [2, 7]]) self.assertAllEqual(concat_out.values, [b"a", b"b", b"e", b"c", b"d", b"f", b"g", b"h"]) self.assertAllEqual(concat_out.shape, [3, 8])
def testConcat2(self): with self.test_session(use_gpu=False) as sess: # concat(A, B): # [ 1 ] # [2 1 ] # [3 4 2 1 0] for sp_a in (self._SparseTensorValue_3x3(), self._SparseTensor_3x3()): for sp_b in (self._SparseTensorValue_3x5(), self._SparseTensor_3x5()): sp_concat = tf.sparse_concat(1, [sp_a, sp_b]) self.assertEqual(sp_concat.indices.get_shape(), [8, 2]) self.assertEqual(sp_concat.values.get_shape(), [8]) self.assertEqual(sp_concat.shape.get_shape(), [2]) concat_out = sess.run(sp_concat) self.assertAllEqual( concat_out.indices, [[0, 2], [1, 0], [1, 4], [2, 0], [2, 2], [2, 3], [2, 6], [2, 7]] ) self.assertAllEqual(concat_out.values, [1, 2, 1, 3, 4, 2, 1, 0]) self.assertAllEqual(concat_out.shape, [3, 8])
def testConcat1(self): with self.test_session(use_gpu=False) as sess: # concat(A): # [ 1] # [2 ] # [3 4] for sp_a in (self._SparseTensorValue_3x3(), self._SparseTensor_3x3()): # Note that we ignore concat_dim in this case since we short-circuit the # single-input case in python. for concat_dim in (-2000, 1, 2000): sp_concat = tf.sparse_concat(concat_dim, [sp_a]) self.assertEqual(sp_concat.indices.get_shape(), [4, 2]) self.assertEqual(sp_concat.values.get_shape(), [4]) self.assertEqual(sp_concat.dense_shape.get_shape(), [2]) concat_out = sess.run(sp_concat) self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [2, 0], [2, 2]]) self.assertAllEqual(concat_out.values, [1, 2, 3, 4]) self.assertAllEqual(concat_out.shape, [3, 3])
def testConcat3(self): with self.test_session(use_gpu=False) as sess: # concat(A, B, C): # [ 1 ] # [2 1 1 ] # [3 4 2 1 0 2 ] sp_a = self._SparseTensor_3x3() sp_b = self._SparseTensor_3x5() sp_c = self._SparseTensor_3x2() sp_concat = tf.sparse_concat(1, [sp_a, sp_b, sp_c]) self.assertEqual(sp_concat.indices.get_shape(), [10, 2]) self.assertEqual(sp_concat.values.get_shape(), [10]) self.assertEqual(sp_concat.shape.get_shape(), [2]) concat_out = sess.run(sp_concat) self.assertAllEqual( concat_out.indices, [[0, 2], [1, 0], [1, 4], [1, 8], [2, 0], [2, 2], [2, 3], [2, 6], [2, 7], [2, 8]] ) self.assertAllEqual(concat_out.values, [1, 2, 1, 1, 3, 4, 2, 1, 0, 2]) self.assertAllEqual(concat_out.shape, [3, 10])
def __init__( self, cfg: InputFeederCfg, fnames, capacity=10000, min_after_deque=10, shuffle=True, trace=False, ): """ :param cap: queue capacity :param batch_size: output batch size """ self.cfg = cfg self.fnames = fnames self.logger = logging.getLogger(__name__) self._values_ph = tf.placeholder( dtype=tf.int32, shape=[None], name="labels" ) self._values_len_ph = tf.placeholder( dtype=tf.int64, shape=[], name="labels_len" ) self._signal_ph = tf.placeholder( dtype=tf.float32, shape=[None, 1], name="signal" ) self._signal_len_ph = tf.placeholder( dtype=tf.int64, shape=[], name="signal_len" ) self.closing = [] self.queue = tf.PaddingFIFOQueue( capacity=capacity, dtypes=[tf.int32, tf.int64, tf.float32, tf.int64], shapes=[ [None], [], [None, 1], [], ] ) # self.closing.append(self.queue.close()) Closed with queue runners...no idea how&why it works self.summaries = [] self.summaries.append( tf.summary.scalar( "paddingFIFOQueue_input", self.queue.size(), family="queue" ) ) if shuffle: self.shuffle_queue = tf.RandomShuffleQueue( capacity=capacity, dtypes=[tf.int32, tf.int64, tf.float32, tf.int64], min_after_dequeue=min_after_deque, ) self.enq = self.shuffle_queue.enqueue([ self._values_ph, self._values_len_ph, self._signal_ph, self._signal_len_ph ]) num_threads = 4 qr = tf.train.QueueRunner( self.queue, [self.queue.enqueue(self.shuffle_queue.dequeue())] * num_threads ) tf.train.add_queue_runner(qr) self.closing.append( self.shuffle_queue.close(cancel_pending_enqueues=True) ) self.summaries.append( tf.summary.scalar( "randomShuffleQueue_input", self.queue.size(), family="queue" ) ) else: self.enq = self.queue.enqueue([ self._values_ph, self._values_len_ph, self._signal_ph, self._signal_len_ph ]) values_op, values_len_op, signal_op, signal_len_op = self.queue.dequeue_many( cfg.batch_size ) if trace: values_op = tf.Print( values_op, [values_op, tf.shape(values_op)], message="values op" ) values_len_op = tf.Print( values_len_op, [values_len_op, tf.shape(values_len_op)], message="values len op" ) sp = [] for label_idx, label_len in zip( tf.split(values_op, cfg.batch_size), tf.split(values_len_op, cfg.batch_size) ): label_len = tf.squeeze(label_len, axis=0) ind = tf.transpose( tf.stack([ tf.zeros(shape=label_len, dtype=tf.int64), tf.range(label_len, dtype=tf.int64), ]) ) sp.append( tf.SparseTensor( indices=ind, values=tf.squeeze(label_idx, axis=0)[:label_len], dense_shape=tf.stack([1, tf.maximum(label_len, 1)], 0) ) ) # labels: An `int32` `SparseTensor`. # `labels.indices[i, :] == [b, t]` means `labels.values[i]` stores # the id for (batch b, time t). # `labels.values[i]` must take on values in `[0, num_labels)`. # See `core/ops/ctc_ops.cc` for more details. # That's ok implemented self.batch_labels = tf.sparse_concat( axis=0, sp_inputs=sp, expand_nonconcat_dim=True ) self.batch_labels_len = values_len_op self.batch_dense_labels = tf.sparse_to_dense( sparse_indices=self.batch_labels.indices, sparse_values=self.batch_labels.values, output_shape=self.batch_labels.dense_shape, default_value=9, ) self.batch_signal = signal_op if trace: self.batch_signal = tf.Print( self.batch_signal, [ self.batch_dense_labels, tf.shape(self.batch_dense_labels), tf.shape(signal_op) ], "dense labels" ) self.batch_signal_len = signal_len_op
def d_attn_head(seq, out_sz, adj_mat, activation, nb_nodes, in_drop=0.0, coef_drop=0.0, residual=False): with tf.name_scope('sp_attn'): if in_drop != 0.0: seq = tf.nn.dropout(seq, 1.0 - in_drop) seq_fts = tf.layers.conv1d(seq, out_sz, 1, use_bias=False) attn = [] for _ in range(out_sz): # simplest self-attention possible f_1 = tf.layers.conv1d(seq_fts, 1, 1) f_2 = tf.layers.conv1d(seq_fts, 1, 1) f_1 = tf.reshape(f_1, (nb_nodes, 1)) f_2 = tf.reshape(f_2, (nb_nodes, 1)) f_1 = adj_mat * f_1 f_2 = adj_mat * tf.transpose(f_2, [1, 0]) logits = tf.sparse_add(f_1, f_2) lrelu = tf.SparseTensor(indices=logits.indices, values=tf.nn.leaky_relu(logits.values), dense_shape=logits.dense_shape) coefs = tf.sparse_softmax(lrelu) if coef_drop != 0.0: coefs = tf.SparseTensor(indices=coefs.indices, values=tf.nn.dropout( coefs.values, 1.0 - coef_drop), dense_shape=coefs.dense_shape) attn.append(tf.sparse_reshape(coefs, (nb_nodes, nb_nodes, 1))) if in_drop != 0.0: seq_fts = tf.nn.dropout(seq_fts, 1.0 - in_drop) # apply attention # coefs = tf.sparse_reshape(coefs, [nb_nodes, nb_nodes]) # seq_fts = tf.squeeze(seq_fts) # vals = tf.sparse_tensor_dense_matmul(coefs, seq_fts) attn = tf.sparse_concat(2, attn) seq_fts = tf.reshape(seq_fts, (nb_nodes, 1, out_sz)) vals = attn * seq_fts vals = tf.sparse_reduce_sum(vals, axis=1) vals = tf.expand_dims(vals, axis=0) vals.set_shape([1, nb_nodes, out_sz]) ret = tf.contrib.layers.bias_add(vals) # residual connection if residual: if seq.shape[-1] != ret.shape[-1]: ret = ret + conv1d(seq, ret.shape[-1], 1) # activation else: seq_fts = ret + seq return activation(ret) # activation # # dimensional attention head # def d_attn_head(seq, out_sz, adj_mat, activation, nb_nodes, in_drop=0.0, coef_drop=0.0, residual=False): # with tf.name_scope('sp_attn'): # if in_drop != 0.0: # seq = tf.nn.dropout(seq, 1.0 - in_drop) # # seq_fts = tf.layers.conv1d(seq, out_sz, 1, use_bias=False) # # # simplest self-attention possible # f_1 = tf.layers.conv1d(seq_fts, out_sz, 1) # f_2 = tf.layers.conv1d(seq_fts, out_sz, 1) # # f_1 = tf.reshape(f_1, (1, nb_nodes, out_sz)) # f_2 = tf.reshape(f_2, (nb_nodes, 1, out_sz)) # adj_mat = tf.sparse_reshape(adj_mat, (nb_nodes, nb_nodes, 1)) # # logits = adj_mat * f_1 * f_2 # # logits = adj_mat # # lrelu = tf.SparseTensor(indices=logits.indices, # values=tf.exp(tf.nn.leaky_relu(logits.values)), # dense_shape=logits.dense_shape) # s = tf.sparse_reduce_sum(lrelu, axis=1, keep_dims=True) # coefs = lrelu/s # # # drop out # if coef_drop != 0.0: # coefs = tf.SparseTensor(indices=coefs.indices, # values=tf.nn.dropout(coefs.values, 1.0 - coef_drop), # dense_shape=coefs.dense_shape) # if in_drop != 0.0: # seq_fts = tf.nn.dropout(seq_fts, 1.0 - in_drop) # # # As tf.sparse_tensor_dense_matmul expects its arguments to have rank-2, # # here we make an assumption that our input is of batch size 1, and reshape appropriately. # # The method will fail in all other cases! # # coefs = tf.sparse_reshape(coefs, [nb_nodes, nb_nodes]) # seq_fts = tf.reshape(seq_fts, (nb_nodes, 1, out_sz)) # vals = coefs*seq_fts # vals = tf.sparse_reduce_sum(vals, axis=1) # # vals = tf.sparse_tensor_dense_matmul(coefs, seq_fts) # vals = tf.expand_dims(vals, axis=0) # vals.set_shape([1, nb_nodes, out_sz]) # ret = tf.contrib.layers.bias_add(vals) # # # residual connection # if residual: # if seq.shape[-1] != ret.shape[-1]: # ret = ret + conv1d(seq, ret.shape[-1], 1) # activation # else: # seq_fts = ret + seq # # return activation(ret) # activation
def call_sparse(self, layer_input_embeds, target_embeds): # input: [num_edge_labels, n, k, 3], [n, d1] # [num_edge_labels, n, k, d1] node_embeds = self.get_node_embeds_from_inputs(layer_input_embeds, target_embeds) split_node_embeds = tf.sparse_split( sp_input=node_embeds, num_split=self.network_params.num_edge_labels, axis=0) transformed_sparse_messages = [] for edge_idx, split_embed in enumerate(split_node_embeds): reshaped_embed = tf.sparse_reshape( sp_input=split_embed, shape=[-1, self.layer_params.node_embed_size]) transformed_embeds = tf.sparse.matmul( sp_a=reshaped_embed, b=self.edge_weights[edge_idx]) indices = tf.cast(tf.where(tf.not_equal(transformed_embeds, 0)), tf.int64) values = tf.gather_nd(transformed_embeds, indices) dense_shape = tf.shape(transformed_embeds, out_type=tf.int64) sparse_transformed = tf.SparseTensor(indices=indices, values=values, dense_shape=dense_shape) #edge_bias = self.edge_biases[edge_idx] + 1e-4 #tiled_edge_bias = tf.tile(edge_bias, [tf.round(tf.size(values) / tf.size(edge_bias))]) #sparse_bias = tf.SparseTensor(indices=indices, values=tiled_edge_bias, # dense_shape=dense_shape) #sparse_transformed = tf.sparse_add( # sparse_transformed, sparse_bias, thresh=1e-6) sparse_transformed_reshaped = tf.sparse_reshape( sp_input=sparse_transformed, shape=split_embed.dense_shape) transformed_sparse_messages.append(sparse_transformed_reshaped) concat_sparse_messages = tf.sparse_concat( axis=0, sp_inputs=transformed_sparse_messages) summed_to_get_num_incoming_edges = tf.sparse.reduce_sum( sp_input=concat_sparse_messages, axis=[0, 3]) num_nonzero = tf.cast( tf.reduce_any(tf.abs(summed_to_get_num_incoming_edges) > 1e-6, axis=1, keepdims=True), tf.float32) # [n, d1] # change this to divide by the number nonzero rather than reduce mean incoming_messages = tf.sparse.reduce_sum( sp_input=concat_sparse_messages, axis=[0, 2]) / num_nonzero incoming_messages = tf.reshape(incoming_messages, [-1, self.layer_params.node_embed_size]) # [n, d1] output_embeds = self.rnn_cell(incoming_messages, target_embeds)[1] return output_embeds
def _dnn_softmax_fn(features, targets, mode): """Creates the prediction, loss, and train ops. Args: features: A dictionary of tensors keyed by the feature name. targets: A tensor representing the labels (in this case, the ratings on the target movie). mode: The execution mode, as defined in tf.contrib.learn.ModeKeys. Returns: ModelFnOps with the mode, prediction, loss, train_op and output_alternatives a dictionary specifying the output for a classification request during serving. Raises: ValueError: When the wrong evaluation type is specified. """ _ = targets # Unused variable. class_weights = tf.get_variable( name='class_weights', shape=[MOVIE_VOCAB_SIZE, hparams.query_hidden_dims[-1]], initializer=tf.contrib.layers.xavier_initializer()) class_biases = tf.get_variable(name='class_biases', shape=[MOVIE_VOCAB_SIZE], initializer=tf.zeros_initializer()) query_embeddings = _embed_query_features(features, mode=mode) tf.summary.scalar('query_embeddings_zero_fraction', tf.nn.zero_fraction(query_embeddings)) # Create layers for target features. if mode != tf.contrib.learn.ModeKeys.INFER: logits_layer = tf.matmul( query_embeddings, tf.transpose(class_weights)) + class_biases target_one_hot = tf.one_hot( indices=features[CANDIDATE_MOVIE_ID].values, depth=MOVIE_VOCAB_SIZE, on_value=1.0) loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=target_one_hot, logits=logits_layer)) if mode == tf.contrib.learn.ModeKeys.TRAIN: train_op = tf.contrib.layers.optimize_loss( loss=loss, global_step=tf.contrib.framework.get_global_step(), learning_rate=hparams.learning_rate, optimizer=hparams.optimizer) return tf.contrib.learn.ModelFnOps(mode=mode, loss=loss, train_op=train_op) elif mode == tf.contrib.learn.ModeKeys.EVAL: if hparams.eval_type == REGRESSION: raise ValueError( 'eval_type must be RANKING for DNN softmax model.') elif hparams.eval_type == RANKING: predictions = tf.matmul( query_embeddings, tf.transpose(class_weights)) + class_biases if hparams.use_ranking_candidate_movie_ids: # Get ranking candidate movie ids to rank our candidate movie # against. ranking_candidate_movie_ids = features[ RANKING_CANDIDATE_MOVIE_IDS] movies_to_rank_condition = tf.sparse_to_indicator( tf.sparse_concat(axis=1, sp_inputs=[ ranking_candidate_movie_ids, features[CANDIDATE_MOVIE_ID] ]), MOVIE_VOCAB_SIZE) predictions = tf.where( movies_to_rank_condition, predictions, tf.fill(tf.shape(predictions), tf.reduce_min(predictions))) return tf.contrib.learn.ModelFnOps(mode=mode, predictions=predictions, loss=loss) elif mode == tf.contrib.learn.ModeKeys.INFER: scores = tf.matmul(query_embeddings, tf.transpose(class_weights)) + class_biases rated_movie_ids = features[QUERY_RATED_MOVIE_IDS] pruned_scores = tf.where( tf.sparse_to_indicator(rated_movie_ids, MOVIE_VOCAB_SIZE), tf.fill(tf.shape(scores), tf.reduce_min(scores)), scores) predictions, output_alternatives = generate_top_k_scores_and_ids( pruned_scores, hparams.top_k_infer) return tf.contrib.learn.ModelFnOps( mode=mode, predictions=predictions, output_alternatives=output_alternatives)
def _matrix_factorization_model_fn(features, target_ratings, mode): """Creates a neighborhood factorization model. Each user is represented by a combination of embeddings of rated items, as described in the paper: "Factorization Meets the Neighborhood: a Multifaceted Collaborative Filtering Model - Yehuda Koren (KDD 2013)". Args: features: A dictionary of tensors keyed by the feature name. target_ratings: A tensor representing the labels (in this case, the ratings on the target movie). mode: The execution mode, as defined in tf.contrib.learn.ModeKeys. Returns: ModelFnOps with the mode, prediction, loss, train_op and output_alternatives a dictionary specifying the output for a classification request during serving. """ _ = target_ratings # Unused on this model. if hparams.embedding_weight_initializer == TRUNCATED_NORMAL: embedding_weight_initializer = tf.truncated_normal_initializer( stddev=0.1) else: embedding_weight_initializer = None query_movie_embedding_weights = tf.get_variable( 'query_movie_ids_embedding_weights', [MOVIE_VOCAB_SIZE, hparams.movie_embedding_dim], initializer=embedding_weight_initializer, regularizer=tf.contrib.layers.l2_regularizer( hparams.l2_weight_decay)) query_movie_ids = features[QUERY_RATED_MOVIE_IDS] query_embeddings = tf.nn.embedding_lookup_sparse( [query_movie_embedding_weights], query_movie_ids, None, combiner='sqrtn', name='query_embedding') query_biases, _, _ = tf.contrib.layers.weighted_sum_from_feature_columns( columns_to_tensors=features, feature_columns=make_query_feature_columns(), num_outputs=1) global_rating_bias = tf.get_variable( name='global_rating_bias', initializer=tf.constant(RATING_BIAS, dtype=tf.float32)) candidate_movie_embedding_weights = tf.get_variable( 'candidate_movie_id_embedding_weights', [MOVIE_VOCAB_SIZE, hparams.movie_embedding_dim], initializer=embedding_weight_initializer, regularizer=tf.contrib.layers.l2_regularizer( hparams.l2_weight_decay)) candidate_biases, _, _ = ( tf.contrib.layers.weighted_sum_from_feature_columns( columns_to_tensors=features, feature_columns=make_candidate_feature_columns(), num_outputs=1)) # Create layers for target features. if mode != tf.contrib.learn.ModeKeys.INFER: candidate_movie_ids = features[CANDIDATE_MOVIE_ID] candidate_embeddings = tf.nn.embedding_lookup_sparse( [candidate_movie_embedding_weights], candidate_movie_ids, None, name='candidate_embedding') predictions = tf.reduce_sum(tf.multiply(query_embeddings, candidate_embeddings), 1, keep_dims=True) if hparams.enable_bias: biases = tf.add(query_biases, candidate_biases) predictions = tf.add(predictions, biases) predictions = tf.add(predictions, global_rating_bias) labels = features[LABEL_RATING_SCORE] loss = tf.losses.mean_squared_error(labels, predictions) if mode == tf.contrib.learn.ModeKeys.TRAIN: train_op = tf.contrib.layers.optimize_loss( loss=loss, global_step=tf.contrib.framework.get_global_step(), learning_rate=hparams.learning_rate, optimizer=hparams.optimizer) return tf.contrib.learn.ModelFnOps(mode=mode, predictions=predictions, loss=loss, train_op=train_op) elif mode == tf.contrib.learn.ModeKeys.EVAL: if hparams.eval_type == REGRESSION: return tf.contrib.learn.ModelFnOps(mode=mode, predictions=predictions, loss=loss) elif hparams.eval_type == RANKING: # For 'RANKING' eval, we are interested in precision@k, recall@k # metrics which require us to compute prediction/ranking scores for # all movies. predictions = tf.matmul(query_embeddings, candidate_movie_embedding_weights, transpose_b=True) if hparams.enable_bias: biases = tf.add(query_biases, candidate_biases) predictions = tf.add(predictions, biases) if hparams.use_ranking_candidate_movie_ids: # Get ranking candidate movie ids to rank our candidate movie # against. ranking_candidate_movie_ids = features[ RANKING_CANDIDATE_MOVIE_IDS] movies_to_rank_condition = tf.sparse_to_indicator( tf.sparse_concat(axis=1, sp_inputs=[ ranking_candidate_movie_ids, candidate_movie_ids ]), MOVIE_VOCAB_SIZE) predictions = tf.where( movies_to_rank_condition, predictions, tf.fill(tf.shape(predictions), tf.reduce_min(predictions))) return tf.contrib.learn.ModelFnOps(mode=mode, predictions=predictions, loss=loss) elif mode == tf.contrib.learn.ModeKeys.INFER: scores = tf.matmul(query_embeddings, candidate_movie_embedding_weights, transpose_b=True) if hparams.enable_bias: biases = tf.add(query_biases, candidate_biases) scores = tf.add(scores, biases) # Eliminate already rated candates. rated_movie_ids = features[QUERY_RATED_MOVIE_IDS] pruned_scores = tf.where( tf.sparse_to_indicator(rated_movie_ids, MOVIE_VOCAB_SIZE), tf.fill(tf.shape(scores), tf.reduce_min(scores)), scores) predictions, output_alternatives = generate_top_k_scores_and_ids( pruned_scores, hparams.top_k_infer) return tf.contrib.learn.ModelFnOps( mode=mode, predictions=predictions, output_alternatives=output_alternatives)
def _add_stop_words(line): STOP = tf.SparseTensor(indices=[[0,0]], values=[chr(0)], dense_shape=[1,1]) return tf.sparse_concat(axis=1, sp_inputs=[line, STOP])
def _build_attention_embedding(self, hparams): PAIR_NUM = hparams.PAIR_NUM batch_size = hparams.batch_size # sparse input # news pair的sparse表示,shape=[attention_num*batch_size, feature_num] attention_news_x = tf.SparseTensor( indices=self.iterator.attention_news_indices, values=self.iterator.attention_news_values, dense_shape=self.iterator.attention_news_shape) # attention_user_feat_index_batch = tf.SparseTensor( indices=self.iterator.attention_user_indices, values=self.iterator.attention_user_values, dense_shape=self.iterator.attention_user_shape) attention_user_feat_weight_batch = tf.SparseTensor( indices=self.iterator.attention_user_indices, values=self.iterator.attention_user_weights, dense_shape=self.iterator.attention_user_shape) attention_news_x_embedding = tf.sparse_tensor_dense_matmul( attention_news_x, self.embedding) split_attention_x_embedding = tf.split(attention_news_x_embedding, num_or_size_splits=PAIR_NUM * batch_size, axis=0) # 将news_embedding分成PAIR_NUM份 # 每份都代表一个field的embedding,size = [batch_size, embedding_dim] attention_news_embedding_pair = [] for pair_idx in range(PAIR_NUM): tmp = [] for i in range(batch_size): tmp.append(split_attention_x_embedding[i * PAIR_NUM + pair_idx]) attention_news_embedding_pair.append(tf.concat(tmp, axis=0)) split_attention_user_feat_index_batch = tf.sparse_split( axis=0, num_split=PAIR_NUM * batch_size, sp_input=attention_user_feat_index_batch) split_attention_user_feat_weight_batch = tf.sparse_split( axis=0, num_split=PAIR_NUM * batch_size, sp_input=attention_user_feat_weight_batch) attention_user_feat_index_batch_pair = [] attention_user_feat_weight_batch_pair = [] for pair_idx in range(PAIR_NUM): tmp1 = [] tmp2 = [] for i in range(batch_size): tmp1.append( split_attention_user_feat_index_batch[i * PAIR_NUM + pair_idx]) tmp2.append( split_attention_user_feat_weight_batch[i * PAIR_NUM + pair_idx]) attention_user_feat_index_batch_pair.append( tf.sparse_concat(axis=0, sp_inputs=tmp1)) attention_user_feat_weight_batch_pair.append( tf.sparse_concat(axis=0, sp_inputs=tmp2)) # #news embedding nn_input = [] attention_nn_params = [] for idx in range(PAIR_NUM): attention_embedding, attention_module_params = self._build_attention_pair( attention_user_feat_index_batch_pair[idx].indices, attention_user_feat_index_batch_pair[idx].dense_shape, attention_user_feat_index_batch_pair[idx], attention_user_feat_weight_batch_pair[idx], attention_news_embedding_pair[idx], self.embedding, idx, hparams) field_nn_input = tf.concat( [attention_embedding, attention_news_embedding_pair[idx]], 1) nn_input.append(field_nn_input) attention_nn_params.extend(attention_module_params) # for param in attention_nn_params: self.layer_params.append(param) concat_nn_input = tf.concat(nn_input, 1) return concat_nn_input, attention_nn_params
def stack(self, arr): concat_arr = [tf.sparse_reshape(x, [1] + self.shape) for x in arr] return tf.sparse_concat(0, concat_arr)
def _matrix_factorization_model_fn(features, target_ratings, mode): """Creates a neighborhood factorization model. Each user is represented by a combination of embeddings of rated items, as described in the paper: "Factorization Meets the Neighborhood: a Multifaceted Collaborative Filtering Model - Yehuda Koren (KDD 2013)". Args: features: A dictionary of tensors keyed by the feature name. target_ratings: A tensor representing the labels (in this case, the ratings on the target movie). mode: The execution mode, as defined in tf.contrib.learn.ModeKeys. Returns: ModelFnOps with the mode, prediction, loss, train_op and output_alternatives a dictionary specifying the output for a classification request during serving. """ _ = target_ratings # Unused on this model. if hparams.embedding_weight_initializer == TRUNCATED_NORMAL: embedding_weight_initializer = tf.truncated_normal_initializer(stddev=0.1) else: embedding_weight_initializer = None query_movie_embedding_weights = tf.get_variable( 'query_movie_ids_embedding_weights', [MOVIE_VOCAB_SIZE, hparams.movie_embedding_dim], initializer=embedding_weight_initializer, regularizer=tf.contrib.layers.l2_regularizer(hparams.l2_weight_decay)) query_movie_ids = features[QUERY_RATED_MOVIE_IDS] query_embeddings = tf.nn.embedding_lookup_sparse( [query_movie_embedding_weights], query_movie_ids, None, combiner='sqrtn', name='query_embedding') query_biases, _, _ = tf.contrib.layers.weighted_sum_from_feature_columns( columns_to_tensors=features, feature_columns=make_query_feature_columns(), num_outputs=1) global_rating_bias = tf.get_variable( name='global_rating_bias', initializer=tf.constant(RATING_BIAS, dtype=tf.float32)) candidate_movie_embedding_weights = tf.get_variable( 'candidate_movie_id_embedding_weights', [MOVIE_VOCAB_SIZE, hparams.movie_embedding_dim], initializer=embedding_weight_initializer, regularizer=tf.contrib.layers.l2_regularizer(hparams.l2_weight_decay)) candidate_biases, _, _ = ( tf.contrib.layers.weighted_sum_from_feature_columns( columns_to_tensors=features, feature_columns=make_candidate_feature_columns(), num_outputs=1)) # Create layers for target features. if mode != tf.contrib.learn.ModeKeys.INFER: candidate_movie_ids = features[CANDIDATE_MOVIE_ID] candidate_embeddings = tf.nn.embedding_lookup_sparse( [candidate_movie_embedding_weights], candidate_movie_ids, None, name='candidate_embedding') predictions = tf.reduce_sum(tf.multiply( query_embeddings, candidate_embeddings), 1, keep_dims=True) if hparams.enable_bias: biases = tf.add(query_biases, candidate_biases) predictions = tf.add(predictions, biases) predictions = tf.add(predictions, global_rating_bias) labels = features[LABEL_RATING_SCORE] loss = tf.losses.mean_squared_error(labels, predictions) if mode == tf.contrib.learn.ModeKeys.TRAIN: train_op = tf.contrib.layers.optimize_loss( loss=loss, global_step=tf.contrib.framework.get_global_step(), learning_rate=hparams.learning_rate, optimizer=hparams.optimizer) return tf.contrib.learn.ModelFnOps( mode=mode, predictions=predictions, loss=loss, train_op=train_op) elif mode == tf.contrib.learn.ModeKeys.EVAL: if hparams.eval_type == REGRESSION: return tf.contrib.learn.ModelFnOps( mode=mode, predictions=predictions, loss=loss) elif hparams.eval_type == RANKING: # For 'RANKING' eval, we are interested in precision@k, recall@k # metrics which require us to compute prediction/ranking scores for # all movies. predictions = tf.matmul(query_embeddings, candidate_movie_embedding_weights, transpose_b=True) if hparams.enable_bias: biases = tf.add(query_biases, candidate_biases) predictions = tf.add(predictions, biases) if hparams.use_ranking_candidate_movie_ids: # Get ranking candidate movie ids to rank our candidate movie # against. ranking_candidate_movie_ids = features[RANKING_CANDIDATE_MOVIE_IDS] movies_to_rank_condition = tf.sparse_to_indicator( tf.sparse_concat( axis=1, sp_inputs=[ranking_candidate_movie_ids, candidate_movie_ids]), MOVIE_VOCAB_SIZE) predictions = tf.where(movies_to_rank_condition, predictions, tf.fill( tf.shape(predictions), tf.reduce_min(predictions))) return tf.contrib.learn.ModelFnOps( mode=mode, predictions=predictions, loss=loss) elif mode == tf.contrib.learn.ModeKeys.INFER: scores = tf.matmul(query_embeddings, candidate_movie_embedding_weights, transpose_b=True) if hparams.enable_bias: biases = tf.add(query_biases, candidate_biases) scores = tf.add(scores, biases) # Eliminate already rated candates. rated_movie_ids = features[QUERY_RATED_MOVIE_IDS] pruned_scores = tf.where( tf.sparse_to_indicator(rated_movie_ids, MOVIE_VOCAB_SIZE), tf.fill(tf.shape(scores), tf.reduce_min(scores)), scores) predictions, output_alternatives = generate_top_k_scores_and_ids( pruned_scores, hparams.top_k_infer) return tf.contrib.learn.ModelFnOps( mode=mode, predictions=predictions, output_alternatives=output_alternatives)
X_xyz = tf.constant(X_xyz_np,tf.float32,shape = (BATCH,Sz*Sz,3)) Adj_sp = tf.SparseTensor(indices=wsp_indices_list, values=wsp_values_list, dense_shape=[Sz*Sz, Sz*Sz]) W_mat_np = np.float32(np.ones((n_spatial_filters*n_features,n_features))) W_mat = tf.constant(W_mat_np,tf.float32,shape = (n_spatial_filters*n_features,n_features)) #Build geodesic sparse matrices W_sp_list = [] for b in range(BATCH): for dim in range(3): W_sp_ = tf.sparse_add((X_xyz[b,:,dim:dim+1]*Adj_sp), Adj_sp*tf.transpose(-1*X_xyz[b,:,dim:dim+1])) W_sp_ = tf.sparse_reshape(W_sp_, (Sz*Sz,Sz*Sz,1)) W_sp_ = tf.sparse_concat(axis = 2,sp_inputs = [W_sp_ for i in range(n_spatial_filters)]) Adj_sp_re = tf.sparse_reshape(Adj_sp, (Sz*Sz,Sz*Sz,1)) Adj_sp_re = tf.sparse_concat(axis = 2,sp_inputs = [Adj_sp_re for i in range(n_spatial_filters)]) alpha_ = tf.reshape(Alfas[:,dim],(1,1,-1)) sigma_ = tf.reshape(Sigmas[:,dim],(1,1,-1)) W_sp_ = tf.sparse_add(W_sp_,Adj_sp_re*(-1*alpha_)) W_sp_ = tf.SparseTensor(indices = W_sp_.indices, values = W_sp_.values**2,dense_shape=[Sz*Sz, Sz*Sz,n_spatial_filters]) W_sp_ = W_sp_*(-0.5/sigma_**2) if dim == 0: W_sp = W_sp_ else: W_sp = tf.sparse_add(W_sp,W_sp_)
def _dnn_softmax_fn(features, targets, mode): """Creates the prediction, loss, and train ops. Args: features: A dictionary of tensors keyed by the feature name. targets: A tensor representing the labels (in this case, the ratings on the target movie). mode: The execution mode, as defined in tf.contrib.learn.ModeKeys. Returns: ModelFnOps with the mode, prediction, loss, train_op and output_alternatives a dictionary specifying the output for a classification request during serving. Raises: ValueError: When the wrong evaluation type is specified. """ _ = targets # Unused variable. class_weights = tf.get_variable( name='class_weights', shape=[MOVIE_VOCAB_SIZE, hparams.query_hidden_dims[-1]], initializer=tf.contrib.layers.xavier_initializer()) class_biases = tf.get_variable( name='class_biases', shape=[MOVIE_VOCAB_SIZE], initializer=tf.zeros_initializer()) query_embeddings = _embed_query_features(features, mode=mode) tf.summary.scalar('query_embeddings_zero_fraction', tf.nn.zero_fraction(query_embeddings)) # Create layers for target features. if mode != tf.contrib.learn.ModeKeys.INFER: logits_layer = tf.matmul( query_embeddings, tf.transpose(class_weights)) + class_biases target_one_hot = tf.one_hot( indices=features[CANDIDATE_MOVIE_ID].values, depth=MOVIE_VOCAB_SIZE, on_value=1.0) loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits( labels=target_one_hot, logits=logits_layer)) if mode == tf.contrib.learn.ModeKeys.TRAIN: train_op = tf.contrib.layers.optimize_loss( loss=loss, global_step=tf.contrib.framework.get_global_step(), learning_rate=hparams.learning_rate, optimizer=hparams.optimizer) return tf.contrib.learn.ModelFnOps( mode=mode, loss=loss, train_op=train_op) elif mode == tf.contrib.learn.ModeKeys.EVAL: if hparams.eval_type == REGRESSION: raise ValueError('eval_type must be RANKING for DNN softmax model.') elif hparams.eval_type == RANKING: predictions = tf.matmul( query_embeddings, tf.transpose(class_weights)) + class_biases if hparams.use_ranking_candidate_movie_ids: # Get ranking candidate movie ids to rank our candidate movie # against. ranking_candidate_movie_ids = features[RANKING_CANDIDATE_MOVIE_IDS] movies_to_rank_condition = tf.sparse_to_indicator( tf.sparse_concat( axis=1, sp_inputs=[ranking_candidate_movie_ids, features[CANDIDATE_MOVIE_ID]]), MOVIE_VOCAB_SIZE) predictions = tf.where(movies_to_rank_condition, predictions, tf.fill( tf.shape(predictions), tf.reduce_min(predictions))) return tf.contrib.learn.ModelFnOps( mode=mode, predictions=predictions, loss=loss) elif mode == tf.contrib.learn.ModeKeys.INFER: scores = tf.matmul( query_embeddings, tf.transpose(class_weights)) + class_biases rated_movie_ids = features[QUERY_RATED_MOVIE_IDS] pruned_scores = tf.where( tf.sparse_to_indicator(rated_movie_ids, MOVIE_VOCAB_SIZE), tf.fill(tf.shape(scores), tf.reduce_min(scores)), scores) predictions, output_alternatives = generate_top_k_scores_and_ids( pruned_scores, hparams.top_k_infer) return tf.contrib.learn.ModelFnOps( mode=mode, predictions=predictions, output_alternatives=output_alternatives)
def define_graph(self): filenames = glob(os.path.join("../data", "cnn_*.tfrecords")) min_after_dequeue = 1000 filename_queue = tf.train.string_input_producer(filenames) document, question, answer, document_shape, question_shape, answer_shape = read_tf_record_file( filename_queue) d_batch, q_batch, ans_batch, document_shape_batch, question_shape_batch, answer_shape_batch = tf.train.shuffle_batch( [ document, question, answer, document_shape, question_shape, answer_shape ], batch_size=self.hparams.batch_size, capacity=min_after_dequeue * 3 + 1, min_after_dequeue=min_after_dequeue) d_q_batch = tf.sparse_concat( axis=1, sp_inputs=[d_batch, q_batch], ) dense_d_q_batch = tf.sparse_to_dense( sparse_indices=d_q_batch.indices, output_shape=d_q_batch.dense_shape, sparse_values=d_q_batch.values, default_value=0, validate_indices=True, name=None) dens_ans_batch = tf.sparse_to_dense(sparse_indices=ans_batch.indices, output_shape=ans_batch.dense_shape, sparse_values=ans_batch.values, default_value=0, validate_indices=True, name=None) d_q_lengths = tf.reduce_sum(tf.concat([ tf.reshape(document_shape_batch, (self.hparams.batch_size, 1)), tf.reshape(question_shape_batch, (self.hparams.batch_size, 1)) ], axis=1), axis=1) self.embedding = tf.get_variable( "embedding", [self.vocab_size, self.hparams.number_of_hidden_units], trainable=True) tf.logging.info(self.embedding.get_shape()) self.inputs = dense_d_q_batch self.y = dens_ans_batch[:, 0] #tf.placeholder(tf.float32, [self.hparams.self.hparams.batch_size, self.vocab_size]) #unstacked_inputs = tf.unstack(self.inputs,axis=1) embedded_inputs = [ tf.nn.embedding_lookup(self.embedding, self.inputs[i]) for i in range(self.hparams.batch_size) ] #embedded_inputs = tf.stack(embedded_inputs) tf.summary.histogram("embeddings", self.embedding) self.__build_deep_lstm_cell() states_series, current_state = tf.nn.dynamic_rnn( cell=self.stacked_cell, inputs=tf.stack(embedded_inputs), sequence_length=d_q_lengths, initial_state=None, dtype=tf.float32, parallel_iterations=None, swap_memory=False, time_major=False, scope=None) self.batch_states = [ layer_state[1] for layer_state in tf.unstack(current_state) ] # tf.stack(states) self.output_size = self.hparams.depth * self.hparams.number_of_hidden_units """startings = tf.concat([ tf.zeros((self.hparams.batch_size,1),dtype=tf.int64), tf.reshape(d_q_lengths,(self.hparams.batch_size,1)) - 1, tf.zeros((self.hparams.batch_size,1),dtype=tf.int64)],axis=1) outputs = [self.batch_states[0][i,d_q_lengths[i],:] for i in range(self.hparams.batch_size)]""" outputs = tf.concat(self.batch_states, axis=1) tf.logging.set_verbosity(tf.logging.INFO) tf.logging.info(outputs) tf.logging.info(self.batch_states) tf.logging.info(current_state[0]) self.outputs = tf.reshape(outputs, [self.hparams.batch_size, self.output_size]) self.W = tf.get_variable("W", [ self.output_size, self.vocab_size, ], trainable=True) tf.summary.histogram("weights", self.W) tf.summary.histogram("output", self.outputs) self.y_ = tf.matmul(self.outputs, self.W) tf.logging.info(self.y_) cross_ent = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=self.y_, labels=self.y) self.train_loss = (tf.reduce_sum(cross_ent) / self.hparams.batch_size) tf.summary.scalar("loss", tf.reduce_mean(self.train_loss)) tf.logging.info(tf.argmax(self.y_, 1)) correct_prediction = tf.equal(self.y, tf.argmax(self.y_, 1)) self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) tf.summary.scalar("accuracy", self.accuracy)
def __call__(self, inputs, state, scope=None): """Gated recurrent unit (GRU) with Graph Convolution. :param inputs: (B, num_nodes * input_dim) :return - Output: A `2-D` tensor with shape `[batch_size x self.output_size]`. - New state: Either a single `2-D` tensor, or a tuple of tensors matching the arity and shapes of `state` """ with tf.variable_scope(scope or "dcgru_cell"): # inputs = tf.ones([self._num_nodes, self._num_nodes]) # x_shape = tf.concat([inputs, state], axis=1) # input_shape = x_shape.get_shape().as_list() # input_rnn = tf.SparseTensor(self.index, self.value, [self._num_nodes, self._num_nodes]) state = tf.reshape(state, [self._num_nodes, -1]) state_shape = state.get_shape().as_list() input_dim = state_shape[1] + self._num_nodes zero = tf.constant(0, dtype=tf.float32) where = tf.not_equal(state, zero) indices = tf.where(where) values = tf.gather_nd(state, indices) sparse_state = tf.SparseTensor(indices, values, state.shape) x1 = tf.sparse_concat(sp_inputs=[self.input_rnn, sparse_state], axis=1) non_zeros_feat = tf.size(x1.values) with tf.variable_scope("gates"): # Reset gate # We start with bias of 1.0 to not reset and not update. r = GraphConvolution_saprse(input_dim=input_dim, output_dim=self._num_units, adj=self.adj, act=tf.nn.sigmoid, dropout=self.dropout, non_zero = non_zeros_feat, bias=self.bias, logging=self.logging)(x1) with tf.variable_scope("gates"): # Update gate u = GraphConvolution_saprse(input_dim=input_dim, output_dim=self._num_units, adj=self.adj, act=tf.nn.sigmoid, dropout=self.dropout, non_zero=non_zeros_feat, bias=self.bias, logging=self.logging)(x1) with tf.variable_scope("candidate"): state_r = r * state where_r = tf.not_equal(state_r, zero) indices_r = tf.where(where_r) values_r = tf.gather_nd(state_r, indices_r) sparse_state_r = tf.SparseTensor(indices_r, values_r, state_r.shape) x2 = tf.sparse_concat(sp_inputs=[self.input_rnn, sparse_state_r], axis=1) c = GraphConvolution_saprse(input_dim=input_dim, output_dim=self._num_units, adj=self.adj, act=tf.nn.sigmoid, dropout=self.dropout, non_zero=non_zeros_feat, bias=self.bias, logging=self.logging)(x2) h = u * state + (1 - u) * c h = tf.reshape(h, [1, -1]) output = new_state = h return output, new_state
tf.to_double() tf.to_float() tf.to_int32() tf.to_int64() tf.trace() tf.trainable_variables() tf.transpose() tf.truncated_normal() tf.truediv() tf.sparse_transpose() tf.sparse_tensor_dense_matmul() tf.sparse_accumulator_apply_gradient() tf.sparse_accumulator_take_gradient() tf.sparse_add() tf.sparse_concat() tf.sparse_conditional_accumulator() tf.sparse_mask() tf.sparse_matmul() tf.sparse_maximum() tf.sparse_merge() tf.sparse_minimum() tf.sparse_reduce_max() tf.sparse_reduce_max_sparse() tf.reduce_all() tf.reduce_any() tf.reduce_join() tf.reduce_logsumexp() tf.reduce_max()
def sp_hete_attn_head(seq, out_sz, adj_mat, adj_type, edge_list, activation, nb_nodes, in_drop=0.0, coef_drop=0.0, residual=False): # input adjacency matrices are TRANSPOSED before feeding! # nb_nodes_j * nb_nodes_i with tf.name_scope('sp_hete_attn'): if in_drop != 0.0: seq = [tf.nn.dropout(seq_i, 1.0 - in_drop) for seq_i in seq] # seq_fts[j][i]: hidden features from group i to group j, center node is j # 1 * nb_nodes_i * out_sz_j W_list = [[None for _ in seq] for _ in seq] seq_fts = [[None for _ in seq] for _ in seq] W_inv_list = [ tf.Variable( tf.glorot_uniform_initializer()(shape=(1, out_sz, int(s.shape[2])))) for s in seq ] for W_type in adj_type: i, j = W_type if W_list[i][j] is None: W_list[i][j] = tf.Variable(tf.glorot_uniform_initializer()( shape=(1, int(seq[i].shape[2]), out_sz))) seq_fts[j][i] = tf.matmul(seq[i], W_list[i][j]) for i in range(len(seq)): for j in range(len(seq)): if W_list[i][j] is not None and W_list[j][i] is not None: loss1 = seq_fts[j][i] @ ( W_inv_list[j] @ W_list[j][i]) - seq_fts[i][i] # 1 * nb_nodes_i * out_sz_i loss1 = loss1 * loss1 loss1 = tf.reduce_sum(loss1) / nb_nodes[i] tf.add_to_collection('loss_loop', loss1) for j in range(len(seq)): if out_sz >= seq[j].shape[1]: loss2 = W_list[j][j] @ W_inv_list[j] - tf.eye(seq[j].shape[2]) else: loss2 = W_inv_list[j] @ W_list[j][j] - tf.eye(out_sz) loss2 = loss2 * loss2 loss2 = tf.reduce_sum(loss2) tf.add_to_collection('loss_inv', loss2) attn_biases = [None for _ in adj_type] for dir_edge in edge_list: attn_bias = tf.Variable(tf.random_normal(shape=(1, out_sz))) attn_biases[dir_edge[0]] = attn_bias if len(dir_edge) == 2: attn_biases[dir_edge[1]] = -attn_bias # for out_sz_j in out_sz coefs_lists = [[] for _ in range(len(seq))] seq_fts_lists = [[] for _ in range(len(seq))] # simplest self-attention possible for adj_ij, type_ij, attn_bias in zip(adj_mat, adj_type, attn_biases): # adj_ij is transposed, nb_nodes_j * nb_nodes_i i, j = type_ij f_1 = tf.reshape(seq_fts[j][j], (nb_nodes[j], out_sz)) f_1 = tf.gather(f_1, adj_ij.indices[:, 0]) f_2 = tf.reshape(seq_fts[j][i], (nb_nodes[i], out_sz)) if attn_bias is not None: f_2 = f_2 + attn_bias f_2 = tf.gather(f_2, adj_ij.indices[:, 1]) f = tf.reduce_sum(tf.multiply(f_1, f_2), 1) coefs = tf.SparseTensor(indices=adj_ij.indices, values=tf.nn.leaky_relu(f), dense_shape=adj_ij.dense_shape) if coef_drop != 0.0: coefs = tf.SparseTensor(indices=coefs.indices, values=tf.nn.dropout( coefs.values, 1.0 - coef_drop), dense_shape=coefs.dense_shape) coefs_lists[j].append(coefs) # transposed, nb_nodes_j * nb_nodes_i if in_drop != 0.0: seq_fts_ij = tf.nn.dropout(seq_fts[j][i], 1.0 - in_drop) seq_fts_lists[j].append( tf.squeeze(seq_fts_ij)) # nb_nodes_i * out_sz_j # As tf.sparse_tensor_dense_matmul expects its arguments to have rank-2, # here we make an assumption that our input is of batch size 1, and reshape appropriately. # The method will fail in all other cases! coefs = [tf.sparse_concat(1, coefs_list) for coefs_list in coefs_lists] coefs = [tf.sparse_softmax(coef) for coef in coefs] seq_fts = [ tf.concat(seq_fts_list, 0) for seq_fts_list in seq_fts_lists ] vals = [ tf.sparse_tensor_dense_matmul(coef, seq_ft) for coef, seq_ft in zip(coefs, seq_fts) ] # nb_nodes_j * out_sz_j vals = [tf.expand_dims(val, axis=0) for val in vals] for i, val in enumerate(vals): val.set_shape([1, nb_nodes[i], out_sz]) ret = [tf.contrib.layers.bias_add(val) for val in vals] # residual connection if residual: ret2 = [] for r, s in zip(ret, seq): if s.shape[-1] != r.shape[-1]: ret2.append(r + tf.layers.conv1d(s, r.shape[-1], 1)) else: ret2.append(r + s) ret = ret2 ret = [activation(r) for r in ret] return ret # activation
def sp_hete_attn_head(seq, out_sz, adj_mat, adj_type, activation, nb_nodes, in_drop=0.0, coef_drop=0.0, residual=False): # input adjacency matrices are TRANSPOSED before feeding! with tf.name_scope('sp_hete_attn'): if in_drop != 0.0: seq = [tf.nn.dropout(seq_i, 1.0 - in_drop) for seq_i in seq] # seq_fts[j][i]: hidden features from group i to group j, center node is j # 1 * nb_nodes_i * out_sz_j seq_fts = [ [ tf.layers.conv1d( seq_i, out_sz, # out_sz_j 1, use_bias=False) for seq_i in seq ] for _ in seq ] # for out_sz_j in out_sz coefs_lists = [[] for _ in range(len(seq))] seq_fts_lists = [[] for _ in range(len(seq))] # simplest self-attention possible for adj_ij, type_ij in zip(adj_mat, adj_type): # transposed, # nb_nodes_j * nb_nodes_i i, j = type_ij f_1 = tf.layers.conv1d(seq_fts[j][j], 1, 1) f_2 = tf.layers.conv1d(seq_fts[j][i], 1, 1) f_1 = tf.reshape(f_1, (nb_nodes[j], 1)) f_2 = tf.reshape(f_2, (nb_nodes[i], 1)) f_1 = adj_ij * f_1 f_2 = adj_ij * tf.transpose(f_2, [1, 0]) logits = tf.sparse_add(f_1, f_2) # nb_nodes_j * nb_nodes_i coefs = tf.SparseTensor(indices=logits.indices, values=tf.nn.leaky_relu(logits.values), dense_shape=logits.dense_shape) # coefs = tf.sparse_softmax(lrelu) if coef_drop != 0.0: coefs = tf.SparseTensor(indices=coefs.indices, values=tf.nn.dropout( coefs.values, 1.0 - coef_drop), dense_shape=coefs.dense_shape) coefs_lists[j].append(coefs) # transposed, nb_nodes_j * nb_nodes_i if in_drop != 0.0: seq_fts_ij = tf.nn.dropout(seq_fts[j][i], 1.0 - in_drop) seq_fts_lists[j].append( tf.squeeze(seq_fts_ij)) # nb_nodes_i * out_sz_j # As tf.sparse_tensor_dense_matmul expects its arguments to have rank-2, # here we make an assumption that our input is of batch size 1, and reshape appropriately. # The method will fail in all other cases! # coefs = tf.sparse_reshape(coefs, [nb_nodes, nb_nodes]) coefs = [tf.sparse_concat(1, coefs_list) for coefs_list in coefs_lists] coefs = [tf.sparse_softmax(coef) for coef in coefs] # seq_fts = tf.squeeze(seq_fts) seq_fts = [ tf.concat(seq_fts_list, 0) for seq_fts_list in seq_fts_lists ] vals = [ tf.sparse_tensor_dense_matmul(coef, seq_ft) for coef, seq_ft in zip(coefs, seq_fts) ] # nb_nodes_j * out_sz_j vals = [tf.expand_dims(val, axis=0) for val in vals] for i, val in enumerate(vals): val.set_shape([1, nb_nodes[i], out_sz]) ret = [tf.contrib.layers.bias_add(val) for val in vals] # residual connection if residual: ret2 = [] for r, s in zip(ret, seq): if s.shape[-1] != r.shape[-1]: ret2.append(r + tf.layers.conv1d(s, r.shape[-1], 1)) else: ret2.append(r + s) ret = ret2 ret = [activation(r) for r in ret] return ret # activation
def body(index, words): next_word = tf.sparse_slice(line, tf.to_int64(index), [1, 1]).values next_word = tf.string_split(next_word, delimiter='') words = tf.sparse_concat(axis=0, sp_inputs=[words, next_word], expand_nonconcat_dim=True) return index+[0, 1], words