def testMismatchedShapesExpandNonconcatDim(self):
with self.test_session(use_gpu=False) as sess:
sp_a = self._SparseTensor_3x3()
sp_b = self._SparseTensor_3x5()
sp_c = self._SparseTensor_3x2()
sp_d = self._SparseTensor_2x3()
sp_concat_dim0 = tf.sparse_concat(0, [sp_a, sp_b, sp_c, sp_d],
expand_nonconcat_dim=True)
sp_concat_dim1 = tf.sparse_concat(1, [sp_a, sp_b, sp_c, sp_d],
expand_nonconcat_dim=True)
sp_concat_dim0_out = sess.run(sp_concat_dim0)
sp_concat_dim1_out = sess.run(sp_concat_dim1)
self.assertAllEqual(
sp_concat_dim0_out.indices,
[[0, 2], [1, 0], [2, 0], [2, 2], [4, 1], [5, 0], [5, 3], [5, 4],
[7, 0], [8, 0], [9, 1], [10, 0], [10, 2]])
self.assertAllEqual(
sp_concat_dim0_out.values,
[1, 2, 3, 4, 1, 2, 1, 0, 1, 2, 1, 1, 2])
self.assertAllEqual(
sp_concat_dim0_out.shape,
[11, 5])
self.assertAllEqual(
sp_concat_dim1_out.indices,
[[0, 2], [0, 11], [1, 0], [1, 4], [1, 8], [1, 10], [1, 12], [2, 0],
[2, 2], [2, 3], [2, 6], [2, 7], [2, 8]])
self.assertAllEqual(
sp_concat_dim1_out.values,
[1, 1, 2, 1, 1, 1, 2, 3, 4, 2, 1, 0, 2])
self.assertAllEqual(
sp_concat_dim1_out.shape,
[3, 13])
def testMismatchedRank(self):
with self.test_session(use_gpu=False):
sp_a = self._SparseTensor_3x3()
sp_e = self._SparseTensor_2x3x4()
# Rank mismatches can be caught at shape-inference time
with self.assertRaises(ValueError):
tf.sparse_concat(1, [sp_a, sp_e])
def testMismatchedRankExpandNonconcatDim(self):
with self.test_session(use_gpu=False):
sp_a = self._SparseTensor_3x3()
sp_e = self._SparseTensor_2x3x4()
# Rank mismatches should be caught at shape-inference time, even for
# expand_nonconcat_dim=True.
with self.assertRaises(ValueError):
tf.sparse_concat(1, [sp_a, sp_e], expand_nonconcat_dim=True)
def testSliceConcat(self):
with self.test_session(use_gpu=False):
sparse_tensors = tf.sparse_split(1, 2, self._SparseTensor_3x4x2())
concat_tensor = tf.sparse_concat(1, sparse_tensors)
expected_output = self._SparseTensor_3x4x2()
self.assertAllEqual(concat_tensor.indices.eval(),
expected_output.indices.eval())
def testSliceConcat(self):
for sp_input in (self._SparseTensorValue_3x4x2(), self._SparseTensor_3x4x2()):
with self.test_session(use_gpu=False):
sparse_tensors = tf.sparse_split(sp_input=sp_input, num_split=2, axis=1)
concat_tensor = tf.sparse_concat(1, sparse_tensors)
expected_output = self._SparseTensor_3x4x2()
self.assertAllEqual(concat_tensor.indices.eval(), expected_output.indices.eval())
def testConcatDim0(self):
with self.test_session(use_gpu=False) as sess:
# concat(A, D):
# [ 1]
# [2 ]
# [3 4]
# [ 1 ]
# [1 2]
sp_a = self._SparseTensor_3x3()
sp_d = self._SparseTensor_2x3()
sp_concat = tf.sparse_concat(0, [sp_a, sp_d])
self.assertEqual(sp_concat.indices.get_shape(), [7, 2])
self.assertEqual(sp_concat.values.get_shape(), [7])
self.assertEqual(sp_concat.shape.get_shape(), [2])
concat_out = sess.run(sp_concat)
self.assertAllEqual(
concat_out.indices,
[[0, 2], [1, 0], [2, 0], [2, 2], [3, 1], [4, 0], [4, 2]])
self.assertAllEqual(
concat_out.values, np.array([1, 2, 3, 4, 1, 1, 2]))
self.assertAllEqual(
concat_out.shape, np.array([5, 3]))
def tensors_to_item(self, keys_to_tensors):
"""Maps the given dictionary of tensors to a concatenated list of bboxes.
Args:
keys_to_tensors: a mapping of TF-Example keys to parsed tensors.
Returns:
[time, num_boxes, 4] tensor of bounding box coordinates, in order
[y_min, x_min, y_max, x_max]. Whether the tensor is a SparseTensor
or a dense Tensor is determined by the return_dense parameter. Empty
positions in the sparse tensor are filled with -1.0 values.
"""
sides = []
for key in self._full_keys:
value = keys_to_tensors[key]
expanded_dims = tf.concat(
[tf.to_int64(tf.shape(value)),
tf.constant([1], dtype=tf.int64)], 0)
side = tf.sparse_reshape(value, expanded_dims)
sides.append(side)
bounding_boxes = tf.sparse_concat(2, sides)
if self._return_dense:
bounding_boxes = tf.sparse_tensor_to_dense(
bounding_boxes, default_value=self._default_value)
return bounding_boxes
def testMismatchedShapes(self):
with self.test_session(use_gpu=False) as sess:
sp_a = self._SparseTensor_3x3()
sp_b = self._SparseTensor_3x5()
sp_c = self._SparseTensor_3x2()
sp_d = self._SparseTensor_2x3()
sp_concat = tf.sparse_concat(1, [sp_a, sp_b, sp_c, sp_d])
# Shape mismatches can only be caught when the op is run
with self.assertRaisesOpError("Input shapes must match"):
sess.run(sp_concat)
def testShapeInferenceUnknownShapes(self):
with self.test_session(use_gpu=False):
sp_inputs = [
self._SparseTensor_UnknownShape(),
self._SparseTensor_UnknownShape(val_shape=[3]),
self._SparseTensor_UnknownShape(ind_shape=[1, 3]),
self._SparseTensor_UnknownShape(shape_shape=[3])]
sp_concat = tf.sparse_concat(0, sp_inputs)
self.assertEqual(sp_concat.indices.get_shape().as_list(), [None, 3])
self.assertEqual(sp_concat.values.get_shape().as_list(), [None])
self.assertEqual(sp_concat.shape.get_shape(), [3])
def testConcat1(self):
with self.test_session(use_gpu=False) as sess:
# concat(A):
# [ 1]
# [2 ]
# [3 4]
for sp_a in (self._SparseTensorValue_3x3(), self._SparseTensor_3x3()):
sp_concat = tf.sparse_concat(1, [sp_a])
self.assertEqual(sp_concat.indices.get_shape(), [4, 2])
self.assertEqual(sp_concat.values.get_shape(), [4])
self.assertEqual(sp_concat.shape.get_shape(), [2])
concat_out = sess.run(sp_concat)
self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [2, 0], [2, 2]])
self.assertAllEqual(concat_out.values, [1, 2, 3, 4])
self.assertAllEqual(concat_out.shape, [3, 3])
def testConcatNonNumeric(self):
with self.test_session(use_gpu=False) as sess:
# concat(A, B):
# [ a ]
# [b e ]
# [c d f g h]
sp_a = self._SparseTensor_String3x3()
sp_b = self._SparseTensor_String3x5()
sp_concat = tf.sparse_concat(1, [sp_a, sp_b])
self.assertEqual(sp_concat.indices.get_shape(), [8, 2])
self.assertEqual(sp_concat.values.get_shape(), [8])
self.assertEqual(sp_concat.shape.get_shape(), [2])
concat_out = sess.run(sp_concat)
self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [1, 4], [2, 0], [2, 2], [2, 3], [2, 6], [2, 7]])
self.assertAllEqual(concat_out.values, [b"a", b"b", b"e", b"c", b"d", b"f", b"g", b"h"])
self.assertAllEqual(concat_out.shape, [3, 8])
def testConcat2(self):
with self.test_session(use_gpu=False) as sess:
# concat(A, B):
# [ 1 ]
# [2 1 ]
# [3 4 2 1 0]
for sp_a in (self._SparseTensorValue_3x3(), self._SparseTensor_3x3()):
for sp_b in (self._SparseTensorValue_3x5(), self._SparseTensor_3x5()):
sp_concat = tf.sparse_concat(1, [sp_a, sp_b])
self.assertEqual(sp_concat.indices.get_shape(), [8, 2])
self.assertEqual(sp_concat.values.get_shape(), [8])
self.assertEqual(sp_concat.shape.get_shape(), [2])
concat_out = sess.run(sp_concat)
self.assertAllEqual(
concat_out.indices, [[0, 2], [1, 0], [1, 4], [2, 0], [2, 2], [2, 3], [2, 6], [2, 7]]
)
self.assertAllEqual(concat_out.values, [1, 2, 1, 3, 4, 2, 1, 0])
self.assertAllEqual(concat_out.shape, [3, 8])
def testConcat1(self):
with self.test_session(use_gpu=False) as sess:
# concat(A):
# [ 1]
# [2 ]
# [3 4]
for sp_a in (self._SparseTensorValue_3x3(), self._SparseTensor_3x3()):
# Note that we ignore concat_dim in this case since we short-circuit the
# single-input case in python.
for concat_dim in (-2000, 1, 2000):
sp_concat = tf.sparse_concat(concat_dim, [sp_a])
self.assertEqual(sp_concat.indices.get_shape(), [4, 2])
self.assertEqual(sp_concat.values.get_shape(), [4])
self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
concat_out = sess.run(sp_concat)
self.assertAllEqual(concat_out.indices,
[[0, 2], [1, 0], [2, 0], [2, 2]])
self.assertAllEqual(concat_out.values, [1, 2, 3, 4])
self.assertAllEqual(concat_out.shape, [3, 3])
def testConcat3(self):
with self.test_session(use_gpu=False) as sess:
# concat(A, B, C):
# [ 1 ]
# [2 1 1 ]
# [3 4 2 1 0 2 ]
sp_a = self._SparseTensor_3x3()
sp_b = self._SparseTensor_3x5()
sp_c = self._SparseTensor_3x2()
sp_concat = tf.sparse_concat(1, [sp_a, sp_b, sp_c])
self.assertEqual(sp_concat.indices.get_shape(), [10, 2])
self.assertEqual(sp_concat.values.get_shape(), [10])
self.assertEqual(sp_concat.shape.get_shape(), [2])
concat_out = sess.run(sp_concat)
self.assertAllEqual(
concat_out.indices, [[0, 2], [1, 0], [1, 4], [1, 8], [2, 0], [2, 2], [2, 3], [2, 6], [2, 7], [2, 8]]
)
self.assertAllEqual(concat_out.values, [1, 2, 1, 1, 3, 4, 2, 1, 0, 2])
self.assertAllEqual(concat_out.shape, [3, 10])
def __init__(
self,
cfg: InputFeederCfg,
fnames,
capacity=10000,
min_after_deque=10,
shuffle=True,
trace=False,
):
"""
:param cap: queue capacity
:param batch_size: output batch size
"""
self.cfg = cfg
self.fnames = fnames
self.logger = logging.getLogger(__name__)
self._values_ph = tf.placeholder(
dtype=tf.int32, shape=[None], name="labels"
)
self._values_len_ph = tf.placeholder(
dtype=tf.int64, shape=[], name="labels_len"
)
self._signal_ph = tf.placeholder(
dtype=tf.float32, shape=[None, 1], name="signal"
)
self._signal_len_ph = tf.placeholder(
dtype=tf.int64, shape=[], name="signal_len"
)
self.closing = []
self.queue = tf.PaddingFIFOQueue(
capacity=capacity,
dtypes=[tf.int32, tf.int64, tf.float32, tf.int64],
shapes=[
[None],
[],
[None, 1],
[],
]
)
# self.closing.append(self.queue.close()) Closed with queue runners...no idea how&why it works
self.summaries = []
self.summaries.append(
tf.summary.scalar(
"paddingFIFOQueue_input", self.queue.size(), family="queue"
)
)
if shuffle:
self.shuffle_queue = tf.RandomShuffleQueue(
capacity=capacity,
dtypes=[tf.int32, tf.int64, tf.float32, tf.int64],
min_after_dequeue=min_after_deque,
)
self.enq = self.shuffle_queue.enqueue([
self._values_ph, self._values_len_ph, self._signal_ph,
self._signal_len_ph
])
num_threads = 4
qr = tf.train.QueueRunner(
self.queue,
[self.queue.enqueue(self.shuffle_queue.dequeue())] * num_threads
)
tf.train.add_queue_runner(qr)
self.closing.append(
self.shuffle_queue.close(cancel_pending_enqueues=True)
)
self.summaries.append(
tf.summary.scalar(
"randomShuffleQueue_input",
self.queue.size(),
family="queue"
)
)
else:
self.enq = self.queue.enqueue([
self._values_ph, self._values_len_ph, self._signal_ph,
self._signal_len_ph
])
values_op, values_len_op, signal_op, signal_len_op = self.queue.dequeue_many(
cfg.batch_size
)
if trace:
values_op = tf.Print(
values_op, [values_op, tf.shape(values_op)],
message="values op"
)
values_len_op = tf.Print(
values_len_op,
[values_len_op, tf.shape(values_len_op)],
message="values len op"
)
sp = []
for label_idx, label_len in zip(
tf.split(values_op, cfg.batch_size),
tf.split(values_len_op, cfg.batch_size)
):
label_len = tf.squeeze(label_len, axis=0)
ind = tf.transpose(
tf.stack([
tf.zeros(shape=label_len, dtype=tf.int64),
tf.range(label_len, dtype=tf.int64),
])
)
sp.append(
tf.SparseTensor(
indices=ind,
values=tf.squeeze(label_idx, axis=0)[:label_len],
dense_shape=tf.stack([1, tf.maximum(label_len, 1)], 0)
)
)
# labels: An `int32` `SparseTensor`.
# `labels.indices[i, :] == [b, t]` means `labels.values[i]` stores
# the id for (batch b, time t).
# `labels.values[i]` must take on values in `[0, num_labels)`.
# See `core/ops/ctc_ops.cc` for more details.
# That's ok implemented
self.batch_labels = tf.sparse_concat(
axis=0, sp_inputs=sp, expand_nonconcat_dim=True
)
self.batch_labels_len = values_len_op
self.batch_dense_labels = tf.sparse_to_dense(
sparse_indices=self.batch_labels.indices,
sparse_values=self.batch_labels.values,
output_shape=self.batch_labels.dense_shape,
default_value=9,
)
self.batch_signal = signal_op
if trace:
self.batch_signal = tf.Print(
self.batch_signal, [
self.batch_dense_labels,
tf.shape(self.batch_dense_labels),
tf.shape(signal_op)
], "dense labels"
)
self.batch_signal_len = signal_len_op
def d_attn_head(seq,
out_sz,
adj_mat,
activation,
nb_nodes,
in_drop=0.0,
coef_drop=0.0,
residual=False):
with tf.name_scope('sp_attn'):
if in_drop != 0.0:
seq = tf.nn.dropout(seq, 1.0 - in_drop)
seq_fts = tf.layers.conv1d(seq, out_sz, 1, use_bias=False)
attn = []
for _ in range(out_sz):
# simplest self-attention possible
f_1 = tf.layers.conv1d(seq_fts, 1, 1)
f_2 = tf.layers.conv1d(seq_fts, 1, 1)
f_1 = tf.reshape(f_1, (nb_nodes, 1))
f_2 = tf.reshape(f_2, (nb_nodes, 1))
f_1 = adj_mat * f_1
f_2 = adj_mat * tf.transpose(f_2, [1, 0])
logits = tf.sparse_add(f_1, f_2)
lrelu = tf.SparseTensor(indices=logits.indices,
values=tf.nn.leaky_relu(logits.values),
dense_shape=logits.dense_shape)
coefs = tf.sparse_softmax(lrelu)
if coef_drop != 0.0:
coefs = tf.SparseTensor(indices=coefs.indices,
values=tf.nn.dropout(
coefs.values, 1.0 - coef_drop),
dense_shape=coefs.dense_shape)
attn.append(tf.sparse_reshape(coefs, (nb_nodes, nb_nodes, 1)))
if in_drop != 0.0:
seq_fts = tf.nn.dropout(seq_fts, 1.0 - in_drop)
# apply attention
# coefs = tf.sparse_reshape(coefs, [nb_nodes, nb_nodes])
# seq_fts = tf.squeeze(seq_fts)
# vals = tf.sparse_tensor_dense_matmul(coefs, seq_fts)
attn = tf.sparse_concat(2, attn)
seq_fts = tf.reshape(seq_fts, (nb_nodes, 1, out_sz))
vals = attn * seq_fts
vals = tf.sparse_reduce_sum(vals, axis=1)
vals = tf.expand_dims(vals, axis=0)
vals.set_shape([1, nb_nodes, out_sz])
ret = tf.contrib.layers.bias_add(vals)
# residual connection
if residual:
if seq.shape[-1] != ret.shape[-1]:
ret = ret + conv1d(seq, ret.shape[-1], 1) # activation
else:
seq_fts = ret + seq
return activation(ret) # activation
# # dimensional attention head
# def d_attn_head(seq, out_sz, adj_mat, activation, nb_nodes, in_drop=0.0, coef_drop=0.0, residual=False):
# with tf.name_scope('sp_attn'):
# if in_drop != 0.0:
# seq = tf.nn.dropout(seq, 1.0 - in_drop)
#
# seq_fts = tf.layers.conv1d(seq, out_sz, 1, use_bias=False)
#
# # simplest self-attention possible
# f_1 = tf.layers.conv1d(seq_fts, out_sz, 1)
# f_2 = tf.layers.conv1d(seq_fts, out_sz, 1)
#
# f_1 = tf.reshape(f_1, (1, nb_nodes, out_sz))
# f_2 = tf.reshape(f_2, (nb_nodes, 1, out_sz))
# adj_mat = tf.sparse_reshape(adj_mat, (nb_nodes, nb_nodes, 1))
#
# logits = adj_mat * f_1 * f_2
# # logits = adj_mat
#
# lrelu = tf.SparseTensor(indices=logits.indices,
# values=tf.exp(tf.nn.leaky_relu(logits.values)),
# dense_shape=logits.dense_shape)
# s = tf.sparse_reduce_sum(lrelu, axis=1, keep_dims=True)
# coefs = lrelu/s
#
# # drop out
# if coef_drop != 0.0:
# coefs = tf.SparseTensor(indices=coefs.indices,
# values=tf.nn.dropout(coefs.values, 1.0 - coef_drop),
# dense_shape=coefs.dense_shape)
# if in_drop != 0.0:
# seq_fts = tf.nn.dropout(seq_fts, 1.0 - in_drop)
#
# # As tf.sparse_tensor_dense_matmul expects its arguments to have rank-2,
# # here we make an assumption that our input is of batch size 1, and reshape appropriately.
# # The method will fail in all other cases!
# # coefs = tf.sparse_reshape(coefs, [nb_nodes, nb_nodes])
# seq_fts = tf.reshape(seq_fts, (nb_nodes, 1, out_sz))
# vals = coefs*seq_fts
# vals = tf.sparse_reduce_sum(vals, axis=1)
# # vals = tf.sparse_tensor_dense_matmul(coefs, seq_fts)
# vals = tf.expand_dims(vals, axis=0)
# vals.set_shape([1, nb_nodes, out_sz])
# ret = tf.contrib.layers.bias_add(vals)
#
# # residual connection
# if residual:
# if seq.shape[-1] != ret.shape[-1]:
# ret = ret + conv1d(seq, ret.shape[-1], 1) # activation
# else:
# seq_fts = ret + seq
#
# return activation(ret) # activation
def call_sparse(self, layer_input_embeds, target_embeds):
# input: [num_edge_labels, n, k, 3], [n, d1]
# [num_edge_labels, n, k, d1]
node_embeds = self.get_node_embeds_from_inputs(layer_input_embeds,
target_embeds)
split_node_embeds = tf.sparse_split(
sp_input=node_embeds,
num_split=self.network_params.num_edge_labels,
axis=0)
transformed_sparse_messages = []
for edge_idx, split_embed in enumerate(split_node_embeds):
reshaped_embed = tf.sparse_reshape(
sp_input=split_embed,
shape=[-1, self.layer_params.node_embed_size])
transformed_embeds = tf.sparse.matmul(
sp_a=reshaped_embed, b=self.edge_weights[edge_idx])
indices = tf.cast(tf.where(tf.not_equal(transformed_embeds, 0)),
tf.int64)
values = tf.gather_nd(transformed_embeds, indices)
dense_shape = tf.shape(transformed_embeds, out_type=tf.int64)
sparse_transformed = tf.SparseTensor(indices=indices,
values=values,
dense_shape=dense_shape)
#edge_bias = self.edge_biases[edge_idx] + 1e-4
#tiled_edge_bias = tf.tile(edge_bias, [tf.round(tf.size(values) / tf.size(edge_bias))])
#sparse_bias = tf.SparseTensor(indices=indices, values=tiled_edge_bias,
# dense_shape=dense_shape)
#sparse_transformed = tf.sparse_add(
# sparse_transformed, sparse_bias, thresh=1e-6)
sparse_transformed_reshaped = tf.sparse_reshape(
sp_input=sparse_transformed, shape=split_embed.dense_shape)
transformed_sparse_messages.append(sparse_transformed_reshaped)
concat_sparse_messages = tf.sparse_concat(
axis=0, sp_inputs=transformed_sparse_messages)
summed_to_get_num_incoming_edges = tf.sparse.reduce_sum(
sp_input=concat_sparse_messages, axis=[0, 3])
num_nonzero = tf.cast(
tf.reduce_any(tf.abs(summed_to_get_num_incoming_edges) > 1e-6,
axis=1,
keepdims=True), tf.float32)
# [n, d1]
# change this to divide by the number nonzero rather than reduce mean
incoming_messages = tf.sparse.reduce_sum(
sp_input=concat_sparse_messages, axis=[0, 2]) / num_nonzero
incoming_messages = tf.reshape(incoming_messages,
[-1, self.layer_params.node_embed_size])
# [n, d1]
output_embeds = self.rnn_cell(incoming_messages, target_embeds)[1]
return output_embeds
def _dnn_softmax_fn(features, targets, mode):
"""Creates the prediction, loss, and train ops.
Args:
features: A dictionary of tensors keyed by the feature name.
targets: A tensor representing the labels (in this case,
the ratings on the target movie).
mode: The execution mode, as defined in tf.contrib.learn.ModeKeys.
Returns:
ModelFnOps with the mode, prediction, loss, train_op and
output_alternatives a dictionary specifying the output for a
classification request during serving.
Raises:
ValueError: When the wrong evaluation type is specified.
"""
_ = targets # Unused variable.
class_weights = tf.get_variable(
name='class_weights',
shape=[MOVIE_VOCAB_SIZE, hparams.query_hidden_dims[-1]],
initializer=tf.contrib.layers.xavier_initializer())
class_biases = tf.get_variable(name='class_biases',
shape=[MOVIE_VOCAB_SIZE],
initializer=tf.zeros_initializer())
query_embeddings = _embed_query_features(features, mode=mode)
tf.summary.scalar('query_embeddings_zero_fraction',
tf.nn.zero_fraction(query_embeddings))
# Create layers for target features.
if mode != tf.contrib.learn.ModeKeys.INFER:
logits_layer = tf.matmul(
query_embeddings, tf.transpose(class_weights)) + class_biases
target_one_hot = tf.one_hot(
indices=features[CANDIDATE_MOVIE_ID].values,
depth=MOVIE_VOCAB_SIZE,
on_value=1.0)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=target_one_hot,
logits=logits_layer))
if mode == tf.contrib.learn.ModeKeys.TRAIN:
train_op = tf.contrib.layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=hparams.learning_rate,
optimizer=hparams.optimizer)
return tf.contrib.learn.ModelFnOps(mode=mode,
loss=loss,
train_op=train_op)
elif mode == tf.contrib.learn.ModeKeys.EVAL:
if hparams.eval_type == REGRESSION:
raise ValueError(
'eval_type must be RANKING for DNN softmax model.')
elif hparams.eval_type == RANKING:
predictions = tf.matmul(
query_embeddings,
tf.transpose(class_weights)) + class_biases
if hparams.use_ranking_candidate_movie_ids:
# Get ranking candidate movie ids to rank our candidate movie
# against.
ranking_candidate_movie_ids = features[
RANKING_CANDIDATE_MOVIE_IDS]
movies_to_rank_condition = tf.sparse_to_indicator(
tf.sparse_concat(axis=1,
sp_inputs=[
ranking_candidate_movie_ids,
features[CANDIDATE_MOVIE_ID]
]), MOVIE_VOCAB_SIZE)
predictions = tf.where(
movies_to_rank_condition, predictions,
tf.fill(tf.shape(predictions),
tf.reduce_min(predictions)))
return tf.contrib.learn.ModelFnOps(mode=mode,
predictions=predictions,
loss=loss)
elif mode == tf.contrib.learn.ModeKeys.INFER:
scores = tf.matmul(query_embeddings,
tf.transpose(class_weights)) + class_biases
rated_movie_ids = features[QUERY_RATED_MOVIE_IDS]
pruned_scores = tf.where(
tf.sparse_to_indicator(rated_movie_ids, MOVIE_VOCAB_SIZE),
tf.fill(tf.shape(scores), tf.reduce_min(scores)), scores)
predictions, output_alternatives = generate_top_k_scores_and_ids(
pruned_scores, hparams.top_k_infer)
return tf.contrib.learn.ModelFnOps(
mode=mode,
predictions=predictions,
output_alternatives=output_alternatives)
def _matrix_factorization_model_fn(features, target_ratings, mode):
"""Creates a neighborhood factorization model.
Each user is represented by a combination of embeddings of rated items,
as described in the paper: "Factorization Meets the Neighborhood:
a Multifaceted Collaborative Filtering Model - Yehuda Koren (KDD 2013)".
Args:
features: A dictionary of tensors keyed by the feature name.
target_ratings: A tensor representing the labels (in this case,
the ratings on the target movie).
mode: The execution mode, as defined in tf.contrib.learn.ModeKeys.
Returns:
ModelFnOps with the mode, prediction, loss, train_op and
output_alternatives a dictionary specifying the output for a
classification request during serving.
"""
_ = target_ratings # Unused on this model.
if hparams.embedding_weight_initializer == TRUNCATED_NORMAL:
embedding_weight_initializer = tf.truncated_normal_initializer(
stddev=0.1)
else:
embedding_weight_initializer = None
query_movie_embedding_weights = tf.get_variable(
'query_movie_ids_embedding_weights',
[MOVIE_VOCAB_SIZE, hparams.movie_embedding_dim],
initializer=embedding_weight_initializer,
regularizer=tf.contrib.layers.l2_regularizer(
hparams.l2_weight_decay))
query_movie_ids = features[QUERY_RATED_MOVIE_IDS]
query_embeddings = tf.nn.embedding_lookup_sparse(
[query_movie_embedding_weights],
query_movie_ids,
None,
combiner='sqrtn',
name='query_embedding')
query_biases, _, _ = tf.contrib.layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=make_query_feature_columns(),
num_outputs=1)
global_rating_bias = tf.get_variable(
name='global_rating_bias',
initializer=tf.constant(RATING_BIAS, dtype=tf.float32))
candidate_movie_embedding_weights = tf.get_variable(
'candidate_movie_id_embedding_weights',
[MOVIE_VOCAB_SIZE, hparams.movie_embedding_dim],
initializer=embedding_weight_initializer,
regularizer=tf.contrib.layers.l2_regularizer(
hparams.l2_weight_decay))
candidate_biases, _, _ = (
tf.contrib.layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=make_candidate_feature_columns(),
num_outputs=1))
# Create layers for target features.
if mode != tf.contrib.learn.ModeKeys.INFER:
candidate_movie_ids = features[CANDIDATE_MOVIE_ID]
candidate_embeddings = tf.nn.embedding_lookup_sparse(
[candidate_movie_embedding_weights],
candidate_movie_ids,
None,
name='candidate_embedding')
predictions = tf.reduce_sum(tf.multiply(query_embeddings,
candidate_embeddings),
1,
keep_dims=True)
if hparams.enable_bias:
biases = tf.add(query_biases, candidate_biases)
predictions = tf.add(predictions, biases)
predictions = tf.add(predictions, global_rating_bias)
labels = features[LABEL_RATING_SCORE]
loss = tf.losses.mean_squared_error(labels, predictions)
if mode == tf.contrib.learn.ModeKeys.TRAIN:
train_op = tf.contrib.layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=hparams.learning_rate,
optimizer=hparams.optimizer)
return tf.contrib.learn.ModelFnOps(mode=mode,
predictions=predictions,
loss=loss,
train_op=train_op)
elif mode == tf.contrib.learn.ModeKeys.EVAL:
if hparams.eval_type == REGRESSION:
return tf.contrib.learn.ModelFnOps(mode=mode,
predictions=predictions,
loss=loss)
elif hparams.eval_type == RANKING:
# For 'RANKING' eval, we are interested in precision@k, recall@k
# metrics which require us to compute prediction/ranking scores for
# all movies.
predictions = tf.matmul(query_embeddings,
candidate_movie_embedding_weights,
transpose_b=True)
if hparams.enable_bias:
biases = tf.add(query_biases, candidate_biases)
predictions = tf.add(predictions, biases)
if hparams.use_ranking_candidate_movie_ids:
# Get ranking candidate movie ids to rank our candidate movie
# against.
ranking_candidate_movie_ids = features[
RANKING_CANDIDATE_MOVIE_IDS]
movies_to_rank_condition = tf.sparse_to_indicator(
tf.sparse_concat(axis=1,
sp_inputs=[
ranking_candidate_movie_ids,
candidate_movie_ids
]), MOVIE_VOCAB_SIZE)
predictions = tf.where(
movies_to_rank_condition, predictions,
tf.fill(tf.shape(predictions),
tf.reduce_min(predictions)))
return tf.contrib.learn.ModelFnOps(mode=mode,
predictions=predictions,
loss=loss)
elif mode == tf.contrib.learn.ModeKeys.INFER:
scores = tf.matmul(query_embeddings,
candidate_movie_embedding_weights,
transpose_b=True)
if hparams.enable_bias:
biases = tf.add(query_biases, candidate_biases)
scores = tf.add(scores, biases)
# Eliminate already rated candates.
rated_movie_ids = features[QUERY_RATED_MOVIE_IDS]
pruned_scores = tf.where(
tf.sparse_to_indicator(rated_movie_ids, MOVIE_VOCAB_SIZE),
tf.fill(tf.shape(scores), tf.reduce_min(scores)), scores)
predictions, output_alternatives = generate_top_k_scores_and_ids(
pruned_scores, hparams.top_k_infer)
return tf.contrib.learn.ModelFnOps(
mode=mode,
predictions=predictions,
output_alternatives=output_alternatives)
def _add_stop_words(line):
STOP = tf.SparseTensor(indices=[[0,0]], values=[chr(0)], dense_shape=[1,1])
return tf.sparse_concat(axis=1, sp_inputs=[line, STOP])
def _build_attention_embedding(self, hparams):
PAIR_NUM = hparams.PAIR_NUM
batch_size = hparams.batch_size
# sparse input
# news pair的sparse表示,shape=[attention_num*batch_size, feature_num]
attention_news_x = tf.SparseTensor(
indices=self.iterator.attention_news_indices,
values=self.iterator.attention_news_values,
dense_shape=self.iterator.attention_news_shape)
#
attention_user_feat_index_batch = tf.SparseTensor(
indices=self.iterator.attention_user_indices,
values=self.iterator.attention_user_values,
dense_shape=self.iterator.attention_user_shape)
attention_user_feat_weight_batch = tf.SparseTensor(
indices=self.iterator.attention_user_indices,
values=self.iterator.attention_user_weights,
dense_shape=self.iterator.attention_user_shape)
attention_news_x_embedding = tf.sparse_tensor_dense_matmul(
attention_news_x, self.embedding)
split_attention_x_embedding = tf.split(attention_news_x_embedding,
num_or_size_splits=PAIR_NUM *
batch_size,
axis=0)
# 将news_embedding分成PAIR_NUM份
# 每份都代表一个field的embedding,size = [batch_size, embedding_dim]
attention_news_embedding_pair = []
for pair_idx in range(PAIR_NUM):
tmp = []
for i in range(batch_size):
tmp.append(split_attention_x_embedding[i * PAIR_NUM +
pair_idx])
attention_news_embedding_pair.append(tf.concat(tmp, axis=0))
split_attention_user_feat_index_batch = tf.sparse_split(
axis=0,
num_split=PAIR_NUM * batch_size,
sp_input=attention_user_feat_index_batch)
split_attention_user_feat_weight_batch = tf.sparse_split(
axis=0,
num_split=PAIR_NUM * batch_size,
sp_input=attention_user_feat_weight_batch)
attention_user_feat_index_batch_pair = []
attention_user_feat_weight_batch_pair = []
for pair_idx in range(PAIR_NUM):
tmp1 = []
tmp2 = []
for i in range(batch_size):
tmp1.append(
split_attention_user_feat_index_batch[i * PAIR_NUM +
pair_idx])
tmp2.append(
split_attention_user_feat_weight_batch[i * PAIR_NUM +
pair_idx])
attention_user_feat_index_batch_pair.append(
tf.sparse_concat(axis=0, sp_inputs=tmp1))
attention_user_feat_weight_batch_pair.append(
tf.sparse_concat(axis=0, sp_inputs=tmp2))
# #news embedding
nn_input = []
attention_nn_params = []
for idx in range(PAIR_NUM):
attention_embedding, attention_module_params = self._build_attention_pair(
attention_user_feat_index_batch_pair[idx].indices,
attention_user_feat_index_batch_pair[idx].dense_shape,
attention_user_feat_index_batch_pair[idx],
attention_user_feat_weight_batch_pair[idx],
attention_news_embedding_pair[idx], self.embedding, idx,
hparams)
field_nn_input = tf.concat(
[attention_embedding, attention_news_embedding_pair[idx]], 1)
nn_input.append(field_nn_input)
attention_nn_params.extend(attention_module_params)
#
for param in attention_nn_params:
self.layer_params.append(param)
concat_nn_input = tf.concat(nn_input, 1)
return concat_nn_input, attention_nn_params
def stack(self, arr): concat_arr = [tf.sparse_reshape(x, [1] + self.shape) for x in arr] return tf.sparse_concat(0, concat_arr)
def _matrix_factorization_model_fn(features, target_ratings, mode):
"""Creates a neighborhood factorization model.
Each user is represented by a combination of embeddings of rated items,
as described in the paper: "Factorization Meets the Neighborhood:
a Multifaceted Collaborative Filtering Model - Yehuda Koren (KDD 2013)".
Args:
features: A dictionary of tensors keyed by the feature name.
target_ratings: A tensor representing the labels (in this case,
the ratings on the target movie).
mode: The execution mode, as defined in tf.contrib.learn.ModeKeys.
Returns:
ModelFnOps with the mode, prediction, loss, train_op and
output_alternatives a dictionary specifying the output for a
classification request during serving.
"""
_ = target_ratings # Unused on this model.
if hparams.embedding_weight_initializer == TRUNCATED_NORMAL:
embedding_weight_initializer = tf.truncated_normal_initializer(stddev=0.1)
else:
embedding_weight_initializer = None
query_movie_embedding_weights = tf.get_variable(
'query_movie_ids_embedding_weights',
[MOVIE_VOCAB_SIZE, hparams.movie_embedding_dim],
initializer=embedding_weight_initializer,
regularizer=tf.contrib.layers.l2_regularizer(hparams.l2_weight_decay))
query_movie_ids = features[QUERY_RATED_MOVIE_IDS]
query_embeddings = tf.nn.embedding_lookup_sparse(
[query_movie_embedding_weights],
query_movie_ids,
None,
combiner='sqrtn',
name='query_embedding')
query_biases, _, _ = tf.contrib.layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=make_query_feature_columns(),
num_outputs=1)
global_rating_bias = tf.get_variable(
name='global_rating_bias',
initializer=tf.constant(RATING_BIAS, dtype=tf.float32))
candidate_movie_embedding_weights = tf.get_variable(
'candidate_movie_id_embedding_weights',
[MOVIE_VOCAB_SIZE, hparams.movie_embedding_dim],
initializer=embedding_weight_initializer,
regularizer=tf.contrib.layers.l2_regularizer(hparams.l2_weight_decay))
candidate_biases, _, _ = (
tf.contrib.layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=make_candidate_feature_columns(),
num_outputs=1))
# Create layers for target features.
if mode != tf.contrib.learn.ModeKeys.INFER:
candidate_movie_ids = features[CANDIDATE_MOVIE_ID]
candidate_embeddings = tf.nn.embedding_lookup_sparse(
[candidate_movie_embedding_weights],
candidate_movie_ids,
None,
name='candidate_embedding')
predictions = tf.reduce_sum(tf.multiply(
query_embeddings, candidate_embeddings), 1, keep_dims=True)
if hparams.enable_bias:
biases = tf.add(query_biases, candidate_biases)
predictions = tf.add(predictions, biases)
predictions = tf.add(predictions, global_rating_bias)
labels = features[LABEL_RATING_SCORE]
loss = tf.losses.mean_squared_error(labels, predictions)
if mode == tf.contrib.learn.ModeKeys.TRAIN:
train_op = tf.contrib.layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=hparams.learning_rate,
optimizer=hparams.optimizer)
return tf.contrib.learn.ModelFnOps(
mode=mode, predictions=predictions, loss=loss, train_op=train_op)
elif mode == tf.contrib.learn.ModeKeys.EVAL:
if hparams.eval_type == REGRESSION:
return tf.contrib.learn.ModelFnOps(
mode=mode, predictions=predictions, loss=loss)
elif hparams.eval_type == RANKING:
# For 'RANKING' eval, we are interested in precision@k, recall@k
# metrics which require us to compute prediction/ranking scores for
# all movies.
predictions = tf.matmul(query_embeddings,
candidate_movie_embedding_weights,
transpose_b=True)
if hparams.enable_bias:
biases = tf.add(query_biases, candidate_biases)
predictions = tf.add(predictions, biases)
if hparams.use_ranking_candidate_movie_ids:
# Get ranking candidate movie ids to rank our candidate movie
# against.
ranking_candidate_movie_ids = features[RANKING_CANDIDATE_MOVIE_IDS]
movies_to_rank_condition = tf.sparse_to_indicator(
tf.sparse_concat(
axis=1,
sp_inputs=[ranking_candidate_movie_ids,
candidate_movie_ids]),
MOVIE_VOCAB_SIZE)
predictions = tf.where(movies_to_rank_condition, predictions,
tf.fill(
tf.shape(predictions),
tf.reduce_min(predictions)))
return tf.contrib.learn.ModelFnOps(
mode=mode,
predictions=predictions,
loss=loss)
elif mode == tf.contrib.learn.ModeKeys.INFER:
scores = tf.matmul(query_embeddings,
candidate_movie_embedding_weights,
transpose_b=True)
if hparams.enable_bias:
biases = tf.add(query_biases, candidate_biases)
scores = tf.add(scores, biases)
# Eliminate already rated candates.
rated_movie_ids = features[QUERY_RATED_MOVIE_IDS]
pruned_scores = tf.where(
tf.sparse_to_indicator(rated_movie_ids, MOVIE_VOCAB_SIZE),
tf.fill(tf.shape(scores), tf.reduce_min(scores)), scores)
predictions, output_alternatives = generate_top_k_scores_and_ids(
pruned_scores, hparams.top_k_infer)
return tf.contrib.learn.ModelFnOps(
mode=mode,
predictions=predictions,
output_alternatives=output_alternatives)
X_xyz = tf.constant(X_xyz_np,tf.float32,shape = (BATCH,Sz*Sz,3))
Adj_sp = tf.SparseTensor(indices=wsp_indices_list, values=wsp_values_list, dense_shape=[Sz*Sz, Sz*Sz])
W_mat_np = np.float32(np.ones((n_spatial_filters*n_features,n_features)))
W_mat = tf.constant(W_mat_np,tf.float32,shape = (n_spatial_filters*n_features,n_features))
#Build geodesic sparse matrices
W_sp_list = []
for b in range(BATCH):
for dim in range(3):
W_sp_ = tf.sparse_add((X_xyz[b,:,dim:dim+1]*Adj_sp), Adj_sp*tf.transpose(-1*X_xyz[b,:,dim:dim+1]))
W_sp_ = tf.sparse_reshape(W_sp_, (Sz*Sz,Sz*Sz,1))
W_sp_ = tf.sparse_concat(axis = 2,sp_inputs = [W_sp_ for i in range(n_spatial_filters)])
Adj_sp_re = tf.sparse_reshape(Adj_sp, (Sz*Sz,Sz*Sz,1))
Adj_sp_re = tf.sparse_concat(axis = 2,sp_inputs = [Adj_sp_re for i in range(n_spatial_filters)])
alpha_ = tf.reshape(Alfas[:,dim],(1,1,-1))
sigma_ = tf.reshape(Sigmas[:,dim],(1,1,-1))
W_sp_ = tf.sparse_add(W_sp_,Adj_sp_re*(-1*alpha_))
W_sp_ = tf.SparseTensor(indices = W_sp_.indices, values = W_sp_.values**2,dense_shape=[Sz*Sz, Sz*Sz,n_spatial_filters])
W_sp_ = W_sp_*(-0.5/sigma_**2)
if dim == 0:
W_sp = W_sp_
else:
W_sp = tf.sparse_add(W_sp,W_sp_)
def _dnn_softmax_fn(features, targets, mode):
"""Creates the prediction, loss, and train ops.
Args:
features: A dictionary of tensors keyed by the feature name.
targets: A tensor representing the labels (in this case,
the ratings on the target movie).
mode: The execution mode, as defined in tf.contrib.learn.ModeKeys.
Returns:
ModelFnOps with the mode, prediction, loss, train_op and
output_alternatives a dictionary specifying the output for a
classification request during serving.
Raises:
ValueError: When the wrong evaluation type is specified.
"""
_ = targets # Unused variable.
class_weights = tf.get_variable(
name='class_weights',
shape=[MOVIE_VOCAB_SIZE, hparams.query_hidden_dims[-1]],
initializer=tf.contrib.layers.xavier_initializer())
class_biases = tf.get_variable(
name='class_biases',
shape=[MOVIE_VOCAB_SIZE],
initializer=tf.zeros_initializer())
query_embeddings = _embed_query_features(features, mode=mode)
tf.summary.scalar('query_embeddings_zero_fraction',
tf.nn.zero_fraction(query_embeddings))
# Create layers for target features.
if mode != tf.contrib.learn.ModeKeys.INFER:
logits_layer = tf.matmul(
query_embeddings, tf.transpose(class_weights)) + class_biases
target_one_hot = tf.one_hot(
indices=features[CANDIDATE_MOVIE_ID].values,
depth=MOVIE_VOCAB_SIZE,
on_value=1.0)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
labels=target_one_hot, logits=logits_layer))
if mode == tf.contrib.learn.ModeKeys.TRAIN:
train_op = tf.contrib.layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=hparams.learning_rate,
optimizer=hparams.optimizer)
return tf.contrib.learn.ModelFnOps(
mode=mode, loss=loss, train_op=train_op)
elif mode == tf.contrib.learn.ModeKeys.EVAL:
if hparams.eval_type == REGRESSION:
raise ValueError('eval_type must be RANKING for DNN softmax model.')
elif hparams.eval_type == RANKING:
predictions = tf.matmul(
query_embeddings, tf.transpose(class_weights)) + class_biases
if hparams.use_ranking_candidate_movie_ids:
# Get ranking candidate movie ids to rank our candidate movie
# against.
ranking_candidate_movie_ids = features[RANKING_CANDIDATE_MOVIE_IDS]
movies_to_rank_condition = tf.sparse_to_indicator(
tf.sparse_concat(
axis=1,
sp_inputs=[ranking_candidate_movie_ids,
features[CANDIDATE_MOVIE_ID]]),
MOVIE_VOCAB_SIZE)
predictions = tf.where(movies_to_rank_condition, predictions,
tf.fill(
tf.shape(predictions),
tf.reduce_min(predictions)))
return tf.contrib.learn.ModelFnOps(
mode=mode, predictions=predictions, loss=loss)
elif mode == tf.contrib.learn.ModeKeys.INFER:
scores = tf.matmul(
query_embeddings, tf.transpose(class_weights)) + class_biases
rated_movie_ids = features[QUERY_RATED_MOVIE_IDS]
pruned_scores = tf.where(
tf.sparse_to_indicator(rated_movie_ids, MOVIE_VOCAB_SIZE),
tf.fill(tf.shape(scores), tf.reduce_min(scores)), scores)
predictions, output_alternatives = generate_top_k_scores_and_ids(
pruned_scores, hparams.top_k_infer)
return tf.contrib.learn.ModelFnOps(
mode=mode,
predictions=predictions,
output_alternatives=output_alternatives)
def define_graph(self):
filenames = glob(os.path.join("../data", "cnn_*.tfrecords"))
min_after_dequeue = 1000
filename_queue = tf.train.string_input_producer(filenames)
document, question, answer, document_shape, question_shape, answer_shape = read_tf_record_file(
filename_queue)
d_batch, q_batch, ans_batch, document_shape_batch, question_shape_batch, answer_shape_batch = tf.train.shuffle_batch(
[
document, question, answer, document_shape, question_shape,
answer_shape
],
batch_size=self.hparams.batch_size,
capacity=min_after_dequeue * 3 + 1,
min_after_dequeue=min_after_dequeue)
d_q_batch = tf.sparse_concat(
axis=1,
sp_inputs=[d_batch, q_batch],
)
dense_d_q_batch = tf.sparse_to_dense(
sparse_indices=d_q_batch.indices,
output_shape=d_q_batch.dense_shape,
sparse_values=d_q_batch.values,
default_value=0,
validate_indices=True,
name=None)
dens_ans_batch = tf.sparse_to_dense(sparse_indices=ans_batch.indices,
output_shape=ans_batch.dense_shape,
sparse_values=ans_batch.values,
default_value=0,
validate_indices=True,
name=None)
d_q_lengths = tf.reduce_sum(tf.concat([
tf.reshape(document_shape_batch, (self.hparams.batch_size, 1)),
tf.reshape(question_shape_batch, (self.hparams.batch_size, 1))
],
axis=1),
axis=1)
self.embedding = tf.get_variable(
"embedding",
[self.vocab_size, self.hparams.number_of_hidden_units],
trainable=True)
tf.logging.info(self.embedding.get_shape())
self.inputs = dense_d_q_batch
self.y = dens_ans_batch[:,
0] #tf.placeholder(tf.float32, [self.hparams.self.hparams.batch_size, self.vocab_size])
#unstacked_inputs = tf.unstack(self.inputs,axis=1)
embedded_inputs = [
tf.nn.embedding_lookup(self.embedding, self.inputs[i])
for i in range(self.hparams.batch_size)
]
#embedded_inputs = tf.stack(embedded_inputs)
tf.summary.histogram("embeddings", self.embedding)
self.__build_deep_lstm_cell()
states_series, current_state = tf.nn.dynamic_rnn(
cell=self.stacked_cell,
inputs=tf.stack(embedded_inputs),
sequence_length=d_q_lengths,
initial_state=None,
dtype=tf.float32,
parallel_iterations=None,
swap_memory=False,
time_major=False,
scope=None)
self.batch_states = [
layer_state[1] for layer_state in tf.unstack(current_state)
] # tf.stack(states)
self.output_size = self.hparams.depth * self.hparams.number_of_hidden_units
"""startings = tf.concat([
tf.zeros((self.hparams.batch_size,1),dtype=tf.int64),
tf.reshape(d_q_lengths,(self.hparams.batch_size,1)) - 1,
tf.zeros((self.hparams.batch_size,1),dtype=tf.int64)],axis=1)
outputs = [self.batch_states[0][i,d_q_lengths[i],:] for i in range(self.hparams.batch_size)]"""
outputs = tf.concat(self.batch_states, axis=1)
tf.logging.set_verbosity(tf.logging.INFO)
tf.logging.info(outputs)
tf.logging.info(self.batch_states)
tf.logging.info(current_state[0])
self.outputs = tf.reshape(outputs,
[self.hparams.batch_size, self.output_size])
self.W = tf.get_variable("W", [
self.output_size,
self.vocab_size,
],
trainable=True)
tf.summary.histogram("weights", self.W)
tf.summary.histogram("output", self.outputs)
self.y_ = tf.matmul(self.outputs, self.W)
tf.logging.info(self.y_)
cross_ent = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.y_, labels=self.y)
self.train_loss = (tf.reduce_sum(cross_ent) / self.hparams.batch_size)
tf.summary.scalar("loss", tf.reduce_mean(self.train_loss))
tf.logging.info(tf.argmax(self.y_, 1))
correct_prediction = tf.equal(self.y, tf.argmax(self.y_, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
tf.summary.scalar("accuracy", self.accuracy)
def __call__(self, inputs, state, scope=None):
"""Gated recurrent unit (GRU) with Graph Convolution.
:param inputs: (B, num_nodes * input_dim)
:return
- Output: A `2-D` tensor with shape `[batch_size x self.output_size]`.
- New state: Either a single `2-D` tensor, or a tuple of tensors matching
the arity and shapes of `state`
"""
with tf.variable_scope(scope or "dcgru_cell"):
# inputs = tf.ones([self._num_nodes, self._num_nodes])
# x_shape = tf.concat([inputs, state], axis=1)
# input_shape = x_shape.get_shape().as_list()
# input_rnn = tf.SparseTensor(self.index, self.value, [self._num_nodes, self._num_nodes])
state = tf.reshape(state, [self._num_nodes, -1])
state_shape = state.get_shape().as_list()
input_dim = state_shape[1] + self._num_nodes
zero = tf.constant(0, dtype=tf.float32)
where = tf.not_equal(state, zero)
indices = tf.where(where)
values = tf.gather_nd(state, indices)
sparse_state = tf.SparseTensor(indices, values, state.shape)
x1 = tf.sparse_concat(sp_inputs=[self.input_rnn, sparse_state], axis=1)
non_zeros_feat = tf.size(x1.values)
with tf.variable_scope("gates"): # Reset gate
# We start with bias of 1.0 to not reset and not update.
r = GraphConvolution_saprse(input_dim=input_dim,
output_dim=self._num_units,
adj=self.adj,
act=tf.nn.sigmoid,
dropout=self.dropout,
non_zero = non_zeros_feat,
bias=self.bias,
logging=self.logging)(x1)
with tf.variable_scope("gates"): # Update gate
u = GraphConvolution_saprse(input_dim=input_dim,
output_dim=self._num_units,
adj=self.adj,
act=tf.nn.sigmoid,
dropout=self.dropout,
non_zero=non_zeros_feat,
bias=self.bias,
logging=self.logging)(x1)
with tf.variable_scope("candidate"):
state_r = r * state
where_r = tf.not_equal(state_r, zero)
indices_r = tf.where(where_r)
values_r = tf.gather_nd(state_r, indices_r)
sparse_state_r = tf.SparseTensor(indices_r, values_r, state_r.shape)
x2 = tf.sparse_concat(sp_inputs=[self.input_rnn, sparse_state_r], axis=1)
c = GraphConvolution_saprse(input_dim=input_dim,
output_dim=self._num_units,
adj=self.adj,
act=tf.nn.sigmoid,
dropout=self.dropout,
non_zero=non_zeros_feat,
bias=self.bias,
logging=self.logging)(x2)
h = u * state + (1 - u) * c
h = tf.reshape(h, [1, -1])
output = new_state = h
return output, new_state
tf.to_double() tf.to_float() tf.to_int32() tf.to_int64() tf.trace() tf.trainable_variables() tf.transpose() tf.truncated_normal() tf.truediv() tf.sparse_transpose() tf.sparse_tensor_dense_matmul() tf.sparse_accumulator_apply_gradient() tf.sparse_accumulator_take_gradient() tf.sparse_add() tf.sparse_concat() tf.sparse_conditional_accumulator() tf.sparse_mask() tf.sparse_matmul() tf.sparse_maximum() tf.sparse_merge() tf.sparse_minimum() tf.sparse_reduce_max() tf.sparse_reduce_max_sparse() tf.reduce_all() tf.reduce_any() tf.reduce_join() tf.reduce_logsumexp() tf.reduce_max()
def sp_hete_attn_head(seq,
out_sz,
adj_mat,
adj_type,
edge_list,
activation,
nb_nodes,
in_drop=0.0,
coef_drop=0.0,
residual=False):
# input adjacency matrices are TRANSPOSED before feeding!
# nb_nodes_j * nb_nodes_i
with tf.name_scope('sp_hete_attn'):
if in_drop != 0.0:
seq = [tf.nn.dropout(seq_i, 1.0 - in_drop) for seq_i in seq]
# seq_fts[j][i]: hidden features from group i to group j, center node is j
# 1 * nb_nodes_i * out_sz_j
W_list = [[None for _ in seq] for _ in seq]
seq_fts = [[None for _ in seq] for _ in seq]
W_inv_list = [
tf.Variable(
tf.glorot_uniform_initializer()(shape=(1, out_sz,
int(s.shape[2]))))
for s in seq
]
for W_type in adj_type:
i, j = W_type
if W_list[i][j] is None:
W_list[i][j] = tf.Variable(tf.glorot_uniform_initializer()(
shape=(1, int(seq[i].shape[2]), out_sz)))
seq_fts[j][i] = tf.matmul(seq[i], W_list[i][j])
for i in range(len(seq)):
for j in range(len(seq)):
if W_list[i][j] is not None and W_list[j][i] is not None:
loss1 = seq_fts[j][i] @ (
W_inv_list[j] @ W_list[j][i]) - seq_fts[i][i]
# 1 * nb_nodes_i * out_sz_i
loss1 = loss1 * loss1
loss1 = tf.reduce_sum(loss1) / nb_nodes[i]
tf.add_to_collection('loss_loop', loss1)
for j in range(len(seq)):
if out_sz >= seq[j].shape[1]:
loss2 = W_list[j][j] @ W_inv_list[j] - tf.eye(seq[j].shape[2])
else:
loss2 = W_inv_list[j] @ W_list[j][j] - tf.eye(out_sz)
loss2 = loss2 * loss2
loss2 = tf.reduce_sum(loss2)
tf.add_to_collection('loss_inv', loss2)
attn_biases = [None for _ in adj_type]
for dir_edge in edge_list:
attn_bias = tf.Variable(tf.random_normal(shape=(1, out_sz)))
attn_biases[dir_edge[0]] = attn_bias
if len(dir_edge) == 2:
attn_biases[dir_edge[1]] = -attn_bias
# for out_sz_j in out_sz
coefs_lists = [[] for _ in range(len(seq))]
seq_fts_lists = [[] for _ in range(len(seq))]
# simplest self-attention possible
for adj_ij, type_ij, attn_bias in zip(adj_mat, adj_type, attn_biases):
# adj_ij is transposed, nb_nodes_j * nb_nodes_i
i, j = type_ij
f_1 = tf.reshape(seq_fts[j][j], (nb_nodes[j], out_sz))
f_1 = tf.gather(f_1, adj_ij.indices[:, 0])
f_2 = tf.reshape(seq_fts[j][i], (nb_nodes[i], out_sz))
if attn_bias is not None:
f_2 = f_2 + attn_bias
f_2 = tf.gather(f_2, adj_ij.indices[:, 1])
f = tf.reduce_sum(tf.multiply(f_1, f_2), 1)
coefs = tf.SparseTensor(indices=adj_ij.indices,
values=tf.nn.leaky_relu(f),
dense_shape=adj_ij.dense_shape)
if coef_drop != 0.0:
coefs = tf.SparseTensor(indices=coefs.indices,
values=tf.nn.dropout(
coefs.values, 1.0 - coef_drop),
dense_shape=coefs.dense_shape)
coefs_lists[j].append(coefs) # transposed, nb_nodes_j * nb_nodes_i
if in_drop != 0.0:
seq_fts_ij = tf.nn.dropout(seq_fts[j][i], 1.0 - in_drop)
seq_fts_lists[j].append(
tf.squeeze(seq_fts_ij)) # nb_nodes_i * out_sz_j
# As tf.sparse_tensor_dense_matmul expects its arguments to have rank-2,
# here we make an assumption that our input is of batch size 1, and reshape appropriately.
# The method will fail in all other cases!
coefs = [tf.sparse_concat(1, coefs_list) for coefs_list in coefs_lists]
coefs = [tf.sparse_softmax(coef) for coef in coefs]
seq_fts = [
tf.concat(seq_fts_list, 0) for seq_fts_list in seq_fts_lists
]
vals = [
tf.sparse_tensor_dense_matmul(coef, seq_ft)
for coef, seq_ft in zip(coefs, seq_fts)
]
# nb_nodes_j * out_sz_j
vals = [tf.expand_dims(val, axis=0) for val in vals]
for i, val in enumerate(vals):
val.set_shape([1, nb_nodes[i], out_sz])
ret = [tf.contrib.layers.bias_add(val) for val in vals]
# residual connection
if residual:
ret2 = []
for r, s in zip(ret, seq):
if s.shape[-1] != r.shape[-1]:
ret2.append(r + tf.layers.conv1d(s, r.shape[-1], 1))
else:
ret2.append(r + s)
ret = ret2
ret = [activation(r) for r in ret]
return ret # activation
def sp_hete_attn_head(seq,
out_sz,
adj_mat,
adj_type,
activation,
nb_nodes,
in_drop=0.0,
coef_drop=0.0,
residual=False):
# input adjacency matrices are TRANSPOSED before feeding!
with tf.name_scope('sp_hete_attn'):
if in_drop != 0.0:
seq = [tf.nn.dropout(seq_i, 1.0 - in_drop) for seq_i in seq]
# seq_fts[j][i]: hidden features from group i to group j, center node is j
# 1 * nb_nodes_i * out_sz_j
seq_fts = [
[
tf.layers.conv1d(
seq_i,
out_sz, # out_sz_j
1,
use_bias=False) for seq_i in seq
] for _ in seq
]
# for out_sz_j in out_sz
coefs_lists = [[] for _ in range(len(seq))]
seq_fts_lists = [[] for _ in range(len(seq))]
# simplest self-attention possible
for adj_ij, type_ij in zip(adj_mat, adj_type):
# transposed, # nb_nodes_j * nb_nodes_i
i, j = type_ij
f_1 = tf.layers.conv1d(seq_fts[j][j], 1, 1)
f_2 = tf.layers.conv1d(seq_fts[j][i], 1, 1)
f_1 = tf.reshape(f_1, (nb_nodes[j], 1))
f_2 = tf.reshape(f_2, (nb_nodes[i], 1))
f_1 = adj_ij * f_1
f_2 = adj_ij * tf.transpose(f_2, [1, 0])
logits = tf.sparse_add(f_1, f_2) # nb_nodes_j * nb_nodes_i
coefs = tf.SparseTensor(indices=logits.indices,
values=tf.nn.leaky_relu(logits.values),
dense_shape=logits.dense_shape)
# coefs = tf.sparse_softmax(lrelu)
if coef_drop != 0.0:
coefs = tf.SparseTensor(indices=coefs.indices,
values=tf.nn.dropout(
coefs.values, 1.0 - coef_drop),
dense_shape=coefs.dense_shape)
coefs_lists[j].append(coefs) # transposed, nb_nodes_j * nb_nodes_i
if in_drop != 0.0:
seq_fts_ij = tf.nn.dropout(seq_fts[j][i], 1.0 - in_drop)
seq_fts_lists[j].append(
tf.squeeze(seq_fts_ij)) # nb_nodes_i * out_sz_j
# As tf.sparse_tensor_dense_matmul expects its arguments to have rank-2,
# here we make an assumption that our input is of batch size 1, and reshape appropriately.
# The method will fail in all other cases!
# coefs = tf.sparse_reshape(coefs, [nb_nodes, nb_nodes])
coefs = [tf.sparse_concat(1, coefs_list) for coefs_list in coefs_lists]
coefs = [tf.sparse_softmax(coef) for coef in coefs]
# seq_fts = tf.squeeze(seq_fts)
seq_fts = [
tf.concat(seq_fts_list, 0) for seq_fts_list in seq_fts_lists
]
vals = [
tf.sparse_tensor_dense_matmul(coef, seq_ft)
for coef, seq_ft in zip(coefs, seq_fts)
]
# nb_nodes_j * out_sz_j
vals = [tf.expand_dims(val, axis=0) for val in vals]
for i, val in enumerate(vals):
val.set_shape([1, nb_nodes[i], out_sz])
ret = [tf.contrib.layers.bias_add(val) for val in vals]
# residual connection
if residual:
ret2 = []
for r, s in zip(ret, seq):
if s.shape[-1] != r.shape[-1]:
ret2.append(r + tf.layers.conv1d(s, r.shape[-1], 1))
else:
ret2.append(r + s)
ret = ret2
ret = [activation(r) for r in ret]
return ret # activation
def body(index, words):
next_word = tf.sparse_slice(line, tf.to_int64(index), [1, 1]).values
next_word = tf.string_split(next_word, delimiter='')
words = tf.sparse_concat(axis=0, sp_inputs=[words, next_word], expand_nonconcat_dim=True)
return index+[0, 1], words