def _parse_example_proto(example_serialized):
# parse record
# decode jpeg
# compute the length of the caption
feature_map = {
'image/encoded': tf.FixedLenFeature([], dtype=tf.string),
'image/path': tf.FixedLenFeature([], dtype=tf.string),
'labels': tf.VarLenFeature(dtype=tf.int64),
}
features = tf.parse_single_example(example_serialized, feature_map)
image = tf.image.decode_jpeg(features['image/encoded'],
channels=3,
try_recover_truncated=True)
labels = features['labels']
label_map = tf.sparse_to_indicator(labels, NUM_CLASSES)
label_map.set_shape([NUM_CLASSES])
label_pairs = _lm2lp(label_map)
label_map = tf.cast(label_map, tf.float32)
labels = tf.sparse_tensor_to_dense(labels)
random_label = tf.random_shuffle(labels)[0]
labels = {}
labels['label_map'] = label_map
labels['label_pair'] = label_pairs
labels['random_label'] = random_label
labels['image'] = image
labels['path'] = features['image/path']
return labels
def prepare_serialized_examples(self, serialized_example):
"""
build the input tensor from serialized_example
:param serialized_example:
:return:
"""
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {
'video_id': tf.FixedLenFeature([], tf.string),
'labels': tf.VarLenFeature(tf.int64)
}
for feature_index in range(num_features):
feature_map[
self.feature_names[feature_index]] = tf.FixedLenFeature(
self.feature_sizes[feature_index], tf.float32)
features = tf.parse_example(serialized_example, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_class)
labels.set_shape([None, self.num_class])
concatenated_features = tf.concat(
[features[feature_name] for feature_name in self.feature_names], 1)
return features["video_id"], concatenated_features, labels, tf.ones(
[tf.shape(serialized_example)[0]])
def prepare_serialized_examples(self, serialized_examples):
"""Parse a single video-level TF Example."""
# set the mapping from the fields to data types in the proto
#num_features = len(self.feature_names)
num_features = 1
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format(
len(self.feature_names), len(self.feature_sizes))
feature_map = {
#"video_id": tf.io.FixedLenFeature([], tf.string),
"labels": tf.io.VarLenFeature(tf.int64)
}
for feature_index in range(num_features):
feature_map[
self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat(
[features[feature_name] for feature_name in self.feature_names], 1)
output_dict = {
"video_ids": features["video_id"],
"video_matrix": concatenated_features,
"labels": labels,
"num_frames": tf.ones([tf.shape(serialized_examples)[0]])
}
return output_dict
def prepare_serialized_examples(self, serialized_examples):
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {
"id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)
}
for feature_index in range(num_features):
feature_map[
self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat(
[features[feature_name] for feature_name in self.feature_names], 1)
return features["id"], concatenated_features, labels, tf.ones(
[tf.shape(serialized_examples)[0]])
def read_and_decode(filename_queue, num_classes):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
context_features = {
'query_plan_row_length': tf.FixedLenFeature([], tf.int64),
'env_segment_number': tf.FixedLenFeature([], tf.int64),
'env_segment_cpu': tf.FixedLenFeature([], tf.int64),
'env_segment_mem': tf.FixedLenFeature([], tf.int64),
'env_segment_storage': tf.FixedLenFeature([], tf.int64)
}
sequence_features = {
'query_plan_ops': tf.VarLenFeature(tf.string),
# 'query_plan_ops_seq': tf.VarLenFeature(tf.int64),
'query_table_size': tf.VarLenFeature(tf.float32)
}
context_parsed, sequence_parsed = tf.parse_single_sequence_example(
serialized=serialized_example,
context_features=context_features,
sequence_features=sequence_features
# Defaults are not specified since both keys are required.
)
query_plan_ops = tf.sparse_to_indicator(sequence_parsed['query_plan_ops'], num_classes)
query_plan_ops.set_shape([None, num_classes])
return context_parsed, sequence_features['query_table_size'], query_plan_ops
def decode(filename_queue):
# Create TFRecords reader
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
# Feature keys in TFRecords example
features = tf.parse_single_example(serialized_example,
features={
'id':
tf.FixedLenFeature([], tf.string),
'vector':
tf.FixedLenFeature([], tf.string),
'label':
tf.VarLenFeature(tf.int64)
})
video_id = features['id']
# Decode vector and pad to fixed size
vector = tf.decode_raw(features['vector'], tf.float32)
vector = tf.reshape(vector, [-1, 300])
vector = tf.pad(vector, [[0, 40 - tf.shape(vector)[0]], [0, 0]])
vector.set_shape([40, 300])
# Get label index
label = tf.sparse_to_indicator(features['label'], 4716)
label.set_shape([4716])
label = tf.cast(label, tf.float32)
return video_id, vector, label
def prepare_reader(self, filename_queue, batch_size=1024):
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
feature_map = {
"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64),
"scores": tf.FixedLenFeature([5 * 4716], tf.float32)
}
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], 4716)
labels.set_shape([None, self.num_classes])
concatenated_features = features['scores']
sparse_labels, label_weights, input_weights = labels, labels, labels
# sparse_labels = features["labels"].values
if self.num_max_labels == 4716:
sparse_labels, label_weights = tf.py_func(gen_sparse_label_batch,
[labels],
[tf.int64, tf.float32])
sparse_labels = tf.reshape(sparse_labels,
[-1, self.num_max_labels])
label_weights = tf.reshape(label_weights,
[-1, self.num_max_labels])
return features[
"video_id"], concatenated_features, labels, sparse_labels, tf.ones(
[tf.shape(serialized_examples)[0]
]), label_weights, input_weights
def _load(self, splitname: str) -> Tuple[tf.Tensor, tf.Tensor, tf.Tensor]:
# create the dataset
filenames = list(self._discover_filenames(splitname))
dataset = tf.data.TFRecordDataset(filenames)
# Parse every sample of the dataset
dataset = dataset.map(self._read_record, num_parallel_calls=8)
# Filter only certain data
dataset = dataset.filter(self._only_music_genre_samples)
# Set the batchsize
dataset = dataset.batch(self.batch_size)
# Start over when we are finished reading the dataset
if self.repeat:
dataset = dataset.repeat()
# Create an iterator
iterator = dataset.make_one_shot_iterator()
# Create your tf representation of the iterator
video_id, features, labels = iterator.get_next()
# Set a fixed shape of features (the first dimension is the batch)
features = tf.reshape(features, [-1, 10, 128])
# Create a one hot array for multilabel classification
labels = tf.sparse_to_indicator(labels, NUM_TOTAL_CLASSES)
# Only take the required music genre classes
labels = tf.gather(labels, self.class_indexes, axis=1)
# cast to a supported data type
labels = tf.cast(labels, tf.float32)
# return ids, features and labels
return video_id, features, labels
def prepare_serialized_examples(self, serialized_examples):
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {
"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)
}
for feature_index in range(num_features):
feature_map[
self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
### Newly
raw_labels = features["labels"]
raw_coarse = tf.SparseTensor(indices=raw_labels.indices,
values=tf.reshape(
tf.gather(
tf.constant(self.label_belongs,
dtype=tf.int64),
raw_labels.values), [-1]),
dense_shape=raw_labels.dense_shape)
coarse_labels = tf.sparse_to_indicator(raw_coarse,
self.num_coarse_classes,
name='coarse_transfer')
coarse_labels.set_shape([None, self.num_coarse_classes])
###
concatenated_features = tf.concat(
[features[feature_name] for feature_name in self.feature_names], 1)
# return features["video_id"], concatenated_features, labels, tf.ones([tf.shape(serialized_examples)[0]])
### Newly
return features[
"video_id"], concatenated_features, labels, coarse_labels, tf.ones(
[tf.shape(serialized_examples)[0]])
def test_t3():
print("----------------------")
sp1 = tf.SparseTensor(indices=[[0, 0], [0, 1], [1, 0]],
values=[1, 123, 1],
dense_shape=(2, 3)) #
sp1_dense = tf.sparse_tensor_to_dense(sp1, default_value=-1)
indicate = tf.sparse_to_indicator(sp1, vocab_size=123 + 1)
sess = tf.Session()
print("sp1:", sess.run(sp1))
print("sp_dense:", sess.run(sp1_dense))
print("indicate:", sess.run(indicate))
def top_k_mask(self, logits, k=20):
values, indices = tf.nn.top_k(logits, k=k)
sp_idx = tf.where(indices > -1)
sp_val = tf.gather_nd(indices, sp_idx)
mask = tf.sparse_to_indicator(sp_input=tf.SparseTensor(
indices=sp_idx,
values=sp_val,
dense_shape=logits.get_shape().as_list()),
vocab_size=FLAGS.vocab_size)
mask = tf.cast(mask, dtype=tf.float32) / k
print("mask", mask)
return mask
def prepare_serialized_examples(serialized_examples):
# set the mapping from the fields to data types in the proto
feature_map = {"id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64),
"mean_rgb": tf.FixedLenFeature([1024], tf.float32),
"mean_audio": tf.FixedLenFeature([128], tf.float32)}
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], num_classes)
labels.set_shape([None, num_classes])
concatenated_features = tf.concat([features["mean_rgb"], features["mean_audio"]], axis=1)
return concatenated_features, labels
def test_sparse():
"""
测试SparseTensor。
:return:
"""
# 位置索引
idx = [[0, 0, 0], [0, 1, 0], [1, 0, 3], [1, 1, 2], [1, 1, 3], [1, 2, 1]]
# 张量值
val = [0, 10, 103, 112, 113, 114]
# 张量形状
shape = [2, 3, 4]
# 创建稀疏张量
sp = tf.SparseTensor(idx, val, shape)
# 将SparseTensor转换为稠密的布尔指示器张量
si = tf.sparse_to_indicator(sp, 200)
si_val = si[1, 1, 113]
test_run_sess("sparse indicator", si)
test_run_sess("sparse indicator value", si_val)
# 稀疏张量叠加
sp1 = tf.SparseTensor([[0, 2], [1, 0], [1, 1]], ['a', 'b', 'c'], [2, 3])
sp2 = tf.SparseTensor([[0, 1], [0, 2]], ['d', 'e'], [2, 4])
sp3 = tf.SparseTensor([[0, 1], [0, 2]], ['d', 'e'], [2, 3])
con1 = tf.sparse_concat(1, [sp1, sp2], name=None)
con2 = tf.sparse_concat(0, [sp1, sp3], name=None)
test_run_sess("sparse concat1", con1)
test_run_sess("sparse concat2", con2)
# 稀疏张量重排序，成为以行为主的标准排序
sp4 = tf.SparseTensor([[0, 3], [0, 1], [3, 1], [2, 0]],
['b', 'a', 'd', 'c'], [4, 5])
rsp4 = tf.sparse_reorder(sp4)
# 保留部分元素
to_retain = [True, False, False, True]
rsp5 = tf.sparse_retain(sp4, to_retain)
# 填充空行
rsp6 = tf.sparse_fill_empty_rows(sp4, 'zz')
test_run_sess("rsp4", rsp4)
test_run_sess("rsp5", rsp5)
test_run_sess("rsp6", rsp6)
def prepare_serialized_examples(self, serialized_examples):
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {
"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)
}
for feature_index in range(num_features):
feature_map[
self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
tf.add_to_collection("features", features)
tf.add_to_collection("mean_rgb", features["mean_rgb"])
tf.add_to_collection("mean_audio", features["mean_audio"])
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
if FLAGS.cbp == False:
concatenated_features = tf.concat([
features[feature_name] for feature_name in self.feature_names
], 1)
elif FLAGS.cbp == True:
cbp_size = FLAGS.cbp_size
concatenated_features = cbp_layer(
tf.reshape(features["mean_rgb"], shape=[-1, 1, 1, 1024]),
tf.reshape(features["mean_audio"], shape=[-1, 1, 1, 128]),
output_dim=cbp_size)
concatenated_features.set_shape([None, cbp_size])
# 10% of Gaussian random noise added.
concatenated_features = concatenated_features + FLAGS.random_noise * tf.random_normal(
shape=tf.shape(concatenated_features))
concatenated_features = tf.nn.l2_normalize(concatenated_features,
dim=1)
tf.add_to_collection("concat_features", concatenated_features)
return features["video_id"], concatenated_features, labels, tf.ones(
[tf.shape(serialized_examples)[0]])
def _parse_example_proto(example_serialized):
"""Parses an Example proto containing a training example of an image.
The output of the build_image_data.py image preprocessing script is a dataset
containing serialized Example protocol buffers. Each Example proto contains
the following fields (values are included as examples):
image/height: 462
image/width: 581
image/colorspace: 'RGB'
image/channels: 3
image/class/label: 615
image/class/synset: 'n03623198'
image/class/text: 'knee pad'
image/object/bbox/xmin: 0.1
image/object/bbox/xmax: 0.9
image/object/bbox/ymin: 0.2
image/object/bbox/ymax: 0.6
image/object/bbox/label: 615
image/format: 'JPEG'
image/filename: 'ILSVRC2012_val_00041207.JPEG'
image/encoded: <JPEG encoded string>
Args:
example_serialized: scalar Tensor tf.string containing a serialized
Example protocol buffer.
Returns:
image_buffer: Tensor tf.string containing the contents of a JPEG file.
label: Tensor tf.int32 containing the label.
bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]
where each coordinate is [0, 1) and the coordinates are arranged as
[ymin, xmin, ymax, xmax].
"""
# Dense features in Example proto.
feature_map = {
'image/encoded': tf.FixedLenFeature([], dtype=tf.string,
default_value=''),
'image/ingrs': tf.VarLenFeature(dtype=tf.int64)
}
features = tf.parse_single_example(example_serialized, feature_map)
label = tf.sparse_to_indicator(features['image/ingrs'], _NUM_CLASSES)
return features['image/encoded'], label
def __parse_example_proto_with_elmo(example_serialized, NUM_CLASS, MAX_PAIRS):
feature_map = {
'raw_text': tf.FixedLenFeature([], tf.string),
'labels': tf.VarLenFeature(dtype=tf.int64)
}
features = tf.parse_single_example(example_serialized, features=feature_map)
raw_text = features['raw_text']
labels = features['labels']
label_map = tf.sparse_to_indicator(labels, NUM_CLASS)
label_pairs = _lm2lp(label_map, MAX_PAIRS)
label_map = tf.cast(label_map, tf.float32)
label_map.set_shape([NUM_CLASS])
label_dict = {}
label_dict['label_pair'] = label_pairs
label_dict['label_map'] = label_map
label_dict['raw_text'] = raw_text
return label_dict
def __init__(self, fileList):
self.num_classes = 3862
self.feature_sizes = [1024, 128]
self.feature_names = ["mean_rgb", "mean_audio"]
reader = tf.TFRecordReader()
feature_map = {
"id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)
}
for feature_index in range(len(self.feature_names)):
feature_map[self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
filename_queue = tf.train.string_input_producer(fileList, num_epochs=1, shuffle=False)
_, serialized_example = reader.read_up_to(filename_queue, 1024)
features = tf.parse_example(serialized_example, feature_map)
#
self.labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
# labels.set_shape([None, self.num_classes])
self.concatenated_features = tf.concat([features[feature_name] for feature_name in self.feature_names], 1)
return
def _parse_fn(self, example_serialized):
"""Parses an Example proto containing a training example of an image.
Args:
example_serialized: scalar Tensor tf.string containing a serialized
Example protocol buffer.
Returns:
image_buffer: Tensor tf.string containing the contents of a JPEG file.
label: Tensor tf.int32 containing the label.
"""
feature_map = {"id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64),
"mean_rgb": tf.FixedLenFeature(1024, tf.float32),
"mean_audio": tf.FixedLenFeature(128, tf.float32)}
features = tf.parse_single_example(example_serialized, features=feature_map)
# features = tf.parse_example(example_serialized, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.n_classes)
concatenated_features = tf.concat([
features[name] for name in self.feature_names], 0)
return concatenated_features, labels
def prepare_reader(self, filename_queue, batch_size=1024):
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
feature_map = {
"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64),
"feats": tf.FixedLenFeature([20 * 256], tf.float32)
}
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], 4716)
labels.set_shape([None, self.num_classes])
concatenated_features = features['feats']
sparse_labels, label_weights, input_weights = labels, labels, labels
return features[
"video_id"], concatenated_features, labels, sparse_labels, tf.ones(
[tf.shape(serialized_examples)[0]
]), label_weights, input_weights
def prepare_serialized_examples(self, serialized_examples):
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {"id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)}
for feature_index in range(num_features):
feature_map[self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat([
features[feature_name] for feature_name in self.feature_names], 1)
return features["id"], concatenated_features, labels, tf.ones([tf.shape(serialized_examples)[0]])
def prepare_serialized_examples(self, serialized_examples):
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
f"length of feature_names (={len(self.feature_names)}) !="\
f" length of feature_sizes (={len(self.feature_sizes)})"
feature_map = {
"id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)
}
for feature_index in range(num_features):
feature_map[self.feature_names[feature_index]] = \
tf.FixedLenFeature([self.feature_sizes[feature_index]],
tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = \
tf.concat([features[feature_name]
for feature_name in self.feature_names], 1)
return (features["id"], concatenated_features, labels,
tf.ones([tf.shape(serialized_examples)[0]]))
def parse_serialized(serialized_example):
features = tf.parse_single_example(serialized_example,
features={
"img":
tf.FixedLenFeature([],
tf.string),
"attributes":
tf.VarLenFeature(tf.string)
})
img = _parse_serialized_img(features["img"], crop_and_rescale)
# Considered attributes as indicator vector.
attributes = attribute_index.lookup(features["attributes"])
attributes = tf.sparse_to_indicator(attributes,
len(considered_attributes) +
1) # +1 for the oov bucket.
attributes = attributes[:
-1] # Skip the oov bucket since those attributes should not be considered.
attributes = tf.cast(attributes, tf.float32)
return img, attributes
def prepare_reader(self, filename_queue, batch_size=1024):
"""Creates a single reader thread for pre-aggregated YouTube 8M Examples.
Args:
filename_queue: A tensorflow queue of filename locations.
Returns:
A tuple of video indexes, features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {
"video_id": tf.FixedLenFeature([], tf.string),
"predictions": tf.FixedLenFeature([self.num_classes], tf.float32),
"labels": tf.VarLenFeature(tf.int64)
}
for feature_index in range(num_features):
feature_map[
self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat(
[features[feature_name] for feature_name in self.feature_names], 1)
return features["video_id"], concatenated_features, labels, tf.ones(
[tf.shape(serialized_examples)[0]]), features["predictions"]
def get_reader(filename_queue, reader_batch_size, num_classes):
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue,
num_records=reader_batch_size)
feature_map = {
"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64),
"mean_rgb": tf.FixedLenFeature(1024, tf.float32)
}
features = tf.parse_example(serialized_examples, features=feature_map)
video_ids = features['video_id']
labels = tf.sparse_to_indicator(features["labels"], num_classes)
labels.set_shape([None, num_classes])
feature_dim = len(features['mean_rgb'].get_shape()) - 1
video_features = tf.nn.l2_normalize(features['mean_rgb'], dim=feature_dim)
return video_ids, video_features, labels, tf.ones([tf.shape(video_ids)[0]])
def AddIntegratedGradientsOps(graph,
attribution_tensors,
output_tensor,
num_evals,
attribution_dims_map,
zero_baseline_tensors=None,
new_output_scope='attribution',
baseline_scope='baseline',
tensors_to_keep=None):
"""Modify graph to create ops for computing integrated gradients.
Function to modify a tensorflow graph by adding ops for attributing the change
in value of a given output tensor, to different input 'attribution_tensors'
(see arxiv.org/abs/1703.01365).
The first dimension of each attribution_tensor and output_tensor is assumed
to be the batch dimension. That is, if we create multiple input values for the
attribution tensors, we should be able to concatenate them along the first
dimension, and the resulting output tensor should have corresponding values
for different values of its first dimension.
The attribution works by interpolating between a given input, and a given
baseline, to create multiple (num_evals) interpolated inputs. At each
interpolated input, we compute the gradient of the output tensor with respect
to each attribution tensor. The gradients for each attribution tensor are
averaged over all interpolated inputs, to get an attribution score for it.
Example Usage: attribution_feed_dict = AddIntegratedGradientsOps(...)
Then to get attribution for a given input (specificed by input_feed_dict,
relative to a baseline given be baseline_feed_dict):
combined_feed_dict = attribution_feed_dict['create_combined_feed_dict'](
input_feed_dict, baseline_feed_dict)
with graph.as_default(), sess.as_default():
attributions = sess.run(
attribution_feed_dict['mean_grads'], combined_feed_dict)
for tensor, attribution in zip(attribution_tensors, attributions):
print('Attribution for %s: %s' % (tensor.op.name, attribution))
Warning: This function is not compatible with tf.cond. If there is a tf.cond
in the graph path between the attribution tensors and the output tensor, the
attribution ops may not work.
# TODO(manasrj): Make attribution ops compatible with tf.cond.
Args:
graph: The tf.Graph to add attribution ops to.
attribution_tensors: Tensors for which to compute attribution scores. The
tensors must satisfy two properties: (1) The output tensor must
be computable given values for attribution tensors. (2) Each
attribution tensor must be computationally independent of the
others, i.e., it should not be the case that one of the
attribution tensor's value is completely determined by the
values of the other attribution tensors. Properties (1) and (2) ensure
the attribution tensors form an input-output cut in the computation
graph.
output_tensor: Tensor for whose value we are performing attribution.
num_evals: Integer scalar. Number of interpolated points at which to
evaluate gradients. Higher values of this parameter increase computation
time, but also increase accuracy of attributions.
attribution_dims_map: Dict mapping attribution tensors to lists of integers.
For each attribution_tensor, we compute a separate gradient value for each
slice along the dims in the list. For example, if we have a rank 3
attribution tensor T that consists of embeddings lookups, with the first
dimension being the batch dimension, and the second dimension being the
sparse ids, then setting attribution_dims_map[T] = [1] will give us a
separate gradient for each sparse id. If an attribution_tensor has no
entry in attribution_dims_map, then the list defaults to [].
zero_baseline_tensors: Set of attribution tensors. For each tensor T in this
set, we compute gradients with respect to T for all interpolated values of
T between the value computed from the input feed, and zero. For each
tensor U not in zero_baseline_tensors, we compute gradients for
interpolated values between the one derived from the input feed, and the
one derived from the baseline feed.
new_output_scope: String. New ops needed for computing the output tensor at
different interpolated values are created under this scope name.
baseline_scope: String. New ops needed for computing attribution tensor
interpolated values are created under this scope name.
tensors_to_keep: Set of tensors. By default, tensors in the graph between
the output_tensor and attribution tensors are copied to a different part
of the graph, and evaluated separately for each interpolation. If we want
a value to be fixed (only computed for the main input instead of each
interpolation), it should be put in tensors_to_keep.
Returns:
attribution_hooks: Dict with the following keys (among others):
mean_grads: List of attribution scores (aligned with attribution_tensors).
create_combined_feed_dict: A Function that takes an input feed dict, and
optionally, a baseline feed dict, and creates a combined feed dict to
pass to sess.run to get attributions.
"""
ops_to_tensors = lambda ops: [op.outputs[0] for op in ops]
attribution_hooks = {}
if tensors_to_keep is None:
tensors_to_keep = []
else:
tensors_to_keep = list(tensors_to_keep)
if zero_baseline_tensors is None:
zero_baseline_tensors = []
with graph.as_default():
# Compute parts of graph and check correctness.
all_ops = graph.get_operations()
constant_ops = tf.contrib.graph_editor.select.select_ops(
all_ops, positive_filter=lambda x: x.type == 'Const')
placeholder_ops = tf.contrib.graph_editor.select.select_ops(
all_ops, positive_filter=lambda x: x.type == 'Placeholder')
var_read_ops = tf.contrib.graph_editor.select.select_ops('/read$',
graph=graph)
attr_ops = [t.op for t in attribution_tensors]
required_ops = set(
tf.contrib.graph_editor.select.get_backward_walk_ops(
output_tensor.op,
stop_at_ts=(tensors_to_keep + list(attribution_tensors) +
ops_to_tensors(var_read_ops) +
ops_to_tensors(placeholder_ops))))
# Check that attribution tensors are sufficient to compute output_tensor.
forward_ops = set(
tf.contrib.graph_editor.select.get_forward_walk_ops(attr_ops +
var_read_ops +
constant_ops))
assert required_ops.issubset(forward_ops)
required_sgv = tf.contrib.graph_editor.subgraph.make_view(required_ops)
attribution_subgraph, attribution_transform_info = (
tf.contrib.graph_editor.transform.copy_with_input_replacements(
required_sgv, {}, graph, new_output_scope))
attribution_hooks['attribution_subgraph'] = attribution_subgraph
attribution_hooks[
'attribution_transform_info'] = attribution_transform_info
# Copy feed to attribution part of graph so we can have one part for
# baseline and one for input.
backward_ops = tf.contrib.graph_editor.select.get_backward_walk_ops(
attr_ops, stop_at_ts=ops_to_tensors(var_read_ops))
backward_sgv = tf.contrib.graph_editor.subgraph.make_view(backward_ops)
_, baseline_transform_info = (
tf.contrib.graph_editor.transform.copy_with_input_replacements(
backward_sgv, {}, graph, baseline_scope))
attribution_hooks['baseline_transform_info'] = baseline_transform_info
# Function to compute combined feed dict. The default setting of
# baseline_transform_info is to get around python's late binding.
def CreateCombinedFeedDict(
input_feed_dict,
baseline_feed_dict=None,
baseline_transform_info=baseline_transform_info):
"""Combine baseline and input feed dicts into a common feed dict."""
combined_feed_dict = input_feed_dict.copy()
if baseline_feed_dict is None:
baseline_feed_dict = input_feed_dict
for key, feed_value in baseline_feed_dict.items():
if isinstance(key, tf.Tensor):
combined_feed_dict[baseline_transform_info.transformed(
key)] = (feed_value)
elif isinstance(key, unicode):
tensor = graph.get_tensor_by_name(key.decode())
combined_feed_dict[baseline_transform_info.transformed(
tensor)] = (feed_value)
elif isinstance(key, tf.SparseTensor):
sparse_transformed_tensor = tf.SparseTensor(
baseline_transform_info.transformed(key.indices),
baseline_transform_info.transformed(key.values),
baseline_transform_info.transformed(key.dense_shape))
combined_feed_dict[sparse_transformed_tensor] = feed_value
else:
raise ValueError('Invalid key type %s in Feed Dict.' %
type(key))
return combined_feed_dict
attribution_hooks['create_combined_feed_dict'] = CreateCombinedFeedDict
# Create new tensors with the multipliers to insert after previous ones.
attribution_hooks['multipliers'] = []
attribution_hooks['weighted_attribution_tensors'] = []
for attribution_tensor in attribution_tensors:
with tf.control_dependencies(
[tf.assert_equal(tf.shape(attribution_tensor)[0], 1)]):
attribution_dims = (attribution_dims_map[attribution_tensor]
if attribution_tensor
in attribution_dims_map else [])
vocab_size = len(attribution_tensor.get_shape())
attribution_dim_cond = tf.sparse_to_indicator(
tf.SparseTensor(
tf.expand_dims(
tf.range(len(attribution_dims), dtype=tf.int64),
1), attribution_dims, [vocab_size]), vocab_size)
base_multiplier_shape = tf.concat([
tf.expand_dims(num_evals, 0),
tf.ones_like(tf.shape(attribution_tensor))[1:]
], 0)
tile_dims = tf.where(
attribution_dim_cond, tf.shape(attribution_tensor),
tf.ones_like(tf.shape(attribution_tensor)))
pre_tile_multiplier = tf.reshape(
tf.range(tf.to_float(num_evals)) /
tf.to_float(num_evals - 1), base_multiplier_shape)
multiplier = tf.tile(pre_tile_multiplier, tile_dims)
if attribution_tensor in zero_baseline_tensors:
weighted_attribution_tensor = multiplier * attribution_tensor
else:
base_attribution_tensor = baseline_transform_info.transformed(
attribution_tensor)
weighted_attribution_tensor = (
multiplier * attribution_tensor +
(1 - multiplier) * base_attribution_tensor)
attribution_hooks['weighted_attribution_tensors'].append(
weighted_attribution_tensor)
attribution_hooks['multipliers'].append(multiplier)
tf.contrib.graph_editor.reroute_ts(
attribution_hooks['weighted_attribution_tensors'],
attribution_tensors,
can_modify=attribution_subgraph.ops)
g = tf.gradients(attribution_transform_info.transformed(output_tensor),
attribution_hooks['multipliers'])
attribution_hooks['mean_grads'] = [
tf.reduce_mean(grad, 0) for grad in g
]
return attribution_hooks
def _matrix_factorization_model_fn(features, target_ratings, mode):
"""Creates a neighborhood factorization model.
Each user is represented by a combination of embeddings of rated items,
as described in the paper: "Factorization Meets the Neighborhood:
a Multifaceted Collaborative Filtering Model - Yehuda Koren (KDD 2013)".
Args:
features: A dictionary of tensors keyed by the feature name.
target_ratings: A tensor representing the labels (in this case,
the ratings on the target movie).
mode: The execution mode, as defined in tf.contrib.learn.ModeKeys.
Returns:
ModelFnOps with the mode, prediction, loss, train_op and
output_alternatives a dictionary specifying the output for a
classification request during serving.
"""
_ = target_ratings # Unused on this model.
if hparams.embedding_weight_initializer == TRUNCATED_NORMAL:
embedding_weight_initializer = tf.truncated_normal_initializer(stddev=0.1)
else:
embedding_weight_initializer = None
query_movie_embedding_weights = tf.get_variable(
'query_movie_ids_embedding_weights',
[MOVIE_VOCAB_SIZE, hparams.movie_embedding_dim],
initializer=embedding_weight_initializer,
regularizer=tf.contrib.layers.l2_regularizer(hparams.l2_weight_decay))
query_movie_ids = features[QUERY_RATED_MOVIE_IDS]
query_embeddings = tf.nn.embedding_lookup_sparse(
[query_movie_embedding_weights],
query_movie_ids,
None,
combiner='sqrtn',
name='query_embedding')
query_biases, _, _ = tf.contrib.layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=make_query_feature_columns(),
num_outputs=1)
global_rating_bias = tf.get_variable(
name='global_rating_bias',
initializer=tf.constant(RATING_BIAS, dtype=tf.float32))
candidate_movie_embedding_weights = tf.get_variable(
'candidate_movie_id_embedding_weights',
[MOVIE_VOCAB_SIZE, hparams.movie_embedding_dim],
initializer=embedding_weight_initializer,
regularizer=tf.contrib.layers.l2_regularizer(hparams.l2_weight_decay))
candidate_biases, _, _ = (
tf.contrib.layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=make_candidate_feature_columns(),
num_outputs=1))
# Create layers for target features.
if mode != tf.contrib.learn.ModeKeys.INFER:
candidate_movie_ids = features[CANDIDATE_MOVIE_ID]
candidate_embeddings = tf.nn.embedding_lookup_sparse(
[candidate_movie_embedding_weights],
candidate_movie_ids,
None,
name='candidate_embedding')
predictions = tf.reduce_sum(tf.multiply(
query_embeddings, candidate_embeddings), 1, keep_dims=True)
if hparams.enable_bias:
biases = tf.add(query_biases, candidate_biases)
predictions = tf.add(predictions, biases)
predictions = tf.add(predictions, global_rating_bias)
labels = features[LABEL_RATING_SCORE]
loss = tf.losses.mean_squared_error(labels, predictions)
if mode == tf.contrib.learn.ModeKeys.TRAIN:
train_op = tf.contrib.layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=hparams.learning_rate,
optimizer=hparams.optimizer)
return tf.contrib.learn.ModelFnOps(
mode=mode, predictions=predictions, loss=loss, train_op=train_op)
elif mode == tf.contrib.learn.ModeKeys.EVAL:
if hparams.eval_type == REGRESSION:
return tf.contrib.learn.ModelFnOps(
mode=mode, predictions=predictions, loss=loss)
elif hparams.eval_type == RANKING:
# For 'RANKING' eval, we are interested in precision@k, recall@k
# metrics which require us to compute prediction/ranking scores for
# all movies.
predictions = tf.matmul(query_embeddings,
candidate_movie_embedding_weights,
transpose_b=True)
if hparams.enable_bias:
biases = tf.add(query_biases, candidate_biases)
predictions = tf.add(predictions, biases)
if hparams.use_ranking_candidate_movie_ids:
# Get ranking candidate movie ids to rank our candidate movie
# against.
ranking_candidate_movie_ids = features[RANKING_CANDIDATE_MOVIE_IDS]
movies_to_rank_condition = tf.sparse_to_indicator(
tf.sparse_concat(
axis=1,
sp_inputs=[ranking_candidate_movie_ids,
candidate_movie_ids]),
MOVIE_VOCAB_SIZE)
predictions = tf.where(movies_to_rank_condition, predictions,
tf.fill(
tf.shape(predictions),
tf.reduce_min(predictions)))
return tf.contrib.learn.ModelFnOps(
mode=mode,
predictions=predictions,
loss=loss)
elif mode == tf.contrib.learn.ModeKeys.INFER:
scores = tf.matmul(query_embeddings,
candidate_movie_embedding_weights,
transpose_b=True)
if hparams.enable_bias:
biases = tf.add(query_biases, candidate_biases)
scores = tf.add(scores, biases)
# Eliminate already rated candates.
rated_movie_ids = features[QUERY_RATED_MOVIE_IDS]
pruned_scores = tf.where(
tf.sparse_to_indicator(rated_movie_ids, MOVIE_VOCAB_SIZE),
tf.fill(tf.shape(scores), tf.reduce_min(scores)), scores)
predictions, output_alternatives = generate_top_k_scores_and_ids(
pruned_scores, hparams.top_k_infer)
return tf.contrib.learn.ModelFnOps(
mode=mode,
predictions=predictions,
output_alternatives=output_alternatives)
def _dnn_softmax_fn(features, targets, mode):
"""Creates the prediction, loss, and train ops.
Args:
features: A dictionary of tensors keyed by the feature name.
targets: A tensor representing the labels (in this case,
the ratings on the target movie).
mode: The execution mode, as defined in tf.contrib.learn.ModeKeys.
Returns:
ModelFnOps with the mode, prediction, loss, train_op and
output_alternatives a dictionary specifying the output for a
classification request during serving.
Raises:
ValueError: When the wrong evaluation type is specified.
"""
_ = targets # Unused variable.
class_weights = tf.get_variable(
name='class_weights',
shape=[MOVIE_VOCAB_SIZE, hparams.query_hidden_dims[-1]],
initializer=tf.contrib.layers.xavier_initializer())
class_biases = tf.get_variable(name='class_biases',
shape=[MOVIE_VOCAB_SIZE],
initializer=tf.zeros_initializer())
query_embeddings = _embed_query_features(features, mode=mode)
tf.summary.scalar('query_embeddings_zero_fraction',
tf.nn.zero_fraction(query_embeddings))
# Create layers for target features.
if mode != tf.contrib.learn.ModeKeys.INFER:
logits_layer = tf.matmul(
query_embeddings, tf.transpose(class_weights)) + class_biases
target_one_hot = tf.one_hot(
indices=features[CANDIDATE_MOVIE_ID].values,
depth=MOVIE_VOCAB_SIZE,
on_value=1.0)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=target_one_hot,
logits=logits_layer))
if mode == tf.contrib.learn.ModeKeys.TRAIN:
train_op = tf.contrib.layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=hparams.learning_rate,
optimizer=hparams.optimizer)
return tf.contrib.learn.ModelFnOps(mode=mode,
loss=loss,
train_op=train_op)
elif mode == tf.contrib.learn.ModeKeys.EVAL:
if hparams.eval_type == REGRESSION:
raise ValueError(
'eval_type must be RANKING for DNN softmax model.')
elif hparams.eval_type == RANKING:
predictions = tf.matmul(
query_embeddings,
tf.transpose(class_weights)) + class_biases
if hparams.use_ranking_candidate_movie_ids:
# Get ranking candidate movie ids to rank our candidate movie
# against.
ranking_candidate_movie_ids = features[
RANKING_CANDIDATE_MOVIE_IDS]
movies_to_rank_condition = tf.sparse_to_indicator(
tf.sparse_concat(axis=1,
sp_inputs=[
ranking_candidate_movie_ids,
features[CANDIDATE_MOVIE_ID]
]), MOVIE_VOCAB_SIZE)
predictions = tf.where(
movies_to_rank_condition, predictions,
tf.fill(tf.shape(predictions),
tf.reduce_min(predictions)))
return tf.contrib.learn.ModelFnOps(mode=mode,
predictions=predictions,
loss=loss)
elif mode == tf.contrib.learn.ModeKeys.INFER:
scores = tf.matmul(query_embeddings,
tf.transpose(class_weights)) + class_biases
rated_movie_ids = features[QUERY_RATED_MOVIE_IDS]
pruned_scores = tf.where(
tf.sparse_to_indicator(rated_movie_ids, MOVIE_VOCAB_SIZE),
tf.fill(tf.shape(scores), tf.reduce_min(scores)), scores)
predictions, output_alternatives = generate_top_k_scores_and_ids(
pruned_scores, hparams.top_k_infer)
return tf.contrib.learn.ModelFnOps(
mode=mode,
predictions=predictions,
output_alternatives=output_alternatives)
def _matrix_factorization_model_fn(features, target_ratings, mode):
"""Creates a neighborhood factorization model.
Each user is represented by a combination of embeddings of rated items,
as described in the paper: "Factorization Meets the Neighborhood:
a Multifaceted Collaborative Filtering Model - Yehuda Koren (KDD 2013)".
Args:
features: A dictionary of tensors keyed by the feature name.
target_ratings: A tensor representing the labels (in this case,
the ratings on the target movie).
mode: The execution mode, as defined in tf.contrib.learn.ModeKeys.
Returns:
ModelFnOps with the mode, prediction, loss, train_op and
output_alternatives a dictionary specifying the output for a
classification request during serving.
"""
_ = target_ratings # Unused on this model.
if hparams.embedding_weight_initializer == TRUNCATED_NORMAL:
embedding_weight_initializer = tf.truncated_normal_initializer(
stddev=0.1)
else:
embedding_weight_initializer = None
query_movie_embedding_weights = tf.get_variable(
'query_movie_ids_embedding_weights',
[MOVIE_VOCAB_SIZE, hparams.movie_embedding_dim],
initializer=embedding_weight_initializer,
regularizer=tf.contrib.layers.l2_regularizer(
hparams.l2_weight_decay))
query_movie_ids = features[QUERY_RATED_MOVIE_IDS]
query_embeddings = tf.nn.embedding_lookup_sparse(
[query_movie_embedding_weights],
query_movie_ids,
None,
combiner='sqrtn',
name='query_embedding')
query_biases, _, _ = tf.contrib.layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=make_query_feature_columns(),
num_outputs=1)
global_rating_bias = tf.get_variable(
name='global_rating_bias',
initializer=tf.constant(RATING_BIAS, dtype=tf.float32))
candidate_movie_embedding_weights = tf.get_variable(
'candidate_movie_id_embedding_weights',
[MOVIE_VOCAB_SIZE, hparams.movie_embedding_dim],
initializer=embedding_weight_initializer,
regularizer=tf.contrib.layers.l2_regularizer(
hparams.l2_weight_decay))
candidate_biases, _, _ = (
tf.contrib.layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=make_candidate_feature_columns(),
num_outputs=1))
# Create layers for target features.
if mode != tf.contrib.learn.ModeKeys.INFER:
candidate_movie_ids = features[CANDIDATE_MOVIE_ID]
candidate_embeddings = tf.nn.embedding_lookup_sparse(
[candidate_movie_embedding_weights],
candidate_movie_ids,
None,
name='candidate_embedding')
predictions = tf.reduce_sum(tf.multiply(query_embeddings,
candidate_embeddings),
1,
keep_dims=True)
if hparams.enable_bias:
biases = tf.add(query_biases, candidate_biases)
predictions = tf.add(predictions, biases)
predictions = tf.add(predictions, global_rating_bias)
labels = features[LABEL_RATING_SCORE]
loss = tf.losses.mean_squared_error(labels, predictions)
if mode == tf.contrib.learn.ModeKeys.TRAIN:
train_op = tf.contrib.layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=hparams.learning_rate,
optimizer=hparams.optimizer)
return tf.contrib.learn.ModelFnOps(mode=mode,
predictions=predictions,
loss=loss,
train_op=train_op)
elif mode == tf.contrib.learn.ModeKeys.EVAL:
if hparams.eval_type == REGRESSION:
return tf.contrib.learn.ModelFnOps(mode=mode,
predictions=predictions,
loss=loss)
elif hparams.eval_type == RANKING:
# For 'RANKING' eval, we are interested in precision@k, recall@k
# metrics which require us to compute prediction/ranking scores for
# all movies.
predictions = tf.matmul(query_embeddings,
candidate_movie_embedding_weights,
transpose_b=True)
if hparams.enable_bias:
biases = tf.add(query_biases, candidate_biases)
predictions = tf.add(predictions, biases)
if hparams.use_ranking_candidate_movie_ids:
# Get ranking candidate movie ids to rank our candidate movie
# against.
ranking_candidate_movie_ids = features[
RANKING_CANDIDATE_MOVIE_IDS]
movies_to_rank_condition = tf.sparse_to_indicator(
tf.sparse_concat(axis=1,
sp_inputs=[
ranking_candidate_movie_ids,
candidate_movie_ids
]), MOVIE_VOCAB_SIZE)
predictions = tf.where(
movies_to_rank_condition, predictions,
tf.fill(tf.shape(predictions),
tf.reduce_min(predictions)))
return tf.contrib.learn.ModelFnOps(mode=mode,
predictions=predictions,
loss=loss)
elif mode == tf.contrib.learn.ModeKeys.INFER:
scores = tf.matmul(query_embeddings,
candidate_movie_embedding_weights,
transpose_b=True)
if hparams.enable_bias:
biases = tf.add(query_biases, candidate_biases)
scores = tf.add(scores, biases)
# Eliminate already rated candates.
rated_movie_ids = features[QUERY_RATED_MOVIE_IDS]
pruned_scores = tf.where(
tf.sparse_to_indicator(rated_movie_ids, MOVIE_VOCAB_SIZE),
tf.fill(tf.shape(scores), tf.reduce_min(scores)), scores)
predictions, output_alternatives = generate_top_k_scores_and_ids(
pruned_scores, hparams.top_k_infer)
return tf.contrib.learn.ModelFnOps(
mode=mode,
predictions=predictions,
output_alternatives=output_alternatives)
def __init__(self, filename_queue, feature_dimensions, batch_size,
embedding_count, layer1_count, layer2_count, label_count,
learning_rate, ps_count, use_reduce_sum, optimizer):
with tf.variable_scope("ads_dnn_input"):
_, batch_example = tf.TFRecordReader().read_up_to(
filename_queue, batch_size)
with tf.variable_scope("ads_dnn_model"):
if sys.platform == "win32":
sample_labels, sample_weights, position_indices, position_values, position_shapes, term_indices, term_values, term_shapes = user_ops.parse_ads_dnn_samples(
batch_example,
len(feature_dimensions) - 1)
else:
ads_dnn_module = tf.load_op_library('./ads_dnn.so')
sample_labels, sample_weights, position_indices, position_values, position_shapes, term_indices, term_values, term_shapes = ads_dnn_module.parse_ads_dnn_samples(
batch_example,
len(feature_dimensions) - 1)
pos_sp_tensor = tf.SparseTensor(position_indices, position_values,
position_shapes)
term_sp_tensor = tf.SparseTensor(term_indices, term_values,
term_shapes)
#y = tf.reshape(tf.bitcast(tf.slice(int_tensor, [0], [batch_size]), tf.float32), [-1, 1])
#sample_weights = tf.bitcast(tf.slice(int_tensor, [batch_size], [batch_size]), tf.float32)
#int64_part = tf.bitcast(tf.reshape(tf.slice(int_tensor, [batch_size * 2], [-1]), [-1, 2]), tf.int64)
#pos_sp_indices= tf.reshape(tf.slice(int64_part, [batch_size], [batch_size * 2]), [-1, 2])
#pos_sp_ids = tf.slice(int64_part, [0], [batch_size])
#pos_sp_shapes = tf.constant([batch_size, 1], dtype=tf.int64)
#pos_sp_tensor = tf.SparseTensor(pos_sp_indices, pos_sp_ids, pos_sp_shapes)
#term_sp_part = tf.reshape(tf.slice(int64_part, [batch_size * 3 + 2], [-1]), [3, -1])
#term_sp_indices = tf.reshape(tf.slice(term_sp_part, [1, 0], [-1, -1]), [-1, 2])
#term_sp_ids = tf.reshape(tf.slice(term_sp_part, [0, 0], [1, -1]), [-1])
#term_sp_shapes = tf.slice(int64_part, [batch_size * 3], [2])
#term_sp_tensor = tf.SparseTensor(term_sp_indices, term_sp_ids, term_sp_shapes)
feature_group_count = len(feature_dimensions)
term_dimension_count = sum(feature_dimensions)
# assign one default feature id for empty group
term_dimension_count += len(
feature_dimensions) + 1 # feature id start from 0
print("total term feature count:", term_dimension_count)
variable_initializer = tf.random_uniform_initializer(-0.05, 0.05)
dense_tensor_size = embedding_count * (
feature_group_count - 1
) * layer1_count + layer1_count * layer2_count + layer2_count * label_count + (
feature_dimensions[0] + 1) * label_count
#term_embedding_ps_weight = batch_size * embedding_count * 100
#dense_ps_weight = dense_tensor_size
#ps_weight_sum = term_embedding_ps_weight + dense_ps_weight
#term_embedding_ps_count = int(term_embedding_ps_weight * ps_count / ps_weight_sum) + 1
#dense_ps_count = int(dense_ps_weight * ps_count / ps_weight_sum) + 1
#if term_embedding_ps_count > ps_count:
# term_embedding_ps_count = ps_count
#if dense_ps_count > ps_count:
# dense_ps_count = ps_count
#print("term embedding ps count:", term_embedding_ps_count, "dense ps count:", dense_ps_count)
embedding_out_offset = 0
embedding_out_size = embedding_count * (feature_group_count -
1) * layer1_count
embedding_out_shape = [
embedding_count * (feature_group_count - 1), layer1_count
]
l1l2_offset = embedding_out_offset + embedding_out_size
l1l2_size = layer1_count * layer2_count
l1l2_shape = [layer1_count, layer2_count]
final_offset = l1l2_offset + l1l2_size
final_size = layer2_count * label_count
final_shape = [layer2_count, label_count]
pos_offset = final_offset + final_size
pos_size = (feature_dimensions[0] + 1) * label_count
pos_shape = [(feature_dimensions[0] + 1), label_count]
W_dense_part = tf.get_variable(
"W_dense_part", [dense_tensor_size],
partitioner=tf.fixed_size_partitioner(ps_count),
initializer=variable_initializer)
W_term_embeddings = tf.get_variable(
"W_term_embeddings", [term_dimension_count, embedding_count],
partitioner=tf.fixed_size_partitioner(ps_count),
initializer=variable_initializer)
#W_embeddings_out = tf.get_variable("W_embedding_out", [embedding_count*(feature_group_count-1), layer1_count], partitioner=tf.fixed_size_partitioner(embedding_out_ps_count), initializer=variable_initializer)
#W_l1_l2 = tf.get_variable("W_l1_l2", [layer1_count, layer2_count], partitioner=tf.fixed_size_partitioner(l1_l2_ps_count), initializer=variable_initializer)
#W_final = tf.get_variable("W_final", [layer2_count, label_count], initializer=variable_initializer)
#W_pos = tf.get_variable("W_pos", [feature_dimensions[0], label_count], initializer=variable_initializer)
global_step = tf.Variable(0, name="global_step", trainable=False)
with tf.device("/cpu:0"):
term_embeddings = tf.nn.embedding_lookup_sparse(
W_term_embeddings,
sp_ids=term_sp_tensor,
sp_weights=None,
combiner="sum")
term_embeddings = tf.reshape(
term_embeddings,
[-1, embedding_count * (feature_group_count - 1)])
W_embeddings_out = tf.reshape(
tf.slice(W_dense_part, [embedding_out_offset],
[embedding_out_size]), embedding_out_shape)
W_l1_l2 = tf.reshape(
tf.slice(W_dense_part, [l1l2_offset], [l1l2_size]),
l1l2_shape)
W_final = tf.reshape(
tf.slice(W_dense_part, [final_offset], [final_size]),
final_shape)
W_pos = tf.reshape(
tf.slice(W_dense_part, [pos_offset], [pos_size]),
pos_shape)
embedding_out = tf.matmul(term_embeddings, W_embeddings_out)
layer1_output = tf.nn.relu(embedding_out)
layer2_input = tf.matmul(layer1_output, W_l1_l2)
layer2_output = tf.nn.relu(layer2_input)
layer3_input1 = tf.matmul(layer2_output, W_final)
layer3_input2 = tf.matmul(
tf.to_float(
tf.sparse_to_indicator(pos_sp_tensor,
feature_dimensions[0] + 1)),
W_pos)
#layer3_input2 = tf.nn.embedding_lookup_sparse(W_pos, sp_ids=pos_sp_tensor, sp_weights=None, combiner="sum")
pred_pre_sigmoid = tf.add(layer3_input1, layer3_input2)
pred = tf.nn.sigmoid(pred_pre_sigmoid)
unweighted_loss = tf.nn.sigmoid_cross_entropy_with_logits(
logits=pred_pre_sigmoid, targets=sample_labels)
#unweighted_loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=pred_pre_sigmoid, labels=sample_labels)
final_loss = tf.multiply(sample_weights, unweighted_loss)
if use_reduce_sum:
loss_fn = tf.reduce_sum(final_loss)
else:
loss_fn = tf.reduce_mean(final_loss)
print("Optimizer type:", optimizer)
if optimizer == "AdaGrad":
train_op = tf.train.AdagradOptimizer(learning_rate).minimize(
loss_fn, global_step=global_step)
elif optimizer == "RMSProp":
train_op = tf.train.RMSPropOptimizer(learning_rate).minimize(
loss_fn, global_step=global_step)
elif optimizer == "FTRL":
train_op = tf.train.FtrlOptimizer(
learning_rate=learning_rate,
l1_regularization_strength=0,
l2_regularization_strength=0).minimize(
loss_fn, global_step=global_step)
else:
train_op = tf.train.GradientDescentOptimizer(
learning_rate).minimize(loss_fn, global_step=global_step)
self._labels = sample_labels
self._weights = sample_weights
self._prediction = pred
self._global_step = global_step
self._loss_fn = loss_fn
self._train_op = train_op
self._final = W_final
self._w_l1_l2 = W_l1_l2
self._layer3_input1 = layer3_input1
self._layer3_input2 = layer3_input2
self._layer2_input = layer2_input
self._layer1_output = layer1_output
self._term_embeddings = term_embeddings
self._w_pos = W_pos
self._pos_sp_tensor = pos_sp_tensor
self._layer2_output = layer2_output
self._embedding_out = embedding_out
self._unweighted_loss = unweighted_loss
self._final_loss = final_loss
self._term_shapes = term_shapes
def _dnn_softmax_fn(features, targets, mode):
"""Creates the prediction, loss, and train ops.
Args:
features: A dictionary of tensors keyed by the feature name.
targets: A tensor representing the labels (in this case,
the ratings on the target movie).
mode: The execution mode, as defined in tf.contrib.learn.ModeKeys.
Returns:
ModelFnOps with the mode, prediction, loss, train_op and
output_alternatives a dictionary specifying the output for a
classification request during serving.
Raises:
ValueError: When the wrong evaluation type is specified.
"""
_ = targets # Unused variable.
class_weights = tf.get_variable(
name='class_weights',
shape=[MOVIE_VOCAB_SIZE, hparams.query_hidden_dims[-1]],
initializer=tf.contrib.layers.xavier_initializer())
class_biases = tf.get_variable(
name='class_biases',
shape=[MOVIE_VOCAB_SIZE],
initializer=tf.zeros_initializer())
query_embeddings = _embed_query_features(features, mode=mode)
tf.summary.scalar('query_embeddings_zero_fraction',
tf.nn.zero_fraction(query_embeddings))
# Create layers for target features.
if mode != tf.contrib.learn.ModeKeys.INFER:
logits_layer = tf.matmul(
query_embeddings, tf.transpose(class_weights)) + class_biases
target_one_hot = tf.one_hot(
indices=features[CANDIDATE_MOVIE_ID].values,
depth=MOVIE_VOCAB_SIZE,
on_value=1.0)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
labels=target_one_hot, logits=logits_layer))
if mode == tf.contrib.learn.ModeKeys.TRAIN:
train_op = tf.contrib.layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=hparams.learning_rate,
optimizer=hparams.optimizer)
return tf.contrib.learn.ModelFnOps(
mode=mode, loss=loss, train_op=train_op)
elif mode == tf.contrib.learn.ModeKeys.EVAL:
if hparams.eval_type == REGRESSION:
raise ValueError('eval_type must be RANKING for DNN softmax model.')
elif hparams.eval_type == RANKING:
predictions = tf.matmul(
query_embeddings, tf.transpose(class_weights)) + class_biases
if hparams.use_ranking_candidate_movie_ids:
# Get ranking candidate movie ids to rank our candidate movie
# against.
ranking_candidate_movie_ids = features[RANKING_CANDIDATE_MOVIE_IDS]
movies_to_rank_condition = tf.sparse_to_indicator(
tf.sparse_concat(
axis=1,
sp_inputs=[ranking_candidate_movie_ids,
features[CANDIDATE_MOVIE_ID]]),
MOVIE_VOCAB_SIZE)
predictions = tf.where(movies_to_rank_condition, predictions,
tf.fill(
tf.shape(predictions),
tf.reduce_min(predictions)))
return tf.contrib.learn.ModelFnOps(
mode=mode, predictions=predictions, loss=loss)
elif mode == tf.contrib.learn.ModeKeys.INFER:
scores = tf.matmul(
query_embeddings, tf.transpose(class_weights)) + class_biases
rated_movie_ids = features[QUERY_RATED_MOVIE_IDS]
pruned_scores = tf.where(
tf.sparse_to_indicator(rated_movie_ids, MOVIE_VOCAB_SIZE),
tf.fill(tf.shape(scores), tf.reduce_min(scores)), scores)
predictions, output_alternatives = generate_top_k_scores_and_ids(
pruned_scores, hparams.top_k_infer)
return tf.contrib.learn.ModelFnOps(
mode=mode,
predictions=predictions,
output_alternatives=output_alternatives)
class YT8MAggregatedFeatureReader(BaseReader):
"""Reads TFRecords of pre-aggregated Examples.
The TFRecords must contain Examples with a sparse int64 'labels' feature and
a fixed length float32 feature, obtained from the features in 'feature_name'.
The float features are assumed to be an average of dequantized values.
"""
def __init__(self,
num_classes=3862,
feature_sizes=[1024, 128],
feature_names=["mean_rgb", "mean_audio"]):
"""Construct a YT8MAggregatedFeatureReader.
Args:
num_classes: a positive integer for the number of classes.
feature_sizes: positive integer(s) for the feature dimensions as a list.
feature_names: the feature name(s) in the tensorflow record as a list.
"""
assert len(feature_names) == len(feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(feature_names), len(feature_sizes))
self.num_classes = num_classes
self.feature_sizes = feature_sizes
self.feature_names = feature_names
def prepare_reader(self, filename_queue, batch_size=1024):
"""Creates a single reader thread for pre-aggregated YouTube 8M Examples.
Args:
filename_queue: A tensorflow queue of filename locations.
Returns:
A tuple of video indexes, features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
tf.add_to_collection("serialized_examples", serialized_examples)
return self.prepare_serialized_examples(serialized_examples)
def prepare_serialized_examples(self, serialized_examples):
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {"id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)}
for feature_index in range(num_features):
feature_map[self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat([
features[feature_name] for feature_name in self.feature_names], 1)
return features["id"], concatenated_features, labels, tf.ones([tf.shape(serialized_examples)[0]])
import tensorflow as tf
import numpy as np
a = tf.SparseTensor(indices=[[0, 1], [0, 3], [2, 0]],
values=[1, 2, 3],
dense_shape=[3, 5])
b = tf.sparse_to_indicator(a, 10)
sess = tf.Session()
print(sess.run(b))
sess.close()
def prepare_serialized_examples(self, serialized_examples):
logging.set_verbosity(tf.logging.DEBUG)
# hardcoded values
len_features_frames = 1024
len_features_audio = 128
name_frames = "mean_rgb"
name_audio = "mean_audio"
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {
"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)
}
logging.debug("self.random_selection es " + str(self.random_selection))
zeros_float = tf.zeros([tf.shape(serialized_examples)[0]])
# Manera cutre de crear un vector de False. Alguna altra manera ha d'haver-hi
is_negative = tf.not_equal(zeros_float, zeros_float)
for feature_index in range(num_features):
feature_map[
self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
features_rgb = features[name_frames]
features_audio = features[name_audio]
labels_audio = tf.sparse_to_indicator(features["labels"],
self.num_classes)
batch_size = tf.shape(features[name_frames])[0]
if self.negative_sampling:
labels = tf.sparse_to_indicator(features["labels"],
self.num_classes)
labels.set_shape([None, self.num_classes])
def return_itself(a, b):
return a, b
# 80% of the samples are negative
number_neg_sample = tf.random_uniform(
[],
minval=0.,
maxval=1.,
dtype=tf.float32,
name="random_number_neg_sample")
constant = tf.constant(self.percentage_negative)
batch_size = tf.shape(features_rgb)[0]
logging.info("-----------------")
logging.info(batch_size)
is_negative = tf.random_uniform([batch_size, 1],
minval=0,
maxval=1)
is_negative = tf.less(is_negative, constant)
features_audio_return, labels_audio = self.sample_negatively(
features, labels, is_negative)
concatenated_features = tf.concat(
[features_rgb, features_audio_return], 1)
else:
# Normal case, leave as it was
# We can use python comparisons because they are checked only when creating the graph
if self.random_selection == 0 | (self.random_selection ==
1 & num_features > 1):
for feature_index in range(num_features):
feature_map[self.feature_names[
feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples,
features=feature_map)
labels = tf.sparse_to_indicator(features["labels"],
self.num_classes)
labels.set_shape([None, self.num_classes])
labels_audio = labels
concatenated_features = tf.concat([
features[feature_name]
for feature_name in self.feature_names
], 1)
# Evaluation with only one of the two features
elif self.random_selection == 1:
feature_map[name_frames] = tf.FixedLenFeature(
[len_features_frames], tf.float32)
feature_map[name_audio] = tf.FixedLenFeature(
[len_features_audio], tf.float32)
features = tf.parse_example(serialized_examples,
features=feature_map)
labels = tf.sparse_to_indicator(features["labels"],
self.num_classes)
labels.set_shape([None, self.num_classes])
# In this point there is only 1 feature_name
# We can use python comparisons because they are checked only when creating the graph
if self.feature_names[0] == name_frames:
concatenated_features = tf.concat([
features[name_frames],
tf.zeros_like(features[name_audio])
], 1)
else:
concatenated_features = tf.concat([
tf.zeros_like(features[name_frames]),
features[name_audio]
], 1)
# Training with thirds
else:
feature_map[name_frames] = tf.FixedLenFeature(
[len_features_frames], tf.float32)
feature_map[name_audio] = tf.FixedLenFeature(
[len_features_audio], tf.float32)
features = tf.parse_example(serialized_examples,
features=feature_map)
labels = tf.sparse_to_indicator(features["labels"],
self.num_classes)
labels.set_shape([None, self.num_classes])
number = tf.random_uniform([],
minval=0.,
maxval=3.,
dtype=tf.float32,
name="random_number")
features_rgb = features[name_frames]
features_audio = features[name_audio]
one = tf.constant(1.)
two = tf.constant(2.)
features_audio = tf.cond(
tf.less(number, one),
lambda: tf.clip_by_value(features_audio, 0, 0),
lambda: features_audio)
features_rgb = tf.cond(
tf.greater(number, two),
lambda: tf.clip_by_value(features_rgb, 0, 0),
lambda: features_rgb)
concatenated_features = tf.concat(
[features_rgb, features_audio], 1, name="concat_features")
return features["video_id"], concatenated_features, labels, tf.ones(
[tf.shape(serialized_examples)[0]]), is_negative, labels_audio
