def decode(self, serialized_example, items=None):
"""Decodes the given serialized TF-example."""
context, sequence = tf.parse_single_sequence_example(
serialized_example, self._context_keys_to_features,
self._sequence_keys_to_features)
# Merge context and sequence features
example = {}
example.update(context)
example.update(sequence)
all_features = {}
all_features.update(self._context_keys_to_features)
all_features.update(self._sequence_keys_to_features)
# Reshape non-sparse elements just once:
for k, value in all_features.items():
if isinstance(value, tf.FixedLenFeature):
example[k] = tf.reshape(example[k], value.shape)
if not items:
items = list(self._items_to_handlers.keys())
outputs = []
for item in items:
handler = self._items_to_handlers[item]
keys_to_tensors = {key: example[key] for key in handler.keys}
outputs.append(handler.tensors_to_item(keys_to_tensors))
return outputs
def decode(self, serialized_example, items=None):
"""Decodes the given serialized TF-SequenceExample.
Args:
serialized_example: A serialized TF-SequenceExample tensor.
items: The list of items to decode. These must be a subset of the item
keys in self._items_to_handlers. If `items` is left as None, then all
of the items in self._items_to_handlers are decoded.
Returns:
The decoded items, a list of tensor.
"""
context, feature_list = tf.parse_single_sequence_example(
serialized_example, self._keys_to_context_features,
self._keys_to_sequence_features)
# Reshape non-sparse elements just once:
for k in self._keys_to_context_features:
v = self._keys_to_context_features[k]
if isinstance(v, tf.FixedLenFeature):
context[k] = tf.reshape(context[k], v.shape)
if not items:
items = self._items_to_handlers.keys()
outputs = []
for item in items:
handler = self._items_to_handlers[item]
keys_to_tensors = {
key: context[key] if key in context else feature_list[key]
for key in handler.keys
}
outputs.append(handler.tensors_to_item(keys_to_tensors))
return outputs
def parse_example(example_proto):
"""Decodes a TFRecords example
Args:
example_proto (tf.train.Example): TFRecords Example
Returns:
tuple(tf.Tensor, int, str): tensor of the video, label and filename of
the video
"""
# Parse the input tf.train.Example using the dictionary above.
context, sequence = tf.parse_single_sequence_example(example_proto,\
context_features=context_features, sequence_features=sequence_features)
# extract the expected shape
shape = (context['temporal'], context['height'], context['width'],
context['depth'])
## the golden while loop ##
# loop through the feature lists and decode each image seperately:
# decoding the first video
video_data = tf.image.decode_image(
tf.gather(sequence['video_frames'], [0])[0])
video_data = tf.expand_dims(video_data, 0)
i = tf.constant(1, dtype=tf.int32)
# condition of when to stop / loop through every frame
cond = lambda i, _: tf.less(i, tf.cast(context['temporal'], tf.int32))
# reading + decoding the i-th image frame
def body(i, video_data):
# get the i-th index
encoded_img = tf.gather(sequence['video_frames'], [i])
# decode the image
img_data = tf.image.decode_image(encoded_img[0])
# append to list using tf operations
video_data = tf.concat([video_data, [img_data]], 0)
# update counter & new video_data
return (tf.add(i, 1), video_data)
# run the loop (use `shape_invariants` since video_data changes size)
_, video_data = tf.while_loop(
cond,
body, [i, video_data],
shape_invariants=[i.get_shape(), tf.TensorShape([None])])
# use this to set the shape + dtype
video_data = tf.reshape(video_data, shape)
video_data = tf.cast(video_data, tf.float32)
label = context['label']
filename = context['filename']
return video_data, label, filename
def read_record(self, record):
"""Parse record TFRecord into a set a set of values, names and types
that can be queued and then read.
Returns:
- queue_values: Dict with tensor values.
- queue_names: Names for each tensor.
- queue_types: Types for each tensor.
"""
# We parse variable length features (bboxes in a image) as sequence
# features
context_example, sequence_example = tf.parse_single_sequence_example(
record,
context_features=self.CONTEXT_FEATURES,
sequence_features=self.SEQUENCE_FEATURES)
# Decode image
image_raw = tf.image.decode_image(context_example['image_raw'],
channels=3)
image = tf.cast(image_raw, tf.float32)
height = tf.cast(context_example['height'], tf.int32)
width = tf.cast(context_example['width'], tf.int32)
image_shape = tf.stack([height, width, 3])
image = tf.reshape(image, image_shape)
label = self._sparse_to_tensor(sequence_example['label'])
xmin = self._sparse_to_tensor(sequence_example['xmin'])
xmax = self._sparse_to_tensor(sequence_example['xmax'])
ymin = self._sparse_to_tensor(sequence_example['ymin'])
ymax = self._sparse_to_tensor(sequence_example['ymax'])
# Stack parsed tensors to define bounding boxes of shape (num_boxes, 5)
bboxes = tf.stack([xmin, ymin, xmax, ymax, label], axis=1)
image, bboxes, preprocessing_details = self.preprocess(image, bboxes)
filename = tf.cast(context_example['filename'], tf.string)
# TODO: Send additional metadata through the queue (scale_factor,
# applied_augmentations)
queue_dtypes = [tf.float32, tf.int32, tf.string, tf.float32]
queue_names = ['image', 'bboxes', 'filename', 'scale_factor']
queue_values = {
'image': image,
'bboxes': bboxes,
'filename': filename,
'scale_factor': preprocessing_details['scale_factor'],
}
return queue_values, queue_dtypes, queue_names
def _decode_record(record, name_to_features):
"""Decodes a record to a TensorFlow example."""
_, example = tf.parse_single_sequence_example(
record, sequence_features=name_to_features)
# tf.Example only supports tf.int64, but the TPU only supports tf.int32.
# So cast all int64 to int32.
for name in list(example.keys()):
t = example[name]
if t.dtype == tf.int64:
t = tf.to_int32(t)
shape = tf.shape(example[name])
# sequence_examples come with dynamic/unknown dimension which we reshape
# to explicit dimension for the fewshot "batch" size.
example[name] = tf.reshape(t, tf.concat([[fewshot_batch], shape[1:]], 0))
return example
def _parse_fn_old(serialized_example):
"""Parses tf.(Sparse)Tensors from the serialized tf.SequenceExample.
Also works with TF versions < 1.12 but is slower than _parse_fn_new.
Args:
serialized_example: A single serialized tf.SequenceExample.
Returns:
A dictionary from name to (Sparse)Tensors of the context and sequence
features.
"""
context, sequence = tf.parse_single_sequence_example(
serialized_example,
context_features=context_features_config,
sequence_features=sequence_features_config,
example_name='parsing_examples')
feature_map = dict()
for k, v in context.items():
feature_map[CONTEXT_KEY_PREFIX + k] = v
for k, v in sequence.items():
feature_map[SEQUENCE_KEY_PREFIX + k] = v
return feature_map
def get_padded_batch(file_list, batch_size, input_size, label_shape=None,
num_enqueuing_threads=4, shuffle=False):
"""Reads batches of SequenceExamples from TFRecords and pads them.
Can deal with variable length SequenceExamples by padding each batch to the
length of the longest sequence with zeros.
Args:
file_list: A list of paths to TFRecord files containing SequenceExamples.
batch_size: The number of SequenceExamples to include in each batch.
input_size: The size of each input vector. The returned batch of inputs
will have a shape [batch_size, num_steps, input_size].
label_shape: Shape for labels. If not specified, will use [].
num_enqueuing_threads: The number of threads to use for enqueuing
SequenceExamples.
shuffle: Whether to shuffle the batches.
Returns:
inputs: A tensor of shape [batch_size, num_steps, input_size] of floats32s.
labels: A tensor of shape [batch_size, num_steps] of int64s.
lengths: A tensor of shape [batch_size] of int32s. The lengths of each
SequenceExample before padding.
Raises:
ValueError: If `shuffle` is True and `num_enqueuing_threads` is less than 2.
"""
file_queue = tf.train.string_input_producer(file_list)
reader = tf.TFRecordReader()
_, serialized_example = reader.read(file_queue)
sequence_features = {
'inputs': tf.FixedLenSequenceFeature(shape=[input_size],
dtype=tf.float32),
'labels': tf.FixedLenSequenceFeature(shape=label_shape or [],
dtype=tf.int64)}
_, sequence = tf.parse_single_sequence_example(
serialized_example, sequence_features=sequence_features)
length = tf.shape(sequence['inputs'])[0]
input_tensors = [sequence['inputs'], sequence['labels'], length]
if shuffle:
if num_enqueuing_threads < 2:
raise ValueError(
'`num_enqueuing_threads` must be at least 2 when shuffling.')
shuffle_threads = int(math.ceil(num_enqueuing_threads) / 2.)
# Since there may be fewer records than SHUFFLE_MIN_AFTER_DEQUEUE, take the
# minimum of that number and the number of records.
min_after_dequeue = count_records(
file_list, stop_at=SHUFFLE_MIN_AFTER_DEQUEUE)
input_tensors = _shuffle_inputs(
input_tensors, capacity=QUEUE_CAPACITY,
min_after_dequeue=min_after_dequeue,
num_threads=shuffle_threads)
num_enqueuing_threads -= shuffle_threads
tf.logging.info(input_tensors)
return tf.train.batch(
input_tensors,
batch_size=batch_size,
capacity=QUEUE_CAPACITY,
num_threads=num_enqueuing_threads,
dynamic_pad=True,
allow_smaller_final_batch=False)
def _parse_function(*args):
"""Parses the tf example."""
serialized_example = args[-1]
context_feature_names = {
dataset_descriptor.image_id: tf.FixedLenFeature([], tf.string),
}
sequence_feature_names = {}
if flags.use_ref_exp:
context_feature_names[REF_EXP_ID] = tf.FixedLenFeature([],
tf.string)
if flags.use_labels:
if dataset_descriptor.has_candidate:
context_feature_names[
SELECTED_CANDIDATE_ID] = tf.FixedLenFeature([], tf.int64)
sequence_feature_names[
ELEMENTS_MASK_ID] = tf.FixedLenSequenceFeature([],
tf.string)
else:
context_feature_names[
dataset_descriptor.label_id] = tf.FixedLenFeature(
[], tf.string)
if dataset_descriptor.has_elements_boxes:
sequence_feature_names[
dataset_descriptor.
elements_box_id] = tf.FixedLenSequenceFeature([4],
dtype=tf.float32)
if flags.use_elements_texts:
sequence_feature_names[
dataset_descriptor.
elements_text_id] = tf.FixedLenSequenceFeature([],
dtype=tf.string)
if flags.use_elements_neighbors:
sequence_feature_names[
ELEMENTS_NEIGHBORS_ID] = tf.FixedLenSequenceFeature(
[], dtype=tf.string)
if flags.use_elements_ref_match:
sequence_feature_names[
ELEMENTS_REF_MATCH_ID] = tf.FixedLenSequenceFeature(
[], dtype=tf.string)
if flags.use_groundtruth_box:
context_feature_names[GROUNDTRUTH_XMIN_ID] = tf.FixedLenFeature(
[], tf.float32)
context_feature_names[GROUNDTRUTH_XMAX_ID] = tf.FixedLenFeature(
[], tf.float32)
context_feature_names[GROUNDTRUTH_YMIN_ID] = tf.FixedLenFeature(
[], tf.float32)
context_feature_names[GROUNDTRUTH_YMAX_ID] = tf.FixedLenFeature(
[], tf.float32)
context_features, sequence_features = tf.parse_single_sequence_example(
serialized_example,
context_features=context_feature_names,
sequence_features=sequence_feature_names,
)
features.update(context_features)
features.update(sequence_features)
if flags.use_elements_texts:
features[ELEMENTS_TEXT_ID] = features.pop(
dataset_descriptor.elements_text_id)
if dataset_descriptor.has_elements_boxes:
features[ELEMENTS_BOX_ID] = features.pop(
dataset_descriptor.elements_box_id)
image = features.pop(dataset_descriptor.image_id)
image = tf.image.decode_image(image, channels=3)
image = tf.cast(image, tf.float32)
mean_pixel = tf.reshape(
feature_extractor.mean_pixel(flags.model_variant), [1, 1, 3])
features[IMAGE_PAD_WEIGHTS_ID] = tf.ones_like(image[:, :, 0:1])
features[IMAGE_PAD_WEIGHTS_ID] = resize_im(
features[IMAGE_PAD_WEIGHTS_ID], flags.image_size, 0, 1)
features[IMAGE_PAD_WEIGHTS_ID] = tf.squeeze(
features[IMAGE_PAD_WEIGHTS_ID], 2)
if dataset_descriptor.has_elements_boxes:
image = resize_im(image, flags.image_size, mean_pixel, 3, features)
else:
image = resize_im(image, flags.image_size, mean_pixel, 3)
if flags.use_labels:
if dataset_descriptor.has_candidate:
features[ELEMENTS_MASK_ID] = tf.map_fn(
process_label,
features.pop(ELEMENTS_MASK_ID),
parallel_iterations=128,
dtype=tf.int32,
name="mask_map")
features[LABEL_ID] = tf.gather_nd(
features[ELEMENTS_MASK_ID],
[features[SELECTED_CANDIDATE_ID]])
else:
label = features.pop(dataset_descriptor.label_id)
label = process_label(label)
features[LABEL_ID] = label
if flags.use_elements_texts:
features[ELEMENTS_EXIST_ID] = tf.ones_like(
features[ELEMENTS_TEXT_ID], dtype=tf.int32)
elif dataset_descriptor.has_elements_boxes:
features[ELEMENTS_EXIST_ID] = tf.ones(tf.shape(
features[ELEMENTS_BOX_ID])[:1],
dtype=tf.int32)
if flags.use_elements_neighbors:
features[ELEMENTS_NEIGHBORS_ID] = convert_string_neighbors(
features[ELEMENTS_NEIGHBORS_ID])
features[IMAGE_ID] = image
return features
def shift_melody(example):
# one example is one melody
_, sequence = tf.parse_single_sequence_example(
serialized=example, sequence_features=sequence_features)
# return melody from first step as input and melody shifted by one step to the left as label
return sequence['inputs'][:-1], sequence['inputs'][1:]
def _parse_function(self, sequence_example_proto):
"""Parse a SequenceExample in the AutoDL/TensorFlow format.
Args:
sequence_example_proto: a SequenceExample with "x_dense_input" or sparse
input representation.
Returns:
An array of tensors. For first edition of AutoDl challenge, returns a
pair `(features, labels)` where `features` is a Tensor of shape
[sequence_size, row_count, col_count, num_channels]
and `labels` a Tensor of shape
[output_dim, ]
"""
sequence_features = {}
for i in range(self.metadata_.get_bundle_size()):
if self.metadata_.is_sparse(i):
sequence_features[self._feature_key(
i, "sparse_col_index")] = tf.VarLenFeature(tf.int64)
sequence_features[self._feature_key(
i, "sparse_row_index")] = tf.VarLenFeature(tf.int64)
sequence_features[self._feature_key(
i, "sparse_channel_index")] = tf.VarLenFeature(tf.int64)
sequence_features[self._feature_key(
i, "sparse_value")] = tf.VarLenFeature(tf.float32)
elif self.metadata_.is_compressed(i):
sequence_features[self._feature_key(
i, "compressed")] = tf.VarLenFeature(tf.string)
else:
sequence_features[self._feature_key(
i, "dense_input")] = tf.FixedLenSequenceFeature(
self.metadata_.get_tensor_size(i), dtype=tf.float32)
# read TFRecord
contexts, features = tf.parse_single_sequence_example(
sequence_example_proto,
context_features={
"label_index": tf.VarLenFeature(tf.int64),
"label_score": tf.VarLenFeature(tf.float32),
},
sequence_features=sequence_features,
)
sample = [] # will contain [features, labels]
for i in range(self.metadata_.get_bundle_size()):
key_dense = self._feature_key(i, "dense_input")
row_count, col_count = self.metadata_.get_matrix_size(i)
num_channels = self.metadata_.get_num_channels(i)
sequence_size = self.metadata_.get_sequence_size()
fixed_matrix_size = row_count > 0 and col_count > 0
row_count = row_count if row_count > 0 else None
col_count = col_count if col_count > 0 else None
if key_dense in features:
f = features[key_dense]
if not fixed_matrix_size:
raise ValueError(
"To parse dense data, the tensor shape should " +
"be known but got {} instead...".format(
(sequence_size, row_count, col_count)))
f = tf.reshape(
f, [sequence_size, row_count, col_count, num_channels])
sample.append(f)
sequence_size = sequence_size if sequence_size > 0 else None
key_compressed = self._feature_key(i, "compressed")
if key_compressed in features:
compressed_images = features[key_compressed].values
decompress_image_func = lambda x: dataset_utils.decompress_image(
x, num_channels=num_channels)
# `images` here is a 4D-tensor of shape [T, H, W, C], some of which might be unknown
images = tf.map_fn(decompress_image_func,
compressed_images,
dtype=tf.float32)
images.set_shape(
[sequence_size, row_count, col_count, num_channels])
sample.append(images)
key_sparse_val = self._feature_key(i, "sparse_value")
if key_sparse_val in features:
key_sparse_col = self._feature_key(i, "sparse_col_index")
key_sparse_row = self._feature_key(i, "sparse_row_index")
key_sparse_channel = self._feature_key(i,
"sparse_channel_index")
sparse_col = features[key_sparse_col].values
sparse_row = features[key_sparse_row].values
try: # For back-compatibility. Before, there was no channel dimension.
sparse_channel = features[key_sparse_channel].values
except:
# I think this won't work, Tensor object has no 'len'
sparse_channel = [0] * len(sparse_col)
sparse_val = features[key_sparse_val].values
if col_count > num_channels:
print("Sparse tabular data")
# TABULAR: [120, 1]
# [1000, 2]
# [1504, 1]
# each row is (index, value)
sparse_col = tf.cast(sparse_col, tf.float32)
sparse_channel = tf.cast(sparse_channel, tf.float32)
tensor = tf.concat([
tf.reshape(sparse_col, [-1, 1]),
tf.reshape(sparse_val, [-1, 1])
], 1)
tensor = tf.reshape(tensor, [1, -1, 2, 1])
tensor = tf.cast(tensor, tf.float32)
# Could use SparseTensor (to dense) because the shape of the dense tensor is known:
# (1, col_count, 1, 1)
else:
print("Sparse text data")
# TEXT: [232, 2, 41]
# each example is a 'time series' of indexes
tensor = tf.reshape(sparse_channel, [-1, 1, 1, 1])
tensor = tf.cast(tensor, tf.float32)
sample.append(tensor)
# TODO: see how we can keep sparse tensors instead of
# returning dense ones.
label_indices = (contexts["label_index"].values, )
label_indices = tf.reshape(label_indices, [-1, 1])
sparse_tensor = tf.sparse.SparseTensor(
indices=label_indices,
values=contexts["label_score"].values,
dense_shape=(self.metadata_.get_output_size(), ),
)
labels = tf.sparse.to_dense(sparse_tensor, validate_indices=False)
sample.append(labels)
return sample