def _reshape_keypoints(self, keys_to_tensors):
"""Reshape keypoints.
The keypoints are reshaped to [num_instances, num_keypoints, 2].
Args:
keys_to_tensors: a dictionary from keys to tensors. Expected keys are:
'image/object/keypoint/x'
'image/object/keypoint/y'
Returns:
A 3-D float tensor of shape [num_instances, num_keypoints, 2] with values
in [0, 1].
"""
y = keys_to_tensors['image/object/keypoint/y']
if isinstance(y, tf.SparseTensor):
y = tf.sparse_tensor_to_dense(y)
y = tf.expand_dims(y, 1)
x = keys_to_tensors['image/object/keypoint/x']
if isinstance(x, tf.SparseTensor):
x = tf.sparse_tensor_to_dense(x)
x = tf.expand_dims(x, 1)
keypoints = tf.concat([y, x], 1)
keypoints = tf.reshape(keypoints, [-1, self._num_keypoints, 2])
return keypoints
def get_optimizer(model_str, model, discriminator, placeholders, pos_weight,
norm, d_real, num_nodes):
if model_str == 'arga_ae':
d_fake = discriminator.construct(model.embeddings, reuse=True)
opt = OptimizerAE(preds=model.reconstructions,
labels=tf.reshape(
tf.sparse_tensor_to_dense(
placeholders['adj_orig'],
validate_indices=False), [-1]),
pos_weight=pos_weight,
norm=norm,
d_real=d_real,
d_fake=d_fake)
elif model_str == 'arga_vae':
opt = OptimizerVAE(preds=model.reconstructions,
labels=tf.reshape(
tf.sparse_tensor_to_dense(
placeholders['adj_orig'],
validate_indices=False), [-1]),
model=model,
num_nodes=num_nodes,
pos_weight=pos_weight,
norm=norm,
d_real=d_real,
d_fake=discriminator.construct(model.embeddings,
reuse=True))
return opt
def decode(self, serialized_example):
parsed_tensors = tf.io.parse_single_example(serialized_example, self._keys_to_features)
for k in parsed_tensors:
if isinstance(parsed_tensors[k], tf.SparseTensor):
if parsed_tensors[k].dtype == tf.string:
parsed_tensors[k] = tf.sparse_tensor_to_dense(
parsed_tensors[k], default_value='')
else:
parsed_tensors[k] = tf.sparse_tensor_to_dense(
parsed_tensors[k], default_value=0)
image = self._decode_image(parsed_tensors) # 解析图像
boxes = self._decode_boxes(parsed_tensors)
areas = self._decode_areas(parsed_tensors)
decode_image_shape = tf.logical_or(tf.equal(parsed_tensors['image/height'], -1),
tf.equal(parsed_tensors['image/width'], -1))
image_shape = tf.cast(tf.shape(image), dtype=tf.int64)
parsed_tensors['image/height'] = tf.where(decode_image_shape, image_shape[0], parsed_tensors['image/height'])
parsed_tensors['image/width'] = tf.where(decode_image_shape, image_shape[1], parsed_tensors['image/width'])
decoded_tensors = {
'image': image,
'height': parsed_tensors['image/height'],
'width': parsed_tensors['image/width'],
'groundtruth_classes': parsed_tensors['image/object/class/label'],
'groundtruth_area': areas,
'groundtruth_boxes': boxes,
}
return decoded_tensors
def _parse_example(serialized_example):
"""Return inputs and targets Tensors from a serialized tf.Example."""
data_fields = {
"inputs": tf.VarLenFeature(tf.int64),
"targets": tf.VarLenFeature(tf.int64)
}
parsed = tf.parse_single_example(serialized_example, data_fields)
inputs = tf.sparse_tensor_to_dense(parsed["inputs"])
targets = tf.sparse_tensor_to_dense(parsed["targets"])
return inputs, targets
def _dense_pose_surface_coordinates(self, keys_to_tensors):
"""Creates a tensor that contains surface coords for each DensePose point.
Args:
keys_to_tensors: a dictionary from keys to tensors.
Returns:
A 3-D float32 tensor of shape [num_instances, num_points, 4] where each
point contains (y, x, v, u) data for each sampled DensePose point. The
(y, x) coordinate has normalized image locations for the point, and (v, u)
contains the surface coordinate (also normalized) for the part. The value
`num_points` corresponds to the maximum number of sampled points across
all instances in the image. Note that instances with less sampled points
will be padded with zeros in dim=1.
"""
num_points_per_instances = keys_to_tensors[
'image/object/densepose/num']
dp_y = keys_to_tensors['image/object/densepose/y']
dp_x = keys_to_tensors['image/object/densepose/x']
dp_v = keys_to_tensors['image/object/densepose/v']
dp_u = keys_to_tensors['image/object/densepose/u']
if isinstance(num_points_per_instances, tf.SparseTensor):
num_points_per_instances = tf.sparse_tensor_to_dense(
num_points_per_instances)
if isinstance(dp_y, tf.SparseTensor):
dp_y = tf.sparse_tensor_to_dense(dp_y)
if isinstance(dp_x, tf.SparseTensor):
dp_x = tf.sparse_tensor_to_dense(dp_x)
if isinstance(dp_v, tf.SparseTensor):
dp_v = tf.sparse_tensor_to_dense(dp_v)
if isinstance(dp_u, tf.SparseTensor):
dp_u = tf.sparse_tensor_to_dense(dp_u)
max_points_per_instance = tf.cast(
tf.math.reduce_max(num_points_per_instances), dtype=tf.int32)
num_points_cumulative = tf.concat(
[[0], tf.math.cumsum(num_points_per_instances)], axis=0)
def pad_surface_coordinates_tensor(instance_ind):
"""Pads DensePose surface coordinates for each instance."""
points_range_start = num_points_cumulative[instance_ind]
points_range_end = num_points_cumulative[instance_ind + 1]
y = dp_y[points_range_start:points_range_end]
x = dp_x[points_range_start:points_range_end]
v = dp_v[points_range_start:points_range_end]
u = dp_u[points_range_start:points_range_end]
# Create [num_points_i, 4] tensor, where num_points_i is the number of
# sampled points for instance i.
unpadded_tensor = tf.stack([y, x, v, u], axis=1)
return shape_utils.pad_or_clip_nd(
unpadded_tensor, output_shape=[max_points_per_instance, 4])
return tf.map_fn(pad_surface_coordinates_tensor,
tf.range(tf.size(num_points_per_instances)),
dtype=tf.float32)
def decode_dataset(self, serialized_example):
example = tf.parse_single_example(
serialized_example,
features={
"s1": tf.VarLenFeature(tf.float32),
"s2": tf.VarLenFeature(tf.float32)
},
)
s1 = tf.sparse_tensor_to_dense(example["s1"])
s2 = tf.sparse_tensor_to_dense(example["s2"])
audios = tf.stack([s1, s2])
return audios
def decode(self, serialized_example):
"""Decode the serialized example.
Args:
serialized_example: a single serialized tf.Example string.
Returns:
decoded_tensors: a dictionary of tensors with the following fields:
- image: a uint8 tensor of shape [None, None, 3].
- height: an integer scalar tensor.
- width: an integer scalar tensor.
- groundtruth_classes: a int64 tensor of shape [None].
- groundtruth_boxes: a float32 tensor of shape [None, 4].
"""
parsed_tensors = tf.io.parse_single_example(
serialized_example, self._keys_to_features
)
for k in parsed_tensors:
if isinstance(parsed_tensors[k], tf.SparseTensor):
if parsed_tensors[k].dtype == tf.string:
parsed_tensors[k] = tf.sparse_tensor_to_dense(
parsed_tensors[k], default_value=""
)
else:
parsed_tensors[k] = tf.sparse_tensor_to_dense(
parsed_tensors[k], default_value=0
)
image = self._decode_image(parsed_tensors)
boxes = self._decode_boxes(parsed_tensors)
decode_image_shape = tf.logical_or(
tf.equal(parsed_tensors["image/height"], -1),
tf.equal(parsed_tensors["image/width"], -1),
)
image_shape = tf.cast(tf.shape(image), dtype=tf.int64)
parsed_tensors["image/height"] = tf.where(
decode_image_shape, image_shape[0], parsed_tensors["image/height"]
)
parsed_tensors["image/width"] = tf.where(
decode_image_shape, image_shape[1], parsed_tensors["image/width"]
)
decoded_tensors = {
"image": image,
"height": parsed_tensors["image/height"],
"width": parsed_tensors["image/width"],
"groundtruth_classes": parsed_tensors["image/object/class/label"],
"groundtruth_boxes": boxes,
}
return decoded_tensors
def _decode_png_instance_masks(self, keys_to_tensors):
"""Decode PNG instance segmentation masks and stack into dense tensor.
The instance segmentation masks are reshaped to [num_instances, height,
width].
Args:
keys_to_tensors: a dictionary from keys to tensors.
Returns:
A 3-D float tensor of shape [num_instances, height, width] with values
in {0, 1}.
"""
def decode_png_mask(image_buffer):
image = tf.squeeze(
tf.image.decode_image(image_buffer, channels=1), axis=2)
image.set_shape([None, None])
image = tf.cast(tf.greater(image, 0), dtype=tf.float32)
return image
png_masks = keys_to_tensors['image/object/mask']
height = keys_to_tensors['image/height']
width = keys_to_tensors['image/width']
if isinstance(png_masks, tf.SparseTensor):
png_masks = tf.sparse_tensor_to_dense(png_masks, default_value='')
return tf.cond(
tf.greater(tf.size(png_masks), 0),
lambda: tf.map_fn(decode_png_mask, png_masks, dtype=tf.float32),
lambda: tf.zeros(tf.cast(tf.stack([0, height, width]), dtype=tf.int32)))
def testAddBOWIntegratedGradientsOps(self):
with tf.Graph().as_default() as graph:
# pyformat: disable
embedding_weights = tf.constant([[1., 3., 5.], [4., 6., 8.],
[4., 5., 4.]])
batch_size = tf.placeholder_with_default(tf.constant(1, tf.int64),
[])
sparse_ids = tf.SparseTensor([[0, 0], [0, 1], [0, 2]], [2, 0, 2],
[batch_size, 3])
# pyformat: enable
sparse_embedding = contrib_layers.safe_embedding_lookup_sparse(
embedding_weights,
sparse_ids,
combiner='sum',
partition_strategy='div')
vector = tf.constant([1., 2., 4.])
output_tensor = tf.reduce_sum(vector * sparse_embedding, axis=1)
embedding_lookup = tf.nn.embedding_lookup(
embedding_weights,
tf.sparse_tensor_to_dense(sparse_ids),
partition_strategy='div')
bow_attribution_hooks = integrated_gradients.AddBOWIntegratedGradientsOps(
graph, [embedding_lookup], [sparse_embedding], [],
output_tensor)
with tf.Session(graph=graph) as sess:
sess.run(tf.global_variables_initializer())
result = sess.run(bow_attribution_hooks['bow_attributions'])
self.assertTupleEqual(result[0].shape, (3, ))
# Since the output is a sum of dot products, attributions are simply dot
# products of the embedding with [1., 2., 4.].
self.assertAlmostEqual(result[0][0], 30.)
self.assertAlmostEqual(result[0][1], 27.)
self.assertAlmostEqual(result[0][2], 30.)
def parse_fn(sequence_example):
"""Parses a clip classification example."""
context_features = {
ms.get_example_id_key():
ms.get_example_id_default_parser(),
ms.get_clip_label_index_key():
ms.get_clip_label_index_default_parser(),
ms.get_clip_label_string_key():
ms.get_clip_label_string_default_parser()
}
sequence_features = {
ms.get_image_encoded_key(): ms.get_image_encoded_default_parser(),
}
parsed_context, parsed_sequence = tf.io.parse_single_sequence_example(
sequence_example, context_features, sequence_features)
example_id = parsed_context[ms.get_example_id_key()]
classification_target = tf.one_hot(
tf.sparse_tensor_to_dense(
parsed_context[ms.get_clip_label_index_key()]), NUM_CLASSES)
images = tf.map_fn(
tf.image.decode_jpeg,
parsed_sequence[ms.get_image_encoded_key()],
back_prop=False,
dtype=tf.uint8)
return {
"id": example_id,
"labels": classification_target,
"images": images,
}
def tensors_to_item(self, keys_to_tensors):
"""Maps the given dictionary of tensors to a concatenated list of bboxes.
Args:
keys_to_tensors: a mapping of TF-Example keys to parsed tensors.
Returns:
[time, num_boxes, 4] tensor of bounding box coordinates, in order
[y_min, x_min, y_max, x_max]. Whether the tensor is a SparseTensor
or a dense Tensor is determined by the return_dense parameter. Empty
positions in the sparse tensor are filled with -1.0 values.
"""
sides = []
for key in self._full_keys:
value = keys_to_tensors[key]
expanded_dims = tf.concat([
tf.to_int64(tf.shape(value)),
tf.constant([1], dtype=tf.int64)
], 0)
side = tf.sparse_reshape(value, expanded_dims)
sides.append(side)
bounding_boxes = tf.sparse_concat(2, sides)
if self._return_dense:
bounding_boxes = tf.sparse_tensor_to_dense(
bounding_boxes, default_value=self._default_value)
return bounding_boxes
def _slice_with_actions(embeddings, actions):
"""Slice a Tensor.
Take embeddings of the form [batch_size, num_actions, embed_dim]
and actions of the form [batch_size, 1], and return the sliced embeddings
like embeddings[:, actions, :].
Args:
embeddings: Tensor of embeddings to index.
actions: int Tensor to use as index into embeddings
Returns:
Tensor of embeddings indexed by actions
"""
batch_size, num_actions = embeddings.get_shape()[:2]
# Values are the 'values' in a sparse tensor we will be setting
act_indx = tf.cast(actions, tf.int64)[:, None]
values = tf.reshape(tf.cast(tf.ones(tf.shape(actions)), tf.bool), [-1])
# Create a range for each index into the batch
act_range = tf.range(0, batch_size, dtype=tf.int64)[:, None]
# Combine this into coordinates with the action indices
indices = tf.concat([act_range, act_indx], 1)
actions_mask = tf.SparseTensor(indices, values, [batch_size, num_actions])
actions_mask = tf.stop_gradient(
tf.sparse_tensor_to_dense(actions_mask, default_value=False))
sliced_emb = tf.boolean_mask(embeddings, actions_mask)
return sliced_emb
def _reshape_keypoint_visibilities(self, keys_to_tensors):
"""Reshape keypoint visibilities.
The keypoint visibilities are reshaped to [num_instances,
num_keypoints].
The raw keypoint visibilities are expected to conform to the
MSCoco definition. See Visibility enum.
The returned boolean is True for the labeled case (either
Visibility.NOT_VISIBLE or Visibility.VISIBLE). These are the same categories
that COCO uses to evaluate keypoint detection performance:
http://cocodataset.org/#keypoints-eval
If image/object/keypoint/visibility is not provided, visibilities will be
set to True for finite keypoint coordinate values, and 0 if the coordinates
are NaN.
Args:
keys_to_tensors: a dictionary from keys to tensors. Expected keys are:
'image/object/keypoint/x'
'image/object/keypoint/visibility'
Returns:
A 2-D bool tensor of shape [num_instances, num_keypoints] with values
in {0, 1}. 1 if the keypoint is labeled, 0 otherwise.
"""
x = keys_to_tensors['image/object/keypoint/x']
vis = keys_to_tensors['image/object/keypoint/visibility']
if isinstance(vis, tf.SparseTensor):
vis = tf.sparse_tensor_to_dense(vis)
if isinstance(x, tf.SparseTensor):
x = tf.sparse_tensor_to_dense(x)
default_vis = tf.where(
tf.math.is_nan(x),
Visibility.UNLABELED.value * tf.ones_like(x, dtype=tf.int64),
Visibility.VISIBLE.value * tf.ones_like(x, dtype=tf.int64))
# Use visibility if provided, otherwise use the default visibility.
vis = tf.cond(tf.equal(tf.size(x), tf.size(vis)),
true_fn=lambda: vis,
false_fn=lambda: default_vis)
vis = tf.math.logical_or(
tf.math.equal(vis, Visibility.NOT_VISIBLE.value),
tf.math.equal(vis, Visibility.VISIBLE.value))
vis = tf.reshape(vis, [-1, self._num_keypoints])
return vis
def dot1(x, y, sparse=False):
"""Wrapper for tf.matmul (sparse vs dense)."""
if sparse:
y = tf.sparse_tensor_to_dense(y)
res = tf.sparse_tensor_dense_matmul(x, y)
else:
res = tf.sparse_tensor_dense_matmul(x, y)
return res
def compute_coarse_matrix_sparse(self, model, A_stencil, A_matrices, grid_size, bb=True):
if bb == True:
P_stencil = model(inputs=A_stencil, black_box=True)
else:
P_stencil = model(inputs=A_stencil, black_box=False, phase="Test")
P_matrix = tf.to_double(self.compute_p2_sparse(P_stencil, P_stencil.shape.as_list()[0], grid_size))
P_matrix_t = tf.sparse_transpose(P_matrix, [0, 2, 1])
A_matrices = tf.squeeze(A_matrices)
temp = tf.sparse_tensor_to_dense(P_matrix_t)
q = tf.matmul(temp, tf.to_double(A_matrices))
A_c = tf.transpose(tf.matmul(temp, tf.transpose(q, [0, 2, 1])), [0, 2, 1])
return A_c, self.compute_stencil(tf.squeeze(A_c), (grid_size // 2)), P_matrix, P_matrix_t
def parser(record):
keys_to_features = {
'dirtymap': tf.VarLenFeature(tf.float32),
'p2': tf.FixedLenFeature((), tf.float32),
'coordinates': tf.VarLenFeature(tf.float32)
}
parsed = tf.parse_single_example(record, keys_to_features)
# Perform additional preprocessing on the parsed data.
dm = tf.reshape(
tf.sparse_tensor_to_dense(parsed['dirtymap']),
[NXSTEPS, NYSTEPS, 1]) # maybe direct from byte string map
coordinates = tf.reshape(
tf.sparse_tensor_to_dense(parsed['coordinates']), [2, 1])
p2 = tf.reshape(parsed["p2"], [1, 1])
return {
"features": dm
}, {
"labels": tf.squeeze(tf.concat([coordinates, p2], 0)),
'coordinates': tf.squeeze(coordinates),
'p2': tf.squeeze(p2)
}
def tf_set_intersection(set_a, set_b):
'''Now with 100% less sparse tensors!
Like tf.contrib.metrics.set_intersection.
'''
assert len(set_a.get_shape()) == len(set_b.get_shape())
rank_one = len(set_a.get_shape()) == 1
if rank_one:
set_a = tf.expand_dims(set_a, 0)
set_b = tf.expand_dims(set_b, 0)
intersection = tf.sparse_tensor_to_dense(
tf.contrib.metrics.set_intersection(set_a, set_b))
if rank_one:
intersection = tf.squeeze(intersection, [0])
return intersection
def _dense_pose_part_indices(self, keys_to_tensors):
"""Creates a tensor that contains part indices for each DensePose point.
Args:
keys_to_tensors: a dictionary from keys to tensors.
Returns:
A 2-D int32 tensor of shape [num_instances, num_points] where each element
contains the DensePose part index (0-23). The value `num_points`
corresponds to the maximum number of sampled points across all instances
in the image. Note that instances with less sampled points will be padded
with zeros in the last dimension.
"""
num_points_per_instances = keys_to_tensors[
'image/object/densepose/num']
part_index = keys_to_tensors['image/object/densepose/part_index']
if isinstance(num_points_per_instances, tf.SparseTensor):
num_points_per_instances = tf.sparse_tensor_to_dense(
num_points_per_instances)
if isinstance(part_index, tf.SparseTensor):
part_index = tf.sparse_tensor_to_dense(part_index)
part_index = tf.cast(part_index, dtype=tf.int32)
max_points_per_instance = tf.cast(
tf.math.reduce_max(num_points_per_instances), dtype=tf.int32)
num_points_cumulative = tf.concat(
[[0], tf.math.cumsum(num_points_per_instances)], axis=0)
def pad_parts_tensor(instance_ind):
points_range_start = num_points_cumulative[instance_ind]
points_range_end = num_points_cumulative[instance_ind + 1]
part_inds = part_index[points_range_start:points_range_end]
return shape_utils.pad_or_clip_nd(
part_inds, output_shape=[max_points_per_instance])
return tf.map_fn(pad_parts_tensor,
tf.range(tf.size(num_points_per_instances)),
dtype=tf.int32)
def get_optimizer(model, discriminator, placeholders, pos_weight, norm, d_real,num_nodes,attr_labels_list):
d_fake = discriminator.construct(model.embeddings, reuse=True)
# pred_attrs=[model.attr0_logits,model.attr1_logits,model.attr2_logits,
# model.attr3_logits,model.attr4_logits]
pred_attrs = [model.attr_logits,model.pri_logits]
opt = OptimizerAE(preds=model.reconstructions,
labels=tf.reshape(tf.sparse_tensor_to_dense(placeholders['adj_orig'],validate_indices=False), [-1]),
pos_weight=pos_weight,
pred_attrs = pred_attrs,
norm=norm,
d_real=d_real,
d_fake=d_fake,
attr_labels_list=attr_labels_list,
sample_list=model.sample)
return opt
def _reshape_context_features(self, keys_to_tensors):
"""Reshape context features.
The instance context_features are reshaped to
[num_context_features, context_feature_length]
Args:
keys_to_tensors: a dictionary from keys to tensors.
Returns:
A 2-D float tensor of shape [num_context_features, context_feature_length]
"""
context_feature_length = keys_to_tensors['image/context_feature_length']
to_shape = tf.cast(tf.stack([-1, context_feature_length]), tf.int32)
context_features = keys_to_tensors['image/context_features']
if isinstance(context_features, tf.SparseTensor):
context_features = tf.sparse_tensor_to_dense(context_features)
context_features = tf.reshape(context_features, to_shape)
return context_features
def unpool_layer2x2_batch(bottom, argmax):
bottom_shape = tf.shape(bottom)
top_shape = [
bottom_shape[0], bottom_shape[1] * 2, bottom_shape[2] * 2,
bottom_shape[3]
]
batch_size = top_shape[0]
height = top_shape[1]
width = top_shape[2]
channels = top_shape[3]
argmax_shape = tf.to_int64([batch_size, height, width, channels])
argmax = unravel_argmax(argmax, argmax_shape)
t1 = tf.to_int64(tf.range(channels))
t1 = tf.tile(t1, [batch_size * (width // 2) * (height // 2)])
t1 = tf.reshape(t1, [-1, channels])
t1 = tf.transpose(t1, perm=[1, 0])
t1 = tf.reshape(t1, [channels, batch_size, height // 2, width // 2, 1])
t1 = tf.transpose(t1, perm=[1, 0, 2, 3, 4])
t2 = tf.to_int64(tf.range(batch_size))
t2 = tf.tile(t2, [channels * (width // 2) * (height // 2)])
t2 = tf.reshape(t2, [-1, batch_size])
t2 = tf.transpose(t2, perm=[1, 0])
t2 = tf.reshape(t2, [batch_size, channels, height // 2, width // 2, 1])
t3 = tf.transpose(argmax, perm=[1, 4, 2, 3, 0])
t = tf.concat(4, [t2, t3, t1])
indices = tf.reshape(t, [(height // 2) *
(width // 2) * channels * batch_size, 4])
x1 = tf.transpose(bottom, perm=[0, 3, 1, 2])
values = tf.reshape(x1, [-1])
delta = tf.SparseTensor(indices, values, tf.to_int64(top_shape))
return tf.sparse_tensor_to_dense(tf.sparse_reorder(delta))
def _reshape_instance_masks(self, keys_to_tensors):
"""Reshape instance segmentation masks.
The instance segmentation masks are reshaped to [num_instances, height,
width].
Args:
keys_to_tensors: a dictionary from keys to tensors.
Returns:
A 3-D float tensor of shape [num_instances, height, width] with values
in {0, 1}.
"""
height = keys_to_tensors['image/height']
width = keys_to_tensors['image/width']
to_shape = tf.cast(tf.stack([-1, height, width]), tf.int32)
masks = keys_to_tensors['image/object/mask']
if isinstance(masks, tf.SparseTensor):
masks = tf.sparse_tensor_to_dense(masks)
masks = tf.reshape(
tf.cast(tf.greater(masks, 0.0), dtype=tf.float32), to_shape)
return tf.cast(masks, tf.float32)
def _mapper(dataset):
"""Tokenizes strings using tf.string_split and truncates by length."""
for k in keys_to_map:
# pylint: disable=g-explicit-length-test
if len(dataset[k].get_shape()) == 0: # Used for questions.
# pylint: enable=g-explicit-length-test
# <string> [num_tokens]
tokens = tf.string_split([dataset[k]]).values
else: # Used for contexts.
# <string> [num_context, num_tokens] (sparse)
sparse_tokens = tf.string_split(dataset[k])
# <string>[num_tokens, max_num_tokens] (dense)
tokens = tf.sparse_tensor_to_dense(sparse_tokens,
default_value="")
dataset[k + suffix] = tokens
# Compute exact length of each context.
dataset[k + suffix + "_len"] = tf.count_nonzero(tokens,
axis=-1,
dtype=tf.int32)
return dataset
def __init__(self, batch_size=None):
net_params, train_params = parser_cfg_file('./net.cfg')
self._model_save_path = str(net_params['model_save_path'])
self.input_img_height = int(net_params['input_height'])
self.input_img_width = int(net_params['input_width'])
if batch_size is None:
self.test_batch_size = int(net_params['test_batch_size'])
else:
self.test_batch_size = batch_size
# 加载label onehot
f = open('./data/word_onehot.txt', 'r', encoding='utf-8')
data = f.read()
words_onehot_dict = eval(data)
self.words_list = list(words_onehot_dict.keys())
self.words_onehot_list = [
words_onehot_dict[self.words_list[i]]
for i in range(len(self.words_list))
]
# 构建网络
self.inputs_tensor = tf.placeholder(tf.float32, [
self.test_batch_size, self.input_img_height, self.input_img_width,
1
])
self.seq_len_tensor = tf.placeholder(tf.int32, [None], name='seq_len')
crnn_net = CRNN(net_params, self.inputs_tensor, self.seq_len_tensor,
self.test_batch_size, True)
net_output, decoded, self.max_char_count = crnn_net.construct_graph()
self.dense_decoded = tf.sparse_tensor_to_dense(decoded[0],
default_value=-1)
self.sess = tf.Session()
saver = tf.train.Saver()
saver.restore(self.sess, "./model/ckpt")
def _stitch(features):
"""Stitch features on the first dimension."""
full_mask = tf.greater(features['task'], 1)
step_mask = tf.reduce_any(full_mask, axis=-1)
step_mask_exclude_last = tf.pad(step_mask, [[0, 0], [0, 1]],
constant_values=False)[:, 1:]
num_sequences = common_layers.shape_list(features['task'])[0]
num_steps = common_layers.shape_list(features['task'])[1]
connectors = tf.constant(PADDED_CONCATENATORS)
# Select connectors
connector_indices = tf.random.uniform([num_sequences * num_steps],
minval=0,
maxval=len(PADDED_CONCATENATORS),
dtype=tf.int32)
selected_connectors = tf.reshape(
tf.gather(connectors, connector_indices),
[num_sequences, num_steps,
len(PADDED_CONCATENATORS[0])])
selected_connectors = tf.multiply(selected_connectors,
tf.expand_dims(
tf.to_int32(step_mask_exclude_last),
2),
name='connector_mask')
features['task'] = tf.concat([features['task'], selected_connectors],
axis=-1)
ref_offsets = tf.expand_dims(
tf.cumsum(tf.reduce_sum(tf.to_int32(tf.greater(features['task'], 1)),
-1),
exclusive=True,
axis=-1), 2)
features['task'] = tf.reshape(features['task'], [num_sequences, -1])
full_mask = tf.greater(features['task'], 1)
full_mask_int = tf.to_int32(full_mask)
indices = tf.where(
tf.sequence_mask(lengths=tf.reduce_sum(full_mask_int, -1)))
values = tf.boolean_mask(tf.reshape(features['task'], [-1]),
tf.reshape(full_mask, [-1]))
sparse_task = tf.sparse.SparseTensor(indices=indices,
values=values,
dense_shape=tf.to_int64(
tf.shape(features['task'])))
# Stitch task and raw_task
stitched_features = {}
stitched_features['task'] = tf.sparse_tensor_to_dense(sparse_task)
max_len = tf.reduce_max(
tf.reduce_sum(tf.to_int32(tf.greater(stitched_features['task'], 1)),
-1))
stitched_features['task'] = stitched_features['task'][:, :max_len]
if 'raw_task' in features:
connector_strs = tf.reshape(
tf.gather(tf.constant(CONCATENATORS_STR), connector_indices),
[num_sequences, num_steps])
masked_connector_strs = tf.where(step_mask_exclude_last,
connector_strs,
tf.fill(tf.shape(connector_strs), ''))
stitched_features['raw_task'] = tf.strings.reduce_join(
tf.strings.reduce_join(tf.concat([
tf.expand_dims(features['raw_task'], 2),
tf.expand_dims(masked_connector_strs, 2)
],
axis=2),
axis=-1), -1)
# Stitch screen sequences
action_lengths = tf.reduce_sum(
tf.to_int32(
tf.greater(features['verb_refs'][:, :, 0, 1],
features['verb_refs'][:, :, 0, 0])), -1)
max_action_length = tf.reduce_max(action_lengths)
def _pad(tensor, padding_value=0):
shape_list = common_layers.shape_list(tensor)
assert len(shape_list) >= 2
padding_list = [[0, 0], [0, 1]] + [[0, 0]] * (len(shape_list) - 2)
return tf.pad(tensor[:, :max_action_length],
padding_list,
constant_values=padding_value)
for key in features.keys():
if key.endswith('_refs'):
features[key] = tf.squeeze(features[key], 2)
ref_mask = tf.expand_dims(
tf.to_int32(
tf.not_equal(features[key][:, :, 0], features[key][:, :,
1])), 2)
stitched_features[key] = tf.multiply((features[key] + ref_offsets),
ref_mask,
name='ref_mask')
stitched_features[key] = _pad(stitched_features[key])
elif key in [
'verbs', 'objects', 'consumed', 'obj_dom_pos', 'obj_text',
'obj_type', 'obj_clickable', 'obj_screen_pos', 'verb_refs',
'obj_refs', 'input_refs', 'obj_dom_dist'
]:
features[key] = tf.squeeze(features[key], 2)
stitched_features[key] = features[key]
stitched_features[key] = _pad(
stitched_features[key],
padding_value=-1 if key == 'obj_type' else 0)
elif key not in ['task', 'raw_task']:
stitched_features[key] = features[key][:, 0]
# Append eos to 'task'
stitched_features['task'] = tf.pad(stitched_features['task'],
[[0, 0], [0, 1]])
task_mask = tf.to_int32(tf.greater(stitched_features['task'], 1))
task_eos_mask = tf.pad(task_mask, [[0, 0], [1, 0]],
constant_values=1)[:, :-1]
stitched_features['task'] = stitched_features['task'] + (task_eos_mask -
task_mask)
# Append eos
verb_mask = tf.to_int32(tf.greater(stitched_features['verbs'], 1))
verb_eos_mask = tf.pad(verb_mask, [[0, 0], [1, 0]],
constant_values=1)[:, :-1]
verb_eos = verb_eos_mask - verb_mask
stitched_features['verbs'] = stitched_features['verbs'] + verb_eos
# Append last step refs to 'verb_refs'
task_lengths = tf.where(tf.equal(stitched_features['task'], 1))[:, 1]
eos_pos = tf.to_int32(tf.stack([task_lengths, task_lengths + 1], axis=1))
action_mask = tf.to_int32(
tf.sequence_mask(action_lengths, max_action_length + 1))
action_and_eos_mask = tf.pad(action_mask, [[0, 0], [1, 0]],
constant_values=1)[:, :-1]
verb_ref_eos = action_and_eos_mask - action_mask
eos_refs = tf.multiply(tf.tile(tf.expand_dims(eos_pos, 1),
[1, max_action_length + 1, 1]),
tf.expand_dims(verb_ref_eos, 2),
name='verb_ref_eos')
stitched_features['verb_refs'] += eos_refs
return stitched_features
# Optimizer
with tf.name_scope('optimizer'):
if model_str == 'gcn_ae':
labels_placeholders = placeholders['adj_orig_values'] , placeholders['adj_orig_l2_splits'] , placeholders['adj_orig_l1_splits']
opt = OptimizerAE(preds=model.reconstructions,batch_size=batchsize,
labels=labels_placeholders
,pos_weight=placeholders['pos_weight'],
norm=placeholders['norm'] , roc=placeholders['ROC_Score'] , ap=placeholders['AP'])
elif model_str == 'gcn_vae':
opt = OptimizerVAE(preds=model.reconstructions,
labels=tf.reshape(tf.sparse_tensor_to_dense(placeholders['adj_orig_values'],
validate_indices=False), [-1]),
model=model, num_nodes=num_nodes,
pos_weight=pos_weight,
norm=norm)
# Initialize session
sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))) #sess = tf.Session()
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
sess.run(tf.global_variables_initializer())
files = os.listdir("C:\\Users\\USER\\Documents\\Projects\\MastersEnv\\GraphAutoEncoder\\gae_batch_update\\myData")
array1 = []
array2 = []
limit = 100000
limit_counter = 0
for file in tqdm(files) :
batch = 0
while end <= len(InputListTest):
batchInputs = []
batchSeqLengths = []
for batchI, origI in enumerate(randIxs[start:end]):
batchInputs.extend(InputListTest[origI])
batchSeqLengths.append(SeqLensTest[origI])
feed = {x: batchInputs, SeqLens: batchSeqLengths}
del batchInputs, batchSeqLengths
Decoded = session.run([decoded], feed_dict=feed)[0]
del feed
trans = session.run(tf.sparse_tensor_to_dense(Decoded[0]))
for i in range(0, cfg.BatchSize):
fileIndex = cfg.BatchSize * batch + i
filename = FilesList[fileIndex].strip()
decodedStr = " "
for j in range(0, len(trans[i])):
if trans[i][j] == 0:
if (j != (len(trans[i]) - 1)):
if trans[i][j + 1] == 0: break
else:
decodedStr = "%s%s" % (decodedStr,
Classes[trans[i][j]])
else:
def decode(self, serialized_example):
"""Decode the serialized example.
Args:
serialized_example: a single serialized tf.Example string.
Returns:
decoded_tensors: a dictionary of tensors with the following fields:
- image: a uint8 tensor of shape [None, None, 3].
- source_id: a string scalar tensor.
- height: an integer scalar tensor.
- width: an integer scalar tensor.
- groundtruth_classes: a int64 tensor of shape [None].
- groundtruth_is_crowd: a bool tensor of shape [None].
- groundtruth_area: a float32 tensor of shape [None].
- groundtruth_boxes: a float32 tensor of shape [None, 4].
- groundtruth_instance_masks: a float32 tensor of shape
[None, None, None].
- groundtruth_instance_masks_png: a string tensor of shape [None].
"""
parsed_tensors = tf.io.parse_single_example(serialized_example,
self._keys_to_features)
for k in parsed_tensors:
if isinstance(parsed_tensors[k], tf.SparseTensor):
if parsed_tensors[k].dtype == tf.string:
parsed_tensors[k] = tf.sparse_tensor_to_dense(
parsed_tensors[k], default_value='')
else:
parsed_tensors[k] = tf.sparse_tensor_to_dense(
parsed_tensors[k], default_value=0)
image = self._decode_image(parsed_tensors)
boxes = self._decode_boxes(parsed_tensors)
areas = self._decode_areas(parsed_tensors)
decode_image_shape = tf.logical_or(
tf.equal(parsed_tensors['image/height'], -1),
tf.equal(parsed_tensors['image/width'], -1))
image_shape = tf.cast(tf.shape(image), dtype=tf.int64)
parsed_tensors['image/height'] = tf.where(
decode_image_shape, image_shape[0], parsed_tensors['image/height'])
parsed_tensors['image/width'] = tf.where(decode_image_shape,
image_shape[1],
parsed_tensors['image/width'])
is_crowds = tf.cond(
tf.greater(tf.shape(parsed_tensors['image/object/is_crowd'])[0], 0),
lambda: tf.cast(parsed_tensors['image/object/is_crowd'], dtype=tf.bool),
lambda: tf.zeros_like(parsed_tensors['image/object/class/label'],
dtype=tf.bool)) # pylint: disable=line-too-long
if self._regenerate_source_id:
source_id = _get_source_id_from_encoded_image(parsed_tensors)
else:
source_id = tf.cond(
tf.greater(
tf.strings.length(parsed_tensors['image/source_id']),
0), lambda: parsed_tensors['image/source_id'],
lambda: _get_source_id_from_encoded_image(parsed_tensors))
if self._include_mask:
masks = self._decode_masks(parsed_tensors)
decoded_tensors = {
'image': image,
'source_id': source_id,
'height': parsed_tensors['image/height'],
'width': parsed_tensors['image/width'],
'groundtruth_classes': parsed_tensors['image/object/class/label'],
'groundtruth_is_crowd': is_crowds,
'groundtruth_area': areas,
'groundtruth_boxes': boxes,
}
if self._include_mask:
decoded_tensors.update({
'groundtruth_instance_masks':
masks,
'groundtruth_instance_masks_png':
parsed_tensors['image/object/mask'],
})
return decoded_tensors
def decode(self, serialized_example):
"""Decode the serialized example.
Args:
serialized_example: a single serialized tf.Example string.
Returns:
decoded_tensors: a dictionary of tensors with the following fields:
- image: a uint8 tensor of shape [None, None, 3].
- source_id: a string scalar tensor.
- height: an integer scalar tensor.
- width: an integer scalar tensor.
- groundtruth_classes: a int64 tensor of shape [None].
- groundtruth_is_crowd: a bool tensor of shape [None].
- groundtruth_area: a float32 tensor of shape [None].
- groundtruth_boxes: a float32 tensor of shape [None, 4].
- groundtruth_attributes - 1D int64 tensor of shape [None].
Optional:
- groundtruth_difficult - 1D bool tensor of shape
[None] indicating if the boxes represent `difficult` instances.
- groundtruth_group_of - 1D bool tensor of shape
[None] indicating if the boxes represent `group_of` instances.
- groundtruth_instance_masks - 3D float32 tensor of
shape [None, None, None] containing instance masks.
- groundtruth_polygons - 1D float tensor of shape [None]
"""
parsed_tensors = tf.io.parse_single_example(
serialized_example, self._keys_to_features)
for k in parsed_tensors:
if isinstance(parsed_tensors[k], tf.SparseTensor):
if parsed_tensors[k].dtype == tf.string:
parsed_tensors[k] = tf.sparse_tensor_to_dense(
parsed_tensors[k], default_value='')
else:
parsed_tensors[k] = tf.sparse_tensor_to_dense(
parsed_tensors[k], default_value=0)
image = self._decode_image(parsed_tensors)
boxes = self._decode_boxes(parsed_tensors)
areas = self._decode_areas(parsed_tensors)
if self._regenerate_source_id:
source_id = _get_source_id_from_encoded_image(parsed_tensors)
else:
source_id = tf.cond(
tf.greater(tf.strings.length(parsed_tensors['image/source_id']),
0), lambda: parsed_tensors['image/source_id'],
lambda: _get_source_id_from_encoded_image(parsed_tensors))
if self._use_instance_mask:
masks = self._decode_masks(parsed_tensors)
decoded_tensors = {
'image': image,
'source_id': source_id,
'height': parsed_tensors['image/height'],
'width': parsed_tensors['image/width'],
'groundtruth_classes': parsed_tensors['image/object/class/label'],
'groundtruth_is_crowd': tf.cast(parsed_tensors['image/object/is_crowd'],
dtype=tf.bool),
'groundtruth_area': areas,
'groundtruth_boxes': boxes,
'groundtruth_attributes': parsed_tensors[
'image/object/attribute/label'],
}
if self._use_instance_mask:
decoded_tensors.update({
'groundtruth_instance_masks': masks,
'groundtruth_polygons': parsed_tensors['image/object/polygon'],
'groundtruth_difficult':
parsed_tensors['image/object/difficult'],
'groundtruth_group_of':
parsed_tensors['image/object/group_of'],
})
return decoded_tensors
def knn_affinity(input_x,
n_nbrs,
scale=None,
scale_nbr=None,
local_scale=None,
verbose=False):
"""Calculates Gaussian affinity matrix.
Calculates the symmetrized Gaussian affinity matrix with k1 nonzero
affinities for each point, scaled by
1) a provided scale,
2) the median distance of the k2-th neighbor of each point in X, or
3) a covariance matrix S where S_ii is the distance of the k2-th
neighbor of each point i, and S_ij = 0 for all i != j
Here, k1 = n_nbrs, k2 = scale_nbr
Args:
input_x: input dataset of size n
n_nbrs: k1
scale: provided scale
scale_nbr: k2, used if scale not provided
local_scale: if True, then we use the aforementioned option 3), else we
use option 2)
verbose: extra printouts
Returns:
n x n affinity matrix
"""
if isinstance(n_nbrs, np.float):
n_nbrs = int(n_nbrs)
elif isinstance(n_nbrs,
tf.Variable) and n_nbrs.dtype.as_numpy_dtype != np.int32:
n_nbrs = tf.cast(n_nbrs, np.int32)
# get squared distance
dist_x = squared_distance(input_x)
# calculate the top k losest neighbors
nn = tf.nn.top_k(-dist_x, n_nbrs, sorted=True)
vals = nn[0]
# apply scale
if scale is None:
# if scale not provided, use local scale
if scale_nbr is None:
scale_nbr = 0
else:
assert scale_nbr > 0 and scale_nbr <= n_nbrs
if local_scale:
scale = -nn[0][:, scale_nbr - 1]
scale = tf.reshape(scale, [-1, 1])
scale = tf.tile(scale, [1, n_nbrs])
scale = tf.reshape(scale, [-1, 1])
vals = tf.reshape(vals, [-1, 1])
if verbose:
vals = tf.Print(vals, [tf.shape(vals), tf.shape(scale)],
'vals, scale shape')
vals = vals / (2 * scale)
vals = tf.reshape(vals, [-1, n_nbrs])
else:
def get_median(scales, m):
with tf.device('/cpu:0'):
scales = tf.nn.top_k(scales, m)[0]
scale = scales[m - 1]
return scale, scales
scales = -vals[:, scale_nbr - 1]
const = tf.shape(input_x)[0] // 2
scale, scales = get_median(scales, const)
vals = vals / (2 * scale)
else:
# otherwise, use provided value for global scale
vals = vals / (2 * scale**2)
# get the affinity
aff_vals = tf.exp(vals)
# flatten this into a single vector of values to shove in a sparse matrix
aff_vals = tf.reshape(aff_vals, [-1])
# get the matrix of indices corresponding to each rank
# with 1 in the first column and k in the kth column
nn_ind = nn[1]
# get the j index for the sparse matrix
j_index = tf.reshape(nn_ind, [-1, 1])
# the i index is just sequential to the j matrix
i_index = tf.range(tf.shape(nn_ind)[0])
i_index = tf.reshape(i_index, [-1, 1])
i_index = tf.tile(i_index, [1, tf.shape(nn_ind)[1]])
i_index = tf.reshape(i_index, [-1, 1])
# concatenate the indices to build the sparse matrix
indices = tf.concat((i_index, j_index), axis=1)
# assemble the sparse weight matrix
weight_mat = tf.SparseTensor(
indices=tf.cast(indices, dtype='int64'),
values=aff_vals,
dense_shape=tf.cast(tf.shape(dist_x), dtype='int64'))
# fix the ordering of the indices
weight_mat = tf.sparse_reorder(weight_mat)
# convert to dense tensor
weight_mat = tf.sparse_tensor_to_dense(weight_mat)
# symmetrize
weight_mat = (weight_mat + tf.transpose(weight_mat)) / 2.0
return weight_mat
