def testMapSparseTensor(self):
with self.test_session():
with self.assertRaises(TypeError):
tf.map_fn(
lambda x: x,
tf.SparseTensor(indices=[[0, 0], [0, 1], [1, 0]], values=tf.constant([0, 1, 2]), shape=[2, 2]),
)
def testMap_Scoped(self):
with self.test_session() as sess:
def double_scoped(x):
"""2x with a dummy 2 that is scoped."""
with tf.variable_scope("body"):
# Dummy variable, just to check that scoping works as intended.
two = tf.get_variable("two", [], dtype=tf.int32,
initializer=tf.constant_initializer(2))
return tf.mul(x, two)
with tf.variable_scope("root") as varscope:
elems = tf.constant([1, 2, 3, 4, 5, 6], name="data")
doubles = np.array([2*x for x in [1, 2, 3, 4, 5, 6]])
r = tf.map_fn(double_scoped, elems)
# Check that we have the one variable we asked for here.
self.assertEqual(len(tf.trainable_variables()), 1)
self.assertEqual(tf.trainable_variables()[0].name, "root/body/two:0")
sess.run([tf.initialize_all_variables()])
self.assertAllEqual(doubles, r.eval())
# Now let's reuse our single variable.
varscope.reuse_variables()
r = tf.map_fn(double_scoped, elems)
self.assertEqual(len(tf.trainable_variables()), 1)
self.assertAllEqual(doubles, r.eval())
def alexnet(image, batch_size=50):
""" Canonical alexnet implementation. See the network detail above
Arg:
images: pixels value in an array (transformed to a tensor)
Return:
logits
"""
x_image = tf.reshape(image, [-1, 32, 32, 3])
# Randomly crop a [height, width] section of the image.
# distorted_image = tf.random_crop(x_image, [height, width, 3])
# Randomly flip the image horizontally.
# distorted_image = tf.image.random_flip_left_right(x_image)
distorted_image = tf.map_fn(lambda image: tf.image.random_flip_left_right(image), x_image)
# Because these operations are not commutative, consider randomizing
# the order their operation.
# NOTE: since per_image_standardization zeros the mean and makes
# the stddev unit, this likely has no effect see tensorflow#1458.
distorted_image = tf.map_fn(lambda image: tf.image.random_brightness(image, max_delta=63), distorted_image)
distorted_image = tf.map_fn(lambda image: tf.image.random_contrast(image, lower=0.2, upper=1.8), distorted_image)
# Subtract off the mean and divide by the variance of the pixels.
float_image = tf.map_fn(lambda image: tf.image.per_image_standardization(image), distorted_image)
# conv1
conv1 = conv_layer(float_image, 3, 64, "conv1")
#poo1
pool1 = tf.nn.max_pool(conv1, ksize=[1,3,3,1], strides=[1,2,2,1], padding='SAME')
#norm1
norm1 = tf.nn.lrn(pool1, depth_radius=4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
#conv2
W_conv2 = weight_variable([5,5,64,64])
conv2 = conv_layer(norm1, 64, 64, "conv2")
#norm2
norm2 = tf.nn.lrn(conv2, depth_radius=4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
#poo2
pool2 = tf.nn.max_pool(norm2, ksize=[1,3,3,1], strides=[1,2,2,1], padding='SAME')
#fc1, fully connected layer
reshape = tf.reshape(pool2, [-1, 8 * 8 * 64])
fc1 = fc_layer(reshape, 8*8*64, 1024, "fc1")
#local4
# weights = weight_variable([384, 192])
# biases = bias_variable([192])
# local4 = tf.nn.relu(tf.matmul(local3, weights) + biases)
# linear layer(WX + b),
logits = fc_layer(fc1, 1024, 10, "output_layer", act=tf.identity)
return logits
def parse_sequence_to_pairs_batch(
serialized_example, preprocess_fn, is_training, num_views, batch_size,
window):
"""Parses a serialized sequence example into a batch of preprocessed data.
Args:
serialized_example: A serialized SequenceExample.
preprocess_fn: A function with the signature (raw_images, is_training) ->
preprocessed_images.
is_training: Boolean, whether or not we're in training.
num_views: Int, the number of simultaneous viewpoints at each timestep in
the dataset.
batch_size: Int, size of the batch to get.
window: Int, only take pairs from a maximium window of this size.
Returns:
preprocessed: A 4-D float32 `Tensor` holding preprocessed images.
anchor_images: A 4-D float32 `Tensor` holding raw anchor images.
pos_images: A 4-D float32 `Tensor` holding raw positive images.
"""
_, views, seq_len = parse_sequence_example(serialized_example, num_views)
# Get random (anchor, positive) timestep and viewpoint indices.
num_pairs = batch_size // 2
ap_time_indices, a_view_indices, p_view_indices = get_tcn_anchor_pos_indices(
seq_len, num_views, num_pairs, window)
# Gather the image strings.
combined_anchor_indices = tf.concat(
[tf.expand_dims(a_view_indices, 1),
tf.expand_dims(ap_time_indices, 1)], 1)
combined_pos_indices = tf.concat(
[tf.expand_dims(p_view_indices, 1),
tf.expand_dims(ap_time_indices, 1)], 1)
anchor_images = tf.gather_nd(views, combined_anchor_indices)
pos_images = tf.gather_nd(views, combined_pos_indices)
# Decode images.
anchor_images = tf.map_fn(
preprocessing.decode_image, anchor_images, dtype=tf.float32)
pos_images = tf.map_fn(
preprocessing.decode_image, pos_images, dtype=tf.float32)
# Concatenate [anchor, postitive] images into a batch and preprocess it.
concatenated = tf.concat([anchor_images, pos_images], 0)
preprocessed = preprocess_fn(concatenated, is_training)
anchor_prepro, positive_prepro = tf.split(preprocessed, num_or_size_splits=2,
axis=0)
# Set static batch dimensions for all image tensors
ims = [anchor_prepro, positive_prepro, anchor_images, pos_images]
ims = [set_image_tensor_batch_dim(i, num_pairs) for i in ims]
[anchor_prepro, positive_prepro, anchor_images, pos_images] = ims
# Assign each anchor and positive the same label.
anchor_labels = tf.range(1, num_pairs+1)
positive_labels = tf.range(1, num_pairs+1)
return (anchor_prepro, positive_prepro, anchor_images, pos_images,
anchor_labels, positive_labels, seq_len)
def testMap_MultiOutputMismatchedDtype(self):
with self.test_session():
nums = np.array([1, 2, 3, 4, 5, 6])
with self.assertRaisesRegexp(
TypeError, r"two structures don't have the same sequence type."):
# lambda emits tuple, but dtype is a list
tf.map_fn(lambda x: ((x + 3) * 2, -(x + 3) * 2), nums,
dtype=[tf.int64, tf.int64])
def loss(self, logits, labels, weights=None, biases=None):
with tf.name_scope("xent"):
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, labels))
l2_reg = 0.0
if weights:
l2_reg += sum(tf.map_fn(tf.nn.l2_loss, weights))
if biases:
l2_reg += sum(tf.map_fn(tf.nn.l2_loss, biases))
loss += self.l2_lambda * l2_reg
return loss
def compare_vs_map(X1, X2, kern, sess):
K12_map = tf.map_fn(lambda x: kern.K(x[0], x[1]), [X1, X2], dtype=settings.float_type)
K12_native = kern.K(X1, X2)
assert_allclose(*sess.run([K12_map, K12_native]))
K_map = tf.map_fn(kern.K, X1, dtype=settings.float_type)
K_native = kern.K(X1)
assert_allclose(*sess.run([K_map, K_native]))
Kdiag_map = tf.map_fn(kern.Kdiag, X1, dtype=settings.float_type)
Kdiag_native = kern.Kdiag(X1)
assert_allclose(*sess.run([Kdiag_map, Kdiag_native]))
def __call__(self, *parents):
if len(parents) != 1:
raise ValueError("Must have one parent")
parent_tensor = parents[0].out_tensor
dense_fn = lambda x: tf.contrib.layers.fully_connected(x, num_outputs=self.out_channels,
activation_fn=tf.nn.sigmoid)
self.out_tensor = tf.map_fn(dense_fn, parent_tensor)
def preprocess_training_images(images, height, width, min_scale, max_scale,
p_scale_up, aug_color=True, fast_mode=True):
"""Preprocesses a batch of images for training.
This applies training-time scale and color augmentation, crops/resizes,
and scales images to the [-1,1] range expected by pre-trained Inception nets.
Args:
images: A 4-D float32 `Tensor` holding raw images to be preprocessed.
height: Int, height in pixels to resize image to.
width: Int, width in pixels to resize image to.
min_scale: Float, minimum scale augmentation allowed, as a fraction of the
central min_side * min_side area of the original image.
max_scale: Float, maximum scale augmentation allowed, as a fraction of the
central min_side * min_side area of the original image.
p_scale_up: Float, fraction of images scaled up.
aug_color: Whether or not to do color augmentation.
fast_mode: Boolean, avoids slower ops (random_hue and random_contrast).
Returns:
preprocessed_images: A 4-D float32 `Tensor` holding preprocessed images.
"""
def _prepro_train(im):
"""Map this preprocessing function over each image in the batch."""
return preprocess_training_image(
im, height, width, min_scale, max_scale, p_scale_up,
aug_color=aug_color, fast_mode=fast_mode)
return tf.map_fn(_prepro_train, images)
def bboxes_sort_all_classes(classes, scores, bboxes, top_k=400, scope=None):
"""Sort bounding boxes by decreasing order and keep only the top_k.
Assume the input Tensors mix-up objects with different classes.
Assume a batch-type input.
Args:
classes: Batch x N Tensor containing integer classes.
scores: Batch x N Tensor containing float scores.
bboxes: Batch x N x 4 Tensor containing boxes coordinates.
top_k: Top_k boxes to keep.
Return:
classes, scores, bboxes: Sorted tensors of shape Batch x Top_k.
"""
with tf.name_scope(scope, 'bboxes_sort', [classes, scores, bboxes]):
scores, idxes = tf.nn.top_k(scores, k=top_k, sorted=True)
# Trick to be able to use tf.gather: map for each element in the batch.
def fn_gather(classes, bboxes, idxes):
cl = tf.gather(classes, idxes)
bb = tf.gather(bboxes, idxes)
return [cl, bb]
r = tf.map_fn(lambda x: fn_gather(x[0], x[1], x[2]),
[classes, bboxes, idxes],
dtype=[classes.dtype, bboxes.dtype],
parallel_iterations=10,
back_prop=False,
swap_memory=False,
infer_shape=True)
classes = r[0]
bboxes = r[1]
return classes, scores, bboxes
def input_fn():
starts = tf.random_uniform([batch_size], maxval=(2 * np.pi), seed=seed)
sin_curves = tf.map_fn(_sin_fn, (starts,), dtype=tf.float32)
inputs = tf.expand_dims(
tf.slice(sin_curves, [0, 0], [batch_size, sequence_length]), 2)
labels = tf.slice(sin_curves, [0, 1], [batch_size, sequence_length])
return {'inputs': inputs}, labels
def _decode_png_instance_masks(self, keys_to_tensors):
"""Decode PNG instance segmentation masks and stack into dense tensor.
The instance segmentation masks are reshaped to [num_instances, height,
width].
Args:
keys_to_tensors: a dictionary from keys to tensors.
Returns:
A 3-D float tensor of shape [num_instances, height, width] with values
in {0, 1}.
"""
def decode_png_mask(image_buffer):
image = tf.squeeze(
tf.image.decode_image(image_buffer, channels=1), axis=2)
image.set_shape([None, None])
image = tf.to_float(tf.greater(image, 0))
return image
png_masks = keys_to_tensors['image/object/mask']
height = keys_to_tensors['image/height']
width = keys_to_tensors['image/width']
if isinstance(png_masks, tf.SparseTensor):
png_masks = tf.sparse_tensor_to_dense(png_masks, default_value='')
return tf.cond(
tf.greater(tf.size(png_masks), 0),
lambda: tf.map_fn(decode_png_mask, png_masks, dtype=tf.float32),
lambda: tf.zeros(tf.to_int32(tf.stack([0, height, width]))))
def _evaluate_objective_multiple(objective_function, arg_batch,
batch_evaluate_objective):
"""Evaluates the objective function on a batch of points.
If `batch_evaluate_objective` is True, returns
`objective function(arg_batch)` else it maps the `objective_function`
across the `arg_batch`.
Args:
objective_function: A Python callable that accepts a single `Tensor` of
rank 'R > 1' and any shape 's' and returns a scalar `Tensor` of real dtype
containing the value of the function at that point. If
`batch a `Tensor` of shape `[batch_size] + s ` where `batch_size` is the
size of the batch of args. In this case, the expected return value is a
`Tensor` of shape `[batch_size]`.
arg_batch: A `Tensor` of real dtype. The batch of arguments at which to
evaluate the `objective_function`. If `batch_evaluate_objective` is False,
`arg_batch` will be unpacked along the zeroth axis and the
`objective_function` will be applied to each element.
batch_evaluate_objective: `bool`. Whether the `objective_function` can
evaluate a batch of arguments at once.
Returns:
A tuple containing:
objective_values: A `Tensor` of real dtype and shape `[batch_size]`.
The value of the objective function evaluated at the supplied
`arg_batch`.
num_evaluations: An `int32` scalar `Tensor`containing the number of
points on which the objective function was evaluated (i.e `batch_size`).
"""
n_points = tf.shape(arg_batch)[0]
if batch_evaluate_objective:
return objective_function(arg_batch), n_points
return tf.map_fn(objective_function, arg_batch), n_points
def normalized_to_image_coordinates(normalized_boxes, image_shape,
parallel_iterations=32):
"""Converts a batch of boxes from normal to image coordinates.
Args:
normalized_boxes: a float32 tensor of shape [None, num_boxes, 4] in
normalized coordinates.
image_shape: a float32 tensor of shape [4] containing the image shape.
parallel_iterations: parallelism for the map_fn op.
Returns:
absolute_boxes: a float32 tensor of shape [None, num_boxes, 4] containg the
boxes in image coordinates.
"""
def _to_absolute_coordinates(normalized_boxes):
return box_list_ops.to_absolute_coordinates(
box_list.BoxList(normalized_boxes),
image_shape[1], image_shape[2], check_range=False).get()
absolute_boxes = tf.map_fn(
_to_absolute_coordinates,
elems=(normalized_boxes),
dtype=tf.float32,
parallel_iterations=parallel_iterations,
back_prop=True)
return absolute_boxes
def standardize(images):
"""Linearly scales `image` to have zero mean and unit norm.
(A mirror to tf.image per_image_standardization)
This op computes `(x - mean) / adjusted_stddev`, where `mean` is the average
of all values in image, and
`adjusted_stddev = max(stddev, 1.0/sqrt(image.NumElements()))`.
`stddev` is the standard deviation of all values in `image`. It is capped
away from zero to protect against division by 0 when handling uniform images.
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
Returns:
The standardized image with same shape as `image`.
Raises:
ValueError: if the shape of 'image' is incompatible with this function.
"""
images_shape = get_shape(images)
if len(images_shape) > 4:
ValueError("'image' must have either 3 or 4 dimensions, "
"received `{}`.".format(images_shape))
if len(images_shape) == 4:
return tf.map_fn(lambda img: tf.image.per_image_standardization(img), images)
return tf.image.per_image_standardization(images)
def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
inputs = self._get_input_tensors(in_layers)
parent_tensor = inputs[0]
training = kwargs['training'] if 'training' in kwargs else 1.0
parent_tensor = parent_tensor / 255.0
if not self.augment:
out_tensor = parent_tensor
else:
def preprocess(img):
img = tf.image.random_flip_left_right(img)
img = tf.image.random_flip_up_down(img)
img = tf.image.rot90(img, k=np.random.randint(0, 4))
if self.distort_color:
img = tf.image.random_brightness(img, max_delta=32. / 255.)
img = tf.image.random_saturation(img, lower=0.5, upper=1.5)
img = tf.clip_by_value(img, 0.0, 1.0)
if self.central_crop:
# sample cut ratio from a clipped gaussian
img = tf.image.central_crop(img,
np.clip(
np.random.normal(1., 0.06), 0.8, 1.))
img = tf.image.resize_bilinear(
tf.expand_dims(img, 0), tf.convert_to_tensor(self.size))[0]
return img
outs = tf.map_fn(preprocess, parent_tensor)
# train/valid differences
out_tensor = training * outs + (1 - training) * parent_tensor
if set_tensors:
self.out_tensor = out_tensor
return out_tensor
def transpose(images):
"""Transpose an image/images by swapping the first and second dimension.
(A mirror to tf.image transpose_image)
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
Returns:
If `image` was 4-D, a 4-D float Tensor of shape
`[batch, target_height, target_width, channels]`
If `image` was 3-D, a 3-D float Tensor of shape
`[target_height, target_width, channels]
Raises:
ValueError: if the shape of `image` not supported.
"""
images_shape = get_shape(images)
if len(images_shape) > 4:
ValueError("'image' must have either 3 or 4 dimensions, "
"received `{}`.".format(images_shape))
if len(images_shape) == 4:
return tf.map_fn(lambda img: tf.image.transpose_image(img), images)
return tf.image.transpose_image(images)
def rotate90(images, k=1, is_random=False, seed=None, name=None):
"""Rotate (randomly) images counter-clockwise by 90 degrees.
(A mirror to tf.image rot90)
Args:
images: 4-D Tensor of shape `[batch, height, width, channels]` or
3-D Tensor of shape `[height, width, channels]`.
k: A scalar integer. The number of times the image is rotated by 90 degrees.
is_random: `bool`, If True, adjust randomly.
seed: A Python integer. Used to create a random seed. See @{tf.set_random_seed}.
name: A name for this operation (optional).
Returns:
If `image` was 4-D, a 4-D float Tensor of shape
`[batch, target_height, target_width, channels]`
If `image` was 3-D, a 3-D float Tensor of shape
`[target_height, target_width, channels]
Raises:
ValueError: if the shape of `image` not supported.
"""
if is_random:
k = random_ops.random_shuffle([0, 1, 2, 3], seed=seed)[0]
images_shape = get_shape(images)
if len(images_shape) > 4:
ValueError("'image' must have either 3 or 4 dimensions, "
"received `{}`.".format(images_shape))
if len(images_shape) == 4:
return tf.map_fn(lambda img: tf.image.rot90(img, k, name), images)
return tf.image.rot90(images, k, name)
def get_exemplar_images(images, exemplar_size, targets_pos=None):
"""Crop exemplar image from input images"""
with tf.name_scope('get_exemplar_image'):
batch_size, x_height, x_width = images.get_shape().as_list()[:3]
z_height, z_width = exemplar_size
if targets_pos is None:
target_pos_single = [[get_center(x_height), get_center(x_width)]]
targets_pos_ = tf.tile(target_pos_single, [batch_size, 1])
else:
targets_pos_ = targets_pos
# convert to top-left corner based coordinates
top = tf.to_int32(tf.round(targets_pos_[:, 0] - get_center(z_height)))
bottom = tf.to_int32(top + z_height)
left = tf.to_int32(tf.round(targets_pos_[:, 1] - get_center(z_width)))
right = tf.to_int32(left + z_width)
def _slice(x):
f, t, l, b, r = x
c = f[t:b, l:r]
return c
exemplar_img = tf.map_fn(_slice, (images, top, left, bottom, right), dtype=images.dtype)
exemplar_img.set_shape([batch_size, z_height, z_width, 3])
return exemplar_img
def build_inputs_and_outputs(self):
if self.frame_features:
serialized_examples = tf.placeholder(tf.string, shape=(None,))
fn = lambda x: self.build_prediction_graph(x)
video_id_output, top_indices_output, top_predictions_output = (
tf.map_fn(fn, serialized_examples,
dtype=(tf.string, tf.int32, tf.float32)))
else:
serialized_examples = tf.placeholder(tf.string, shape=(None,))
video_id_output, top_indices_output, top_predictions_output = (
self.build_prediction_graph(serialized_examples))
inputs = {"example_bytes":
saved_model_utils.build_tensor_info(serialized_examples)}
outputs = {
"video_id": saved_model_utils.build_tensor_info(video_id_output),
"class_indexes": saved_model_utils.build_tensor_info(top_indices_output),
"predictions": saved_model_utils.build_tensor_info(top_predictions_output)}
return inputs, outputs
def attentive_matching(input_sentence, att_matrix, weights):
"""
Parameters
----------
input_sentence: Tensor
Tensor of shape (batch_size, num_sentence_words, rnn_hidden_dim)
att_matrix: Tensor
Tensor of shape (batch_size, num_sentence_words, rnn_hidden_dim)
"""
def single_instance(inputs):
# Shapes: (num_sentence_words, rnn_hidden_dim)
sentence_a_single = inputs[0]
sentence_b_single_att = inputs[1]
# Shapes: (num_sentence_words, multiperspective_dims, rnn_hidden_dim)
expanded_sentence_a_single = multi_perspective_expand_for_2D(
sentence_a_single, weights)
expanded_sentence_b_single_att = multi_perspective_expand_for_2D(
sentence_b_single_att, weights)
# Shape: (num_sentence_words, multiperspective_dims)
return cosine_distance(expanded_sentence_a_single,
expanded_sentence_b_single_att)
elems = (input_sentence, att_matrix)
# Shape: (batch_size, num_sentence_words, multiperspective_dims)
return tf.map_fn(single_instance, elems, dtype="float")
def lowercase(self, raw_post):
split_chars = tf.string_split(tf.reshape(raw_post, [-1]), delimiter="").values
upchar_inds = self.upchars_lut.lookup(split_chars)
return tf.reduce_join(tf.map_fn(lambda x: tf.cond(x[0] > 25,
lambda: x[1],
lambda: self.lchars[x[0]]),
(upchar_inds, split_chars), dtype=tf.string))
def input_images(self):
num_images = (self._sqlen + self._args.lookback_length) * self._bsize
images = tf.map_fn(lambda x: tf.image.decode_jpeg(tf.read_file(x)),
tf.reshape(self._imfiles, shape=[num_images]),
dtype=tf.uint8)
images.set_shape([None, ds.HEIGHT, ds.WIDTH, ds.CHANNELS])
return images
def batch_logits(indices, acts):
init_outs = tf.zeros([1, FLAGS.wvs, 1])
def logits_continue(*parms):
cur, idxs, _, _, _ = parms
return tf.less(cur, tf.size(idxs), name='batch_done')
def logits_batch_body(*parms):
i, idxs, ptr, css, act = parms
i_s = tf.reshape(tf.slice(idxs, tf.pack([i]), [1]), [])
start, size = get_bounds(i_s)
outs = forward_prop_nodes(start, size, acts, ptr)
new_css = tf.cond(tf.equal(i, iZERO),
lambda: outs,
lambda: tf.concat(0, [css, outs]))
return i + iONE, indices, ptr + size, new_css, acts
with tf.device('/cpu:0'):
iZ = tf.convert_to_tensor(0, dtype=tf.int32)
zero_activations(acts)
while_parms = [iZ, indices, iZ, init_outs, acts]
_, _, _, outs, _ = tf.while_loop(logits_continue, logits_batch_body, while_parms,
parallel_iterations=1, name='batch_logits')
lumpy_logits = tf.map_fn(activation_to_logits, outs, name='raw_logits')
logits = tf.squeeze(lumpy_logits, [2], name='logits')
return logits
def _sample_n(self, n, seed=None):
n_draws = tf.cast(self.total_count, dtype=tf.int32)
k = self.event_shape_tensor()[0]
# broadcast the total_count and logits to same shape
n_draws = tf.ones_like(
self.logits[..., 0], dtype=n_draws.dtype) * n_draws
logits = tf.ones_like(
n_draws[..., tf.newaxis], dtype=self.logits.dtype) * self.logits
# flatten the total_count and logits
flat_logits = tf.reshape(logits, [-1, k]) # [B1B2...Bm, k]
flat_ndraws = n * tf.reshape(n_draws, [-1]) # [B1B2...Bm]
# computes each total_count and logits situation by map_fn
def _sample_single(args):
logits, n_draw = args[0], args[1] # [K], []
x = tf.multinomial(logits[tf.newaxis, ...], n_draw,
seed) # [1, n*n_draw]
x = tf.reshape(x, shape=[n, -1]) # [n, n_draw]
x = tf.reduce_sum(tf.one_hot(x, depth=k), axis=-2) # [n, k]
return x
x = tf.map_fn(
_sample_single, [flat_logits, flat_ndraws],
dtype=self.dtype) # [B1B2...Bm, n, k]
# reshape the results to proper shape
x = tf.transpose(x, perm=[1, 0, 2])
final_shape = tf.concat([[n], self.batch_shape_tensor(), [k]], 0)
x = tf.reshape(x, final_shape) # [n, B1, B2,..., Bm, k]
return x
def preprocess_for_inception(images):
"""Preprocess images for inception.
Args:
images: images minibatch. Shape [batch size, width, height,
channels]. Values are in [0..255].
Returns:
preprocessed_images
"""
# Images should have 3 channels.
assert images.shape[3].value == 3
# tfgan_eval.preprocess_image function takes values in [0, 1], so rescale.
with tf.control_dependencies([tf.assert_greater_equal(images, 0.0),
tf.assert_less_equal(images, 255.0)]):
images = tf.identity(images)
preprocessed_images = tf.map_fn(
fn=tfgan_eval.preprocess_image,
elems=images,
back_prop=False
)
return preprocessed_images
def testMap_MultiInputSingleOutput(self):
with self.test_session():
nums = np.array([1, 2, 3, 4, 5, 6])
r = tf.map_fn(lambda x: x[0] * x[1][0] + x[1][1], (nums, (nums, -nums)), dtype=tf.int64)
self.assertEqual((6,), r.get_shape())
received = r.eval()
self.assertAllEqual(nums * nums + (-nums), received)
def _get_bbox_pred(self, proposed_boxes, gt_boxes_per_class):
"""Computes valid bbox_pred from proposals and gt_boxes for each class.
Args:
proposed_boxes: Tensor with shape (num_proposals, 5).
gt_boxes_per_class: Tensor holding the ground truth boxes for each
class. Has shape (num_classes, num_gt_boxes_per_class, 4).
Returns:
A tensor with shape (num_proposals, num_classes * 4), holding the
correct bbox_preds.
"""
def bbox_encode(gt_boxes):
return encode(
proposed_boxes, gt_boxes
)
bbox_pred_tensor = tf.map_fn(
bbox_encode, gt_boxes_per_class,
dtype=tf.float32
)
# We need to explicitly unstack the tensor so that tf.concat works
# properly.
bbox_pred_list = tf.unstack(bbox_pred_tensor)
return tf.concat(bbox_pred_list, 1)
def get_labels_from_annotation_batch(annotation_batch_tensor, class_labels):
batch_labels = tf.map_fn(fn=lambda x: get_labels_from_annotation(annotation_tensor=x, class_labels=class_labels),
elems=annotation_batch_tensor,
dtype=tf.float32)
return batch_labels
def alignment_summary(features, labels, predictions, mode, params):
image = tf.cast(features, tf.uint8)
prediction = predictions['prediction']
label_points = tf.stack([labels[:, ::2], labels[:, 1::2]], axis=2)
predict_points = tf.stack(
[prediction[:, ::2], prediction[:, 1::2]], axis=2)
def draw_points(args):
image, label_points, predict_points = args
def draw_points_pyfn(image, points, color, radius=1):
if image.shape[-1] == 1:
image = np.squeeze(image, axis=-1)
image_pil = Image.fromarray(np.uint8(image)).convert('RGB')
draw = ImageDraw.Draw(image_pil)
im_width, im_height = image_pil.size
for point in points:
x = point[0] * im_width
y = point[1] * im_height
draw.ellipse([(x - radius, y - radius),
(x + radius, y + radius)],
outline=color, fill=color)
image = np.array(image_pil)
return image
image = tf.py_func(
functools.partial(draw_points_pyfn, color=(0, 255, 0)),
(image, label_points), tf.uint8)
image = tf.py_func(
functools.partial(draw_points_pyfn, color=(255, 0, 0)),
(image, predict_points), tf.uint8)
return image
image = tf.map_fn(
draw_points, (image, label_points, predict_points),
dtype=tf.uint8, back_prop=False)
tf.summary.image('image', image, 10)
add_trainable_variables_histogram()
def main():
if a.seed is None:
a.seed = random.randint(0, 2**31 - 1)
tf.set_random_seed(a.seed)
np.random.seed(a.seed)
random.seed(a.seed)
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
if a.mode == "test" or a.mode == "export":
if a.checkpoint is None:
raise Exception("checkpoint required for test mode")
# load some options from the checkpoint
options = {"which_direction", "ngf", "ndf", "lab_colorization"}
with open(os.path.join(a.checkpoint, "options.json")) as f:
for key, val in json.loads(f.read()).items():
if key in options:
print("loaded", key, "=", val)
setattr(a, key, val)
# disable these features in test mode
a.scale_size = CROP_SIZE
a.flip = False
for k, v in a._get_kwargs():
print(k, "=", v)
with open(os.path.join(a.output_dir, "options.json"), "w") as f:
f.write(json.dumps(vars(a), sort_keys=True, indent=4))
examples = load_examples()
print("examples count = %d" % examples.count)
# inputs and targets are [batch_size, height, width, channels]
model = create_model(examples.inputs, examples.targets)
# undo colorization splitting on images that we use for display/output
if a.lab_colorization:
if a.which_direction == "AtoB":
# inputs is brightness, this will be handled fine as a grayscale image
# need to augment targets and outputs with brightness
targets = augment(examples.targets, examples.inputs)
outputs = augment(model.outputs, examples.inputs)
# inputs can be deprocessed normally and handled as if they are single channel
# grayscale images
inputs = deprocess(examples.inputs)
elif a.which_direction == "BtoA":
# inputs will be color channels only, get brightness from targets
inputs = augment(examples.inputs, examples.targets)
targets = deprocess(examples.targets)
outputs = deprocess(model.outputs)
else:
raise Exception("invalid direction")
else:
inputs = deprocess(examples.inputs)
targets = deprocess(examples.targets)
outputs = deprocess(model.outputs)
def convert(image):
if a.aspect_ratio != 1.0:
# upscale to correct aspect ratio
size = [CROP_SIZE, int(round(CROP_SIZE * a.aspect_ratio))]
image = tf.image.resize_images(
image, size=size, method=tf.image.ResizeMethod.BICUBIC)
return tf.image.convert_image_dtype(image,
dtype=tf.uint8,
saturate=True)
# reverse any processing on images so they can be written to disk or displayed to user
with tf.name_scope("convert_inputs"):
converted_inputs = convert(inputs)
with tf.name_scope("convert_targets"):
converted_targets = convert(targets)
with tf.name_scope("convert_outputs"):
converted_outputs = convert(outputs)
with tf.name_scope("encode_images"):
display_fetches = {
"paths":
examples.paths,
"inputs":
tf.map_fn(tf.image.encode_png,
converted_inputs,
dtype=tf.string,
name="input_pngs"),
"targets":
tf.map_fn(tf.image.encode_png,
converted_targets,
dtype=tf.string,
name="target_pngs"),
"outputs":
tf.map_fn(tf.image.encode_png,
converted_outputs,
dtype=tf.string,
name="output_pngs"),
}
# summaries
with tf.name_scope("inputs_summary"):
tf.summary.image("inputs", converted_inputs)
with tf.name_scope("targets_summary"):
tf.summary.image("targets", converted_targets)
with tf.name_scope("outputs_summary"):
tf.summary.image("outputs", converted_outputs)
with tf.name_scope("predict_real_summary"):
tf.summary.image(
"predict_real",
tf.image.convert_image_dtype(model.predict_real, dtype=tf.uint8))
with tf.name_scope("predict_fake_summary"):
tf.summary.image(
"predict_fake",
tf.image.convert_image_dtype(model.predict_fake, dtype=tf.uint8))
tf.summary.scalar("discriminator_loss", model.discrim_loss)
tf.summary.scalar("generator_loss_GAN", model.gen_loss_GAN)
tf.summary.scalar("generator_loss_L1", model.gen_loss_L1)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/values", var)
for grad, var in model.discrim_grads_and_vars + model.gen_grads_and_vars:
tf.summary.histogram(var.op.name + "/gradients", grad)
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum(
[tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=1)
logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None
sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None)
with sv.managed_session() as sess:
print("parameter_count =", sess.run(parameter_count))
if a.checkpoint is not None:
print("#############################")
print("loading model from checkpoint")
print("#############################")
try:
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
saver.restore(sess, checkpoint)
except:
print("loading was unsuccessful")
print("#############################")
max_steps = 2**32
if a.max_epochs is not None:
max_steps = examples.steps_per_epoch * a.max_epochs
if a.max_steps is not None:
max_steps = a.max_steps
if a.mode == "test":
# testing
# at most, process the test data once
start = time.time()
max_steps = min(examples.steps_per_epoch, max_steps)
for step in range(max_steps):
results = sess.run(display_fetches)
filesets = save_images(results)
for i, f in enumerate(filesets):
print("evaluated image", f["name"])
index_path = append_index(filesets)
print("wrote index at", index_path)
print("rate", (time.time() - start) / max_steps)
else:
# training
start = time.time()
for step in range(max_steps):
def should(freq):
return freq > 0 and ((step + 1) % freq == 0
or step == max_steps - 1)
options = None
run_metadata = None
if should(a.trace_freq):
options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
fetches = {
"train": model.train,
"global_step": sv.global_step,
}
if should(a.progress_freq):
fetches["discrim_loss"] = model.discrim_loss
fetches["gen_loss_GAN"] = model.gen_loss_GAN
fetches["gen_loss_L1"] = model.gen_loss_L1
if should(a.summary_freq):
fetches["summary"] = sv.summary_op
if should(a.display_freq):
fetches["display"] = display_fetches
results = sess.run(fetches,
options=options,
run_metadata=run_metadata)
if should(a.summary_freq):
print("recording summary")
sv.summary_writer.add_summary(results["summary"],
results["global_step"])
if should(a.display_freq):
print("saving display images")
filesets = save_images(results["display"],
step=results["global_step"])
append_index(filesets, step=True)
if should(a.trace_freq):
print("recording trace")
sv.summary_writer.add_run_metadata(
run_metadata, "step_%d" % results["global_step"])
if should(a.progress_freq):
# global_step will have the correct step count if we resume from a checkpoint
train_epoch = math.ceil(results["global_step"] /
examples.steps_per_epoch)
train_step = (results["global_step"] -
1) % examples.steps_per_epoch + 1
rate = (step + 1) * a.batch_size / (time.time() - start)
remaining = (max_steps - step) * a.batch_size / rate
print(
"progress epoch %d step %d image/sec %0.1f remaining %dm"
% (train_epoch, train_step, rate, remaining / 60))
print("discrim_loss", results["discrim_loss"])
print("gen_loss_GAN", results["gen_loss_GAN"])
print("gen_loss_L1", results["gen_loss_L1"])
if results["gen_loss_L1"] < float(a.desired_l1_loss):
print("###################")
print("Reached desired error")
print(
"progress epoch %d step %d image/sec %0.1f remaining %dm"
% (train_epoch, train_step, rate, remaining / 60))
print("saving model")
saver.save(sess,
os.path.join(a.output_dir, "model"),
global_step=sv.global_step)
break
if should(a.save_freq):
print("saving model")
saver.save(sess,
os.path.join(a.output_dir, "model"),
global_step=sv.global_step)
if sv.should_stop():
break
def esmm_model_fn(features, labels, mode, params):
batch_weight = tf.feature_column.input_layer(features, params['weight_columns'])
inputs, shared_weights = build_input(features, params)
hidden_units = params['hidden_units']
linear_parent_scope = 'linear'
dnn_parent_scope = 'dnn'
is_dynamic = params['dynamic']
print("is_dynamic:", is_dynamic)
reg = 1e-4
if params['model'] == 'linear':
with tf.variable_scope(linear_parent_scope, values=tuple(six.itervalues(features)), reuse=tf.AUTO_REUSE):
with tf.variable_scope('linear_ctr'):
ctr_logit_fn = linear._linear_logit_fn_builder(1, params['linear_columns'])
ctr_logits = ctr_logit_fn(features=features)
with tf.variable_scope('linear_cvr'):
cvr_logit_fn = linear._linear_logit_fn_builder(1, params['linear_columns'])
cvr_logits = cvr_logit_fn(features=features)
if params['model'] == 'dnn':
with tf.variable_scope(dnn_parent_scope):
with tf.variable_scope('dnn_ctr'):
ctr_logits = build_deep_layers(inputs, hidden_units, mode, params['ctr_reg'])
#ctr_logit_fn = dnn._dnn_logit_fn_builder(1, hidden_units, params['dnn_columns'], tf.nn.relu, None, None, True)
#ctr_logits = ctr_logit_fn(features=features, mode=mode)
with tf.variable_scope('dnn_cvr'):
cvr_logits = build_deep_layers(inputs, hidden_units, mode, params['cvr_reg'])
#cvr_logit_fn = dnn._dnn_logit_fn_builder(1, hidden_units, params['dnn_columns'], tf.nn.relu, None, None, True)
#cvr_logits = cvr_logit_fn(features=features, mode=mode)
logits = {'ctr': ctr_logits, 'ctcvr': ctr_logits*cvr_logits}
ctr_preds = tf.nn.sigmoid(ctr_logits)
cvr_preds = tf.nn.sigmoid(cvr_logits)
#ctcvr_preds = tf.stop_gradient(ctr_preds) * cvr_preds
ctcvr_preds = ctr_preds * cvr_preds
tf.summary.histogram("esmm/ctr_preds", ctr_preds)
tf.summary.histogram("esmm/ctcvr_preds", ctcvr_preds)
if mode == tf.estimator.ModeKeys.PREDICT:
#redundant_items = ctr_preds
predictions = {
'prob': tf.concat([ctcvr_preds, ctr_preds], 1)
}
export_outputs = {
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: tf.estimator.export.PredictOutput(predictions) #线上预测需要的
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions, export_outputs=export_outputs)
else:
#shared_weights = tf.trainable_variables(dnn_parent_scope + '/SharedLayer/kernel')[0]
ctr_labels = labels['ctr']
ctcvr_labels = labels['ctcvr']
linear_optimizer = tf.train.FtrlOptimizer(0.01, l1_regularization_strength=0.001, l2_regularization_strength=0.001)
dnn_optimizer = optimizers.get_optimizer_instance('Adam', params['learning_rate'])
loss_optimizer = optimizers.get_optimizer_instance('Adam', 0.001)
ctr_loss = tf.losses.log_loss(ctr_labels, ctr_preds, reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE, weights=batch_weight)
ctcvr_loss = tf.losses.log_loss(ctcvr_labels, ctcvr_preds, reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE)
#reg_loss = tf.reduce_sum(ops.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
ctr_auc = tf.metrics.auc(labels=ctr_labels, predictions=ctr_preds, weights=batch_weight)
ctcvr_auc = tf.metrics.auc(labels=ctcvr_labels, predictions=ctcvr_preds)
mask = tf.map_fn(lambda x:tf.cond(tf.equal(x, 1), lambda: True, lambda: False), tf.squeeze(labels['ctr']), dtype=tf.bool)
cvr_preds = tf.boolean_mask(cvr_preds, mask)
cvr_labels = tf.boolean_mask(labels['ctcvr'], mask)
cvr_auc = tf.metrics.auc(labels=cvr_labels, predictions=cvr_preds)
cvr_loss = tf.losses.log_loss(cvr_labels, cvr_preds, reduction=tf.losses.Reduction.SUM_OVER_NONZERO_WEIGHTS)
tf.summary.scalar("cvr_auc", cvr_auc[1])
tf.summary.scalar("cvr_loss", cvr_loss)
tf.summary.scalar('ctr_loss', ctr_loss)
tf.summary.scalar('ctcvr_loss', ctcvr_loss)
tf.summary.scalar('ctr_auc', ctr_auc[1])
tf.summary.scalar('ctcvr_auc', ctcvr_auc[1])
loss = tf.add_n([ctr_loss, ctcvr_loss])
#weight_loss, update_list, w_list, loss_gradnorm = get_weight_loss([ctr_loss, ctcvr_loss], is_dynamic, shared_weights)
#print("get_weight_loss:", weight_loss, update_list)
def _train_op_fn(loss):
train_ops = []
global_step = tf.train.get_global_step()
if params['model'] in ('dnn'):
fm_var_list = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES, scope='fm')
dnn_var_list = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES, scope=dnn_parent_scope) + ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES, scope='dnn_embed')
train_ops.append(
dnn_optimizer.minimize(
loss,
var_list=dnn_var_list))
train_ops.append(
linear_optimizer.minimize(
loss,
var_list=fm_var_list))
if params['model'] in ('linear'):
train_ops.append(
linear_optimizer.minimize(
loss,
var_list=tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES,
scope=linear_parent_scope)))
'''
if w_list is not None and update_list is not None and loss_gradnorm is not None:
train_ops.append(
loss_optimizer.minimize(
loss_gradnorm,
var_list=w_list))
train_ops.append(update_list)
'''
train_op = control_flow_ops.group(*train_ops)
with ops.control_dependencies([train_op]):
return state_ops.assign_add(global_step, 1).op
hooks = tf.train.LoggingTensorHook({'ctr_loss':ctr_loss, 'ctcvr_loss':ctcvr_loss, 'cvr_loss':cvr_loss}, every_n_iter=10000)
train_op = _train_op_fn(loss)
train_op = head_v1._append_update_ops(train_op)
metrics = {'ctr_auc': ctr_auc, 'ctcvr_auc': ctcvr_auc, 'cvr_auc': cvr_auc}
#return _TPUEstimatorSpec(mode, loss=loss, train_op=train_op).as_estimator_spec()
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op, eval_metric_ops=metrics)
def build_graph(self, data_paths, batch_size, graph_mod):
"""Builds generic graph for training or eval."""
tensors = GraphReferences()
is_training = graph_mod == GraphMod.TRAIN
if data_paths:
tensors.keys, tensors.examples = util.read_examples(
data_paths,
batch_size,
shuffle=is_training,
num_epochs=None if is_training else 2)
else:
logging.info("No data path")
tensors.examples = tf.placeholder(tf.string,
name='input',
shape=(None, ))
if graph_mod == GraphMod.PREDICT:
pass
else:
# For training and evaluation we assume data is preprocessed, so the
# inputs are tf-examples.
# Generate placeholders for examples.
with tf.name_scope('inputs'):
feature_map = {
'image_uri':
tf.FixedLenFeature(shape=[],
dtype=tf.string,
default_value=['']),
'image_bytes':
tf.FixedLenFeature(shape=[],
dtype=tf.string,
default_value=['']),
'label':
tf.FixedLenFeature(shape=[1],
dtype=tf.int64,
default_value=[self.label_count])
}
#tensors.examples = tf.Print(tensors.examples, [tf.shape(tensors.examples)], message="Parsing examples: ")
parsed = tf.parse_example(tensors.examples,
features=feature_map)
labels = tf.squeeze(parsed['label'])
uris = tf.squeeze(parsed['image_uri'])
images_str_tensor = parsed['image_bytes']
def decode_and_resize(image_str_tensor):
"""Decodes jpeg string, resizes it and returns a uint8 tensor."""
image = tf.image.decode_jpeg(image_str_tensor, channels=1)
# Note resize expects a batch_size, but tf_map supresses that index,
# thus we have to expand then squeeze. Resize returns float32 in the
# range [0, uint8_max]
"""
image = tf.expand_dims(image, 0)
image = tf.image.resize_bilinear(
image, [height, width], align_corners=False)*/
image = tf.squeeze(image, squeeze_dims=[0])
"""
image = tf.cast(image, dtype=tf.uint8)
# convert_image_dtype, also scales [0, uint8_max] -> [0 ,1).
return tf.image.convert_image_dtype(image, dtype=tf.float32)
#images_str_tensor = tf.Print(images_str_tensor, [tf.shape(images_str_tensor)], message="Decoding images: ")
images = tf.map_fn(decode_and_resize,
images_str_tensor,
back_prop=False,
dtype=tf.float32)
# We assume a default label, so the total number of labels is equal to
# label_count+1.
all_labels_count = self.label_count + 1
with tf.name_scope('model'):
fc_padding = 'VALID'
with tf.variable_scope('model', 'vgg_16', [images]) as sc:
end_points_collection = sc.original_name_scope + '_end_points'
# Collect outputs for conv2d, fully_connected and max_pool2d.
with slim.arg_scope(
[slim.conv2d, slim.fully_connected, slim.max_pool2d],
outputs_collections=end_points_collection):
# images = tf.Print(images, [tf.shape(images)], message="Shape of input: ", summarize=4)
net = slim.repeat(images,
2,
slim.conv2d,
64, [3, 3],
scope='conv1')
#net = tf.Print(net, [tf.shape(net)], summarize=4, message='conv1')
net = slim.max_pool2d(net, [2, 2], scope='pool1')
net = slim.repeat(net,
2,
slim.conv2d,
128, [3, 3],
scope='conv2')
#net = tf.Print(net, [tf.shape(net)], summarize=4, message='conv2')
net = slim.max_pool2d(net, [2, 2], scope='pool2')
net = slim.repeat(net,
3,
slim.conv2d,
256, [3, 3],
scope='conv3')
#net = tf.Print(net, [tf.shape(net)], summarize=4, message='conv3')
net = slim.max_pool2d(net, [2, 2], scope='pool3')
net = slim.repeat(net,
3,
slim.conv2d,
512, [3, 3],
scope='conv4')
#net = tf.Print(net, [tf.shape(net)], summarize=4, message='conv4')
net = slim.max_pool2d(net, [2, 2], scope='pool4')
net = slim.repeat(net,
3,
slim.conv2d,
512, [3, 3],
scope='conv5')
#net = tf.Print(net, [tf.shape(net)], summarize=4, message='conv5')
net = slim.max_pool2d(net, [2, 2], scope='pool5')
# Use conv2d instead of fully_connected layers.
net = slim.conv2d(net,
4096, [7, 7],
padding=fc_padding,
scope='fc6')
#net = tf.Print(net, [tf.shape(net)], summarize=4, message='fc6')
net = slim.dropout(net,
0.5,
is_training=True,
scope='dropout6')
net = slim.conv2d(net, 4096, [1, 1], scope='fc7')
#net = tf.Print(net, [tf.shape(net)], summarize=4, message='fc7')
# Convert end_points_collection into a end_point dict.
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
net = slim.dropout(net,
0.5,
is_training=True,
scope='dropout7')
net = slim.conv2d(net,
all_labels_count, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='fc8')
#net = tf.Print(net, [tf.shape(net)], summarize=4, message='fc8')
net = tf.squeeze(net, [1, 2], name='fc8/squeezed')
end_points[sc.name + '/fc8'] = net
logits = net
softmax = tf.nn.softmax(logits)
#softmax = tf.Print(softmax, [tf.shape(softmax)], summarize=4, message='softmax')
# Prediction is the index of the label with the highest score. We are
# interested only in the top score.
prediction = tf.argmax(softmax, 1)
tensors.predictions = [prediction, softmax, images]
if graph_mod == GraphMod.PREDICT:
return tensors
with tf.name_scope('evaluate'):
loss_value = loss(logits, labels)
# Add to the Graph the Ops that calculate and apply gradients.
if is_training:
tensors.train, tensors.global_step = training(loss_value)
else:
tensors.global_step = tf.Variable(0,
name='global_step',
trainable=False)
# Add means across all batches.
loss_updates, loss_op = util.loss(loss_value)
accuracy_updates, accuracy_op = util.accuracy(logits, labels)
if not is_training:
#accuracy_op = tf.Print(accuracy_op, [accuracy_op], message="Accuracy")
#loss_op = tf.Print(loss_op, [loss_op], message="Loss")
tf.summary.scalar('accuracy', accuracy_op)
tf.summary.scalar('loss', loss_op)
tensors.metric_updates = loss_updates + accuracy_updates
tensors.metric_values = [loss_op, accuracy_op]
return tensors
def augment_image(image):
image = tf.image.resize_images(image, [self._augment_size, self._augment_size])
image = tf.random_crop(image, [self._batch_size] + self._image_shape, seed=seed)
image = tf.map_fn(lambda x: tf.image.random_flip_left_right(x, seed), image)
return image
def build_template(self):
model_config = self.model_config
track_config = self.track_config
size_z = model_config['z_image_size']
ratio = self.target_size[0] / self.target_size[1]
# Exemplar image lies at the center of the search image in the first frame
center_scale = int(get_center(track_config['num_scales']))
search_images = tf.expand_dims(self.search_images[center_scale], 0)
exemplar_images = get_exemplar_images(
search_images, [size_z, size_z],
np.array([[
get_center(track_config['x_image_size']),
get_center(track_config['x_image_size'])
]]))
def boundary_suppression(embeds, embeds2, ratio):
offsets = tf.cond(
tf.greater(ratio, 1.5), lambda: [0, 4, 0, 4],
lambda: tf.cond(tf.less(ratio, 0.67), lambda: [4, 0, 4, 0],
lambda: [2, 2, 2, 2]))
embeds = tf.image.resize_image_with_crop_or_pad(
embeds, t_shape[1] - offsets[0], t_shape[2] - offsets[1])
embeds = tf.image.resize_image_with_crop_or_pad(
embeds, t_shape[1], t_shape[2])
embeds2 = tf.image.resize_image_with_crop_or_pad(
embeds2, t_shape2[1] - offsets[2], t_shape2[2] - offsets[3])
embeds2 = tf.image.resize_image_with_crop_or_pad(
embeds2, t_shape2[1], t_shape2[2])
return embeds, embeds2
def background_suppression(embeds, ratio):
offsets = tf.cond(
tf.greater(ratio, 1.5), # 1.2 / 0.83; 1.5 / 0.67
lambda: [1., 1.2 / ratio],
lambda: tf.cond(
tf.less(ratio, 0.67), lambda: [1.2 * ratio, 1.], lambda: tf
.cond(
tf.greater(ratio, 1.2), lambda: [1., 1.1 / ratio],
lambda: tf.cond(tf.less(ratio, 0.83), lambda: [
1.1 * ratio, 1.
], lambda: [0.7, 0.7]))))
h = tf.cast(size_z * offsets[0], tf.int32)
w = tf.cast(size_z * offsets[1], tf.int32)
embeds_mean = tf.reduce_mean(embeds, axis=(0, 1), keepdims=True)
embeds = embeds - embeds_mean
embeds = tf.image.resize_image_with_crop_or_pad(embeds, h, w)
embeds = tf.image.resize_image_with_crop_or_pad(
embeds, size_z, size_z)
return embeds + embeds_mean
exemplar_images = tf.map_fn(
lambda x: background_suppression(x[0], x[1]),
(exemplar_images, tf.expand_dims(ratio, 0)),
dtype=exemplar_images.dtype)
self.exemplar_images = exemplar_images
templates, templates2 = self.get_image_embedding(exemplar_images)
t_shape = templates.get_shape().as_list()
t_shape2 = templates2.get_shape().as_list()
templates, templates2 = tf.map_fn(
lambda x: boundary_suppression(x[0], x[1], x[2]),
(templates, templates2, tf.expand_dims(ratio, 0)),
dtype=(templates.dtype, templates2.dtype))
templates = templates
templates2 = templates2
with tf.variable_scope('target_template'):
# Store template in Variable such that we don't have to feed this template every time.
with tf.variable_scope('State'):
state = tf.get_variable('exemplar',
initializer=tf.zeros(
templates.get_shape().as_list(),
dtype=templates.dtype),
trainable=False)
state2 = tf.get_variable('exemplar2',
initializer=tf.zeros(
templates2.get_shape().as_list(),
dtype=templates2.dtype),
trainable=False)
with tf.control_dependencies([templates, templates2]):
self.init = tf.assign(state,
templates,
validate_shape=True)
self.init2 = tf.assign(state2,
templates2,
validate_shape=True)
self.templates = state
self.templates2 = state2
# Store Pseudo Templates
def _euc_distance(x, z):
z = tf.expand_dims(z, 0)
return tf.reduce_sum(tf.abs(x - z), -1)
num_k = 3 # 3
state_pseu = []
state_pseu2 = []
image_pseu = []
self.init_pseu = []
self.init2_pseu = []
self.init_pseu_img = []
for i in range(num_k):
state_pseu.append(
tf.get_variable('exemplar_pseu' + str(i),
initializer=tf.zeros(
templates.get_shape().as_list(),
dtype=templates.dtype),
trainable=False))
state_pseu2.append(
tf.get_variable('exemplar2_pseu' + str(i),
initializer=tf.zeros(
templates2.get_shape().as_list(),
dtype=templates2.dtype),
trainable=False))
image_pseu.append(
tf.get_variable(
'exemplar_pseu_image' + str(i),
initializer=tf.zeros(
exemplar_images.get_shape().as_list(),
dtype=exemplar_images.dtype),
trainable=False))
with tf.control_dependencies(
[templates, templates2, exemplar_images]):
self.init_pseu.append(
tf.assign(state_pseu[i],
templates,
validate_shape=True))
self.init2_pseu.append(
tf.assign(state_pseu2[i],
templates2,
validate_shape=True))
self.init_pseu_img.append(
tf.assign(image_pseu[i],
exemplar_images,
validate_shape=True))
self.image_pseu = image_pseu
self.pseu_temp = state_pseu
self.pseu_temp2 = state_pseu2
state_pseus = tf.concat([self.templates] + state_pseu +
[templates], 0)
sp_shape = state_pseus.get_shape().as_list()[0]
state_pseus_c = tf.reshape(state_pseus, [sp_shape, -1])
state_pseus_dis = tf.map_fn(
lambda x: _euc_distance(state_pseus_c, x),
state_pseus_c,
dtype=state_pseus_c.dtype)
state_pseus_dis = tf.reshape(state_pseus_dis,
[sp_shape, sp_shape])[1:, :]
state_pseus_dis = tf.reduce_sum(state_pseus_dis, -1)
self.state_pseus_dis = state_pseus_dis
_, state_pseus_idx = tf.nn.top_k(state_pseus_dis,
k=len(state_pseu))
image_pseu_extra = tf.concat(image_pseu + [exemplar_images], 0)
state_pseus2 = tf.concat(state_pseu2 + [templates2], 0)
self.up_img = []
self.up_pseu = []
self.up2_pseu = []
for i in range(len(state_pseu)):
with tf.control_dependencies([
state_pseus_idx, image_pseu_extra, state_pseus,
state_pseus2
]):
self.up_pseu.append(
tf.assign(state_pseu[i],
tf.expand_dims(
state_pseus[state_pseus_idx[i] + 1],
0),
validate_shape=True))
self.up2_pseu.append(
tf.assign(state_pseu2[i],
tf.expand_dims(
state_pseus2[state_pseus_idx[i]], 0),
validate_shape=True))
self.up_img.append(
tf.assign(image_pseu[i],
tf.expand_dims(
image_pseu_extra[state_pseus_idx[i]],
0),
validate_shape=True))
def call(self, inputs):
if self.r_num == 1:
# if there is no routing (and this is so when r_num is 1 and all c are equal)
# then this is a common convolution
outputs = K.conv2d(
K.reshape(inputs,
(-1, self.h_i, self.w_i, self.ch_i * self.n_i)),
K.reshape(
self.w, self.kernel_size +
(self.ch_i * self.n_i, self.ch_j * self.n_j)),
data_format='channels_last',
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate)
outputs = squeeze(
K.reshape(outputs,
((-1, self.h_j, self.w_j, self.ch_j, self.n_j))))
else:
bt = K.shape(inputs)[0]
ksz = self.kernel_size[0] * self.kernel_size[1]
xr = K.reshape(inputs,
(-1, self.h_i, self.w_i, self.ch_i * self.n_i))
pt = tf.extract_image_patches(xr, (1, ) + self.kernel_size + (1, ),
(1, ) + self.strides + (1, ), (
1,
1,
1,
1,
), 'VALID')
pt = K.reshape(pt, (-1, ksz * self.ch_i, self.n_i))
wr = K.reshape(self.w,
(ksz * self.ch_i, self.n_i, self.ch_j * self.n_j))
global useGPU
# it sometimes works faster on GPU when batch is devided into two parts
# bp = K.expand_dims(bt // 2, axis=0) if useGPU else K.constant([2], dtype=tf.int32)
bp = K.expand_dims(bt // 1, axis=0) if useGPU else K.constant(
[2], dtype=tf.int32)
if self.strides != (1, 1):
zr_shape = K.concatenate([
bp,
K.constant([self.w_j, ksz * self.ch_i * self.ch_j],
dtype=tf.int32)
])
zr = tf.zeros(shape=zr_shape)
zc_shape = K.concatenate([
bp,
K.constant([self.ah_j, ksz * self.ch_i * self.ch_j],
dtype=tf.int32)
])
zc = tf.zeros(shape=zc_shape)
def rt(ptb):
ptb = K.reshape(ptb, (-1, ksz * self.ch_i, self.n_i))
if useGPU:
ub = tf.einsum('bin,inj->bij', ptb, wr)
else:
ul = []
for i in range(ksz * self.ch_i):
ul.append(K.dot(ptb[:, i], wr[i]))
ub = K.stack(ul, axis=1)
#b = tf.constant_initializer(0.)((bp, self.h_i*self.w_i*self.ch_i,
# ksz * self.ch_j))
b = 0.0
j_all = self.h_j * self.w_j * self.ch_j
j_add = j_all - ksz * self.ch_j
for r in range(self.r_num):
ex = K.exp(b * self.b_alphas[r])
if r > 0:
c = ex / (
(K.sum(ex, axis=-1, keepdims=True) + K.epsilon()) +
j_add) * j_all
c = K.reshape(c, (-1, self.h_i, self.w_i,
self.ch_i * ksz * self.ch_j))
c = K.stop_gradient(c)
pc = tf.extract_image_patches(
c, (1, ) + self.kernel_size + (1, ),
(1, ) + self.strides + (1, ), (
1,
1,
1,
1,
), 'VALID')
pc = K.reshape(pc, (-1, self.h_j, self.w_j, ksz,
self.ch_i, self.kernel_size[0] *
self.kernel_size[1], self.ch_j))
pcl = []
for n in range(ksz):
pcl.append(
pc[:, :, :, n, :,
self.kernel_size[0] * self.kernel_size[1] -
1 - n])
pcc = K.stack(pcl, axis=3)
if useGPU:
pcc = K.reshape(pcc,
(-1, self.h_j * self.w_j * ksz *
self.ch_i * self.ch_j, 1))
ub = K.reshape(ub,
(-1, self.h_j * self.w_j * ksz *
self.ch_i * self.ch_j, self.n_j))
cu = pcc * ub
else:
pcc = K.reshape(pcc, (-1, 1))
ub = K.reshape(ub, (-1, self.n_j, 1))
cul = []
for n in range(self.n_j):
cul.append(ub[:, n] * pcc)
cu = K.stack(cul, axis=-2)
else:
cu = ub
cu = K.reshape(cu, (-1, self.h_j * self.w_j,
ksz * self.ch_i, self.ch_j, self.n_j))
s = K.sum(cu, axis=-3)
v = squeeze(s)
if r == self.r_num - 1:
break
v = K.stop_gradient(v)
ubr = K.reshape(K.stop_gradient(ub),
(-1, self.h_j * self.w_j, ksz * self.ch_i,
self.ch_j, self.n_j))
if True:
#if useGPU:
a = tf.einsum('bjck,bjick->bjic', v, ubr)
else:
al = []
for i in range(ksz * self.ch_i):
al.append(
K.batch_dot(
K.reshape(ubr[:, :, i],
(-1, self.h_j * self.w_j *
self.ch_j, 1, self.n_j)),
K.reshape(v, (-1, self.h_j * self.w_j *
self.ch_j, self.n_j, 1))))
a = K.stack(al, axis=1)
a = K.reshape(a, (-1, ksz * self.ch_i,
self.h_j * self.w_j, self.ch_j))
a = K.permute_dimensions(a, [0, 2, 1, 3])
ph, pw = 2, 2
a = K.reshape(
a,
(-1, self.h_j, self.w_j, ksz * self.ch_i * self.ch_j))
if self.strides == (1, 1):
aa = a
else:
rl = []
for r in range(self.ah_j):
rl.append(zr if r % ph else a[:, r // ph])
rs = K.stack(rl, axis=1)
cl = []
for c in range(self.aw_j):
cl.append(zc if c % pw else rs[:, :, c // pw])
aa = K.stack(cl, axis=-2)
aa = K.spatial_2d_padding(aa, ((ph, ph), (pw, pw)),
data_format='channels_last')
pa = tf.extract_image_patches(
aa,
(1, ) + self.kernel_size + (1, ),
(
1,
1,
1,
1,
), #(1,)+strides+(1,),
(
1,
1,
1,
1,
),
'VALID')
pa = K.reshape(pa, (-1, self.h_i * self.w_i, ksz, ksz,
self.ch_i, self.ch_j))
pal = []
for n in range(ksz):
pal.append(pa[:, :, n, ksz - 1 - n])
paa = K.stack(pal, axis=3)
paa = K.reshape(
paa,
(-1, self.h_i * self.w_i * self.ch_i, ksz * self.ch_j))
b = b + paa
return v
v = tf.map_fn(rt,
K.reshape(pt, (-1, bp[0], self.h_j, self.w_j,
ksz * self.ch_i, self.n_i)),
parallel_iterations=100,
back_prop=True,
infer_shape=False)
outputs = v
outputs = K.reshape(outputs,
(-1, self.h_j, self.w_j, self.ch_j, self.n_j))
return outputs
def parse_data(proto,
include_flow,
height=None,
width=None,
include_occlusion=False,
include_invalid=False,
resize_gt_flow=True,
gt_flow_shape=None):
"""Parse a data proto with flow.
Args:
proto: path to data proto file
include_flow: bool, whether or not to include flow in the output
height: int or None height to resize image to
width: int or None width to resize image to
include_occlusion: bool, whether or not to also return occluded pixels (will
throw error if occluded pixels are not present)
include_invalid: bool, whether or not to also return invalid pixels (will
throw error if invalid pixels are not present)
resize_gt_flow: bool, wether or not to resize flow ground truth as the image
gt_flow_shape: list, shape of the original ground truth flow (only required
to set a fixed ground truth flow shape for tensorflow estimator in case of
supervised training at full resolution resize_gt_flow=False)
Returns:
images, flow: A tuple of (image1, image2), flow
"""
# Parse context and image sequence from protobuffer.
context_features = {
'height': tf.io.FixedLenFeature([], tf.int64),
'width': tf.io.FixedLenFeature([], tf.int64),
}
sequence_features = {
'images': tf.io.FixedLenSequenceFeature([], tf.string),
}
if include_invalid:
sequence_features['invalid_masks'] = tf.io.FixedLenSequenceFeature(
[], tf.string)
if include_flow:
pass # context_features['flow_uv'] = tf.io.FixedLenFeature([], tf.string)
if include_occlusion:
context_features['occlusion_mask'] = tf.io.FixedLenFeature([],
tf.string)
context_parsed, sequence_parsed = tf.io.parse_single_sequence_example(
proto,
context_features=context_features,
sequence_features=sequence_features,
)
def deserialize(s, dtype, dims):
return tf.reshape(
tf.io.decode_raw(s, dtype),
[context_parsed['height'], context_parsed['width'], dims])
images = tf.map_fn(lambda s: deserialize(s, tf.uint8, 3),
sequence_parsed['images'],
dtype=tf.uint8)
images = tf.image.convert_image_dtype(images, tf.float32)
if height is not None and width is not None:
images = uflow_utils.resize(images, height, width, is_flow=False)
output = [images]
if include_flow:
'''
flow_uv = deserialize(context_parsed['flow_uv'], tf.float32, 2)
flow_uv = flow_uv[Ellipsis, ::-1]
if height is not None and width is not None and resize_gt_flow:
flow_uv = uflow_utils.resize(flow_uv, height, width, is_flow=True)
else:
if gt_flow_shape is not None:
flow_uv.set_shape(gt_flow_shape)
# To be consistent with uflow internals, we flip the ordering of flow.
output.append(flow_uv)
# create valid mask
flow_valid = tf.ones_like(flow_uv[Ellipsis, :1], dtype=tf.float32)
output.append(flow_valid)
'''
output.append([])
output.append([])
if include_occlusion:
occlusion_mask = deserialize(context_parsed['occlusion_mask'],
tf.uint8, 1)
if height is not None and width is not None:
occlusion_mask = uflow_utils.resize(occlusion_mask,
height,
width,
is_flow=False)
output.append(occlusion_mask)
if include_invalid:
invalid_masks = tf.map_fn(lambda s: deserialize(s, tf.uint8, 1),
sequence_parsed['invalid_masks'],
dtype=tf.uint8)
if height is not None and width is not None:
invalid_masks = uflow_utils.resize(invalid_masks,
height,
width,
is_flow=False)
output.append(invalid_masks)
# Only put the output in a list if there are more than one items in there.
if len(output) == 1:
output = output[0]
# print("output.shape: ", output.shape)
return output
def _conditional(Xnew,
feat,
kern,
f,
*,
full_cov=False,
full_output_cov=False,
q_sqrt=None,
white=False):
"""
Multi-output GP with independent GP priors.
Number of latent processes equals the number of outputs (L = P).
The covariance matrices used to calculate the conditional have the following shape:
- Kuu: P x M x M
- Kuf: P x M x N
- Kff: P x N or P x N x N
Further reference
-----------------
- See `gpflow.conditionals._conditional` for a detailed explanation of
conditional in the single-output case.
- See the multiouput notebook for more information about the multiouput framework.
- See above for the parameters and the return value.
"""
logger.debug(
"conditional: object, SharedIndependentMof, SeparateIndependentMok, object"
)
# Following are: P x M x M - P x M x N - P x N(x N)
Kmms = Kuu(feat, kern, jitter=settings.numerics.jitter_level) # P x M x M
Kmns = Kuf(feat, kern, Xnew) # P x M x N
kern_list = kern.kernels if isinstance(
kern, Combination) else [kern.kern] * len(feat.feat_list)
Knns = tf.stack(
[k.K(Xnew) if full_cov else k.Kdiag(Xnew) for k in kern_list], axis=0)
fs = tf.transpose(f)[:, :, None] # P x M x 1
# P x 1 x M x M or P x M x 1
q_sqrts = tf.transpose(
q_sqrt)[:, :, None] if q_sqrt.shape.ndims == 2 else q_sqrt[:,
None, :, :]
def single_gp_conditional(t):
Kmm, Kmn, Knn, f, q_sqrt = t
return base_conditional(Kmn,
Kmm,
Knn,
f,
full_cov=full_cov,
q_sqrt=q_sqrt,
white=white)
rmu, rvar = tf.map_fn(
single_gp_conditional, (Kmms, Kmns, Knns, fs, q_sqrts),
(settings.float_type,
settings.float_type)) # P x N x 1, P x 1 x N x N or P x N x 1
fmu = tf.matrix_transpose(rmu[:, :, 0]) # N x P
if full_cov:
fvar = rvar[:, 0, :, :] # P x N x N
else:
fvar = tf.transpose(rvar[..., 0]) # N x P
return fmu, _expand_independent_outputs(fvar, full_cov, full_output_cov)
def conditional(self, X, test, full_cov=False):
"""
A multisample conditional, where X is shape (S,N,D_out), independent over samples S
if full_cov is True
mean is (S,N,D_out), var is (S,N,N,D_out)
if full_cov is False
mean and var are both (S,N,D_out)
:param X: The input locations (S,N,D_in)
:param full_cov: Whether to calculate full covariance or just diagonal
:return: mean (S,N,D_out), var (S,N,D_out or S,N,N,D_out)
"""
def single_sample_conditional(X, full_cov=False):
if test == 0:
select = tf.random_shuffle(tf.range(tf.shape(
self.q_mu)[1]))[:tf.cast(
(1.0 - self.dropout) *
tf.cast(tf.shape(self.q_mu)[1], tf.float64), tf.int32)]
select = tf.contrib.framework.sort(select)
q_mu_temp = tf.gather(self.q_mu, select, axis=1)
q_sqrt_temp = tf.gather(self.q_sqrt, select, axis=0)
'''
select = np.random.choice((tf.convert_to_tensor(self.q_mu.shape[1])).eval()-1, size=int((1-self.dropout)*((tf.convert_to_tensor(self.q_mu.shape[1])).eval()-1)), replace = False)
select = np.sort(select)
q_mu_temp = np.take((self.q_mu).eval(),select,axis=1)
q_sqrt_temp = np.take((self.q_sqrt).eval(),select,axis=0)
transform = transforms.LowerTriangular((tf.convert_to_tensor(self.feature.Z.shape[0])).eval(), num_matrices=q_mu_temp.shape[1])
q_sqrt_temp = Parameter(q_sqrt_temp, transform=transform)
q_mu_temp = tf.constant(q_mu_temp)
q_sqrt_temp = tf.constant(q_sqrt_temp)
'''
else:
q_mu_temp = self.q_mu
q_sqrt_temp = self.q_sqrt
select = []
self.q_mu_temp = q_mu_temp
self.q_sqrt_temp = q_sqrt_temp
mean, var = my_conditional(X,
self.feature.Z,
self.kern,
q_mu_temp,
q_sqrt=q_sqrt_temp,
full_cov=full_cov,
white=True)
return mean + self.mean_function(X), var, select
if full_cov is True:
f = lambda a: single_sample_conditional(a, full_cov=full_cov)
mean, var, select = tf.map_fn(f, X, dtype=(tf.float64, tf.float64))
return tf.stack(mean), tf.stack(var), select
else:
S, N, D = tf.shape(X)[0], tf.shape(X)[1], tf.shape(X)[2]
X_flat = tf.reshape(X, [S * N, D])
mean, var, select = single_sample_conditional(X_flat)
mean = tf.reshape(mean, [S, N, -1])
var = tf.reshape(var, [S, N, -1])
return mean, var, select
def draw_bounding_boxes_on_image_tensors(images,
boxes,
classes,
scores,
category_index,
instance_masks=None,
keypoints=None,
max_boxes_to_draw=20,
min_score_thresh=0.2,
use_normalized_coordinates=True):
"""Draws bounding boxes, masks, and keypoints on batch of image tensors.
Args:
images: A 4D uint8 image tensor of shape [N, H, W, C]. If C > 3, additional
channels will be ignored.
boxes: [N, max_detections, 4] float32 tensor of detection boxes.
classes: [N, max_detections] int tensor of detection classes. Note that
classes are 1-indexed.
scores: [N, max_detections] float32 tensor of detection scores.
category_index: a dict that maps integer ids to category dicts. e.g.
{1: {1: 'dog'}, 2: {2: 'cat'}, ...}
instance_masks: A 4D uint8 tensor of shape [N, max_detection, H, W] with
instance masks.
keypoints: A 4D float32 tensor of shape [N, max_detection, num_keypoints, 2]
with keypoints.
max_boxes_to_draw: Maximum number of boxes to draw on an image. Default 20.
min_score_thresh: Minimum score threshold for visualization. Default 0.2.
use_normalized_coordinates: Whether to assume boxes and kepoints are in
normalized coordinates (as opposed to absolute coordiantes).
Default is True.
Returns:
4D image tensor of type uint8, with boxes drawn on top.
"""
# Additional channels are being ignored.
images = images[:, :, :, 0:3]
visualization_keyword_args = {
'use_normalized_coordinates': use_normalized_coordinates,
'max_boxes_to_draw': max_boxes_to_draw,
'min_score_thresh': min_score_thresh,
'agnostic_mode': False,
'line_thickness': 4
}
if instance_masks is not None and keypoints is None:
visualize_boxes_fn = functools.partial(_visualize_boxes_and_masks,
category_index=category_index,
**visualization_keyword_args)
elems = [images, boxes, classes, scores, instance_masks]
elif instance_masks is None and keypoints is not None:
visualize_boxes_fn = functools.partial(_visualize_boxes_and_keypoints,
category_index=category_index,
**visualization_keyword_args)
elems = [images, boxes, classes, scores, keypoints]
elif instance_masks is not None and keypoints is not None:
visualize_boxes_fn = functools.partial(
_visualize_boxes_and_masks_and_keypoints,
category_index=category_index,
**visualization_keyword_args)
elems = [images, boxes, classes, scores, instance_masks, keypoints]
else:
visualize_boxes_fn = functools.partial(_visualize_boxes,
category_index=category_index,
**visualization_keyword_args)
elems = [images, boxes, classes, scores]
def draw_boxes(image_and_detections):
"""Draws boxes on image."""
image_with_boxes = tf.py_func(visualize_boxes_fn, image_and_detections,
tf.uint8)
return image_with_boxes
images = tf.map_fn(draw_boxes, elems, dtype=tf.uint8, back_prop=False)
return images
def decode(self, knowledge_rep, dropout=None):
"""
takes in a knowledge representation
and output a probability estimation over
all paragraph tokens on which token should be
the start of the answer span, and which should be
the end of the answer span.
:param knowledge_rep: it is a representation of the paragraph and question,
decided by how you choose to implement the encoder
:return:
"""
# Approach 1
# Run final Uniderectional LSTM one final inputs representation
#finalInputs, contextLen = knowledge_rep
#with vs.variable_scope("FinalLSTMs", reuse = False):
#outputs_s, _ = tf.nn.dynamic_rnn(self.LSTMcellCompresser, inputs=finalInputs, sequence_length=contextLen, dtype=tf.float32)
# _, (_, output_s) = tf.nn.dynamic_rnn(self.LSTMcell, inputs=finalInputs, sequence_length=contextLen, dtype=tf.float32)
#with vs.variable_scope("FinalLSTMe", reuse=False):
#outputs_e, _ = tf.nn.dynamic_rnn(self.LSTMcellCompresser, inputs=finalInputs, sequence_length=contextLen, dtype=tf.float32)
# _, (_, output_e) = tf.nn.dynamic_rnn(self.LSTMcell, inputs=finalInputs, sequence_length=contextLen, dtype=tf.float32)
#return(output_s, output_e)
#outputs_s = outputs[:, :, 0]
#outputs_e = outputs[:, :, 1]
#outputs_s = tf.reshape(outputs_s, [-1, self.output_size])
#outputs_e = tf.reshape(outputs_e, [-1, self.output_size])
#return(outputs_s, outputs_e)
# Approach 2
#final_inputs, contextLen = knowledge_rep
#with vs.variable_scope("Decoder", reuse=False):
# output_s = tf.contrib.layers.fully_connected(inputs=final_inputs, num_outputs=self.output_size,
# weights_initializer=tf.contrib.layers.xavier_initializer())
# output_e = tf.contrib.layers.fully_connected(inputs=final_inputs, num_outputs=self.output_size,
# weights_initializer=tf.contrib.layers.xavier_initializer())
#return (output_s, output_e)
# Approach 3
#_, max_length, encoded_size = knowledge_rep.get_shape().as_list()
final_inputs, contextLen = knowledge_rep
with vs.variable_scope("Decoder", reuse=None):
W_s = tf.get_variable("W_s", shape=(final_inputs.get_shape()[2], 1), dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())
W_e = tf.get_variable("W_e", shape=(final_inputs.get_shape()[2], 1), dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())
tmp_s = tf.map_fn(lambda current_output: tf.matmul(current_output, W_s), final_inputs)
tmp_e = tf.map_fn(lambda current_output: tf.matmul(current_output, W_e), final_inputs)
#tmp_s = tf.map_fn(lambda current_output: tf.matmul(tf.nn.dropout(current_output, dropout), W_s), final_inputs)
#tmp_e = tf.map_fn(lambda current_output: tf.matmul(tf.nn.dropout(current_output, dropout), W_e), final_inputs)
outputs_s = tf.reshape(tmp_s, [-1, self.output_size])
outputs_e = tf.reshape(tmp_e, [-1, self.output_size])
return(outputs_s, outputs_e)
def call(self, y_pred, mask=None):
'''
Returns:
3D tensor of shape `(batch_size, top_k, 6)`. The second axis is
zero-padded
to always yield `top_k` predictions per batch item. The last
axis contains
the coordinates for each predicted box in the format
`[class_id, confidence, xmin, ymin, xmax, ymax]`.
'''
#######################################################################
# 1. Convert the box coordinates from predicted anchor box offsets
# to predicted
# absolute coordinates
#######################################################################
# Convert anchor box offsets to image offsets.
cx = y_pred[..., -12] * y_pred[..., -4] * y_pred[..., -6] + y_pred[
..., -8] # cx = cx_pred * cx_variance * w_anchor + cx_anchor
cy = y_pred[..., -11] * y_pred[..., -3] * y_pred[..., -5] + y_pred[
..., -7] # cy = cy_pred * cy_variance * h_anchor + cy_anchor
w = tf.exp(y_pred[..., -10] * y_pred[..., -2]) * y_pred[
..., -6] # w = exp(w_pred * variance_w) * w_anchor
h = tf.exp(y_pred[..., -9] * y_pred[..., -1]) * y_pred[
..., -5] # h = exp(h_pred * variance_h) * h_anchor
# Convert 'centroids' to 'corners'.
xmin = cx - 0.5 * w
ymin = cy - 0.5 * h
xmax = cx + 0.5 * w
ymax = cy + 0.5 * h
# If the model predicts box coordinates relative to the image
# dimensions and they are supposed
# to be converted back to absolute coordinates, do that.
def normalized_coords():
xmin1 = tf.expand_dims(xmin * self.tf_img_width, axis=-1)
ymin1 = tf.expand_dims(ymin * self.tf_img_height, axis=-1)
xmax1 = tf.expand_dims(xmax * self.tf_img_width, axis=-1)
ymax1 = tf.expand_dims(ymax * self.tf_img_height, axis=-1)
return xmin1, ymin1, xmax1, ymax1
def non_normalized_coords():
return tf.expand_dims(xmin, axis=-1), tf.expand_dims(ymin,
axis=-1), \
tf.expand_dims(
xmax, axis=-1), tf.expand_dims(ymax, axis=-1)
xmin, ymin, xmax, ymax = tf.cond(self.tf_normalize_coords,
normalized_coords,
non_normalized_coords)
# Concatenate the one-hot class confidences and the converted box
# coordinates to form the decoded predictions tensor.
y_pred = tf.concat(values=[y_pred[..., :-12], xmin, ymin, xmax, ymax],
axis=-1)
#######################################################################
# 2. Perform confidence thresholding, per-class non-maximum
# suppression, and
# top-k filtering.
#######################################################################
batch_size = tf.shape(y_pred)[0] # Output dtype: tf.int32
n_boxes = tf.shape(y_pred)[1]
n_classes = y_pred.shape[2] - 4
class_indices = tf.range(1, n_classes)
# Create a function that filters the predictions for the given batch
# item. Specifically, it performs:
# - confidence thresholding
# - non-maximum suppression (NMS)
# - top-k filtering
def filter_predictions(batch_item):
# Create a function that filters the predictions for one single
# class.
def filter_single_class(index):
# From a tensor of shape (n_boxes, n_classes + 4
# coordinates) extract
# a tensor of shape (n_boxes, 1 + 4 coordinates) that
# contains the
# confidnece values for just one class, determined by `index`.
confidences = tf.expand_dims(batch_item[..., index], axis=-1)
class_id = tf.fill(dims=tf.shape(confidences),
value=tf.to_float(index))
box_coordinates = batch_item[..., -4:]
single_class = tf.concat(
[class_id, confidences, box_coordinates], axis=-1)
# Apply confidence thresholding with respect to the class
# defined by `index`.
threshold_met = single_class[:, 1] > self.tf_confidence_thresh
single_class = tf.boolean_mask(tensor=single_class,
mask=threshold_met)
# If any boxes made the threshold, perform NMS.
def perform_nms():
scores = single_class[..., 1]
# `tf.image.non_max_suppression()` needs the box
# coordinates in the format `(ymin, xmin, ymax, xmax)`.
xmin = tf.expand_dims(single_class[..., -4], axis=-1)
ymin = tf.expand_dims(single_class[..., -3], axis=-1)
xmax = tf.expand_dims(single_class[..., -2], axis=-1)
ymax = tf.expand_dims(single_class[..., -1], axis=-1)
boxes = tf.concat(values=[ymin, xmin, ymax, xmax], axis=-1)
maxima_indices = tf.image.non_max_suppression(
boxes=boxes,
scores=scores,
max_output_size=self.tf_nms_max_output_size,
iou_threshold=self.iou_threshold,
name='non_maximum_suppresion')
maxima = tf.gather(params=single_class,
indices=maxima_indices,
axis=0)
return maxima
def no_confident_predictions():
return tf.constant(value=0.0, shape=(1, 6))
single_class_nms = tf.cond(tf.equal(tf.size(single_class), 0),
no_confident_predictions,
perform_nms)
# Make sure `single_class` is exactly
# `self.nms_max_output_size` elements long.
padded_single_class = tf.pad(tensor=single_class_nms,
paddings=[[
0,
self.tf_nms_max_output_size -
tf.shape(single_class_nms)[0]
], [0, 0]],
mode='CONSTANT',
constant_values=0.0)
return padded_single_class
# Iterate `filter_single_class()` over all class indices.
filtered_single_classes = tf.map_fn(
fn=lambda i: filter_single_class(i),
elems=tf.range(1, n_classes),
dtype=tf.float32,
parallel_iterations=128,
back_prop=False,
swap_memory=False,
infer_shape=True,
name='loop_over_classes')
# Concatenate the filtered results for all individual classes to
# one tensor.
filtered_predictions = tf.reshape(tensor=filtered_single_classes,
shape=(-1, 6))
# Perform top-k filtering for this batch item or pad it in case
# there are
# fewer than `self.top_k` boxes left at this point. Either way,
# produce a
# tensor of length `self.top_k`. By the time we return the final
# results tensor
# for the whole batch, all batch items must have the same number
# of predicted
# boxes so that the tensor dimensions are homogenous. If fewer
# than `self.top_k`
# predictions are left after the filtering process above,
# we pad the missing
# predictions with zeros as dummy entries.
def top_k():
return tf.gather(params=filtered_predictions,
indices=tf.nn.top_k(filtered_predictions[:,
1],
k=self.tf_top_k,
sorted=True).indices,
axis=0)
def pad_and_top_k():
padded_predictions = tf.pad(
tensor=filtered_predictions,
paddings=[[
0, self.tf_top_k - tf.shape(filtered_predictions)[0]
], [0, 0]],
mode='CONSTANT',
constant_values=0.0)
return tf.gather(params=padded_predictions,
indices=tf.nn.top_k(padded_predictions[:, 1],
k=self.tf_top_k,
sorted=True).indices,
axis=0)
top_k_boxes = tf.cond(
tf.greater_equal(
tf.shape(filtered_predictions)[0], self.tf_top_k), top_k,
pad_and_top_k)
return top_k_boxes
# Iterate `filter_predictions()` over all batch items.
output_tensor = tf.map_fn(fn=lambda x: filter_predictions(x),
elems=y_pred,
dtype=None,
parallel_iterations=128,
back_prop=False,
swap_memory=False,
infer_shape=True,
name='loop_over_batch')
return output_tensor
################ inputs do grafo ##################
tf_x_input = tf.placeholder(tf.float32, [None, 40, 140], name='X_input')
tf_X = tf.reshape(tf_x_input, shape=[
-1, 40, 140, 1
]) # reformata input para forma de imagem requerida pelo tf (h, w, d)
tf_y_input = tf.placeholder(tf.int64, [None, seq_len, n_classes],
name='y_input')
tf_keep_prob = tf.placeholder(tf.float32) # probabilidade de drop_out no grafo
# augment dataset if training
tf_X = tf.cond(
tf.less_equal(tf_keep_prob,
0.98), # usa prob de dropout para saber se está treinando
lambda: tf.map_fn(lambda image: augment(image), tf_X
), # caso treinando, aumenta os dados
lambda: tf.map_fn(lambda image: center_crop(image), tf_X)
) # caso testanto, corta imagen no centro
################ Variáveis do grafo ################
# Inicialização Xavier dos parâmetros
init_wc1 = np.sqrt(6.0 / (img_w * img_h * 1 + img_w * img_h * CL1_depth))
init_wc2 = np.sqrt(
6.0 / (img_w * img_h * CL1_depth + img_w / 2 * img_h / 2 * CL2_depth))
init_wc3 = np.sqrt(
6.0 /
(img_w / 2 * img_h / 2 * CL2_depth + img_w / 4 * img_h / 4 * CL3_depth))
init_wc4 = np.sqrt(
6.0 /
(img_w / 4 * img_h / 4 * CL3_depth + img_w / 8 * img_h / 8 * CL4_depth))
init_wfc1 = np.sqrt(6.0 / (img_w / 8 * img_h / 8 * CL4_depth + DL1_size))
# Weight for output layers
with tf.name_scope("linear_layer_weights"):
with tf.name_scope("W_linear"):
Wl = tf.Variable(
tf.truncated_normal([hidden_layer_size, num_classes],
mean=0,
stddev=.01))
variable_summaries(Wl)
with tf.name_scope("Bias_linear"):
bl = tf.Variable(tf.truncated_normal([num_classes], mean=0,
stddev=.01))
variable_summaries(bl)
with tf.name_scope("linear_layer_weights"):
#Iterate accross time and apply linear layer to all RNN outputs
all_outputs = tf.map_fn(get_linear_layer, all_hidden_states)
# Get last output
output = all_outputs[-1]
tf.summary.histogram("outputs", output)
with tf.name_scope("cross_entropy"):
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=y))
tf.summary.scalar("cross_entropy", cross_entropy)
with tf.name_scope("train"):
# Uses RMSPropOptimizer
# Minimize the error
train_step = tf.train.RMSPropOptimizer(1e-3, 0.9).minimize(cross_entropy)
with tf.name_scope("accuracy"):
def train(x, y, num_epochs, batch_size, graph_type='regular'):
"""
Train a specified network architecture on a dataset
:param x: Inputs
:param y: Labels
:param num_epochs: Passes through data
:param batch_size: Mini batch size
:param graph_type: Network architecture
:return:
"""
train_dataset = tf.data.Dataset.from_tensor_slices((x, y))
train_dataset = train_dataset.shuffle(x.shape[0])
train_dataset = train_dataset.repeat(num_epochs)
train_dataset = train_dataset.batch(batch_size)
iterator = train_dataset.make_initializable_iterator()
inputs, labels = iterator.get_next()
inputs = tf.map_fn(lambda frame: tf.image.per_image_standardization(frame),
inputs,
dtype=tf.float32)
if graph_type == 'regular':
logits = regular_net(inputs, 10)
elif graph_type == 'tf':
logits = weightnorm_net(inputs, 10)
elif graph_type == 'keras':
logits = weightnorm_keras_net(inputs, 10)
else:
raise ValueError
loss_op = tf.losses.sparse_softmax_cross_entropy(logits=logits,
labels=labels)
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=momentum)
train_op = optimizer.minimize(loss_op)
init = tf.global_variables_initializer()
step = 0
running_loss = 0
running_loss_array = []
with tf.Session() as sess:
sess.run(init)
sess.run(iterator.initializer)
graph_path = os.path.join(os.getcwd(), 'train', graph_type)
tf.summary.FileWriter(graph_path, sess.graph)
while True:
try:
_, loss = sess.run([train_op, loss_op])
step += 1
running_loss += (loss / batch_size)
if step % 32 == (32 - 1):
running_loss_array.append(running_loss)
running_loss = 0.0
except tf.errors.OutOfRangeError:
return running_loss_array
def call(self, y_pred, mask=None):
'''
Returns:
3D tensor of shape `(batch_size, top_k, 6)`. The second axis is zero-padded
to always yield `top_k` predictions per batch item. The last axis contains
the coordinates for each predicted box in the format
`[class_id, confidence, xmin, ymin, xmax, ymax]`.
'''
#####################################################################################
# 1. Convert the box coordinates from predicted anchor box offsets to predicted
# absolute coordinates
#####################################################################################
# Extract the predicted class IDs as the indices of the highest confidence values.
class_ids = tf.expand_dims(tf.to_float(
tf.argmax(y_pred[..., :-12], axis=-1)),
axis=-1)
# Extract the confidences of the maximal classes.
confidences = tf.reduce_max(y_pred[..., :-12], axis=-1, keep_dims=True)
# Convert anchor box offsets to image offsets.
cx = y_pred[..., -12] * y_pred[..., -4] * y_pred[..., -6] + y_pred[
..., -8] # cx = cx_pred * cx_variance * w_anchor + cx_anchor
cy = y_pred[..., -11] * y_pred[..., -3] * y_pred[..., -5] + y_pred[
..., -7] # cy = cy_pred * cy_variance * h_anchor + cy_anchor
w = tf.exp(y_pred[..., -10] * y_pred[..., -2]) * y_pred[
..., -6] # w = exp(w_pred * variance_w) * w_anchor
h = tf.exp(y_pred[..., -9] * y_pred[..., -1]) * y_pred[
..., -5] # h = exp(h_pred * variance_h) * h_anchor
# Convert 'centroids' to 'corners'.
xmin = cx - 0.5 * w
ymin = cy - 0.5 * h
xmax = cx + 0.5 * w
ymax = cy + 0.5 * h
# If the model predicts box coordinates relative to the image dimensions and they are supposed
# to be converted back to absolute coordinates, do that.
def normalized_coords():
xmin1 = tf.expand_dims(xmin * self.tf_img_width, axis=-1)
ymin1 = tf.expand_dims(ymin * self.tf_img_height, axis=-1)
xmax1 = tf.expand_dims(xmax * self.tf_img_width, axis=-1)
ymax1 = tf.expand_dims(ymax * self.tf_img_height, axis=-1)
return xmin1, ymin1, xmax1, ymax1
def non_normalized_coords():
return tf.expand_dims(xmin, axis=-1), tf.expand_dims(
ymin,
axis=-1), tf.expand_dims(xmax,
axis=-1), tf.expand_dims(ymax,
axis=-1)
xmin, ymin, xmax, ymax = tf.cond(self.tf_normalize_coords,
normalized_coords,
non_normalized_coords)
# Concatenate the one-hot class confidences and the converted box coordinates to form the decoded predictions tensor.
y_pred = tf.concat(
values=[class_ids, confidences, xmin, ymin, xmax, ymax], axis=-1)
#####################################################################################
# 2. Perform confidence thresholding, non-maximum suppression, and top-k filtering.
#####################################################################################
batch_size = tf.shape(y_pred)[0] # Output dtype: tf.int32
n_boxes = tf.shape(y_pred)[1]
n_classes = y_pred.shape[2] - 4
class_indices = tf.range(1, n_classes)
# Create a function that filters the predictions for the given batch item. Specifically, it performs:
# - confidence thresholding
# - non-maximum suppression (NMS)
# - top-k filtering
def filter_predictions(batch_item):
# Keep only the non-background boxes.
positive_boxes = tf.not_equal(batch_item[..., 0], 0.0)
predictions = tf.boolean_mask(tensor=batch_item,
mask=positive_boxes)
def perform_confidence_thresholding():
# Apply confidence thresholding.
threshold_met = predictions[:, 1] > self.tf_confidence_thresh
return tf.boolean_mask(tensor=predictions, mask=threshold_met)
def no_positive_boxes():
return tf.constant(value=0.0, shape=(1, 6))
# If there are any positive predictions, perform confidence thresholding.
predictions_conf_thresh = tf.cond(
tf.equal(tf.size(predictions), 0), no_positive_boxes,
perform_confidence_thresholding)
def perform_nms():
scores = predictions_conf_thresh[..., 1]
# `tf.image.non_max_suppression()` needs the box coordinates in the format `(ymin, xmin, ymax, xmax)`.
xmin = tf.expand_dims(predictions_conf_thresh[..., -4],
axis=-1)
ymin = tf.expand_dims(predictions_conf_thresh[..., -3],
axis=-1)
xmax = tf.expand_dims(predictions_conf_thresh[..., -2],
axis=-1)
ymax = tf.expand_dims(predictions_conf_thresh[..., -1],
axis=-1)
boxes = tf.concat(values=[ymin, xmin, ymax, xmax], axis=-1)
maxima_indices = tf.image.non_max_suppression(
boxes=boxes,
scores=scores,
max_output_size=self.tf_nms_max_output_size,
iou_threshold=self.iou_threshold,
name='non_maximum_suppresion')
maxima = tf.gather(params=predictions_conf_thresh,
indices=maxima_indices,
axis=0)
return maxima
def no_confident_predictions():
return tf.constant(value=0.0, shape=(1, 6))
# If any boxes made the threshold, perform NMS.
predictions_nms = tf.cond(
tf.equal(tf.size(predictions_conf_thresh), 0),
no_confident_predictions, perform_nms)
# Perform top-k filtering for this batch item or pad it in case there are
# fewer than `self.top_k` boxes left at this point. Either way, produce a
# tensor of length `self.top_k`. By the time we return the final results tensor
# for the whole batch, all batch items must have the same number of predicted
# boxes so that the tensor dimensions are homogenous. If fewer than `self.top_k`
# predictions are left after the filtering process above, we pad the missing
# predictions with zeros as dummy entries.
def top_k():
return tf.gather(params=predictions_nms,
indices=tf.nn.top_k(predictions_nms[:, 1],
k=self.tf_top_k,
sorted=True).indices,
axis=0)
def pad_and_top_k():
padded_predictions = tf.pad(tensor=predictions_nms,
paddings=[[
0, self.tf_top_k -
tf.shape(predictions_nms)[0]
], [0, 0]],
mode='CONSTANT',
constant_values=0.0)
return tf.gather(params=padded_predictions,
indices=tf.nn.top_k(padded_predictions[:, 1],
k=self.tf_top_k,
sorted=True).indices,
axis=0)
top_k_boxes = tf.cond(
tf.greater_equal(tf.shape(predictions_nms)[0], self.tf_top_k),
top_k, pad_and_top_k)
return top_k_boxes
# Iterate `filter_predictions()` over all batch items.
output_tensor = tf.map_fn(fn=lambda x: filter_predictions(x),
elems=y_pred,
dtype=None,
parallel_iterations=128,
back_prop=False,
swap_memory=False,
infer_shape=True,
name='loop_over_batch')
return output_tensor
def test_lanenet(dataset_dir, image_path, weights_path, use_gpu, image_list, batch_size, save_dir):
"""
:param image_path:
:param weights_path:
:param use_gpu:
:return:
"""
test_dataset = lanenet_data_processor_test.DataSet(image_path, batch_size, dataset_dir)
input_tensor = tf.placeholder(dtype=tf.string, shape=[None], name='input_tensor')
imgs = tf.map_fn(test_dataset.process_img, input_tensor, dtype=tf.float32)
phase_tensor = tf.constant('test', tf.string)
net = lanenet_merge_model.LaneNet()
binary_seg_ret, instance_seg_ret = net.test_inference(imgs, phase_tensor, 'lanenet_loss')
initial_var = tf.global_variables()
final_var = initial_var[:-1]
saver = tf.train.Saver(final_var)
# Set sess configuration
if use_gpu:
sess_config = tf.ConfigProto(device_count={'GPU': 1})
else:
sess_config = tf.ConfigProto(device_count={'GPU': 0})
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess_config.gpu_options.allocator_type = 'BFC'
sess = tf.Session(config=sess_config)
with sess.as_default():
sess.run(tf.global_variables_initializer())
saver.restore(sess=sess, save_path=weights_path)
for i in range(math.ceil(len(image_list) / batch_size)):
print("Inferring batch {} with batch size {}".format(i, batch_size))
paths = test_dataset.next_batch()
instance_seg_image, existence_output = sess.run([binary_seg_ret, instance_seg_ret],
feed_dict={input_tensor: paths})
# pudb.set_trace()
for cnt, input_image_path in enumerate(paths):
print("Generating output for input image {}".format(input_image_path))
input_image = cv2.imread(input_image_path)
input_image = cv2.resize(input_image, (CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT))
all_lanes_image = np.zeros((CFG.TRAIN.IMG_HEIGHT, CFG.TRAIN.IMG_WIDTH))
parent_path = os.path.dirname(input_image_path)
output_dir = os.path.join(parent_path, save_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
file_exist = open(os.path.join(output_dir, os.path.basename(input_image_path)[:-3] + 'exist.txt'), 'w')
for cnt_img in range(4):
filename = os.path.join(output_dir, os.path.basename(input_image_path)[:-4] + '_' + str(cnt_img + 1) + '_avg.png')
lane_image = (instance_seg_image[cnt, :, :, cnt_img + 1] * 255).astype(int)
cv2.imwrite(filename, lane_image)
all_lanes_image = all_lanes_image + lane_image
if existence_output[cnt, cnt_img] > 0.5:
file_exist.write('1 ')
else:
file_exist.write('0 ')
file_exist.close()
# Make all_lanes_image into color image (3 channels)
all_lanes_color_image = np.zeros((CFG.TRAIN.IMG_HEIGHT, CFG.TRAIN.IMG_WIDTH, 3))
all_lanes_color_image[:, :, 2] = all_lanes_image[:, :]
all_lanes_color_image_path = os.path.join(output_dir, os.path.basename(input_image_path)[:-3] + 'lanes.jpg')
resized_input_image_path = os.path.join(output_dir, os.path.basename(input_image_path)[:-3] + 'resized_input.jpg')
overlay_image_path = os.path.join(output_dir, os.path.basename(input_image_path)[:-3] + 'overlay.jpg')
cv2.imwrite(resized_input_image_path, input_image)
cv2.imwrite(all_lanes_color_image_path, all_lanes_color_image)
cv2.imwrite(overlay_image_path, all_lanes_color_image + input_image)
sess.close()
return
def __init__(self, models_path, vehicle_name='drone0'):
self.g = tf.Graph()
self.vehicle_name = vehicle_name
self.iter=0
self.models_path = models_path
self.first_frame = True
self.last_frame = []
with self.g.as_default():
# stat_writer_path = './adda/return_plot/'
# loss_writer_path = './adda/return_plot/'+ self.vehicle_name + '/loss/'
# self.stat_writer = tf.summary.FileWriter(stat_writer_path)
# name_array = 'D:/train/loss'+'/'+name
# self.loss_writer = tf.summary.FileWriter(loss_writer_path)
self.batch_size = tf.placeholder(tf.int32, shape=())
self.learning_rate = tf.placeholder(tf.float32, shape=())
self.X1 = tf.placeholder(tf.float32, [None, 103,103, 3], name='States')
self.X1T = tf.placeholder(tf.float32, [None, 103,103, 3], name='StatesT')
self.X = tf.map_fn(lambda frame: tf.image.per_image_standardization(frame), self.X1)
self.XT = tf.map_fn(lambda frame: tf.image.per_image_standardization(frame), self.X1T)
# self.X = tf.map_fn(lambda frame: tf.image.per_image_standardization(frame), self.X1)
self.Y = tf.placeholder(tf.float32, shape=[None], name='Qvals')
self.YT = tf.placeholder(tf.float32, shape=[None], name='Qvals')
self.Yactions = tf.placeholder(tf.int32, shape=[None], name='Actions')
self.YTactions = tf.placeholder(tf.int32, shape=[None], name='Actions')
# create graph
nn = adda.ADDA(25)
# for source domain
self.feat_s = nn.s_encoder(self.X, reuse=False, trainable=False)
self.logits_s = nn.classifier(self.feat_s, reuse=False, trainable=False)
self.disc_s = nn.discriminator(self.feat_s, reuse=False)
# for target domain
self.feat_t = nn.t_encoder(self.XT, reuse=False)
self.logits_t = nn.classifier(self.feat_t, reuse=True, trainable=False)
self.disc_t = nn.discriminator(self.feat_t, reuse=True)
# build loss
self.g_loss, self.d_loss = nn.build_ad_loss(self.disc_s, self.disc_t)
# g_loss,d_loss = nn.build_w_loss(disc_s,disc_t)
# create optimizer for two task
self.var_t_en = tf.trainable_variables(nn.t_e)
self.optim_g = tf.train.AdamOptimizer(0.00001, beta1=0.5, beta2=0.999).minimize(self.g_loss, var_list=self.var_t_en)
self.var_d = tf.trainable_variables(nn.d)
self.optim_d = tf.train.AdamOptimizer(0.00001, beta1=0.5, beta2=0.999).minimize(self.d_loss, var_list=self.var_d)
# create acuuracy op with training batch
self.acc_tr_s = nn.eval( self.logits_s, self.Yactions)
self.acc_tr_t = nn.eval( self.logits_t, self.YTactions)
# create source saver for restore s_encoder
encoder_path = tf.train.latest_checkpoint(models_path + "/encoder/")
classifier_path = tf.train.latest_checkpoint(models_path + "/classifier/")
if encoder_path is None:
raise ValueError("Don't exits in this dir")
if classifier_path is None:
raise ValueError("Don't exits in this dir")
self.source_var = tf.contrib.framework.list_variables(encoder_path)
self.var_s_g = tf.global_variables(scope=nn.s_e)
self.var_c_g = tf.global_variables(scope=nn.c)
self. var_t_g = tf.trainable_variables(scope=nn.t_e)
self.encoder_saver = tf.train.Saver(var_list=self.var_s_g)
self.classifier_saver = tf.train.Saver(var_list=self.var_c_g)
self.encoder_saver_target = tf.train.Saver(var_list=self.var_s_g)
dict_var = {}
for i in self.source_var:
for j in self.var_t_g:
if i[0][1:] in j.name[1:]:
dict_var[i[0]] = j
# print(dict_var)
self.fine_turn_saver = tf.train.Saver(var_list=dict_var)
# dict_var2 = {}
# for i in self.var_t_g:
# for j in self.source_var:
# if i[0][1:] in j.name[1:]:
# dict_var[i[0]] = j
# print(dict_var)
# self.fine_turn_saver2= tf.train.Saver(var_list=dict_var2)
# self.fine_turn_saver = tf.train.Saver(var_list=dict_var)
# assert False
# create this model saver
self.best_saver = tf.train.Saver(max_to_keep=3)
self.sess = tf.InteractiveSession()
self.merge = tf.summary.merge_all()
tf.global_variables_initializer().run()
tf.local_variables_initializer().run()
self.sess.graph.finalize()
self.filewriter = tf.summary.FileWriter(logdir="./adda/logs/", graph=self.sess.graph)
self.load_network()
print("model init successfully!")
def chi_sqr(z_true,z_pred):
numerator = K.square(z_true - z_pred)
denominator = tf.map_fn(lambda x: tf.where(tf.less(x, 1e-7), 1., x), z_true, dtype='float32')
true_mean = tf.tensordot(z_true, distribution_elements_row, 1)
hist_weights = 0.01 + K.square(distribution_elements_row - true_mean)
return tf.tensordot(tf.div(numerator,denominator), hist_weights, 1)
def main():
################ check directory #########################################
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
if a.checkpoint is None:
raise Exception("checkpoint required for test mode")
############### display/save setting #####################################
for k, v in a._get_kwargs():
print(k, "=", v)
with open(os.path.join(a.output_dir, "options.json"), "w") as f:
f.write(json.dumps(vars(a), sort_keys=True, indent=4))
################ read TFRecord dataset ###################################
input_batch, iterator = read_tfrecord()
################ creat model ############################################
# inputs and targets are of size [batch_size, height, width, channels]
model = create_model(input_batch.images, input_batch.labels, a.class_num)
################ summaries ###############################################
# undo colorization splitting on images that we use for display/output
images = deprocess(input_batch.images)
# YuhangLi: only save a part of images for saving driver space.
with tf.name_scope("encode_images"):
display_fetches = {
#"paths": input_batch.paths,
"images":
tf.map_fn(tf.image.encode_png,
convert(images),
dtype=tf.string,
name="images"),
"label":
input_batch.labels
}
################ configuration ##############################################
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum(
[tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=1)
logdir = a.output_dir
sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
f = open("log.txt", mode='wb')
############### session ######################################################
with sv.managed_session(config=sess_config) as sess:
print("parameter_count =", sess.run(parameter_count))
if a.checkpoint is not None:
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
saver.restore(sess, checkpoint)
max_steps = 2**32
if a.max_epochs is not None:
max_steps = input_batch.steps_per_epoch * a.max_epochs
if a.max_steps is not None:
max_steps = a.max_steps
######################## test ############################
if a.mode == "test":
# at most, process the test data once
max_steps = min(input_batch.steps_per_epoch, max_steps)
for step in range(max_steps):
results = sess.run(display_fetches)
filesets = save_images(results, step=step)
for i, f in enumerate(filesets):
print("evaluated image", f["name"])
# temporaly commented, error for unknown reason
# index_path = append_index(filesets)
print("wrote index at", index_path)
return
def call(self, vector):
"""
Parameters
vector: tensor of affine components
"""
return tf.map_fn(self._single_conversion, vector, dtype=tf.float32)
def _split_target(sequence_batch: tf.Tensor) -> Tuple[tf.Tensor, tf.Tensor]:
"""Split a N + 1 sequence into shifted-by-1 sequences for input and output."""
input_text = tf.map_fn(lambda x: x[:-1], sequence_batch)
target_text = tf.map_fn(lambda x: x[1:], sequence_batch)
return (input_text, target_text)
infer_decoder, output_time_major=True, swap_memory=True)
preds = infer_decoder_outputs.sample_id
return preds
int_to_vocab, vocab_size = load_vocab(FLAGS.vocab_file)
vocab = lookup_ops.index_table_from_file(FLAGS.vocab_file,
default_value=FLAGS.unk_id)
eos_id = tf.cast(vocab.lookup(tf.constant(FLAGS.eos)), tf.int32)
raw_sequence_op = tf.placeholder(tf.string, [1, None])
sequence = tf.cast(vocab.lookup(raw_sequence_op), tf.int32)
sequence = tf.map_fn(lambda x: tf.concat((x, [eos_id]), 0), sequence)
source_sequence_length = tf.map_fn(lambda x: tf.size(x), sequence)
preds = create_network(sequence, FLAGS.sos, FLAGS.eos, vocab,
source_sequence_length, vocab_size, FLAGS.hidden_size,
FLAGS.batch_size, FLAGS.encoder_num_layers,
FLAGS.decoder_num_layers)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
saver = tf.train.Saver()
latest_checkpoint = tf.train.latest_checkpoint('checkpoint_bahdanau')
if latest_checkpoint and tf.train.checkpoint_exists(latest_checkpoint):
def convert_alpha(self, alpha):
one_over_sqrt_alpha = tf.map_fn(lambda x: 1 / (tf.math.sqrt(x)),
alpha,
dtype=tf.float64)
return one_over_sqrt_alpha
def filter_predictions(batch_item):
# Create a function that filters the predictions for one single class.
def filter_single_class(index):
# From a tensor of shape (n_boxes, n_classes + 4 coordinates) extract
# a tensor of shape (n_boxes, 1 + 4 coordinates) that contains the
# confidnece values for just one class, determined by `index`.
confidences = tf.expand_dims(batch_item[..., index], axis=-1)
# revised
class_id = tf.fill(dims=tf.shape(confidences),
value=tf.cast(index, dtype=tf.float32))
box_coordinates = batch_item[..., -4:]
single_class = tf.concat([class_id, confidences, box_coordinates],
axis=-1)
# Apply confidence thresholding with respect to the class defined by `index`.
threshold_met = single_class[:, 1] > tf_confidence_thresh
single_class = tf.boolean_mask(tensor=single_class,
mask=threshold_met)
# If any boxes made the threshold, perform NMS.
def perform_nms():
scores = single_class[..., 1]
# `tf.image.non_max_suppression()` needs the box coordinates in the format `(ymin, xmin, ymax, xmax)`.
xmin = tf.expand_dims(single_class[..., -4], axis=-1)
ymin = tf.expand_dims(single_class[..., -3], axis=-1)
xmax = tf.expand_dims(single_class[..., -2], axis=-1)
ymax = tf.expand_dims(single_class[..., -1], axis=-1)
boxes = tf.concat(values=[ymin, xmin, ymax, xmax], axis=-1)
maxima_indices = tf.image.non_max_suppression(
boxes=boxes,
scores=scores,
max_output_size=tf_nms_max_output_size,
iou_threshold=iou_threshold,
name='non_maximum_suppresion')
maxima = tf.gather(params=single_class,
indices=maxima_indices,
axis=0)
return maxima
def no_confident_predictions():
return tf.constant(value=0.0, shape=(1, 6), dtype=tf.float32)
single_class_nms = tf.cond(tf.equal(tf.size(single_class), 0),
no_confident_predictions, perform_nms)
# Make sure `single_class` is exactly `self.nms_max_output_size` elements long.
padded_single_class = tf.pad(tensor=single_class_nms,
paddings=[[
0, tf_nms_max_output_size -
tf.shape(single_class_nms)[0]
], [0, 0]],
mode='CONSTANT',
constant_values=0.0)
return padded_single_class
# Iterate `filter_single_class()` over all class indices.
filtered_single_classes = tf.map_fn(
fn=lambda i: filter_single_class(i),
elems=tf.range(1, n_classes),
dtype=tf.float32,
parallel_iterations=128,
back_prop=False,
swap_memory=False,
infer_shape=True,
name='loop_over_classes')
# Concatenate the filtered results for all individual classes to one tensor.
filtered_predictions = tf.reshape(tensor=filtered_single_classes,
shape=(-1, 6))
# Perform top-k filtering for this batch item or pad it in case there are
# fewer than `self.top_k` boxes left at this point. Either way, produce a
# tensor of length `self.top_k`. By the time we return the final results tensor
# for the whole batch, all batch items must have the same number of predicted
# boxes so that the tensor dimensions are homogenous. If fewer than `self.top_k`
# predictions are left after the filtering process above, we pad the missing
# predictions with zeros as dummy entries.
def top_k():
return tf.gather(params=filtered_predictions,
indices=tf.nn.top_k(filtered_predictions[:, 1],
k=tf_top_k,
sorted=True).indices,
axis=0)
def pad_and_top_k():
padded_predictions = tf.pad(
tensor=filtered_predictions,
paddings=[[0, tf_top_k - tf.shape(filtered_predictions)[0]],
[0, 0]],
mode='CONSTANT',
constant_values=0.0)
return tf.gather(params=padded_predictions,
indices=tf.nn.top_k(padded_predictions[:, 1],
k=tf_top_k,
sorted=True).indices,
axis=0)
top_k_boxes = tf.cond(
tf.greater_equal(tf.shape(filtered_predictions)[0], tf_top_k),
top_k, pad_and_top_k)
return top_k_boxes
def main():
if dataset == 'cifar10':
x_train, y_train = cifar_input.get_cifar_data(data_path, option='train')
x_test, y_test = cifar_input.get_cifar_data(data_path, option='test')
elif dataset == 'svhn':
x_train, y_train, x_test, y_test = svhn_input.get_svhn_data('../datasets')
model_x_train = []
model_y_train = []
sorted_x_train, sorted_y_train = noniid.generate_nonIID(x_train, y_train)
temp_idx = [0] * num_classes
cls_idx_ex = [] * 10
cls_idx_ex.append([9, 7, 3, 6, 4, 8, 0, 5, 1, 2])
cls_idx_ex.append([2, 0, 6, 7, 5, 8, 3, 4, 1, 9])
cls_idx_ex.append([4, 3, 1, 0, 6, 9, 5, 8, 2, 7])
cls_idx_ex.append([3, 0, 1, 4, 9, 5, 2, 8, 7, 6])
cls_idx_ex.append([8, 4, 6, 1, 2, 9, 0, 5, 7, 3])
cls_idx_ex.append([6, 9, 8, 4, 7, 2, 3, 5, 0, 1])
cls_idx_ex.append([8, 6, 2, 4, 1, 7, 3, 0, 9, 5])
cls_idx_ex.append([5, 1, 7, 6, 9, 3, 2, 0, 8, 4])
cls_idx_ex.append([5, 6, 1, 3, 9, 0, 7, 2, 4, 8])
cls_idx_ex.append([8, 0, 9, 1, 3, 4, 5, 7, 6, 2])
temp_cls_idx = 0
cls_idx = cls_idx_ex[temp_cls_idx]
for i in range(num_learners):
temp_x = np.concatenate([sorted_x_train[cls_idx[j]]
[temp_idx[cls_idx[j]]:
temp_idx[
cls_idx[j]] + num_training_data_examples_per_learner / num_classes_per_learner]
for j in range(num_classes_per_learner)])
temp_y = np.concatenate([sorted_y_train[cls_idx[j]]
[temp_idx[cls_idx[j]]:
temp_idx[
cls_idx[j]] + num_training_data_examples_per_learner / num_classes_per_learner]
for j in range(num_classes_per_learner)])
for j in range(num_classes_per_learner):
temp_idx[cls_idx[j]] += num_training_data_examples_per_learner / num_classes_per_learner
model_x_train.append(temp_x)
model_y_train.append(temp_y)
cls_idx = np.delete(cls_idx, np.s_[:num_classes_per_learner], axis=0)
if len(cls_idx) == 0 and i < num_learners - 1:
temp_cls_idx += 1
cls_idx = cls_idx_ex[temp_cls_idx]
del sorted_x_train
del sorted_y_train
config = tf.ConfigProto()
config.gpu_options.allow_growth = False
config.allow_soft_placement = True
model_batch_x = []
model_batch_y = []
model_pred = []
model_cost = []
model_schedule = []
model_opt = []
model_train_op = []
model_accuracy = []
model_params = []
update_ops = []
variable_set = []
moving_variances = []
moving_mean_squares = []
moving_square_means = []
moving_means = []
model_is_train = []
num_conv_chs = [3, 16 * num_convs_width_multi, 128, 256]
num_fc_outdims = [num_classes, 256, 256 * num_fcs_width_multi]
for i in range(num_learners + 2):
with tf.variable_scope('model{}'.format(i)):
with tf.name_scope('input'):
model_batch_x.insert(i, tf.placeholder(tf.float32, shape=[None, 32, 32, 3], name='x-input'))
model_batch_y.insert(i, tf.placeholder(tf.float32, shape=[None, num_classes], name='y-input'))
model_schedule.insert(i, tf.placeholder_with_default(0.0, shape=None))
model_is_train.insert(i, tf.placeholder_with_default(False, shape=None, name="is_train"))
if is_aug is True:
out = tf.cond(model_is_train[i],
lambda: image_augmentation(model_batch_x[i]),
lambda: tf.map_fn(lambda img: tf.image.per_image_standardization(img), model_batch_x[i]))
else:
out = tf.map_fn(lambda img: tf.image.per_image_standardization(img), model_batch_x[i])
with tf.variable_scope('conv_g1'):
with tf.variable_scope('conv1'):
out = _conv(out, num_conv_chs[0], num_conv_chs[1], 1)
if is_bn is True: out = _batchnorm(out, model_is_train[i])
if is_brn is True: out = _batchrnorm(out, model_is_train[i])
out = tf.nn.relu(out)
if is_dropout is True: out = tf.layers.dropout(out, dropout_rate1, training=model_is_train[i])
for j in six.moves.range(1, num_convs):
with tf.variable_scope('conv%d' % (j + 1)):
out = _conv(out, num_conv_chs[1], num_conv_chs[1], 1)
if is_bn is True: out = _batchnorm(out, model_is_train[i])
if is_brn is True: out = _batchrnorm(out, model_is_train[i])
out = tf.nn.relu(out)
if is_dropout is True: out = tf.layers.dropout(out, dropout_rate1, training=model_is_train[i])
out = tf.nn.max_pool(out, ksize=(1, 3, 3, 1), strides=[1, 2, 2, 1], padding='SAME')
with tf.variable_scope('conv_g2'):
with tf.variable_scope('conv1'):
out = _conv(out, num_conv_chs[1], num_conv_chs[2], 2)
if is_bn is True: out = _batchnorm(out, model_is_train[i])
if is_brn is True: out = _batchrnorm(out, model_is_train[i])
out = tf.nn.relu(out)
if is_dropout is True: out = tf.layers.dropout(out, dropout_rate1, training=model_is_train[i])
for j in six.moves.range(1, num_convs):
with tf.variable_scope('conv%d' % (j + 1)):
out = _conv(out, num_conv_chs[2], num_conv_chs[2], 2)
if is_bn is True: out = _batchnorm(out, model_is_train[i])
if is_brn is True: out = _batchrnorm(out, model_is_train[i])
out = tf.nn.relu(out)
if is_dropout is True: out = tf.layers.dropout(out, dropout_rate1, training=model_is_train[i])
out = tf.nn.max_pool(out, ksize=(1, 3, 3, 1), strides=[1, 2, 2, 1], padding='SAME')
with tf.variable_scope('conv_g3'):
with tf.variable_scope('conv1'):
out = _conv(out, num_conv_chs[2], num_conv_chs[3], 3)
if is_bn is True: out = _batchnorm(out, model_is_train[i])
if is_brn is True: out = _batchrnorm(out, model_is_train[i])
out = tf.nn.relu(out)
if is_dropout is True: out = tf.layers.dropout(out, dropout_rate1, training=model_is_train[i])
for j in six.moves.range(1, num_convs):
with tf.variable_scope('conv%d' % (j + 1)):
out = _conv(out, num_conv_chs[3], num_conv_chs[3], 3)
if is_bn is True: out = _batchnorm(out, model_is_train[i])
if is_brn is True: out = _batchrnorm(out, model_is_train[i])
out = tf.nn.relu(out)
if is_dropout is True: out = tf.layers.dropout(out, dropout_rate1, training=model_is_train[i])
if last_pooling == 'max':
out = tf.nn.max_pool(out, ksize=(1, 3, 3, 1), strides=[1, last_stride, last_stride, 1], padding='SAME')
elif last_pooling == 'avg':
out = _global_avg_pool(out)
n = 1
for j in range(1, len(out.get_shape().as_list())):
n = n * out.get_shape().as_list()[j]
out = tf.reshape(out, shape=[-1, n])
for j in six.moves.range(1, num_fcs):
temp_outdims = num_fc_outdims[2]
if j == num_fcs - 1: temp_outdims = num_fc_outdims[1]
with tf.variable_scope('fc%d' % (j)):
out = _fc(out, temp_outdims)
if is_bn is True: out = _batchnorm(out, model_is_train[i])
if is_brn is True: out = _batchrnorm(out, model_is_train[i])
out = tf.nn.relu(out)
if is_dropout is True: out = tf.layers.dropout(out, dropout_rate2, training=model_is_train[i])
with tf.variable_scope('fc%d' % num_fcs):
logits = _fc(out, num_fc_outdims[0])
model_pred.insert(i, tf.nn.softmax(logits=logits))
with tf.name_scope('cost'):
xent1 = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=model_batch_y[i])
model_cost.insert(i, tf.reduce_mean(xent1))
lr = lr_init * model_schedule[i]
wd = wd_init * model_schedule[i]
with tf.name_scope('opt'):
if opt_algo == 'pure':
model_opt.insert(i, tf.contrib.opt.MomentumWOptimizer(weight_decay=wd,
learning_rate=lr,
momentum=0.0))
elif opt_algo == 'nmom-wb' or opt_algo == 'nmom-a':
model_opt.insert(i, tf.contrib.opt.MomentumWOptimizer(weight_decay=wd,
learning_rate=lr,
momentum=0.9,
use_nesterov=True))
elif opt_algo == 'adam-wb' or opt_algo == 'adam-a':
model_opt.insert(i, tf.contrib.opt.AdamWOptimizer(weight_decay=wd,
learning_rate=lr))
update_ops.insert(i, tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope='model{}'.format(i)))
with tf.control_dependencies(update_ops[i]):
model_train_op.insert(i, model_opt[i].minimize(model_cost[i]))
with tf.name_scope('Accuracy'):
predictions = tf.argmax(model_pred[i], axis=1)
model_accuracy.insert(i, tf.reduce_mean(
tf.to_float(tf.equal(predictions, tf.argmax(model_batch_y[i], axis=1)))))
model_params.insert(i, tf.contrib.framework.get_trainable_variables(scope='model{}'.format(i)))
variable_set.insert(i, tf.contrib.framework.get_variables(scope='model{}'.format(i),
collection=tf.GraphKeys.VARIABLES))
moving_means.insert(i, tf.contrib.framework.get_variables(scope='model{}'.format(i),
suffix='moving_mean',
collection=tf.GraphKeys.VARIABLES))
moving_variances.insert(i, tf.contrib.framework.get_variables(scope='model{}'.format(i),
suffix='moving_variance',
collection=tf.GraphKeys.VARIABLES))
moving_mean_squares.insert(i, [tf.math.square(v) for v in tf.contrib.framework.get_variables(scope='model{}'.format(i),
suffix='moving_mean',
collection=tf.GraphKeys.VARIABLES)])
moving_square_means.insert(i, [var + msq for var, msq in zip(moving_variances[i], moving_mean_squares[i])])
init_params = []
avg_assign_global = []
avg_assign_wb = []
avg_assign_a = []
assign_var_params = []
for i in range(num_learners + 2):
init_params.insert(i, [])
avg_assign_wb.insert(i, [])
avg_assign_a.insert(i, [])
for idx in range(len(model_params[0])):
init_params[i].insert(idx, tf.assign(model_params[i][idx], model_params[0][idx]))
variable_avg = []
for idx in range(len(variable_set[0])):
variable_avg.insert(idx, 0)
for i in range(num_learners):
variable_avg[idx] += variable_set[i][idx] / num_learners
temp_idx = 0
for idx in variable_set[0]:
avg_assign_global.insert(temp_idx,
tf.assign(variable_set[num_learners][temp_idx], variable_avg[temp_idx]))
temp_idx += 1
temp_idx = 0
for idx in variable_set[0]:
for i in range(num_learners + 2):
if idx.op.name.find(r'MomentumW') == -1 and idx.op.name.find(r'AdamW') == -1:
avg_assign_wb[i].insert(temp_idx, tf.assign(variable_set[i][temp_idx],
variable_set[num_learners][temp_idx]))
else:
avg_assign_wb[i].insert(temp_idx, tf.assign(variable_set[i][temp_idx],
variable_set[i][temp_idx]))
temp_idx += 1
temp_idx = 0
for idx in variable_set[0]:
for i in range(num_learners + 2):
avg_assign_a[i].insert(temp_idx,
tf.assign(variable_set[i][temp_idx],
variable_set[num_learners][temp_idx]))
temp_idx += 1
moving_var_avg = []
moving_mean_avg = []
for idx in range(len(moving_means[0])):
moving_mean_avg.insert(idx, 0)
for i in range(num_learners):
moving_mean_avg[idx] += moving_means[i][idx] / num_learners
for idx in range(len(moving_variances[0])):
moving_var_avg.insert(idx, 0)
for i in range(num_learners):
moving_var_avg[idx] += moving_square_means[i][idx] / num_learners
moving_var_avg[idx] -= tf.math.square(moving_mean_avg[idx])
temp_idx = 0
for idx in moving_variances[0]:
assign_var_params.insert(temp_idx,
tf.assign(moving_variances[num_learners][temp_idx], moving_var_avg[temp_idx]))
temp_idx += 1
with tf.train.MonitoredTrainingSession(config=config) as sess:
sess.run(init_params)
training_loss = np.zeros([num_learners], dtype=np.float32)
best_accuracy = 0.0
best_accuracy_round = 0
comm_round = 1
while comm_round <= training_rounds:
print('====================Round {0}===================='.format(comm_round))
if num_learners == 1:
if comm_round < 51:
schedule = 1.0
elif comm_round >= 51 and comm_round < 76:
schedule = 0.1
elif comm_round >= 76:
schedule = 0.01
elif num_learners == 10 and num_classes_per_learner == 10:
if comm_round < 101:
schedule = 1.0
elif comm_round >= 101 and comm_round < 151:
schedule = 0.1
elif comm_round >= 151:
schedule = 0.01
elif num_learners == 10 and num_classes_per_learner == 2:
if comm_round < 151:
schedule = 1.0
elif comm_round >= 151 and comm_round < 226:
schedule = 0.1
elif comm_round >= 226:
schedule = 0.01
elif num_learners == 10 and num_classes_per_learner == 1:
if comm_round < 151:
schedule = 1.0
elif comm_round >= 151 and comm_round < 226:
schedule = 0.1
elif comm_round >= 226:
schedule = 0.01
training_loss.fill(0.0)
shuffle_idx = [] * num_learners
for i in range(num_learners):
shuffle_idx.insert(i, np.arange(num_training_data_examples_per_learner, dtype=np.int32))
np.random.shuffle(shuffle_idx[i])
for j in range(local_steps):
for i in range(num_learners):
batch_xs = model_x_train[i][shuffle_idx[i][:batch_size]]
batch_ys = model_y_train[i][shuffle_idx[i][:batch_size]]
shuffle_idx[i] = np.delete(shuffle_idx[i], np.s_[:batch_size], axis=0)
if len(shuffle_idx[i]) == 0:
shuffle_idx[i] = np.arange(num_training_data_examples_per_learner, dtype=np.int32)
np.random.shuffle(shuffle_idx[i])
_, loss = sess.run([model_train_op[i], model_cost[i]],
feed_dict={model_batch_x[i]: batch_xs, model_batch_y[i]: batch_ys,
model_is_train[i]: True, model_schedule[i]: schedule})
training_loss[i] += loss / local_steps
print('[1-1] Training Loss (Each Learner, Mean): {0}, {1}'.format(training_loss, np.mean(training_loss)))
print(' ')
if num_learners != 1:
shuffle_idx = np.arange(num_training_data_examples, dtype=np.int32)
np.random.shuffle(shuffle_idx)
for idx in range(local_steps):
batch_xs = x_train[shuffle_idx[:batch_size]]
batch_ys = y_train[shuffle_idx[:batch_size]]
shuffle_idx = np.delete(shuffle_idx, np.s_[:batch_size], axis=0)
if len(shuffle_idx) == 0:
shuffle_idx = np.arange(num_training_data_examples)
np.random.shuffle(shuffle_idx)
sess.run(model_train_op[num_learners + 1],
feed_dict={model_batch_x[num_learners + 1]: batch_xs,
model_batch_y[num_learners + 1]: batch_ys,
model_is_train[num_learners + 1]: True,
model_schedule[num_learners + 1]: schedule})
model_params_per_layer = [] * (num_learners + 2)
model_params_whole = [] * (num_learners + 2)
for i in range(num_learners + 2):
temp_params = sess.run(model_params[i])
temp_params_per_layer = []
temp_params_whole = np.array((), dtype=float)
temp_idx = 0
for idx in model_params[0]:
if idx.op.name.find(r'weight') > 0:
temp_params_per_layer.append(temp_params[temp_idx].flatten())
temp_params_whole = np.append(temp_params_whole, temp_params[temp_idx].flatten())
temp_idx += 1
model_params_per_layer.insert(i, temp_params_per_layer)
model_params_whole.insert(i, temp_params_whole)
dist_per_layer = [0.0] * len(model_params_per_layer[0])
for i in range(num_learners):
for j in range(num_learners):
if i != j:
for idx in range(len(model_params_per_layer[0])):
dist_per_layer[idx] += distance.cosine(
model_params_per_layer[i][idx], model_params_per_layer[j][idx]
) / (num_learners * num_learners - num_learners)
print('[2-1] PD_Ls (Each Layer): {0}'.format(dist_per_layer))
dist_per_layer = [0.0] * len(model_params_per_layer[0])
for i in range(num_learners):
for idx in range(len(model_params_per_layer[0])):
dist_per_layer[idx] += distance.cosine(
model_params_per_layer[i][idx], model_params_per_layer[num_learners + 1][idx]
) / num_learners
print('[2-2] PD_VL (Each Layer): {0}'.format(dist_per_layer))
sess.run(avg_assign_global)
sess.run(assign_var_params)
test_accuracy = 0.0
test_iters = 100
test_batch_size = num_test_data_examples / test_iters
for j in range(test_iters):
temp_accuracy = sess.run(model_accuracy[num_learners],
feed_dict={model_batch_x[num_learners]:
x_test[test_batch_size * j:test_batch_size * (j + 1)],
model_batch_y[num_learners]:
y_test[test_batch_size * j:test_batch_size * (j + 1)],
model_is_train[num_learners]: False})
test_accuracy += temp_accuracy
test_accuracy = test_accuracy / float(test_iters)
if test_accuracy > best_accuracy:
best_accuracy = test_accuracy
best_accuracy_round = comm_round
print('[3-1] Test Accuracy (Global Model) - Current Round : {0}'.format(test_accuracy))
print('[3-2] Test Accuracy (Global Model) - Best : {0}'.format(best_accuracy))
print('[3-3] Round - Best Test Accuracy (Global Model) : {0}'.format(best_accuracy_round))
print(' ')
if opt_algo == 'pure' or opt_algo == 'nmom-wb' or opt_algo == 'adam-wb':
for i in range(num_learners):
sess.run(avg_assign_wb[i])
sess.run(avg_assign_wb[num_learners + 1])
elif opt_algo == 'nmom-a' or opt_algo == 'adam-a':
for i in range(num_learners):
sess.run(avg_assign_a[i])
sess.run(avg_assign_a[num_learners + 1])
comm_round += 1
print('Done')
print('Session closed cleanly')
def DecodeDetections(
y_pred,
confidence_thresh=0.01,
iou_threshold=0.45,
top_k=200,
nms_max_output_size=400,
coords='centroids',
normalize_coords=True,
img_height=None,
img_width=None,
):
'''
All default argument values follow the Caffe implementation.
Arguments:
confidence_thresh (float, optional): A float in [0,1), the minimum classification confidence in a specific
positive class in order to be considered for the non-maximum suppression stage for the respective class.
A lower value will result in a larger part of the selection process being done by the non-maximum suppression
stage, while a larger value will result in a larger part of the selection process happening in the confidence
thresholding stage.
iou_threshold (float, optional): A float in [0,1]. All boxes with a Jaccard similarity of greater than `iou_threshold`
with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers
to the box score.
top_k (int, optional): The number of highest scoring predictions to be kept for each batch item after the
non-maximum suppression stage.
nms_max_output_size (int, optional): The maximum number of predictions that will be left after performing non-maximum
suppression.
coords (str, optional): The box coordinate format that the model outputs. Must be 'centroids'
i.e. the format `(cx, cy, w, h)` (box center coordinates, width, and height). Other coordinate formats are
currently not supported.
normalize_coords (bool, optional): Set to `True` if the model outputs relative coordinates (i.e. coordinates in [0,1])
and you wish to transform these relative coordinates back to absolute coordinates. If the model outputs
relative coordinates, but you do not want to convert them back to absolute coordinates, set this to `False`.
Do not set this to `True` if the model already outputs absolute coordinates, as that would result in incorrect
coordinates. Requires `img_height` and `img_width` if set to `True`.
img_height (int, optional): The height of the input images. Only needed if `normalize_coords` is `True`.
img_width (int, optional): The width of the input images. Only needed if `normalize_coords` is `True`.
Returns:
3D tensor of shape `(batch_size, top_k, 6)`. The second axis is zero-padded
to always yield `top_k` predictions per batch item. The last axis contains
the coordinates for each predicted box in the format
`[class_id, confidence, xmin, ymin, xmax, ymax]`.
'''
# We need these members for TensorFlow.
tf_confidence_thresh = tf.constant(confidence_thresh,
dtype=tf.float32,
name='confidence_thresh')
tf_iou_threshold = tf.constant(iou_threshold,
dtype=tf.float32,
name='iou_threshold')
tf_top_k = tf.constant(top_k, name='top_k')
tf_normalize_coords = tf.constant(normalize_coords,
name='normalize_coords')
tf_img_height = tf.constant(img_height,
dtype=tf.float32,
name='img_height')
tf_img_width = tf.constant(img_width, dtype=tf.float32, name='img_width')
tf_nms_max_output_size = tf.constant(nms_max_output_size,
name='nms_max_output_size')
#####################################################################################
# 1. Convert the box coordinates from predicted anchor box offsets to predicted
# absolute coordinates
#####################################################################################
# Convert anchor box offsets to image offsets.
cx = y_pred[..., -12] * y_pred[..., -4] * y_pred[..., -6] + y_pred[
..., -8] # cx = cx_pred * cx_variance * w_anchor + cx_anchor
cy = y_pred[..., -11] * y_pred[..., -3] * y_pred[..., -5] + y_pred[
..., -7] # cy = cy_pred * cy_variance * h_anchor + cy_anchor
w = tf.exp(y_pred[..., -10] * y_pred[..., -2]) * y_pred[
..., -6] # w = exp(w_pred * variance_w) * w_anchor
h = tf.exp(y_pred[..., -9] * y_pred[..., -1]) * y_pred[
..., -5] # h = exp(h_pred * variance_h) * h_anchor
# Convert 'centroids' to 'corners'.
xmin = cx - 0.5 * w
ymin = cy - 0.5 * h
xmax = cx + 0.5 * w
ymax = cy + 0.5 * h
# If the model predicts box coordinates relative to the image dimensions and they are supposed
# to be converted back to absolute coordinates, do that.
def normalized_coords():
xmin1 = tf.expand_dims(xmin * tf_img_width, axis=-1)
ymin1 = tf.expand_dims(ymin * tf_img_height, axis=-1)
xmax1 = tf.expand_dims(xmax * tf_img_width, axis=-1)
ymax1 = tf.expand_dims(ymax * tf_img_height, axis=-1)
return xmin1, ymin1, xmax1, ymax1
def non_normalized_coords():
return tf.expand_dims(xmin, axis=-1), tf.expand_dims(ymin, axis=-1), \
tf.expand_dims(xmax, axis=-1), tf.expand_dims(ymax, axis=-1)
xmin, ymin, xmax, ymax = tf.cond(tf_normalize_coords, normalized_coords,
non_normalized_coords)
# Concatenate the one-hot class confidences and the converted box coordinates to form the decoded predictions tensor.
y_pred = tf.concat(values=[y_pred[..., :-12], xmin, ymin, xmax, ymax],
axis=-1)
#####################################################################################
# 2. Perform confidence thresholding, per-class non-maximum suppression, and
# top-k filtering.
#####################################################################################
batch_size = tf.shape(y_pred)[0] # Output dtype: tf.int32
n_boxes = tf.shape(y_pred)[1]
n_classes = y_pred.shape[2] - 4
class_indices = tf.range(1, n_classes)
# Create a function that filters the predictions for the given batch item. Specifically, it performs:
# - confidence thresholding
# - non-maximum suppression (NMS)
# - top-k filtering
def filter_predictions(batch_item):
# Create a function that filters the predictions for one single class.
def filter_single_class(index):
# From a tensor of shape (n_boxes, n_classes + 4 coordinates) extract
# a tensor of shape (n_boxes, 1 + 4 coordinates) that contains the
# confidnece values for just one class, determined by `index`.
confidences = tf.expand_dims(batch_item[..., index], axis=-1)
# revised
class_id = tf.fill(dims=tf.shape(confidences),
value=tf.cast(index, dtype=tf.float32))
box_coordinates = batch_item[..., -4:]
single_class = tf.concat([class_id, confidences, box_coordinates],
axis=-1)
# Apply confidence thresholding with respect to the class defined by `index`.
threshold_met = single_class[:, 1] > tf_confidence_thresh
single_class = tf.boolean_mask(tensor=single_class,
mask=threshold_met)
# If any boxes made the threshold, perform NMS.
def perform_nms():
scores = single_class[..., 1]
# `tf.image.non_max_suppression()` needs the box coordinates in the format `(ymin, xmin, ymax, xmax)`.
xmin = tf.expand_dims(single_class[..., -4], axis=-1)
ymin = tf.expand_dims(single_class[..., -3], axis=-1)
xmax = tf.expand_dims(single_class[..., -2], axis=-1)
ymax = tf.expand_dims(single_class[..., -1], axis=-1)
boxes = tf.concat(values=[ymin, xmin, ymax, xmax], axis=-1)
maxima_indices = tf.image.non_max_suppression(
boxes=boxes,
scores=scores,
max_output_size=tf_nms_max_output_size,
iou_threshold=iou_threshold,
name='non_maximum_suppresion')
maxima = tf.gather(params=single_class,
indices=maxima_indices,
axis=0)
return maxima
def no_confident_predictions():
return tf.constant(value=0.0, shape=(1, 6), dtype=tf.float32)
single_class_nms = tf.cond(tf.equal(tf.size(single_class), 0),
no_confident_predictions, perform_nms)
# Make sure `single_class` is exactly `self.nms_max_output_size` elements long.
padded_single_class = tf.pad(tensor=single_class_nms,
paddings=[[
0, tf_nms_max_output_size -
tf.shape(single_class_nms)[0]
], [0, 0]],
mode='CONSTANT',
constant_values=0.0)
return padded_single_class
# Iterate `filter_single_class()` over all class indices.
filtered_single_classes = tf.map_fn(
fn=lambda i: filter_single_class(i),
elems=tf.range(1, n_classes),
dtype=tf.float32,
parallel_iterations=128,
back_prop=False,
swap_memory=False,
infer_shape=True,
name='loop_over_classes')
# Concatenate the filtered results for all individual classes to one tensor.
filtered_predictions = tf.reshape(tensor=filtered_single_classes,
shape=(-1, 6))
# Perform top-k filtering for this batch item or pad it in case there are
# fewer than `self.top_k` boxes left at this point. Either way, produce a
# tensor of length `self.top_k`. By the time we return the final results tensor
# for the whole batch, all batch items must have the same number of predicted
# boxes so that the tensor dimensions are homogenous. If fewer than `self.top_k`
# predictions are left after the filtering process above, we pad the missing
# predictions with zeros as dummy entries.
def top_k():
return tf.gather(params=filtered_predictions,
indices=tf.nn.top_k(filtered_predictions[:, 1],
k=tf_top_k,
sorted=True).indices,
axis=0)
def pad_and_top_k():
padded_predictions = tf.pad(
tensor=filtered_predictions,
paddings=[[0, tf_top_k - tf.shape(filtered_predictions)[0]],
[0, 0]],
mode='CONSTANT',
constant_values=0.0)
return tf.gather(params=padded_predictions,
indices=tf.nn.top_k(padded_predictions[:, 1],
k=tf_top_k,
sorted=True).indices,
axis=0)
top_k_boxes = tf.cond(
tf.greater_equal(tf.shape(filtered_predictions)[0], tf_top_k),
top_k, pad_and_top_k)
return top_k_boxes
# Iterate `filter_predictions()` over all batch items.
output_tensor = tf.map_fn(fn=lambda x: filter_predictions(x),
elems=y_pred,
dtype=None,
parallel_iterations=128,
back_prop=False,
swap_memory=False,
infer_shape=True,
name='loop_over_batch')
return output_tensor
def build_graph(train):
with tf.device("/cpu:0"):
with tf.name_scope('X'):
x = tf.placeholder(tf.float32, [None, 125, 160, 1], name='x')
mlp = x
if train:
with tf.name_scope('RANDOM-CROP-FLIP'):
crop_x = tf.map_fn(lambda img: tf.random_crop(img, [112, 144, 1]), mlp)
crop_x = tf.map_fn(lambda img: tf.image.random_flip_up_down(img), crop_x)
mlp = crop_x
else:
with tf.name_scope('CENTER-CROP'):
crop_x = tf.map_fn(lambda img: tf.image.resize_image_with_crop_or_pad(img, 112, 144), mlp)
mlp = crop_x
with tf.name_scope('CONV-1'):
c1 = 16
res = tf.pad(mlp, [[0, 0], [0, 0], [0, 0], [0, c1-1]])
mlp = conv(mlp, weight([3, 3, 1, c1], name='w11')) + positive_bias([c1], name='b11')
mlp = tf.nn.relu(mlp, name='conv1')
mlp = conv(mlp, weight([3, 3, c1, c1], name='w12')) + positive_bias([c1], name='b12')
mlp = tf.nn.relu(mlp, name='conv2')
mlp = conv(mlp, weight([3, 3, c1, c1], name='w13')) + positive_bias([c1], name='b13')
mlp = tf.nn.relu(mlp, name='conv3')
# mlp = conv(mlp, weight([3, 3, c1, c1], name='w14')) + positive_bias([c1], name='b14')
# mlp = tf.nn.relu(mlp, name='conv4')
mlp = tf.add(mlp, res, name='res')
mlp = pool(mlp, name='pool')
with tf.name_scope('CONV-2'):
c2 = 32
res = tf.pad(mlp, [[0, 0], [0, 0], [0, 0], [0, c2-c1]])
mlp = conv(mlp, weight([3, 3, c1, c2], name='w21')) + positive_bias([c2], name='b21')
mlp = tf.nn.relu(mlp, name='conv1')
mlp = conv(mlp, weight([3, 3, c2, c2], name='w22')) + positive_bias([c2], name='b22')
mlp = tf.nn.relu(mlp, name='conv2')
mlp = conv(mlp, weight([3, 3, c2, c2], name='w23')) + positive_bias([c2], name='b23')
mlp = tf.nn.relu(mlp, name='conv3')
# mlp = conv(mlp, weight([3, 3, c2, c2], name='w24')) + positive_bias([c2], name='b24')
# mlp = tf.nn.relu(mlp, name='conv4')
mlp = tf.add(mlp, res, name='res')
mlp = pool(mlp, name='pool')
with tf.name_scope('CONV-3'):
c3 = 64
res = tf.pad(mlp, [[0, 0], [0, 0], [0, 0], [0, c3-c2]])
mlp = conv(mlp, weight([3, 3, c2, c3], name='w31')) + positive_bias([c3], name='b31')
mlp = tf.nn.relu(mlp, name='conv1')
mlp = conv(mlp, weight([3, 3, c3, c3], name='w32')) + positive_bias([c3], name='b32')
mlp = tf.nn.relu(mlp, name='conv2')
# mlp = conv(mlp, weight([3, 3, c3, c3], name='w33')) + positive_bias([c3], name='b33')
# mlp = tf.nn.relu(mlp, name='conv3')
# mlp = conv(mlp, weight([3, 3, c3, c3], name='w34')) + positive_bias([c3], name='b34')
# mlp = tf.nn.relu(mlp, name='conv4')
mlp = tf.add(mlp, res, name='res')
mlp = pool(mlp, name='pool')
with tf.name_scope('CONV-4'):
c4 = 128
res = tf.pad(mlp, [[0, 0], [0, 0], [0, 0], [0, c4-c3]])
mlp = conv(mlp, weight([3, 3, c3, c4], name='w41')) + positive_bias([c4], name='b41')
mlp = tf.nn.relu(mlp, name='conv1')
mlp = conv(mlp, weight([3, 3, c4, c4], name='w42')) + positive_bias([c4], name='b42')
mlp = tf.nn.relu(mlp, name='conv2')
# mlp = conv(mlp, weight([3, 3, c4, c4], name='w43')) + positive_bias([c4], name='b43')
# mlp = tf.nn.relu(mlp, name='conv3')
# mlp = conv(mlp, weight([3, 3, c4, c4], name='w44')) + positive_bias([c4], name='b44')
# mlp = tf.nn.relu(mlp, name='conv4')
mlp = tf.add(mlp, res, name='res')
mlp = pool(mlp, name='pool')
with tf.name_scope('MASK'):
ca = 66
mask = tf.reshape(mlp, [-1, c4])
mask = tf.nn.xw_plus_b(mask, weight([c4, ca], 'w5'), zero_bias([ca], 'b5'))
mask = tf.tanh(mask)
mask = tf.nn.xw_plus_b(mask, weight([ca, 1], 'w6'), zero_bias([1], 'b6'))
mask = tf.reshape(mask, [-1, 7*9])
mask = tf.nn.softmax(mask)
mask = tf.reshape(mask, [-1, 7, 9, 1])
mlp = tf.mul(mlp, mask)
mlp = tf.reduce_sum(mlp, [1, 2], True)
if train:
with tf.name_scope('DROPOUT'):
mlp = tf.nn.dropout(mlp, 0.5, noise_shape=tf.shape(mlp)*[1, 0, 0, 1]+[0, 1, 1, 0], name='dropout') # dropout by map
with tf.name_scope('FLAT'):
mlp = tf.reshape(mlp, [-1, c4], name='flat')
'''
if train:
with tf.name_scope('DROPOUT'):
mlp = tf.nn.dropout(mlp, 0.5, name='dropout')
'''
# 1FC
with tf.name_scope('FC'):
mlp = tf.nn.xw_plus_b(mlp, weight([c4, 7], name='w7'), zero_bias([7], name='b7'), name='fc')
with tf.name_scope('Y'):
y = tf.placeholder(tf.float32, [None, 7], name='y')
with tf.name_scope('SOFTMAX-WITH-LOSS'):
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(mlp, y), name='loss')
with tf.name_scope('SOFTMAX'):
prob = tf.nn.softmax(mlp, name='prob')
with tf.name_scope('ACC'):
acc = tf.equal(tf.argmax(prob, 1), tf.argmax(y, 1), name='correct')
acc = tf.reduce_mean(tf.cast(acc, tf.float32), name='acc')
if train:
with tf.name_scope('OPT'):
opt = tf.train.AdamOptimizer(name='opt')
train_op = opt.minimize(loss, name='train_op')
else:
train_op = None
# 创建summary
with tf.name_scope('SUM'):
# mask 为方便展示仅尺度变化
# mask_m = tf.reduce_min(mask, [1, 2], True)
mask_M = tf.reduce_max(mask, [1, 2], True)
mask_visual = mask / mask_M * 255.0 # mask_visual = (mask-mask_m) / (mask_M-mask_m) * 255.0
# prj_mask 为方便展示仅尺度变化
prj_mask = mask
prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
prj_mask = tf.image.resize_nearest_neighbor(prj_mask, tf.shape(prj_mask)[1:3]*2)
prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
prj_mask = tf.image.resize_nearest_neighbor(prj_mask, tf.shape(prj_mask)[1:3]*2)
prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
prj_mask = tf.image.resize_nearest_neighbor(prj_mask, tf.shape(prj_mask)[1:3]*2)
prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
prj_mask = tf.image.resize_nearest_neighbor(prj_mask, tf.shape(prj_mask)[1:3]*2)
# prj_mask_m = tf.reduce_min(prj_mask, [1, 2], True)
prj_mask_M = tf.reduce_max(prj_mask, [1, 2], True)
prj_mask_visual = prj_mask / prj_mask_M * 255.0 # prj_mask_visual = (prj_mask-prj_mask_m) / (prj_mask_M-prj_mask_m) * 255.0
# mask_crop_x 为方便展示动态范围变化
mask_crop_x = prj_mask * crop_x
mask_crop_x_m = tf.reduce_min(mask_crop_x, [1, 2], True)
mask_crop_x_M = tf.reduce_max(mask_crop_x, [1, 2], True)
mask_crop_x_visual = (mask_crop_x - mask_crop_x_m) / (mask_crop_x_M - mask_crop_x_m) * 255.0
# y
y_visual = tf.reshape(y, [-1, 1, 7, 1]) * 255.0
# prob
prob_visual = tf.reshape(prob, [-1, 1, 7, 1]) * 255.0
if train:
summary = tf.merge_summary([
tf.image_summary('train mask', mask_visual), # 1 7 9 1
tf.image_summary('train prj_mask', prj_mask_visual), # 1 112 144 1
tf.image_summary('train crop_x', crop_x), # 1 112 144 1
tf.image_summary('train mask_crop_x', mask_crop_x_visual), # 1 112 144 1
tf.image_summary('train y', y_visual), # 1 1 7 1
tf.image_summary('train prob', prob_visual), # 1 1 7 1
tf.scalar_summary('train loss', loss),
tf.scalar_summary('train acc', acc),
])
else:
summary = tf.merge_summary([
tf.image_summary('val mask', mask_visual), # 1 7 9 1
tf.image_summary('val prj_mask', prj_mask_visual), # 1 112 144 1
tf.image_summary('val crop_x', crop_x), # 1 112 144 1
tf.image_summary('val mask_crop_x', mask_crop_x_visual), # 1 112 144 1
tf.image_summary('val y', y_visual), # 1 1 7 1
tf.image_summary('val prob', prob_visual), # 1 1 7 1
tf.scalar_summary('val loss', loss),
tf.scalar_summary('val acc', acc),
])
return [x, y, loss, prob, acc, train_op, summary, crop_x, mask]
def call(self, inputs):
if self.r_num == 1:
outputs = K.dot(
K.reshape(inputs, (-1, self.ch_i * self.n_i)),
K.reshape(self.w,
(self.ch_i * self.n_i, self.ch_j * self.n_j)))
outputs = squeeze(K.reshape(outputs, (-1, self.ch_j, self.n_j)))
else:
wr = K.reshape(self.w, (self.ch_i, self.n_i, self.ch_j * self.n_j))
u = tf.transpose(tf.matmul(tf.transpose(inputs, [1, 0, 2]), wr),
[1, 0, 2])
u = K.reshape(u, (-1, self.ch_i, self.ch_j, self.n_j))
def rt(ub):
ub = K.reshape(ub, (-1, self.ch_i, self.ch_j, self.n_j))
ub_wo_g = K.stop_gradient(ub)
b = 0.0
for r in range(self.r_num):
if r > 0:
c = K.expand_dims(
K.softmax(b * self.b_alphas[r])
) * self.ch_j # distribution of weighs of capsules in I across capsules in J
c = K.stop_gradient(c)
else:
c = 1.0
if r == self.r_num - 1:
cub = c * ub
else:
cub = c * ub_wo_g
s = K.sum(cub, axis=-3) # vectors of capsules in J
v = squeeze(s) # squeezed vectors of capsules in J
if r == self.r_num - 1:
break
v = K.stop_gradient(v)
a = tf.einsum('bjk,bijk->bij', v, ub) # a = v dot u
#a = K.matmul(K.reshape(v, (-1, 1, J, 1, n_j)),
# K.reshape(u, (-1, I, J, n_j, 1))).reshape((-1, I, J))
b = b + a # increase those b[i,j] where v[j] dot b[i,j] is larger
return v
u = K.reshape(u, (-1, self.ch_i * self.ch_j * self.n_j))
global useGPU
if useGPU:
outputs = rt(u)
else:
outputs = tf.map_fn(rt,
u,
parallel_iterations=100,
back_prop=True,
infer_shape=False)
outputs = K.reshape(outputs, (-1, self.ch_j, self.n_j))
return outputs
def build_detection(self):
self.embeds, self.embeds2 = self.get_image_embedding(
self.search_images, reuse=True)
alpha = 0.5 # 0.5
beta = 0.2 # 0.3
with tf.variable_scope('detection'):
def _translation_match(
x, z): # translation match for one example within a batch
x = tf.expand_dims(x,
0) # [1, in_height, in_width, in_channels]
z = tf.expand_dims(
z, -1) # [filter_height, filter_width, in_channels, 1]
y = tf.nn.conv2d(x,
z,
strides=[1, 1, 1, 1],
padding='VALID',
name='translation_match')
return y
#def _dep_translation_match(x, z): # translation match for one example within a batch
#x = tf.expand_dims(x, 0) # [1, in_height, in_width, in_channels]
#z = tf.expand_dims(z, -1) # [filter_height, filter_width, in_channels, 1]
#y = tf.nn.depthwise_conv2d(x, z, strides=[1, 1, 1, 1], padding='VALID', name='dep_translation_match')
#return y
#@functools.wraps(attention_module)
#def att_fn(images, reuse=False):
#with slim.arg_scope(self.arg_scope):
#y, _ = attention_module(images, reuse=reuse)
#return y
with tf.variable_scope('patch_matching'):
template = tf.add_n(self.pseu_temp) / (len(self.pseu_temp))
template = alpha * self.templates + (1. - alpha) * template
embeds = tf.split(self.embeds, 2, axis=-1)
template = tf.split(template, 2, axis=-1)
#embeds = self.embeds
#output = tf.map_fn(lambda x: _translation_match(x, template[0]),
#embeds,
#dtype=embeds.dtype)
#output = tf.squeeze(output, [1]) # of shape e.g., [b, h, w, n]
#output = tf.map_fn(lambda x: _dep_translation_match(x, template[0]),
#embeds,
#dtype=embeds.dtype)
#output = tf.squeeze(output, [1]) # of shape e.g., [b, h, w, n]
#output = att_fn(output)
output_s1 = tf.map_fn(
lambda x: _translation_match(x, template[0][0]),
embeds[0],
dtype=self.embeds.dtype)
output_s1 = tf.squeeze(output_s1,
[1]) # of shape e.g., [b, h, w, 1]
output_s2 = tf.map_fn(
lambda x: _translation_match(x, template[1][0]),
embeds[1],
dtype=self.embeds.dtype)
output_s2 = tf.squeeze(output_s2,
[1]) # of shape e.g., [b, h, w, 1]
with tf.variable_scope('patch_matching2'):
template2 = tf.add_n(self.pseu_temp2) / (len(self.pseu_temp2))
template2 = alpha * self.templates2 + (1. - alpha) * template2
embeds2 = tf.split(self.embeds2, 2, axis=-1)
template2 = tf.split(template2, 2, axis=-1)
#embeds2 = self.embeds2
#output2 = tf.map_fn(lambda x: _translation_match(x, template2[0]),
#embeds2,
#dtype=embeds2.dtype)
#output2 = tf.squeeze(output2, [1]) # of shape e.g., [b, h, w, n]
#output2 = tf.map_fn(lambda x: _dep_translation_match(x, template2[0]),
#embeds2,
#dtype=embeds2.dtype)
#output2 = tf.squeeze(output2, [1]) # of shape e.g., [b, h, w, n]
#output2 = att_fn(output2)
output2_s1 = tf.map_fn(
lambda x: _translation_match(x, template2[0][0]),
embeds2[0],
dtype=self.embeds2.dtype)
output2_s1 = tf.squeeze(output2_s1,
[1]) # of shape e.g., [b, h, w, 1]
output2_s2 = tf.map_fn(
lambda x: _translation_match(x, template2[1][0]),
embeds2[1],
dtype=self.embeds2.dtype)
output2_s2 = tf.squeeze(output2_s2,
[1]) # of shape e.g., [b, h, w, 1]
with slim.arg_scope(self.arg_scope):
#output = slim.batch_norm(output, scope='bn_patch')
#output2 = slim.batch_norm(output2, scope='bn_patch2')
output_s1 = slim.batch_norm(output_s1, scope='bn_patch_s1')
output_s2 = slim.batch_norm(output_s2, scope='bn_patch_s2')
output2_s1 = slim.batch_norm(output2_s1, scope='bn_patch2_s1')
output2_s2 = slim.batch_norm(output2_s2, scope='bn_patch2_s2')
#output = slim.batch_norm(output, scope='bn_patch_ext')
#output2 = slim.batch_norm(output2, scope='bn_patch2_ext')
output = output_s1 * beta + output_s2 * (1. - beta)
output2 = output2_s1 * beta + output2_s2 * (1. - beta)
outputF_all = tf.squeeze(output * 0.5 + output2 * 0.5, -1)
self.response = outputF_all