def blend_images(data_folder1, data_folder2, out_folder, alpha=.5):
filename_queue = tf.placeholder(dtype=tf.string)
label = tf.placeholder(dtype=tf.int32)
tensor_image = tf.read_file(filename_queue)
image = tf.image.decode_jpeg(tensor_image, channels=3)
multiplier = tf.div(tf.constant(224, tf.float32),
tf.cast(tf.maximum(tf.shape(image)[0], tf.shape(image)[1]), tf.float32))
x = tf.cast(tf.round(tf.mul(tf.cast(tf.shape(image)[0], tf.float32), multiplier)), tf.int32)
y = tf.cast(tf.round(tf.mul(tf.cast(tf.shape(image)[1], tf.float32), multiplier)), tf.int32)
image = tf.image.resize_images(image, [x, y])
image = tf.image.rot90(image, k=label)
image = tf.image.resize_image_with_crop_or_pad(image, 224, 224)
sess = tf.Session()
sess.run(tf.local_variables_initializer())
for root, folders, files in os.walk(data_folder1):
for each in files:
if each.find('.jpg') >= 0:
img1 = Image.open(os.path.join(root, each))
img2_path = os.path.join(root.replace(data_folder1, data_folder2), each.split("-")[-1])
rotation = int(each.split("-")[1])
img2 = sess.run(image, feed_dict={filename_queue: img2_path, label: rotation})
imsave(os.path.join(os.getcwd(), "temp", "temp.jpg"), img2)
img2 = Image.open(os.path.join(os.getcwd(), "temp", "temp.jpg"))
out_image = Image.blend(img1, img2, alpha)
outfile = os.path.join(root.replace(data_folder1, out_folder), each)
if not os.path.exists(os.path.split(outfile)[0]):
os.makedirs(os.path.split(outfile)[0])
out_image.save(outfile)
else:
print(each)
sess.close()
def _compute_average_correct(input_layer, labels, per_example_weights, topk=1):
"""Returns the numerator and denominator of classifier accuracy."""
dtype = tf.float32
if topk == 1:
true_labels = tf.argmax(input_layer, 1)
predictions = tf.argmax(labels, 1)
in_topk = tf.equal(true_labels, predictions)
else:
_, true_labels = tf.nn.top_k(labels, k=1)
true_labels = tf.reshape(true_labels, [-1])
in_topk = tf.nn.in_top_k(tf.cast(input_layer, dtype), true_labels, k=topk)
correct_predictions = tf.cast(in_topk, dtype)
# If individual examples are weighted, then we want to normalize by that.
if per_example_weights:
per_example_weights = tf.convert_to_tensor(per_example_weights,
name='per_example_weights')
if ((input_layer.get_shape() and not per_example_weights.get_shape(
).is_compatible_with([input_layer.get_shape().dims[0]])) or
per_example_weights.get_shape().ndims != 1):
raise ValueError(
'per_example_weights must be a vector of the same length as '
'labels: was %s but expected (%s,)' % (
per_example_weights.get_shape()), input_layer[0])
float_weights = tf.cast(per_example_weights, dtype)
# TODO(eiderman): This should use an op that doesn't support broadcasting.
correct_predictions *= float_weights
num_examples = tf.reduce_sum(float_weights)
else:
# shape only holds ints, but we want to always return the same type
# for num_examples to make everything compatible.
num_examples = tf.cast(tf.gather(tf.shape(input_layer), 0), dtype)
return tf.reduce_sum(correct_predictions), num_examples
def to_normalized_coordinates(boxlist, height, width,
check_range=True, scope=None):
"""Converts absolute box coordinates to normalized coordinates in [0, 1].
Usually one uses the dynamic shape of the image or conv-layer tensor:
boxlist = box_list_ops.to_normalized_coordinates(boxlist,
tf.shape(images)[1],
tf.shape(images)[2]),
This function raises an assertion failed error at graph execution time when
the maximum coordinate is smaller than 1.01 (which means that coordinates are
already normalized). The value 1.01 is to deal with small rounding errors.
Args:
boxlist: BoxList with coordinates in terms of pixel-locations.
height: Maximum value for height of absolute box coordinates.
width: Maximum value for width of absolute box coordinates.
check_range: If True, checks if the coordinates are normalized or not.
scope: name scope.
Returns:
boxlist with normalized coordinates in [0, 1].
"""
with tf.name_scope(scope, 'ToNormalizedCoordinates'):
height = tf.cast(height, tf.float32)
width = tf.cast(width, tf.float32)
if check_range:
max_val = tf.reduce_max(boxlist.get())
max_assert = tf.Assert(tf.greater(max_val, 1.01),
['max value is lower than 1.01: ', max_val])
with tf.control_dependencies([max_assert]):
width = tf.identity(width)
return scale(boxlist, 1 / height, 1 / width)
def _apply(self, grad, var, indices=None):
lr = tf.cast(self._learning_rate_tensor, var.dtype.base_dtype)
m = self.get_slot(var, "m")
v = self.get_slot(var, "v")
beta1_t = tf.cast(self._beta1_t, var.dtype.base_dtype)
beta2_t = tf.cast(self._beta2_t, var.dtype.base_dtype)
epsilon_t = tf.cast(self._epsilon_t, var.dtype.base_dtype)
# m_t = beta1 * m + (1 - beta1) * g_t
m_scaled_g_values = grad * (1 - beta1_t)
m_t = tf.assign(m, m * beta1_t, use_locking=self._use_locking)
with tf.control_dependencies([m_t]):
m_t = self._assign_add(m, updates=m_scaled_g_values, indices=indices)
m_gathered = self._gather(m_t, indices=indices)
# Also see tf.nn.moments.
variance = tf.squared_difference(grad, m_gathered)
# v_t = beta2 * v + (1 - beta2) * variance
v_scaled_new_values = variance * (1 - beta2_t)
v_t = tf.assign(v, v * beta2_t, use_locking=self._use_locking)
with tf.control_dependencies([v_t]):
v_t = self._assign_add(v, updates=v_scaled_new_values, indices=indices)
v_gathered = self._gather(v_t, indices=indices)
factor = v_gathered / (variance + epsilon_t)
update = lr * grad * tf.minimum(factor, 1.0)
var_update = self._assign_sub(ref=var, updates=update, indices=indices)
return tf.group(*[var_update, m_t])
def boston_input_fn():
boston = tf.contrib.learn.datasets.load_boston()
features = tf.cast(
tf.reshape(tf.constant(boston.data), [-1, 13]), tf.float32)
labels = tf.cast(
tf.reshape(tf.constant(boston.target), [-1, 1]), tf.float32)
return features, labels
def extract_images_and_targets(read_data):
"""Extract images and targets from the input dict."""
image = read_data[fields.InputDataFields.image]
key = ''
if fields.InputDataFields.source_id in read_data:
key = read_data[fields.InputDataFields.source_id]
location_gt = read_data[fields.InputDataFields.groundtruth_boxes]
classes_gt = tf.cast(read_data[fields.InputDataFields.groundtruth_classes],
tf.int32)
classes_gt -= label_id_offset
if merge_multiple_label_boxes and use_multiclass_scores:
raise ValueError(
'Using both merge_multiple_label_boxes and use_multiclass_scores is'
'not supported'
)
if merge_multiple_label_boxes:
location_gt, classes_gt, _ = util_ops.merge_boxes_with_multiple_labels(
location_gt, classes_gt, num_classes)
elif use_multiclass_scores:
classes_gt = tf.cast(read_data[fields.InputDataFields.multiclass_scores],
tf.float32)
else:
classes_gt = util_ops.padded_one_hot_encoding(
indices=classes_gt, depth=num_classes, left_pad=0)
masks_gt = read_data.get(fields.InputDataFields.groundtruth_instance_masks)
keypoints_gt = read_data.get(fields.InputDataFields.groundtruth_keypoints)
if (merge_multiple_label_boxes and (
masks_gt is not None or keypoints_gt is not None)):
raise NotImplementedError('Multi-label support is only for boxes.')
weights_gt = read_data.get(
fields.InputDataFields.groundtruth_weights)
return (image, key, location_gt, classes_gt, masks_gt, keypoints_gt,
weights_gt)
def visualize_boxes_in_image(image, boxlist, normalized=False, scope=None):
"""Overlay bounding box list on image.
Currently this visualization plots a 1 pixel thick red bounding box on top
of the image. Note that tf.image.draw_bounding_boxes essentially is
1 indexed.
Args:
image: an image tensor with shape [height, width, 3]
boxlist: a BoxList
normalized: (boolean) specify whether corners are to be interpreted
as absolute coordinates in image space or normalized with respect to the
image size.
scope: name scope.
Returns:
image_and_boxes: an image tensor with shape [height, width, 3]
"""
with tf.name_scope(scope, 'VisualizeBoxesInImage'):
if not normalized:
height, width, _ = tf.unstack(tf.shape(image))
boxlist = scale(boxlist,
1.0 / tf.cast(height, tf.float32),
1.0 / tf.cast(width, tf.float32))
corners = tf.expand_dims(boxlist.get(), 0)
image = tf.expand_dims(image, 0)
return tf.squeeze(tf.image.draw_bounding_boxes(image, corners), [0])
def get_test_iterator(src_dataset, src_vocab_table, batch_size, config):
src_eos_id = tf.cast(src_vocab_table.lookup(tf.constant(config.eos)), tf.int32)
src_dataset = src_dataset.map(lambda src: tf.string_split([src]).values)
src_dataset = src_dataset.map(lambda src: src[:config.src_max_len])
src_dataset = src_dataset.map(
lambda src: tf.cast(src_vocab_table.lookup(src), tf.int32))
if config.reverse_src:
src_dataset = src_dataset.map(lambda src: tf.reverse(src, axis=[0]))
src_dataset = src_dataset.map(lambda src: (src, tf.size(src)))
def batching_func(x):
return x.padded_batch(
config.batch_size,
padded_shapes=(tf.TensorShape([None]),
tf.TensorShape([])),
padding_values=(src_eos_id,
0))
batched_dataset = batching_func(src_dataset)
batched_iter = batched_dataset.make_initializable_iterator()
src_ids, src_seq_len = batched_iter.get_next()
return BatchedInput(
initializer=batched_iter.initializer,
source=src_ids,
target_input=None,
target_output=None,
source_sequence_length=src_seq_len,
target_sequence_length=None)
def clip(x, min_value, max_value):
'''Element-wise value clipping.
'''
if max_value < min_value:
max_value = min_value
return tf.clip_by_value(x, tf.cast(min_value, dtype=_FLOATX),
tf.cast(max_value, dtype=_FLOATX))
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
dense_keys=['image_raw', 'label'],
# Defaults are not specified since both keys are required.
dense_types=[tf.string, tf.int64])
# Convert from a scalar string tensor (whose single string has
# length mnist.IMAGE_PIXELS) to a uint8 tensor with shape
# [mnist.IMAGE_PIXELS].
image = tf.decode_raw(features['image_raw'], tf.uint8)
image.set_shape([mnist.IMAGE_PIXELS])
# OPTIONAL: Could reshape into a 28x28 image and apply distortions
# here. Since we are not applying any distortions in this
# example, and the next step expects the image to be flattened
# into a vector, we don't bother.
# Convert from [0, 255] -> [-0.5, 0.5] floats.
image = tf.cast(image, tf.float32) * (1. / 255) - 0.5
# Convert label from a scalar uint8 tensor to an int32 scalar.
label = tf.cast(features['label'], tf.int32)
return image, label
def entropy(self, n, p):
# Note that given n and p where p is a probability vector of
# length k, the entropy requires a sum over all
# possible configurations of a k-vector which sums to n. It's
# expensive.
# http://stackoverflow.com/questions/36435754/generating-a-numpy-array-with-all-combinations-of-numbers-that-sum-to-less-than
sess = tf.Session()
n = sess.run(tf.cast(tf.squeeze(n), dtype=tf.int32))
sess.close()
p = tf.cast(tf.squeeze(p), dtype=tf.float32)
if isinstance(n, np.int32):
k = get_dims(p)[0]
max_range = np.zeros(k, dtype=np.int32) + n
x = np.array([i for i in product(*(range(i+1) for i in max_range))
if sum(i)==n])
logpmf = self.logpmf(x, n, p)
return tf.reduce_sum(tf.mul(tf.exp(logpmf), logpmf))
else:
out = []
for j in range(n.shape[0]):
k = get_dims(p)[0]
max_range = np.zeros(k, dtype=np.int32) + n[j]
x = np.array([i for i in product(*(range(i+1) for i in max_range))
if sum(i)==n[j]])
logpmf = self.logpmf(x, n[j], p[j, :])
out += [tf.reduce_sum(tf.mul(tf.exp(logpmf), logpmf))]
return tf.pack(out)
def siamese_cosine_loss(left, right, y, scope="cosine_loss"):
r"""Loss for Siamese networks (cosine version).
Same as :func:`contrastive_loss` but with different similarity measurement.
.. math::
[\frac{l \cdot r}{\lVert l\rVert \lVert r\rVert} - (2y-1)]^2
Args:
left (tf.Tensor): left feature vectors of shape [Batch, N].
right (tf.Tensor): right feature vectors of shape [Batch, N].
y (tf.Tensor): binary labels of shape [Batch]. 1: similar, 0: not similar.
Returns:
tf.Tensor: cosine-loss as a scalar tensor.
"""
def l2_norm(t, eps=1e-12):
"""
Returns:
tf.Tensor: norm of 2D input tensor on axis 1
"""
with tf.name_scope("l2_norm"):
return tf.sqrt(tf.reduce_sum(tf.square(t), 1) + eps)
with tf.name_scope(scope):
y = 2 * tf.cast(y, tf.float32) - 1
pred = tf.reduce_sum(left * right, 1) / (l2_norm(left) * l2_norm(right) + 1e-10)
return tf.nn.l2_loss(y - pred) / tf.cast(tf.shape(left)[0], tf.float32)
def _create_classification_weights(self,
match,
positive_class_weight=1.0,
negative_class_weight=1.0):
"""Create classification weights for each anchor.
Positive (matched) anchors are associated with a weight of
positive_class_weight and negative (unmatched) anchors are associated with
a weight of negative_class_weight. When anchors are ignored, weights are set
to zero. By default, both positive/negative weights are set to 1.0,
but they can be adjusted to handle class imbalance (which is almost always
the case in object detection).
Args:
match: a matcher.Match object that provides a matching between anchors
and groundtruth boxes.
positive_class_weight: weight to be associated to positive anchors
negative_class_weight: weight to be associated to negative anchors
Returns:
cls_weights: a float32 tensor with shape [num_anchors] representing
classification weights.
"""
matched_indicator = tf.cast(match.matched_column_indicator(), tf.float32)
ignore_indicator = tf.cast(match.ignored_column_indicator(), tf.float32)
unmatched_indicator = 1.0 - matched_indicator - ignore_indicator
cls_weights = (positive_class_weight * matched_indicator
+ negative_class_weight * unmatched_indicator)
return cls_weights
def _compute_log_moment(self, sigma, q, moment_order):
"""Compute high moment of privacy loss.
Args:
sigma: the noise sigma, in the multiples of the sensitivity.
q: the sampling ratio.
moment_order: the order of moment.
Returns:
log E[exp(moment_order * X)]
"""
assert moment_order <= self._max_moment_order, ("The order of %d is out "
"of the upper bound %d."
% (moment_order,
self._max_moment_order))
binomial_table = tf.slice(self._binomial_table, [moment_order, 0],
[1, moment_order + 1])
# qs = [1 q q^2 ... q^L] = exp([0 1 2 ... L] * log(q))
qs = tf.exp(tf.constant([i * 1.0 for i in range(moment_order + 1)],
dtype=tf.float64) * tf.cast(
tf.log(q), dtype=tf.float64))
moments0 = self._differential_moments(sigma, 0.0, moment_order)
term0 = tf.reduce_sum(binomial_table * qs * moments0)
moments1 = self._differential_moments(sigma, 1.0, moment_order)
term1 = tf.reduce_sum(binomial_table * qs * moments1)
return tf.squeeze(tf.log(tf.cast(q * term0 + (1.0 - q) * term1,
tf.float64)))
def __init__(self,num_classes, learning_rate, batch_size, decay_steps, decay_rate,sequence_length,
vocab_size,embed_size,is_training,initializer=tf.random_normal_initializer(stddev=0.1)):
"""init all hyperparameter here"""
# set hyperparamter
self.num_classes = num_classes
self.batch_size = batch_size
self.sequence_length=sequence_length
self.vocab_size=vocab_size
self.embed_size=embed_size
self.hidden_size=embed_size
self.is_training=is_training
self.learning_rate=learning_rate
self.initializer=initializer
self.num_sampled=20
# add placeholder (X,label)
self.input_x = tf.placeholder(tf.int32, [None, self.sequence_length], name="input_x") # X
self.input_y = tf.placeholder(tf.int32,[None], name="input_y") # y [None,num_classes]
self.dropout_keep_prob=tf.placeholder(tf.float32,name="dropout_keep_prob")
self.global_step = tf.Variable(0, trainable=False, name="Global_Step")
self.epoch_step=tf.Variable(0,trainable=False,name="Epoch_Step")
self.epoch_increment=tf.assign(self.epoch_step,tf.add(self.epoch_step,tf.constant(1)))
self.decay_steps, self.decay_rate = decay_steps, decay_rate
self.instantiate_weights()
self.logits = self.inference() #[None, self.label_size]. main computation graph is here.
if not is_training:
return
self.loss_val = self.loss() #-->self.loss_nce()
self.train_op = self.train()
self.predictions = tf.argmax(self.logits, axis=1, name="predictions") # shape:[None,]
correct_prediction = tf.equal(tf.cast(self.predictions,tf.int32), self.input_y) #tf.argmax(self.logits, 1)-->[batch_size]
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name="Accuracy") # shape=()
def dist_info_sym(self, obs_var, state_info_vars):
n_batches = tf.shape(obs_var)[0]
n_steps = tf.shape(obs_var)[1]
obs_var = tf.reshape(obs_var, tf.pack([n_batches, n_steps, -1]))
obs_var = tf.cast(obs_var, tf.float32)
if self.state_include_action:
prev_action_var = state_info_vars["prev_action"]
prev_action_var = tf.cast(prev_action_var, tf.float32)
all_input_var = tf.concat(2, [obs_var, prev_action_var])
else:
all_input_var = obs_var
if self.feature_network is None:
return dict(
prob=L.get_output(
self.prob_network.output_layer,
{self.l_input: all_input_var}
)
)
else:
flat_input_var = tf.reshape(all_input_var, (-1, self.input_dim))
return dict(
prob=L.get_output(
self.prob_network.output_layer,
{self.l_input: all_input_var, self.feature_network.input_layer: flat_input_var}
)
)
def GetLengths(self, dim=2, factor=1):
"""Returns the lengths of the batch of elements in the given dimension.
WARNING: The returned sizes may not exactly match TF's calculation.
Args:
dim: dimension to get the sizes of, in [1,2]. batch, depth not allowed.
factor: A scalar value to multiply by.
Returns:
The original heights/widths scaled by the current scaling of the model and
the given factor.
Raises:
ValueError: If the args are invalid.
"""
if dim == 1:
lengths = self.heights
elif dim == 2:
lengths = self.widths
else:
raise ValueError('Invalid dimension given to GetLengths')
lengths = tf.cast(lengths, tf.float32)
if self.reduction_factors[dim] is not None:
lengths = tf.div(lengths, self.reduction_factors[dim])
else:
lengths = tf.ones_like(lengths)
if factor != 1:
lengths = tf.mul(lengths, tf.cast(factor, tf.float32))
return tf.cast(lengths, tf.int32)
def parse_record(raw_record):
"""Parse CIFAR-10 image and label from a raw record."""
# Every record consists of a label followed by the image, with a fixed number
# of bytes for each.
label_bytes = 1
image_bytes = _HEIGHT * _WIDTH * _DEPTH
record_bytes = label_bytes + image_bytes
# Convert bytes to a vector of uint8 that is record_bytes long.
record_vector = tf.decode_raw(raw_record, tf.uint8)
# The first byte represents the label, which we convert from uint8 to int32
# and then to one-hot.
label = tf.cast(record_vector[0], tf.int32)
label = tf.one_hot(label, _NUM_CLASSES)
# The remaining bytes after the label represent the image, which we reshape
# from [depth * height * width] to [depth, height, width].
depth_major = tf.reshape(
record_vector[label_bytes:record_bytes], [_DEPTH, _HEIGHT, _WIDTH])
# Convert from [depth, height, width] to [height, width, depth], and cast as
# float32.
image = tf.cast(tf.transpose(depth_major, [1, 2, 0]), tf.float32)
return image, label
def kl_multivariate_normal(loc_one, scale_one, loc_two=0.0, scale_two=1.0):
"""Calculate the KL of multivariate normal distributions with
diagonal covariances.
Parameters
----------
loc_one : tf.Tensor
A 0-D tensor, 1-D tensor of length n, or 2-D tensor of shape M
x n where each row represents the mean of a n-dimensional
Gaussian.
scale_one : tf.Tensor
A tensor of same shape as ``loc_one``, representing the
standard deviation.
loc_two : tf.Tensor, optional
A tensor of same shape as ``loc_one``, representing the
mean of another Gaussian.
scale_two : tf.Tensor, optional
A tensor of same shape as ``loc_one``, representing the
standard deviation of another Gaussian.
Returns
-------
tf.Tensor
For 0-D or 1-D tensor inputs, outputs the 0-D tensor
``KL( N(z; loc_one, scale_one) || N(z; loc_two, scale_two) )``
For 2-D tensor inputs, outputs the 1-D tensor
``[KL( N(z; loc_one[m,:], scale_one[m,:]) || N(z; loc_two[m,:], scale_two[m,:]) )]_{m=1}^M``
Raises
------
InvalidArgumentError
If the location variables have Inf or NaN values, or if the scale
variables are not positive.
"""
dependencies = [tf.verify_tensor_all_finite(loc_one, msg=''),
tf.verify_tensor_all_finite(loc_two, msg=''),
tf.assert_positive(scale_one),
tf.assert_positive(scale_two)]
loc_one = control_flow_ops.with_dependencies(dependencies, loc_one)
scale_one = control_flow_ops.with_dependencies(dependencies, scale_one)
loc_one = tf.cast(loc_one, tf.float32)
scale_one = tf.cast(scale_one, tf.float32)
if loc_two == 0.0 and scale_two == 1.0:
# With default arguments, we can avoid some intermediate computation.
out = tf.square(scale_one) + tf.square(loc_one) - \
1.0 - 2.0 * tf.log(scale_one)
else:
loc_two = control_flow_ops.with_dependencies(dependencies, loc_two)
scale_two = control_flow_ops.with_dependencies(dependencies, scale_two)
loc_two = tf.cast(loc_two, tf.float32)
scale_two = tf.cast(scale_two, tf.float32)
out = tf.square(scale_one/scale_two) + \
tf.square((loc_two - loc_one)/scale_two) - \
1.0 + 2.0 * tf.log(scale_two) - 2.0 * tf.log(scale_one)
if len(out.get_shape()) <= 1: # scalar or vector
return 0.5 * tf.reduce_sum(out)
else: # matrix
return 0.5 * tf.reduce_sum(out, 1)
def calc_reward(outputs):
outputs = outputs[-1] # look at ONLY THE END of the sequence
outputs = tf.reshape(outputs, (batch_size, cell_out_size))
h_a_out = weight_variable((cell_out_size, n_classes))
p_y = tf.nn.softmax(tf.matmul(outputs, h_a_out))
max_p_y = tf.arg_max(p_y, 1)
correct_y = tf.cast(labels_placeholder, tf.int64)
R = tf.cast(tf.equal(max_p_y, correct_y), tf.float32) # reward per example
reward = tf.reduce_mean(R) # overall reward
p_loc = gaussian_pdf(mean_locs, sampled_locs)
p_loc = tf.reshape(p_loc, (batch_size, glimpses * 2))
R = tf.reshape(R, (batch_size, 1))
J = tf.concat(1, [tf.log(p_y + 1e-5) * onehot_labels_placeholder, tf.log(
p_loc + 1e-5) * R])
J = tf.reduce_sum(J, 1)
J = tf.reduce_mean(J, 0)
cost = -J
optimizer = tf.train.AdamOptimizer(lr)
train_op = optimizer.minimize(cost)
return cost, reward, max_p_y, correct_y, train_op
def iou(self, boxes1, boxes2):
"""calculate ious
Args:
boxes1: 4-D tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL, 4] ====> (x_center, y_center, w, h)
boxes2: 1-D tensor [4] ===> (x_center, y_center, w, h)
Return:
iou: 3-D tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL]
"""
boxes1 = tf.stack([boxes1[:, :, :, 0] - boxes1[:, :, :, 2] / 2, boxes1[:, :, :, 1] - boxes1[:, :, :, 3] / 2,
boxes1[:, :, :, 0] + boxes1[:, :, :, 2] / 2, boxes1[:, :, :, 1] + boxes1[:, :, :, 3] / 2])
boxes1 = tf.transpose(boxes1, [1, 2, 3, 0])
boxes2 = tf.stack([boxes2[0] - boxes2[2] / 2, boxes2[1] - boxes2[3] / 2,
boxes2[0] + boxes2[2] / 2, boxes2[1] + boxes2[3] / 2])
#calculate the left up point
lu = tf.maximum(boxes1[:, :, :, 0:2], boxes2[0:2])
rd = tf.minimum(boxes1[:, :, :, 2:], boxes2[2:])
#intersection
intersection = rd - lu
inter_square = intersection[:, :, :, 0] * intersection[:, :, :, 1]
mask = tf.cast(intersection[:, :, :, 0] > 0, tf.float32) * tf.cast(intersection[:, :, :, 1] > 0, tf.float32)
inter_square = mask * inter_square
#calculate the boxs1 square and boxs2 square
square1 = (boxes1[:, :, :, 2] - boxes1[:, :, :, 0]) * (boxes1[:, :, :, 3] - boxes1[:, :, :, 1])
square2 = (boxes2[2] - boxes2[0]) * (boxes2[3] - boxes2[1])
return inter_square/(square1 + square2 - inter_square + 1e-6)
def m_body(i, ta_tp, ta_fp, gmatch):
# Jaccard score with groundtruth bboxes.
rbbox = bboxes[i]
jaccard = bboxes_jaccard(rbbox, gbboxes)
jaccard = jaccard * tf.cast(tf.equal(glabels, rlabel), dtype=jaccard.dtype)
# Best fit, checking it's above threshold.
idxmax = tf.cast(tf.argmax(jaccard, axis=0), tf.int32)
jcdmax = jaccard[idxmax]
match = jcdmax > matching_threshold
existing_match = gmatch[idxmax]
not_difficult = tf.logical_not(gdifficults[idxmax])
# TP: match & no previous match and FP: previous match | no match.
# If difficult: no record, i.e FP=False and TP=False.
tp = tf.logical_and(not_difficult,
tf.logical_and(match, tf.logical_not(existing_match)))
ta_tp = ta_tp.write(i, tp)
fp = tf.logical_and(not_difficult,
tf.logical_or(existing_match, tf.logical_not(match)))
ta_fp = ta_fp.write(i, fp)
# Update grountruth match.
mask = tf.logical_and(tf.equal(grange, idxmax),
tf.logical_and(not_difficult, match))
gmatch = tf.logical_or(gmatch, mask)
return [i+1, ta_tp, ta_fp, gmatch]
def read_cifar_files(filename_queue, distort_images = True):
reader = tf.FixedLengthRecordReader(record_bytes=record_length)
key, record_string = reader.read(filename_queue)
record_bytes = tf.decode_raw(record_string, tf.uint8)
image_label = tf.cast(tf.slice(record_bytes, [0], [1]), tf.int32)
# Extract image
image_extracted = tf.reshape(tf.slice(record_bytes, [1], [image_vec_length]),
[num_channels, image_height, image_width])
# Reshape image
image_uint8image = tf.transpose(image_extracted, [1, 2, 0])
reshaped_image = tf.cast(image_uint8image, tf.float32)
# Randomly Crop image
final_image = tf.image.resize_image_with_crop_or_pad(reshaped_image, crop_width, crop_height)
if distort_images:
# Randomly flip the image horizontally, change the brightness and contrast
final_image = tf.image.random_flip_left_right(final_image)
final_image = tf.image.random_brightness(final_image,max_delta=63)
final_image = tf.image.random_contrast(final_image,lower=0.2, upper=1.8)
# Normalize whitening
final_image = tf.image.per_image_standardization(final_image)
return(final_image, image_label)
def _fn(*args):
p = tf.identity(proposal_log_prob_fn(*args), name="proposal_log_prob")
t = tf.identity(target_log_prob_fn(*args), name="target_log_prob")
dtype = p.dtype.base_dtype
beta = tf.cast(iter_ + 1, dtype) / tf.cast(num_steps, dtype)
return tf.identity(beta * t + (1. - beta) * p,
name="convex_combined_log_prob")
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'image_raw': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64),
'height': tf.FixedLenFeature([], tf.int64),
'width': tf.FixedLenFeature([], tf.int64),
'depth': tf.FixedLenFeature([], tf.int64)
})
image = tf.decode_raw(features['image_raw'], tf.uint8)
img_height = tf.cast(features['height'], tf.int32)
img_width = tf.cast(features['width'], tf.int32)
img_depth = tf.cast(features['depth'], tf.int32)
# Convert label from a scalar uint8 tensor to an int32 scalar.
label = tf.cast(features['label'], tf.int32)
image.set_shape([IMG_PIXELS])
image = tf.reshape(image, [IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS])
# Convert from [0, 255] -> [-0.5, 0.5] floats.
image = tf.cast(image, tf.float32) * (1. / 255) - 0.5
return image, label
def drawGraph(self, n_row, n_latent, n_col):
with tf.name_scope('matDecomp'):
self._p = tf.placeholder(tf.float32, shape=[None, n_col])
self._c = tf.placeholder(tf.float32, shape=[None, n_col])
self._lambda = tf.placeholder(tf.float32)
self._index = tf.placeholder(tf.float32, shape=[None, n_row])
self._A = tf.Variable(tf.truncated_normal([n_row, n_latent]))
self._B = tf.Variable(tf.truncated_normal([n_latent, n_col]))
self._h = tf.matmul(tf.matmul(self._index, self._A), self._B)
weighted_loss = tf.reduce_mean(tf.mul(self._c, tf.squared_difference(self._p, self._h)))
self._weighted_loss = weighted_loss
l2_A = tf.reduce_sum(tf.square(self._A))
l2_B = tf.reduce_sum(tf.square(self._B))
n_w = tf.constant(n_row * n_latent + n_latent * n_col, tf.float32)
l2 = tf.truediv(tf.add(l2_A, l2_B), n_w)
reg_term = tf.mul(self._lambda, l2)
self._loss = tf.add(weighted_loss, reg_term)
self._mask = tf.placeholder(tf.float32, shape=[n_row, n_col])
one = tf.constant(1, tf.float32)
pred = tf.cast(tf.greater_equal(tf.matmul(self._A, self._B), one), tf.float32)
cor = tf.mul(tf.cast(tf.equal(pred, self._p), tf.float32), self._c)
self._vali_err = tf.reduce_sum(tf.mul(cor, self._mask))
self._saver = tf.train.Saver([v for v in tf.all_variables() if v.name.find('matDecomp') != -1])
tf.scalar_summary('training_weighted_loss_l2', self._loss)
tf.scalar_summary('validation_weighted_loss', self._weighted_loss)
merged = tf.merge_all_summaries()
def compute_accuracy(y_hat, labels, sparse=False):
"""Compute accuracy for a 3-dimensional outputs.
The prediction is assumed to be made by argmax.
Parameters
----------
y_hat : tensor, shape (batch_size, n_samples, n_outputs)
Raw predictions of a neural network. It is not required to convert it
to softmax, because softmax is a monotonous transform.
labels : tensor
True labels. It can have shape (batch_size, n_samples), then each
values should be an index within [0, n_classes). Or alternatively
it can have shape (batch_size, n_samples, n_outputs), then for each
sample a probability distribution with n_outputs values should be
provided (this case also handles one-hot label encoding). In the
latter case the correct label is also selected by argmax. Set `sparse`
parameter to select an appropriate setting.
sparse : bool, default False
Whether `labels` are indices or full distributions.
Returns
-------
accuracy : scalar tensor
Computed accuracy.
"""
prediction = tf.arg_max(y_hat, 2)
if sparse:
labels = tf.cast(labels, prediction.dtype)
else:
labels = tf.arg_max(labels, 2)
return tf.reduce_mean(tf.cast(tf.equal(prediction, labels), tf.float32))
def _add_layer( self, input, n_in, n_out, activation=None, weights=None, bias=None, dropout=None, l2_reg=False ):
if( weights is None ):
''' Xavier init '''
init_range = math.sqrt(6.0 / (n_in + n_out))
init_w = tf.random_uniform( [n_in,n_out], -init_range, init_range)
weights = tf.cast( tf.Variable( init_w ), tf.float32 )
self.weights.append( weights )
if( bias is None ):
bias = tf.cast( tf.Variable( tf.zeros( [ n_out ] ) ), tf.float32 )
self.bias.append( bias )
if( l2_reg ):
''' L2 regularization '''
l2_reg = tf.nn.l2_loss( weights )
self.l2_reg += l2_reg
layer = tf.matmul( input, weights ) + bias
if( activation is not None ):
layer = activation( layer )
if( dropout is not None ):
''' Dropout + scaling '''
layer = tf.nn.dropout( layer, 1-dropout ) * 1/( 1- dropout )
return layer
def ValidArcAndTokenMasks(lengths, max_length, dtype=tf.float32):
r"""Returns 0/1 masks for valid arcs and tokens.
Args:
lengths: [B] vector of input sequence lengths.
max_length: Scalar maximum input sequence length, aka M.
dtype: Data type for output mask.
Returns:
[B,M,M] tensor A with 0/1 indicators of valid arcs. Specifically,
A_{b,t,s} = t,s < lengths[b] ? 1 : 0
[B,M] matrix T with 0/1 indicators of valid tokens. Specifically,
T_{b,t} = t < lengths[b] ? 1 : 0
"""
lengths_bx1 = tf.expand_dims(lengths, 1)
sequence_m = tf.range(tf.cast(max_length, lengths.dtype.base_dtype))
sequence_1xm = tf.expand_dims(sequence_m, 0)
# Create vectors of 0/1 indicators for valid tokens. Note that the comparison
# operator will broadcast from [1,M] and [B,1] to [B,M].
valid_token_bxm = tf.cast(sequence_1xm < lengths_bx1, dtype)
# Compute matrices of 0/1 indicators for valid arcs as the outer product of
# the valid token indicator vector with itself.
valid_arc_bxmxm = tf.matmul(
tf.expand_dims(valid_token_bxm, 2), tf.expand_dims(valid_token_bxm, 1))
return valid_arc_bxmxm, valid_token_bxm
def _smallest_size_at_least(height, width, smallest_side):
"""Computes new shape with the smallest side equal to `smallest_side`.
Computes new shape with the smallest side equal to `smallest_side` while
preserving the original aspect ratio.
Args:
height: an int32 scalar tensor indicating the current height.
width: an int32 scalar tensor indicating the current width.
smallest_side: A python integer or scalar `Tensor` indicating the size of
the smallest side after resize.
Returns:
new_height: an int32 scalar tensor indicating the new height.
new_width: an int32 scalar tensor indicating the new width.
"""
smallest_side = tf.cast(smallest_side, tf.float32)
height = tf.cast(height, tf.float32)
width = tf.cast(width, tf.float32)
smaller_dim = tf.minimum(height, width)
scale_ratio = smallest_side / smaller_dim
new_height = tf.cast(height * scale_ratio, tf.int32)
new_width = tf.cast(width * scale_ratio, tf.int32)
return new_height, new_width
def __init__(self,
sess,
batch_size,
max_step,
n_layers,
n_hidden,
n_cmd_type,
num_actor_vars,
dim_emb=64,
dim_laser=[666, 3],
dim_goal=2,
dim_cmd=1,
dim_action=2,
gpu_num=1,
tau=0.1,
learning_rate=1e-3):
self.sess = sess
self.max_step = max_step
self.batch_size = batch_size
self.n_layers = n_layers
self.n_hidden = n_hidden
self.n_cmd_type = n_cmd_type
self.dim_emb = dim_emb
self.dim_laser = dim_laser
self.dim_goal = dim_goal
self.dim_cmd = dim_cmd
self.dim_action = dim_action
self.gpu_num = gpu_num
self.tau = tau
self.num_actor_vars = num_actor_vars
self.learning_rate = learning_rate
with tf.variable_scope('critic'):
# training input
self.input_laser = tf.placeholder(
tf.float32,
shape=[None, dim_laser[0], dim_laser[1]],
name='input_laser')
self.input_cmd = tf.placeholder(tf.int64,
shape=[None, dim_cmd],
name='input_cmd')
self.input_cmd_next = tf.placeholder(tf.int64,
shape=[None, dim_cmd],
name='input_cmd_next')
self.prev_action = tf.placeholder(tf.float32,
shape=[None, dim_action],
name='prev_action')
self.input_obj_goal = tf.placeholder(tf.float32,
shape=[None, dim_goal],
name='input_obj_goal')
self.input_action = tf.placeholder(tf.float32,
shape=[None, dim_action],
name='input_action')
self.length = tf.placeholder(tf.int32, [self.batch_size],
name='length') # b
# build model with multi-gpu parallely
inputs = [
self.input_laser, self.input_cmd, self.input_cmd_next,
self.prev_action, self.input_obj_goal, self.input_action
]
with tf.variable_scope('online'):
self.pred_q = self.Model(inputs)
self.network_params = tf.trainable_variables()[num_actor_vars:]
with tf.variable_scope('target'):
self.q_target = self.Model(inputs)
self.target_network_params = tf.trainable_variables()[(
len(self.network_params) + num_actor_vars):]
self.y = tf.placeholder(tf.float32,
[self.batch_size, self.max_step, 1],
name='y')
self.mask = tf.expand_dims(tf.sequence_mask(self.length,
maxlen=self.max_step,
dtype=tf.float32),
axis=2) # b, l, 1
self.square_diff = tf.pow(
(self.y - tf.reshape(self.pred_q,
(self.batch_size, self.max_step, 1))) *
self.mask, 2) # b, l, 1
self.loss_t = tf.reduce_sum(self.square_diff,
reduction_indices=1) / tf.cast(
self.length, tf.float32) # b, 1
self.loss_n = tf.reduce_sum(self.loss_t,
reduction_indices=0) / self.batch_size # 1
self.gradient = tf.gradients(self.loss_n, self.network_params)
self.opt = tf.train.AdamOptimizer(self.learning_rate)
self.optimize = self.opt.apply_gradients(
zip(self.gradient, self.network_params))
mask_reshape = tf.reshape(
self.mask, (self.batch_size * self.max_step, 1)) # b*l, 1
self.a_gradient_mask = tf.tile(mask_reshape, [1, 2]) # b*l, 2
self.action_grads = tf.gradients(
self.pred_q, self.input_action) * self.a_gradient_mask
# Op for periodically updating target network with online network weights
self.update_target_network_params = \
[self.target_network_params[i].assign(tf.multiply(self.network_params[i], self.tau) + \
tf.multiply(self.target_network_params[i], 1. - self.tau))
for i in range(len(self.target_network_params))]
def _GuidedBackProp(op, grad):
dtype = op.inputs[0].dtype
return grad * tf.cast(grad > 0., dtype)*tf.cast(op.inputs[0] > 0., dtype)
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# output layer: softmax -----------------------------------------------------------------------
W_fc2 = weight_varible([1024, 10])
b_fc2 = bias_variable([10])
y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
y_ = tf.placeholder(tf.float32, [None, 10])
# model training -----------------------------------------------------------------------
cross_entropy = -tf.reduce_sum(y_ * tf.log(y_conv))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.arg_max(y_conv, 1), tf.arg_max(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
sess.run(tf.initialize_all_variables())
for i in range(300):
batch = mnist.train.next_batch(50)
if i % 100 == 0:
train_accuacy = accuracy.eval(feed_dict={
x: batch[0],
y_: batch[1],
keep_prob: 1.0
})
print("step %d, training accuracy %g" % (i, train_accuacy))
train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
def main(args):
mnist = input_data.read_data_sets("data/",
reshape=False,
one_hot=True,
validation_size=0)
learning_rate = 0.001
training_iterations = 5000
batch_size = 100
display_iterations = 250
drop_out_rate = 0.4
input_layer = tf.placeholder(tf.float32, [None, 28, 28, 1])
# input_layer = tf.reshape(raw_data, [-1, 28, 28, 1])
output_layer = tf.placeholder(tf.float32, [None, 10])
training = tf.placeholder(tf.bool)
conv1 = tf.layers.conv2d(inputs=input_layer,
filters=32,
kernel_size=[5, 5],
padding="same",
activation=tf.nn.relu)
norm1 = tf.layers.batch_normalization(conv1)
pool1 = tf.layers.max_pooling2d(inputs=norm1, pool_size=[2, 2], strides=2)
conv2 = tf.layers.conv2d(inputs=pool1,
filters=64,
kernel_size=[5, 5],
padding="same",
activation=tf.nn.relu)
norm2 = tf.layers.batch_normalization(conv2)
pool2 = tf.layers.max_pooling2d(inputs=norm2, pool_size=[2, 2], strides=2)
pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])
dense1 = tf.layers.dense(inputs=pool2_flat,
units=1024,
activation=tf.nn.relu)
dropout = tf.layers.dropout(inputs=dense1,
rate=drop_out_rate,
training=training)
dense2 = tf.layers.dense(inputs=dropout, units=1024, activation=tf.nn.relu)
logits = tf.layers.dense(inputs=dense2, units=10)
with tf.name_scope('Model'):
predictions = tf.nn.softmax(logits)
with tf.name_scope('Loss'):
cost = tf.reduce_mean(-tf.reduce_sum(
output_layer * tf.log(predictions), reduction_indices=1))
with tf.name_scope('Accuracy'):
accuracy = tf.equal(tf.argmax(predictions, 1),
tf.argmax(output_layer, 1))
accuracy = tf.reduce_mean(tf.cast(accuracy, tf.float32))
with tf.name_scope('SGD'):
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(
cost)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
avg_cost = 0.
for iteration in range(training_iterations):
batch_xs, batch_ys = mnist.train.next_batch(batch_size)
_, c = sess.run([optimizer, cost],
feed_dict={
input_layer: batch_xs,
output_layer: batch_ys,
training: True
})
avg_cost += c / display_iterations
if (iteration + 1) % display_iterations == 0:
tf.logging.info("Iteration: %s\tcost=%s", (iteration + 1),
avg_cost)
avg_cost = 0.
tf.logging.info(
"Accuracy: %s",
accuracy.eval({
input_layer: mnist.test.images,
output_layer: mnist.test.labels,
training: False
}))
def train():
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
global_step,
DECAY_STEPS,
LEARNING_RATE_DECAY_FACTOR,
staircase=True)
logits, inputs, targets, seq_len, W, b = get_train_model()
loss = tf.nn.ctc_loss(labels=targets,
inputs=logits,
sequence_length=seq_len)
cost = tf.reduce_mean(loss)
#optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,momentum=MOMENTUM).minimize(cost, global_step=global_step)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(
loss, global_step=global_step)
decoded, log_prob = tf.nn.ctc_beam_search_decoder(logits,
seq_len,
merge_repeated=False)
acc = tf.reduce_mean(
tf.edit_distance(tf.cast(decoded[0], tf.int32), targets))
init = tf.global_variables_initializer()
def do_report():
test_inputs, test_targets, test_seq_len = get_next_batch(BATCH_SIZE)
test_feed = {
inputs: test_inputs,
targets: test_targets,
seq_len: test_seq_len
}
dd, log_probs, accuracy = session.run([decoded[0], log_prob, acc],
test_feed)
report_accuracy(dd, test_targets)
# decoded_list = decode_sparse_tensor(dd)
def do_batch():
train_inputs, train_targets, train_seq_len = get_next_batch(BATCH_SIZE)
feed = {
inputs: train_inputs,
targets: train_targets,
seq_len: train_seq_len
}
b_loss, b_targets, b_logits, b_seq_len, b_cost, steps, _ = session.run(
[loss, targets, logits, seq_len, cost, global_step, optimizer],
feed)
#print b_loss
#print b_targets, b_logits, b_seq_len
print b_cost, steps
if steps > 0 and steps % REPORT_STEPS == 0:
do_report()
#save_path = saver.save(session, "ocr.model", global_step=steps)
# print(save_path)
return b_cost, steps
with tf.Session() as session:
session.run(init)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=100)
for curr_epoch in xrange(num_epochs):
print("Epoch.......", curr_epoch)
train_cost = train_ler = 0
for batch in xrange(BATCHES):
start = time.time()
c, steps = do_batch()
train_cost += c * BATCH_SIZE
seconds = time.time() - start
print("Step:", steps, ", batch seconds:", seconds)
train_cost /= TRAIN_SIZE
train_inputs, train_targets, train_seq_len = get_next_batch(
BATCH_SIZE)
val_feed = {
inputs: train_inputs,
targets: train_targets,
seq_len: train_seq_len
}
val_cost, val_ler, lr, steps = session.run(
[cost, acc, learning_rate, global_step], feed_dict=val_feed)
log = "Epoch {}/{}, steps = {}, train_cost = {:.3f}, train_ler = {:.3f}, val_cost = {:.3f}, val_ler = {:.3f}, time = {:.3f}s, learning_rate = {}"
print(
log.format(curr_epoch + 1, num_epochs, steps, train_cost,
train_ler, val_cost, val_ler,
time.time() - start, lr))
def distorted_inputs(filenames, batch_size):
"""Construct distorted input for training using the Reader ops.
Args:
:param filenames: list - [str1, str2, ...].
:param batch_size: int.
Returns:
:returns: tuple - (patches_x, patches_y, labels).
patches_x, patches_y: tensors - (batch_size*num_patches, patch_size, patch_size, depth).
lables: tensors - (batch_size).
"""
for f in filenames:
if not tf.gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
with tf.variable_scope('input'):
# Create a queue that produces the filenames to read.
filename_queue = tf.train.string_input_producer(string_tensor=filenames)
# Even when reading in multiple threads, share the filename queue.
result = read_and_decode(filename_queue)
# OPTIONAL: Could reshape into a image and apply distortionshere.
reshaped_image_x = tf.reshape(result.image_x, [HEIGHT, WIDTH, DEPTH])
reshaped_image_y = tf.reshape(result.image_y, [HEIGHT, WIDTH, DEPTH])
# Convert from [0, 255] -> [-0.5, 0.5] floats.
distorted_image_x = tf.cast(reshaped_image_x, tf.float32) * (1. / 255) - 0.5
distorted_image_y = tf.cast(reshaped_image_y, tf.float32) * (1. / 255) - 0.5
# # Randomly flip the image horizontally.
# distorted_image = tf.image.random_flip_left_right(distorted_image)
#
# # Because these operations are not commutative, consider randomizing
# # the order their operation.
# distorted_image = tf.image.random_brightness(distorted_image,
# max_delta=63)
# distorted_image = tf.image.random_contrast(distorted_image,
# lower=0.2, upper=1.8)
#
# # Subtract off the mean and divide by the variance of the pixels.
# distorted_image = tf.image.per_image_standardization(distorted_image)
#
# # Set the shapes of tensors.
# distorted_image.set_shape([patch_size, patch_size, result.depth])
# result.label.set_shape([1])
label = result.label
# Ensure that the random shuffling has good mixing properties.
min_queue_examples = 1000
# print('Filling queue with %d mnist images before starting to train or validation. '
# 'This will take a few minutes.' % min_queue_examples)
# Generate a batch of images and labels by building up a queue of examples.
images_x, images_y, labels = \
_generate_image_and_label_batch(image=(distorted_image_x, distorted_image_y), label=label,
min_queue_examples=min_queue_examples,
batch_size=batch_size,
shuffle=True)
# Random crop patches from images
patches_x, patches_y = random_sample(images_x, images_y, PATCH_SIZE, NUM_PATCHES_PER_IMAGE)
# Display the training images in the visualizer.
#tf.summary.image('patches_x', tensor=patches_x, max_outputs=4)
#tf.summary.image('patches_y', tensor=patches_y, max_outputs=4)
return patches_x, patches_y, labels
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
# optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
with tf.control_dependencies(extra_update_ops):
training_op = optimizer.minimize(loss=loss) # minimize 负责计算梯度
# 应用梯度裁剪 (在递归神经网络中 非常有用)
# threshhold = 1
# grads_and_vars = optimizer.compute_gradients(loss=loss)
# capped_gvs = [(tf.clip_by_value(grad,-threshhold,threshhold),var)for grad,var in grads_and_vars]
# training_op = optimizer.apply_gradients(capped_gvs)
#
# 评估模型 精度
with tf.name_scope('eval') :
correct = tf.nn.in_top_k(predictions=logists, targets=y, k=1)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
'''
执行阶段
'''
# 加载数据
from tensorflow.examples.tutorials.mnist import input_data
from datetime import datetime
# tensorboard 可视化
now = datetime.utcnow().strftime("%Y%m%d%H%M%S")
root_logdir = r"D://tf_logs"
logdir = "{}/run-{}/".format(root_logdir, now)
xentropy_summary = tf.summary.scalar('xentropy', loss)
file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())
def model_fn(features, labels, mode, params): # pylint: disable=unused-argument
"""The `model_fn` for TPUEstimator."""
tf.logging.info("*** Features ***")
for name in sorted(features.keys()):
tf.logging.info(" name = %s, shape = %s" %
(name, features[name].shape))
input_ids = features["input_ids"]
input_mask = features["input_mask"]
segment_ids = features["segment_ids"]
label_ids = features["label_ids"]
is_real_example = None
if "is_real_example" in features:
is_real_example = tf.cast(features["is_real_example"],
dtype=tf.float32)
else:
is_real_example = tf.ones(tf.shape(label_ids), dtype=tf.float32)
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
(total_loss, per_example_loss, logits,
probabilities) = create_model(bert_config, is_training, input_ids,
input_mask, segment_ids, label_ids,
num_labels, use_one_hot_embeddings)
tvars = tf.trainable_variables()
initialized_variable_names = {}
scaffold_fn = None
if init_checkpoint:
(assignment_map, initialized_variable_names
) = modeling.get_assignment_map_from_checkpoint(
tvars, init_checkpoint)
if use_tpu:
def tpu_scaffold():
tf.train.init_from_checkpoint(init_checkpoint,
assignment_map)
return tf.train.Scaffold()
scaffold_fn = tpu_scaffold
else:
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
tf.logging.info("**** Trainable Variables ****")
for var in tvars:
init_string = ""
if var.name in initialized_variable_names:
init_string = ", *INIT_FROM_CKPT*"
tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape,
init_string)
output_spec = None
if mode == tf.estimator.ModeKeys.TRAIN:
train_op = optimization.create_optimizer(total_loss, learning_rate,
num_train_steps,
num_warmup_steps, use_tpu)
output_spec = tf.contrib.tpu.TPUEstimatorSpec(
mode=mode,
loss=total_loss,
train_op=train_op,
scaffold_fn=scaffold_fn)
elif mode == tf.estimator.ModeKeys.EVAL:
def metric_fn(per_example_loss, label_ids, logits,
is_real_example):
predictions = tf.argmax(logits, axis=-1, output_type=tf.int32)
accuracy = tf.metrics.accuracy(labels=label_ids,
predictions=predictions,
weights=is_real_example)
loss = tf.metrics.mean(values=per_example_loss,
weights=is_real_example)
return {
"eval_accuracy": accuracy,
"eval_loss": loss,
}
eval_metrics = (metric_fn, [
per_example_loss, label_ids, logits, is_real_example
])
output_spec = tf.contrib.tpu.TPUEstimatorSpec(
mode=mode,
loss=total_loss,
eval_metrics=eval_metrics,
scaffold_fn=scaffold_fn)
else:
output_spec = tf.contrib.tpu.TPUEstimatorSpec(
mode=mode,
predictions={"probabilities": probabilities},
scaffold_fn=scaffold_fn)
return output_spec
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)) as sess:
sess.run(init)
#Training cycle
for epoch in range(training_epochs):
avg_cost = 0.
total_batch = int(len(Y_train)/batch_size)
#Loop oveer all batches
for i in range(total_batch):
batch_x_aud, batch_x_img, batch_ys, finish = data.next_batch_multi(X_aud_train, X_img_train, Y_train, batch_size, len(Y_train))
# Fit traning using batch data
sess.run(optimizer, feed_dict = {x_aud: batch_x_aud, x_img: batch_x_img, y: batch_ys, keep_prob: dropout})
# Compute average loss
avg_cost += sess.run(cost, feed_dict = {x_aud: batch_x_aud, x_img: batch_x_img, y: batch_ys, keep_prob: 1.}) / total_batch
#Shuffling
if finish:
p = np.random.permutation(len(Y_train))
X_aud_train = X_aud_train[p]
X_img_train = X_img_train[p]
Y_train = Y_train[p]
# Display logs per epoch step
if epoch % display_step == 0:
print "Epoch:", '%04d' % (epoch+1), "cost=", "{:.9f}".format(avg_cost)
print "Optimization Finished!"
# Test model
correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
# Calculate accuracy
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print "Accuracy:", accuracy.eval({x_aud: X_aud_test, x_img: X_img_test, y: Y_test, keep_prob: 1.})
print 'MM.py'
def call(self, net, training):
keep_prob = self.keep_prob
dropblock_size = self.dropblock_size
data_format = self.data_format
if not training or keep_prob is None:
return net
tf.logging.info(
"Applying DropBlock: dropblock_size {}, net.shape {}".format(
dropblock_size, net.shape))
if data_format == "channels_last":
_, width, height, _ = net.get_shape().as_list()
else:
_, _, width, height = net.get_shape().as_list()
if width != height:
raise ValueError(
"Input tensor with width!=height is not supported.")
dropblock_size = min(dropblock_size, width)
# seed_drop_rate is the gamma parameter of DropBlcok.
seed_drop_rate = ((1.0 - keep_prob) * width**2 / dropblock_size**2 /
(width - dropblock_size + 1)**2)
# Forces the block to be inside the feature map.
w_i, h_i = tf.meshgrid(tf.range(width), tf.range(width))
valid_block_center = tf.logical_and(
tf.logical_and(w_i >= int(dropblock_size // 2),
w_i < width - (dropblock_size - 1) // 2),
tf.logical_and(h_i >= int(dropblock_size // 2),
h_i < width - (dropblock_size - 1) // 2),
)
valid_block_center = tf.expand_dims(valid_block_center, 0)
valid_block_center = tf.expand_dims(
valid_block_center, -1 if data_format == "channels_last" else 0)
randnoise = tf.random_uniform(net.shape, dtype=tf.float32)
block_pattern = (
1 - tf.cast(valid_block_center, dtype=tf.float32) + tf.cast(
(1 - seed_drop_rate), dtype=tf.float32) + randnoise) >= 1
block_pattern = tf.cast(block_pattern, dtype=tf.float32)
if dropblock_size == width:
block_pattern = tf.reduce_min(
block_pattern,
axis=[1, 2] if data_format == "channels_last" else [2, 3],
keepdims=True,
)
else:
if data_format == "channels_last":
ksize = [1, dropblock_size, dropblock_size, 1]
else:
ksize = [1, 1, dropblock_size, dropblock_size]
block_pattern = -tf.nn.max_pool(
-block_pattern,
ksize=ksize,
strides=[1, 1, 1, 1],
padding="SAME",
data_format="NHWC"
if data_format == "channels_last" else "NCHW",
)
percent_ones = tf.cast(tf.reduce_sum(
(block_pattern)), tf.float32) / tf.cast(tf.size(block_pattern),
tf.float32)
net = net / tf.cast(percent_ones, net.dtype) * tf.cast(
block_pattern, net.dtype)
return net
def perprocess(x, y):
x = tf.cast(x, dtype=tf.float32) / 255.0
x = tf.reshape(x, [-1, 28 * 28])
y = tf.cast(y, dtype=tf.int32)
y = tf.one_hot(y, depth=num_classes)
return x, y
support = [preprocess_adj(adj)]
t_features = tf.SparseTensor(*features)
t_y_train = tf.convert_to_tensor(y_train)
t_y_val = tf.convert_to_tensor(y_val)
t_y_test = tf.convert_to_tensor(y_test)
tm_train_mask = tf.convert_to_tensor(train_mask)
tm_val_mask = tf.convert_to_tensor(val_mask)
tm_test_mask = tf.convert_to_tensor(test_mask)
t_support = []
for i in range(len(support)):
t_support.append(tf.cast(tf.SparseTensor(*support[i]), dtype=tf.float64))
# Create model
model = GCN(input_dim=features[2][1], output_dim=y_train.shape[1], num_features_nonzero=features[1].shape)
# Loss and optimizer
optimizer = optimizers.Adam(lr=cfg.learning_rate)
cost_val = []
for epoch in range(cfg.epochs):
t = time.time()
def parse_example_proto(example_serialized):
"""Parses an Example proto containing a training example of an image.
The output of the build_image_data.py image preprocessing script is a dataset
containing serialized Example protocol buffers. Each Example proto contains
the following fields:
image/height: 462
image/width: 581
image/colorspace: 'RGB'
image/channels: 3
image/class/label: 615
image/class/synset: 'n03623198'
image/class/text: 'knee pad'
image/object/bbox/xmin: 0.1
image/object/bbox/xmax: 0.9
image/object/bbox/ymin: 0.2
image/object/bbox/ymax: 0.6
image/object/bbox/label: 615
image/format: 'JPEG'
image/filename: 'ILSVRC2012_val_00041207.JPEG'
image/encoded: <JPEG encoded string>
Args:
example_serialized: scalar Tensor tf.string containing a serialized
Example protocol buffer.
Returns:
image_buffer: Tensor tf.string containing the contents of a JPEG file.
label: Tensor tf.int32 containing the label.
bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]
where each coordinate is [0, 1) and the coordinates are arranged as
[ymin, xmin, ymax, xmax].
text: Tensor tf.string containing the human-readable label.
"""
# Dense features in Example proto.
feature_map = {
'image/encoded': tf.FixedLenFeature([], dtype=tf.string,
default_value=''),
'image/class/label': tf.FixedLenFeature([1], dtype=tf.int64,
default_value=-1),
'image/class/text': tf.FixedLenFeature([], dtype=tf.string,
default_value=''),
}
sparse_float32 = tf.VarLenFeature(dtype=tf.float32)
# Sparse features in Example proto.
feature_map.update(
{k: sparse_float32 for k in ['image/object/bbox/xmin',
'image/object/bbox/ymin',
'image/object/bbox/xmax',
'image/object/bbox/ymax']})
features = tf.parse_single_example(example_serialized, feature_map)
label = tf.cast(features['image/class/label'], dtype=tf.int32)
xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0)
ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 0)
xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 0)
ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 0)
# Note that we impose an ordering of (y, x) just to make life difficult.
bbox = tf.concat([ymin, xmin, ymax, xmax], 0)
# Force the variable number of bounding boxes into the shape
# [1, num_boxes, coords].
bbox = tf.expand_dims(bbox, 0)
bbox = tf.transpose(bbox, [0, 2, 1])
return features['image/encoded'], label, bbox, features['image/class/text']
global_step,
steps_per_epoch * epochs_per_decay,
decay_factor,
staircase=staircase)
with tf.name_scope('inputs') as scope:
image, label = read_and_decode_single_example(train_files, label_type="label_normal", normalize=False)
X_def, y_def = tf.train.shuffle_batch([image, label], batch_size=batch_size, capacity=2000,
min_after_dequeue=1000)
# Placeholders
X = tf.placeholder_with_default(X_def, shape=[None, 299, 299, 1])
y = tf.placeholder_with_default(y_def, shape=[None])
X = tf.cast(X, dtype=tf.float32)
# Input stem
conv1 = _conv2d_batch_norm(X, filters=32, stride=(1, 1), training=training, padding="VALID", name="1.1")
# Max pooling layer 1
with tf.name_scope('pool1') as scope:
pool1 = tf.layers.max_pooling2d(
conv1, # Input
pool_size=(2, 2),
strides=(2, 2),
padding='VALID',
name='pool1'
)
# Layer 2 branch 1
def eval_image(image,
height,
width,
batch_position,
resize_method,
summary_verbosity=0):
"""Get the image for model evaluation.
We preprocess the image simiarly to Slim, see
https://github.com/tensorflow/models/blob/master/slim/preprocessing/vgg_preprocessing.py
Validation images do not have bounding boxes, so to crop the image, we first
resize the image such that the aspect ratio is maintained and the resized
height and width are both at least 1.15 times `height` and `width`
respectively. Then, we do a central crop to size (`height`, `width`).
Args:
image: 3-D float Tensor representing the image.
height: The height of the image that will be returned.
width: The width of the image that will be returned.
batch_position: position of the image in a batch, which affects how images
are distorted and resized. NOTE: this argument can be an integer or a
tensor
resize_method: one of the strings 'round_robin', 'nearest', 'bilinear',
'bicubic', or 'area'.
summary_verbosity: Verbosity level for summary ops. Pass 0 to disable both
summaries and checkpoints.
Returns:
An image of size (output_height, output_width, 3) that is resized and
cropped as described above.
"""
# TODO(reedwm): Currently we resize then crop. Investigate if it's faster to
# crop then resize.
with tf.name_scope('eval_image'):
if summary_verbosity >= 3:
tf.summary.image(
'original_image', tf.expand_dims(image, 0))
shape = tf.shape(image)
image_height = shape[0]
image_width = shape[1]
image_height_float = tf.cast(image_height, tf.float32)
image_width_float = tf.cast(image_width, tf.float32)
scale_factor = 1.15
# Compute resize_height and resize_width to be the minimum values such that
# 1. The aspect ratio is maintained (i.e. resize_height / resize_width is
# image_height / image_width), and
# 2. resize_height >= height * `scale_factor`, and
# 3. resize_width >= width * `scale_factor`
max_ratio = tf.maximum(height / image_height_float,
width / image_width_float)
resize_height = tf.cast(image_height_float * max_ratio * scale_factor,
tf.int32)
resize_width = tf.cast(image_width_float * max_ratio * scale_factor,
tf.int32)
# Resize the image to shape (`resize_height`, `resize_width`)
image_resize_method = get_image_resize_method(resize_method, batch_position)
distorted_image = tf.image.resize_images(image,
[resize_height, resize_width],
image_resize_method,
align_corners=False)
# Do a central crop of the image to size (height, width).
total_crop_height = (resize_height - height)
crop_top = total_crop_height // 2
total_crop_width = (resize_width - width)
crop_left = total_crop_width // 2
distorted_image = tf.slice(distorted_image, [crop_top, crop_left, 0],
[height, width, 3])
distorted_image.set_shape([height, width, 3])
if summary_verbosity >= 3:
tf.summary.image(
'cropped_resized_image', tf.expand_dims(distorted_image, 0))
image = distorted_image
return image
loss = criterion(batchy, out) # 注意真实值在前，预测值在后
grads = tape.gradient(loss, model.trainable_variables) # 对指定param求取梯度
optimizer.apply_gradients(zip(grads, model.trainable_variables)) # 优化器更新梯度
if step % 100 == 0:
train_losses.append(loss.numpy())
if step % 500 == 0:
# 测试
total_correct = 0
for batchx_, batchy_ in test_dataloader:
out_ = model(batchx_)
prob = tf.nn.softmax(out_, axis=1)
pred = tf.argmax(prob, axis=1)
y = tf.argmax(batchy_, axis=1) # 注意次数batchy_是独热编码后的数据，要还原到原始标签
correct = tf.cast(tf.equal(pred, y), dtype=tf.int32)
correct = tf.reduce_sum(correct).numpy()
total_correct += correct
accuracy = total_correct / x_test.shape[0]
test_accuracy.append(accuracy)
print('step:{},accuracy:{}'.format(step, accuracy))
plt.figure()
plt.plot(train_losses)
plt.show()
plt.figure()
plt.plot(test_accuracy)
plt.show()
def train(self):
"""Training code.
"""
args = self.args
self.max_iter = args.num_steps
self.checkpoint_dir = args.snapshot_dir
self.imgflist = read_data_list(os.path.join(args.data_dir, 'img'), args.data_list, '.jpg')
self.labelmapflist = read_data_list(os.path.join(args.data_dir, DIR_ANNOTATION), args.data_list, '.png')
## Image, Labelmap loader
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
loader_img = load_single_image(args, input_size)
caller_imgloader = [loader_img['output_img'], loader_img['output_labelmap']]
## Point sampler
pt_sampler = load_batch_samplepts()
caller_sampler = [pt_sampler['out_batchx2d'], pt_sampler['out_weightmat']]
# Pixel-wise softmax loss.
l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
self.loss = tf.add_n(l2_losses)
# Processed predictions: for visualisation.
pred = tf.cast(tf.image.resize_bilinear(self.netcontainer['out_visimg'], input_size), tf.uint8)
# Image summary.
images_summary = tf.py_func(inv_preprocess, [tf.expand_dims(self.netcontainer['input_img'], dim=0), args.save_num_images, IMG_MEAN], tf.uint8)
total_summary = tf.summary.image('images', tf.concat(axis=2, values=[images_summary, pred]),
max_outputs=args.save_num_images) # Concatenate row-wise.
summary_writer = tf.summary.FileWriter(args.snapshot_dir, graph=tf.get_default_graph())
self.setup_optimizer()
self.step = self.train_container['step_ph']
tf.global_variables_initializer().run()
self.load(self.checkpoint_dir)
start_time = time.time()
start_iter = self.step.eval()
nimg = len(self.imgflist)
# Iterate over training steps.
for step in range(0, args.num_steps):
start_time = time.time()
feed_dict = { self.step : step }
loss_value = 0
# Clear the accumulated gradients.
sess.run(self.train_container['zero_op'], feed_dict=feed_dict)
# Image loading
feed_dict_imgloader = {loader_img['input_img_name']: self.imgflist[step%nimg],
loader_img['input_lbm_name']: self.labelmapflist[step%nimg]}
cur_image, cur_labelmap = sess.run(caller_imgloader, feed_dict=feed_dict_imgloader)
if len(np.unique(cur_labelmap)) < NINST:
continue
print('Loaded image: %s' % self.imgflist[step%nimg])
# Accumulate gradients.
for i in range(args.grad_update_every):
# Point sampling
feed_dict_sampler = {pt_sampler['input_labelmap']: cur_labelmap}
batchx2d, weightmat = sess.run(caller_sampler, feed_dict=feed_dict_sampler)
# print('Sampled %d' % i)
feed_dict_backprob = {self.netcontainer['input_img']: cur_image,
self.netcontainer['input_weightmat']: weightmat,
self.netcontainer['input_samplepts']: batchx2d,
self.step : step}
_, l_val = sess.run([self.train_container['acc_grads_op'], self.loss], feed_dict=feed_dict_backprob)
loss_value += l_val
# Normalise the loss.
loss_value /= args.grad_update_every
# Apply gradients.
if step % args.save_pred_every == 0:
print('Summary')
feed_dict_summary = {self.netcontainer['input_img']: cur_image,
self.netcontainer['input_weightmat']: weightmat,
self.netcontainer['input_samplepts']: batchx2d,
self.step : step}
summary, _ = sess.run([total_summary, self.train_container['train_op']], feed_dict=feed_dict_summary)
summary_writer.add_summary(summary, step)
self.save(self.checkpoint_dir, step)
else:
sess.run(self.train_container['train_op'], feed_dict=feed_dict)
duration = time.time() - start_time
print('step {:d} \t loss = {:.5f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
def train_image(image_buffer,
height,
width,
bbox,
batch_position,
resize_method,
distortions,
scope=None,
summary_verbosity=0,
distort_color_in_yiq=False,
fuse_decode_and_crop=False):
"""Distort one image for training a network.
Distorting images provides a useful technique for augmenting the data
set during training in order to make the network invariant to aspects
of the image that do not effect the label.
Args:
image_buffer: scalar string Tensor representing the raw JPEG image buffer.
height: integer
width: integer
bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]
where each coordinate is [0, 1) and the coordinates are arranged
as [ymin, xmin, ymax, xmax].
batch_position: position of the image in a batch, which affects how images
are distorted and resized. NOTE: this argument can be an integer or a
tensor
resize_method: round_robin, nearest, bilinear, bicubic, or area.
distortions: If true, apply full distortions for image colors.
scope: Optional scope for op_scope.
summary_verbosity: Verbosity level for summary ops. Pass 0 to disable both
summaries and checkpoints.
distort_color_in_yiq: distort color of input images in YIQ space.
fuse_decode_and_crop: fuse the decode/crop operation.
Returns:
3-D float Tensor of distorted image used for training.
"""
# with tf.op_scope([image, height, width, bbox], scope, 'distort_image'):
# with tf.name_scope(scope, 'distort_image', [image, height, width, bbox]):
with tf.name_scope(scope or 'distort_image'):
# A large fraction of image datasets contain a human-annotated bounding box
# delineating the region of the image containing the object of interest. We
# choose to create a new bounding box for the object which is a randomly
# distorted version of the human-annotated bounding box that obeys an
# allowed range of aspect ratios, sizes and overlap with the human-annotated
# bounding box. If no box is supplied, then we assume the bounding box is
# the entire image.
sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
tf.image.extract_jpeg_shape(image_buffer),
bounding_boxes=bbox,
min_object_covered=0.1,
aspect_ratio_range=[0.75, 1.33],
area_range=[0.05, 1.0],
max_attempts=100,
use_image_if_no_bounding_boxes=True)
bbox_begin, bbox_size, distort_bbox = sample_distorted_bounding_box
if summary_verbosity >= 3:
image = tf.image.decode_jpeg(image_buffer, channels=3,
dct_method='INTEGER_FAST')
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
image_with_distorted_box = tf.image.draw_bounding_boxes(
tf.expand_dims(image, 0), distort_bbox)
tf.summary.image(
'images_with_distorted_bounding_box',
image_with_distorted_box)
# Crop the image to the specified bounding box.
if fuse_decode_and_crop:
offset_y, offset_x, _ = tf.unstack(bbox_begin)
target_height, target_width, _ = tf.unstack(bbox_size)
crop_window = tf.stack([offset_y, offset_x, target_height, target_width])
image = tf.image.decode_and_crop_jpeg(
image_buffer, crop_window, channels=3)
else:
image = tf.image.decode_jpeg(image_buffer, channels=3,
dct_method='INTEGER_FAST')
image = tf.slice(image, bbox_begin, bbox_size)
distorted_image = tf.image.random_flip_left_right(image)
# This resizing operation may distort the images because the aspect
# ratio is not respected.
image_resize_method = get_image_resize_method(resize_method, batch_position)
distorted_image = tf.image.resize_images(
distorted_image, [height, width],
image_resize_method,
align_corners=False)
# Restore the shape since the dynamic slice based upon the bbox_size loses
# the third dimension.
distorted_image.set_shape([height, width, 3])
if summary_verbosity >= 3:
tf.summary.image('cropped_resized_maybe_flipped_image',
tf.expand_dims(distorted_image, 0))
if distortions:
distorted_image = tf.cast(distorted_image, dtype=tf.float32)
# Images values are expected to be in [0,1] for color distortion.
distorted_image /= 255.
# Randomly distort the colors.
distorted_image = distort_color(distorted_image, batch_position,
distort_color_in_yiq=distort_color_in_yiq)
# Note: This ensures the scaling matches the output of eval_image
distorted_image *= 255
if summary_verbosity >= 3:
tf.summary.image(
'final_distorted_image',
tf.expand_dims(distorted_image, 0))
return distorted_image
def build_graph(self, x, bboxes_xyz, bboxes_lwh, box3d_pts_label,
semantic_labels, heading_labels, heading_residuals,
size_labels, size_residuals):
# def build_graph(self, x, bboxes_xyz, bboxes_lwh, semantic_labels, heading_labels, heading_residuals, size_labels, size_residuals):
l0_xyz = x
l0_points = None
# Set Abstraction layers
l1_xyz, l1_points, l1_indices = pointnet_sa_module(l0_xyz,
l0_points,
npoint=2048,
radius=0.2,
nsample=64,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
scope='sa1')
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=1024,
radius=0.4,
nsample=64,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
scope='sa2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=512,
radius=0.8,
nsample=64,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
scope='sa3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz,
l3_points,
npoint=256,
radius=1.2,
nsample=64,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
scope='sa4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [256, 256],
scope='fp1')
seeds_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
scope='fp2')
seeds_xyz = l2_xyz
# Voting Module layers
offset = self.hough_voting_mlp(seeds_points)
votes_xyz_points = tf.concat([seeds_xyz, seeds_points], 2) + offset
votes_xyz, votes_points = tf.slice(votes_xyz_points, (0, 0, 0), (-1, -1, 3)), \
tf.slice(votes_xyz_points, (0, 0, 3), (-1, -1, -1))
vote_reg_loss = self.vote_reg_loss(seeds_xyz, votes_xyz, bboxes_xyz,
box3d_pts_label)
# Proposal Module layers
# Farthest point sampling on seeds
proposals_xyz, proposals_output, _ = pointnet_sa_module(
votes_xyz,
votes_points,
npoint=config.PROPOSAL_NUM,
radius=0.3,
nsample=64,
mlp=[128, 128, 128],
mlp2=[128, 128, 5 + 2 * config.NH + 4 * config.NS + config.NC],
group_all=False,
scope='proposal')
nms_iou = tf.get_variable('nms_iou',
shape=[],
initializer=tf.constant_initializer(0.25),
trainable=False)
if not get_current_tower_context().is_training:
def get_3d_bbox(box_size, heading_angle, center):
batch_size = tf.shape(heading_angle)[0]
c = tf.cos(heading_angle)
s = tf.sin(heading_angle)
zeros = tf.zeros_like(c)
ones = tf.ones_like(c)
rotation = tf.reshape(
tf.stack([c, zeros, s, zeros, ones, zeros, -s, zeros, c],
-1), tf.stack([batch_size, -1, 3, 3]))
l, w, h = box_size[..., 0], box_size[..., 1], box_size[
..., 2] # lwh(xzy) order!!!
corners = tf.reshape(
tf.stack([
l / 2, l / 2, -l / 2, -l / 2, l / 2, l / 2, -l / 2,
-l / 2, h / 2, h / 2, h / 2, h / 2, -h / 2, -h / 2,
-h / 2, -h / 2, w / 2, -w / 2, -w / 2, w / 2, w / 2,
-w / 2, -w / 2, w / 2
], -1), tf.stack([batch_size, -1, 3, 8]))
return tf.einsum('ijkl,ijlm->ijmk',
rotation, corners) + tf.expand_dims(
center, 2) # B * N * 8 * 3
class_mean_size_tf = tf.constant(class_mean_size)
size_cls_pred = tf.argmax(
proposals_output[..., 5 + 2 * config.NH:5 + 2 * config.NH +
config.NS],
axis=-1)
size_cls_pred_onehot = tf.one_hot(size_cls_pred,
depth=config.NS,
axis=-1) # B * N * NS
size_residual_pred = tf.reduce_sum(
tf.expand_dims(size_cls_pred_onehot, -1) * tf.reshape(
proposals_output[..., 5 + 2 * config.NH + config.NS:5 +
2 * config.NH + 4 * config.NS],
(-1, config.PROPOSAL_NUM, config.NS, 3)),
axis=2)
size_pred = tf.gather_nd(
class_mean_size_tf,
tf.expand_dims(size_cls_pred, -1)) * tf.maximum(
1 + size_residual_pred, 1e-6) # B * N * 3: size
# with tf.control_dependencies([tf.print(size_pred[0, 0, 2])]):
center_pred = proposals_xyz + proposals_output[...,
2:5] # B * N * 3
heading_cls_pred = tf.argmax(proposals_output[...,
5:5 + config.NH],
axis=-1)
heading_cls_pred_onehot = tf.one_hot(heading_cls_pred,
depth=config.NH,
axis=-1)
heading_residual_pred = tf.reduce_sum(
heading_cls_pred_onehot *
proposals_output[..., 5 + config.NH:5 + 2 * config.NH],
axis=2)
heading_pred = tf.floormod(
(tf.cast(heading_cls_pred, tf.float32) * 2 +
heading_residual_pred) * np.pi / config.NH, 2 * np.pi)
# with tf.control_dependencies([tf.print(size_residual_pred[0, :10, :]), tf.print(size_pred[0, :10, :])]):
bboxes = get_3d_bbox(
size_pred, heading_pred,
center_pred) # B * N * 8 * 3, lhw(xyz) order!!!
# bbox_corners = tf.concat([bboxes[:, :, 6, :], bboxes[:, :, 0, :]], axis=-1) # B * N * 6, lhw(xyz) order!!!
# with tf.control_dependencies([tf.print(bboxes[0, 0])]):
nms_idx = NMS3D(bboxes,
tf.reduce_max(proposals_output[..., -config.NC:],
axis=-1), proposals_output[..., :2],
nms_iou) # Nnms * 2
bboxes_pred = tf.gather_nd(bboxes, nms_idx,
name='bboxes_pred') # Nnms * 8 * 3
class_scores_pred = tf.gather_nd(
proposals_output[..., -config.NC:],
nms_idx,
name='class_scores_pred') # Nnms * C
batch_idx = tf.identity(
nms_idx[:, 0], name='batch_idx'
) # Nnms, this is used to identify between batches
return
# calculate positive and negative proposal idxes
bboxes_xyz_gt = bboxes_xyz # B * BB * 3
bboxes_labels_gt = semantic_labels # B * BB
bboxes_heading_labels_gt = heading_labels
bboxes_heading_residuals_gt = heading_residuals
bboxes_size_labels_gt = size_labels
bboxes_size_residuals_gt = size_residuals
dist_mat = tf.norm(tf.expand_dims(proposals_xyz, 2) -
tf.expand_dims(bboxes_xyz_gt, 1),
axis=-1) # B * PR * BB
bboxes_assignment = tf.argmin(dist_mat, axis=-1) # B * PR
min_dist = tf.reduce_min(dist_mat, axis=-1)
thres_mid = tf.reduce_mean(min_dist, axis=-1, keepdims=True)
thres_min = tf.reduce_min(min_dist, axis=-1, keepdims=True)
thres_max = tf.reduce_max(min_dist, axis=-1, keepdims=True)
POSITIVE_THRES, NEGATIVE_THRES = (thres_mid + thres_min) / 2.0, (
thres_mid + thres_max) / 2.0
positive_idxes = tf.where(min_dist < POSITIVE_THRES)
negative_idxes = tf.where(min_dist > NEGATIVE_THRES)
positive_gt_idxes = tf.stack([
positive_idxes[:, 0],
tf.gather_nd(bboxes_assignment, positive_idxes)
],
axis=1)
# objectiveness loss
pos_obj_cls_score = tf.gather_nd(proposals_output[..., :2],
positive_idxes)
pos_obj_cls_gt = tf.ones([tf.shape(positive_idxes)[0]], dtype=tf.int32)
neg_obj_cls_score = tf.gather_nd(proposals_output[..., :2],
negative_idxes)
neg_obj_cls_gt = tf.zeros([tf.shape(negative_idxes)[0]],
dtype=tf.int32)
obj_cls_loss = tf.identity(
(tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=pos_obj_cls_score, labels=pos_obj_cls_gt)) +
tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=neg_obj_cls_score, labels=neg_obj_cls_gt))) / 2.0,
name='obj_cls_loss')
obj_correct = tf.concat([
tf.cast(tf.nn.in_top_k(pos_obj_cls_score, pos_obj_cls_gt, 1),
tf.float32),
tf.cast(tf.nn.in_top_k(neg_obj_cls_score, neg_obj_cls_gt, 1),
tf.float32)
],
axis=0,
name='obj_correct')
obj_accuracy = tf.reduce_mean(obj_correct, name='obj_accuracy')
# center regression losses
center_gt = tf.gather_nd(bboxes_xyz_gt, positive_gt_idxes)
delta_predicted = tf.gather_nd(proposals_output[..., 2:5],
positive_idxes)
delta_gt = center_gt - tf.gather_nd(proposals_xyz, positive_idxes)
center_loss = tf.reduce_mean(
tf.reduce_sum(tf.losses.huber_loss(
labels=delta_gt,
predictions=delta_predicted,
reduction=tf.losses.Reduction.NONE),
axis=-1))
# Appendix A1: chamfer loss, assignment at one bbox to each gt bbox
bboxes_assignment_dual = tf.argmin(dist_mat, axis=1) # B * BB
batch_idx = tf.tile(
tf.expand_dims(tf.range(
tf.shape(bboxes_assignment_dual, out_type=tf.int64)[0]),
axis=-1),
[1, tf.shape(bboxes_assignment_dual)[1]]) # B * BB
delta_gt_dual = bboxes_xyz_gt - tf.gather_nd(
proposals_xyz,
tf.stack([batch_idx, bboxes_assignment_dual],
axis=-1)) # B * BB * 3
delta_predicted_dual = tf.gather_nd(
proposals_output[..., 2:5],
tf.stack([batch_idx, bboxes_assignment_dual],
axis=-1)) # B * BB * 3)
center_loss_dual = tf.reduce_mean(
tf.reduce_sum(tf.losses.huber_loss(
labels=delta_gt_dual,
predictions=delta_predicted_dual,
reduction=tf.losses.Reduction.NONE),
axis=-1))
# add up
center_loss += center_loss_dual
center_loss = tf.identity(center_loss, 'center_loss')
# heading classification loss
heading_cls_gt = tf.gather_nd(bboxes_heading_labels_gt,
positive_gt_idxes)
heading_cls_score = tf.gather_nd(
proposals_output[..., 5:5 + config.NH], positive_idxes)
heading_cls_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=heading_cls_score, labels=heading_cls_gt),
name='heading_cls_loss')
# heading residual loss
heading_cls_gt_onehot = tf.one_hot(heading_cls_gt,
depth=config.NH,
on_value=1,
off_value=0,
axis=-1) # Np * NH
heading_residual_gt = tf.gather_nd(bboxes_heading_residuals_gt,
positive_gt_idxes) / (
np.pi / config.NH) # Np
heading_residual_predicted = tf.gather_nd(
proposals_output[..., 5 + config.NH:5 + 2 * config.NH],
positive_idxes) # Np * NH
heading_residual_loss = tf.losses.huber_loss(labels=heading_residual_gt,
predictions=tf.reduce_sum(heading_residual_predicted * \
tf.to_float(heading_cls_gt_onehot),
axis=1),
reduction=tf.losses.Reduction.MEAN)
heading_residual_loss = tf.identity(heading_residual_loss,
name='heading_residual_loss')
# Size loss
size_cls_gt = tf.gather_nd(bboxes_size_labels_gt, positive_gt_idxes)
size_cls_score = tf.gather_nd(
proposals_output[...,
5 + 2 * config.NH:5 + 2 * config.NH + config.NS],
positive_idxes)
size_cls_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=size_cls_score, labels=size_cls_gt),
name='size_cls_loss')
# size residual loss
size_cls_gt_onehot = tf.one_hot(size_cls_gt,
depth=config.NS,
on_value=1,
off_value=0,
axis=-1) # Np * NS
size_cls_gt_onehot = tf.tile(
tf.expand_dims(tf.to_float(size_cls_gt_onehot), -1),
[1, 1, 3]) # Np * NS * 3
mean_size_arr_expand = tf.expand_dims(
tf.constant(class_mean_size, dtype=tf.float32), 0) # (1, NS, 3)
mean_size_label = tf.reduce_sum(size_cls_gt_onehot *
mean_size_arr_expand,
axis=[1]) # (P, 3)
size_residual_gt = tf.gather_nd(
bboxes_size_residuals_gt,
positive_gt_idxes) / mean_size_label # Np * 3
size_residual_predicted = tf.reshape(
tf.gather_nd(
proposals_output[..., 5 + 2 * config.NH + config.NS:5 +
2 * config.NH + 4 * config.NS],
positive_idxes), (-1, config.NS, 3)) # Np * NS * 3
size_residual_loss = tf.reduce_mean(tf.reduce_sum(tf.losses.huber_loss(
labels=size_residual_gt,
predictions=tf.reduce_sum(size_residual_predicted *
tf.to_float(size_cls_gt_onehot),
axis=1),
reduction=tf.losses.Reduction.NONE),
axis=-1),
name='size_residual_loss')
box_loss = tf.identity(center_loss + 0.1 * heading_cls_loss +
heading_residual_loss + 0.1 * size_cls_loss +
size_residual_loss,
name='box_loss')
# semantic loss
sem_cls_score = tf.gather_nd(proposals_output[..., -config.NC:],
positive_idxes)
sem_cls_gt = tf.gather_nd(bboxes_labels_gt, positive_gt_idxes) # Np
sem_cls_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=sem_cls_score, labels=sem_cls_gt),
name='sem_cls_loss')
sem_correct = tf.cast(tf.nn.in_top_k(sem_cls_score, sem_cls_gt, 1),
tf.float32,
name='sem_correct')
sem_accuracy = tf.reduce_mean(sem_correct, name='sem_accuracy')
# This will monitor training error & accuracy (in a moving average fashion). The value will be automatically
# 1. written to tensosrboard
# 2. written to stat.json
# 3. printed after each epoch
# summary.add_moving_summary(obj_accuracy, sem_accuracy)
# Use a regex to find parameters to apply weight decay.
# Here we apply a weight decay on all W (weight matrix) of all fc layers
# If you don't like regex, you can certainly define the cost in any other methods.
# no weight decay
wd_cost = tf.multiply(1e-5,
regularize_cost('.*/W', tf.nn.l2_loss),
name='regularize_loss')
total_cost = vote_reg_loss + 0.5 * obj_cls_loss + 1. * box_loss + 0.1 * sem_cls_loss
total_cost = tf.identity(total_cost, name='total_loss')
summary.add_moving_summary(total_cost,
vote_reg_loss,
obj_cls_loss,
box_loss,
center_loss,
heading_cls_loss,
heading_residual_loss,
size_cls_loss,
size_residual_loss,
sem_cls_loss,
wd_cost,
obj_accuracy,
sem_accuracy,
decay=0)
# monitor histogram of all weight (of conv and fc layers) in tensorboard
summary.add_param_summary(('.*/W', ['histogram', 'rms']))
# the function should return the total cost to be optimized
return total_cost
bias = tf.Variable(tf.zeros([10]))
a = tf.matmul(x, weight) + bias
y_head = tf.nn.softmax(a)
# # 定义二次损失函数并依据梯度下降法进行训练 -- 这样梯度下降的train就变成了x和y的函数
learning_rate = 0.1
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=y_head, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate)
train = optimizer.minimize(loss)
init = tf.global_variables_initializer()
correct_prediction = tf.equal(tf.argmax(y, 1),
tf.argmax(y_head, 1)) # tf.argmax找到x中等于1的最大的id
correction = tf.reduce_mean(tf.cast(
correct_prediction, tf.float32)) # tf.cast 转换类型，将bool转为float，从而求得准确率
# 模型保存
saver = tf.train.Saver()
DIR = "D://tensorboardLogDir/eightWeek/"
# 迭代100次，进行mini_batch梯度下降
epoch_n = 100
with tf.Session() as session:
session.run(init)
for step in range(epoch_n):
for batch in range(batch_n):
batch_x, batch_y = mnist.train.next_batch(batch_size)
session.run(train, feed_dict={x: batch_x, y: batch_y}) # 此处是最小化
corr = session.run(correction,
feed_dict={
def __init__(self,
rateTrain=0.0,
lr=1e-3,
index=0,
size=0,
dataMode='test'):
# parameter ----
if dataMode == 'large':
self.xDim = 20
self.yDim = 20
elif dataMode == 'middle':
self.xDim = 15
self.yDim = 15
elif dataMode == 'small':
self.xDim = 10
self.yDim = 10
self.tDim = 71
self.index = index
self.lambdaDim = 2
# ----
# Dataset ----
self.myData = burgers2Ddata.Data(pdeMode='burgers2d',
dataMode=dataMode)
# allx,ally[51,1] t[201,1] x[xdim or ydim,1] u,v[xdim,ydim,tdim] nu[data,]
self.alltestX, self.alltestY, self.testT, self.testX, self.testY, self.testU, self.testV, self.testNU, self.idx, self.idy = self.myData.traintest(
size=size)
# ----
#pdb.set_trace()
# Placeholder ----
# input u,v
self.inobsu = tf.compat.v1.placeholder(
tf.float32, shape=[None, self.xDim, self.yDim, self.tDim])
self.inobsv = tf.compat.v1.placeholder(
tf.float32, shape=[None, self.xDim, self.yDim, self.tDim])
self.inobs = tf.concat([self.inobsu, self.inobsv], -1)
# output u,v
self.outobsu = tf.compat.v1.placeholder(
tf.float64, shape=[None, self.xDim, self.yDim, self.tDim])
self.outobsv = tf.compat.v1.placeholder(
tf.float64, shape=[None, self.xDim, self.yDim, self.tDim])
# initial condition u,v
self.initu = tf.compat.v1.placeholder(
tf.float64,
shape=[None, self.alltestX.shape[0], self.alltestY.shape[0], 1])
self.initv = tf.compat.v1.placeholder(
tf.float64,
shape=[None, self.alltestX.shape[0], self.alltestY.shape[0], 1])
# param nu
self.y = tf.compat.v1.placeholder(tf.float32,
shape=[None, self.lambdaDim])
# ----
#pdb.set_trace()
# Restore neural network ----
# pred nu [ndata,]
self.predparam = self.lambdaNN(self.inobs)
# ----
# optimizer ----
config = tf.compat.v1.ConfigProto(gpu_options=tf.compat.v1.GPUOptions(
per_process_gpu_memory_fraction=0.1, allow_growth=True))
self.saver = tf.compat.v1.train.Saver()
self.sess = tf.compat.v1.Session(config=config)
# ----
#pdb.set_trace()
# Restore model ----
ckptpath = os.path.join('model', f'{dataMode}burgers2d_d{size}_part2')
ckpt = tf.train.get_checkpoint_state(ckptpath)
lastmodel = ckpt.model_checkpoint_path
self.saver.restore(self.sess, lastmodel)
print('>>> Restore train model')
# ----
# float32 -> float64
self.predparam = tf.cast(self.predparam, tf.float64)
# for predparam
self.placeparam = tf.compat.v1.placeholder(
tf.float64, shape=[None, self.lambdaDim])
# PDE ----
# output: u
self.predu, self.predv = pdeburgers2D.burgers2d(
self.placeparam[:, None, 0], self.placeparam[:, None, 1],
self.initu, self.initv)
self.predu_first, self.predv_first = pdeburgers2D.burgers2d(
self.predparam[:, None, 0], self.predparam[:, None, 1], self.initu,
self.initv)
# ----
# space data -> [none, self.xDim, t] ----
self.indx = tf.compat.v1.placeholder(tf.int32, shape=[self.xDim, 1])
self.indy = tf.compat.v1.placeholder(tf.int32, shape=[self.yDim, 1])
# 1) [x,data,y,t] for slice xdim u,v
trans_predu = tf.transpose(self.predu, perm=[1, 0, 2, 3])
trans_predu_first = tf.transpose(self.predu_first, perm=[1, 0, 2, 3])
trans_predv = tf.transpose(self.predv, perm=[1, 0, 2, 3])
trans_predv_first = tf.transpose(self.predv_first, perm=[1, 0, 2, 3])
# 2) slice xdim [xdim,data,y,t]
gather_predu = tf.gather_nd(trans_predu, self.indx)
gather_predu_first = tf.gather_nd(trans_predu_first, self.indx)
gather_predv = tf.gather_nd(trans_predv, self.indx)
gather_predv_first = tf.gather_nd(trans_predv_first, self.indx)
# 1)' [y,data,x,t]
trans_predu_ = tf.transpose(gather_predu, perm=[2, 1, 0, 3])
trans_predu_first_ = tf.transpose(gather_predu_first,
perm=[2, 1, 0, 3])
trans_predv_ = tf.transpose(gather_predv, perm=[2, 1, 0, 3])
trans_predv_first_ = tf.transpose(gather_predv_first,
perm=[2, 1, 0, 3])
# 2) slice ydim [ydim,data,xdim,t]
gather_predu_ = tf.gather_nd(trans_predu_, self.indy)
gather_predu_first_ = tf.gather_nd(trans_predu_first_, self.indy)
gather_predv_ = tf.gather_nd(trans_predv_, self.indy)
gather_predv_first_ = tf.gather_nd(trans_predv_first_, self.indy)
# reshape [data,ydim,xdim,t]
trans_u = tf.transpose(gather_predu_, perm=[1, 0, 2, 3])
trans_u_first = tf.transpose(gather_predu_first_, perm=[1, 0, 2, 3])
trans_v = tf.transpose(gather_predv_, perm=[1, 0, 2, 3])
trans_v_first = tf.transpose(gather_predv_first_, perm=[1, 0, 2, 3])
# reshape [data,xdim,ydim,t]
space_predu = tf.transpose(trans_u, perm=[0, 2, 1, 3])
space_predu_first = tf.transpose(trans_u_first, perm=[0, 2, 1, 3])
space_predv = tf.transpose(trans_v, perm=[0, 2, 1, 3])
space_predv_first = tf.transpose(trans_v_first, perm=[0, 2, 1, 3])
# ----
# loss param ----
self.loss1 = tf.reduce_mean(
tf.square(
tf.cast(self.y[:, None, 0], tf.float64) -
self.placeparam[:, None, 0]))
self.loss2 = tf.reduce_mean(
tf.square(
tf.cast(self.y[:, None, 1], tf.float64) -
self.placeparam[:, None, 1]))
self.loss1_first = tf.reduce_mean(
tf.square(
tf.cast(self.y[:, None, 0], tf.float64) -
self.predparam[:, None, 0]))
self.loss2_first = tf.reduce_mean(
tf.square(
tf.cast(self.y[:, None, 1], tf.float64) -
self.predparam[:, None, 1]))
# ----
#pdb.set_trace()
# loss uv ----
self.lossu = tf.reduce_mean(tf.square(self.outobsu - space_predu))
self.lossu_first = tf.reduce_mean(
tf.square(self.outobsu - space_predu_first))
self.lossv = tf.reduce_mean(tf.square(self.outobsv - space_predv))
self.lossv_first = tf.reduce_mean(
tf.square(self.outobsv - space_predv_first))
self.lossuv = self.lossu + self.lossv
self.lossuv_first = self.lossu_first + self.lossv_first
# ----
#pdb.set_trace()
self.grad_all = tf.gradients(tf.square(self.outobsu - space_predu),
self.placeparam)
# gradient ----
self.alpha = tf.compat.v1.placeholder(tf.float64, shape=[1])
self.grad = tf.gradients(self.lossuv, self.placeparam)[0]
self.grad_first = tf.gradients(
self.lossuv_first,
self.predparam)[0] # for first time(use predict parameter)
self.nextparam1 = self.placeparam[:, 0] - (tf.reduce_sum(self.grad) *
self.alpha)
self.nextparam2 = self.placeparam[:, 1] - (tf.reduce_sum(self.grad) *
self.alpha)
self.nextparam1_first = self.predparam[:, 0] - (
tf.reduce_sum(self.grad_first) * self.alpha) # for first time
self.nextparam2_first = self.predparam[:, 1] - (
tf.reduce_sum(self.grad_first) * self.alpha) # for first time
def createModel():
# Create the model
x = tf.placeholder(tf.float32, [None, 784])
y_ = tf.placeholder(tf.float32, [None, 10])
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
y = tf.nn.softmax(tf.matmul(x, W) + b)
W_conv1 = weight_variable([5, 5, 1, 32])
b_conv1 = bias_variable([32])
x_image = tf.reshape(x, [-1,28,28,1])
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)
W_conv2 = weight_variable([5, 5, 32, 64])
b_conv2 = bias_variable([64])
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool_2x2(h_conv2)
W_fc1 = weight_variable([7 * 7 * 64, 1024])
b_fc1 = bias_variable([1024])
h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
W_fc2 = weight_variable([1024, 10])
b_fc2 = bias_variable([10])
y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
# Define loss and optimizer
cross_entropy = -tf.reduce_sum(y_*tf.log(y_conv))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
"""
Train the model and save the model to disk as a model.ckpt file
file is stored in the same directory as this python script is started
Based on the documentatoin at
https://www.tensorflow.org/versions/master/how_tos/variables/index.html
"""
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
for i in range(20000):
batch = mnist.train.next_batch(50)
if i%100 == 0:
train_accuracy = accuracy.eval(feed_dict={
x:batch[0], y_: batch[1], keep_prob: 1.0})
print("step %d, training accuracy %g"%(i, train_accuracy))
train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
save_path = saver.save(sess, os.path.join(FLAGS.log_dir, 'model.ckpt'))
print ("Model saved in file: ", save_path)
def cast(x, dtype):
return tf.cast(x, dtype)
def evaluation(logits, labels): pred = tf.nn.softmax(logits) correct = tf.nn.in_top_k(pred, labels, 1) return tf.reduce_mean(tf.cast(correct, tf.float32))
def rnn(step_function,
inputs,
initial_states,
go_backwards=False,
mask=None,
constants=None):
'''Iterates over the time dimension of a tensor.
# Arguments
inputs: tensor of temporal data of shape (samples, time, ...)
(at least 3D).
step_function:
Parameters:
input: tensor with shape (samples, ...) (no time dimension),
representing input for the batch of samples at a certain
time step.
states: list of tensors.
Returns:
output: tensor with shape (samples, ...) (no time dimension),
new_states: list of tensors, same length and shapes
as 'states'.
initial_states: tensor with shape (samples, ...) (no time dimension),
containing the initial values for the states used in
the step function.
go_backwards: boolean. If True, do the iteration over
the time dimension in reverse order.
mask: binary tensor with shape (samples, time, 1),
with a zero for every element that is masked.
constants: a list of constant values passed at each step.
# Returns
A tuple (last_output, outputs, new_states).
last_output: the latest output of the rnn, of shape (samples, ...)
outputs: tensor with shape (samples, time, ...) where each
entry outputs[s, t] is the output of the step function
at time t for sample s.
new_states: list of tensors, latest states returned by
the step function, of shape (samples, ...).
'''
ndim = len(inputs.get_shape())
assert ndim >= 3, "Input should be at least 3D."
axes = [1, 0] + list(range(2, ndim))
inputs = tf.transpose(inputs, (axes))
input_list = tf.unpack(inputs)
if constants is None:
constants = []
states = initial_states
successive_states = []
successive_outputs = []
if go_backwards:
input_list.reverse()
if mask is not None:
# Transpose not supported by bool tensor types, hence round-trip to uint8.
mask = tf.cast(mask, tf.uint8)
if len(mask.get_shape()) == ndim - 1:
mask = expand_dims(mask)
mask = tf.cast(tf.transpose(mask, axes), tf.bool)
mask_list = tf.unpack(mask)
if go_backwards:
mask_list.reverse()
for input, mask_t in zip(input_list, mask_list):
output, new_states = step_function(input, states + constants)
# tf.select needs its condition tensor to be the same shape as its two
# result tensors, but in our case the condition (mask) tensor is
# (nsamples, 1), and A and B are (nsamples, ndimensions). So we need to
# broadcast the mask to match the shape of A and B. That's what the
# tile call does, is just repeat the mask along its second dimension
# ndimensions times.
tiled_mask_t = tf.tile(mask_t, tf.pack([1, tf.shape(output)[1]]))
if len(successive_outputs) == 0:
prev_output = zeros_like(output)
else:
prev_output = successive_outputs[-1]
output = tf.select(tiled_mask_t, output, prev_output)
return_states = []
for state, new_state in zip(states, new_states):
# (see earlier comment for tile explanation)
tiled_mask_t = tf.tile(mask_t,
tf.pack([1, tf.shape(new_state)[1]]))
return_states.append(tf.select(tiled_mask_t, new_state, state))
states = return_states
successive_outputs.append(output)
successive_states.append(states)
else:
for input in input_list:
output, states = step_function(input, states + constants)
successive_outputs.append(output)
successive_states.append(states)
last_output = successive_outputs[-1]
outputs = tf.pack(successive_outputs)
new_states = successive_states[-1]
axes = [1, 0] + list(range(2, len(outputs.get_shape())))
outputs = tf.transpose(outputs, axes)
return last_output, outputs, new_states
def run(self):
'''
Runs the model according to the specified settings
- If mode = Train: Train a GRU model using the training data
- If mode = Val: Load the saved GRU model and evaluate it on the validation fold
- If mode = Test: Load the saved GRU model and evaluate it on the blind test set
'''
self.is_train = (self.mode == 'Train')
if not os.path.exists(self.path):
os.mkdir(self.path)
# Load the training data
with open('train_data.pkl', 'rb') as f:
data_sequences = pkl.load(f)
with open('train_labels.pkl', 'rb') as f:
data_labels = pkl.load(f)
dictionary, reverse_dictionary, data_lengths, self.max_seq_len, enc_sequences = build_dictionary(
data_sequences)
self.dictionary = sorted(dictionary.items(),
key=operator.itemgetter(1))
print(self.dictionary)
self.vocabulary_size = len(dictionary)
self.val_size = len(data_sequences) // self.folds
fold = self.mask
print('Training fold number %d. Each fold of size %d' %
(fold, len(data_sequences) // self.folds))
# Truncates sequences at length 2000 and returns descriptive statistics.
# This is done by concatenating the first 1900 and the last 100 amino acids.
if self.is_train:
self.max_seq_len = 2000
original_lengths = copy(data_lengths)
data_sequences = enc_sequences[:, :self.max_seq_len]
for i in range(len(data_lengths)):
if data_lengths[i] > self.max_seq_len:
data_sequences[i] = np.concatenate(
(enc_sequences[i, :self.max_seq_len - 100],
enc_sequences[i, -100:]),
axis=0)
data_lengths[i] = self.max_seq_len
if self.folds == 1:
val_mask = np.array([False])
else:
val_mask = np.arange(self.val_size * (fold - 1),
self.val_size * (fold))
# Use seed to ensure same randomisation is applied for each fold
np.random.seed(4)
perm = np.random.permutation(len(data_sequences))
data_labels = np.array(data_labels)
data_sequences = data_sequences[perm]
data_labels = data_labels[perm]
data_lenghts = data_lengths[perm]
original_lengths = original_lengths[perm]
self.val_data = data_sequences[val_mask]
self.val_labels = data_labels[val_mask]
self.val_lengths = data_lengths[val_mask]
self.val_original_lengths = original_lengths[val_mask]
self.train_data = np.delete(data_sequences, val_mask, axis=0)
self.train_labels = np.delete(data_labels, val_mask, axis=0)
self.train_lengths = np.delete(data_lengths, val_mask, axis=0)
self.train_original_lengths = np.delete(original_lengths,
val_mask,
axis=0)
self.train_statistics, self.train_frame = self.summary_stats(
self.train_lengths, self.train_labels, 'train')
if self.folds == 1:
self.val_statistics = np.array([])
self.val_frame = np.array([])
self.val_original_lengths = np.array([])
else:
self.val_statistics, self.val_frame = self.summary_stats(
self.val_lengths, self.val_labels, 'validation')
this_data = [
self.train_data, self.train_labels, self.train_lengths,
self.val_data, self.val_labels, self.val_lengths,
self.train_statistics, self.train_frame, self.val_statistics,
self.val_frame, self.train_original_lengths,
self.val_original_lengths
]
with open(self.path + 'this_data.pkl', 'wb') as f:
pkl.dump(this_data, f)
else:
with open(self.path + 'this_data.pkl', 'rb') as f:
self.train_data, self.train_labels, self.train_lengths, self.val_data, self.val_labels, self.val_lengths, self.train_statistics, self.train_frame, self.val_statistics, self.val_frame, self.train_original_lengths, self.val_original_lengths = pkl.load(
f)
# Now construct the Tensorflow graph
print('\r~~~~~~~ Building model ~~~~~~~\r')
# Define placeholders and variables
initializer = tf.random_normal_initializer()
self.word_embeddings = tf.get_variable(
'embeddings', [self.vocabulary_size, self.embedding_size],
tf.float32,
initializer=initializer)
sequences = tf.placeholder(tf.int32, [None, None], "sequences")
sequences_lengths = tf.placeholder(tf.int32, [None],
"sequences_lengths")
labels = tf.placeholder(tf.int64, [None], "labels")
keep_prob_dropout = tf.placeholder(tf.float32, name='dropout')
global_step = tf.Variable(0, name='global_step', trainable=False)
# Embed and encode sequences
sequences_embedded = self.embed_data(sequences)
encoded_sequences = self.encoder(sequences_embedded,
sequences_lengths,
keep_prob_dropout,
bidirectional=self.bidirectional)
# Take last hidden state of GRU and put them through a nonlinear and a linear FC layer
with tf.name_scope('non_linear_layer'):
encoded_sentences_BN = self.batch_norm_wrapper(
encoded_sequences, self.is_train)
non_linear = tf.nn.dropout(tf.nn.relu(
tf.contrib.layers.linear(encoded_sentences_BN, 64)),
keep_prob=keep_prob_dropout)
with tf.name_scope('final_layer'):
non_linear_BN = self.batch_norm_wrapper(non_linear, self.is_train)
logits = tf.contrib.layers.linear(non_linear_BN, 4)
# Compute mean loss on this batch, consisting of cross entropy loss and L2 loss
CE_loss = self.get_CE_loss(labels, logits)
L2_loss = self.get_L2_loss()
loss = CE_loss + L2_loss
# Perform training operation
learning_rate = tf.train.exponential_decay(self.learning_rate,
global_step,
100,
0.96,
staircase=True)
opt_op = tf.contrib.layers.optimize_loss(loss=loss,
global_step=global_step,
learning_rate=learning_rate,
optimizer='Adam',
clip_gradients=2.0,
learning_rate_decay_fn=None,
summaries=None)
# Define scalars for Tensorboard
tf.summary.scalar('CE_loss', CE_loss)
tf.summary.scalar('L2_loss', L2_loss)
tf.summary.scalar('loss', loss)
tf.summary.scalar('learning_rate', learning_rate)
# Compute accuracy of prediction
probs = tf.nn.softmax(logits)
with tf.name_scope('accuracy'):
pred = tf.argmax(logits, 1)
correct_prediction = tf.equal(labels, pred)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# If in training mode:
# - shuffle data set before each epoch
# - train model using mini batches
# - track performance on train and validation set throughout training
if self.is_train == True:
with tf.Session() as session:
train_loss_writer = tf.summary.FileWriter(
str(self.path + 'tensorboard/train_loss'), session.graph)
train_summary_writer = tf.summary.FileWriter(
str(self.path + 'tensorboard/train_summary'),
session.graph)
val_summary_writer = tf.summary.FileWriter(
str(self.path + 'tensorboard/val_summary'), session.graph)
# Use the same LOG_DIR where you stored your checkpoint.
embedding_writer = tf.summary.FileWriter(
str(self.path + 'tensorboard/'), session.graph)
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = self.word_embeddings.name
# Link this tensor to its metadata file (e.g. labels).
embedding.metadata_path = os.path.join('./metadata.tsv')
# Saves a configuration file that TensorBoard will read during startup.
projector.visualize_embeddings(embedding_writer, config)
merged = tf.summary.merge_all()
print('\r~~~~~~~ Initializing variables ~~~~~~~\r')
tf.global_variables_initializer().run()
start_time = time.time()
min_train_loss = np.inf
batch_times = []
n = self.train_data.shape[0]
print('\r~~~~~~~ Starting training ~~~~~~~\r')
try:
train_summaryIndex = -1
for epoch in range(self.num_epochs):
self.is_train = True
epoch_time = time.time()
print('----- Epoch', epoch, '-----')
print('Shuffling dataset')
perm = np.random.permutation(len(self.train_data))
self.train_data_perm = self.train_data[perm]
self.train_labels_perm = self.train_labels[perm]
self.train_lengths_perm = self.train_lengths[perm]
total_loss = 0
for i in range(n // self.batch_size):
batch_start = time.time()
batch_data = self.train_data_perm[i *
self.batch_size:
(i + 1) *
self.batch_size]
batch_lengths = self.train_lengths_perm[
i * self.batch_size:(i + 1) * self.batch_size]
batch_labels = self.train_labels_perm[
i * self.batch_size:(i + 1) * self.batch_size]
train_dict = {
sequences: batch_data,
sequences_lengths: batch_lengths,
labels: batch_labels,
keep_prob_dropout: self.keep_prob_dropout
}
_, batch_loss, batch_accuracy, batch_summary = session.run(
[opt_op, loss, accuracy, merged],
feed_dict=train_dict)
total_loss += batch_loss
batch_times.append(time.time() - batch_start)
train_loss_writer.add_summary(
batch_summary,
i + (n // self.batch_size) * epoch)
if i % 10 == 0 and i > 0:
# Print loss every 10 batches
time_per_epoch = np.mean(batch_times) * (
n // self.batch_size)
remaining_time = int(time_per_epoch -
time.time() + epoch_time)
string_out = '\rEnd of batch ' + str(
i) + ' Train loss: ' + str(
total_loss / (i * self.batch_size)
) + ' Accuracy: ' + str(
batch_accuracy)
string_out += ' Elapsed training time : ' + str(
int(time.time() - start_time)) + "s, "
string_out += str(
remaining_time
) + "s remaining for this epoch"
string_out += ' (' + str(
time_per_epoch * 100 / 60 // 1 /
100) + ' min/epoch)'
stdout.write(string_out)
# Train accuracy
train_dict = {
sequences: self.train_data_perm[:1000],
sequences_lengths: self.train_lengths_perm[:1000],
labels: self.train_labels_perm[:1000],
keep_prob_dropout: 1.0
}
train_summary, train_loss, train_accuracy = session.run(
[merged, loss, accuracy], feed_dict=train_dict)
train_summary_writer.add_summary(train_summary, epoch)
print('\nEpoch train loss: ', train_loss,
'Epoch train accuracy: ', train_accuracy)
# Val accuracy
val_dict = {
sequences: self.val_data,
sequences_lengths: self.val_lengths,
labels: self.val_labels,
keep_prob_dropout: 1.0
}
val_summary, val_loss, val_accuracy = session.run(
[merged, loss, accuracy], feed_dict=val_dict)
val_summary_writer.add_summary(val_summary, epoch)
print('\nEpoch val loss: ', val_loss,
'Epoch val accuracy: ', val_accuracy)
self.save_model(session, epoch)
saver = tf.train.Saver(
write_version=tf.train.SaverDef.V2)
saver.save(
session,
os.path.join(self.path + '/tensorboard/',
'model.ckpt'))
except KeyboardInterrupt:
save = input('save?')
if 'y' in save:
self.save_model(session, epoch)
# If in validation mode:
# - Load saved model and evaluate on validation fold
# - Return list containing confusion matrices, and accuracy measures such as FPR and TPR
elif self.mode == 'Val':
with tf.Session() as session:
print('Restoring model...')
saver = tf.train.Saver(write_version=tf.train.SaverDef.V2)
saver.restore(session, self.path + 'tensorboard/model.ckpt')
print('Model restored!')
val_dict = {
sequences: self.val_data,
sequences_lengths: self.val_lengths,
labels: self.val_labels,
keep_prob_dropout: 1.0
}
self.val_pred, self.val_accuracy, self.val_probs = session.run(
[pred, accuracy, probs], feed_dict=val_dict)
_ = self.summary_stats(self.val_lengths, self.val_labels,
'val')
print('\nConfusion matrix (all sequence lengths):')
val_confusion_1 = self.confusion(
gold=self.val_labels,
prediction=self.val_pred,
lengths=self.val_original_lengths,
min_length=0,
max_length=np.inf)
print(val_confusion_1)
print('\nConfusion matrix (sequence length < 2000):')
val_confusion_2 = self.confusion(
gold=self.val_labels,
prediction=self.val_pred,
lengths=self.val_original_lengths,
min_length=0,
max_length=2000)
print(val_confusion_2)
print('\nConfusion matrix (sequence length > 2000):')
val_confusion_3 = self.confusion(
gold=self.val_labels,
prediction=self.val_pred,
lengths=self.val_original_lengths,
min_length=2000,
max_length=np.inf)
print(val_confusion_3)
print('\n Val accuracy:', self.val_accuracy)
print(
'\n Val accuracy when length <2000:',
np.sum((self.val_pred == self.val_labels) *
(self.val_original_lengths <= 2000)) /
np.sum(self.val_original_lengths <= 2000))
print(
'\n Val accuracy when length >2000:',
np.sum((self.val_pred == self.val_labels) *
(self.val_original_lengths > 2000)) /
np.sum(self.val_original_lengths > 2000))
this_sum = np.zeros([3, 5])
this_auc = np.zeros([1, 5])
this_TPR = []
this_FPR = []
total_tp = 0
total_fp = 0
total_fn = 0
total_tn = 0
for i in range(4):
tp = np.sum((self.val_labels == i) * (self.val_pred == i))
fp = np.sum((self.val_labels != i) * (self.val_pred == i))
fn = np.sum((self.val_labels == i) * (self.val_pred != i))
tn = np.sum((self.val_labels != i) * (self.val_pred != i))
total_tp += tp
total_fp += fp
total_fn += fn
total_tn += tn
prec = tp / (tp + fp) if (tp + fp) > 0 else 0.0
recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
f1 = 2 * prec * recall / (
prec + recall) if prec * recall > 0 else 0.0
this_sum[:, i] = np.array([prec, recall, f1])
this_auc[:, i] = roc_auc_score(self.val_labels == i,
self.val_pred == i)
if i < 4:
this_FPR.append(
roc_curve(self.val_labels == i,
self.val_probs[:, i])[0])
this_TPR.append(
roc_curve(self.val_labels == i,
self.val_probs[:, i])[1])
prec = total_tp / (total_tp + total_fp) if (
total_tp + total_fp) > 0 else 0.0
recall = total_tp / (total_tp + total_fn) if (
total_tp + total_fn) > 0 else 0.0
f1 = 2 * prec * recall / (prec +
recall) if prec * recall > 0 else 0.0
this_sum[:, 4] = np.array([prec, recall, f1])
this_sum = np.concatenate((this_sum, this_auc), 0)
self.this_sum = pd.DataFrame(this_sum)
self.this_sum.index = pd.Index(
['Precision', 'Recall', 'F1', 'AUC'])
self.this_sum.columns = pd.Index(
['cyto', 'secreted', 'mito', 'nucleus', 'Total'])
print(self.this_sum)
if self.is_train == False:
return [
val_confusion_1, val_confusion_2, val_confusion_3,
self.this_sum, this_FPR, this_TPR
]
# If in test model:
# - Load saved model and evaluate on test set
# - Print predicted probabilities for each protein in the test set
elif self.mode == 'Test':
with tf.Session() as session:
print('Restoring model...')
saver = tf.train.Saver(write_version=tf.train.SaverDef.V2)
saver.restore(session, self.path + 'model.checkpoint')
print('Model restored!')
with open('test_data.pkl', 'rb') as f:
test_sequences = pkl.load(f)
with open('test_labels.pkl', 'rb') as f:
test_labels = pkl.load(f)
_, _, data_lengths, _, enc_sequences = build_dictionary(
test_sequences, vocab=dictionary)
test_dict = {
sequences: enc_sequences,
sequences_lengths: data_lengths,
keep_prob_dropout: 1.0
}
self.probs, self.pred = session.run([probs, pred],
feed_dict=test_dict)
result = pd.DataFrame(
np.concatenate((self.probs, np.expand_dims(self.pred, 1)),
1))
result.columns = pd.Index(
['cyto', 'secreted', 'mito', 'nucleus', 'prediction'])
print(result)
def sqrt(x):
x = tf.clip_by_value(x, tf.cast(0., dtype=_FLOATX),
tf.cast(np.inf, dtype=_FLOATX))
return tf.sqrt(x)
def hard_sigmoid(x):
x = (0.2 * x) + 0.5
x = tf.clip_by_value(x, tf.cast(0., dtype=_FLOATX),
tf.cast(1., dtype=_FLOATX))
return x
def mean(x, axis=None, keepdims=False):
axis = normalize_axis(axis, ndim(x))
if x.dtype.base_dtype == tf.bool:
x = tf.cast(x, _FLOATX)
return tf.reduce_mean(x, reduction_indices=axis, keep_dims=keepdims)
def clip(x, min_value, max_value):
if max_value < min_value:
max_value = min_value
return tf.clip_by_value(x, tf.cast(min_value, dtype=_FLOATX),
tf.cast(max_value, dtype=_FLOATX))
