def bhattacharyya(self):
"""Approximate bhattacharyya distance between cover and non-cover distances.
Similar to Mahalanobis distance, but for distributions with different variances.
Assumes normality, hence approximate.
Returns:
tf.Tensor: bhattacharyya distance between distributions of the cover
and non-cover pairs' distances.
tf.Tensor: mean cover pair distance
tf.Tensor: mean non-cover pair distance
"""
y_A, y_B = self.subnet_A[-1], self.subnet_B[-1]
squared_dists = tf.reduce_sum(tf.square(y_A - y_B),
reduction_indices=1, )
cover_pairs = tf.where(tf.equal(self.is_cover, tf.ones_like(self.is_cover)))
non_cover_pairs = tf.where(tf.equal(self.is_cover, tf.zeros_like(self.is_cover)))
pair_dists = tf.sqrt(tf.gather(squared_dists, tf.reshape(cover_pairs, [-1])))
non_pair_dists = tf.sqrt(tf.gather(squared_dists, tf.reshape(non_cover_pairs, [-1])))
mu_pairs, sigma2_pairs = tf.nn.moments(pair_dists, axes=[0], name='d_pairs')
mu_non_pairs, sigma2_non_pairs = tf.nn.moments(non_pair_dists, axes=[0], name='d_non_pairs')
bhatt = tf.add( 0.25 * tf.log(0.25 * (sigma2_pairs/sigma2_non_pairs + sigma2_non_pairs/sigma2_pairs + 2)),
0.25 * (mu_pairs - mu_non_pairs)**2 / (sigma2_pairs + sigma2_non_pairs), name='bhatt')
return bhatt, mu_pairs, mu_non_pairs
def make_tf_top(x_shape, loss='sigmoid_ce'):
"""
builds the top layer, i.e. the loss layer.
"""
with tf.name_scope('top') as scope:
x = tf.placeholder(tf.float32, shape=x_shape, name='input')
y = tf.placeholder(tf.float32, shape=x_shape, name='output')
if loss=='sigmoid_ce':
L = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(x, y))
correct_prediction = tf.equal(tf.round( tf.sigmoid(x) ), tf.round( y ))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
accuracy_summary = [tf.summary.scalar('accuracy', accuracy)]
elif loss=='softmax_ce':
L = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(x, y))
correct_prediction = tf.equal(tf.argmax( tf.nn.softmax(x), 1 ), tf.argmax( y, 1 ))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
accuracy_summary = [tf.summary.scalar('accuracy', accuracy)]
elif loss=='sigmoid_l2':
L = tf.nn.l2_loss(tf.sigmoid(x) - y)
accuracy = None
accuracy_summary = []
elif loss=='l2':
L = tf.nn.l2_loss(x - y)
accuracy = None
accuracy_summary = []
loss_summary = tf.summary.scalar('log_loss', tf.log(L))
dx = tf.gradients(L, x)[0]
return L, dx, tf.summary.merge([loss_summary] + accuracy_summary), accuracy
def train(self, sentences):
token_ids, token_values, token_dense_shape = self._tokenize(sentences)
tokens_sparse = tf.sparse.SparseTensor(
indices=token_ids, values=token_values, dense_shape=token_dense_shape)
tokens = tf.sparse.to_dense(tokens_sparse, default_value="")
sparse_lookup_ids = tf.sparse.SparseTensor(
indices=tokens_sparse.indices,
values=self._words_to_indices(tokens_sparse.values),
dense_shape=tokens_sparse.dense_shape)
lookup_ids = tf.sparse.to_dense(sparse_lookup_ids, default_value=0)
# Targets are the next word for each word of the sentence.
tokens_ids_seq = lookup_ids[:, 0:-1]
tokens_ids_target = lookup_ids[:, 1:]
tokens_prefix = tokens[:, 0:-1]
# Mask determining which positions we care about for a loss: all positions
# that have a valid non-terminal token.
mask = tf.logical_and(
tf.logical_not(tf.equal(tokens_prefix, "")),
tf.logical_not(tf.equal(tokens_prefix, "<E>")))
input_mask = tf.cast(mask, tf.int32)
with tf.GradientTape() as t:
sentence_embeddings = tf.nn.embedding_lookup(self._embeddings,
tokens_ids_seq)
lstm_initial_state = self._lstm_cell.get_initial_state(
sentence_embeddings)
lstm_output = self._rnn_layer(
inputs=sentence_embeddings, initial_state=lstm_initial_state)
# Stack LSTM outputs into a batch instead of a 2D array.
lstm_output = tf.reshape(lstm_output, [-1, self._lstm_cell.output_size])
logits = self._logit_layer(lstm_output)
targets = tf.reshape(tokens_ids_target, [-1])
weights = tf.cast(tf.reshape(input_mask, [-1]), tf.float32)
losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=targets, logits=logits)
# Final loss is the mean loss for all token losses.
final_loss = tf.math.divide(
tf.reduce_sum(tf.multiply(losses, weights)),
tf.reduce_sum(weights),
name="final_loss")
watched = t.watched_variables()
gradients = t.gradient(final_loss, watched)
for w, g in zip(watched, gradients):
w.assign_sub(g)
return final_loss
def make_minibatch(self, valid_anchors):
with tf.variable_scope('rpn_minibatch'):
# in labels(shape is [N, ]): 1 is positive, 0 is negative, -1 is ignored
labels, anchor_matched_gtboxes, object_mask = \
self.rpn_find_positive_negative_samples(valid_anchors) # [num_of_valid_anchors, ]
positive_indices = tf.reshape(tf.where(tf.equal(labels, 1.0)), [-1]) # use labels is same as object_mask
num_of_positives = tf.minimum(tf.shape(positive_indices)[0],
tf.cast(self.rpn_mini_batch_size * self.rpn_positives_ratio, tf.int32))
# num of positives <= minibatch_size * 0.5
positive_indices = tf.random_shuffle(positive_indices)
positive_indices = tf.slice(positive_indices, begin=[0], size=[num_of_positives])
# positive_anchors = tf.gather(self.anchors, positive_indices)
negative_indices = tf.reshape(tf.where(tf.equal(labels, 0.0)), [-1])
num_of_negatives = tf.minimum(self.rpn_mini_batch_size - num_of_positives,
tf.shape(negative_indices)[0])
negative_indices = tf.random_shuffle(negative_indices)
negative_indices = tf.slice(negative_indices, begin=[0], size=[num_of_negatives])
# negative_anchors = tf.gather(self.anchors, negative_indices)
minibatch_indices = tf.concat([positive_indices, negative_indices], axis=0)
minibatch_indices = tf.random_shuffle(minibatch_indices)
minibatch_anchor_matched_gtboxes = tf.gather(anchor_matched_gtboxes, minibatch_indices)
object_mask = tf.gather(object_mask, minibatch_indices)
labels = tf.cast(tf.gather(labels, minibatch_indices), tf.int32)
labels_one_hot = tf.one_hot(labels, depth=2)
return minibatch_indices, minibatch_anchor_matched_gtboxes, object_mask, labels_one_hot
def _decode_and_resize(image_tensor):
"""Decodes jpeg string, resizes it and returns a uint8 tensor."""
# These constants are set by Inception v3's expectations.
height = 299
width = 299
channels = 3
image_tensor = tf.where(tf.equal(image_tensor, ''), IMAGE_DEFAULT_STRING, image_tensor)
# Fork by whether image_tensor value is a file path, or a base64 encoded string.
slash_positions = tf.equal(tf.string_split([image_tensor], delimiter="").values, '/')
is_file_path = tf.cast(tf.count_nonzero(slash_positions), tf.bool)
# The following two functions are required for tf.cond. Note that we can not replace them
# with lambda. According to TF docs, if using inline lambda, both branches of condition
# will be executed. The workaround is to use a function call.
def _read_file():
return tf.read_file(image_tensor)
def _decode_base64():
return tf.decode_base64(image_tensor)
image = tf.cond(is_file_path, lambda: _read_file(), lambda: _decode_base64())
image = tf.image.decode_jpeg(image, channels=channels)
image = tf.expand_dims(image, 0)
image = tf.image.resize_bilinear(image, [height, width], align_corners=False)
image = tf.squeeze(image, squeeze_dims=[0])
image = tf.cast(image, dtype=tf.uint8)
return image
def call(self, inputs):
batch_shape = tf.shape(inputs)[:-1]
length = tf.shape(inputs)[-1]
ngram_range_counts = []
for n in range(self.minval, self.maxval):
# Reshape inputs from [..., length] to [..., 1, length // n, n], dropping
# remainder elements. Each n-vector is an ngram.
reshaped_inputs = tf.reshape(
inputs[..., :(n * (length // n))],
tf.concat([batch_shape, [1], (length // n)[tf.newaxis], [n]], 0))
# Count the number of times each ngram appears in the input. We do so by
# checking whether each n-vector in the input is equal to each n-vector
# in a Tensor of all possible ngrams. The comparison is batched between
# the input Tensor of shape [..., 1, length // n, n] and the ngrams Tensor
# of shape [..., input_dim**n, 1, n].
ngrams = tf.reshape(
list(np.ndindex((self.input_dim,) * n)),
[1] * (len(inputs.shape)-1) + [self.input_dim**n, 1, n])
is_ngram = tf.equal(
tf.reduce_sum(tf.cast(tf.equal(reshaped_inputs, ngrams), tf.int32),
axis=-1),
n)
ngram_counts = tf.reduce_sum(tf.cast(is_ngram, tf.float32), axis=-1)
ngram_range_counts.append(ngram_counts)
return tf.concat(ngram_range_counts, axis=-1)
def getRpRnTpTnForTrain0OrVal1(self, y, training0OrValidation1):
# The returned list has (numberOfClasses)x4 integers: >numberOfRealPositives, numberOfRealNegatives, numberOfTruePredictedPositives, numberOfTruePredictedNegatives< for each class (incl background).
# Order in the list is the natural order of the classes (ie class-0 RP,RN,TPP,TPN, class-1 RP,RN,TPP,TPN, class-2 RP,RN,TPP,TPN ...)
# param y: y = T.itensor4('y'). Dimensions [batchSize, r, c, z]
yPredToUse = self.y_pred_train if training0OrValidation1 == 0 else self.y_pred_val
returnedListWithNumberOfRpRnTpTnForEachClass = []
for class_i in range(0, self._numberOfOutputClasses) :
#Number of Real Positive, Real Negatives, True Predicted Positives and True Predicted Negatives are reported PER CLASS (first for WHOLE).
tensorOneAtRealPos = tf.equal(y, class_i)
tensorOneAtRealNeg = tf.logical_not(tensorOneAtRealPos)
tensorOneAtPredictedPos = tf.equal(yPredToUse, class_i)
tensorOneAtPredictedNeg = tf.logical_not(tensorOneAtPredictedPos)
tensorOneAtTruePos = tf.logical_and(tensorOneAtRealPos,tensorOneAtPredictedPos)
tensorOneAtTrueNeg = tf.logical_and(tensorOneAtRealNeg,tensorOneAtPredictedNeg)
returnedListWithNumberOfRpRnTpTnForEachClass.append( tf.reduce_sum( tf.cast(tensorOneAtRealPos, dtype="int32")) )
returnedListWithNumberOfRpRnTpTnForEachClass.append( tf.reduce_sum( tf.cast(tensorOneAtRealNeg, dtype="int32")) )
returnedListWithNumberOfRpRnTpTnForEachClass.append( tf.reduce_sum( tf.cast(tensorOneAtTruePos, dtype="int32")) )
returnedListWithNumberOfRpRnTpTnForEachClass.append( tf.reduce_sum( tf.cast(tensorOneAtTrueNeg, dtype="int32")) )
return returnedListWithNumberOfRpRnTpTnForEachClass
def focal_loss(onehot_labels, cls_preds,
alpha=0.25, gamma=2.0, name=None, scope=None):
"""Compute softmax focal loss between logits and onehot labels
logits and onehot_labels must have same shape [batchsize, num_classes] and
the same data type (float16, 32, 64)
Args:
onehot_labels: Each row labels[i] must be a valid probability distribution
cls_preds: Unscaled log probabilities
alpha: The hyperparameter for adjusting biased samples, default is 0.25
gamma: The hyperparameter for penalizing the easy labeled samples
name: A name for the operation (optional)
Returns:
A 1-D tensor of length batch_size of same type as logits with softmax focal loss
"""
with tf.name_scope(scope, 'focal_loss', [cls_preds, onehot_labels]) as sc:
logits = tf.convert_to_tensor(cls_preds)
onehot_labels = tf.convert_to_tensor(onehot_labels)
precise_logits = tf.cast(logits, tf.float32) if (
logits.dtype == tf.float16) else logits
onehot_labels = tf.cast(onehot_labels, precise_logits.dtype)
predictions = tf.nn.sigmoid(logits)
predictions_pt = tf.where(tf.equal(onehot_labels, 1), predictions, 1. - predictions)
# add small value to avoid 0
epsilon = 1e-8
alpha_t = tf.scalar_mul(alpha, tf.ones_like(onehot_labels, dtype=tf.float32))
alpha_t = tf.where(tf.equal(onehot_labels, 1.0), alpha_t, 1 - alpha_t)
losses = tf.reduce_sum(
-alpha_t * tf.pow(1. - predictions_pt, gamma) * onehot_labels * tf.log(predictions_pt + epsilon),
name=name, axis=1)
return losses
def fpn_map_rois_to_levels(boxes):
"""
Assign boxes to level 2~5.
Args:
boxes (nx4):
Returns:
[tf.Tensor]: 4 tensors for level 2-5. Each tensor is a vector of indices of boxes in its level.
[tf.Tensor]: 4 tensors, the gathered boxes in each level.
Be careful that the returned tensor could be empty.
"""
sqrtarea = tf.sqrt(tf_area(boxes))
level = tf.to_int32(tf.floor(
4 + tf.log(sqrtarea * (1. / 224) + 1e-6) * (1.0 / np.log(2))))
# RoI levels range from 2~5 (not 6)
level_ids = [
tf.where(level <= 2),
tf.where(tf.equal(level, 3)), # == is not supported
tf.where(tf.equal(level, 4)),
tf.where(level >= 5)]
level_ids = [tf.reshape(x, [-1], name='roi_level{}_id'.format(i + 2))
for i, x in enumerate(level_ids)]
num_in_levels = [tf.size(x, name='num_roi_level{}'.format(i + 2))
for i, x in enumerate(level_ids)]
add_moving_summary(*num_in_levels)
level_boxes = [tf.gather(boxes, ids) for ids in level_ids]
return level_ids, level_boxes
def get_accuracy_loss(arg,x,y,y_):
'''
Note: when the task is regression accuracy = loss but for classification
loss = cross_entropy,svm_loss, surrogate_loss, etc and accuracy = 1 - {0-1 loss}.
'''
with tf.name_scope("loss_and_acc") as scope:
# loss
if arg.softmax:
#cross_entropy = tf.reduce_mean(-tf.rduce_sum(y_ * tf.log(y), reduction_indices=[1]))
diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)
cross_entropy = tf.reduce_mean(diff)
loss = cross_entropy
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) # list of booleans indicating correct predictions
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
else:
l2_loss = tf.reduce_sum( tf.reduce_mean(tf.square(y_-y), 0))
loss = l2_loss
y = tf.cast(tf.sign(y),tf.float32)
y_ = tf.cast(tf.sign(y_),tf.float32)
correct_prediction = tf.equal(y, y_) # list of booleans indicating correct predictions
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# accuracy
# if arg.classification:
# correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) # list of booleans indicating correct predictions
# accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# else:
# accuracy = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
##
tf.summary.scalar('loss', loss)
tf.summary.scalar('accuracy', accuracy)
return loss, accuracy
def remap_keys(sparse_tensor):
# Current indices of our SparseTensor that we need to fix
bad_indices = sparse_tensor.indices # shape = (current_batch_size * (number_of_items/users[i] + 1), 2)
# Current values of our SparseTensor that we need to fix
bad_values = sparse_tensor.values # shape = (current_batch_size * (number_of_items/users[i] + 1),)
# Since batch is ordered, the last value for a batch index is the user
# Find where the batch index chages to extract the user rows
# 1 where user, else 0
user_mask = tf.concat(values = [bad_indices[1:,0] - bad_indices[:-1,0], tf.constant(value = [1], dtype = tf.int64)], axis = 0) # shape = (current_batch_size * (number_of_items/users[i] + 1), 2)
# Mask out the user rows from the values
good_values = tf.boolean_mask(tensor = bad_values, mask = tf.equal(x = user_mask, y = 0)) # shape = (current_batch_size * number_of_items/users[i],)
item_indices = tf.boolean_mask(tensor = bad_indices, mask = tf.equal(x = user_mask, y = 0)) # shape = (current_batch_size * number_of_items/users[i],)
user_indices = tf.boolean_mask(tensor = bad_indices, mask = tf.equal(x = user_mask, y = 1))[:, 1] # shape = (current_batch_size,)
good_user_indices = tf.gather(params = user_indices, indices = item_indices[:,0]) # shape = (current_batch_size * number_of_items/users[i],)
# User and item indices are rank 1, need to make rank 1 to concat
good_user_indices_expanded = tf.expand_dims(input = good_user_indices, axis = -1) # shape = (current_batch_size * number_of_items/users[i], 1)
good_item_indices_expanded = tf.expand_dims(input = item_indices[:, 1], axis = -1) # shape = (current_batch_size * number_of_items/users[i], 1)
good_indices = tf.concat(values = [good_user_indices_expanded, good_item_indices_expanded], axis = 1) # shape = (current_batch_size * number_of_items/users[i], 2)
remapped_sparse_tensor = tf.SparseTensor(indices = good_indices, values = good_values, dense_shape = sparse_tensor.dense_shape)
return remapped_sparse_tensor
def _compute_sparse_average_correct(input_, labels, per_example_weights, topk=1):
"""Returns the numerator and denominator of classifier accuracy."""
labels = tf.to_int64(labels)
labels.get_shape().assert_is_compatible_with([input_.get_shape()[0], None])
if topk == 1:
predictions = tf.reshape(tf.argmax(input_, 1), [-1, 1])
in_topk = tf.reduce_any(tf.equal(labels, predictions), reduction_indices=[1])
else:
# Use broadcasting to check if ANY of the predictions are in the top k.
# TODO(eiderman): For a multi-label top k, what does accuracy mean?
predictions = tf.reshape(tf.nn.top_k(input_, topk)[1], [-1, 1, topk])
labels = tf.expand_dims(labels, [-1])
in_topk = tf.reduce_any(tf.equal(tf.cast(labels, predictions.dtype), predictions), reduction_indices=[1, 2])
correct_predictions = tf.to_float(in_topk)
# If individual examples are weighted, then we want to normalize by that.
if per_example_weights is not None:
per_example_weights = _convert_and_assert_per_example_weights_compatible(
input_, per_example_weights, dtype=None
)
float_weights = tf.to_float(per_example_weights)
# TODO(eiderman): This should use an op that doesn't support broadcasting.
correct_predictions *= float_weights
num_examples = tf.reduce_sum(float_weights)
else:
# shape only holds ints, but we want to always return the same type
# for num_examples to make everything compatible.
num_examples = tf.to_float(tf.gather(tf.shape(input_), 0))
return tf.reduce_sum(correct_predictions), num_examples
def _get_mask_of_label(label, background, ignore_label):
mask_fg = 1 - tf.to_float(tf.equal(label, background))
mask_ig = 1 - tf.to_float(tf.equal(label, ignore_label))
# mask_fg = 1 - label.eq(background)
# mask_ig = 1 - label.eq(ignore_label)
mask = mask_fg * mask_ig
return mask, mask_ig
def to_binary_tf(bar_or_track_bar, threshold=0.0, track_mode=False, melody=False):
"""Return the binarize tensor of the input tensor (be careful of the channel order!)"""
if track_mode:
# melody track
if melody:
melody_is_max = tf.equal(bar_or_track_bar, tf.reduce_max(bar_or_track_bar, axis=2, keep_dims=True))
melody_pass_threshold = (bar_or_track_bar > threshold)
out_tensor = tf.logical_and(melody_is_max, melody_pass_threshold)
# non-melody track
else:
out_tensor = (bar_or_track_bar > threshold)
return out_tensor
else:
if len(bar_or_track_bar.get_shape()) == 4:
melody_track = tf.slice(bar_or_track_bar, [0, 0, 0, 0], [-1, -1, -1, 1])
other_tracks = tf.slice(bar_or_track_bar, [0, 0, 0, 1], [-1, -1, -1, -1])
elif len(bar_or_track_bar.get_shape()) == 5:
melody_track = tf.slice(bar_or_track_bar, [0, 0, 0, 0, 0], [-1, -1, -1, -1, 1])
other_tracks = tf.slice(bar_or_track_bar, [0, 0, 0, 0, 1], [-1, -1, -1, -1, -1])
# melody track
melody_is_max = tf.equal(melody_track, tf.reduce_max(melody_track, axis=2, keep_dims=True))
melody_pass_threshold = (melody_track > threshold)
out_tensor_melody = tf.logical_and(melody_is_max, melody_pass_threshold)
# other tracks
out_tensor_others = (other_tracks > threshold)
if len(bar_or_track_bar.get_shape()) == 4:
return tf.concat([out_tensor_melody, out_tensor_others], 3)
elif len(bar_or_track_bar.get_shape()) == 5:
return tf.concat([out_tensor_melody, out_tensor_others], 4)
def m_body(i, ta_tp, ta_fp, gmatch):
# Jaccard score with groundtruth bboxes.
rbbox = bboxes[i]
jaccard = bboxes_jaccard(rbbox, gbboxes)
jaccard = jaccard * tf.cast(tf.equal(glabels, rlabel), dtype=jaccard.dtype)
# Best fit, checking it's above threshold.
idxmax = tf.cast(tf.argmax(jaccard, axis=0), tf.int32)
jcdmax = jaccard[idxmax]
match = jcdmax > matching_threshold
existing_match = gmatch[idxmax]
not_difficult = tf.logical_not(gdifficults[idxmax])
# TP: match & no previous match and FP: previous match | no match.
# If difficult: no record, i.e FP=False and TP=False.
tp = tf.logical_and(not_difficult,
tf.logical_and(match, tf.logical_not(existing_match)))
ta_tp = ta_tp.write(i, tp)
fp = tf.logical_and(not_difficult,
tf.logical_or(existing_match, tf.logical_not(match)))
ta_fp = ta_fp.write(i, fp)
# Update grountruth match.
mask = tf.logical_and(tf.equal(grange, idxmax),
tf.logical_and(not_difficult, match))
gmatch = tf.logical_or(gmatch, mask)
return [i+1, ta_tp, ta_fp, gmatch]
def activation_image(activations, label_onehot):
"""Make a row sorted by class for each activation. Put a black line around the activations."""
labels = tf.argmax(label_onehot, axis=1)
_, n_classes = label_onehot.shape.as_list()
mean, var = tf.nn.moments(activations, [0, 1])
activations = (activations - mean)/tf.sqrt(var+1e-5)
activations = tf.clip_by_value(activations, -1, 1)
activations = (activations + 1.0) / 2.0 # shift to [0, 1]
canvas = []
for i in xrange(n_classes):
inds = tf.where(tf.equal(labels, i))
def _gather():
return tf.squeeze(tf.gather(activations, inds), 1)
def _empty():
return tf.zeros([0, activations.shape.as_list()[1]], dtype=tf.float32)
assert inds.shape.as_list()[0] is None
x = tf.cond(tf.equal(tf.shape(inds)[0], 0), _empty, _gather)
canvas.append(x)
canvas.append(tf.zeros([1, activations.shape.as_list()[1]]))
canvas = tf.concat(canvas, 0)
canvas = tf.reshape(canvas, [1, activations.shape.as_list()[0]+n_classes, canvas.shape.as_list()[1], 1])
return canvas
def weights_concatenated(labels):
"""Assign weight 1.0 to the "target" part of the concatenated labels.
The labels look like:
source English I love you . ID1 target French Je t'aime . ID1 source
English the cat ID1 target French le chat ID1 source English ...
We want to assign weight 1.0 to all words in the target text (including the
ID1 end symbol), but not to the source text or the boilerplate. In the
above example, the target words that get positive weight are:
Je t'aime . ID1 le chat ID1
Args:
labels: a Tensor
Returns:
a Tensor
"""
eos_mask = tf.to_int32(tf.equal(labels, 1))
sentence_num = tf.cumsum(eos_mask, axis=1, exclusive=True)
in_target = tf.equal(tf.mod(sentence_num, 2), 1)
# first two tokens of each sentence are boilerplate.
sentence_num_plus_one = sentence_num + 1
shifted = tf.pad(sentence_num_plus_one, [[0, 0], [2, 0], [0, 0],
[0, 0]])[:, :-2, :, :]
nonboilerplate = tf.equal(sentence_num_plus_one, shifted)
ret = tf.to_float(tf.logical_and(nonboilerplate, in_target))
return ret
def sorted_images(images, label_onehot):
# images is [bs, x, y, c]
labels = tf.argmax(label_onehot, axis=1)
_, n_classes = label_onehot.shape.as_list()
to_stack = []
for i in xrange(n_classes):
inds = tf.where(tf.equal(labels, i))
def _gather():
return tf.squeeze(tf.gather(images, inds), 1)
def _empty():
return tf.zeros([0] + images.shape.as_list()[1:], dtype=tf.float32)
assert inds.shape.as_list()[0] is None
x = tf.cond(tf.equal(tf.shape(inds)[0], 0), _empty, _gather)
to_stack.append(x)
# pad / trim all up to 10.
padded = []
for t in to_stack:
n_found = tf.shape(t)[0]
pad = tf.pad(t[0:10], tf.stack([tf.stack([0,tf.maximum(0, 10-n_found)]), [0,0], [0,0], [0,0]]))
padded.append(pad)
xs = [tf.concat(tf.split(p, 10), axis=1) for p in padded]
ys = tf.concat(xs, axis=2)
ys = tf.cast(tf.clip_by_value(ys, 0., 1.) * 255., tf.uint8)
return ys
def evaluate_precision_recall(
input_layer, labels, threshold=0.5, per_example_weights=None, name=PROVIDED, phase=Phase.train
):
"""Computes the precision and recall of the prediction vs the labels.
Args:
input_layer: A Pretty Tensor object.
labels: The target labels to learn as a float tensor.
threshold: The threshold to use to decide if the prediction is true.
per_example_weights: A Tensor with a weight per example.
name: An optional name.
phase: The phase of this model; non training phases compute a total across
all examples.
Returns:
Precision and Recall.
"""
_ = name # Eliminate warning, name used for namescoping by PT.
selected, sum_retrieved, sum_relevant = _compute_precision_recall(
input_layer, labels, threshold, per_example_weights
)
if phase != Phase.train:
dtype = tf.float32
# Create the variables in all cases so that the load logic is easier.
relevant_count = tf.get_variable(
"relevant_count",
[],
dtype,
tf.zeros_initializer,
collections=[bookkeeper.GraphKeys.TEST_VARIABLES],
trainable=False,
)
retrieved_count = tf.get_variable(
"retrieved_count",
[],
dtype,
tf.zeros_initializer,
collections=[bookkeeper.GraphKeys.TEST_VARIABLES],
trainable=False,
)
selected_count = tf.get_variable(
"selected_count",
[],
dtype,
tf.zeros_initializer,
collections=[bookkeeper.GraphKeys.TEST_VARIABLES],
trainable=False,
)
with input_layer.g.device(selected_count.device):
selected = tf.assign_add(selected_count, selected)
with input_layer.g.device(retrieved_count.device):
sum_retrieved = tf.assign_add(retrieved_count, sum_retrieved)
with input_layer.g.device(relevant_count.device):
sum_relevant = tf.assign_add(relevant_count, sum_relevant)
return (
tf.select(tf.equal(sum_retrieved, 0), tf.zeros_like(selected), selected / sum_retrieved),
tf.select(tf.equal(sum_relevant, 0), tf.zeros_like(selected), selected / sum_relevant),
)
def streaming_f1(self, labels, predictions, n_classes, weights=None, type='macro'):
labels_and_predictions_by_class = [(tf.equal(labels, c), tf.equal(predictions, c)) for c in range(0, n_classes)]
tp_by_class_val, tp_by_class_update_op = zip(*[tf.metrics.true_positives(label, prediction, weights=weights)
for label, prediction in labels_and_predictions_by_class])
fn_by_class_val, fn_by_class_update_op = zip(*[tf.metrics.false_negatives(label, prediction, weights=weights)
for label, prediction in labels_and_predictions_by_class])
fp_by_class_val, fp_by_class_update_op = zip(*[tf.metrics.false_positives(label, prediction, weights=weights)
for label, prediction in labels_and_predictions_by_class])
f1_update_op = tf.group(*chain(tp_by_class_update_op, fn_by_class_update_op, fp_by_class_update_op))
if type == 'macro':
epsilon = [10e-6 for _ in range(n_classes)]
f1_val = tf.multiply(2., tp_by_class_val) / (tf.reduce_sum([tf.multiply(2., tp_by_class_val),
fp_by_class_val, fn_by_class_val, epsilon],
axis=0))
f1_val = tf.reduce_mean(f1_val)
else:
epsilon = 10e-6
total_tp = tf.reduce_sum(tp_by_class_val)
total_fn = tf.reduce_sum(fn_by_class_val)
total_fp = tf.reduce_sum(fp_by_class_val)
f1_val = tf.squeeze(tf.multiply(2., total_tp) / (tf.multiply(2., total_tp) +
total_fp + total_fn + epsilon,
))
return f1_val, f1_update_op
def check_integrity_and_batch(*datasets):
"""Checks whether a sequence of frames are from the same video.
Args:
*datasets: datasets each skipping 1 frame from the previous one.
Returns:
batched data and the integrity flag.
"""
not_broken = tf.constant(True)
if "frame_number" in datasets[0]:
frame_numbers = [dataset["frame_number"][0] for dataset in datasets]
not_broken = tf.equal(
frame_numbers[-1] - frame_numbers[0], num_frames-1)
if self.only_keep_videos_from_0th_frame:
not_broken = tf.logical_and(not_broken,
tf.equal(frame_numbers[0], 0))
else:
tf.logging.warning("use_not_breaking_batching is True but "
"no frame_number is in the dataset.")
features = {}
for key in datasets[0].keys():
values = [dataset[key] for dataset in datasets]
batch = tf.stack(values)
features[key] = batch
return features, not_broken
def training_control(global_step, print_span, evaluation_span, max_step, name=None):
with tf.name_scope(name, "training_control"):
return {
"step": global_step,
"time_to_print": tf.equal(tf.mod(global_step, print_span), 0),
"time_to_evaluate": tf.equal(tf.mod(global_step, evaluation_span), 0),
"time_to_stop": tf.greater_equal(global_step, max_step),
}
def body(i, prev_base_states, prev_high_states, prev_ys, prev_embs, cost, ys_array, p_array):
# get predictions from all models and sum the log probs
sum_log_probs = None
base_states = [None] * len(models)
high_states = [None] * len(models)
for j in range(len(models)):
d = models[j].decoder
states1 = d.grustep1.forward(prev_base_states[j], prev_embs[j])
att_ctx = d.attstep.forward(states1)
base_states[j] = d.grustep2.forward(states1, att_ctx)
if d.high_gru_stack == None:
stack_output = base_states[j]
high_states[j] = []
else:
if d.high_gru_stack.context_state_size == 0:
stack_output, high_states[j] = d.high_gru_stack.forward_single(
prev_high_states[j], base_states[j])
else:
stack_output, high_states[j] = d.high_gru_stack.forward_single(
prev_high_states[j], base_states[j], context=att_ctx)
logits = d.predictor.get_logits(prev_embs[j], stack_output,
att_ctx, multi_step=False)
log_probs = tf.nn.log_softmax(logits) # shape (batch, vocab_size)
if sum_log_probs == None:
sum_log_probs = log_probs
else:
sum_log_probs += log_probs
# set cost of EOS to zero for completed sentences so that they are in top k
# Need to make sure only EOS is selected because a completed sentence might
# kill ongoing sentences
sum_log_probs = tf.where(tf.equal(prev_ys, 0), eos_log_probs, sum_log_probs)
all_costs = sum_log_probs + tf.expand_dims(cost, axis=1) # TODO: you might be getting NaNs here since -inf is in log_probs
all_costs = tf.reshape(all_costs,
shape=[-1, target_vocab_size * beam_size])
values, indices = tf.nn.top_k(all_costs, k=beam_size) #the sorted option is by default True, is this needed?
new_cost = tf.reshape(values, shape=[batch_size])
offsets = tf.range(
start = 0,
delta = beam_size,
limit = batch_size,
dtype=tf.int32)
offsets = tf.expand_dims(offsets, axis=1)
survivor_idxs = (indices // target_vocab_size) + offsets
new_ys = indices % target_vocab_size
survivor_idxs = tf.reshape(survivor_idxs, shape=[batch_size])
new_ys = tf.reshape(new_ys, shape=[batch_size])
new_embs = [m.decoder.y_emb_layer.forward(new_ys, factor=0) for m in models]
new_base_states = [tf.gather(s, indices=survivor_idxs) for s in base_states]
new_high_states = [[tf.gather(s, indices=survivor_idxs) for s in states] for states in high_states]
new_cost = tf.where(tf.equal(new_ys, 0), tf.abs(new_cost), new_cost)
ys_array = ys_array.write(i, value=new_ys)
p_array = p_array.write(i, value=survivor_idxs)
return i+1, new_base_states, new_high_states, new_ys, new_embs, new_cost, ys_array, p_array
def expected_classification_loss_under_sampling(batch_cls_targets, cls_losses,
desired_negative_sampling_ratio,
minimum_negative_sampling):
"""Computes classification loss by background/foreground weighting.
The weighting is such that the effective background/foreground weight ratio
is the desired_negative_sampling_ratio. if p_i is the foreground probability
of anchor a_i, L(a_i) is the anchors loss, N is the number of anchors, and M
is the sum of foreground probabilities across anchors, then the total loss L
is calculated as:
beta = K*M/(N-M)
L = sum_{i=1}^N [p_i + beta * (1 - p_i)] * (L(a_i))
Args:
batch_cls_targets: A tensor with shape [batch_size, num_anchors,
num_classes + 1], where 0'th index is the background class, containing
the class distrubution for the target assigned to a given anchor.
cls_losses: Float tensor of shape [batch_size, num_anchors]
representing anchorwise classification losses.
desired_negative_sampling_ratio: The desired background/foreground weight
ratio.
minimum_negative_sampling: Minimum number of effective negative samples.
Used only when there are no positive examples.
Returns:
The classification loss.
"""
num_anchors = tf.cast(tf.shape(batch_cls_targets)[1], tf.float32)
# find the p_i
foreground_probabilities = (
foreground_probabilities_from_targets(batch_cls_targets))
foreground_sum = tf.reduce_sum(foreground_probabilities, axis=-1)
k = desired_negative_sampling_ratio
# compute beta
denominators = (num_anchors - foreground_sum)
beta = tf.where(
tf.equal(denominators, 0), tf.zeros_like(foreground_sum),
k * foreground_sum / denominators)
# where the foreground sum is zero, use a minimum negative weight.
min_negative_weight = 1.0 * minimum_negative_sampling / num_anchors
beta = tf.where(
tf.equal(foreground_sum, 0), min_negative_weight * tf.ones_like(beta),
beta)
beta = tf.reshape(beta, [-1, 1])
cls_loss_weights = foreground_probabilities + (
1 - foreground_probabilities) * beta
weighted_losses = cls_loss_weights * cls_losses
cls_losses = tf.reduce_sum(weighted_losses, axis=-1)
return cls_losses
def _extract_features(self, preprocessed_inputs):
"""Extract features from preprocessed inputs.
Args:
preprocessed_inputs: a [batch, height, width, channels] float tensor
representing a batch of images.
Returns:
feature_maps: a list of tensors where the ith tensor has shape
[batch, height_i, width_i, depth_i]
Raises:
ValueError: if image height or width are not 256 pixels.
"""
image_shape = preprocessed_inputs.get_shape()
image_shape.assert_has_rank(4)
image_height = image_shape[1].value
image_width = image_shape[2].value
if image_height is None or image_width is None:
shape_assert = tf.Assert(
tf.logical_and(tf.equal(tf.shape(preprocessed_inputs)[1], 256),
tf.equal(tf.shape(preprocessed_inputs)[2], 256)),
['image size must be 256 in both height and width.'])
with tf.control_dependencies([shape_assert]):
preprocessed_inputs = tf.identity(preprocessed_inputs)
elif image_height != 256 or image_width != 256:
raise ValueError('image size must be = 256 in both height and width;'
' image dim = %d,%d' % (image_height, image_width))
feature_map_layout = {
'from_layer': [
'Conv2d_11_pointwise', 'Conv2d_13_pointwise', '', '', ''
],
'layer_depth': [-1, -1, 512, 256, 256],
'conv_kernel_size': [-1, -1, 3, 3, 2],
'use_explicit_padding': self._use_explicit_padding,
'use_depthwise': self._use_depthwise,
}
with slim.arg_scope(self._conv_hyperparams):
with slim.arg_scope([slim.batch_norm], fused=False):
with tf.variable_scope('MobilenetV1',
reuse=self._reuse_weights) as scope:
_, image_features = mobilenet_v1.mobilenet_v1_base(
ops.pad_to_multiple(preprocessed_inputs, self._pad_to_multiple),
final_endpoint='Conv2d_13_pointwise',
min_depth=self._min_depth,
depth_multiplier=self._depth_multiplier,
scope=scope)
feature_maps = feature_map_generators.multi_resolution_feature_maps(
feature_map_layout=feature_map_layout,
depth_multiplier=self._depth_multiplier,
min_depth=self._min_depth,
insert_1x1_conv=True,
image_features=image_features)
return feature_maps.values()
def train_test_and_save_model():
X_train, y_train, X_test, y_test = load_and_preprocess_imdb_data(N_GRAM)
model = FastTextModel(
vocab_size=VOCAB_SIZE + N_BUCKETS,
embedding_size=EMBEDDING_SIZE,
n_labels=2,
)
optimizer = tf.optimizers.Adam(learning_rate=LEARNING_RATE)
if os.path.exists(MODEL_FILE_PATH):
# loading pre-trained model if applicable
model.load_weights(MODEL_FILE_PATH)
else:
# training
model.train()
for epoch in range(N_EPOCH):
start_time = time.time()
print('Epoch %d/%d' % (epoch + 1, N_EPOCH))
train_accuracy = list()
for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size=BATCH_SIZE, shuffle=True):
# forward and define the loss function
# TODO: use tf.function to speed up
with tf.GradientTape() as tape:
y_pred = model(tl.prepro.pad_sequences(X_batch))
cost = tl.cost.cross_entropy(y_pred, y_batch, name='cost')
# backward, calculate gradients and update the weights
grad = tape.gradient(cost, model.weights)
optimizer.apply_gradients(zip(grad, model.weights))
# calculate the accuracy
predictions = tf.argmax(y_pred, axis=1, output_type=tf.int32)
are_predictions_correct = tf.equal(predictions, y_batch)
accuracy = tf.reduce_mean(tf.cast(are_predictions_correct, tf.float32))
train_accuracy.append(accuracy)
if len(train_accuracy) % N_STEPS_TO_PRINT == 0:
print("\t[%d/%d][%d]accuracy " % (epoch + 1, N_EPOCH, len(train_accuracy)),
np.mean(train_accuracy[-N_STEPS_TO_PRINT:]))
print("\tSummary: time %.5fs, overall accuracy" % (time.time() - start_time),
np.mean(train_accuracy))
# evaluation and testing
model.eval()
# forward and calculate the accuracy
y_pred = model(tl.prepro.pad_sequences(X_test))
predictions = tf.argmax(y_pred, axis=1, output_type=tf.int32)
are_predictions_correct = tf.equal(predictions, y_test)
test_accuracy = tf.reduce_mean(tf.cast(are_predictions_correct, tf.float32))
print('Test accuracy: %.5f' % test_accuracy)
# saving the model
model.save_weights(MODEL_FILE_PATH)
def _build_once(self, dataset, feature_transformer):
with tf.device(self._local_device):
tr_batch = dataset()
te_batch = dataset()
num_classes = tr_batch.label_onehot.shape.as_list()[1]
all_batch = utils.structure_map_multi(lambda x: tf.concat(x, 0),
[tr_batch, te_batch])
features = feature_transformer(all_batch)
trX, teX = utils.structure_map_split(lambda x: tf.split(x, 2, axis=0),
features)
trY = tf.to_int64(tr_batch.label)
trY_onehot = tf.to_int32(tr_batch.label_onehot)
teY = tf.to_int64(te_batch.label)
teY_shape = teY.shape.as_list()
def blackbox((trX, trY, teX, teY)):
trY = tf.to_int32(tf.rint(trY))
teY = tf.to_int32(tf.rint(teY))
tf_fn = build_fit(
self._local_device,
self._get_model,
num_classes=num_classes,
probs=self.probs)
if self.probs:
trP, teP, teP_probs = tf_fn(trX, trY, teX)
else:
trP, teP = tf_fn(trX, trY, teX)
teY.set_shape(teY_shape)
if self.probs:
onehot = tf.one_hot(teY, num_classes)
crossent = -tf.reduce_sum(onehot * teP_probs, [1])
return tf.reduce_mean(crossent)
else:
# use error rate as the loss if no surrogate is avalible.
return 1 - tf.reduce_mean(
tf.to_float(tf.equal(teY, tf.to_int32(teP))))
test_loss = blackbox((trX, tf.to_float(trY), teX, tf.to_float(teY)))
stats = {}
tf_fn = build_fit(
self._local_device,
self._get_model,
num_classes=num_classes,
probs=self.probs)
if self.probs:
trP, teP, teP_probs = tf_fn(trX, trY, teX)
else:
trP, teP = tf_fn(trX, trY, teX)
stats["%s/accuracy_train" % self.name] = tf.reduce_mean(
tf.to_float(tf.equal(tf.to_int32(trY), tf.to_int32(trP))))
stats["%s/accuracy_test" % self.name] = tf.reduce_mean(
tf.to_float(tf.equal(tf.to_int32(teY), tf.to_int32(teP))))
stats["%s/test_loss" % self.name] = test_loss
return test_loss, stats
def _count_condition(name, labels_value, predicted_value):
"""Creates a counter for given values of predictions and labels."""
count = _metric_variable(name, [], tf.float32)
is_equal = tf.to_float(
tf.logical_and(
tf.equal(labels, labels_value),
tf.equal(predicted_labels, predicted_value)))
update_op = tf.assign_add(count, tf.reduce_sum(weights * is_equal))
return count.read_value(), update_op
def accuracy(log, w1, w2, w3):
with tf.name_scope('accuracy') as scope:
c1 = tf.equal(tf.argmax(log, 1), tf.argmax(w1, 1))
c2 = tf.equal(tf.argmax(log, 1), tf.argmax(w2, 1))
c3 = tf.equal(tf.argmax(log, 1), tf.argmax(w3, 1))
correct_prediction = tf.logical_or(tf.logical_or(c1,c2),c3)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
tf.scalar_summary("accuracy", accuracy)
return accuracy
def get_pos_and_neg_masks(labels):
if config.train_with_ignored:
pos_mask = labels >= 0
neg_mask = tf.logical_not(pos_mask)
else:
pos_mask = tf.equal(labels, 1)
neg_mask = tf.equal(labels, -1)
return pos_mask, neg_mask
def equal(x, y):
return tf.equal(x, y)
def main():
# [b, 32, 32, 3] => [b, 1, 1, 512]
conv_net = Sequential(conv_layers)
fc_net = Sequential([
layers.Dense(256, activation=tf.nn.relu),
layers.Dense(128, activation=tf.nn.relu),
layers.Dense(100, activation=None),
])
conv_net.build(input_shape=[None, 32, 32, 3])
fc_net.build(input_shape=[None, 512])
optimizer = optimizers.Adam(lr=1e-4)
# [1, 2] + [3, 4] => [1, 2, 3, 4]
variables = conv_net.trainable_variables + fc_net.trainable_variables
for epoch in range(50):
for step, (x, y) in enumerate(train_db):
with tf.GradientTape() as tape:
# [b, 32, 32, 3] => [b, 1, 1, 512]
out = conv_net(x)
# flatten, => [b, 512]
out = tf.reshape(out, [-1, 512])
# [b, 512] => [b, 100]
logits = fc_net(out)
# [b] => [b, 100]
y_onehot = tf.one_hot(y, depth=100)
# compute loss
loss = tf.losses.categorical_crossentropy(y_onehot,
logits,
from_logits=True)
loss = tf.reduce_mean(loss)
grads = tape.gradient(loss, variables)
optimizer.apply_gradients(zip(grads, variables))
if step % 100 == 0:
print(epoch, step, 'loss:', float(loss))
total_num = 0
total_correct = 0
for x, y in test_db:
out = conv_net(x)
out = tf.reshape(out, [-1, 512])
logits = fc_net(out)
prob = tf.nn.softmax(logits, axis=1)
pred = tf.argmax(prob, axis=1)
pred = tf.cast(pred, dtype=tf.int32)
correct = tf.cast(tf.equal(pred, y), dtype=tf.int32)
correct = tf.reduce_sum(correct)
total_num += x.shape[0]
total_correct += int(correct)
acc = total_correct / total_num
print(epoch, 'acc:', acc)
with tf.name_scope('Loss'):
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, labels=y_))
l2 = tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
loss = cost + l2
tf.summary.scalar('loss', loss)
with tf.name_scope('Train_step'):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_step = tf.train.MomentumOptimizer(learning_rate, momentum=momentum, use_nesterov=True).minimize(loss,
global_step)
with tf.name_scope('Accuracy'):
y_conv_softmax = tf.nn.softmax(y_conv)
distribution = [tf.arg_max(y_, 1), tf.arg_max(y_conv_softmax, 1)]
correct_prediction = tf.equal(distribution[0], distribution[1])
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
with tf.variable_scope('Saver'):
var_list = tf.trainable_variables()
g_list = tf.global_variables()
bn_moving_vars = [g for g in g_list if 'moving_mean' in g.name]
bn_moving_vars += [g for g in g_list if 'moving_variance' in g.name]
var_list += bn_moving_vars
saver = tf.train.Saver(var_list=var_list, max_to_keep=epoch)
with tf.variable_scope('Summary_writer'):
merged = tf.summary.merge_all()
writer_training = tf.summary.FileWriter(checkpoint_dir, sess.graph)
if Load_Data:
#gradient descent
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_step)
with tf.control_dependencies([train_step]):
train_op = tf.no_op(name = 'train')
'''
learning_rate = tf.train.exponential_decay(LEARING_RATE_BASE, global_step, 100,
LEARNING_RATE_DECAY)
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
# Create optimizer and apply gradient descent to the trainable variables
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients,
global_step=global_step)
correct_prediction = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
saver = tf.train.Saver()
with tf.Session() as sess:
tf.initialize_all_variables().run()
#load the trained model
if os.path.isfile(MODEL_SAVE_PATH + 'checkpoint'):
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = int(
ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1])
print("Continue training!")
else:
def train():
avg_accuracy = 0
mnist = input_data.read_data_sets(FLAGS.data_dir,
fake_data=FLAGS.fake_data)
sess = tf.InteractiveSession()
# Input placeholders
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, 784], name='x-input')
x_quantized = fake_quantize_tensor(x,
input_quantization,
0,
1,
name="quantized_input")
y_ = tf.placeholder(tf.int64, [None], name='y-input')
with tf.name_scope('input_reshape'):
image_shaped_input = tf.Variable(
[-1, 28, 28, 1], collections=[tf.GraphKeys.GLOBAL_VARIABLES])
image_shaped_input = tf.reshape(x, [-1, 28, 28, 1])
tf.summary.image('input', image_shaped_input, 10)
# Functions to create fc and conv layers depending on quantization settings
def create_fc_layer(input_tensor,
input_dim,
output_dim,
bits_w,
max_w,
bits_b,
max_b,
bits_a,
max_a,
noise_stddev,
layer_name,
act=tf.nn.relu):
if (quantization_enabled == False):
return fc_layer(input_tensor, input_dim, output_dim, layer_name,
act)
elif (noise_enabled_fc == False):
return fc_layer_quantized(input_tensor, input_dim, output_dim,
bits_w, max_w, bits_b, max_b, bits_a,
max_a, layer_name, act)
else:
return fc_layer_quantized_add_noise(input_tensor, input_dim,
output_dim, bits_w, max_w,
bits_b, max_b, bits_a, max_a,
noise_stddev, layer_name, act)
def create_conv_layer(input_data, num_input_channels, num_filters,
filter_shape, pool_shape, bits_w, max_w, bits_b,
max_b, bits_a, max_a, noise_stddev, layer_name):
if (quantization_enabled == False):
return conv_layer(input_data, num_input_channels, num_filters,
filter_shape, pool_shape, layer_name)
elif (noise_enabled_conv == False):
return conv_layer_quantized(input_data, num_input_channels,
num_filters, filter_shape, pool_shape,
bits_w, max_w, bits_b, max_b, bits_a,
max_a, layer_name)
else:
return conv_layer_quantized_add_noise(input_data,
num_input_channels,
num_filters, filter_shape,
pool_shape, bits_w, max_w,
bits_b, max_b, bits_a, max_a,
noise_stddev, layer_name)
if (conv_enabled):
if (quantization_enabled):
image_shaped_input_quantized = fake_quantize_tensor(
image_shaped_input,
input_quantization,
0,
1,
name="quantized_input")
else:
image_shaped_input_quantized = image_shaped_input
layer1 = create_conv_layer(image_shaped_input_quantized,
1,
32, [5, 5], [2, 2],
conv1_w_bits,
conv1_w_max,
conv1_b_bits,
conv1_b_max,
conv1_a_bits,
conv1_a_max,
noise_stddev,
layer_name='conv1')
layer2 = create_conv_layer(layer1,
32,
64, [5, 5], [2, 2],
conv2_w_bits,
conv2_w_max,
conv2_b_bits,
conv2_b_max,
conv2_a_bits,
conv2_a_max,
noise_stddev,
layer_name='conv2')
with tf.name_scope('flatten'):
x_flattened = tf.reshape(layer2, [-1, 7 * 7 * 64])
x_shape = 7 * 7 * 64
else:
if (quantization_enabled):
x_flattened = fake_quantize_tensor(x,
input_quantization,
0,
1,
name="quantized_input")
else:
x_flattened = x
x_shape = 28 * 28
if (num_layers == 3):
hidden1 = create_fc_layer(x_flattened, x_shape, fc1_depth, fc1_w_bits,
fc1_w_max, fc1_b_bits, fc1_b_max, fc1_a_bits,
fc1_a_max, noise_stddev, 'fully_connected1')
with tf.name_scope('dropout'):
keep_prob = tf.placeholder(tf.float32)
tf.summary.scalar('dropout_keep_probability', keep_prob)
dropped = tf.nn.dropout(hidden1, keep_prob)
# Middle Layer (using settings 3)
hidden2 = create_fc_layer(dropped, fc1_depth, fc3_depth, fc3_w_bits,
fc3_w_max, fc3_b_bits, fc3_b_max, fc3_a_bits,
fc3_a_max, noise_stddev,
'fully_connected_middle')
with tf.name_scope('dropout2'):
keep_prob2 = tf.placeholder(tf.float32)
tf.summary.scalar('dropout2_keep_probability', keep_prob2)
dropped2 = tf.nn.dropout(hidden2, keep_prob2)
# Do not apply softmax activation yet, see below.
y = create_fc_layer(dropped2,
fc3_depth,
fc2_depth,
fc2_w_bits,
fc2_w_max,
fc2_b_bits,
fc2_b_max,
fc2_a_bits,
fc2_a_max,
noise_stddev,
'fully_connected2',
act=tf.identity)
elif (num_layers == 1):
keep_prob = tf.placeholder(tf.float32) # Need to create placeholders
keep_prob2 = tf.placeholder(
tf.float32) # Even though they wont be used
y = create_fc_layer(x_flattened,
x_shape,
fc2_depth,
fc2_w_bits,
fc2_w_max,
fc2_b_bits,
fc2_b_max,
fc2_a_bits,
fc2_a_max,
noise_stddev,
layer_name='fully_connected',
act=tf.identity)
else: # Otherwise create 2 layers
hidden1 = create_fc_layer(x_flattened, x_shape, fc1_depth, fc1_w_bits,
fc1_w_max, fc1_b_bits, fc1_b_max, fc1_a_bits,
fc1_a_max, noise_stddev, 'fully_connected1')
with tf.name_scope('dropout'):
keep_prob = tf.placeholder(tf.float32)
tf.summary.scalar('dropout_keep_probability', keep_prob)
dropped = tf.nn.dropout(hidden1, keep_prob)
keep_prob2 = tf.placeholder(tf.float32) # Need to create a placholder
y = create_fc_layer(dropped,
fc1_depth,
fc2_depth,
fc2_w_bits,
fc2_w_max,
fc2_b_bits,
fc2_b_max,
fc2_a_bits,
fc2_a_max,
noise_stddev,
'fully_connected2',
act=tf.identity)
with tf.name_scope('cross_entropy'):
# The raw formulation of cross-entropy can be numerically unstable.
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.softmax(y)),
# reduction_indices=[1]))
#
# So here we use tf.losses.sparse_softmax_cross_entropy on the
# raw logit outputs of the nn_layer above, and then average across the batch.
with tf.name_scope('total'):
cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_,
logits=y)
tf.summary.scalar('cross_entropy', cross_entropy)
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(
FLAGS.learning_rate).minimize(cross_entropy)
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
correct_prediction = tf.equal(tf.argmax(y, 1), y_)
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# Merge all the summaries and write them out
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test')
tf.global_variables_initializer().run()
# Train the model, and also write summaries.
# Every 10th step, measure test-set accuracy, and write test summaries
# All other steps, run train_step on training data, & add training summaries
def feed_dict(train, batch_size=100): # 128
"""Make a TensorFlow feed_dict: maps data onto Tensor placeholders."""
if train or FLAGS.fake_data:
xs, ys = mnist.train.next_batch(batch_size,
fake_data=FLAGS.fake_data)
k = FLAGS.dropout
else:
xs, ys = mnist.test.images, mnist.test.labels
k = 1.0
return {x: xs, y_: ys, keep_prob: k, keep_prob2: k}
for i in range(FLAGS.max_steps + 5):
if (i >= FLAGS.max_steps):
sess.run([merged, train_step], feed_dict=feed_dict(True))
summary, acc = sess.run([merged, accuracy],
feed_dict=feed_dict(False))
test_writer.add_summary(summary, i)
print('Accuracy test %s: %s' % (i - FLAGS.max_steps, acc))
avg_accuracy += acc
elif (i % 100 == 99 and record_summaries == True
and i <= 1000): # Record summaries and test-set accuracy
summary, acc = sess.run([merged, accuracy],
feed_dict=feed_dict(False))
test_writer.add_summary(summary, i)
print(datetime.datetime.now().strftime("%H:%M:%S"),
'Accuracy at step %s: %s' % (i, acc))
elif (i % 1000 == 999): # Record summaries and test-set accuracy
summary, acc = sess.run([merged, accuracy],
feed_dict=feed_dict(False))
test_writer.add_summary(summary, i)
print(datetime.datetime.now().strftime("%H:%M:%S"),
'Accuracy at step %s: %s' % (i, acc))
elif (i % 100 == 0):
# print(datetime.datetime.now().strftime("%H:%M:%S"), "Adding run metadata for Step: ", i)
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
summary, _ = sess.run([merged, train_step],
feed_dict=feed_dict(True),
options=run_options,
run_metadata=run_metadata)
train_writer.add_run_metadata(run_metadata, 'step%03d' % i)
train_writer.add_summary(summary, i)
else: # Train and record summary
summary, _ = sess.run([merged, train_step],
feed_dict=feed_dict(True))
train_writer.add_summary(summary, i)
print("Training Completed: ", datetime.datetime.now().strftime("%H:%M:%S"))
print('Final accuracy: ', avg_accuracy / 5)
x_in, x_flat, y_out = sess.run([x, x_flattened, y],
feed_dict=feed_dict(True, batch_size=1))
np.savetxt("input.csv", x_in[0], delimiter=",", fmt='%f')
np.savetxt("input_reshaped_quantized.csv",
x_flat[0],
delimiter=",",
fmt='%f')
np.savetxt("output.csv", y_out[0], delimiter=",", fmt='%f')
for var in tf.global_variables():
if num_layers == 1:
if '/weights/Variable:0' in var.name:
print(var)
v = sess.run(var) #get_weights(sess.run(var))
np.savetxt("Parameters/weights.csv",
v,
delimiter=",",
fmt='%f')
np.savetxt("Parameters/Q.csv",
calculate_Q(0.066667, v).astype(int),
delimiter=",",
fmt='%i')
elif '/biases/Variable:0' in var.name:
print(var)
v = sess.run(var) # get_biases(sess.run(var))
np.savetxt("Parameters/floating_biases.csv",
v,
delimiter=",",
fmt='%f') # Store the floating point biases
np.savetxt("Parameters/biases.csv",
get_biases(v),
delimiter=",",
fmt='%f') # Store the fixed point biases
else:
print(
"Fixed point inference not implemented yet for networks with more than one layer"
)
if 'fully_connected1/weights/Variable:0' in var.name:
print(var)
elif 'fully_connected1/biases/Variable:0' in var.name:
print(var)
elif 'fully_connected2/weights/Variable:0' in var.name:
print(var)
elif 'fully_connected2/biases/Variable:0' in var.name:
print(var)
elif 'fully_connected3/weights/Variable:0' in var.name:
print(var)
elif 'fully_connected3/biases/Variable:0' in var.name:
print(var)
train_writer.close()
model = tf.matmul(L2, W3) + b3
tf.summary.histogram('W3', W3)
with tf.name_scope('optimizer'):
base_cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=model, labels=Y))
lossL2 = tf.reduce_mean(tf.nn.l2_loss(W1) +
tf.nn.l2_loss(W2) + tf.nn.l2_loss(W3)) * L2beta
cost = base_cost + lossL2
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
tf.summary.scalar('cost', cost)
with tf.name_scope("accuracy"):
is_correct = tf.equal(tf.argmax(model, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(is_correct, tf.float32))
tf.summary.scalar("accuracy", accuracy)
summ = tf.summary.merge_all()
saver = tf.train.Saver()
sess = tf.Session()
total_model_test = 10
for model in range(total_model_test):
random_learning_rate = HP_np[model][1]
random_L2beta = HP_np[model][2]
sess.run(tf.global_variables_initializer())
out_flat = tf.nn.relu(out_flat)
out_final = dense_layer(out_flat,
shape_w=[1024, n_class],
shape_b=[n_class],
name='last')
print("out final: ", out_final.get_shape())
pred = tf.nn.softmax(out_final)
err1 = tf.nn.softmax_cross_entropy_with_logits_v2(labels=target,
logits=out_final)
err = tf.reduce_mean(err1) + tf.losses.get_regularization_loss()
err_val = tf.reduce_mean(err1)
print("error: ", err1.get_shape(), err.get_shape())
correct_pred = tf.cast(tf.equal(tf.argmax(pred, 1), tf.argmax(target, 1)),
tf.float32)
accuracy = tf.reduce_mean(correct_pred)
global_step = tf.train.get_or_create_global_step()
train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(
err, global_step=global_step)
saver = tf.train.Saver(max_to_keep=10)
init = tf.global_variables_initializer()
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(MDPATH)
if ckpt and ckpt.model_checkpoint_path:
print(ckpt, ckpt.model_checkpoint_path)
sess.run(init)
optimistic_restore(sess, MDPATH)
x_data = [[1, 2], [2, 3], [3, 1], [4, 3], [5, 3], [6, 2]]
y_data = [[0], [0], [0], [1], [1], [1]]
X = tf.placeholder(tf.float32, shape=[None, 2])
Y = tf.placeholder(tf.float32, shape=[None, 1])
W = tf.Variable(tf.random_normal([2, 1]), name='weight')
b = tf.Variable(tf.random_normal([1]), name='bias')
hypothesis = tf.sigmoid(tf.matmul(X, W) + b)
cost = -1 * tf.reduce_mean(Y * tf.log(hypothesis) + (1 - Y) * tf.log(1 - hypothesis))
train = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(cost)
predicted = tf.cast(hypothesis > 0.5, dtype=tf.float32)
accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, Y), dtype=tf.float32))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for step in range(10001):
cost_val, _ = sess.run([cost, train],
feed_dict={X: x_data, Y: y_data})
if step % 200 == 0:
print(step, cost_val)
h, c, a = sess.run([hypothesis, cost, accuracy],
feed_dict={X: x_data, Y: y_data})
print("\nhypothesis : ", h,
"\ncost : ", c,
"\naccuracy : ", a)
num_outputs=fc_layer_size,
use_relu=True)
layer_fc2 = create_fc_layer(input=layer_fc1,
num_inputs=fc_layer_size,
num_outputs=num_classes,
use_relu=False)
y_pred = tf.nn.softmax(layer_fc2, name='y_pred')
y_pred_cls = tf.argmax(y_pred, dimension=1)
session.run(tf.global_variables_initializer())
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=layer_fc2,
labels=y_true)
cost = tf.reduce_mean(cross_entropy)
optimizer = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cost) #学习率
correct_prediction = tf.equal(y_pred_cls, y_true_cls)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
session.run(tf.global_variables_initializer())
def show_progress(epoch, feed_dict_train, feed_dict_validate, val_loss):
acc = session.run(accuracy, feed_dict=feed_dict_train)
val_acc = session.run(accuracy, feed_dict=feed_dict_validate)
msg = "Training Epoch {0} --- Training Accuracy: {1:>6.1%}, Validation Accuracy: {2:>6.1%}, Validation Loss: {3:.3f}"
print(msg.format(epoch + 1, acc, val_acc, val_loss))
total_iterations = 0
saver = tf.train.Saver()
b1=tf.Variable(tf.zeros([n_hidden]))
hidden=tf.sigmoid(tf.matmul(input, W1)+b1)
W2=tf.Variable(tf.zeros([n_hidden, 10]))
b2=tf.Variable(tf.zeros([10]))
output=tf.nn.softmax(tf.matmul(hidden,W2)+b2)
loss=-tf.reduce_sum(label*tf.log(output))
train_step=tf.train.AdamOptimizer().minimize(loss)
correct_prediction=tf.equal(tf.argmax(output,1), tf.argmax(label,1))
accuracy=tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
plt.title('Hidden layer size %d'%n_hidden)
plt.xlabel('Train step')
plt.ylabel('Accuracy')
for k in range(5):
tf.global_variables_initializer().run()
log=np.zeros([2,101])
for i in range(10000):
if i%100==0:
accuracy_eval=accuracy.eval({input:mnist.test.images, label:mnist.test.labels})
print('step %d, accuracy %s'%(i,accuracy_eval))
log[0][i/100]=i
log[1][i/100]=accuracy_eval
x, y=mnist.train.next_batch(100)
def contrastive_loss(hidden,
num_replicas,
normalize_hidden,
temperature,
model,
weight_decay):
"""Computes contrastive loss.
Args:
hidden: embedding of video clips after projection head.
num_replicas: number of distributed replicas.
normalize_hidden: whether or not to l2 normalize the hidden vector.
temperature: temperature in the InfoNCE contrastive loss.
model: keras model for calculating weight decay.
weight_decay: weight decay parameter.
Returns:
A loss scalar.
The logits for contrastive prediction task.
The labels for contrastive prediction task.
"""
large_num = 1e9
hidden1, hidden2 = tf.split(hidden, num_or_size_splits=2, axis=0)
if normalize_hidden:
hidden1 = tf.math.l2_normalize(hidden1, -1)
hidden2 = tf.math.l2_normalize(hidden2, -1)
batch_size = tf.shape(hidden1)[0]
if num_replicas == 1:
# This is the local version
hidden1_large = hidden1
hidden2_large = hidden2
labels = tf.one_hot(tf.range(batch_size), batch_size * 2)
masks = tf.one_hot(tf.range(batch_size), batch_size)
else:
# This is the cross-tpu version.
hidden1_large = tpu_cross_replica_concat(hidden1, num_replicas)
hidden2_large = tpu_cross_replica_concat(hidden2, num_replicas)
enlarged_batch_size = tf.shape(hidden1_large)[0]
replica_id = tf.cast(tf.cast(xla.replica_id(), tf.uint32), tf.int32)
labels_idx = tf.range(batch_size) + replica_id * batch_size
labels = tf.one_hot(labels_idx, enlarged_batch_size * 2)
masks = tf.one_hot(labels_idx, enlarged_batch_size)
logits_aa = tf.matmul(hidden1, hidden1_large, transpose_b=True) / temperature
logits_aa = logits_aa - tf.cast(masks, logits_aa.dtype) * large_num
logits_bb = tf.matmul(hidden2, hidden2_large, transpose_b=True) / temperature
logits_bb = logits_bb - tf.cast(masks, logits_bb.dtype) * large_num
logits_ab = tf.matmul(hidden1, hidden2_large, transpose_b=True) / temperature
logits_ba = tf.matmul(hidden2, hidden1_large, transpose_b=True) / temperature
loss_a = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
labels, tf.concat([logits_ab, logits_aa], 1)))
loss_b = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
labels, tf.concat([logits_ba, logits_bb], 1)))
loss = loss_a + loss_b
l2_loss = weight_decay * tf.add_n([
tf.nn.l2_loss(v)
for v in model.trainable_variables
if 'kernel' in v.name
])
total_loss = loss + tf.cast(l2_loss, loss.dtype)
contrast_prob = tf.nn.softmax(logits_ab)
contrast_entropy = - tf.reduce_mean(
tf.reduce_sum(contrast_prob * tf.math.log(contrast_prob + 1e-8), -1))
contrast_acc = tf.equal(tf.argmax(labels, 1), tf.argmax(logits_ab, axis=1))
contrast_acc = tf.reduce_mean(tf.cast(contrast_acc, tf.float32))
return {
'total_loss': total_loss,
'contrastive_loss': loss,
'reg_loss': l2_loss,
'contrast_acc': contrast_acc,
'contrast_entropy': contrast_entropy,
}
sess.run(tf.global_variables_initializer())
# parameters
training_epochs = 15
batch_size = 100
# Check out https://www.tensorflow.org/get_started/mnist/beginners for
# more information about the mnist dataset
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
nb_classes = 10
# train my model
print('Learning stared. It takes sometime.')
for epoch in range(training_epochs):
avg_cost = 0
total_batch = int(mnist.train.num_examples / batch_size)
for i in range(total_batch):
batch_xs, batch_ys = mnist.train.next_batch(batch_size)
feed_dict = {X: batch_xs, Y: batch_ys}
c, _, = sess.run([cost, optimizer], feed_dict=feed_dict)
avg_cost += c / total_batch
print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_cost))
print('Learning Finished!')
# Test model and check accuracy
correct_prediction = tf.equal(tf.argmax(hypothesis, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print('Accuracy:', sess.run(accuracy, feed_dict={X: mnist.test.images, Y: mnist.test.labels}))
def bboxes_matching(label, scores, bboxes,
glabels, gbboxes, gdifficults,
matching_threshold=0.5, scope=None):
"""Matching a collection of detected boxes with groundtruth values.
Does not accept batched-inputs.
The algorithm goes as follows: for every detected box, check
if one grountruth box is matching. If none, then considered as False Positive.
If the grountruth box is already matched with another one, it also counts
as a False Positive. We refer the Pascal VOC documentation for the details.
Args:
rclasses, rscores, rbboxes: N(x4) Tensors. Detected objects, sorted by score;
glabels, gbboxes: Groundtruth bounding boxes. May be zero padded, hence
zero-class objects are ignored.
matching_threshold: Threshold for a positive match.
Return: Tuple of:
n_gbboxes: Scalar Tensor with number of groundtruth boxes (may difer from
size because of zero padding).
tp_match: (N,)-shaped boolean Tensor containing with True Positives.
fp_match: (N,)-shaped boolean Tensor containing with False Positives.
"""
with tf.name_scope(scope, 'bboxes_matching_single',
[scores, bboxes, glabels, gbboxes]):
rsize = tf.size(scores)
rshape = tf.shape(scores)
rlabel = tf.cast(label, glabels.dtype)
# Number of groundtruth boxes.
gdifficults = tf.cast(gdifficults, tf.bool)
n_gbboxes = tf.count_nonzero(tf.logical_and(tf.equal(glabels, label),
tf.logical_not(gdifficults)))
# Grountruth matching arrays.
gmatch = tf.zeros(tf.shape(glabels), dtype=tf.bool)
grange = tf.range(tf.size(glabels), dtype=tf.int32)
# True/False positive matching TensorArrays.
sdtype = tf.bool
ta_tp_bool = tf.TensorArray(sdtype, size=rsize, dynamic_size=False, infer_shape=True)
ta_fp_bool = tf.TensorArray(sdtype, size=rsize, dynamic_size=False, infer_shape=True)
# Loop over returned objects.
def m_condition(i, ta_tp, ta_fp, gmatch):
r = tf.less(i, rsize)
return r
def m_body(i, ta_tp, ta_fp, gmatch):
# Jaccard score with groundtruth bboxes.
rbbox = bboxes[i]
jaccard = bboxes_jaccard(rbbox, gbboxes)
jaccard = jaccard * tf.cast(tf.equal(glabels, rlabel), dtype=jaccard.dtype)
# Best fit, checking it's above threshold.
idxmax = tf.cast(tf.argmax(jaccard, axis=0), tf.int32)
jcdmax = jaccard[idxmax]
match = jcdmax > matching_threshold
existing_match = gmatch[idxmax]
not_difficult = tf.logical_not(gdifficults[idxmax])
# TP: match & no previous match and FP: previous match | no match.
# If difficult: no record, i.e FP=False and TP=False.
tp = tf.logical_and(not_difficult,
tf.logical_and(match, tf.logical_not(existing_match)))
ta_tp = ta_tp.write(i, tp)
fp = tf.logical_and(not_difficult,
tf.logical_or(existing_match, tf.logical_not(match)))
ta_fp = ta_fp.write(i, fp)
# Update grountruth match.
mask = tf.logical_and(tf.equal(grange, idxmax),
tf.logical_and(not_difficult, match))
gmatch = tf.logical_or(gmatch, mask)
return [i+1, ta_tp, ta_fp, gmatch]
# Main loop definition.
i = 0
[i, ta_tp_bool, ta_fp_bool, gmatch] = \
tf.while_loop(m_condition, m_body,
[i, ta_tp_bool, ta_fp_bool, gmatch],
parallel_iterations=1,
back_prop=False)
# TensorArrays to Tensors and reshape.
tp_match = tf.reshape(ta_tp_bool.stack(), rshape)
fp_match = tf.reshape(ta_fp_bool.stack(), rshape)
# Some debugging information...
# tp_match = tf.Print(tp_match,
# [n_gbboxes,
# tf.reduce_sum(tf.cast(tp_match, tf.int64)),
# tf.reduce_sum(tf.cast(fp_match, tf.int64)),
# tf.reduce_sum(tf.cast(gmatch, tf.int64))],
# 'Matching (NG, TP, FP, GM): ')
return n_gbboxes, tp_match, fp_match
# third dense layer,full connected
w_fc1 = weight_variable([1 * tmp_shape * L2, L3])
b_fc1 = bias_variable([L3])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# fourth layer, output
w_fc2 = weight_variable([L3, classes])
b_fc2 = bias_variable([classes])
y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, w_fc2) + b_fc2)
cost = tf.reduce_mean(-tf.reduce_sum(ys * tf.log(y_conv), reduction_indices=[1]))
optimizer = tf.train.AdamOptimizer(learning_date).minimize(cost)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(ys, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
init = tf.global_variables_initializer()
sess.run(init)
### cnn start train##################################
for epoch in range(epochs):
avg_cost = 0.
avg_acc = 0.
for batch in range(len(train_data) // batch_size):
offset = (batch * batch_size) % len(train_data)
batch_data = train_data[offset:(offset + batch_size)]
batch_labels = train_labels[offset:(offset + batch_size)]
_, c, acc = sess.run([optimizer, cost, accuracy],
feed_dict={xs: batch_data, ys: batch_labels, keep_prob: 0.5})
def construct_batched_adjacency_and_feature_matrices(
size,
adj_row,
adj_column,
adj_values,
adj_elem_len,
adj_degrees,
feature_row,
feature_column,
feature_values,
feature_elem_len,
input_dim,
max_degree=5,
normalize=True,
split_adj=False,
):
"""
Constructs a batched, sparse adjacency matrix.
For example to make a batch of two adjacency matrices of 2 and 3 nodes:
```
Example:
>>> # first adjacency matrix: [[1, 1], [1, 1]]
>>> # second adjacency matrix: [[1, 1, 0], [1, 1, 1], [0, 1, 1]]
>>> import tensorflow as tf
>>> tf.enable_eager_execution()
>>> size = tf.contant([2, 3], tf.int64)
>>> adj_row = tf.constant([0, 0, 1, 1, 0, 0, 1, 1, 1, 2, 2], tf.int64)
>>> adj_column = tf.constant([0, 1, 0, 1, 0, 1, 0, 1, 2, 1, 2], tf.int64)
>>> adj_values = tf.constant([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], tf.float32)
>>> adj_elem_len = tf.constant([2, 3], tf.int64)
>>> feature_row = tf.constant([0, 1, 0, 1, 2], tf.int64)
>>> feature_column = tf.constant([2, 3, 1, 2, 3], tf.int64)
>>> feature_values = tf.constant([4, 5, 1, 2, 3], tf.int64)
>>> feature_elem_len = tf.constant([2, 3], tf.int64)
>>> diagonalized_adj, feature_mat = construct_batched_adjacency_matrix(size, adj_row, adj_column, adj_values, adj_elem_len, adj_degrees, feature_row, feature_column, feature_values, feature_elem_len, 10, normalize=False, split_adj=False)
>>> tf.sparse.to_dense(diagonalized_adj[0]).numpy()
array([[1., 1., 0., 0., 0].,
[1., 1., 0., 0., 0].,
[0., 0., 1., 1., 0].,
[0., 0., 1., 1., 1].,
[0., 0., 0., 1., 1]]
>>> feature_mat.numpy()
array([[0, 0, 4, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 5, 0, 0, 0, 0, 0, 0],
[0, 1, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 2, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 3, 0, 0, 0, 0, 0, 0], dtype=int32)
Parameters:
size: sizes of adjacency matrices.
adj_row: concatenated row indices of all matrices in a batch.
adj_column: concatenated column indices of all matrices in a batch.
adj_values: concatenated values of elements of all matrices in a batch
adj_elem_len: number of non-zero elements in all matrices.
adj_degrees: degree of each node
feature_row: concatenated row indices of all feature matrices.
feature_column: concatenated column indices of all feature matrices.
feature_values: concatenated values in all feature matrices.
feature_elem_len: number of non-zero elements n all feature matrices.
input_dim: dimension of each node in feature matrices.
normalize: normalizes the adjacency matrix if True.
split_adj: splits the adjacency matrix based on degrees of nodes if True.
Returns:
Batched adjacency matrix.
"""
with tf.device("/cpu:0"):
cumsum = tf.cumsum(size)
adj_row = adj_row
adj_col = adj_column
adj_elem_len = adj_elem_len
start = tf.cumsum(adj_elem_len, exclusive=True)
offset = tf.cumsum(size, exclusive=True)
start_size_offset = tf.stack([start, adj_elem_len, offset], 1)
def offset_index(row_or_column):
def split_sum(a, x):
padded_index = tf.concat(
[
tf.zeros(x[0], tf.int64),
row_or_column[x[0]:x[0] + x[1]] + x[2],
tf.zeros(
tf.shape(row_or_column, out_type=tf.int64)[0] - x[0] -
x[1],
tf.int64,
),
],
0,
)
return padded_index
return split_sum
with tf.device("/cpu:0"):
padded_rows = tf.scan(offset_index(adj_row),
start_size_offset,
initializer=adj_row)
padded_columns = tf.scan(offset_index(adj_col),
start_size_offset,
initializer=adj_col)
diagonal_row = tf.reduce_sum(padded_rows, axis=0)
diagonal_col = tf.reduce_sum(padded_columns, axis=0)
adj_shape = [tf.reduce_sum(size), tf.reduce_sum(size)]
if normalize:
diagonalized_adj = tf.SparseTensor(
indices=tf.transpose(tf.stack([diagonal_row, diagonal_col])),
values=adj_values,
dense_shape=adj_shape,
)
degree_hat = tf.sparse.reduce_sum(diagonalized_adj, axis=0)
diagonalized_adj = (diagonalized_adj / tf.sqrt(degree_hat) /
tf.expand_dims(tf.sqrt(degree_hat), 1))
diagonalized_adj = [diagonalized_adj] # number of channel is 1.
elif split_adj:
adj_degrees = adj_degrees
adj_degrees = tf.clip_by_value(
adj_degrees, 0, max_degree
) # degree is the number of edges on each node, including the one to itself. A node with degree 1 is not connected with any other node.
diagonalized_adj = []
for degree in range(1, max_degree + 1):
row_deg = tf.boolean_mask(diagonal_row,
tf.equal(adj_degrees, degree))
row_col = tf.boolean_mask(diagonal_col,
tf.equal(adj_degrees, degree))
diagonalized_adj.append(
tf.SparseTensor(
indices=tf.transpose(tf.stack([row_deg, row_col])),
values=tf.boolean_mask(adj_values,
tf.equal(adj_degrees, degree)),
dense_shape=adj_shape,
))
diagonalized_adj.append(tf.sparse.eye(
adj_shape[0])) # connection to self
else:
diagonalized_adj = tf.SparseTensor(
indices=tf.transpose(tf.stack([diagonal_row, diagonal_col])),
values=adj_values,
dense_shape=adj_shape,
)
diagonalized_adj = [diagonalized_adj]
start_feature = tf.cumsum(feature_elem_len, exclusive=True)
start_size_offset_feature = tf.stack(
[start_feature, feature_elem_len, offset], 1)
with tf.device("/cpu:0"):
padded_rows_feature = tf.scan(
offset_index(feature_row),
start_size_offset_feature,
initializer=feature_row,
)
stacked_row = tf.reduce_sum(padded_rows_feature, axis=0)
net = tf.SparseTensor(
indices=tf.transpose(tf.stack([stacked_row, feature_column])),
values=feature_values,
dense_shape=[tf.reduce_sum(size), input_dim],
)
net = tf.sparse_reorder(net)
net = tf.sparse_tensor_to_dense(net)
return diagonalized_adj, net
def model_eval(sess,
x,
y,
predictions,
X_test=None,
Y_test=None,
feed=None,
args=None):
"""
Compute the accuracy of a TF model on some data
:param sess: TF session to use
:param x: input placeholder
:param y: output placeholder (for labels)
:param predictions: model output predictions
:param X_test: numpy array with training inputs
:param Y_test: numpy array with training outputs
:param feed: An optional dictionary that is appended to the feeding
dictionary before the session runs. Can be used to feed
the learning phase of a Keras model for instance.
:param args: dict or argparse `Namespace` object.
Should contain `batch_size`
:return: a float with the accuracy value
"""
global _model_eval_cache
args = _ArgsWrapper(args or {})
assert args.batch_size, "Batch size was not given in args dict"
if X_test is None or Y_test is None:
raise ValueError("X_test argument and Y_test argument "
"must be supplied.")
# Define accuracy symbolically
key = (y, predictions)
if key in _model_eval_cache:
correct_preds = _model_eval_cache[key]
else:
correct_preds = tf.equal(tf.argmax(y, axis=-1),
tf.argmax(predictions, axis=-1))
_model_eval_cache[key] = correct_preds
# Init result var
accuracy = 0.0
with sess.as_default():
# Compute number of batches
nb_batches = int(math.ceil(float(len(X_test)) / args.batch_size))
assert nb_batches * args.batch_size >= len(X_test)
X_cur = np.zeros((args.batch_size, ) + X_test.shape[1:],
dtype=X_test.dtype)
Y_cur = np.zeros((args.batch_size, ) + Y_test.shape[1:],
dtype=Y_test.dtype)
for batch in range(nb_batches):
if batch % 100 == 0 and batch > 0:
_logger.debug("Batch " + str(batch))
# Must not use the `batch_indices` function here, because it
# repeats some examples.
# It's acceptable to repeat during training, but not eval.
start = batch * args.batch_size
end = min(len(X_test), start + args.batch_size)
# The last batch may be smaller than all others. This should not
# affect the accuarcy disproportionately.
cur_batch_size = end - start
X_cur[:cur_batch_size] = X_test[start:end]
Y_cur[:cur_batch_size] = Y_test[start:end]
feed_dict = {x: X_cur, y: Y_cur}
if feed is not None:
feed_dict.update(feed)
cur_corr_preds = correct_preds.eval(feed_dict=feed_dict)
accuracy += cur_corr_preds[:cur_batch_size].sum()
assert end >= len(X_test)
# Divide by number of examples to get final value
accuracy /= len(X_test)
return accuracy
def train(self):
train_graph = tf.Graph()
with train_graph.as_default():
output, x, is_training = self.lstm_model()
# 训练集的标签
y = tf.placeholder(shape=(None, ), name='label', dtype=tf.float64)
print(y.name)
# 参数L2正则化减少过拟合
reg = tf.contrib.layers.apply_regularization(tf.contrib.layers.l2_regularizer(self.numda), tf.trainable_variables())
# 损失函数计算（交叉熵加L2正则化）
cse = tf.convert_to_tensor(value=0, dtype=tf.float64)
for i in range(self.batch_size):
cse = cse - 0.61022*tf.multiply(y[i], tf.log(output[i][0])) - 2.76825*tf.multiply(1 - y[i], tf.log(output[i][1]))
cse = cse/self.batch_size
cse = cse + reg
# 准确率计算
acc_num = tf.convert_to_tensor(0)
n = tf.convert_to_tensor(0)
for i in range(self.batch_size):
acc1 = tf.convert_to_tensor(0)
acc2 = tf.convert_to_tensor(0)
#acc3 = tf.convert_to_tensor(0)
n = tf.cond(pred=tf.equal(y[i], 0),
true_fn=lambda: tf.add(n, 1), false_fn=lambda: tf.add(n, 0))
acc1 = tf.cond(pred=tf.logical_and(tf.equal(y[i], 1),
tf.greater(output[i][0], output[i][1])
),
true_fn=lambda: tf.add(acc1, 1), false_fn=lambda: tf.add(acc1, 0))
acc2 = tf.cond(pred=tf.logical_and(tf.equal(y[i], 0),
tf.greater(output[i][1], output[i][0])
),
true_fn=lambda: tf.add(acc2, 1), false_fn=lambda: tf.add(acc2, 0))
acc_num = acc_num + acc1 + acc2
#acc_num = acc_num + acc2
acc = acc_num / self.batch_size
#acc = acc_num / n
#acc3 = tf.cond(pred=tf.logical_and(tf.equal(y[i][2], 1),
# tf.logical_and(tf.greater(output[i][2], output[i][0]),
# tf.greater(output[i][2], output[i][1]))),
# true_fn=lambda: tf.add(acc3, 1), false_fn=lambda: tf.add(acc3, 0))
#acc_num = acc_num + acc1 + acc2 + acc3
# 批量梯度下降(Adam?)优化损失函数，好像Adam更好一些
optimizer_1 = tf.train.GradientDescentOptimizer(learning_rate=self.learning_rate)
optimizer_2 = tf.train.AdamOptimizer(learning_rate=self.learning_rate)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op_1 = optimizer_1.minimize(loss=cse)
train_op_2 = optimizer_2.minimize(loss=cse)
'''
var_list = []
list_2 = []
for v in tf.global_variables():
if v in tf.trainable_variables():
var_list.append(v)
else:
list_2.append(v)
saver = tf.train.Saver(var_list=var_list)
'''
saver = tf.train.Saver()
# 开启session，载入最后报存的模型\初始化模型
sess = tf.Session()
# 以下两行可用作开启第一次训练与延续上一次训练之间切换
#sess.run(tf.global_variables_initializer())
saver.restore(sess=sess, save_path="C:\\Users\\Jason\\Desktop\\hengqin quant finance\\saved_model1_0\\lstm_model-00.3")
#sess.run(tf.variables_initializer(list_2))
for i in range(self.epoch):
# 获取minibatch
batch_x, batch_y = self.get_mini_batch(self.batch_size)
# 训练，返回损失函数值
if i == 0:
print(sess.run((cse, sess.graph.get_tensor_by_name('fully_connection/relu:0'), output), {x: batch_x, y: batch_y, is_training: True}))
sess.run(fetches=train_op_2, feed_dict={x: batch_x, y: batch_y, is_training: True})
else:
print(sess.run((cse, acc), {x: batch_x, y: batch_y, is_training: True}))
sess.run(fetches=train_op_2, feed_dict={x: batch_x, y: batch_y, is_training: True})
saver.save(sess=sess,
save_path='C:\\Users\\Jason\\Desktop\\hengqin quant finance\\saved_model1_0\\lstm_model-00.3')
#saver.save(sess=sess,
# save_path='./lstm_model-00.1')
'''
builder = tf.saved_model.builder.SavedModelBuilder('./lstm_saved_model')
builder.add_meta_graph_and_variables(sess, [tf.saved_model.tag_constants.TRAINING])
builder.save()
'''
varlist = [v.name[:-2] for v in tf.global_variables()]
varlist.append('fully_connection/output')
varlist.append('lstm_model/input')
varlist.append('label')
varlist.append('lstm_model/is_training')
print(varlist)
constant_graph = graph_util.convert_variables_to_constants(sess, sess.graph_def, varlist)
with tf.gfile.FastGFile('./lstm_model_5.pb', mode='wb') as f:
f.write(constant_graph.SerializeToString())
sess.close()
def evaluate_pictures(n_epochs=200,batch_size=10,dataset='E:/bishe/cifar-100-python'):
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# 计算各数据集的batch个数
n_train_batches = train_set_x.shape[0]
n_train_batches = int(n_train_batches / batch_size)
print("... building the model")
x = tf.placeholder(tf.float32, shape=[None, 3072])
y = tf.placeholder(tf.float32, shape=[None, 100])
keep_prob = tf.placeholder(tf.float32)
x_image = tf.transpose(tf.reshape(x, [-1, 3, 32, 32]),perm=[0,2,3,1])##
W_conv1 = weight_variable([5, 5, 3, 64])
b_conv1 = bias_variable([64])
h_pool1 = max_pool_2x2(tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1))
input1=tf.nn.local_response_normalization(h_pool1)
W_conv2 = weight_variable([5, 5, 64, 128])
b_conv2 = bias_variable([128])
h_pool2 = mean_pool_2x2(tf.nn.relu(conv2d(input1, W_conv2) + b_conv2))
input2 = tf.nn.local_response_normalization(h_pool2)
W_conv3 = weight_variable([5, 5, 128, 500])
b_conv3 = bias_variable([500])
h_pool3 = mean_pool_2x2(tf.nn.relu(conv2d(input2, W_conv3) + b_conv3))
input3 = tf.nn.local_response_normalization(h_pool3)
h_fc1_drop = tf.nn.dropout(input3, keep_prob)
W_fc1 = weight_variable([4 * 4 * 500, 100])
b_fc1 = bias_variable([100])
y_conv = tf.matmul(tf.reshape(h_fc1_drop, [-1, 4 * 4 * 500]), W_fc1) + b_fc1
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_conv))
train_step = tf.train.AdadeltaOptimizer(0.1).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
tf.summary.scalar('accuracy', accuracy)
tf.summary.histogram('h_pool1', y_conv)
sess=tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
best_validation_acc = np.inf
epoch = 0
done_looping = False
print("... training")
summary_op=tf.summary.merge_all()
summary_writer=tf.summary.FileWriter(FLAGS.summaries_dir,graph_def=sess.graph_def)
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
iter = (epoch - 1) * n_train_batches + minibatch_index
_,acc=sess.run([train_step, accuracy],feed_dict={x: train_set_x[minibatch_index * batch_size: (minibatch_index + 1) * batch_size],
y: train_set_y[minibatch_index * batch_size: (minibatch_index + 1) * batch_size], keep_prob: 0.2})
print('epoch %i, step %d,minibatch %i/%i, train acc %g' %
(epoch, iter, minibatch_index + 1, n_train_batches,acc))
if (iter + 1) % 1000 == 0:
# compute zero-one loss on validation set
validation_acc = accuracy.eval(feed_dict={x: valid_set_x, y: valid_set_y, keep_prob: 0.2})
print(' validation acc %g' %(validation_acc ))
# test it on the test set
summary_str, acc = sess.run([summary_op, accuracy],
feed_dict={x: test_set_x, y: test_set_y, keep_prob: 0.2})
summary_writer.add_summary(summary_str, iter)
print(' test acc %g' % (acc))
# if we got the best validation score until now
if validation_acc > best_validation_acc:
# save best validation score and iteration number
best_validation_acc = validation_acc
save_params(W_conv1, b_conv1, W_conv2, b_conv2,W_fc1,b_fc1) # 保存参数
print('Optimization complete.')
print("test accuracy %g" % accuracy.eval(feed_dict={
x: test_set_x, y: test_set_y, keep_prob: 1.0}))
def dice_score(predicted_labels, labels, dim_output, type_unet):
####### Dice score for binary classification #######
### predicted_labels -- shape (num_batch, height, width)
### labels -- shape (num_batch, height, width)
print('shape of predicted labels')
print(predicted_labels)
print('shape of actual labels')
print(labels)
shape_of_data = labels.get_shape().as_list()
indices_predictions = tf.round(
tf.reshape(predicted_labels, [-1, dim_output]))
if type_unet == '3D':
indices_predictions = tf.reshape(
indices_predictions,
[-1, shape_of_data[1] * shape_of_data[2] * shape_of_data[3] * 1])
else:
indices_predictions = tf.reshape(
indices_predictions, [-1, shape_of_data[1] * shape_of_data[2] * 1])
indices_labels = tf.round(tf.reshape(labels, [-1, dim_output]))
if type_unet == '3D':
indices_labels = tf.reshape(
indices_labels,
[-1, shape_of_data[1] * shape_of_data[2] * shape_of_data[3] * 1])
else:
indices_labels = tf.reshape(
indices_labels, [-1, shape_of_data[1] * shape_of_data[2] * 1])
print('after transofrmation')
print(indices_predictions)
print(indices_labels)
dice_score = defaultdict()
for _ in range(2):
shared_bool = tf.logical_and(
tf.equal(
tf.cast(indices_predictions, tf.float32),
tf.ones_like(indices_predictions, dtype=tf.float32) *
tf.cast(_, tf.float32)),
tf.equal(
tf.cast(indices_labels, tf.float32),
tf.ones_like(indices_predictions, dtype=tf.float32) *
tf.cast(_, tf.float32)))
area_shared = tf.reduce_sum(tf.cast(shared_bool, tf.float32), 1)
predictions_bool = tf.equal(
tf.cast(indices_predictions, tf.float32),
tf.ones_like(indices_predictions, dtype=tf.float32) *
tf.cast(_, tf.float32))
area_predictions = tf.reduce_sum(tf.cast(predictions_bool, tf.float32),
1)
labels_bool = tf.equal(
tf.cast(indices_labels, tf.float32),
tf.ones_like(indices_predictions, dtype=tf.float32) *
tf.cast(_, tf.float32))
area_labels = tf.reduce_sum(tf.cast(labels_bool, tf.float32), 1)
dice_score[_] = tf.reduce_mean((2.0 * area_shared + 1e-6) /
(area_predictions + area_labels + 1e-6))
return dice_score
Ylogits = tf.matmul(Y4, W5) + B5
Y = tf.nn.softmax(Ylogits)
# 3. Define the loss function
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=Ylogits, labels=Y_)
cross_entropy = tf.reduce_mean(cross_entropy)*100
# 4. Define the accuracy
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
correct_prediction = tf.equal(tf.argmax(Y,1), tf.argmax(Y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print(sess.run(accuracy, feed_dict={X: fashion_mnist.test.images, Y_: fashion_mnist.test.labels}))
# 5. Define an optimizer
#train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
train_step = tf.train.AdamOptimizer(0.003).minimize(cross_entropy)
# initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
outputs, final_state = tf.nn.dynamic_rnn(lstm_cell, X_in, initial_state=init_state, time_major=False)
# 最后，output_layer和result的值
# 方式一输出结果
result = tf.matmul(final_state[1], weights['out'] + biases['out'])
# 方式二输出结果
# outputs = tf.unstack(tf.transpose(outputs, [1, 0, 2]))
# result = tf.matmul(outputs[-1], weights['out'] + biases['out'])
return result
pre = rnn(x, weight, biases)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pre, labels=y))
train_op = tf.train.AdamOptimizer(lr).minimize(cost)
correct_pre = tf.equal(tf.argmax(pre, 1), tf.argmax(y, 1))
accuary = tf.reduce_mean(tf.cast(correct_pre, tf.float32))
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
step = 0
while step * batch_size < training_iters:
batch_xs, batch_ys = mnist.train.next_batch(batch_size)
batch_xs = batch_xs.reshape([batch_size, row_inputs, col_inputs])
sess.run([train_op], feed_dict={x: batch_xs, y: batch_ys, })
if step % 20 == 0:
print(sess.run(accuary, feed_dict={x: batch_xs, y: batch_ys, }))
def tf_hog_descriptor(images, cell_size = 8, block_size = 2, block_stride = 1, n_bins = 9,
grayscale = False, oriented = False):
batch_size, height, width, depth = images.shape
half_pi = tf.constant(np.pi/2, name="pi_half")
eps = tf.constant(1e-6, name="eps")
scale_factor = tf.constant(np.pi * n_bins * 0.99999, name="scale_factor")
img = tf.constant(images, name="ImgBatch", dtype=tf.float32)
# gradients
if grayscale:
gray = tf.image.rgb_to_grayscale(img, name="ImgGray")
grad = tf_deriv(gray)
else:
grad = tf_deriv(img)
g_x = grad[:,:,:,0::2]
g_y = grad[:,:,:,1::2]
# maximum norm gradient selection
g_norm = tf.sqrt(tf.square(g_x) + tf.square(g_y), "GradNorm")
idx = tf.argmax(g_norm, 3)
g_norm = tf.expand_dims(tf_select_by_idx(g_norm, idx), -1)
g_x = tf.expand_dims(tf_select_by_idx(g_x, idx), -1)
g_y = tf.expand_dims(tf_select_by_idx(g_y, idx), -1)
# orientation and binning
if oriented:
# atan2 implementation needed
# lots of conditional indexing required
raise NotImplementedError("`oriented` gradient not supported yet")
else:
g_dir = tf.atan(g_y / (g_x + eps)) + half_pi
g_bin = tf.to_int32(g_dir / scale_factor, name="Bins")
# cells partitioning
cell_norm = tf.space_to_depth(g_norm, cell_size, name="GradCells")
cell_bins = tf.space_to_depth(g_bin, cell_size, name="BinsCells")
# cells histograms
hist = list()
zero = tf.zeros(cell_bins.get_shape())
for i in range(n_bins):
mask = tf.equal(cell_bins, tf.constant(i, name="%i"%i))
hist.append(tf.reduce_sum(tf.select(mask, cell_norm, zero), 3))
hist = tf.transpose(tf.pack(hist), [1,2,3,0], name="Hist")
# blocks partitioning
block_hist = tf.extract_image_patches(hist,
ksizes = [1, block_size, block_size, 1],
strides = [1, block_stride, block_stride, 1],
rates = [1, 1, 1, 1],
padding = 'VALID',
name = "BlockHist")
# block normalization
block_hist = tf.nn.l2_normalize(block_hist, 3, epsilon=1.0)
# HOG descriptor
hog_descriptor = tf.reshape(block_hist,
[int(block_hist.get_shape()[0]),
int(block_hist.get_shape()[1]) * \
int(block_hist.get_shape()[2]) * \
int(block_hist.get_shape()[3])],
name='HOGDescriptor')
return hog_descriptor
def multihead_attention(queries,
keys,
num_units=None,
num_heads=8,
dropout_rate=0,
is_training=True,
causality=False,
scope="multihead_attention",
reuse=None):
'''Applies multihead attention. Forked from https://github.com/Kyubyong/transformer
Args:
queries: A 3d tensor with shape of [N, T_q, C_q].
keys: A 3d tensor with shape of [N, T_k, C_k].
num_units: A scalar. Attention size.
dropout_rate: A floating point number.
is_training: Boolean. Controller of mechanism for dropout.
causality: Boolean. If true, units that reference the future are masked.
num_heads: An int. Number of heads.
scope: Optional scope for `variable_scope`.
reuse: Boolean, whether to reuse the weights of a previous layer
by the same name.
Returns
A 3d tensor with shape of (N, T_q, C)
'''
with tf.variable_scope(scope, reuse=reuse):
# Set the fall back option for num_units
if num_units is None:
num_units = queries.get_shape().as_list()[-1]
# Linear projections
Q = tf.layers.dense(queries, num_units, activation=tf.nn.relu) # (N, T_q, C)
K = tf.layers.dense(keys, num_units, activation=tf.nn.relu) # (N, T_k, C)
V = tf.layers.dense(keys, num_units, activation=tf.nn.relu) # (N, T_k, C)
# Split and concat
Q_ = tf.concat(tf.split(Q, num_heads, axis=2), axis=0) # (h*N, T_q, C/h)
K_ = tf.concat(tf.split(K, num_heads, axis=2), axis=0) # (h*N, T_k, C/h)
V_ = tf.concat(tf.split(V, num_heads, axis=2), axis=0) # (h*N, T_k, C/h)
# Multiplication
outputs = tf.matmul(Q_, tf.transpose(K_, [0, 2, 1])) # (h*N, T_q, T_k)
# Scale
outputs = outputs / (K_.get_shape().as_list()[-1] ** 0.5)
# Key Masking
key_masks = tf.sign(tf.reduce_sum(tf.abs(keys), axis=-1)) # (N, T_k)
key_masks = tf.tile(key_masks, [num_heads, 1]) # (h*N, T_k)
key_masks = tf.tile(tf.expand_dims(key_masks, 1), [1, tf.shape(queries)[1], 1]) # (h*N, T_q, T_k)
paddings = tf.ones_like(outputs)*(-2**32+1)
outputs = tf.where(tf.equal(key_masks, 0), paddings, outputs) # (h*N, T_q, T_k)
# Causality = Future blinding
if causality:
diag_vals = tf.ones_like(outputs[0, :, :]) # (T_q, T_k)
tril = tf.linalg.LinearOperatorLowerTriangular(diag_vals).to_dense() # (T_q, T_k)
masks = tf.tile(tf.expand_dims(tril, 0), [tf.shape(outputs)[0], 1, 1]) # (h*N, T_q, T_k)
paddings = tf.ones_like(masks)*(-2**32+1)
outputs = tf.where(tf.equal(masks, 0), paddings, outputs) # (h*N, T_q, T_k)
# Activation
outputs = tf.nn.softmax(outputs) # (h*N, T_q, T_k)
# Query Masking
query_masks = tf.sign(tf.reduce_sum(tf.abs(queries), axis=-1)) # (N, T_q)
query_masks = tf.tile(query_masks, [num_heads, 1]) # (h*N, T_q)
query_masks = tf.tile(tf.expand_dims(query_masks, -1), [1, 1, tf.shape(keys)[1]]) # (h*N, T_q, T_k)
outputs *= query_masks # broadcasting. (N, T_q, C)
# Dropouts
outputs = tf.layers.dropout(outputs, rate=dropout_rate, training=tf.convert_to_tensor(is_training))
# Weighted sum
outputs = tf.matmul(outputs, V_) # ( h*N, T_q, C/h)
# Restore shape
outputs = tf.concat(tf.split(outputs, num_heads, axis=0), axis=2 ) # (N, T_q, C)
# Residual connection
outputs += queries
# Normalize
outputs = normalize(outputs) # (N, T_q, C)
return outputs
def dice_score_multiclass(predicted_labels, labels, num_classes, type_unet):
#### Dice Score for at least 3 classes #####
### predicted_labels -- shape (num_batch, height, width, depth, num_classes)
### labels -- shape (num_batch, height, width, depth, num_classes)
print('shape of predicted labels')
print(predicted_labels)
print('shape of actual labels')
print(labels)
shape_of_data = labels.get_shape().as_list()
if type_unet == '3D':
indices_predictions = tf.argmax(tf.reshape(predicted_labels,
[-1, shape_of_data[4]]),
axis=-1)
indices_predictions = tf.reshape(
indices_predictions,
[-1, shape_of_data[1] * shape_of_data[2] * shape_of_data[3] * 1])
indices_labels = tf.argmax(tf.reshape(labels, [-1, shape_of_data[4]]),
axis=-1)
indices_labels = tf.reshape(
indices_labels,
[-1, shape_of_data[1] * shape_of_data[2] * shape_of_data[3] * 1])
else:
indices_predictions = tf.argmax(tf.reshape(predicted_labels,
[-1, shape_of_data[3]]),
axis=-1)
indices_predictions = tf.reshape(
indices_predictions, [-1, shape_of_data[1] * shape_of_data[2] * 1])
indices_labels = tf.argmax(tf.reshape(labels, [-1, shape_of_data[3]]),
axis=-1)
indices_labels = tf.reshape(
indices_labels, [-1, shape_of_data[1] * shape_of_data[2] * 1])
print('after transformation')
print(indices_predictions)
print(indices_labels)
dice_score = defaultdict()
for _ in range(num_classes):
shared_bool = tf.logical_and(
tf.equal(
tf.cast(indices_predictions, tf.float32),
tf.ones_like(indices_predictions, dtype=tf.float32) *
tf.cast(_, tf.float32)),
tf.equal(
tf.cast(indices_labels, tf.float32),
tf.ones_like(indices_predictions, dtype=tf.float32) *
tf.cast(_, tf.float32)))
area_shared = tf.reduce_sum(tf.cast(shared_bool, tf.float32), 1)
predictions_bool = tf.equal(
tf.cast(indices_predictions, tf.float32),
tf.ones_like(indices_predictions, dtype=tf.float32) *
tf.cast(_, tf.float32))
area_predictions = tf.reduce_sum(tf.cast(predictions_bool, tf.float32),
1)
labels_bool = tf.equal(
tf.cast(indices_labels, tf.float32),
tf.ones_like(indices_predictions, dtype=tf.float32) *
tf.cast(_, tf.float32))
area_labels = tf.reduce_sum(tf.cast(labels_bool, tf.float32), 1)
dice_score[_] = tf.reduce_mean((2.0 * area_shared + 1e-6) /
(area_predictions + area_labels + 1e-6))
return dice_score
def tf_select_by_idx(a, idx):
return tf.select(tf.equal(idx, 2),
a[:,:,:,2],
tf.select(tf.equal(idx, 1),
a[:,:,:,1],
a[:,:,:,0]))
def evaluate(sess, X, Y):
# 对训练得到的模型进行评估
predicted = tf.cast(inference(X) > 0.5, tf.float32)
print(sess.run(tf.reduce_mean(tf.cast(tf.equal(predicted, Y),
tf.float32))))
def model(X_train, Y_train, X_test, Y_test, learning_rate=0.009,
num_epochs=100, minibatch_size=64, print_cost=True):
"""
Implements a three-layer ConvNet in Tensorflow:
CONV2D -> RELU -> MAXPOOL -> CONV2D -> RELU -> MAXPOOL -> FLATTEN -> FULLYCONNECTED
Arguments:
X_train -- training set, of shape (None, 240, 240, 3)
Y_train -- test set, of shape (None, n_y = 4)
X_test -- training set, of shape (None, 240, 240, 3)
Y_test -- test set, of shape (None, n_y = 4)
learning_rate -- learning rate of the optimization
num_epochs -- number of epochs of the optimization loop
minibatch_size -- size of a minibatch
print_cost -- True to print the cost every 100 epochs
Returns:
train_accuracy -- real number, accuracy on the train set (X_train)
test_accuracy -- real number, testing accuracy on the test set (X_test)
parameters -- parameters learnt by the model. They can then be used to predict.
"""
ops.reset_default_graph() # to be able to rerun the model without overwriting tf variables
tf.set_random_seed(1) # to keep results consistent (tensorflow seed)
seed = 3 # to keep results consistent (numpy seed)
(m, n_H0, n_W0, n_C0) = X_train.shape
n_y = Y_train.shape[1]
costs = [] # To keep track of the cost
# Create Placeholders of the correct shape
### START CODE HERE ### (1 line)
X, Y = create_placeholders(n_H0, n_W0, n_C0, n_y)
### END CODE HERE ###
# Initialize parameters
### START CODE HERE ### (1 line)
parameters = initialize_parameters()
# W1 = tf.get_variable("W1", [m, n_H0, n_W0, n_C0], initializer = tf.contrib.layers.xavier_initializer(seed = 0))
# W2 = tf.get_variable("W2", [m, n_H0, n_W0, n_C0], initializer = tf.contrib.layers.xavier_initializer(seed = 0))
# parameters = {"W1": W1,
# "W2": W2}
### END CODE HERE ###
# Forward propagation: Build the forward propagation in the tensorflow graph
### START CODE HERE ### (1 line)
Z3 = forward_propagation(X, parameters)
### END CODE HERE ###
# Cost function: Add cost function to tensorflow graph
### START CODE HERE ### (1 line)
cost = compute_cost(Z3, Y)
### END CODE HERE ###
# Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer that minimizes the cost.
### START CODE HERE ### (1 line)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
### END CODE HERE ###
# Initialize all the variables globally
init = tf.global_variables_initializer()
# Start the session to compute the tensorflow graph
with tf.Session() as sess:
# Run the initialization
sess.run(init)
# Do the training loop
for epoch in range(num_epochs):
minibatch_cost = 0.
num_minibatches = int(
m / minibatch_size) # number of minibatches of size minibatch_size in the train set
seed = seed + 1
minibatches = random_mini_batches(X_train, Y_train, minibatch_size, seed)
for minibatch in minibatches:
# Select a minibatch
(minibatch_X, minibatch_Y) = minibatch
"""
# IMPORTANT: The line that runs the graph on a minibatch.
# Run the session to execute the optimizer and the cost.
# The feedict should contain a minibatch for (X,Y).
"""
### START CODE HERE ### (1 line)
_, temp_cost = sess.run(fetches=[optimizer, cost],
feed_dict={X: minibatch_X, Y: minibatch_Y}
)
### END CODE HERE ###
minibatch_cost += temp_cost / num_minibatches
# Print the cost every epoch
if print_cost == True and epoch % 5 == 0:
print("Cost after epoch %i: %f" % (epoch, minibatch_cost))
if print_cost == True and epoch % 1 == 0:
costs.append(minibatch_cost)
# plot the cost
plt.plot(np.squeeze(costs))
plt.ylabel('cost')
plt.xlabel('iterations (per tens)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
# Calculate the correct predictions
predict_op = tf.argmax(Z3, 1)
correct_prediction = tf.equal(predict_op, tf.argmax(Y, 1))
# Calculate accuracy on the test set
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print(accuracy)
train_accuracy = accuracy.eval({X: X_train, Y: Y_train})
test_accuracy = accuracy.eval({X: X_test, Y: Y_test})
print("Train Accuracy:", train_accuracy)
print("Test Accuracy:", test_accuracy)
return train_accuracy, test_accuracy, parameters
def call(self, x1, x2, mask=None):
r"""
Args:
x1: low-level feature map with larger size [b, h, w, c].
x2: high-level feature map with smaller size [b, h/2, w/2, c].
mask: Input mask, 1 for missing regions 0 for known regions.
Returns:
tf.Tensor: reconstructed feature map
"""
# downsample input feature maps if needed due to limited GPU memory
if self.rescale:
x1 = _resize(x1,
scale=1. / 2,
func=tf.compat.v1.image.resize_nearest_neighbor)
x2 = _resize(x2,
scale=1. / 2,
func=tf.compat.v1.image.resize_nearest_neighbor)
# get shapes
raw_x1s = tf.shape(x1)
int_x1s = x1.get_shape().as_list()
int_x2s = x2.get_shape().as_list()
# extract patches from low-level feature maps for reconstruction
kernel = 2 * self.rate
raw_w = tf.compat.v1.extract_image_patches(
x1, [1, kernel, kernel, 1],
[1, self.rate * self.stride, self.rate * self.stride, 1],
[1, 1, 1, 1],
padding='SAME')
raw_w = tf.reshape(raw_w, [int_x1s[0], -1, kernel, kernel, int_x1s[3]])
# transpose to [b, kernel, kernel, c, hw]
raw_w = tf.transpose(raw_w, [0, 2, 3, 4, 1])
raw_w_groups = tf.split(raw_w, int_x1s[0], axis=0)
# extract patches from high-level feature maps for matching and attending
x2_groups = tf.split(x2, int_x2s[0], axis=0)
w = tf.compat.v1.extract_image_patches(
x2, [1, self.ksize, self.ksize, 1],
[1, self.stride, self.stride, 1], [1, 1, 1, 1],
padding='SAME')
w = tf.reshape(w, [int_x2s[0], -1, self.ksize, self.ksize, int_x2s[3]])
# transpose to [b, ksize, ksize, c, hw/4]
w = tf.transpose(w, [0, 2, 3, 4, 1])
w_groups = tf.split(w, int_x2s[0], axis=0)
# resize and extract patches from masks
mask = _resize(mask,
to_shape=int_x2s[1:3],
func=tf.compat.v1.image.resize_nearest_neighbor)
m = tf.compat.v1.extract_image_patches(
mask, [1, self.ksize, self.ksize, 1],
[1, self.stride, self.stride, 1], [1, 1, 1, 1],
padding='SAME')
m = tf.reshape(m, [1, -1, self.ksize, self.ksize, 1])
# transpose to [1, ksize, ksize, 1, hw/4]
m = tf.transpose(m, [0, 2, 3, 4, 1])
m = m[0]
mm = tf.cast(
tf.equal(tf.reduce_mean(m, axis=[0, 1, 2], keepdims=True), 0.),
tf.float32)
# matching and attending hole and non-hole patches
y = []
scale = self.softmax_scale
for xi, wi, raw_wi in zip(x2_groups, w_groups, raw_w_groups):
# matching on high-level feature maps
wi = wi[0]
wi_normed = wi / \
tf.maximum(tf.sqrt(tf.reduce_sum(
tf.square(wi), axis=[0, 1, 2])), 1e-4)
yi = tf.nn.conv2d(xi,
wi_normed,
strides=[1, 1, 1, 1],
padding="SAME")
yi = tf.reshape(yi, [
1, int_x2s[1], int_x2s[2],
(int_x2s[1] // self.stride) * (int_x2s[2] // self.stride)
])
# apply softmax to obtain attention score
yi *= mm # mask
yi = tf.nn.softmax(yi * scale, 3)
yi *= mm # mask
# transfer non-hole features into holes according to the atttention score
wi_center = raw_wi[0]
yi = tf.nn.conv2d_transpose(yi,
wi_center,
tf.concat([[1], raw_x1s[1:]], axis=0),
strides=[
1, self.rate * self.stride,
self.rate * self.stride, 1
]) / 4.
y.append(yi)
y = tf.concat(y, axis=0)
y.set_shape(int_x1s)
# refine filled feature map after matching and attending
y1 = self.conv1(y)
y2 = self.conv2(y)
y3 = self.conv3(y)
y4 = self.conv4(y)
y = tf.concat([y1, y2, y3, y4], axis=3)
if self.rescale:
y = _resize(y,
scale=2.,
func=tf.compat.v1.image.resize_nearest_neighbor)
return y
def construct_network(self, frame_feat_input, batch_labels, weights, reuse, is_training):
"""
:param frame_feat_input:[self.train_batch_size, SHOT_NUM, SHOT_FEAT_LENGTH]
:param batch_labels:
:param weights:
:param reuse:
:param is_training:
:return:
"""
with tf.variable_scope('video_level', reuse=reuse) as sc:
nets = tf.layers.conv1d(frame_feat_input, 1024, kernel_size=3, name='conv_1')
nets = slim.batch_norm(nets,
decay=0.9997,
epsilon=0.001,
is_training=is_training)
nets = tf.nn.relu(nets)
nets = tf.layers.max_pooling1d(nets, pool_size=2, strides=2, name='pool1d_1')
nets = tf.layers.conv1d(nets, filters=256, kernel_size=3, name='conv1d_2')
nets = slim.batch_norm(nets,
decay=0.9997,
epsilon=0.001,
is_training=is_training)
nets = tf.nn.relu(nets)
# layer 3
nets = tf.layers.conv1d(nets, filters=256, kernel_size=3, name='conv1d_3')
nets = slim.batch_norm(nets,
decay=0.9997,
epsilon=0.001,
is_training=is_training)
nets = tf.nn.relu(nets)
# test flat
nets = tf.layers.flatten(nets)
fc_frame = tf.layers.dense(nets, 512, name='fc1')
video_vector = tf.layers.dropout(fc_frame, FLAGS.dropout_rate, training=is_training)
video_vector = tf.nn.relu(video_vector)
video_vector = tf.layers.dense(video_vector, 512, name='dense_layer_1')
video_vector = tf.layers.dropout(video_vector, FLAGS.dropout_rate, training=is_training)
video_vector = tf.nn.relu(video_vector)
video_vector = tf.layers.dense(video_vector, 1024, name='dense_layer_2')
video_vector = tf.nn.relu(video_vector)
logits = tf.layers.dense(video_vector, 2, name='dense_layer_3')
predict_confidence = tf.nn.softmax(logits, name='confidence') # [batch_size, 2]
with tf.name_scope('loss'):
cost = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=batch_labels,
weights=weights))
L2_frame = 0
for w in tl.layers.get_variables_with_name('video_level', True, True):
L2_frame += tf.contrib.layers.l2_regularizer(1.0)(w)
loss = cost + 0.001 * L2_frame
with tf.name_scope('accuracy'):
predict_index = tf.argmax(logits, 1)
predicts = tf.equal(predict_index, batch_labels)
accuracy = tf.reduce_mean(tf.cast(predicts, np.float32))
tf.summary.scalar('accuracy', accuracy)
end_point = {'L2_frame': L2_frame, 'loss': loss, 'cost': cost, 'accuracy': accuracy,
'logits': predict_confidence, 'predict': predict_index}
return end_point
# Loss function
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = y_true, logits = y_pred))
# Optimizer
optimizer = tf.train.AdamOptimizer(learning_rate = 0.001)
train = optimizer.minimize(cross_entropy)
init = tf.global_variables_initializer()
steps = 5000
with tf.Session() as sess:
sess.run(init)
for i in range(steps):
batch_x, batch_y = mnist.train.next_batch(50)
sess.run(train, feed_dict = {x:batch_x, y_true: batch_y, hold_prob:0.5})
if i%100 == 0:
print("ON Step: {}".format(i))
print "Accuracy: "
matches = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))
acc = tf.reduce_mean(tf.cast(matches, tf.float32))
print(sess.run(acc, feed_dict = {x:mnist.test.images, y_true:mnist.test.labels, hold_prob: 1.0}))
print '\n'