def fast_rcnn_minibatch(self, reference_boxes):
with tf.variable_scope('fast_rcnn_minibatch'):
reference_boxes_mattached_gtboxes, object_mask, label = \
self.fast_rcnn_find_positive_negative_samples(reference_boxes)
positive_indices = tf.reshape(tf.where(tf.not_equal(object_mask, 0.)), [-1])
num_of_positives = tf.minimum(tf.shape(positive_indices)[0],
tf.cast(self.fast_rcnn_minibatch_size*self.fast_rcnn_positives_ratio, tf.int32))
positive_indices = tf.random_shuffle(positive_indices)
positive_indices = tf.slice(positive_indices, begin=[0], size=[num_of_positives])
negative_indices = tf.reshape(tf.where(tf.equal(object_mask, 0.)), [-1])
num_of_negatives = tf.minimum(tf.shape(negative_indices)[0],
self.fast_rcnn_minibatch_size - num_of_positives)
negative_indices = tf.random_shuffle(negative_indices)
negative_indices = tf.slice(negative_indices, begin=[0], size=[num_of_negatives])
minibatch_indices = tf.concat([positive_indices, negative_indices], axis=0)
minibatch_indices = tf.random_shuffle(minibatch_indices)
minibatch_reference_boxes_mattached_gtboxes = tf.gather(reference_boxes_mattached_gtboxes,
minibatch_indices)
object_mask = tf.gather(object_mask, minibatch_indices)
label = tf.gather(label, minibatch_indices)
label_one_hot = tf.one_hot(label, self.num_classes + 1)
return minibatch_indices, minibatch_reference_boxes_mattached_gtboxes, object_mask, label_one_hot
def ternarize(x, thresh=0.05):
"""
Implemented Trained Ternary Quantization:
https://arxiv.org/abs/1612.01064
Code modified from the authors' at:
https://github.com/czhu95/ternarynet/blob/master/examples/Ternary-Net/ternary.py
"""
shape = x.get_shape()
thre_x = tf.stop_gradient(tf.reduce_max(tf.abs(x)) * thresh)
w_p = tf.get_variable('Wp', initializer=1.0, dtype=tf.float32)
w_n = tf.get_variable('Wn', initializer=1.0, dtype=tf.float32)
tf.summary.scalar(w_p.op.name + '-summary', w_p)
tf.summary.scalar(w_n.op.name + '-summary', w_n)
mask = tf.ones(shape)
mask_p = tf.where(x > thre_x, tf.ones(shape) * w_p, mask)
mask_np = tf.where(x < -thre_x, tf.ones(shape) * w_n, mask_p)
mask_z = tf.where((x < thre_x) & (x > - thre_x), tf.zeros(shape), mask)
@tf.custom_gradient
def _sign_mask(x):
return tf.sign(x) * mask_z, lambda dy: dy
w = _sign_mask(x)
w = w * mask_np
tf.summary.histogram(w.name, w)
return w
def unwrap(p, discont=np.pi, axis=-1):
"""Unwrap a cyclical phase tensor.
Args:
p: Phase tensor.
discont: Float, size of the cyclic discontinuity.
axis: Axis of which to unwrap.
Returns:
unwrapped: Unwrapped tensor of same size as input.
"""
dd = diff(p, axis=axis)
ddmod = tf.mod(dd + np.pi, 2.0 * np.pi) - np.pi
idx = tf.logical_and(tf.equal(ddmod, -np.pi), tf.greater(dd, 0))
ddmod = tf.where(idx, tf.ones_like(ddmod) * np.pi, ddmod)
ph_correct = ddmod - dd
idx = tf.less(tf.abs(dd), discont)
ddmod = tf.where(idx, tf.zeros_like(ddmod), dd)
ph_cumsum = tf.cumsum(ph_correct, axis=axis)
shape = p.get_shape().as_list()
shape[axis] = 1
ph_cumsum = tf.concat([tf.zeros(shape, dtype=p.dtype), ph_cumsum], axis=axis)
unwrapped = p + ph_cumsum
return unwrapped
def _survival_function(self, y):
low = self._low
high = self._high
# Recall the promise:
# survival_function(y) := P[Y > y]
# = 0, if y >= high,
# = 1, if y < low,
# = P[X > y], otherwise.
# P[Y > j] = P[ceiling(Y) > j] since mass is only at integers, not in
# between.
j = tf.ceil(y)
# P[X > j], used when low < X < high.
result_so_far = self.distribution.survival_function(j)
# Broadcast, because it's possible that this is a single distribution being
# evaluated on a number of samples, or something like that.
j += tf.zeros_like(result_so_far)
# Re-define values at the cutoffs.
if low is not None:
result_so_far = tf.where(j < low, tf.ones_like(result_so_far),
result_so_far)
if high is not None:
result_so_far = tf.where(j >= high, tf.zeros_like(result_so_far),
result_so_far)
return result_so_far
def fpn_map_rois_to_levels(boxes):
"""
Assign boxes to level 2~5.
Args:
boxes (nx4):
Returns:
[tf.Tensor]: 4 tensors for level 2-5. Each tensor is a vector of indices of boxes in its level.
[tf.Tensor]: 4 tensors, the gathered boxes in each level.
Be careful that the returned tensor could be empty.
"""
sqrtarea = tf.sqrt(tf_area(boxes))
level = tf.to_int32(tf.floor(
4 + tf.log(sqrtarea * (1. / 224) + 1e-6) * (1.0 / np.log(2))))
# RoI levels range from 2~5 (not 6)
level_ids = [
tf.where(level <= 2),
tf.where(tf.equal(level, 3)), # == is not supported
tf.where(tf.equal(level, 4)),
tf.where(level >= 5)]
level_ids = [tf.reshape(x, [-1], name='roi_level{}_id'.format(i + 2))
for i, x in enumerate(level_ids)]
num_in_levels = [tf.size(x, name='num_roi_level{}'.format(i + 2))
for i, x in enumerate(level_ids)]
add_moving_summary(*num_in_levels)
level_boxes = [tf.gather(boxes, ids) for ids in level_ids]
return level_ids, level_boxes
def bottleneck(self, x):
hparams = self.hparams
b, _ = super(AutoencoderDualDiscrete, self).bottleneck(x)
if hparams.mode == tf.estimator.ModeKeys.EVAL:
return b, 0.0
bt, bi = tf.split(b, 2, axis=0)
if self.hparams.mode != tf.estimator.ModeKeys.TRAIN:
return tf.concat([bi, bi], axis=0), 0.0
# Share the first hparams.bottleneck_shared_bits.
shared = (bt + bi) / 2 # -1 if both -1, 1 if both were 1, 0 if disagree.
rand = tf.random_uniform(common_layers.shape_list(bt))
br = tf.where(rand < 0.5, bt, bi) # Break ties at random.
bs = tf.where(shared == 0, br, shared)
bs = tf.concat([bs, bs], axis=0)
n = hparams.bottleneck_shared_bits
step = tf.train.get_global_step()
zero = tf.constant(0, dtype=tf.int64)
if step is None:
step = zero
step = tf.maximum(zero, step - hparams.bottleneck_shared_bits_start_warmup)
f = common_layers.inverse_lin_decay(
hparams.bottleneck_shared_bits_stop_warmup, min_value=0.1, step=step)
n = tf.where(step > 1, n * f, n)
n = tf.cast(n, tf.int64)
b_shape = common_layers.shape_list(b)
b = tf.concat([bs[..., :n], b[..., n:]], axis=-1)
b = tf.reshape(b, b_shape)
return b, 0.0
def _log_cdf(self, y):
low = self._low
high = self._high
# Recall the promise:
# cdf(y) := P[Y <= y]
# = 1, if y >= high,
# = 0, if y < low,
# = P[X <= y], otherwise.
# P[Y <= j] = P[floor(Y) <= j] since mass is only at integers, not in
# between.
j = tf.floor(y)
result_so_far = self.distribution.log_cdf(j)
# Broadcast, because it's possible that this is a single distribution being
# evaluated on a number of samples, or something like that.
j += tf.zeros_like(result_so_far)
# Re-define values at the cutoffs.
if low is not None:
neg_inf = -np.inf * tf.ones_like(result_so_far)
result_so_far = tf.where(j < low, neg_inf, result_so_far)
if high is not None:
result_so_far = tf.where(j >= high, tf.zeros_like(result_so_far),
result_so_far)
return result_so_far
def convert_f0(f0, src, trg):
mu_s, std_s = np.fromfile(os.path.join('./etc', '{}.npf'.format(src)), np.float32)
mu_t, std_t = np.fromfile(os.path.join('./etc', '{}.npf'.format(trg)), np.float32)
lf0 = tf.where(f0 > 1., tf.log(f0), f0)
lf0 = tf.where(lf0 > 1., (lf0 - mu_s)/std_s * std_t + mu_t, lf0)
lf0 = tf.where(lf0 > 1., tf.exp(lf0), lf0)
return lf0
def make_minibatch(self, valid_anchors):
with tf.variable_scope('rpn_minibatch'):
# in labels(shape is [N, ]): 1 is positive, 0 is negative, -1 is ignored
labels, anchor_matched_gtboxes, object_mask = \
self.rpn_find_positive_negative_samples(valid_anchors) # [num_of_valid_anchors, ]
positive_indices = tf.reshape(tf.where(tf.equal(labels, 1.0)), [-1]) # use labels is same as object_mask
num_of_positives = tf.minimum(tf.shape(positive_indices)[0],
tf.cast(self.rpn_mini_batch_size * self.rpn_positives_ratio, tf.int32))
# num of positives <= minibatch_size * 0.5
positive_indices = tf.random_shuffle(positive_indices)
positive_indices = tf.slice(positive_indices, begin=[0], size=[num_of_positives])
# positive_anchors = tf.gather(self.anchors, positive_indices)
negative_indices = tf.reshape(tf.where(tf.equal(labels, 0.0)), [-1])
num_of_negatives = tf.minimum(self.rpn_mini_batch_size - num_of_positives,
tf.shape(negative_indices)[0])
negative_indices = tf.random_shuffle(negative_indices)
negative_indices = tf.slice(negative_indices, begin=[0], size=[num_of_negatives])
# negative_anchors = tf.gather(self.anchors, negative_indices)
minibatch_indices = tf.concat([positive_indices, negative_indices], axis=0)
minibatch_indices = tf.random_shuffle(minibatch_indices)
minibatch_anchor_matched_gtboxes = tf.gather(anchor_matched_gtboxes, minibatch_indices)
object_mask = tf.gather(object_mask, minibatch_indices)
labels = tf.cast(tf.gather(labels, minibatch_indices), tf.int32)
labels_one_hot = tf.one_hot(labels, depth=2)
return minibatch_indices, minibatch_anchor_matched_gtboxes, object_mask, labels_one_hot
def mm_inverse_cdf(epsilon, n=10):
acc = tf.zeros(epsilon.shape)
a_acc = tf.zeros(epsilon.shape)
K = tf.zeros(epsilon.shape)
for ki in xrange(n):
with tf.variable_scope("a_scope"):
a = tf.get_variable("a{}".format(ki), epsilon.get_shape(), dtype = tf.float32)
K += a/n
for ki in xrange(n):
with tf.variable_scope("a_scope", reuse=True):
a = tf.get_variable("a{}".format(ki), epsilon.get_shape(), dtype = tf.float32)
a_prev = a_acc
a_acc += a
a_next = a_acc
v0 = tf.where(e >= a_prev/K, e, tf.zeros(e.shape))
v1 = tf.where(v0 <= a_next/K, v0, tf.zeros(e.shape))
acc += v1 #* a # (ki/K + (K/a) * (e - start))
return acc
def tf_format_mnist_images(X, Y, Y_, n=100, lines=10):
correct_prediction = tf.equal(tf.argmax(Y,1), tf.argmax(Y_,1))
correctly_recognised_indices = tf.squeeze(tf.where(correct_prediction), [1]) # indices of correctly recognised images
incorrectly_recognised_indices = tf.squeeze(tf.where(tf.logical_not(correct_prediction)), [1]) # indices of incorrectly recognised images
everything_incorrect_first = tf.concat([incorrectly_recognised_indices, correctly_recognised_indices], 0) # images reordered with indeces of unrecognised images first
everything_incorrect_first = tf.slice(everything_incorrect_first, [0], [n]) # compute first 100 only - no space to display more anyway
# compute n=100 digits to display only
Xs = tf.gather(X, everything_incorrect_first)
Ys = tf.gather(Y, everything_incorrect_first)
Ys_ = tf.gather(Y_, everything_incorrect_first)
correct_prediction_s = tf.gather(correct_prediction, everything_incorrect_first)
digits_left = tf.image.grayscale_to_rgb(tensorflowvisu_digits.digits_left())
correct_tags = tf.gather(digits_left, tf.argmax(Ys_, 1)) # correct digits to be printed on the images
digits_right = tf.image.grayscale_to_rgb(tensorflowvisu_digits.digits_right())
computed_tags = tf.gather(digits_right, tf.argmax(Ys, 1)) # computed digits to be printed on the images
#superimposed_digits = correct_tags+computed_tags
superimposed_digits = tf.where(correct_prediction_s, tf.zeros_like(correct_tags),correct_tags+computed_tags) # only pring the correct and computed digits on unrecognised images
correct_bkg = tf.reshape(tf.tile([1.3,1.3,1.3], [28*28]), [1, 28,28,3]) # white background
incorrect_bkg = tf.reshape(tf.tile([1.3,1.0,1.0], [28*28]), [1, 28,28,3]) # red background
recognised_bkg = tf.gather(tf.concat([incorrect_bkg, correct_bkg], 0), tf.cast(correct_prediction_s, tf.int32)) # pick either the red or the white background depending on recognised status
I = tf.image.grayscale_to_rgb(Xs)
I = ((1-(I+superimposed_digits))*recognised_bkg)/1.3 # stencil extra data on top of images and reorder them unrecognised first
I = tf.image.convert_image_dtype(I, tf.uint8, saturate=True)
Islices = [] # 100 images => 10x10 image block
for imslice in range(lines):
Islices.append(tf.concat(tf.unstack(tf.slice(I, [imslice*n//lines,0,0,0], [n//lines,28,28,3])), 1))
I = tf.concat(Islices, 0)
return I
def apply_updates(self, allow_no_op: bool = False) -> tf.Operation:
# Construct training op to update the registered variables based on their gradients
tfutil.assert_tf_initialized()
assert not self._updates_applied
self._updates_applied = True
all_ops = []
# Check for no-op
if allow_no_op and len(self._devices) == 0:
with tfutil.absolute_name_scope(self.scope):
return tf.no_op(name="TrainingOp")
# Clean up gradients
for device_idx, device in enumerate(self._devices.values()):
with tfutil.absolute_name_scope(self.scope + "/Clean%d" %
device_idx), tf.device(
device.name):
for var, grad in device.grad_raw.items():
# Filter out disconnected gradients and convert to float32
grad = [g for g in grad if g is not None]
grad = [tf.cast(g, tf.float32) for g in grad]
# Sum within the device
if len(grad) == 0:
# No gradients => zero
grad = tf.zeros(var.shape)
elif len(grad) == 1:
# Single gradient => use as is
grad = grad[0]
else:
# Multiple gradients => sum
grad = tf.add_n(grad)
# Scale as needed
scale = 1.0 / len(device.grad_raw[var]) / len(
self._devices)
scale = tf.constant(scale, dtype=tf.float32, name="scale")
if self.minibatch_multiplier is not None:
scale /= tf.cast(self.minibatch_multiplier, tf.float32)
scale = self.undo_loss_scaling(scale)
device.grad_clean[var] = grad * scale
# Sum gradients across devices
if len(self._devices) > 1:
with tfutil.absolute_name_scope(self.scope +
"/Broadcast"), tf.device(None):
for all_vars in zip(*[
device.grad_clean.keys()
for device in self._devices.values()
]):
if len(all_vars) > 0 and all(
dim > 0 for dim in all_vars[0].shape.as_list()
): # NCCL does not support zero-sized tensors
all_grads = [
device.grad_clean[var] for device, var in zip(
self._devices.values(), all_vars)
]
all_grads = nccl_ops.all_sum(all_grads)
for device, var, grad in zip(self._devices.values(),
all_vars, all_grads):
device.grad_clean[var] = grad
# Apply updates separately on each device
for device_idx, device in enumerate(self._devices.values()):
with tfutil.absolute_name_scope(self.scope + "/Apply%d" %
device_idx), tf.device(
device.name):
# Accumulate gradients over time
if self.minibatch_multiplier is None:
acc_ok = tf.constant(True, name="acc_ok")
device.grad_acc = OrderedDict(device.grad_clean)
else:
# Create variables
with tf.control_dependencies(None):
for var in device.grad_clean.keys():
device.grad_acc_vars[var] = tf.Variable(
tf.zeros(var.shape),
trainable=False,
name="grad_acc_var")
device.grad_acc_count = tf.Variable(
tf.zeros([]),
trainable=False,
name="grad_acc_count")
# Track counter
count_cur = device.grad_acc_count + 1.0
count_inc_op = lambda: tf.assign(device.grad_acc_count,
count_cur)
count_reset_op = lambda: tf.assign(device.grad_acc_count,
tf.zeros([]))
acc_ok = (count_cur >= tf.cast(self.minibatch_multiplier,
tf.float32))
all_ops.append(
tf.cond(acc_ok, count_reset_op, count_inc_op))
# Track gradients
for var, grad in device.grad_clean.items():
acc_var = device.grad_acc_vars[var]
acc_cur = acc_var + grad
device.grad_acc[var] = acc_cur
with tf.control_dependencies([acc_cur]):
acc_inc_op = lambda: tf.assign(acc_var, acc_cur)
acc_reset_op = lambda: tf.assign(
acc_var, tf.zeros(var.shape))
all_ops.append(
tf.cond(acc_ok, acc_reset_op, acc_inc_op))
# No overflow => apply gradients
all_ok = tf.reduce_all(
tf.stack([acc_ok] + [
tf.reduce_all(tf.is_finite(g))
for g in device.grad_acc.values()
]))
apply_op = lambda: device.optimizer.apply_gradients(
[(tf.cast(grad, var.dtype), var)
for var, grad in device.grad_acc.items()])
all_ops.append(tf.cond(all_ok, apply_op, tf.no_op))
# Adjust loss scaling
if self.use_loss_scaling:
ls_inc_op = lambda: tf.assign_add(device.loss_scaling_var,
self.loss_scaling_inc)
ls_dec_op = lambda: tf.assign_sub(device.loss_scaling_var,
self.loss_scaling_dec)
ls_update_op = lambda: tf.group(
tf.cond(all_ok, ls_inc_op, ls_dec_op))
all_ops.append(tf.cond(acc_ok, ls_update_op, tf.no_op))
# Last device => report statistics
if device_idx == len(self._devices) - 1:
all_ops.append(
autosummary.autosummary(self.id + "/learning_rate",
self.learning_rate))
all_ops.append(
autosummary.autosummary(self.id +
"/overflow_frequency",
tf.where(all_ok, 0, 1),
condition=acc_ok))
if self.use_loss_scaling:
all_ops.append(
autosummary.autosummary(
self.id + "/loss_scaling_log2",
device.loss_scaling_var))
# Initialize variables
self.reset_optimizer_state()
if self.use_loss_scaling:
tfutil.init_uninitialized_vars(
[device.loss_scaling_var for device in self._devices.values()])
if self.minibatch_multiplier is not None:
tfutil.run([
var.initializer for device in self._devices.values()
for var in list(device.grad_acc_vars.values()) +
[device.grad_acc_count]
])
# Group everything into a single op
with tfutil.absolute_name_scope(self.scope):
return tf.group(*all_ops, name="TrainingOp")
def fix_grad(grad):
return tf.where(is_bad(grad), tf.zeros_like(grad),
grad) if grad is not None else None
def _build_sampler(self):
"""Build the sampler ops and the log_prob ops."""
print("-" * 80)
print("Build controller sampler")
anchors = []
anchors_w_1 = []
arc_seq = []
entropys = []
log_probs = []
skip_count = []
skip_penaltys = []
prev_c = [
tf.zeros([1, self.lstm_size], tf.float32)
for _ in xrange(self.lstm_num_layers)
]
prev_h = [
tf.zeros([1, self.lstm_size], tf.float32)
for _ in xrange(self.lstm_num_layers)
]
inputs = self.g_emb
skip_targets = tf.constant([1.0 - self.skip_target, self.skip_target],
dtype=tf.float32)
for layer_id in xrange(self.num_layers):
if self.search_whole_channels:
next_c, next_h = stack_lstm(inputs, prev_c, prev_h,
self.w_lstm)
prev_c, prev_h = next_c, next_h
logit = tf.matmul(next_h[-1], self.w_soft)
if self.temperature is not None:
logit /= self.temperature
if self.tanh_constant is not None:
logit = self.tanh_constant * tf.tanh(logit)
if self.search_for == "macro" or self.search_for == "branch":
branch_id = tf.multinomial(logit, 1) # Deprecated
# branch_id = tf.random.categorical(logit, 1)
branch_id = tf.cast(branch_id, tf.int32)
branch_id = tf.reshape(branch_id, [1])
elif self.search_for == "connection":
branch_id = tf.constant([0], dtype=tf.int32)
else:
raise ValueError("Unknown search_for {}".format(
self.search_for))
arc_seq.append(branch_id)
log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logit, labels=branch_id)
log_probs.append(log_prob)
entropy = tf.stop_gradient(log_prob * tf.exp(-log_prob))
entropys.append(entropy)
inputs = tf.nn.embedding_lookup(self.w_emb, branch_id)
else:
for branch_id in xrange(self.num_branches):
next_c, next_h = stack_lstm(inputs, prev_c, prev_h,
self.w_lstm)
prev_c, prev_h = next_c, next_h
logit = tf.matmul(next_h[-1],
self.w_soft["start"][branch_id])
if self.temperature is not None:
logit /= self.temperature
if self.tanh_constant is not None:
logit = self.tanh_constant * tf.tanh(logit)
start = tf.multinomial(logit, 1) # Deprecated
# start = tf.random.categorical(logit, 1)
start = tf.cast(start, tf.int32)
start = tf.reshape(start, [1])
arc_seq.append(start)
log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logit, labels=start)
log_probs.append(log_prob)
entropy = tf.stop_gradient(log_prob * tf.exp(-log_prob))
entropys.append(entropy)
inputs = tf.nn.embedding_lookup(
self.w_emb["start"][branch_id], start)
next_c, next_h = stack_lstm(inputs, prev_c, prev_h,
self.w_lstm)
prev_c, prev_h = next_c, next_h
logit = tf.matmul(next_h[-1],
self.w_soft["count"][branch_id])
if self.temperature is not None:
logit /= self.temperature
if self.tanh_constant is not None:
logit = self.tanh_constant * tf.tanh(logit)
mask = tf.range(0,
limit=self.out_filters - 1,
delta=1,
dtype=tf.int32)
mask = tf.reshape(mask, [1, self.out_filters - 1])
mask = tf.less_equal(mask, self.out_filters - 1 - start)
logit = tf.where(mask,
x=logit,
y=tf.fill(tf.shape(logit), -np.inf))
count = tf.multinomial(logit, 1) # Deprecated
# count = tf.random.categorical(logit, 1)
count = tf.cast(count, tf.int32)
count = tf.reshape(count, [1])
arc_seq.append(count + 1)
log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logit, labels=count)
log_probs.append(log_prob)
entropy = tf.stop_gradient(log_prob * tf.exp(-log_prob))
entropys.append(entropy)
inputs = tf.nn.embedding_lookup(
self.w_emb["count"][branch_id], count)
next_c, next_h = stack_lstm(inputs, prev_c, prev_h, self.w_lstm)
prev_c, prev_h = next_c, next_h
if layer_id > 0:
query = tf.concat(anchors_w_1, axis=0)
query = tf.tanh(query + tf.matmul(next_h[-1], self.w_attn_2))
query = tf.matmul(query, self.v_attn)
logit = tf.concat([-query, query], axis=1)
if self.temperature is not None:
logit /= self.temperature
if self.tanh_constant is not None:
logit = self.tanh_constant * tf.tanh(logit)
skip = tf.multinomial(logit, 1) # Deprecated
# skip = tf.random.categorical(logit, 1)
skip = tf.cast(skip, tf.int32)
skip = tf.reshape(skip, [layer_id])
arc_seq.append(skip)
skip_prob = tf.sigmoid(logit)
kl = skip_prob * tf.log(skip_prob / skip_targets)
kl = tf.reduce_sum(kl)
skip_penaltys.append(kl)
log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logit, labels=skip)
log_probs.append(tf.reduce_sum(log_prob, keepdims=True))
entropy = tf.stop_gradient(
tf.reduce_sum(log_prob * tf.exp(-log_prob), keepdims=True))
entropys.append(entropy)
skip = tf.cast(skip, tf.float32)
skip = tf.reshape(skip, [1, layer_id])
skip_count.append(tf.reduce_sum(skip))
inputs = tf.matmul(skip, tf.concat(anchors, axis=0))
inputs /= 1.0 + tf.reduce_sum(skip)
else:
inputs = self.g_emb
anchors.append(next_h[-1])
anchors_w_1.append(tf.matmul(next_h[-1], self.w_attn_1))
arc_seq = tf.concat(arc_seq, axis=0)
self.sample_arc = tf.reshape(arc_seq, [-1])
entropys = tf.stack(entropys)
self.sample_entropy = tf.reduce_sum(entropys)
log_probs = tf.stack(log_probs)
self.sample_log_prob = tf.reduce_sum(log_probs)
skip_count = tf.stack(skip_count)
self.skip_count = tf.reduce_sum(skip_count)
skip_penaltys = tf.stack(skip_penaltys)
self.skip_penaltys = tf.reduce_mean(skip_penaltys)
def _generate_panoptic_masks(self, boxes, scores, classes,
detections_masks, segmentation_mask):
"""Generates panoptic masks for a single image.
This function implements the following steps to merge instance and semantic
segmentation masks described in https://arxiv.org/pdf/1901.02446.pdf
Steps:
1. resolving overlaps between different instances based on their
confidence scores
2. resolving overlaps between instance and semantic segmentation
outputs in favor of instances
3. removing any stuff regions labeled other or under a given area
threshold.
Args:
boxes: A `tf.Tensor` of shape [num_rois, 4], representing the bounding
boxes for detected objects.
scores: A `tf.Tensor` of shape [num_rois], representing the
confidence scores for each object.
classes: A `tf.Tensor` of shape [num_rois], representing the class
for each object.
detections_masks: A `tf.Tensor` of shape
[num_rois, mask_height, mask_width, 1], representing the cropped mask
for each object.
segmentation_mask: A `tf.Tensor` of shape [height, width], representing
the semantic segmentation output.
Returns:
Dict with the following keys:
- category_mask: A `tf.Tensor` for category masks.
- instance_mask: A `tf.Tensor for instance masks.
"""
# Offset stuff class predictions
segmentation_mask = tf.where(
tf.logical_or(
tf.equal(segmentation_mask, self._things_class_label),
tf.equal(segmentation_mask, self._void_class_label)),
segmentation_mask, segmentation_mask + self._stuff_classes_offset)
# sort instances by their scores
sorted_indices = tf.argsort(scores, direction='DESCENDING')
mask_shape = self._output_size + [1]
category_mask = tf.ones(mask_shape,
dtype=tf.float32) * self._void_class_label
instance_mask = tf.ones(mask_shape,
dtype=tf.float32) * self._void_instance_id
# filter instances with low confidence
sorted_scores = tf.sort(scores, direction='DESCENDING')
valid_indices = tf.where(sorted_scores > self._score_threshold)
# if no instance has sufficient confidence score, skip merging
# instance segmentation masks
if tf.shape(valid_indices)[0] > 0:
loop_end_idx = valid_indices[-1, 0] + 1
loop_end_idx = tf.minimum(tf.cast(loop_end_idx, dtype=tf.int32),
self._max_num_detections)
pasted_masks = self._paste_masks_fn(
(detections_masks[:loop_end_idx], boxes[:loop_end_idx]))
# add things segmentation to panoptic masks
for i in range(loop_end_idx):
# we process instances in decending order, which will make sure
# the overlaps are resolved based on confidence score
instance_idx = sorted_indices[i]
pasted_mask = pasted_masks[instance_idx]
class_id = tf.cast(classes[instance_idx], dtype=tf.float32)
# convert sigmoid scores to binary values
binary_mask = tf.greater(pasted_mask,
self._mask_binarize_threshold)
# filter empty instance masks
if not tf.reduce_sum(tf.cast(binary_mask, tf.float32)) > 0:
continue
overlap = tf.logical_and(
binary_mask,
tf.not_equal(category_mask, self._void_class_label))
binary_mask_area = tf.reduce_sum(
tf.cast(binary_mask, dtype=tf.float32))
overlap_area = tf.reduce_sum(tf.cast(overlap,
dtype=tf.float32))
# skip instance that have a big enough overlap with instances with
# higer scores
if overlap_area / binary_mask_area > self._things_overlap_threshold:
continue
# fill empty regions in category_mask represented by
# void_class_label with class_id of the instance.
category_mask = tf.where(
tf.logical_and(
binary_mask,
tf.equal(category_mask, self._void_class_label)),
tf.ones_like(category_mask) * class_id, category_mask)
# fill empty regions in the instance_mask represented by
# void_instance_id with the id of the instance, starting from 1
instance_mask = tf.where(
tf.logical_and(
binary_mask,
tf.equal(instance_mask, self._void_instance_id)),
tf.ones_like(instance_mask) *
tf.cast(instance_idx + 1, tf.float32), instance_mask)
stuff_class_ids = tf.unique(tf.reshape(segmentation_mask, [-1])).y
for stuff_class_id in stuff_class_ids:
if stuff_class_id == self._things_class_label:
continue
stuff_mask = tf.logical_and(
tf.equal(segmentation_mask, stuff_class_id),
tf.equal(category_mask, self._void_class_label))
stuff_mask_area = tf.reduce_sum(
tf.cast(stuff_mask, dtype=tf.float32))
if stuff_mask_area < self._stuff_area_threshold:
continue
category_mask = tf.where(
stuff_mask,
tf.ones_like(category_mask) * stuff_class_id, category_mask)
results = {
'category_mask': category_mask[:, :, 0],
'instance_mask': instance_mask[:, :, 0]
}
return results
train_opt = opt.minimize(loss)
return train_opt
input_placeholder = tf.placeholder(tf.int32, [None, None])
input_labels = tf.placeholder(tf.int64, [None])
keep_prob = tf.placeholder(tf.float32)
logits_layer, embedding_init,\
embedding_placeholder = model(input_placeholder, keep_prob)
loss_layer = add_loss(logits_layer, input_labels)
train_op = optimize(loss_layer)
predictions = tf.argmax(logits_layer, 1)
acc = tf.reduce_sum(
tf.where(tf.equal(predictions, input_labels),
tf.ones(tf.shape(predictions)), tf.zeros(tf.shape(predictions))))
sess = tf.Session()
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
sess.run(embedding_init, feed_dict={embedding_placeholder: embedding_matrix})
batch_size = FLAGS.batch_size
num_epochs = FLAGS.num_epochs
for epoch in range(num_epochs):
perm = np.random.permutation(np.arange(len(x_train)))
permutation_train = np.take(x_train, perm, axis=0)
permutation_labels = np.take(y_train, perm, axis=0)
overall_avg_loss = 0.0
def build_loss(self):
"""Builds the loss Tensor.
Outputs:
self.total_loss
"""
all_sen_embs = self.thought_vectors
if FLAGS.dropout:
mask_shp = [1, self.config.encoder_dim]
bin_mask = tf.random_uniform(mask_shp) > FLAGS.dropout_rate
bin_mask = tf.where(bin_mask, tf.ones(mask_shp),
tf.zeros(mask_shp))
src = all_sen_embs[0] * bin_mask
dst = all_sen_embs[1] * bin_mask
scores = tf.matmul(src, dst, transpose_b=True)
else:
scores = tf.matmul(all_sen_embs[0],
all_sen_embs[1],
transpose_b=True)
# Ignore source sentence
scores = tf.matrix_set_diag(scores, np.zeros(FLAGS.batch_size))
# Targets
targets_np = np.zeros((FLAGS.batch_size, FLAGS.batch_size))
ctxt_sent_pos = list(range(-FLAGS.context_size,
FLAGS.context_size + 1))
ctxt_sent_pos.remove(0)
for ctxt_pos in ctxt_sent_pos:
targets_np += np.eye(FLAGS.batch_size, k=ctxt_pos)
targets_np_sum = np.sum(targets_np, axis=1, keepdims=True)
targets_np = targets_np / targets_np_sum
targets = tf.constant(targets_np, dtype=tf.float32)
# Forward and backward scores
f_scores = scores[:-1]
b_scores = scores[1:]
self.f_scores = f_scores
self.b_scores = b_scores
losses = tf.nn.softmax_cross_entropy_with_logits(labels=targets,
logits=scores)
loss = tf.reduce_mean(losses)
tf.summary.scalar("losses/ent_loss", loss)
self.total_loss = loss
if self.mode == "eval":
f_max = tf.to_int64(tf.argmax(f_scores, axis=1))
b_max = tf.to_int64(tf.argmax(b_scores, axis=1))
targets = list(range(FLAGS.batch_size - 1))
targets = tf.constant(targets, dtype=tf.int64)
fwd_targets = targets + 1
names_to_values, names_to_updates = tf.contrib.slim.metrics.aggregate_metric_map(
{
"Acc/Fwd Acc":
tf.contrib.slim.metrics.streaming_accuracy(
f_max, fwd_targets),
"Acc/Bwd Acc":
tf.contrib.slim.metrics.streaming_accuracy(b_max, targets)
})
for name, value in names_to_values.items():
tf.summary.scalar(name, value)
self.eval_op = names_to_updates.values()
def _build_single_target(self, proposals, gt_boxes, gt_class_ids, img_shape):
'''
Args
---
proposals: [num_proposals, (y1, x1, y2, x2)] in regular coordinates.
gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)]
gt_class_ids: [num_gt_boxes]
img_shape: np.ndarray. [2]. (img_height, img_width)
Returns
---
rois: [num_rois, (y1, x1, y2, x2)]
target_matchs: [num_positive_rois]
target_deltas: [num_positive_rois, (dy, dx, log(dh), log(dw))]
'''
# remove padded proposals and gt boxes if any
proposals, _ = trim_zeros(proposals)
gt_boxes, non_zeros = trim_zeros(gt_boxes)
gt_boxes = tf.cast(gt_boxes, proposals.dtype)
gt_labels = tf.boolean_mask(gt_class_ids, non_zeros)
noise_mean = 5.0
noisy_gt_boxes = tf.add(gt_boxes,
tf.random.truncated_normal(tf.shape(gt_boxes), noise_mean, 0.1, dtype=proposals.dtype))
proposals_gt = tf.concat([proposals, noisy_gt_boxes], axis=0)
iou = geometry.compute_overlaps(proposals_gt, gt_boxes) # [rois_size, gt_bboxes_size]
max_overlaps = tf.reduce_max(iou, axis=1) # [rois_size, ]
gt_assignment = tf.argmax(iou, axis=1) # [rois_size, ]
labels = tf.gather(gt_labels, gt_assignment) # [rois_size, ]
# get FG and BG
fg_inds = tf.where(max_overlaps >= self.pos_iou_thr)[:, 0]
bg_inds = tf.where(tf.logical_and(max_overlaps < self.pos_iou_thr,
max_overlaps >= self.neg_iou_thr))[:, 0]
# filter FG/BG
if tf.size(fg_inds) > self._max_pos_samples:
fg_inds = tf.random.shuffle(fg_inds)[:self._max_pos_samples]
remaining = self.num_rcnn_deltas - tf.size(fg_inds)
num_bg = tf.size(bg_inds)
if tf.greater_equal(num_bg, remaining):
bg_inds = tf.random.shuffle(bg_inds)[:remaining]
else:
# sample with replacement from very poor overlaps if number of backgrounds is not enough
bg_inds = tf.where(max_overlaps < self.pos_iou_thr)[:, 0]
bg_inds = tf.random.shuffle(bg_inds)[:remaining]
num_bg = tf.size(bg_inds)
while remaining > num_bg:
dups = remaining - num_bg
dup_bgs = tf.random.shuffle(bg_inds)[:dups]
bg_inds = tf.concat([bg_inds, dup_bgs], axis=0)
num_bg = tf.size(bg_inds)
# tf.print('proposal target generated %d fgs and %d bgs.' % (tf.size(fg_inds), tf.size(bg_inds)))
keep_inds = tf.concat([fg_inds, bg_inds], axis=0)
final_rois = tf.gather(proposals_gt, keep_inds) # rois[keep_inds]
final_labels = tf.gather(labels, keep_inds) # labels[keep_inds]
zero_indices = tf.expand_dims(tf.range(tf.size(fg_inds), tf.size(keep_inds), dtype=tf.int32), axis=1)
zero_labels = tf.zeros(tf.shape(zero_indices)[0], dtype=tf.int32)
final_labels = tf.tensor_scatter_nd_update(final_labels, zero_indices, zero_labels)
# inside weights - positive examples are set, rest are zeros
bbox_inside_weights = tf.zeros((tf.size(keep_inds), self.num_classes, 4), dtype=tf.float32)
if tf.size(fg_inds) > 0:
if self.reg_class_agnostic:
cur_index = tf.transpose(tf.stack([tf.range(tf.size(fg_inds)), tf.zeros(tf.size(fg_inds), dtype=tf.int32)]))
else:
cur_index = tf.stack([tf.range(tf.size(fg_inds)), tf.gather(labels, fg_inds)], axis=1)
bbox_inside_weights = tf.tensor_scatter_nd_update(bbox_inside_weights,
cur_index,
tf.ones([tf.size(fg_inds), 4], bbox_inside_weights.dtype))
bbox_inside_weights = tf.reshape(bbox_inside_weights, [-1, self.num_classes * 4])
final_bbox_targets = tf.zeros((tf.size(keep_inds), self.num_classes, 4), dtype=tf.float32)
if tf.size(fg_inds) > 0:
bbox_targets = transforms.bbox2delta(
tf.gather(final_rois, tf.range(tf.size(fg_inds))),
tf.gather(gt_boxes, tf.gather(gt_assignment, fg_inds)),
target_stds=self.target_stds, target_means=self.target_means)
if self.reg_class_agnostic:
final_bbox_targets = tf.tensor_scatter_nd_update(
final_bbox_targets,
tf.transpose(tf.stack([tf.range(tf.size(fg_inds)),
tf.zeros(tf.size(fg_inds), dtype=tf.int32)])),
bbox_targets)
else:
final_bbox_targets = tf.tensor_scatter_nd_update(
final_bbox_targets,
tf.stack([tf.range(tf.size(fg_inds)),
tf.gather(labels, fg_inds)], axis=1), bbox_targets)
final_bbox_targets = tf.reshape(final_bbox_targets, [-1, self.num_classes * 4])
bbox_outside_weights = tf.ones_like(bbox_inside_weights, dtype=bbox_inside_weights.dtype) * 1.0 / self.num_rcnn_deltas
fg_assignments = tf.gather(gt_assignment, keep_inds)
return (tf.stop_gradient(final_rois), tf.stop_gradient(final_labels), tf.stop_gradient(final_bbox_targets),
tf.stop_gradient(bbox_inside_weights), tf.stop_gradient(bbox_outside_weights), tf.stop_gradient(fg_assignments))
def huber_loss(x, delta=1.0):
return tf.where(tf.abs(x) < delta, 0.5 * tf.square(x), delta * tf.abs(x) - 0.5* delta)
def __init__(self, args, is_training=True):
self.graph = tf.Graph()
with self.graph.as_default():
with tf.variable_scope("input") as scope:
self.inputs = tf.placeholder(tf.int32, shape=[None, args.max_length])
self.seq_length = tf.placeholder(tf.int32, shape=[None])
self.real_sample = tf.placeholder(dtype=tf.int32, shape=[None, args.max_length])
self.real_seq_length = tf.placeholder(tf.int32, shape=[None])
# embedding
with tf.variable_scope("generator"):
self.false_sample, self.translate_id, self.mapping_id, self.all_phn_distribution = idx2phn_distribution(self.inputs, self.seq_length, args.idx_size, args.phn_size)
with tf.variable_scope("discriminator"):
emb = creating_embedding_matrix(args.phn_size, args.discriminator_hidden_units, 'emb')
real_sample = tf.one_hot(self.real_sample, args.phn_size, dtype=tf.float32)
#mask
mask = tf.sequence_mask(self.real_seq_length, tf.shape(real_sample)[1])
mask = tf.tile(tf.expand_dims(mask,-1),[1,1,args.phn_size])
paddings = tf.zeros_like(real_sample)
real_sample = tf.where(mask, real_sample, paddings)
true_sample_pred = weak_discriminator(real_sample, emb, args.discriminator_hidden_units)
false_sample_pred = weak_discriminator(self.false_sample, emb, args.discriminator_hidden_units, reuse=True)
alpha = tf.random_uniform(
shape=[tf.shape(real_sample)[0],1,1],
minval=0.,
maxval=1.)
differences = self.false_sample - real_sample
interpolates = real_sample + (alpha*differences)
interpolates_sample_pred = weak_discriminator(interpolates, emb, args.discriminator_hidden_units, reuse=True)
gradients = tf.gradients(interpolates_sample_pred,[interpolates])[0]
slopes = tf.sqrt(tf.reduce_sum(tf.square(gradients), reduction_indices=[1,2]))
gradient_penalty = tf.reduce_mean((slopes-1.)**2)
#loss
if is_training:
self.learning_rate = tf.placeholder(tf.float32, shape=[])
with tf.variable_scope("discriminator_loss") as scope:
self.true_sample_score = tf.reduce_mean(true_sample_pred)
self.false_sample_score = tf.reduce_mean(false_sample_pred)
self.discriminator_loss = -(self.true_sample_score - self.false_sample_score) + 10.0*gradient_penalty
with tf.variable_scope("generator_loss") as scope:
self.generator_loss = self.true_sample_score - self.false_sample_score
self.discriminator_variables = [v for v in tf.trainable_variables() if v.name.startswith("discriminator")]
self.generator_variables = [v for v in tf.trainable_variables() if v.name.startswith("generator")]
#discriminator optimizer
self.discriminator_global_step = tf.Variable(0, name='discriminator_global_step', trainable=False)
train_discriminator_op = tf.train.AdamOptimizer(self.learning_rate) #adam is better
gradients, variables = zip(*train_discriminator_op.compute_gradients(self.discriminator_loss,\
var_list=self.discriminator_variables))
gradients, _ = tf.clip_by_global_norm(gradients, 5.0)
self.train_discriminator_op = train_discriminator_op.apply_gradients(zip(gradients, variables), global_step=self.discriminator_global_step)
#generator optimizer
self.generator_global_step = tf.Variable(0, name='generator_global_step', trainable=False)
train_generator_op = tf.train.AdamOptimizer(self.learning_rate)
gradients, variables = zip(*train_generator_op.compute_gradients(self.generator_loss,\
var_list=self.generator_variables))
gradients, _ = tf.clip_by_global_norm(gradients, 5.0)
self.train_generator_op = train_generator_op.apply_gradients(zip(gradients, variables), global_step=self.generator_global_step)
#write summary
discriminator_summary = tf.summary.scalar("discriminator_loss", self.discriminator_loss)
generator_summary = tf.summary.scalar("generator_loss", self.generator_loss)
self.discriminator_summary = tf.summary.merge([discriminator_summary])
self.generator_summary = tf.summary.merge([generator_summary])
def bits(
self,
factory: Optional[AbstractFactory] = None,
ensure_positive_interpretation: bool = False
) -> AbstractTensor:
"""Convert to a pure bits representation."""
factory = factory or self.factory
with tf.name_scope('to_bits'):
# we will extract the bits in chunks of 16 as that's reasonable for the explicit CRT
max_chunk_bitsize = 16
q, r = BITSIZE // max_chunk_bitsize, BITSIZE % max_chunk_bitsize
chunk_bitsizes = [max_chunk_bitsize] * q + ([r]
if r > 0 else [])
chunks_modulus = [2**bitsize for bitsize in chunk_bitsizes]
remaining = self
if ensure_positive_interpretation:
# To get the right bit pattern for negative numbers we need to apply a correction
# to the first chunk. Unfortunately, this isn't known until all bits have been
# extracted and hence we extract bits both with and without the correction and
# select afterwards. Although these two versions could be computed independently
# we here combine them into a single tensor to keep the graph smaller.
shape = self.shape.as_list()
shape_value = [1] + shape
shape_correction = [2] + [1] * len(shape)
# this means that chunk[0] is uncorrected and chunk[1] is corrected
correction_raw = [0, self.modulus % chunks_modulus[0]]
correction = tf.constant(correction_raw,
shape=shape_correction,
dtype=self.factory.native_type)
remaining = remaining.reshape(shape_value)
# extract chunks
chunks = []
apply_correction = ensure_positive_interpretation
for chunk_modulus in chunks_modulus:
# extract chunk from remaining
chunk = crt_mod(remaining.backing, chunk_modulus)
# apply correction only to the first chunk
if apply_correction:
chunk = (chunk + correction) % chunk_modulus
apply_correction = False
# save for below
chunks.append(chunk)
# perform right shift on remaining
shifted = (remaining - master_factory.tensor(chunk))
remaining = shifted * inverse(chunk_modulus, self.modulus)
if ensure_positive_interpretation:
# pick between corrected and uncorrected based on MSB
msb = chunks[-1][0] >= (chunks_modulus[-1]) // 2
chunks = [
tf.where(
msb,
chunk[1], # corrected
chunk[0], # uncorrected
) for chunk in chunks
]
# extract bits from chunks
chunks_bits = [
binarize(chunk, chunk_bitsize)
for chunk, chunk_bitsize in zip(chunks, chunk_bitsizes)
]
# combine bits of chunks
bits = tf.concat(chunks_bits, axis=-1)
return factory.tensor(bits)
def smooth_L1(self, x):
return tf.where(tf.less_equal(tf.abs(x), 1.0),
tf.multiply(0.5, tf.pow(x, 2.0)),
tf.subtract(tf.abs(x), 0.5))
def _static_subsample(self, indicator, batch_size, labels):
"""Returns subsampled minibatch.
Args:
indicator: boolean tensor of shape [N] whose True entries can be sampled.
N should be a complie time constant.
batch_size: desired batch size. This scalar cannot be None.
labels: boolean tensor of shape [N] denoting positive(=True) and negative
(=False) examples. N should be a complie time constant.
Returns:
sampled_idx_indicator: boolean tensor of shape [N], True for entries which
are sampled. It ensures the length of output of the subsample is always
batch_size, even when number of examples set to True in indicator is
less than batch_size.
Raises:
ValueError: if labels and indicator are not 1D boolean tensors.
"""
# Check if indicator and labels have a static size.
if not indicator.shape.is_fully_defined():
raise ValueError(
'indicator must be static in shape when is_static is'
'True')
if not labels.shape.is_fully_defined():
raise ValueError('labels must be static in shape when is_static is'
'True')
if not isinstance(batch_size, int):
raise ValueError(
'batch_size has to be an integer when is_static is'
'True.')
input_length = tf.shape(indicator)[0]
# Set the number of examples set True in indicator to be at least
# batch_size.
num_true_sampled = tf.reduce_sum(tf.cast(indicator, tf.float32))
additional_false_sample = tf.less_equal(
tf.cumsum(tf.cast(tf.logical_not(indicator), tf.float32)),
batch_size - num_true_sampled)
indicator = tf.logical_or(indicator, additional_false_sample)
# Shuffle indicator and label. Need to store the permutation to restore the
# order post sampling.
permutation = tf.random_shuffle(tf.range(input_length))
indicator = ops.matmul_gather_on_zeroth_axis(
tf.cast(indicator, tf.float32), permutation)
labels = ops.matmul_gather_on_zeroth_axis(tf.cast(labels, tf.float32),
permutation)
# index (starting from 1) when indicator is True, 0 when False
indicator_idx = tf.where(tf.cast(indicator, tf.bool),
tf.range(1, input_length + 1),
tf.zeros(input_length, tf.int32))
# Replace -1 for negative, +1 for positive labels
signed_label = tf.where(
tf.cast(labels, tf.bool), tf.ones(input_length, tf.int32),
tf.scalar_mul(-1, tf.ones(input_length, tf.int32)))
# negative of index for negative label, positive index for positive label,
# 0 when indicator is False.
signed_indicator_idx = tf.multiply(indicator_idx, signed_label)
sorted_signed_indicator_idx = tf.nn.top_k(signed_indicator_idx,
input_length,
sorted=True).values
[num_positive_samples, num_negative_samples
] = self._get_num_pos_neg_samples(sorted_signed_indicator_idx,
batch_size)
sampled_idx = self._get_values_from_start_and_end(
sorted_signed_indicator_idx, num_positive_samples,
num_negative_samples, batch_size)
# Shift the indices to start from 0 and remove any samples that are set as
# False.
sampled_idx = tf.abs(sampled_idx) - tf.ones(batch_size, tf.int32)
sampled_idx = tf.multiply(
tf.cast(tf.greater_equal(sampled_idx, tf.constant(0)), tf.int32),
sampled_idx)
sampled_idx_indicator = tf.cast(
tf.reduce_sum(tf.one_hot(sampled_idx, depth=input_length), axis=0),
tf.bool)
# project back the order based on stored permutations
reprojections = tf.one_hot(permutation,
depth=input_length,
dtype=tf.float32)
return tf.cast(
tf.tensordot(tf.cast(sampled_idx_indicator, tf.float32),
reprojections,
axes=[0, 0]), tf.bool)
def calibration_mmce_w_loss(logits, correct_labels):
"""Function to compute the MMCE_w loss."""
predicted_probs = tf.nn.softmax(logits)
range_index = tf.to_int64(
tf.expand_dims(tf.range(0,
tf.shape(predicted_probs)[0]), 1))
predicted_labels = tf.argmax(predicted_probs, axis=1)
gather_index = tf.concat(
[range_index, tf.expand_dims(predicted_labels, 1)], axis=1)
predicted_probs = tf.reduce_max(predicted_probs, 1)
correct_mask = tf.where(tf.equal(correct_labels, predicted_labels),
tf.ones(tf.shape(correct_labels)),
tf.zeros(tf.shape(correct_labels)))
sigma = 0.2
def tf_kernel(matrix):
"""Kernel was taken to be a laplacian kernel with sigma = 0.4."""
return tf.exp(-1.0 * tf.abs(matrix[:, :, 0] - matrix[:, :, 1]) /
(2 * 0.2))
k = tf.to_int32(tf.reduce_sum(correct_mask))
k_p = tf.to_int32(tf.reduce_sum(1.0 - correct_mask))
cond_k = tf.where(tf.equal(k, 0), 0, 1)
cond_k_p = tf.where(tf.equal(k_p, 0), 0, 1)
k = tf.maximum(k, 1) * cond_k * cond_k_p + (1 - cond_k * cond_k_p) * 2
k_p = tf.maximum(k_p, 1) * cond_k_p * cond_k + (
(1 - cond_k_p * cond_k) * (tf.shape(correct_mask)[0] - 2))
correct_prob, _ = tf.nn.top_k(predicted_probs * correct_mask, k)
incorrect_prob, _ = tf.nn.top_k(predicted_probs * (1 - correct_mask), k_p)
def get_pairs(tensor1, tensor2):
correct_prob_tiled = tf.expand_dims(
tf.tile(tf.expand_dims(tensor1, 1), [1, tf.shape(tensor1)[0]]), 2)
incorrect_prob_tiled = tf.expand_dims(
tf.tile(tf.expand_dims(tensor2, 1), [1, tf.shape(tensor2)[0]]), 2)
correct_prob_pairs = tf.concat(
[correct_prob_tiled,
tf.transpose(correct_prob_tiled, [1, 0, 2])],
axis=2)
incorrect_prob_pairs = tf.concat([
incorrect_prob_tiled,
tf.transpose(incorrect_prob_tiled, [1, 0, 2])
],
axis=2)
correct_prob_tiled_1 = tf.expand_dims(
tf.tile(tf.expand_dims(tensor1, 1), [1, tf.shape(tensor2)[0]]), 2)
incorrect_prob_tiled_1 = tf.expand_dims(
tf.tile(tf.expand_dims(tensor2, 1), [1, tf.shape(tensor1)[0]]), 2)
correct_incorrect_pairs = tf.concat([
correct_prob_tiled_1,
tf.transpose(incorrect_prob_tiled_1, [1, 0, 2])
],
axis=2)
return correct_prob_pairs, incorrect_prob_pairs, correct_incorrect_pairs
correct_prob_pairs, incorrect_prob_pairs,\
correct_incorrect_pairs = get_pairs(correct_prob, incorrect_prob)
correct_kernel = tf_kernel(correct_prob_pairs)
incorrect_kernel = tf_kernel(incorrect_prob_pairs)
correct_incorrect_kernel = tf_kernel(correct_incorrect_pairs)
sampling_weights_correct = tf.matmul(
tf.expand_dims(1.0 - correct_prob, 1),
tf.transpose(tf.expand_dims(1.0 - correct_prob, 1)))
correct_correct_vals = get_out_tensor(correct_kernel,
sampling_weights_correct)
sampling_weights_incorrect = tf.matmul(
tf.expand_dims(incorrect_prob, 1),
tf.transpose(tf.expand_dims(incorrect_prob, 1)))
incorrect_incorrect_vals = get_out_tensor(incorrect_kernel,
sampling_weights_incorrect)
sampling_correct_incorrect = tf.matmul(
tf.expand_dims(1.0 - correct_prob, 1),
tf.transpose(tf.expand_dims(incorrect_prob, 1)))
correct_incorrect_vals = get_out_tensor(correct_incorrect_kernel,
sampling_correct_incorrect)
correct_denom = tf.reduce_sum(1.0 - correct_prob)
incorrect_denom = tf.reduce_sum(incorrect_prob)
m = tf.reduce_sum(correct_mask)
n = tf.reduce_sum(1.0 - correct_mask)
mmd_error = 1.0 / (m * m + 1e-5) * tf.reduce_sum(correct_correct_vals)
mmd_error += 1.0 / (n * n + 1e-5) * tf.reduce_sum(incorrect_incorrect_vals)
mmd_error -= 2.0 / (m * n + 1e-5) * tf.reduce_sum(correct_incorrect_vals)
return tf.maximum(tf.stop_gradient(tf.to_float(cond_k*cond_k_p))*\
tf.sqrt(mmd_error + 1e-10), 0.0)
def build_graph(self, is_training=True, scope='generator'):
bn_decay = 0.95
self.step = tf.Variable(0, trainable=False)
self.pointclouds_input = tf.placeholder(tf.float32,
shape=(BATCH_SIZE, NUM_POINT,
3))
self.pointclouds_radius = tf.placeholder(tf.float32,
shape=(BATCH_SIZE))
self.pointclouds_poisson = tf.placeholder(tf.float32,
shape=(BATCH_SIZE, NUM_POINT,
3))
# self.pointclouds_dist = tf.placeholder(tf.float32, shape=(BATCH_SIZE, NUM_POINT))
self.pointclouds_idx = tf.placeholder(tf.int32,
shape=(BATCH_SIZE, NUM_POINT, 2))
self.pointclouds_edge = tf.placeholder(tf.float32,
shape=(BATCH_SIZE, NUM_EDGE, 6))
self.pointclouds_plane = tf.placeholder(tf.float32,
shape=(BATCH_SIZE, NUM_FACE,
9))
self.pointclouds_plane_normal = tf.placeholder(tf.float32,
shape=(BATCH_SIZE,
NUM_FACE, 3))
# create the generator model
self.pred_dist, self.pred_coord, self.idx, self.transform = MODEL_GEN.get_gen_model(
self.pointclouds_input,
is_training,
scope=scope,
bradius=1.0,
num_addpoint=NUM_ADDPOINT,
reuse=None,
use_normal=False,
use_bn=False,
use_ibn=False,
bn_decay=bn_decay,
up_ratio=UP_RATIO,
idx=self.pointclouds_idx,
is_crop=IS_CROP)
###calculate the distance ground truth of upsample_point
self.pointclouds_dist = model_utils.distance_point2edge(
self.pred_coord, self.pointclouds_edge)
self.pointclouds_dist = tf.sqrt(
tf.reduce_min(self.pointclouds_dist, axis=-1))
self.pointclouds_dist_truncated = tf.minimum(0.5,
self.pointclouds_dist)
self.pred_dist = tf.minimum(0.5, tf.maximum(0.0, self.pred_dist))
# gather the edge
self.pred_edgecoord = tf.gather_nd(self.pred_coord, self.idx)
self.pred_edgedist = tf.gather_nd(self.pred_dist, self.idx)
self.edgedist = tf.gather_nd(self.pointclouds_dist_truncated, self.idx)
# ## The following code is okay when the batch size is 1
self.edge_threshold = tf.constant(
0.05, tf.float32, [1]) # select a small value when use 1k points
if True:
indics = tf.where(
tf.less_equal(self.pred_edgedist, self.edge_threshold)) #(?,2)
self.select_pred_edgecoord = tf.gather_nd(self.pred_edgecoord,
indics) #(?,3)
self.select_pred_edgedist = tf.gather_nd(self.pred_edgedist,
indics) #(?,3)
else:
indics = tf.where(
tf.less_equal(self.pointclouds_dist_truncated,
self.edge_threshold)) # (?,2)
self.select_pred_edgecoord = tf.gather_nd(self.pred_coord,
indics) # (?,3)
self.select_pred_edgedist = tf.gather_nd(self.pred_dist,
indics) # (?,3)
# if is_training is False:
# input_dist = model_utils.distance_point2edge(self.pointclouds_input,self.pointclouds_edge)
# input_dist = tf.sqrt(tf.reduce_min(input_dist,axis=-1))
# indics = tf.where(tf.less_equal(input_dist,self.edge_threshold))
# self.select_input_edge = tf.gather_nd(self.pointclouds_input, indics)
# self.select_input_edgedist = tf.gather_nd(input_dist,indics)
if is_training is False:
return
self.dist_mseloss = 1.0 / (0.4 + self.pointclouds_dist_truncated) * (
self.pointclouds_dist_truncated - self.pred_dist)**2
self.dist_mseloss = 5 * tf.reduce_mean(
self.dist_mseloss /
tf.expand_dims(self.pointclouds_radius**2, axis=-1))
tf.summary.scalar('loss/dist_loss', self.dist_mseloss)
tf.summary.histogram('dist/gt', self.pointclouds_dist_truncated)
tf.summary.histogram('dist/edge_dist', self.edgedist)
tf.summary.histogram('dist/pred', self.pred_dist)
# weight = tf.pow(0.98, tf.to_float(tf.div(self.step,200)))
weight = tf.maximum(0.5 - tf.to_float(self.step) / 200000.0, 0.0)
self.edgemask = tf.to_float(
tf.less_equal(
weight * self.edgedist + (1 - weight) * self.pred_edgedist,
0.15))
# self.edgemask = tf.to_float(tf.less_equal(self.edgedist,0.45))
self.edge_loss = 50 * tf.reduce_sum(
self.edgemask * self.edgedist**2 /
tf.expand_dims(self.pointclouds_radius**2, axis=-1)) / (
tf.reduce_sum(self.edgemask) + 1.0)
tf.summary.scalar('weight', weight)
tf.summary.histogram('loss/edge_mask', self.edgemask)
tf.summary.scalar('loss/edge_loss', self.edge_loss)
with tf.device('/gpu:0'):
self.plane_dist = model_utils.distance_point2mesh(
self.pred_coord, self.pointclouds_plane)
self.plane_dist = tf.reduce_min(self.plane_dist, axis=2)
# idx = tf.argmin(self.plane_dist, axis=2,output_type=tf.int32)
# idx0 = tf.tile(tf.reshape(tf.range(BATCH_SIZE), (BATCH_SIZE, 1)), (1, NUM_POINT*UP_RATIO/2))
# face_normal = tf.gather_nd(self.pointclouds_plane_normal,tf.stack([idx0,idx],axis=-1))
# dist = tf.where(tf.is_nan(dist),tf.zeros_like(dist),dist)
self.plane_loss = 500 * tf.reduce_mean(
self.plane_dist /
tf.expand_dims(self.pointclouds_radius**2, axis=-1))
tf.summary.scalar('loss/plane_loss', self.plane_loss)
#self.perulsionloss = 10*model_utils.get_uniform_loss1_orthdistance(self.pred_coord,face_normal, numpoint=NUM_POINT*UP_RATIO)
self.perulsionloss = 500 * model_utils.get_perulsion_loss1(
self.pred_coord, numpoint=NUM_POINT * UP_RATIO)
tf.summary.scalar('loss/perulsion_loss', self.perulsionloss)
# # Enforce the transformation as orthogonal matrix
# K = transform.get_shape()[1].value # BxKxK
# mat_diff = tf.matmul(transform, tf.transpose(transform, perm=[0, 2, 1]))
# mat_diff -= tf.constant(np.eye(K), dtype=tf.float32)
# self.mat_diff_loss = 0.01*tf.nn.l2_loss(mat_diff)
# tf.summary.scalar('loss/mat_loss', self.mat_diff_loss)
self.total_loss = self.dist_mseloss + self.plane_loss + self.edge_loss + self.perulsionloss + tf.losses.get_regularization_loss(
)
gen_update_ops = [
op for op in tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if op.name.startswith(scope)
]
gen_tvars = [
var for var in tf.trainable_variables()
if var.name.startswith(scope)
]
with tf.control_dependencies(gen_update_ops):
self.pre_gen_train = tf.train.AdamOptimizer(
BASE_LEARNING_RATE,
beta1=0.9).minimize(self.total_loss,
var_list=gen_tvars,
colocate_gradients_with_ops=False,
global_step=self.step)
# merge summary and add pointclouds summary
tf.summary.scalar('loss/regularation',
tf.losses.get_regularization_loss())
tf.summary.scalar('loss/total_loss', self.total_loss)
self.merged = tf.summary.merge_all()
self.pointclouds_image_input = tf.placeholder(
tf.float32, shape=[None, 500, 1500, 1])
pointclouds_input_summary = tf.summary.image(
'1_input', self.pointclouds_image_input, max_outputs=1)
self.pointclouds_image_pred = tf.placeholder(
tf.float32, shape=[None, 500, 1500, 1])
pointclouds_pred_summary = tf.summary.image(
'2_pred', self.pointclouds_image_pred, max_outputs=1)
self.pointclouds_image_gt = tf.placeholder(tf.float32,
shape=[None, 500, 1500, 1])
pointclouds_gt_summary = tf.summary.image('3_edge',
self.pointclouds_image_gt,
max_outputs=1)
self.image_merged = tf.summary.merge([
pointclouds_input_summary, pointclouds_pred_summary,
pointclouds_gt_summary
])
def tf_update_batch(self, loss_per_instance):
"""
Updates priority memory by performing the following steps:
1. Use saved indices from prior retrieval to reconstruct the batch
elements which will have their priorities updated.
2. Compute priorities for these elements.
3. Insert buffer elements to memory, potentially overwriting existing elements.
4. Update priorities of existing memory elements
5. Resort memory.
6. Update buffer insertion index.
Note that this implementation could be made more efficient by maintaining
a sorted version via sum trees.
:param loss_per_instance: Losses from recent batch to perform priority update
"""
# 1. We reconstruct the batch from the buffer and the priority memory via
# the TensorFlow variables holding the respective indices.
mask = tf.not_equal(
x=self.batch_indices,
y=tf.zeros(shape=tf.shape(input=self.batch_indices), dtype=tf.int32)
)
priority_indices = tf.reshape(tensor=tf.where(condition=mask), shape=[-1])
# These are elements from the buffer which first need to be inserted into the main memory.
sampled_buffer_batch = self.tf_retrieve_indices(
buffer_elements=self.last_batch_buffer_elems,
priority_indices=priority_indices
)
# Extract batch elements.
states = sampled_buffer_batch['states']
internals = sampled_buffer_batch['internals']
actions = sampled_buffer_batch['actions']
terminal = sampled_buffer_batch['terminal']
reward = sampled_buffer_batch['reward']
# 2. Compute priorities for all batch elements.
priorities = loss_per_instance ** self.prioritization_weight
assignments = list()
# 3. Insert the buffer elements from the recent batch into memory,
# overwrite memory if full.
memory_end_index = self.memory_index + self.last_batch_buffer_elems
memory_insert_indices = tf.range(
start=self.memory_index,
limit=memory_end_index
) % self.capacity
for name in sorted(states):
assignments.append(tf.scatter_update(
ref=self.states_memory[name],
indices=memory_insert_indices,
# Only buffer elements from batch.
updates=states[name][0:self.last_batch_buffer_elems])
)
for name in sorted(internals):
assignments.append(tf.scatter_update(
ref=self.internals_buffer[name],
indices=memory_insert_indices,
updates=internals[name][0:self.last_batch_buffer_elems]
))
assignments.append(tf.scatter_update(
ref=self.priorities,
indices=memory_insert_indices,
updates=priorities[0:self.last_batch_buffer_elems]
))
assignments.append(tf.scatter_update(
ref=self.terminal_memory,
indices=memory_insert_indices,
updates=terminal[0:self.last_batch_buffer_elems])
)
assignments.append(tf.scatter_update(
ref=self.reward_memory,
indices=memory_insert_indices,
updates=reward[0:self.last_batch_buffer_elems])
)
for name in sorted(actions):
assignments.append(tf.scatter_update(
ref=self.actions_memory[name],
indices=memory_insert_indices,
updates=actions[name][0:self.last_batch_buffer_elems]
))
# 4.Update the priorities of the elements already in the memory.
# Slice out remaining elements - [] if all batch elements were from buffer.
main_memory_priorities = priorities[self.last_batch_buffer_elems:]
# Note that priority indices can have a different shape because multiple
# samples can be from the same index.
main_memory_priorities = main_memory_priorities[0:tf.shape(priority_indices)[0]]
assignments.append(tf.scatter_update(
ref=self.priorities,
indices=priority_indices,
updates=main_memory_priorities
))
with tf.control_dependencies(control_inputs=assignments):
# 5. Re-sort memory according to priorities.
assignments = list()
# Obtain sorted order and indices.
sorted_priorities, sorted_indices = tf.nn.top_k(
input=self.priorities,
k=self.capacity,
sorted=True
)
# Re-assign elements according to priorities.
# Priorities was the tensor we used to sort, so this can be directly assigned.
assignments.append(tf.assign(ref=self.priorities, value=sorted_priorities))
# All other memory variables are assigned via scatter updates using the indices
# returned by the sort:
assignments.append(tf.scatter_update(
ref=self.terminal_memory,
indices=sorted_indices,
updates=self.terminal_memory
))
for name in sorted(self.states_memory):
assignments.append(tf.scatter_update(
ref=self.states_memory[name],
indices=sorted_indices,
updates=self.states_memory[name]
))
for name in sorted(self.actions_memory):
assignments.append(tf.scatter_update(
ref=self.actions_memory[name],
indices=sorted_indices,
updates=self.actions_memory[name]
))
for name in sorted(self.internals_memory):
assignments.append(tf.scatter_update(
ref=self.internals_memory[name],
indices=sorted_indices,
updates=self.internals_memory[name]
))
assignments.append(tf.scatter_update(
ref=self.reward_memory,
indices=sorted_indices,
updates=self.reward_memory
))
# 6. Reset buffer index and increment memory index by inserted elements.
with tf.control_dependencies(control_inputs=assignments):
assignments = list()
# Decrement pointer of last elements used.
assignments.append(tf.assign_sub(ref=self.buffer_index, value=self.last_batch_buffer_elems))
# Keep track of memory size as to know whether we can sample from the main memory.
# Since the memory pointer can set to 0, we want to know if we are at capacity.
total_inserted_elements = self.memory_size + self.last_batch_buffer_elems
assignments.append(tf.assign(
ref=self.memory_size,
value=tf.minimum(x=total_inserted_elements, y=self.capacity))
)
# Update memory insertion index.
assignments.append(tf.assign(ref=self.memory_index, value=memory_end_index))
# Reset batch indices.
assignments.append(tf.assign(
ref=self.batch_indices,
value=tf.zeros(shape=tf.shape(self.batch_indices), dtype=tf.int32)
))
with tf.control_dependencies(control_inputs=assignments):
return tf.no_op()
def indicate(v):
shape = v.get_shape()
mask = tf.ones(shape, tf.float32)
return tf.where(v <= tf.constant(0, tf.float32), -mask, mask)
def train(n_token, cutoffs, ps_device):
##### Get input function and model function
train_input_fn, train_record_info = data_utils.get_input_fn(
record_info_dir=FLAGS.record_info_dir,
split="train",
per_host_bsz=FLAGS.train_batch_size,
tgt_len=FLAGS.tgt_len,
num_core_per_host=FLAGS.num_core_per_host,
num_hosts=1,
use_tpu=False)
tf.logging.info("num of batches {}".format(train_record_info["num_batch"]))
##### Create computational graph
train_set = train_input_fn({
"batch_size": FLAGS.train_batch_size,
"data_dir": FLAGS.data_dir
})
input_feed, label_feed = train_set.make_one_shot_iterator().get_next()
inputs = tf.split(input_feed, FLAGS.num_core_per_host, 0)
labels = tf.split(label_feed, FLAGS.num_core_per_host, 0)
per_core_bsz = FLAGS.train_batch_size // FLAGS.num_core_per_host
tower_mems, tower_losses, tower_new_mems, tower_grads_and_vars = [], [], [], []
for i in range(FLAGS.num_core_per_host):
reuse = True if i > 0 else None
with tf.device(assign_to_gpu(i, ps_device)), \
tf.variable_scope(tf.get_variable_scope(), reuse=reuse):
mems_i = [
tf.placeholder(tf.float32,
[FLAGS.mem_len, per_core_bsz, FLAGS.d_model])
for _ in range(FLAGS.n_layer)
]
loss_i, new_mems_i, grads_and_vars_i = single_core_graph(
n_token=n_token,
cutoffs=cutoffs,
is_training=True,
inp=inputs[i],
tgt=labels[i],
mems=mems_i)
tower_mems.append(mems_i)
tower_losses.append(loss_i)
tower_new_mems.append(new_mems_i)
tower_grads_and_vars.append(grads_and_vars_i)
## average losses and gradients across towers
if len(tower_losses) > 1:
loss = tf.add_n(tower_losses) / len(tower_losses)
grads_and_vars = average_grads_and_vars(tower_grads_and_vars)
else:
loss = tower_losses[0]
grads_and_vars = tower_grads_and_vars[0]
grads, all_vars = zip(*grads_and_vars)
## clip gradient
clipped, gnorm = tf.clip_by_global_norm(grads, FLAGS.clip)
grads_and_vars = list(zip(clipped, all_vars))
## configure the optimizer
global_step = tf.train.get_or_create_global_step()
# warmup stage: increase the learning rate linearly
if FLAGS.warmup_steps > 0:
warmup_lr = tf.to_float(global_step) / tf.to_float(FLAGS.warmup_steps) \
* FLAGS.learning_rate
else:
warmup_lr = 0.0
# decay stage: decay the learning rate using the cosine schedule
decay_lr = tf.train.cosine_decay(
FLAGS.learning_rate,
global_step=global_step - FLAGS.warmup_steps,
decay_steps=FLAGS.train_steps - FLAGS.warmup_steps,
alpha=FLAGS.min_lr_ratio)
# choose warmup or decay
learning_rate = tf.where(global_step < FLAGS.warmup_steps, warmup_lr,
decay_lr)
# get the train op
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars, global_step)
##### Training loop
tower_mems_np = [[
np.zeros([FLAGS.mem_len, per_core_bsz, FLAGS.d_model],
dtype=np.float32) for layer in range(FLAGS.n_layer)
] for core in range(FLAGS.num_core_per_host)]
saver = tf.train.Saver()
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
sess.run(tf.global_variables_initializer())
if FLAGS.warm_start_path is not None:
tf.logging.info("warm start from {}".format(FLAGS.warm_start_path))
saver.restore(sess, FLAGS.warm_start_path)
fetches = [
loss, tower_new_mems, global_step, gnorm, learning_rate, train_op
]
total_loss, prev_step = 0., -1
while True:
feed_dict = {}
for i in range(FLAGS.num_core_per_host):
for m, m_np in zip(tower_mems[i], tower_mems_np[i]):
feed_dict[m] = m_np
fetched = sess.run(fetches, feed_dict=feed_dict)
loss_np, tower_mems_np, curr_step = fetched[:3]
total_loss += loss_np
if curr_step > 0 and curr_step % FLAGS.iterations == 0:
curr_loss = total_loss / (curr_step - prev_step)
tf.logging.info(
"[{}] | gnorm {:.2f} lr {:8.6f} "
"| loss {:.2f} | pplx {:>7.2f}, bpc {:>7.4f}".format(
curr_step, fetched[-3], fetched[-2], curr_loss,
math.exp(curr_loss), curr_loss / math.log(2)))
total_loss, prev_step = 0., curr_step
if curr_step > 0 and curr_step % FLAGS.save_steps == 0:
save_path = os.path.join(FLAGS.model_dir, "model.ckpt")
saver.save(sess, save_path)
tf.logging.info("Model saved in path: {}".format(save_path))
if curr_step == FLAGS.train_steps:
break
def ssd_losses(logits, localisations,
gclasses, glocalisations, gscores,
match_threshold=0.5,
negative_ratio=3.,
alpha=1.,
label_smoothing=0.,
scope=None):
"""Loss functions for training the SSD 300 VGG network.
This function defines the different loss components of the SSD, and
adds them to the TF loss collection.
Arguments:
logits: (list of) predictions logits Tensors;
localisations: (list of) localisations Tensors;
gclasses: (list of) groundtruth labels Tensors;
glocalisations: (list of) groundtruth localisations Tensors;
gscores: (list of) groundtruth score Tensors;
"""
with tf.name_scope(scope, 'ssd_losses'):
l_cross_pos = []
l_cross_neg = []
l_loc = []
for i in range(len(logits)):
dtype = logits[i].dtype
with tf.name_scope('block_%i' % i):
# Determine weights Tensor.
pmask = gscores[i] > match_threshold
fpmask = tf.cast(pmask, dtype)
n_positives = tf.reduce_sum(fpmask)
# Select some random negative entries.
# n_entries = np.prod(gclasses[i].get_shape().as_list())
# r_positive = n_positives / n_entries
# r_negative = negative_ratio * n_positives / (n_entries - n_positives)
# Negative mask.
no_classes = tf.cast(pmask, tf.int32)
predictions = slim.softmax(logits[i])
nmask = tf.logical_and(tf.logical_not(pmask),
gscores[i] > -0.5)
fnmask = tf.cast(nmask, dtype)
nvalues = tf.where(nmask,
predictions[:, :, :, :, 0],
1. - fnmask)
nvalues_flat = tf.reshape(nvalues, [-1])
# Number of negative entries to select.
n_neg = tf.cast(negative_ratio * n_positives, tf.int32)
n_neg = tf.maximum(n_neg, tf.size(nvalues_flat) // 8)
n_neg = tf.maximum(n_neg, tf.shape(nvalues)[0] * 4)
max_neg_entries = 1 + tf.cast(tf.reduce_sum(fnmask), tf.int32)
n_neg = tf.minimum(n_neg, max_neg_entries)
val, idxes = tf.nn.top_k(-nvalues_flat, k=n_neg)
minval = val[-1]
# Final negative mask.
nmask = tf.logical_and(nmask, -nvalues > minval)
fnmask = tf.cast(nmask, dtype)
# Add cross-entropy loss.
with tf.name_scope('cross_entropy_pos'):
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits[i],
labels=gclasses[i])
loss = tf.losses.compute_weighted_loss(loss, fpmask)
l_cross_pos.append(loss)
with tf.name_scope('cross_entropy_neg'):
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits[i],
labels=no_classes)
loss = tf.losses.compute_weighted_loss(loss, fnmask)
l_cross_neg.append(loss)
# Add localization loss: smooth L1, L2, ...
with tf.name_scope('localization'):
# Weights Tensor: positive mask + random negative.
weights = tf.expand_dims(alpha * fpmask, axis=-1)
loss = custom_layers.abs_smooth(localisations[i] - glocalisations[i])
loss = tf.losses.compute_weighted_loss(loss, weights)
l_loc.append(loss)
# Additional total losses...
with tf.name_scope('total'):
total_cross_pos = tf.add_n(l_cross_pos, 'cross_entropy_pos')
total_cross_neg = tf.add_n(l_cross_neg, 'cross_entropy_neg')
total_cross = tf.add(total_cross_pos, total_cross_neg, 'cross_entropy')
total_loc = tf.add_n(l_loc, 'localization')
# Add to EXTRA LOSSES TF.collection
tf.add_to_collection('EXTRA_LOSSES', total_cross_pos)
tf.add_to_collection('EXTRA_LOSSES', total_cross_neg)
tf.add_to_collection('EXTRA_LOSSES', total_cross)
tf.add_to_collection('EXTRA_LOSSES', total_loc)
def train_model(self):
"""
Network training loop
"""
## 获得data
data_layer = get_data_layer(self.roidb, self.imdb.num_classes)
## 需要获取的值并计算损失，分为RPN 和 fast-rcnn
## RPN
## classification loss
## anchors predicted
rpn_cls_score = tf.reshape(self.net.get_output('rpn_cls_score_reshape'), \
[-1,2])
## anchors targets
rpn_label = tf.reshape(self.net.get_output('rpn-data')[0], [-1])
rpn_cls_score = tf.reshape(
tf.gather(rpn_cls_score, tf.where(tf.not_equal(rpn_label, -1))),
[-1, 2])
rpn_label = tf.reshape(
tf.gather(rpn_label, tf.where(tf.not_equal(rpn_label, -1))), [-1])
### cross_entropy loss
rpn_cross_entropy = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=rpn_cls_score, labels=rpn_label))
## rpn bbox_reg L1 loss
## anchors predicted
rpn_bbox_pred = self.net.get_output('rpn_bbox_pred')
## anchors targets
rpn_bbox_targets = tf.transpose(
self.net.get_output('rpn-data')[1], [0, 2, 3, 1])
rpn_bbox_inside_weights = tf.transpose(
self.net.get_output('rpn-data')[2], [0, 2, 3, 1])
rpn_bbox_outside_weights = tf.transpose(
self.net.get_output('rpn-data')[3], [0, 2, 3, 1])
rpn_smooth_l1 = self._modified_smooth_l1(3.0, rpn_bbox_pred,
rpn_bbox_targets,
rpn_bbox_inside_weights,
rpn_bbox_outside_weights)
rpn_loss_box = tf.reduce_mean(
tf.reduce_sum(rpn_smooth_l1, reduction_indices=[
1,
2,
3,
]))
## fast-rcnn
## classification loss
## bbox predicted
## bbox targets
cls_score = self.net.get_output('cls_score')
label = tf.reshape(self.net.get_output('roi-data')[1], [-1])
cross_entropy = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=cls_score,
labels=label))
## bbox_reg L1 loss
## bbox_res predicted
bbox_pred = self.net.get_output('bbox_pred')
## bbox_reg targets
bbox_targets = self.net.get_output('roi-data')[2]
bbox_inside_weights = self.net.get_output('roi-data')[3]
bbox_outside_weights = self.net.get_output('roi-data')[4]
smoothl1 = self._modified_smooth_l1(1.0, bbox_pred, bbox_targets,
bbox_inside_weights,
bbox_outside_weights)
loss_box = tf.reduce_mean(
tf.reduce_sum(smoothl1, reduction_indices=[1]))
## total loss
loss = cross_entropy + loss_box + rpn_cross_entropy + rpn_loss_box
## learning rate,optimizer
global_step = tf.Variable(0, trainable=False)
lr = tf.train.exponential_decay(cfg.TRAIN.LEARNING_RATE,
global_step,
cfg.TRAIN.STEPSIZE,
0.1,
staircase=True)
momentum = cfg.TRAIN.MOMENTUM
train_op = tf.train.MomentumOptimizer(lr, momentum).minimize(
loss, global_step=global_step)
## initializer variables
self.sess.run(tf.global_variables_initializer())
## load pretrained_model
if self.pretrained_model is not None:
print('Loading pretrained model '
'weights from {:s}'.format(self.pretrained_model))
self.net.load(self.pretrained_model, self.sess, self.saver, True)
last_snapshot_iter = -1
timer = Timer()
## iters
for iter in range(self.max_iters):
# get one batch
blobs = data_layer.forward()
feed_dict = {
self.net.data: blobs['data'],
self.net.im_info: blobs['im_info'],
self.net.keep_prob: 0.5,
self.net.gt_boxes: blobs['gt_boxes']
}
run_options = None
run_metadata = None
if cfg.TRAIN.DEBUG_TIMELINE:
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = rf.RunMetadata()
timer.tic()
rpn_loss_cls_value, rpn_loss_box_value, loss_cls_value, loss_box_value, _ = \
self.sess.run([rpn_cross_entropy, rpn_loss_box, cross_entropy, loss_box, train_op],
feed_dict=feed_dict,
options=run_options,
run_metadata=run_metadata)
timer.toc()
if (iter + 1) % (cfg.TRAIN.DISPLAY) == 0:
print ('iter: %d / %d, total loss: %.4f, rpn_loss_cls: %.4f, rpn_loss_box: %.4f, loss_cls: %.4f, loss_box: %.4f, lr: %f'%\
(iter+1, self.max_iters, rpn_loss_cls_value + rpn_loss_box_value + loss_cls_value + loss_box_value ,rpn_loss_cls_value, rpn_loss_box_value,loss_cls_value, loss_box_value, lr.eval()))
print('speed: {:.3f}s / iter'.format(timer.average_time))
if (iter + 1) % cfg.TRAIN.SNAPSHOT_ITERS == 0:
last_snapshot_iter = iter
self.snapshot(iter)
if last_snapshot_iter != iter:
self.snapshot(iter)
def call(self, inputs, mask=None):
base_out = super(MaskedDense, self).call(inputs)
if mask is None:
return base_out
zeros = tf.zeros_like(base_out)
return tf.where(mask, base_out, zeros)
def __init__(self, graph_path, target_size=(320, 240), tf_config=None):
self.target_size = target_size
# load graph
logger.info('loading graph from %s(default size=%dx%d)' % (graph_path, target_size[0], target_size[1]))
with tf.gfile.GFile(graph_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
self.graph = tf.get_default_graph()
tf.import_graph_def(graph_def, name='TfPoseEstimator')
self.persistent_sess = tf.Session(graph=self.graph, config=tf_config)
# for op in self.graph.get_operations():
# print(op.name)
# for ts in [n.name for n in tf.get_default_graph().as_graph_def().node]:
# print(ts)
self.tensor_image = self.graph.get_tensor_by_name('TfPoseEstimator/image:0')
self.tensor_output = self.graph.get_tensor_by_name('TfPoseEstimator/Openpose/concat_stage7:0')
self.tensor_heatMat = self.tensor_output[:, :, :, :19]
self.tensor_pafMat = self.tensor_output[:, :, :, 19:]
self.upsample_size = tf.placeholder(dtype=tf.int32, shape=(2,), name='upsample_size')
self.tensor_heatMat_up = tf.image.resize_area(self.tensor_output[:, :, :, :19], self.upsample_size,
align_corners=False, name='upsample_heatmat')
self.tensor_pafMat_up = tf.image.resize_area(self.tensor_output[:, :, :, 19:], self.upsample_size,
align_corners=False, name='upsample_pafmat')
smoother = Smoother({'data': self.tensor_heatMat_up}, 25, 3.0)
gaussian_heatMat = smoother.get_output()
max_pooled_in_tensor = tf.nn.pool(gaussian_heatMat, window_shape=(3, 3), pooling_type='MAX', padding='SAME')
self.tensor_peaks = tf.where(tf.equal(gaussian_heatMat, max_pooled_in_tensor), gaussian_heatMat,
tf.zeros_like(gaussian_heatMat))
self.heatMat = self.pafMat = None
# warm-up
self.persistent_sess.run(tf.variables_initializer(
[v for v in tf.global_variables() if
v.name.split(':')[0] in [x.decode('utf-8') for x in
self.persistent_sess.run(tf.report_uninitialized_variables())]
])
)
self.persistent_sess.run(
[self.tensor_peaks, self.tensor_heatMat_up, self.tensor_pafMat_up],
feed_dict={
self.tensor_image: [np.ndarray(shape=(target_size[1], target_size[0], 3), dtype=np.float32)],
self.upsample_size: [target_size[1], target_size[0]]
}
)
self.persistent_sess.run(
[self.tensor_peaks, self.tensor_heatMat_up, self.tensor_pafMat_up],
feed_dict={
self.tensor_image: [np.ndarray(shape=(target_size[1], target_size[0], 3), dtype=np.float32)],
self.upsample_size: [target_size[1] // 2, target_size[0] // 2]
}
)
self.persistent_sess.run(
[self.tensor_peaks, self.tensor_heatMat_up, self.tensor_pafMat_up],
feed_dict={
self.tensor_image: [np.ndarray(shape=(target_size[1], target_size[0], 3), dtype=np.float32)],
self.upsample_size: [target_size[1] // 4, target_size[0] // 4]
}
)
def BatchNormV1(x, use_local_stat=None, decay=0.9, epsilon=1e-5):
shape = x.get_shape().as_list()
assert len(shape) in [2, 4]
n_out = shape[-1] # channel
assert n_out is not None
beta = tf.get_variable('beta', [n_out],
initializer=tf.constant_initializer())
gamma = tf.get_variable('gamma', [n_out],
initializer=tf.constant_initializer(1.0))
if len(shape) == 2:
batch_mean, batch_var = tf.nn.moments(x, [0], keep_dims=False)
else:
batch_mean, batch_var = tf.nn.moments(x, [0, 1, 2], keep_dims=False)
# just to make a clear name.
batch_mean = tf.identity(batch_mean, 'mean')
batch_var = tf.identity(batch_var, 'variance')
emaname = 'EMA'
ctx = get_current_tower_context()
if use_local_stat is None:
use_local_stat = ctx.is_training
if use_local_stat != ctx.is_training:
logger.warn("[BatchNorm] use_local_stat != is_training")
if use_local_stat:
# training tower
if ctx.is_training:
# reuse = tf.get_variable_scope().reuse
with tf.variable_scope(tf.get_variable_scope(), reuse=False):
# BatchNorm in reuse scope can be tricky! Moving mean/variance are not reused
with tf.name_scope(None): # https://github.com/tensorflow/tensorflow/issues/2740
# if reuse=True, try to find and use the existing statistics
# how to use multiple tensors to update one EMA? seems impossbile
ema = tf.train.ExponentialMovingAverage(decay=decay, name=emaname)
ema_apply_op = ema.apply([batch_mean, batch_var])
ema_mean, ema_var = ema.average(batch_mean), ema.average(batch_var)
if ctx.is_main_training_tower:
# inside main training tower
add_model_variable(ema_mean)
add_model_variable(ema_var)
else:
# no apply() is called here, no magic vars will get created,
# no reuse issue will happen
assert not ctx.is_training
with tf.name_scope(None):
ema = tf.train.ExponentialMovingAverage(decay=decay, name=emaname)
mean_var_name = ema.average_name(batch_mean)
var_var_name = ema.average_name(batch_var)
if ctx.is_main_tower:
# main tower, but needs to use global stat. global stat must be from outside
# when reuse=True, the desired variable name could
# actually be different, because a different var is created
# for different reuse tower
ema_mean = tf.get_variable('mean/' + emaname, [n_out])
ema_var = tf.get_variable('variance/' + emaname, [n_out])
else:
# use statistics in another tower
G = tf.get_default_graph()
ema_mean = ctx.find_tensor_in_main_tower(G, mean_var_name + ':0')
ema_var = ctx.find_tensor_in_main_tower(G, var_var_name + ':0')
if use_local_stat:
batch = tf.cast(tf.shape(x)[0], tf.float32)
mul = tf.where(tf.equal(batch, 1.0), 1.0, batch / (batch - 1))
batch_var = batch_var * mul # use unbiased variance estimator in training
with tf.control_dependencies([ema_apply_op] if ctx.is_training else []):
# only apply EMA op if is_training
return tf.nn.batch_normalization(
x, batch_mean, batch_var, beta, gamma, epsilon, 'output')
else:
return tf.nn.batch_normalization(
x, ema_mean, ema_var, beta, gamma, epsilon, 'output')
def build_graph(self, *inputs):
comb_state, self.action, reward, isOver, human = inputs
comb_state = tf.cast(comb_state, tf.float32)
state = tf.slice(comb_state, [0, 0, 0, 0, 0],
[-1, -1, -1, -1, self.channel],
name='state')
# Standard DQN loss
self.predict_value = self.get_DQN_prediction(state)
if not get_current_tower_context().is_training:
return
reward = tf.clip_by_value(reward, -1, 1)
next_state = tf.slice(comb_state, [0, 0, 0, 0, 1],
[-1, -1, -1, -1, self.channel],
name='next_state')
self.action_onehot = tf.one_hot(self.action, self.num_actions, 1.0,
0.0)
pred_action_value = tf.reduce_sum(self.predict_value *
self.action_onehot, 1) # N,
max_pred_reward = tf.reduce_mean(tf.reduce_max(self.predict_value, 1),
name='predict_reward')
summary.add_moving_summary(max_pred_reward)
with tf.variable_scope('target'):
targetQ_predict_value = self.get_DQN_prediction(next_state) # NxA
if 'Double' not in self.method:
# DQN or Dueling
best_v = tf.reduce_max(targetQ_predict_value, 1) # N,
else:
# Double-DQN or DuelingDouble
next_predict_value = self.get_DQN_prediction(next_state)
self.greedy_choice = tf.argmax(next_predict_value, 1) # N,
predict_onehot = tf.one_hot(self.greedy_choice, self.num_actions,
1.0, 0.0)
best_v = tf.reduce_sum(targetQ_predict_value * predict_onehot, 1)
target = reward + (1.0 - tf.cast(
isOver, tf.float32)) * self.gamma * tf.stop_gradient(best_v)
cost = tf.losses.huber_loss(target,
pred_action_value,
reduction=tf.losses.Reduction.MEAN)
###############################################################################
# HITL UPDATE: Margin classification loss
# This can only be calculated on the Human generated samples.
# Q(s,A_E) (The Q value of the action that was take by the human in that state)
action_value_1 = tf.multiply(self.predict_value,
self.action_onehot,
name='action_value_1')
tar = tf.reduce_sum(action_value_1, 1)
# l(a_E,a) here penalise every action plus 0.8 except the action that the
# human took which gets 0
mar = tf.one_hot(self.action, self.num_actions, 0.0, 0.8)
# max[Q(s,a) + l(a_E,a)]
# Q(s,a) = self.predict_value
mar_1 = tf.add(mar, self.predict_value)
margin = tf.reduce_max(mar_1, 1)
margin_loss = tf.subtract(margin, tar)
# this has been applied to all the transitions now need to set the
# margin classification loss to 0 for the transition which were
# not generated by a Human
margin_loss = tf.where(human, margin_loss,
tf.zeros_like(margin_loss, dtype=tf.float32))
margin_loss = tf.reduce_mean(margin_loss)
cost = tf.add(cost, margin_loss)
###############################################################################
summary.add_param_summary(
('conv.*/W', ['histogram', 'rms']),
('fc.*/W', ['histogram', 'rms'])) # monitor all W
summary.add_moving_summary(cost)
logger.info("Cost: {}".format(cost))
return cost
def matching(self, gt_bbox, gt_labels, num_crowd=None, threshold_pos=0.5, threshold_neg=0.4, threshold_crowd=0.7):
"""
:param gt_bbox:
:param gt_labels:
:return:
Args:
num_crowd:
threshold_pos:
threshold_neg:
threshold_crowd:
pos_iou_threshold:
num_crowd:
neg_iou_threshold:
"""
if num_crowd > 0:
# split the gt_bbox
gt_bbox = gt_bbox[:-num_crowd]
crowd_gt_bbox = gt_bbox[-num_crowd:]
else:
crowd_gt_bbox = tf.zeros_like(gt_bbox)
# Matching only for non-crowd annotation
# --------------------------------------------------------------------------------------------------------------
num_gt = tf.shape(gt_bbox)[0]
# tf.print("num gt", num_gt)
# pairwise IoU
pairwise_iou = self._pairwise_iou(gt_bbox=gt_bbox, is_crowd=False)
# assign the max overlap gt index for each anchor
max_iou_for_anchors = tf.reduce_max(pairwise_iou, axis=-1)
max_id_for_anchors = tf.math.argmax(pairwise_iou, axis=-1)
# force the anchors which is the best matched of each gt to predict the correspond gt
forced_update_id = tf.cast(tf.range(0, num_gt), tf.int64)
# force the iou over threshold for not wasting any training data
forced_update_iou = tf.reduce_max(pairwise_iou, axis=0)
forced_update_indice = tf.expand_dims(tf.math.argmax(pairwise_iou, axis=0), axis=-1)
max_iou_for_anchors = tf.tensor_scatter_nd_update(max_iou_for_anchors, forced_update_indice, forced_update_iou)
max_id_for_anchors = tf.tensor_scatter_nd_update(max_id_for_anchors, forced_update_indice, forced_update_id)
# decide the anchors to be positive or negative based on the IoU and given threshold
pos_iou = tf.where(max_iou_for_anchors > threshold_pos)
max_iou_for_anchors = tf.tensor_scatter_nd_update(max_iou_for_anchors, pos_iou, tf.ones(tf.size(pos_iou)))
neg_iou = tf.where(max_iou_for_anchors < threshold_neg)
max_iou_for_anchors = tf.tensor_scatter_nd_update(max_iou_for_anchors, neg_iou, tf.zeros(tf.size(neg_iou)))
neu_iou = tf.where(
tf.math.logical_and((max_iou_for_anchors <= threshold_pos), max_iou_for_anchors >= threshold_neg))
max_iou_for_anchors = tf.tensor_scatter_nd_update(max_iou_for_anchors, neu_iou, -1 * tf.ones(tf.size(neu_iou)))
# deal with crowd annotations, only affect non-positive
# --------------------------------------------------------------------------------------------------------------
if num_crowd > 0 and threshold_crowd < 1:
# crowd pairwise IoU
crowd_pairwise_iou = self._pairwise_iou(gt_bbox=crowd_gt_bbox, is_crowd=True)
# assign the max overlap gt index for each anchor
crowd_max_iou_for_anchors = tf.reduce_max(crowd_pairwise_iou, axis=-1)
# assign neutral for those neg iou that over crowd threshold
crowd_neu_iou = tf.where(
tf.math.logical_and((max_iou_for_anchors <= 0), crowd_max_iou_for_anchors > threshold_crowd))
# reassigh from negative to neutral
max_iou_for_anchors = tf.tensor_scatter_nd_update(max_iou_for_anchors, crowd_neu_iou,
-1 * tf.ones(tf.size(crowd_neu_iou)))
match_positiveness = max_iou_for_anchors
# create class target
# map idx to label[idx]
# match_labels = tf.map_fn(lambda x: gt_labels[x], max_id_for_anchors)
match_labels = tf.gather(gt_labels, max_id_for_anchors)
"""
element-wise multiplication of label[idx] and positiveness:
1. positive sample will have correct label
2. negative sample will have 0 * label[idx] = 0
3. neural sample will have -1 * label[idx] = -1 * label[idx]
it can be useful to distinguish positive sample during loss calculation
"""
target_cls = tf.multiply(tf.cast(match_labels, tf.float32), match_positiveness)
# create loc target
# map_loc = tf.map_fn(lambda x: gt_bbox[x], max_id_for_anchors, dtype=tf.float32)
map_loc = tf.gather(gt_bbox, max_id_for_anchors)
# convert to center form [cx, cy, w, h]
# center_anchors = tf.map_fn(lambda x: map_to_center_form(x), self.anchors)
h = self.anchors[:, 2] - self.anchors[:, 0]
w = self.anchors[:, 3] - self.anchors[:, 1]
center_anchors = tf.stack([self.anchors[:, 1] + (w / 2), self.anchors[:, 0] + (h / 2), w, h], axis=-1)
# center_gt = tf.map_fn(lambda x: map_to_center_form(x), map_loc)
h = map_loc[:, 2] - map_loc[:, 0]
w = map_loc[:, 3] - map_loc[:, 1]
center_gt = tf.stack([map_loc[:, 1] + (w / 2), map_loc[:, 0] + (h / 2), w, h], axis=-1)
variances = [0.1, 0.2]
# calculate offset
# target_loc = tf.map_fn(lambda x: map_to_offset(x), tf.stack([center_gt, center_anchors], axis=-1))
g_hat_cx = (center_gt[:, 0] - center_anchors[:, 0]) / center_anchors[:, 2] / variances[0]
g_hat_cy = (center_gt[:, 1] - center_anchors[:, 1]) / center_anchors[:, 3] / variances[0]
tf.debugging.assert_non_negative(center_anchors[:, 2] / center_gt[:, 2])
tf.debugging.assert_non_negative(center_anchors[:, 3] / center_gt[:, 3])
g_hat_w = tf.math.log(center_gt[:, 2] / center_anchors[:, 2]) / variances[1]
g_hat_h = tf.math.log(center_gt[:, 3] / center_anchors[:, 3]) / variances[1]
target_loc = tf.stack([g_hat_cx, g_hat_cy, g_hat_w, g_hat_h], axis=-1)
return target_cls, target_loc, max_id_for_anchors, match_positiveness
def calc_energy_weights(t_energy):
lower_cut = 0.5
w = tf.where(t_energy > 10., 1., ((t_energy-lower_cut) / 10.)*10./(10.-lower_cut))
return tf.nn.relu(w)
def ssd_losses(logits,
localisations,
gclasses,
glocalisations,
gscores,
match_threshold=0.5,
negative_ratio=3.,
alpha=1.,
label_smoothing=0.,
device='/cpu:0',
scope=None):
with tf.name_scope(scope, 'ssd_losses'):
lshape = tfe.get_shape(logits[0], 5)
num_classes = lshape[-1]
batch_size = lshape[0]
# Flatten out all vectors!
flogits = []
fgclasses = []
fgscores = []
flocalisations = []
fglocalisations = []
for i in range(len(logits)):
flogits.append(tf.reshape(logits[i], [-1, num_classes]))
fgclasses.append(tf.reshape(gclasses[i], [-1]))
fgscores.append(tf.reshape(gscores[i], [-1]))
flocalisations.append(tf.reshape(localisations[i], [-1, 4]))
fglocalisations.append(tf.reshape(glocalisations[i], [-1, 4]))
# And concat the crap!
logits = tf.concat(flogits, axis=0)
gclasses = tf.concat(fgclasses, axis=0)
gscores = tf.concat(fgscores, axis=0)
localisations = tf.concat(flocalisations, axis=0)
glocalisations = tf.concat(fglocalisations, axis=0)
dtype = logits.dtype
# Compute positive matching mask...
pmask = gscores > match_threshold
fpmask = tf.cast(pmask, dtype)
n_positives = tf.reduce_sum(fpmask)
# Hard negative mining...
no_classes = tf.cast(pmask, tf.int32)
predictions = slim.softmax(logits)
nmask = tf.logical_and(tf.logical_not(pmask), gscores > -0.5)
fnmask = tf.cast(nmask, dtype)
nvalues = tf.where(nmask, predictions[:, 0], 1. - fnmask)
nvalues_flat = tf.reshape(nvalues, [-1])
# Number of negative entries to select.
max_neg_entries = tf.cast(tf.reduce_sum(fnmask), tf.int32)
# 困难样本挖掘，保持正样本与负样本之间的比例是1:3
n_neg = tf.cast(negative_ratio * n_positives, tf.int32) + batch_size
n_neg = tf.minimum(n_neg, max_neg_entries)
val, idxes = tf.nn.top_k(-nvalues_flat, k=n_neg)
max_hard_pred = -val[-1]
# Final negative mask.
nmask = tf.logical_and(nmask, nvalues < max_hard_pred)
fnmask = tf.cast(nmask, dtype)
# Add cross-entropy loss.
# 正例交叉熵损失、负例交叉熵损失(分类损失)
with tf.name_scope('cross_entropy_pos'):
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=gclasses)
loss = tf.div(tf.reduce_sum(loss * fpmask),
batch_size,
name='value')
tf.losses.add_loss(loss)
with tf.name_scope('cross_entropy_neg'):
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=no_classes)
loss = tf.div(tf.reduce_sum(loss * fnmask),
batch_size,
name='value')
tf.losses.add_loss(loss)
# 定位损失
# Add localization loss: smooth L1, L2, ...
with tf.name_scope('localization'):
# Weights Tensor: positive mask + random negative.
weights = tf.expand_dims(alpha * fpmask, axis=-1)
loss = custom_layers.abs_smooth(localisations - glocalisations)
# 除以样本总数
loss = tf.div(tf.reduce_sum(loss * weights),
batch_size,
name='value')
# 总损失(分类损失与定位损失的加权求和)
tf.losses.add_loss(loss)
def safe_div(numerator, denominator):
"""Safe division, return 0 if denominator is 0"""
numerator, denominator = tf.to_float(numerator), tf.to_float(denominator)
zeros = tf.zeros_like(numerator, dtype=numerator.dtype)
denominator_is_zero = tf.equal(denominator, zeros)
return tf.where(denominator_is_zero, zeros, numerator / denominator)
def inference(actor_ids, run_ids, env_outputs, raw_rewards):
# Reset the actors that had their first run or crashed.
previous_run_ids = actor_run_ids.read(actor_ids)
actor_run_ids.replace(actor_ids, run_ids)
reset_indices = tf.where(tf.not_equal(previous_run_ids, run_ids))[:, 0]
actors_needing_reset = tf.gather(actor_ids, reset_indices)
if tf.not_equal(tf.shape(actors_needing_reset)[0], 0):
tf.print('Actor ids needing reset:', actors_needing_reset)
actor_infos.reset(actors_needing_reset)
store.reset(actors_needing_reset)
initial_agent_states = agent.initial_state(
tf.shape(actors_needing_reset)[0])
first_agent_states.replace(actors_needing_reset, initial_agent_states)
agent_states.replace(actors_needing_reset, initial_agent_states)
actions.reset(actors_needing_reset)
tf.debugging.assert_non_positive(
tf.cast(env_outputs.abandoned, tf.int32),
'Abandoned done states are not supported in SAC.')
# Update steps and return.
actor_infos.add(actor_ids, (0, env_outputs.reward, raw_rewards))
done_ids = tf.gather(actor_ids, tf.where(env_outputs.done)[:, 0])
info_queue.enqueue_many(actor_infos.read(done_ids))
actor_infos.reset(done_ids)
actor_infos.add(actor_ids, (FLAGS.num_action_repeats, 0., 0.))
# Inference.
prev_actions = parametric_action_distribution.postprocess(
actions.read(actor_ids))
input_ = encode((prev_actions, env_outputs))
prev_agent_states = agent_states.read(actor_ids)
def make_inference_fn(inference_device):
def device_specific_inference_fn():
with tf.device(inference_device):
@tf.function
def agent_inference(*args):
return agent(*decode(args),
is_training=False,
postprocess_action=False)
return agent_inference(*input_, prev_agent_states)
return device_specific_inference_fn
# Distribute the inference calls among the inference cores.
branch_index = inference_iteration.assign_add(1) % len(
inference_devices)
agent_actions, curr_agent_states = tf.switch_case(
branch_index, {
i: make_inference_fn(inference_device)
for i, inference_device in enumerate(inference_devices)
})
# Append the latest outputs to the unroll and insert completed unrolls in
# queue.
completed_ids, unrolls = store.append(
actor_ids, (prev_actions, env_outputs, agent_actions))
unrolls = Unroll(first_agent_states.read(completed_ids), *unrolls)
unroll_queue.enqueue_many(unrolls)
first_agent_states.replace(completed_ids,
agent_states.read(completed_ids))
# Update current state.
agent_states.replace(actor_ids, curr_agent_states)
actions.replace(actor_ids, agent_actions)
# Return environment actions to actors.
return parametric_action_distribution.postprocess(agent_actions)
def cnn(x_train, x_test, y_train, y_test):
x_train_size = np.size(x_train, 0)
#定义两个placeholder
x = tf.placeholder(tf.float32, [None, INPUT_NODE])#23*20
y = tf.placeholder(tf.float32, [None, OUTPUT_NODE])
#改变x的格式转为４Ｄ的向量【batch, in_height, in_width, in_channels]
x_image = tf.reshape(x,[-1, X_SIZE, Y_SIZE, 1])
#初始化第一个卷积层的权值和偏量
W_conv1 = weight_variable([CONV1_SIZE,CONV1_SIZE,NUM_CHANNELS,CONV1_DEEP])#5*5的采样窗口，３２个卷积核从4个平面抽取特征
b_conv1 = bias_variable([CONV1_DEEP])#每一个卷积核一个偏置值
#把x_image和权值向量进行卷积，再加上偏置值，然后应用于relu激活函数
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
h_pool1 = max_pool(h_conv1)#进行max-pooling,12-by-40
#初始化第二个卷积层的权值和偏置
W_conv2 = weight_variable([CONV2_SIZE,CONV2_SIZE,CONV1_DEEP,CONV2_DEEP]) #5*5的采样窗口，64个卷积核从32个平面抽取特征
b_conv2 = bias_variable([CONV2_DEEP]) #每一个卷积核一个偏置值
#把H_pool1和权值向量进行卷积，再加上偏置值，然后应用于relu激活函数
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool(h_conv2)#6-by-5
#23*20的图片第一次卷积后还是23*20,第一次池化后变为12*10
#第二次卷积后为12*10,第二次池化后变为6*5
#进过上面操作后得到64张6*5的平面
#初始化第一全链接层的权值
W_fc1 = weight_variable([6*5*CONV2_DEEP,FC_SIZE]) #上一层有6*10*64个神经元,全连接层有1024个神经元
b_fc1 = bias_variable([FC_SIZE])
#把池化层2的输出扁平化为1维
h_pool2_flat = tf.reshape(h_pool2,[-1,6*5*CONV2_DEEP])
#求第一个全连接层的输出
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
#keep_prob用了表示神经元的输出概率
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
#初始化第二个全连接层
W_fc2 = weight_variable([FC_SIZE,OUTPUT_NODE])
b_fc2 = bias_variable([OUTPUT_NODE])
#计算输出
prediction = tf.nn.softmax(tf.matmul(h_fc1_drop,W_fc2) + b_fc2)
# 结果存放在一个布尔列表中
correct_prediction = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
#交叉熵代价函数
#cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y,logits=prediction))
#自定义损失函数，因为结合位点的标签是[0,1]共有3778，非结合位点的标签是[1,0]有53570，是非平衡数据集，
cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(labels=y, logits=prediction)
y1 = tf.argmax(y,1)
yshape = tf.shape(y)
a = tf.ones([yshape[0]],dtype=tf.int64)
loss = tf.reduce_mean( tf.where( tf.greater_equal( y1,a), cross_entropy * LOSS_COEF[1], cross_entropy * LOSS_COEF[0]))
#使用AdamOptimizer进行优化
train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss)
#求准确率
accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
n_batch = math.ceil(x_train_size/BATCH_SIZE)
for epoch in range(TRAINING_STEPS):
for batch in range(n_batch):
start = (batch * BATCH_SIZE) % x_train_size
end = min(start + BATCH_SIZE, x_train_size)
batch_xs = x_train[start:end]
batch_ys = y_train[start:end]
sess.run(train_step,feed_dict={x:batch_xs, y: batch_ys, keep_prob: 0.5})
acc = sess.run(accuracy, feed_dict={x: x_test, y: y_test, keep_prob: 1.0})
print("Iter " + str(epoch) + " Testing Accuracy=" + str(acc))
pred = sess.run(prediction, feed_dict={x: x_test, y: y_test, keep_prob: 1.0})
return pred
def multihead_attention(queries,
keys,
num_units,
num_heads,
dropout_rate=0,
scope="multihead_attention",
reuse=None):
with tf.variable_scope(scope, reuse=reuse):
# Linear projections
Q = tf.layers.dense(queries,
num_units,
kernel_initializer=initializer()) # (N, T_q, C)
K = tf.layers.dense(keys, num_units,
kernel_initializer=initializer()) # (N, T_k, C)
V = tf.layers.dense(keys, num_units,
kernel_initializer=initializer()) # (N, T_k, C)
# Split and concat
Q_ = tf.concat(tf.split(Q, num_heads, axis=2),
axis=0) # (h*N, T_q, C/h)
K_ = tf.concat(tf.split(K, num_heads, axis=2),
axis=0) # (h*N, T_k, C/h)
V_ = tf.concat(tf.split(V, num_heads, axis=2),
axis=0) # (h*N, T_k, C/h)
# Multiplication
outputs = tf.matmul(Q_, tf.transpose(K_, [0, 2, 1])) # (h*N, T_q, T_k)
# Scale
outputs /= K_.get_shape().as_list()[-1]**0.5
# Key Masking
key_masks = tf.sign(tf.abs(tf.reduce_sum(keys, axis=-1))) # (N, T_k)
key_masks = tf.tile(key_masks, [num_heads, 1]) # (h*N, T_k)
key_masks = tf.tile(tf.expand_dims(key_masks, 1),
[1, tf.shape(queries)[1], 1]) # (h*N, T_q, T_k)
paddings = tf.ones_like(outputs) * (-2**32 + 1)
outputs = tf.where(tf.equal(key_masks, 0), paddings,
outputs) # (h*N, T_q, T_k)
# Activation
alphas = tf.nn.softmax(outputs) # (h*N, T_q, T_k)
# Query Masking
query_masks = tf.sign(tf.abs(tf.reduce_sum(queries,
axis=-1))) # (N, T_q)
query_masks = tf.tile(query_masks, [num_heads, 1]) # (h*N, T_q)
query_masks = tf.tile(tf.expand_dims(query_masks, -1),
[1, 1, tf.shape(keys)[1]]) # (h*N, T_q, T_k)
alphas *= query_masks # broadcasting. (N, T_q, C)
# Dropouts
alphas = tf.layers.dropout(alphas,
rate=dropout_rate,
training=tf.convert_to_tensor(True))
# Weighted sum
outputs = tf.matmul(alphas, V_) # ( h*N, T_q, C/h)
# Restore shape
outputs = tf.concat(tf.split(outputs, num_heads, axis=0),
axis=2) # (N, T_q, C)
# Linear
outputs = tf.layers.dense(outputs,
num_units,
activation=tf.nn.relu,
kernel_initializer=initializer())
# Residual connection
outputs += queries
# Normalize
outputs = layer_norm(outputs) # (N, T_q, C)
return outputs, alphas