def __init__(self, model: CANTRIPModel, optimizer):
self.model = model
self.optimizer = optimizer
# Batch-level confusion matrix
self.tp = tf.count_nonzero(model.y * model.labels, dtype=tf.int32)
self.tn = tf.count_nonzero((model.y - 1) * (model.labels - 1), dtype=tf.int32)
self.fp = tf.count_nonzero(model.y * (model.labels - 1), dtype=tf.int32)
self.fn = tf.count_nonzero((model.y - 1) * model.labels, dtype=tf.int32)
# Batch-level binary classification metrics
self.precision = self.tp / (self.tp + self.fp)
self.recall = self.tp / (self.tp + self.fn)
self.accuracy = (self.tp + self.tn) / model.batch_size
self.specificity = self.tn / (self.tn + self.fp)
self.f1 = 2 * self.precision * self.recall / (self.precision + self.recall)
self.f2 = 5 * self.precision * self.recall / (4 * self.precision + self.recall)
self.fhalf = 1.25 * self.precision * self.recall / (.25 * self.precision + self.recall)
# Dict of all metrics to make fetching more convenient
self.batch_metrics = {
'TP': self.tp,
'TN': self.tn,
'FP': self.fp,
'FN': self.fn,
'Precision': self.precision,
'Recall': self.recall,
'Accuracy': self.accuracy,
'Specificity': self.specificity,
'F1': self.f1,
'F2': self.f2,
'F.5': self.fhalf,
}
if self.optimizer:
self.batch_metrics['Loss'] = optimizer.loss
# Group all batch-level metrics in the same pane in TensorBoard using a name scope
with tf.name_scope('Batch'):
summaries = [
tf.summary.scalar('Accuracy', self.accuracy),
tf.summary.scalar('Precision', self.precision),
tf.summary.scalar('Recall', self.recall),
tf.summary.scalar('F1', self.f1),
tf.summary.scalar('F2', self.f2),
tf.summary.scalar('F.5', self.fhalf),
tf.summary.scalar('Specificity', self.specificity)
]
if self.optimizer:
summaries.append(tf.summary.scalar('Loss', self.optimizer.loss))
self.batch_summary = tf.summary.merge([summaries])
# Specific training/development/testing summarizers
self.train = _CANTRIPModeSummarizer('train', model)
self.devel = _CANTRIPModeSummarizer('devel', model)
self.test = _CANTRIPModeSummarizer('test', model)
def _prec_rec(conf_gt, conf_out, reg_gt, reg_out, config):
"""
Creates precision and recall metrics.
Returns (precision, recall)
conf_gt
Ground truth confidence, i.e. 1 for close anchors, 0 for anchors
that are too far off and -1 for anchors to be ignored. Must have
shape (?, fh, fw, k).
conf_out
PPN confidence output, must have shape (?, fh, fw, k).
reg_gt
Ground truth point offsets, need only have valid values for the
anchors with conf_gt of 1. Must have shape (?, fh, fw, 2k).
reg_out
PPN anchor offset output, must have shape (?, fh, fw, 2k).
config
The configuration dictionary. See ppn.config.ppn_config.
"""
import tensorflow.compat.v1 as tf
score_thr = config['score_thr']
dist_thr = config['dist_thr']
# mask positive outputs and ground truths
gt_pos_mask = tf.equal(conf_gt, 1)
gt_pos_count = tf.count_nonzero(gt_pos_mask, dtype=tf.int32)
out_pos_mask = tf.greater_equal(conf_out, score_thr)
out_pos_count = tf.count_nonzero(out_pos_mask, dtype=tf.int32)
# calculate regression distances
dist = reg_gt - reg_out
dist *= dist
dist = tf.reduce_sum(dist, axis=3)
close_mask = tf.less_equal(dist, dist_thr*dist_thr) # uses squared distance
# calculate number of correct predictions
correct_mask = tf.logical_and(tf.logical_and(gt_pos_mask, out_pos_mask),
close_mask)
correct_count = tf.count_nonzero(correct_mask, dtype=tf.float32)
precision = tf.where(tf.equal(out_pos_count, 0),
0.0,
correct_count / tf.cast(out_pos_count, tf.float32))
recall = tf.where(tf.equal(out_pos_count, 0),
0.0,
correct_count / tf.cast(gt_pos_count, tf.float32))
return tf.stop_gradient(precision), tf.stop_gradient(recall)
def _custom_recall_at_k(labels_as_multi_hot, predictions, k):
"""Calculates recall_at_k metric with multi-hot labels.
For each example which contains at least one label, a recall-at-k is
calculated by assessing what proportion of these labels are in the top k
predictions. This metric is the mean of these values.
Args:
labels_as_multi_hot: a tensor of [batch_size, num_output_classes] where
elements are zero (absent) or one (present).
predictions: a tensor of [batch_size, num_output_classes] where elemenents
are floats indicating the probability of class membership.
k: number of top predictions to consider (must be <= num_output_classes).
Returns:
mean: A scalar `Tensor` representing the current mean, the value of `total`
divided by `count` (of finite values).
update_op: An operation that increments the `total` and `count` variables
appropriately and whose (scalar) value matches the mean_value.
"""
labels_as_multi_hot = tf.cast(labels_as_multi_hot, tf.float32)
num_output_classes = tf.shape(labels_as_multi_hot)[1]
_, indices = tf.math.top_k(predictions, k=k)
predictions_top_k_as_multi_hot = _indices_to_multihot(
indices, num_output_classes)
true_positives_tensor = tf.math.logical_and(
tf.cast(labels_as_multi_hot, tf.bool),
tf.cast(predictions_top_k_as_multi_hot, tf.bool))
false_negatives_tensor = tf.math.greater(labels_as_multi_hot,
predictions_top_k_as_multi_hot)
true_positives_per_example = tf.count_nonzero(true_positives_tensor,
axis=1)
false_negatives_per_example = tf.count_nonzero(false_negatives_tensor,
axis=1)
recall_per_example = true_positives_per_example / (
true_positives_per_example + false_negatives_per_example)
is_finite = tf.is_finite(
recall_per_example) # To filter out no label cases.
recall_per_example_finite_only = tf.boolean_mask(recall_per_example,
is_finite)
return tf.metrics.mean(recall_per_example_finite_only)
def create_model(self, optimizer):
features = tf.placeholder(tf.int32, [None, self.seq_len], name='features')
labels = tf.placeholder(tf.int64, [None,], name='labels')
embs = tf.Variable(self.emb_arr, dtype=tf.float32, trainable=False)
x = tf.nn.embedding_lookup(embs, features)
forward_cell1, backward_cell1 = rnn.BasicGRUCell(64), rnn.BasicGRUCell(64)
forward_cell2, backward_cell2 = rnn.BasicGRUCell(32), rnn.BasicGRUCell(32)
(birnn_fw1, birnn_bw1), _, _ = dynamic_birnn(forward_cell1, backward_cell1, x, dtype=tf.float32)
print("shape:", birnn_fw1.shape, birnn_bw1.shape)
stacked_lstm = rnn.MultiRNNCell(
[rnn.BasicLSTMCell(self.n_hidden) for _ in range(2)])
outputs, _ = tf.nn.dynamic_rnn(stacked_lstm, x, dtype=tf.float32)
fc1 = tf.layers.dense(inputs=outputs[:,-1,:], units=30)
pred = tf.squeeze(tf.layers.dense(inputs=fc1, units=1))
loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=labels, logits=pred)
#optimizer = tf.train.AdamOptimizer(learning_rate=self.lr)
grads_and_vars = optimizer.compute_gradients(loss)
grads, _ = zip(*grads_and_vars)
train_op = optimizer.apply_gradients(grads_and_vars, global_step=tf.train.get_global_step())
correct_pred = tf.equal(tf.to_int64(tf.greater(pred,0)), labels)
eval_metric_ops = tf.count_nonzero(correct_pred)
return features, labels, train_op, grads, eval_metric_ops, loss
def create_model(self, optimizer):
"""Model function for Logistic Regression."""
features = tf.placeholder(tf.float32, shape=[None, 784], name='features')
labels = tf.placeholder(tf.int64, shape=[None,], name='labels')
input_layer = tf.reshape(features, [-1, 28, 28, 1])
conv1 = tf.layers.conv2d(
inputs=input_layer,
filters=32,
kernel_size=[5, 5],
padding="same",
activation=tf.nn.relu)
pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)
conv2 = tf.layers.conv2d(
inputs=pool1,
filters=64,
kernel_size=[5, 5],
padding="same",
activation=tf.nn.relu)
pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)
pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])
dense = tf.layers.dense(inputs=pool2_flat, units=512, activation=tf.nn.relu)
logits = tf.layers.dense(inputs=dense, units=self.num_classes)
predictions = {
"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
grads_and_vars = optimizer.compute_gradients(loss)
grads, _ = zip(*grads_and_vars)
train_op = optimizer.apply_gradients(grads_and_vars, global_step=tf.train.get_global_step())
eval_metric_ops = tf.count_nonzero(tf.equal(labels, predictions["classes"]))
return features, labels, train_op, grads, eval_metric_ops, loss
def create_model(self, optimizer):
"""Model function for Logistic Regression."""
features = tf.placeholder(tf.float32,
shape=[None, 784],
name='features')
labels = tf.placeholder(tf.int64, shape=[
None,
], name='labels')
logits = tf.layers.dense(
inputs=features,
units=self.num_classes,
kernel_regularizer=tf.keras.regularizers.l2(0.001))
predictions = {
"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits, \
reduction=tf.losses.Reduction.MEAN)
grads_and_vars = optimizer.compute_gradients(loss)
grads, _ = zip(*grads_and_vars)
train_op = optimizer.apply_gradients(
grads_and_vars, global_step=tf.train.get_global_step())
eval_metric_ops = tf.count_nonzero(
tf.equal(labels, predictions["classes"]))
return features, labels, train_op, grads, eval_metric_ops, loss
def per_instance_loss(i, current_ta_term_0, current_ta_term_1,
current_ta_term_2):
target_instances_mask = masked_bit(instances_bitwise, i * 2)
other_instances_mask = masked_bit(instances_bitwise, i * 2 + 1)
current_terms = tf.cond(
tf.greater(tf.count_nonzero(target_instances_mask),
0), lambda: cosine_embedding_single_instance_loss(
embeddings,
target_instances_mask,
other_instances_mask,
normalized_embeddings,
term_1_2_normalization=term_1_2_normalization,
term_0_squared=term_0_squared,
term_1_squared=term_1_squared,
return_h_norm=return_h_norm,
is_background=is_background,
data_format=data_format), lambda:
(tf.zeros([0], dtype=embeddings.dtype),
tf.zeros([0], dtype=embeddings.dtype)) if not return_h_norm else
(tf.zeros([0], dtype=embeddings.dtype),
tf.zeros([0], dtype=embeddings.dtype),
tf.zeros([embeddings.shape[0]], dtype=embeddings.dtype)))
current_ta_term_0 = current_ta_term_0.write(i, current_terms[0])
current_ta_term_1 = current_ta_term_1.write(i, current_terms[1])
if return_h_norm:
current_ta_term_2 = current_ta_term_2.write(i, current_terms[2])
return i + 1, current_ta_term_0, current_ta_term_1, current_ta_term_2
def create_model(self, optimizer):
features = tf.placeholder(tf.int32, [None, self.seq_len])
embedding = tf.get_variable("embedding", [self.num_classes, 8])
x = tf.nn.embedding_lookup(embedding, features)
labels = tf.placeholder(tf.int32, [None, self.num_classes])
stacked_lstm = rnn.MultiRNNCell(
[rnn.BasicLSTMCell(self.n_hidden) for _ in range(2)])
outputs, _ = tf.nn.dynamic_rnn(stacked_lstm, x, dtype=tf.float32)
pred = tf.layers.dense(inputs=outputs[:, -1, :],
units=self.num_classes)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(logits=pred,
labels=labels))
grads_and_vars = optimizer.compute_gradients(loss)
grads, _ = zip(*grads_and_vars)
train_op = optimizer.apply_gradients(
grads_and_vars, global_step=tf.train.get_global_step())
correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(labels, 1))
eval_metric_ops = tf.count_nonzero(correct_pred)
return features, labels, train_op, grads, eval_metric_ops, loss
def z_e_sample(self):
"""Sample from the distribution of probability of the latent embeddings."""
#z_e = tf.identity(self.z_e, name="z_e")
z_e = self.z_e.sample()
z_e = tf.identity(z_e, name="z_e")
tf.summary.histogram("count_nonzeros_z_e", tf.count_nonzero(z_e, -1))
return z_e
def binary_classification_summaries(x_quant, x_tilde_quant):
tp = tf.count_nonzero(x_tilde_quant * x_quant, dtype=tf.float32)
tn = tf.count_nonzero((x_tilde_quant - 1) * (x_quant - 1),
dtype=tf.float32)
fp = tf.count_nonzero(x_tilde_quant * (x_quant - 1), dtype=tf.float32)
fn = tf.count_nonzero((x_tilde_quant - 1) * x_quant, dtype=tf.float32)
precision = tp / (tp + fp)
recall = tp / (tp + fn)
accuracy = (tp + tn) / (tp + tn + fp + fn)
specificity = tn / (tn + fp)
f1_score = (2 * precision * recall) / (precision + recall)
tf.summary.scalar("bc/precision", precision)
tf.summary.scalar("bc/recall", recall)
tf.summary.scalar("bc/accuracy", accuracy)
tf.summary.scalar("bc/specificity", specificity)
tf.summary.scalar("bc/f1_score", f1_score)
def get_input(self, training=False):
SET_TRAIN_FLAG(training)
dataset = tf.data.Dataset.from_tensor_slices((self.x, self.y))
dataset = dataset.shuffle(buffer_size=SHUF_BUF_SZ)
dataset = dataset.batch(50)
dataset = dataset.map(lambda x, y: ({
'word': x,
'lengths': count_nonzero(x, axis=1)
}, y))
dataset = dataset.prefetch(NUM_PREFETCH)
return dataset
def select_random(x):
"""Selectes a random elements from each row of x."""
def to_float(x):
return tf.cast(x, tf.float32)
def to_int(x):
return tf.cast(x, tf.int64)
batch_size = tf.shape(x)[0]
rn = tf.range(batch_size)
nnz = to_float(tf.count_nonzero(x >= 0, axis=1))
rnd = tf.random_uniform([batch_size])
ids = tf.stack([to_int(rn), to_int(nnz * rnd)], axis=1)
return to_int(tf.gather_nd(x, ids))
def _grad_sparsity(self):
"""Gradient sparsity."""
# If the sparse minibatch gradient has 10 percent of its entries
# non-zero, its sparsity is 0.1.
# The norm of dense gradient averaged from full dataset
# are roughly estimated norm of minibatch
# sparse gradient norm * sqrt(sparsity)
# An extension maybe only correct the sparse blob.
non_zero_cnt = tf.add_n([tf.count_nonzero(g) for g in self._grad])
all_entry_cnt = tf.add_n([tf.size(g) for g in self._grad])
self._sparsity = tf.cast(non_zero_cnt, self._grad[0].dtype)
self._sparsity /= tf.cast(all_entry_cnt, self._grad[0].dtype)
avg_op = self._moving_averager.apply([
self._sparsity,
])
with tf.control_dependencies([avg_op]):
self._sparsity_avg = self._moving_averager.average(self._sparsity)
return avg_op
def per_instance_loss(i, current_ta_term_0, current_ta_term_1,
current_ta_term_2):
# TODO: simplify condition, use instance_axis somehow
if data_format == 'channels_first':
target_instances_mask = tf.expand_dims(tf.equal(
instances[i, ...], 1),
axis=instance_axis)
other_instances_mask = tf.expand_dims(tf.equal(
instances[i, ...], 2),
axis=instance_axis)
else:
target_instances_mask = tf.expand_dims(tf.equal(
instances[..., i], 1),
axis=instance_axis)
other_instances_mask = tf.expand_dims(tf.equal(
instances[..., i], 2),
axis=instance_axis)
current_terms = tf.cond(
tf.greater(tf.count_nonzero(target_instances_mask),
0), lambda: cosine_embedding_single_instance_loss(
embeddings,
target_instances_mask,
other_instances_mask,
normalized_embeddings,
term_1_2_normalization=term_1_2_normalization,
term_0_squared=term_0_squared,
term_1_squared=term_1_squared,
return_h_norm=return_h_norm,
is_background=is_background,
data_format=data_format), lambda:
(tf.zeros([0], dtype=embeddings.dtype),
tf.zeros([0], dtype=embeddings.dtype)) if not return_h_norm else
(tf.zeros([0], dtype=embeddings.dtype),
tf.zeros([0], dtype=embeddings.dtype),
tf.zeros([embeddings.shape[0]], dtype=embeddings.dtype)))
current_ta_term_0 = current_ta_term_0.write(i, current_terms[0])
current_ta_term_1 = current_ta_term_1.write(i, current_terms[1])
if return_h_norm:
current_ta_term_2 = current_ta_term_2.write(i, current_terms[2])
return i + 1, current_ta_term_0, current_ta_term_1, current_ta_term_2
def create_model(self, optimizer):
"""Model function for Logistic Regression."""
features = tf.placeholder(tf.float32,
shape=[None, 100],
name='features')
labels = tf.placeholder(tf.float32, shape=[None, 1], name='labels')
W = tf.Variable(tf.zeros([100, 1]))
b = tf.Variable(tf.zeros([1]))
y_pred = tf.matmul(features, W) + b
loss = 0.01 * tf.reduce_sum(tf.square(W)) + tf.reduce_mean(
tf.maximum(tf.zeros_like(labels), 1 - labels * y_pred))
grads_and_vars = optimizer.compute_gradients(loss)
grads, _ = zip(*grads_and_vars)
train_op = optimizer.apply_gradients(
grads_and_vars, global_step=tf.train.get_global_step())
eval_metric_ops = tf.count_nonzero(tf.equal(labels, tf.sign(y_pred)))
return features, labels, train_op, grads, eval_metric_ops, loss, tf.sign(
y_pred)
def _mapper(dataset):
"""Tokenizes strings using tf.string_split and truncates by length."""
for k in keys_to_map:
# pylint: disable=g-explicit-length-test
if len(dataset[k].get_shape()) == 0: # Used for questions.
# pylint: enable=g-explicit-length-test
# <string> [num_tokens]
tokens = tf.string_split([dataset[k]]).values
else: # Used for contexts.
# <string> [num_context, num_tokens] (sparse)
sparse_tokens = tf.string_split(dataset[k])
# <string>[num_tokens, max_num_tokens] (dense)
tokens = tf.sparse_tensor_to_dense(sparse_tokens,
default_value="")
dataset[k + suffix] = tokens
# Compute exact length of each context.
dataset[k + suffix + "_len"] = tf.count_nonzero(tokens,
axis=-1,
dtype=tf.int32)
return dataset
def create_model(self, optimizer):
"""Model function for Logistic Regression."""
features = tf.placeholder(tf.float32,
shape=[None, 784],
name='features')
labels = tf.placeholder(tf.int64, shape=[
None,
], name='labels')
images = tf.reshape(features, shape=[tf.shape(features)[0], 28, 28, 1])
conv1 = tf.layers.conv2d(inputs=images,
filters=24,
kernel_size=5,
activation=tf.nn.relu)
pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=2, strides=2)
pool1_flatten = tf.layers.flatten(pool1)
fc2 = tf.layers.dense(
inputs=pool1_flatten,
units=256,
activation=tf.nn.relu,
kernel_regularizer=tf.keras.regularizers.l2(0.001))
logits = tf.layers.dense(
inputs=fc2,
units=self.num_classes,
kernel_regularizer=tf.keras.regularizers.l2(0.001))
predictions = {
"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels,
logits=logits)
grads_and_vars = optimizer.compute_gradients(loss)
grads, _ = zip(*grads_and_vars)
train_op = optimizer.apply_gradients(
grads_and_vars, global_step=tf.train.get_global_step())
eval_metric_ops = tf.count_nonzero(
tf.equal(labels, predictions["classes"]))
return features, labels, train_op, grads, eval_metric_ops, loss
def create_model(self, q, optimizer):
input_ph = tf.placeholder(tf.float32,
shape=(None, IMAGE_SIZE, IMAGE_SIZE, 3))
output2 = tf.placeholder(tf.float32,
shape=[None, self.num_classes],
name='output2')
out = input_ph
for _ in range(4):
out = tf.layers.conv2d(out, 32, 3, padding='same')
out = tf.layers.batch_normalization(out, training=True)
out = tf.layers.max_pooling2d(out, 2, 2, padding='same')
out = tf.nn.relu(out)
out = tf.reshape(out, (-1, int(np.prod(out.get_shape()[1:]))))
logits = tf.layers.dense(out, self.num_classes)
label_ph = tf.placeholder(tf.int64, shape=(None, ))
loss = tf.losses.sparse_softmax_cross_entropy(labels=label_ph,
logits=logits)
predictions = {
"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
grads_and_vars = optimizer.compute_gradients(loss)
grads, _ = zip(*grads_and_vars)
train_op = optimizer.apply_gradients(
grads_and_vars, global_step=tf.train.get_global_step())
eval_metric_ops = tf.count_nonzero(
tf.equal(label_ph, tf.argmax(input=logits, axis=1)))
kl_loss = tf.keras.losses.KLD(
predictions['probabilities'], output2) + tf.keras.losses.KLD(
output2, predictions['probabilities'])
kl_grads_and_vars = optimizer.compute_gradients(kl_loss)
kl_grads, _ = zip(*kl_grads_and_vars)
return input_ph, label_ph, output2, train_op, grads, kl_grads, eval_metric_ops, loss, \
kl_loss, predictions['probabilities'], predictions['classes']
def _loss_function(conf_gt, conf_logits, reg_gt, reg_logits, config):
"""
Creates the PPN loss function.
Returns (conf_loss, point_loss)
conf_gt:
Ground truth confidence, i.e. 1 for close anchors, 0 for anchors
that are too far off and -1 for anchors to be ignored. Must have
shape (?, fh, fw, k).
conf_logits:
PPN confidence output, must have shape (?, fh, fw, k).
reg_gt:
Ground truth point offsets, need only have valid values for the
anchors with conf_gt of 1. Must have shape (?, fh, fw, 2k).
reg_logits:
PPN anchor offset output, must have shape (?, fh, fw, 2k).
config
The configuration dictionary. See ppn.config.ppn_config.
"""
import tensorflow.compat.v1 as tf
# mask out the invalid anchors:
# only penalize confidence of valid (i.e. not ignored) anchors
# only penalize points of positive anchors
valid_mask = tf.stop_gradient(tf.not_equal(conf_gt, -1))
pos_mask = tf.stop_gradient(tf.equal(conf_gt, 1))
num_valid = tf.stop_gradient(tf.count_nonzero(valid_mask, dtype=tf.int32))
num_pos = tf.stop_gradient(tf.count_nonzero(pos_mask, dtype=tf.int32))
valid_conf_gt = tf.boolean_mask(conf_gt, valid_mask)
valid_conf_logits = tf.boolean_mask(conf_logits, valid_mask)
pos_reg_gt = tf.boolean_mask(reg_gt, pos_mask)
pos_reg_logits = tf.boolean_mask(reg_logits, pos_mask)
if config['loss_function'] == 'crossentropy':
# get the confidence loss using sigmoidal cross entropy
conf_loss = tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.cast(valid_conf_gt, tf.float32),
logits=valid_conf_logits)
else:
# get the confidence loss using focal loss
conf_loss = _binary_focal_loss_with_logits(
labels=tf.cast(valid_conf_gt, tf.float32),
logits=valid_conf_logits,
gamma=config['focal_gamma'],
pos_weight=config['focal_pos_weight'])
if config['focal_normalized']:
# normalize according to number of positive anchors
conf_loss = conf_loss / tf.cast(num_valid, tf.float32)
conf_loss = tf.reduce_sum(conf_loss)
# get the point loss using MSE
point_loss = tf.losses.mean_squared_error(
labels=pos_reg_gt,
predictions=pos_reg_logits,
reduction=tf.losses.Reduction.SUM)
# zero out the losses if there were no valid points
conf_loss = tf.where(tf.equal(num_valid, 0),
0.0,
conf_loss,
name='conf_loss')
point_loss = tf.where(tf.equal(num_pos, 0),
0.0,
point_loss,
name='point_loss')
# normalize losses to contribute equally and add
N_conf, N_reg = config['N_conf'], config['N_reg']
return ((1.0/N_conf) * conf_loss, (1.0/N_reg) * point_loss)
def step(index, scores_sum, scores_num):
"""Single step."""
index %= epoch_length # Only needed in eval runs.
# Note - the only way to ensure making a copy of tensor is to run simple
# operation. We are waiting for tf.copy:
# https://github.com/tensorflow/tensorflow/issues/11186
obs_copy = batch_env.observ + 0
value_fun_shape = (num_agents, )
if distributional_size > 1:
value_fun_shape = (num_agents, distributional_size)
def env_step(arg1, arg2, arg3): # pylint: disable=unused-argument
"""Step of the environment."""
(logits, value_function) = get_policy(obs_copy, ppo_hparams,
batch_env.action_space,
distributional_size)
action = common_layers.sample_with_temperature(
logits, sampling_temp)
action = tf.cast(action, tf.int32)
action = tf.reshape(action, shape=(num_agents, ))
reward, done = batch_env.simulate(action)
pdf = tfp.distributions.Categorical(logits=logits).prob(action)
pdf = tf.reshape(pdf, shape=(num_agents, ))
value_function = tf.reshape(value_function,
shape=value_fun_shape)
done = tf.reshape(done, shape=(num_agents, ))
with tf.control_dependencies([reward, done]):
return tf.identity(pdf), tf.identity(value_function), \
tf.identity(done)
# TODO(piotrmilos): while_body is executed at most once,
# thus should be replaced with tf.cond
pdf, value_function, top_level_done = tf.while_loop(
lambda _1, _2, _3: tf.equal(speculum.size(), 0),
env_step,
[
tf.constant(0.0, shape=(num_agents, )),
tf.constant(0.0, shape=value_fun_shape),
tf.constant(False, shape=(num_agents, ))
],
parallel_iterations=1,
back_prop=False,
)
with tf.control_dependencies([pdf, value_function]):
obs, reward, done, action = speculum.dequeue()
to_save = [obs, reward, done, action, pdf, value_function]
save_ops = [
tf.scatter_update(memory_slot, index, value)
for memory_slot, value in zip(memory, to_save)
]
cumulate_rewards_op = cumulative_rewards.assign_add(reward)
agent_indices_to_reset = tf.where(top_level_done)[:, 0]
with tf.control_dependencies([cumulate_rewards_op]):
# TODO(piotrmilos): possibly we need cumulative_rewards.read_value()
scores_sum_delta = tf.reduce_sum(
tf.gather(cumulative_rewards.read_value(),
agent_indices_to_reset))
scores_num_delta = tf.count_nonzero(done, dtype=tf.int32)
with tf.control_dependencies(save_ops +
[scores_sum_delta, scores_num_delta]):
reset_env_op = batch_env.reset(agent_indices_to_reset)
reset_cumulative_rewards_op = tf.scatter_update(
cumulative_rewards, agent_indices_to_reset,
tf.gather(zeros_tensor, agent_indices_to_reset))
with tf.control_dependencies(
[reset_env_op, reset_cumulative_rewards_op]):
return [
index + 1, scores_sum + scores_sum_delta,
scores_num + scores_num_delta
]
def compute_loss(self, y_true, y_pred):
'''
Compute the loss of the SSD model prediction against the ground truth.
Arguments:
y_true (array): A Numpy array of shape `(batch_size, #boxes, #classes + 12)`,
where `#boxes` is the total number of boxes that the model predicts
per image. Be careful to make sure that the index of each given
box in `y_true` is the same as the index for the corresponding
box in `y_pred`. The last axis must have length `#classes + 12` and contain
`[classes one-hot encoded, 4 ground truth box coordinate offsets, 8 arbitrary entries]`
in this order, including the background class. The last eight entries of the
last axis are not used by this function and therefore their contents are
irrelevant, they only exist so that `y_true` has the same shape as `y_pred`,
where the last four entries of the last axis contain the anchor box
coordinates, which are needed during inference. Important: Boxes that
you want the cost function to ignore need to have a one-hot
class vector of all zeros.
y_pred (Keras tensor): The model prediction. The shape is identical
to that of `y_true`, i.e. `(batch_size, #boxes, #classes + 12)`.
The last axis must contain entries in the format
`[classes one-hot encoded, 4 predicted box coordinate offsets, 8 arbitrary entries]`.
Returns:
A scalar, the total multitask loss for classification and localization.
'''
self.neg_pos_ratio = tf.constant(self.neg_pos_ratio)
self.n_neg_min = tf.constant(self.n_neg_min)
self.alpha = tf.constant(self.alpha)
batch_size = tf.shape(y_pred)[0] # Output dtype: tf.int32
n_boxes = tf.shape(
y_pred
)[1] # Output dtype: tf.int32, note that `n_boxes` in this context denotes the total number of boxes per image, not the number of boxes per cell
# 1: Compute the losses for class and box predictions for every box
classification_loss = tf.to_float(
self.log_loss(
y_true[:, :, :-12],
y_pred[:, :, :-12])) # Output shape: (batch_size, n_boxes)
localization_loss = tf.to_float(
self.smooth_L1_loss(
y_true[:, :, -12:-8],
y_pred[:, :, -12:-8])) # Output shape: (batch_size, n_boxes)
# 2: Compute the classification losses for the positive and negative targets
# Create masks for the positive and negative ground truth classes
negatives = y_true[:, :, 0] # Tensor of shape (batch_size, n_boxes)
positives = tf.to_float(
tf.reduce_max(y_true[:, :, 1:-12],
axis=-1)) # Tensor of shape (batch_size, n_boxes)
# Count the number of positive boxes (classes 1 to n) in y_true across the whole batch
n_positive = tf.reduce_sum(positives)
# Now mask all negative boxes and sum up the losses for the positive boxes PER batch item
# (Keras loss functions must output one scalar loss value PER batch item, rather than just
# one scalar for the entire batch, that's why we're not summing across all axes)
pos_class_loss = tf.reduce_sum(
classification_loss * positives,
axis=-1) # Tensor of shape (batch_size,)
# Compute the classification loss for the negative default boxes (if there are any)
# First, compute the classification loss for all negative boxes
neg_class_loss_all = classification_loss * negatives # Tensor of shape (batch_size, n_boxes)
n_neg_losses = tf.count_nonzero(
neg_class_loss_all, dtype=tf.int32
) # The number of non-zero loss entries in `neg_class_loss_all`
# What's the point of `n_neg_losses`? For the next step, which will be to compute which negative boxes enter the classification
# loss, we don't just want to know how many negative ground truth boxes there are, but for how many of those there actually is
# a positive (i.e. non-zero) loss. This is necessary because `tf.nn.top-k()` in the function below will pick the top k boxes with
# the highest losses no matter what, even if it receives a vector where all losses are zero. In the unlikely event that all negative
# classification losses ARE actually zero though, this behavior might lead to `tf.nn.top-k()` returning the indices of positive
# boxes, leading to an incorrect negative classification loss computation, and hence an incorrect overall loss computation.
# We therefore need to make sure that `n_negative_keep`, which assumes the role of the `k` argument in `tf.nn.top-k()`,
# is at most the number of negative boxes for which there is a positive classification loss.
# Compute the number of negative examples we want to account for in the loss
# We'll keep at most `self.neg_pos_ratio` times the number of positives in `y_true`, but at least `self.n_neg_min` (unless `n_neg_loses` is smaller)
n_negative_keep = tf.minimum(
tf.maximum(self.neg_pos_ratio * tf.to_int32(n_positive),
self.n_neg_min), n_neg_losses)
# In the unlikely case when either (1) there are no negative ground truth boxes at all
# or (2) the classification loss for all negative boxes is zero, return zero as the `neg_class_loss`
def f1():
return tf.zeros([batch_size])
# Otherwise compute the negative loss
def f2():
# Now we'll identify the top-k (where k == `n_negative_keep`) boxes with the highest confidence loss that
# belong to the background class in the ground truth data. Note that this doesn't necessarily mean that the model
# predicted the wrong class for those boxes, it just means that the loss for those boxes is the highest.
# To do this, we reshape `neg_class_loss_all` to 1D...
neg_class_loss_all_1D = tf.reshape(
neg_class_loss_all,
[-1]) # Tensor of shape (batch_size * n_boxes,)
# ...and then we get the indices for the `n_negative_keep` boxes with the highest loss out of those...
values, indices = tf.nn.top_k(neg_class_loss_all_1D,
n_negative_keep,
False) # We don't need sorting
# ...and with these indices we'll create a mask...
negatives_keep = tf.scatter_nd(
tf.expand_dims(indices, axis=1),
updates=tf.ones_like(indices, dtype=tf.int32),
shape=tf.shape(neg_class_loss_all_1D
)) # Tensor of shape (batch_size * n_boxes,)
negatives_keep = tf.to_float(
tf.reshape(negatives_keep,
[batch_size, n_boxes
])) # Tensor of shape (batch_size, n_boxes)
# ...and use it to keep only those boxes and mask all other classification losses
neg_class_loss = tf.reduce_sum(
classification_loss * negatives_keep,
axis=-1) # Tensor of shape (batch_size,)
return neg_class_loss
neg_class_loss = tf.cond(tf.equal(n_neg_losses, tf.constant(0)), f1,
f2)
class_loss = pos_class_loss + neg_class_loss # Tensor of shape (batch_size,)
# 3: Compute the localization loss for the positive targets
# We don't penalize localization loss for negative predicted boxes (obviously: there are no ground truth boxes they would correspond to)
loc_loss = tf.reduce_sum(localization_loss * positives,
axis=-1) # Tensor of shape (batch_size,)
# 4: Compute the total loss
#self.class_loss = class_loss / tf.maximum(1.0, n_positive)
#self.loc_loss = (self.alpha * loc_loss) / tf.maximum(1.0, n_positive)
total_loss = (class_loss + self.alpha * loc_loss) / tf.maximum(
1.0, n_positive) # In case `n_positive == 0`
# Keras has the annoying habit of dividing the loss by the batch size, which sucks in our case
# because the relevant criterion to average our loss over is the number of positive boxes in the batch
# (by which we're dividing in the line above), not the batch size. So in order to revert Keras' averaging
# over the batch size, we'll have to multiply by it.
total_loss *= tf.to_float(batch_size)
#print "Classification loss: ", self.class_loss
#print "Localization loss: ", self.loc_loss
return total_loss
def create_low_latency_svdf_model(fingerprint_input, model_settings,
is_training, runtime_settings):
"""Builds an SVDF model with low compute requirements.
This is based in the topology presented in the 'Compressing Deep Neural
Networks using a Rank-Constrained Topology' paper:
https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43813.pdf
Here's the layout of the graph:
(fingerprint_input)
v
[SVDF]<-(weights)
v
[BiasAdd]<-(bias)
v
[Relu]
v
[MatMul]<-(weights)
v
[BiasAdd]<-(bias)
v
[MatMul]<-(weights)
v
[BiasAdd]<-(bias)
v
[MatMul]<-(weights)
v
[BiasAdd]<-(bias)
v
This model produces lower recognition accuracy than the 'conv' model above,
but requires fewer weight parameters and, significantly fewer computations.
During training, dropout nodes are introduced after the relu, controlled by a
placeholder.
Args:
fingerprint_input: TensorFlow node that will output audio feature vectors.
The node is expected to produce a 2D Tensor of shape:
[batch, model_settings['dct_coefficient_count'] *
model_settings['spectrogram_length']]
with the features corresponding to the same time slot arranged contiguously,
and the oldest slot at index [:, 0], and newest at [:, -1].
model_settings: Dictionary of information about the model.
is_training: Whether the model is going to be used for training.
runtime_settings: Dictionary of information about the runtime.
Returns:
TensorFlow node outputting logits results, and optionally a dropout
placeholder.
Raises:
ValueError: If the inputs tensor is incorrectly shaped.
"""
if is_training:
dropout_prob = tf.placeholder(tf.float32, name='dropout_prob')
input_frequency_size = model_settings['dct_coefficient_count']
input_time_size = model_settings['spectrogram_length']
# Validation.
input_shape = fingerprint_input.get_shape()
if len(input_shape) != 2:
raise ValueError('Inputs to `SVDF` should have rank == 2.')
if input_shape[-1].value is None:
raise ValueError('The last dimension of the inputs to `SVDF` '
'should be defined. Found `None`.')
if input_shape[-1].value % input_frequency_size != 0:
raise ValueError(
'Inputs feature dimension %d must be a multiple of '
'frame size %d', fingerprint_input.shape[-1].value,
input_frequency_size)
# Set number of units (i.e. nodes) and rank.
rank = 2
num_units = 1280
# Number of filters: pairs of feature and time filters.
num_filters = rank * num_units
# Create the runtime memory: [num_filters, batch, input_time_size]
batch = 1
memory = tf.Variable(tf.zeros([num_filters, batch, input_time_size]),
trainable=False,
name='runtime-memory')
# Determine the number of new frames in the input, such that we only operate
# on those. For training we do not use the memory, and thus use all frames
# provided in the input.
# new_fingerprint_input: [batch, num_new_frames*input_frequency_size]
if is_training:
num_new_frames = input_time_size
else:
window_stride_ms = int(model_settings['window_stride_samples'] * 1000 /
model_settings['sample_rate'])
num_new_frames = tf.cond(
tf.equal(tf.count_nonzero(memory), 0), lambda: input_time_size,
lambda: int(runtime_settings['clip_stride_ms'] / window_stride_ms))
new_fingerprint_input = fingerprint_input[:, -num_new_frames *
input_frequency_size:]
# Expand to add input channels dimension.
new_fingerprint_input = tf.expand_dims(new_fingerprint_input, 2)
# Create the frequency filters.
weights_frequency = tf.Variable(
tf.truncated_normal([input_frequency_size, num_filters], stddev=0.01))
# Expand to add input channels dimensions.
# weights_frequency: [input_frequency_size, 1, num_filters]
weights_frequency = tf.expand_dims(weights_frequency, 1)
# Convolve the 1D feature filters sliding over the time dimension.
# activations_time: [batch, num_new_frames, num_filters]
activations_time = tf.nn.conv1d(new_fingerprint_input, weights_frequency,
input_frequency_size, 'VALID')
# Rearrange such that we can perform the batched matmul.
# activations_time: [num_filters, batch, num_new_frames]
activations_time = tf.transpose(activations_time, perm=[2, 0, 1])
# Runtime memory optimization.
if not is_training:
# We need to drop the activations corresponding to the oldest frames, and
# then add those corresponding to the new frames.
new_memory = memory[:, :, num_new_frames:]
new_memory = tf.concat([new_memory, activations_time], 2)
tf.assign(memory, new_memory)
activations_time = new_memory
# Create the time filters.
weights_time = tf.Variable(
tf.truncated_normal([num_filters, input_time_size], stddev=0.01))
# Apply the time filter on the outputs of the feature filters.
# weights_time: [num_filters, input_time_size, 1]
# outputs: [num_filters, batch, 1]
weights_time = tf.expand_dims(weights_time, 2)
outputs = tf.matmul(activations_time, weights_time)
# Split num_units and rank into separate dimensions (the remaining
# dimension is the input_shape[0] -i.e. batch size). This also squeezes
# the last dimension, since it's not used.
# [num_filters, batch, 1] => [num_units, rank, batch]
outputs = tf.reshape(outputs, [num_units, rank, -1])
# Sum the rank outputs per unit => [num_units, batch].
units_output = tf.reduce_sum(outputs, axis=1)
# Transpose to shape [batch, num_units]
units_output = tf.transpose(units_output)
# Appy bias.
bias = tf.Variable(tf.zeros([num_units]))
first_bias = tf.nn.bias_add(units_output, bias)
# Relu.
first_relu = tf.nn.relu(first_bias)
if is_training:
first_dropout = tf.nn.dropout(first_relu, dropout_prob)
else:
first_dropout = first_relu
first_fc_output_channels = 256
first_fc_weights = tf.Variable(
tf.truncated_normal([num_units, first_fc_output_channels],
stddev=0.01))
first_fc_bias = tf.Variable(tf.zeros([first_fc_output_channels]))
first_fc = tf.matmul(first_dropout, first_fc_weights) + first_fc_bias
if is_training:
second_fc_input = tf.nn.dropout(first_fc, dropout_prob)
else:
second_fc_input = first_fc
second_fc_output_channels = 256
second_fc_weights = tf.Variable(
tf.truncated_normal(
[first_fc_output_channels, second_fc_output_channels],
stddev=0.01))
second_fc_bias = tf.Variable(tf.zeros([second_fc_output_channels]))
second_fc = tf.matmul(second_fc_input, second_fc_weights) + second_fc_bias
if is_training:
final_fc_input = tf.nn.dropout(second_fc, dropout_prob)
else:
final_fc_input = second_fc
label_count = model_settings['label_count']
final_fc_weights = tf.Variable(
tf.truncated_normal([second_fc_output_channels, label_count],
stddev=0.01))
final_fc_bias = tf.Variable(tf.zeros([label_count]))
final_fc = tf.matmul(final_fc_input, final_fc_weights) + final_fc_bias
if is_training:
return final_fc, dropout_prob
else:
return final_fc
y = hidden_layer(x, weight1, bias1, weight2, bias2, weight3, bias3)
#加入正则化
#regularizer = tf.contrib.layers.l2_regularizer(0.01)
regularizer = tf.keras.regularizers.l2(0.001)
regularization = regularizer(weight1) + regularizer(weight2) + regularizer(
weight3)
tf.add_to_collection("losses", regularization)
#定义损失函数
error_loss = tf.reduce_sum(tf.pow(y_ - y, 2)) / sample_size
tf.add_to_collection("losses", error_loss)
loss = tf.add_n(tf.get_collection("losses"))
#定义准确率
accuracy = tf.count_nonzero(tf.less(tf.pow(y_ - y, 2), 0.25)) / sample_size_yz
#定义优化器
train_op = tf.train.AdamOptimizer(0.05).minimize(loss)
#train_op = tf.train.GradientDescentOptimizer(0.05).minimize(loss)
with tf.Session() as sess:
tf.global_variables_initializer().run()
for i in range(training_steps):
sess.run(train_op, feed_dict={x: data, y_: label})
if i % 2000 == 0:
#计算损失函数
loss_value = sess.run(loss, feed_dict={x: data, y_: label})
#计算准确度
test_accuracy = sess.run(accuracy,
feed_dict={
def save_reduce_mean(x, axis=None, keepdims=False):
return save_divide(
tf.reduce_sum(x, axis=axis, keepdims=keepdims),
tf.cast(tf.count_nonzero(x, axis=axis, keepdims=keepdims), x.dtype))
def _build(self, probs, all_anchors, gt_boxes):
"""
Args:
all_anchors: A Tensor with anchors for all of SSD's features.
The shape of the Tensor is (num_anchors, 4).
gt_boxes: A Tensor with the ground truth boxes for the image.
The shape of the Tensor is (num_gt, 5), having the truth label
as the last value for each box.
Returns:
class_targets: Either a truth value of the anchor (a value
between 0 and num_classes, with 0 being background), or -1 when
the anchor is to be ignored in the minibatch.
The shape of the Tensor is (num_anchors, 1).
bbox_offsets_targets: A bounding box regression target for each of
the anchors that have a greater than zero label. For every
other anchors we return zeros.
The shape of the Tensor is (num_anchors, 4).
"""
all_anchors = tf.cast(all_anchors, tf.float32)
gt_boxes = tf.cast(gt_boxes, tf.float32)
# We are going to label each anchor based on the IoU with
# `gt_boxes`. Start by filling the labels with -1, marking them as
# unknown.
anchors_label_shape = tf.gather(tf.shape(all_anchors), [0])
anchors_label = tf.fill(dims=anchors_label_shape, value=-1.)
overlaps = bbox_overlap_tf(all_anchors, gt_boxes[:, :4])
max_overlaps = tf.reduce_max(overlaps, axis=1)
# Get the index of the best gt_box for each anchor.
best_gtbox_for_anchors_idx = tf.argmax(overlaps, axis=1)
# Having the index of the gt bbox with the best label we need to get
# the label for each gt box and sum 1 to it because 0 is used for
# background.
best_fg_labels_for_anchors = tf.add(
tf.gather(gt_boxes[:, 4], best_gtbox_for_anchors_idx), 1.)
iou_is_fg = tf.greater_equal(max_overlaps, self._foreground_threshold)
# We update anchors_label with the value in
# best_fg_labels_for_anchors only when the box is foreground.
# TODO: Replace with a sparse_to_dense with -1 default_value
anchors_label = tf.where(condition=iou_is_fg,
x=best_fg_labels_for_anchors,
y=anchors_label)
best_anchor_idxs = tf.argmax(overlaps, axis=0)
is_best_box = tf.sparse_to_dense(sparse_indices=best_anchor_idxs,
sparse_values=True,
default_value=False,
output_shape=tf.cast(
anchors_label_shape, tf.int64),
validate_indices=False)
# Now we need to find the anchors that are the best for each of the
# gt_boxes. We overwrite the previous anchors_label with this
# because setting the best anchor for each gt_box has priority.
best_anchors_gt_labels = tf.sparse_to_dense(
sparse_indices=best_anchor_idxs,
sparse_values=gt_boxes[:, 4] + 1,
default_value=-1,
output_shape=tf.cast(anchors_label_shape, tf.int64),
validate_indices=False,
name="get_right_labels_for_bestboxes")
anchors_label = tf.where(condition=is_best_box,
x=best_anchors_gt_labels,
y=anchors_label,
name="update_labels_for_bestbox_anchors")
# Use the worst backgrounds (the bgs whose probability of being fg is
# the greatest).
cls_probs = probs[:, 1:]
max_cls_probs = tf.reduce_max(cls_probs, axis=1)
# Exclude boxes with IOU > `background_threshold_high` with any GT.
iou_less_than_bg_tresh_high_filter = tf.less_equal(
max_overlaps, self._background_threshold_high)
bg_anchors = tf.less_equal(anchors_label, 0)
bg_overlaps_filter = tf.logical_and(iou_less_than_bg_tresh_high_filter,
bg_anchors)
max_cls_probs = tf.where(
condition=bg_overlaps_filter,
x=max_cls_probs,
y=tf.fill(dims=anchors_label_shape, value=-1.),
)
# We calculate up to how many backgrounds we desire based on the
# final number of foregrounds and the hard minning ratio.
num_fg_mask = tf.greater(anchors_label, 0.0)
num_fg = tf.cast(tf.count_nonzero(num_fg_mask), tf.float32)
num_bg = tf.cast(num_fg * self._hard_negative_ratio, tf.int32)
top_k_bg = tf.nn.top_k(max_cls_probs, k=num_bg)
set_bg = tf.sparse_to_dense(sparse_indices=top_k_bg.indices,
sparse_values=True,
default_value=False,
output_shape=anchors_label_shape,
validate_indices=False)
anchors_label = tf.where(condition=set_bg,
x=tf.fill(dims=anchors_label_shape, value=0.),
y=anchors_label)
# Next step is to calculate the proper bbox targets for the labeled
# anchors based on the values of the ground-truth boxes.
# We have to use only the anchors labeled >= 1, each matching with
# the proper gt_boxes
# Get the ids of the anchors that mater for bbox_target comparison.
is_anchor_with_target = tf.greater(anchors_label, 0)
anchors_with_target_idx = tf.where(condition=is_anchor_with_target)
# Get the corresponding ground truth box only for the anchors with
# target.
gt_boxes_idxs = tf.gather(best_gtbox_for_anchors_idx,
anchors_with_target_idx)
# Get the values of the ground truth boxes.
anchors_gt_boxes = tf.gather_nd(gt_boxes[:, :4], gt_boxes_idxs)
# We create the same array but with the anchors
anchors_with_target = tf.gather_nd(all_anchors,
anchors_with_target_idx)
# We create our targets with bbox_transform
bbox_targets = encode(anchors_with_target,
anchors_gt_boxes,
variances=self._variances)
# We unmap targets to anchor_labels (containing the length of
# anchors)
bbox_targets = tf.scatter_nd(indices=anchors_with_target_idx,
updates=bbox_targets,
shape=tf.cast(tf.shape(all_anchors),
tf.int64))
return anchors_label, bbox_targets
def train_function(training_method, loss, cross_loss, reg_loss, output_dir,
use_tpu):
"""Training script for resnet model.
Args:
training_method: string indicating pruning method used to compress model.
loss: tensor float32 of the cross entropy + regularization losses.
cross_loss: tensor, only cross entropy loss, passed for logging.
reg_loss: tensor, only regularization loss, passed for logging.
output_dir: string tensor indicating the directory to save summaries.
use_tpu: boolean indicating whether to run script on a tpu.
Returns:
host_call: summary tensors to be computed at each training step.
train_op: the optimization term.
"""
global_step = tf.train.get_global_step()
steps_per_epoch = FLAGS.num_train_images / FLAGS.train_batch_size
current_epoch = (tf.cast(global_step, tf.float32) / steps_per_epoch)
learning_rate = lr_schedule(current_epoch)
if FLAGS.use_adam:
# We don't use step decrease for the learning rate.
learning_rate = FLAGS.base_learning_rate * (FLAGS.train_batch_size /
256.0)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
else:
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=FLAGS.momentum,
use_nesterov=True)
if use_tpu:
# use CrossShardOptimizer when using TPU.
optimizer = contrib_tpu.CrossShardOptimizer(optimizer)
if training_method == 'set':
# We override the train op to also update the mask.
optimizer = sparse_optimizers.SparseSETOptimizer(
optimizer,
begin_step=FLAGS.maskupdate_begin_step,
end_step=FLAGS.maskupdate_end_step,
grow_init=FLAGS.grow_init,
frequency=FLAGS.maskupdate_frequency,
drop_fraction=FLAGS.drop_fraction,
drop_fraction_anneal=FLAGS.drop_fraction_anneal,
stateless_seed_offset=FLAGS.seed)
elif training_method == 'static':
# We override the train op to also update the mask.
optimizer = sparse_optimizers.SparseStaticOptimizer(
optimizer,
begin_step=FLAGS.maskupdate_begin_step,
end_step=FLAGS.maskupdate_end_step,
grow_init=FLAGS.grow_init,
frequency=FLAGS.maskupdate_frequency,
drop_fraction=FLAGS.drop_fraction,
drop_fraction_anneal=FLAGS.drop_fraction_anneal,
stateless_seed_offset=FLAGS.seed)
elif training_method == 'momentum':
# We override the train op to also update the mask.
optimizer = sparse_optimizers.SparseMomentumOptimizer(
optimizer,
begin_step=FLAGS.maskupdate_begin_step,
end_step=FLAGS.maskupdate_end_step,
momentum=FLAGS.s_momentum,
frequency=FLAGS.maskupdate_frequency,
drop_fraction=FLAGS.drop_fraction,
grow_init=FLAGS.grow_init,
stateless_seed_offset=FLAGS.seed,
drop_fraction_anneal=FLAGS.drop_fraction_anneal,
use_tpu=use_tpu)
elif training_method == 'rigl':
# We override the train op to also update the mask.
optimizer = sparse_optimizers.SparseRigLOptimizer(
optimizer,
begin_step=FLAGS.maskupdate_begin_step,
end_step=FLAGS.maskupdate_end_step,
grow_init=FLAGS.grow_init,
frequency=FLAGS.maskupdate_frequency,
drop_fraction=FLAGS.drop_fraction,
stateless_seed_offset=FLAGS.seed,
drop_fraction_anneal=FLAGS.drop_fraction_anneal,
initial_acc_scale=FLAGS.rigl_acc_scale,
use_tpu=use_tpu)
elif training_method == 'snip':
optimizer = sparse_optimizers.SparseSnipOptimizer(
optimizer,
mask_init_method=FLAGS.mask_init_method,
custom_sparsity_map=CUSTOM_SPARSITY_MAP,
default_sparsity=FLAGS.end_sparsity,
use_tpu=use_tpu)
elif training_method in ('scratch', 'baseline'):
pass
else:
raise ValueError('Unsupported pruning method: %s' %
FLAGS.training_method)
# UPDATE_OPS needs to be added as a dependency due to batch norm
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops), tf.name_scope('train'):
train_op = optimizer.minimize(loss, global_step)
metrics = {
'global_step': tf.train.get_or_create_global_step(),
'loss': loss,
'cross_loss': cross_loss,
'reg_loss': reg_loss,
'learning_rate': learning_rate,
'current_epoch': current_epoch,
}
# Logging drop_fraction if dynamic sparse training.
if training_method in ('set', 'momentum', 'rigl', 'static'):
metrics['drop_fraction'] = optimizer.drop_fraction
# Let's log some statistics from a single parameter-mask couple.
# This is useful for debugging.
test_var = pruning.get_weights()[0]
test_var_mask = pruning.get_masks()[0]
metrics.update({
'fw_nz_weight': tf.count_nonzero(test_var),
'fw_nz_mask': tf.count_nonzero(test_var_mask),
'fw_l1_weight': tf.reduce_sum(tf.abs(test_var))
})
masks = pruning.get_masks()
global_sparsity = sparse_utils.calculate_sparsity(masks)
metrics['global_sparsity'] = global_sparsity
metrics.update(
utils.mask_summaries(masks[:4] + masks[-1:],
with_img=FLAGS.log_mask_imgs_each_iteration))
host_call = (functools.partial(utils.host_call_fn,
output_dir), utils.format_tensors(metrics))
return host_call, train_op
def build_optimizer(self,
learning_rate=0.001,
weight_decay=0.0005,
momentum=0.9,
global_step=None):
self.labels = tf.placeholder(tf.float32,
name='labels',
shape=[None, None, self.num_vars])
with tf.variable_scope('ground_truth'):
# Split the ground truth tensor
# Classification ground truth tensor
# Shape: (batch_size, num_anchors, num_classes)
gt_cl = self.labels[:, :, :self.num_classes]
# Localization ground truth tensor
# Shape: (batch_size, num_anchors, 4)
gt_loc = self.labels[:, :, self.num_classes:]
# Batch size
# Shape: scalar
batch_size = tf.shape(gt_cl)[0]
# Compute match counters
with tf.variable_scope('match_counters'):
# Number of anchors per sample
# Shape: (batch_size)
total_num = tf.ones([batch_size], dtype=tf.int64) * \
tf.to_int64(self.preset.num_anchors)
# Number of negative (not-matched) anchors per sample, computed
# by counting boxes of the background class in each sample.
# Shape: (batch_size)
negatives_num = tf.count_nonzero(gt_cl[:, :, -1], axis=1)
# Number of positive (matched) anchors per sample
# Shape: (batch_size)
positives_num = total_num - negatives_num
# Number of positives per sample that is division-safe
# Shape: (batch_size)
positives_num_safe = tf.where(tf.equal(positives_num, 0),
tf.ones([batch_size]) * 10e-15,
tf.to_float(positives_num))
# Compute masks
with tf.variable_scope('match_masks'):
# Boolean tensor determining whether an anchor is a positive
# Shape: (batch_size, num_anchors)
positives_mask = tf.equal(gt_cl[:, :, -1], 0)
# Boolean tensor determining whether an anchor is a negative
# Shape: (batch_size, num_anchors)
negatives_mask = tf.logical_not(positives_mask)
# Compute the confidence loss
with tf.variable_scope('confidence_loss'):
# Cross-entropy tensor - all of the values are non-negative
# Shape: (batch_size, num_anchors)
ce = tf.nn.softmax_cross_entropy_with_logits_v2(labels=gt_cl,
logits=self.logits)
# Sum up the loss of all the positive anchors
# Positives - the loss of negative anchors is zeroed out
# Shape: (batch_size, num_anchors)
positives = tf.where(positives_mask, ce, tf.zeros_like(ce))
# Total loss of positive anchors
# Shape: (batch_size)
positives_sum = tf.reduce_sum(positives, axis=-1)
# Figure out what the negative anchors with highest confidence loss
# are
# Negatives - the loss of positive anchors is zeroed out
# Shape: (batch_size, num_anchors)
negatives = tf.where(negatives_mask, ce, tf.zeros_like(ce))
# Top negatives - sorted confience loss with the highest one first
# Shape: (batch_size, num_anchors)
negatives_top = tf.nn.top_k(negatives, self.preset.num_anchors)[0]
# Fugure out what the number of negatives we want to keep is
# Maximum number of negatives to keep per sample - we keep at most
# 3 times as many as we have positive anchors in the sample
# Shape: (batch_size)
negatives_num_max = tf.minimum(negatives_num, 3 * positives_num)
# Mask out superfluous negatives and compute the sum of the loss
# Transposed vector of maximum negatives per sample
# Shape (batch_size, 1)
negatives_num_max_t = tf.expand_dims(negatives_num_max, 1)
# Range tensor: [0, 1, 2, ..., num_anchors-1]
# Shape: (num_anchors)
rng = tf.range(0, self.preset.num_anchors, 1)
# Row range, the same as above, but int64 and a row of a matrix
# Shape: (1, num_anchors)
range_row = tf.to_int64(tf.expand_dims(rng, 0))
# Mask of maximum negatives - first `negative_num_max` elements
# in corresponding row are `True`, the rest is false
# Shape: (batch_size, num_anchors)
negatives_max_mask = tf.less(range_row, negatives_num_max_t)
# Max negatives - all the positives and superfluous negatives are
# zeroed out.
# Shape: (batch_size, num_anchors)
negatives_max = tf.where(negatives_max_mask, negatives_top,
tf.zeros_like(negatives_top))
# Sum of max negatives for each sample
# Shape: (batch_size)
negatives_max_sum = tf.reduce_sum(negatives_max, axis=-1)
# Compute the confidence loss for each element
# Total confidence loss for each sample
# Shape: (batch_size)
confidence_loss = tf.add(positives_sum, negatives_max_sum)
# Total confidence loss normalized by the number of positives
# per sample
# Shape: (batch_size)
confidence_loss = tf.where(
tf.equal(positives_num, 0), tf.zeros([batch_size]),
tf.div(confidence_loss, positives_num_safe))
# Mean confidence loss for the batch
# Shape: scalar
self.confidence_loss = tf.reduce_mean(confidence_loss,
name='confidence_loss')
# Compute the localization loss
with tf.variable_scope('localization_loss'):
# Element-wise difference between the predicted localization loss
# and the ground truth
# Shape: (batch_size, num_anchors, 4)
loc_diff = tf.subtract(self.locator, gt_loc)
# Smooth L1 loss
# Shape: (batch_size, num_anchors, 4)
loc_loss = smooth_l1_loss(loc_diff)
# Sum of localization losses for each anchor
# Shape: (batch_size, num_anchors)
loc_loss_sum = tf.reduce_sum(loc_loss, axis=-1)
# Positive locs - the loss of negative anchors is zeroed out
# Shape: (batch_size, num_anchors)
positive_locs = tf.where(positives_mask, loc_loss_sum,
tf.zeros_like(loc_loss_sum))
# Total loss of positive anchors
# Shape: (batch_size)
localization_loss = tf.reduce_sum(positive_locs, axis=-1)
# Total localization loss normalized by the number of positives
# per sample
# Shape: (batch_size)
localization_loss = tf.where(
tf.equal(positives_num, 0), tf.zeros([batch_size]),
tf.div(localization_loss, positives_num_safe))
# Mean localization loss for the batch
# Shape: scalar
self.localization_loss = tf.reduce_mean(localization_loss,
name='localization_loss')
# Compute total loss
with tf.variable_scope('total_loss'):
# Sum of the localization and confidence loss
# Shape: (batch_size)
self.conf_and_loc_loss = tf.add(self.confidence_loss,
self.localization_loss,
name='sum_losses')
# L2 loss
# Shape: scalar
self.l2_loss = tf.multiply(weight_decay,
self.l2_loss,
name='l2_loss')
# Final loss
# Shape: scalar
self.loss = tf.add(self.conf_and_loc_loss,
self.l2_loss,
name='loss')
# Build the optimizer
with tf.variable_scope('optimizer'):
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
optimizer = optimizer.minimize(self.loss,
global_step=global_step,
name='optimizer')
# Store the tensors
self.optimizer = optimizer
self.losses = {
'total': self.loss,
'localization': self.localization_loss,
'confidence': self.confidence_loss,
'l2': self.l2_loss
}