def _get_testing(rnn_logits,sequence_length,label,label_length):
"""Create ops for testing (all scalars):
loss: CTC loss function value,
label_error: Batch-normalized edit distance on beam search max
sequence_error: Batch-normalized sequence error rate
"""
with tf.name_scope("train"):
loss = model.ctc_loss_layer(rnn_logits,label,sequence_length)
with tf.name_scope("test"):
predictions,_ = tf.nn.ctc_beam_search_decoder(rnn_logits,
sequence_length,
beam_width=128,
top_paths=1,
merge_repeated=True)
hypothesis = tf.cast(predictions[0], tf.int32) # for edit_distance
label_errors = tf.edit_distance(hypothesis, label, normalize=False)
sequence_errors = tf.count_nonzero(label_errors,axis=0)
total_label_error = tf.reduce_sum( label_errors )
total_labels = tf.reduce_sum( label_length )
label_error = tf.truediv( total_label_error,
tf.cast(total_labels, tf.float32 ),
name='label_error')
sequence_error = tf.truediv( tf.cast( sequence_errors, tf.int32 ),
tf.shape(label_length)[0],
name='sequence_error')
tf.summary.scalar( 'loss', loss )
tf.summary.scalar( 'label_error', label_error )
tf.summary.scalar( 'sequence_error', sequence_error )
return loss, label_error, sequence_error
def rpn_losses(anchor_labels, anchor_boxes, label_logits, box_logits):
"""
Args:
anchor_labels: fHxfWxNA
anchor_boxes: fHxfWxNAx4, encoded
label_logits: fHxfWxNA
box_logits: fHxfWxNAx4
Returns:
label_loss, box_loss
"""
with tf.device('/cpu:0'):
valid_mask = tf.stop_gradient(tf.not_equal(anchor_labels, -1))
pos_mask = tf.stop_gradient(tf.equal(anchor_labels, 1))
nr_valid = tf.stop_gradient(tf.count_nonzero(valid_mask, dtype=tf.int32), name='num_valid_anchor')
nr_pos = tf.count_nonzero(pos_mask, dtype=tf.int32, name='num_pos_anchor')
valid_anchor_labels = tf.boolean_mask(anchor_labels, valid_mask)
valid_label_logits = tf.boolean_mask(label_logits, valid_mask)
with tf.name_scope('label_metrics'):
valid_label_prob = tf.nn.sigmoid(valid_label_logits)
summaries = []
with tf.device('/cpu:0'):
for th in [0.5, 0.2, 0.1]:
valid_prediction = tf.cast(valid_label_prob > th, tf.int32)
nr_pos_prediction = tf.reduce_sum(valid_prediction, name='num_pos_prediction')
pos_prediction_corr = tf.count_nonzero(
tf.logical_and(
valid_label_prob > th,
tf.equal(valid_prediction, valid_anchor_labels)),
dtype=tf.int32)
summaries.append(tf.truediv(
pos_prediction_corr,
nr_pos, name='recall_th{}'.format(th)))
precision = tf.to_float(tf.truediv(pos_prediction_corr, nr_pos_prediction))
precision = tf.where(tf.equal(nr_pos_prediction, 0), 0.0, precision, name='precision_th{}'.format(th))
summaries.append(precision)
add_moving_summary(*summaries)
label_loss = tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.to_float(valid_anchor_labels), logits=valid_label_logits)
label_loss = tf.reduce_mean(label_loss, name='label_loss')
pos_anchor_boxes = tf.boolean_mask(anchor_boxes, pos_mask)
pos_box_logits = tf.boolean_mask(box_logits, pos_mask)
delta = 1.0 / 9
box_loss = tf.losses.huber_loss(
pos_anchor_boxes, pos_box_logits, delta=delta,
reduction=tf.losses.Reduction.SUM) / delta
box_loss = tf.div(
box_loss,
tf.cast(nr_valid, tf.float32), name='box_loss')
add_moving_summary(label_loss, box_loss, nr_valid, nr_pos)
return label_loss, box_loss
def one_bp_iteration(self, xe_v2c_pre_iter, H_sumC_to_V, H_sumV_to_C, xe_0):
xe_tanh = tf.tanh(tf.to_double(tf.truediv(xe_v2c_pre_iter, [2.0])))
xe_tanh = tf.to_float(xe_tanh)
xe_tanh_temp = tf.sign(xe_tanh)
xe_sum_log_img = tf.matmul(H_sumC_to_V, tf.multiply(tf.truediv((1 - xe_tanh_temp), [2.0]), [3.1415926]))
xe_sum_log_real = tf.matmul(H_sumC_to_V, tf.log(1e-8 + tf.abs(xe_tanh)))
xe_sum_log_complex = tf.complex(xe_sum_log_real, xe_sum_log_img)
xe_product = tf.real(tf.exp(xe_sum_log_complex))
xe_product_temp = tf.multiply(tf.sign(xe_product), -2e-7)
xe_pd_modified = tf.add(xe_product, xe_product_temp)
xe_v_sumc = tf.multiply(self.atanh(xe_pd_modified), [2.0])
xe_c_sumv = tf.add(xe_0, tf.matmul(H_sumV_to_C, xe_v_sumc))
return xe_v_sumc, xe_c_sumv
def build_graph(self, state, action, futurereward, action_prob):
logits, value = self._get_NN_prediction(state)
value = tf.squeeze(value, [1], name='pred_value') # (B,)
policy = tf.nn.softmax(logits, name='policy')
is_training = get_current_tower_context().is_training
if not is_training:
return
log_probs = tf.log(policy + 1e-6)
log_pi_a_given_s = tf.reduce_sum(
log_probs * tf.one_hot(action, NUM_ACTIONS), 1)
advantage = tf.subtract(tf.stop_gradient(value), futurereward, name='advantage')
pi_a_given_s = tf.reduce_sum(policy * tf.one_hot(action, NUM_ACTIONS), 1) # (B,)
importance = tf.stop_gradient(tf.clip_by_value(pi_a_given_s / (action_prob + 1e-8), 0, 10))
policy_loss = tf.reduce_sum(log_pi_a_given_s * advantage * importance, name='policy_loss')
xentropy_loss = tf.reduce_sum(policy * log_probs, name='xentropy_loss')
value_loss = tf.nn.l2_loss(value - futurereward, name='value_loss')
pred_reward = tf.reduce_mean(value, name='predict_reward')
advantage = tf.sqrt(tf.reduce_mean(tf.square(advantage)), name='rms_advantage')
entropy_beta = tf.get_variable('entropy_beta', shape=[],
initializer=tf.constant_initializer(0.01), trainable=False)
cost = tf.add_n([policy_loss, xentropy_loss * entropy_beta, value_loss])
cost = tf.truediv(cost, tf.cast(tf.shape(futurereward)[0], tf.float32), name='cost')
summary.add_moving_summary(policy_loss, xentropy_loss,
value_loss, pred_reward, advantage,
cost, tf.reduce_mean(importance, name='importance'))
return cost
def _init_training(self, optimizer):
with self.model_graph.as_default():
# счётчик обработанных батчей
self.batches_processed = tf.Variable(
initial_value=0, trainable=False
)
increment_batches = self.batches_processed.assign_add(1)
# аккумулятор для среднего значения потерь
self.average_loss = tf.Variable(
initial_value=0.0, trainable=False
)
# рекуррентный пересчёт среднего значения функции потерь
updated_loss = tf.truediv(
tf.add(
self.average_loss * tf.to_float(self.batches_processed),
self.loss
),
tf.to_float(self.batches_processed) + 1.0
)
update_average_loss = self.average_loss.assign(updated_loss)
opt_op = optimizer.minimize(self.loss)
# группируем операции оптимизации и обновления счётчиков в одну
with tf.control_dependencies([opt_op]):
self.train_op = tf.group(
update_average_loss, increment_batches
)
def __init__(self, model, mask, prob, coords, offset_xy_min, offset_xy_max, areas):
self.model = model
with tf.name_scope('true'):
self.mask = tf.identity(mask, name='mask')
self.prob = tf.identity(prob, name='prob')
self.coords = tf.identity(coords, name='coords')
self.offset_xy_min = tf.identity(offset_xy_min, name='offset_xy_min')
self.offset_xy_max = tf.identity(offset_xy_max, name='offset_xy_max')
self.areas = tf.identity(areas, name='areas')
with tf.name_scope('iou') as name:
_offset_xy_min = tf.maximum(model.offset_xy_min, self.offset_xy_min, name='_offset_xy_min')
_offset_xy_max = tf.minimum(model.offset_xy_max, self.offset_xy_max, name='_offset_xy_max')
_wh = tf.maximum(_offset_xy_max - _offset_xy_min, 0.0, name='_wh')
_areas = tf.reduce_prod(_wh, -1, name='_areas')
areas = tf.maximum(self.areas + model.areas - _areas, 1e-10, name='areas')
iou = tf.truediv(_areas, areas, name=name)
with tf.name_scope('mask'):
best_box_iou = tf.reduce_max(iou, 2, True, name='best_box_iou')
best_box = tf.to_float(tf.equal(iou, best_box_iou), name='best_box')
mask_best = tf.identity(self.mask * best_box, name='mask_best')
mask_normal = tf.identity(1 - mask_best, name='mask_normal')
with tf.name_scope('dist'):
iou_dist = tf.square(model.iou - mask_best, name='iou_dist')
coords_dist = tf.square(model.coords - self.coords, name='coords_dist')
prob_dist = tf.square(model.prob - self.prob, name='prob_dist')
with tf.name_scope('objectives'):
cnt = np.multiply.reduce(iou_dist.get_shape().as_list())
self['iou_best'] = tf.identity(tf.reduce_sum(mask_best * iou_dist) / cnt, name='iou_best')
self['iou_normal'] = tf.identity(tf.reduce_sum(mask_normal * iou_dist) / cnt, name='iou_normal')
self['coords'] = tf.identity(tf.reduce_sum(tf.expand_dims(mask_best, -1) * coords_dist) / cnt, name='coords')
self['prob'] = tf.identity(tf.reduce_sum(tf.expand_dims(self.mask, -1) * prob_dist) / cnt, name='prob')
def drawGraph(self, n_row, n_latent, n_col):
with tf.name_scope('matDecomp'):
self._p = tf.placeholder(tf.float32, shape=[None, n_col])
self._c = tf.placeholder(tf.float32, shape=[None, n_col])
self._lambda = tf.placeholder(tf.float32)
self._index = tf.placeholder(tf.float32, shape=[None, n_row])
self._A = tf.Variable(tf.truncated_normal([n_row, n_latent]))
self._B = tf.Variable(tf.truncated_normal([n_latent, n_col]))
self._h = tf.matmul(tf.matmul(self._index, self._A), self._B)
weighted_loss = tf.reduce_mean(tf.mul(self._c, tf.squared_difference(self._p, self._h)))
self._weighted_loss = weighted_loss
l2_A = tf.reduce_sum(tf.square(self._A))
l2_B = tf.reduce_sum(tf.square(self._B))
n_w = tf.constant(n_row * n_latent + n_latent * n_col, tf.float32)
l2 = tf.truediv(tf.add(l2_A, l2_B), n_w)
reg_term = tf.mul(self._lambda, l2)
self._loss = tf.add(weighted_loss, reg_term)
self._mask = tf.placeholder(tf.float32, shape=[n_row, n_col])
one = tf.constant(1, tf.float32)
pred = tf.cast(tf.greater_equal(tf.matmul(self._A, self._B), one), tf.float32)
cor = tf.mul(tf.cast(tf.equal(pred, self._p), tf.float32), self._c)
self._vali_err = tf.reduce_sum(tf.mul(cor, self._mask))
self._saver = tf.train.Saver([v for v in tf.all_variables() if v.name.find('matDecomp') != -1])
tf.scalar_summary('training_weighted_loss_l2', self._loss)
tf.scalar_summary('validation_weighted_loss', self._weighted_loss)
merged = tf.merge_all_summaries()
def _build_graph(self, inputs):
state, action, futurereward = inputs
logits, self.value = self._get_NN_prediction(state)
self.value = tf.squeeze(self.value, [1], name='pred_value') # (B,)
self.policy = tf.nn.softmax(logits, name='policy')
expf = tf.get_variable('explore_factor', shape=[],
initializer=tf.constant_initializer(1), trainable=False)
policy_explore = tf.nn.softmax(logits * expf, name='policy_explore')
is_training = get_current_tower_context().is_training
if not is_training:
return
log_probs = tf.log(self.policy + 1e-6)
log_pi_a_given_s = tf.reduce_sum(
log_probs * tf.one_hot(action, NUM_ACTIONS), 1)
advantage = tf.subtract(tf.stop_gradient(self.value), futurereward, name='advantage')
policy_loss = tf.reduce_sum(log_pi_a_given_s * advantage, name='policy_loss')
xentropy_loss = tf.reduce_sum(
self.policy * log_probs, name='xentropy_loss')
value_loss = tf.nn.l2_loss(self.value - futurereward, name='value_loss')
pred_reward = tf.reduce_mean(self.value, name='predict_reward')
advantage = symbf.rms(advantage, name='rms_advantage')
entropy_beta = tf.get_variable('entropy_beta', shape=[],
initializer=tf.constant_initializer(0.01), trainable=False)
self.cost = tf.add_n([policy_loss, xentropy_loss * entropy_beta, value_loss])
self.cost = tf.truediv(self.cost,
tf.cast(tf.shape(futurereward)[0], tf.float32),
name='cost')
summary.add_moving_summary(policy_loss, xentropy_loss,
value_loss, pred_reward, advantage, self.cost)
def lamb_func(logit, logic, lamb):
logit_pos = tf.boolean_mask(logit, logic)
logit_neg = tf.boolean_mask(logit, tf.logical_not(logic))
logit_neg_exp = tf.exp(logit_neg * lamb)
z = tf.reduce_mean(logit_neg_exp)
left = tf.truediv(tf.reduce_mean(logit_neg * logit_neg_exp), z)
right = tf.reduce_mean(logit_pos)
return left, right
def fastrcnn_losses(labels, label_logits, fg_boxes, fg_box_logits):
"""
Args:
labels: n,
label_logits: nxC
fg_boxes: nfgx4, encoded
fg_box_logits: nfgxCx4 or nfgx1x4 if class agnostic
Returns:
label_loss, box_loss
"""
label_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=labels, logits=label_logits)
label_loss = tf.reduce_mean(label_loss, name='label_loss')
fg_inds = tf.where(labels > 0)[:, 0]
fg_labels = tf.gather(labels, fg_inds)
num_fg = tf.size(fg_inds, out_type=tf.int64)
empty_fg = tf.equal(num_fg, 0)
if int(fg_box_logits.shape[1]) > 1:
indices = tf.stack(
[tf.range(num_fg), fg_labels], axis=1) # #fgx2
fg_box_logits = tf.gather_nd(fg_box_logits, indices)
else:
fg_box_logits = tf.reshape(fg_box_logits, [-1, 4])
with tf.name_scope('label_metrics'), tf.device('/cpu:0'):
prediction = tf.argmax(label_logits, axis=1, name='label_prediction')
correct = tf.to_float(tf.equal(prediction, labels)) # boolean/integer gather is unavailable on GPU
accuracy = tf.reduce_mean(correct, name='accuracy')
fg_label_pred = tf.argmax(tf.gather(label_logits, fg_inds), axis=1)
num_zero = tf.reduce_sum(tf.to_int64(tf.equal(fg_label_pred, 0)), name='num_zero')
false_negative = tf.where(
empty_fg, 0., tf.to_float(tf.truediv(num_zero, num_fg)), name='false_negative')
fg_accuracy = tf.where(
empty_fg, 0., tf.reduce_mean(tf.gather(correct, fg_inds)), name='fg_accuracy')
box_loss = tf.losses.huber_loss(
fg_boxes, fg_box_logits, reduction=tf.losses.Reduction.SUM)
box_loss = tf.truediv(
box_loss, tf.to_float(tf.shape(labels)[0]), name='box_loss')
add_moving_summary(label_loss, box_loss, accuracy,
fg_accuracy, false_negative, tf.to_float(num_fg, name='num_fg_label'))
return label_loss, box_loss
def mini_batch_rmse(estimated, target):
with tf.name_scope('evaluation'):
with tf.control_dependencies([tf.assert_equal(count(tf.to_int32(target) - tf.to_int32(target)), 0.)]):
squared_difference = tf.pow(estimated - target, 2, name='squared_difference')
square_error = tf.reduce_sum(squared_difference, name='summing_square_errors')
square_error = tf.to_float(square_error)
mse = tf.truediv(square_error, count(target), name='meaning_error')
rmse = tf.sqrt(mse)
return rmse
def scale_bboxes(bbox, img_shape):
"""Scale bboxes to [0, 1). bbox format [ymin, xmin, ymax, xmax]
Args:
bbox: 2-D with shape '[num_bbox, 4]'
img_shape: 1-D with shape '[4]'
Return:
sclaed_bboxes: scaled bboxes
"""
img_h = tf.cast(img_shape[0], dtype=tf.float32)
img_w = tf.cast(img_shape[1], dtype=tf.float32)
shape = bbox.get_shape().as_list()
_axis = 1 if len(shape) > 1 else 0
[y_min, x_min, y_max, x_max] = tf.unstack(bbox, axis=_axis)
y_1 = tf.truediv(y_min, img_h)
x_1 = tf.truediv(x_min, img_w)
y_2 = tf.truediv(y_max, img_h)
x_2 = tf.truediv(x_max, img_w)
return tf.stack([y_1, x_1, y_2, x_2], axis=_axis)
def to_chroma_tf(bar_or_track_bar, is_normalize=True):
"""Return the chroma tensor of the input tensor"""
out_shape = tf.stack([tf.shape(bar_or_track_bar)[0], bar_or_track_bar.get_shape()[1], 12, 7,
bar_or_track_bar.get_shape()[3]])
chroma = tf.reduce_sum(tf.reshape(tf.cast(bar_or_track_bar, tf.float32), out_shape), axis=3)
if is_normalize:
chroma_max = tf.reduce_max(chroma, axis=(1, 2, 3), keep_dims=True)
chroma_min = tf.reduce_min(chroma, axis=(1, 2, 3), keep_dims=True)
return tf.truediv(chroma - chroma_min, (chroma_max - chroma_min + 1e-15))
else:
return chroma
def _create(self, encoder_output, decoder_state_size, **kwargs):
""" Creates decoder's initial RNN states according to
`decoder_state_size`.
Do linear transformations to encoder output/state and map the
structure to `decoder_state_size`.
If params[`bridge_input`] == "output", first average the encoder
output tensor over timesteps.
Args:
encoder_output: An instance of `collections.namedtuple`
from `Encoder.encode()`.
decoder_state_size: RNN decoder state size.
**kwargs:
Returns: The decoder states with the structure determined
by `decoder_state_size`.
Raises:
ValueError: if `encoder_output` has no attribute named
params[`bridge_input`].
"""
if not hasattr(encoder_output, self.params["bridge_input"]):
raise ValueError("encoder output has not attribute: {}, "
"only final_state and outputs available"
.format(self.params["bridge_input"]))
if self.params["bridge_input"] == "outputs":
# [batch_size, max_time, num_units]
context = encoder_output.outputs
mask = tf.sequence_mask(
lengths=tf.to_int32(encoder_output.attention_length),
maxlen=tf.shape(context)[1],
dtype=tf.float32)
# [batch_size, num_units]
bridge_input = tf.truediv(
tf.reduce_sum(context * tf.expand_dims(mask, 2), axis=1),
tf.expand_dims(
tf.to_float(encoder_output.attention_length), 1))
elif self.params["bridge_input"] == "final_states":
bridge_input = nest.flatten(_final_states(encoder_output.final_states))
bridge_input = tf.concat(bridge_input, 1)
else:
raise ValueError("Unrecognized value of bridge_input: {}, "
"should be outputs or final_state".format(self.params["bridge_input"]))
state_size_splits = nest.flatten(decoder_state_size)
total_decoder_state_size = sum(state_size_splits)
# [batch_size, total_decoder_state_size]
init_state = fflayer(inputs=bridge_input,
output_size=total_decoder_state_size,
activation=self._activation,
name="init_state_trans")
init_state = nest.pack_sequence_as(
decoder_state_size,
tf.split(init_state, state_size_splits, axis=1))
return init_state
def _build(self, input_tensor):
dtype = input_tensor.dtype
tensor = input_tensor.unwrap()
if 'int' in input_tensor.dtype:
dtype = luchador.get_nn_dtype()
tensor = tf.cast(tensor, dtype)
if self._denom is None:
self._instantiate_denominator(dtype)
output = tf.truediv(tensor, self._denom, 'ouptut')
return Tensor(output, name='output')
def losses(input_mask, labels, ious, box_delta_input, pred_class_probs, pred_conf, pred_box_delta):
batch_size = tf.shape(input_mask)[0]
num_objects = tf.reduce_sum(input_mask, name='num_objects')
with tf.name_scope('class_regression') as scope:
# cross-entropy: q * -log(p) + (1-q) * -log(1-p)
# add a small value into log to prevent blowing up
class_loss = tf.truediv(
tf.reduce_sum(
(labels * (-tf.log(pred_class_probs + config.EPSILON))
+ (1 - labels) * (-tf.log(1 - pred_class_probs + config.EPSILON)))
* input_mask * config.LOSS_COEF_CLASS),
num_objects,
name='class_loss'
)
tf.losses.add_loss(class_loss)
with tf.name_scope('confidence_score_regression') as scope:
input_mask_ = tf.reshape(input_mask, [batch_size, config.ANCHORS])
conf_loss = tf.reduce_mean(
tf.reduce_sum(
tf.square((ious - pred_conf))
* (input_mask_ * config.LOSS_COEF_CONF_POS / num_objects
+ (1 - input_mask_) * config.LOSS_COEF_CONF_NEG / (config.ANCHORS - num_objects)),
reduction_indices=[1]
),
name='confidence_loss'
)
tf.losses.add_loss(conf_loss)
with tf.name_scope('bounding_box_regression') as scope:
bbox_loss = tf.truediv(
tf.reduce_sum(
config.LOSS_COEF_BBOX * tf.square(
input_mask * (pred_box_delta - box_delta_input))),
num_objects,
name='bbox_loss'
)
tf.losses.add_loss(bbox_loss)
def build_graph(self, input, nextinput):
is_training = get_current_tower_context().is_training
initializer = tf.random_uniform_initializer(-0.05, 0.05)
def get_basic_cell():
cell = rnn.BasicLSTMCell(num_units=HIDDEN_SIZE, forget_bias=0.0, reuse=tf.get_variable_scope().reuse)
if is_training:
cell = rnn.DropoutWrapper(cell, output_keep_prob=1 - DROPOUT)
return cell
cell = rnn.MultiRNNCell([get_basic_cell() for _ in range(NUM_LAYER)])
def get_v(n):
return tf.get_variable(n, [BATCH, HIDDEN_SIZE],
trainable=False,
initializer=tf.constant_initializer())
state_var = [rnn.LSTMStateTuple(
get_v('c{}'.format(k)), get_v('h{}'.format(k))) for k in range(NUM_LAYER)]
self.state = state_var = tuple(state_var)
embeddingW = tf.get_variable('embedding', [VOCAB_SIZE, HIDDEN_SIZE], initializer=initializer)
input_feature = tf.nn.embedding_lookup(embeddingW, input) # B x seqlen x hiddensize
input_feature = Dropout(input_feature, keep_prob=1 - DROPOUT)
with tf.variable_scope('LSTM', initializer=initializer):
input_list = tf.unstack(input_feature, num=SEQ_LEN, axis=1) # seqlen x (Bxhidden)
outputs, last_state = rnn.static_rnn(cell, input_list, state_var, scope='rnn')
# update the hidden state after a rnn loop completes
update_state_ops = []
for k in range(NUM_LAYER):
update_state_ops.extend([
tf.assign(state_var[k].c, last_state[k].c),
tf.assign(state_var[k].h, last_state[k].h)])
# seqlen x (Bxrnnsize)
output = tf.reshape(tf.concat(outputs, 1), [-1, HIDDEN_SIZE]) # (Bxseqlen) x hidden
logits = FullyConnected('fc', output, VOCAB_SIZE,
activation=tf.identity, kernel_initializer=initializer,
bias_initializer=initializer)
xent_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=tf.reshape(nextinput, [-1]))
with tf.control_dependencies(update_state_ops):
cost = tf.truediv(tf.reduce_sum(xent_loss),
tf.cast(BATCH, tf.float32), name='cost') # log-perplexity
perpl = tf.exp(cost / SEQ_LEN, name='perplexity')
summary.add_moving_summary(perpl, cost)
return cost
def _safe_div(numerator, denominator):
"""Divides two tensors element-wise, returning 0 if the denominator is <= 0.
Args:
numerator: A real `Tensor`.
denominator: A real `Tensor`, with dtype matching `numerator`.
Returns:
0 if `denominator` <= 0, else `numerator` / `denominator`
"""
t = tf.truediv(numerator, denominator)
zero = tf.zeros_like(t, dtype=denominator.dtype)
condition = tf.greater(denominator, zero)
zero = tf.cast(zero, t.dtype)
return tf.where(condition, t, zero)
def gaussian2d(x, y, cx, cy, a, b, dtype = tf.float32):
"""
This cunction calcuate sum of N 2D Gaussian probability density
functions in m points
y, x : m x n 2D tensor. Position of calculation points
m is number of calculation points
n is number of Gaussian functions
cx, cy, a, b : m x n 2D tensor.
Parameters of Gaussian function
cx and cy are center position
a and b are the width in x and y firection
"""
# A = 1/(2*pi*a*b)
A = tf.inv(tf.mul(tf.constant(2.0*np.pi, dtype), tf.mul(a, b)))
# powerX = (x-xc)^2 / (2*a^2)
powerX = tf.truediv(tf.pow(tf.sub(x, cx) , tf.constant(2.0, dtype)),
tf.mul(tf.constant(2.0, dtype),tf.pow(a, tf.constant(2.0, dtype))))
# powerY = (y-yc)^2 / (2*b^2)
powerY = tf.truediv(tf.pow(tf.sub(y, cy) , tf.constant(2.0, dtype)),
tf.mul(tf.constant(2.0, dtype),tf.pow(a, tf.constant(2.0, dtype))))
# p = A*exp(- powerX - powerY) standard 2D Gaussian distribution
probability = tf.reduce_sum(
tf.mul(A, tf.exp(tf.neg(tf.add(powerX, powerY)))), 1)
return probability
def _apply_gradients(self, grads_and_vars, **_):
decay, ep = self.args['decay'], self.args['epsilon']
updates, new_grads_and_vars = [], []
for grad, var in grads_and_vars:
rms = self._create_slot_var(var, 'rms')
new_rms = rms + (1. - decay) * (tf.square(grad) - rms)
new_grad = tf.truediv(grad, tf.sqrt(new_rms + ep) + ep)
updates.append(rms.assign(new_rms))
new_grads_and_vars.append((new_grad, var))
updates.append(self.optimizer.apply_gradients(new_grads_and_vars))
return Operation(tf.group(*updates))
def batch_iou_fast(anchors, bboxes):
""" Compute iou of two batch of boxes. Box format '[y_min, x_min, y_max, x_max]'.
Args:
anchors: know shape
bboxes: dynamic shape
Return:
ious: 2-D with shape '[num_bboxes, num_anchors]'
"""
num_anchors = anchors.get_shape().as_list()[0]
num_bboxes = tf.shape(bboxes)[0]
box_indices = tf.reshape(tf.range(num_bboxes), shape=[-1, 1])
box_indices = tf.reshape(tf.stack([box_indices] * num_anchors, axis=1), shape=[-1, 1]) # use tf.tile instead
# box_indices = tf.tile(box_indices, [num_anchors, 1])
# box_indices = tf.Print(box_indices, [box_indices], "box_indices", summarize=100)
bboxes_m = tf.gather_nd(bboxes, box_indices)
# bboxes_m = tf.Print(bboxes_m, [bboxes_m], "bboxes_m", summarize=100)
anchors_m = tf.tile(anchors, [num_bboxes, 1])
# anchors_m = tf.Print(anchors_m, [anchors_m], "anchors_m", summarize=100)
lr = tf.maximum(
tf.minimum(bboxes_m[:, 3], anchors_m[:, 3]) -
tf.maximum(bboxes_m[:, 1], anchors_m[:, 1]),
0
)
tb = tf.maximum(
tf.minimum(bboxes_m[:, 2], anchors_m[:, 2]) -
tf.maximum(bboxes_m[:, 0], anchors_m[:, 0]),
0
)
intersection = tf.multiply(tb, lr)
union = tf.subtract(
tf.multiply((bboxes_m[:, 3] - bboxes_m[:, 1]), (bboxes_m[:, 2] - bboxes_m[:, 0])) +
tf.multiply((anchors_m[:, 3] - anchors_m[:, 1]), (anchors_m[:, 2] - anchors_m[:, 0])),
intersection
)
ious = tf.truediv(intersection, union)
ious = tf.reshape(ious, shape=[num_bboxes, num_anchors])
return ious
def pairwise_iou(boxlist1, boxlist2):
"""Computes pairwise intersection-over-union between box collections.
Args:
boxlist1: Nx4 floatbox
boxlist2: Mx4
Returns:
a tensor with shape [N, M] representing pairwise iou scores.
"""
intersections = pairwise_intersection(boxlist1, boxlist2)
areas1 = area(boxlist1)
areas2 = area(boxlist2)
unions = (
tf.expand_dims(areas1, 1) + tf.expand_dims(areas2, 0) - intersections)
return tf.where(
tf.equal(intersections, 0.0),
tf.zeros_like(intersections), tf.truediv(intersections, unions))
def ioa(boxlist1, boxlist2, scope=None):
"""Computes pairwise intersection-over-area between box collections.
intersection-over-area (IOA) between two boxes box1 and box2 is defined as
their intersection area over box2's area. Note that ioa is not symmetric,
that is, ioa(box1, box2) != ioa(box2, box1).
Args:
boxlist1: BoxList holding N boxes
boxlist2: BoxList holding M boxes
scope: name scope.
Returns:
a tensor with shape [N, M] representing pairwise ioa scores.
"""
with tf.name_scope(scope, 'IOA'):
intersections = intersection(boxlist1, boxlist2)
areas = tf.expand_dims(area(boxlist2), 0)
return tf.truediv(intersections, areas)
def east_iou(boxlist1, boxlist2, scope=None):
"""Computes pairwise intersection-over-union between box collections.
Args:
boxlist1: BoxList holding N boxes
boxlist2: BoxList holding M boxes
scope: name scope.
Returns:
a tensor with shape [N, M] representing pairwise iou scores.
"""
with tf.name_scope(scope, 'EAST_IOU'):
intersections = intersection(boxlist1, boxlist2)
areas1 = area(boxlist1)
areas2 = area(boxlist2)
unions = areas2
return tf.where(
tf.equal(intersections, 0.0),
tf.zeros_like(intersections), tf.truediv(intersections, unions))
def cross_entropy(z, y):
"""
:param z: output of nn
:param y: ground truth
"""
z = tf.reshape(z, tf.pack([tf.shape(z)[0], -1]))
y = tf.reshape(y, tf.pack([tf.shape(y)[0], -1]))
count_neg = tf.reduce_sum(1. - y)
count_pos = tf.reduce_sum(y)
total = tf.add(count_neg, count_pos)
beta = tf.truediv(count_neg, total)
eps = 1e-8
loss_pos = tf.mul(-beta, tf.reduce_sum(tf.mul(tf.log(tf.abs(z) + eps), y), 1))
loss_neg = tf.mul(1. - beta, tf.reduce_sum(tf.mul(tf.log(tf.abs(1. - z) + eps), 1. - y), 1))
cost = tf.sub(loss_pos, loss_neg)
cost = tf.reduce_mean(cost, name='cost')
return cost
def make_test_node(self, hypers_name):
outputs = self.tf_nodes[hypers_name]["outputs"]
deltas = []
for var_name, output_node in outputs.iteritems():
data_node = self.tf_nodes[hypers_name]["placeholders"][var_name]
output_rank = output_node.get_shape().ndims
if output_rank == 1:
output_node = tf.tile(tf.expand_dims(output_node, 0), [tf.shape(data_node)[0], 1])
deltas.append(
tf.to_int32(tf.argmax(output_node, dimension=1)) - data_node)
zero_if_correct = tf.reduce_sum(tf.pack(deltas), reduction_indices=0)
zero_elements = tf.equal(zero_if_correct, tf.zeros_like(zero_if_correct))
n_correct = tf.reduce_sum(tf.to_int32(zero_elements))
n_total = tf.shape(zero_if_correct)[0]
accuracy = tf.truediv(n_correct, n_total)
self.summary_nodes["test"] = tf.scalar_summary('test_accuracy', accuracy)
self.tf_nodes[hypers_name]["accuracy"] = accuracy
def matched_iou(boxlist1, boxlist2, scope=None):
"""Compute intersection-over-union between corresponding boxes in boxlists.
Args:
boxlist1: BoxList holding N boxes
boxlist2: BoxList holding N boxes
scope: name scope.
Returns:
a tensor with shape [N] representing pairwise iou scores.
"""
with tf.name_scope(scope, 'MatchedIOU'):
intersections = matched_intersection(boxlist1, boxlist2)
areas1 = area(boxlist1)
areas2 = area(boxlist2)
unions = areas1 + areas2 - intersections
return tf.where(
tf.equal(intersections, 0.0),
tf.zeros_like(intersections), tf.truediv(intersections, unions))
def proposal_metrics(iou):
"""
Add summaries for RPN proposals.
Args:
iou: nxm, #proposal x #gt
"""
# find best roi for each gt, for summary only
best_iou = tf.reduce_max(iou, axis=0)
mean_best_iou = tf.reduce_mean(best_iou, name='best_iou_per_gt')
summaries = [mean_best_iou]
with tf.device('/cpu:0'):
for th in [0.3, 0.5]:
recall = tf.truediv(
tf.count_nonzero(best_iou >= th),
tf.size(best_iou, out_type=tf.int64),
name='recall_iou{}'.format(th))
summaries.append(recall)
add_moving_summary(*summaries)
def _arccosine(self, slist1, slist2, tf_embs):
"""
Uses an arccosine kernel of degree 0 to calculate
the similarity matrix between two vectors of embeddings.
This is just cosine similarity projected into the [0,1] interval.
"""
dot = self._dot(slist1, slist2, tf_embs)
# This calculation corresponds to an arc-cosine with
# degree 0. It can be interpreted as cosine
# similarity but projected into a [0,1] interval.
# TODO: arc-cosine with degree 1.
tf_pi = tf.constant(np.pi, dtype=tf.float64)
tf_norms = tf.constant(self.norms, dtype=tf.float64, name='norms')
normlist1 = tf.gather(tf_norms, slist1, name='normlist1')
normlist2 = tf.matrix_transpose(tf.gather(tf_norms, slist2, name='normlist2'))
norms = tf.batch_matmul(normlist1, normlist2)
cosine = tf.clip_by_value(tf.truediv(dot, norms), -1, 1)
angle = tf.acos(cosine)
angle = tf.select(tf.is_nan(angle), tf.ones_like(angle) * tf_pi, angle)
return 1 - (angle / tf_pi)
def _apply_gradients(self, grads_and_vars, **_):
d1, d2 = self.args['decay1'], self.args['decay2']
ep = self.args['epsilon']
updates, new_grads_and_vars = [], []
for grad, var in grads_and_vars:
mean_grad1 = self._create_slot_var(var, 'grad_mean')
mean_grad2 = self._create_slot_var(var, 'grad_squared_mean')
new_mean_grad1 = d1 * mean_grad1 + (1.0 - d1) * grad
new_mean_grad2 = d2 * mean_grad2 + (1.0 - d2) * tf.square(grad)
rms = tf.sqrt(new_mean_grad2 - tf.square(new_mean_grad1) + ep)
new_grad = tf.truediv(grad, rms)
updates.append(mean_grad1.assign(new_mean_grad1))
updates.append(mean_grad2.assign(new_mean_grad2))
new_grads_and_vars.append((new_grad, var))
updates.append(self.optimizer.apply_gradients(new_grads_and_vars))
return Operation(tf.group(*updates))
#8강 - 주요함수 실습하기 a = tf.constant(16) b = tf.constant(14) ##덧셈함수 사용하기 c = tf.add(a, b) sess.run(c) ##뺄셈함수 사용하기 c = tf.subtract(a, b) sess.run(c) ##곱셈함수 사용하기 c = tf.multiply(a, b) sess.run(c) ##나눗셈 함수 사용하기 c = tf.truediv(a, b) sess.run(c) ##나눗셈의 나머지 함수 사용하기 c = tf.mod(a, b) sess.run(c) ##절대값 함수 사용 c = tf.abs(-a) sess.run(c) #a,b변수 다시 초기화 a = tf.constant(17.5) b = tf.constant(5.0) ##음수함수사용 c = tf.negative(a) sess.run(c) #부호함수 사용
# Initialize placeholders
x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32)
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)
# Create variables for linear regression
A = tf.Variable(tf.random_normal(shape=[1,1]))
b = tf.Variable(tf.random_normal(shape=[1,1]))
# Declare model operations
model_output = tf.add(tf.matmul(x_data, A), b)
# Declare Demming loss function
demming_numerator = tf.abs(tf.sub(y_target, tf.add(tf.matmul(x_data, A), b)))
demming_denominator = tf.sqrt(tf.add(tf.square(A),1))
loss = tf.reduce_mean(tf.truediv(demming_numerator, demming_denominator))
# Initialize variables
init = tf.initialize_all_variables()
sess.run(init)
# Declare optimizer
my_opt = tf.train.GradientDescentOptimizer(0.1)
train_step = my_opt.minimize(loss)
# Training loop
loss_vec = []
for i in range(250):
rand_index = np.random.choice(len(x_vals), size=batch_size)
rand_x = np.transpose([x_vals[rand_index]])
rand_y = np.transpose([y_vals[rand_index]])
# Initialize placeholders
x_data = tf.compat.v1.placeholder(shape=[None, 1], dtype=tf.float32)
y_target = tf.compat.v1.placeholder(shape=[None, 1], dtype=tf.float32)
# Create variables for linear regression
A = tf.Variable(tf.random.normal(shape=[1,1]))
b = tf.Variable(tf.random.normal(shape=[1,1]))
# Declare model operations
model_output = tf.add(tf.matmul(x_data, A), b)
# Declare Deming loss function
deming_numerator = tf.abs(tf.subtract(y_target, tf.add(tf.matmul(x_data, A), b)))
deming_denominator = tf.sqrt(tf.add(tf.square(A),1))
loss = tf.reduce_mean(input_tensor=tf.truediv(deming_numerator, deming_denominator))
# Declare optimizer
my_opt = tf.compat.v1.train.GradientDescentOptimizer(0.15)
train_step = my_opt.minimize(loss)
# Initialize variables
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
# Training loop
loss_vec = []
for i in range(1000):
rand_index = np.random.choice(len(x_vals), size=batch_size)
rand_x = np.transpose([x_vals[rand_index]])
rand_y = np.transpose([y_vals[rand_index]])
def rpn_losses(anchor_labels, anchor_boxes, label_logits, box_logits):
"""
Args:
anchor_labels: fHxfWxNA
anchor_boxes: fHxfWxNAx4, encoded
label_logits: fHxfWxNA
box_logits: fHxfWxNAx4
Returns:
label_loss, box_loss
"""
with tf.device('/cpu:0'):
valid_mask = tf.stop_gradient(tf.not_equal(anchor_labels, -1))
pos_mask = tf.stop_gradient(tf.equal(anchor_labels, 1))
nr_valid = tf.stop_gradient(tf.count_nonzero(valid_mask,
dtype=tf.int32),
name='num_valid_anchor')
nr_pos = tf.identity(tf.count_nonzero(pos_mask, dtype=tf.int32),
name='num_pos_anchor')
# nr_pos is guaranteed >0 in C4. But in FPN. even nr_valid could be 0.
valid_anchor_labels = tf.boolean_mask(anchor_labels, valid_mask)
valid_label_logits = tf.boolean_mask(label_logits, valid_mask)
with tf.name_scope('label_metrics'):
valid_label_prob = tf.nn.sigmoid(valid_label_logits)
summaries = []
with tf.device('/cpu:0'):
for th in [0.5, 0.2, 0.1]:
valid_prediction = tf.cast(valid_label_prob > th, tf.int32)
nr_pos_prediction = tf.reduce_sum(valid_prediction,
name='num_pos_prediction')
pos_prediction_corr = tf.count_nonzero(tf.logical_and(
valid_label_prob > th,
tf.equal(valid_prediction, valid_anchor_labels)),
dtype=tf.int32)
placeholder = 0.5 # A small value will make summaries appear lower.
recall = tf.to_float(tf.truediv(pos_prediction_corr, nr_pos))
recall = tf.where(tf.equal(nr_pos, 0),
placeholder,
recall,
name='recall_th{}'.format(th))
precision = tf.to_float(
tf.truediv(pos_prediction_corr, nr_pos_prediction))
precision = tf.where(tf.equal(nr_pos_prediction, 0),
placeholder,
precision,
name='precision_th{}'.format(th))
summaries.extend([precision, recall])
add_moving_summary(*summaries)
# Per-level loss summaries in FPN may appear lower due to the use of a small placeholder.
# But the total RPN loss will be fine. TODO make the summary op smarter
placeholder = 0.
label_loss = tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.to_float(valid_anchor_labels), logits=valid_label_logits)
label_loss = tf.reduce_sum(label_loss) * (1. / cfg.RPN.BATCH_PER_IM)
label_loss = tf.where(tf.equal(nr_valid, 0),
placeholder,
label_loss,
name='label_loss')
pos_anchor_boxes = tf.boolean_mask(anchor_boxes, pos_mask)
pos_box_logits = tf.boolean_mask(box_logits, pos_mask)
delta = 1.0 / 9
box_loss = tf.losses.huber_loss(pos_anchor_boxes,
pos_box_logits,
delta=delta,
reduction=tf.losses.Reduction.SUM) / delta
box_loss = box_loss * (1. / cfg.RPN.BATCH_PER_IM)
box_loss = tf.where(tf.equal(nr_pos, 0),
placeholder,
box_loss,
name='box_loss')
add_moving_summary(label_loss, box_loss, nr_valid, nr_pos)
return label_loss, box_loss
def main():
"""Create the model and start the training."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
#tf.set_random_seed(args.random_seed)
coord = tf.train.Coordinator()
with tf.Graph().as_default(), tf.device('/cpu:0'):
# Using Poly learning rate policy
base_lr = tf.constant(args.learning_rate)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.train.exponential_decay(base_lr,
step_ph,
20000,
0.5,
staircase=True)
tf.summary.scalar('lr', learning_rate)
opt = tf.train.MomentumOptimizer(learning_rate, 0.9)
#opt = tf.train.RMSPropOptimizer(learning_rate, 0.9, momentum=0.9, epsilon=1e-10)
#opt = tf.train.AdamOptimizer(learning_rate)
losses = []
train_op = []
total_batch_size = args.batch_size * args.gpu_nums
with tf.name_scope('DeepLabResNetModel') as scope:
with tf.name_scope("create_inputs"):
reader = ImageReader(args.data_dir, args.data_list, input_size,
args.random_blur, args.random_scale,
args.random_mirror, args.random_rotate,
args.ignore_label, IMG_MEAN, coord)
image_batch, label_batch = reader.dequeue(total_batch_size)
images_splits = tf.split(axis=0,
num_or_size_splits=args.gpu_nums,
value=image_batch)
labels_splits = tf.split(axis=0,
num_or_size_splits=args.gpu_nums,
value=label_batch)
net = DeepLabResNetModel({'data': images_splits},
is_training=True,
num_classes=args.num_classes)
raw_output_list = net.layers['fc_voc12']
num_valide_pixel = 0
for i in range(len(raw_output_list)):
with tf.device('/gpu:%d' % i):
raw_output_up = tf.image.resize_bilinear(
raw_output_list[i],
size=input_size,
align_corners=True)
tf.summary.image('images_{}'.format(i),
images_splits[i] + IMG_MEAN,
max_outputs=4)
tf.summary.image('labels_{}'.format(i),
labels_splits[i],
max_outputs=4)
tf.summary.image('predict_{}'.format(i),
tf.cast(
tf.expand_dims(
tf.argmax(raw_output_up, -1), 3),
tf.float32),
max_outputs=4)
all_trainable = [v for v in tf.trainable_variables()]
# Predictions: ignoring all predictions with labels greater or equal than n_classes
raw_prediction = tf.reshape(raw_output_up,
[-1, args.num_classes])
label_proc = prepare_label(
labels_splits[i],
tf.stack(raw_output_up.get_shape()[1:3]),
num_classes=args.num_classes,
one_hot=False) # [batch_size, h, w]
raw_gt = tf.reshape(label_proc, [
-1,
])
#indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1)
indices = tf.where(
tf.logical_and(tf.less(raw_gt, args.num_classes),
tf.greater_equal(raw_gt, 0)))
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
prediction = tf.gather(raw_prediction, indices)
mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(
tf.argmax(tf.nn.softmax(prediction), axis=-1),
gt,
num_classes=args.num_classes)
tf.summary.scalar('mean IoU_{}'.format(i), mIoU)
train_op.append(update_op)
# Pixel-wise softmax loss.
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=prediction, labels=gt)
num_valide_pixel += tf.shape(gt)[0]
losses.append(tf.reduce_sum(loss))
l2_losses = [
args.weight_decay * tf.nn.l2_loss(v)
for v in tf.trainable_variables() if 'weights' in v.name
]
reduced_loss = tf.truediv(
tf.reduce_sum(losses), tf.cast(
num_valide_pixel, tf.float32)) + tf.add_n(l2_losses)
tf.summary.scalar('average_loss', reduced_loss)
grads = tf.gradients(reduced_loss,
all_trainable,
colocate_gradients_with_ops=True)
variable_averages = tf.train.ExponentialMovingAverage(0.99, step_ph)
variables_to_average = (tf.trainable_variables() +
tf.moving_average_variables())
variables_averages_op = variable_averages.apply(variables_to_average)
train_op = tf.group(opt.apply_gradients(zip(grads, all_trainable)),
*train_op)
train_op = tf.group(train_op, variables_averages_op)
summary_op = tf.summary.merge_all()
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.allow_soft_placement = True
sess = tf.Session(config=config)
init = [
tf.global_variables_initializer(),
tf.local_variables_initializer()
]
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=2)
#restore from resnet imagenet, bised and local_step is in moving_average
restore_var = [
v for v in tf.trainable_variables() if 'fc' not in v.name
] + [
v for v in tf.global_variables()
if ('moving_mean' in v.name or 'moving_variance' in v.name) and
('biased' not in v.name and 'local_step' not in v.name)
]
ckpt = tf.train.get_checkpoint_state(args.restore_from)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found.')
"""
#restore from snapshot
restore_var = tf.global_variables()
ckpt = tf.train.get_checkpoint_state(args.snapshot_dir)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=restore_var, allow_empty=True)
load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found.')
load_step = 0
"""
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
summary_writer = tf.summary.FileWriter(args.snapshot_dir,
graph=sess.graph)
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
feed_dict = {step_ph: step}
if step % args.save_pred_every == 0 and step != 0:
loss_value, _ = sess.run([reduced_loss, train_op],
feed_dict=feed_dict)
save(saver, sess, args.snapshot_dir, step)
elif step % 100 == 0:
summary_str, loss_value, _, IOU = sess.run(
[summary_op, reduced_loss, train_op, mIoU],
feed_dict=feed_dict)
duration = time.time() - start_time
summary_writer.add_summary(summary_str, step)
print(
'step {:d} \t loss = {:.3f}, mean_IoU = {:.3f}, ({:.3f} sec/step)'
.format(step, loss_value, IOU, duration))
else:
loss_value, _ = sess.run([reduced_loss, train_op],
feed_dict=feed_dict)
coord.request_stop()
coord.join(threads)
# Michael, in your previous code, it was different and it is being fixed by you in the lambda layer
# I not very sure about the size of matrix in your new code R_Q_D_p = merge([query_sem, pos_doc_sem], mode = "cos")
pos_logits = tf.reduce_sum(cosine(query_sem, pos_doc_sem)) # 3.02
neg_logits = [
tf.reduce_sum(cosine(query_sem, neg_docs_sem[i]))
for i in range(n_docs)
] # [1.01 0.04 0.35] since, n_docs = 3
# Here in logits, I have simply converted the cosine matrix of size [B1, B2] to a single number [1]
neg_exp = [tf.exp(neg_logits[i]) for i in range(n_docs)]
pos_exp = tf.exp(pos_logits)
neg_summation = tf.add_n(neg_exp)
total_summation = tf.add(neg_summation, pos_exp)
prob_Dplus_given_Q = tf.truediv(pos_exp, total_summation)
# prob_Dplus_given_Q give the softmax without the gamma term included in it
loss = -tf.log(prob_Dplus_given_Q)
# here is where I am stuck!! I have a query and a pos_doc and 3 neg_docs in this graph in each step. How am I supposed to find the loss ?
'''training'''
global_step = tf.Variable(0, name="global_step", trainable=False)
decay_step = 1
decay_rate = 0.9
learning_r = tf.train.exponential_decay(learning_rate, global_step,
decay_step, decay_rate)
opt = tf.train.AdamOptimizer(learning_r)
train_step = opt.minimize(loss, global_step=global_step)
def rnn_model():
"""
Build the multi-layer RNN model for the speed prediction task.
Description:
Build the multi-layer RNN model based on different input cell_types.
Choosable options are LSTM and GRU.
batch_size, learning_rate and dropout_keep_prob are set as
placeholders, so different values could be input during training and
testing stages.
- batch_size: In this script, the training batch of 1 and testing
batch of all test samples are used.
- learning_rate: Due to the big change of error value during
training, adaptive learning_rate is applied. The initial
learning rate is set to be 1e-3. When the test error is less
than 15%, reduce the error_rate to 1e-4.
- dropout_keep_prob: dropout could be applied during training, but
it must be 1 during testing.
"""
# x: [batch_size, num_steps, num_inputs]
# y: [batch_size, num_classes]
x = tf.placeholder(tf.float32, [None, num_steps, num_inputs], name='X')
y = tf.placeholder(tf.float32, [None, num_classes], name='Y')
batch_size = tf.placeholder(tf.int32, [], name='batch_size')
learning_rate = tf.placeholder(tf.float32, [], name='learning_rate')
dropout_keep_prob = tf.placeholder(tf.float32, [],
name='dropout_keep_prob')
# x_fc_input: [batch_size * num_steps, num_inputs]
x_fc_input = tf.reshape(x, [-1, num_inputs])
# fc_output_0: [batch_size * num_steps, state_size]
fc_output_0 = fc_layer(x_fc_input, num_inputs, state_size, "fc_0")
# x_rnn_input: [batch_size, num_steps, state_size]
x_rnn_input = tf.reshape(fc_output_0, [-1, num_steps, state_size])
if cell_type == "LSTM":
cell = tf.nn.rnn_cell.LSTMCell(state_size, state_is_tuple=True)
cell = tf.nn.rnn_cell.DropoutWrapper(cell,
input_keep_prob=dropout_keep_prob)
elif cell_type == "GRU":
cell = tf.nn.rnn_cell.GRUCell(state_size)
cell = tf.nn.rnn_cell.DropoutWrapper(cell,
input_keep_prob=dropout_keep_prob)
multi_cell = tf.nn.rnn_cell.MultiRNNCell([cell] * num_layers,
state_is_tuple=True)
init_state = multi_cell.zero_state(batch_size, dtype=tf.float32)
# rnn_outputs: [batch_size, num_steps, state_size].
# final_state: [batch_size, hidden_layers * state_size].
rnn_outputs, final_state = tf.nn.dynamic_rnn(multi_cell,
x_rnn_input,
initial_state=init_state)
# Unstack the rnn_outputs to list [num_steps * [batch_size, state_size]].
# Get the last element in the list, that is the last output.
# last_output [batch_size, state_size].
last_output = tf.unstack(rnn_outputs, axis=1)[-1]
fc_output_1 = fc_layer(last_output, state_size, 10, "fc_1")
fc_output_2 = output_layer(fc_output_1, 10, num_classes, "fc_2")
logits = tf.identity(fc_output_2, name="logits")
with tf.name_scope("cost"):
cost = tf.nn.l2_loss(tf.subtract(logits, y))
tf.summary.scalar("cost", cost)
with tf.name_scope("train"):
train_step = tf.train.AdamOptimizer(learning_rate).minimize(cost)
with tf.name_scope("error"):
error = tf.reduce_mean(abs(tf.truediv(logits, y) - 1), name="error")
tf.summary.scalar("error", error)
saver = tf.train.Saver()
return dict(x=x,
y=y,
cost=cost,
saver=saver,
error=error,
logits=logits,
batch_size=batch_size,
train_step=train_step,
learning_rate=learning_rate,
dropout_keep_prob=dropout_keep_prob)
def define_loss(x, y, g_list, weights, biases, params, phase, keep_prob):
"""Define the (unregularized) loss functions for the training.
Arguments:
x -- placeholder for input
y -- list of outputs of network for each shift (each prediction step)
g_list -- list of output of encoder for each shift (encoding each step in x)
weights -- dictionary of weights for all networks
biases -- dictionary of biases for all networks
params -- dictionary of parameters for experiment
phase -- boolean placeholder for dropout: training phase or not training phase
keep_prob -- probability that weight is kept during dropout
Returns:
loss1 -- autoencoder loss function
loss2 -- dynamics/prediction loss function
loss3 -- linearity loss function
loss_Linf -- inf norm on autoencoder loss and one-step prediction loss
loss -- sum of above four losses
Side effects:
None
"""
# Minimize the mean squared errors.
# subtraction and squaring element-wise, then average over both dimensions
# n columns
# average of each row (across columns), then average the rows
denominator_nonzero = 10**(-5)
# autoencoder loss
if params['relative_loss']:
loss1_denominator = tf.reduce_mean(
tf.reduce_mean(tf.square(tf.squeeze(x[0, :, :])),
1)) + denominator_nonzero
else:
loss1_denominator = tf.to_double(1.0)
mean_squared_error = tf.reduce_mean(
tf.reduce_mean(tf.square(y[0] - tf.squeeze(x[0, :, :])), 1))
loss1 = params['recon_lam'] * tf.truediv(mean_squared_error,
loss1_denominator)
# gets dynamics/prediction
loss2 = tf.zeros([
1,
], dtype=tf.float64)
if params['num_shifts'] > 0:
for j in np.arange(params['num_shifts']):
# xk+1, xk+2, xk+3
shift = params['shifts'][j]
if params['relative_loss']:
loss2_denominator = tf.reduce_mean(
tf.reduce_mean(tf.square(tf.squeeze(x[shift, :, :])),
1)) + denominator_nonzero
else:
loss2_denominator = tf.to_double(1.0)
loss2 = loss2 + params['recon_lam'] * tf.truediv(
tf.reduce_mean(
tf.reduce_mean(
tf.square(y[j + 1] - tf.squeeze(x[shift, :, :])), 1)),
loss2_denominator)
loss2 = loss2 / params['num_shifts']
# K linear
loss3 = tf.zeros([
1,
], dtype=tf.float64)
count_shifts_middle = 0
if params['num_shifts_middle'] > 0:
# generalization of: next_step = tf.matmul(g_list[0], L_pow)
omegas = net.omega_net_apply(phase, keep_prob, params, g_list[0],
weights, biases)
next_step = net.varying_multiply(g_list[0], omegas, params['delta_t'],
params['num_real'],
params['num_complex_pairs'])
# multiply g_list[0] by L (j+1) times
for j in np.arange(max(params['shifts_middle'])):
if (j + 1) in params['shifts_middle']:
if params['relative_loss']:
loss3_denominator = tf.reduce_mean(
tf.reduce_mean(
tf.square(
tf.squeeze(g_list[count_shifts_middle + 1])),
1)) + denominator_nonzero
else:
loss3_denominator = tf.to_double(1.0)
loss3 = loss3 + params['mid_shift_lam'] * tf.truediv(
tf.reduce_mean(
tf.reduce_mean(
tf.square(next_step -
g_list[count_shifts_middle + 1]), 1)),
loss3_denominator)
count_shifts_middle += 1
omegas = net.omega_net_apply(phase, keep_prob, params, next_step,
weights, biases)
next_step = net.varying_multiply(next_step, omegas,
params['delta_t'],
params['num_real'],
params['num_complex_pairs'])
loss3 = loss3 / params['num_shifts_middle']
# inf norm on autoencoder error and one prediction step
if params['relative_loss']:
Linf1_den = tf.norm(tf.norm(tf.squeeze(x[0, :, :]), axis=1,
ord=np.inf),
ord=np.inf) + denominator_nonzero
Linf2_den = tf.norm(tf.norm(tf.squeeze(x[1, :, :]), axis=1,
ord=np.inf),
ord=np.inf) + denominator_nonzero
else:
Linf1_den = tf.to_double(1.0)
Linf2_den = tf.to_double(1.0)
Linf1_penalty = tf.truediv(
tf.norm(tf.norm(y[0] - tf.squeeze(x[0, :, :]), axis=1, ord=np.inf),
ord=np.inf), Linf1_den)
Linf2_penalty = tf.truediv(
tf.norm(tf.norm(y[1] - tf.squeeze(x[1, :, :]), axis=1, ord=np.inf),
ord=np.inf), Linf2_den)
loss_Linf = params['Linf_lam'] * (Linf1_penalty + Linf2_penalty)
loss = loss1 + loss2 + loss3 + loss_Linf
return loss1, loss2, loss3, loss_Linf, loss
def call(self, x):
input_image, y_pred, y_true, true_boxes = x
# adjust the shape of the y_predict [batch, grid_h, grid_w, 3, 4+1+nb_class]
y_pred = tf.reshape(
y_pred,
tf.concat([tf.shape(y_pred)[:3],
tf.constant([3, -1])], axis=0))
# initialize the masks
object_mask = tf.expand_dims(y_true[..., 4], 4)
# the variable to keep track of number of batches processed
batch_seen = tf.Variable(0.)
# compute grid factor and net factor
grid_h = tf.shape(y_true)[1]
grid_w = tf.shape(y_true)[2]
grid_factor = tf.reshape(tf.cast([grid_w, grid_h], tf.float32),
[1, 1, 1, 1, 2])
net_h = tf.shape(input_image)[1]
net_w = tf.shape(input_image)[2]
net_factor = tf.reshape(tf.cast([net_w, net_h], tf.float32),
[1, 1, 1, 1, 2])
"""
Adjust prediction
"""
pred_box_xy = (self.cell_grid[:, :grid_h, :grid_w, :, :] +
tf.sigmoid(y_pred[..., :2])) # sigma(t_xy) + c_xy
pred_box_wh = y_pred[..., 2:4] # t_wh
pred_box_conf = tf.expand_dims(tf.sigmoid(y_pred[..., 4]),
4) # adjust confidence
pred_box_class = y_pred[..., 5:] # adjust class probabilities
"""
Adjust ground truth
"""
true_box_xy = y_true[..., 0:2] # (sigma(t_xy) + c_xy)
true_box_wh = y_true[..., 2:4] # t_wh
true_box_conf = tf.expand_dims(y_true[..., 4], 4)
true_box_class = tf.argmax(y_true[..., 5:], -1)
"""
Compare each predicted box to all true boxes
"""
# initially, drag all objectness of all boxes to 0
conf_delta = pred_box_conf - 0
# then, ignore the boxes which have good overlap with some true box
true_xy = true_boxes[..., 0:2] / grid_factor
true_wh = true_boxes[..., 2:4] / net_factor
true_wh_half = true_wh / 2.
true_mins = true_xy - true_wh_half
true_maxes = true_xy + true_wh_half
pred_xy = tf.expand_dims(pred_box_xy / grid_factor, 4)
pred_wh = tf.expand_dims(
tf.exp(pred_box_wh) * self.anchors / net_factor, 4)
pred_wh_half = pred_wh / 2.
pred_mins = pred_xy - pred_wh_half
pred_maxes = pred_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_wh[..., 0] * true_wh[..., 1]
pred_areas = pred_wh[..., 0] * pred_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
best_ious = tf.reduce_max(iou_scores, axis=4)
conf_delta *= tf.expand_dims(
tf.to_float(best_ious < self.ignore_thresh), 4)
"""
Compute some online statistics
"""
true_xy = true_box_xy / grid_factor
true_wh = tf.exp(true_box_wh) * self.anchors / net_factor
true_wh_half = true_wh / 2.
true_mins = true_xy - true_wh_half
true_maxes = true_xy + true_wh_half
pred_xy = pred_box_xy / grid_factor
pred_wh = tf.exp(pred_box_wh) * self.anchors / net_factor
pred_wh_half = pred_wh / 2.
pred_mins = pred_xy - pred_wh_half
pred_maxes = pred_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_wh[..., 0] * true_wh[..., 1]
pred_areas = pred_wh[..., 0] * pred_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
iou_scores = object_mask * tf.expand_dims(iou_scores, 4)
count = tf.reduce_sum(object_mask)
count_noobj = tf.reduce_sum(1 - object_mask)
detect_mask = tf.to_float((pred_box_conf * object_mask) >= 0.5)
class_mask = tf.expand_dims(
tf.to_float(tf.equal(tf.argmax(pred_box_class, -1),
true_box_class)), 4)
recall50 = tf.reduce_sum(
tf.to_float(iou_scores >= 0.5) * detect_mask *
class_mask) / (count + 1e-3)
recall75 = tf.reduce_sum(
tf.to_float(iou_scores >= 0.75) * detect_mask *
class_mask) / (count + 1e-3)
avg_iou = tf.reduce_sum(iou_scores) / (count + 1e-3)
avg_obj = tf.reduce_sum(pred_box_conf * object_mask) / (count + 1e-3)
avg_noobj = tf.reduce_sum(pred_box_conf *
(1 - object_mask)) / (count_noobj + 1e-3)
avg_cat = tf.reduce_sum(object_mask * class_mask) / (count + 1e-3)
"""
Warm-up training
"""
batch_seen = tf.assign_add(batch_seen, 1.)
true_box_xy, true_box_wh, xywh_mask = tf.cond(
tf.less(batch_seen, self.warmup_batches + 1), lambda: [
true_box_xy +
(0.5 + self.cell_grid[:, :grid_h, :grid_w, :, :]) *
(1 - object_mask), true_box_wh + tf.zeros_like(true_box_wh) *
(1 - object_mask),
tf.ones_like(object_mask)
], lambda: [true_box_xy, true_box_wh, object_mask])
"""
Compare each true box to all anchor boxes
"""
wh_scale = tf.exp(true_box_wh) * self.anchors / net_factor
wh_scale = tf.expand_dims(
2 - wh_scale[..., 0] * wh_scale[..., 1],
axis=4) # the smaller the box, the bigger the scale
xy_delta = xywh_mask * (pred_box_xy -
true_box_xy) * wh_scale * self.xywh_scale
wh_delta = xywh_mask * (pred_box_wh -
true_box_wh) * wh_scale * self.xywh_scale
conf_delta = object_mask * (
pred_box_conf - true_box_conf) * self.obj_scale + (
1 - object_mask) * conf_delta * self.noobj_scale
class_delta = object_mask * \
tf.expand_dims(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=true_box_class, logits=pred_box_class), 4) * \
self.class_scale
loss_xy = tf.reduce_sum(tf.square(xy_delta), list(range(1, 5)))
loss_wh = tf.reduce_sum(tf.square(wh_delta), list(range(1, 5)))
loss_conf = tf.reduce_sum(tf.square(conf_delta), list(range(1, 5)))
loss_class = tf.reduce_sum(class_delta, list(range(1, 5)))
loss = loss_xy + loss_wh + loss_conf + loss_class
if debug:
loss = tf.Print(loss, [grid_h, avg_obj],
message='avg_obj \t\t',
summarize=1000)
loss = tf.Print(loss, [grid_h, avg_noobj],
message='avg_noobj \t\t',
summarize=1000)
loss = tf.Print(loss, [grid_h, avg_iou],
message='avg_iou \t\t',
summarize=1000)
loss = tf.Print(loss, [grid_h, avg_cat],
message='avg_cat \t\t',
summarize=1000)
loss = tf.Print(loss, [grid_h, recall50],
message='recall50 \t',
summarize=1000)
loss = tf.Print(loss, [grid_h, recall75],
message='recall75 \t',
summarize=1000)
loss = tf.Print(loss, [grid_h, count],
message='count \t',
summarize=1000)
loss = tf.Print(loss, [
grid_h,
tf.reduce_sum(loss_xy),
tf.reduce_sum(loss_wh),
tf.reduce_sum(loss_conf),
tf.reduce_sum(loss_class)
],
message='loss xy, wh, conf, class: \t',
summarize=1000)
return loss * self.grid_scale
def avg_rec(y_true, y_pred):
mask_shape = tf.shape(y_true)[:4]
cell_x = tf.to_float(tf.reshape(tf.tile(tf.range(GRID_W), [GRID_H]), (1, GRID_H, GRID_W, 1, 1)))
cell_y = tf.transpose(cell_x, (0,2,1,3,4))
cell_grid = tf.tile(tf.concat([cell_x,cell_y], -1), [BATCH_SIZE, 1, 1, 5, 1])
coord_mask = tf.zeros(mask_shape)
conf_mask = tf.zeros(mask_shape)
class_mask = tf.zeros(mask_shape)
seen = tf.Variable(0.)
total_recall = tf.Variable(0.)
"""
Adjust prediction
"""
### adjust x and y
pred_box_xy = tf.sigmoid(y_pred[..., :2]) + cell_grid
### adjust w and h
pred_box_wh = tf.exp(y_pred[..., 2:4]) * np.reshape(ANCHORS, [1,1,1,BOX,2])
### adjust confidence
pred_box_conf = tf.sigmoid(y_pred[..., 4])
### adjust class probabilities
pred_box_class = y_pred[..., 5:]
"""
Adjust ground truth
"""
### adjust x and y
true_box_xy = y_true[..., 0:2] # relative position to the containing cell
### adjust w and h
true_box_wh = y_true[..., 2:4] # number of cells accross, horizontally and vertically
### adjust confidence
true_wh_half = true_box_wh / 2.
true_mins = true_box_xy - true_wh_half
true_maxes = true_box_xy + true_wh_half
pred_wh_half = pred_box_wh / 2.
pred_mins = pred_box_xy - pred_wh_half
pred_maxes = pred_box_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_box_wh[..., 0] * true_box_wh[..., 1]
pred_areas = pred_box_wh[..., 0] * pred_box_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
true_box_conf = iou_scores * y_true[..., 4]
### adjust class probabilities
true_box_class = tf.argmax(y_true[..., 5:], -1)
"""
Determine the masks
"""
### coordinate mask: simply the position of the ground truth boxes (the predictors)
coord_mask = tf.expand_dims(y_true[..., 4], axis=-1) * COORD_SCALE
### confidence mask: penelize predictors + penalize boxes with low IOU
# penalize the confidence of the boxes, which have IOU with some ground truth box < 0.6
true_xy = true_boxes[..., 0:2]
true_wh = true_boxes[..., 2:4]
true_wh_half = true_wh / 2.
true_mins = true_xy - true_wh_half
true_maxes = true_xy + true_wh_half
pred_xy = tf.expand_dims(pred_box_xy, 4)
pred_wh = tf.expand_dims(pred_box_wh, 4)
pred_wh_half = pred_wh / 2.
pred_mins = pred_xy - pred_wh_half
pred_maxes = pred_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_wh[..., 0] * true_wh[..., 1]
pred_areas = pred_wh[..., 0] * pred_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
best_ious = tf.reduce_max(iou_scores, axis=4)
conf_mask = conf_mask + tf.to_float(best_ious < 0.6) * (1 - y_true[..., 4]) * NO_OBJECT_SCALE
# penalize the confidence of the boxes, which are reponsible for corresponding ground truth box
conf_mask = conf_mask + y_true[..., 4] * OBJECT_SCALE
### class mask: simply the position of the ground truth boxes (the predictors)
class_mask = y_true[..., 4] * tf.gather(CLASS_WEIGHTS, true_box_class) * CLASS_SCALE
"""
Warm-up training
"""
no_boxes_mask = tf.to_float(coord_mask < COORD_SCALE/2.)
seen = tf.assign_add(seen, 1.)
true_box_xy, true_box_wh, coord_mask = tf.cond(tf.less(seen, WARM_UP_BATCHES),
lambda: [true_box_xy + (0.5 + cell_grid) * no_boxes_mask,
true_box_wh + tf.ones_like(true_box_wh) * np.reshape(ANCHORS, [1,1,1,BOX,2]) * no_boxes_mask,
tf.ones_like(coord_mask)],
lambda: [true_box_xy,
true_box_wh,
coord_mask])
"""
Finalize the loss
"""
nb_coord_box = tf.reduce_sum(tf.to_float(coord_mask > 0.0))
nb_conf_box = tf.reduce_sum(tf.to_float(conf_mask > 0.0))
nb_class_box = tf.reduce_sum(tf.to_float(class_mask > 0.0))
loss_xy = tf.reduce_sum(tf.square(true_box_xy-pred_box_xy) * coord_mask) / (nb_coord_box + 1e-6) / 2.
loss_wh = tf.reduce_sum(tf.square(true_box_wh-pred_box_wh) * coord_mask) / (nb_coord_box + 1e-6) / 2.
loss_conf = tf.reduce_sum(tf.square(true_box_conf-pred_box_conf) * conf_mask) / (nb_conf_box + 1e-6) / 2.
loss_class = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=true_box_class, logits=pred_box_class)
loss_class = tf.reduce_sum(loss_class * class_mask) / (nb_class_box + 1e-6)
loss = loss_xy + loss_wh + loss_conf + loss_class
nb_true_box = tf.reduce_sum(y_true[..., 4])
nb_pred_box = tf.reduce_sum(tf.to_float(true_box_conf > 0.5) * tf.to_float(pred_box_conf > 0.3))
"""
Debugging code
"""
current_recall = nb_pred_box/(nb_true_box + 1e-6)
total_recall = tf.assign_add(total_recall, current_recall)
return total_recall/seen
def Optimizer(self, lr, dError_dy):
train_list = []
a = []
var_list = []
self.classifier['real_cost'] = []
self.classifier['updates'] = []
for i in xrange(len(self.Layer_cost)):
cost, weight, bias, fan_in, batch_size, layer_in, dlayer,\
backward = self.Layer_cost[i]
cost_clamp, weight_clamp, bias_clamp = self.Layer_cost_Clamp[i]
self.classifier['real_cost'].append(
tf.nn.l2_loss(cost - cost_clamp))
if i < (len(self.Layer_cost) - 1):
temp_mul = tf.matmul(backward, dlayer)
# temp_mul = backward
dError_dhidden = tf.matmul(dError_dy, temp_mul)
reshaped_layer_in = tf.reshape(layer_in,
[batch_size, fan_in, 1])
upd_weight = tf.reduce_mean(
tf.matmul(reshaped_layer_in, dError_dhidden), 0)
upd_bias = tf.reduce_mean(dError_dhidden, 0)
#### Gradient Descent update
dir_weight_update = (
(tf.gradients(self.classifier['real_cost'][i], weight)[0])
- (tf.gradients(self.classifier['real_cost'][i],
weight_clamp)[0]))
self.classifier['updates'].append(
tf.nn.l2_loss(dir_weight_update))
dir_weight_update = tf.truediv(
dir_weight_update, (tf.norm(dir_weight_update) + 0.001))
dir_bias_update = ((tf.gradients(self.classifier['real_cost'][i], bias)[0])\
- (tf.gradients(self.classifier['real_cost'][i], bias_clamp)[0]))
dir_bias_update = tf.truediv(
dir_bias_update, (tf.norm(dir_bias_update) + 0.001))
weight_update = tf.add(
(upd_weight), 0.01 * dir_weight_update) + 0.0001 * weight
bias_update = tf.add(
(upd_bias[0]), 0.01 * dir_bias_update) + 0.0001 * bias
# weight_update = upd_weight + 0.0001*weight
# bias_update = upd_bias[0] + 0.0001*bias
self.classifier['updates'].append(tf.nn.l2_loss(weight_update))
else:
reshaped_layer_in = tf.reshape(layer_in,
[batch_size, fan_in, 1])
dError_dhidden = tf.matmul(dError_dy, dlayer)
upd_weight = tf.reduce_mean(
tf.matmul(reshaped_layer_in, dError_dhidden), 0)
upd_bias = tf.reduce_mean(dError_dy, 0)
weight_update = upd_weight + 0.0001 * weight
bias_update = upd_bias[0] + 0.0001 * bias
self.classifier['updates'].append(tf.nn.l2_loss(weight_update))
# Generate the updated variables
a.append(weight_update)
a.append(bias_update)
train_list.append(
(tf.placeholder("float32", shape=weight_update.get_shape())))
train_list.append((tf.placeholder("float32",
shape=bias_update.get_shape())))
var_list.append(weight)
var_list.append(bias)
return a, train_list, tf.train.AdamOptimizer(lr).\
apply_gradients([(e, var_list[i]) for i, e in enumerate(train_list)])
def create_ema_op():
with self.cached_name_scope():
avg_size = tf.truediv(tf.add_n(self._size_ops), len(self._size_ops), name='avg_stagingarea_size')
return add_moving_summary(avg_size, collection=None)[0].op
def _create_model(self, mode, input_ids, input_mask, segment_ids, labels,
labels_mask):
"""Creates a LaserTagger model."""
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
model = modeling.BertModel(
config=self._config,
is_training=is_training,
input_ids=input_ids,
input_mask=input_mask,
token_type_ids=segment_ids,
use_one_hot_embeddings=self._use_one_hot_embeddings)
final_hidden = model.get_sequence_output()
if self._config.use_t2t_decoder:
# Size of the output vocabulary which contains the tags + begin and end
# tokens used by the Transformer decoder.
output_vocab_size = self._num_tags + 2 # TODO 为什么加２，begin and end
params = _get_decoder_params(self._config, self._use_tpu,
self._max_seq_length,
output_vocab_size)
decoder = transformer_decoder.TransformerDecoder(
params, is_training)
logits = decoder(input_mask, final_hidden,
labels) # labels is the id of operation
else:
if is_training:
# I.e., 0.1 dropout
final_hidden = tf.nn.dropout(final_hidden, keep_prob=0.9)
logits = tf.layers.dense(
final_hidden,
self._num_tags,
kernel_initializer=tf.truncated_normal_initializer(
stddev=0.02),
name="output_projection")
with tf.variable_scope("loss"):
loss = None
per_example_loss = None
outputs_outputs = tf.constant(100) # TODO
print("mode:", mode)
if mode != tf.estimator.ModeKeys.PREDICT:
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=labels, logits=logits)
per_example_loss = tf.truediv(
tf.reduce_sum(loss, axis=1),
tf.cast(tf.reduce_sum(labels_mask, axis=1), tf.float32))
loss = tf.reduce_mean(per_example_loss)
pred = tf.cast(tf.argmax(logits, axis=-1), tf.int32)
else:
if self._config.use_t2t_decoder:
# pred = logits["outputs"]
# # Transformer decoder reserves the first two IDs to the begin and the
# # end token so we shift the IDs back.
# pred -= 2
outputs_outputs = logits["outputs"]
pred = outputs_outputs - 2
else:
pred = tf.cast(tf.argmax(logits, axis=-1), tf.int32)
return (loss, per_example_loss, pred, outputs_outputs)
import tensorflow as tf
import numpy as np
# 计算图的其他操作
sess = tf.Session()
#div()函数返回值的数据类型与输入数据类型一致
print(sess.run(tf.div(3, 4)))
#truediv()函数会将计算结果强制转换为浮点数类型
print(sess.run(tf.truediv(3, 4)))
#floordiv()函数，会将计算结果向下取整
print(sess.run(tf.floordiv(5, 4)))
#mod()取模运算，返回除法的余数
print(sess.run(tf.mod(22.0, 5.0)))
#通过cross()函数计算两个张量级的点积。只为三维向量定义。
print(sess.run(tf.cross([1., 0., 0.], [0., 1., 0.])))
# 组合预处理函数生成自定义函数
print(sess.run(tf.div(tf.sin(3.1416 / 4.), tf.cos(3.1416 / 4.))))
# 延伸学习
#创建一个自定义二次多项式函数，3x^2-x+10
def custom_polynomial(value):
return (tf.subtract(3 * tf.square(value), value) + 10)
doc_y,
tf.slice(temp, [rand, 0], [BS - rand, -1]),
tf.slice(temp, [0, 0], [rand, -1])
], 0)
with tf.name_scope('Cosine_Similarity'):
# Cosine similarity
query_norm = tf.tile(tf.sqrt(tf.reduce_sum(tf.square(query_y), 1, True)),
[NEG + 1, 1])
doc_norm = tf.sqrt(tf.reduce_sum(tf.square(doc_y), 1, True))
prod = tf.reduce_sum(tf.multiply(tf.tile(query_y, [NEG + 1, 1]), doc_y), 1,
True)
norm_prod = tf.multiply(query_norm, doc_norm)
cos_sim_raw = tf.truediv(prod, norm_prod)
cos_sim = tf.transpose(tf.reshape(tf.transpose(cos_sim_raw),
[NEG + 1, BS])) * 20
with tf.name_scope('Loss'):
# Train Loss
# prob BS * 51 matrix
# prob = tf.nn.softmax((cos_sim))
# y BS * 51 matrix too
label_value = np.array([[1] + [0] * NEG] * BS)
loss = tf.losses.softmax_cross_entropy(onehot_labels=label_value,
logits=cos_sim +
np.finfo(np.float32).eps)
tf.summary.scalar('loss', loss)
with tf.name_scope('Evaluate'):
total_series_length = len(k)
num_batches = total_series_length // batch_size // truncated_backprop_length
k_series[n] = tf.unstack(batchk_placeholder, axis=1)
t_series[n] = tf.unstack(batcht_placeholder, axis=1)
# Forward pass
current_h = init_h
for i in range(len(k_series)):
current_k = k_series[n][i]
current_t = t_series[n][i]
numerator = -omega * current_t
denominator = current_k
frac = tf.truediv(numerator, denominator)
kernel = tf.exp(frac)
next_h = tf.exp((alpha * current_h + beta * kernel + gamma))
h_series[n].append(next_h)
current_h = next_h
G_loss[n] = tf.reduce_sum(
tf.exp(a[n] * h_series[n][-1] + b[n] * t_series[n][-1] + c[n]) / b[n] -
tf.exp(a[n] * h_series[n][-1] + b[n] * t_series[n][0] + c[n]) /
b[n]) - sum([
tf.reduce_sum(a[n] * h_series[n][-1] + b[n] *
(t_series[n][k1 + 1] - t_series[n][k1]) + c[n])
for k1 in range(len(t_series) - 1)
])
regularizer = tf.add_n([
def QueueTime_loss(y_true,
y_pred): # should be a BS * CELL_ROW * CELL_COL * 5 tensor
# each one of them should now be batch*10*10*5
print("[INFO] ytrue", y_true)
print("[INFO] ypred", y_pred)
y_true = K.reshape(y_true, [-1, 10, 10, 5])
y_pred = K.reshape(y_pred, [-1, 10, 10, 5])
print("[INFO] ytrue", y_true)
print("[INFO] ypred", y_pred)
coord = 3
noobj = 0.1
indicator = y_true[..., 0]
print("[INFO] indicator", indicator)
x_loss = K.square(y_true[..., 1] - y_pred[..., 1])
# print("[INFO] x loss", x_loss.eval())
y_loss = K.square(y_true[..., 2] - y_pred[..., 2])
# print("[INFO] y loss", y_loss.eval())
xy_loss = coord * indicator * (y_loss + x_loss)
print("[INFO] xy_loss ? 10 10 [[[1]]]", xy_loss)
w_loss = K.square(K.sqrt(y_true[..., 3]) -
K.sqrt(y_pred[..., 3])) #hard code now
h_loss = K.square(K.sqrt(y_true[..., 4]) -
K.sqrt(y_pred[..., 4])) #hard code now
wh_loss = coord * indicator * (w_loss + h_loss)
pred_box_xy = y_pred[..., 1:3]
print("[INFO] pred_box_xy ?, 10, 10, 2", pred_box_xy)
pred_box_wh = y_pred[..., 3:5]
true_box_xy = y_true[..., 1:3] # relative position to the containing cell
true_box_wh = y_true[
..., 3:5] # number of cells accross, horizontally and vertically
print("[INFO] pred_box_wh ? 10 10 2", pred_box_wh)
### adjust confidence
true_wh_half = true_box_wh / 2.
true_mins = true_box_xy - true_wh_half
true_maxes = true_box_xy + true_wh_half
print("[INFO] true_maxes ? 10 10 2", true_maxes)
pred_wh_half = pred_box_wh / 2.
pred_mins = pred_box_xy - pred_wh_half
pred_maxes = pred_box_xy + pred_wh_half
intersect_mins = K.maximum(pred_mins, true_mins)
intersect_maxes = K.minimum(pred_maxes, true_maxes)
intersect_wh = K.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
print("[INFO] intersect_areas ? 10 10", intersect_areas)
true_areas = true_box_wh[..., 0] * true_box_wh[..., 1] #may equal to 0
pred_areas = pred_box_wh[..., 0] * pred_box_wh[..., 1] #may equal to 0
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas + 1e-10)
# true_box_conf = iou_scores * y_true[..., 0]
# print("[INFO] true_box_conf ? 10 10", true_box_conf) #expect ?*10*10 here
pr_loss_pos = indicator * K.square(iou_scores * y_true[..., 0] -
y_pred[..., 0])
pr_loss_neg = noobj * (
1 - indicator) * K.square(iou_scores * y_true[..., 0] - y_pred[..., 0])
print("[INFO] pr_loss_neg ? 10 10", pr_loss_neg) #expect ?*10*10 here
# m = K.int_shape(y_true)
# print("[INFO] y_true is ", y_true, ",m is ", m, "xy_loss is", xy_loss[0])
loss = (xy_loss + wh_loss + pr_loss_neg +
pr_loss_pos) / 16.0 #hard code BS now
# fake_loss = pr_loss_neg
real_loss = K.sum(K.sum(K.sum(loss, 0), 0), 0, True)
print("[INFO] real_loss", real_loss)
return real_loss
def __rtruediv__(self, other):
return tf.truediv(other, self)
def rpn_losses(anchor_labels, anchor_boxes, label_logits, box_logits):
"""
:param anchor_labels: fHxfWxNA
:param anchor_boxes: fHxfWxNAx4, encoded
:param label_logits: fHxfWxNA
:param box_logits: fHxfWxNAx4
:return: label_loss, box_loss
"""
valid_mask = tf.not_equal(anchor_labels, -1)
pos_mask = tf.equal(anchor_labels, 1)
nr_valid = tf.count_nonzero(valid_mask, dtype=tf.int32)
nr_pos = tf.count_nonzero(pos_mask, dtype=tf.int32)
valid_anchor_labels = tf.boolean_mask(anchor_labels, valid_mask)
valid_label_logits = tf.boolean_mask(label_logits, valid_mask)
valid_label_prob = tf.nn.sigmoid(valid_label_logits)
for th in [0.5, 0.2, 0.1]:
valid_prediction = tf.cast(tf.greater(valid_label_prob, th), tf.int32)
nr_pos_prediction = tf.reduce_sum(valid_prediction,
name='num_pos_prediction')
pos_prediction_corr = tf.count_nonzero(tf.logical_and(
tf.greater(valid_label_prob, th),
tf.equal(valid_prediction, valid_anchor_labels)),
dtype=tf.int32)
recall = tf.cast(tf.truediv(pos_prediction_corr, nr_pos), tf.float32)
placeholder = 0.5
recall = tf.where(tf.equal(nr_pos, 0),
placeholder,
recall,
name="recall_th{}".format(th))
precision = tf.cast(tf.truediv(pos_prediction_corr, nr_pos_prediction),
tf.float32)
precision = tf.where(tf.equal(nr_pos_prediction, 0),
placeholder,
precision,
name='precision_th{}'.format(th))
tf.summary.scalar('precision_th{}'.format(th), precision)
tf.summary.scalar('recall_th{}'.format(th), recall)
placeholder = 0.0
label_loss = tf.losses.sigmoid_cross_entropy(
tf.cast(valid_anchor_labels, tf.float32),
valid_label_logits,
reduction=tf.losses.Reduction.SUM)
label_loss = label_loss / _C.RPN.BATCH_PER_IM
label_loss = tf.where(tf.equal(nr_valid, 0),
placeholder,
label_loss,
name="label_loss")
pos_anchor_boxes = tf.boolean_mask(anchor_boxes, pos_mask)
pos_box_logits = tf.boolean_mask(box_logits, pos_mask)
delta = 1.0 / 9
box_loss = tf.losses.huber_loss(pos_anchor_boxes,
pos_box_logits,
delta=delta,
reduction=tf.losses.Reduction.SUM) / delta
box_loss = box_loss * (1. / _C.RPN.BATCH_PER_IM)
box_loss = tf.where(tf.equal(nr_pos, 0), placeholder, box_loss)
#print_op = tf.print({'label_loss':label_loss, 'box_loss':box_loss})
#with tf.control_dependencies([print_op]):
label_loss = tf.identity(label_loss)
box_loss = tf.identity(box_loss)
return [label_loss, box_loss]
if FLAGS.train:
# similarity regularization
sim_reg_list = []
for i in range(FLAGS.batch_size):
for j in range(i + 1, FLAGS.batch_size):
a_0 = tf.reduce_sum(c_one_hot_0[i] * c_one_hot_0[j])
a_1 = tf.reduce_sum(c_one_hot_1[i] * c_one_hot_1[j])
a_2 = tf.reduce_sum(c_one_hot_2[i] * c_one_hot_2[j])
a_3 = tf.reduce_sum(c_one_hot_3[i] * c_one_hot_3[j])
sim = tf.sqrt(tf.reduce_sum(tf.square(G_fake[i] - G_fake[j])))
sim_reg_list.append(a_0 * sim + (1. - a_0) / (sim + 1e-5))
sim_reg_list.append(a_1 * sim + (1. - a_1) / (sim + 1e-5))
sim_reg_list.append(a_2 * sim + (1. - a_2) / (sim + 1e-5))
sim_reg_list.append(a_3 * sim + (1. - a_3) / (sim + 1e-5))
sim_reg = tf.truediv(tf.add_n(sim_reg_list),
FLAGS.batch_size * (FLAGS.batch_size - 1.))
g_loss += FLAGS.lambd * sim_reg
# trainable variable
t_vars = tf.trainable_variables()
d_vars = [var for var in t_vars if 'discriminator' in var.op.name]
g_vars = [var for var in t_vars if 'generator' in var.op.name]
# optimizers
d_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
g_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, 'generator') + \
tf.get_collection(tf.GraphKeys.UPDATE_OPS, 'discriminator_1')
with tf.control_dependencies(d_update_ops):
d_optim = tf.train.AdamOptimizer(FLAGS.learning_rate,
beta1=FLAGS.adam_beta1,
beta2=FLAGS.adam_beta2).minimize(
# Declare model operations
model_output = tf.add(tf.matmul(x_data, A), b)
###
# Loss Functions
###
# Select appropriate loss function based on regression type
if regression_type == 'LASSO':
# Declare Lasso loss function
# Lasso Loss = L2_Loss + heavyside_step,
# Where heavyside_step ~ 0 if A < constant, otherwise ~ 99
lasso_param = tf.constant(0.9)
heavyside_step = tf.truediv(1., tf.add(1., tf.exp(tf.multiply(-50., tf.subtract(A, lasso_param)))))
regularization_param = tf.multiply(heavyside_step, 99.)
loss = tf.add(tf.reduce_mean(tf.square(y_target - model_output)), regularization_param)
elif regression_type == 'Ridge':
# Declare the Ridge loss function
# Ridge loss = L2_loss + L2 norm of slope
ridge_param = tf.constant(1.)
ridge_loss = tf.reduce_mean(tf.square(A))
loss = tf.expand_dims(tf.add(tf.reduce_mean(tf.square(y_target - model_output)), tf.multiply(ridge_param, ridge_loss)), 0)
else:
print('Invalid regression_type parameter value',file=sys.stderr)
###
def build_model(self, learning_rate=[0.001, 0.01]):
'''
Model - wide and deep - built using zqtflearn
'''
n_cc = len(self.continuous_columns)
n_categories = 1 # two categories: is_idv and is_not_idv
#为什么不是len(self.CATEGORICAL_COLUMNS)
input_shape = [None, n_cc]
if self.verbose:
print ("="*77 + " Model %s (type=%s)" % (self.name, self.model_type))
print (" Input placeholder shape=%s" % str(input_shape))
wide_inputs = zqtflearn.input_data(shape=input_shape, name="wide_X")
if not isinstance(learning_rate, list):
learning_rate = [learning_rate, learning_rate] # wide, deep
if self.verbose:
print (" Learning rates (wide, deep)=%s" % learning_rate)
with tf.name_scope("Y"): # placeholder for target variable (i.e. trainY input)
Y_in = tf.placeholder(shape=[None, 1], dtype=tf.float32, name="Y")
with tf.variable_op_scope([wide_inputs], None, "cb_unit", reuse=False) as scope:
central_bias = zqtflearn.variables.variable('central_bias', shape=[1],
initializer=tf.constant_initializer(np.random.randn()),
trainable=True, restore=True)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/cb_unit', central_bias)
if 'wide' in self.model_type:
wide_network = self.wide_model(wide_inputs, n_cc)
network = wide_network
wide_network_with_bias = tf.add(wide_network, central_bias, name="wide_with_bias")
if 'deep' in self.model_type:
deep_network = self.deep_model(wide_inputs, n_cc) #这里面应该是deep inputs
deep_network_with_bias = tf.add(deep_network, central_bias, name="deep_with_bias")
if 'wide' in self.model_type:
network = tf.add(wide_network, deep_network)
if self.verbose:
print ("Wide + deep model network %s" % network)
else:
network = deep_network
network = tf.add(network, central_bias, name="add_central_bias")
# add validation monitor summaries giving confusion matrix entries
with tf.name_scope('Monitors'):
predictions = tf.cast(tf.greater(network, 0), tf.int64)
print ("predictions=%s" % predictions)
Ybool = tf.cast(Y_in, tf.bool)
print ("Ybool=%s" % Ybool)
pos = tf.boolean_mask(predictions, Ybool)
neg = tf.boolean_mask(predictions, ~Ybool)
psize = tf.cast(tf.shape(pos)[0], tf.int64)
nsize = tf.cast(tf.shape(neg)[0], tf.int64)
true_positive = tf.reduce_sum(pos, name="true_positive")
false_negative = tf.subtract(psize, true_positive, name="false_negative")
false_positive = tf.reduce_sum(neg, name="false_positive")
true_negative = tf.subtract(nsize, false_positive, name="true_negative")
overall_accuracy = tf.truediv(tf.add(true_positive, true_negative), tf.add(nsize, psize), name="overall_accuracy")
vmset = [true_positive, true_negative, false_positive, false_negative, overall_accuracy]
trainable_vars = tf.trainable_variables()
tv_deep = [v for v in trainable_vars if v.name.startswith('deep_')]
tv_wide = [v for v in trainable_vars if v.name.startswith('wide_')]
if self.verbose:
print ("DEEP trainable_vars")
for v in tv_deep:
print (" Variable %s: %s" % (v.name, v))
print ("WIDE trainable_vars")
for v in tv_wide:
print (" Variable %s: %s" % (v.name, v))
if 'wide' in self.model_type:
if not 'deep' in self.model_type:
tv_wide.append(central_bias)
zqtflearn.regression(wide_network_with_bias,
placeholder=Y_in,
optimizer='sgd',
#loss='roc_auc_score',
loss='binary_crossentropy',
metric="accuracy",
learning_rate=learning_rate[0],
validation_monitors=vmset,
trainable_vars=tv_wide,
op_name="wide_regression",
name="Y")
if 'deep' in self.model_type:
if not 'wide' in self.model_type:
tv_wide.append(central_bias)
zqtflearn.regression(deep_network_with_bias,
placeholder=Y_in,
optimizer='adam',
#loss='roc_auc_score',
loss='binary_crossentropy',
metric="accuracy",
learning_rate=learning_rate[1],
validation_monitors=vmset if not 'wide' in self.model_type else None,
trainable_vars=tv_deep,
op_name="deep_regression",
name="Y")
if self.model_type=='wide+deep': # learn central bias separately for wide+deep
zqtflearn.regression(network,
placeholder=Y_in,
optimizer='adam',
loss='binary_crossentropy',
metric="accuracy",
learning_rate=learning_rate[0], # use wide learning rate
trainable_vars=[central_bias],
op_name="central_bias_regression",
name="Y")
self.model = zqtflearn.DNN(network,
tensorboard_verbose=self.tensorboard_verbose,
max_checkpoints=5,
checkpoint_path="%s/%s.tfl" % (self.checkpoints_dir, self.name),
)
if self.verbose:
print ("Target variables:")
for v in tf.get_collection(tf.GraphKeys.TARGETS):
print (" variable %s: %s" % (v.name, v))
print ("="*77)
def custom_loss(self, y_true, y_pred):
mask_shape = tf.shape(y_true)[:4]
cell_x = tf.to_float(tf.reshape(tf.tile(tf.range(self.grid_w), [self.grid_h]), (1, self.grid_h, self.grid_w, 1, 1)))
cell_y = tf.transpose(cell_x, (0,2,1,3,4))
cell_grid = tf.tile(tf.concat([cell_x,cell_y], -1), [self.batch_size, 1, 1, self.nb_box, 1])
coord_mask = tf.zeros(mask_shape)
conf_mask = tf.zeros(mask_shape)
class_mask = tf.zeros(mask_shape)
seen = tf.Variable(0.)
total_recall = tf.Variable(0.)
"""
Adjust prediction
"""
### adjust x and y
pred_box_xy = tf.sigmoid(y_pred[..., :2]) + cell_grid
### adjust w and h
pred_box_wh = tf.exp(y_pred[..., 2:4]) * np.reshape(self.anchors, [1,1,1,self.nb_box,2])
### adjust confidence
pred_box_conf = tf.sigmoid(y_pred[..., 4])
### adjust class probabilities
pred_box_class = y_pred[..., 5:]
"""
Adjust ground truth
"""
### adjust x and y
true_box_xy = y_true[..., 0:2] # relative position to the containing cell
### adjust w and h
true_box_wh = y_true[..., 2:4] # number of cells accross, horizontally and vertically
### adjust confidence
true_wh_half = true_box_wh / 2.
true_mins = true_box_xy - true_wh_half
true_maxes = true_box_xy + true_wh_half
pred_wh_half = pred_box_wh / 2.
pred_mins = pred_box_xy - pred_wh_half
pred_maxes = pred_box_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_box_wh[..., 0] * true_box_wh[..., 1]
pred_areas = pred_box_wh[..., 0] * pred_box_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
true_box_conf = iou_scores * y_true[..., 4]
### adjust class probabilities
true_box_class = tf.argmax(y_true[..., 5:], -1)
"""
Determine the masks
"""
### coordinate mask: simply the position of the ground truth boxes (the predictors)
coord_mask = tf.expand_dims(y_true[..., 4], axis=-1) * self.coord_scale
### confidence mask: penelize predictors + penalize boxes with low IOU
# penalize the confidence of the boxes, which have IOU with some ground truth box < 0.6
true_xy = self.true_boxes[..., 0:2]
true_wh = self.true_boxes[..., 2:4]
true_wh_half = true_wh / 2.
true_mins = true_xy - true_wh_half
true_maxes = true_xy + true_wh_half
pred_xy = tf.expand_dims(pred_box_xy, 4)
pred_wh = tf.expand_dims(pred_box_wh, 4)
pred_wh_half = pred_wh / 2.
pred_mins = pred_xy - pred_wh_half
pred_maxes = pred_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_wh[..., 0] * true_wh[..., 1]
pred_areas = pred_wh[..., 0] * pred_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
best_ious = tf.reduce_max(iou_scores, axis=4)
conf_mask = conf_mask + tf.to_float(best_ious < 0.6) * (1 - y_true[..., 4]) * self.no_object_scale
# penalize the confidence of the boxes, which are reponsible for corresponding ground truth box
conf_mask = conf_mask + y_true[..., 4] * self.object_scale
### class mask: simply the position of the ground truth boxes (the predictors)
class_mask = y_true[..., 4] * tf.gather(self.class_wt, true_box_class) * self.class_scale
"""
Warm-up training
"""
no_boxes_mask = tf.to_float(coord_mask < self.coord_scale/2.)
seen = tf.assign_add(seen, 1.)
true_box_xy, true_box_wh, coord_mask = tf.cond(tf.less(seen, self.warmup_batches+1),
lambda: [true_box_xy + (0.5 + cell_grid) * no_boxes_mask,
true_box_wh + tf.ones_like(true_box_wh) * \
np.reshape(self.anchors, [1,1,1,self.nb_box,2]) * \
no_boxes_mask,
tf.ones_like(coord_mask)],
lambda: [true_box_xy,
true_box_wh,
coord_mask])
"""
Finalize the loss
"""
nb_coord_box = tf.reduce_sum(tf.to_float(coord_mask > 0.0))
nb_conf_box = tf.reduce_sum(tf.to_float(conf_mask > 0.0))
nb_class_box = tf.reduce_sum(tf.to_float(class_mask > 0.0))
loss_xy = tf.reduce_sum(tf.square(true_box_xy-pred_box_xy) * coord_mask) / (nb_coord_box + 1e-6) / 2.
loss_wh = tf.reduce_sum(tf.square(true_box_wh-pred_box_wh) * coord_mask) / (nb_coord_box + 1e-6) / 2.
loss_conf = tf.reduce_sum(tf.square(true_box_conf-pred_box_conf) * conf_mask) / (nb_conf_box + 1e-6) / 2.
loss_class = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=true_box_class, logits=pred_box_class)
loss_class = tf.reduce_sum(loss_class * class_mask) / (nb_class_box + 1e-6)
loss = tf.cond(tf.less(seen, self.warmup_batches+1),
lambda: loss_xy + loss_wh + loss_conf + loss_class + 10,
lambda: loss_xy + loss_wh + loss_conf + loss_class)
if self.debug:
nb_true_box = tf.reduce_sum(y_true[..., 4])
nb_pred_box = tf.reduce_sum(tf.to_float(true_box_conf > 0.5) * tf.to_float(pred_box_conf > 0.3))
current_recall = nb_pred_box/(nb_true_box + 1e-6)
total_recall = tf.assign_add(total_recall, current_recall)
loss = tf.Print(loss, [loss_xy], message='Loss XY \t', summarize=1000)
loss = tf.Print(loss, [loss_wh], message='Loss WH \t', summarize=1000)
loss = tf.Print(loss, [loss_conf], message='Loss Conf \t', summarize=1000)
loss = tf.Print(loss, [loss_class], message='Loss Class \t', summarize=1000)
loss = tf.Print(loss, [loss], message='Total Loss \t', summarize=1000)
loss = tf.Print(loss, [current_recall], message='Current Recall \t', summarize=1000)
loss = tf.Print(loss, [total_recall/seen], message='Average Recall \t', summarize=1000)
return loss
def __init__(self, envs, workerid, target_task):
"""
An implementation of the A3C algorithm that is reasonably well-tuned for the VNC environments.
Below, we will have a modest amount of complexity due to the way TensorFlow handles data parallelism.
But overall, we'll define the model, specify its inputs, and describe how the policy gradients step
should be computed.
"""
self.env = envs
self.num_tasks = num_tasks = 2 # only suitable when number of tasks equal to 2.
self.target_task = target_task
self.aux_tasks_id = int(1 - target_task)
self.workerid = workerid
self.network = [None] * self.num_tasks
self.local_logitProjnet = [None] * self.num_tasks
self.local_network = [None] * self.num_tasks
self.global_step = [None] * self.num_tasks
self.logitProjnet = [None] * self.num_tasks
self.T1 = [100000000, 3000000] # [400, 5000] #[4000000, 6000000]
self.T2 = [100000000, 4000000] # [400, 5000] #[4000000, 6000000]
pi = [None] * self.num_tasks
worker_device = "/job:worker/task:{}/cpu:0".format(workerid)
with tf.device(
tf.train.replica_device_setter(1,
worker_device=worker_device)):
with tf.variable_scope("global" + str(0)): # Pong
self.network[0] = LSTMPolicy(envs[0].observation_space.shape,
envs[0].action_space.n)
self.global_step[0] = tf.get_variable(
"global_step", [],
tf.int32,
initializer=tf.zeros_initializer,
trainable=False)
with tf.variable_scope("global" + str(1)): #bowling
self.network[1] = LSTMPolicy(envs[1].observation_space.shape,
envs[1].action_space.n)
self.global_step[1] = tf.get_variable(
"global_step", [],
tf.int32,
initializer=tf.zeros_initializer,
trainable=False)
with tf.variable_scope(
"globallogits0"): # network for projection logits.
self.logitProjnet[0] = logitsProj(envs[0].action_space.n)
with tf.variable_scope(
"globallogits1"): # network for projection logits.
self.logitProjnet[1] = logitsProj(envs[1].action_space.n)
with tf.device(worker_device):
with tf.variable_scope("local" + str(0)):
self.local_network[0] = pi[0] = LSTMPolicy(
envs[0].observation_space.shape, envs[0].action_space.n)
pi[0].global_step = self.global_step[0]
with tf.variable_scope("local" + str(1)):
self.local_network[1] = pi[1] = LSTMPolicy(
envs[1].observation_space.shape, envs[1].action_space.n)
pi[1].global_step = self.global_step[1]
with tf.variable_scope("local" + "logits0"):
self.local_logitProjnet[0] = logitsProj(envs[0].action_space.n)
with tf.variable_scope("local" + "logits1"):
self.local_logitProjnet[1] = logitsProj(envs[1].action_space.n)
self.ac = [None] * num_tasks
self.adv = [None] * num_tasks
self.r = [None] * num_tasks
log_prob_tf = [None] * num_tasks
prob_tf = [None] * num_tasks
pi_loss = [None] * num_tasks
vf_loss = [None] * num_tasks
entropy = [None] * num_tasks
bs = [None] * num_tasks
self.loss = [None] * num_tasks
self.runner = [None] * num_tasks
grads = [None] * num_tasks
self.summary_op = [[None, None] for i in np.arange(num_tasks)
] #[[None, None], [None, None]] 2 tasks
self.sync = [None] * num_tasks
grads_and_vars = [None] * num_tasks
self.inc_step = [None] * num_tasks
opt = [None] * num_tasks
self.train_op = [None] * num_tasks
self.target_logits = [None] * num_tasks
soft_p_temperature = [None] * num_tasks
soft_t_temperature = [None] * num_tasks
self.KD_trainop = [None] * num_tasks
kl_loss = [None] * num_tasks
grads_kd = [None] * num_tasks
grads_and_vars_kd = [None] * num_tasks
optkd = [None] * num_tasks
self.sync_logits = [None] * num_tasks
self.logits_stu = [None] * num_tasks
soft_student_logits = [None] * num_tasks
soft_teacher_logits = [None] * num_tasks
self.proj_loss = [None] * num_tasks
grad_logproj = [None] * num_tasks
grads_and_vars_logproj = [None] * num_tasks
optlgproj = [None] * num_tasks
self.lgproj_trainop = [None] * num_tasks
self.summary_op_proj = [None] * num_tasks
for ii in np.arange(num_tasks):
# start to build loss for target network
self.ac[ii] = tf.placeholder(tf.float32,
[None, envs[ii].action_space.n],
name="ac" + str(ii))
self.adv[ii] = tf.placeholder(tf.float32, [None],
name="adv" + str(ii))
self.r[ii] = tf.placeholder(tf.float32, [None],
name="r" + str(ii))
log_prob_tf[ii] = tf.nn.log_softmax(pi[ii].logits)
prob_tf[ii] = tf.nn.softmax(pi[ii].logits)
# the "policy gradients" loss: its derivative is precisely the policy gradient
# notice that self.ac is a placeholder that is provided externally.
# ac will contain the advantages, as calculated in process_rollout
pi_loss[ii] = -tf.reduce_sum(
tf.reduce_sum(log_prob_tf[ii] * self.ac[ii], [1]) *
self.adv[ii])
# loss of value function
vf_loss[ii] = 0.5 * tf.reduce_sum(
tf.square(pi[ii].vf - self.r[ii]))
entropy[ii] = -tf.reduce_sum(prob_tf[ii] * log_prob_tf[ii])
bs[ii] = tf.to_float(tf.shape(pi[ii].x)[0])
self.loss[
ii] = pi_loss[ii] + 0.5 * vf_loss[ii] - entropy[ii] * 0.01
# 20 represents the number of "local steps": the number of timesteps
# we run the policy before we update the parameters.
# The larger local steps is, the lower is the variance in our policy gradients estimate
# on the one hand; but on the other hand, we get less frequent parameter updates, which
# slows down learning. In this code, we found that making local steps be much
# smaller than 20 makes the algorithm more difficult to tune and to get to work.
# name = "worker"+str(workerid)+"task"+str(ii)
name = "task" + str(ii)
self.runner[ii] = RunnerThread(
envs[ii], pi[ii], 20,
name) # todo local step should be task specific
grads[ii] = tf.gradients(self.loss[ii], pi[ii].var_list)
summaries1 = list() # summary when it's target tasks
summaries1.append(
tf.scalar_summary("model/policy_loss" + str(ii),
pi_loss[ii] / bs[ii]))
summaries1.append(
tf.scalar_summary("model/value_loss" + str(ii),
vf_loss[ii] / bs[ii]))
summaries1.append(
tf.scalar_summary("model/entropy" + str(ii),
entropy[ii] / bs[ii]))
summaries1.append(
tf.image_summary("model/state" + str(ii), pi[ii].x))
summaries1.append(
tf.scalar_summary("model/grad_global_norm" + str(ii),
tf.global_norm(grads[ii])))
summaries1.append(
tf.scalar_summary("model/var_global_norm" + str(ii),
tf.global_norm(pi[ii].var_list)))
summaries1.append(
tf.histogram_summary("model/action_weight" + str(ii),
prob_tf[ii]))
summaries2 = list() # summary when it's aux tasks.
summaries2.append(
tf.histogram_summary("model/action_weight" + str(ii),
prob_tf[ii]))
summaries2.append(
tf.scalar_summary("model/entropy" + str(ii),
entropy[ii] / bs[ii]))
self.summary_op[ii][0] = tf.merge_summary(summaries1)
self.summary_op[ii][1] = tf.merge_summary(summaries2)
grads[ii], _ = tf.clip_by_global_norm(grads[ii], 40.0)
# self.sync = [None] * self.num_tasks
zipvars_lp = zip(pi[ii].var_list, self.network[ii].var_list)
self.sync[ii] = tf.group(
*[v1.assign(v2) for v1, v2 in zipvars_lp])
grads_and_vars[ii] = list(
zip(grads[ii], self.network[ii].var_list))
self.inc_step[ii] = self.global_step[ii].assign_add(
tf.shape(pi[ii].x)[0])
# each worker has a different set of adam optimizer parameters
opt[ii] = tf.train.AdamOptimizer(1e-4)
self.train_op[ii] = tf.group(
opt[ii].apply_gradients(grads_and_vars[ii]),
self.inc_step[ii])
# knowledge distillation
self.target_logits[ii] = tf.placeholder(
tf.float32, [None, envs[ii].action_space.n],
name="target_logits") # logits from teacher
Tao = 1.0 # temperature used for distillation.
# soft_p_temperature[ii] = tf.nn.softmax(tf.truediv(pi[ii].logits_fordistill, Tao)) # todo this is wrong if tau !=1
soft_p_temperature[ii] = tf.nn.softmax(
pi[ii].logits_fordistill)
soft_t_temperature[ii] = tf.nn.softmax(
tf.truediv(self.target_logits[ii], Tao))
kl_loss[ii] = tf.reduce_mean(
tf.reduce_sum(
soft_t_temperature[ii] * tf.log(1e-10 + tf.truediv(
soft_t_temperature[ii], soft_p_temperature[ii])),
1))
grads_kd[ii] = tf.gradients(kl_loss[ii], pi[ii].var_list)
grads_kd[ii], _ = tf.clip_by_global_norm(grads_kd[ii], 40.0)
grads_and_vars_kd[ii] = list(
zip(grads_kd[ii], self.network[ii].var_list))
optkd[ii] = tf.train.AdamOptimizer(1e-4)
self.KD_trainop[ii] = optkd[ii].apply_gradients(
grads_and_vars_kd[ii])
'learning logits projection'
zipvars_lp = zip(self.local_logitProjnet[ii].var_list,
self.logitProjnet[ii].var_list)
self.sync_logits[ii] = tf.group(
*[v1.assign(v2) for v1, v2 in zipvars_lp])
# soft_student_logits = tf.nn.softmax(pi[target_task].logits)
self.logits_stu[ii] = tf.placeholder(
tf.float32, [None, envs[ii].action_space.n])
soft_student_logits[ii] = tf.nn.softmax(self.logits_stu[ii])
soft_teacher_logits[ii] = tf.nn.softmax(
self.local_logitProjnet[ii].logits_out)
self.proj_loss[ii] = proj_loss = tf.reduce_mean(
tf.reduce_sum( # todo verify this in tensorboard
soft_teacher_logits[ii] * tf.log(1e-10 + tf.truediv(
soft_teacher_logits[ii], soft_student_logits[ii])),
1)) # target task --> student
grad_logproj[ii] = tf.gradients(
proj_loss, self.local_logitProjnet[ii].var_list)
grad_logproj[ii], _ = tf.clip_by_global_norm(
grad_logproj[ii], 40.0)
grads_and_vars_logproj[ii] = list(
zip(grad_logproj[ii], self.logitProjnet[ii].var_list))
optlgproj[ii] = tf.train.AdamOptimizer(1e-4)
self.lgproj_trainop[ii] = optlgproj[ii].apply_gradients(
grads_and_vars_logproj[ii])
self.summary_op_proj[ii] = tf.scalar_summary(
"model/proj_loss" + str(ii), self.proj_loss[ii])
self.summary_writer = None
self.local_steps = 0
def build_graph(mode, hparams, encoder_decoder, sequence_example_file=None):
"""Builds the TensorFlow graph.
Args:
mode: 'train', 'eval', or 'generate'. Only mode related ops are added to
the graph.
hparams: A tf_lib.HParams object containing the hyperparameters to use.
encoder_decoder: The MelodyEncoderDecoder being used by the model.
sequence_example_file: A string path to a TFRecord file containing
tf.train.SequenceExamples. Only needed for training and evaluation.
Returns:
A tf.Graph instance which contains the TF ops.
Raises:
ValueError: If mode is not 'train', 'eval', or 'generate', or if
sequence_example_file does not match a file when mode is 'train' or
'eval'.
"""
if mode not in ('train', 'eval', 'generate'):
raise ValueError('The mode parameter must be \'train\', \'eval\', '
'or \'generate\'. The mode parameter was: %s' % mode)
tf.logging.info('hparams = %s', hparams.values())
input_size = encoder_decoder.input_size
num_classes = encoder_decoder.num_classes
no_event_label = encoder_decoder.no_event_label
with tf.Graph().as_default() as graph:
inputs, labels, lengths, = None, None, None
state_is_tuple = True
if mode == 'train' or mode == 'eval':
inputs, labels, lengths = sequence_example_lib.get_padded_batch(
[sequence_example_file], hparams.batch_size, input_size)
elif mode == 'generate':
inputs = tf.placeholder(tf.float32,
[hparams.batch_size, None, input_size])
# If state_is_tuple is True, the output RNN cell state will be a tuple
# instead of a tensor. During training and evaluation this improves
# performance. However, during generation, the RNN cell state is fed
# back into the graph with a feed dict. Feed dicts require passed in
# values to be tensors and not tuples, so state_is_tuple is set to False.
state_is_tuple = False
cells = []
for num_units in hparams.rnn_layer_sizes:
cell = tf.nn.rnn_cell.BasicLSTMCell(num_units,
state_is_tuple=state_is_tuple)
cell = tf.nn.rnn_cell.DropoutWrapper(
cell, output_keep_prob=hparams.dropout_keep_prob)
cells.append(cell)
cell = tf.nn.rnn_cell.MultiRNNCell(cells,
state_is_tuple=state_is_tuple)
if hparams.attn_length:
cell = tf.contrib.rnn.AttentionCellWrapper(
cell, hparams.attn_length, state_is_tuple=state_is_tuple)
initial_state = cell.zero_state(hparams.batch_size, tf.float32)
outputs, final_state = tf.nn.dynamic_rnn(cell,
inputs,
lengths,
initial_state,
parallel_iterations=1,
swap_memory=True)
outputs_flat = tf.reshape(outputs, [-1, hparams.rnn_layer_sizes[-1]])
logits_flat = tf.contrib.layers.linear(outputs_flat, num_classes)
if mode == 'train' or mode == 'eval':
if hparams.skip_first_n_losses:
logits = tf.reshape(logits_flat,
[hparams.batch_size, -1, num_classes])
logits = logits[:, hparams.skip_first_n_losses:, :]
logits_flat = tf.reshape(logits, [-1, num_classes])
labels = labels[:, hparams.skip_first_n_losses:]
labels_flat = tf.reshape(labels, [-1])
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits_flat, labels_flat))
perplexity = tf.exp(loss)
correct_predictions = tf.to_float(
tf.nn.in_top_k(logits_flat, labels_flat, 1))
accuracy = tf.reduce_mean(correct_predictions) * 100
event_positions = tf.to_float(
tf.not_equal(labels_flat, no_event_label))
event_accuracy = tf.truediv(
tf.reduce_sum(tf.mul(correct_predictions, event_positions)),
tf.reduce_sum(event_positions)) * 100
no_event_positions = tf.to_float(
tf.equal(labels_flat, no_event_label))
no_event_accuracy = tf.truediv(
tf.reduce_sum(tf.mul(correct_predictions, no_event_positions)),
tf.reduce_sum(no_event_positions)) * 100
global_step = tf.Variable(0, trainable=False, name='global_step')
tf.add_to_collection('loss', loss)
tf.add_to_collection('perplexity', perplexity)
tf.add_to_collection('accuracy', accuracy)
tf.add_to_collection('global_step', global_step)
summaries = [
tf.scalar_summary('loss', loss),
tf.scalar_summary('perplexity', perplexity),
tf.scalar_summary('accuracy', accuracy),
tf.scalar_summary('event_accuracy', event_accuracy),
tf.scalar_summary('no_event_accuracy', no_event_accuracy),
]
if mode == 'train':
learning_rate = tf.train.exponential_decay(
hparams.initial_learning_rate,
global_step,
hparams.decay_steps,
hparams.decay_rate,
staircase=True,
name='learning_rate')
opt = tf.train.AdamOptimizer(learning_rate)
params = tf.trainable_variables()
gradients = tf.gradients(loss, params)
clipped_gradients, _ = tf.clip_by_global_norm(
gradients, hparams.clip_norm)
train_op = opt.apply_gradients(zip(clipped_gradients, params),
global_step)
tf.add_to_collection('learning_rate', learning_rate)
tf.add_to_collection('train_op', train_op)
summaries.append(
tf.scalar_summary('learning_rate', learning_rate))
if mode == 'eval':
summary_op = tf.merge_summary(summaries)
tf.add_to_collection('summary_op', summary_op)
elif mode == 'generate':
if hparams.temperature != 1.0:
logits_flat /= hparams.temperature
softmax_flat = tf.nn.softmax(logits_flat)
softmax = tf.reshape(softmax_flat,
[hparams.batch_size, -1, num_classes])
tf.add_to_collection('inputs', inputs)
tf.add_to_collection('initial_state', initial_state)
tf.add_to_collection('final_state', final_state)
tf.add_to_collection('softmax', softmax)
return graph
def v2_loss(outs, anchorcoords, classes):
# Refer to the following darkflow loss
# https://github.com/thtrieu/darkflow/blob/master/darkflow/net/yolov2/train.py
sprob = 1.
sconf = 5.
snoob = 1.
scoor = 1.
H = int(outs.shape[1]) if tf_later_than('2') else outs.shape[1].value
W = int(outs.shape[2]) if tf_later_than('2') else outs.shape[2].value
cells = H * W
sizes = np.array([[[[W, H]]]], dtype=np.float32)
anchors = len(anchorcoords) // 2
anchorcoords = np.reshape(anchorcoords, [1, 1, anchors, 2])
_, _probs, _confs, _coord, _proid, _areas, _ul, _br = outs.inputs[:8]
# Extract the coordinate prediction from net.out
outs = tf.reshape(outs, [-1, H, W, anchors, (5 + classes)])
coords = tf.reshape(outs[:, :, :, :, :4], [-1, cells, anchors, 4])
adj_xy = 1. / (1. + tf.exp(-coords[:, :, :, 0:2]))
adj_wh = tf.sqrt(tf.exp(coords[:, :, :, 2:4]) * anchorcoords / sizes)
adj_c = 1. / (1. + tf.exp(-outs[:, :, :, :, 4]))
adj_c = tf.reshape(adj_c, [-1, cells, anchors, 1])
adj_prob = tf.reshape(tf.nn.softmax(outs[:, :, :, :, 5:]),
[-1, cells, anchors, classes])
adj_outs = tf.concat([adj_xy, adj_wh, adj_c, adj_prob], 3)
coords = tf.concat([adj_xy, adj_wh], 3)
wh = tf.pow(coords[:, :, :, 2:4], 2) * sizes
area_pred = wh[:, :, :, 0] * wh[:, :, :, 1]
centers = coords[:, :, :, 0:2]
floor = centers - (wh * .5)
ceil = centers + (wh * .5)
# calculate the intersection areas
intersect_upleft = tf.maximum(floor, _ul)
intersect_botright = tf.minimum(ceil, _br)
intersect_wh = intersect_botright - intersect_upleft
intersect_wh = tf.maximum(intersect_wh, 0.0)
intersect = tf.multiply(intersect_wh[:, :, :, 0], intersect_wh[:, :, :, 1])
# calculate the best IOU, set 0.0 confidence for worse boxes
iou = tf.truediv(intersect, _areas + area_pred - intersect)
best_box = tf.equal(iou, tf.reduce_max(iou, [2], True))
best_box = tf.to_float(best_box)
confs = tf.multiply(best_box, _confs)
# take care of the weight terms
conid = snoob * (1. - confs) + sconf * confs
weight_coo = tf.concat(4 * [tf.expand_dims(confs, -1)], 3)
cooid = scoor * weight_coo
weight_pro = tf.concat(classes * [tf.expand_dims(confs, -1)], 3)
proid = sprob * weight_pro
true = tf.concat([_coord, tf.expand_dims(confs, 3), _probs], 3)
wght = tf.concat([cooid, tf.expand_dims(conid, 3), proid], 3)
loss = tf.pow(adj_outs - true, 2)
loss = tf.multiply(loss, wght)
loss = tf.reshape(loss, [-1, cells * anchors * (5 + classes)])
loss = tf.reduce_sum(loss, 1)
return .5 * tf.reduce_mean(loss) + tf.losses.get_regularization_loss()
def bbox_loss(y_true, y_pred):
"""
This function defines the custom bounding box loss. Heavily inspired by
https://github.com/allanzelener/YAD2K/blob/master/yad2k/models/keras_yolo.py
"""
mask_shape = tf.shape(y_true)[:4]
cell_x = tf.to_float(
tf.reshape(tf.tile(tf.range(GRID_W), [GRID_H]),
(1, GRID_H, GRID_W, 1, 1)))
cell_y = tf.transpose(cell_x, (0, 2, 1, 3, 4))
cell_grid = tf.tile(tf.concat([cell_x, cell_y], -1),
[BATCH_SIZE, 1, 1, 5, 1])
coord_mask = tf.zeros(mask_shape)
conf_mask = tf.zeros(mask_shape)
class_mask = tf.zeros(mask_shape)
seen = tf.Variable(0.)
# adjust x and y
pred_box_xy = tf.sigmoid(y_pred[..., :2]) + cell_grid
true_box_xy = y_true[..., 0:2]
# adjust w and h
pred_box_wh = tf.exp(y_pred[..., 2:4]) * np.reshape(
ANCHORS, [1, 1, 1, BOXES, 2])
true_box_wh = y_true[..., 2:4]
# adjust class probabilities
pred_box_class = y_pred[..., 5:]
# adjust confidence
pred_box_conf = tf.sigmoid(y_pred[..., 4])
true_wh_half = true_box_wh / 2.
true_mins = true_box_xy - true_wh_half
true_maxes = true_box_xy + true_wh_half
pred_wh_half = pred_box_wh / 2.
pred_mins = pred_box_xy - pred_wh_half
pred_maxes = pred_box_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_box_wh[..., 0] * true_box_wh[..., 1]
pred_areas = pred_box_wh[..., 0] * pred_box_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
true_box_conf = iou_scores * y_true[..., 4]
# adjust class probabilities
true_box_class = tf.argmax(y_true[..., 5:], -1)
# coordinate mask: simply the position of the ground truth boxes (the predictors)
coord_mask = tf.expand_dims(y_true[..., 4], axis=-1) * COORD_SCALE
# confidence mask: penelize predictors + penalize boxes with low IOU
# penalize the confidence of the boxes, which have IOU with some ground truth box < 0.6
true_xy = gt_boxes[..., 0:2]
true_wh = gt_boxes[..., 2:4]
true_wh_half = true_wh / 2.
true_mins = true_xy - true_wh_half
true_maxes = true_xy + true_wh_half
pred_xy = tf.expand_dims(pred_box_xy, 4)
pred_wh = tf.expand_dims(pred_box_wh, 4)
pred_wh_half = pred_wh / 2.
pred_mins = pred_xy - pred_wh_half
pred_maxes = pred_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_wh[..., 0] * true_wh[..., 1]
pred_areas = pred_wh[..., 0] * pred_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
best_ious = tf.reduce_max(iou_scores, axis=4)
conf_mask = conf_mask + tf.to_float(
best_ious < 0.6) * (1 - y_true[..., 4]) * NO_OBJECT_SCALE
# penalize the confidence of the boxes, which are reponsible for corresponding ground truth box
conf_mask = conf_mask + y_true[..., 4] * OBJECT_SCALE
# class mask: simply the position of the ground truth boxes (the predictors)
class_mask = y_true[..., 4] * tf.gather(CLASS_WEIGHTS,
true_box_class) * CLASS_SCALE
no_boxes_mask = tf.to_float(coord_mask < COORD_SCALE / 2.)
seen = tf.assign_add(seen, 1.)
true_box_xy, true_box_wh, coord_mask = tf.cond(
tf.less(seen, WARM_UP_BATCHES), lambda: [
true_box_xy + (0.5 + cell_grid) * no_boxes_mask,
true_box_wh + tf.ones_like(true_box_wh) * np.reshape(
ANCHORS, [1, 1, 1, BOXES, 2]) * no_boxes_mask,
tf.ones_like(coord_mask)
], lambda: [true_box_xy, true_box_wh, coord_mask])
nb_coord_box = tf.reduce_sum(tf.to_float(coord_mask > 0.0))
nb_conf_box = tf.reduce_sum(tf.to_float(conf_mask > 0.0))
nb_class_box = tf.reduce_sum(tf.to_float(class_mask > 0.0))
loss_xy = tf.reduce_sum(tf.square(true_box_xy - pred_box_xy) *
coord_mask) / (nb_coord_box + 1e-6) / 2.
loss_wh = tf.reduce_sum(tf.square(true_box_wh - pred_box_wh) *
coord_mask) / (nb_coord_box + 1e-6) / 2.
loss_conf = tf.reduce_sum(
tf.square(true_box_conf - pred_box_conf) * conf_mask) / (nb_conf_box +
1e-6) / 2.
loss_class = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=true_box_class, logits=pred_box_class)
loss_class = tf.reduce_sum(loss_class * class_mask) / (nb_class_box + 1e-6)
loss = loss_xy + loss_wh + loss_conf + loss_class
return loss
def __truediv__(self, other):
return tf.truediv(self, other)
def normalize_to_target(inputs,
target_norm_value,
dim,
epsilon=1e-7,
trainable=True,
scope='NormalizeToTarget',
summarize=True):
"""L2 normalizes the inputs across the specified dimension to a target norm.
This op implements the L2 Normalization layer introduced in
Liu, Wei, et al. "SSD: Single Shot MultiBox Detector."
and Liu, Wei, Andrew Rabinovich, and Alexander C. Berg.
"Parsenet: Looking wider to see better." and is useful for bringing
activations from multiple layers in a convnet to a standard scale.
Note that the rank of `inputs` must be known and the dimension to which
normalization is to be applied should be statically defined.
TODO: Add option to scale by L2 norm of the entire input.
Args:
inputs: A `Tensor` of arbitrary size.
target_norm_value: A float value that specifies an initial target norm or
a list of floats (whose length must be equal to the depth along the
dimension to be normalized) specifying a per-dimension multiplier
after normalization.
dim: The dimension along which the input is normalized.
epsilon: A small value to add to the inputs to avoid dividing by zero.
trainable: Whether the norm is trainable or not
scope: Optional scope for variable_scope.
summarize: Whether or not to add a tensorflow summary for the op.
Returns:
The input tensor normalized to the specified target norm.
Raises:
ValueError: If dim is smaller than the number of dimensions in 'inputs'.
ValueError: If target_norm_value is not a float or a list of floats with
length equal to the depth along the dimension to be normalized.
"""
with tf.variable_scope(scope, 'NormalizeToTarget', [inputs]):
if not inputs.get_shape():
raise ValueError('The input rank must be known.')
input_shape = inputs.get_shape().as_list()
input_rank = len(input_shape)
if dim < 0 or dim >= input_rank:
raise ValueError(
'dim must be non-negative but smaller than the input rank.')
if not input_shape[dim]:
raise ValueError('input shape should be statically defined along '
'the specified dimension.')
depth = input_shape[dim]
if not (isinstance(target_norm_value, float) or
(isinstance(target_norm_value, list)
and len(target_norm_value) == depth)
and all([isinstance(val, float)
for val in target_norm_value])):
raise ValueError(
'target_norm_value must be a float or a list of floats '
'with length equal to the depth along the dimension to '
'be normalized.')
if isinstance(target_norm_value, float):
initial_norm = depth * [target_norm_value]
else:
initial_norm = target_norm_value
target_norm = tf.contrib.framework.model_variable(
name='weights',
dtype=tf.float32,
initializer=tf.constant(initial_norm, dtype=tf.float32),
trainable=trainable)
if summarize:
mean = tf.reduce_mean(target_norm)
mean = tf.Print(mean, ['NormalizeToTarget:', mean])
tf.summary.scalar(tf.get_variable_scope().name, mean)
lengths = epsilon + tf.sqrt(tf.reduce_sum(tf.square(inputs), dim,
True))
mult_shape = input_rank * [1]
mult_shape[dim] = depth
return tf.reshape(target_norm, mult_shape) * tf.truediv(
inputs, lengths)
def darkeras_loss(net_out):
sprob = float(cfg.class_scale)
sconf = float(cfg.object_scale)
snoob = float(cfg.noobject_scale)
scoor = float(cfg.coord_scale)
S, B, C = cfg.cell_size, cfg.boxes_per_cell, cfg.num_classes
SS = S * S # number of grid cells
size1 = [None, SS, C]
size2 = [None, SS, B]
# return the below placeholders
_probs = tf.placeholder(tf.float32, size1)
_confs = tf.placeholder(tf.float32, size2)
_coord = tf.placeholder(tf.float32, size2 + [4])
# weights term for L2 loss
_proid = tf.placeholder(tf.float32, size1)
# material calculating IOU
_areas = tf.placeholder(tf.float32, size2)
_upleft = tf.placeholder(tf.float32, size2 + [2])
_botright = tf.placeholder(tf.float32, size2 + [2])
placeholders = {
'probs': _probs,
'confs': _confs,
'coord': _coord,
'proid': _proid,
'areas': _areas,
'upleft': _upleft,
'botright': _botright
}
# Extract the coordinate prediction from net.out
coords = net_out[:, SS * (C + B):]
coords = tf.reshape(coords, [-1, SS, B, 4])
wh = tf.pow(coords[:, :, :, 2:4], 2) * S # unit: grid cell
area_pred = wh[:, :, :, 0] * wh[:, :, :, 1] # unit: grid cell^2
centers = coords[:, :, :, 0:2] # [batch, SS, B, 2]
floor = centers - (wh * .5) # [batch, SS, B, 2]
ceil = centers + (wh * .5) # [batch, SS, B, 2]
# calculate the intersection areas
intersect_upleft = tf.maximum(floor, _upleft)
intersect_botright = tf.minimum(ceil, _botright)
intersect_wh = intersect_botright - intersect_upleft
intersect_wh = tf.maximum(intersect_wh, 0.0)
intersect = tf.multiply(intersect_wh[:, :, :, 0], intersect_wh[:, :, :, 1])
# calculate the best IOU, set 0.0 confidence for worse boxes
iou = tf.truediv(intersect, _areas + area_pred - intersect)
best_box = tf.equal(iou, tf.reduce_max(iou, [2], True))
best_box = tf.to_float(best_box)
confs = tf.multiply(best_box, _confs)
# take care of the weight terms
conid = snoob * (1. - confs) + sconf * confs
weight_coo = tf.concat(4 * [tf.expand_dims(confs, -1)], 3)
cooid = scoor * weight_coo
proid = sprob * _proid
# flatten 'em all
probs = slim.flatten(_probs)
proid = slim.flatten(proid)
confs = slim.flatten(confs)
conid = slim.flatten(conid)
coord = slim.flatten(_coord)
cooid = slim.flatten(cooid)
# reshape 1 dim vevtor
# probs = tf.reshape(_probs, [-1])
# proid = tf.reshape(proid, [-1])
# confs = tf.reshape(confs, [-1])
# conid = tf.reshape(conid, [-1])
# coord = tf.reshape(_coord, [-1])
# cooid = tf.reshape(cooid, [-1])
true = tf.concat([probs, confs, coord], 1)
wght = tf.concat([proid, conid, cooid], 1)
print('Building {} loss'.format(cfg.model_name))
loss = tf.pow(net_out - true, 2)
loss = tf.multiply(loss, wght)
loss = tf.reduce_sum(loss, 1)
return placeholders, .5 * tf.reduce_mean(loss)
