def loss_function(target_subtoken, y_pred):
# prediction is a probability, log probability for speed and smoothness
print("Model objective: y_pred.shape: {}".format(y_pred.shape))
# I_C = vector of a target subtoken exist in the input token - TODO probably not ok, debug using TF eager
I_C = K.expand_dims(
K.cast(K.any(K.equal(input_code_subtoken,
K.cast(target_subtoken, 'int32')),
axis=-1),
dtype='float32'), -1)
print("Model objective: I_C.shape: {}".format(I_C.shape))
# I_C shape = [batch_size, token, max_char_len, 1]
# TODO should I add a penality if there is no subtokens appearing in the model ? Yes
probability_correct_copy = K.log(copy_probability) + K.log(
K.sum(I_C * copy_weights) + mu)
print("Model objective: probability_correct_copy.shape: {}".format(
probability_correct_copy.shape))
# penalise the model when cnn-attention predicts unknown
# but the value can be predicted from the copy mechanism.
mask_unknown = K.cast(K.equal(target_subtoken, unknown_id),
dtype='float32') * mu
probability_target_token = K.sum(
K.log(1 - copy_probability) + K.log(y_pred) + mask_unknown, -1,
True)
print("Model objective: probability_target_token.shape: {}".format(
probability_target_token.shape))
loss = K.logsumexp(
[probability_correct_copy, probability_target_token])
return K.mean(loss)
def call(self, x, mask=None):
embedding_dim = self.embedding_dim
sequence_length = self.sequence_length
eij = K.reshape(
K.dot(K.reshape(x, (-1, embedding_dim)),
K.reshape(self.W, (embedding_dim, 1))),
(-1, sequence_length))
if self.bias:
eij += self.b
eij = K.tanh(eij)
a = K.exp(eij)
if mask is not None:
a *= K.cast(mask, K.floatx())
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
weighted_input = x * K.expand_dims(a)
output = K.sum(weighted_input, axis=1)
if self.return_attentions:
return output, a
else:
return output
def weighted_bce_dice_loss(y_true, y_pred):
y_true = K.cast(y_true, 'float32')
y_pred = K.cast(y_pred, 'float32')
# if we want to get same size of output, kernel size must be odd number
if K.int_shape(y_pred)[1] == 128:
kernel_size = 11
elif K.int_shape(y_pred)[1] == 256:
kernel_size = 21
elif K.int_shape(y_pred)[1] == 512:
kernel_size = 21
elif K.int_shape(y_pred)[1] == 1024:
kernel_size = 41
else:
raise ValueError('Unexpected image size')
averaged_mask = K.pool2d(y_true,
pool_size=(kernel_size, kernel_size),
strides=(1, 1),
padding='same',
pool_mode='avg')
border = K.cast(K.greater(averaged_mask, 0.005), 'float32') * K.cast(
K.less(averaged_mask, 0.995), 'float32')
weight = K.ones_like(averaged_mask)
w0 = K.sum(weight)
weight += border * 2
w1 = K.sum(weight)
weight *= (w0 / w1)
loss = weighted_bce_loss(y_true, y_pred, weight) + (
1 - weighted_dice_coeff(y_true, y_pred, weight))
return loss
def compute_loss(y_true, y_pred):
'''
Computes "recall-weighted" F1 score: wF1 = 2 * (P * R) / (((1-w) * P) + (w * R))
The coefficient w is calculated from recall_weight.
:param y_true: Ground truth labels
:param y_pred: Model's predictions
:return: The loss, which is calculated from weighted F1 score
'''
# Cast predictions and labels, then calculate quantities in numerator and denominator for precision and recall.
y_pred = K.cast(y_pred, dtype='float32')
y_true = K.cast(y_true, dtype='float32')
actual_positives = K.sum(y_true, axis=0)
predicted_positives = K.sum(y_pred, axis=0)
true_positives = K.sum(y_true * y_pred, axis=0)
# Calculate precision and recall
precision = (true_positives + K.epsilon()) / (predicted_positives + K.epsilon())
recall = (true_positives + K.epsilon()) / (actual_positives + K.epsilon())
# Calculate coefficients to be applied in denominator of F1 score to weigh precision and recall accordingly.
recall_weight = 2 / (recall_factor + 1)
precision_weight = 2 - recall_weight
# Calculate weighted F1 score (wF1)
weighted_f1 = 2 * (precision * recall) / ((precision_weight * precision) + (recall_weight * recall) +
K.epsilon())
loss = 1 - weighted_f1 # Minimizing this quantity will maximize wF1
return broadcast_to(loss, shape(y_true)) # Broadcast to shape that takes into account batch size
def call(self, inputs, **kwargs):
inputs, memory_length = inputs
memory_length = K.cast(memory_length[0][0], 'int32')
batch_size = K.cast(K.shape(inputs)[0], 'int32')
seq_len = K.cast(K.shape(inputs)[1], 'int32')
# Build new memory
pad = K.tile(inputs[0:1, ...], (self.batch_size - batch_size, 1, 1))
padded = K.concatenate([inputs, pad], axis=0) # (self.batch_size, seq_len, output_dim)
new_memory = K.concatenate([self.memory, padded], axis=1) # (self.batch_size, self.memory_len + self.target_len + seq_len, ...)
new_memory = tf.slice( # (self.batch_size, self.memory_len + self.target_len, output_dim)
new_memory,
(0, seq_len, 0),
(self.batch_size, self.memory_len + self.target_len, self.output_dim),
)
self.add_update(K.update(self.memory, new_memory), inputs)
# Build output
old_memory = tf.slice( # (batch_size, memory_length, output_dim)
new_memory,
(0, K.maximum(0, self.memory_len + self.target_len - seq_len - memory_length), 0),
(batch_size, K.minimum(self.memory_len, memory_length), self.output_dim),
)
return old_memory
def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False):
"""Convert final layer features to bounding box parameters."""
num_anchors = len(anchors)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2])
grid_shape = K.shape(feats)[1:3] # height, width
grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
[grid_shape[0], 1, 1, 1])
grid = K.concatenate([grid_x, grid_y])
grid = K.cast(grid, K.dtype(feats))
feats = K.reshape(
feats,
[-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])
# Adjust preditions to each spatial grid point and anchor size.
box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(
grid_shape[::-1], K.dtype(feats))
box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(
input_shape[::-1], K.dtype(feats))
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.sigmoid(feats[..., 5:])
if calc_loss == True:
return grid, feats, box_xy, box_wh
return box_xy, box_wh, box_confidence, box_class_probs
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay *
K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
(1. - K.pow(self.beta_1, t)))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1) for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
# Learning rate multipliers
if self.multipliers:
multiplier = [
mult for mult in self.multipliers if mult in p.name
]
else:
multiplier = None
if multiplier:
new_lr_t = lr_t * self.multipliers[multiplier[0]]
if self.debug_verbose:
print('Setting {} to learning rate {}'.format(
multiplier[0], new_lr_t))
print(K.get_value(new_lr_t))
else:
new_lr_t = lr_t
if self.debug_verbose:
print('No change in learning rate {}'.format(p.name))
print(K.get_value(new_lr_t))
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
p_t = p - new_lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)
self.updates.append(K.update(vhat, vhat_t))
else:
p_t = p - new_lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
def _kernel_constraint(self, kernel):
"""Radially constraints a kernel with shape (height, width, channels)."""
padding = K.constant([[1, 1], [1, 1]], dtype='int32')
kernel_shape = K.shape(kernel)[0]
start = K.cast(kernel_shape / 2, 'int32')
kernel_new = K.switch(
K.cast(math_ops.floormod(kernel_shape, 2), 'bool'),
lambda: kernel[start - 1:start, start - 1:start],
lambda: kernel[start - 1:start, start - 1:start] + K.zeros( # pylint: disable=g-long-lambda
(2, 2), dtype=kernel.dtype))
index = K.switch(K.cast(math_ops.floormod(kernel_shape, 2),
'bool'), lambda: K.constant(0, dtype='int32'),
lambda: K.constant(1, dtype='int32'))
while_condition = lambda index, *args: K.less(index, start)
def body_fn(i, array):
return i + 1, array_ops.pad(array,
padding,
constant_values=kernel[start + i,
start + i])
_, kernel_new = control_flow_ops.while_loop(
while_condition,
body_fn, [index, kernel_new],
shape_invariants=[
index.get_shape(),
tensor_shape.TensorShape([None, None])
])
return kernel_new
def fractional_accuracy(y_true, y_pred):
equal = K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1))
X = K.mean(K.sum(K.cast(equal, tf.float32), axis=-1))
not_equal = K.not_equal(K.argmax(y_true, axis=-1), K.argmax(y_pred,
axis=-1))
Y = K.mean(K.sum(K.cast(not_equal, tf.float32), axis=-1))
return X / (X + Y)
def call(self, inputs, **kwargs):
orig_dtype = inputs.dtype
x = K.cast(inputs, tf.float32)
x -= K.mean(x, axis=[2, 3], keepdims=True)
x /= K.sqrt(K.mean(K.square(x), axis=[2, 3], keepdims=True) + 1e-8)
x = K.cast(x, orig_dtype)
return x
def triplet_loss(y_true, y_pred):
# Euclidean dist between all pairs
dist = K.expand_dims(y_pred, axis=1) - K.expand_dims(y_pred, axis=0)
dist_mat = K.cast(K.sqrt(K.sum(K.square(dist), axis=-1) + K.epsilon()),
dtype='float32')
self_mask = K.cast(K.equal(K.expand_dims(y_true, axis=1),
K.expand_dims(y_true, axis=0)),
dtype='float32')
# Reverse the the positive mask
neg_mask = K.cast(tf.abs(self_mask - 1), dtype='float32')
# Make the sample do not match with itself
diag = tf.linalg.diag_part(self_mask) - tf.linalg.diag_part(self_mask)
pos_mask = K.cast(tf.linalg.set_diag(self_mask, diag), dtype='float32')
# Pick the top K pairs for each positive/negative example(furthest positives and closest negatives)
top_k_pos = tf.nn.top_k(dist_mat * pos_mask, k).values
top_k_neg = tf.abs(
tf.nn.top_k(-1 * (dist_mat * neg_mask + 1e10 * self_mask),
k).values)
loss = K.mean(margin + K.expand_dims(top_k_pos, axis=-1) -
K.expand_dims(top_k_neg, axis=-2))
loss = K.maximum(loss, 0)
return loss
def __init__(self, model, layer_name):
self.model = model
self.layer_name = layer_name
dream = model.input
# Get the symbolic outputs of each "key" layer (we gave them unique names).
layers_all = [layer.name for layer in model.layers]
if layer_name not in layers_all:
raise ValueError('Layer ' + layer_name + ' not found in model.')
# Define the loss.
loss = K.variable(0.)
for layer_local in model.layers:
if layer_local.name == layer_name:
x = layer_local.output
# We avoid border artifacts by only involving non-border pixels in the loss.
if K.image_data_format() == 'channels_first':
scaling = K.prod(K.cast(K.shape(x), 'float32'))
loss = loss + K.sum(K.square(x[:, :, 2:-2,
2:-2])) / scaling
else:
scaling = K.prod(K.cast(K.shape(x), 'float32'))
loss = loss + K.sum(K.square(x[:, 2:-2,
2:-2, :])) / scaling
# Compute the gradients of the dream wrt the loss.
grads = K.gradients(loss, dream)[0]
# Normalize gradients.
grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon())
# Set up function to retrieve the value
# of the loss and gradients given an input image.
outputs = [loss, grads]
self.fetch_loss_and_grads = K.function([dream], outputs)
def YOLOCorrectBoxes(box_xy, box_wh, input_shape, image_shape):
'''Get Corrected Boxes.'''
box_yx = box_xy[..., ::-1]
box_hw = box_wh[..., ::-1]
input_shape = K.cast(input_shape, K.dtype(box_yx))
image_shape = K.cast(image_shape, K.dtype(box_yx))
new_shape = K.round(image_shape * K.min(input_shape / image_shape))
offset = (input_shape - new_shape) / 2. / input_shape
scale = input_shape / new_shape
box_yx = (box_yx - offset) * scale
box_hw *= scale
box_mins = box_yx - (box_hw / 2.)
box_max = box_yx + (box_hw / 2.)
boxes = K.concatenate([
box_mins[..., 0:1], #y min
box_mins[..., 1:2], #x min
box_max[..., 0:1], #y max
box_max[..., 1:2] #x max
])
#Scale boxes back to original image shape
boxes *= K.concatenate([image_shape, image_shape])
return boxes
def call(self, x, mask=None):
eij = dot_product(x, self.W)
if self.bias:
eij += self.b
eij = K.tanh(eij)
a = K.exp(eij)
# apply mask after the exp. will be re-normalized next
if mask is not None:
# Cast the mask to floatX to avoid float64 upcasting in theano
a *= K.cast(mask, K.floatx())
# in some cases especially in the early stages of training the sum may be almost zero
# and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
# a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
weighted_input = x * K.expand_dims(a)
result = K.sum(weighted_input, axis=1)
if self.return_attention:
return [result, a]
return result
def yolo_correct_boxes(box_xy, box_wh, input_shape,
image_shape): #调整框的相对大小以适应原始图像的长宽比
'''Get corrected boxes'''
box_yx = box_xy[..., ::-1]
box_hw = box_wh[..., ::-1]
input_shape = K.cast(input_shape, K.dtype(box_yx))
image_shape = K.cast(image_shape, K.dtype(box_yx))
new_shape = K.round(image_shape * K.min(input_shape / image_shape))
offset = (input_shape - new_shape) / 2. / input_shape
scale = input_shape / new_shape
box_yx = (box_yx - offset) * scale
box_hw *= scale
box_mins = box_yx - (box_hw / 2.)
box_maxes = box_yx + (box_hw / 2.)
boxes = K.concatenate([
box_mins[..., 0:1], # y_min
box_mins[..., 1:2], # x_min
box_maxes[..., 0:1], # y_max
box_maxes[..., 1:2] # x_max
])
# Scale boxes back to original image shape.
boxes *= K.concatenate([image_shape, image_shape])
return boxes
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay *
K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
# Applies bounds on actual learning rate
step_size = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
(1. - K.pow(self.beta_1, t)))
final_lr = self.final_lr * lr / self.base_lr
lower_bound = final_lr * (1. - 1. / (self.gamma * t + 1.))
upper_bound = final_lr * (1. + 1. / (self.gamma * t))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsbound:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1) for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
# apply weight decay
if self.weight_decay != 0.:
g += self.weight_decay * K.stop_gradient(p)
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
if self.amsbound:
vhat_t = K.maximum(vhat, v_t)
denom = (K.sqrt(vhat_t) + self.epsilon)
self.updates.append(K.update(vhat, vhat_t))
else:
denom = (K.sqrt(v_t) + self.epsilon)
# Compute the bounds
step_size_p = step_size * K.ones_like(denom)
step_size_p_bound = step_size_p / denom
bounded_lr_t = m_t * K.minimum(
K.maximum(step_size_p_bound, lower_bound), upper_bound)
p_t = p - bounded_lr_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
def call(self, inputs, mask=None, training=None):
inputs, relatives, memories, bias_context, bias_relative = inputs
full = K.concatenate([memories, inputs], axis=1) # (batch, prev_len + seq_len, units)
w_q = K.dot(inputs, self.kernel_q) # (batch, seq_len, units)
w_kv = K.dot(full, self.kernel_kv) # (batch, prev_len + seq_len, units * 2)
w_r = K.dot(relatives, self.kernel_r) # (batch, prev_len + seq_len, units)
if self.use_bias:
w_q = K.bias_add(w_q, self.bias_q)
w_kv = K.bias_add(w_kv, self.bias_kv)
w_r = K.bias_add(w_r, self.bias_r)
if self.activation is not None:
w_q = self.activation(w_q)
w_kv = self.activation(w_kv)
w_r = self.activation(w_r)
w_k = w_kv[:, :, :self.units] # (batch, prev_len + seq_len, units)
w_v = w_kv[:, :, self.units:] # (batch, prev_len + seq_len, units)
w_qc = K.bias_add(w_q, bias_context)
w_qc = self._reshape_to_batches(w_qc) # (batch * n_head, seq_len, units_head)
w_k = self._reshape_to_batches(w_k) # (batch * n_head, prev_len + seq_len, units_head)
a_context = K.batch_dot(w_qc, w_k, axes=2) # (batch * n_head, seq_len, prev_len + seq_len)
w_qr = K.bias_add(w_q, bias_relative)
w_qr = self._reshape_to_batches(w_qr) # (batch * n_head, seq_len, units_head)
w_r = self._reshape_to_batches(w_r) # (batch * n_head, prev_len + seq_len, units_head)
a_relative = K.batch_dot(w_qr, w_r, axes=2) # (batch * n_head, seq_len, prev_len + seq_len)
a_relative = self._relative_shift(a_relative) # (batch * n_head, seq_len, prev_len + seq_len)
att = (a_context + a_relative) / K.sqrt(K.constant(self.units_head, dtype=K.floatx()))
exp = K.exp(att - K.max(att, axis=-1, keepdims=True))
q_len, k_len = K.shape(w_q)[1], K.shape(w_k)[1]
indices = K.expand_dims(K.arange(0, k_len), axis=0)
upper = K.expand_dims(K.arange(k_len - q_len, k_len), axis=-1)
exp *= K.expand_dims(K.cast(indices <= upper, K.floatx()), axis=0)
if mask is not None and mask[0] is not None:
mask = K.cast(mask[0], K.floatx())
mask = K.concatenate([K.ones_like(memories[:, :, 0]), mask], axis=1)
exp *= K.expand_dims(self._reshape_mask(mask), axis=1)
att = exp / K.sum(exp, axis=-1, keepdims=True)
if self.att_drop_layer is not None:
att = self.att_drop_layer(att, training=training)
w_v = self._reshape_to_batches(w_v) # (batch * n_head, prev_len + seq_len, units_head)
w_o = K.batch_dot(att, w_v) # (batch * n_head, seq_len, units_head)
w_o = self._reshape_from_batches(w_o) # (batch, seq_len, units)
w_o = K.dot(w_o, self.kernel_o) # (batch, seq_len, units)
if self.use_bias:
w_o = K.bias_add(w_o, self.bias_o)
if self.activation is not None:
w_o = self.activation(w_o)
# Add shape information to tensor when using `tf.keras`
input_shape = K.int_shape(inputs)
if input_shape[1] is not None:
w_o = K.reshape(w_o, (-1,) + input_shape[1:])
return w_o
def class2_accuracy(y_true, y_pred):
class_id_true = K.argmax(y_true, axis=-1)
class_id_preds = K.argmax(y_pred, axis=-1)
accuracy_mask = K.cast(K.equal(class_id_preds, 2), 'int32')
class_acc_tensor = K.cast(K.equal(class_id_true, class_id_preds),
'int32') * accuracy_mask
class_acc = K.sum(class_acc_tensor) / K.maximum(K.sum(accuracy_mask), 1)
return class_acc
def loss(y_true, y_pred):
y_true = K.cast(y_true, K.floatx())
mask = K.equal(y_true, mask_value)
mask = 1 - K.cast(mask, K.floatx())
y_true = y_true * mask
loss = K.sparse_categorical_crossentropy(y_true, y_pred) * mask
return K.sum(loss) / K.sum(mask)
def accuracy(y_true, y_pred):
# reshape in case it's in shape (num_samples, 1) instead of (num_samples,)
if K.ndim(y_true) == K.ndim(y_pred):
y_true = K.squeeze(y_true, -1)
# convert dense predictions to labels
y_pred_labels = K.argmax(y_pred, axis=-1)
y_pred_labels = K.cast(y_pred_labels, K.floatx())
return K.cast(K.equal(y_true, y_pred_labels), K.floatx())
def _roi_align(args):
boxes = args[0]
scores = args[1]
fpn = args[2]
# compute from which level to get features from
target_levels = self.map_to_level(boxes)
# process each pyramid independently
rois, ordered_indices = [], []
for i in range(len(fpn)):
# select the boxes and classification from this pyramid level
indices = tf.where(K.equal(target_levels, i))
ordered_indices.append(indices)
level_boxes = tf.gather_nd(boxes, indices)
fpn_shape = K.cast(K.shape(fpn[i]), dtype=K.floatx())
# convert to expected format for crop_and_resize
x1 = level_boxes[:, 0]
y1 = level_boxes[:, 1]
x2 = level_boxes[:, 2]
y2 = level_boxes[:, 3]
level_boxes = K.stack([
(y1 / image_shape[1] * fpn_shape[0]) / (fpn_shape[0] - 1),
(x1 / image_shape[2] * fpn_shape[1]) / (fpn_shape[1] - 1),
(y2 / image_shape[1] * fpn_shape[0] - 1) / (fpn_shape[0] - 1),
(x2 / image_shape[2] * fpn_shape[1] - 1) / (fpn_shape[1] - 1),
], axis=1)
if(len(fpn[i].get_shape()) >=4):
unstack = tf.unstack(fpn[i], axis=3)
temp_stack=[]
for j in unstack:
temp = tf.image.crop_and_resize(
K.expand_dims(j, axis=3),
level_boxes,
tf.zeros((K.shape(level_boxes)[0],), dtype='int32'),
(self.crop_size[0], self.crop_size[1]))
temp_stack.append(temp)
rois.append(temp_stack)
else:
rois.append(tf.image.crop_and_resize(
K.expand_dims(fpn[i], axis=0),
level_boxes,
tf.zeros((K.shape(level_boxes)[0],), dtype='int32'),
self.crop_size
))
# concatenate rois to one blob
rois = K.concatenate(rois, axis=0)
# reorder rois back to original order
indices = K.concatenate(ordered_indices, axis=0)
rois = tf.scatter_nd(indices, rois, K.cast(K.shape(rois), 'int64'))
return rois
def _mask_batch(y_true, y_pred,
iou_threshold=0.5,
mask_size=(28, 28),
parallel_iterations=32):
if K.ndim(y_pred) == 4:
y_pred_shape = tf.shape(y_pred)
new_y_pred_shape = [y_pred_shape[0] * y_pred_shape[1],
y_pred_shape[2], y_pred_shape[3]]
y_pred = tf.reshape(y_pred, new_y_pred_shape)
y_true_shape = tf.shape(y_true)
new_y_true_shape = [y_true_shape[0] * y_true_shape[1],
y_true_shape[2], y_true_shape[3]]
y_true = tf.reshape(y_true, new_y_true_shape)
# split up the different predicted blobs
boxes = y_pred[:, :, :4]
masks = y_pred[:, :, 4:]
# split up the different blobs
annotations = y_true[:, :, :5]
width = K.cast(y_true[0, 0, 5], dtype='int32')
height = K.cast(y_true[0, 0, 6], dtype='int32')
masks_target = y_true[:, :, 7:]
# reshape the masks back to their original size
masks_target = K.reshape(masks_target, (K.shape(masks_target)[0],
K.shape(masks_target)[1],
height, width))
masks = K.reshape(masks, (K.shape(masks)[0], K.shape(masks)[1],
mask_size[0], mask_size[1], -1))
def _mask(args):
boxes = args[0]
masks = args[1]
annotations = args[2]
masks_target = args[3]
return compute_mask_loss(
boxes,
masks,
annotations,
masks_target,
width,
height,
iou_threshold=iou_threshold,
mask_size=mask_size,
)
mask_batch_loss = tf.map_fn(
_mask,
elems=[boxes, masks, annotations, masks_target],
dtype=K.floatx(),
parallel_iterations=parallel_iterations
)
return K.mean(mask_batch_loss)
def lr_decay_schedule_generator(iter):
return tf.where(
tf.greater(
iter, K.cast(number_of_iters_until_decay_generator,
'int64')),
tf.maximum(
0., 1. - (K.cast(iter, 'float32') -
number_of_iters_until_decay_generator) /
number_of_iters_after_decay_generator), 1)
def mask_acc(y_true, y_pred):
y_true_class = K.argmax(y_true, axis=-1)
y_pred_class = K.argmax(y_pred, axis=-1)
ignore_mask = K.cast(K.not_equal(y_true_class, 0), "int32")
matches = K.cast(K.equal(y_true_class, y_pred_class),
"int32") * ignore_mask
accuracy = K.sum(matches) / K.maximum(K.sum(ignore_mask), 1)
return accuracy
def dice(y_true, y_pred):
eps = K.constant(1e-6)
truelabels = tf.argmax(y_true, axis=-1, output_type=tf.int32)
predictions = tf.argmax(y_pred, axis=-1, output_type=tf.int32)
# cast->型変換,minimum2つのテンソルの要素ごとの最小値,equal->boolでかえってくる
intersection = K.cast(K.sum(K.minimum(K.cast(K.equal(predictions, truelabels), tf.int32), truelabels)), tf.float32)
union = tf.count_nonzero(predictions, dtype=tf.float32) + tf.count_nonzero(truelabels, dtype=tf.float32)
dice = 2. * intersection / (union + eps)
return dice
def fallback_metric(self, y_true, y_pred):
#grab the most confident prediction
predictions = K.max(y_pred, axis=-1)
#fill a tensor with our threshold_value
threshold_tensor = tf.fill(tf.shape(predictions), self.threshold)
#Are we confident in our prediction?
threshold_high = predictions > threshold_tensor
threshold_high = tf.cast(threshold_high, tf.int32)
#Do we have low confidence in our prediction?
threshold_low = predictions <= threshold_tensor
threshold_low = tf.cast(threshold_low, tf.int32)
idx_true = K.argmax(y_true, -1)
idx_pred = K.argmax(y_pred, -1)
#For our confident predictions, compare the top prediction to the label of the true value
high_correct = math_ops.equal(idx_true, idx_pred)
high_correct = tf.cast(high_correct, tf.int32)
#For our less confident predictions, grab the top 2 most confident predictions
_, max_pred = tf.math.top_k(y_pred, k=2)
#Gather the lineages of those top 2 predictions using the transpose of the hierarchy's adjaency matrix because the adjacency only points from ancestor to descendant
lineages = tf.gather(K.transpose(self.hierarchy.A), max_pred)
lineages = K.cast(lineages, tf.int32)
#Grab the first two columns of this matrix
fallback = tf.bitwise.bitwise_and(lineages[:, 0], lineages[:, 1])
#Gather the lineage of the true value
actual = tf.gather(K.transpose(self.hierarchy.A), K.argmax(y_true))
actual = K.cast(actual, tf.int32)
#Multiply the two together
overlap_score = K.batch_dot(fallback, actual)
#Are either of the top 2 predictions in the lineage of the true value? If so, overlap_score should be >1 and we count the result as correct
low_correct = overlap_score > 1
low_correct = tf.cast(low_correct, tf.int32)
low_correct = tf.squeeze(low_correct)
#results for the high confidence predictions
high_accuracy = tf.math.multiply(threshold_high, high_correct)
#results for the low confidence predictions
low_accuracy = tf.math.multiply(threshold_low, low_correct)
# total accuracy vector
correct = high_accuracy + low_accuracy
#return batch accuracy value
return K.mean(K.cast(correct, tf.float32))
def f(y_true, y_pred):
class_id_true = K.argmax(y_true, axis=-1)
class_id_preds = K.argmax(y_pred, axis=-1)
# Replace class_id_preds with class_id_true for recall here
accuracy_mask = K.cast(K.equal(class_id_preds, interested_class_id),
'int32')
class_acc_tensor = K.cast(K.equal(class_id_true, class_id_preds),
'int32') * accuracy_mask
class_acc = K.sum(class_acc_tensor) / K.maximum(
K.sum(accuracy_mask), 1)
return class_acc
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
'''Bias corrections according to the Adam paper
'''
lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
(1. - K.pow(self.beta_1, t)))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
####################################################
# Add a lr multiplier for vars outside excluded_vars
if p.name in self.excluded_vars:
multiplied_lr_t = lr_t
else:
multiplied_lr_t = lr_t * self.lr_mult
###################################################
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
'''Schedule multiplier eta_t = 1 for simple AdamW
According to the AdamW paper, eta_t can be fixed, decay, or
also be used for warm restarts (AdamWR to come).
'''
eta_t = 1.
p_t = p - eta_t * (multiplied_lr_t * m_t / (K.sqrt(v_t) + self.epsilon))
if self.weight_decay != 0:
'''Normalized weight decay according to the AdamW paper
'''
w_d = self.weight_decay * K.sqrt(self.batch_size / (self.samples_per_epoch * self.epochs))
p_t = p_t - eta_t * (w_d * p)
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
def custom_loss(y_true, y_pred, loss_weights = loss_weights): # Verified
zero_index = K.zeros_like(y_true[:, 0])
ones_index = K.ones_like(y_true[:, 0])
# Classifier
labels = y_true[:, 0]
class_preds = y_pred[:, 0]
bi_crossentropy_loss = -labels * K.log(class_preds) - (1 - labels) * K.log(1 - class_preds)
classify_valid_index = tf.where(K.less(y_true[:, 0], 0), zero_index, ones_index)
classify_keep_num = K.cast(tf.cast(tf.reduce_sum(classify_valid_index), tf.float32) * SAMPLE_KEEP_RATIO, dtype = tf.int32)
# For classification problem, only pick 70% of the valid samples.
classify_loss_sum = bi_crossentropy_loss * tf.cast(classify_valid_index, bi_crossentropy_loss.dtype)
classify_loss_sum_filtered, _ = tf.nn.top_k(classify_loss_sum, k = classify_keep_num)
classify_loss = tf.where(K.equal(classify_keep_num, 0), tf.constant(0, dtype = tf.float32), K.mean(classify_loss_sum_filtered))
# Bounding box regressor
rois = y_true[:, 1: 5]
roi_preds = y_pred[:, 1: 5]
roi_raw_mean_square_error = K.sum(K.square(rois - roi_preds), axis = 1) # mse
# roi_raw_smooth_l1_loss = K.mean(tf.where(K.abs(rois - roi_preds) < 1, 0.5 * K.square(rois - roi_preds), K.abs(rois - roi_preds) - 0.5)) # L1 Smooth Loss
roi_valid_index = tf.where(K.equal(K.abs(y_true[:, 0]), 1), ones_index, zero_index)
roi_keep_num = K.cast(tf.reduce_sum(roi_valid_index), dtype = tf.int32)
roi_valid_mean_square_error = roi_raw_mean_square_error * tf.cast(roi_valid_index, roi_raw_mean_square_error.dtype)
roi_filtered_mean_square_error, _ = tf.nn.top_k(roi_valid_mean_square_error, k = roi_keep_num)
roi_loss = tf.where(K.equal(roi_keep_num, 0), tf.constant(0, dtype = tf.float32), K.mean(roi_filtered_mean_square_error))
# roi_valid_smooth_l1_loss = roi_raw_smooth_l1_loss * roi_valid_index
# roi_filtered_smooth_l1_loss, _ = tf.nn.top_k(roi_valid_smooth_l1_loss, k = roi_keep_num)
# roi_loss = K.mean(roi_filtered_smooth_l1_loss)
# Landmark regressor
pts = y_true[:, 5: 17]
pt_preds = y_pred[:, 5: 17]
pts_raw_mean_square_error = K.sum(K.square(pts - pt_preds), axis = 1) # mse
# pts_raw_smooth_l1_loss = K.mean(tf.where(K.abs(pts - pt_preds) < 1, 0.5 * K.square(pts - pt_preds), K.abs(pts - pt_preds) - 0.5)) # L1 Smooth Loss
pts_valid_index = tf.where(K.equal(y_true[:, 0], -2), ones_index, zero_index)
pts_keep_num = K.cast(tf.reduce_sum(pts_valid_index), dtype = tf.int32)
pts_valid_mean_square_error = pts_raw_mean_square_error * tf.cast(pts_valid_index, tf.float32)
pts_filtered_mean_square_error, _ = tf.nn.top_k(pts_valid_mean_square_error, k = pts_keep_num)
pts_loss = tf.where(K.equal(pts_keep_num, 0), tf.constant(0, dtype = tf.float32), K.mean(pts_filtered_mean_square_error))
# pts_valid_smooth_l1_loss = pts_raw_smooth_l1_loss * pts_valid_index
# pts_filtered_smooth_l1_loss, _ = tf.nn.top_k(pts_valid_smooth_l1_loss, k = pts_keep_num)
# pts_loss = K.mean(pts_filtered_smooth_l1_loss)
loss = classify_loss * loss_weights[0] + roi_loss * loss_weights[1] + pts_loss * loss_weights[2]
return loss
def f1(y_true, y_pred):
y_pred = K.round(y_pred)
tp = K.sum(K.cast(y_true * y_pred, 'float'), axis=0)
fp = K.sum(K.cast((1 - y_true) * y_pred, 'float'), axis=0)
fn = K.sum(K.cast(y_true * (1 - y_pred), 'float'), axis=0)
p = tp / (tp + fp + K.epsilon())
r = tp / (tp + fn + K.epsilon())
f1 = 2 * p * r / (p + r + K.epsilon())
f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)
return f1
def trivial_model(num_classes, dtype='float32'):
"""Trivial model for ImageNet dataset."""
input_shape = (224, 224, 3)
img_input = layers.Input(shape=input_shape, dtype=dtype)
x = layers.Lambda(lambda x: backend.reshape(x, [-1, 224 * 224 * 3]),
name='reshape')(img_input)
x = layers.Dense(1, name='fc1')(x)
x = layers.Dense(num_classes, name='fc1000')(x)
# TODO(reedwm): Remove manual casts once mixed precision can be enabled with a
# single line of code.
x = backend.cast(x, 'float32')
x = layers.Activation('softmax')(x)
return models.Model(img_input, x, name='trivial')
def get_real_batch_size(self, dataset_batch):
"""Returns the number of elements in a potentially partial batch."""
if isinstance(dataset_batch, (tuple, list)):
dataset_batch = dataset_batch[0]
assert nest.flatten(dataset_batch)
def _find_any_tensor(batch_features):
tensors = [
x for x in nest.flatten(batch_features) if tensor_util.is_tensor(x)
]
if not tensors:
raise ValueError('Cannot find any Tensor in features dict.')
return tensors[0]
return K.cast(K.shape(_find_any_tensor(dataset_batch))[0],
dtype='int64')
def resnet50(num_classes, dtype='float32'):
# TODO(tfboyd): add training argument, just lik resnet56.
"""Instantiates the ResNet50 architecture.
Args:
num_classes: `int` number of classes for image classification.
Returns:
A Keras model instance.
"""
input_shape = (224, 224, 3)
img_input = layers.Input(shape=input_shape, dtype=dtype)
if backend.image_data_format() == 'channels_first':
x = layers.Lambda(lambda x: backend.permute_dimensions(x, (0, 3, 1, 2)),
name='transpose')(img_input)
bn_axis = 1
else: # channels_last
x = img_input
bn_axis = 3
x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(x)
x = layers.Conv2D(64, (7, 7),
strides=(2, 2),
padding='valid', use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name='conv1')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name='bn_conv1')(x)
x = layers.Activation('relu')(x)
x = layers.ZeroPadding2D(padding=(1, 1), name='pool1_pad')(x)
x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = layers.Dense(
num_classes,
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
bias_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name='fc1000')(x)
# TODO(reedwm): Remove manual casts once mixed precision can be enabled with a
# single line of code.
x = backend.cast(x, 'float32')
x = layers.Activation('softmax')(x)
# Create model.
return models.Model(img_input, x, name='resnet50')
