def sentence_accuracy(y_true, y_pred):
y_true_class = K.argmax(y_true, axis=-1)
y_pred_class = K.argmax(y_pred, axis=-1)
ignore_mask = K.cast(K.equal(y_true_class, to_ignore), 'int32')
matches = K.cast(K.equal(y_true_class, y_pred_class), 'int32')
matches = K.any(K.stack([matches, ignore_mask], axis=0),
axis=0) # This is logic OR
accuracy = K.sum(K.cast(K.all(matches, axis=1), 'int32')) / K.sum(
K.cast(K.any(matches, axis=1), 'int32'))
return accuracy
def decorator(self, x):
# Only call layer if there are input spikes. This is to prevent
# accumulation of bias.
self.impulse = tf.cond(k.any(k.not_equal(x, 0)), lambda: call(self, x),
lambda: k.zeros_like(self.mem))
return self.update_neurons()
def compute_mask(self, inputs, mask=None):
if len(inputs[1].shape) == 3:
output_mask = K.any(K.not_equal(inputs[1], self.mask_value),
axis=-1)
else:
output_mask = K.not_equal(inputs[1], self.mask_value)
return output_mask
def _fd_conditional(y_true, y_pred):
# if there are no masks annotations, return 0; else, compute fdl loss
return tf.cond(
K.any(K.equal(K.shape(y_true), 0)),
lambda: K.cast_to_floatx(0.0),
lambda: _fd_batch(y_true, y_pred,
iou_threshold=self.fdl_iou_threshold,
parallel_iterations=self.parallel_iterations))
def call(self, inputs):
if len(inputs[1].shape) == 3:
boolean_mask = K.any(K.not_equal(inputs[1], self.mask_value),
axis=-1,
keepdims=True)
else:
boolean_mask = K.expand_dims(
K.not_equal(inputs[1], self.mask_value))
return inputs[0] * K.cast(boolean_mask, K.dtype(inputs[0]))
def wrapper(y_true, y_pred):
if (usesoftmax):
y_pred = tf.keras.activations.softmax(y_pred)
y_pred = backend.clip(y_pred, _EPSILON, 1.0 - _EPSILON)
""" here all calculations will be based on the class greater than 0, except accuracy"""
avgIOU = 0.0
for i in range(batch_size):
numUnion = 1.0
recall = 0.0
numClass = 0.0
IOU = 0.0
mask = backend.argmax(y_true[i], -1)
pred = backend.argmax(y_pred[i], -1)
for c in np.arange(1, num_classes, 1):
msk_equal = backend.cast(backend.equal(mask, c),
dtype='float32')
masks_sum = backend.sum(msk_equal)
predictions_sum = backend.sum(
backend.cast(backend.equal(pred, c), 'float32'))
numTrue = backend.sum(
backend.cast(backend.equal(pred, c), 'float32') *
backend.cast(backend.equal(mask, c), 'float32'))
unionSize = masks_sum + predictions_sum - numTrue
maskhaslabel = backend.greater(masks_sum, 0)
predhaslabel = backend.greater(predictions_sum, 0)
predormaskexistlabel = backend.any(backend.stack(
[maskhaslabel, predhaslabel], axis=0),
axis=0)
IOU = backend.switch(predormaskexistlabel,
lambda: IOU + numTrue / unionSize,
lambda: IOU)
numUnion = backend.switch(predormaskexistlabel,
lambda: numUnion + 1,
lambda: numUnion)
recall = backend.switch(maskhaslabel,
lambda: recall + numTrue / masks_sum,
lambda: recall)
numClass = backend.switch(maskhaslabel, lambda: numClass + 1,
lambda: numClass)
IOU = IOU / numUnion
avgIOU = avgIOU + IOU
avgIOU = avgIOU / batch_size
iou_loss = 1.0 - avgIOU
# print(np.shape(y_true), np.shape(y_pred))
main_loss = backend.mean(weighted_loss_fn(y_true, y_pred))
# dice_loss = soft_dice_loss(y_true, y_pred)
return main_loss + 0.1 * iou_loss
def new_update(x, new_x):
if x is var and self._do_lazy_optimization(x):
if indices is None:
r = K.any(K.not_equal(grad, 0.),
axis=-1,
keepdims=True)
new_x = x + (new_x - x) * K.cast(r, K.floatx())
return old_update(x, new_x)
else:
return self._resource_scatter_add(
x, indices, K.gather(new_x - x, indices))
return old_update(x, new_x)
def padded_categorical_crossentropy(
y_true: np.ndarray,
y_pred: np.ndarray
) -> float:
""" A padded categorical crossentropy loss function
Expect the last dimension of y_true and y_pred be one hot encoded vectors.
Vectors without any one hot entry (all zeroes) are considered as padding.
# Arguments:
y_true (np.ndarray): ground truth
y_pred (np.ndarray): predictions
# Returns:
the mean categorical crossentropy
"""
mask = K.any(y_true, -1)
mask = K.cast(mask, y_true.dtype)
loss = K.categorical_crossentropy(y_true, y_pred) * mask
return K.sum(loss) / K.sum(mask)
def padded_categorical_hinge(
y_true: np.ndarray,
y_pred: np.ndarray
) -> float:
""" A padded categorical hinge loss function
Expect the last dimension of y_true and y_pred be one hot encoded vectors.
Vectors without any one hot entry (all zeroes) are considered as padding.
# Arguments:
y_true (np.ndarray): ground truth
y_pred (np.ndarray): predictions
# Returns:
the mean categorical hinge
"""
mask = K.any(y_true, axis=-1)
mask = K.cast(mask, y_true.dtype)
pos = K.sum(y_true * y_pred, axis=-1)
neg = K.sum((1.0 - y_true) * y_pred, axis=-1)
loss = K.maximum(0.0, neg - pos + 1) * mask
return K.sum(loss) / K.sum(mask)
def __apply_special_masking(self, tensors):
'''
Private method for concatenating a temporal tensor and a static tensor
mantaining the masking in the temporal tensor. This method is employed
by a lambda layer in the construction of the bifurcating model.
Args
- tensors: a list of keras tensors masked and not masked
Return:
- a masked concatenation of the two tesnors
'''
masked_tensor = tensors[0]
non_masked_tensor = tensors[1]
concat = K.concatenate([masked_tensor, non_masked_tensor])
mask = K.any(K.not_equal(masked_tensor, self.masked), axis=2)
mask = K.cast(mask, K.dtype(concat))
mask = K.stack([mask], axis=2)
return concat * mask
def get_updates(self, loss, params):
# Only for initialization (仅初始化)
self.optimizer.get_updates(loss, params)
# Common updates (常规更新)
dense_params = [p for p in params if p not in self.embeddings]
self.updates = self.optimizer.get_updates(loss, dense_params)
# Sparse update (稀疏更新)
sparse_params = self.embeddings
sparse_grads = self.get_gradients(loss, sparse_params)
sparse_flags = [
K.any(K.not_equal(g, 0), axis=-1, keepdims=True)
for g in sparse_grads
]
original_lr = self.optimizer.lr
for f, p in zip(sparse_flags, sparse_params):
self.optimizer.lr = original_lr * K.cast(f, 'float32')
# updates only when gradients are not equal to zeros.
# (gradients are equal to zeros means these words are not sampled very likely.)
# 仅更新梯度不为0的Embedding（梯度为0意味着这些词很可能是没被采样到的）
self.updates.extend(self.optimizer.get_updates(loss, [p]))
self.optimizer.lr = original_lr
return self.updates
def compute_mask(self, inputs, mask=None):
if mask is None:
return mask
return K.any(mask, axis=1)
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
# first update the number of iterations
self.updates = [K.update_add(self.iterations, 1)]
if self.decay_epochs:
ite_casted = K.cast(self.iterations, K.dtype(self.decay_epochs))
hit_decay_epoch = K.any(K.equal(ite_casted, self.decay_epochs))
#print(hit_decay_epoch)
lr = K.switch(hit_decay_epoch, self.lr['all']*self.decay['all'],
self.lr['all'])
#K.print_tensor(self.lr['all'])
#a = K.switch(hit_decay_epoch,
# K.print_tensor(self.lr['all'],message='Decays:'),
# K.print_tensor(self.lr['all'],message=' '))
self.updates.append(K.update(self.lr['all'],lr))
shapes = [K.int_shape(p) for p in params]
moments = [K.zeros(s) for s in shapes]
self.weights = [self.iterations] + moments
#print(self.weights)
for p, g, m in zip(params, grads, moments):
#print("HEREEEE:", p.name, g, m)
lrptrkey= set_pattern_find(p.name,self.lr.keys())
if lrptrkey:
if self.verbose>0:
print("Setting different learning rate for ", p.name, " : ", K.eval(self.lr[lrptrkey]))
lr = self.lr[lrptrkey]
dcptrkey=set_pattern_find(p.name,self.decay.keys())
if self.decay_epochs and dcptrkey:
lr = K.switch(hit_decay_epoch, self.lr[lrptrkey]*self.decay[dcptrkey],
self.lr[lrptrkey])
self.updates.append(K.update(self.lr[lrptrkey],lr))
if self.verbose>0:
print("Added decay to ", p.name, ": ", K.eval(lr),",",self.decay[dcptrkey])
elif self.decay_epochs:
lr = K.switch(hit_decay_epoch, self.lr[lrptrkey]*self.decay['all'],self.lr[lrptrkey])
self.updates.append(K.update(self.lr[lrptrkey],lr))
if self.verbose>0:
print("Added decay to ", p.name, ": ", K.eval(lr),",",self.decay['all'])
else:
lr = self.lr[lrptrkey]
else:
lr = self.lr['all']
momptrkey = set_pattern_find(p.name,self.momentum.keys())
if momptrkey:
if self.verbose>0:
print("Setting different momentum for ", p.name, " , ",
K.eval(self.momentum[momptrkey]))
momentum = self.momentum[momptrkey]
else:
momentum = self.momentum['all']
v = momentum * m - lr * g # velocity
self.updates.append(K.update(m, v))
if self.nesterov:
new_p = p + momentum * (momentum * m - lr * g) - lr * g
else:
new_p = p + momentum * m - lr * g
# CHANGE CLIP
_to_tensor = K.tensorflow_backend._to_tensor
_clip_by_val = K.tf.clip_by_value
margin = K.mean(K.abs(p*K.constant(self.UPCLIP)))
min_value = _to_tensor(p-margin, p.dtype.base_dtype)
max_value = _to_tensor(p+margin, p.dtype.base_dtype)
max_v = K.maximum(min_value, max_value)
min_v = K.minimum(min_value, max_value)
new_p = _clip_by_val(new_p, min_v, max_v)
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
clptrkey = set_pattern_find(p.name,self.clips.keys())
if self.clips_val and clptrkey:
if self.verbose>0:
print("Clipping variable",p.name," to ", self.clips[clptrkey])
c = K.eval(self.clips[clptrkey])
new_p = K.clip(new_p, c[0], c[1])
#print("updates for ", p.name, " lr: ", K.eval(lr), " mom:", K.eval(momentum))
self.updates.append(K.update(p, new_p))
return self.updates
def build_grided_gt(y_true, mask, size, classes, true_shape, dtype,
use_tie_breaker):
"""
convert ground truth for use in loss functions
Args:
y_true: tf.Tensor[] ground truth [box coords[0:4], classes_onehot[0:-1], best_fit_anchor_box]
mask: list of the anchor boxes choresponding to the output, ex. [1, 2, 3] tells this layer to predict only the first 3 anchors in the total.
size: the dimensions of this output, for regular, it progresses from 13, to 26, to 52
Return:
tf.Tensor[] of shape [batch, size, size, #of_anchors, 4, 1, num_classes]
"""
boxes = tf.cast(y_true["bbox"], dtype)
classes = tf.one_hot(tf.cast(y_true["classes"], dtype=tf.int32),
depth=classes,
dtype=dtype)
anchors = tf.cast(y_true["best_anchors"], dtype)
batches = tf.shape(boxes)[0]
num_boxes = tf.shape(boxes)[1]
len_masks = tf.shape(mask)[0]
full = tf.zeros([batches, size, size, len_masks, true_shape[-1]],
dtype=dtype)
depth_track = tf.zeros((batches, size, size, len_masks), dtype=tf.int32)
x = tf.cast(boxes[..., 0] * tf.cast(size, dtype=dtype), dtype=tf.int32)
y = tf.cast(boxes[..., 1] * tf.cast(size, dtype=dtype), dtype=tf.int32)
anchors = tf.repeat(tf.expand_dims(anchors, axis=-1), len_masks, axis=-1)
update_index = tf.TensorArray(tf.int32, size=0, dynamic_size=True)
update = tf.TensorArray(dtype, size=0, dynamic_size=True)
const = tf.cast(tf.convert_to_tensor([1.]), dtype=dtype)
mask = tf.cast(mask, dtype=dtype)
i = 0
anchor_id = 0
for batch in range(batches):
for box_id in range(num_boxes):
if K.all(tf.math.equal(boxes[batch, box_id, 2:4], 0)):
continue
if K.any(tf.math.less(boxes[batch, box_id, 0:2], 0.0)) or K.any(
tf.math.greater_equal(boxes[batch, box_id, 0:2], 1.0)):
continue
if use_tie_breaker:
for anchor_id in range(tf.shape(anchors)[-1]):
index = tf.math.equal(anchors[batch, box_id, anchor_id],
mask)
if K.any(index):
#tf.print(anchor_id, anchors[batch, box_id, anchor_id])
p = tf.cast(K.argmax(tf.cast(index, dtype=tf.int32)),
dtype=tf.int32)
uid = 1
used = depth_track[batch, y[batch, box_id],
x[batch, box_id], p]
if anchor_id == 0:
# write the box to the update list
# the boxes output from yolo are for some reason have the x and y indexes swapped for some reason, I am not sure why
"""peculiar"""
update_index = update_index.write(
i,
[batch, y[batch, box_id], x[batch, box_id], p])
value = K.concatenate([
boxes[batch, box_id], const, classes[batch,
box_id]
])
update = update.write(i, value)
elif tf.math.equal(used, 2) or tf.math.equal(used, 0):
uid = 2
# write the box to the update list
# the boxes output from yolo are for some reason have the x and y indexes swapped for some reason, I am not sure why
"""peculiar"""
update_index = update_index.write(
i,
[batch, y[batch, box_id], x[batch, box_id], p])
value = K.concatenate([
boxes[batch, box_id], const, classes[batch,
box_id]
])
update = update.write(i, value)
depth_track = tf.tensor_scatter_nd_update(
depth_track,
[(batch, y[batch, box_id], x[batch, box_id], p)],
[uid])
i += 1
else:
index = tf.math.equal(anchors[batch, box_id, 0], mask)
if K.any(index):
#tf.print(0, anchors[batch, box_id, 0])
p = tf.cast(K.argmax(tf.cast(index, dtype=tf.int32)),
dtype=tf.int32)
update_index = update_index.write(
i, [batch, y[batch, box_id], x[batch, box_id], p])
value = K.concatenate(
[boxes[batch, box_id], const, classes[batch, box_id]])
update = update.write(i, value)
i += 1
# if the size of the update list is not 0, do an update, other wise, no boxes and pass an empty grid
if tf.math.greater(update_index.size(), 0):
update_index = update_index.stack()
update = update.stack()
full = tf.tensor_scatter_nd_add(full, update_index, update)
#tf.print(K.sum(full), K.sum(y_true))
return full
def call(self, inputs):
boolean_mask = K.any(K.not_equal(inputs[1], self.mask_value),
axis=-1, keepdims=True)
return inputs[0] * K.cast(boolean_mask, K.dtype(inputs[0]))
def compute_mask(self, input, mask=None):
if mask is not None and self.learn_mode == 'join':
return K.any(mask, axis=1)
return mask
def loss(y_true, y_pred):
y_pred = backend.clip(y_pred, _EPSILON, 1.0 - _EPSILON)
print('---=ytrue', np.shape(y_true))
print('---=ypred', np.shape(y_pred))
""" here all calculations will be based on the class greater than 0, except accuracy"""
avgIOU = backend.variable(0.0)
for i in range(batch_size):
numUnion = backend.variable(1.0)
recall = backend.variable(0.0)
numClass = backend.variable(0.0)
IOU = backend.variable(0.0)
mask = backend.argmax(y_true[i], -1)
pred = backend.argmax(y_pred[i], -1)
print('---=mask', np.shape(mask))
print('---=pred', np.shape(pred))
mask_shape1 = backend.shape(mask)
mask_shape1 = backend.print_tensor(mask_shape1,
message='mask_shape1 = ')
for c in np.arange(1, num_classes, 1):
msk_equal = backend.cast(backend.equal(mask, c),
dtype='float32')
# msks_shape1 = backend.shape(msk_equal)
# msks_shape1 = backend.print_tensor(msks_shape1, message='msk_equal shape = ' )
masks_sum = backend.sum(msk_equal)
msks_shape2 = backend.shape(masks_sum)
msks_shape2 = backend.print_tensor(
msks_shape2, message='masks_sum shape = ')
masks_sum = backend.print_tensor(masks_sum,
message='masks_sum = ')
predictions_sum = backend.sum(
backend.cast(backend.equal(pred, c), 'float32'))
predictions_sum = backend.print_tensor(
predictions_sum, message='predictions_sum = ')
print('---=masks_sum', np.shape(masks_sum))
print('---=predictions_sum', np.shape(predictions_sum))
numTrue = backend.sum(
backend.cast(backend.equal(pred, c), 'float32') *
backend.cast(backend.equal(mask, c), 'float32'))
unionSize = masks_sum + predictions_sum - numTrue
# unionSize = tf.Print(unionSize, [unionSize], "union size : ")
unionSize = backend.print_tensor(unionSize,
message='unionSize = ')
maskhaslabel = backend.greater(masks_sum, 0)
predhaslabel = backend.greater(predictions_sum, 0)
maskhaslabel = backend.print_tensor(maskhaslabel,
message='maskhaslabel = ')
predhaslabel = backend.print_tensor(predhaslabel,
message='maskhaslabel = ')
predormaskexistlabel = backend.any(
backend.stack([maskhaslabel, predhaslabel], axis=0),
axis=0
) # backend.cond(backend.logical_or(maskhaslabel, predhaslabel), lambda:True,lambda:False)
predormaskexistlabel = backend.print_tensor(
predormaskexistlabel, message='predormaskexistlabel = ')
IOU = backend.switch(predormaskexistlabel,
lambda: IOU + numTrue / unionSize,
lambda: IOU)
numUnion = backend.switch(predormaskexistlabel,
lambda: numUnion + 1,
lambda: numUnion)
recall = backend.switch(maskhaslabel,
lambda: recall + numTrue / masks_sum,
lambda: recall)
numClass = backend.switch(maskhaslabel, lambda: numClass + 1,
lambda: numClass)
IOU = IOU / numUnion
avgIOU = avgIOU + IOU
avgIOU = avgIOU / batch_size
iou_loss = 1.0 - avgIOU
# iou_loss = backend.print_tensor(iou_loss, message='\n\n\niouloss = ')
return iou_loss
def bool_match(y_true, y_pred):
return K.switch(K.any(y_true - y_pred.round()), K.variable(0),
K.variable(1))
def pfn_mask_func(X, mask_val=mask_val):
# map mask_val to zero and return 1 elsewhere
return K.cast(K.any(K.not_equal(X, mask_val), axis=-1), K.dtype(X))
def build_grided_gt(y_true, mask, size):
batches = tf.shape(y_true)[0]
num_boxes = tf.shape(y_true)[1]
len_masks = tf.shape(mask)[0]
finshape = tf.convert_to_tensor(
[batches, size, size, len_masks * tf.shape(y_true)[-1]])
full = tf.zeros([batches, size, size, len_masks, tf.shape(y_true)[-1]])
depth_track = tf.zeros((batches, size, size, len_masks), dtype=tf.int32)
x = tf.cast(tf.math.floor(y_true[..., 0] *
tf.cast(size, dtype=tf.float32)),
dtype=tf.int32)
y = tf.cast(tf.math.floor(y_true[..., 1] *
tf.cast(size, dtype=tf.float32)),
dtype=tf.int32)
anchors = tf.repeat(tf.expand_dims(y_true[..., -1], axis=-1),
len_masks,
axis=-1)
update_index = tf.TensorArray(tf.int32, size=0, dynamic_size=True)
update = tf.TensorArray(tf.float32, size=0, dynamic_size=True)
i = 0
for batch in range(batches):
for box_id in range(num_boxes):
if tf.math.equal(y_true[batch, box_id, 3], 0):
continue
index = tf.math.equal(anchors[batch, box_id], mask)
if K.any(index):
p = tf.cast(K.argmax(tf.cast(index, dtype=tf.int8)),
dtype=tf.int32)
# start code for tie breaker, temp check performance
uid = 1
used = depth_track[batch, x[batch, box_id], y[batch, box_id],
p]
count = 0
while tf.math.equal(used, 1) and tf.math.less(count, 3):
uid = 2
count += 1
p = (p + 1) % 3
used = depth_track[batch, x[batch, box_id], y[batch,
box_id], p]
if tf.math.equal(used, 1):
tf.print("skipping")
continue
depth_track = tf.tensor_scatter_nd_update(
depth_track,
[(batch, x[batch, box_id], y[batch, box_id], p)], [uid])
#end code for tie breaker
if x[batch, box_id] < size and y[batch, box_id] < size:
update_index = update_index.write(
i, [batch, x[batch, box_id], y[batch, box_id], p])
test = K.concatenate([
y_true[batch, box_id, 0:4],
tf.convert_to_tensor([1.]), y_true[batch, box_id, 4:-1]
])
update = update.write(i, test)
i += 1
#tf.print(x[batch, box_id], y[batch, box_id], y_true[batch, box_id, 0:2])
"""
used can be:
0 not used
1 used with the correct anchor
2 used with offset anchor
if used is 0 or 2:
do not enter tie breaker (count = 0)
edit box index with the most recent box
if tie breaker was used:
set used to 2
else:
set used to 1
E tensorflow/core/grappler/optimizers/dependency_optimizer.cc:741] Iteration = 0, topological sort failed with message: The graph couldn't be sorted in topological order.
raised likely due to a memory issue? reduced batch size to 2 and it solved the problem? odd
W tensorflow/core/grappler/optimizers/loop_optimizer.cc:906] Skipping loop optimization for Merge node with control input: cond/branch_executed/_11
idk should look into this
18 seconds for 2000 images
"""
if tf.math.greater(update_index.size(), 0):
update_index = update_index.stack()
update = update.stack()
full = tf.tensor_scatter_nd_update(full, update_index, update)
# full = tf.reshape(full, finshape)
# tf.print("gtsum 1", K.sum(y_true))
# tf.print("gtsum full",K.sum(full))
return full
def build_grided_gt(y_true, mask, size):
"""
convert ground truth for use in loss functions
Args:
y_true: tf.Tensor[] ground truth [box coords[0:4], classes_onehot[0:-1], best_fit_anchor_box]
mask: list of the anchor boxes choresponding to the output, ex. [1, 2, 3] tells this layer to predict only the first 3 anchors in the total.
size: the dimensions of this output, for regular, it progresses from 13, to 26, to 52
Return:
tf.Tensor[] of shape [batch, size, size, #of_anchors, 4, 1, num_classes]
"""
batches = tf.shape(y_true)[0]
num_boxes = tf.shape(y_true)[1]
len_masks = tf.shape(mask)[0]
# finshape = tf.convert_to_tensor([batches, size, size, len_masks * tf.shape(y_true)[-1]])
full = tf.zeros([batches, size, size, len_masks, tf.shape(y_true)[-1]])
depth_track = tf.zeros((batches, size, size, len_masks), dtype=tf.int32)
x = tf.cast(y_true[..., 0] * tf.cast(size, dtype=tf.float32),
dtype=tf.int32)
y = tf.cast(y_true[..., 1] * tf.cast(size, dtype=tf.float32),
dtype=tf.int32)
anchors = tf.repeat(tf.expand_dims(y_true[..., -1], axis=-1),
len_masks,
axis=-1)
update_index = tf.TensorArray(tf.int32, size=0, dynamic_size=True)
update = tf.TensorArray(tf.float32, size=0, dynamic_size=True)
i = 0
for batch in range(batches):
for box_id in range(num_boxes):
# if tf.math.equal(K.sum(y_true[batch, box_id, 0:4]), 0.0): # VALUE ERROR is (None, 25)
# continue
if K.all(tf.math.equal(y_true[batch, box_id, 2:4],
0)): # VALUE ERROR is (None, 25)
#tf.print("outer zero: ",y_true[batch, box_id, 0:2])
continue
if K.any(tf.math.less(y_true[batch, box_id, 0:2], 0.0)) or K.any(
tf.math.greater_equal(y_true[batch, box_id, 0:2], 1.0)):
#tf.print("outer vals: ",y_true[batch, box_id, 0:2])
continue
index = tf.math.equal(anchors[batch, box_id], mask)
if K.any(index):
p = tf.cast(K.argmax(tf.cast(index, dtype=tf.int32)),
dtype=tf.int32)
# start code for tie breaker, temp check performance
# find the index of the box
uid = 1
used = depth_track[batch, y[batch, box_id], x[batch, box_id],
p]
count = 0
# check if the next anchor is used used == 1, if so find another box
while tf.math.equal(used, 1) and tf.math.less(count, 3):
uid = 2
count += 1
p = (p + 1) % 3
used = depth_track[batch, x[batch, box_id], y[batch,
box_id], p]
if tf.math.equal(used, 1):
tf.print("skipping")
continue
# set the current index to used = 2, to indicate that it is occupied by something that should not be there, so if another box fits that anchor
# it will be prioritized over the current box.
depth_track = tf.tensor_scatter_nd_update(
depth_track,
[(batch, y[batch, box_id], x[batch, box_id], p)], [uid])
#end code for tie breaker
# write the box to the update list
# the boxes output from yolo are for some reason have the x and y indexes swapped for some reason, I am not sure why
"""peculiar"""
update_index = update_index.write(
i, [batch, y[batch, box_id], x[batch, box_id], p])
value = K.concatenate([
y_true[batch, box_id, 0:4],
tf.convert_to_tensor([1.]), y_true[batch, box_id, 4:-1]
])
update = update.write(i, value)
i += 1
"""
used can be:
0 not used
1 used with the correct anchor
2 used with offset anchor
if used is 0 or 2:
do not enter tie breaker (count = 0)
edit box index with the most recent box
if tie breaker was used:
set used to 2
else:
set used to 1
E tensorflow/core/grappler/optimizers/dependency_optimizer.cc:741] Iteration = 0, topological sort failed with message: The graph couldn't be sorted in topological order.
raised likely due to a memory issue? reduced batch size to 2 and it solved the problem? odd
W tensorflow/core/grappler/optimizers/loop_optimizer.cc:906] Skipping loop optimization for Merge node with control input: cond/branch_executed/_11
idk should look into this
18 seconds for 2000 images
"""
# if the seize of the update list is not 0, do an update, other wise, no boxes and pass an empty grid
if tf.math.greater(update_index.size(), 0):
update_index = update_index.stack()
update = update.stack()
full = tf.tensor_scatter_nd_add(full, update_index, update)
#debug
#tf.print("gtsum: ", K.sum(y_true))
#tf.print("gtsum full",size, " ", K.sum(full))
return full
def count_labels(y):
return K.sum(K.cast(K.any(K.not_equal(y, 0), axis=-1), dtype=K.floatx()))
def compute_mask(self, input, mask=None):
if mask is not None:
return K.any(mask, axis=1)
return mask
def get_updates(self, loss, params):
self.updates = []
self.updates.append(K.update_add(self.state_counter, 1))
self.updates.append(K.update_add(self.iterator, 1))
self.updates.append(K.update_add(self.iterations, 1))
lr = self.lr
## lr exponential decay
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay *
K.cast(self.iterations, K.dtype(self.decay))))
shapes = [K.int_shape(p) for p in params]
x = [K.update(K.zeros(shape), p) for shape, p in zip(shapes, params)]
mu = [K.update(K.zeros(shape), p) for shape, p in zip(shapes, params)]
grads = self.get_gradients(loss, params)
moments = [
K.zeros(shape, name='moment_' + str(i))
for (i, shape) in enumerate(shapes)
]
for x_i, x_prime_i, mu_i, g, m in zip(x, params, mu, grads, moments):
## we update x_prime (if we are in LAngevin steps, we update, otherwise we switch to parameters x_i)
dx_prime_i = g - self.gamma * (x_i - x_prime_i)
x_prime_update_i = K.switch(
K.any(K.stack([
K.equal(self.state_counter, 0),
K.equal(self.num_steps, self.iterator)
],
axis=0),
axis=0), x_i, x_prime_i - self.sgld_step * dx_prime_i +
K.sqrt(self.sgld_step) * self.sgld_noise *
K.random_normal(K.int_shape(x_prime_i)))
# Apply constraints.
if getattr(x_prime_i, 'constraint', None) is not None:
x_prime_update_i = x_prime_i.constraint(x_prime_update_i)
self.updates.append(K.update(x_prime_i, x_prime_update_i))
## We update mu (if we are in LAngevin steps, we update otherwise we switch to parameters x_i)
mu_update_i = K.switch(K.equal(self.state_counter,
0), x_i, (1 - self.alpha) * mu_i +
self.alpha * x_prime_i)
self.updates.append(K.update(mu_i, mu_update_i))
## As they described in the paper, we remove the gamma from the update because it interferes with the learning annealing
## After each outer loop update we apply an exponential decay on gamma
## The following lines concerns the outer loop updates
## Nesterov's momentum
gradient = (x_i - mu_i)
v = self.momentum * m - lr * gradient # velocity
self.updates.append(
K.update(
m, K.switch(K.equal(self.state_counter, self.L + 1), v,
m)))
if self.nesterov:
new_x_i = x_i + self.momentum * v - lr * gradient
else:
new_x_i = x_i + v
x_i_update = K.switch(K.equal(self.state_counter, self.L + 1),
new_x_i, x_i)
self.updates.append(K.update(x_i, x_i_update))
## Gamma scoping
gamma_update = K.switch(K.equal(self.state_counter,
self.L + 1), self.gamma,
self.gamma * (1. + self.scoping))
self.updates.append(K.update(self.gamma, gamma_update))
counter = K.switch(K.equal(self.state_counter, self.L + 2),
K.constant(0, dtype='int64'), self.state_counter)
self.updates.append(K.update(self.state_counter, counter))
return self.updates
def build_batch_grided_gt(y_true, mask, size, num_classes, dtype,
use_tie_breaker):
"""
convert ground truth for use in loss functions
Args:
y_true: tf.Tensor[] ground truth [batch, box coords[0:4], classes_onehot[0:-1], best_fit_anchor_box]
mask: list of the anchor boxes choresponding to the output, ex. [1, 2, 3] tells this layer to predict only the first 3 anchors in the total.
size: the dimensions of this output, for regular, it progresses from 13, to 26, to 52
num_classes: `integer` for the number of classes
dtype: expected output datatype
use_tie_breaker: boolean value for wether or not to use the tie_breaker
Return:
tf.Tensor[] of shape [batch, size, size, #of_anchors, 4, 1, num_classes]
"""
# unpack required components from the input ground truth
boxes = tf.cast(y_true['bbox'], dtype)
classes = tf.expand_dims(tf.cast(y_true['classes'], dtype=dtype), axis=-1)
anchors = tf.cast(y_true['best_anchors'], dtype)
# get the batch size
batches = tf.shape(boxes)[0]
# get the number of boxes in the ground truth boxs
num_boxes = tf.shape(boxes)[1]
# get the number of anchor boxes used for this anchor scale
len_masks = tf.shape(mask)[0]
# init a fixed memeory size grid for this prediction scale
# [batch, size, size, # of anchors, 1 + 1 + number of anchors per scale]
full = tf.zeros([batches, size, size, len_masks, 1 + 4 + 1], dtype=dtype)
# init a grid to use to track which locations have already
# been used before (for the tie breaker)
depth_track = tf.zeros((batches, size, size, len_masks), dtype=tf.int32)
# rescale the x and y centers to the size of the grid [size, size]
x = tf.cast(boxes[..., 0] * tf.cast(size, dtype=dtype), dtype=tf.int32)
y = tf.cast(boxes[..., 1] * tf.cast(size, dtype=dtype), dtype=tf.int32)
# init all the tensorArrays to be used in storeing the index and the values
# to be used to update both depth_track and full
update_index = tf.TensorArray(tf.int32, size=0, dynamic_size=True)
update = tf.TensorArray(dtype, size=0, dynamic_size=True)
# init constants and match data types before entering loop
i = 0
anchor_id = 0
const = tf.cast(tf.convert_to_tensor([1.]), dtype=dtype)
mask = tf.cast(mask, dtype=dtype)
rand_update = 0.0
for batch in range(batches):
for box_id in range(num_boxes):
# if the width or height of the box is zero, skip it
if K.all(tf.math.equal(boxes[batch, box_id, 2:4], 0)):
continue
# after pre processing, if the box is not in the image bounds anymore
# skip the box
if K.any(tf.math.less(boxes[batch, box_id, 0:2], 0.0)) or K.any(
tf.math.greater_equal(boxes[batch, box_id, 0:2], 1.0)):
continue
if use_tie_breaker:
for anchor_id in range(tf.shape(anchors)[-1]):
index = tf.math.equal(anchors[batch, box_id, anchor_id],
mask)
if K.any(index):
# using the boolean index mask to determine exactly which anchor box was used
p = tf.cast(K.argmax(tf.cast(index, dtype=tf.int32)),
dtype=tf.int32)
# determine if the index was used or not
used = depth_track[batch, y[batch, box_id],
x[batch, box_id], p]
# defualt used upadte value
uid = 1
# if anchor_id is 0, this is the best matched anchor for this box
# with the highest IOU
if anchor_id == 0:
# create random number to trigger a replacment if the cell is used already
if tf.math.equal(used, 1):
rand_update = tf.random.uniform([], maxval=1)
else:
rand_update = 1.0
if rand_update > 0.5:
# write the box to the update list
update_index = update_index.write(
i, [
batch, y[batch, box_id], x[batch,
box_id], p
])
value = K.concatenate([
boxes[batch, box_id], const,
classes[batch, box_id]
])
update = update.write(i, value)
# if used is 2, this cell is filled with a non-optimal box
# if used is 0, the cell in the ground truth is not yet consumed
# in either case you can replace that cell with a new box, as long
# as it is not consumed by an optimal box with anchor_id = 0
elif tf.math.equal(used, 2) or tf.math.equal(used, 0):
uid = 2
# write the box to the update list
update_index = update_index.write(
i,
[batch, y[batch, box_id], x[batch, box_id], p])
value = K.concatenate([
boxes[batch, box_id], const, classes[batch,
box_id]
])
update = update.write(i, value)
# update the used index for where and how the box was placed
depth_track = tf.tensor_scatter_nd_update(
depth_track,
[(batch, y[batch, box_id], x[batch, box_id], p)],
[uid])
i += 1
else:
index = tf.math.equal(anchors[batch, box_id, 0], mask)
if K.any(index):
# if any there is an index match
p = tf.cast(K.argmax(tf.cast(index, dtype=tf.int32)),
dtype=tf.int32)
# write the box to the update list
update_index = update_index.write(
i, [batch, y[batch, box_id], x[batch, box_id], p])
value = K.concatenate(
[boxes[batch, box_id], const, classes[batch, box_id]])
update = update.write(i, value)
i += 1
# if the size of the update list is not 0, do an update, other wise, no boxes and pass an empty grid
if tf.math.greater(update_index.size(), 0):
update_index = update_index.stack()
update = update.stack()
full = tf.tensor_scatter_nd_update(full, update_index, update)
return full
def compute_mask(self, inputs, mask=None):
# pylint: disable=unused-argument
if mask is None:
return None
return K.any(mask, axis=self.axis)
