def sentence_accuracy(y_true, y_pred): y_true_class = K.argmax(y_true, axis=-1) y_pred_class = K.argmax(y_pred, axis=-1) ignore_mask = K.cast(K.equal(y_true_class, to_ignore), 'int32') matches = K.cast(K.equal(y_true_class, y_pred_class), 'int32') matches = K.any(K.stack([matches, ignore_mask], axis=0), axis=0) # This is logic OR accuracy = K.sum(K.cast(K.all(matches, axis=1), 'int32')) / K.sum( K.cast(K.any(matches, axis=1), 'int32')) return accuracy
def decorator(self, x): # Only call layer if there are input spikes. This is to prevent # accumulation of bias. self.impulse = tf.cond(k.any(k.not_equal(x, 0)), lambda: call(self, x), lambda: k.zeros_like(self.mem)) return self.update_neurons()
def compute_mask(self, inputs, mask=None): if len(inputs[1].shape) == 3: output_mask = K.any(K.not_equal(inputs[1], self.mask_value), axis=-1) else: output_mask = K.not_equal(inputs[1], self.mask_value) return output_mask
def _fd_conditional(y_true, y_pred): # if there are no masks annotations, return 0; else, compute fdl loss return tf.cond( K.any(K.equal(K.shape(y_true), 0)), lambda: K.cast_to_floatx(0.0), lambda: _fd_batch(y_true, y_pred, iou_threshold=self.fdl_iou_threshold, parallel_iterations=self.parallel_iterations))
def call(self, inputs): if len(inputs[1].shape) == 3: boolean_mask = K.any(K.not_equal(inputs[1], self.mask_value), axis=-1, keepdims=True) else: boolean_mask = K.expand_dims( K.not_equal(inputs[1], self.mask_value)) return inputs[0] * K.cast(boolean_mask, K.dtype(inputs[0]))
def wrapper(y_true, y_pred): if (usesoftmax): y_pred = tf.keras.activations.softmax(y_pred) y_pred = backend.clip(y_pred, _EPSILON, 1.0 - _EPSILON) """ here all calculations will be based on the class greater than 0, except accuracy""" avgIOU = 0.0 for i in range(batch_size): numUnion = 1.0 recall = 0.0 numClass = 0.0 IOU = 0.0 mask = backend.argmax(y_true[i], -1) pred = backend.argmax(y_pred[i], -1) for c in np.arange(1, num_classes, 1): msk_equal = backend.cast(backend.equal(mask, c), dtype='float32') masks_sum = backend.sum(msk_equal) predictions_sum = backend.sum( backend.cast(backend.equal(pred, c), 'float32')) numTrue = backend.sum( backend.cast(backend.equal(pred, c), 'float32') * backend.cast(backend.equal(mask, c), 'float32')) unionSize = masks_sum + predictions_sum - numTrue maskhaslabel = backend.greater(masks_sum, 0) predhaslabel = backend.greater(predictions_sum, 0) predormaskexistlabel = backend.any(backend.stack( [maskhaslabel, predhaslabel], axis=0), axis=0) IOU = backend.switch(predormaskexistlabel, lambda: IOU + numTrue / unionSize, lambda: IOU) numUnion = backend.switch(predormaskexistlabel, lambda: numUnion + 1, lambda: numUnion) recall = backend.switch(maskhaslabel, lambda: recall + numTrue / masks_sum, lambda: recall) numClass = backend.switch(maskhaslabel, lambda: numClass + 1, lambda: numClass) IOU = IOU / numUnion avgIOU = avgIOU + IOU avgIOU = avgIOU / batch_size iou_loss = 1.0 - avgIOU # print(np.shape(y_true), np.shape(y_pred)) main_loss = backend.mean(weighted_loss_fn(y_true, y_pred)) # dice_loss = soft_dice_loss(y_true, y_pred) return main_loss + 0.1 * iou_loss
def new_update(x, new_x): if x is var and self._do_lazy_optimization(x): if indices is None: r = K.any(K.not_equal(grad, 0.), axis=-1, keepdims=True) new_x = x + (new_x - x) * K.cast(r, K.floatx()) return old_update(x, new_x) else: return self._resource_scatter_add( x, indices, K.gather(new_x - x, indices)) return old_update(x, new_x)
def padded_categorical_crossentropy( y_true: np.ndarray, y_pred: np.ndarray ) -> float: """ A padded categorical crossentropy loss function Expect the last dimension of y_true and y_pred be one hot encoded vectors. Vectors without any one hot entry (all zeroes) are considered as padding. # Arguments: y_true (np.ndarray): ground truth y_pred (np.ndarray): predictions # Returns: the mean categorical crossentropy """ mask = K.any(y_true, -1) mask = K.cast(mask, y_true.dtype) loss = K.categorical_crossentropy(y_true, y_pred) * mask return K.sum(loss) / K.sum(mask)
def padded_categorical_hinge( y_true: np.ndarray, y_pred: np.ndarray ) -> float: """ A padded categorical hinge loss function Expect the last dimension of y_true and y_pred be one hot encoded vectors. Vectors without any one hot entry (all zeroes) are considered as padding. # Arguments: y_true (np.ndarray): ground truth y_pred (np.ndarray): predictions # Returns: the mean categorical hinge """ mask = K.any(y_true, axis=-1) mask = K.cast(mask, y_true.dtype) pos = K.sum(y_true * y_pred, axis=-1) neg = K.sum((1.0 - y_true) * y_pred, axis=-1) loss = K.maximum(0.0, neg - pos + 1) * mask return K.sum(loss) / K.sum(mask)
def __apply_special_masking(self, tensors): ''' Private method for concatenating a temporal tensor and a static tensor mantaining the masking in the temporal tensor. This method is employed by a lambda layer in the construction of the bifurcating model. Args - tensors: a list of keras tensors masked and not masked Return: - a masked concatenation of the two tesnors ''' masked_tensor = tensors[0] non_masked_tensor = tensors[1] concat = K.concatenate([masked_tensor, non_masked_tensor]) mask = K.any(K.not_equal(masked_tensor, self.masked), axis=2) mask = K.cast(mask, K.dtype(concat)) mask = K.stack([mask], axis=2) return concat * mask
def get_updates(self, loss, params): # Only for initialization (仅初始化) self.optimizer.get_updates(loss, params) # Common updates (常规更新) dense_params = [p for p in params if p not in self.embeddings] self.updates = self.optimizer.get_updates(loss, dense_params) # Sparse update (稀疏更新) sparse_params = self.embeddings sparse_grads = self.get_gradients(loss, sparse_params) sparse_flags = [ K.any(K.not_equal(g, 0), axis=-1, keepdims=True) for g in sparse_grads ] original_lr = self.optimizer.lr for f, p in zip(sparse_flags, sparse_params): self.optimizer.lr = original_lr * K.cast(f, 'float32') # updates only when gradients are not equal to zeros. # (gradients are equal to zeros means these words are not sampled very likely.) # 仅更新梯度不为0的Embedding(梯度为0意味着这些词很可能是没被采样到的) self.updates.extend(self.optimizer.get_updates(loss, [p])) self.optimizer.lr = original_lr return self.updates
def compute_mask(self, inputs, mask=None): if mask is None: return mask return K.any(mask, axis=1)
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) # first update the number of iterations self.updates = [K.update_add(self.iterations, 1)] if self.decay_epochs: ite_casted = K.cast(self.iterations, K.dtype(self.decay_epochs)) hit_decay_epoch = K.any(K.equal(ite_casted, self.decay_epochs)) #print(hit_decay_epoch) lr = K.switch(hit_decay_epoch, self.lr['all']*self.decay['all'], self.lr['all']) #K.print_tensor(self.lr['all']) #a = K.switch(hit_decay_epoch, # K.print_tensor(self.lr['all'],message='Decays:'), # K.print_tensor(self.lr['all'],message=' ')) self.updates.append(K.update(self.lr['all'],lr)) shapes = [K.int_shape(p) for p in params] moments = [K.zeros(s) for s in shapes] self.weights = [self.iterations] + moments #print(self.weights) for p, g, m in zip(params, grads, moments): #print("HEREEEE:", p.name, g, m) lrptrkey= set_pattern_find(p.name,self.lr.keys()) if lrptrkey: if self.verbose>0: print("Setting different learning rate for ", p.name, " : ", K.eval(self.lr[lrptrkey])) lr = self.lr[lrptrkey] dcptrkey=set_pattern_find(p.name,self.decay.keys()) if self.decay_epochs and dcptrkey: lr = K.switch(hit_decay_epoch, self.lr[lrptrkey]*self.decay[dcptrkey], self.lr[lrptrkey]) self.updates.append(K.update(self.lr[lrptrkey],lr)) if self.verbose>0: print("Added decay to ", p.name, ": ", K.eval(lr),",",self.decay[dcptrkey]) elif self.decay_epochs: lr = K.switch(hit_decay_epoch, self.lr[lrptrkey]*self.decay['all'],self.lr[lrptrkey]) self.updates.append(K.update(self.lr[lrptrkey],lr)) if self.verbose>0: print("Added decay to ", p.name, ": ", K.eval(lr),",",self.decay['all']) else: lr = self.lr[lrptrkey] else: lr = self.lr['all'] momptrkey = set_pattern_find(p.name,self.momentum.keys()) if momptrkey: if self.verbose>0: print("Setting different momentum for ", p.name, " , ", K.eval(self.momentum[momptrkey])) momentum = self.momentum[momptrkey] else: momentum = self.momentum['all'] v = momentum * m - lr * g # velocity self.updates.append(K.update(m, v)) if self.nesterov: new_p = p + momentum * (momentum * m - lr * g) - lr * g else: new_p = p + momentum * m - lr * g # CHANGE CLIP _to_tensor = K.tensorflow_backend._to_tensor _clip_by_val = K.tf.clip_by_value margin = K.mean(K.abs(p*K.constant(self.UPCLIP))) min_value = _to_tensor(p-margin, p.dtype.base_dtype) max_value = _to_tensor(p+margin, p.dtype.base_dtype) max_v = K.maximum(min_value, max_value) min_v = K.minimum(min_value, max_value) new_p = _clip_by_val(new_p, min_v, max_v) # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) clptrkey = set_pattern_find(p.name,self.clips.keys()) if self.clips_val and clptrkey: if self.verbose>0: print("Clipping variable",p.name," to ", self.clips[clptrkey]) c = K.eval(self.clips[clptrkey]) new_p = K.clip(new_p, c[0], c[1]) #print("updates for ", p.name, " lr: ", K.eval(lr), " mom:", K.eval(momentum)) self.updates.append(K.update(p, new_p)) return self.updates
def build_grided_gt(y_true, mask, size, classes, true_shape, dtype, use_tie_breaker): """ convert ground truth for use in loss functions Args: y_true: tf.Tensor[] ground truth [box coords[0:4], classes_onehot[0:-1], best_fit_anchor_box] mask: list of the anchor boxes choresponding to the output, ex. [1, 2, 3] tells this layer to predict only the first 3 anchors in the total. size: the dimensions of this output, for regular, it progresses from 13, to 26, to 52 Return: tf.Tensor[] of shape [batch, size, size, #of_anchors, 4, 1, num_classes] """ boxes = tf.cast(y_true["bbox"], dtype) classes = tf.one_hot(tf.cast(y_true["classes"], dtype=tf.int32), depth=classes, dtype=dtype) anchors = tf.cast(y_true["best_anchors"], dtype) batches = tf.shape(boxes)[0] num_boxes = tf.shape(boxes)[1] len_masks = tf.shape(mask)[0] full = tf.zeros([batches, size, size, len_masks, true_shape[-1]], dtype=dtype) depth_track = tf.zeros((batches, size, size, len_masks), dtype=tf.int32) x = tf.cast(boxes[..., 0] * tf.cast(size, dtype=dtype), dtype=tf.int32) y = tf.cast(boxes[..., 1] * tf.cast(size, dtype=dtype), dtype=tf.int32) anchors = tf.repeat(tf.expand_dims(anchors, axis=-1), len_masks, axis=-1) update_index = tf.TensorArray(tf.int32, size=0, dynamic_size=True) update = tf.TensorArray(dtype, size=0, dynamic_size=True) const = tf.cast(tf.convert_to_tensor([1.]), dtype=dtype) mask = tf.cast(mask, dtype=dtype) i = 0 anchor_id = 0 for batch in range(batches): for box_id in range(num_boxes): if K.all(tf.math.equal(boxes[batch, box_id, 2:4], 0)): continue if K.any(tf.math.less(boxes[batch, box_id, 0:2], 0.0)) or K.any( tf.math.greater_equal(boxes[batch, box_id, 0:2], 1.0)): continue if use_tie_breaker: for anchor_id in range(tf.shape(anchors)[-1]): index = tf.math.equal(anchors[batch, box_id, anchor_id], mask) if K.any(index): #tf.print(anchor_id, anchors[batch, box_id, anchor_id]) p = tf.cast(K.argmax(tf.cast(index, dtype=tf.int32)), dtype=tf.int32) uid = 1 used = depth_track[batch, y[batch, box_id], x[batch, box_id], p] if anchor_id == 0: # write the box to the update list # the boxes output from yolo are for some reason have the x and y indexes swapped for some reason, I am not sure why """peculiar""" update_index = update_index.write( i, [batch, y[batch, box_id], x[batch, box_id], p]) value = K.concatenate([ boxes[batch, box_id], const, classes[batch, box_id] ]) update = update.write(i, value) elif tf.math.equal(used, 2) or tf.math.equal(used, 0): uid = 2 # write the box to the update list # the boxes output from yolo are for some reason have the x and y indexes swapped for some reason, I am not sure why """peculiar""" update_index = update_index.write( i, [batch, y[batch, box_id], x[batch, box_id], p]) value = K.concatenate([ boxes[batch, box_id], const, classes[batch, box_id] ]) update = update.write(i, value) depth_track = tf.tensor_scatter_nd_update( depth_track, [(batch, y[batch, box_id], x[batch, box_id], p)], [uid]) i += 1 else: index = tf.math.equal(anchors[batch, box_id, 0], mask) if K.any(index): #tf.print(0, anchors[batch, box_id, 0]) p = tf.cast(K.argmax(tf.cast(index, dtype=tf.int32)), dtype=tf.int32) update_index = update_index.write( i, [batch, y[batch, box_id], x[batch, box_id], p]) value = K.concatenate( [boxes[batch, box_id], const, classes[batch, box_id]]) update = update.write(i, value) i += 1 # if the size of the update list is not 0, do an update, other wise, no boxes and pass an empty grid if tf.math.greater(update_index.size(), 0): update_index = update_index.stack() update = update.stack() full = tf.tensor_scatter_nd_add(full, update_index, update) #tf.print(K.sum(full), K.sum(y_true)) return full
def call(self, inputs): boolean_mask = K.any(K.not_equal(inputs[1], self.mask_value), axis=-1, keepdims=True) return inputs[0] * K.cast(boolean_mask, K.dtype(inputs[0]))
def compute_mask(self, input, mask=None): if mask is not None and self.learn_mode == 'join': return K.any(mask, axis=1) return mask
def loss(y_true, y_pred): y_pred = backend.clip(y_pred, _EPSILON, 1.0 - _EPSILON) print('---=ytrue', np.shape(y_true)) print('---=ypred', np.shape(y_pred)) """ here all calculations will be based on the class greater than 0, except accuracy""" avgIOU = backend.variable(0.0) for i in range(batch_size): numUnion = backend.variable(1.0) recall = backend.variable(0.0) numClass = backend.variable(0.0) IOU = backend.variable(0.0) mask = backend.argmax(y_true[i], -1) pred = backend.argmax(y_pred[i], -1) print('---=mask', np.shape(mask)) print('---=pred', np.shape(pred)) mask_shape1 = backend.shape(mask) mask_shape1 = backend.print_tensor(mask_shape1, message='mask_shape1 = ') for c in np.arange(1, num_classes, 1): msk_equal = backend.cast(backend.equal(mask, c), dtype='float32') # msks_shape1 = backend.shape(msk_equal) # msks_shape1 = backend.print_tensor(msks_shape1, message='msk_equal shape = ' ) masks_sum = backend.sum(msk_equal) msks_shape2 = backend.shape(masks_sum) msks_shape2 = backend.print_tensor( msks_shape2, message='masks_sum shape = ') masks_sum = backend.print_tensor(masks_sum, message='masks_sum = ') predictions_sum = backend.sum( backend.cast(backend.equal(pred, c), 'float32')) predictions_sum = backend.print_tensor( predictions_sum, message='predictions_sum = ') print('---=masks_sum', np.shape(masks_sum)) print('---=predictions_sum', np.shape(predictions_sum)) numTrue = backend.sum( backend.cast(backend.equal(pred, c), 'float32') * backend.cast(backend.equal(mask, c), 'float32')) unionSize = masks_sum + predictions_sum - numTrue # unionSize = tf.Print(unionSize, [unionSize], "union size : ") unionSize = backend.print_tensor(unionSize, message='unionSize = ') maskhaslabel = backend.greater(masks_sum, 0) predhaslabel = backend.greater(predictions_sum, 0) maskhaslabel = backend.print_tensor(maskhaslabel, message='maskhaslabel = ') predhaslabel = backend.print_tensor(predhaslabel, message='maskhaslabel = ') predormaskexistlabel = backend.any( backend.stack([maskhaslabel, predhaslabel], axis=0), axis=0 ) # backend.cond(backend.logical_or(maskhaslabel, predhaslabel), lambda:True,lambda:False) predormaskexistlabel = backend.print_tensor( predormaskexistlabel, message='predormaskexistlabel = ') IOU = backend.switch(predormaskexistlabel, lambda: IOU + numTrue / unionSize, lambda: IOU) numUnion = backend.switch(predormaskexistlabel, lambda: numUnion + 1, lambda: numUnion) recall = backend.switch(maskhaslabel, lambda: recall + numTrue / masks_sum, lambda: recall) numClass = backend.switch(maskhaslabel, lambda: numClass + 1, lambda: numClass) IOU = IOU / numUnion avgIOU = avgIOU + IOU avgIOU = avgIOU / batch_size iou_loss = 1.0 - avgIOU # iou_loss = backend.print_tensor(iou_loss, message='\n\n\niouloss = ') return iou_loss
def bool_match(y_true, y_pred): return K.switch(K.any(y_true - y_pred.round()), K.variable(0), K.variable(1))
def pfn_mask_func(X, mask_val=mask_val): # map mask_val to zero and return 1 elsewhere return K.cast(K.any(K.not_equal(X, mask_val), axis=-1), K.dtype(X))
def build_grided_gt(y_true, mask, size): batches = tf.shape(y_true)[0] num_boxes = tf.shape(y_true)[1] len_masks = tf.shape(mask)[0] finshape = tf.convert_to_tensor( [batches, size, size, len_masks * tf.shape(y_true)[-1]]) full = tf.zeros([batches, size, size, len_masks, tf.shape(y_true)[-1]]) depth_track = tf.zeros((batches, size, size, len_masks), dtype=tf.int32) x = tf.cast(tf.math.floor(y_true[..., 0] * tf.cast(size, dtype=tf.float32)), dtype=tf.int32) y = tf.cast(tf.math.floor(y_true[..., 1] * tf.cast(size, dtype=tf.float32)), dtype=tf.int32) anchors = tf.repeat(tf.expand_dims(y_true[..., -1], axis=-1), len_masks, axis=-1) update_index = tf.TensorArray(tf.int32, size=0, dynamic_size=True) update = tf.TensorArray(tf.float32, size=0, dynamic_size=True) i = 0 for batch in range(batches): for box_id in range(num_boxes): if tf.math.equal(y_true[batch, box_id, 3], 0): continue index = tf.math.equal(anchors[batch, box_id], mask) if K.any(index): p = tf.cast(K.argmax(tf.cast(index, dtype=tf.int8)), dtype=tf.int32) # start code for tie breaker, temp check performance uid = 1 used = depth_track[batch, x[batch, box_id], y[batch, box_id], p] count = 0 while tf.math.equal(used, 1) and tf.math.less(count, 3): uid = 2 count += 1 p = (p + 1) % 3 used = depth_track[batch, x[batch, box_id], y[batch, box_id], p] if tf.math.equal(used, 1): tf.print("skipping") continue depth_track = tf.tensor_scatter_nd_update( depth_track, [(batch, x[batch, box_id], y[batch, box_id], p)], [uid]) #end code for tie breaker if x[batch, box_id] < size and y[batch, box_id] < size: update_index = update_index.write( i, [batch, x[batch, box_id], y[batch, box_id], p]) test = K.concatenate([ y_true[batch, box_id, 0:4], tf.convert_to_tensor([1.]), y_true[batch, box_id, 4:-1] ]) update = update.write(i, test) i += 1 #tf.print(x[batch, box_id], y[batch, box_id], y_true[batch, box_id, 0:2]) """ used can be: 0 not used 1 used with the correct anchor 2 used with offset anchor if used is 0 or 2: do not enter tie breaker (count = 0) edit box index with the most recent box if tie breaker was used: set used to 2 else: set used to 1 E tensorflow/core/grappler/optimizers/dependency_optimizer.cc:741] Iteration = 0, topological sort failed with message: The graph couldn't be sorted in topological order. raised likely due to a memory issue? reduced batch size to 2 and it solved the problem? odd W tensorflow/core/grappler/optimizers/loop_optimizer.cc:906] Skipping loop optimization for Merge node with control input: cond/branch_executed/_11 idk should look into this 18 seconds for 2000 images """ if tf.math.greater(update_index.size(), 0): update_index = update_index.stack() update = update.stack() full = tf.tensor_scatter_nd_update(full, update_index, update) # full = tf.reshape(full, finshape) # tf.print("gtsum 1", K.sum(y_true)) # tf.print("gtsum full",K.sum(full)) return full
def build_grided_gt(y_true, mask, size): """ convert ground truth for use in loss functions Args: y_true: tf.Tensor[] ground truth [box coords[0:4], classes_onehot[0:-1], best_fit_anchor_box] mask: list of the anchor boxes choresponding to the output, ex. [1, 2, 3] tells this layer to predict only the first 3 anchors in the total. size: the dimensions of this output, for regular, it progresses from 13, to 26, to 52 Return: tf.Tensor[] of shape [batch, size, size, #of_anchors, 4, 1, num_classes] """ batches = tf.shape(y_true)[0] num_boxes = tf.shape(y_true)[1] len_masks = tf.shape(mask)[0] # finshape = tf.convert_to_tensor([batches, size, size, len_masks * tf.shape(y_true)[-1]]) full = tf.zeros([batches, size, size, len_masks, tf.shape(y_true)[-1]]) depth_track = tf.zeros((batches, size, size, len_masks), dtype=tf.int32) x = tf.cast(y_true[..., 0] * tf.cast(size, dtype=tf.float32), dtype=tf.int32) y = tf.cast(y_true[..., 1] * tf.cast(size, dtype=tf.float32), dtype=tf.int32) anchors = tf.repeat(tf.expand_dims(y_true[..., -1], axis=-1), len_masks, axis=-1) update_index = tf.TensorArray(tf.int32, size=0, dynamic_size=True) update = tf.TensorArray(tf.float32, size=0, dynamic_size=True) i = 0 for batch in range(batches): for box_id in range(num_boxes): # if tf.math.equal(K.sum(y_true[batch, box_id, 0:4]), 0.0): # VALUE ERROR is (None, 25) # continue if K.all(tf.math.equal(y_true[batch, box_id, 2:4], 0)): # VALUE ERROR is (None, 25) #tf.print("outer zero: ",y_true[batch, box_id, 0:2]) continue if K.any(tf.math.less(y_true[batch, box_id, 0:2], 0.0)) or K.any( tf.math.greater_equal(y_true[batch, box_id, 0:2], 1.0)): #tf.print("outer vals: ",y_true[batch, box_id, 0:2]) continue index = tf.math.equal(anchors[batch, box_id], mask) if K.any(index): p = tf.cast(K.argmax(tf.cast(index, dtype=tf.int32)), dtype=tf.int32) # start code for tie breaker, temp check performance # find the index of the box uid = 1 used = depth_track[batch, y[batch, box_id], x[batch, box_id], p] count = 0 # check if the next anchor is used used == 1, if so find another box while tf.math.equal(used, 1) and tf.math.less(count, 3): uid = 2 count += 1 p = (p + 1) % 3 used = depth_track[batch, x[batch, box_id], y[batch, box_id], p] if tf.math.equal(used, 1): tf.print("skipping") continue # set the current index to used = 2, to indicate that it is occupied by something that should not be there, so if another box fits that anchor # it will be prioritized over the current box. depth_track = tf.tensor_scatter_nd_update( depth_track, [(batch, y[batch, box_id], x[batch, box_id], p)], [uid]) #end code for tie breaker # write the box to the update list # the boxes output from yolo are for some reason have the x and y indexes swapped for some reason, I am not sure why """peculiar""" update_index = update_index.write( i, [batch, y[batch, box_id], x[batch, box_id], p]) value = K.concatenate([ y_true[batch, box_id, 0:4], tf.convert_to_tensor([1.]), y_true[batch, box_id, 4:-1] ]) update = update.write(i, value) i += 1 """ used can be: 0 not used 1 used with the correct anchor 2 used with offset anchor if used is 0 or 2: do not enter tie breaker (count = 0) edit box index with the most recent box if tie breaker was used: set used to 2 else: set used to 1 E tensorflow/core/grappler/optimizers/dependency_optimizer.cc:741] Iteration = 0, topological sort failed with message: The graph couldn't be sorted in topological order. raised likely due to a memory issue? reduced batch size to 2 and it solved the problem? odd W tensorflow/core/grappler/optimizers/loop_optimizer.cc:906] Skipping loop optimization for Merge node with control input: cond/branch_executed/_11 idk should look into this 18 seconds for 2000 images """ # if the seize of the update list is not 0, do an update, other wise, no boxes and pass an empty grid if tf.math.greater(update_index.size(), 0): update_index = update_index.stack() update = update.stack() full = tf.tensor_scatter_nd_add(full, update_index, update) #debug #tf.print("gtsum: ", K.sum(y_true)) #tf.print("gtsum full",size, " ", K.sum(full)) return full
def count_labels(y): return K.sum(K.cast(K.any(K.not_equal(y, 0), axis=-1), dtype=K.floatx()))
def compute_mask(self, input, mask=None): if mask is not None: return K.any(mask, axis=1) return mask
def get_updates(self, loss, params): self.updates = [] self.updates.append(K.update_add(self.state_counter, 1)) self.updates.append(K.update_add(self.iterator, 1)) self.updates.append(K.update_add(self.iterations, 1)) lr = self.lr ## lr exponential decay if self.initial_decay > 0: lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay)))) shapes = [K.int_shape(p) for p in params] x = [K.update(K.zeros(shape), p) for shape, p in zip(shapes, params)] mu = [K.update(K.zeros(shape), p) for shape, p in zip(shapes, params)] grads = self.get_gradients(loss, params) moments = [ K.zeros(shape, name='moment_' + str(i)) for (i, shape) in enumerate(shapes) ] for x_i, x_prime_i, mu_i, g, m in zip(x, params, mu, grads, moments): ## we update x_prime (if we are in LAngevin steps, we update, otherwise we switch to parameters x_i) dx_prime_i = g - self.gamma * (x_i - x_prime_i) x_prime_update_i = K.switch( K.any(K.stack([ K.equal(self.state_counter, 0), K.equal(self.num_steps, self.iterator) ], axis=0), axis=0), x_i, x_prime_i - self.sgld_step * dx_prime_i + K.sqrt(self.sgld_step) * self.sgld_noise * K.random_normal(K.int_shape(x_prime_i))) # Apply constraints. if getattr(x_prime_i, 'constraint', None) is not None: x_prime_update_i = x_prime_i.constraint(x_prime_update_i) self.updates.append(K.update(x_prime_i, x_prime_update_i)) ## We update mu (if we are in LAngevin steps, we update otherwise we switch to parameters x_i) mu_update_i = K.switch(K.equal(self.state_counter, 0), x_i, (1 - self.alpha) * mu_i + self.alpha * x_prime_i) self.updates.append(K.update(mu_i, mu_update_i)) ## As they described in the paper, we remove the gamma from the update because it interferes with the learning annealing ## After each outer loop update we apply an exponential decay on gamma ## The following lines concerns the outer loop updates ## Nesterov's momentum gradient = (x_i - mu_i) v = self.momentum * m - lr * gradient # velocity self.updates.append( K.update( m, K.switch(K.equal(self.state_counter, self.L + 1), v, m))) if self.nesterov: new_x_i = x_i + self.momentum * v - lr * gradient else: new_x_i = x_i + v x_i_update = K.switch(K.equal(self.state_counter, self.L + 1), new_x_i, x_i) self.updates.append(K.update(x_i, x_i_update)) ## Gamma scoping gamma_update = K.switch(K.equal(self.state_counter, self.L + 1), self.gamma, self.gamma * (1. + self.scoping)) self.updates.append(K.update(self.gamma, gamma_update)) counter = K.switch(K.equal(self.state_counter, self.L + 2), K.constant(0, dtype='int64'), self.state_counter) self.updates.append(K.update(self.state_counter, counter)) return self.updates
def build_batch_grided_gt(y_true, mask, size, num_classes, dtype, use_tie_breaker): """ convert ground truth for use in loss functions Args: y_true: tf.Tensor[] ground truth [batch, box coords[0:4], classes_onehot[0:-1], best_fit_anchor_box] mask: list of the anchor boxes choresponding to the output, ex. [1, 2, 3] tells this layer to predict only the first 3 anchors in the total. size: the dimensions of this output, for regular, it progresses from 13, to 26, to 52 num_classes: `integer` for the number of classes dtype: expected output datatype use_tie_breaker: boolean value for wether or not to use the tie_breaker Return: tf.Tensor[] of shape [batch, size, size, #of_anchors, 4, 1, num_classes] """ # unpack required components from the input ground truth boxes = tf.cast(y_true['bbox'], dtype) classes = tf.expand_dims(tf.cast(y_true['classes'], dtype=dtype), axis=-1) anchors = tf.cast(y_true['best_anchors'], dtype) # get the batch size batches = tf.shape(boxes)[0] # get the number of boxes in the ground truth boxs num_boxes = tf.shape(boxes)[1] # get the number of anchor boxes used for this anchor scale len_masks = tf.shape(mask)[0] # init a fixed memeory size grid for this prediction scale # [batch, size, size, # of anchors, 1 + 1 + number of anchors per scale] full = tf.zeros([batches, size, size, len_masks, 1 + 4 + 1], dtype=dtype) # init a grid to use to track which locations have already # been used before (for the tie breaker) depth_track = tf.zeros((batches, size, size, len_masks), dtype=tf.int32) # rescale the x and y centers to the size of the grid [size, size] x = tf.cast(boxes[..., 0] * tf.cast(size, dtype=dtype), dtype=tf.int32) y = tf.cast(boxes[..., 1] * tf.cast(size, dtype=dtype), dtype=tf.int32) # init all the tensorArrays to be used in storeing the index and the values # to be used to update both depth_track and full update_index = tf.TensorArray(tf.int32, size=0, dynamic_size=True) update = tf.TensorArray(dtype, size=0, dynamic_size=True) # init constants and match data types before entering loop i = 0 anchor_id = 0 const = tf.cast(tf.convert_to_tensor([1.]), dtype=dtype) mask = tf.cast(mask, dtype=dtype) rand_update = 0.0 for batch in range(batches): for box_id in range(num_boxes): # if the width or height of the box is zero, skip it if K.all(tf.math.equal(boxes[batch, box_id, 2:4], 0)): continue # after pre processing, if the box is not in the image bounds anymore # skip the box if K.any(tf.math.less(boxes[batch, box_id, 0:2], 0.0)) or K.any( tf.math.greater_equal(boxes[batch, box_id, 0:2], 1.0)): continue if use_tie_breaker: for anchor_id in range(tf.shape(anchors)[-1]): index = tf.math.equal(anchors[batch, box_id, anchor_id], mask) if K.any(index): # using the boolean index mask to determine exactly which anchor box was used p = tf.cast(K.argmax(tf.cast(index, dtype=tf.int32)), dtype=tf.int32) # determine if the index was used or not used = depth_track[batch, y[batch, box_id], x[batch, box_id], p] # defualt used upadte value uid = 1 # if anchor_id is 0, this is the best matched anchor for this box # with the highest IOU if anchor_id == 0: # create random number to trigger a replacment if the cell is used already if tf.math.equal(used, 1): rand_update = tf.random.uniform([], maxval=1) else: rand_update = 1.0 if rand_update > 0.5: # write the box to the update list update_index = update_index.write( i, [ batch, y[batch, box_id], x[batch, box_id], p ]) value = K.concatenate([ boxes[batch, box_id], const, classes[batch, box_id] ]) update = update.write(i, value) # if used is 2, this cell is filled with a non-optimal box # if used is 0, the cell in the ground truth is not yet consumed # in either case you can replace that cell with a new box, as long # as it is not consumed by an optimal box with anchor_id = 0 elif tf.math.equal(used, 2) or tf.math.equal(used, 0): uid = 2 # write the box to the update list update_index = update_index.write( i, [batch, y[batch, box_id], x[batch, box_id], p]) value = K.concatenate([ boxes[batch, box_id], const, classes[batch, box_id] ]) update = update.write(i, value) # update the used index for where and how the box was placed depth_track = tf.tensor_scatter_nd_update( depth_track, [(batch, y[batch, box_id], x[batch, box_id], p)], [uid]) i += 1 else: index = tf.math.equal(anchors[batch, box_id, 0], mask) if K.any(index): # if any there is an index match p = tf.cast(K.argmax(tf.cast(index, dtype=tf.int32)), dtype=tf.int32) # write the box to the update list update_index = update_index.write( i, [batch, y[batch, box_id], x[batch, box_id], p]) value = K.concatenate( [boxes[batch, box_id], const, classes[batch, box_id]]) update = update.write(i, value) i += 1 # if the size of the update list is not 0, do an update, other wise, no boxes and pass an empty grid if tf.math.greater(update_index.size(), 0): update_index = update_index.stack() update = update.stack() full = tf.tensor_scatter_nd_update(full, update_index, update) return full
def compute_mask(self, inputs, mask=None): # pylint: disable=unused-argument if mask is None: return None return K.any(mask, axis=self.axis)