def nn_distance(pc1, pc2, l1smooth=False, delta=1.0, l1=False):
"""
Input:
pc1: (B,N,C) torch tensor
pc2: (B,M,C) torch tensor
l1smooth: bool, whether to use l1smooth loss
delta: scalar, the delta used in l1smooth loss
Output:
dist1: (B,N) torch float32 tensor
idx1: (B,N) torch int64 tensor
dist2: (B,M) torch float32 tensor
idx2: (B,M) torch int64 tensor
"""
N = K.int_shape(pc1)[1]
M = K.int_shape(pc2)[1]
pc1_expand = K.tile(K.expand_dims(pc1, 2), [1, 1, M, 1])
pc2_expand = K.tile(K.expand_dims(pc2, 1), [1, N, 1, 1])
pc_diff = pc1_expand - pc2_expand
if l1smooth:
pc_dist = K.sum(huber_loss(pc_diff, delta), axis=-1)
elif l1:
pc_dist = K.sum(K.abs(pc_diff), axis=-1)
else:
pc_dist = K.sum(K.square(pc_diff), axis=-1) # (B,N,M)
dist1 = K.min(pc_dist, axis=2, keepdims=False) #(B,N)
idx1 = K.argmin(pc_dist, axis=2) #
dist2 = K.min(pc_dist, axis=1, keepdims=False) #(B,M)
idx2 = K.argmin(pc_dist, axis=1)
return dist1, idx1, dist2, idx2
def update_state(self, y_false, y_pred, sample_weight=None):
y_false = K.argmin(y_false, axis=-1)
y_pred = K.argmin(y_pred, axis=-1)
y_false = K.flatten(y_false)
false_poss = K.sum(K.cast((K.equal(y_false, y_pred)), dtype=tf.float32))
self.cat_false_positives.assign_add(false_poss)
def _process_channel(args):
__kps, __hm_hp = args
thresh = 0.1
__hm_scores, __hm_inds = tf.math.top_k(__hm_hp, k=k, sorted=True)
__hm_xs = K.cast(__hm_inds % width, 'float32')
__hm_ys = K.cast(K.cast(__hm_inds / width, 'int32'), 'float32')
__hp_offset = K.gather(_hp_offset, __hm_inds)
__hm_xs = __hm_xs + __hp_offset[..., 0]
__hm_ys = __hm_ys + __hp_offset[..., 1]
mask = K.cast(__hm_scores > thresh, 'float32')
__hm_scores = (1. - mask) * -1. + mask * __hm_scores
__hm_xs = (1. - mask) * -10000. + mask * __hm_xs
__hm_ys = (1. - mask) * -10000. + mask * __hm_ys
__hm_kps = K.stack([__hm_xs, __hm_ys], -1) # k x 2
__broadcast_hm_kps = K.expand_dims(__hm_kps, 1) # k x 1 x 2
__broadcast_kps = K.expand_dims(__kps, 0) # 1 x k x 2
dist = K.sqrt(
K.sum(K.pow(__broadcast_kps - __broadcast_hm_kps, 2),
2)) # k, k
min_dist = K.min(dist, 0)
min_ind = K.argmin(dist, 0)
__hm_scores = K.gather(__hm_scores, min_ind)
__hm_kps = K.gather(__hm_kps, min_ind)
mask = (K.cast(__hm_kps[..., 0] < _x1, 'float32') +
K.cast(__hm_kps[..., 0] > _x2, 'float32') +
K.cast(__hm_kps[..., 1] < _y1, 'float32') +
K.cast(__hm_kps[..., 1] > _y2, 'float32') +
K.cast(__hm_scores < thresh, 'float32') + K.cast(
min_dist > 0.3 *
(K.maximum(_wh[..., 0], _wh[..., 1])), 'float32'))
mask = K.expand_dims(mask, -1)
mask = K.cast(mask > 0, 'float32')
__kps = (1. - mask) * __hm_kps + mask * __kps
return __kps
def step(self, input_energy_t, states, return_logZ=True):
# not in the following `prev_target_val` has shape = (B, F)
# where B = batch_size, F = output feature dim
# Note: `i` is of float32, due to the behavior of `K.rnn`
prev_target_val, i, chain_energy = states[:3]
t = K.cast(i[0, 0], dtype='int32')
if len(states) > 3:
if K.backend() == 'theano':
m = states[3][:, t:(t + 2)]
else:
m = K.tf.slice(states[3], [0, t], [-1, 2])
input_energy_t = input_energy_t * K.expand_dims(m[:, 0])
chain_energy = chain_energy * K.expand_dims(
K.expand_dims(
m[:, 0] * m[:, 1])) # (1, F, F)*(B, 1, 1) -> (B, F, F)
if return_logZ:
energy = chain_energy + K.expand_dims(
input_energy_t - prev_target_val,
2) # shapes: (1, B, F) + (B, F, 1) -> (B, F, F)
new_target_val = K.logsumexp(-energy, 1) # shapes: (B, F)
return new_target_val, [new_target_val, i + 1]
else:
energy = chain_energy + K.expand_dims(
input_energy_t + prev_target_val, 2)
min_energy = K.min(energy, 1)
argmin_table = K.cast(K.argmin(energy, 1),
K.floatx()) # cast for tf-version `K.rnn`
return argmin_table, [min_energy, i + 1]
def step(self, input_energy_t, states, return_logZ=True):
prev_target_val, i, chain_energy = states[:3]
t = K.cast(i[0, 0], dtype='int32')
if len(states) > 3:
if K.backend() == 'theano':
m = states[3][:, t:(t + 2)]
else:
m = K.tf.slice(states[3], [0, t], [-1, 2])
input_energy_t = input_energy_t * K.expand_dims(m[:, 0])
chain_energy = chain_energy * K.expand_dims(
K.expand_dims(
m[:, 0] * m[:, 1])) # (1, F, F)*(B, 1, 1) -> (B, F, F)
if return_logZ:
energy = chain_energy + K.expand_dims(
input_energy_t - prev_target_val,
2) # shapes: (1, B, F) + (B, F, 1) -> (B, F, F)
new_target_val = K.logsumexp(-energy, 1) # shapes: (B, F)
return new_target_val, [new_target_val, i + 1]
else:
energy = chain_energy + K.expand_dims(
input_energy_t + prev_target_val, 2)
min_energy = K.min(energy, 1)
argmin_table = K.cast(K.argmin(energy, 1),
K.floatx()) # cast for tf-version `K.rnn`
return argmin_table, [min_energy, i + 1]
def metric(y_true, y_pred):
val = K.cast(K.one_hot(
K.argmin(K.mean(K.square(new_img - af_img), (2, 3, 4),
keepdims=True)[:, :, 0, 0, 0],
axis=1), 4),
dtype='float32')
return keras.metrics.categorical_accuracy(val, y_pred)
def rgb2hsv_convert(x):
x = K.cast(x, dtype=K.floatx())
max_rgb = K.max(x, axis=-1)
min_rgb = K.min(x, axis=-1)
max_which = K.argmax(x, axis=-1)
min_which = K.argmin(x, axis=-1)
R = x[:, :, :, 0]
B = x[:, :, :, 1]
G = x[:, :, :, 2]
# print(K.eval(R))
# print(K.eval(max_which))
H1 = (G - B) / (max_rgb - min_rgb)
H2 = 2 + (B - R) / (max_rgb - min_rgb)
H3 = 4 + (R - G) / (max_rgb - min_rgb)
H = K.cast((max_which - 1) * (max_which - 2), K.floatx()) * H1 / 2 + \
K.cast((max_which - 0) * (max_which - 2), K.floatx()) * H2 / (-1) + \
K.cast((max_which - 0) * (max_which - 1), K.floatx()) * H3 / 2
H = H / 6
# print(K.eval(H))
H = H + K.cast(H < 0, dtype=K.floatx())
V = max_rgb / 255
S = (max_rgb - min_rgb) / max_rgb
HSV = K.stack([H, S, V], axis=-1)
return HSV
def precision_class(y_true, y_pred, class_id):
"""
Precision for a specifc class. This is only verified for 2-class one-hot encoding to be working.
:param y_true: True labels
:param y_pred: Prediction labels
:param class_id: Class we want to get recall on (located at [_, class] on the one-hot encoding)
:return: Precision for specified class
"""
class_id_true = K.argmax(y_true, axis=-1)
class_id_preds = K.argmax(y_pred, axis=-1)
# Generate maths for predictions for classes true positive labels
mask_positive_self = K.cast(
K.equal(class_id_true, class_id),
'int32') # This is own class true positive plus false negative
true_positive_tensor = K.cast(K.equal(class_id_true, class_id_preds),
'int32') * mask_positive_self
# Generate maths for predictions for other classes false negatives (meaning our false positives)
mask_positive_other = K.cast(
K.equal(class_id_true, ((class_id - 1) * -1)), 'int32'
) # This is other class true positive plus false negative (so our true negatives and false positives)
false_positive_tensor = K.cast(
K.not_equal(class_id_true, K.argmin(y_pred, axis=-1)),
'int32') * mask_positive_other
# Now we have our true positives and false positives
true_positive_count = K.sum(true_positive_tensor)
false_positive_count = K.sum(false_positive_tensor)
class_precision = true_positive_count / K.maximum(
(true_positive_count + false_positive_count), 1)
return class_precision
def handle_arg_min(cls, node, input_dict):
data = input_dict[node.inputs[0]]
axis = node.attrs["axis"]
keepdims = node.attrs.get("keepdims", 1)
if keepdims == 1:
warnings.warn(
"Definition of ArgMin with keepdims enabled is "
"incompatible between onnx and keras.", UserWarning)
return [Lambda(lambda x: K.argmin(x, axis=axis))(data)]
def call(self, inputs):
"""inputs.shape=[None, m, m, dim]
"""
l2_inputs = K.sum(inputs**2, -1, keepdims=True)
l2_embeddings = K.sum(self.embeddings**2, -1)
for _ in range(3):
l2_embeddings = K.expand_dims(l2_embeddings, 0)
embeddings = K.transpose(self.embeddings)
dot = K.dot(inputs, embeddings)
distance = l2_inputs + l2_embeddings - 2 * dot
codes = K.cast(K.argmin(distance, -1), 'int32')
code_vecs = K.gather(self.embeddings, codes)
return [codes, code_vecs]
def retriv_acc(s_im):
# For a minibatch of sentence and image embeddings,
# compute the retrieval accuracy
s = s_im[0]
im = s_im[1]
s2 = K.expand_dims(s, 1)
im2 = K.expand_dims(im, 0)
order_violation = K.pow(K.maximum(0.0, s2 - im2), 2)
errors = K.sum(order_violation, axis=2)
inds = K.argmin(errors, axis=1)
inds = tf.cast(inds, tf.int32)
inds_true = tf.range(tf.shape(s)[0])
elements_equal_to_value = tf.equal(inds, inds_true)
as_ints = tf.cast(elements_equal_to_value, tf.int32)
results = tf.reduce_sum(as_ints)
results = tf.cast(results, tf.float32)
return results
def prune_empty_boxes(self, coord_t, class_t):
"""
Removes boxes that don't contain any predictions/true boxes.
(1) Boxes are sorted by confidence
(2) For each batch the index of the first smallest element is calculated
(3) Boxes that are higher than the overall highest index are pruned. That way
the dimension for each batch is the same and some of the boxes are still
empty but the overall size of the tensor is drastically reduced.
:param coord_t: tensor(#boxes,4) true bounding box coordinates in minmax-format
:param class_t: tensor(#boxes,#classes) true class confidences one-hot encoded
:return: same as input but with reduced size
"""
coord_sorted, class_sorted = self._sort_by_conf(coord_t, class_t)
conf_true = K.max(class_sorted, -1)
min_idx = K.argmin(conf_true, axis=-1)
max_min_idx = K.cast(K.max(min_idx), K.tf.int32)
coord_sorted = coord_sorted[:, :max_min_idx + 1]
class_sorted = class_sorted[:, :max_min_idx + 1]
return coord_sorted, class_sorted
def fls_neg(y_true, y_pred):
axis = K.ndim(y_true) - 1
ytam = K.argmin(y_true, axis=axis)
ypam = K.argmin(y_pred, axis=axis)
diff = ypam - ytam
return K.sum(K.clip(diff, 0, 1))
def tru_neg(y_true, y_pred):
axis = K.ndim(y_true) - 1
ytam = K.argmin(y_true, axis=axis)
ypam = K.argmin(y_pred, axis=axis)
prod = ytam * ypam
return K.sum(prod)
def _call(self, inputs, **kwargs):
if self.proto_number == self.capsule_number:
return inputs
else:
signals = inputs[0]
diss = inputs[1]
signal_shape = mixed_shape(signals)
if self.use_for_loop:
diss_stack = []
signals_stack = []
sub_idx = None
with K.name_scope('for_loop'):
for p in self._proto_distrib:
with K.name_scope('compute_slices'):
diss_ = diss[:, p[0]:(p[-1]+1)]
signals_ = K.reshape(signals[:, p[0]:(p[-1]+1), :],
[signal_shape[0] * len(p)] + list(signal_shape[2:]))
with K.name_scope('competition'):
if len(p) > 1:
with K.name_scope('competition_indices'):
argmin_idx = K.argmin(diss_, axis=-1)
if sub_idx is None:
sub_idx = K.arange(0, signal_shape[0], dtype=argmin_idx.dtype)
argmin_idx = argmin_idx + len(p) * sub_idx
with K.name_scope('dissimilarity_competition'):
diss_stack.append(K.expand_dims(K.gather(K.flatten(diss_), argmin_idx), -1))
with K.name_scope('signal_competition'):
signals_stack.append(K.gather(signals_, argmin_idx))
else:
diss_stack.append(diss_)
signals_stack.append(signals_)
diss = K.concatenate(diss_stack, 1)
with K.name_scope('signal_concatenation'):
signals = K.concatenate(signals_stack, 1)
signals = K.reshape(signals, [signal_shape[0], self.capsule_number] + list(signal_shape[2:]))
else:
with K.name_scope('dissimilarity_preprocessing'):
# extend if it is not equally distributed
if not self._equally_distributed:
# permute to first dimension is prototype (protos x batch)
diss = K.permute_dimensions(diss, [1, 0])
# gather regarding extension (preparing for reshape to block)
diss = K.gather(diss, self._proto_extension)
# permute back (max_proto_number x (max_proto_number * batch))
diss = K.permute_dimensions(diss, [1, 0])
# reshape to block form
diss = K.reshape(diss, [signal_shape[0] * self.capsule_number, self._max_proto_number_in_capsule])
with K.name_scope('competition_indices'):
# get minimal idx in each class and batch for element selection in diss and signals
argmin_idx = K.argmin(diss, axis=-1)
argmin_idx = argmin_idx + self._max_proto_number_in_capsule * \
K.arange(0, signal_shape[0] * self.capsule_number, dtype=argmin_idx.dtype)
with K.name_scope('dissimilarity_competition'):
# get minimal values in the form (batch x capsule)
diss = K.gather(K.flatten(diss), argmin_idx)
diss = K.reshape(diss, [signal_shape[0], self.capsule_number])
with K.name_scope('signal_preprocessing'):
# apply the same steps as above for signals
# get signals in: (batch x protos x dim1 x ... x dimN) --> out: (batch x capsule x dim1 x ... x dimN)
# extend if is not equally distributed
if not self._equally_distributed:
signals = K.permute_dimensions(signals, [1, 0] + list(range(2, len(signal_shape))))
signals = K.gather(signals, self._proto_extension)
signals = K.permute_dimensions(signals, [1, 0] + list(range(2, len(signal_shape))))
signals = K.reshape(signals,
[signal_shape[0] * self.capsule_number * self._max_proto_number_in_capsule]
+ list(signal_shape[2:]))
with K.name_scope('signal_competition'):
signals = K.gather(signals, argmin_idx)
signals = K.reshape(signals, [signal_shape[0], self.capsule_number] + list(signal_shape[2:]))
return {0: signals, 1: diss}
def call(self, inputs, **kwargs):
return K.cast(K.argmin(inputs, axis=1), dtype=K.floatx())
def loss(y_true, y_pred):
index = K.cast(K.argmin(K.mean(K.square(new_img - a_img), (2, 3, 4),
keepdims=True)[:, :, 0, 0, 0],
axis=1)[0],
dtype='int64')
return keras.losses.mse(new_img[:, index, :, :, :], p_image)
def ranked_prediction(y_pred):
extra_column = K.zeros((K.shape(y_pred)[0], 1), dtype='int32')
threshold = K.constant(0.5, dtype=K.floatx())
over_threshold = K.cast(K.greater(y_pred, threshold), 'int32')
concatted = K.concatenate([over_threshold, extra_column], axis=1)
return K.cast(K.argmin(concatted, axis=1), 'int32')
def tnr(y_true, y_pred): true=K.argmin(y_true, axis=1) pred=K.argmin(y_pred, axis=1) num_true=K.sum(true) true_negative=K.sum(true*pred) return true_negative/(num_true)#+K.epsilon())
def DistanceMetric(y_true, y_pred):
e = K.equal(K.argmax(y_true, axis=1), K.argmin(y_pred, axis=1))
s = tf.reduce_sum(tf.cast(e, tf.float32))
n = tf.cast(K.shape(y_true)[0], tf.float32)
return s / n