def input_channel_swap(tensor):
if len(swap_channels) == 3:
return K.stack(
[tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]]], axis=-1)
elif len(swap_channels) == 4:
return K.stack([tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]],
tensor[..., swap_channels[3]]], axis=-1)
def call(self, inputs):
input_shape = self.in_shape
if self.data_format == 'channels_first':
z = K.arange(0, input_shape[1], dtype=K.floatx())
x = K.arange(0, input_shape[2], dtype=K.floatx())
y = K.arange(0, input_shape[3], dtype=K.floatx())
else:
z = K.arange(0, input_shape[0], dtype=K.floatx())
x = K.arange(0, input_shape[1], dtype=K.floatx())
y = K.arange(0, input_shape[2], dtype=K.floatx())
x = x / K.max(x)
y = y / K.max(y)
z = z / K.max(z)
loc_z, loc_x, loc_y = tf.meshgrid(z, x, y, indexing='ij')
if self.data_format == 'channels_first':
loc = K.stack([loc_z, loc_x, loc_y], axis=0)
else:
loc = K.stack([loc_z, loc_x, loc_y], axis=-1)
location = K.expand_dims(loc, axis=0)
if self.data_format == 'channels_first':
location = K.permute_dimensions(location, pattern=[0, 2, 3, 4, 1])
location = tf.tile(location, [K.shape(inputs)[0], 1, 1, 1, 1])
if self.data_format == 'channels_first':
location = K.permute_dimensions(location, pattern=[0, 4, 1, 2, 3])
return location
def call(self, input, **kwargs):
#w = K.reverse(self.kernel, axes=-1)
#n_filter_length = w.shape[-1]
# arrange as 4D array for conv2d
# input of the convolution
#xx = K.reshape(input, (-1, n_filter_length, 1, 1))
# repeat, and arrange as 4D-array for conv2d
# kernel of the convolution
#ww = K.reshape(K.stack((w, w)), (2 * n_filter_length, 1, 1, 1))
#dims: n_eval_batches, n_filter_length
#outputs = K.squeeze(K.squeeze(K.conv2d(xx, ww, strides=(1, 1), padding="same"), -1), -1)
#outputs = K.expand_dims(outputs, axis=1)
################################################### 2d-convolution #################################################
w = K.reverse(self.kernel, axes=-1)
w1 = K.reshape(w, shape=(-1, w.shape[1], self.n_antennas_MS, self.n_antennas_BS))
w1 = K.permute_dimensions(w1, pattern=(0, 1, 3, 2))
n_filter_length = w1.shape[-2]
n_filter_width = w1.shape[-1]
n_filter_mult = n_filter_length * n_filter_width
# arrange as 4D array for conv2d
xx1 = K.reshape(input, (-1, n_filter_mult, 1, 1))
xx1 = K.reshape(xx1, shape=(-1, self.n_antennas_MS, self.n_antennas_BS, 1))
xx1 = K.permute_dimensions(xx1, pattern=(0, 2, 1, 3))
# repeat, and arrange as 4D-array for conv2d
wtemp = K.reshape(K.stack((w1,w1),axis = 2),(1,1,2*n_filter_length,-1))
wtemp = K.reshape(K.stack((wtemp,wtemp),axis=3),(1,1,-1,2*n_filter_width))
ww1 = K.permute_dimensions(wtemp, pattern=(2,3,0,1))
# dims: n_eval_batches, n_filter_length
outputs1 = K.conv2d(xx1, ww1, padding="same")
outputs1 = K.permute_dimensions(outputs1, pattern=(0, 2, 1, 3))
outputs1 = K.reshape(outputs1, shape=(-1,n_filter_mult,1))
outputs = K.permute_dimensions(outputs1,pattern=(0,2,1))
#outputs = K.squeeze(K.conv3d(xx1, ww, strides=(1, 1, 1), padding="same"), -1)
#stop = 0
if self.use_bias:
outputs = K.bias_add(
outputs,
self.bias)
if self.activation is not None:
outputs = self.activation(outputs)
if self.output_type is not None:
outputs = K.cast(outputs, self.output_type)
return outputs
def competition_coef(y_true, y_pred, smooth=1):
y_pred = K.argmax(y_pred, axis=-1)
y_true = K.argmax(y_true, axis=-1)
# try:
# Compute tumor+kidney Dice
tk_pd = K.greater(y_pred, 0)
tk_gt = K.greater(y_true, 0)
intersection = K.all(K.stack([tk_gt, tk_pd], axis=3), axis=3)
tk_dice = (2 * K.sum(K.cast(intersection, K.floatx())) + smooth)/ (
K.sum(K.cast(tk_pd, K.floatx())) + K.sum(K.cast(tk_gt, K.floatx())) + smooth
)
# except ZeroDivisionError:
# return 0
# try:
# Compute tumor Dice
tu_pd = K.greater(y_pred, 1)
tu_gt = K.greater(y_true, 1)
intersection = K.all(K.stack([tu_pd, tu_gt], axis=3), axis=3)
tu_dice = (2 * K.sum(K.cast(intersection, K.floatx())) + smooth)/ (
K.sum(K.cast(tu_pd, K.floatx())) + K.sum(K.cast(tu_gt, K.floatx())) + smooth
)
# except ZeroDivisionError:
# return tk_dice / 2.0
return (tk_dice+tu_dice) / 2.0
def hd_loss_fn(y_true: tf.Tensor,
y_pred: tf.Tensor,
weights: tf.Tensor = None) -> tf.Tensor:
y_true_np = K.eval(y_true)
y_pred_np = K.eval(y_pred)
shape = K.eval(K.shape(y_true))
max_hd = np.sqrt(np.sum(shape[1:3]**2))
hds = []
for batch in range(shape[0]):
temp = []
for ch_index in range(shape[-1]):
t_ch = y_true_np[batch, ..., ch_index]
p_ch = y_pred_np[batch, ..., ch_index]
t_sum = np.sum(t_ch)
p_sum = np.sum(p_ch)
if t_sum and p_sum: # true positive
hd_np = _hd_percentile(t_ch, p_ch)
elif not t_sum and p_sum: # false positive
hd_np = max_hd
elif t_sum and not p_sum: # false negative
hd_np = max_hd
else: # true negative
hd_np = 0
temp.append(K.constant(hd_np / max_hd))
hds.append(K.stack(temp))
tf_hd = K.stack(hds)
if weights is not None:
return K.sum(weights * tf_hd)
return K.sum(tf_hd)
def call(self, inputs):
input_shape = K.shape(inputs)
if self.data_format == 'channels_first':
x = K.arange(0, input_shape[2], dtype=inputs.dtype)
y = K.arange(0, input_shape[3], dtype=inputs.dtype)
else:
x = K.arange(0, input_shape[1], dtype=inputs.dtype)
y = K.arange(0, input_shape[2], dtype=inputs.dtype)
x = x / K.max(x)
y = y / K.max(y)
loc_x, loc_y = tf.meshgrid(x, y, indexing='ij')
if self.data_format == 'channels_first':
loc = K.stack([loc_x, loc_y], axis=0)
else:
loc = K.stack([loc_x, loc_y], axis=-1)
location = K.expand_dims(loc, axis=0)
if self.data_format == 'channels_first':
location = K.permute_dimensions(location, pattern=[0, 2, 3, 1])
location = tf.tile(location, [input_shape[0], 1, 1, 1])
if self.data_format == 'channels_first':
location = K.permute_dimensions(location, pattern=[0, 3, 1, 2])
return location
def call(self, inputs):
user_behavior_inputs, interests_weights = inputs
user_behavior_inputs = K.stack(user_behavior_inputs, axis=1)
interests_weights = K.stack(interests_weights, axis=1)
sum_pooling = K.sum(tf.multiply(user_behavior_inputs,
interests_weights),
axis=1)
return sum_pooling
def space_cnn_part(self, input_data):
# add message passing #
# top to down #
feature_list_new = []
nB, H, W, C = input_data.get_shape().as_list()
slices = [input_data[:, i:(i + 1), :, :] for i in range(H)]
out = [slices[0]]
for i in range(1, len(slices)):
tem = Conv2D(filters=128,
kernel_size=[1, 9],
strides=1,
padding='SAME')(out[i - 1])
tem = ReLU()(tem)
conv_6 = Add()([tem, slices[i]])
out.append(conv_6)
slices = out[::-1]
out = [slices[0]]
for i in range(1, len(slices)):
tem = Conv2D(filters=128,
kernel_size=[1, 9],
strides=1,
padding='SAME')(out[i - 1])
tem = ReLU()(tem)
conv_6 = Add()([tem, slices[i]])
out.append(conv_6)
slices = out[::-1]
feature_list_new = K.stack(slices, axis=1)
feature_list_new = K.squeeze(feature_list_new, axis=2)
slices = [feature_list_new[:, :, i:(i + 1), :] for i in range(W)]
out = [slices[0]]
for i in range(1, len(slices)):
tem = Conv2D(filters=128,
kernel_size=[9, 1],
strides=1,
padding='SAME')(out[i - 1])
tem = ReLU()(tem)
conv_6 = Add()([tem, slices[i]])
out.append(conv_6)
slices = slices[::-1]
out = [slices[0]]
for i in range(1, len(slices)):
tem = Conv2D(
filters=128,
kernel_size=[9, 1],
strides=1,
padding='SAME',
)(out[i - 1])
tem = ReLU()(tem)
conv_6 = Add()([tem, slices[i]])
out.append(conv_6)
slices = out[::-1]
feature_list_new = K.stack(slices, axis=2)
feature_list_new = K.squeeze(feature_list_new, axis=3)
return feature_list_new
def call(self, inputs, **kwargs):
if self.axis == 1:
# If channels first, force it to be channels last for these ops
inputs = K.permute_dimensions(inputs, [0, 2, 3, 1])
q, k, v = tf.split(inputs, [self.depth_k, self.depth_k, self.depth_v],
axis=-1)
q = self.split_heads_2d(q)
k = self.split_heads_2d(k)
v = self.split_heads_2d(v)
# scale query
depth_k_heads = self.depth_k / self.num_heads
q *= (depth_k_heads**-0.5)
# [Batch, num_heads, height * width, depth_k or depth_v] if axis == -1
qk_shape = [
self._batch, self.num_heads, self._height * self._width,
self.depth_k // self.num_heads
]
v_shape = [
self._batch, self.num_heads, self._height * self._width,
self.depth_v // self.num_heads
]
flat_q = K.reshape(q, K.stack(qk_shape))
flat_k = K.reshape(k, K.stack(qk_shape))
flat_v = K.reshape(v, K.stack(v_shape))
# [Batch, num_heads, HW, HW]
logits = tf.matmul(flat_q, flat_k, transpose_b=True)
# Apply relative encodings
if self.relative:
h_rel_logits, w_rel_logits = self.relative_logits(q)
logits += h_rel_logits
logits += w_rel_logits
weights = K.softmax(logits, axis=-1)
attn_out = tf.matmul(weights, flat_v)
attn_out_shape = [
self._batch, self.num_heads, self._height, self._width,
self.depth_v // self.num_heads
]
attn_out_shape = K.stack(attn_out_shape)
attn_out = K.reshape(attn_out, attn_out_shape)
attn_out = self.combine_heads_2d(attn_out)
# [batch, height, width, depth_v]
if self.axis == 1:
# return to [batch, depth_v, height, width] for channels first
attn_out = K.permute_dimensions(attn_out, [0, 3, 1, 2])
attn_out.set_shape(self.compute_output_shape(self._shape))
return attn_out
def compute_mask_loss(boxes,
masks,
annotations,
masks_target,
width,
height,
iou_threshold=0.5,
mask_size=(28, 28)):
"""compute overlap of boxes with annotations"""
iou = overlap(boxes, annotations)
argmax_overlaps_inds = K.argmax(iou, axis=1)
max_iou = K.max(iou, axis=1)
# filter those with IoU > 0.5
indices = tf.where(K.greater_equal(max_iou, iou_threshold))
boxes = tf.gather_nd(boxes, indices)
masks = tf.gather_nd(masks, indices)
argmax_overlaps_inds = K.cast(tf.gather_nd(argmax_overlaps_inds, indices), 'int32')
labels = K.cast(K.gather(annotations[:, 4], argmax_overlaps_inds), 'int32')
# make normalized boxes
x1 = boxes[:, 0]
y1 = boxes[:, 1]
x2 = boxes[:, 2]
y2 = boxes[:, 3]
boxes = K.stack([
y1 / (K.cast(height, dtype=K.floatx()) - 1),
x1 / (K.cast(width, dtype=K.floatx()) - 1),
(y2 - 1) / (K.cast(height, dtype=K.floatx()) - 1),
(x2 - 1) / (K.cast(width, dtype=K.floatx()) - 1),
], axis=1)
# crop and resize masks_target
# append a fake channel dimension
masks_target = K.expand_dims(masks_target, axis=3)
masks_target = tf.image.crop_and_resize(
masks_target,
boxes,
argmax_overlaps_inds,
mask_size
)
masks_target = masks_target[:, :, :, 0] # remove fake channel dimension
# gather the predicted masks using the annotation label
masks = tf.transpose(masks, (0, 3, 1, 2))
label_indices = K.stack([tf.range(K.shape(labels)[0]), labels], axis=1)
masks = tf.gather_nd(masks, label_indices)
# compute mask loss
mask_loss = K.binary_crossentropy(masks_target, masks)
normalizer = K.shape(masks)[0] * K.shape(masks)[1] * K.shape(masks)[2]
normalizer = K.maximum(K.cast(normalizer, K.floatx()), 1)
mask_loss = K.sum(mask_loss) / normalizer
return mask_loss
def build_model(embedder):
main_input = Input(shape=(MAX_SEQUENCE_LENGTH,))
net = embedder(main_input)
for i in range(0, NUM_LAYERS):
net = GRU(128, return_sequences=True, activation='relu', name='embed' + str(i))(net)
net = GRU(128, activation='relu', name='embed' + str(i+1))(net)
if USE_L2_NORM:
net = Lambda(l2Norm, output_shape=[128])(net)
base_model = Model(embedder.input, net, name='triplet_model')
base_model.summary()
input_shape=(MAX_SEQUENCE_LENGTH,)
input_anchor = Input(shape=input_shape, name='input_anchor')
input_positive = Input(shape=input_shape, name='input_pos')
input_negative = Input(shape=input_shape, name='input_neg')
net_anchor = base_model(input_anchor)
net_positive = base_model(input_positive)
net_negative = base_model(input_negative)
positive_dist = Lambda(euclidean_distance, name='pos_dist', output_shape=(1,))([net_anchor, net_positive])
negative_dist = Lambda(euclidean_distance, name='neg_dist', output_shape=(1,))([net_anchor, net_negative])
if USE_ANGULAR_LOSS:
n_c = Lambda(n_c_angular_distance, name='nc_angular_dist')([net_anchor, net_positive, net_negative])
a_p = Lambda(a_p_angular_distance, name='ap_angular_dist')([net_anchor, net_positive, net_negative])
stacked_dists = Lambda(
lambda vects: K.stack(vects, axis=1),
name='stacked_dists', output_shape=(3, 1)
)([a_p, n_c])
model = Model([input_anchor, input_positive, input_negative], stacked_dists, name='triple_siamese')
model.compile(optimizer="rmsprop", loss=angular_loss, metrics=[accuracy])
else:
exemplar_negative_dist = Lambda(euclidean_distance, name='exemplar_neg_dist', output_shape=(1,))([net_positive, net_negative])
stacked_dists = Lambda(
# lambda vects: C.splice(*vects, axis=C.Axis.new_leading_axis()).eval(vects),
lambda vects: K.stack(vects, axis=1),
name='stacked_dists', output_shape=(3, 1)
)([positive_dist, negative_dist, exemplar_negative_dist])
model = Model([input_anchor, input_positive, input_negative], stacked_dists, name='triple_siamese')
print('>>>>>>>>>>>>>>>>>>>>>>>>>>', LOSS_FUNCTION)
model.compile(optimizer="rmsprop", loss=LOSS_FUNCTION, metrics=[accuracy])
test_positive_model = Model([input_anchor, input_positive, input_negative], positive_dist)
test_negative_model = Model([input_anchor, input_positive, input_negative], negative_dist)
inter_model = Model(input_anchor, net_anchor)
print("output_shapes")
model.summary()
# print(positive_dist.output_shape)
# print(negative_dist.output_shape)
# print(exemplar_negative_dist)
# print(neg_dist.output_shape)
return model, test_positive_model, test_negative_model, inter_model
def compute_kernel(x, y):
x_size = K.shape(x)[0]
y_size = K.shape(y)[0]
dim = K.shape(x)[1]
tiled_x = K.tile(K.reshape(x, K.stack([x_size, 1, dim])),
K.stack([1, y_size, 1]))
tiled_y = K.tile(K.reshape(y, K.stack([1, y_size, dim])),
K.stack([x_size, 1, 1]))
return K.exp(-K.mean(K.square(tiled_x - tiled_y), axis=2) /
K.cast(dim, 'float32'))
def learn(self):
states, actions, rewards, next_states, done = zip(*self.experiences)
with tf.GradientTape(persistent=True) as tape:
states = K.stack(states)
actions = K.constant(actions, dtype='int32')
rewards = K.expand_dims(K.constant(rewards, dtype=K.floatx()), 1)
next_states = K.stack(next_states)
done = K.expand_dims(K.constant(done, dtype=K.floatx()), 1)
action_probs = self.actor(states)
action_log_probs = K.log(action_probs)
actions_one_hot = K.one_hot(actions, self.action_num)
forward_losses, pred_actions = self.icm(
[states, actions_one_hot, next_states])
forward_loss = K.mean(forward_losses)
inverse_loss = self.icm_loss_func(actions_one_hot, pred_actions)
icm_loss = (self.icm_beta * forward_loss +
(1 - self.icm_beta) * inverse_loss)
rewards = rewards + self.eta * forward_losses
critic_value = self.critic(states)
critic_target = rewards + self.gamma * self.critic(next_states) * (
1 - done)
critic_loss = self.critic_loss_func(critic_target, critic_value)
indices = K.stack((np.arange(len(action_probs)), actions), 1)
action_log_prob = K.expand_dims(
tf.gather_nd(action_log_probs, indices), 1)
prev_action_prob = K.expand_dims(
tf.gather_nd(self.prev_actor(states), indices), 1)
prob_ratio = K.exp(action_log_prob -
K.log(prev_action_prob + 1e-10))
advantage = critic_target - critic_value
entropy = -K.sum(action_probs * action_log_probs, 1, keepdims=True)
actor_loss = -K.mean(
K.minimum(
prob_ratio * advantage,
K.clip(prob_ratio, 1 - self.actor_loss_epsilon, 1 +
self.actor_loss_epsilon) * advantage) +
entropy * self.entropy_coef)
self.prev_actor.set_weights(self.actor.get_weights())
actor_grads = tape.gradient(actor_loss, self.actor.trainable_weights)
self.actor_optimizer.apply_gradients(
zip(actor_grads, self.actor.trainable_weights))
critic_grads = tape.gradient(critic_loss,
self.critic.trainable_weights)
self.critic_optimizer.apply_gradients(
zip(critic_grads, self.critic.trainable_weights))
icm_grads = tape.gradient(icm_loss, self.icm.trainable_weights)
self.icm_optimizer.apply_gradients(
zip(icm_grads, self.icm.trainable_weights))
self.entropy_coef = self.entropy_coef * self.entropy_decay
self.experiences.clear()
return actor_loss.numpy(), critic_loss.numpy(), icm_loss.numpy()
def noised():
Ni = K.shape(x)[0] #This is the number in the batch
#get an angle to shift each image in the batch
angles = K.clip( self.amount*K.random_normal((Ni,)), self.lower, self.upper)
#We are going to post multiply the vector (x'=xR) with the matrix
#rather than the normal way (x'=Rx)
#so we use the transpose of what is shown in literature for R
R = K.stack( (K.stack((K.cos(angles),K.sin(angles)),axis=1) ,
K.stack((-K.sin(angles),K.cos(angles)),axis=1)) ,
axis=1)
return tf.matmul(x,R)
def get_2_max(y):
best = K.argmax(y, axis=-1)
y_min_best = y - K.one_hot(best, K.shape(y)[-1])
second_best = K.argmax(y_min_best, axis=-1)
best_value = K.max(y, axis=-1)
second_best_value = K.max(y_min_best, axis=-1)
return K.stack([
K.stack([K.cast(best, 'float32'), best_value]),
K.stack([K.cast(second_best, 'float32'), second_best_value])
])
def seasonality_model(thetas, backcast_length, forecast_length, is_forecast):
p = thetas.get_shape().as_list()[-1]
p1, p2 = (p // 2, p // 2) if p % 2 == 0 else (p // 2, p // 2 + 1)
t = linear_space(backcast_length, forecast_length, fwd_looking=is_forecast)
s1 = K.stack([K.cos(2 * np.pi * i * t) for i in range(p1)], axis=0)
s2 = K.stack([K.sin(2 * np.pi * i * t) for i in range(p2)], axis=0)
if p == 1:
s = s2
else:
s = K.concatenate([s1, s2], axis=0)
s = K.cast(s, np.float32)
return K.dot(thetas, s)
def rel_to_abs(self, x):
shape = K.shape(x)
shape = [shape[i] for i in range(3)]
B, Nh, L, = shape
col_pad = K.zeros(K.stack([B, Nh, L, 1]))
x = K.concatenate([x, col_pad], axis=3)
flat_x = K.reshape(x, [B, Nh, L * 2 * L])
flat_pad = K.zeros(K.stack([B, Nh, L - 1]))
flat_x_padded = K.concatenate([flat_x, flat_pad], axis=2)
final_x = K.reshape(flat_x_padded, [B, Nh, L + 1, 2 * L - 1])
final_x = final_x[:, :, :L, L - 1:]
return final_x
def compute_kernel(x, y, sigma=1.0):
x_size = K.shape(x)[0]
y_size = K.shape(x)[1]
dim = K.shape(x)[1]
x_tiled = K.tile(K.reshape(x, K.stack([x_size, 1, dim])),
K.stack([1, y_size, 1]))
y_tiled = K.tile(K.reshape(y, K.stack([1, y_size, dim])),
K.stack([x_size, 1, 1]))
denominator = 2.0 * K.square(sigma)
kernel_value = K.exp(
-K.mean(K.square(x_tiled - y_tiled) / denominator, axis=3) /
K.cast(dim, 'float32'))
return kernel_value
def call(self, x):
range_x, range_y = self.meshgrid(self.feature_map_size)
expected_x = K.sum(x * range_x, axis=self.axes)
expected_y = K.sum(x * range_y, axis=self.axes)
keypoints_stack = K.stack([expected_x, expected_y], -1)
keypoints = K.reshape(keypoints_stack, [-1, self.num_keypoints, 2])
return keypoints
def yolo_eval(yolo_outputs,
image_shape,
max_boxes=10,
score_threshold=.6,
iou_threshold=.5):
"""Evaluate YOLO model on given input batch and return filtered boxes."""
box_confidence, box_xy, box_wh, box_class_probs = yolo_outputs
boxes = yolo_boxes_to_corners(box_xy, box_wh)
boxes, scores, classes = yolo_filter_boxes(box_confidence,
boxes,
box_class_probs,
threshold=score_threshold)
# Scale boxes back to original image shape.
height = image_shape[0]
width = image_shape[1]
image_dims = K.stack([height, width, height, width])
image_dims = K.reshape(image_dims, [1, 4])
boxes = boxes * image_dims
# TODO: Something must be done about this ugly hack!
max_boxes_tensor = K.variable(max_boxes, dtype='int32')
K.get_session().run(tf.variables_initializer([max_boxes_tensor]))
nms_index = tf.image.non_max_suppression(boxes,
scores,
max_boxes_tensor,
iou_threshold=iou_threshold)
boxes = K.gather(boxes, nms_index)
scores = K.gather(scores, nms_index)
classes = K.gather(classes, nms_index)
return boxes, scores, classes
def cartpole_next_state_fn(input, action):
gravity = 9.8
masscart = 1.0
masspole = 0.1
total_mass = (masspole + masscart)
length = 0.5 # actually half the pole's length
polemass_length = (masspole * length)
force_mag = 10.0
tau = 0.02 # seconds between state updates
x = input[:, 0]
x_dot = input[:, 1]
theta = input[:, 2]
theta_dot = input[:, 3]
costheta = K.cos(theta)
sintheta = K.sin(theta)
force = force_mag if action == 1 else -force_mag
temp = (force +
polemass_length * theta_dot * theta_dot * sintheta) / total_mass
thetaacc = (gravity * sintheta - costheta * temp) / (
length * (4.0 / 3.0 - masspole * costheta * costheta / total_mass))
xacc = temp - polemass_length * thetaacc * costheta / total_mass
x = x + tau * x_dot
x_dot = x_dot + tau * xacc
theta = theta + tau * theta_dot
theta_dot = theta_dot + tau * thetaacc
return K.stack([x, x_dot, theta, theta_dot], axis=1)
def boxes_from_deltas(pred_box_delta, config):
"""
Converts prediction deltas to bounding boxes
Arguments:
pred_box_delta {[type]} -- tensor of deltas
config {[type]} -- hyperparameter dict
Returns:
[type] -- tensor of bounding boxes
"""
# Keras backend allows no unstacking
delta_x = pred_box_delta[:, :, 0]
delta_y = pred_box_delta[:, :, 1]
delta_w = pred_box_delta[:, :, 2]
delta_h = pred_box_delta[:, :, 3]
# get the coordinates and sizes of the anchor boxes from config
anchor_x = config.ANCHOR_BOX[:, 0]
anchor_y = config.ANCHOR_BOX[:, 1]
anchor_w = config.ANCHOR_BOX[:, 2]
anchor_h = config.ANCHOR_BOX[:, 3]
# as we only predict the deltas, we need to transform the anchor box values before computing the loss
box_center_x = tf.identity(
anchor_x + delta_x * anchor_w)
box_center_y = tf.identity(
anchor_y + delta_y * anchor_h)
box_width = tf.identity(
anchor_w * safe_exp(delta_w, config.EXP_THRESH))
box_height = tf.identity(
anchor_h * safe_exp(delta_h, config.EXP_THRESH))
# tranform into a real box with four coordinates
xmins, ymins, xmaxs, ymaxs = bbox_transform([box_center_x, box_center_y, box_width, box_height])
# trim boxes if predicted outside
xmins = K.minimum(
K.maximum(0.0, xmins), config.IMAGE_WIDTH - 1.0)
ymins = K.minimum(
K.maximum(0.0, ymins), config.IMAGE_HEIGHT - 1.0)
xmaxs = K.maximum(
K.minimum(config.IMAGE_WIDTH - 1.0, xmaxs), 0.0)
ymaxs = K.maximum(
K.minimum(config.IMAGE_HEIGHT - 1.0, ymaxs), 0.0)
det_boxes = K.permute_dimensions(
K.stack(bbox_transform_inv([xmins, ymins, xmaxs, ymaxs])),
(1, 2, 0)
)
return (det_boxes)
def numpy_mean_iou(y_true, y_pred):
prec = []
for t in np.arange(0.5, 1.0, 0.5):
y_pred_ = tf.cast(y_pred > t, tf.int32)
score = tf.py_function(numpy_iou, [y_true, y_pred_], tf.float64)
prec.append(score)
return K.mean(K.stack(prec), axis=0)
def call(self, inputs, output_shape=None):
updates, mask = inputs[0], inputs[1]
with K.tf.variable_scope(self.name):
mask = K.cast(mask, 'int32')
input_shape = K.tf.shape(updates, out_type='int32')
# calculation new shape
if output_shape is None:
output_shape = (input_shape[0],
input_shape[1] * self.up_size[0],
input_shape[2] * self.up_size[1],
input_shape[3])
# calculation indices for batch, height, width and feature maps
one_like_mask = K.ones_like(mask, dtype='int32')
batch_shape = K.concatenate([[input_shape[0]], [1], [1], [1]],
axis=0)
batch_range = K.reshape(K.tf.range(output_shape[0], dtype='int32'),
shape=batch_shape)
b = one_like_mask * batch_range
y = mask // (output_shape[2] * output_shape[3])
x = (mask // output_shape[3]) % output_shape[2]
feature_range = K.tf.range(output_shape[3], dtype='int32')
f = one_like_mask * feature_range
# transpose indices & reshape update values to one dimension
updates_size = K.tf.size(updates)
indices = K.transpose(
K.reshape(K.stack([b, y, x, f]), [4, updates_size]))
values = K.reshape(updates, [updates_size])
ret = K.tf.scatter_nd(indices, values, output_shape)
return ret
def _diagonalize(xyz, mass):
rsq = K.expand_dims(K.sum(xyz**2, axis=-1, keepdims=True), -1)
# xyz::(..., num_neighbors, 3)
# f1, f2::(..., num_neighbors, 3, 3)
f1 = np.eye(3) * rsq
f2 = K.expand_dims(xyz, -2) * K.expand_dims(xyz, -1)
# mass::(..., num_neighbors)
expanded_mass = K.expand_dims(K.expand_dims(mass, -1), -1)
# I_s::(..., 3, 3)
I_s = K.sum((f1 - f2) * expanded_mass, -3)
# rotations::(..., 3, 3)
rotations = _custom_eigvecsh(I_s)
# swap z for -z when an inversion occurs
det_sign = tf.linalg.det(rotations)
inversions = K.stack(
[K.ones_like(det_sign),
K.ones_like(det_sign), det_sign], axis=-1)
rotations = rotations * K.expand_dims(inversions, -2)
rotated_xyz = K.sum(
K.expand_dims(xyz, -1) * K.expand_dims(rotations, -3), -2)
return rotated_xyz, I_s, rotations
def call(self, inputs, **kwargs):
pi = []
for i in range(self.time_steps):
# slice
block = tf.strided_slice(inputs,
begin=[0, i, 0],
end=[
self.batch_size,
i + self.norm_window_size,
self.nb_features
],
strides=[1, 1, 1])
# compute mean & standard deviation
mean = K.mean(block, axis=1, keepdims=False)
std = K.std(inputs[:, i:i + self.norm_window_size],
axis=1,
keepdims=False)
# normlization
input = tf.strided_slice(
inputs,
begin=[0, i + self.norm_window_size - 1, 0],
end=[
self.batch_size, i + self.norm_window_size,
self.nb_features
],
strides=[1, 1, 1])
res = (K.squeeze(input, axis=1) - mean) / (std + K.epsilon())
pi.append(res)
return K.stack(pi, axis=1)
def _compute_valid_seed_region(self):
positions = K.concatenate([
K.expand_dims(K.tile(K.expand_dims(K.arange(self.height), axis=1),
[1, self.width]),
axis=-1),
K.expand_dims(K.tile(K.expand_dims(K.arange(self.width), axis=0),
[self.height, 1]),
axis=-1),
],
axis=-1)
half_block_size = self.block_size // 2
valid_seed_region = K.switch(
K.all(
K.stack(
[
positions[:, :, 0] >= half_block_size,
positions[:, :, 1] >= half_block_size,
positions[:, :, 0] < self.height - half_block_size,
positions[:, :, 1] < self.width - half_block_size,
],
axis=-1,
),
axis=-1,
),
self.ones,
self.zeros,
)
return K.expand_dims(K.expand_dims(valid_seed_region, axis=0), axis=-1)
def shift(shape, stride, anchors):
"""Produce shifted anchors based on shape of the map and stride size.
Args:
shape (tuple): Shape to shift the anchors over.
stride (int): Stride to shift the anchors with over the shape.
anchors (numpy.array): The anchors to apply at each location.
Returns:
numpy.array: shifted anchors
"""
shift_x = (K.arange(0, shape[1], dtype=K.floatx()) +
K.constant(0.5, dtype=K.floatx())) * stride
shift_y = (K.arange(0, shape[0], dtype=K.floatx()) +
K.constant(0.5, dtype=K.floatx())) * stride
shift_x, shift_y = tf.meshgrid(shift_x, shift_y)
shift_x = K.reshape(shift_x, [-1])
shift_y = K.reshape(shift_y, [-1])
shifts = K.stack([shift_x, shift_y, shift_x, shift_y], axis=0)
shifts = K.transpose(shifts)
number_of_anchors = K.shape(anchors)[0]
k = K.shape(shifts)[0] # number of base points = feat_h * feat_w
shifts = K.cast(K.reshape(shifts, [k, 1, 4]), K.floatx())
shifted_anchors = K.reshape(anchors, [1, number_of_anchors, 4]) + shifts
shifted_anchors = K.reshape(shifted_anchors, [k * number_of_anchors, 4])
return shifted_anchors
def call(self, input_tensor, training=None):
input_transposed = tf.transpose(input_tensor, [3, 0, 1, 2, 4])
input_shape = K.shape(input_transposed)
input_tensor_reshaped = K.reshape(input_transposed, [
input_shape[0] * input_shape[1], self.input_height, self.input_width, self.input_num_atoms])
input_tensor_reshaped.set_shape((None, self.input_height, self.input_width, self.input_num_atoms))
conv = K.conv2d(input_tensor_reshaped, self.W, (self.strides, self.strides),
padding=self.padding, data_format='channels_last')
votes_shape = K.shape(conv)
_, conv_height, conv_width, _ = conv.get_shape()
votes = K.reshape(conv, [input_shape[1], input_shape[0], votes_shape[1], votes_shape[2],
self.num_capsule, self.num_atoms])
votes.set_shape((None, self.input_num_capsule, conv_height, conv_width,
self.num_capsule, self.num_atoms))
logit_shape = K.stack([
input_shape[1], input_shape[0], votes_shape[1], votes_shape[2], self.num_capsule])
biases_replicated = K.tile(self.b, [conv_height, conv_width, 1, 1])
activations = update_routing(
votes=votes,
biases=biases_replicated,
logit_shape=logit_shape,
num_dims=6,
input_dim=self.input_num_capsule,
output_dim=self.num_capsule,
num_routing=self.routings)
return activations
def cell_offset_table(scale_size):
# Dynamic implementation of conv dims for fully convolutional model.
# In YOLO the height index is the inner most iteration.
conv_height_index = K.arange(0, stop=scale_size)
conv_width_index = K.arange(0, stop=scale_size)
conv_height_index = K.tile(conv_height_index, [scale_size]) # 늘어놓는 함수 tile -> 같은걸 N번 반복함
# 결과 -> 0~12, 0~12, ...., 0~12
# TODO: Repeat_elements and tf.split doesn't support dynamic splits.
# conv_width_index = K.repeat_elements(conv_width_index, conv_dims[1], axis=0)
conv_width_index = K.tile(
K.expand_dims(conv_width_index, 0), [scale_size, 1]) # tile을 [n, m] 쓰면 dims 2로 만들어줌
# 결과 -> [0~12], [0~12], [0~12], ...
conv_width_index = K.flatten(K.transpose(conv_width_index))
# 결과 -> 0, 0, 0, 0, 0, 0, 0 (13개), 1, 1, 1, 1, 1, 1, 1 (13개), ...
conv_index = K.transpose(K.stack([conv_height_index, conv_width_index]))
# 결과 -> [0, 0], [1, 0], [2, 0], ..., [11, 12], [12, 12]
conv_index = K.reshape(conv_index, [1, scale_size, scale_size, 1, 2])
# 결과 -> 1 * 13 * 13 에 있는 [1 * 2]의 conv index item이 만들어짐
# 각각 [1 * 2]의 값은 [0, 0], [1, 0], [2, 0], ..., [11, 12], [12, 12]
# 이런 식으로 이루어져 있음 -> Mask를 만들기 위한 과정
# 결과 shape -> 1, 13, 13, 1, 2
conv_index = K.cast(conv_index, tf.float32)
diff = (1 / scale_size * 416)
conv_index = conv_index * diff
return conv_index
