def call(self, inputs, **kwargs):
image, boxes = inputs
shape = K.cast(K.shape(image), K.floatx())
if K.ndim(image) == 4:
if self.data_format == "channels_first":
height = shape[2]
width = shape[3]
else:
height = shape[1]
width = shape[2]
elif K.ndim(image) == 5:
if self.data_format == "channels_first":
height = shape[3]
width = shape[4]
else:
height = shape[2]
width = shape[3]
if K.ndim(image) == 4:
x1 = tf.clip_by_value(boxes[:, :, 0], 0, width)
y1 = tf.clip_by_value(boxes[:, :, 1], 0, height)
x2 = tf.clip_by_value(boxes[:, :, 2], 0, width)
y2 = tf.clip_by_value(boxes[:, :, 3], 0, height)
return K.stack([x1, y1, x2, y2], axis=2)
elif K.ndim(image) == 5:
x1 = tf.clip_by_value(boxes[:, :, :, 0], 0, width)
y1 = tf.clip_by_value(boxes[:, :, :, 1], 0, height)
x2 = tf.clip_by_value(boxes[:, :, :, 2], 0, width)
y2 = tf.clip_by_value(boxes[:, :, :, 3], 0, height)
return K.stack([x1, y1, x2, y2], axis=3)
def call(self, inputs):
input_shape = self.in_shape
if self.data_format == 'channels_first':
x = K.arange(0, input_shape[1], dtype=K.floatx())
y = K.arange(0, input_shape[2], dtype=K.floatx())
else:
x = K.arange(0, input_shape[0], dtype=K.floatx())
y = K.arange(0, input_shape[1], dtype=K.floatx())
x = x / K.max(x)
y = y / K.max(y)
loc_x, loc_y = tf.meshgrid(x, y, indexing='ij')
if self.data_format == 'channels_first':
loc = K.stack([loc_x, loc_y], axis=0)
else:
loc = K.stack([loc_x, loc_y], axis=-1)
location = K.expand_dims(loc, axis=0)
if self.data_format == 'channels_first':
location = K.permute_dimensions(location, pattern=[0, 2, 3, 1])
location = tf.tile(location, [K.shape(inputs)[0], 1, 1, 1])
if self.data_format == 'channels_first':
location = K.permute_dimensions(location, pattern=[0, 3, 1, 2])
return location
def call(self, inputs):
X = inputs[0] # Node features (B x N x F)
A = inputs[1] # Adjacency matrix (B x N x N)
X_dims = X.get_shape().as_list()
B, N, F = X_dims
outputs = []
attentions = []
for head in range(self.attn_heads):
# W in the paper (F x F")
kernel = self.kernels[head]
# Compute inputs to attention network
features = K.dot(X, kernel) # (B x N x F")
dropout_feat = Dropout(self.dropout_rate)(features) # (B x N x F")
neighbor_kernel = self.neighbor_kernels[head]
attn_kernel = self.attn_kernels[head]
neighbor_features = K.dot(X, neighbor_kernel)
dropout_neighbor = Dropout(self.dropout_rate)(neighbor_features)
merged = tf.matmul(K.dot(dropout_feat, attn_kernel),
tf.transpose(dropout_neighbor, (0, 2, 1)))
attention = tf.nn.tanh(merged)
attention = K.reshape(attention, (-1, N, N))
mask = -10e9 * (1.0 - A)
attention += mask
attention = tf.nn.softmax(attention)
dropout_attn = Dropout(self.dropout_rate)(attention)
node_features = tf.matmul(dropout_attn, dropout_feat)
if self.use_bias:
node_features = K.bias_add(node_features, self.biases[head])
if self.return_attention:
attentions.append(attention)
# Add output of attention head to final output
outputs.append(node_features)
# Aggregate the heads" output according to the reduction method
if self.attn_heads_reduction == "concat":
output = K.concatenate(outputs, axis=-1) # (B x N x KF")
else:
output = K.mean(K.stack(outputs), axis=0) # (B x N x F")
# If "average", compute the activation here (Eq. 6)
output = self.activation(output)
if self.return_attention:
attentions = K.stack(attentions, axis=1)
return (output, attentions)
else:
return output
def compute_mask_loss(boxes,
masks,
annotations,
masks_target,
width,
height,
iou_threshold=0.5,
mask_size=(28, 28)):
"""compute overlap of boxes with annotations"""
iou = overlap(boxes, annotations)
argmax_overlaps_inds = K.argmax(iou, axis=1)
max_iou = K.max(iou, axis=1)
# filter those with IoU > 0.5
indices = tf.where(K.greater_equal(max_iou, iou_threshold))
boxes = tf.gather_nd(boxes, indices)
masks = tf.gather_nd(masks, indices)
argmax_overlaps_inds = K.cast(tf.gather_nd(argmax_overlaps_inds, indices), 'int32')
labels = K.cast(K.gather(annotations[:, 4], argmax_overlaps_inds), 'int32')
# make normalized boxes
x1 = boxes[:, 0]
y1 = boxes[:, 1]
x2 = boxes[:, 2]
y2 = boxes[:, 3]
boxes = K.stack([
y1 / (K.cast(height, dtype=K.floatx()) - 1),
x1 / (K.cast(width, dtype=K.floatx()) - 1),
(y2 - 1) / (K.cast(height, dtype=K.floatx()) - 1),
(x2 - 1) / (K.cast(width, dtype=K.floatx()) - 1),
], axis=1)
# crop and resize masks_target
# append a fake channel dimension
masks_target = K.expand_dims(masks_target, axis=3)
masks_target = tf.image.crop_and_resize(
masks_target,
boxes,
argmax_overlaps_inds,
mask_size
)
masks_target = masks_target[:, :, :, 0] # remove fake channel dimension
# gather the predicted masks using the annotation label
masks = tf.transpose(masks, (0, 3, 1, 2))
label_indices = K.stack([tf.range(K.shape(labels)[0]), labels], axis=1)
masks = tf.gather_nd(masks, label_indices)
# compute mask loss
mask_loss = K.binary_crossentropy(masks_target, masks)
normalizer = K.shape(masks)[0] * K.shape(masks)[1] * K.shape(masks)[2]
normalizer = K.maximum(K.cast(normalizer, K.floatx()), 1)
mask_loss = K.sum(mask_loss) / normalizer
return mask_loss
def input_channel_swap(tensor):
if len(swap_channels) == 3:
return K.stack([
tensor[..., swap_channels[0]], tensor[..., swap_channels[1]],
tensor[..., swap_channels[2]]
],
axis=-1)
elif len(swap_channels) == 4:
return K.stack([
tensor[..., swap_channels[0]], tensor[..., swap_channels[1]],
tensor[..., swap_channels[2]], tensor[..., swap_channels[3]]
],
axis=-1)
def __call__(self, w):
w_shape = w.shape
if w_shape.rank is None or w_shape.rank != 4:
raise ValueError(
'The weight tensor must be of rank 4, but is of shape: %s' %
w_shape)
height, width, channels, kernels = w_shape
w = K.reshape(w, (height, width, channels * kernels))
# TODO(cpeter): Switch map_fn for a faster tf.vectorized_map once K.switch
# is supported.
w = K.map_fn(self._kernel_constraint,
K.stack(array_ops.unstack(w, axis=-1), axis=0))
return K.reshape(K.stack(array_ops.unstack(w, axis=0), axis=-1),
(height, width, channels, kernels))
def shift(shape, stride, anchors):
"""Produce shifted anchors based on shape of the map and stride size.
Args:
shape: Shape to shift the anchors over.
stride: Stride to shift the anchors with over the shape.
anchors: The anchors to apply at each location.
Returns:
shifted anchors
"""
shift_x = (K.arange(0, shape[1], dtype=K.floatx()) +
K.constant(0.5, dtype=K.floatx())) * stride
shift_y = (K.arange(0, shape[0], dtype=K.floatx()) +
K.constant(0.5, dtype=K.floatx())) * stride
shift_x, shift_y = tf.meshgrid(shift_x, shift_y)
shift_x = K.reshape(shift_x, [-1])
shift_y = K.reshape(shift_y, [-1])
shifts = K.stack([shift_x, shift_y, shift_x, shift_y], axis=0)
shifts = K.transpose(shifts)
number_of_anchors = K.shape(anchors)[0]
k = K.shape(shifts)[0] # number of base points = feat_h * feat_w
shifts = K.cast(K.reshape(shifts, [k, 1, 4]), K.floatx())
shifted_anchors = K.reshape(anchors, [1, number_of_anchors, 4]) + shifts
shifted_anchors = K.reshape(shifted_anchors, [k * number_of_anchors, 4])
return shifted_anchors
def call(self, inputs, output_shape=None):
updates, mask = inputs[0], inputs[1]
mask = tf.cast(mask, 'int32')
input_shape = tf.shape(updates, out_type='int32')
# calculation new shape
if output_shape is None:
output_shape = (input_shape[0], input_shape[1] * self.size[0],
input_shape[2] * self.size[1], input_shape[3])
# calculation indices for batch, height, width and feature maps
one_like_mask = K.ones_like(mask, dtype='int32')
batch_shape = K.concatenate([[input_shape[0]], [1], [1], [1]], axis=0)
batch_range = K.reshape(tf.range(output_shape[0], dtype='int32'),
shape=batch_shape)
b = one_like_mask * batch_range
y = mask // (output_shape[2] * output_shape[3])
x = (mask // output_shape[3]) % output_shape[2]
feature_range = tf.range(output_shape[3], dtype='int32')
f = one_like_mask * feature_range
# transpose indices & reshape update values to one dimension
updates_size = tf.size(updates)
indices = K.transpose(
K.reshape(K.stack([b, y, x, f]), [4, updates_size]))
values = K.reshape(updates, [updates_size])
ret = tf.scatter_nd(indices, values, output_shape)
return ret
def gram_matrix(x, norm_by_channels=False):
'''
Returns the Gram matrix of the tensor x.
'''
if K.ndim(x) == 3:
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
shape = K.shape(x)
C, H, W = shape[0], shape[1], shape[2]
gram = K.dot(features, K.transpose(features))
elif K.ndim(x) == 4:
# Swap from (H, W, C) to (B, C, H, W)
x = K.permute_dimensions(x, (0, 3, 1, 2))
shape = K.shape(x)
B, C, H, W = shape[0], shape[1], shape[2], shape[3]
# Reshape as a batch of 2D matrices with vectorized channels
features = K.reshape(x, K.stack([B, C, H * W]))
# This is a batch of Gram matrices (B, C, C).
gram = K.batch_dot(features, features, axes=2)
else:
raise ValueError(
'The input tensor should be either a 3d (H, W, C) or 4d (B, H, W, C) tensor.'
)
# Normalize the Gram matrix
if norm_by_channels:
denominator = C * H * W # Normalization from Johnson
else:
denominator = H * W # Normalization from Google
gram = gram / K.cast(denominator, x.dtype)
return gram
def call(self, inputs):
X = inputs[0] # Node features (N x F)
A = inputs[1] # Adjacency matrix (N x N)
outputs = []
for head in range(self.attn_heads):
kernel = self.kernels[head] # W in the paper (F x F")
attention_kernel = self.attn_kernels[
head] # Attention kernel a in the paper (2F" x 1)
# Compute inputs to attention network
features = K.dot(X, kernel) # (N x F")
# Compute feature combinations
# Note: [[a_1], [a_2]]^T [[Wh_i], [Wh_2]] = [a_1]^T [Wh_i] + [a_2]^T [Wh_j]
attn_for_self = K.dot(
features, attention_kernel[0]) # (N x 1), [a_1]^T [Wh_i]
attn_for_neighs = K.dot(
features, attention_kernel[1]) # (N x 1), [a_2]^T [Wh_j]
# Attention head a(Wh_i, Wh_j) = a^T [[Wh_i], [Wh_j]]
dense = attn_for_self + K.transpose(
attn_for_neighs) # (N x N) via broadcasting
# Add nonlinearty
dense = LeakyReLU(alpha=0.2)(dense)
# Mask values before activation (Vaswani et al., 2017)
mask = -10e9 * (1.0 - A)
dense += mask
# Apply softmax to get attention coefficients
dense = K.softmax(dense) # (N x N)
# Apply dropout to features and attention coefficients
dropout_attn = Dropout(self.dropout_rate)(dense) # (N x N)
dropout_feat = Dropout(self.dropout_rate)(features) # (N x F")
# Linear combination with neighbors" features
node_features = K.dot(dropout_attn, dropout_feat) # (N x F")
if self.use_bias:
node_features = K.bias_add(node_features, self.biases[head])
if self.attn_heads_reduction == "concat":
# If "concat", compute the activation here (Eq. 5)
node_features = self.activation(node_features)
# Add output of attention head to final output
outputs.append(node_features)
# Aggregate the heads" output according to the reduction method
if self.attn_heads_reduction == "concat":
output = K.concatenate(outputs) # (N x KF")
else:
output = K.mean(K.stack(outputs), axis=0) # N x F")
output = self.activation(output)
return output
def greater3(x):
# x = K.print_tensor(x, 'input: ')
x1 = greater3_on_axis(x, 1)
x2 = greater3_on_axis(x, 2)
x3 = greater3_on_axis(x, 3)
x = K.stack([x1, x2, x3])
# x = K.print_tensor(x, 'output: ')
return K.any(x)
def scale_boxes(boxes, image_shape):
""" Scales the predicted boxes in order to be drawable on the image"""
height = image_shape[0]
width = image_shape[1]
image_dims = K.stack([height, width, height, width])
image_dims = K.reshape(image_dims, [1, 4])
boxes = boxes * image_dims
return boxes
def diaggreater3(x):
# x = K.print_tensor(x, 'input: ')
x1 = diaggreater3_on_axis(x, 0, 1, 2)
x2 = diaggreater3_on_axis(x, 1, 0, 2)
x3 = diaggreater3_on_axis(x, 2, 0, 1)
x = K.stack([x1, x2, x3])
# x = tf.Print(x, [x], summarize=64, message='diagonals: ')
return K.any(x)
def _roi_align(args):
boxes = args[0]
scores = args[1]
fpn = args[2]
# compute from which level to get features from
target_levels = self.map_to_level(boxes)
# process each pyramid independently
rois, ordered_indices = [], []
for i in range(len(fpn)):
# select the boxes and classification from this pyramid level
indices = tf.where(K.equal(target_levels, i))
ordered_indices.append(indices)
level_boxes = tf.gather_nd(boxes, indices)
fpn_shape = K.cast(K.shape(fpn[i]), dtype=K.floatx())
# convert to expected format for crop_and_resize
x1 = level_boxes[:, 0]
y1 = level_boxes[:, 1]
x2 = level_boxes[:, 2]
y2 = level_boxes[:, 3]
level_boxes = K.stack([
(y1 / image_shape[1] * fpn_shape[0]) / (fpn_shape[0] - 1),
(x1 / image_shape[2] * fpn_shape[1]) / (fpn_shape[1] - 1),
(y2 / image_shape[1] * fpn_shape[0] - 1) / (fpn_shape[0] - 1),
(x2 / image_shape[2] * fpn_shape[1] - 1) / (fpn_shape[1] - 1),
], axis=1)
if(len(fpn[i].get_shape()) >=4):
unstack = tf.unstack(fpn[i], axis=3)
temp_stack=[]
for j in unstack:
temp = tf.image.crop_and_resize(
K.expand_dims(j, axis=3),
level_boxes,
tf.zeros((K.shape(level_boxes)[0],), dtype='int32'),
(self.crop_size[0], self.crop_size[1]))
temp_stack.append(temp)
rois.append(temp_stack)
else:
rois.append(tf.image.crop_and_resize(
K.expand_dims(fpn[i], axis=0),
level_boxes,
tf.zeros((K.shape(level_boxes)[0],), dtype='int32'),
self.crop_size
))
# concatenate rois to one blob
rois = K.concatenate(rois, axis=0)
# reorder rois back to original order
indices = K.concatenate(ordered_indices, axis=0)
rois = tf.scatter_nd(indices, rois, K.cast(K.shape(rois), 'int64'))
return rois
def mean_iou(self, y_true, y_pred):
"""Implementation of mean IoU metric"""
prec = []
t = 0.5
y_pred_ = tf.to_int32(y_pred > t)
score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, self.nb_classes)
K.get_session().run(tf.local_variables_initializer())
with tf.control_dependencies([up_opt]):
score = tf.identity(score)
prec.append(score)
return K.mean(K.stack(prec), axis=0)
def fulldiaggreater3(x):
xs = []
for filt in [FULLDIAG1, FULLDIAG2, FULLDIAG3, FULLDIAG4]:
x1 = x * tf.constant(filt)
x1 = K.sum(x1, axis=1)
x1 = K.sum(x1, axis=1)
x1 = K.sum(x1, axis=1)
x1 = K.greater(x1, 3)
xs.append(x1)
x = K.stack(xs)
return K.any(x)
def yolo_eval(yolo_outputs,
image_shape,
max_boxes=10,
score_threshold=.6,
iou_threshold=.5):
""" Evaluate YOLO model on given input batch and return filtered boxes.
Parameters
----------
yolo_outputs: Tuple
Contains box_confidence, box_xy, box_wh, box_class_probs variables from yolo_head function
image_shape: tf.Tensor
A Tensor contains image shapes
max_boxes: int
Maximum boxes
score_threshold: float
Probability threshold value
iou_threshold: float
IOU threshold value
Returns
-------
boxes, scores, classes: (tf.Tensor, tf.Tensor, tf.Tensor)
A tuple of Tensors contains boxes, scores and classes.
"""
box_confidence, box_xy, box_wh, box_class_probs = yolo_outputs
boxes = boxes_to_corners(box_xy, box_wh)
boxes, scores, classes = filter_boxes(box_confidence,
boxes,
box_class_probs,
threshold=score_threshold)
# Scale boxes back to original image shape.
height, width = image_shape[0], image_shape[1]
image_dims = K.stack([height, width, height, width])
image_dims = K.reshape(image_dims, [1, 4])
boxes = boxes * image_dims
# TODO: Something must be done about this ugly hack!
max_boxes_tensor = K.variable(max_boxes, dtype='int32')
K.get_session().run(tf.variables_initializer([max_boxes_tensor]))
nms_index = tf.image.non_max_suppression(boxes,
scores,
max_boxes_tensor,
iou_threshold=iou_threshold)
boxes = K.gather(boxes, nms_index)
scores = K.gather(scores, nms_index)
classes = K.gather(classes, nms_index)
return boxes, scores, classes
def _time_distributed_dense(x,
w,
b=None,
dropout=None,
input_dim=None,
output_dim=None,
timesteps=None,
training=None):
"""Apply `y . w + b` for every temporal slice y of x.
# Arguments
x: input tensor.
w: weight matrix.
b: optional bias vector.
dropout: wether to apply dropout (same dropout mask
for every temporal slice of the input).
input_dim: integer; optional dimensionality of the input.
output_dim: integer; optional dimensionality of the output.
timesteps: integer; optional number of timesteps.
training: training phase tensor or boolean.
# Returns
Output tensor.
"""
if not input_dim:
input_dim = K.shape(x)[2]
if not timesteps:
timesteps = K.shape(x)[1]
if not output_dim:
output_dim = K.int_shape(w)[1]
if dropout is not None and 0. < dropout < 1.:
# apply the same dropout pattern at every timestep
ones = K.ones_like(K.reshape(x[:, 0, :], (-1, input_dim)))
dropout_matrix = K.dropout(ones, dropout)
expanded_dropout_matrix = K.repeat(dropout_matrix, timesteps)
x = K.in_train_phase(x * expanded_dropout_matrix, x, training=training)
# collapse time dimension and batch dimension together
x = K.reshape(x, (-1, input_dim))
x = K.dot(x, w)
if b is not None:
x = K.bias_add(x, b)
# reshape to 3D tensor
if K.backend() == 'tensorflow':
x = K.reshape(x, K.stack([-1, timesteps, output_dim]))
x.set_shape([None, None, output_dim])
else:
x = K.reshape(x, (-1, timesteps, output_dim))
return x
def _pad(batch):
"""Helper function to pad nested data within each batch elements."""
padded_dict_batch = {}
if isinstance(batch, dict):
for key, value in six.iteritems(batch):
padded_dict_batch[key] = _pad(value)
return padded_dict_batch
rank = len(batch.shape)
assert rank > 0
missing_count = (self.padded_batch_size -
self.get_real_batch_size(batch))
padding = K.stack([[0, missing_count]] + [[0, 0]] * (rank - 1))
return array_ops.pad(batch, padding, 'constant')
def channel_attention_m(x, residual=False, stream=False):
if not stream:
# dims: BxHxWxCxM (M streams)
if isinstance(x, list):
x = Lambda(lambda var: K.stack(var, axis=4))(x)
y = GlobalMaxPooling3D()(x)
y = Lambda(lambda var: K.expand_dims(
K.expand_dims(K.expand_dims(var, axis=1), axis=2), axis=3))(y)
y = Conv3D(filters=int(K.int_shape(x)[-1] / 2),
kernel_size=1,
strides=1)(y)
y = Activation("relu")(y)
y = Conv3D(filters=K.int_shape(x)[-1], kernel_size=1, strides=1)(y)
y = Activation("softmax")(y)
y = Lambda(lambda var: tf.multiply(*var))([x, y])
if residual:
y = Add()([y, x])
else:
# dims: BxHxWxCxM (M streams)
y = GlobalMaxPooling3D()(x)
y = Lambda(lambda var: K.expand_dims(
K.expand_dims(K.expand_dims(var, axis=1), axis=2), axis=3))(y)
y = Conv3D(filters=int(K.int_shape(x)[-1] / 2),
kernel_size=1,
strides=1)(y)
y = Activation("relu")(y)
y = Conv3D(filters=2, kernel_size=1, strides=1)(y)
y = Activation("sigmoid")(y)
y_l = []
c = int(x.get_shape().as_list()[-1] / 2)
for i in range(2):
ind_st = i * c
ind_end = (i + 1) * c
x_sub = Lambda(slicing,
arguments={
'index': ind_st,
'index_end': ind_end
})(x)
y_sub = Lambda(slicing, arguments={
'index': i,
'index_end': i + 1
})(y)
y = Lambda(lambda var: tf.multiply(*var))([x_sub, y_sub])
if residual:
y = Add()([y, x_sub])
y_l.append(y)
y = concatenate(y_l)
return y
def diaggreater3_on_axis(x, i, j, k):
assert j < k
# x = tf.Print(x, [x], summarize=64, message='initial x: ')
diag = tf.diag(np.array([1, 1, 1, 1], dtype=np.int32))
# x = tf.Print(x, [diag], summarize=64, message='diagnonal: ')
diag = K.stack([diag, diag, diag, diag], axis=i)
# x = tf.Print(x, [diag], summarize=64, message='diagnonal: ')
x = x * diag
# x = tf.Print(x, [x], summarize=64, message='x * diag: ')
x = K.sum(x, axis=k + 1)
# x = tf.Print(x, [x], summarize=64, message='x after first sum: ')
x = K.sum(x, axis=j + 1)
# x = tf.Print(x, [x], summarize=64, message='x after second sum: ')
x = K.greater(x, 3)
x = K.any(x)
return x
def call(self, inputs, **kwargs):
input_shape = K.int_shape(inputs)
tensor_input_shape = K.shape(inputs)
# Prepare broadcasting shape.
reduction_axes = list(range(len(input_shape)))
del reduction_axes[self.axis]
broadcast_shape = [1] * len(input_shape)
broadcast_shape[self.axis] = input_shape[self.axis] // self.groups
broadcast_shape.insert(1, self.groups)
reshape_group_shape = K.shape(inputs)
group_axes = [reshape_group_shape[i] for i in range(len(input_shape))]
group_axes[self.axis] = input_shape[self.axis] // self.groups
group_axes.insert(1, self.groups)
# reshape inputs to new group shape
group_shape = [group_axes[0], self.groups] + group_axes[2:]
group_shape = K.stack(group_shape)
inputs = K.reshape(inputs, group_shape)
group_reduction_axes = list(range(len(group_axes)))
mean, variance = nn_impl.moments(inputs,
group_reduction_axes[2:],
shift=None,
keep_dims=True)
inputs = (inputs - mean) / (K.sqrt(variance + self.epsilon))
# prepare broadcast shape
inputs = K.reshape(inputs, group_shape)
outputs = inputs
# In this case we must explicitly broadcast all parameters.
if self.scale:
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
outputs = outputs * broadcast_gamma
if self.center:
broadcast_beta = K.reshape(self.beta, broadcast_shape)
outputs = outputs + broadcast_beta
# finally we reshape the output back to the input shape
outputs = K.reshape(outputs, tensor_input_shape)
return outputs
def call(self, inputs, **kwargs):
image, boxes = inputs
shape = K.cast(K.shape(image), K.floatx())
ndim = K.ndim(image)
if self.data_format == "channels_first":
height = shape[ndim - 2]
width = shape[ndim - 1]
else:
height = shape[ndim - 3]
width = shape[ndim - 2]
x1, y1, x2, y2 = tf.unstack(boxes, axis=-1)
x1 = tf.clip_by_value(x1, 0, width - 1)
y1 = tf.clip_by_value(y1, 0, height - 1)
x2 = tf.clip_by_value(x2, 0, width - 1)
y2 = tf.clip_by_value(y2, 0, height - 1)
return K.stack([x1, y1, x2, y2], axis=ndim - 2)
def loss(y_true, y_pred):
y_true = K.squeeze(y_true, axis=-1)
# Squeeze y_pred; i.e remove last layer which is of dim 1
y_pred_squeezed = K.squeeze(y_pred, axis=-1)
"""
Reconstructing x_org
"""
# reversed_wnorm = Lambda(lambda x: )
# reversed_wnorm = dict(map(reversed, wnorm.items()))
# x_org = Lambda(lambda x: [tf.reshape(tf.where(tf.equal(wnorm, word)), [-1])[0] for sent in x for word in sent])(y_true)
# x_org =
# x_org = [reversed_wnorm.get(word) for sent in y_true for word in sent]
x_org = raw_input
print(raw_input.shape)
x_temp = Lambda(lambda x: tf.cast(tf.reshape(x, [-1, ]), dtype=tf.int32))(x_org)
K.print_tensor(K.shape(y_pred_squeezed), message='y_pred_squeezed are ')
print(f'Inside decoder....After reshape of x_norm is {y_pred_squeezed.shape}')
# Calc prob logits
print(type(y_pred_squeezed))
print(type(wnorm))
print(f'wnorm shape is {wnorm.shape}')
prob_logits = K.batch_dot(y_pred_squeezed, wnorm, axes=[2, 1])
prob = Lambda(lambda x: tf.nn.log_softmax(x * 100, axis=-1, name='prob_lambda'))(prob_logits)
print(f'Prob shape is {prob.shape}')
prob = Lambda(lambda x: tf.reshape(x, [-1, n_words]))(prob)
# prob = K.reshape(prob, [-1, wnorm.shape[0]])
print(f'Prob reshaped is {prob.shape}')
"""
Get prob of all the words
"""
idx = Lambda(lambda x: tf.range(K.shape(x)[0], K.shape(x)[1]))(y_pred_squeezed)
all_idx = K.transpose(K.stack([idx, x_temp]))
all_prob = Lambda(lambda prob_idx_list: tf.gather_nd(prob_idx_list[0], prob_idx_list[1]))([prob, all_idx])
K.print_tensor(K.shape(all_prob), message='all_prob shape is: ')
recons_loss = Lambda(lambda x: -tf.reduce_mean(x))(all_prob)
# K.print_tensor(loss, message='Loss is: ')
# weighted_recons_loss = loss_weight * recons_loss
return recons_loss
def get_relation_vectors(x):
objects = []
relations = []
shape = K.int_shape(x)
k = 25 # Hyperparameter which controls how many objects are considered
keys = []
while k > 0:
i = ra.randint(0, shape[1] - 1)
j = ra.randint(0, shape[2] - 1)
if not (i, j) in keys:
keys.append((i, j))
objects.append(x[:, i, j, :])
k -= 1
for i in range(len(objects)):
for j in range(i, len(objects)):
relations.append(K.concatenate([objects[i], objects[j]], axis=1))
return K.permute_dimensions(K.stack([r for r in relations], axis=0), [1, 0, 2])
def antidiaggreater3(x):
x1 = x * tf.constant(ANTIDIAG23)
x1 = K.sum(x1, axis=2)
x1 = K.sum(x1, axis=2)
x1 = K.greater(x1, 3)
x1 = K.any(x1)
x2 = x * tf.constant(ANTIDIAG13)
x2 = K.sum(x2, axis=2)
x2 = K.sum(x2, axis=2)
x2 = K.greater(x2, 3)
x2 = K.any(x2)
x3 = x * tf.constant(ANTIDIAG12)
x3 = K.sum(x3, axis=2)
x3 = K.sum(x3, axis=2)
x3 = K.greater(x3, 3)
x3 = K.any(x3)
x = K.stack([x1, x2, x3])
return K.any(x)
def bbox_transform_inv(boxes, deltas, mean=None, std=None):
"""Applies deltas (usually regression results) to boxes (usually anchors).
Before applying the deltas to the boxes, the normalization that was
previously applied (in the generator) has to be removed.
The mean and std are the mean and std as applied in the generator.
They are unnormalized in this function and then applied to the boxes.
Args:
boxes: np.array of shape (B, N, 4), where B is the batch size,
N the number of boxes and 4 values for (x1, y1, x2, y2).
deltas: np.array of same shape as boxes. These deltas
(d_x1, d_y1, d_x2, d_y2) are a factor of the width/height.
mean: The mean value used when computing deltas
(defaults to [0, 0, 0, 0]).
std: The standard deviation used when computing deltas
(defaults to [0.2, 0.2, 0.2, 0.2]).
Returns:
A np.array of the same shape as boxes with deltas applied to each box.
The mean and std are used during training to normalize the
regression values (networks love normalization).
"""
if mean is None:
mean = [0, 0, 0, 0]
if std is None:
std = [0.2, 0.2, 0.2, 0.2]
width = boxes[:, :, 2] - boxes[:, :, 0]
height = boxes[:, :, 3] - boxes[:, :, 1]
x1 = boxes[:, :, 0] + (deltas[:, :, 0] * std[0] + mean[0]) * width
y1 = boxes[:, :, 1] + (deltas[:, :, 1] * std[1] + mean[1]) * height
x2 = boxes[:, :, 2] + (deltas[:, :, 2] * std[2] + mean[2]) * width
y2 = boxes[:, :, 3] + (deltas[:, :, 3] * std[3] + mean[3]) * height
pred_boxes = K.stack([x1, y1, x2, y2], axis=2)
return pred_boxes
def call(self, inputs, training=None, mask=None):
"""
Compute the derivative for every species, on the device given by self.deviceIdx (set at initialization of the object).
:param inputs: 1d-array or tensor of the concentration of input species
:param training: legacy, do not use
:param mask: legacy, do not use
:return: The derivative for every species in a Tensor.
"""
# with tf.device(self.deviceIdx):
if (not self.isSparse):
#outputs = []
# for idx,mask2 in enumerate(self.maskIdxList):
# # The following expression is described precisely in the docs.
# outputs += [backend.sum(tf.multiply(tf.log(tf.multiply(mask2,X)+self.maskComplementaryList[idx]),self.stoichioList[idx]),axis=1)]
# x = tf.multiply(
# tf.exp(backend.stack(outputs,axis=0)),
# self.Karray)
x = tf.multiply(
tf.exp(
backend.sum(tf.multiply(
tf.log(
tf.multiply(self.tfmask, inputs) +
self.tfMaskComplementary), self.tfStoichio),
axis=2)), self.Karray)
derivatives = backend.sum(x, axis=0) + self.derivLeak
return tf.reshape(derivatives, (1, derivatives.shape[0]))
else:
outputs = []
x2 = tf.log(inputs)
for idx, mask2 in enumerate(self.masklist):
outputs += [
tf.exp(
tf.nn.embedding_lookup_sparse(x2,
mask2,
self.maskWeightList[idx],
combiner="sum"))
]
x = tf.multiply(backend.stack(outputs, axis=0), self.Karray)
return backend.sum(x, axis=0) + self.derivLeak
def _roi_align(args):
boxes = args[0]
fpn = args[1] # process the feature map
x1 = boxes[:, 0]
y1 = boxes[:, 1]
x2 = boxes[:, 2]
y2 = boxes[:, 3]
fpn_shape = K.cast(K.shape(fpn), dtype=K.floatx())
norm_boxes = K.stack([
(y1 / image_shape[1] * fpn_shape[0]) / (fpn_shape[0] - 1),
(x1 / image_shape[2] * fpn_shape[1]) / (fpn_shape[1] - 1),
(y2 / image_shape[1] * fpn_shape[0] - 1) / (fpn_shape[0] - 1),
(x2 / image_shape[2] * fpn_shape[1] - 1) / (fpn_shape[1] - 1)
],
axis=1)
rois = tf.image.crop_and_resize(
K.expand_dims(fpn, axis=0), norm_boxes,
tf.zeros((K.shape(norm_boxes)[0], ), dtype='int32'),
self.crop_size)
return rois
def _filter_detections(scores, labels):
# threshold based on score
indices = tf.where(K.greater(scores, score_threshold))
if nms:
filtered_boxes = tf.gather_nd(boxes, indices)
filtered_scores = K.gather(scores, indices)[:, 0]
# perform NMS
nms_indices = tf.image.non_max_suppression(
filtered_boxes,
filtered_scores,
max_output_size=max_detections,
iou_threshold=nms_threshold)
# filter indices based on NMS
indices = K.gather(indices, nms_indices)
# add indices to list of all indices
labels = tf.gather_nd(labels, indices)
indices = K.stack([indices[:, 0], labels], axis=1)
return indices
