def yolo_eval(yolo_outputs,
anchors,
num_classes,
image_shape,
max_boxes=20,
score_threshold=.6,
iou_threshold=.5):
"""Evaluate YOLO model on given input and return filtered boxes."""
num_layers = len(yolo_outputs)
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [
[3, 4, 5], [1, 2, 3]] # default setting
input_shape = K.shape(yolo_outputs[0])[1:3] * 32
boxes = []
box_scores = []
for l in range(num_layers):
_boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l],
anchors[anchor_mask[l]],
num_classes,
input_shape,
image_shape)
boxes.append(_boxes)
box_scores.append(_box_scores)
boxes = K.concatenate(boxes, axis=0)
box_scores = K.concatenate(box_scores, axis=0)
mask = box_scores >= score_threshold
max_boxes_tensor = K.constant(max_boxes, dtype='int32')
boxes_ = []
scores_ = []
classes_ = []
for c in range(num_classes):
# TODO: use keras backend instead of tf.
class_boxes = tf.boolean_mask(boxes, mask[:, c])
class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])
nms_index = tf.image.non_max_suppression(
class_boxes, class_box_scores, max_boxes_tensor,
iou_threshold=iou_threshold)
class_boxes = K.gather(class_boxes, nms_index)
class_box_scores = K.gather(class_box_scores, nms_index)
classes = K.ones_like(class_box_scores, 'int32') * c
boxes_.append(class_boxes)
scores_.append(class_box_scores)
classes_.append(classes)
boxes_ = K.concatenate(boxes_, axis=0)
scores_ = K.concatenate(scores_, axis=0)
classes_ = K.concatenate(classes_, axis=0)
# Apply identity to tensor so they can be identified by name
boxes_ = K.identity(boxes_, name='boxes')
scores_ = K.identity(scores_, name='scores')
classes_ = K.identity(classes_, name='classes')
return boxes_, scores_, classes_
def boxes_from_deltas(pred_box_delta, config):
"""
Converts prediction deltas to bounding boxes
Arguments:
pred_box_delta {[type]} -- tensor of deltas
config {[type]} -- hyperparameter dict
Returns:
[type] -- tensor of bounding boxes
"""
# Keras backend allows no unstacking
delta_x = pred_box_delta[:, :, 0]
delta_y = pred_box_delta[:, :, 1]
delta_w = pred_box_delta[:, :, 2]
delta_h = pred_box_delta[:, :, 3]
# get the coordinates and sizes of the anchor boxes from config
anchor_x = config.ANCHOR_BOX[:, 0]
anchor_y = config.ANCHOR_BOX[:, 1]
anchor_w = config.ANCHOR_BOX[:, 2]
anchor_h = config.ANCHOR_BOX[:, 3]
# as we only predict the deltas, we need to transform the anchor box values before computing the loss
box_center_x = K.identity(anchor_x + delta_x * anchor_w)
box_center_y = K.identity(anchor_y + delta_y * anchor_h)
box_width = K.identity(anchor_w * safe_exp(delta_w, config.EXP_THRESH))
box_height = K.identity(anchor_h * safe_exp(delta_h, config.EXP_THRESH))
# tranform into a real box with four coordinates
xmins, ymins, xmaxs, ymaxs = bbox_transform(
[box_center_x, box_center_y, box_width, box_height])
# trim boxes if predicted outside
xmins = K.minimum(K.maximum(0.0, xmins), config.IMAGE_WIDTH - 1.0)
ymins = K.minimum(K.maximum(0.0, ymins), config.IMAGE_HEIGHT - 1.0)
xmaxs = K.maximum(K.minimum(config.IMAGE_WIDTH - 1.0, xmaxs), 0.0)
ymaxs = K.maximum(K.minimum(config.IMAGE_HEIGHT - 1.0, ymaxs), 0.0)
det_boxes = K.permute_dimensions(
K.stack(bbox_transform_inv([xmins, ymins, xmaxs, ymaxs])), (1, 2, 0))
return (det_boxes)
def orthogonal(w):
w_kw = K.int_shape(w)[0]
w_kh = K.int_shape(w)[1]
# w_w = K.int_shape(w)[2]
# w_h = K.int_shape(w)[3]
temp = 0
for i in range(w_kw):
for j in range(w_kh):
wwt = tf.matmul(tf.transpose(w[i, j]), w[i, j])
mi = K.ones_like(wwt) - K.identity(wwt)
a = wwt * mi
a = tf.matmul(tf.transpose(a), a)
a = a * K.identity(a)
temp += K.sum(a)
return 2e-6 * temp
def calculate_pre_cov(self, x):
"""1
4D tensor to 3D (N, nb_filter, col* row)
:param x: Keras.tensor (N, nb_filter, col, row) data being called
:return: Keras.tensor (N, nb_filter, col* row)
"""
xf = self.reshape_tensor3d(x)
xf_mean = K.mean(xf, axis=2, keepdims=True)
if self.normalization == 'mean':
xf_normal = xf - xf_mean
else:
xf_normal = xf
if self.use_kernel:
# Parametric Covariance matrix computation
tx = K.dot(xf_normal, tf.matrix_diag(self.kernel))
tx = K.batch_dot(tx, K.permute_dimensions(xf_normal, [0, 2, 1]))
else:
tx = K.batch_dot(xf_normal, K.permute_dimensions(xf_normal, [0, 2, 1]))
# tx = K.sum(K.multiply(K.expand_dims(xf_normal, dim=1),
# K.expand_dims(xf_normal, dim=2)),
# axis=3)
if self.cov_mode == 'channel' or self.cov_mode == 'mean' or self.cov_mode == 'pmean':
cov = tx / (self.rows * self.cols - 1)
# cov = tx / (self.rows * self.cols )
else:
cov = tx / (self.nb_filter - 1)
if self.normalization == None:
# cov /= (self.rows * self.cols - 1)
cov /= (self.rows * self.cols)
cov = K.identity(cov, 'pre_cov')
return cov, xf_mean
def __init__(self, currLayer, nextAct):
self.layer = currLayer
self.name = currLayer.name
self.type = self.layer.__class__.__name__
self.act = K.identity(nextAct)
self.maxValue = K.epsilon()
self.minValue = -K.epsilon()
def call(self, inputs, **kwargs):
# init_state: [batch_size, n_node, embed_dim]
# adj_matrix: [batch_size, n_edge, n_node, n_node]
init_state, adj_matrix = inputs
n_node = K.shape(init_state)[1]
expand_alpha = K.expand_dims(K.expand_dims(self.alpha, axis=-1),
axis=-1)
weighted_adj_matrix = adj_matrix * K.sigmoid(expand_alpha)
cur_state = K.identity(init_state)
for _ in range(self.n_step):
h = K.dot(cur_state,
self.w) + self.b # [batch_size, n_node, n_edge, units]
neigh_state = []
for edge_idx in range(self.n_edge):
neigh_state.append(
K.batch_dot(weighted_adj_matrix[:, edge_idx, :, :],
h[:, :, edge_idx, :],
axes=(2, 1))) # [batch_size, n_node, units]
neigh_state = K.concatenate(
neigh_state, axis=-1) # [batch_size, n_node, units*n_edge]
gru_inputs = K.reshape(neigh_state, (-1, self.units * self.n_edge))
gru_states = K.reshape(cur_state, (-1, self.units))
# should look up into GRUCell's implementation
gru_output, _ = self.gru_cell.call(inputs=gru_inputs,
states=[gru_states])
cur_state = K.reshape(gru_output, (-1, n_node, self.units))
return cur_state
def call(self, inputs):
segment, memory = inputs
full = K.concatenate([K.zeros_like(memory[:, :, 0]), segment], axis=1)
relative = K.not_equal(K.expand_dims(segment, axis=-1),
K.expand_dims(full, axis=1))
relative = K.one_hot(K.cast(relative, 'uint8'), 2)
return [relative, K.identity(self.embeddings)]
def compute_gradient(model, output_index, input_image):
"""
Computes and returns the gradients of the input image from a single
back-propogation pass through the model, with respect to the output index
args:
model: The model to perform back-propogation on
output_index: The class which to obtain gradients from
input_image: The input which the gradients will come from
"""
# Grab input and outputs
input_tensor = model.input
output_tensor = model.output
wrt_tensor = K.identity(input_tensor)
# Define loss
loss_fn = K.mean(output_tensor[:, output_index])
# Compute gradient
grad_fn = K.gradients(loss_fn, input_tensor)[0]
# Normalize gradients
grad_fn = K.l2_normalize(grad_fn)
# Define the function to compute the gradients
compute_fn = K.function([input_tensor], [loss_fn, grad_fn, wrt_tensor])
# Perform gradient descent
computed_values = compute_fn([input_image])
loss, grads, wrt_value = computed_values
print(grads)
# "Deprocess input"
return grads
def __call__(self, p):
# p = theano.shared(p)
# n, m =p.shape
# print n, m
# p *= T.identity_like(p)
# TODO: review the identity_like funciton
p *= K.identity(p)
return p
def nrmse_a(y_true, y_pred):
"""
Root relative squared error: divide MSE by some variation of y_pred
RRSE = sqrt( MSE / MESS) with ESS = sum((y_true - mean(y_true))**2), MESS = 1/N*ESS
"""
return K.sqrt(
K.mean(K.square(y_true - y_pred)) /
K.mean(K.square(y_pred - K.mean(K.identity(y_true)))))
def r2(y_true, y_pred):
"""
R2 score
$$ 1 - \frac{MSE}{TSS}, \quad TSS= \sum\limits_{i}^{N} (y_i - \bar{y})^2) $$
$$ \bar{y} = \frac{1}{N} sum\limits_{i}^{N} y_i $$
"""
return 1. - K.mean(K.sum(K.square(y_true - y_pred))) / K.sum(
K.square(y_true - K.mean(K.identity(y_true))))
def __init__(self,
input_tensor,
losses,
input_range=(0, 255),
wrt_tensor=None,
norm_grads=True):
"""Creates an optimizer that minimizes weighted loss function.
Args:
input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
losses: List of ([Loss](vis.losses.md#Loss), weight) tuples.
input_range: Specifies the input range as a `(min, max)` tuple. This is used to rescale the
final optimized input to the given range. (Default value=(0, 255))
wrt_tensor: Short for, with respect to. This instructs the optimizer that the aggregate loss from `losses`
should be minimized with respect to `wrt_tensor`.
`wrt_tensor` can be any tensor that is part of the model graph. Default value is set to None
which means that loss will simply be minimized with respect to `input_tensor`.
norm_grads: True to normalize gradients. Normalization avoids very small or large gradients and ensures
a smooth gradient gradient descent process. If you want the actual gradient
(for example, visualizing attention), set this to false.
"""
self.input_tensor = input_tensor
self.input_range = input_range
self.loss_names = []
self.loss_functions = []
self.wrt_tensor = self.input_tensor if wrt_tensor is None else wrt_tensor
if self.input_tensor is self.wrt_tensor:
self.wrt_tensor_is_input_tensor = True
self.wrt_tensor = K.identity(self.wrt_tensor)
else:
self.wrt_tensor_is_input_tensor = False
overall_loss = None
for loss, weight in losses:
# Perf optimization. Don't build loss function with 0 weight.
if weight != 0:
loss_fn = weight * loss.build_loss()
overall_loss = loss_fn if overall_loss is None else overall_loss + loss_fn
self.loss_names.append(loss.name)
self.loss_functions.append(loss_fn)
# Compute gradient of overall with respect to `wrt` tensor.
if self.wrt_tensor_is_input_tensor:
grads1 = K.gradients(overall_loss, self.input_tensor)
grads = K.gradients(grads1, self.input_tensor)[0]
else:
grads1 = K.gradients(overall_loss, self.wrt_tensor)[0]
grads = K.gradients(grads1, self.wrt_tensor)[0]
if norm_grads:
grads = K.l2_normalize(grads)
# The main function to compute various quantities in optimization loop.
self.compute_fn = K.function(
[self.input_tensor, K.learning_phase()],
self.loss_functions + [grads1, grads, self.wrt_tensor])
def modelOutput(model, testInput):
# 값을 입력받아 레이어의 출력을 구하도록 함수 설정하기
input_ = model.input
output_ = [K.identity(layer.output) for layer in model.layers]
func = K.function([input_, K.learning_phase()], output_)
testInput = np.array(testInput)
result = func([testInput, 1]) # Neural Network에 넣고 결과 받아오기
return result # 결과 반환
def bias_initializer(_, *args, **kwargs):
f_init = ChronoInitializer(self.max_timestep)(
(self.units, ), *args, **kwargs)
i_init = -K.identity(f_init)
return K.concatenate([
i_init,
f_init,
self.bias_initializer((self.units * 2, ), *args,
**kwargs),
])
def lstm_chrono_bias_initializer(_, *args, **kwargs):
"""Chrono init for bias vector in LSTMs"""
f_init = \
ChronoInitializer(cfg.keras_cfg['max_time_step'])((int(n_units),),
*args, **kwargs)
i_init = -K.identity(f_init)
return K.concatenate([
i_init,
f_init,
initializers.Zeros()((int(n_units) * 2,), *args, **kwargs),
])
def convert_deltas_to_bboxes(pred_box_delta):
"""
Converts prediction deltas to bounding boxes
:param pred_box_delta: tensor of deltas
:returns: tensor of bounding boxes
"""
def exp_helper_tensor(w, thresh=1.0):
""" Safe exponential function. """
slope = np.exp(thresh)
lin_bool = w > thresh
lin_region = K.cast(lin_bool, dtype='float32')
lin_out = slope * (w - thresh + 1.)
exp_out = K.exp(K.switch(lin_bool, K.zeros_like(w), w))
out = lin_region * lin_out + (1. - lin_region) * exp_out
return out
# Get the coordinates and sizes of the anchor boxes and deltas
anchor_x = para.ANCHOR_BOX[:, 0]
anchor_y = para.ANCHOR_BOX[:, 1]
anchor_w = para.ANCHOR_BOX[:, 2]
anchor_h = para.ANCHOR_BOX[:, 3]
delta_x = pred_box_delta[:, :, 0]
delta_y = pred_box_delta[:, :, 1]
delta_w = pred_box_delta[:, :, 2]
delta_h = pred_box_delta[:, :, 3]
# Get the anchor box values
box_cx = K.identity(anchor_x + delta_x * anchor_w)
box_cy = K.identity(anchor_y + delta_y * anchor_h)
box_w = K.identity(anchor_w * exp_helper_tensor(delta_w))
box_h = K.identity(anchor_h * exp_helper_tensor(delta_h))
# Tranform into a [xmin, ymin, xmax, ymax] format
xmins, ymins, xmaxs, ymaxs = convert_bbox_diagonal(
[box_cx, box_cy, box_w, box_h])
xmins = K.minimum(K.maximum(0.0, xmins), para.IMAGE_WIDTH - 1.0)
ymins = K.minimum(K.maximum(0.0, ymins), para.IMAGE_HEIGHT - 1.0)
xmaxs = K.maximum(K.minimum(para.IMAGE_WIDTH - 1.0, xmaxs), 0.0)
ymaxs = K.maximum(K.minimum(para.IMAGE_HEIGHT - 1.0, ymaxs), 0.0)
det_boxes = K.permute_dimensions(
K.stack(convert_bbox_center([xmins, ymins, xmaxs, ymaxs])), (1, 2, 0))
return (det_boxes)
def call(self, tensor, **kwargs):
base_pooled_tensor = self.base_pool_layer(tensor)
base_pooled_shape = base_pooled_tensor.get_shape()
row_reduce_steps = base_pooled_shape[1] - self.output_size[0]
col_reduce_steps = base_pooled_shape[2] - self.output_size[1]
result = 0
for iter_n in range(self.iterations):
pooled_tensor = K.identity(base_pooled_tensor)
pooled_tensor = self._random_reduce(pooled_tensor,
seed_up=iter_n,
row_steps=row_reduce_steps,
col_steps=col_reduce_steps)
result += pooled_tensor
return result / self.iterations
def __init__(self, input_tensor, losses, input_range=(0, 255), wrt_tensor=None, norm_grads=True):
"""Creates an optimizer that minimizes weighted loss function.
Args:
input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
losses: List of ([Loss](vis.losses#Loss), weight) tuples.
input_range: Specifies the input range as a `(min, max)` tuple. This is used to rescale the
final optimized input to the given range. (Default value=(0, 255))
wrt_tensor: Short for, with respect to. This instructs the optimizer that the aggregate loss from `losses`
should be minimized with respect to `wrt_tensor`.
`wrt_tensor` can be any tensor that is part of the model graph. Default value is set to None
which means that loss will simply be minimized with respect to `input_tensor`.
norm_grads: True to normalize gradients. Normalization avoids very small or large gradients and ensures
a smooth gradient gradient descent process. If you want the actual gradient
(for example, visualizing attention), set this to false.
"""
self.input_tensor = input_tensor
self.input_range = input_range
self.loss_names = []
self.loss_functions = []
self.wrt_tensor = self.input_tensor if wrt_tensor is None else wrt_tensor
if self.input_tensor is self.wrt_tensor:
self.wrt_tensor_is_input_tensor = True
self.wrt_tensor = K.identity(self.wrt_tensor)
else:
self.wrt_tensor_is_input_tensor = False
overall_loss = None
for loss, weight in losses:
# Perf optimization. Don't build loss function with 0 weight.
if weight != 0:
loss_fn = weight * loss.build_loss()
overall_loss = loss_fn if overall_loss is None else overall_loss + loss_fn
self.loss_names.append(loss.name)
self.loss_functions.append(loss_fn)
# Compute gradient of overall with respect to `wrt` tensor.
if self.wrt_tensor_is_input_tensor:
grads = K.gradients(overall_loss, self.input_tensor)[0]
else:
grads = K.gradients(overall_loss, self.wrt_tensor)[0]
if norm_grads:
grads = K.l2_normalize(grads)
# The main function to compute various quantities in optimization loop.
self.compute_fn = K.function([self.input_tensor, K.learning_phase()],
self.loss_functions + [overall_loss, grads, self.wrt_tensor])
def identity(t, name=None):
"""
identity An alias function for Keras naming convention
Parameters
----------
t : backend.Tensor
name: string, optional
Name for the variable to create., by default None
Returns
-------
backend.Tensor
A tensor of zeros has the same shape of input tensor
"""
if backend == 'keras':
return K.identity(t, name=name)
elif backend == 'tf.keras':
return tf.identity(t, name=name)
def __run_rules(self):
"""METHOD::__RUN_RULES: computes the relevance tensor for all the model layers.
---
Returns:
>- {NONE}."""
vrb.print_msg('Run Rules...')
vrb.print_msg('--------------------------')
R = {}
R[self.rules[0].name] = K.identity(self.outputR[-1])
self.__print_rule_information(R[self.rules[0].name])
for k in range(len(self.rules)):
if k != len(self.rules) - 1:
R[self.rules[k + 1].name] = self.rules[k].run(
R[self.rules[k].name], ignoreBias=False)
self.__print_rule_information(R[self.rules[k + 1].name])
else:
R['input'] = self.rules[k].run(R[self.rules[k].name],
ignoreBias=False)
self.__print_rule_information(R['input'])
vrb.print_msg('========== DONE ==========\n')
self.R = R
def call(self, x, mask=None):
if not self.built:
raise Exception("Secondary stat layer not built")
logging.debug('Secondary_stat parameter', type(x)) # Confirm the type of x is indeed tensor4D
cov_mat, x_mean = self.calculate_pre_cov(x)
# print('call during second {}'.format(self.eps))
# cov_mat += self.eps * self.b
if self.robust:
""" Implement the robust estimate, by apply an elementwise function to it. """
if K.backend() != 'tensorflow':
raise RuntimeError("Not support for theano now")
import tensorflow as tf
# with tf.device('/cpu:0'):
s, u = tf.self_adjoint_eig(cov_mat)
comp = tf.zeros_like(s)
s = tf.where(tf.less(s, comp), comp, s)
# s = tf.Print(s, [s], message='s:', summarize=self.out_dim)
inner = robust_estimate_eigenvalues(s, alpha=self.cov_alpha)
inner = tf.identity(inner, 'RobustEigen')
# inner = tf.Print(inner, [inner], message='inner:', summarize=self.out_dim)
cov_mat = tf.matmul(u, tf.matmul(tf.matrix_diag(inner), tf.transpose(u, [0, 2, 1])))
if self.cov_mode == 'mean' or self.cov_mode == 'pmean':
# Encode mean into Cov mat.
addition_array = K.mean(x_mean, axis=1, keepdims=True)
addition_array /= addition_array # Make it 1
if self.cov_mode == 'pmean':
x_mean = self.mean_p * x_mean
new_cov = K.concatenate(
[cov_mat + K.batch_dot(x_mean, K.permute_dimensions(x_mean, (0, 2, 1))), x_mean])
else:
new_cov = K.concatenate([cov_mat, x_mean])
tmp = K.concatenate([K.permute_dimensions(x_mean, (0, 2, 1)), addition_array])
new_cov = K.concatenate([new_cov, tmp], axis=1)
cov_mat = K.identity(new_cov, 'final_cov_mat')
return cov_mat
def call(self, inputs):
"""Generates a new memory based on thequestion and
current inputs.
Parameters
----------
inputs : (list) of (K.Tensor)
A list of size two, where each element is a tensor. The first one is
the facts vector, and the second - the question vector.
Returns
-------
K.Tensor
A memory generated from the question and fact_vectors
"""
facts = inputs[0]
question = inputs[1]
memory = K.identity(question) # Initialize memory to the question
fact_list = tf.unstack(facts, axis=1)
for step in range(self.memory_steps):
# Adapted from
# https://github.com/barronalex/Dynamic-Memory-Networks-in-TensorFlow/
attentions = [tf.squeeze(
self.compute_attention_gate(fact, question, memory), axis=1)
for i, fact in enumerate(fact_list)]
attentions = tf.transpose(tf.stack(attentions))
attentions = tf.expand_dims(attentions, axis=-1)
episode, _, _ = K.rnn(self.step,
inputs=K.concatenate([facts, attentions], axis=2),
constants=[],
initial_states=[memory])
memory = self.memory_activation(K.dot(K.concatenate(
[memory, episode, question], axis=1), self.memory_net) + self.memory_bias)
return memory
def _initialize_params(self, model, use_logits, input_layer, output_layer,
custom_activation):
"""
Initialize most parameters of the classifier. This is a convenience function called by `__init__` and
`__setstate__` to avoid code duplication.
:param model: Keras model
:type model: `keras.models.Model`
:param use_logits: True if the output of the model are the logits.
:type use_logits: `bool`
:param input_layer: Which layer to consider as the Input when the model has multple input layers.
:type input_layer: `int`
:param output_layer: Which layer to consider as the Output when the model has multiple output layers.
:type output_layer: `int`
:param custom_activation: True if the model uses the last activation other than softmax and requires to use the
output probability rather than the logits by attacks.
:type custom_activation: `bool`
"""
import keras.backend as k
if hasattr(model, 'inputs'):
self._input_layer = input_layer
self._input = model.inputs[input_layer]
else:
self._input = model.input
self._input_layer = 0
if hasattr(model, 'outputs'):
self._output = model.outputs[output_layer]
self._output_layer = output_layer
else:
self._output = model.output
self._output_layer = 0
_, self._nb_classes = k.int_shape(self._output)
self._input_shape = k.int_shape(self._input)[1:]
self._custom_activation = custom_activation
logger.debug(
'Inferred %i classes and %s as input shape for Keras classifier.',
self.nb_classes, str(self.input_shape))
# Get predictions and loss function
label_ph = k.placeholder(shape=self._output.shape)
if not hasattr(self._model, 'loss'):
logger.warning(
'Keras model has no loss set. Trying to use `k.sparse_categorical_crossentropy`.'
)
loss_function = k.sparse_categorical_crossentropy
else:
if isinstance(self._model.loss, six.string_types):
loss_function = getattr(k, self._model.loss)
else:
loss_function = getattr(k, self._model.loss.__name__)
self._use_logits = use_logits
if not use_logits:
if k.backend() == 'tensorflow':
if custom_activation:
preds = self._output
loss_ = loss_function(label_ph, preds, from_logits=False)
else:
# We get a list of tensors that comprise the final "layer"
# Take the last element
preds = self._output.op.inputs[-1]
loss_ = loss_function(label_ph, preds, from_logits=True)
else:
loss_ = loss_function(label_ph,
self._output,
from_logits=use_logits)
# Convert predictions to logits for consistency with the other cases
eps = 10e-8
preds = k.log(k.clip(self._output, eps, 1. - eps))
else:
preds = self._output
loss_ = loss_function(label_ph,
self._output,
from_logits=use_logits)
if preds == self._input: # recent Tensorflow version does not allow a model with an output same as the input.
preds = k.identity(preds)
loss_grads = k.gradients(loss_, self._input)
if k.backend() == 'tensorflow':
loss_grads = loss_grads[0]
elif k.backend() == 'cntk':
raise NotImplementedError(
'Only TensorFlow and Theano support is provided for Keras.')
# Set loss, grads and prediction functions
self._preds_op = preds
self._loss = loss_
self._loss_grads = k.function([self._input, label_ph], [loss_grads])
self._preds = k.function([self._input], [preds])
# Get the internal layer
self._layer_names = self._get_layers()
def identity(x):
return K.identity(x)
def nrmse_c(y_true, y_pred):
return K.sqrt(
K.mean(K.sum(K.square(y_true - y_pred))) / K.abs(K.identity(y_true)))
def nrmse_b(y_true, y_pred):
" If this value is larger than 1, you 'd obtain a better model by simply generating a random time series " \
"of the same mean and standard deviation as Y."
return K.sqrt(K.mean(K.sum(K.square(y_true - y_pred)))) / K.std(
K.identity(y_true))
def __init__(self,
clip_values,
model,
use_logits=False,
channel_index=3,
defences=None,
preprocessing=(0, 1),
input_layer=0,
output_layer=0,
custom_activation=False):
"""
Create a `Classifier` instance from a Keras model. Assumes the `model` passed as argument is compiled.
:param clip_values: Tuple of the form `(min, max)` representing the minimum and maximum values allowed
for features.
:type clip_values: `tuple`
:param model: Keras model
:type model: `keras.models.Model`
:param use_logits: True if the output of the model are the logits.
:type use_logits: `bool`
:param channel_index: Index of the axis in data containing the color channels or features.
:type channel_index: `int`
:param defences: Defences to be activated with the classifier.
:type defences: `str` or `list(str)`
:param preprocessing: Tuple of the form `(substractor, divider)` of floats or `np.ndarray` of values to be
used for data preprocessing. The first value will be substracted from the input. The input will then
be divided by the second one.
:type preprocessing: `tuple`
:param input_layer: Which layer to consider as the Input when the model has multple input layers.
:type input_layer: `int`
:param output_layer: Which layer to consider as the Output when the model has multiple output layers.
:type output_layer: `int`
:param custom_activation: True if the model uses the last activation other than softmax and requires to use the
output probability rather than the logits by attacks.
:type custom_activation: `bool`
"""
import keras.backend as k
super(KerasClassifier, self).__init__(clip_values=clip_values,
channel_index=channel_index,
defences=defences,
preprocessing=preprocessing)
self._model = model
if hasattr(model, 'inputs'):
self._input = model.inputs[input_layer]
else:
self._input = model.input
if hasattr(model, 'outputs'):
self._output = model.outputs[output_layer]
else:
self._output = model.output
_, self._nb_classes = k.int_shape(self._output)
self._input_shape = k.int_shape(self._input)[1:]
self._custom_activation = custom_activation
logger.debug(
'Inferred %i classes and %s as input shape for Keras classifier.',
self.nb_classes, str(self.input_shape))
# Get predictions and loss function
label_ph = k.placeholder(shape=(None, ))
if not use_logits:
if k.backend() == 'tensorflow':
if custom_activation:
preds = self._output
loss = k.sparse_categorical_crossentropy(label_ph,
preds,
from_logits=False)
else:
preds, = self._output.op.inputs
loss = k.sparse_categorical_crossentropy(label_ph,
preds,
from_logits=True)
else:
loss = k.sparse_categorical_crossentropy(
label_ph, self._output, from_logits=use_logits)
# Convert predictions to logits for consistency with the other cases
eps = 10e-8
preds = k.log(k.clip(self._output, eps, 1. - eps))
else:
preds = self._output
loss = k.sparse_categorical_crossentropy(label_ph,
self._output,
from_logits=use_logits)
if preds == self._input: # recent Tensorflow version does not allow a model with an output same as the input.
preds = k.identity(preds)
loss_grads = k.gradients(loss, self._input)
if k.backend() == 'tensorflow':
loss_grads = loss_grads[0]
elif k.backend() == 'cntk':
raise NotImplementedError(
'Only TensorFlow and Theano support is provided for Keras.')
# Set loss, grads and prediction functions
self._preds_op = preds
self._loss = k.function([self._input], [loss])
self._loss_grads = k.function([self._input, label_ph], [loss_grads])
self._preds = k.function([self._input], [preds])
# Get the internal layer
self._layer_names = self._get_layers()
def build_heatmap_inference(in_tensor, config, names = None):
'''
detections : [N, 100, 6 {y1, x1, y2, x2}]
'''
# rois_per_image = 32
num_detections = config.DETECTION_MAX_INSTANCES
h, w = config.IMAGE_SHAPE[:2]
img_h, img_w = config.IMAGE_SHAPE[:2]
batch_size = config.BATCH_SIZE
num_classes = config.NUM_CLASSES
print('\n')
print(' > BUILD_HEATMAP_INFERENCE() for ' , names)
# rois per image is determined by size of input tensor
# detection mode: config.TRAIN_ROIS_PER_IMAGE
# ground_truth : config.DETECTION_MAX_INSTANCES
print(' in_tensor shape : ', in_tensor.shape)
in_tensor = in_tensor[:,:,:,2:7]
print(' modified in_tensor shape : ', in_tensor.get_shape())
rois_per_image = tf.to_int32(in_tensor.shape[2])
strt_cls = 0 if rois_per_image == 32 else 1
print(' num of bboxes per class is : ', rois_per_image)
## Build mesh-grid to hold pixel coordinates ----------------------------------
X = tf.range(img_w, dtype=tf.int32)
Y = tf.range(img_h, dtype=tf.int32)
X, Y = tf.meshgrid(X, Y)
# print(' X/Y shapes :', X.get_shape(), Y.get_shape())
# print(' X : \n',X.eval())
# print(' Y : \n',Y.eval())
## repeat X and Y batch_size x rois_per_image times
ones = tf.ones([batch_size, rois_per_image,1, 1], dtype = tf.int32)
rep_X = ones * X
rep_Y = ones * Y
# print(' Ones: ',ones.shape)
# print(' ones_exp * X', ones.shape, '*', X.shape, '= ',rep_X.shape)
# print(' ones_exp * Y', ones.shape, '*', Y.shape, '= ',rep_Y.shape)
# # stack the X and Y grids
bef_pos = tf.to_float(tf.stack([rep_X,rep_Y], axis = -1))
# print(' before transpse ', bef_pos.get_shape())
pos_grid = tf.transpose(bef_pos,[2,3,0,1,4])
print(' after transpose ', pos_grid.get_shape())
# pt2_reshape = tf.reshape( in_tensor , [batch_size, num_classes * rois_per_image ,8])
# print(' pt2_reshape shape is : ', pt2_reshape.get_shape())
# print(pt2_reshape[0].eval())
# print(pt2_reshape[1].eval())
# print(pt2_reshape[2].eval())
## Stack non_zero bboxes from in_tensor into pt2_dense --------------------------
pt2_sum = tf.reduce_sum(tf.abs(in_tensor[:,:,:,:-1]), axis=-1)
print(' pt2_sum shape ',pt2_sum.shape)
# print(pt2_sum[0].eval())
pt2_mask = tf.greater(pt2_sum , 0)
# print(' pt2_mask shape ', pt2_mask.get_shape())
# print(pt2_mask.eval())
pt2_ind = tf.where(pt2_mask)
# print(' pt2_ind shape ', pt2_ind.get_shape())
# print(pt2_ind.eval())
# pt2_ind_float = tf.to_float(pt2_ind[:,0:1])
pt2_dense = tf.gather_nd( in_tensor, pt2_ind)
# print(' dense shape ',pt2_dense.get_shape())
# print(dense.eval())
pt2_dense = tf.concat([tf.to_float(pt2_ind[:,0:1]), pt2_dense],axis=1)
print(' dense shape ',pt2_dense.get_shape())
# print(dense.eval())
# print(dense[1].eval())
# print(dense[2].eval())
# print(dense[3].eval())
stacked_list = tf.dynamic_partition(pt2_dense, tf.to_int32(pt2_ind[:,0]),num_partitions = batch_size )
# print(len(dyn_part))
## Build Stacked output from dynamically partitioned lists ----------------------
print(' Build Stacked output from dynamically partitioned lists --------------')
stacked_output=[]
for img, item in enumerate(stacked_list) :
# rois_in_image, cols = tf.shape(stacked_list[img]).eval()
# print(' img ', img, ' stacked_list[img] ', tf.shape(item).eval() )
rois_in_image = tf.shape(item)[0]
# print(stacked_list[img].eval())
pad_item = tf.pad(item,[[0, rois_per_image - rois_in_image ],[0,0]])
stacked_output.append(pad_item)
# print()
# print(' ===> list item #', img)
# print(' stacked_list[img] shape: ',rois_in_image)
# print(' tensor_list item pos padding :', tf.shape(pad_item))
# print(stacked_list[img].eval())
print()
stacked_tensor = tf.stack(stacked_output)
# print(' stacked_tensor shape : ', tf.shape(stacked_tensor), stacked_tensor.shape, stacked_tensor.get_shape())
# # print(' -- Stacked output contents --------------')
# # for img, item in enumerate(stacked_output) :
# # print('\n ===> list item #', img)
# print(' img ', img, ' stacked_list[img] ', tf.shape(item).eval() )
# print(' img ', img, ' stacked_list[img] ', tf.shape(item).eval()[0] )
width = stacked_tensor[:,:,4] - stacked_tensor[:,:,2]
height = stacked_tensor[:,:,3] - stacked_tensor[:,:,1]
cx = stacked_tensor[:,:,2] + ( width / 2.0)
cy = stacked_tensor[:,:,1] + ( height / 2.0)
means = tf.stack((cx,cy),axis = -1)
covar = tf.stack((width * 0.5 , height * 0.5), axis = -1)
covar = tf.sqrt(covar)
# print(means.eval())
# print(covar.eval())
# print('width shape ',width.get_shape())
# print(mns.eval())
tfd = tf.contrib.distributions
mvn = tfd.MultivariateNormalDiag( loc = means, scale_diag = covar)
prob_grid = mvn.prob(pos_grid)
trans_grid = tf.transpose(prob_grid,[2,3,0,1])
# print(' means shape ', means.get_shape(),' covar shape ', covar.get_shape())
# print(' from MVN : mns shape : ', means.shape, means.get_shape())
# print(' from MVN : cov shape : ', covar.shape, covar.get_shape())
# print(' from MVN : mean shape : ', mvn.mean().get_shape(), '\t stddev shape', mvn.stddev().get_shape())
# print(' from MVN : mvn.batch_shape: ', mvn.batch_shape , '\t mvn.event_shape ', mvn.event_shape)
# print(' from Linear : op shape : ', mvn.scale.shape, ' Linear Op batch shape ',mvn.scale.batch_shape)
# print(' from Linear : op Range Dim : ', mvn.scale.range_dimension)
# print(' from Linear : op Domain Dim : ', mvn.scale.domain_dimension)
print(' >> input to MVN.PROB: pos_grid (meshgrid) shape: ', pos_grid.get_shape())
print(' << output probabilities shape:' , prob_grid.get_shape())
# print(prob_grid.eval())
#--------------------------------------------------------------------------------
# kill distributions of NaN boxes (resulting from bboxes with height/width of zero
# which cause singular sigma cov matrices
#--------------------------------------------------------------------------------
gauss_grid = tf.where(tf.is_nan(trans_grid), tf.zeros_like(trans_grid), trans_grid)
## scatter out the probability distributions based on class ---------------------------
print('\n Scatter out the probability distributions based on class --------------')
class_inds = tf.to_int32(stacked_tensor[:,:,-1])
batch_grid, roi_grid = tf.meshgrid( tf.range(batch_size, dtype=tf.int32), tf.range(rois_per_image, dtype=tf.int32),
indexing = 'ij' )
scatter_classes = tf.stack([batch_grid, class_inds, roi_grid ],axis = -1)
gauss_scatt = tf.scatter_nd(scatter_classes, gauss_grid, [batch_size, num_classes, rois_per_image, img_w, img_h])
print(' gaussian_grid : ', gauss_grid.shape)
print(' class shape : ', class_inds.shape)
print(' roi_grid shape : ', roi_grid.get_shape() )
print(' batch_grid shape : ', batch_grid.get_shape())
print(' scatter_classes : ', scatter_classes.get_shape())
print(' gaussian scattered : ', gauss_scatt.shape)
## sum based on class -----------------------------------------------------------------
print('\n Reduce sum based on class ---------------------------------------------')
gauss_sum = tf.reduce_sum(gauss_scatt, axis=2, name='pred_heatmap')
gauss_sum = tf.where(gauss_sum > 1e-6, gauss_sum,tf.zeros_like(gauss_sum))
gauss_sum = tf.transpose(gauss_sum,[0,2,3,1], name = names[0])
print(' gaussian sum type/name : ', type(gauss_sum), gauss_sum.name, names[0])
print(' gaussian_sum shape : ', gauss_sum.get_shape(), 'Keras tensor ', KB.is_keras_tensor(gauss_sum) )
## L2 normalization -----------------------------------------------------------------
print('\n L2 normalization ------------------------------------------------------')
heatmap_shape=KB.shape(gauss_sum)
print(' pred_shape: KB.shape:' , heatmap_shape, ' tf.get_shape(): ', heatmap_shape.get_shape(), ' pred_maks.shape:',
gauss_sum.shape, 'tf.shape :', tf.shape(gauss_sum))
gauss_flatten = KB.reshape(gauss_sum, (heatmap_shape[0], -1, heatmap_shape[-1]) )
output_norm = KB.l2_normalize(gauss_flatten, axis = 1)
gauss_norm = KB.identity(KB.reshape(output_norm, heatmap_shape ) , name = names[0]+'_norm')
print(' gauss_flatten : ', KB.int_shape(gauss_flatten) , gauss_flatten.get_shape(),' Keras tensor ', KB.is_keras_tensor(gauss_flatten) )
print(' gauss_norm1 : ', KB.int_shape(output_norm) , output_norm.get_shape(),' Keras tensor ', KB.is_keras_tensor(output_norm) )
print(' gauss_norm final : ', KB.int_shape(gauss_norm) , gauss_norm.get_shape(),' Keras tensor ', KB.is_keras_tensor(gauss_norm) )
print(' complete')
return gauss_sum, gauss_norm # [gauss_sum, gauss_scatt, means, covar]
def _apply_map(self, x):
return K.identity(x)
def call(self, inputs):
return K.identity(self.embeddings)
def identity(x):
return K.identity(x)
def fpn_classifier_graph(rois, feature_maps, image_shape, pool_size, num_classes):
'''
Builds the computation graph of the feature pyramid network classifier
and regressor heads.
Inputs:
-------
rois: [batch, num_rois, 4 ]
Proposal boxes in normalized coordinates (y1, x1, y2, x2)
feature_maps: List of feature maps from diffent layers of the pyramid,
[P2, P3, P4, P5]. Each has a different resolution.
image_shape: [height, width, depth]
pool_size: The width of the square feature map generated from ROI Pooling.
num_classes: number of classes, which determines the depth of the results
Returns:
--------
logits: [N, NUM_CLASSES] classifier logits (before softmax)
probs: [N, NUM_CLASSES] classifier probabilities
bbox_deltas: [N, (dy, dx, log(dh), log(dw))]
Deltas to apply to proposal boxes
'''
print('\n>>> FPN Classifier Graph ')
print(' rois shape :', rois.get_shape())
print(' No of feature_maps :', len(feature_maps))
for item in feature_maps:
print(' feature_maps shape :', item.get_shape())
print(' input_shape :', image_shape)
print(' pool_size :', pool_size)
# ROI Pooling
# Shape: [batch, num_boxes, pool_height, pool_width, channels]
x = PyramidROIAlign([pool_size, pool_size], image_shape, name="roi_align_classifier")([rois] + feature_maps)
print(' roi_align_classifier output shape is : ' ,x.get_shape(), x.shape)
# Two 1024 FC layers (implemented with Conv2D for consistency)
x = KL.TimeDistributed(KL.Conv2D(1024, (pool_size, pool_size), padding="valid"), name="mrcnn_class_conv1")(x)
print(' mrcnn_class_conv1 output shape is : ' ,x.get_shape())
x = KL.TimeDistributed(BatchNorm(axis=3), name='mrcnn_class_bn1')(x)
print(' mrcnn_class_bn1 output shape is : ' ,x.get_shape())
x = KL.Activation('relu')(x)
print(' mrcnn_class_relu1 output shape is : ' ,x.get_shape())
# x = KL.Dropout(0.5)(x)
x = KL.TimeDistributed(KL.Conv2D(1024, (1, 1)), name="mrcnn_class_conv2")(x)
print(' mrcnn_class_conv2 output shape is : ' ,x.get_shape())
x = KL.TimeDistributed(BatchNorm(axis=3), name='mrcnn_class_bn2')(x)
print(' mrcnn_class_bn2 output shape is : ' ,x.get_shape())
x = KL.Activation('relu')(x)
print(' mrcnn_class_relu2 output shape is : ' ,x.get_shape())
shared = KL.Lambda(lambda x: KB.squeeze(KB.squeeze(x, 3), 2), name="pool_squeeze")(x)
print(' pool_squeeze(Shared) output shape is : ' , shared.get_shape())
# Classifier head
x = KL.TimeDistributed(KL.Dense(num_classes))(shared)
mrcnn_class_logits = KL.Lambda(lambda x: KB.identity(x, name = 'mrcnn_class_logits'), name='mrcnn_class_logits')(x)
x = KL.TimeDistributed(KL.Activation("softmax"))(mrcnn_class_logits)
mrcnn_probs = KL.Lambda(lambda x: KB.identity(x, name = 'mrcnn_class'), name='mrcnn_class')(x)
print(' mrcnn_class_logits output shape is : ' , mrcnn_class_logits.get_shape())
print(' mrcnn_class_probs output shape is : ' , mrcnn_probs.get_shape())
# BBox head
# [batch, boxes, num_classes * (dy, dx, log(dh), log(dw))]
x = KL.TimeDistributed(KL.Dense(num_classes * 4, activation='linear'),name='mrcnn_bbox_fc')(shared)
print(' mrcnn_bbox_fc output shape is : ' , x.get_shape())
# Reshape to [batch, boxes, num_classes, (dy, dx, log(dh), log(dw))]
s = KB.int_shape(x)
x = KL.Reshape((s[1], num_classes, 4) )(x)
mrcnn_bbox = KL.Lambda(lambda x: KB.identity(x, name = 'mrcnn_bbox'), name="mrcnn_bbox_regression") (x)
print(' mrcnn_bbox output shape is : ' , mrcnn_bbox.get_shape())
return mrcnn_class_logits, mrcnn_probs, mrcnn_bbox
