def call(self, inputs, mask=None):
input_shape = K.int_shape(inputs)
if input_shape[0]:
# batch size matters, use rnn-based implementation
def step(x, _):
output = self.layer.call(x)
return output, []
_, outputs, _ = K.rnn(
step,
inputs,
initial_states=[],
input_length=input_shape[1],
unroll=False)
y = outputs
else:
# No batch size specified, therefore the layer will be able
# to process batches of any size.
# We can go with reshape-based implementation for performance.
input_length = input_shape[1]
if not input_length:
input_length = K.shape(inputs)[1]
# Shape: (num_samples * timesteps, ...)
inputs = K.reshape(inputs, (-1,) + input_shape[2:])
y = self.layer.call(inputs) # (num_samples * timesteps, ...)
# Shape: (num_samples, timesteps, ...)
output_shape = self._compute_output_shape(input_shape).as_list() # pylint: disable=protected-access
y = K.reshape(y, [-1, input_length] + output_shape[2:])
# Apply activity regularizer if any:
if (hasattr(self.layer, 'activity_regularizer') and
self.layer.activity_regularizer is not None):
regularization_loss = self.layer.activity_regularizer(y)
self.add_loss(regularization_loss, inputs)
return y
def normalize_inference():
if needs_broadcasting:
# In this case we must explictly broadcast all parameters.
broadcast_moving_mean = K.reshape(self.moving_mean,
broadcast_shape)
broadcast_moving_variance = K.reshape(
self.moving_variance, broadcast_shape)
if self.center:
broadcast_beta = K.reshape(self.beta, broadcast_shape)
else:
broadcast_beta = None
if self.scale:
broadcast_gamma = K.reshape(self.gamma,
broadcast_shape)
else:
broadcast_gamma = None
return K.batch_normalization(inputs,
broadcast_moving_mean,
broadcast_moving_variance,
broadcast_beta,
broadcast_gamma,
epsilon=self.epsilon)
else:
return K.batch_normalization(inputs,
self.moving_mean,
self.moving_variance,
self.beta,
self.gamma,
epsilon=self.epsilon)
def get_constants(self, inputs, training=None):
constants = []
if self.implementation != 0 and 0 < self.dropout < 1:
input_shape = K.int_shape(inputs)
input_dim = input_shape[-1]
ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1)))
ones = K.tile(ones, (1, int(input_dim)))
def dropped_inputs():
return K.dropout(ones, self.dropout)
dp_mask = [
K.in_train_phase(dropped_inputs, ones, training=training)
for _ in range(3)
]
constants.append(dp_mask)
else:
constants.append([K.cast_to_floatx(1.) for _ in range(3)])
if 0 < self.recurrent_dropout < 1:
ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1)))
ones = K.tile(ones, (1, self.units))
def dropped_inputs(): # pylint: disable=function-redefined
return K.dropout(ones, self.recurrent_dropout)
rec_dp_mask = [
K.in_train_phase(dropped_inputs, ones, training=training)
for _ in range(3)
]
constants.append(rec_dp_mask)
else:
constants.append([K.cast_to_floatx(1.) for _ in range(3)])
return constants
def normalize_inference():
if needs_broadcasting:
# In this case we must explictly broadcast all parameters.
broadcast_moving_mean = K.reshape(self.moving_mean, broadcast_shape)
broadcast_moving_variance = K.reshape(self.moving_variance,
broadcast_shape)
if self.center:
broadcast_beta = K.reshape(self.beta, broadcast_shape)
else:
broadcast_beta = None
if self.scale:
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
else:
broadcast_gamma = None
return K.batch_normalization(
inputs,
broadcast_moving_mean,
broadcast_moving_variance,
broadcast_beta,
broadcast_gamma,
epsilon=self.epsilon)
else:
return K.batch_normalization(
inputs,
self.moving_mean,
self.moving_variance,
self.beta,
self.gamma,
epsilon=self.epsilon)
def call(self, inputs, mask=None):
input_shape = K.int_shape(inputs)
if input_shape[0]:
# batch size matters, use rnn-based implementation
def step(x, _):
output = self.layer.call(x)
return output, []
_, outputs, _ = K.rnn(
step,
inputs,
initial_states=[],
input_length=input_shape[1],
unroll=False)
y = outputs
else:
# No batch size specified, therefore the layer will be able
# to process batches of any size.
# We can go with reshape-based implementation for performance.
input_length = input_shape[1]
if not input_length:
input_length = K.shape(inputs)[1]
# Shape: (num_samples * timesteps, ...)
inputs = K.reshape(inputs, (-1,) + input_shape[2:])
y = self.layer.call(inputs) # (num_samples * timesteps, ...)
# Shape: (num_samples, timesteps, ...)
output_shape = self._compute_output_shape(input_shape).as_list() # pylint: disable=protected-access
y = K.reshape(y, [-1, input_length] + output_shape[2:])
# Apply activity regularizer if any:
if (hasattr(self.layer, 'activity_regularizer') and
self.layer.activity_regularizer is not None):
regularization_loss = self.layer.activity_regularizer(y)
self.add_loss(regularization_loss, inputs)
return y
def get_constants(self, inputs, training=None):
constants = []
if self.implementation != 0 and 0 < self.dropout < 1:
input_shape = K.int_shape(inputs)
input_dim = input_shape[-1]
ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1)))
ones = K.tile(ones, (1, int(input_dim)))
def dropped_inputs():
return K.dropout(ones, self.dropout)
dp_mask = [
K.in_train_phase(dropped_inputs, ones, training=training)
for _ in range(3)
]
constants.append(dp_mask)
else:
constants.append([K.cast_to_floatx(1.) for _ in range(3)])
if 0 < self.recurrent_dropout < 1:
ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1)))
ones = K.tile(ones, (1, self.units))
def dropped_inputs(): # pylint: disable=function-redefined
return K.dropout(ones, self.recurrent_dropout)
rec_dp_mask = [
K.in_train_phase(dropped_inputs, ones, training=training)
for _ in range(3)
]
constants.append(rec_dp_mask)
else:
constants.append([K.cast_to_floatx(1.) for _ in range(3)])
return constants
def call(self, inputs):
if self._reshape_required:
reshaped_inputs = []
input_ndims = list(map(K.ndim, inputs))
if None not in input_ndims:
# If ranks of all inputs are available,
# we simply expand each of them at axis=1
# until all of them have the same rank.
max_ndim = max(input_ndims)
for x in inputs:
x_ndim = K.ndim(x)
for _ in range(max_ndim - x_ndim):
x = K.expand_dims(x, 1)
reshaped_inputs.append(x)
return self._merge_function(reshaped_inputs)
else:
# Transpose all inputs so that batch size is the last dimension.
# (batch_size, dim1, dim2, ... ) -> (dim1, dim2, ... , batch_size)
transposed = False
for x in inputs:
x_ndim = K.ndim(x)
if x_ndim is None:
x_shape = K.shape(x)
batch_size = x_shape[0]
new_shape = K.concatenate([x_shape[1:], K.expand_dims(batch_size)])
x_transposed = K.reshape(x,
K.stack([batch_size,
K.prod(x_shape[1:])]))
x_transposed = K.permute_dimensions(x_transposed, (1, 0))
x_transposed = K.reshape(x_transposed, new_shape)
reshaped_inputs.append(x_transposed)
transposed = True
elif x_ndim > 1:
dims = list(range(1, x_ndim)) + [0]
reshaped_inputs.append(K.permute_dimensions(x, dims))
transposed = True
else:
# We don't transpose inputs if they are 1D vectors or scalars.
reshaped_inputs.append(x)
y = self._merge_function(reshaped_inputs)
y_ndim = K.ndim(y)
if transposed:
# If inputs have been transposed, we have to transpose the output too.
if y_ndim is None:
y_shape = K.shape(y)
y_ndim = K.shape(y_shape)[0]
batch_size = y_shape[y_ndim - 1]
new_shape = K.concatenate(
[K.expand_dims(batch_size), y_shape[:y_ndim - 1]])
y = K.reshape(y, (-1, batch_size))
y = K.permute_dimensions(y, (1, 0))
y = K.reshape(y, new_shape)
elif y_ndim > 1:
dims = [y_ndim - 1] + list(range(y_ndim - 1))
y = K.permute_dimensions(y, dims)
return y
else:
return self._merge_function(inputs)
def call(self, inputs, training=None, mask=None):
kwargs = {}
if has_arg(self.layer.call, 'training'):
kwargs['training'] = training
uses_learning_phase = False # pylint: disable=redefined-outer-name
input_shape = K.int_shape(inputs)
if input_shape[0]:
# batch size matters, use rnn-based implementation
def step(x, _):
global uses_learning_phase # pylint: disable=global-variable-undefined
output = self.layer.call(x, **kwargs)
if hasattr(output, '_uses_learning_phase'):
uses_learning_phase = (output._uses_learning_phase or
uses_learning_phase)
return output, []
_, outputs, _ = K.rnn(
step,
inputs,
initial_states=[],
unroll=False)
y = outputs
else:
# No batch size specified, therefore the layer will be able
# to process batches of any size.
# We can go with reshape-based implementation for performance.
input_length = input_shape[1]
if not input_length:
input_length = K.shape(inputs)[1]
# Shape: (num_samples * timesteps, ...). And track the
# transformation in self._input_map.
input_uid = tf_base_layers._object_list_uid(inputs)
inputs = K.reshape(inputs, (-1,) + input_shape[2:])
self._input_map[input_uid] = inputs
# (num_samples * timesteps, ...)
y = self.layer.call(inputs, **kwargs)
if hasattr(y, '_uses_learning_phase'):
uses_learning_phase = y._uses_learning_phase
# Shape: (num_samples, timesteps, ...)
output_shape = self._compute_output_shape(input_shape).as_list()
y = K.reshape(y, (-1, input_length) + tuple(output_shape[2:]))
# Apply activity regularizer if any:
if (hasattr(self.layer, 'activity_regularizer') and
self.layer.activity_regularizer is not None):
regularization_loss = self.layer.activity_regularizer(y)
self.add_loss(regularization_loss, inputs)
if uses_learning_phase:
y._uses_learning_phase = True
return y
def call(self, inputs, training=None, mask=None):
kwargs = {}
if has_arg(self.layer.call, 'training'):
kwargs['training'] = training
uses_learning_phase = False # pylint: disable=redefined-outer-name
input_shape = K.int_shape(inputs)
if input_shape[0]:
# batch size matters, use rnn-based implementation
def step(x, _):
global uses_learning_phase # pylint: disable=global-variable-undefined
output = self.layer.call(x, **kwargs)
if hasattr(output, '_uses_learning_phase'):
uses_learning_phase = (output._uses_learning_phase
or uses_learning_phase)
return output, []
_, outputs, _ = K.rnn(step,
inputs,
initial_states=[],
unroll=False)
y = outputs
else:
# No batch size specified, therefore the layer will be able
# to process batches of any size.
# We can go with reshape-based implementation for performance.
input_length = input_shape[1]
if not input_length:
input_length = K.shape(inputs)[1]
# Shape: (num_samples * timesteps, ...). And track the
# transformation in self._input_map.
input_uid = tf_base_layers._object_list_uid(inputs)
inputs = K.reshape(inputs, (-1, ) + input_shape[2:])
self._input_map[input_uid] = inputs
# (num_samples * timesteps, ...)
y = self.layer.call(inputs, **kwargs)
if hasattr(y, '_uses_learning_phase'):
uses_learning_phase = y._uses_learning_phase
# Shape: (num_samples, timesteps, ...)
output_shape = self._compute_output_shape(input_shape).as_list()
y = K.reshape(y, (-1, input_length) + tuple(output_shape[2:]))
# Apply activity regularizer if any:
if (hasattr(self.layer, 'activity_regularizer')
and self.layer.activity_regularizer is not None):
regularization_loss = self.layer.activity_regularizer(y)
self.add_loss(regularization_loss, inputs)
if uses_learning_phase:
y._uses_learning_phase = True
return y
def _time_distributed_dense(x,
w,
b=None,
dropout=None,
input_dim=None,
output_dim=None,
timesteps=None,
training=None):
"""Apply `y . w + b` for every temporal slice y of x.
Arguments:
x: input tensor.
w: weight matrix.
b: optional bias vector.
dropout: wether to apply dropout (same dropout mask
for every temporal slice of the input).
input_dim: integer; optional dimensionality of the input.
output_dim: integer; optional dimensionality of the output.
timesteps: integer; optional number of timesteps.
training: training phase tensor or boolean.
Returns:
Output tensor.
"""
if not input_dim:
input_dim = K.shape(x)[2]
if not timesteps:
timesteps = K.shape(x)[1]
if not output_dim:
output_dim = K.shape(w)[1]
if dropout is not None and 0. < dropout < 1.:
# apply the same dropout pattern at every timestep
ones = K.ones_like(K.reshape(x[:, 0, :], (-1, input_dim)))
dropout_matrix = K.dropout(ones, dropout)
expanded_dropout_matrix = K.repeat(dropout_matrix, timesteps)
x = K.in_train_phase(x * expanded_dropout_matrix, x, training=training)
# collapse time dimension and batch dimension together
x = K.reshape(x, (-1, input_dim))
x = K.dot(x, w)
if b is not None:
x = K.bias_add(x, b)
# reshape to 3D tensor
if K.backend() == 'tensorflow':
x = K.reshape(x, K.stack([-1, timesteps, output_dim]))
x.set_shape([None, None, output_dim])
else:
x = K.reshape(x, (-1, timesteps, output_dim))
return x
def _time_distributed_dense(x,
w,
b=None,
dropout=None,
input_dim=None,
output_dim=None,
timesteps=None,
training=None):
"""Apply `y . w + b` for every temporal slice y of x.
Arguments:
x: input tensor.
w: weight matrix.
b: optional bias vector.
dropout: wether to apply dropout (same dropout mask
for every temporal slice of the input).
input_dim: integer; optional dimensionality of the input.
output_dim: integer; optional dimensionality of the output.
timesteps: integer; optional number of timesteps.
training: training phase tensor or boolean.
Returns:
Output tensor.
"""
if not input_dim:
input_dim = K.shape(x)[2]
if not timesteps:
timesteps = K.shape(x)[1]
if not output_dim:
output_dim = K.shape(w)[1]
if dropout is not None and 0. < dropout < 1.:
# apply the same dropout pattern at every timestep
ones = K.ones_like(K.reshape(x[:, 0, :], (-1, input_dim)))
dropout_matrix = K.dropout(ones, dropout)
expanded_dropout_matrix = K.repeat(dropout_matrix, timesteps)
x = K.in_train_phase(x * expanded_dropout_matrix, x, training=training)
# collapse time dimension and batch dimension together
x = K.reshape(x, (-1, input_dim))
x = K.dot(x, w)
if b is not None:
x = K.bias_add(x, b)
# reshape to 3D tensor
if K.backend() == 'tensorflow':
x = K.reshape(x, K.stack([-1, timesteps, output_dim]))
x.set_shape([None, None, output_dim])
else:
x = K.reshape(x, (-1, timesteps, output_dim))
return x
def yolo_eval(yolo_outputs,
image_shape,
max_boxes=10,
score_threshold=.6,
iou_threshold=.5):
"""Evaluate YOLO model on given input batch and return filtered boxes."""
box_xy, box_wh, box_confidence, box_class_probs = yolo_outputs
boxes = yolo_boxes_to_corners(box_xy, box_wh)
boxes, scores, classes = yolo_filter_boxes(boxes,
box_confidence,
box_class_probs,
threshold=score_threshold)
# Scale boxes back to original image shape.
height = image_shape[0]
width = image_shape[1]
image_dims = K.stack([height, width, height, width])
image_dims = K.reshape(image_dims, [1, 4])
boxes = boxes * image_dims
# TODO: Something must be done about this ugly hack!
max_boxes_tensor = K.variable(max_boxes, dtype='int32')
K.get_session().run(tf.variables_initializer([max_boxes_tensor]))
nms_index = tf.image.non_max_suppression(boxes,
scores,
max_boxes_tensor,
iou_threshold=iou_threshold)
boxes = K.gather(boxes, nms_index)
scores = K.gather(scores, nms_index)
classes = K.gather(classes, nms_index)
return boxes, scores, classes
def call(self, inputs):
# In case the target shape is not fully defined,
# we need access to the shape of x.
target_shape = self.target_shape
if -1 in target_shape:
# target shape not fully defined
target_shape = self._compute_output_shape(inputs.get_shape())
target_shape = target_shape.as_list()[1:]
return K.reshape(inputs, (-1,) + tuple(target_shape))
def call(self, inputs):
stride = self.strides[0]
output_length, feature_dim, filters = self.kernel_shape
xs = []
for i in range(output_length):
slice_length = slice(i * stride, i * stride + self.kernel_size[0])
xs.append(K.reshape(inputs[:, slice_length, :], (1, -1, feature_dim)))
x_aggregate = K.concatenate(xs, axis=0)
# Shape: `(output_length, batch_size, filters)`.
output = K.batch_dot(x_aggregate, self.kernel)
output = K.permute_dimensions(output, (1, 0, 2))
if self.use_bias:
output += K.reshape(self.bias, (1, output_length, filters))
if self.activation is not None:
output = self.activation(output)
return output
def call(self, inputs):
# In case the target shape is not fully defined,
# we need access to the shape of x.
target_shape = self.target_shape
if -1 in target_shape:
# target shape not fully defined
target_shape = self._compute_output_shape(inputs.get_shape())
target_shape = target_shape.as_list()[1:]
return K.reshape(inputs, (-1,) + tuple(target_shape))
def call(self, inputs):
stride = self.strides[0]
output_length, feature_dim, filters = self.kernel_shape
xs = []
for i in range(output_length):
slice_length = slice(i * stride, i * stride + self.kernel_size[0])
xs.append(K.reshape(inputs[:, slice_length, :], (1, -1, feature_dim)))
x_aggregate = K.concatenate(xs, axis=0)
# Shape: `(output_length, batch_size, filters)`.
output = K.batch_dot(x_aggregate, self.kernel)
output = K.permute_dimensions(output, (1, 0, 2))
if self.use_bias:
output += K.reshape(self.bias, (1, output_length, filters))
if self.activation is not None:
output = self.activation(output)
return output
def call(self, inputs):
stride_row, stride_col = self.strides
_, feature_dim, filters = self.kernel_shape
if self.data_format == 'channels_first':
if K.backend() == 'theano':
output = []
for i in range(self.output_row):
for j in range(self.output_col):
slice_row = slice(i * stride_row,
i * stride_row + self.kernel_size[0])
slice_col = slice(j * stride_col,
j * stride_col + self.kernel_size[1])
x_flatten = K.reshape(inputs[:, :, slice_row, slice_col],
(1, -1, feature_dim))
output.append(
K.dot(x_flatten, self.kernel[i * self.output_col + j, :, :]))
output = K.concatenate(output, axis=0)
else:
xs = []
for i in range(self.output_row):
for j in range(self.output_col):
slice_row = slice(i * stride_row,
i * stride_row + self.kernel_size[0])
slice_col = slice(j * stride_col,
j * stride_col + self.kernel_size[1])
xs.append(
K.reshape(inputs[:, :, slice_row, slice_col], (1, -1,
feature_dim)))
x_aggregate = K.concatenate(xs, axis=0)
output = K.batch_dot(x_aggregate, self.kernel)
output = K.reshape(output, (self.output_row, self.output_col, -1,
filters))
output = K.permute_dimensions(output, (2, 3, 0, 1))
elif self.data_format == 'channels_last':
xs = []
for i in range(self.output_row):
for j in range(self.output_col):
slice_row = slice(i * stride_row,
i * stride_row + self.kernel_size[0])
slice_col = slice(j * stride_col,
j * stride_col + self.kernel_size[1])
xs.append(
K.reshape(inputs[:, slice_row, slice_col, :], (1, -1, feature_dim
)))
x_aggregate = K.concatenate(xs, axis=0)
output = K.batch_dot(x_aggregate, self.kernel)
output = K.reshape(output, (self.output_row, self.output_col, -1,
filters))
output = K.permute_dimensions(output, (2, 0, 1, 3))
if self.use_bias:
if self.data_format == 'channels_first':
output += K.reshape(self.bias, (1, filters, self.output_row,
self.output_col))
elif self.data_format == 'channels_last':
output += K.reshape(self.bias, (1, self.output_row, self.output_col,
filters))
output = self.activation(output)
return output
def call(self, inputs):
stride_row, stride_col = self.strides
_, feature_dim, filters = self.kernel_shape
if self.data_format == 'channels_first':
if K.backend() == 'theano':
output = []
for i in range(self.output_row):
for j in range(self.output_col):
slice_row = slice(i * stride_row,
i * stride_row + self.kernel_size[0])
slice_col = slice(j * stride_col,
j * stride_col + self.kernel_size[1])
x_flatten = K.reshape(inputs[:, :, slice_row, slice_col],
(1, -1, feature_dim))
output.append(
K.dot(x_flatten, self.kernel[i * self.output_col + j, :, :]))
output = K.concatenate(output, axis=0)
else:
xs = []
for i in range(self.output_row):
for j in range(self.output_col):
slice_row = slice(i * stride_row,
i * stride_row + self.kernel_size[0])
slice_col = slice(j * stride_col,
j * stride_col + self.kernel_size[1])
xs.append(
K.reshape(inputs[:, :, slice_row, slice_col], (1, -1,
feature_dim)))
x_aggregate = K.concatenate(xs, axis=0)
output = K.batch_dot(x_aggregate, self.kernel)
output = K.reshape(output, (self.output_row, self.output_col, -1,
filters))
output = K.permute_dimensions(output, (2, 3, 0, 1))
elif self.data_format == 'channels_last':
xs = []
for i in range(self.output_row):
for j in range(self.output_col):
slice_row = slice(i * stride_row,
i * stride_row + self.kernel_size[0])
slice_col = slice(j * stride_col,
j * stride_col + self.kernel_size[1])
xs.append(
K.reshape(inputs[:, slice_row, slice_col, :], (1, -1, feature_dim
)))
x_aggregate = K.concatenate(xs, axis=0)
output = K.batch_dot(x_aggregate, self.kernel)
output = K.reshape(output, (self.output_row, self.output_col, -1,
filters))
output = K.permute_dimensions(output, (2, 0, 1, 3))
if self.use_bias:
if self.data_format == 'channels_first':
output += K.reshape(self.bias, (1, filters, self.output_row,
self.output_col))
elif self.data_format == 'channels_last':
output += K.reshape(self.bias, (1, self.output_row, self.output_col,
filters))
output = self.activation(output)
return output
