def __shuffle_channels(tensor, n_groups):
tensor_shape = K.int_shape(tensor)
n_timesteps, side_dim1, side_dim2, n_channels = tensor_shape
n_channels_per_group = n_channels / n_groups
assert n_channels_per_group * n_groups == n_channels
# shuffle channels
tensor = ReshapeLayer((n_timesteps, side_dim1, side_dim2, n_groups,
n_channels_per_group))(tensor)
tensor = TransposeLayer((0, 1, 2, 3, 5, 4))(tensor)
tensor = ReshapeLayer(
(n_timesteps, side_dim1, side_dim2, n_channels))(tensor)
return tensor
def timeception_temporal_convolutions_parallelized(tensor,
n_layers,
n_groups,
expansion_factor,
is_dilated=True):
input_shape = K.int_shape(tensor)
assert len(input_shape) == 5
raise Exception('Sorry, not implemented now')
_, n_timesteps, side_dim, side_dim, n_channels_in = input_shape
# collapse regions in one dim
tensor = ReshapeLayer(
(n_timesteps, side_dim * side_dim, 1, n_channels_in))(tensor)
for i in range(n_layers):
# add global pooling as regions
tensor = __global_spatial_pooling(tensor)
# temporal conv (inception-style, shuffled)
n_channels_per_branch, n_channels_out = __get_n_channels_per_branch(
n_groups, expansion_factor, n_channels_in)
if is_dilated:
tensor = __timeception_shuffled_depthwise_dilated_parallelized(
tensor, n_groups, n_channels_per_branch)
else:
tensor = __timeception_shuffled_depthwise_parallelized(
tensor, n_groups, n_channels_per_branch)
tensor = MaxPooling3D(pool_size=(2, 1, 1))(tensor)
n_channels_in = n_channels_out
return tensor
def call(self, input, mask=None):
n_layers = self.n_layers
is_dilated = self.is_dilated
_, n_timesteps, side_dim, side_dim, n_channels_in = K.int_shape(input)
# collapse regions in one dim
tensor = ReshapeLayer(
(n_timesteps, side_dim * side_dim, 1, n_channels_in))(input)
# create n laters
for i in range(n_layers):
# add global pooling as regions
tensor = self.__global_spatial_pooling(tensor)
# temporal conv (inception-style, shuffled)
n_channels_per_branch, n_channels_out = self.__get_n_channels_per_branch(
n_channels_in)
if is_dilated:
tensor = self.__inception_style_temporal_layer_shuffled_depthwise_dilated_complicated(
tensor, n_channels_per_branch)
else:
tensor = self.__inception_style_temporal_layer_shuffled_depthwise_complicated(
tensor, n_channels_per_branch)
# temporal max pool
tensor = MaxPooling3D(pool_size=(2, 1, 1))(tensor)
n_channels_in = n_channels_out
return tensor
def __convolve_nodes(tensor, n_nodes, layer_id, kernel_size):
"""
Input size (None, 100, 7, 7, 1024)
"""
# unreveal nodes dimension
_, n_timesteps, side_dim, side_dim, n_channels_in = K.int_shape(tensor) # (None, 64, 7, 7, 1024)
tensor = ReshapeLayer((n_nodes, n_timesteps, side_dim, side_dim, n_channels_in))(tensor) # (None, 100, 64, 7, 7, 1024)
# hide temporal dimension
tensor = TransposeLayer((0, 2, 1, 3, 4, 5))(tensor) # (None, 64, 100, 7, 7, 1024)
tensor = ReshapeLayer((n_nodes, side_dim, side_dim, n_channels_in))(tensor) # (None, 100, 7, 7, 1024)
# node conv
tensor = DepthwiseConv1DLayer(kernel_size, padding='SAME', name='conv_n_%s' % (layer_id))(tensor) # (None, 100, 7, 7, 1024)
return tensor
def __load_model_mlp_classifier_action_vlad(n_classes, input_shape, n_gpus,
is_load_weights, weight_path):
"""
Model
"""
# optimizer and loss
loss = keras_utils.LOSSES[0]
metrics = [keras_utils.METRICS[0]]
output_activation = keras_utils.ACTIVATIONS[3]
optimizer = SGD(lr=0.01)
optimizer = Adam(lr=0.01, epsilon=1e-8)
optimizer = Adam(lr=0.01, epsilon=1e-4)
expansion_factor = 5.0 / 4.0
_, n_timesteps, side_dim, _, n_channels_in = input_shape
input_shape = (input_shape[1:])
t_input = Input(shape=input_shape) # (None, 7, 7, 1024)
tensor = t_input
# spatial convolution
n_channels_out = 512
tensor = Conv3D(n_channels_out, kernel_size=(1, 1, 1),
padding='same')(tensor)
tensor = BatchNormalization()(tensor)
tensor = Activation('relu')(tensor)
n_channels_in = n_channels_out
# reshape for vlad
tensor = ReshapeLayer((n_channels_in, ))(tensor)
# vlad layer
max_samples = n_timesteps * side_dim * side_dim
tensor = NetVLAD(n_channels_in, max_samples, 32)(tensor)
# dense layers
tensor = Dropout(0.5)(tensor)
tensor = Dense(256)(tensor)
tensor = BatchNormalization()(tensor)
tensor = LeakyReLU(alpha=0.2)(tensor)
tensor = Dropout(0.25)(tensor)
tensor = Dense(n_classes)(tensor)
t_output = Activation(output_activation)(tensor)
model = Model(input=t_input, output=t_output)
if is_load_weights:
model.load_weights(weight_path)
if n_gpus == 1:
model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
parallel_model = model
else:
parallel_model = multi_gpu_utils.multi_gpu_model(model, n_gpus)
parallel_model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
return model, parallel_model
def graph_embedding(tensor, n_layers, n_avg_size, n_kernel_size, t_kernel_size, n_max_size, t_max_size):
"""
Graph embedding.
:param tensor:
:param n_layers:
:return:
"""
input_shape = K.int_shape(tensor)
_, n_odes, n_timesteps, side_dim, side_dim, n_channels_in = input_shape
# hide temporal dimension
tensor = TransposeLayer((0, 2, 1, 3, 4, 5))(tensor) # (None, 64, 100, 7, 7, 1024)
tensor = ReshapeLayer((n_odes, side_dim, side_dim, n_channels_in))(tensor)
# pool over node
tensor = AveragePooling3D(pool_size=(n_avg_size, 1, 1), name='pool_n')(tensor)
_, n_odes, side_dim, side_dim, n_channels_in = K.int_shape(tensor)
# recover node dimension
tensor = ReshapeLayer((n_timesteps, n_odes, side_dim, side_dim, n_channels_in))(tensor) # (None, 64, 100, 7, 7, 1024)
tensor = TransposeLayer((0, 2, 1, 3, 4, 5))(tensor) # (None, 100, 64, 7, 7, 1024)
# hide the node dimension
tensor = ReshapeLayer((n_timesteps, side_dim, side_dim, n_channels_in))(tensor) # (None, 64, 7, 7, 1024)
# 2 layers spatio-temporal conv
for i in range(n_layers):
layer_id = '%d' % (i + 1)
# spatial conv
tensor = Conv3D(n_channels_in, (1, 1, 1), padding='SAME', name='conv_s_%s' % (layer_id))(tensor) # (None, 64, 7, 7, 1024)
# temporal conv
tensor = DepthwiseConv1DLayer(t_kernel_size, padding='SAME', name='conv_t_%s' % (layer_id))(tensor) # (None, 64, 7, 7, 1024)
# node conv
tensor = __convolve_nodes(tensor, n_odes, layer_id, n_kernel_size) # (None, 100, 7, 7, 1024)
# activation
tensor = BatchNormalization()(tensor)
tensor = LeakyReLU(alpha=0.2)(tensor)
# max_pool over nodes
tensor = MaxPooling3D(pool_size=(n_max_size, 1, 1), name='pool_n_%s' % (layer_id))(tensor) # (None, 100, 7, 7, 1024)
_, n_odes, side_dim, side_dim, n_channels_in = K.int_shape(tensor)
# get back temporal dimension and hide node dimension
tensor = ReshapeLayer((n_timesteps, n_odes, side_dim, side_dim, n_channels_in))(tensor) # (None, 64, 100, 7, 7, 1024)
tensor = TransposeLayer((0, 2, 1, 3, 4, 5))(tensor) # (None, 100, 64, 7, 7, 1024)
tensor = ReshapeLayer((n_timesteps, side_dim, side_dim, n_channels_in))(tensor) # (None, 64, 7, 7, 1024)
# max_pool over time
tensor = MaxPooling3D(pool_size=(t_max_size, 1, 1), name='pool_t_%s' % (layer_id))(tensor) # (None, 64, 7, 7, 1024)
_, n_timesteps, side_dim, side_dim, n_channels_in = K.int_shape(tensor) # (None, 64, 7, 7, 1024)
# recover nodes dimension
tensor = ReshapeLayer((n_odes, n_timesteps, side_dim, side_dim, n_channels_in))(tensor)
return tensor
def __inception_style_temporal_layer_shuffled_depthwise_dilated_complicated(
self, tensor_input, n_channels_per_branch):
n_groups = self.n_groups
_, n_timesteps, side_dim1, side_dim2, n_channels_in = K.int_shape(
tensor_input)
assert n_channels_in % n_groups == 0
n_branches = 5
n_channels_per_group_in = n_channels_in / n_groups
n_channels_out = n_groups * n_branches * n_channels_per_branch
n_channels_per_group_out = n_channels_out / n_groups
assert n_channels_out % n_groups == 0
# slice maps into groups
tensors = Lambda(lambda x: [
x[:, :, :, :, i * n_channels_per_group_in:
(i + 1) * n_channels_per_group_in] for i in range(n_groups)
])(tensor_input)
t_outputs = []
for idx_group in range(n_groups):
tensor_group = tensors[idx_group]
# branch 1: dimension reduction only and no temporal conv
t_0 = Conv3D(n_channels_per_branch,
kernel_size=(1, 1, 1),
padding='same')(tensor_group)
t_0 = BatchNormalization()(t_0)
# branch 2: dimension reduction followed by depth-wise temp conv (kernel-size 3, dilation 1)
t_3 = Conv3D(n_channels_per_branch,
kernel_size=(1, 1, 1),
padding='same')(tensor_group)
t_3 = DepthwiseDilatedConv1DLayer(3, 1, padding='same')(t_3)
t_3 = BatchNormalization()(t_3)
# branch 3: dimension reduction followed by depth-wise temp conv (kernel-size 3, dilation 2)
t_5 = Conv3D(n_channels_per_branch,
kernel_size=(1, 1, 1),
padding='same')(tensor_group)
t_5 = DepthwiseDilatedConv1DLayer(3, 2, padding='same')(t_5)
t_5 = BatchNormalization()(t_5)
# branch 4: dimension reduction followed by depth-wise temp conv (kernel-size 3, dilation 3)
t_7 = Conv3D(n_channels_per_branch,
kernel_size=(1, 1, 1),
padding='same')(tensor_group)
t_7 = DepthwiseDilatedConv1DLayer(3, 3, padding='same')(t_7)
t_7 = BatchNormalization()(t_7)
# # branch 5: dimension reduction followed by temporal max pooling
t_1 = Conv3D(n_channels_per_branch,
kernel_size=(1, 1, 1),
padding='same')(tensor_group)
t_1 = MaxPooling3D(pool_size=(2, 1, 1),
strides=(1, 1, 1),
padding='same')(t_1)
t_1 = BatchNormalization()(t_1)
# concatenate channels of branches
tensor = Concatenate(axis=4)([t_0, t_3, t_5, t_7, t_1])
t_outputs.append(tensor)
# concatenate channels of groups
tensor = Concatenate(axis=4)(t_outputs)
tensor = Activation('relu')(tensor)
# shuffle channels
tensor = ReshapeLayer((n_timesteps, side_dim1, side_dim2, n_groups,
n_channels_per_group_out))(tensor)
tensor = TransposeLayer((0, 1, 2, 3, 5, 4))(tensor)
tensor = ReshapeLayer(
(n_timesteps, side_dim1, side_dim2, n_channels_out))(tensor)
return tensor
def __timeception_shuffled_depthwise_parallelized(tensor_input, n_groups,
n_channels_per_branch):
_, n_timesteps, side_dim1, side_dim2, n_channels_in = tensor_input.get_shape(
).as_list()
assert n_channels_in % n_groups == 0
n_branches = 5
n_channels_per_group_in = n_channels_in / n_groups
n_channels_out = n_groups * n_branches * n_channels_per_branch
n_channels_per_group_out = n_channels_out / n_groups
assert n_channels_out % n_groups == 0
# define all layers
l_01 = Conv3D(n_channels_per_branch, kernel_size=(1, 1, 1), padding='same')
l_02 = BatchNormalization()
l_31 = Conv3D(n_channels_per_branch, kernel_size=(1, 1, 1), padding='same')
l_32 = DepthwiseConv1DLayer(3, padding='same')
l_33 = BatchNormalization()
l_51 = Conv3D(n_channels_per_branch, kernel_size=(1, 1, 1), padding='same')
l_52 = DepthwiseConv1DLayer(5, padding='same')
l_53 = BatchNormalization()
l_71 = Conv3D(n_channels_per_branch, kernel_size=(1, 1, 1), padding='same')
l_72 = DepthwiseConv1DLayer(7, padding='same')
l_73 = BatchNormalization()
l_11 = Conv3D(n_channels_per_branch, kernel_size=(1, 1, 1), padding='same')
l_12 = MaxPooling3D(pool_size=(2, 1, 1), strides=(1, 1, 1), padding='same')
l_13 = BatchNormalization()
keras_layers = [
l_01, l_02, l_31, l_32, l_33, l_51, l_52, l_53, l_71, l_72, l_73, l_11,
l_12, l_13
]
tensor = TimeceptionTemporalBlock(keras_layers, n_groups,
n_channels_per_group_in,
n_channels_per_branch)(tensor_input)
# slice maps into groups
tensors = Lambda(lambda x: [
x[:, :, :, :, i * n_channels_per_group_in:
(i + 1) * n_channels_per_group_in] for i in range(n_groups)
])(tensor_input)
t_outputs = []
for idx_group in range(n_groups):
tensor_group = tensors[idx_group]
# branch 1: dimension reduction only and no temporal conv
t_0 = l_01(tensor_group)
t_0 = l_02(t_0)
# branch 2: dimension reduction followed by depth-wise temp conv (kernel-size 3)
t_3 = l_31(tensor_group)
t_3 = l_32(t_3)
t_3 = l_33(t_3)
# branch 3: dimension reduction followed by depth-wise temp conv (kernel-size 5)
t_5 = l_51(tensor_group)
t_5 = l_52(t_5)
t_5 = l_53(t_5)
# branch 4: dimension reduction followed by depth-wise temp conv (kernel-size 7)
t_7 = l_71(tensor_group)
t_7 = l_72(t_7)
t_7 = l_73(t_7)
# branch 5: dimension reduction followed by temporal max pooling
t_1 = l_11(tensor_group)
t_1 = l_12(t_1)
t_1 = l_13(t_1)
# concatenate channels of branches
tensor = Concatenate(axis=4)([t_0, t_3, t_5, t_7, t_1])
t_outputs.append(tensor)
# concatenate channels of groups
tensor = Concatenate(axis=4)(t_outputs)
tensor = Activation('relu')(tensor)
# shuffle channels
tensor = ReshapeLayer((n_timesteps, side_dim1, side_dim2, n_groups,
n_channels_per_group_out))(tensor)
tensor = TransposeLayer((0, 1, 2, 3, 5, 4))(tensor)
tensor = ReshapeLayer(
(n_timesteps, side_dim1, side_dim2, n_channels_out))(tensor)
return tensor