def construct_simple_model(inputs,
conv_layers_count=3,
conv_filters=64,
kernel_sizes=3,
strides=1,
paddings='same',
fc_layers_count=2,
fc_sizes=500,
activations='elu',
regularizations=0,
batch_norms=True):
if type(conv_filters) != list:
conv_filters = [conv_filters] * conv_layers_count
if type(kernel_sizes) != list:
kernel_sizes = [kernel_sizes] * conv_layers_count
if type(strides) != list:
strides = [strides] * conv_layers_count
if type(paddings) != list:
paddings = [paddings] * conv_layers_count
if type(fc_sizes) != list:
fc_sizes = [fc_sizes] * fc_layers_count
if type(activations) != list:
activations = [activations] * (conv_layers_count + fc_layers_count)
if type(regularizations) != list:
regularizations = [regularizations] * (conv_layers_count + fc_layers_count)
if type(batch_norms) != list:
batch_norms = [batch_norms] * (conv_layers_count + fc_layers_count)
layer = concat(inputs)
for i in range(conv_layers_count):
layer = conv2d(
layer,
conv_filters[i],
kernel_size=kernel_sizes[i],
batchnorm=batch_norms[i],
padding=paddings[i],
kernel_initializer='glorot_normal',
strides=strides[i],
activation=activations[i],
activity_regularizer=l2(regularizations[i]))
layer = Flatten(name='flatten1')(layer)
for i in range(fc_layers_count):
layer = Dense(
fc_sizes[i],
kernel_initializer='glorot_normal',
activation=activations[i + conv_layers_count],
activity_regularizer=l2(regularizations[i]))(layer)
r_x = Dense(1, name='r_x')(layer)
r_y = Dense(1, name='r_y')(layer)
r_z = Dense(1, name='r_z')(layer)
t_x = Dense(1, name='t_x')(layer)
t_y = Dense(1, name='t_y')(layer)
t_z = Dense(1, name='t_z')(layer)
outputs = [r_x, r_y, r_z, t_x, t_y, t_z]
return outputs
def construct_resnet50_model(inputs,
weights='imagenet',
kernel_initializer='glorot_normal'):
inputs = concat(inputs)
conv0 = Conv2D(3, kernel_size=7, padding='same', activation='relu',
kernel_initializer=kernel_initializer, name='conv0')(inputs)
base_model = ResNet50(weights=weights, include_top=False, pooling=None)
outputs = base_model(conv0)
embedding = Flatten()(outputs)
fc1 = Dense(500, activation='relu',
kernel_initializer=kernel_initializer, name='fc1')(embedding)
fc2 = Dense(500, activation='relu',
kernel_initializer=kernel_initializer, name='fc2')(fc1)
r_x = Dense(1, name='r_x')(fc2)
r_y = Dense(1, name='r_y')(fc2)
r_z = Dense(1, name='r_z')(fc2)
t_x = Dense(1, name='t_x')(fc2)
t_y = Dense(1, name='t_y')(fc2)
t_z = Dense(1, name='t_z')(fc2)
outputs = [r_x, r_y, r_z, t_x, t_y, t_z]
return outputs
def construct_ls_vo_rt_model(inputs,
cropping=((0, 0), (0, 0)),
hidden_size=500,
regularization=0,
kernel_initializer='glorot_normal'):
inputs = concat(inputs)
features, bottleneck = construct_encoder(
inputs, kernel_initializer=kernel_initializer)
reconstructed_flow = construct_flow_decoder(
bottleneck, cropping=cropping, output_channels=inputs.shape[-1].value)
fc2_rotation = construct_double_fc(features,
hidden_size=hidden_size,
regularization=regularization,
kernel_initializer=kernel_initializer,
name='rotation')
fc2_translation = construct_double_fc(
features,
hidden_size=hidden_size,
regularization=regularization,
kernel_initializer=kernel_initializer,
name='translation')
outputs = construct_outputs(
fc2_rotation, fc2_translation,
regularization=regularization) + [reconstructed_flow]
return outputs
def construct_constant_model(inputs,
rot_and_trans_array):
inputs = concat(inputs)
mean_r_x, mean_r_y, mean_r_z, mean_t_x, mean_t_y, mean_t_z = rot_and_trans_array.mean(axis=0)
r_x = ConstLayer(mean_r_x, name='r_x')(inputs)
r_y = ConstLayer(mean_r_y, name='r_y')(inputs)
r_z = ConstLayer(mean_r_z, name='r_z')(inputs)
t_x = ConstLayer(mean_t_x, name='t_x')(inputs)
t_y = ConstLayer(mean_t_y, name='t_y')(inputs)
t_z = ConstLayer(mean_t_z, name='t_z')(inputs)
outputs = [r_x, r_y, r_z, t_x, t_y, t_z]
return outputs
def construct_ls_vo_rt_no_decoder_model(inputs,
hidden_size=500,
regularization=0,
kernel_initializer='glorot_normal'):
inputs = concat(inputs)
features, _ = construct_encoder(inputs,
kernel_initializer=kernel_initializer)
fc2_rotation = construct_double_fc(features,
hidden_size=hidden_size,
regularization=regularization,
kernel_initializer=kernel_initializer,
name='rotation')
fc2_translation = construct_double_fc(
features,
hidden_size=hidden_size,
regularization=regularization,
kernel_initializer=kernel_initializer,
name='translation')
outputs = construct_outputs(fc2_rotation,
fc2_translation,
regularization=regularization)
return outputs
def construct_rigidity_model(inputs,
batchnorm=True,
add_grid_layer=True,
f_x=1,
f_y=1,
c_x=0.5,
c_y=0.5,
kernel_initializer='he_uniform'):
inputs = concat(inputs)
if add_grid_layer:
inputs = AddGridLayer(f_x=f_x, f_y=f_y, c_x=c_x, c_y=c_y)(inputs)
conv1 = conv2d(inputs, 32, kernel_size=7, batchnorm=batchnorm, strides=2,
padding='same', activation='relu', kernel_initializer=kernel_initializer)
conv2 = conv2d(conv1, 64, kernel_size=7, batchnorm=batchnorm, strides=2,
padding='same', activation='relu', kernel_initializer=kernel_initializer)
conv3 = conv2d(conv2, 128, kernel_size=5, batchnorm=batchnorm, strides=2,
padding='same', activation='relu', kernel_initializer=kernel_initializer)
conv4 = conv2d(conv3, 256, kernel_size=5, batchnorm=batchnorm, strides=2,
padding='same', activation='relu', kernel_initializer=kernel_initializer)
conv5 = conv2d(conv4, 512, kernel_size=3, batchnorm=batchnorm, strides=2,
padding='same', activation='relu', kernel_initializer=kernel_initializer)
conv6 = conv2d(conv5, 1024, kernel_size=3, batchnorm=batchnorm,
padding='same', activation='relu', kernel_initializer=kernel_initializer)
conv7 = Conv2D(6, kernel_size=1, kernel_initializer=kernel_initializer)(conv6)
pool = GlobalAveragePooling2D()(conv7)
r_x = Lambda(lambda x: x[:,0:1], name='r_x')(pool)
r_y = Lambda(lambda x: x[:,1:2], name='r_y')(pool)
r_z = Lambda(lambda x: x[:,2:3], name='r_z')(pool)
t_x = Lambda(lambda x: x[:,3:4], name='t_x')(pool)
t_y = Lambda(lambda x: x[:,4:5], name='t_y')(pool)
t_z = Lambda(lambda x: x[:,5:6], name='t_z')(pool)
outputs = [r_x, r_y, r_z, t_x, t_y, t_z]
return outputs
def construct_st_vo_model(inputs, kernel_initializer='glorot_normal'):
inputs = concat(inputs)
conv1 = Conv2D(64,
kernel_size=3,
strides=2,
kernel_initializer=kernel_initializer,
name='conv1')(inputs)
pool1 = MaxPooling2D(pool_size=4, strides=4, name='pool1')(conv1)
conv2 = Conv2D(20,
kernel_size=3,
kernel_initializer=kernel_initializer,
name='conv2')(pool1)
pool2 = MaxPooling2D(pool_size=2, strides=2, name='pool2')(conv2)
flatten1 = Flatten(name='flatten1')(pool1)
flatten2 = Flatten(name='flatten2')(pool2)
merged = concatenate([flatten1, flatten2], axis=1)
activation = Activation('relu')(merged)
fc = construct_fc(activation,
kernel_initializer=kernel_initializer,
name='fc')
outputs = construct_outputs(fc, fc)
return outputs
def construct_encoder(inputs,
use_depth=True,
use_flow=True,
use_association_layer=True,
concat_axis=3,
merge_filters=256,
merge_stride=2,
regularization_depth=0,
depth_multiplicator=None,
use_batchnorm_depth=False,
use_batchnorm_flow=False,
add_grid_layer=False,
f_x=1,
f_y=1,
c_x=0.5,
c_y=0.5,
kernel_initializer='glorot_normal'):
# flow convolutional branch
if use_flow:
flow_xy = concat(inputs[:2])
if use_batchnorm_flow:
flow_xy = BatchNormalization()(flow_xy)
if add_grid_layer:
flow_xy = AddGridLayer(f_x=f_x, f_y=f_y, c_x=c_x, c_y=c_y)(flow_xy)
conv1_flow = conv2d(flow_xy,
64,
kernel_size=3,
strides=2,
kernel_initializer=kernel_initializer,
name='conv1_flow')
conv2_flow = conv2d(conv1_flow,
128,
kernel_size=3,
strides=2,
kernel_initializer=kernel_initializer,
name='conv2_flow')
conv3_flow = conv2d(conv2_flow,
256,
kernel_size=3,
strides=2,
kernel_initializer=kernel_initializer,
name='conv3_flow')
conv4_flow = conv2d(conv3_flow,
512,
kernel_size=3,
strides=2,
kernel_initializer=kernel_initializer,
name='conv4_flow')
# depth convolutional branch
if use_depth:
if use_association_layer: # pass flow_z as input
depths = AssociationLayer()(concat(inputs))
if use_batchnorm_depth:
depths = BatchNormalization()(depths)
else:
depths = concat(inputs[2:])
if add_grid_layer:
depths = AddGridLayer(f_x=f_x, f_y=f_y, c_x=c_x, c_y=c_y)(depths)
conv1_depth = conv2d(depths,
64,
kernel_size=3,
strides=2,
kernel_initializer=kernel_initializer,
name='conv1_depth')
conv2_depth = conv2d(conv1_depth,
128,
kernel_size=3,
strides=2,
kernel_initializer=kernel_initializer,
name='conv2_depth')
conv3_depth = conv2d(conv2_depth,
256,
kernel_size=3,
strides=2,
kernel_initializer=kernel_initializer,
name='conv3_depth')
conv4_depth = conv2d(conv3_depth,
512,
kernel_size=3,
strides=2,
kernel_initializer=kernel_initializer,
kernel_regularizer=l2(regularization_depth),
bias_regularizer=l2(regularization_depth),
name='conv4_depth')
if depth_multiplicator is not None:
conv4_depth = Lambda(lambda x: x * depth_multiplicator)(conv4_depth)
if use_flow and use_depth:
concatenated = Concatenate(axis=concat_axis)([conv4_flow, conv4_depth])
elif use_flow:
concatenated = conv4_flow
elif use_depth:
concatenated = conv4_depth
merged = conv2d(concatenated,
merge_filters,
kernel_size=1,
strides=merge_stride,
kernel_initializer=kernel_initializer,
name='merge')
flatten = Flatten(name='flatten')(merged)
return flatten