def call(self, x):
"""
Apply the layer to the node embeddings in x. These embeddings are either:
* A list of two tensors of shape (N, M) being the embeddings for each of the nodes in the link,
where N is the number of links, and M is the node embedding size.
* A single tensor of shape (..., N, 2, M) where the axis second from last indexes the nodes
in the link and N is the number of links and M the embedding size.
"""
# Currently GraphSAGE & HinSage output a list of two tensors being the embeddings
# for each of the nodes in the link. However, GCN, GAT & other full-batch methods
# return a tensor of shape (1, N, 2, M).
# Detect and support both inputs
if isinstance(x, (list, tuple)):
if len(x) != 2:
raise ValueError(
"Expecting a list of length 2 for link embedding")
x0, x1 = x
elif isinstance(x, tf.Tensor):
if int(x.shape[self.axis]) != 2:
raise ValueError(
"Expecting a tensor of shape 2 along specified axis for link embedding"
)
x0, x1 = tf.unstack(x, axis=self.axis)
else:
raise TypeError("Expected a list, tuple, or Tensor as input")
# Apply different ways to combine the node embeddings to a link embedding.
if self.method in ["ip", "dot"]:
out = tf.reduce_sum(x0 * x1, axis=-1, keepdims=True)
elif self.method == "l1":
# l1(u,v)_i = |u_i - v_i| - vector of the same size as u,v
out = tf.abs(x0 - x1)
elif self.method == "l2":
# l2(u,v)_i = (u_i - v_i)^2 - vector of the same size as u,v
out = tf.square(x0 - x1)
elif self.method in ["mul", "hadamard"]:
out = tf.multiply(x0, x1)
elif self.method == "concat":
out = Concatenate()([x0, x1])
elif self.method == "avg":
out = Average()([x0, x1])
else:
raise NotImplementedError()
# "{}: the requested method '{}' is not known/not implemented".format(
# name, edge_embedding_method
# )
# )
# Apply activation function
out = self.activation(out)
return out
def create_mu_block(x_list, config):
if config.n_feature_sets > 1:
if config.last_layer == 'concatenate':
x = Concatenate()(x_list)
else:
# pad first if prorated
max_len = np.max(config.feature_units)
# print("Max len {}".format(max_len))
for i, a in enumerate(x_list):
# print(i, a)
if (max_len - config.feature_units[i]) > 0:
left_pad = (int)((max_len - config.feature_units[i]) / 2)
right_pad = max_len - config.feature_units[i] - left_pad
x_list[i] = tf.pad(a, [[0, 0], [left_pad, right_pad]])
x = Average()(x_list)
if config.task == 'regression':
x = Dense(1)(x)
elif config.task == 'classification':
x = Dense(config.n_classes, activation='softmax')(x)
else:
if config.task == 'regression':
x = Dense(1)(x_list[0])
elif config.task == 'classification':
x = Dense(config.n_classes, activation='softmax')(x_list[0])
return x
def create_model(image_shape=(224, 224, 3),
restart_checkpoint=None,
backbone='mobilnetv2',
feature_len=128,
freeze=False):
"""
Creates an image encoder.
Args:
image_shape: input image shape (use [None, None] for resizable network)
restart_checkpoint: snapshot to be restored
backbone: the backbone CNN (one of mobilenetv2, densent121, custom)
feature_len: the length of the additional feature layer
freeze: freeze the backbone
"""
input_img = Input(shape=image_shape)
# add the backbone
backbone_name = backbone
if backbone_name == 'densenet121':
print('Using DenseNet121 backbone.')
backbone = DenseNet121(input_tensor=input_img, include_top=False)
backbone.layers.pop()
if freeze:
for layer in backbone.layers:
layer.trainable = False
backbone = backbone.output
elif backbone_name == 'mobilenetv2':
print('Using MobileNetV2 backbone.')
backbone = MobileNetV2(input_tensor=input_img, include_top=False)
backbone.layers.pop()
if freeze:
for layer in backbone.layers:
layer.trainable = False
backbone = backbone.output
elif backbone_name == 'custom':
backbone = custom_backbone(input_tensor=input_img)
else:
raise Exception('Unknown backbone: {}'.format(backbone_name))
# add the head layers
gmax = GlobalMaxPool2D()(backbone)
gavg = GlobalAvgPool2D()(backbone)
gmul = Multiply()([gmax, gavg])
ggavg = Average()([gmax, gavg])
backbone = Concatenate()([gmax, gavg, gmul, ggavg])
backbone = BatchNormalization()(backbone)
backbone = Dense(feature_len)(backbone)
backbone = Activation('sigmoid')(backbone)
encoder = Model(input_img, backbone)
if restart_checkpoint:
print('Loading weights from {}'.format(restart_checkpoint))
encoder.load_weights(restart_checkpoint,
by_name=True,
skip_mismatch=True)
return encoder
def UnetPlusPlus_mobilenetv2(config, num_classes, weights, base_trainable):
base_model, preprocessing_function = classification_networks[
'mobilenetv2'](input_shape=config.input_shape,
weights=weights,
include_top=False)
base_model.trainable = base_trainable
img_height = config.height
img_width = config.width
conv1_1 = base_model.get_layer('block_1_expand_relu').output
conv2_1 = base_model.get_layer('block_3_expand_relu').output
conv3_1 = base_model.get_layer('block_6_expand_relu').output
conv4_1 = base_model.get_layer('block_13_expand_relu').output
conv5_1 = base_model.get_layer('out_relu').output
conv1_2, conv1_3, conv1_4, conv1_5 = ret_model_output(
config, conv1_1, conv2_1, conv3_1, conv4_1, conv5_1)
output_list = get_output_list(config, num_classes, conv1_2, conv1_3,
conv1_4, conv1_5)
o = 1
if (len(output_list) > 1):
o = Average()(output_list)
else:
o = output_list[0]
o = Upsample(img_height, img_width)(o)
model = Model(inputs=base_model.input,
outputs=o,
name='UnetPlusPlus_mobilenetv2')
return model, preprocessing_function
def build_CNN_ensemble_model(params):
"""
Builds ensemble of FFCNN's
Parameters
----------
params : dict
complete hyper parameter dictionary for model generation and training
Required Parameters
----------
ensemble_size : integer
Number of feed forward CNNs based on params to include in ensemble
Returns
-------
model
"""
x_in = Input(shape=params['data_shape'], name='X')
members = []
for i in range(params['ensemble_size']):
m = build_CNN_model(params)
members.append(m(x_in))
outputs = Average()(members)
model = Model(x_in, outputs)
return model
def edge_function(x):
x0 = x[0]
x1 = x[1]
if edge_embedding_method == "ip" or edge_embedding_method == "dot":
out = Lambda(
lambda x: K.sum(x[0] * x[1], axis=-1, keepdims=False))(
[x0, x1])
out = Activation(output_act)(out)
if output_dim != 1:
warnings.warn(
"Inner product is a scalar, but output_dim is set to {}. Reverting output_dim to be 1."
.format(output_dim))
out = Reshape((1, ))(out)
elif edge_embedding_method == "l1":
# l1(u,v)_i = |u_i - v_i| - vector of the same size as u,v
le = Lambda(lambda x: K.abs(x[0] - x[1]))([x0, x1])
# add dense layer to convert le to the desired output:
out = Dense(output_dim, activation=output_act)(le)
out = Reshape((output_dim, ))(out)
elif edge_embedding_method == "l2":
# l2(u,v)_i = (u_i - v_i)^2 - vector of the same size as u,v
le = Lambda(lambda x: K.square(x[0] - x[1]))([x0, x1])
# add dense layer to convert le to the desired output:
out = Dense(output_dim, activation=output_act)(le)
out = Reshape((output_dim, ))(out)
elif edge_embedding_method == "mul" or edge_embedding_method == "hadamard":
le = Multiply()([x0, x1])
# add dense layer to convert le to the desired output:
out = Dense(output_dim, activation=output_act)(le)
out = Reshape((output_dim, ))(out)
elif edge_embedding_method == "concat":
le = Concatenate()([x0, x1])
# add dense layer to convert le to the desired output:
out = Dense(output_dim, activation=output_act)(le)
out = Reshape((output_dim, ))(out)
elif edge_embedding_method == "avg":
le = Average()([x0, x1])
# add dense layer to convert le to the desired output:
out = Dense(output_dim, activation=output_act)(le)
out = Reshape((output_dim, ))(out)
else:
raise NotImplementedError(
"{}: the requested method '{}' is not known/not implemented".
format(name, edge_embedding_method))
if clip_limits:
out = LeakyClippedLinear(low=clip_limits[0],
high=clip_limits[1],
alpha=0.1)(out)
return out
def average(models: List[training.Model],
model_input: Tensor) -> training.Model:
outputs = [model.outputs[0] for model in models]
y = Average()(outputs)
model = Model(model_input, y, name='ensemble')
return model
def classify(**args):
"""
Main method that prepares dataset, builds model, executes training and displays results.
:param args: keyword arguments passed from cli parser
"""
# only allow print-outs if execution has no repetitions
allow_print = args['repetitions'] == 1
# determine classification targets and parameters to construct datasets properly
cls_target, cls_str = set_classification_targets(args['cls_choice'])
d = prepare_dataset(args['dataset_choice'], cls_target, args['batch_size'])
print('\n\tTask: Classify «{}» using «{}»\n'.format(
cls_str, d['data_str']))
print_dataset_info(d)
# build and train
inputs = Input(shape=(7810, ))
models = [
build_model(i, d['num_classes'], inputs=inputs)
for i in range(args['num_models'])
]
# combine outputs of all models
y = Average()([m.outputs[0] for m in models])
model = Model(inputs, outputs=y, name='multiple')
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
if allow_print:
model.summary()
print('')
plot_model(model, to_file='img/multiple_mlp.png')
model.fit(d['train_data'],
steps_per_epoch=d['train_steps'],
epochs=args['epochs'],
verbose=1,
class_weight=d['class_weights'])
# evaluation model
print('Evaluate ...')
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.evaluate(d['eval_data'], steps=d['test_steps'], verbose=1)
# predict on testset and calculate classification report and confusion matrix for diagnosis
print('Test ...')
pred = model.predict(d['test_data'], steps=d['test_steps'])
if allow_print:
diagnose_output(d['test_labels'], pred.argmax(axis=1),
d['classes_trans'])
return balanced_accuracy_score(d['test_labels'], pred.argmax(axis=1))
def deep_fuse_layer(inputs, filters, kernel_size, strides):
if type(inputs) is not list:
raise Exception()
avg = Average()(inputs)
conv_list = []
for _ in range(len(inputs)):
conv_list.append(create_res_conv_block(avg, filters, kernel_size, strides))
return conv_list
def fusion_block(tensors, name, type='add'):
if (type == 'add'):
return Add(name='add_' + name)(tensors)
if (type == 'max'):
return Maximum(name='max_' + name)(tensors)
if (type == 'con'):
return Concatenate(name='conc_' + name)(tensors)
if (type == 'avg'):
return Average(name='avg_' + name)(tensors)
def call(self, claim, evidence):
if self.merging_type == 'concat':
new_row = Concatenate(axis=1)([claim, evidence])
elif self.merging_type == 'sum':
new_row = Add()([claim, evidence])
elif self.merging_type == 'mean':
new_row = Average()([claim, evidence])
if self.cosine_sim_flag:
cosine_sim = Dot(1, normalize=True)([claim, evidence])
new_row = Concatenate(axis=1)([new_row, cosine_sim])
return new_row
def discriminator_model():
dm_input = Input(shape=patch_shape)
dm = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(2, 2),
padding="same")(dm_input)
dm = BatchNormalization()(dm)
dm = LeakyReLU(alpha=0.2)(dm)
dm = Convolution2D(filters=2 * 64,
kernel_size=(3, 3),
strides=(2, 2),
padding="same")(dm)
dm = BatchNormalization()(dm)
dm = LeakyReLU(alpha=0.2)(dm)
dm = Convolution2D(filters=4 * 64,
kernel_size=(3, 3),
strides=(2, 2),
padding="same")(dm)
dm = BatchNormalization()(dm)
dm = LeakyReLU(alpha=0.2)(dm)
dm = Convolution2D(filters=4 * 64,
kernel_size=(3, 3),
strides=(2, 2),
padding="same")(dm)
dm = BatchNormalization()(dm)
dm = LeakyReLU(alpha=0.2)(dm)
dm = Flatten()(dm)
outputs = Dense(units=1, activation='sigmoid')(dm)
model = Model(inputs=dm_input, outputs=outputs, name='Discriminator_Model')
# discriminator
inputs = Input(shape=image_shape)
list_row_idx = [(64 * i, (i + 1) * 64)
for i in range(int(image_shape[0] / patch_shape[0]))]
list_col_idx = [(64 * i, (i + 1) * 64)
for i in range(int(image_shape[1] / patch_shape[1]))]
list_patch = []
for row_idx in list_row_idx:
for col_idx in list_col_idx:
x_patch = Lambda(lambda z: z[:, row_idx[0]:row_idx[1], col_idx[0]:
col_idx[1], :])(inputs)
list_patch.append(x_patch)
x = [model(patch) for patch in list_patch]
outputs = Average()(x)
model = Model(inputs=inputs, outputs=outputs, name='Discriminator')
return model
def call(self, inputs):
attention_layers = []
# apply all attention layers to inputs
for layer in self.attn_heads:
attention_layers.append(layer(inputs))
# now average all their outputs
if self.num_heads > 1:
return Average()(attention_layers)
else:
return attention_layers[0]
def build_extractor_29layers_v2(self, name, block, layers):
in_img = Input(shape=(*self.in_size_hw, 1))
X = self._mfm(in_img, name = name + '_mfm1', out_channels=48, kernel_size=5, strides=1)
X = Average()([MaxPooling2D(pool_size=2, padding='same')(X), AveragePooling2D(pool_size=2, padding='same')(X)])
X = self._make_layer(X, name = name + '_layers1', block=block, num_blocks=layers[0], out_channels=48)
X = self._group(X, name = name + '_group1', in_channels=48, out_channels=96, kernel_size=3, strides=1)
X = Average()([MaxPooling2D(pool_size=2, padding='same')(X), AveragePooling2D(pool_size=2, padding='same')(X)])
X = self._make_layer(X, name = name + '_layers2', block=block, num_blocks=layers[1], out_channels=96)
X = self._group(X, name = name + '_group2', in_channels=96, out_channels=192, kernel_size=3, strides=1)
X = Average()([MaxPooling2D(pool_size=2, padding='same')(X), AveragePooling2D(pool_size=2, padding='same')(X)])
X = self._make_layer(X, name = name + '_layers3', block=block, num_blocks=layers[2], out_channels=192)
X = self._group(X, name = name + '_group3', in_channels=192, out_channels=128, kernel_size=3, strides=1)
X = self._make_layer(X, name = name + '_layers4', block=block, num_blocks=layers[3], out_channels=128)
X = self._group(X, name = name + '_group4', in_channels=128, out_channels=128, kernel_size=3, strides=1)
X = Average()([MaxPooling2D(pool_size=2, padding='same')(X), AveragePooling2D(pool_size=2, padding='same')(X)])
feat = Dense(256, name = name + '_dense1', kernel_regularizer=regularizers.l2(0.0005))(Flatten()(X))
ret_extractor = Model(inputs=in_img, outputs=feat, name=name)
ret_extractor.summary()
return ret_extractor
def __tail__(self, x_2, x_1, x, output_size, dropout=False):
x_2 = GlobalAveragePooling2D()(x_2)
x_1 = GlobalAveragePooling2D()(x_1)
x = GlobalAveragePooling2D()(x)
if dropout:
x_2 = Dropout(.5)(x_2)
x_1 = Dropout(.5)(x_1)
x = Dropout(.5)(x)
x_2 = Dense(output_size, activation='softmax', name='out_2')(x_2)
x_1 = Dense(output_size, activation='softmax', name='out_1')(x_1)
x = Dense(output_size, activation='softmax', name='out')(x)
softmax_avg = Average()([x_2, x_1, x])
return softmax_avg
def define_ensemble_model(models):
# Wrap each model so that names are different and do not collide.
models = [
Model(inputs=model.input,
outputs=model.output,
name=f"{model.name}_{model_idx}")
for model_idx, model in enumerate(models)
]
input = Input(IMAGE_SIZE + (3, ))
ensemble_outputs = [model(input) for model in models]
avg = Average()(ensemble_outputs)
model = Model(inputs=input, outputs=avg)
return model
def create_model(backbone, input_shape, lr, metrics):
base = create_model_base(backbone, input_shape)
# define the 2 inputs (left and right eyes)
left_input = Input(shape=input_shape)
right_input = Input(shape=input_shape)
# get the 2 outputs using shared layers
out_left = base(left_input)
out_right = base(right_input)
# average the predictions
merged = Average()([out_left, out_right])
model = Model(inputs=[right_input, left_input], outputs=merged)
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=lr), metrics=metrics)
model.summary()
return model
def build(self, **kwargs) -> Model:
opticalflow_model = kwargs['OpticalflowModel']
rgb_model = kwargs['RGBModel']
opticalflow_input = Input(shape=self.opticalflow_input_shape,
name='opticalflow_inputs')
rgb_input = Input(shape=self.rgb_input_shape, name='rgb_inputs')
opticalflow_target = opticalflow_model(opticalflow_input)
rgb_target = rgb_model(rgb_input)
# average opticalflow and rgb models outputs
outputs = Average(name='nsdm_output')([opticalflow_target, rgb_target])
# ensemble model
nsdm_model = Model(inputs=[opticalflow_input, rgb_input],
outputs=outputs,
name='nsdm_model')
return nsdm_model
def nn_model():
i = Input(shape=(15, ))
# Branch 1
x1 = Dense(128, activation='relu')(i)
x1 = BatchNormalization()(x1)
x1 = Dense(64, activation='relu')(x1)
x1 = BatchNormalization()(x1)
x1 = Dense(32, activation='relu')(x1)
x1 = BatchNormalization()(x1)
x1 = Dense(16, activation='relu')(x1)
x1 = BatchNormalization()(x1)
x1_output = Dense(1, activation='linear')(x1)
# Branch 2
x2 = Dense(128, activation='relu')(i)
x2 = BatchNormalization()(x2)
x2 = Dense(64, activation='relu')(x2)
x2 = BatchNormalization()(x2)
x2 = Dense(32, activation='relu')(x2)
x2 = BatchNormalization()(x2)
x2 = Dense(16, activation='relu')(x2)
x2 = BatchNormalization()(x2)
x2_output = Dense(1, activation='linear')(x2)
# Branch 3
x3 = Dense(128, activation='relu')(i)
x3 = BatchNormalization()(x3)
x3 = Dense(64, activation='relu')(x3)
x3 = BatchNormalization()(x3)
x3 = Dense(32, activation='relu')(x3)
x3 = BatchNormalization()(x3)
x3 = Dense(16, activation='relu')(x3)
x3 = BatchNormalization()(x3)
x3_output = Dense(1, activation='linear')(x3)
scalars = Average()([x1_output, x2_output, x3_output])
return Model(inputs=[i], outputs=[scalars])
def create_model(pretrained_model, num_labels, output_offset, layer_index):
model_inputs = pretrained_model.inputs[:2]
if is_signed_digit(layer_index):
layer_index = int(layer_index)
pretrained_output = get_bert_output(pretrained_model, layer_index,
output_offset)
elif layer_index in ('avg', 'concat'):
outputs = []
for i in count(1):
try:
outputs.append(
get_bert_output(pretrained_model, i, output_offset))
except ValueError:
break # assume past last layer
if layer_index == 'avg':
pretrained_output = Average()(outputs)
else:
assert layer_index == 'concat'
pretrained_output = Concatenate()(outputs)
model_output = keras.layers.Dense(num_labels,
activation='softmax')(pretrained_output)
model = keras.models.Model(inputs=model_inputs, outputs=model_output)
return model
def create_model(model_type, dataset, validation_dataset, test_dataset,
callbacks, batch_size, num_epochs, multi_modal, augmented,
perform_test_only, perform_early_fusion, perform_late_fusion,
pe_block, path_prefix):
time_callback = TimeHistory()
input_shape = (vol_x, vol_y, vol_z, n_channels)
mirrored_strategy = tf.distribute.MirroredStrategy()
with mirrored_strategy.scope():
print_section("Creating and compiling " + model_type + " model")
inputs_bmode, inputs_pd = handle_input_fusion(multi_modal, input_shape,
batch_size)
if multi_modal and perform_late_fusion:
# Treat as separate single modal models
model_bmode = get_model(model_type, False, False, pe_block,
inputs_bmode, None, True)
model_pd = get_model(model_type, False, False, pe_block, inputs_pd,
None, True)
output_conv = Average()([model_bmode, model_pd])
output = Conv3D(1, 1, activation='sigmoid',
padding='same')(output_conv)
model = Model(inputs=[inputs_bmode, inputs_pd], outputs=[output])
else:
model = get_model(model_type, multi_modal, perform_early_fusion,
pe_block, inputs_bmode, inputs_pd)
model.compile(optimizer=Adam(lr=learning_rate,
beta_1=beta_1,
beta_2=beta_2,
epsilon=epsilon,
decay=decay),
loss=dice_loss,
metrics=["binary_crossentropy", dice_coe])
print_section("Generating model summary")
model.summary()
if not perform_test_only:
print_section("Generate model graph image")
plot_model(model,
to_file=generate_model_image_path(path_prefix),
show_shapes=True,
show_layer_names=True,
rankdir='LR',
expand_nested=False,
dpi=96)
print_section("Training model")
num_train_samples = train_samples_total if augmented else train_samples
model.fit(dataset,
epochs=num_epochs,
verbose=1,
shuffle=True,
steps_per_epoch=calculated_steps_per_epoch(
num_train_samples, batch_size),
validation_data=validation_dataset,
validation_steps=calculated_steps_per_epoch(
validation_samples, batch_size),
validation_freq=1,
callbacks=[time_callback] + callbacks)
print_section("Loading model weights")
model.load_weights(generate_checkpoint_path(path_prefix))
print_section('Testing model')
imgs_mask_test = model.predict(test_dataset,
batch_size=batch_size,
verbose=1)
print_section('Saving predictions')
pred_label_list = save_test_images(imgs_mask_test, path_prefix)
print_section('Evaluating against ground truth')
test_df = pd.read_csv(os.path.join(dataset_output_path, 'test.csv'))
ground_truth_label_list = test_df['label'].values.tolist()
perf_metrics = compare_segmentations(pred_label_list,
ground_truth_label_list)
print_section('Saving results')
create_results_csv(path_prefix, perf_metrics,
list(zip(pred_label_list, ground_truth_label_list)),
sum(time_callback.times))
if not perform_test_only:
print_section("Printing stats")
print("Each epoch time", time_callback.times)
print("Total Time Taken (s)", sum(time_callback.times))
def get_test_model_exhaustive():
"""Returns a exhaustive test model."""
input_shapes = [(2, 3, 4, 5, 6), (2, 3, 4, 5, 6), (7, 8, 9, 10),
(7, 8, 9, 10), (11, 12, 13), (11, 12, 13), (14, 15),
(14, 15), (16, ),
(16, ), (2, ), (1, ), (2, ), (1, ), (1, 3), (1, 4),
(1, 1, 3), (1, 1, 4), (1, 1, 1, 3), (1, 1, 1, 4),
(1, 1, 1, 1, 3), (1, 1, 1, 1, 4), (26, 28, 3), (4, 4, 3),
(4, 4, 3), (4, ), (2, 3), (1, ), (1, ), (1, ), (2, 3),
(9, 16, 1), (1, 9, 16)]
inputs = [Input(shape=s) for s in input_shapes]
outputs = []
outputs.append(Conv1D(1, 3, padding='valid')(inputs[6]))
outputs.append(Conv1D(2, 1, padding='same')(inputs[6]))
outputs.append(Conv1D(3, 4, padding='causal', dilation_rate=2)(inputs[6]))
outputs.append(ZeroPadding1D(2)(inputs[6]))
outputs.append(Cropping1D((2, 3))(inputs[6]))
outputs.append(MaxPooling1D(2)(inputs[6]))
outputs.append(MaxPooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(MaxPooling1D(2, data_format="channels_first")(inputs[6]))
outputs.append(AveragePooling1D(2)(inputs[6]))
outputs.append(AveragePooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(
AveragePooling1D(2, data_format="channels_first")(inputs[6]))
outputs.append(GlobalMaxPooling1D()(inputs[6]))
outputs.append(GlobalMaxPooling1D(data_format="channels_first")(inputs[6]))
outputs.append(GlobalAveragePooling1D()(inputs[6]))
outputs.append(
GlobalAveragePooling1D(data_format="channels_first")(inputs[6]))
outputs.append(Conv2D(4, (3, 3))(inputs[4]))
outputs.append(Conv2D(4, (3, 3), use_bias=False)(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4]))
outputs.append(SeparableConv2D(3, (3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((1, 2))(inputs[4]))
outputs.append(MaxPooling2D((2, 2))(inputs[4]))
# todo: check if TensorFlow >= 2.1 supports this
# outputs.append(MaxPooling2D((2, 2), data_format="channels_first")(inputs[4])) # Default MaxPoolingOp only supports NHWC on device type CPU
outputs.append(
MaxPooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(AveragePooling2D((2, 2))(inputs[4]))
# todo: check if TensorFlow >= 2.1 supports this
# outputs.append(AveragePooling2D((2, 2), data_format="channels_first")(inputs[4])) # Default AvgPoolingOp only supports NHWC on device type CPU
outputs.append(
AveragePooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(GlobalAveragePooling2D()(inputs[4]))
outputs.append(
GlobalAveragePooling2D(data_format="channels_first")(inputs[4]))
outputs.append(GlobalMaxPooling2D()(inputs[4]))
outputs.append(GlobalMaxPooling2D(data_format="channels_first")(inputs[4]))
outputs.append(Permute((3, 4, 1, 5, 2))(inputs[0]))
outputs.append(Permute((1, 5, 3, 2, 4))(inputs[0]))
outputs.append(Permute((3, 4, 1, 2))(inputs[2]))
outputs.append(Permute((2, 1, 3))(inputs[4]))
outputs.append(Permute((2, 1))(inputs[6]))
outputs.append(Permute((1, ))(inputs[8]))
outputs.append(Permute((3, 1, 2))(inputs[31]))
outputs.append(Permute((3, 1, 2))(inputs[32]))
outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[31])))
outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[32])))
outputs.append(BatchNormalization()(inputs[0]))
outputs.append(BatchNormalization(axis=1)(inputs[0]))
outputs.append(BatchNormalization(axis=2)(inputs[0]))
outputs.append(BatchNormalization(axis=3)(inputs[0]))
outputs.append(BatchNormalization(axis=4)(inputs[0]))
outputs.append(BatchNormalization(axis=5)(inputs[0]))
outputs.append(BatchNormalization()(inputs[2]))
outputs.append(BatchNormalization(axis=1)(inputs[2]))
outputs.append(BatchNormalization(axis=2)(inputs[2]))
outputs.append(BatchNormalization(axis=3)(inputs[2]))
outputs.append(BatchNormalization(axis=4)(inputs[2]))
outputs.append(BatchNormalization()(inputs[4]))
# todo: check if TensorFlow >= 2.1 supports this
# outputs.append(BatchNormalization(axis=1)(inputs[4])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now.
outputs.append(BatchNormalization(axis=2)(inputs[4]))
outputs.append(BatchNormalization(axis=3)(inputs[4]))
outputs.append(BatchNormalization()(inputs[6]))
outputs.append(BatchNormalization(axis=1)(inputs[6]))
outputs.append(BatchNormalization(axis=2)(inputs[6]))
outputs.append(BatchNormalization()(inputs[8]))
outputs.append(BatchNormalization(axis=1)(inputs[8]))
outputs.append(BatchNormalization()(inputs[27]))
outputs.append(BatchNormalization(axis=1)(inputs[27]))
outputs.append(BatchNormalization()(inputs[14]))
outputs.append(BatchNormalization(axis=1)(inputs[14]))
outputs.append(BatchNormalization(axis=2)(inputs[14]))
outputs.append(BatchNormalization()(inputs[16]))
# todo: check if TensorFlow >= 2.1 supports this
# outputs.append(BatchNormalization(axis=1)(inputs[16])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now.
outputs.append(BatchNormalization(axis=2)(inputs[16]))
outputs.append(BatchNormalization(axis=3)(inputs[16]))
outputs.append(BatchNormalization()(inputs[18]))
outputs.append(BatchNormalization(axis=1)(inputs[18]))
outputs.append(BatchNormalization(axis=2)(inputs[18]))
outputs.append(BatchNormalization(axis=3)(inputs[18]))
outputs.append(BatchNormalization(axis=4)(inputs[18]))
outputs.append(BatchNormalization()(inputs[20]))
outputs.append(BatchNormalization(axis=1)(inputs[20]))
outputs.append(BatchNormalization(axis=2)(inputs[20]))
outputs.append(BatchNormalization(axis=3)(inputs[20]))
outputs.append(BatchNormalization(axis=4)(inputs[20]))
outputs.append(BatchNormalization(axis=5)(inputs[20]))
outputs.append(Dropout(0.5)(inputs[4]))
outputs.append(ZeroPadding2D(2)(inputs[4]))
outputs.append(ZeroPadding2D((2, 3))(inputs[4]))
outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Cropping2D(2)(inputs[4]))
outputs.append(Cropping2D((2, 3))(inputs[4]))
outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Dense(3, use_bias=True)(inputs[13]))
outputs.append(Dense(3, use_bias=True)(inputs[14]))
outputs.append(Dense(4, use_bias=False)(inputs[16]))
outputs.append(Dense(4, use_bias=False, activation='tanh')(inputs[18]))
outputs.append(Dense(4, use_bias=False)(inputs[20]))
outputs.append(Reshape(((2 * 3 * 4 * 5 * 6), ))(inputs[0]))
outputs.append(Reshape((2, 3 * 4 * 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4 * 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4, 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4, 5, 6))(inputs[0]))
outputs.append(Reshape((16, ))(inputs[8]))
outputs.append(Reshape((2, 8))(inputs[8]))
outputs.append(Reshape((2, 2, 4))(inputs[8]))
outputs.append(Reshape((2, 2, 2, 2))(inputs[8]))
outputs.append(Reshape((2, 2, 1, 2, 2))(inputs[8]))
outputs.append(RepeatVector(3)(inputs[8]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='bilinear')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='bilinear')(inputs[4]))
outputs.append(ReLU()(inputs[0]))
for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]:
outputs.append(Concatenate(axis=axis)([inputs[0], inputs[1]]))
for axis in [-4, -3, -2, -1, 1, 2, 3, 4]:
outputs.append(Concatenate(axis=axis)([inputs[2], inputs[3]]))
for axis in [-3, -2, -1, 1, 2, 3]:
outputs.append(Concatenate(axis=axis)([inputs[4], inputs[5]]))
for axis in [-2, -1, 1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[6], inputs[7]]))
for axis in [-1, 1]:
outputs.append(Concatenate(axis=axis)([inputs[8], inputs[9]]))
for axis in [-1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]]))
for axis in [-1, 3]:
outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]]))
for axis in [-1, 4]:
outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]]))
for axis in [-1, 5]:
outputs.append(Concatenate(axis=axis)([inputs[20], inputs[21]]))
outputs.append(UpSampling1D(size=2)(inputs[6]))
# outputs.append(UpSampling1D(size=2)(inputs[8])) # ValueError: Input 0 of layer up_sampling1d_1 is incompatible with the layer: expected ndim=3, found ndim=2. Full shape received: [None, 16]
outputs.append(Multiply()([inputs[10], inputs[11]]))
outputs.append(Multiply()([inputs[11], inputs[10]]))
outputs.append(Multiply()([inputs[11], inputs[13]]))
outputs.append(Multiply()([inputs[10], inputs[11], inputs[12]]))
outputs.append(Multiply()([inputs[11], inputs[12], inputs[13]]))
shared_conv = Conv2D(1, (1, 1),
padding='valid',
name='shared_conv',
activation='relu')
up_scale_2 = UpSampling2D((2, 2))
x1 = shared_conv(up_scale_2(inputs[23])) # (1, 8, 8)
x2 = shared_conv(up_scale_2(inputs[24])) # (1, 8, 8)
x3 = Conv2D(1, (1, 1),
padding='valid')(up_scale_2(inputs[24])) # (1, 8, 8)
x = Concatenate()([x1, x2, x3]) # (3, 8, 8)
outputs.append(x)
x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8)
outputs.append(x)
x = Dropout(0.5)(x)
outputs.append(x)
x = Concatenate()([MaxPooling2D((2, 2))(x),
AveragePooling2D((2, 2))(x)]) # (6, 4, 4)
outputs.append(x)
x = Flatten()(x) # (1, 1, 96)
x = Dense(4, use_bias=False)(x)
outputs.append(x)
x = Dense(3)(x) # (1, 1, 3)
outputs.append(x)
outputs.append(Add()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Subtract()([inputs[26], inputs[30]]))
outputs.append(Multiply()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Average()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Maximum()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Concatenate()([inputs[26], inputs[30], inputs[30]]))
intermediate_input_shape = (3, )
intermediate_in = Input(intermediate_input_shape)
intermediate_x = intermediate_in
intermediate_x = Dense(8)(intermediate_x)
intermediate_x = Dense(5, name='duplicate_layer_name')(intermediate_x)
intermediate_model = Model(inputs=[intermediate_in],
outputs=[intermediate_x],
name='intermediate_model')
intermediate_model.compile(loss='mse', optimizer='nadam')
x = intermediate_model(x) # (1, 1, 5)
intermediate_model_2 = Sequential()
intermediate_model_2.add(Dense(7, input_shape=(5, )))
intermediate_model_2.add(Dense(5, name='duplicate_layer_name'))
intermediate_model_2.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_2(x) # (1, 1, 5)
intermediate_model_3_nested = Sequential()
intermediate_model_3_nested.add(Dense(7, input_shape=(6, )))
intermediate_model_3_nested.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
intermediate_model_3 = Sequential()
intermediate_model_3.add(Dense(6, input_shape=(5, )))
intermediate_model_3.add(intermediate_model_3_nested)
intermediate_model_3.add(Dense(8))
intermediate_model_3.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_3(x) # (1, 1, 8)
x = Dense(3)(x) # (1, 1, 3)
shared_activation = Activation('tanh')
outputs = outputs + [
Activation('tanh')(inputs[25]),
Activation('hard_sigmoid')(inputs[25]),
Activation('selu')(inputs[25]),
Activation('sigmoid')(inputs[25]),
Activation('softplus')(inputs[25]),
Activation('softmax')(inputs[25]),
Activation('relu')(inputs[25]),
Activation('relu6')(inputs[25]),
Activation('swish')(inputs[25]),
Activation('exponential')(inputs[25]),
Activation('gelu')(inputs[25]),
Activation('softsign')(inputs[25]),
LeakyReLU()(inputs[25]),
ReLU()(inputs[25]),
ReLU(max_value=0.4, negative_slope=1.1, threshold=0.3)(inputs[25]),
ELU()(inputs[25]),
PReLU()(inputs[24]),
PReLU()(inputs[25]),
PReLU()(inputs[26]),
shared_activation(inputs[25]),
Activation('linear')(inputs[26]),
Activation('linear')(inputs[23]),
x,
shared_activation(x),
]
model = Model(inputs=inputs, outputs=outputs, name='test_model_exhaustive')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 2
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, epochs=10)
return model
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow.keras.layers import Add, Activation, Concatenate, Conv1D, Conv2D, Conv3D, \
Input, Subtract, Multiply, Average, Maximum, Minimum, LeakyReLU, PReLU, ELU, ThresholdedReLU, Softmax, \
ZeroPadding1D, ZeroPadding2D, ZeroPadding3D, Reshape
DEFAULT_TF_GRAPH = tf.get_default_graph()
@pytest.mark.parametrize('layer, input_layer, batch_size', [
(Concatenate(), [Input((6, 6, 5)), Input((6, 6, 5))], 32),
(Add(), [Input((6, 6, 5)), Input((6, 6, 5))], 32),
(Add(), [Input((6, 6, 5)), Input((1, 1, 5))], 32),
(Subtract(), [Input((6, 6, 5)), Input((6, 6, 5))], 32),
(Multiply(), [Input((6, 6, 5)), Input((6, 6, 5))], 32),
(Average(), [Input(
(6, 6, 5)), Input(
(6, 6, 5)), Input((6, 6, 5))], 32),
(Maximum(), [Input((6, 6, 5)), Input((6, 6, 5))], 32),
(Minimum(), [Input((6, 6, 5)), Input((6, 6, 5))], 32),
(Activation('relu'), Input((6, 6, 5)), 32),
(Reshape((36, 5)), Input((6, 6, 5)), 32),
(Reshape((20, 6, 5)), Input((6, 5, 4, 5)), 32),
(GatherLayer([1, 4, 2], axis=1), Input((6, 5)), 32),
(GatherLayer([1, 4, 2], axis=2), Input((6, 5)), 32),
(GatherLayer([1, 4, 2], axis=2), Input((6, 7, 5)), 32),
(LeakyReLU(), Input((6, 6, 5)), 32),
(ELU(), Input((6, 6, 5)), 32),
(ThresholdedReLU(), Input((6, 6, 5)), 32),
(Softmax(), Input((6, 6, 5)), 32),
(DownShiftLayer(), Input((6, 6, 5)), 32),
conv_3_flow = Conv2D(512, (3, 3), strides=1, activation='relu')(pool_2_flow)
#Layer4
conv_4_flow = Conv2D(512, (3, 3), strides=1, activation='relu')(conv_3_flow)
#Layer5
conv_5_flow = Conv2D(512, (3, 3), strides=1, activation='relu')(conv_4_flow)
pool_3_flow = MaxPooling2D((2, 2))(conv_5_flow)
flat_flow = Flatten()(pool_3_flow)
#Layer6
fc_1_flow = Dense(4096, activation='relu')(flat_flow)
#Layer7
fc_2_flow = Dense(2048, activation='relu')(fc_1_flow)
#output layer
out_flow = Dense(101, activation='softmax')(fc_2_flow)
#Taking the output of both the streams and combining them
out = Average()([out_frames, out_flow])
model = Model(inputs=[inp_frames, inp_flow], outputs=out)
opti_flow = tf.keras.optimizers.Adam(learning_rate=1e-5)
#compiling the model by using categorical_crossentropy loss
model.compile(optimizer=opti_flow,
loss='categorical_crossentropy',
metrics=['mae', 'accuracy'])
model.summary()
#visualizing the model on tensorboard
tensorboard = TensorBoard(log_dir="logs\{}".format(time()), write_graph=True)
#calling the datagenerator and passing the inputs to our model for training
i = 0
hist_frames = []
def get_test_model_exhaustive():
"""Returns a exhaustive test model."""
input_shapes = [
(2, 3, 4, 5, 6),
(2, 3, 4, 5, 6),
(7, 8, 9, 10),
(7, 8, 9, 10),
(11, 12, 13),
(11, 12, 13),
(14, 15),
(14, 15),
(16, ),
(16, ),
(2, ),
(1, ),
(2, ),
(1, ),
(1, 3),
(1, 4),
(1, 1, 3),
(1, 1, 4),
(1, 1, 1, 3),
(1, 1, 1, 4),
(1, 1, 1, 1, 3),
(1, 1, 1, 1, 4),
(26, 28, 3),
(4, 4, 3),
(4, 4, 3),
(4, ),
(2, 3),
(1, ),
(1, ),
(1, ),
(2, 3),
]
inputs = [Input(shape=s) for s in input_shapes]
outputs = []
outputs.append(Conv1D(1, 3, padding='valid')(inputs[6]))
outputs.append(Conv1D(2, 1, padding='same')(inputs[6]))
outputs.append(Conv1D(3, 4, padding='causal', dilation_rate=2)(inputs[6]))
outputs.append(ZeroPadding1D(2)(inputs[6]))
outputs.append(Cropping1D((2, 3))(inputs[6]))
outputs.append(MaxPooling1D(2)(inputs[6]))
outputs.append(MaxPooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(AveragePooling1D(2)(inputs[6]))
outputs.append(AveragePooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(GlobalMaxPooling1D()(inputs[6]))
outputs.append(GlobalMaxPooling1D(data_format="channels_first")(inputs[6]))
outputs.append(GlobalAveragePooling1D()(inputs[6]))
outputs.append(
GlobalAveragePooling1D(data_format="channels_first")(inputs[6]))
outputs.append(Conv2D(4, (3, 3))(inputs[4]))
outputs.append(Conv2D(4, (3, 3), use_bias=False)(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4]))
outputs.append(SeparableConv2D(3, (3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((1, 2))(inputs[4]))
outputs.append(MaxPooling2D((2, 2))(inputs[4]))
outputs.append(
MaxPooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(AveragePooling2D((2, 2))(inputs[4]))
outputs.append(
AveragePooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(GlobalAveragePooling2D()(inputs[4]))
outputs.append(
GlobalAveragePooling2D(data_format="channels_first")(inputs[4]))
outputs.append(GlobalMaxPooling2D()(inputs[4]))
outputs.append(GlobalMaxPooling2D(data_format="channels_first")(inputs[4]))
outputs.append(BatchNormalization()(inputs[4]))
outputs.append(Dropout(0.5)(inputs[4]))
outputs.append(ZeroPadding2D(2)(inputs[4]))
outputs.append(ZeroPadding2D((2, 3))(inputs[4]))
outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Cropping2D(2)(inputs[4]))
outputs.append(Cropping2D((2, 3))(inputs[4]))
outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Dense(3, use_bias=True)(inputs[13]))
outputs.append(Dense(3, use_bias=True)(inputs[14]))
outputs.append(Dense(4, use_bias=False)(inputs[16]))
outputs.append(Dense(4, use_bias=False, activation='tanh')(inputs[18]))
outputs.append(Dense(4, use_bias=False)(inputs[20]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='bilinear')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='bilinear')(inputs[4]))
for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]:
outputs.append(Concatenate(axis=axis)([inputs[0], inputs[1]]))
for axis in [-4, -3, -2, -1, 1, 2, 3, 4]:
outputs.append(Concatenate(axis=axis)([inputs[2], inputs[3]]))
for axis in [-3, -2, -1, 1, 2, 3]:
outputs.append(Concatenate(axis=axis)([inputs[4], inputs[5]]))
for axis in [-2, -1, 1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[6], inputs[7]]))
for axis in [-1, 1]:
outputs.append(Concatenate(axis=axis)([inputs[8], inputs[9]]))
for axis in [-1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]]))
for axis in [-1, 3]:
outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]]))
for axis in [-1, 4]:
outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]]))
for axis in [-1, 5]:
outputs.append(Concatenate(axis=axis)([inputs[20], inputs[21]]))
outputs.append(UpSampling1D(size=2)(inputs[6]))
outputs.append(Multiply()([inputs[10], inputs[11]]))
outputs.append(Multiply()([inputs[11], inputs[10]]))
outputs.append(Multiply()([inputs[11], inputs[13]]))
outputs.append(Multiply()([inputs[10], inputs[11], inputs[12]]))
outputs.append(Multiply()([inputs[11], inputs[12], inputs[13]]))
shared_conv = Conv2D(1, (1, 1),
padding='valid',
name='shared_conv',
activation='relu')
up_scale_2 = UpSampling2D((2, 2))
x1 = shared_conv(up_scale_2(inputs[23])) # (1, 8, 8)
x2 = shared_conv(up_scale_2(inputs[24])) # (1, 8, 8)
x3 = Conv2D(1, (1, 1),
padding='valid')(up_scale_2(inputs[24])) # (1, 8, 8)
x = Concatenate()([x1, x2, x3]) # (3, 8, 8)
outputs.append(x)
x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8)
outputs.append(x)
x = Dropout(0.5)(x)
outputs.append(x)
x = Concatenate()([MaxPooling2D((2, 2))(x),
AveragePooling2D((2, 2))(x)]) # (6, 4, 4)
outputs.append(x)
x = Flatten()(x) # (1, 1, 96)
x = Dense(4, use_bias=False)(x)
outputs.append(x)
x = Dense(3)(x) # (1, 1, 3)
outputs.append(x)
outputs.append(Add()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Subtract()([inputs[26], inputs[30]]))
outputs.append(Multiply()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Average()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Maximum()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Concatenate()([inputs[26], inputs[30], inputs[30]]))
intermediate_input_shape = (3, )
intermediate_in = Input(intermediate_input_shape)
intermediate_x = intermediate_in
intermediate_x = Dense(8)(intermediate_x)
intermediate_x = Dense(5)(intermediate_x)
intermediate_model = Model(inputs=[intermediate_in],
outputs=[intermediate_x],
name='intermediate_model')
intermediate_model.compile(loss='mse', optimizer='nadam')
x = intermediate_model(x) # (1, 1, 5)
intermediate_model_2 = Sequential()
intermediate_model_2.add(Dense(7, input_shape=(5, )))
intermediate_model_2.add(Dense(5))
intermediate_model_2.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_2(x) # (1, 1, 5)
x = Dense(3)(x) # (1, 1, 3)
shared_activation = Activation('tanh')
outputs = outputs + [
Activation('tanh')(inputs[25]),
Activation('hard_sigmoid')(inputs[25]),
Activation('selu')(inputs[25]),
Activation('sigmoid')(inputs[25]),
Activation('softplus')(inputs[25]),
Activation('softmax')(inputs[25]),
Activation('relu')(inputs[25]),
LeakyReLU()(inputs[25]),
ELU()(inputs[25]),
PReLU()(inputs[24]),
PReLU()(inputs[25]),
PReLU()(inputs[26]),
shared_activation(inputs[25]),
Activation('linear')(inputs[26]),
Activation('linear')(inputs[23]),
x,
shared_activation(x),
]
model = Model(inputs=inputs, outputs=outputs, name='test_model_exhaustive')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 1
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, epochs=10)
return model
def build_xnet(use_backbone,
backbone,
classes,
skip_connection_layers,
decoder_filters=(256, 128, 64, 32),
upsample_rates=(2, 2, 2, 2),
n_upsample_blocks=4,
block_type='upsampling',
activation='sigmoid',
input_shape=(None, None, 3),
use_batchnorm=True,
attention=False,
deep_supervision=False):
downterm = [None] * (n_upsample_blocks + 1)
dsterm = [None] * (n_upsample_blocks)
interm = [[None] * (n_upsample_blocks - i + 1)
for i in range(n_upsample_blocks + 1)]
if block_type == 'transpose':
up_block = Transpose2D_block
else:
up_block = Upsample2D_block
# Using Backbone for the Encoder
if use_backbone:
input = backbone.input
downsampling_layers = skip_connection_layers
output = backbone.output if 'vgg' not in backbone.name else backbone.layers[
-2].output
# convert layer names to indices
downsampling_idx = ([
get_layer_number(backbone, l) if isinstance(l, str) else l
for l in downsampling_layers
])
downsampling_list = [
backbone.layers[downsampling_idx[i]].output
for i in range(len(downsampling_idx))
]
for i in range(len(downsampling_idx)):
downterm[n_upsample_blocks - i - 1] = downsampling_list[i]
downterm[-1] = output
# Using Conv+relu for the Encoder
else:
encoder_filters = (decoder_filters[0] * 2, ) + decoder_filters
input = Input(shape=input_shape)
for i in range(n_upsample_blocks + 1):
if i == 0:
x = conv_block(encoder_filters[n_upsample_blocks - i],
i,
0,
amount=2,
use_batchnorm=use_batchnorm,
type_block='encoder')(input)
else:
down_rate = to_tuple(upsample_rates[n_upsample_blocks - i - 1])
x = MaxPool2D(pool_size=down_rate,
name='encoder_stage{}-0_maxpool'.format(i))(x)
x = conv_block(encoder_filters[n_upsample_blocks - i],
i,
0,
amount=2,
use_batchnorm=use_batchnorm,
type_block='encoder')(x)
downterm[i] = x
skip_connection_layers = tuple([i.name for i in downterm])
output = downterm[-1]
for i in range(len(skip_connection_layers)):
interm[i][0] = downterm[i]
interm[-1][0] = output
down_rate = to_tuple(upsample_rates[-1])
for j in range(n_upsample_blocks):
for i in range(n_upsample_blocks - j):
upsample_rate = to_tuple(upsample_rates[n_upsample_blocks - i - 1])
if deep_supervision and i == 0:
dsterm[j - 1] = interm[i][j]
if attention:
interm[i][j] = attention_block(
decoder_filters[n_upsample_blocks - i - 1], interm[i][j],
i, j)(interm[i + 1][j])
if interm[i][j + 1] is None:
interm[i][j + 1] = up_block(
decoder_filters[n_upsample_blocks - i - 1],
i,
j + 1,
upsample_rate=upsample_rate,
skip=interm[i][:j + 1],
use_batchnorm=use_batchnorm)(interm[i + 1][j])
# Deep Supervision
if deep_supervision and n_upsample_blocks > 1:
# Currently only VGG or not using backbone
dsterm[-1] = interm[0][-1]
x = DeepSupervision(classes)(dsterm)
x = Average(name='average_ds')(x)
else:
if use_backbone and 'vgg' not in backbone.name:
x = up_block(decoder_filters[-1],
0,
n_upsample_blocks + 1,
upsample_rate=down_rate,
use_batchnorm=use_batchnorm)(interm[0][-1])
else:
x = interm[0][-1]
x = Conv2D(classes, (3, 3), padding='same', name='final_conv')(x)
x = Activation(activation, name=activation)(x)
return Model(input, x)
def __init__(self,
config,
latent_code_garms_sz=1024,
garmparams_sz=config.PCA_,
name=None):
super(PoseShapeOffsetModel, self).__init__(name=name)
self.config = config
self.latent_code_garms_sz = latent_code_garms_sz
self.garmparams_sz = garmparams_sz
self.latent_code_betas_sz = 128
# ToDo: Minor: Remove hard coded colors. Should be same as rendered colors in input
self.colormap = tf.cast([
np.array([255, 255, 255]),
np.array([65, 0, 65]),
np.array([0, 65, 65]),
np.array([145, 65, 0]),
np.array([145, 0, 65]),
np.array([0, 145, 65])
], tf.float32) / 255.
with open('assets/hresMapping.pkl', 'rb') as f:
_, self.faces = pkl.load(f, encoding="latin1")
self.faces = np.int32(self.faces)
# Define network layers
self.top_ = SingleImageNet(self.latent_code_garms_sz,
self.latent_code_betas_sz)
for n in self.config.garmentKeys:
gn = GarmentNet(self.config.PCA_, n, self.garmparams_sz)
self.garmentModels.append(gn)
self.smpl = SMPL('assets/neutral_smpl.pkl',
theta_in_rodrigues=False,
theta_is_perfect_rotmtx=False,
isHres=True,
scale=True)
self.smpl_J = SmplBody25Layer(theta_in_rodrigues=False,
theta_is_perfect_rotmtx=False,
isHres=True)
self.J_layers = [NameLayer('J_{}'.format(i)) for i in range(NUM)]
self.lat_betas = Dense(self.latent_code_betas_sz,
kernel_initializer=initializers.RandomNormal(
0, 0.00005),
activation='relu')
self.betas = Dense(10,
kernel_initializer=initializers.RandomNormal(
0, 0.000005),
name='betas')
init_trans = np.array([0, 0.2, -2.])
init_pose = np.load('assets/mean_a_pose.npy')
init_pose[:3] = 0
init_pose = tf.reshape(
batch_rodrigues(init_pose.reshape(-1, 3).astype(np.float32)),
(-1, ))
self.pose_trans = tf.concat((init_pose, init_trans), axis=0)
self.lat_pose = Dense(self.latent_code_betas_sz,
kernel_initializer=initializers.RandomNormal(
0, 0.000005),
activation='relu')
self.lat_pose_layer = Dense(
24 * 3 * 3 + 3,
kernel_initializer=initializers.RandomNormal(0, 0.000005),
name='pose_trans')
self.cut_trans = Lambda(lambda z: z[:, -3:])
self.trans_layers = [
NameLayer('trans_{}'.format(i)) for i in range(NUM)
]
self.cut_poses = Lambda(lambda z: z[:, :-3])
self.reshape_pose = Reshape((24, 3, 3))
self.pose_layers = [NameLayer('pose_{}'.format(i)) for i in range(NUM)]
# Optional: Condition garment on betas, probably not
self.latent_code_offset_ShapeMerged = Dense(self.latent_code_garms_sz +
self.latent_code_betas_sz,
activation='relu')
self.latent_code_offset_ShapeMerged_2 = Dense(
self.latent_code_garms_sz + self.latent_code_betas_sz,
activation='relu',
name='latent_code_offset_ShapeMerged')
self.avg = Average()
self.flatten = Flatten()
self.concat = Concatenate()
self.scatters = []
for vs in self.vertSpread:
self.scatters.append(Scatter_(vs, self.config.NVERTS))
def unetpp(multi_modal,
early_fusion,
project_excite,
inputs_bmode,
inputs_pd,
cascade=False):
set_image_data_format('channels_last')
multi_stage_fusion = multi_modal and not early_fusion
# Input
if multi_modal and early_fusion:
conv, pool, pool_b = fusion_encoder_block(n_filters[0] // 2,
multi_modal, project_excite,
inputs_bmode, inputs_pd)
conv0_0, pool0, pool0_b = fusion_encoder_block(n_filters[0],
multi_stage_fusion,
project_excite, conv,
None)
else:
conv0_0, pool0, pool0_b = fusion_encoder_block(n_filters[0],
multi_stage_fusion,
project_excite,
inputs_bmode, inputs_pd)
# Fused Encode Block 1
conv1_0, pool1, pool1_b = fusion_encoder_block(n_filters[1],
multi_stage_fusion,
project_excite, pool0,
pool0_b)
# Dense Conv Block 0_1
up0_1 = concatenate([
Conv3DTranspose(n_filters[0], 2, strides=(2, 2, 2),
padding='same')(conv1_0), conv0_0
],
axis=-1)
conv0_1 = conv_block(up0_1, n_filters[0])
# Fused Encode Block 2
conv2_0, pool2, pool2_b = fusion_encoder_block(n_filters[2],
multi_stage_fusion,
project_excite, pool1,
pool1_b)
# Dense Conv Block 1_1
up1_1 = concatenate([
Conv3DTranspose(n_filters[1], 2, strides=(2, 2, 2),
padding='same')(conv2_0), conv1_0
],
axis=-1)
conv1_1 = conv_block(up1_1, n_filters[1])
# Dense Conv Block 0_2
up0_2 = concatenate([
Conv3DTranspose(n_filters[0], 2, strides=(2, 2, 2),
padding='same')(conv1_1), conv0_1
],
axis=-1)
conv0_2 = conv_block(up0_2, n_filters[0])
# Fused Encode Block 3
conv3_0, pool3, pool3_b = fusion_encoder_block(n_filters[3],
multi_stage_fusion,
project_excite, pool2,
pool2_b)
# Dense Conv Block 2_1
up2_1 = concatenate([
Conv3DTranspose(n_filters[2], 2, strides=(2, 2, 2),
padding='same')(conv3_0), conv2_0
],
axis=-1)
conv2_1 = conv_block(up2_1, n_filters[2])
# Dense Conv Block 1_2
up1_2 = concatenate([
Conv3DTranspose(n_filters[1], 2, strides=(2, 2, 2),
padding='same')(conv2_1), conv1_1
],
axis=-1)
conv1_2 = conv_block(up1_2, n_filters[1])
# Dense Conv Block 0_3
up0_3 = concatenate([
Conv3DTranspose(n_filters[0], 2, strides=(2, 2, 2),
padding='same')(conv1_2), conv0_2
],
axis=-1)
conv0_3 = conv_block(up0_3, n_filters[0])
# Bottom Conv Block
conv4_0 = conv_block(pool3, n_filters[4], project_excite)
# Decode stage 1
up3_1 = concatenate([
Conv3DTranspose(n_filters[3], 2, strides=(2, 2, 2),
padding='same')(conv4_0), conv3_0
],
axis=-1)
conv3_1 = conv_block(up3_1, n_filters[3], project_excite)
# Decode stage 2
up2_2 = concatenate([
Conv3DTranspose(n_filters[2], 2, strides=(2, 2, 2),
padding='same')(conv3_1), conv2_1
],
axis=-1)
conv2_2 = conv_block(up2_2, n_filters[2], project_excite)
# Decode stage 3
up1_3 = concatenate([
Conv3DTranspose(n_filters[1], 2, strides=(2, 2, 2),
padding='same')(conv2_2), conv1_2
],
axis=-1)
conv1_3 = conv_block(up1_3, n_filters[1], project_excite)
# Decode stage 4
up0_4 = concatenate([
Conv3DTranspose(n_filters[0], 2, strides=(2, 2, 2),
padding='same')(conv1_3), conv0_3
],
axis=-1)
conv0_4 = conv_block(up0_4, n_filters[0], project_excite)
if cascade:
return Average()([conv0_1, conv0_2, conv0_3, conv0_4])
# Average all segmentation branches from top row
output_1 = Conv3D(1,
1,
activation='sigmoid',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(1e-4))(conv0_1)
output_2 = Conv3D(1,
1,
activation='sigmoid',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(1e-4))(conv0_2)
output_3 = Conv3D(1,
1,
activation='sigmoid',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(1e-4))(conv0_3)
output_4 = Conv3D(1,
1,
activation='sigmoid',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(1e-4))(conv0_4)
# If training with deep_supervision, average all skip connection convolution layers in the top row
if deep_supervision:
output = Average()([output_1, output_2, output_3, output_4])
else:
output = output_4
if multi_modal:
model = Model(inputs=[inputs_bmode, inputs_pd], outputs=[output])
else:
model = Model(inputs=[inputs_bmode], outputs=[output])
return model
def model(self, inputs, training_schedule, trainable=True, build=False):
_concat_axis = 3
alpha = 1.0
if 'warped' in inputs and 'flow' in inputs and 'brightness_error' in inputs:
concat_inputs = tf.concat([
inputs['input_a'], inputs['input_b'], inputs['warped'],
inputs['flow'], inputs['brightness_error']
],
axis=_concat_axis)
else:
concat_inputs = tf.concat([inputs['input_a'], inputs['input_b']],
axis=_concat_axis)
if build is True:
#INPUT_SHAPE = #(384, 512,6)
concat_inputs = tf.concat(
[inputs['input_a'], inputs['input_b']],
axis=_concat_axis) #Input(shape=INPUT_SHAPE)
#################################################
# ENCODER
#################################################
if trainable is False:
# all new operations will be in test mode from now on
K.set_learning_phase(0)
# Conv1_dw / Conv1
conv1_dw = _depthwise_convolution2D(concat_inputs,
alpha,
6, (3, 3),
strides=(2, 2),
training=trainable)
conv1 = _convolution2D(conv1_dw,
alpha,
32, (1, 1),
strides=(1, 1),
training=trainable)
# Conv2_dw / Conv2
conv2_dw = _depthwise_convolution2D(conv1,
alpha,
32, (3, 3),
strides=(2, 2),
training=trainable)
conv2 = _convolution2D(conv2_dw,
alpha,
64, (1, 1),
strides=(1, 1),
training=trainable)
# Conv3_dw / Conv3
conv3_dw = _depthwise_convolution2D(conv2,
alpha,
64, (3, 3),
strides=(2, 2),
training=trainable)
conv3 = _convolution2D(conv3_dw,
alpha,
128, (1, 1),
strides=(1, 1),
training=trainable)
# Conv4a_dw / Conv4a
conv4a_dw = _depthwise_convolution2D(conv3,
alpha,
128, (3, 3),
strides=(2, 2),
training=trainable)
conv4a = _convolution2D(conv4a_dw,
alpha,
256, (1, 1),
strides=(1, 1),
training=trainable)
# Conv4b_dw / Conv4b
conv4b_dw = _depthwise_convolution2D(conv4a,
alpha,
256, (3, 3),
strides=(1, 1),
training=trainable)
conv4b = _convolution2D(conv4b_dw,
alpha,
256, (1, 1),
strides=(1, 1),
training=trainable)
# Conv5a_dw / Conv5a
conv5a_dw = _depthwise_convolution2D(conv4b,
alpha,
256, (3, 3),
strides=(2, 2),
training=trainable)
conv5a = _convolution2D(conv5a_dw,
alpha,
512, (1, 1),
strides=(1, 1),
training=trainable)
# Conv5b_dw / Conv5b
conv5b_dw = _depthwise_convolution2D(conv5a,
alpha,
512, (3, 3),
strides=(1, 1),
training=trainable)
conv5b = _convolution2D(conv5b_dw,
alpha,
512, (1, 1),
strides=(1, 1),
training=trainable)
# Conv6a_dw / Conv6a
conv6a_dw = _depthwise_convolution2D(conv5b,
alpha,
512, (3, 3),
strides=(2, 2),
training=trainable)
conv6a = _convolution2D(conv6a_dw,
alpha,
1024, (1, 1),
strides=(1, 1),
training=trainable)
# Conv6b_dw / Conv6b
conv6b_dw = _depthwise_convolution2D(conv6a,
alpha,
1024, (3, 3),
strides=(1, 1),
training=trainable)
conv6b = _convolution2D(conv6b_dw,
alpha,
1024, (1, 1),
strides=(1, 1),
training=trainable)
#################################################
# DECODER
#################################################
# Conv7_dw / Conv7
conv7_dw = _depthwise_convolution2D(conv6b,
alpha,
1024, (3, 3),
strides=(1, 1),
training=trainable)
conv7 = _convolution2D(conv7_dw,
alpha,
256, (1, 1),
strides=(1, 1),
training=trainable)
# Conv8_dw /Conv8
conv7_resized_tensor = Lambda(resize_like,
arguments={
'ref_tensor': conv7,
'scale': 2
})(conv7)
concat_op1 = Concatenate(axis=_concat_axis)(
[conv7_resized_tensor, conv5b])
conv8_dw = _depthwise_convolution2D(concat_op1,
alpha,
768, (3, 3),
strides=(1, 1),
training=trainable)
conv8 = _convolution2D(conv8_dw,
alpha,
128, (1, 1),
strides=(1, 1),
training=trainable)
# Conv9_dw /Conv9
conv8_resized_tensor = Lambda(resize_like,
arguments={
'ref_tensor': conv8,
'scale': 2
})(conv8)
concat_op2 = Concatenate(axis=_concat_axis)(
[conv8_resized_tensor, conv4b])
conv9_dw = _depthwise_convolution2D(concat_op2,
alpha,
384, (3, 3),
strides=(1, 1),
training=trainable)
conv9 = _convolution2D(conv9_dw,
alpha,
64, (1, 1),
strides=(1, 1),
training=trainable)
# Conv10_dw / Conv10
coonv9_resized_tensor = Lambda(resize_like,
arguments={
'ref_tensor': conv9,
'scale': 2
})(conv9)
concat_op3 = Concatenate(axis=_concat_axis)(
[coonv9_resized_tensor, conv3])
conv10_dw = _depthwise_convolution2D(concat_op3,
alpha,
192, (3, 3),
strides=(1, 1),
training=trainable)
conv10 = _convolution2D(conv10_dw,
alpha,
32, (1, 1),
strides=(1, 1),
training=trainable)
# Conv11_dw / Con11
conv10_resized_tensor = Lambda(resize_like,
arguments={
'ref_tensor': conv10,
'scale': 2
})(conv10)
concat_op3 = Concatenate(axis=_concat_axis)(
[conv10_resized_tensor, conv2])
conv11_dw = _depthwise_convolution2D(concat_op3,
alpha,
96, (3, 3),
strides=(1, 1),
training=trainable)
conv11 = _convolution2D(conv11_dw,
alpha,
16, (1, 1),
strides=(1, 1),
training=trainable)
##################################################
# Optical Flow Predictions
##################################################
# Conv12_dw / conv12
conv12_dw = _depthwise_convolution2D(conv7,
alpha,
256, (3, 3),
strides=(1, 1),
training=trainable)
conv12 = _convolution2D(conv12_dw,
alpha,
2, (1, 1),
strides=(1, 1),
training=trainable)
# Conv13_dw / conv13
conv13_dw = _depthwise_convolution2D(conv8,
alpha,
128, (3, 3),
strides=(1, 1),
training=trainable)
conv13 = _convolution2D(conv13_dw,
alpha,
2, (1, 1),
strides=(1, 1),
training=trainable)
# Conv14_dw / conv14
conv14_dw = _depthwise_convolution2D(conv9,
alpha,
64, (3, 3),
strides=(1, 1),
training=trainable)
conv14 = _convolution2D(conv14_dw,
alpha,
2, (1, 1),
strides=(1, 1),
training=trainable)
# Conv15_dw / con15
conv15_dw = _depthwise_convolution2D(conv10,
alpha,
32, (3, 3),
strides=(1, 1),
training=trainable)
conv15 = _convolution2D(conv15_dw,
alpha,
2, (1, 1),
strides=(1, 1),
training=trainable)
# Conv16_dw / conv16
conv16_dw = _depthwise_convolution2D(conv11,
alpha,
16, (3, 3),
strides=(1, 1),
training=trainable)
conv16 = _convolution2D(conv16_dw,
alpha,
2, (1, 1),
strides=(1, 1),
training=trainable)
###################################################
# Multiple Optical Flow Predictions Fusion
###################################################
conv12_resized_tensor_x16 = Lambda(resize_like,
arguments={
'ref_tensor': conv12,
'scale': 16
})(conv12)
conv13_resized_tensor_x8 = Lambda(resize_like,
arguments={
'ref_tensor': conv13,
'scale': 8
})(conv13)
conv14_resized_tensor_x4 = Lambda(resize_like,
arguments={
'ref_tensor': conv14,
'scale': 4
})(conv14)
conv15_resized_tensor_x2 = Lambda(resize_like,
arguments={
'ref_tensor': conv15,
'scale': 2
})(conv15)
# 96x128x2
flow = Average(name='average_layer')([
conv12_resized_tensor_x16, conv13_resized_tensor_x8,
conv14_resized_tensor_x4, conv15_resized_tensor_x2, conv16
])
#flow = tf.image.resize_bilinear(average,
# tf.stack([height, width]),
# align_corners=True)
# Fuse groundtrunth resolution with prediction
#flow = Lambda(resize_like, arguments={'ref_tensor':average, 'scale': 4})(average)
return {'inputs': concat_inputs, 'flow': flow}
def create_range_view_branch(input_shapes=[(375, 1242, 3),
(375, 1242, 1),
(375, 1242, 1),
(375, 1242, 1),
(224, 256 , 2),
(112, 128 , 2),
(56 , 64 , 2),
(224, 256 , 3),
(112, 128 , 3),
(56 , 64 , 3),],
input_names=['rgb_img_input',
'depth_map_input',
'intensity_map_input',
'height_map_input',
'mapping_2x',
'mapping_4x',
'mapping_8x',
'geo_2x',
'geo_4x',
'geo_8x',],
input_types=['float32',
'float32',
'float32',
'float32',
'int32',
'int32',
'int32',
'float32',
'float32',
'float32',]):
init_inputs = []
inputs_dict = {}
for name, shape, dtype in zip(input_names, input_shapes, input_types):
inp = create_input_layer(input_shape=shape, name=name, dtype=dtype)
init_inputs.append(inp)
inputs_dict[name] = inp
conv_inputs = []
for input in init_inputs:
if 'mapping' not in input.name and 'geo' not in input.name:
conv_inputs.append(conv_block(input, 32, 7, 2)) # /2
x = Average()(conv_inputs)
l1 = x
for i in range(4): # same res
x = create_res_conv_block(x, 32, 3)
l2 = x
for i in range(4): # /4
x = create_res_conv_block(x, 64, 3, i==0)
l3 = x
for i in range(6): # /8
x = create_res_conv_block(x, 128, 3, i==0)
l4 = x
out_map_sz = (112, 128)
TOP_DOWN_PYRAMID_SIZE = 96
L2 = tf.image.resize(l2, (l3.shape[1], l3.shape[2]), 'nearest')
L2 = conv_block(L2, TOP_DOWN_PYRAMID_SIZE, 3, 1)
L2 = conv_block(L2, TOP_DOWN_PYRAMID_SIZE, 1, 1)
L3 = conv_block(l3, TOP_DOWN_PYRAMID_SIZE, 1, 1)
L4 = tf.image.resize(l4, (l3.shape[1], l3.shape[2]), 'nearest')
L4 = conv_block(L4, TOP_DOWN_PYRAMID_SIZE, 3, 1)
L4 = conv_block(L4, TOP_DOWN_PYRAMID_SIZE, 1, 1)
RV_OUT = Add()([L2, L3, L4])
RV2BEV_2x = Lambda(tf.gather_nd, arguments={'indices': inputs_dict['mapping_2x'], 'batch_dims': 1})(RV_OUT)
RV2BEV_2x = Concatenate()([RV2BEV_2x, inputs_dict['geo_2x']]) # add geometric feature
RV2BEV_2x = transform_rangeview_to_bev(RV2BEV_2x, 128)
RV2BEV_4x = Lambda(tf.gather_nd, arguments={'indices': inputs_dict['mapping_4x'], 'batch_dims': 1})(RV_OUT)
RV2BEV_4x = Concatenate()([RV2BEV_4x, inputs_dict['geo_4x']]) # add geometric feature
RV2BEV_4x = transform_rangeview_to_bev(RV2BEV_4x, 192)
RV2BEV_8x = Lambda(tf.gather_nd, arguments={'indices': inputs_dict['mapping_8x'], 'batch_dims': 1})(RV_OUT)
RV2BEV_8x = Concatenate()([RV2BEV_8x, inputs_dict['geo_8x']]) # add geometric feature
RV2BEV_8x = transform_rangeview_to_bev(RV2BEV_8x, 256)
# print(RV2BEV_2x.shape, RV2BEV_4x.shape, RV2BEV_8x.shape)
return {
'inputs': init_inputs,
'outputs': [
RV2BEV_2x,
RV2BEV_4x,
RV2BEV_8x
],
}
