def test_merge_mask_2d():
rand = lambda *shape: np.asarray(np.random.random(shape) > 0.5,
dtype='int32')
# inputs
input_a = layers.Input(shape=(3, ))
input_b = layers.Input(shape=(3, ))
# masks
masked_a = layers.Masking(mask_value=0)(input_a)
masked_b = layers.Masking(mask_value=0)(input_b)
# three different types of merging
merged_sum = legacy_layers.merge([masked_a, masked_b], mode='sum')
merged_concat = legacy_layers.merge([masked_a, masked_b],
mode='concat',
concat_axis=1)
merged_concat_mixed = legacy_layers.merge([masked_a, input_b],
mode='concat',
concat_axis=1)
# test sum
model_sum = models.Model([input_a, input_b], [merged_sum])
model_sum.compile(loss='mse', optimizer='sgd')
model_sum.fit([rand(2, 3), rand(2, 3)], [rand(2, 3)], epochs=1)
# test concatenation
model_concat = models.Model([input_a, input_b], [merged_concat])
model_concat.compile(loss='mse', optimizer='sgd')
model_concat.fit([rand(2, 3), rand(2, 3)], [rand(2, 6)], epochs=1)
# test concatenation with masked and non-masked inputs
model_concat = models.Model([input_a, input_b], [merged_concat_mixed])
model_concat.compile(loss='mse', optimizer='sgd')
model_concat.fit([rand(2, 3), rand(2, 3)], [rand(2, 6)], epochs=1)
def test_merge_mask_2d():
rand = lambda *shape: np.asarray(np.random.random(shape) > 0.5, dtype='int32')
# inputs
input_a = layers.Input(shape=(3,))
input_b = layers.Input(shape=(3,))
# masks
masked_a = layers.Masking(mask_value=0)(input_a)
masked_b = layers.Masking(mask_value=0)(input_b)
# three different types of merging
merged_sum = legacy_layers.merge([masked_a, masked_b], mode='sum')
merged_concat = legacy_layers.merge([masked_a, masked_b], mode='concat', concat_axis=1)
merged_concat_mixed = legacy_layers.merge([masked_a, input_b], mode='concat', concat_axis=1)
# test sum
model_sum = models.Model([input_a, input_b], [merged_sum])
model_sum.compile(loss='mse', optimizer='sgd')
model_sum.fit([rand(2, 3), rand(2, 3)], [rand(2, 3)], epochs=1)
# test concatenation
model_concat = models.Model([input_a, input_b], [merged_concat])
model_concat.compile(loss='mse', optimizer='sgd')
model_concat.fit([rand(2, 3), rand(2, 3)], [rand(2, 6)], epochs=1)
# test concatenation with masked and non-masked inputs
model_concat = models.Model([input_a, input_b], [merged_concat_mixed])
model_concat.compile(loss='mse', optimizer='sgd')
model_concat.fit([rand(2, 3), rand(2, 3)], [rand(2, 6)], epochs=1)
def covariance_block_residual(input_tensor,
nb_class,
stage,
block,
epsilon=0,
parametric=[],
denses=[],
wv=True,
wv_param=None,
activation='relu',
o2tconstraints=None,
vectorization='wv',
**kwargs):
if epsilon > 0:
cov_name_base = 'cov' + str(stage) + block + '_branch_epsilon' + str(
epsilon)
else:
cov_name_base = 'cov' + str(stage) + block + ''
o2t_name_base = 'o2t' + str(stage) + block
dense_name_base = 'dense' + str(stage) + block + ''
second_layer = SecondaryStatistic(name=cov_name_base,
eps=epsilon,
**kwargs)
x = second_layer(input_tensor)
cov_dim = second_layer.out_dim
input_cov = x
for id, param in enumerate(parametric):
x = O2Transform(param, activation='relu',
name=o2t_name_base + str(id))(x)
if not param == cov_dim:
x = O2Transform(cov_dim,
activation='relu',
name=o2t_name_base + str(id) + 'r')(x)
x = merge([x, input_cov],
mode='sum',
name='residualsum_{}_{}'.format(str(id), str(block)))
if wv:
if wv_param is None:
wv_param = nb_class
x = WeightedVectorization(wv_param,
activation=activation,
name='wv' + str(stage) + block)(x)
else:
x = Flatten()(x)
for id, param in enumerate(denses):
x = Dense(param, activation=activation,
name=dense_name_base + str(id))(x)
x = Dense(nb_class, activation=activation, name=dense_name_base)(x)
return x
def build(timeSteps, variables, classes, nlstms):
#CONV=>POOL
inputNet = Input(shape=(1, timeSteps,
variables)) #,batch_shape=(32,1, 7, 5))
conv1 = Conv2D(20, (2, 5), padding="same")(inputNet)
conv1 = Activation("relu")(conv1)
conv1 = AveragePooling2D(pool_size=(2, 1), strides=(1, 1))(conv1)
conv2 = Conv2D(nlstms, (3, 3), padding="same")(conv1)
conv2 = AveragePooling2D(pool_size=(2, 1), strides=(1, 1))(conv2)
conv2 = Activation("relu")(conv2)
channels = Dropout(0.40)(conv2)
lstmsVec = []
for x in range(0, nlstms):
filterImg = Lambda(lambda element: element[:, x, :, :])(channels)
#lstm=Bidirectional(LSTM(50),merge_mode='concat')(filterImg)
#lstm=LSTM(50,stateful=True)(filterImg) #batch_shape=(512, 7, 5)
#denselayers=Dense(400)(lstm)
#denselayers=Activation("relu")(denselayers)
#denselayers=Dropout(0.5)(denselayers)
#denselayers=Dense(150)(denselayers)
#denselayers=Activation("relu")(denselayers)
#denselayers=Dropout(0.8)(denselayers)
#classificationLayer=Dense(classes)(lstm)
#classificationLayer=Activation("softmax")(classificationLayer)
#lstmsVec.append(classificationLayer)
lstmsVec.append(filterImg)
#base example for 2 lstms
#x0 = Lambda(lambda x : x[:,0,:,:])(out1)
#x1 = Lambda(lambda x : x[:,1,:,:])(out1)
#lstm1=LSTM(40,input_shape=(5,5))(x0)
#lstm2=LSTM(40,input_shape=(5,5))(x1)
merged = merge(lstmsVec, mode='concat', concat_axis=2)
# a softmax classifier
#flat=Flatten()(merged)
lstm = Bidirectional(LSTM(20, dropout=0.35),
merge_mode='concat')(merged)
#lstm=LSTM(20,dropout=0.35,batch_input_shape=(4, 5, 50),stateful=True)(merged)
classificationLayer = Dense(classes)(lstm)
classificationLayer = Activation("softmax")(classificationLayer)
model = Model(inputNet, classificationLayer)
#model=Model(inputNet,lstmsVec)
return model
def get_f_strided(args, data_format, input_shape, x_shape):
inpt = Input(input_shape)
hidden = inpt
hidden = Conv2D(args.n_kernels, (5, 5),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = Conv2D(args.n_kernels, (5, 5),
padding='same',
strides=(2, 2),
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = Conv2D(args.n_kernels, (5, 5),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = Conv2DTranspose(args.n_kernels, (5, 5),
padding='same',
strides=(2, 2),
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = Lambda(
lambda x: x[:, :x_shape[0], :x_shape[1], :],
# Cut off additional rows
output_shape=x_shape[:2] + (args.n_kernels, ))(hidden)
hidden = Conv2D(args.n_kernels, (5, 5),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = merge([inpt, hidden], mode='concat', concat_axis=-1)
hidden = Conv2D(args.n_kernels, (1, 1),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
output = Conv2D(3, (1, 1),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation='sigmoid')(hidden)
f = Model(inpt, output, name='f')
return f
def get_f_simple(args, data_format, input_shape):
# Same as D , but without the 1x1-Convolution before the merge
inpt = Input(input_shape)
hidden = inpt
hidden = Conv2D(args.n_kernels, (5, 5),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = Conv2D(args.n_kernels, (5, 5),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = Conv2D(args.n_kernels, (5, 5),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = Conv2D(args.n_kernels, (7, 7),
padding='same',
dilation_rate=(1, 1),
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = Conv2D(args.n_kernels, (5, 5),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = merge([inpt, hidden], mode='concat', concat_axis=-1)
hidden = Conv2D(args.n_kernels, (1, 1),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
output = Conv2D(3, (1, 1),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation='sigmoid')(hidden)
f = Model(inpt, output, name='f')
return f
def get_f_dilated(args, data_format, input_shape):
inpt = Input(input_shape)
hidden = inpt
hidden = Conv2D(args.n_kernels, (5, 5),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = Conv2D(args.n_kernels, (5, 5),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = Conv2D(args.n_kernels, (5, 5),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = Conv2D(args.n_kernels, (7, 7),
padding='same',
dilation_rate=(2, 2),
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = Conv2D(args.n_kernels, (5, 5),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
hidden = merge([inpt, hidden], mode='concat', concat_axis=-1)
hidden = Conv2D(args.n_kernels, (1, 1),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation=args.activation)(hidden)
output = Conv2D(3, (1, 1),
padding='same',
data_format=data_format,
kernel_initializer=args.initializer,
activation='sigmoid')(hidden)
f = Model(inpt, output, name='f')
return f
def test_merge_mask_3d():
rand = lambda *shape: np.asarray(np.random.random(shape) > 0.5, dtype='int32')
# embeddings
input_a = layers.Input(shape=(3,), dtype='int32')
input_b = layers.Input(shape=(3,), dtype='int32')
embedding = layers.Embedding(3, 4, mask_zero=True)
embedding_a = embedding(input_a)
embedding_b = embedding(input_b)
# rnn
rnn = layers.SimpleRNN(3, return_sequences=True)
rnn_a = rnn(embedding_a)
rnn_b = rnn(embedding_b)
# concatenation
merged_concat = legacy_layers.merge([rnn_a, rnn_b], mode='concat', concat_axis=-1)
model = models.Model([input_a, input_b], [merged_concat])
model.compile(loss='mse', optimizer='sgd')
model.fit([rand(2, 3), rand(2, 3)], [rand(2, 3, 6)])
def test_merge_mask_3d():
rand = lambda *shape: np.asarray(np.random.random(shape) > 0.5,
dtype='int32')
# embeddings
input_a = layers.Input(shape=(3, ), dtype='int32')
input_b = layers.Input(shape=(3, ), dtype='int32')
embedding = layers.Embedding(3, 4, mask_zero=True)
embedding_a = embedding(input_a)
embedding_b = embedding(input_b)
# rnn
rnn = layers.SimpleRNN(3, return_sequences=True)
rnn_a = rnn(embedding_a)
rnn_b = rnn(embedding_b)
# concatenation
merged_concat = legacy_layers.merge([rnn_a, rnn_b],
mode='concat',
concat_axis=-1)
model = models.Model([input_a, input_b], [merged_concat])
model.compile(loss='mse', optimizer='sgd')
model.fit([rand(2, 3), rand(2, 3)], [rand(2, 3, 6)])
def test_merge():
# test modes: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot'.
input_shapes = [(3, 2), (3, 2)]
inputs = [np.random.random(shape) for shape in input_shapes]
# test functional API
for mode in ['sum', 'mul', 'concat', 'ave', 'max']:
print(mode)
input_a = layers.Input(shape=input_shapes[0][1:])
input_b = layers.Input(shape=input_shapes[1][1:])
merged = legacy_layers.merge([input_a, input_b], mode=mode)
model = models.Model([input_a, input_b], merged)
model.compile('rmsprop', 'mse')
expected_output_shape = model.compute_output_shape(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = models.Model.from_config(config)
model.compile('rmsprop', 'mse')
# test Merge (#2460)
merged = legacy_layers.Merge(mode=mode)([input_a, input_b])
model = models.Model([input_a, input_b], merged)
model.compile('rmsprop', 'mse')
expected_output_shape = model.compute_output_shape(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
# test lambda with output_shape lambda
input_a = layers.Input(shape=input_shapes[0][1:])
input_b = layers.Input(shape=input_shapes[1][1:])
merged = legacy_layers.merge(
[input_a, input_b],
mode=lambda tup: K.concatenate([tup[0], tup[1]]),
output_shape=lambda tup: tup[0][:-1] + (tup[0][-1] + tup[1][-1], ))
model = models.Model([input_a, input_b], merged)
expected_output_shape = model.compute_output_shape(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = models.Model.from_config(config)
model.compile('rmsprop', 'mse')
# test function with output_shape function
def fn_mode(tup):
x, y = tup
return K.concatenate([x, y], axis=1)
def fn_output_shape(tup):
s1, s2 = tup
return (s1[0], s1[1] + s2[1]) + s1[2:]
input_a = layers.Input(shape=input_shapes[0][1:])
input_b = layers.Input(shape=input_shapes[1][1:])
merged = legacy_layers.merge([input_a, input_b],
mode=fn_mode,
output_shape=fn_output_shape)
model = models.Model([input_a, input_b], merged)
expected_output_shape = model.compute_output_shape(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = models.Model.from_config(config)
model.compile('rmsprop', 'mse')
# test function with output_mask function
# time dimension is required for masking
input_shapes = [(4, 3, 2), (4, 3, 2)]
inputs = [np.random.random(shape) for shape in input_shapes]
def fn_output_mask(tup):
x_mask, y_mask = tup
return K.concatenate([x_mask, y_mask])
input_a = layers.Input(shape=input_shapes[0][1:])
input_b = layers.Input(shape=input_shapes[1][1:])
a = layers.Masking()(input_a)
b = layers.Masking()(input_b)
merged = legacy_layers.merge([a, b],
mode=fn_mode,
output_shape=fn_output_shape,
output_mask=fn_output_mask)
model = models.Model([input_a, input_b], merged)
expected_output_shape = model.compute_output_shape(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = models.Model.from_config(config)
model.compile('rmsprop', 'mse')
mask_inputs = (np.zeros(input_shapes[0][:-1]),
np.ones(input_shapes[1][:-1]))
expected_mask_output = np.concatenate(mask_inputs, axis=-1)
mask_input_placeholders = [
K.placeholder(shape=input_shape[:-1]) for input_shape in input_shapes
]
mask_output = model.layers[-1]._output_mask(mask_input_placeholders)
assert np.all(
K.function(mask_input_placeholders, [mask_output])(mask_inputs)[0] ==
expected_mask_output)
# test lambda with output_mask lambda
input_a = layers.Input(shape=input_shapes[0][1:])
input_b = layers.Input(shape=input_shapes[1][1:])
a = layers.Masking()(input_a)
b = layers.Masking()(input_b)
merged = legacy_layers.merge(
[a, b],
mode=lambda tup: K.concatenate([tup[0], tup[1]], axis=1),
output_shape=lambda tup:
(tup[0][0], tup[0][1] + tup[1][1]) + tup[0][2:],
output_mask=lambda tup: K.concatenate([tup[0], tup[1]]))
model = models.Model([input_a, input_b], merged)
expected_output_shape = model.compute_output_shape(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = models.Model.from_config(config)
model.compile('rmsprop', 'mse')
mask_output = model.layers[-1]._output_mask(mask_input_placeholders)
assert np.all(
K.function(mask_input_placeholders, [mask_output])(mask_inputs)[0] ==
expected_mask_output)
# test with arguments
input_shapes = [(3, 2), (3, 2)]
inputs = [np.random.random(shape) for shape in input_shapes]
def fn_mode(tup, a, b):
x, y = tup
return x * a + y * b
input_a = layers.Input(shape=input_shapes[0][1:])
input_b = layers.Input(shape=input_shapes[1][1:])
merged = legacy_layers.merge([input_a, input_b],
mode=fn_mode,
output_shape=lambda s: s[0],
arguments={
'a': 0.7,
'b': 0.3
})
model = models.Model([input_a, input_b], merged)
output = model.predict(inputs)
config = model.get_config()
model = models.Model.from_config(config)
assert np.all(model.predict(inputs) == output)
def dcov_model_wrapper_v2(base_model,
parametrics=[],
mode=0,
nb_classes=1000,
basename='',
cov_mode='channel',
cov_branch='o2transform',
cov_branch_output=None,
freeze_conv=False,
cov_regularizer=None,
nb_branch=1,
concat='concat',
last_conv_feature_maps=[],
upsample_method='conv',
regroup=False,
**kwargs):
"""
Wrapper for any base model, attach right after the last layer of given model
Parameters
----------
base_model
parametrics
mode
nb_classes
input_shape
load_weights
cov_mode
cov_branch
cov_branch_output
cov_block_mode
last_avg
freeze_conv
Returns
-------
"""
cov_branch_mode = cov_branch
# Function name
covariance_block = get_cov_block(cov_branch)
if cov_branch_output is None:
cov_branch_output = nb_classes
x = base_model.output
x = upsample_wrapper_v1(x,
last_conv_feature_maps,
upsample_method,
kernel=[1, 1])
def split_keras_tensor_according_axis(x, nb_split, axis, axis_dim):
outputs = []
split_dim = axis_dim / nb_split
split_loc = [split_dim * i for i in range(nb_split)]
split_loc.append(-1)
for i in range(nb_split):
outputs.append(x[:, :, :, split_loc[i]:split_loc[i + 1]])
return outputs
cov_input = SeparateConvolutionFeatures(nb_branch)(x)
if regroup:
with tf.device('/gpu:0'):
cov_input = Regrouping(None)(cov_input)
cov_outputs = []
for ind, x in enumerate(cov_input):
if mode == 0:
x = Flatten()(x)
for ind, param in enumerate(parametrics):
x = Dense(param, activation='relu', name='fc{}'.format(ind))(x)
# x = Dense(nb_classes, activation='softmax')(x)
if mode == 1:
cov_branch = covariance_block(x,
cov_branch_output,
stage=5,
block=str(ind),
parametric=parametrics,
cov_mode=cov_mode,
cov_regularizer=cov_regularizer,
**kwargs)
x = cov_branch
# x = Dense(nb_classes, activation='softmax', name='predictions')(cov_branch)
elif mode == 2:
cov_branch = covariance_block(x,
cov_branch_output,
stage=5,
block=str(ind),
parametric=parametrics,
cov_regularizer=cov_regularizer,
**kwargs)
x = Flatten()(x)
x = Dense(nb_classes, activation='relu', name='fc')(x)
x = merge([x, cov_branch], mode='concat', name='concat')
# x = Dense(nb_classes, activation='softmax', name='predictions')(x)
elif mode == 3:
cov_branch = covariance_block(x,
cov_branch_output,
stage=5,
block=str(ind),
parametric=parametrics,
cov_mode=cov_mode,
cov_regularizer=cov_regularizer,
o2t_constraints='UnitNorm',
**kwargs)
x = cov_branch
cov_outputs.append(x)
if concat == 'concat':
if cov_branch_mode == 'o2t_no_wv' or cov_branch_mode == 'corr_no_wv':
x = MatrixConcat(cov_outputs,
name='Matrix_diag_concat')(cov_outputs)
x = WeightedVectorization(cov_branch_output * nb_branch,
name='WV_big')(x)
else:
x = merge(cov_outputs, mode='concat', name='merge')
elif concat == 'sum':
x = merge(cov_outputs, mode='sum', name='sum')
if cov_branch_mode == 'o2t_no_wv':
x = WeightedVectorization(cov_branch_output, name='wv_sum')(x)
elif concat == 'ave':
x = merge(cov_outputs, mode='ave', name='ave')
if cov_branch_mode == 'o2t_no_wv':
x = WeightedVectorization(cov_branch_output, name='wv_sum')(x)
else:
raise RuntimeError("concat mode not support : " + concat)
if freeze_conv:
toggle_trainable_layers(base_model, not freeze_conv)
# x = Dense(cov_branch_output * nb_branch, activation='relu', name='Dense_b')(x)
x = Dense(nb_classes, activation='softmax')(x)
model = Model(base_model.input, x, name=basename)
return model
def dcov_model_wrapper_v1(base_model,
parametrics=[],
mode=0,
nb_classes=1000,
basename='',
cov_mode='channel',
cov_branch='o2transform',
cov_branch_output=None,
freeze_conv=False,
cov_regularizer=None,
nb_branch=1,
concat='concat',
last_conv_feature_maps=[],
upsample_method='conv',
regroup=False,
**kwargs):
"""
Wrapper for any base model, attach right after the last layer of given model
Parameters
----------
base_model
parametrics
mode
nb_classes
input_shape
load_weights
cov_mode
cov_branch
cov_branch_output
cov_block_mode
last_avg
freeze_conv
Returns
-------
"""
# Function name
covariance_block = get_cov_block(cov_branch)
if cov_branch_output is None:
cov_branch_output = nb_classes
x = base_model.output
x = upsample_wrapper_v1(x,
last_conv_feature_maps,
upsample_method,
kernel=[1, 1])
cov_input = x
if mode == 0:
x = Flatten()(x)
for ind, param in enumerate(parametrics):
x = Dense(param, activation='relu', name='fc{}'.format(ind))(x)
x = Dense(nb_classes, activation='softmax')(x)
if mode == 1:
if nb_branch == 1:
cov_branch = covariance_block(cov_input,
cov_branch_output,
stage=5,
block='a',
parametric=parametrics,
cov_mode=cov_mode,
cov_regularizer=cov_regularizer,
**kwargs)
x = Dense(nb_classes, activation='softmax',
name='predictions')(cov_branch)
elif nb_branch > 1:
pass
elif mode == 2:
cov_branch = covariance_block(cov_input,
cov_branch_output,
stage=5,
block='a',
parametric=parametrics,
cov_regularizer=cov_regularizer,
**kwargs)
x = Flatten()(x)
x = Dense(nb_classes, activation='relu', name='fc')(x)
x = merge([x, cov_branch], mode='concat', name='concat')
x = Dense(nb_classes, activation='softmax', name='predictions')(x)
elif mode == 3:
if nb_branch == 1:
cov_branch = covariance_block(cov_input,
cov_branch_output,
stage=5,
block='a',
parametric=parametrics,
cov_mode=cov_mode,
cov_regularizer=cov_regularizer,
o2t_constraints='UnitNorm',
**kwargs)
x = Dense(nb_classes, activation='softmax',
name='predictions')(cov_branch)
elif nb_branch > 1:
pass
if freeze_conv:
toggle_trainable_layers(base_model, not freeze_conv)
model = Model(base_model.input, x, name=basename)
return model
def covariance_block_sobn_multi_o2t(input_tensor,
nb_class,
stage,
block,
epsilon=0,
parametric=[],
activation='relu',
cov_mode='channel',
cov_regularizer=None,
vectorization=None,
o2t_constraints=None,
nb_o2t=1,
o2t_concat='concat',
so_mode=2,
**kwargs):
if epsilon > 0:
cov_name_base = 'cov' + str(stage) + block + '_branch_epsilon' + str(
epsilon)
else:
cov_name_base = 'cov' + str(stage) + block + '_branch'
o2t_name_base = 'o2t' + str(stage) + block + '_branch'
dense_name_base = 'fc' + str(stage) + block + '_branch'
wp_name_base = 'wp' + str(stage) + block + '_branch'
x = SecondaryStatistic(name=cov_name_base,
eps=epsilon,
cov_mode=cov_mode,
cov_regularizer=cov_regularizer,
**kwargs)(input_tensor)
# Try to implement multiple o2t layers out of the same x.
cov_input = x
cov_br = []
for i in range(nb_o2t):
x = cov_input
for id, param in enumerate(parametric):
x = SecondOrderBatchNormalization(so_mode=so_mode,
momentum=0.8,
axis=-1)(x)
x = O2Transform(param,
activation='relu',
name=o2t_name_base + str(id) + '_' + str(i))(x)
if vectorization == 'wv':
x = WeightedVectorization(nb_class,
activation=activation,
name=wp_name_base + str(id) + '_' +
str(i))(x)
elif vectorization == 'dense':
x = Flatten()(x)
x = Dense(nb_class, activation=activation, name=dense_name_base)(x)
elif vectorization == 'flatten':
x = Flatten()(x)
elif vectorization == 'mat_flatten':
x = FlattenSymmetric()(x)
elif vectorization is None:
pass
else:
ValueError("vectorization parameter not recognized : {}".format(
vectorization))
cov_br.append(x)
if o2t_concat == 'concat' and vectorization is None:
# use matrix concat
x = MatrixConcat(cov_br)(cov_br)
x = WeightedVectorization(nb_class * nb_o2t / 2)(x)
elif o2t_concat == 'concat':
# use vector concat
x = merge(cov_br, mode='concat')
else:
raise NotImplementedError
return x
def dcov_multi_out_model_wrapper(base_model,
parametrics=[],
mode=0,
nb_classes=1000,
basename='',
cov_mode='channel',
cov_branch='o2t_no_wv',
cov_branch_output=None,
freeze_conv=False,
cov_regularizer=None,
nb_branch=1,
concat='concat',
last_conv_feature_maps=[],
upsample_method='conv',
regroup=False,
**kwargs):
"""
Wrapper for any multi output base model, attach right after the last layer of given model
Parameters
----------
base_model
parametrics
mode
nb_classes
input_shape
load_weights
cov_mode
cov_branch
cov_branch_output
cov_block_mode
last_avg
freeze_conv
mode 1: 1x1 reduce dim
Returns
-------
"""
cov_branch_mode = cov_branch
# Function name
covariance_block = get_cov_block(cov_branch)
if cov_branch_output is None:
cov_branch_output = nb_classes
# 256, 512, 512
block1, block2, block3 = outputs = base_model.outputs
print("===================")
cov_outputs = []
if mode == 1:
print("Model design : ResNet_o2_multi_branch 1x1 conv to reduce dim ")
""" 1x1 conv to reduce dim """
# Starting from block3
block3 = upsample_wrapper_v1(block3, [1024, 512])
block2 = upsample_wrapper_v1(block2, [512])
block2 = MaxPooling2D()(block2)
block1 = MaxPooling2D(pool_size=(4, 4))(block1)
outputs = [block1, block2, block3]
for ind, x in enumerate(outputs):
cov_branch = covariance_block(x,
cov_branch_output,
stage=5,
block=str(ind),
parametric=parametrics,
cov_mode=cov_mode,
cov_regularizer=cov_regularizer,
**kwargs)
x = cov_branch
cov_outputs.append(x)
elif mode == 2 or mode == 3:
""" Use branchs to reduce dim """
block3 = SeparateConvolutionFeatures(4)(block3)
block2 = SeparateConvolutionFeatures(2)(block2)
block1 = MaxPooling2D()(block1)
block1 = [block1]
outputs = [block1, block2, block3]
for ind, outs in enumerate(outputs):
block_outs = []
for ind2, x in enumerate(outs):
cov_branch = covariance_block(x,
cov_branch_output,
stage=5,
block=str(ind) + '_' + str(ind2),
parametric=parametrics,
cov_mode=cov_mode,
cov_regularizer=cov_regularizer,
**kwargs)
x = cov_branch
block_outs.append(x)
if mode == 3:
""" Sum block covariance output """
if len(block_outs) > 1:
o = merge(block_outs,
mode='sum',
name='multibranch_sum_{}'.format(ind))
o = WeightedVectorization(cov_branch_output)(o)
cov_outputs.append(o)
else:
a = block_outs[0]
if 'o2t' in a.name:
a = WeightedVectorization(cov_branch_output)(a)
cov_outputs.append(a)
else:
cov_outputs.extend(block_outs)
elif mode == 4:
""" Use the similar structure to Feature Pyramid Network """
# supplimentary stream
block1 = upsample_wrapper_v1(block1, [256], stage='block1')
block2 = upsample_wrapper_v1(block2, [256], stage='block2')
# main stream
block3 = upsample_wrapper_v1(block3, [512], stage='block3')
cov_input = SeparateConvolutionFeatures(nb_branch)(block3)
cov_outputs = []
for ind, x in enumerate(cov_input):
cov_branch = covariance_block(x,
cov_branch_output,
stage=5,
block=str(ind),
parametric=parametrics,
cov_mode=cov_mode,
cov_regularizer=cov_regularizer,
normalization=False,
**kwargs)
x = cov_branch
cov_outputs.append(x)
x = MatrixConcat(cov_outputs, name='Matrix_diag_concat')(cov_outputs)
x = O2Transform(64, activation='relu', name='o2t_mainst_1')(x)
block2 = SecondaryStatistic(name='cov_block2',
cov_mode='pmean',
robust=False,
eps=1e-5)(block2)
block2 = O2Transform(64, activation='relu', name='o2t_block2')(block2)
# fuse = merge([block2, x], mode='sum')
# fuse = O2Transform(64, activation='relu', name='o2t_mainst_2')(fuse)
block1 = SecondaryStatistic(name='cov_block1',
cov_mode='pmean',
robust=False,
eps=1e-5)(block1)
block1 = O2Transform(64, activation='relu', name='o2t_block1')(block1)
# fuse = merge([fuse, block1], mode='sum')
x = MatrixConcat([x, block1, block2],
name='Matrix_diag_concat_all')([x, block1, block2])
x = WeightedVectorization(128, activation='relu', name='wv_fuse')(x)
# Merge the last matrix for matrix concat
if freeze_conv:
toggle_trainable_layers(base_model, not freeze_conv)
x = Dense(nb_classes, activation='softmax')(x)
model = Model(base_model.input, x, name=basename)
return model
input_img = Input(
shape=(1, 28,
28)) # adapt this if using `channels_first` image data format
x = Conv2D(5, (3, 3), activation='relu', padding='same')(input_img)
#x = MaxPooling2D((2, 2), padding='same')(x)
#x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
#x = MaxPooling2D((2, 2), padding='same')(x)
#x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
encoded = AveragePooling2D(pool_size=(2, 1), strides=(2, 1),
padding='valid')(x)
lstmsVec = []
for x in range(0, 5):
filterImg = Lambda(lambda element: element[:, x, :, :])(encoded)
lstmsVec.append(filterImg)
merged = merge(lstmsVec, mode='concat', concat_axis=2)
# at this point the representation is (4, 4, 8) i.e. 128-dimensional
x = Conv2D(5, (3, 3), activation='relu', padding='same')(encoded)
#x = UpSampling2D((2, 2))(x)
#x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
#x = UpSampling2D((2, 2))(x)
#x = Conv2D(16, (3, 3), activation='relu')(x)
x = UpSampling2D((2, 1))(x)
decoded = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(x)
encoder = Model(input_img, encoded)
encoderHstack = Model(input_img, merged)
def test_merge():
# test modes: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot'.
input_shapes = [(3, 2), (3, 2)]
inputs = [np.random.random(shape) for shape in input_shapes]
# test functional API
for mode in ['sum', 'mul', 'concat', 'ave', 'max']:
print(mode)
input_a = layers.Input(shape=input_shapes[0][1:])
input_b = layers.Input(shape=input_shapes[1][1:])
merged = legacy_layers.merge([input_a, input_b], mode=mode)
model = models.Model([input_a, input_b], merged)
model.compile('rmsprop', 'mse')
expected_output_shape = model.compute_output_shape(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = models.Model.from_config(config)
model.compile('rmsprop', 'mse')
# test Merge (#2460)
merged = legacy_layers.Merge(mode=mode)([input_a, input_b])
model = models.Model([input_a, input_b], merged)
model.compile('rmsprop', 'mse')
expected_output_shape = model.compute_output_shape(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
# test lambda with output_shape lambda
input_a = layers.Input(shape=input_shapes[0][1:])
input_b = layers.Input(shape=input_shapes[1][1:])
merged = legacy_layers.merge(
[input_a, input_b],
mode=lambda tup: K.concatenate([tup[0], tup[1]]),
output_shape=lambda tup: tup[0][:-1] + (tup[0][-1] + tup[1][-1],))
model = models.Model([input_a, input_b], merged)
expected_output_shape = model.compute_output_shape(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = models.Model.from_config(config)
model.compile('rmsprop', 'mse')
# test function with output_shape function
def fn_mode(tup):
x, y = tup
return K.concatenate([x, y], axis=1)
def fn_output_shape(tup):
s1, s2 = tup
return (s1[0], s1[1] + s2[1]) + s1[2:]
input_a = layers.Input(shape=input_shapes[0][1:])
input_b = layers.Input(shape=input_shapes[1][1:])
merged = legacy_layers.merge([input_a, input_b],
mode=fn_mode,
output_shape=fn_output_shape)
model = models.Model([input_a, input_b], merged)
expected_output_shape = model.compute_output_shape(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = models.Model.from_config(config)
model.compile('rmsprop', 'mse')
# test function with output_mask function
# time dimension is required for masking
input_shapes = [(4, 3, 2), (4, 3, 2)]
inputs = [np.random.random(shape) for shape in input_shapes]
def fn_output_mask(tup):
x_mask, y_mask = tup
return K.concatenate([x_mask, y_mask])
input_a = layers.Input(shape=input_shapes[0][1:])
input_b = layers.Input(shape=input_shapes[1][1:])
a = layers.Masking()(input_a)
b = layers.Masking()(input_b)
merged = legacy_layers.merge([a, b], mode=fn_mode, output_shape=fn_output_shape, output_mask=fn_output_mask)
model = models.Model([input_a, input_b], merged)
expected_output_shape = model.compute_output_shape(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = models.Model.from_config(config)
model.compile('rmsprop', 'mse')
mask_inputs = (np.zeros(input_shapes[0][:-1]), np.ones(input_shapes[1][:-1]))
expected_mask_output = np.concatenate(mask_inputs, axis=-1)
mask_input_placeholders = [K.placeholder(shape=input_shape[:-1]) for input_shape in input_shapes]
mask_output = model.layers[-1]._output_mask(mask_input_placeholders)
assert np.all(K.function(mask_input_placeholders, [mask_output])(mask_inputs)[0] == expected_mask_output)
# test lambda with output_mask lambda
input_a = layers.Input(shape=input_shapes[0][1:])
input_b = layers.Input(shape=input_shapes[1][1:])
a = layers.Masking()(input_a)
b = layers.Masking()(input_b)
merged = legacy_layers.merge(
[a, b], mode=lambda tup: K.concatenate([tup[0], tup[1]], axis=1),
output_shape=lambda tup: (tup[0][0], tup[0][1] + tup[1][1]) + tup[0][2:],
output_mask=lambda tup: K.concatenate([tup[0], tup[1]]))
model = models.Model([input_a, input_b], merged)
expected_output_shape = model.compute_output_shape(input_shapes)
actual_output_shape = model.predict(inputs).shape
assert expected_output_shape == actual_output_shape
config = model.get_config()
model = models.Model.from_config(config)
model.compile('rmsprop', 'mse')
mask_output = model.layers[-1]._output_mask(mask_input_placeholders)
assert np.all(K.function(mask_input_placeholders, [mask_output])(mask_inputs)[0] == expected_mask_output)
# test with arguments
input_shapes = [(3, 2), (3, 2)]
inputs = [np.random.random(shape) for shape in input_shapes]
def fn_mode(tup, a, b):
x, y = tup
return x * a + y * b
input_a = layers.Input(shape=input_shapes[0][1:])
input_b = layers.Input(shape=input_shapes[1][1:])
merged = legacy_layers.merge([input_a, input_b], mode=fn_mode, output_shape=lambda s: s[0], arguments={'a': 0.7, 'b': 0.3})
model = models.Model([input_a, input_b], merged)
output = model.predict(inputs)
config = model.get_config()
model = models.Model.from_config(config)
assert np.all(model.predict(inputs) == output)
