def call(self, inputs):
PMW_Score = tf.nn.conv2d(inputs,
self.PWM,
strides=[1, 1, 1, 1],
padding='VALID')
PMWrc_Score = tf.nn.conv2d(inputs,
self.PWMrc,
strides=[1, 1, 1, 1],
padding='VALID')
Indicator_f = K.cast(K.greater(PMW_Score, self.score_cut),
self.dtype_now)
Indicator_r = K.cast(K.greater(PMWrc_Score, self.score_cut),
self.dtype_now)
Indicator = Maximum()([Indicator_r, Indicator_f])
S_relu = Maximum()([PMW_Score, PMWrc_Score])
S_i_S_max = S_relu - self.max_s
S_i_S_max_lam = S_i_S_max * self.kernel_1
K_i_m_n = tf.math.exp(S_i_S_max_lam)
K_relu = K_i_m_n * Indicator
Ko_relu = tf.pad(K_relu, self.paddings, 'CONSTANT') * self.kernel_2
return Ko_relu
def get_convo_nn2(no_word=200, n_gram=21, no_char=178):
input1 = Input(shape=(n_gram, ))
input2 = Input(shape=(n_gram, ))
a = Embedding(no_char, 32, input_length=n_gram)(input1)
a = SpatialDropout1D(0.15)(a)
a = BatchNormalization()(a)
a_concat = []
for i in range(1, 9):
a_concat.append(conv_unit(a, n_gram, no_word, window=i))
for i in range(9, 12):
a_concat.append(conv_unit(a, n_gram, no_word - 50, window=i))
a_concat.append(conv_unit(a, n_gram, no_word - 100, window=12))
a_sum = Maximum()(a_concat)
b = Embedding(12, 12, input_length=n_gram)(input2)
b = SpatialDropout1D(0.15)(b)
x = Concatenate(axis=-1)([a, a_sum, b])
#x = Concatenate(axis=-1)([a_sum, b])
x = BatchNormalization()(x)
x = Flatten()(x)
x = Dense(100, activation='relu')(x)
out = Dense(1, activation='sigmoid')(x)
model = Model(inputs=[input1, input2], outputs=out)
model.compile(optimizer=Adam(),
loss='binary_crossentropy',
metrics=['acc'])
return model
def phrase_embeddings_graph(question_features, dropout_rate, embedding_size=512):
"""
Three n-gram models are approximate with convolutions of kernel sizes 1, 2, 3.
These are applied on the word embeddings with 'same' padding. Then from these
convolution results of 'embedding size' the maximum is chosen, which again
results in a tensor of 'embedding size' containing the maximum embedding value
from the n-grams.
For example, the convolutions may result in three tensors of
unigram: (1, 6, 512) -> with value -0.06274356 for the first word
bigram : (1, 6, 512) -> with value -0.31123462 for the first word
trigram: (1, 6, 512) -> with value 1.3757347 for the first word
then the maximum value is chosen over all n-grams
maximum: (1, 6, 512) -> with value 1.3757347 for the first word
This is similar to 1d pooling, but over an extra dimension (which is not possible with keras).
A note on the activation:
It remains unclear, when the original paper is performing the activation.
In the paper they mention directly at the convolution, but in the implementation
they perform the activation only after the max pooling. This might be a performance
tweak, because they choose the maximum values and only have to perform the
activation on the maximum values and not each convolution output (reduction by 3).
Nevertheless, the it remains unclear, if the network would learn a better
convolution, when the activation is applied right after instead of a linear one.
"""
ugraph = Conv1D(embedding_size, kernel_size=1, name="phrase_unigram")(question_features)
bgraph = Conv1D(embedding_size, kernel_size=2, padding="same", name="phrase_bigram")(question_features)
tgraph = Conv1D(embedding_size, kernel_size=3, padding="same", name="phrase_trigram")(question_features)
pgraph = Maximum(name="phrase_max")([ugraph, bgraph, tgraph])
pgraph = Activation("tanh", name="phrase_activation")(pgraph)
pgraph = Dropout(rate=dropout_rate, name="phrase_dropout")(pgraph)
return pgraph
def _build_model(self):
# Neural Net for Deep-Q learning
input_ = Input(shape=(self.board_size,self.board_size,5))
print("input:",input_.shape)
red = Maximum()([input_[:,:,:,0],input_[:,:,:,1]])
print("red:",red.shape)
#blue = Maximum()([input_[:,:,2],input_[:,:,3]])
conv_1 = Conv2D(32, kernel_size=3, activation='relu')(input_)
conv_2 = Conv2D(32, kernel_size=3, activation='relu')(conv_1)
deconv_3 = Conv2DTranspose(32, kernel_size=3, activation='relu')(conv_2)
deconv_4 = Conv2DTranspose(32, kernel_size=3, activation='relu')(deconv_3)
flatten_1 = Flatten()(deconv_4)
print("flatten:", flatten_1.shape)
dense_6a = Dense(self.board_size**2, activation='relu')(flatten_1[:,0:self.board_size**2])
dense_6b = Dense(self.board_size**2, activation='relu')(flatten_1[:,self.board_size**2:])
print("dense:", dense_6b.shape)
reshape_7a = Reshape((self.board_size, self.board_size))(dense_6a)
print("reshape a:", reshape_7a.shape)
reshape_7b = Reshape((self.board_size, self.board_size,1))(dense_6b)
min_a = Minimum()([reshape_7a, red])
flatten_2a = Flatten()(min_a)
softmax_a = Softmax(axis=1)(flatten_2a)
flatten_2b = Flatten()(reshape_7b)
softmax_b = Softmax(axis=1)(flatten_2b)
reshape_a = Reshape((self.board_size, self.board_size,1))(softmax_a)
reshape_b = Reshape((self.board_size, self.board_size,1))(softmax_b)
concat = Concatenate(axis=-1)([reshape_a, reshape_b])
model = Model(inputs=input_, outputs=concat)
model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate))
return model
def classify(**args):
"""
Main method that prepares dataset, builds model, executes training and displays results.
:param args: keyword arguments passed from cli parser
"""
# only allow print-outs if execution has no repetitions
allow_print = args['repetitions'] == 1
# determine classification targets and parameters to construct datasets properly
cls_target, cls_str = set_classification_targets(args['cls_choice'])
d = prepare_dataset(args['dataset_choice'], cls_target, args['batch_size'])
print('\n\tTask: Classify «{}» using «{}»\n'.format(
cls_str, d['data_str']))
print_dataset_info(d)
# build and train
models = list()
inputs = Input(shape=(7810, ))
for i in range(args['num_models']):
print(f'\nTrain model {i+1} ...')
model = build_model(i, d['num_classes'], inputs=inputs)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(d['train_data'],
steps_per_epoch=d['train_steps'],
epochs=args['epochs'],
verbose=1,
class_weight=d['class_weights'])
models.append(model)
# combine outputs of all previous models, treat outputs
multi_output = [m.outputs[0] for m in models]
y = Maximum()(multi_output)
model = Model(inputs, outputs=y, name='ensemble')
if allow_print:
plot_model(model, to_file='img/ensemble_mlp.png')
# compile and evaluation model
print('Evaluate ...')
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.evaluate(d['eval_data'], steps=d['test_steps'], verbose=1)
# predict on testset and calculate classification report and confusion matrix for diagnosis
print('Test ...')
pred = model.predict(d['test_data'], steps=d['test_steps'])
if allow_print:
diagnose_output(d['test_labels'], pred.argmax(axis=1),
d['classes_trans'])
return balanced_accuracy_score(d['test_labels'], pred.argmax(axis=1))
def build_generator(self):
example = Input(shape=(self.apifeature_dims, ))
noise = Input(shape=(self.z_dims, ))
x = Concatenate(axis=1)([example, noise])
for dim in self.generator_layers[1:]:
x = Dense(dim)(x)
x = Activation(activation='tanh')(x)
x = Maximum()([example, x])
generator = Model([example, noise], x, name='generator')
generator.summary()
return generator
def fusion_block(tensors, name, type='add'):
if (type == 'add'):
return Add(name='add_' + name)(tensors)
if (type == 'max'):
return Maximum(name='max_' + name)(tensors)
if (type == 'con'):
return Concatenate(name='conc_' + name)(tensors)
if (type == 'avg'):
return Average(name='avg_' + name)(tensors)
def createMultiModelMaximum(max_seq_len, bert_ckpt_file, bert_config_file,
NUM_CLASS):
with GFile(bert_config_file, "r") as reader:
bc = StockBertConfig.from_json_string(reader.read())
bert_params = map_stock_config_to_params(bc)
bert_params.adapter_size = None
bert_layer = BertModelLayer.from_params(bert_params, name="bert")
bert_in = Input(shape=(max_seq_len, ),
dtype='int32',
name="input_ids_bert")
bert_inter = bert_layer(bert_in)
cls_out = Lambda(lambda seq: seq[:, 0, :])(bert_inter)
cls_out = Dropout(0.5)(cls_out)
bert_out = Dense(units=768, activation="tanh")(cls_out) # 768 before
load_stock_weights(bert_layer, bert_ckpt_file)
# image models:
inceptionv3 = InceptionV3(weights='imagenet', include_top=False)
resnet50 = ResNet50(weights='imagenet', include_top=False)
res_out = resnet50.output
res_out = GlobalAveragePooling2D()(res_out)
res_out = Dropout(0.5)(res_out)
res_out = Dense(2048)(res_out)
res_out = Dropout(0.5)(res_out)
res_out = Dense(768)(res_out)
inc_out = inceptionv3.output
inc_out = GlobalAveragePooling2D()(inc_out)
inc_out = Dropout(0.5)(inc_out)
inc_out = Dense(2048)(inc_out)
inc_out = Dropout(0.5)(inc_out)
inc_out = Dense(768)(inc_out)
# merge = Concatenate()([res_out, inc_out, bert_out])
merge = Maximum()([res_out, inc_out, bert_out])
# restliche Layer
x = Dense(2048)(merge)
x = Dropout(0.5)(x)
x = Dense(1024)(x)
x = Dropout(0.5)(x)
x = Dense(512)(x)
x = Dropout(0.5)(x)
output = Dense(NUM_CLASS, activation='softmax', name='output_layer')(x)
model = Model(inputs=[resnet50.input, inceptionv3.input, bert_in],
outputs=output)
plot_model(model,
to_file='multiple_inputs_text.png',
show_shapes=True,
dpi=600,
expand_nested=False)
return model, 17
def model():
up_0 = Input(shape=input_shape, name='up_stream_0')
up_1 = Input(shape=input_shape, name='up_stream_1')
down_0 = Input(shape=input_shape, name='down_stream_0')
down_1 = Input(shape=input_shape, name='down_stream_1')
up_stream = share_stream(x_shape=input_shape)
down_stream = share_stream(x_shape=input_shape)
up_feature_0 = up_stream(up_0)
up_feature_1 = up_stream(up_1)
down_feature_0 = down_stream(down_0)
down_feature_1 = down_stream(down_1)
up_feature_0 = Flatten()(up_feature_0)
up_feature_1 = Flatten()(up_feature_1)
down_feature_0 = Flatten()(down_feature_0)
down_feature_1 = Flatten()(down_feature_1)
up_feature = Maximum()([up_feature_0, up_feature_1]) # element wise maximum among two skeleton in case of multiperson maxout
down_feature = Maximum()([down_feature_0, down_feature_1]) # element wise maximum among two skeleton in case of multiperson maxout
feature = concatenate([up_feature, down_feature])
fc_1 = Dense(units=256, activation='relu', use_bias=True, kernel_regularizer=l2(0.001))(feature)
fc_1 = Dropout(0.5)(fc_1)
fc_2 = Dense(units=128, activation='relu', use_bias=True)(fc_1)
fc_3 = Dense(units=96, activation='relu', use_bias=True)(fc_2)
fc_4 = Dense(units=60, activation='softmax', use_bias=True)(fc_3)
#network = Model(input=[up_0, up_1, down_0, down_1], outputs=fc_4)
#network = tf.keras.Model(input=[up_0, up_1, down_0, down_1], outputs=fc_4)
network = tf.keras.Model(inputs=[up_0, up_1, down_0, down_1], outputs=fc_4)
return network
def _mfm(self, X, name, out_channels, kernel_size=3, strides=1, dense=False):
"""
private func for creating mfm layer.
Todo:
* maybe more natural if implemented as custom layer like the comment out code at the bottom of this file.
"""
if dense:
X = Dense(out_channels*2, name = name + '_dense1', kernel_regularizer=regularizers.l2(0.0005))(X)
else:
X = Conv2D(out_channels*2, name = name + '_conv2d1', kernel_size=kernel_size, kernel_regularizer=regularizers.l2(0.0005), strides=strides, padding='same')(X)
X = Maximum()([Lambda(lambda x, c: x[..., :c], arguments={'c':out_channels})(X), Lambda(lambda x, c: x[..., c:], arguments={'c':out_channels})(X)])
return X
def get_convo_nn2(no_word=200, n_gram=21, no_char=178):
input1 = Input(shape=(n_gram, ))
input2 = Input(shape=(n_gram, ))
a = Embedding(no_char, 32, input_length=n_gram)(input1)
a = SpatialDropout1D(0.15)(a)
a = BatchNormalization()(a)
a_concat = []
for i in range(1, 9):
a_concat.append(conv_unit(a, n_gram, no_word, window=i))
for i in range(9, 12):
a_concat.append(conv_unit(a, n_gram, no_word - 50, window=i))
a_concat.append(conv_unit(a, n_gram, no_word - 100, window=12))
a_sum = Maximum()(a_concat)
b = Embedding(12, 12, input_length=n_gram)(input2)
b = SpatialDropout1D(0.15)(b)
x = Concatenate(axis=-1)([a, a_sum, b])
#x = Concatenate(axis=-1)([a_sum, b])
x = BatchNormalization()(x)
x = Flatten()(x)
x = Dense(100, activation='relu')(x)
########################
#out = Dense(1, activation='sigmoid')(x)
# crf = CRF(n_gram) ## ???????
# out = crf(x)
# out = x.add(CRF(100))
crf = CRF(2, sparse_target=False) # num_label
loss = crf.loss_function
out = crf(x)
##########################
model = Model(inputs=[input1, input2], outputs=out)
####################
# model.compile(optimizer=Adam(),
# loss='binary_crossentropy', metrics=['acc'])
model.compile(optimizer="adam", loss=loss)
#####################
return model
def __init__(self, settings, svcnns):
super(MultiViewCNN, self).__init__()
self.merge_mode = settings.merge_mode
self.svcnns = svcnns
self.model_inputs = []
self.svcnn_outs = []
for svcnn in svcnns:
self.model_inputs.append(svcnn.model.input)
self.svcnn_outs.append(svcnn.model.output)
# region View Pooling & Output Layers
if self.merge_mode == 'max':
self.view_pooling_layer = Maximum(name='view_pool_max')(
self.svcnn_outs)
elif self.merge_mode == 'concat':
self.view_pooling_layer = Concatenate(axis=1,
name='view_pool_concat')(
self.svcnn_outs)
self.x = Dense(512, name='mvcnn_fc1')(self.view_pooling_layer)
self.model_output = Dense(settings.num_classes,
activation='softmax',
name='mvcnn_output')(self.x)
self.model = Model(inputs=self.model_inputs, outputs=self.model_output)
def get_test_model_exhaustive():
"""Returns a exhaustive test model."""
input_shapes = [(2, 3, 4, 5, 6), (2, 3, 4, 5, 6), (7, 8, 9, 10),
(7, 8, 9, 10), (11, 12, 13), (11, 12, 13), (14, 15),
(14, 15), (16, ),
(16, ), (2, ), (1, ), (2, ), (1, ), (1, 3), (1, 4),
(1, 1, 3), (1, 1, 4), (1, 1, 1, 3), (1, 1, 1, 4),
(1, 1, 1, 1, 3), (1, 1, 1, 1, 4), (26, 28, 3), (4, 4, 3),
(4, 4, 3), (4, ), (2, 3), (1, ), (1, ), (1, ), (2, 3),
(9, 16, 1), (1, 9, 16)]
inputs = [Input(shape=s) for s in input_shapes]
outputs = []
outputs.append(Conv1D(1, 3, padding='valid')(inputs[6]))
outputs.append(Conv1D(2, 1, padding='same')(inputs[6]))
outputs.append(Conv1D(3, 4, padding='causal', dilation_rate=2)(inputs[6]))
outputs.append(ZeroPadding1D(2)(inputs[6]))
outputs.append(Cropping1D((2, 3))(inputs[6]))
outputs.append(MaxPooling1D(2)(inputs[6]))
outputs.append(MaxPooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(MaxPooling1D(2, data_format="channels_first")(inputs[6]))
outputs.append(AveragePooling1D(2)(inputs[6]))
outputs.append(AveragePooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(
AveragePooling1D(2, data_format="channels_first")(inputs[6]))
outputs.append(GlobalMaxPooling1D()(inputs[6]))
outputs.append(GlobalMaxPooling1D(data_format="channels_first")(inputs[6]))
outputs.append(GlobalAveragePooling1D()(inputs[6]))
outputs.append(
GlobalAveragePooling1D(data_format="channels_first")(inputs[6]))
outputs.append(Conv2D(4, (3, 3))(inputs[4]))
outputs.append(Conv2D(4, (3, 3), use_bias=False)(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4]))
outputs.append(SeparableConv2D(3, (3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((1, 2))(inputs[4]))
outputs.append(MaxPooling2D((2, 2))(inputs[4]))
# todo: check if TensorFlow >= 2.1 supports this
# outputs.append(MaxPooling2D((2, 2), data_format="channels_first")(inputs[4])) # Default MaxPoolingOp only supports NHWC on device type CPU
outputs.append(
MaxPooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(AveragePooling2D((2, 2))(inputs[4]))
# todo: check if TensorFlow >= 2.1 supports this
# outputs.append(AveragePooling2D((2, 2), data_format="channels_first")(inputs[4])) # Default AvgPoolingOp only supports NHWC on device type CPU
outputs.append(
AveragePooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(GlobalAveragePooling2D()(inputs[4]))
outputs.append(
GlobalAveragePooling2D(data_format="channels_first")(inputs[4]))
outputs.append(GlobalMaxPooling2D()(inputs[4]))
outputs.append(GlobalMaxPooling2D(data_format="channels_first")(inputs[4]))
outputs.append(Permute((3, 4, 1, 5, 2))(inputs[0]))
outputs.append(Permute((1, 5, 3, 2, 4))(inputs[0]))
outputs.append(Permute((3, 4, 1, 2))(inputs[2]))
outputs.append(Permute((2, 1, 3))(inputs[4]))
outputs.append(Permute((2, 1))(inputs[6]))
outputs.append(Permute((1, ))(inputs[8]))
outputs.append(Permute((3, 1, 2))(inputs[31]))
outputs.append(Permute((3, 1, 2))(inputs[32]))
outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[31])))
outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[32])))
outputs.append(BatchNormalization()(inputs[0]))
outputs.append(BatchNormalization(axis=1)(inputs[0]))
outputs.append(BatchNormalization(axis=2)(inputs[0]))
outputs.append(BatchNormalization(axis=3)(inputs[0]))
outputs.append(BatchNormalization(axis=4)(inputs[0]))
outputs.append(BatchNormalization(axis=5)(inputs[0]))
outputs.append(BatchNormalization()(inputs[2]))
outputs.append(BatchNormalization(axis=1)(inputs[2]))
outputs.append(BatchNormalization(axis=2)(inputs[2]))
outputs.append(BatchNormalization(axis=3)(inputs[2]))
outputs.append(BatchNormalization(axis=4)(inputs[2]))
outputs.append(BatchNormalization()(inputs[4]))
# todo: check if TensorFlow >= 2.1 supports this
# outputs.append(BatchNormalization(axis=1)(inputs[4])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now.
outputs.append(BatchNormalization(axis=2)(inputs[4]))
outputs.append(BatchNormalization(axis=3)(inputs[4]))
outputs.append(BatchNormalization()(inputs[6]))
outputs.append(BatchNormalization(axis=1)(inputs[6]))
outputs.append(BatchNormalization(axis=2)(inputs[6]))
outputs.append(BatchNormalization()(inputs[8]))
outputs.append(BatchNormalization(axis=1)(inputs[8]))
outputs.append(BatchNormalization()(inputs[27]))
outputs.append(BatchNormalization(axis=1)(inputs[27]))
outputs.append(BatchNormalization()(inputs[14]))
outputs.append(BatchNormalization(axis=1)(inputs[14]))
outputs.append(BatchNormalization(axis=2)(inputs[14]))
outputs.append(BatchNormalization()(inputs[16]))
# todo: check if TensorFlow >= 2.1 supports this
# outputs.append(BatchNormalization(axis=1)(inputs[16])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now.
outputs.append(BatchNormalization(axis=2)(inputs[16]))
outputs.append(BatchNormalization(axis=3)(inputs[16]))
outputs.append(BatchNormalization()(inputs[18]))
outputs.append(BatchNormalization(axis=1)(inputs[18]))
outputs.append(BatchNormalization(axis=2)(inputs[18]))
outputs.append(BatchNormalization(axis=3)(inputs[18]))
outputs.append(BatchNormalization(axis=4)(inputs[18]))
outputs.append(BatchNormalization()(inputs[20]))
outputs.append(BatchNormalization(axis=1)(inputs[20]))
outputs.append(BatchNormalization(axis=2)(inputs[20]))
outputs.append(BatchNormalization(axis=3)(inputs[20]))
outputs.append(BatchNormalization(axis=4)(inputs[20]))
outputs.append(BatchNormalization(axis=5)(inputs[20]))
outputs.append(Dropout(0.5)(inputs[4]))
outputs.append(ZeroPadding2D(2)(inputs[4]))
outputs.append(ZeroPadding2D((2, 3))(inputs[4]))
outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Cropping2D(2)(inputs[4]))
outputs.append(Cropping2D((2, 3))(inputs[4]))
outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Dense(3, use_bias=True)(inputs[13]))
outputs.append(Dense(3, use_bias=True)(inputs[14]))
outputs.append(Dense(4, use_bias=False)(inputs[16]))
outputs.append(Dense(4, use_bias=False, activation='tanh')(inputs[18]))
outputs.append(Dense(4, use_bias=False)(inputs[20]))
outputs.append(Reshape(((2 * 3 * 4 * 5 * 6), ))(inputs[0]))
outputs.append(Reshape((2, 3 * 4 * 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4 * 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4, 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4, 5, 6))(inputs[0]))
outputs.append(Reshape((16, ))(inputs[8]))
outputs.append(Reshape((2, 8))(inputs[8]))
outputs.append(Reshape((2, 2, 4))(inputs[8]))
outputs.append(Reshape((2, 2, 2, 2))(inputs[8]))
outputs.append(Reshape((2, 2, 1, 2, 2))(inputs[8]))
outputs.append(RepeatVector(3)(inputs[8]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='bilinear')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='bilinear')(inputs[4]))
outputs.append(ReLU()(inputs[0]))
for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]:
outputs.append(Concatenate(axis=axis)([inputs[0], inputs[1]]))
for axis in [-4, -3, -2, -1, 1, 2, 3, 4]:
outputs.append(Concatenate(axis=axis)([inputs[2], inputs[3]]))
for axis in [-3, -2, -1, 1, 2, 3]:
outputs.append(Concatenate(axis=axis)([inputs[4], inputs[5]]))
for axis in [-2, -1, 1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[6], inputs[7]]))
for axis in [-1, 1]:
outputs.append(Concatenate(axis=axis)([inputs[8], inputs[9]]))
for axis in [-1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]]))
for axis in [-1, 3]:
outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]]))
for axis in [-1, 4]:
outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]]))
for axis in [-1, 5]:
outputs.append(Concatenate(axis=axis)([inputs[20], inputs[21]]))
outputs.append(UpSampling1D(size=2)(inputs[6]))
# outputs.append(UpSampling1D(size=2)(inputs[8])) # ValueError: Input 0 of layer up_sampling1d_1 is incompatible with the layer: expected ndim=3, found ndim=2. Full shape received: [None, 16]
outputs.append(Multiply()([inputs[10], inputs[11]]))
outputs.append(Multiply()([inputs[11], inputs[10]]))
outputs.append(Multiply()([inputs[11], inputs[13]]))
outputs.append(Multiply()([inputs[10], inputs[11], inputs[12]]))
outputs.append(Multiply()([inputs[11], inputs[12], inputs[13]]))
shared_conv = Conv2D(1, (1, 1),
padding='valid',
name='shared_conv',
activation='relu')
up_scale_2 = UpSampling2D((2, 2))
x1 = shared_conv(up_scale_2(inputs[23])) # (1, 8, 8)
x2 = shared_conv(up_scale_2(inputs[24])) # (1, 8, 8)
x3 = Conv2D(1, (1, 1),
padding='valid')(up_scale_2(inputs[24])) # (1, 8, 8)
x = Concatenate()([x1, x2, x3]) # (3, 8, 8)
outputs.append(x)
x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8)
outputs.append(x)
x = Dropout(0.5)(x)
outputs.append(x)
x = Concatenate()([MaxPooling2D((2, 2))(x),
AveragePooling2D((2, 2))(x)]) # (6, 4, 4)
outputs.append(x)
x = Flatten()(x) # (1, 1, 96)
x = Dense(4, use_bias=False)(x)
outputs.append(x)
x = Dense(3)(x) # (1, 1, 3)
outputs.append(x)
outputs.append(Add()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Subtract()([inputs[26], inputs[30]]))
outputs.append(Multiply()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Average()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Maximum()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Concatenate()([inputs[26], inputs[30], inputs[30]]))
intermediate_input_shape = (3, )
intermediate_in = Input(intermediate_input_shape)
intermediate_x = intermediate_in
intermediate_x = Dense(8)(intermediate_x)
intermediate_x = Dense(5, name='duplicate_layer_name')(intermediate_x)
intermediate_model = Model(inputs=[intermediate_in],
outputs=[intermediate_x],
name='intermediate_model')
intermediate_model.compile(loss='mse', optimizer='nadam')
x = intermediate_model(x) # (1, 1, 5)
intermediate_model_2 = Sequential()
intermediate_model_2.add(Dense(7, input_shape=(5, )))
intermediate_model_2.add(Dense(5, name='duplicate_layer_name'))
intermediate_model_2.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_2(x) # (1, 1, 5)
intermediate_model_3_nested = Sequential()
intermediate_model_3_nested.add(Dense(7, input_shape=(6, )))
intermediate_model_3_nested.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
intermediate_model_3 = Sequential()
intermediate_model_3.add(Dense(6, input_shape=(5, )))
intermediate_model_3.add(intermediate_model_3_nested)
intermediate_model_3.add(Dense(8))
intermediate_model_3.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_3(x) # (1, 1, 8)
x = Dense(3)(x) # (1, 1, 3)
shared_activation = Activation('tanh')
outputs = outputs + [
Activation('tanh')(inputs[25]),
Activation('hard_sigmoid')(inputs[25]),
Activation('selu')(inputs[25]),
Activation('sigmoid')(inputs[25]),
Activation('softplus')(inputs[25]),
Activation('softmax')(inputs[25]),
Activation('relu')(inputs[25]),
Activation('relu6')(inputs[25]),
Activation('swish')(inputs[25]),
Activation('exponential')(inputs[25]),
Activation('gelu')(inputs[25]),
Activation('softsign')(inputs[25]),
LeakyReLU()(inputs[25]),
ReLU()(inputs[25]),
ReLU(max_value=0.4, negative_slope=1.1, threshold=0.3)(inputs[25]),
ELU()(inputs[25]),
PReLU()(inputs[24]),
PReLU()(inputs[25]),
PReLU()(inputs[26]),
shared_activation(inputs[25]),
Activation('linear')(inputs[26]),
Activation('linear')(inputs[23]),
x,
shared_activation(x),
]
model = Model(inputs=inputs, outputs=outputs, name='test_model_exhaustive')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 2
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, epochs=10)
return model
Input, Subtract, Multiply, Average, Maximum, Minimum, LeakyReLU, PReLU, ELU, ThresholdedReLU, Softmax, \
ZeroPadding1D, ZeroPadding2D, ZeroPadding3D, Reshape
DEFAULT_TF_GRAPH = tf.get_default_graph()
@pytest.mark.parametrize('layer, input_layer, batch_size', [
(Concatenate(), [Input((6, 6, 5)), Input((6, 6, 5))], 32),
(Add(), [Input((6, 6, 5)), Input((6, 6, 5))], 32),
(Add(), [Input((6, 6, 5)), Input((1, 1, 5))], 32),
(Subtract(), [Input((6, 6, 5)), Input((6, 6, 5))], 32),
(Multiply(), [Input((6, 6, 5)), Input((6, 6, 5))], 32),
(Average(), [Input(
(6, 6, 5)), Input(
(6, 6, 5)), Input((6, 6, 5))], 32),
(Maximum(), [Input((6, 6, 5)), Input((6, 6, 5))], 32),
(Minimum(), [Input((6, 6, 5)), Input((6, 6, 5))], 32),
(Activation('relu'), Input((6, 6, 5)), 32),
(Reshape((36, 5)), Input((6, 6, 5)), 32),
(Reshape((20, 6, 5)), Input((6, 5, 4, 5)), 32),
(GatherLayer([1, 4, 2], axis=1), Input((6, 5)), 32),
(GatherLayer([1, 4, 2], axis=2), Input((6, 5)), 32),
(GatherLayer([1, 4, 2], axis=2), Input((6, 7, 5)), 32),
(LeakyReLU(), Input((6, 6, 5)), 32),
(ELU(), Input((6, 6, 5)), 32),
(ThresholdedReLU(), Input((6, 6, 5)), 32),
(Softmax(), Input((6, 6, 5)), 32),
(DownShiftLayer(), Input((6, 6, 5)), 32),
(DownShiftLayer(), Input((6, 5)), 32),
(RightShiftLayer(), Input((6, 6, 5)), 32),
(VectorToComplexNumber(), Input((6, 6, 2)), 32),
Discriminator = Model(outputs=Score, inputs=_input) Discriminator.trainable = True Discriminator.compile(loss='mse', optimizer='adam') #### Combine the two networks to become MetricGAN Discriminator.trainable = False Clean_reference = Input(shape=(257,None,1)) Noisy_LP = Input(shape=(257,None,1)) Min_mask = Input(shape=(257,None,1)) Reshape_de_model_output=Reshape((257, -1, 1))(de_model.output) Mask=Maximum()([Reshape_de_model_output, Min_mask]) Enhanced = Multiply()([Mask, Noisy_LP]) Discriminator_input= Concatenate(axis=-1)([Enhanced, Clean_reference]) # Here the input of Discriminator is (Noisy, Clean) pair, so a clean reference is needed!! Predicted_score=Discriminator(Discriminator_input) MetricGAN= Model(inputs=[de_model.input, Noisy_LP, Clean_reference, Min_mask], outputs=Predicted_score) MetricGAN.compile(loss='mse', optimizer='adam') ######## Model define end ######### Test_PESQ=[] Test_STOI=[] Test_Predicted_STOI_list=[] Train_Predicted_STOI_list=[] Previous_Discriminator_training_list=[]
def get_test_model_exhaustive():
"""Returns a exhaustive test model."""
input_shapes = [
(2, 3, 4, 5, 6),
(2, 3, 4, 5, 6),
(7, 8, 9, 10),
(7, 8, 9, 10),
(11, 12, 13),
(11, 12, 13),
(14, 15),
(14, 15),
(16, ),
(16, ),
(2, ),
(1, ),
(2, ),
(1, ),
(1, 3),
(1, 4),
(1, 1, 3),
(1, 1, 4),
(1, 1, 1, 3),
(1, 1, 1, 4),
(1, 1, 1, 1, 3),
(1, 1, 1, 1, 4),
(26, 28, 3),
(4, 4, 3),
(4, 4, 3),
(4, ),
(2, 3),
(1, ),
(1, ),
(1, ),
(2, 3),
]
inputs = [Input(shape=s) for s in input_shapes]
outputs = []
outputs.append(Conv1D(1, 3, padding='valid')(inputs[6]))
outputs.append(Conv1D(2, 1, padding='same')(inputs[6]))
outputs.append(Conv1D(3, 4, padding='causal', dilation_rate=2)(inputs[6]))
outputs.append(ZeroPadding1D(2)(inputs[6]))
outputs.append(Cropping1D((2, 3))(inputs[6]))
outputs.append(MaxPooling1D(2)(inputs[6]))
outputs.append(MaxPooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(AveragePooling1D(2)(inputs[6]))
outputs.append(AveragePooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(GlobalMaxPooling1D()(inputs[6]))
outputs.append(GlobalMaxPooling1D(data_format="channels_first")(inputs[6]))
outputs.append(GlobalAveragePooling1D()(inputs[6]))
outputs.append(
GlobalAveragePooling1D(data_format="channels_first")(inputs[6]))
outputs.append(Conv2D(4, (3, 3))(inputs[4]))
outputs.append(Conv2D(4, (3, 3), use_bias=False)(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4]))
outputs.append(SeparableConv2D(3, (3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((1, 2))(inputs[4]))
outputs.append(MaxPooling2D((2, 2))(inputs[4]))
outputs.append(
MaxPooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(AveragePooling2D((2, 2))(inputs[4]))
outputs.append(
AveragePooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(GlobalAveragePooling2D()(inputs[4]))
outputs.append(
GlobalAveragePooling2D(data_format="channels_first")(inputs[4]))
outputs.append(GlobalMaxPooling2D()(inputs[4]))
outputs.append(GlobalMaxPooling2D(data_format="channels_first")(inputs[4]))
outputs.append(BatchNormalization()(inputs[4]))
outputs.append(Dropout(0.5)(inputs[4]))
outputs.append(ZeroPadding2D(2)(inputs[4]))
outputs.append(ZeroPadding2D((2, 3))(inputs[4]))
outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Cropping2D(2)(inputs[4]))
outputs.append(Cropping2D((2, 3))(inputs[4]))
outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Dense(3, use_bias=True)(inputs[13]))
outputs.append(Dense(3, use_bias=True)(inputs[14]))
outputs.append(Dense(4, use_bias=False)(inputs[16]))
outputs.append(Dense(4, use_bias=False, activation='tanh')(inputs[18]))
outputs.append(Dense(4, use_bias=False)(inputs[20]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='bilinear')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='bilinear')(inputs[4]))
for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]:
outputs.append(Concatenate(axis=axis)([inputs[0], inputs[1]]))
for axis in [-4, -3, -2, -1, 1, 2, 3, 4]:
outputs.append(Concatenate(axis=axis)([inputs[2], inputs[3]]))
for axis in [-3, -2, -1, 1, 2, 3]:
outputs.append(Concatenate(axis=axis)([inputs[4], inputs[5]]))
for axis in [-2, -1, 1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[6], inputs[7]]))
for axis in [-1, 1]:
outputs.append(Concatenate(axis=axis)([inputs[8], inputs[9]]))
for axis in [-1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]]))
for axis in [-1, 3]:
outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]]))
for axis in [-1, 4]:
outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]]))
for axis in [-1, 5]:
outputs.append(Concatenate(axis=axis)([inputs[20], inputs[21]]))
outputs.append(UpSampling1D(size=2)(inputs[6]))
outputs.append(Multiply()([inputs[10], inputs[11]]))
outputs.append(Multiply()([inputs[11], inputs[10]]))
outputs.append(Multiply()([inputs[11], inputs[13]]))
outputs.append(Multiply()([inputs[10], inputs[11], inputs[12]]))
outputs.append(Multiply()([inputs[11], inputs[12], inputs[13]]))
shared_conv = Conv2D(1, (1, 1),
padding='valid',
name='shared_conv',
activation='relu')
up_scale_2 = UpSampling2D((2, 2))
x1 = shared_conv(up_scale_2(inputs[23])) # (1, 8, 8)
x2 = shared_conv(up_scale_2(inputs[24])) # (1, 8, 8)
x3 = Conv2D(1, (1, 1),
padding='valid')(up_scale_2(inputs[24])) # (1, 8, 8)
x = Concatenate()([x1, x2, x3]) # (3, 8, 8)
outputs.append(x)
x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8)
outputs.append(x)
x = Dropout(0.5)(x)
outputs.append(x)
x = Concatenate()([MaxPooling2D((2, 2))(x),
AveragePooling2D((2, 2))(x)]) # (6, 4, 4)
outputs.append(x)
x = Flatten()(x) # (1, 1, 96)
x = Dense(4, use_bias=False)(x)
outputs.append(x)
x = Dense(3)(x) # (1, 1, 3)
outputs.append(x)
outputs.append(Add()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Subtract()([inputs[26], inputs[30]]))
outputs.append(Multiply()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Average()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Maximum()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Concatenate()([inputs[26], inputs[30], inputs[30]]))
intermediate_input_shape = (3, )
intermediate_in = Input(intermediate_input_shape)
intermediate_x = intermediate_in
intermediate_x = Dense(8)(intermediate_x)
intermediate_x = Dense(5)(intermediate_x)
intermediate_model = Model(inputs=[intermediate_in],
outputs=[intermediate_x],
name='intermediate_model')
intermediate_model.compile(loss='mse', optimizer='nadam')
x = intermediate_model(x) # (1, 1, 5)
intermediate_model_2 = Sequential()
intermediate_model_2.add(Dense(7, input_shape=(5, )))
intermediate_model_2.add(Dense(5))
intermediate_model_2.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_2(x) # (1, 1, 5)
x = Dense(3)(x) # (1, 1, 3)
shared_activation = Activation('tanh')
outputs = outputs + [
Activation('tanh')(inputs[25]),
Activation('hard_sigmoid')(inputs[25]),
Activation('selu')(inputs[25]),
Activation('sigmoid')(inputs[25]),
Activation('softplus')(inputs[25]),
Activation('softmax')(inputs[25]),
Activation('relu')(inputs[25]),
LeakyReLU()(inputs[25]),
ELU()(inputs[25]),
PReLU()(inputs[24]),
PReLU()(inputs[25]),
PReLU()(inputs[26]),
shared_activation(inputs[25]),
Activation('linear')(inputs[26]),
Activation('linear')(inputs[23]),
x,
shared_activation(x),
]
model = Model(inputs=inputs, outputs=outputs, name='test_model_exhaustive')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 1
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, epochs=10)
return model
def test_delete_channels_merge_others(channel_index, data_format):
layer_test_helper_merge_2d(Add(), channel_index, data_format)
layer_test_helper_merge_2d(Multiply(), channel_index, data_format)
layer_test_helper_merge_2d(Average(), channel_index, data_format)
layer_test_helper_merge_2d(Maximum(), channel_index, data_format)
def get_model(global_version=True):
""" Return the parallelized VGG16 for the multi-patch subnet
Inputs: - global_version (bool): True when training only the multi-path subnet
Outputs: - model: keras model
"""
K.clear_session()
param = get_param()
input_shape = (param['nb_crops'], param['img_size'], param['img_size'],
param['nb_channels'])
vgg16 = VGG16(include_top=False,
weights='imagenet',
input_shape=(param['img_size'], param['img_size'],
param['nb_channels']))
flatten = Flatten()(vgg16.output)
fc1_vgg = Dense(4096, activation='relu', name='fc1_vgg')(flatten)
dropout_vgg = Dropout(rate=0.5, name='dropout_vgg')(fc1_vgg)
fc2_vgg = Dense(4096, activation='relu', name='fc2_vgg')(dropout_vgg)
backbone = Model(vgg16.input, fc2_vgg, name='backbone')
backbone.trainable = True
for idx, layer in enumerate(backbone.layers):
if idx < 15:
layer.trainable = False
else:
if isinstance(layer, Conv2D) or isinstance(layer, Dense):
layer.add_loss(lambda: l2(param['weight_decay_coeff'])
(layer.kernel))
if hasattr(layer, 'bias_regularizer') and layer.use_bias:
layer.add_loss(lambda: l2(param['weight_decay_coeff'])
(layer.bias))
inputs = Input(shape=input_shape, name='input')
in1, in2, in3, in4, in5 = inputs[:,
0], inputs[:,
1], inputs[:,
2], inputs[:,
3], inputs[:,
4]
out1, out2, out3, out4, out5 = backbone(in1), backbone(in2), backbone(
in3), backbone(in4), backbone(in5)
# agregation
agg_avg = Average(name='average')([out1, out2, out3, out4, out5])
agg_max = Maximum(name='max')([out1, out2, out3, out4, out5])
agg = concatenate([agg_avg, agg_max], name='agregation_layer')
fc1 = Dense(units=4096, activation='relu', name='fc1')(agg)
dropout = Dropout(rate=0.5, name='dropout')(fc1)
fc2 = Dense(units=4096, activation='relu', name='fc2')(
dropout) # output of the multi-patch subnet in the A-Lamp model
if global_version:
out = fc2
else:
out = Dense(units=1, activation='sigmoid', name='predictions')(
fc2) # we first train the multi-patch subnet alone
model = Model(inputs=inputs, outputs=out)
for layer in model.layers:
if isinstance(layer, Conv2D) or isinstance(layer, Dense):
layer.add_loss(lambda: l2(param['weight_decay_coeff'])
(layer.kernel))
if hasattr(layer, 'bias_regularizer') and layer.use_bias:
layer.add_loss(lambda: l2(param['weight_decay_coeff'])(layer.bias))
return model