def __init__(self, dim, batch_norm, dropout, rec_dropout, partition,
ihm_pos, target_repl=False, depth=1, input_dim=76, **kwargs):
print("==> not used params in network class:", kwargs.keys())
self.dim = dim
self.batch_norm = batch_norm
self.dropout = dropout
self.rec_dropout = rec_dropout
self.depth = depth
# Input layers and masking
X = Input(shape=(None, input_dim), name='X')
mX = Masking()(X)
# Masks
ihm_M = Input(shape=(1,), name='ihm_M')
decomp_M = Input(shape=(None,), name='decomp_M')
los_M = Input(shape=(None,), name='los_M')
inputs = [X, ihm_M, decomp_M, los_M]
# Main part of the network
for i in range(depth):
mX = LSTM(units=dim,
activation='tanh',
return_sequences=True,
recurrent_dropout=rec_dropout,
dropout=dropout)(mX)
L = mX
if dropout > 0:
L = Dropout(dropout)(L)
# Output modules
outputs = []
# ihm output
# NOTE: masking for ihm prediction works this way:
# if ihm_M = 1 then we will calculate an error term
# if ihm_M = 0, our prediction will be 0 and as the label
# will also be 0 then error_term will be 0.
if target_repl > 0:
ihm_seq = TimeDistributed(Dense(1, activation='sigmoid'), name='ihm_seq')(L)
ihm_y = GetTimestep(ihm_pos)(ihm_seq)
ihm_y = Multiply(name='ihm_single')([ihm_y, ihm_M])
outputs += [ihm_y, ihm_seq]
else:
ihm_seq = TimeDistributed(Dense(1, activation='sigmoid'))(L)
ihm_y = GetTimestep(ihm_pos)(ihm_seq)
ihm_y = Multiply(name='ihm')([ihm_y, ihm_M])
outputs += [ihm_y]
# decomp output
decomp_y = TimeDistributed(Dense(1, activation='sigmoid'))(L)
decomp_y = ExtendMask(name='decomp', add_epsilon=True)([decomp_y, decomp_M])
outputs += [decomp_y]
# los output
if partition == 'none':
los_y = TimeDistributed(Dense(1, activation='relu'))(L)
else:
los_y = TimeDistributed(Dense(10, activation='softmax'))(L)
los_y = ExtendMask(name='los', add_epsilon=True)([los_y, los_M])
outputs += [los_y]
# pheno output
if target_repl:
pheno_seq = TimeDistributed(Dense(25, activation='sigmoid'), name='pheno_seq')(L)
pheno_y = LastTimestep(name='pheno_single')(pheno_seq)
outputs += [pheno_y, pheno_seq]
else:
pheno_seq = TimeDistributed(Dense(25, activation='sigmoid'))(L)
pheno_y = LastTimestep(name='pheno')(pheno_seq)
outputs += [pheno_y]
super(Network, self).__init__(inputs=inputs, outputs=outputs)
def __init__(self):
emb_dim = 512
inputs = Input(shape=(64, None, 1), name="input_enc")
label = Input(shape=(emb_dim, ), )
emb = Reshape((1, 1, emb_dim))(label)
x = MyConcat()([inputs, emb])
conv1 = Conv2D(filters=16,
kernel_size=(3, 9),
strides=(1, 1),
padding="same")(x)
conv1_bn = BatchNormalization(axis=1)(conv1)
conv1_gated = Conv2D(filters=16,
kernel_size=(3, 9),
strides=(1, 1),
padding="same")(x)
conv1_gated_bn = BatchNormalization(axis=1)(conv1_gated)
conv1_sigmoid = Multiply()(
[conv1_bn, Activation(sigmoid)(conv1_gated_bn)])
x = MyConcat()([conv1_sigmoid, emb])
conv2 = Conv2D(filters=32,
kernel_size=(4, 8),
strides=(2, 2),
padding="same")(x)
conv2_bn = BatchNormalization(axis=1)(conv2)
conv2_gated = Conv2D(filters=32,
kernel_size=(4, 8),
strides=(2, 2),
padding="same")(x)
conv2_gated_bn = BatchNormalization(axis=1)(conv2_gated)
conv2_sigmoid = Multiply()(
[conv2_bn, Activation(sigmoid)(conv2_gated_bn)])
x = MyConcat()([conv2_sigmoid, emb])
conv3 = Conv2D(filters=32,
kernel_size=(4, 8),
strides=(2, 2),
padding="same")(x)
conv3_bn = BatchNormalization(axis=1)(conv3)
conv3_gated = Conv2D(filters=32,
kernel_size=(4, 8),
strides=(2, 2),
padding="same")(x)
conv3_gated_bn = BatchNormalization(axis=1)(conv3_gated)
conv3_sigmoid = Multiply()(
[conv3_bn, Activation(sigmoid)(conv3_gated_bn)])
x = MyConcat()([conv3_sigmoid, emb])
padding = tf.constant([[0, 0], [0, 0], [2, 2], [0, 0]])
x = tf.pad(x, padding, "CONSTANT")
conv4_mu = Conv2D(filters=8,
kernel_size=(16, 5),
strides=(16, 1),
padding="valid")(x)
conv4_logvar = Conv2D(filters=8,
kernel_size=(16, 5),
strides=(16, 1),
padding="valid")(x)
self.encoder = Model(inputs=[inputs, label],
outputs=[conv4_mu, conv4_logvar])
# Decoder
inputs = Input(shape=(1, None, 8), name="input_dec")
label = Input(shape=(512, ), name="label_dec")
emb = Reshape((1, 1, emb_dim))(label)
f0 = Input(shape=(1, None, 257), name="log_f0")
x = MyConcat()([inputs, emb])
x = Concatf0(4)([x, f0])
upconv1 = Conv2DTranspose(filters=32,
kernel_size=(16, 5),
strides=(16, 1),
padding="valid")(x)
upconv1_bn = BatchNormalization(axis=1)(upconv1)
upconv1_gated = Conv2DTranspose(filters=32,
kernel_size=(16, 5),
strides=(16, 1),
padding="valid")(x)
upconv1_gated_bn = BatchNormalization(axis=1)(upconv1_gated)
upconv1_sigmoid = Multiply()(
[upconv1_bn, Activation(sigmoid)(upconv1_gated_bn)])
upconv1_sigmoid = upconv1_sigmoid[:, :, 2:-2, :]
x = MyConcat()([upconv1_sigmoid, emb])
x = Concatf0(4)([x, f0])
upconv2 = Conv2DTranspose(filters=32,
kernel_size=(4, 8),
strides=(2, 2),
padding="same")(x)
upconv2_bn = BatchNormalization(axis=1)(upconv2)
upconv2_gated = Conv2DTranspose(filters=32,
kernel_size=(4, 8),
strides=(2, 2),
padding="same")(x)
upconv2_gated_bn = BatchNormalization(axis=1)(upconv2_gated)
upconv2_sigmoid = Multiply()(
[upconv2_bn, Activation(sigmoid)(upconv2_gated_bn)])
x = MyConcat()([upconv2_sigmoid, emb])
x = Concatf0(2)([x, f0])
upconv3 = Conv2DTranspose(filters=16,
kernel_size=(4, 8),
strides=(2, 2),
padding="same")(x)
upconv3_bn = BatchNormalization(axis=1)(upconv3)
upconv3_gated = Conv2DTranspose(filters=16,
kernel_size=(4, 8),
strides=(2, 2),
padding="same")(x)
upconv3_gated_bn = BatchNormalization(axis=1)(upconv3_gated)
upconv3_sigmoid = Multiply()(
[upconv3_bn, Activation(sigmoid)(upconv3_gated_bn)])
x = MyConcat()([upconv3_sigmoid, emb])
x = Concatf0()([x, f0])
upconv4_mu = Conv2DTranspose(filters=1,
kernel_size=(3, 9),
strides=(1, 1),
padding="same")(x)
self.decoder = Model(inputs=[inputs, label, f0], outputs=upconv4_mu)
#reconstruction model
inputs = Input(shape=(64, None, 1))
emb_s = Input(shape=(512, ))
emb_t = Input(shape=(512, ))
f0_t = Input(shape=(1, None, 257))
mu_enc, logvar_enc = self.encoder([inputs, emb_s])
z_enc = self.reparameterize(mu_enc, logvar_enc)
mu_dec = self.decoder([z_enc, emb_t, f0_t])
self.model = Model(inputs=[inputs, emb_s, emb_t, f0_t],
outputs=[mu_dec, mu_enc, logvar_enc])
def double_stream_6_subs_64_filters_remapfactor_32(self):
num_filters = 64
kernel_initializer = 'he_normal'
padding = 'same'
activation_fn = ReLU()
remap_factor = 32 # must be an even number!
subtract_5 = Subtract(name='subtract_5')(
[self.curr_center_block2_relu, self.prev_center_block2_relu])
conv_low_1 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv_low_1')(subtract_5))
conv_low_2 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv_low_2')(conv_low_1))
up_low = (UpSampling2D(size=(2, 2), name='fuse_up_low')(conv_low_2))
conv_low_3 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv_low_3')(up_low))
subtract_6 = Subtract(name='subtract_6')(
[self.curr_decoder_stage0b_relu, self.prev_decoder_stage0b_relu])
merge10 = concatenate([subtract_6, conv_low_3],
axis=3, name='fuse_cat10')
conv10_1 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv_10_1')(merge10))
conv10_2 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv_10_2')(conv10_1))
up10 = (UpSampling2D(size=(2, 2), name='fuse_up_10')(conv10_2))
conv10_3 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv10_3')(up10))
subtract_7 = Subtract(name='subtract_7')(
[self.curr_decoder_stage1b_relu, self.prev_decoder_stage1b_relu])
merge11 = concatenate([subtract_7, conv10_3],
axis=3, name='fuse_cat11')
conv11_1 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv11_1')(merge11))
conv11_2 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv11_2')(conv11_1))
up11 = (UpSampling2D(size=(2, 2), name='fuse_up11')(conv11_2))
conv11_3 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv11_3')(up11))
subtract_8 = Subtract(name='subtract_8')(
[self.curr_decoder_stage2b_relu, self.prev_decoder_stage2b_relu])
merge12 = concatenate([subtract_8, conv11_3],
axis=3, name='fuse_cat12')
conv12_1 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv12_1')(merge12))
conv12_2 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv12_2')(conv12_1))
up12 = (UpSampling2D(size=(2, 2), name='fuse_up12')(conv12_2))
conv12_3 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv12_3')(up12))
subtract_9 = Subtract(name='subtract_9')(
[self.curr_decoder_stage3b_relu, self.prev_decoder_stage3b_relu])
merge13 = concatenate([subtract_9, conv12_3],
axis=3, name='fuse_cat13')
conv13_1 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv13_1')(merge13))
conv13_2 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv13_2')(conv13_1))
up13 = (UpSampling2D(size=(2, 2), name='fuse_up13')(conv13_2))
conv13_3 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv13_3')(up13))
subtract_10 = Subtract(name='subtract_10')(
[self.curr_decoder_stage4b_relu, self.prev_decoder_stage4b_relu])
merge14 = concatenate([subtract_10, conv13_3],
axis=3, name='fuse_cat14')
conv14_1 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv14_1')(merge14))
conv14_2 = activation_fn(Conv2D(num_filters, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv14_2')(conv14_1))
# use a multiplication and addition layer to segment the current image
# The multiplication layer uses sigmoid activation
conv15_1 = activation_fn(Conv2D(16, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv15_1')(conv14_2))
conv15_2 = activation_fn(Conv2D(8, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv15_2')(conv15_1))
conv15_3 = activation_fn(Conv2D(1, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='fuse_conv15_3')(conv15_2))
############# Stream 1 ###################
# predict if the image is changing or not
stream1_pool1_1 = MaxPooling2D(
2, strides=2, name='stream1_pool1_1')(conv15_3)
stream1_conv1_1 = activation_fn(Conv2D(16, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='stream1_conv1_1')(stream1_pool1_1))
stream1_conv1_2 = activation_fn(Conv2D(32, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='stream1_conv1_2')(stream1_conv1_1))
stream1_pool2_1 = MaxPooling2D(
2, strides=2, name='stream1_pool2_1')(stream1_conv1_2)
stream1_conv2_1 = activation_fn(Conv2D(64, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='stream1_conv2_1')(stream1_pool2_1))
stream1_conv2_2 = activation_fn(Conv2D(128, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='stream1_conv2_2')(stream1_conv2_1))
stream1_pool3_1 = MaxPooling2D(
2, strides=2, name='stream1_pool3_1')(stream1_conv2_2)
stream1_conv3_1 = activation_fn(Conv2D(256, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='stream1_conv3_1')(stream1_pool3_1))
stream1_conv3_2 = activation_fn(Conv2D(512, 3, activation=None, padding=padding,
kernel_initializer=kernel_initializer, name='stream1_conv3_2')(stream1_conv3_1))
# apply global avg pooling and FC layer
stream1_gav = GlobalAveragePooling2D(
name='stream1_classify_global_avg_pool')(stream1_conv3_2)
stream1_dense1 = activation_fn(
Dense(512, activation=None, name='stream1_classify_dense1')(stream1_gav))
############# Stream 2 ###################
# compute min value and subtract it from tensor
min_value_seg_mask = Lambda(lambda x: tf.math.reduce_min(
x), name='seg_process_2')(conv15_3)
seg_positive_reduced = Lambda(lambda x: tf.math.subtract(
x[0], x[1]), name='seg_process_3')([conv15_3, min_value_seg_mask])
# compute max value and normalize tensor with it -> values are in range [0, 1]
max_value_seg_mask = Lambda(lambda x: tf.math.reduce_max(
x), name='seg_process_4')(seg_positive_reduced)
seg_positive_reduced_normalized = Lambda(lambda x: tf.math.divide_no_nan(
x[0], x[1]), name='seg_process_5')([seg_positive_reduced, max_value_seg_mask])
# map values from [0, 1] to [-X, X]
seg_positive_reduced_normalized_mult = Lambda(lambda x: tf.math.multiply_no_nan(
x, remap_factor), name='seg_process_6')(seg_positive_reduced_normalized)
seg_positive_reduced_normalized_remapped = Lambda(lambda x: tf.math.subtract(
x, remap_factor/2), name='seg_process_7')(seg_positive_reduced_normalized_mult)
# apply sigmoid function
seg_positive_reduced_normalized_remapped_sigmoid = Activation(
activation='sigmoid', name='seg_process_sigmoid')(seg_positive_reduced_normalized_remapped)
# now we can segment the current image with that that mask
image_current_seg_mult = Multiply(name='seg_process_multiply')(
[seg_positive_reduced_normalized_remapped_sigmoid, self.image_current])
# classify the segmented image. use pretrained model as a classifier
classifier_model = VGG16(
include_top=False,
weights='imagenet',
input_shape=self.input_size,
pooling=None)
classifier_output = classifier_model(image_current_seg_mult)
# apply global avg pool and add fc layers
gav = GlobalAveragePooling2D(
name='classify_global_avg_pool')(classifier_output)
dense1 = activation_fn(
Dense(1024, activation=None, name='classify_dense1')(gav))
# combine the two streams and apply two more fc layers
cat_streams = Concatenate(
axis=-1, name='cat_streams')([dense1, stream1_dense1])
dense2 = activation_fn(
Dense(512, activation=None, name='classify_dense2')(cat_streams))
dense3 = Dense((self.num_classes), name='classify_dense3')(dense2)
output_feature_comparison = Activation(
activation='softmax', name='output_activation')(dense3)
model = Model(inputs=[
self.image_previous, self.image_current], outputs=output_feature_comparison)
return model
def apply_unet(input_tensor,
output_name='output',
params={},
output_mask_logit=False):
""" Apply a convolutionnal U-net to model a single instrument (one U-net
is used for each instrument).
:param input_tensor:
:param output_name: (Optional) , default to 'output'
:param params: (Optional) , default to empty dict.
:param output_mask_logit: (Optional) , default to False.
"""
logging.info(f'Apply unet for {output_name}')
conv_n_filters = params.get('conv_n_filters', [16, 32, 64, 128, 256, 512])
conv_activation_layer = _get_conv_activation_layer(params)
deconv_activation_layer = _get_deconv_activation_layer(params)
kernel_initializer = he_uniform(seed=50)
conv2d_factory = partial(Conv2D,
strides=(2, 2),
padding='same',
kernel_initializer=kernel_initializer)
# First layer.
conv1 = conv2d_factory(conv_n_filters[0], (5, 5))(input_tensor)
batch1 = BatchNormalization(axis=-1)(conv1)
rel1 = conv_activation_layer(batch1)
# Second layer.
conv2 = conv2d_factory(conv_n_filters[1], (5, 5))(rel1)
batch2 = BatchNormalization(axis=-1)(conv2)
rel2 = conv_activation_layer(batch2)
# Third layer.
conv3 = conv2d_factory(conv_n_filters[2], (5, 5))(rel2)
batch3 = BatchNormalization(axis=-1)(conv3)
rel3 = conv_activation_layer(batch3)
# Fourth layer.
conv4 = conv2d_factory(conv_n_filters[3], (5, 5))(rel3)
batch4 = BatchNormalization(axis=-1)(conv4)
rel4 = conv_activation_layer(batch4)
# Fifth layer.
conv5 = conv2d_factory(conv_n_filters[4], (5, 5))(rel4)
batch5 = BatchNormalization(axis=-1)(conv5)
rel5 = conv_activation_layer(batch5)
# Sixth layer
conv6 = conv2d_factory(conv_n_filters[5], (5, 5))(rel5)
batch6 = BatchNormalization(axis=-1)(conv6)
_ = conv_activation_layer(batch6)
#
#
conv2d_transpose_factory = partial(Conv2DTranspose,
strides=(2, 2),
padding='same',
kernel_initializer=kernel_initializer)
#
up1 = conv2d_transpose_factory(conv_n_filters[4], (5, 5))((conv6))
up1 = deconv_activation_layer(up1)
batch7 = BatchNormalization(axis=-1)(up1)
drop1 = Dropout(0.5)(batch7)
merge1 = Concatenate(axis=-1)([conv5, drop1])
#
up2 = conv2d_transpose_factory(conv_n_filters[3], (5, 5))((merge1))
up2 = deconv_activation_layer(up2)
batch8 = BatchNormalization(axis=-1)(up2)
drop2 = Dropout(0.5)(batch8)
merge2 = Concatenate(axis=-1)([conv4, drop2])
#
up3 = conv2d_transpose_factory(conv_n_filters[2], (5, 5))((merge2))
up3 = deconv_activation_layer(up3)
batch9 = BatchNormalization(axis=-1)(up3)
drop3 = Dropout(0.5)(batch9)
merge3 = Concatenate(axis=-1)([conv3, drop3])
#
up4 = conv2d_transpose_factory(conv_n_filters[1], (5, 5))((merge3))
up4 = deconv_activation_layer(up4)
batch10 = BatchNormalization(axis=-1)(up4)
merge4 = Concatenate(axis=-1)([conv2, batch10])
#
up5 = conv2d_transpose_factory(conv_n_filters[0], (5, 5))((merge4))
up5 = deconv_activation_layer(up5)
batch11 = BatchNormalization(axis=-1)(up5)
merge5 = Concatenate(axis=-1)([conv1, batch11])
#
up6 = conv2d_transpose_factory(1, (5, 5), strides=(2, 2))((merge5))
up6 = deconv_activation_layer(up6)
batch12 = BatchNormalization(axis=-1)(up6)
# Last layer to ensure initial shape reconstruction.
if not output_mask_logit:
up7 = Conv2D(2, (4, 4),
dilation_rate=(2, 2),
activation='sigmoid',
padding='same',
kernel_initializer=kernel_initializer)((batch12))
output = Multiply(name=output_name)([up7, input_tensor])
return output
return Conv2D(2, (4, 4),
dilation_rate=(2, 2),
padding='same',
kernel_initializer=kernel_initializer)((batch12))
def get_resnet34_se(self, input_shape, nb_classes, convfilt, ksize, depth,
drop):
OUTPUT_CLASS = nb_classes # output classes
input1 = Input(shape=input_shape, name='input_ecg')
k = 1 # increment every 4th residual block
p = True # pool toggle every other residual block (end with 2^8)
convfilt = convfilt
encoder_confilt = 64 # encoder filters' num
convstr = 1
ksize = ksize
poolsize = 2
poolstr = 2
drop = drop
se_ratio = 8
depth = depth
# First convolutional block (conv,BN, relu)
lcount = 0
x = Conv1D(filters=convfilt,
kernel_size=ksize,
padding='same',
strides=convstr,
kernel_initializer='he_normal',
name='layer' + str(lcount))(input1)
lcount += 1
squeeze = GlobalAveragePooling1D()(x)
excitation = Dense(convfilt * k // se_ratio)(squeeze)
excitation = Activation('relu')(excitation)
excitation = Dense(convfilt * k)(excitation)
excitation = Activation('sigmoid')(excitation)
excitation = Reshape((1, convfilt * k))(excitation)
x = Multiply()([x, excitation])
x = BatchNormalization(name='layer' + str(lcount))(x)
lcount += 1
x = Activation('relu')(x)
## Second convolutional block (conv, BN, relu, dropout, conv) with residual net
# Left branch (convolutions)
x1 = Conv1D(filters=convfilt,
kernel_size=ksize,
padding='same',
strides=convstr,
kernel_initializer='he_normal',
name='layer' + str(lcount))(x)
lcount += 1
x1 = BatchNormalization(name='layer' + str(lcount))(x1)
lcount += 1
x1 = Activation('relu')(x1)
x1 = Dropout(drop)(x1)
x1 = Conv1D(filters=convfilt,
kernel_size=ksize,
padding='same',
strides=convstr,
kernel_initializer='he_normal',
name='layer' + str(lcount))(x1)
lcount += 1
squeeze = GlobalAveragePooling1D()(x1)
excitation = Dense(convfilt * k // se_ratio)(squeeze)
excitation = Activation('relu')(excitation)
excitation = Dense(convfilt * k)(excitation)
excitation = Activation('sigmoid')(excitation)
excitation = Reshape((1, convfilt * k))(excitation)
x1 = Multiply()([x1, excitation])
x1 = MaxPooling1D(pool_size=poolsize, strides=poolstr,
padding='same')(x1)
# Right branch, shortcut branch pooling
x2 = MaxPooling1D(pool_size=poolsize, strides=poolstr,
padding='same')(x)
# Merge both branches
x = keras.layers.add([x1, x2])
del x1, x2
## Main loop
p = not p
for l in range(depth):
if (l % 4 == 0) and (
l > 0): # increment k on every fourth residual block
k += 1
# increase depth by 1x1 Convolution case dimension shall change
xshort = Conv1D(filters=convfilt * k,
kernel_size=1,
name='layer' + str(lcount))(x)
lcount += 1
else:
xshort = x
# Left branch (convolutions)
# notice the ordering of the operations has changed
x1 = BatchNormalization(name='layer' + str(lcount))(x)
lcount += 1
x1 = Activation('relu')(x1)
x1 = Dropout(drop)(x1)
x1 = Conv1D(filters=convfilt * k,
kernel_size=ksize,
padding='same',
strides=convstr,
kernel_initializer='he_normal',
name='layer' + str(lcount))(x1)
lcount += 1
x1 = BatchNormalization(name='layer' + str(lcount))(x1)
lcount += 1
x1 = Activation('relu')(x1)
x1 = Dropout(drop)(x1)
x1 = Conv1D(filters=convfilt * k,
kernel_size=ksize,
padding='same',
strides=convstr,
kernel_initializer='he_normal',
name='layer' + str(lcount))(x1)
lcount += 1
squeeze = GlobalAveragePooling1D()(x1)
excitation = Dense(convfilt * k // se_ratio)(squeeze)
excitation = Activation('relu')(excitation)
excitation = Dense(convfilt * k)(excitation)
excitation = Activation('sigmoid')(excitation)
excitation = Reshape((1, convfilt * k))(excitation)
x1 = Multiply()([x1, excitation])
if p:
x1 = MaxPooling1D(pool_size=poolsize,
strides=poolstr,
padding='same')(x1)
# Right branch: shortcut connection
if p:
x2 = MaxPooling1D(pool_size=poolsize,
strides=poolstr,
padding='same')(xshort)
else:
x2 = xshort # pool or identity
# Merging branches
x = keras.layers.add([x1, x2])
# change parameters
p = not p # toggle pooling
# x = Conv1D(filters=convfilt * k, kernel_size=ksize, padding='same', strides=convstr, kernel_initializer='he_normal')(x)
# x_reg = Conv1D(filters=convfilt * k, kernel_size=1, padding='same', strides=convstr, kernel_initializer='he_normal')(x)
# Final bit
x = BatchNormalization(name='layer' + str(lcount))(x)
lcount += 1
x = Activation('relu')(x)
x_ecg = Flatten()(x)
out1 = Dense(OUTPUT_CLASS, activation='softmax',
name='main_output')(x_ecg)
model = Model(inputs=input1, outputs=out1)
return model
segment_input = Input(shape=(max_len, ), dtype=tf.int32)
Bert_model = TFBertModel.from_pretrained('bert-base-chinese')
output = Bert_model([tokens_input, mask_input, segment_input])
mask = Lambda(lambda x: x[:, 1:text_maxlen + 1])(mask_input)
mask = K.cast_to_floatx(mask)
answer = Lambda(lambda x: x[:, 0, :])(output[0])
answer = Dense(1, activation='sigmoid', name='answerable')(answer)
output_start = Lambda(lambda x: x[:, 1:text_maxlen + 1, :])(output[0])
output_start = Dense(1)(output_start)
output_start = K.squeeze(output_start, axis=-1)
output_start = Multiply()([output_start, mask])
output_start = Activation('softmax', name='start')(output_start)
output_end = Lambda(lambda x: x[:, 1:text_maxlen + 1, :])(output[0])
output_end = Dropout(0.2)(output_end)
output_end = Dense(1)(output_end)
output_end = K.squeeze(output_end, axis=-1)
output_end = Multiply()([output_end, mask])
output_end = Activation('softmax', name='end')(output_end)
# model = Model([output,mask_input] ,[answer,output_start,output_end])
model = Model([tokens_input, mask_input, segment_input],
[answer, output_start, output_end])
model = Model([tokens_input, mask_input, segment_input],
def unet_save(activation,
batch_norm,
bn_first,
depth,
drop_3,
dropouts,
f_size,
filters,
inputs,
kernel_init,
m_pool,
ndims,
pad,
use_upsample,
mask_classes,
supervision=False):
"""
unet 2d or 3d for the functional tf.keras api
:param activation:
:param batch_norm:
:param bn_first:
:param depth:
:param drop_3:
:param dropouts:
:param f_size:
:param filters:
:param inputs:
:param kernel_init:
:param m_pool:
:param ndims:
:param pad:
:param use_upsample:
:param mask_classes:
:return:
"""
filters_init = filters
encoder = list()
decoder = list()
dropouts = dropouts.copy()
Conv = getattr(kl, 'Conv{}D'.format(ndims))
one_by_one = (1, 1, 1)[-ndims:]
# build the encoder
for l in range(depth):
if len(encoder) == 0:
# first block
input_tensor = inputs
else:
# all other blocks, use the max-pooled output of the previous encoder block
# remember the max-pooled output from the previous layer
input_tensor = encoder[-1][1]
encoder.append(
ownkl.downsampling_block(inputs=input_tensor,
filters=filters,
f_size=f_size,
activation=activation,
drop=dropouts[l],
batch_norm=batch_norm,
kernel_init=kernel_init,
pad=pad,
m_pool=m_pool,
bn_first=bn_first,
ndims=ndims))
filters *= 2
# middle part
input_tensor = encoder[-1][1]
fully = ownkl.conv_layer(inputs=input_tensor,
filters=filters,
f_size=f_size,
activation=activation,
batch_norm=batch_norm,
kernel_init=kernel_init,
pad=pad,
bn_first=bn_first,
ndims=ndims)
fully = Dropout(drop_3)(fully)
fully = ownkl.conv_layer(inputs=fully,
filters=filters,
f_size=f_size,
activation=activation,
batch_norm=batch_norm,
kernel_init=kernel_init,
pad=pad,
bn_first=bn_first,
ndims=ndims)
# build the decoder
decoder.append(fully)
for l in range(depth):
# take the output of the previous decoder block and the output of the corresponding
# encoder block
input_lower = decoder[-1]
input_skip = encoder.pop()[0]
filters //= 2
decoder.append(
ownkl.upsampling_block(lower_input=input_lower,
conv_input=input_skip,
use_upsample=use_upsample,
filters=filters,
f_size=f_size,
activation=activation,
drop=dropouts.pop(),
batch_norm=batch_norm,
up_size=m_pool,
bn_first=bn_first,
ndims=ndims))
# deep supervision
# current test
if supervision:
try:
UpSampling = getattr(ownkl, 'UpSampling{}DInterpol'.format(ndims))
except Exception as e:
logging.info(
'own Upsampling layer not available, fallback to tensorflow upsampling'
)
UpSampling = getattr(kl, 'UpSampling{}D'.format(ndims))
# mask from the pre-last upsampling block
lower_mask = decoder[-2]
lower_mask = Conv(filters_init,
one_by_one,
padding=pad,
kernel_initializer=kernel_init,
activation=activation)(lower_mask)
lower_mask = UpSampling(size=m_pool)(lower_mask)
outputs = decoder[-1]
if supervision:
# outputs = concatenate([lower_mask, outputs])
outputs = Multiply()([lower_mask, outputs])
# use either a 3x3 conv or 1x1
# 16 or 4 filters
# outputs = Conv(mask_classes, f_size, padding=pad, kernel_initializer=kernel_init, activation=activation)(outputs)
return outputs
def call(self, x):
y = self.squeeze(x)
y = self.dense_rel(y)
y = self.dense_sig(y)
return Multiply()([x, y])
def TRepNet(n_steps, n_features, activation='elu'):
codings_size = get_output_dim(n_steps * n_features)
dilation_rates = [2**i for i in range(10)] * 1
skips = []
encoder_input = Input(shape=[n_steps, n_features])
# Convolution
conv = encoder_input
for dilation_rate in dilation_rates:
conv = keras.layers.GaussianNoise(0.01)(conv)
conv = Conv1D(16, 1, activation=activation, padding='same')(conv)
conv_filter = Conv1D(filters=128, kernel_size=3, padding='causal', activation=activation, dilation_rate=dilation_rate)(conv)
conv_filter = Dropout(0.1)(conv_filter)
conv_gate = Conv1D(filters=128, kernel_size=3, padding='causal', activation=activation, dilation_rate=dilation_rate)(conv)
conv_gate = Dropout(0.1)(conv_gate)
mul = Multiply()([Activation('tanh')(conv_filter), Activation('sigmoid')(conv_gate)])
skip = Conv1D(16, 1, padding='same', activation=activation)(mul)
conv = Add()([conv, skip])
skips.append(skip)
conv = Activation(activation)(Add()(skips))
conv = Conv1D(16, 1, activation=activation, padding='same')(conv)
conv = MaxPool1D(pool_size=2)(conv)
conv = Flatten()(conv)
# CNN
cnn = Conv1D(32, kernel_size=1, padding="SAME", activation=activation)(encoder_input)
cnn = MaxPool1D(pool_size=2)(cnn)
cnn = Conv1D(64, kernel_size=3, padding="SAME", activation=activation)(cnn)
cnn = MaxPool1D(pool_size=2)(cnn)
cnn = Conv1D(128, kernel_size=5, padding="SAME", activation=activation)(cnn)
cnn = Conv1D(16, 1, activation=activation, padding='same')(cnn)
cnn = MaxPool1D(pool_size=2)(cnn)
cnn = Flatten()(cnn)
z1 = Dense(codings_size, kernel_initializer='lecun_normal', activation='selu')(conv)
z3 = Dense(codings_size, kernel_initializer='lecun_normal', activation='selu')(cnn)
z = Add()([z1, z3])
encoder_output = Dense(codings_size, activation='sigmoid')(z)
encoder = Model(inputs=[encoder_input], outputs=[encoder_output])
# Decoder
decoder_input = Input(shape=[codings_size])
noise_input = keras.layers.GaussianNoise(0.01)(decoder_input)
dconv = keras.layers.Reshape([codings_size, 1, 1])(noise_input)
dconv = Conv2DTranspose(filters=32, kernel_size=3, activation=activation)(dconv)
dconv = Conv2DTranspose(filters=16, kernel_size=1, activation=activation)(dconv)
dconv = Flatten()(dconv)
x = Dense(n_steps * n_features)(dconv)
decoder_output = keras.layers.Reshape([n_steps, n_features])(x)
decoder = Model(inputs=[decoder_input], outputs=[decoder_output])
return encoder, decoder
def attention_block(input,
input_channels=None,
output_channels=None,
encoder_depth=1):
p = 1
t = 2
r = 1
if input_channels is None:
input_channels = input.get_shape()[-1]
if output_channels is None:
output_channels = input_channels
# First Residual Block
for i in range(p):
input = residual_block(input)
# Trunc Branch
output_trunk = input
for i in range(t):
output_trunk = residual_block(output_trunk)
# Soft Mask Branch
## encoder
### first down sampling
output_soft_mask = MaxPool2D(padding='same')(input)
for i in range(r):
output_soft_mask = residual_block(output_soft_mask)
skip_connections = []
for i in range(encoder_depth - 1):
## skip connections
output_skip_connection = residual_block(output_soft_mask)
skip_connections.append(output_skip_connection)
# print ('skip shape:', output_skip_connection.get_shape())
## down sampling
output_soft_mask = MaxPool2D(padding='same')(output_soft_mask)
for _ in range(r):
output_soft_mask = residual_block(output_soft_mask)
skip_connections = list(reversed(skip_connections))
for i in range(encoder_depth - 1):
for _ in range(r):
output_soft_mask = residual_block(output_soft_mask)
output_soft_mask = UpSampling2D()(output_soft_mask)
output_soft_mask = Add()([output_soft_mask, skip_connections[i]])
for i in range(r):
output_soft_mask = residual_block(output_soft_mask)
output_soft_mask = UpSampling2D()(output_soft_mask)
## Output
output_soft_mask = Conv2D(input_channels, (1, 1))(output_soft_mask)
output_soft_mask = Conv2D(input_channels, (1, 1))(output_soft_mask)
output_soft_mask = Activation('sigmoid')(output_soft_mask)
# Attention: (1 + output_soft_mask) * output_trunk
output = Lambda(lambda x: x + 1)(output_soft_mask)
output = Multiply()([output, output_trunk]) #
# Last Residual Block
for i in range(p):
output = residual_block(output)
return output
def create_tf_lite_model(self, weights_file, target_name, use_dynamic_range_quant=False):
'''
Method to create a tf lite model folder from a weights file.
The conversion creates two models, one for each separation core.
Tf lite does not support complex numbers yet. Some processing must be
done outside the model.
For further information and how real time processing can be
implemented see "real_time_processing_tf_lite.py".
The conversion only works with TF 2.3.
'''
# check for type
if weights_file.find('_norm_') != -1:
norm_stft = True
num_elements_first_core = 2 + self.numLayer * 3 + 2
else:
norm_stft = False
num_elements_first_core = self.numLayer * 3 + 2
# build model
self.build_DTLN_model_stateful(norm_stft=norm_stft)
# load weights
self.model.load_weights(weights_file)
#### Model 1 ##########################
mag = Input(batch_shape=(1, 1, (self.blockLen//2+1)))
states_in_1 = Input(batch_shape=(1, self.numLayer, self.numUnits, 2))
# normalizing log magnitude stfts to get more robust against level variations
if norm_stft:
mag_norm = InstantLayerNormalization()(tf.math.log(mag + 1e-7))
else:
# behaviour like in the paper
mag_norm = mag
# predicting mask with separation kernel
mask_1, states_out_1 = self.seperation_kernel_with_states(self.numLayer,
(self.blockLen//2+1),
mag_norm, states_in_1)
model_1 = Model(inputs=[mag, states_in_1], outputs=[mask_1, states_out_1])
#### Model 2 ###########################
estimated_frame_1 = Input(batch_shape=(1, 1, (self.blockLen)))
states_in_2 = Input(batch_shape=(1, self.numLayer, self.numUnits, 2))
# encode time domain frames to feature domain
encoded_frames = Conv1D(self.encoder_size,1,strides=1,
use_bias=False)(estimated_frame_1)
# normalize the input to the separation kernel
encoded_frames_norm = InstantLayerNormalization()(encoded_frames)
# predict mask based on the normalized feature frames
mask_2, states_out_2 = self.seperation_kernel_with_states(self.numLayer,
self.encoder_size,
encoded_frames_norm,
states_in_2)
# multiply encoded frames with the mask
estimated = Multiply()([encoded_frames, mask_2])
# decode the frames back to time domain
decoded_frame = Conv1D(self.blockLen, 1, padding='causal',
use_bias=False)(estimated)
model_2 = Model(inputs=[estimated_frame_1, states_in_2],
outputs=[decoded_frame, states_out_2])
# set weights to submodels
weights = self.model.get_weights()
model_1.set_weights(weights[:num_elements_first_core])
model_2.set_weights(weights[num_elements_first_core:])
# convert first model
converter = tf.lite.TFLiteConverter.from_keras_model(model_1)
if use_dynamic_range_quant:
converter.optimizations = [tf.lite.Optimize.DEFAULT]
tflite_model = converter.convert()
with tf.io.gfile.GFile(target_name + '_1.tflite', 'wb') as f:
f.write(tflite_model)
# convert second model
converter = tf.lite.TFLiteConverter.from_keras_model(model_2)
if use_dynamic_range_quant:
converter.optimizations = [tf.lite.Optimize.DEFAULT]
tflite_model = converter.convert()
with tf.io.gfile.GFile(target_name + '_2.tflite', 'wb') as f:
f.write(tflite_model)
print('TF lite conversion complete!')
def _build_model(self, x, y):
"""Construct ASAC model using feature and label statistics.
Args:
- x: temporal feature
- y: labels
Returns:
- model: asac model
"""
# Parameters
h_dim = self.h_dim
n_layer = self.n_layer
dim = len(x[0, 0, :])
max_seq_len = len(x[0, :, 0])
# Build one input, two outputs model
main_input = Input(shape=(max_seq_len, dim), dtype='float32')
mask_layer = Masking(mask_value=-1.)(main_input)
previous_input = Input(shape=(max_seq_len, dim), dtype='float32')
previous_mask_layer = Masking(mask_value=-1.)(previous_input)
select_layer = rnn_layer(previous_mask_layer,
self.model_type,
h_dim,
return_seq=True)
for _ in range(n_layer):
select_layer = rnn_layer(select_layer,
self.model_type,
h_dim,
return_seq=True)
select_layer = TimeDistributed(Dense(
dim, activation='sigmoid'))(select_layer)
# Sampling the selection
select_layer = Lambda(lambda x: x - 0.5)(select_layer)
select_layer = Activation('relu')(select_layer)
select_out = Lambda(lambda x: x * 2, name='select')(select_layer)
# Second output
pred_layer = Multiply()([mask_layer, select_out])
for _ in range(n_layer - 1):
pred_layer = rnn_layer(pred_layer,
self.model_type,
h_dim,
return_seq=True)
return_seq_bool = len(y.shape) == 3
pred_layer = rnn_layer(pred_layer, self.model_type, h_dim,
return_seq_bool)
if self.task == 'classification': act_fn = 'sigmoid'
elif self.task == 'regression': act_fn = 'linear'
if len(y.shape) == 3:
pred_out = TimeDistributed(Dense(y.shape[-1], activation=act_fn),
name='predict')(pred_layer)
elif len(y.shape) == 2:
pred_out = Dense(y.shape[-1], activation=act_fn,
name='predict')(pred_layer)
model = Model(inputs=[main_input, previous_input],
outputs=[select_out, pred_out])
# Optimizer
adam = tf.keras.optimizers.Adam(learning_rate=self.learning_rate,
beta_1=0.9,
beta_2=0.999,
amsgrad=False)
# Model compile
if self.task == 'classification':
model.compile(loss={
'select': select_loss,
'predict': binary_cross_entropy_loss
},
optimizer=adam,
loss_weights={
'select': 0.01,
'predict': 1
})
elif self.task == 'regression':
model.compile(loss={
'select': select_loss,
'predict': rmse_loss
},
optimizer=adam,
loss_weights={
'select': 0.01,
'predict': 1
})
return model
def get_test_model_exhaustive():
"""Returns a exhaustive test model."""
input_shapes = [(2, 3, 4, 5, 6), (2, 3, 4, 5, 6), (7, 8, 9, 10),
(7, 8, 9, 10), (11, 12, 13), (11, 12, 13), (14, 15),
(14, 15), (16, ),
(16, ), (2, ), (1, ), (2, ), (1, ), (1, 3), (1, 4),
(1, 1, 3), (1, 1, 4), (1, 1, 1, 3), (1, 1, 1, 4),
(1, 1, 1, 1, 3), (1, 1, 1, 1, 4), (26, 28, 3), (4, 4, 3),
(4, 4, 3), (4, ), (2, 3), (1, ), (1, ), (1, ), (2, 3),
(9, 16, 1), (1, 9, 16)]
inputs = [Input(shape=s) for s in input_shapes]
outputs = []
outputs.append(Conv1D(1, 3, padding='valid')(inputs[6]))
outputs.append(Conv1D(2, 1, padding='same')(inputs[6]))
outputs.append(Conv1D(3, 4, padding='causal', dilation_rate=2)(inputs[6]))
outputs.append(ZeroPadding1D(2)(inputs[6]))
outputs.append(Cropping1D((2, 3))(inputs[6]))
outputs.append(MaxPooling1D(2)(inputs[6]))
outputs.append(MaxPooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(MaxPooling1D(2, data_format="channels_first")(inputs[6]))
outputs.append(AveragePooling1D(2)(inputs[6]))
outputs.append(AveragePooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(
AveragePooling1D(2, data_format="channels_first")(inputs[6]))
outputs.append(GlobalMaxPooling1D()(inputs[6]))
outputs.append(GlobalMaxPooling1D(data_format="channels_first")(inputs[6]))
outputs.append(GlobalAveragePooling1D()(inputs[6]))
outputs.append(
GlobalAveragePooling1D(data_format="channels_first")(inputs[6]))
for axis in range(1, 6):
shape = input_shapes[0][axis - 1]
outputs.append(
Normalization(axis=axis,
mean=np.random.rand(shape),
variance=np.random.rand(shape))(inputs[0]))
outputs.append(Normalization(axis=None, mean=2.1, variance=2.2)(inputs[4]))
outputs.append(Normalization(axis=-1, mean=2.1, variance=2.2)(inputs[6]))
outputs.append(Conv2D(4, (3, 3))(inputs[4]))
outputs.append(Conv2D(4, (3, 3), use_bias=False)(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4]))
outputs.append(SeparableConv2D(3, (3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((1, 2))(inputs[4]))
outputs.append(MaxPooling2D((2, 2))(inputs[4]))
# todo: check if TensorFlow >= 2.8 supports this
# outputs.append(MaxPooling2D((2, 2), data_format="channels_first")(inputs[4]))
outputs.append(
MaxPooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(AveragePooling2D((2, 2))(inputs[4]))
# todo: check if TensorFlow >= 2.8 supports this
# outputs.append(AveragePooling2D((2, 2), data_format="channels_first")(inputs[4]))
outputs.append(
AveragePooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(GlobalAveragePooling2D()(inputs[4]))
outputs.append(
GlobalAveragePooling2D(data_format="channels_first")(inputs[4]))
outputs.append(GlobalMaxPooling2D()(inputs[4]))
outputs.append(GlobalMaxPooling2D(data_format="channels_first")(inputs[4]))
outputs.append(Permute((3, 4, 1, 5, 2))(inputs[0]))
outputs.append(Permute((1, 5, 3, 2, 4))(inputs[0]))
outputs.append(Permute((3, 4, 1, 2))(inputs[2]))
outputs.append(Permute((2, 1, 3))(inputs[4]))
outputs.append(Permute((2, 1))(inputs[6]))
outputs.append(Permute((1, ))(inputs[8]))
outputs.append(Permute((3, 1, 2))(inputs[31]))
outputs.append(Permute((3, 1, 2))(inputs[32]))
outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[31])))
outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[32])))
outputs.append(BatchNormalization()(inputs[0]))
outputs.append(BatchNormalization(axis=1)(inputs[0]))
outputs.append(BatchNormalization(axis=2)(inputs[0]))
outputs.append(BatchNormalization(axis=3)(inputs[0]))
outputs.append(BatchNormalization(axis=4)(inputs[0]))
outputs.append(BatchNormalization(axis=5)(inputs[0]))
outputs.append(BatchNormalization()(inputs[2]))
outputs.append(BatchNormalization(axis=1)(inputs[2]))
outputs.append(BatchNormalization(axis=2)(inputs[2]))
outputs.append(BatchNormalization(axis=3)(inputs[2]))
outputs.append(BatchNormalization(axis=4)(inputs[2]))
outputs.append(BatchNormalization()(inputs[4]))
# todo: check if TensorFlow >= 2.1 supports this
# outputs.append(BatchNormalization(axis=1)(inputs[4])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now.
outputs.append(BatchNormalization(axis=2)(inputs[4]))
outputs.append(BatchNormalization(axis=3)(inputs[4]))
outputs.append(BatchNormalization()(inputs[6]))
outputs.append(BatchNormalization(axis=1)(inputs[6]))
outputs.append(BatchNormalization(axis=2)(inputs[6]))
outputs.append(BatchNormalization()(inputs[8]))
outputs.append(BatchNormalization(axis=1)(inputs[8]))
outputs.append(BatchNormalization()(inputs[27]))
outputs.append(BatchNormalization(axis=1)(inputs[27]))
outputs.append(BatchNormalization()(inputs[14]))
outputs.append(BatchNormalization(axis=1)(inputs[14]))
outputs.append(BatchNormalization(axis=2)(inputs[14]))
outputs.append(BatchNormalization()(inputs[16]))
# todo: check if TensorFlow >= 2.1 supports this
# outputs.append(BatchNormalization(axis=1)(inputs[16])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now.
outputs.append(BatchNormalization(axis=2)(inputs[16]))
outputs.append(BatchNormalization(axis=3)(inputs[16]))
outputs.append(BatchNormalization()(inputs[18]))
outputs.append(BatchNormalization(axis=1)(inputs[18]))
outputs.append(BatchNormalization(axis=2)(inputs[18]))
outputs.append(BatchNormalization(axis=3)(inputs[18]))
outputs.append(BatchNormalization(axis=4)(inputs[18]))
outputs.append(BatchNormalization()(inputs[20]))
outputs.append(BatchNormalization(axis=1)(inputs[20]))
outputs.append(BatchNormalization(axis=2)(inputs[20]))
outputs.append(BatchNormalization(axis=3)(inputs[20]))
outputs.append(BatchNormalization(axis=4)(inputs[20]))
outputs.append(BatchNormalization(axis=5)(inputs[20]))
outputs.append(Dropout(0.5)(inputs[4]))
outputs.append(ZeroPadding2D(2)(inputs[4]))
outputs.append(ZeroPadding2D((2, 3))(inputs[4]))
outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Cropping2D(2)(inputs[4]))
outputs.append(Cropping2D((2, 3))(inputs[4]))
outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Dense(3, use_bias=True)(inputs[13]))
outputs.append(Dense(3, use_bias=True)(inputs[14]))
outputs.append(Dense(4, use_bias=False)(inputs[16]))
outputs.append(Dense(4, use_bias=False, activation='tanh')(inputs[18]))
outputs.append(Dense(4, use_bias=False)(inputs[20]))
outputs.append(Reshape(((2 * 3 * 4 * 5 * 6), ))(inputs[0]))
outputs.append(Reshape((2, 3 * 4 * 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4 * 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4, 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4, 5, 6))(inputs[0]))
outputs.append(Reshape((16, ))(inputs[8]))
outputs.append(Reshape((2, 8))(inputs[8]))
outputs.append(Reshape((2, 2, 4))(inputs[8]))
outputs.append(Reshape((2, 2, 2, 2))(inputs[8]))
outputs.append(Reshape((2, 2, 1, 2, 2))(inputs[8]))
outputs.append(RepeatVector(3)(inputs[8]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='bilinear')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='bilinear')(inputs[4]))
outputs.append(ReLU()(inputs[0]))
for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]:
outputs.append(Concatenate(axis=axis)([inputs[0], inputs[1]]))
for axis in [-4, -3, -2, -1, 1, 2, 3, 4]:
outputs.append(Concatenate(axis=axis)([inputs[2], inputs[3]]))
for axis in [-3, -2, -1, 1, 2, 3]:
outputs.append(Concatenate(axis=axis)([inputs[4], inputs[5]]))
for axis in [-2, -1, 1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[6], inputs[7]]))
for axis in [-1, 1]:
outputs.append(Concatenate(axis=axis)([inputs[8], inputs[9]]))
for axis in [-1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]]))
for axis in [-1, 3]:
outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]]))
for axis in [-1, 4]:
outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]]))
for axis in [-1, 5]:
outputs.append(Concatenate(axis=axis)([inputs[20], inputs[21]]))
outputs.append(UpSampling1D(size=2)(inputs[6]))
# outputs.append(UpSampling1D(size=2)(inputs[8])) # ValueError: Input 0 of layer up_sampling1d_1 is incompatible with the layer: expected ndim=3, found ndim=2. Full shape received: [None, 16]
outputs.append(Multiply()([inputs[10], inputs[11]]))
outputs.append(Multiply()([inputs[11], inputs[10]]))
outputs.append(Multiply()([inputs[11], inputs[13]]))
outputs.append(Multiply()([inputs[10], inputs[11], inputs[12]]))
outputs.append(Multiply()([inputs[11], inputs[12], inputs[13]]))
shared_conv = Conv2D(1, (1, 1),
padding='valid',
name='shared_conv',
activation='relu')
up_scale_2 = UpSampling2D((2, 2))
x1 = shared_conv(up_scale_2(inputs[23])) # (1, 8, 8)
x2 = shared_conv(up_scale_2(inputs[24])) # (1, 8, 8)
x3 = Conv2D(1, (1, 1),
padding='valid')(up_scale_2(inputs[24])) # (1, 8, 8)
x = Concatenate()([x1, x2, x3]) # (3, 8, 8)
outputs.append(x)
x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8)
outputs.append(x)
x = Dropout(0.5)(x)
outputs.append(x)
x = Concatenate()([MaxPooling2D((2, 2))(x),
AveragePooling2D((2, 2))(x)]) # (6, 4, 4)
outputs.append(x)
x = Flatten()(x) # (1, 1, 96)
x = Dense(4, use_bias=False)(x)
outputs.append(x)
x = Dense(3)(x) # (1, 1, 3)
outputs.append(x)
outputs.append(Add()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Subtract()([inputs[26], inputs[30]]))
outputs.append(Multiply()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Average()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Maximum()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Concatenate()([inputs[26], inputs[30], inputs[30]]))
intermediate_input_shape = (3, )
intermediate_in = Input(intermediate_input_shape)
intermediate_x = intermediate_in
intermediate_x = Dense(8)(intermediate_x)
intermediate_x = Dense(5, name='duplicate_layer_name')(intermediate_x)
intermediate_model = Model(inputs=[intermediate_in],
outputs=[intermediate_x],
name='intermediate_model')
intermediate_model.compile(loss='mse', optimizer='nadam')
x = intermediate_model(x) # (1, 1, 5)
intermediate_model_2 = Sequential()
intermediate_model_2.add(Dense(7, input_shape=(5, )))
intermediate_model_2.add(Dense(5, name='duplicate_layer_name'))
intermediate_model_2.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_2(x) # (1, 1, 5)
intermediate_model_3_nested = Sequential()
intermediate_model_3_nested.add(Dense(7, input_shape=(6, )))
intermediate_model_3_nested.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
intermediate_model_3 = Sequential()
intermediate_model_3.add(Dense(6, input_shape=(5, )))
intermediate_model_3.add(intermediate_model_3_nested)
intermediate_model_3.add(Dense(8))
intermediate_model_3.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_3(x) # (1, 1, 8)
x = Dense(3)(x) # (1, 1, 3)
shared_activation = Activation('tanh')
outputs = outputs + [
Activation('tanh')(inputs[25]),
Activation('hard_sigmoid')(inputs[25]),
Activation('selu')(inputs[25]),
Activation('sigmoid')(inputs[25]),
Activation('softplus')(inputs[25]),
Activation('softmax')(inputs[25]),
Activation('relu')(inputs[25]),
Activation('relu6')(inputs[25]),
Activation('swish')(inputs[25]),
Activation('exponential')(inputs[25]),
Activation('gelu')(inputs[25]),
Activation('softsign')(inputs[25]),
LeakyReLU()(inputs[25]),
ReLU()(inputs[25]),
ReLU(max_value=0.4, negative_slope=1.1, threshold=0.3)(inputs[25]),
ELU()(inputs[25]),
PReLU()(inputs[24]),
PReLU()(inputs[25]),
PReLU()(inputs[26]),
shared_activation(inputs[25]),
Activation('linear')(inputs[26]),
Activation('linear')(inputs[23]),
x,
shared_activation(x),
]
model = Model(inputs=inputs, outputs=outputs, name='test_model_exhaustive')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 2
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, epochs=10)
return model
def __init__(self, n_generators=1, n_discriminators=1, **kwargs):
"""Construct a GAN.
In addition to the parameters listed below, this class accepts all the
keyword arguments from KerasModel.
Parameters
----------
n_generators: int
the number of generators to include
n_discriminators: int
the number of discriminators to include
"""
self.n_generators = n_generators
self.n_discriminators = n_discriminators
# Create the inputs.
self.noise_input = Input(shape=self.get_noise_input_shape())
self.data_inputs = []
for shape in self.get_data_input_shapes():
self.data_inputs.append(Input(shape=shape))
self.conditional_inputs = []
for shape in self.get_conditional_input_shapes():
self.conditional_inputs.append(Input(shape=shape))
# Create the generators.
self.generators = []
self.gen_variables = []
generator_outputs = []
for i in range(n_generators):
generator = self.create_generator()
self.generators.append(generator)
generator_outputs.append(
generator(
_list_or_tensor([self.noise_input] + self.conditional_inputs)))
self.gen_variables += generator.trainable_variables
# Create the discriminators.
self.discriminators = []
self.discrim_variables = []
discrim_train_outputs = []
discrim_gen_outputs = []
for i in range(n_discriminators):
discriminator = self.create_discriminator()
self.discriminators.append(discriminator)
discrim_train_outputs.append(
discriminator(
_list_or_tensor(self.data_inputs + self.conditional_inputs)))
for gen_output in generator_outputs:
if isinstance(gen_output, tf.Tensor):
gen_output = [gen_output]
discrim_gen_outputs.append(
discriminator(
_list_or_tensor(gen_output + self.conditional_inputs)))
self.discrim_variables += discriminator.trainable_variables
# Compute the loss functions.
gen_losses = [self.create_generator_loss(d) for d in discrim_gen_outputs]
discrim_losses = []
for i in range(n_discriminators):
for j in range(n_generators):
discrim_losses.append(
self.create_discriminator_loss(
discrim_train_outputs[i],
discrim_gen_outputs[i * n_generators + j]))
if n_generators == 1 and n_discriminators == 1:
total_gen_loss = gen_losses[0]
total_discrim_loss = discrim_losses[0]
else:
# Create learnable weights for the generators and discriminators.
gen_alpha = layers.Variable(np.ones((1, n_generators)), dtype=tf.float32)
# We pass an input to the Variable layer to work around a bug in TF 1.14.
gen_weights = Softmax()(gen_alpha([self.noise_input]))
discrim_alpha = layers.Variable(
np.ones((1, n_discriminators)), dtype=tf.float32)
discrim_weights = Softmax()(discrim_alpha([self.noise_input]))
# Compute the weighted errors
weight_products = Reshape((n_generators * n_discriminators,))(Multiply()([
Reshape((n_discriminators, 1))(discrim_weights),
Reshape((1, n_generators))(gen_weights)
]))
stacked_gen_loss = layers.Stack(axis=0)(gen_losses)
stacked_discrim_loss = layers.Stack(axis=0)(discrim_losses)
total_gen_loss = Lambda(lambda x: tf.reduce_sum(x[0] * x[1]))(
[stacked_gen_loss, weight_products])
total_discrim_loss = Lambda(lambda x: tf.reduce_sum(x[0] * x[1]))(
[stacked_discrim_loss, weight_products])
self.gen_variables += gen_alpha.trainable_variables
self.discrim_variables += gen_alpha.trainable_variables
self.discrim_variables += discrim_alpha.trainable_variables
# Add an entropy term to the loss.
entropy = Lambda(lambda x: -(tf.reduce_sum(tf.log(x[0]))/n_generators +
tf.reduce_sum(tf.log(x[1]))/n_discriminators))([gen_weights, discrim_weights])
total_discrim_loss = Lambda(lambda x: x[0] + x[1])(
[total_discrim_loss, entropy])
# Create the Keras model.
inputs = [self.noise_input] + self.data_inputs + self.conditional_inputs
outputs = [total_gen_loss, total_discrim_loss]
self.gen_loss_fn = lambda outputs, labels, weights: outputs[0]
self.discrim_loss_fn = lambda outputs, labels, weights: outputs[1]
model = tf.keras.Model(inputs=inputs, outputs=outputs)
super(GAN, self).__init__(model, self.gen_loss_fn, **kwargs)
def dense_embedding_quantized(n_features=6,
n_features_cat=2,
number_of_pupcandis=100,
embedding_input_dim={0: 13, 1: 3},
emb_out_dim=2,
with_bias=True,
t_mode=0,
logit_total_bits=7,
logit_int_bits=2,
activation_total_bits=7,
logit_quantizer='quantized_bits',
activation_quantizer='quantized_relu',
activation_int_bits=2,
alpha=1,
use_stochastic_rounding=False,
units=[64, 32, 16]):
n_dense_layers = len(units)
logit_quantizer = getattr(qkeras.quantizers, logit_quantizer)(logit_total_bits, logit_int_bits, alpha=alpha, use_stochastic_rounding=use_stochastic_rounding)
activation_quantizer = getattr(qkeras.quantizers, activation_quantizer)(activation_total_bits, activation_int_bits)
inputs_cont = Input(shape=(number_of_pupcandis, n_features-2), name='input')
pxpy = Input(shape=(number_of_pupcandis, 2), name='input_pxpy')
embeddings = []
inputs = [inputs_cont, pxpy]
for i_emb in range(n_features_cat):
input_cat = Input(shape=(number_of_pupcandis, 1), name='input_cat{}'.format(i_emb))
inputs.append(input_cat)
embedding = Embedding(
input_dim=embedding_input_dim[i_emb],
output_dim=emb_out_dim,
embeddings_initializer=initializers.RandomNormal(
mean=0,
stddev=0.4/emb_out_dim),
name='embedding{}'.format(i_emb))(input_cat)
embedding = Reshape((number_of_pupcandis, emb_out_dim))(embedding)
embeddings.append(embedding)
x = Concatenate()([inputs_cont] + [emb for emb in embeddings])
for i_dense in range(n_dense_layers):
x = QDense(units[i_dense], kernel_quantizer=logit_quantizer, bias_quantizer=logit_quantizer, kernel_initializer='lecun_uniform')(x)
x = BatchNormalization(momentum=0.95)(x)
x = QActivation(activation=activation_quantizer)(x)
if t_mode == 0:
x = qkeras.qpooling.QGlobalAveragePooling1D(name='pool', quantizer=logit_quantizer)(x)
# pool size?
outputs = QDense(2, name='output', bias_quantizer=logit_quantizer, kernel_quantizer=logit_quantizer, activation='linear')(x)
if t_mode == 1:
if with_bias:
b = QDense(2, name='met_bias', kernel_quantizer=logit_quantizer, bias_quantizer=logit_quantizer, kernel_initializer=initializers.VarianceScaling(scale=0.02))(x)
b = QActivation(activation='linear')(b)
pxpy = Add()([pxpy, b])
w = QDense(1, name='met_weight', kernel_quantizer=logit_quantizer, bias_quantizer=logit_quantizer, kernel_initializer=initializers.VarianceScaling(scale=0.02))(x)
w = QActivation(activation='linear')(w)
w = BatchNormalization(trainable=False, name='met_weight_minus_one', epsilon=False)(w)
x = Multiply()([w, pxpy])
x = GlobalAveragePooling1D(name='output')(x)
outputs = x
keras_model = Model(inputs=inputs, outputs=outputs)
keras_model.get_layer('met_weight_minus_one').set_weights([np.array([1.]), np.array([-1.]), np.array([0.]), np.array([1.])])
return keras_model
def __init__(self, masked_outputs=True):
super(HCIModel, self).__init__()
self.masked_outputs = masked_outputs
self.normalize_images = Lambda(lambda x: x / 256 - 0.5)
####################################################################################################################
# First 10 layers of VGG19
self.vgg_conv_1 = conv(64, 3, 'vgg_conv_1')
self.vgg_conv_2 = conv(64, 3, 'vgg_conv_2')
self.vgg_pool_1 = MaxPooling2D((2, 2),
strides=(2, 2),
name='vgg_pool_1')
self.vgg_conv_3 = conv(128, 3, 'vgg_conv_3')
self.vgg_conv_4 = conv(128, 3, 'vgg_conv_4')
self.vgg_pool_2 = MaxPooling2D((2, 2),
strides=(2, 2),
name='vgg_pool_2')
self.vgg_conv_5 = conv(256, 3, 'vgg_conv_5')
self.vgg_conv_6 = conv(256, 3, 'vgg_conv_6')
self.vgg_conv_7 = conv(256, 3, 'vgg_conv_7')
self.vgg_conv_8 = conv(256, 3, 'vgg_conv_8')
self.vgg_pool_3 = MaxPooling2D((2, 2),
strides=(2, 2),
name='vgg_pool_3')
self.vgg_conv_9 = conv(512, 3, 'vgg_conv_9')
self.vgg_conv_10 = conv(512, 3, 'vgg_conv_10')
# Convolutional Pose Machine (CPM) Layers
self.cpm_conv_1 = conv(256, 3, 'cpm_conv_1')
self.cpm_conv_2 = conv(128, 3, 'cpm_conv_2')
# HCIPose Stage 1
self.stage_1_branch_1_conv_1 = conv(128, 3, "stage_1_branch_1_conv_1")
self.stage_1_branch_1_conv_2 = conv(128, 3, "stage_1_branch_1_conv_2")
self.stage_1_branch_1_conv_3 = conv(128, 3, "stage_1_branch_1_conv_3")
self.stage_1_branch_1_conv_4 = conv(512, 1, "stage_1_branch_1_conv_4")
self.stage_1_branch_1_conv_5 = conv(19,
1,
"stage_1_branch_1_conv_5",
activation=None)
self.stage_1_branch_2_conv_1 = conv(128, 3, "stage_1_branch_2_conv_1")
self.stage_1_branch_2_conv_2 = conv(128, 3, "stage_1_branch_2_conv_2")
self.stage_1_branch_2_conv_3 = conv(128, 3, "stage_1_branch_2_conv_3")
self.stage_1_branch_2_conv_4 = conv(512, 1, "stage_1_branch_2_conv_4")
self.stage_1_branch_2_conv_5 = conv(38,
1,
"stage_1_branch_2_conv_5",
activation=None)
self.stage_1_concat = Concatenate(axis=3)
# HCIPose Stage 2
self.stage_2_branch_1_conv_1 = conv(128, 7, "stage_2_branch_1_conv_1")
self.stage_2_branch_1_conv_2 = conv(128, 7, "stage_2_branch_1_conv_2")
self.stage_2_branch_1_conv_3 = conv(128, 7, "stage_2_branch_1_conv_3")
self.stage_2_branch_1_conv_4 = conv(128, 7, "stage_2_branch_1_conv_4")
self.stage_2_branch_1_conv_5 = conv(128, 7, "stage_2_branch_1_conv_5")
self.stage_2_branch_1_conv_6 = conv(128, 1, "stage_2_branch_1_conv_6")
self.stage_2_branch_1_conv_7 = conv(19,
1,
"stage_2_branch_1_conv_7",
activation=None)
self.stage_2_branch_2_conv_1 = conv(128, 7, "stage_2_branch_2_conv_1")
self.stage_2_branch_2_conv_2 = conv(128, 7, "stage_2_branch_2_conv_2")
self.stage_2_branch_2_conv_3 = conv(128, 7, "stage_2_branch_2_conv_3")
self.stage_2_branch_2_conv_4 = conv(128, 7, "stage_2_branch_2_conv_4")
self.stage_2_branch_2_conv_5 = conv(128, 7, "stage_2_branch_2_conv_5")
self.stage_2_branch_2_conv_6 = conv(128, 1, "stage_2_branch_2_conv_6")
self.stage_2_branch_2_conv_7 = conv(38,
1,
"stage_2_branch_2_conv_7",
activation=None)
self.stage_2_concat = Concatenate(axis=3)
# HCIPose Stage 3
self.stage_3_branch_1_conv_1 = conv(128, 7, "stage_3_branch_1_conv_1")
self.stage_3_branch_1_conv_2 = conv(128, 7, "stage_3_branch_1_conv_2")
self.stage_3_branch_1_conv_3 = conv(128, 7, "stage_3_branch_1_conv_3")
self.stage_3_branch_1_conv_4 = conv(128, 7, "stage_3_branch_1_conv_4")
self.stage_3_branch_1_conv_5 = conv(128, 7, "stage_3_branch_1_conv_5")
self.stage_3_branch_1_conv_6 = conv(128, 1, "stage_3_branch_1_conv_6")
self.stage_3_branch_1_conv_7 = conv(19,
1,
"stage_3_branch_1_conv_7",
activation=None)
self.stage_3_branch_2_conv_1 = conv(128, 7, "stage_3_branch_2_conv_1")
self.stage_3_branch_2_conv_2 = conv(128, 7, "stage_3_branch_2_conv_2")
self.stage_3_branch_2_conv_3 = conv(128, 7, "stage_3_branch_2_conv_3")
self.stage_3_branch_2_conv_4 = conv(128, 7, "stage_3_branch_2_conv_4")
self.stage_3_branch_2_conv_5 = conv(128, 7, "stage_3_branch_2_conv_5")
self.stage_3_branch_2_conv_6 = conv(128, 1, "stage_3_branch_2_conv_6")
self.stage_3_branch_2_conv_7 = conv(38,
1,
"stage_3_branch_2_conv_7",
activation=None)
self.stage_3_concat = Concatenate(axis=3)
# HCIPose Stage 4
self.stage_4_branch_1_conv_1 = conv(128, 7, "stage_4_branch_1_conv_1")
self.stage_4_branch_1_conv_2 = conv(128, 7, "stage_4_branch_1_conv_2")
self.stage_4_branch_1_conv_3 = conv(128, 7, "stage_4_branch_1_conv_3")
self.stage_4_branch_1_conv_4 = conv(128, 7, "stage_4_branch_1_conv_4")
self.stage_4_branch_1_conv_5 = conv(128, 7, "stage_4_branch_1_conv_5")
self.stage_4_branch_1_conv_6 = conv(128, 1, "stage_4_branch_1_conv_6")
self.stage_4_branch_1_conv_7 = conv(19,
1,
"stage_4_branch_1_conv_7",
activation=None)
self.stage_4_branch_2_conv_1 = conv(128, 7, "stage_4_branch_2_conv_1")
self.stage_4_branch_2_conv_2 = conv(128, 7, "stage_4_branch_2_conv_2")
self.stage_4_branch_2_conv_3 = conv(128, 7, "stage_4_branch_2_conv_3")
self.stage_4_branch_2_conv_4 = conv(128, 7, "stage_4_branch_2_conv_4")
self.stage_4_branch_2_conv_5 = conv(128, 7, "stage_4_branch_2_conv_5")
self.stage_4_branch_2_conv_6 = conv(128, 1, "stage_4_branch_2_conv_6")
self.stage_4_branch_2_conv_7 = conv(38,
1,
"stage_4_branch_2_conv_7",
activation=None)
self.stage_4_concat = Concatenate(axis=3)
# HCIPose Stage 5
self.stage_5_branch_1_conv_1 = conv(128, 7, "stage_5_branch_1_conv_1")
self.stage_5_branch_1_conv_2 = conv(128, 7, "stage_5_branch_1_conv_2")
self.stage_5_branch_1_conv_3 = conv(128, 7, "stage_5_branch_1_conv_3")
self.stage_5_branch_1_conv_4 = conv(128, 7, "stage_5_branch_1_conv_4")
self.stage_5_branch_1_conv_5 = conv(128, 7, "stage_5_branch_1_conv_5")
self.stage_5_branch_1_conv_6 = conv(128, 1, "stage_5_branch_1_conv_6")
self.stage_5_branch_1_conv_7 = conv(19,
1,
"stage_5_branch_1_conv_7",
activation=None)
self.stage_5_branch_2_conv_1 = conv(128, 7, "stage_5_branch_2_conv_1")
self.stage_5_branch_2_conv_2 = conv(128, 7, "stage_5_branch_2_conv_2")
self.stage_5_branch_2_conv_3 = conv(128, 7, "stage_5_branch_2_conv_3")
self.stage_5_branch_2_conv_4 = conv(128, 7, "stage_5_branch_2_conv_4")
self.stage_5_branch_2_conv_5 = conv(128, 7, "stage_5_branch_2_conv_5")
self.stage_5_branch_2_conv_6 = conv(128, 1, "stage_5_branch_2_conv_6")
self.stage_5_branch_2_conv_7 = conv(38,
1,
"stage_5_branch_2_conv_7",
activation=None)
self.stage_5_concat = Concatenate(axis=3)
# HCIPose Stage 6
self.stage_6_branch_1_conv_1 = conv(128, 7, "stage_6_branch_1_conv_1")
self.stage_6_branch_1_conv_2 = conv(128, 7, "stage_6_branch_1_conv_2")
self.stage_6_branch_1_conv_3 = conv(128, 7, "stage_6_branch_1_conv_3")
self.stage_6_branch_1_conv_4 = conv(128, 7, "stage_6_branch_1_conv_4")
self.stage_6_branch_1_conv_5 = conv(128, 7, "stage_6_branch_1_conv_5")
self.stage_6_branch_1_conv_6 = conv(128, 1, "stage_6_branch_1_conv_6")
self.stage_6_branch_1_conv_7 = conv(19,
1,
"stage_6_branch_1_conv_7",
activation=None)
self.stage_6_branch_2_conv_1 = conv(128, 7, "stage_6_branch_2_conv_1")
self.stage_6_branch_2_conv_2 = conv(128, 7, "stage_6_branch_2_conv_2")
self.stage_6_branch_2_conv_3 = conv(128, 7, "stage_6_branch_2_conv_3")
self.stage_6_branch_2_conv_4 = conv(128, 7, "stage_6_branch_2_conv_4")
self.stage_6_branch_2_conv_5 = conv(128, 7, "stage_6_branch_2_conv_5")
self.stage_6_branch_2_conv_6 = conv(128, 1, "stage_6_branch_2_conv_6")
self.stage_6_branch_2_conv_7 = conv(38,
1,
"stage_6_branch_2_conv_7",
activation=None)
if self.masked_outputs:
self.masked_1_branch_1 = Multiply()
self.masked_2_branch_1 = Multiply()
self.masked_3_branch_1 = Multiply()
self.masked_4_branch_1 = Multiply()
self.masked_5_branch_1 = Multiply()
self.masked_6_branch_1 = Multiply()
self.masked_1_branch_2 = Multiply()
self.masked_2_branch_2 = Multiply()
self.masked_3_branch_2 = Multiply()
self.masked_4_branch_2 = Multiply()
self.masked_5_branch_2 = Multiply()
self.masked_6_branch_2 = Multiply()
def attention_3d_block(inputs):
a = Permute((2, 1))(inputs)
a = Dense(3, activation='softmax')(a)
a_probs = Permute((2, 1))(a)
output_attention_mul = Multiply()([inputs, a_probs])
return output_attention_mul
def yoloNano(anchors,
input_size=416,
include_attention=True,
num_classes=1,
expension=.75,
decay=0.0005):
#f**k tensorflow 2.x
#backbone
input_0 = Input(shape=(None, None, 3))
input_gt = [
Input(shape=(None, None, len(anchors) // 3, num_classes + 5))
for l in range(3)
]
x = Conv2D(filters=12,
strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(input_0)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=24,
strides=(2, 2),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_0 = LeakyReLU(alpha=0.1)(x)
#PEP(7)(208x208x24)
x = Conv2D(filters=7,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_0)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(24 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=24,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = Add()([x_0, x])
#EP(104x104x70)
x = Conv2D(filters=math.ceil(70 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(2, 2),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=70,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x_1 = BatchNormalization()(x)
#PEP(25)(104x104x70)
x = Conv2D(filters=25,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_1)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(70 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=70,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_2 = Add()([x_1, x])
# PEP(24)(104x104x70)
x = Conv2D(filters=24,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_2)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(70 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=70,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = Add()([x_2, x])
# EP(52x52x150)
x = Conv2D(filters=math.ceil(150 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(2, 2),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=150,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x_3 = BatchNormalization()(x)
# PEP(56)(52x52x150)
x = Conv2D(filters=56,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_3)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(150 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=150,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = Add()([x_3, x])
#Conv1x1
x = Conv2D(filters=150,
kernel_size=(1, 1),
strides=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_4 = LeakyReLU(alpha=0.1)(x)
#FCA(8)
if include_attention:
x = GlobalAveragePooling2D()(x_4)
x = Dense(units=150 // 8,
activation='relu',
use_bias=False,
kernel_regularizer=l2(l=decay))(x)
x = Dense(units=150,
activation='sigmoid',
use_bias=False,
kernel_regularizer=l2(l=decay))(x)
x_5 = Multiply()([x_4, x])
else:
x_5 = x_4
#PEP(73)(52x52x150)
x = Conv2D(filters=73,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_5)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(150 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=150,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_6 = Add()([x_5, x])
# PEP(71)(52x52x150)
x = Conv2D(filters=71,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_6)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(150 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=150,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_7 = Add()([x_6, x])
# PEP(75)(52x52x150)
x = Conv2D(filters=75,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_7)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(150 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=150,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_8 = Add()([x_7, x]) #output 52x52x150
#EP(26x26x325)
x = Conv2D(filters=math.ceil(325 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_8)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(2, 2),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=325,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x_9 = BatchNormalization()(x)
# PEP(132)(26x26x325)
x = Conv2D(filters=132,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_9)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=325,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_10 = Add()([x_9, x])
# PEP(124)(26x26x325)
x = Conv2D(filters=124,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_10)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=325,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_11 = Add()([x_10, x])
# PEP(141)(26x26x325)
x = Conv2D(filters=141,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_11)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=325,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_12 = Add()([x_11, x])
# PEP(140)(26x26x325)
x = Conv2D(filters=140,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_12)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=325,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_13 = Add()([x_12, x])
# PEP(137)(26x26x325)
x = Conv2D(filters=137,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_13)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=325,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_14 = Add()([x_13, x])
# PEP(135)(26x26x325)
x = Conv2D(filters=135,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_14)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=325,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_15 = Add()([x_14, x])
# PEP(133)(26x26x325)
x = Conv2D(filters=133,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_15)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=325,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_16 = Add()([x_15, x])
# PEP(140)(26x26x325)
x = Conv2D(filters=140,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_16)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=325,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_17 = Add()([x_16, x]) #output 26x26x325
# EP(13x13x545)
x = Conv2D(filters=math.ceil(545 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_17)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(2, 2),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=545,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x_18 = BatchNormalization()(x)
# PEP(276)(13x13x545)
x = Conv2D(filters=276,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_18)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(545 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=545,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_19 = Add()([x_18, x])
#Conv1x1
x = Conv2D(filters=230,
kernel_size=(1, 1),
strides=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_19)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
# EP(13x13x489)
x = Conv2D(filters=math.ceil(489 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=489,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# PEP(213)(13x13x469)
x = Conv2D(filters=213,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(469 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=469,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# Conv1x1
x = Conv2D(filters=189,
kernel_size=(1, 1),
strides=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_20 = LeakyReLU(alpha=0.1)(x) #output 13x13x189
# EP(13x13x462)
x = Conv2D(filters=math.ceil(462 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_20)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=462,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# feature 13x13x[(num_classes+5)x3]
feature_13x13 = Conv2D(filters=3 * (num_classes + 5),
kernel_size=(1, 1),
strides=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
# Conv1x1
x = Conv2D(filters=105,
kernel_size=(1, 1),
strides=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_20)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
# upsampling 26x26x105
x = UpSampling2D()(x)
# concatenate
x = Concatenate()([x, x_17])
# PEP(113)(26x26x325)
x = Conv2D(filters=113,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=325,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# PEP(99)(26x26x207)
x = Conv2D(filters=99,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(207 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=207,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# Conv1x1
x = Conv2D(filters=98,
kernel_size=(1, 1),
strides=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_21 = LeakyReLU(alpha=0.1)(x)
# EP(13x13x183)
x = Conv2D(filters=math.ceil(183 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_21)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=183,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# feature 26x26x[(num_classes+5)x3]
feature_26x26 = Conv2D(filters=3 * (num_classes + 5),
kernel_size=(1, 1),
strides=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
# Conv1x1
x = Conv2D(filters=47,
kernel_size=(1, 1),
strides=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x_21)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
#upsampling
x = UpSampling2D()(x)
#concatenate
x = Concatenate()([x, x_8])
# PEP(58)(52x52x132)
x = Conv2D(filters=58,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(132 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=132,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# PEP(52)(52x52x87)
x = Conv2D(filters=52,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(87 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=87,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# PEP(47)(52x52x93)
x = Conv2D(filters=47,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=math.ceil(93 * expension),
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = DepthwiseConv2D(strides=(1, 1),
kernel_size=(3, 3),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = Conv2D(filters=93,
strides=(1, 1),
kernel_size=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
feature_52x52 = Conv2D(filters=3 * (num_classes + 5),
kernel_size=(1, 1),
strides=(1, 1),
use_bias=False,
padding='same',
kernel_regularizer=l2(l=decay))(x)
#loss layer
loss = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': num_classes,
'ignore_thresh': 0.5
})([feature_13x13, feature_26x26, feature_52x52, *input_gt])
debug_model = tf.keras.Model(
inputs=input_0, outputs=[feature_13x13, feature_26x26, feature_52x52])
train_model = tf.keras.Model(inputs=[input_0, *input_gt], outputs=loss)
return train_model, debug_model
# import numpy as np
# anchors = np.array([[6.,9.],[8.,13.],[11.,16.],[14.,22.],[17.,37.],[21.,26.],[29.,38.],[39.,62.],[79.,99.]],dtype='float32')
# model,_ = yoloNano(anchors,input_size=416,num_classes=1)
# model.summary()
def hard_swish(x):
return Multiply()([Activation(hard_sigmoid)(x), x])
batch_y_train[current_size] = spectClear[k]
current_size += 1
if current_size >= batch_size:
break
return batch_x_train, batch_y_train
print('Build model...')
if os.path.exists(model_name):
print("Load: " + model_name)
model = load_model(model_name)
else:
main_input = Input(shape=(int(frame_length / 2 + 1)), name='main_input')
x = main_input
x = Dense(int(frame_length / 2 + 1), activation='sigmoid')(x)
x = Multiply()([x, main_input])
model = Model(inputs=main_input, outputs=x)
tf.keras.utils.plot_model(model,
to_file='model_dense_gain.png',
show_shapes=True)
model.compile(loss='mse', metrics='mse', optimizer='adam')
print('Train...')
history = model.fit(MySequence(x_train, x_train_count, batch_size),
epochs=epochs,
steps_per_epoch=(x_train_count // batch_size))
model.save(model_name)
metrics = history.history
plt.plot(history.epoch, metrics['mse'])
plt.legend(['mse'])
def ResCNN(inputshape, cfg):
"""Build a SampleCNN model."""
# Variable-length input for feature visualization.
# x_in = Input(shape=(None, 1), name='input')
x_in = Input(inputshape, name='input')
# RES block
num_features = 512
x = Conv1D(num_features,
kernel_size=3,
strides=3,
padding='same',
use_bias=True,
kernel_regularizer=l2(cfg.weight_decay),
kernel_initializer=taejun_uniform(scale=1.),
name='conv0')(x_in)
shortcut = BatchNormalization(name='norm0')(x)
x = Activation('relu', name='relu0')(shortcut) #(None,512)
if cfg.amplifying_ratio:
x = Multiply(name=f're_scale')(
[x, squeeze_excitation(x, cfg.amplifying_ratio, name='re')])
x = Add(name=f're_scut')([shortcut, x])
x = Activation('relu', name=f're_relu')(x)
x = MaxPool1D(pool_size=3, name=f're_pool')(x)
x = Lambda(lambda y: K.expand_dims(y, -1), name='expand_dim')(x)
# ResCNN
x = Conv2D(8, (3, 3), strides=(3, 3), padding='same', name='conv1')(x)
x = BatchNormalization(name='norm1')(x)
x = Activation('relu', name='relu1')(x)
# x = MaxPool2D((3,3),strides=(3,3),padding='same',name='pool1')(x)
x = Conv2D(8, (3, 3), strides=(3, 3), padding='same', name='conv2')(x)
x = BatchNormalization(name='norm2')(x)
x = Activation('relu', name='relu2')(x)
x = MaxPool2D((3, 3), strides=(3, 3), padding='same', name='pool2')(x)
x = res_conv_block(x, [64, 64, 256],
strides=(2, 2),
cfg=cfg,
name='block1')
x = res_conv_block(x, [64, 64, 256],
strides=(2, 2),
cfg=cfg,
name='block2')
x = res_conv_block(x, [64, 64, 256],
strides=(2, 2),
cfg=cfg,
name='block3')
x = res_conv_block(x, [128, 128, 512],
strides=(2, 2),
cfg=cfg,
name='block4')
x = res_conv_block(x, [128, 128, 512],
strides=(2, 2),
cfg=cfg,
name='block5')
x = res_conv_block(x, [128, 128, 512],
strides=(2, 2),
cfg=cfg,
name='block6')
# 减少维数
x = Lambda(lambda y: K.mean(y, axis=[1, 2]), name='avgpool')(x)
# The final two FCs.
x = Dense(x.shape[-1].value,
kernel_initializer='glorot_uniform',
name='final_fc')(x)
x = BatchNormalization(name='final_norm')(x)
x = Activation('relu', name='final_relu')(x)
if cfg.dropout > 0.:
# x = Dropout(cfg.dropout, name='final_drop')(x)
x = Dropout(0.2, name='final_drop')(x)
x = Dense(cfg.num_classes,
kernel_initializer='glorot_uniform',
name='logit')(x)
x = Activation(cfg.activation, name='pred')(x)
return Model(inputs=[x_in], outputs=[x], name='ResCNN')
h = Dense(q, activation='relu', trainable=False)(inputs)
outputs = Dense(1)(h)
elif architecture == 'RF_gaussian':
h = Dense(q,
activation=tf.cos,
trainable=False,
kernel_initializer=tf.keras.initializers.RandomNormal(
mean=0., stddev=4 / np.sqrt(input_shape[0])),
bias_initializer=tf.keras.initializers.RandomUniform(
minval=0., maxval=2 * np.pi))(inputs)
outputs = Dense(1)(h)
elif architecture == 'GaLU':
h = Dense(q)(inputs)
gate = Dense(q, trainable=False, bias_initializer='zeros')(inputs)
gate = Lambda(lambda x: K.cast(K.greater(x, 0), 'float32'))(gate)
h = Multiply()([gate, h])
outputs = Dense(1)(h)
model = keras.Model(inputs=inputs, outputs=outputs, name="mnist_model")
model.compile(loss=keras.losses.hinge,
optimizer=keras.optimizers.Adadelta(),
metrics=[binary_accuracy])
scores = []
for e in range(epochs):
cur_x_train, cur_y_train = build_dataset(x_train, y_train, k)
cur_x_test, cur_y_test = build_dataset(x_test, y_test, k)
model.fit(cur_x_train,
cur_y_train,
def ernet():
inputs = Input(shape=(7, None, None, 1)) # 32
fea3d = res(inputs, 64, (1, 3, 3), (3, 1, 1))
att_avg3d = GlobalAveragePooling3D()(fea3d)
att_max3d = GlobalMaxPooling3D()(fea3d)
att_avg3d = Dense(64, activation='relu')(att_avg3d)
att_avg3d = Dense(64, activation='relu')(att_avg3d)
att_max3d = Dense(64, activation='relu')(att_max3d)
att_max3d = Dense(64, activation='relu')(att_max3d)
att3d = Add()([att_avg3d, att_max3d])
fea3d = Multiply()([fea3d, att3d])
att = layers.Conv3D(128, (7, 1, 1), activation='relu')(fea3d)
fea = Lambda(lambda x: x[:, 0, :, :, :])(att)
att_avg = GlobalAveragePooling2D()(fea)
att_max = GlobalMaxPooling2D()(fea)
att_avg = Dense(128, activation='relu')(att_avg)
att_avg = Dense(128, activation='relu')(att_avg)
att_max = Dense(128, activation='relu')(att_max)
att_max = Dense(128, activation='relu')(att_max)
att = Add()([att_avg, att_max])
fea = Multiply()([fea, att])
avg_pool = Lambda(lambda x: K.mean(x, axis=3, keepdims=True))(fea)
max_pool = Lambda(lambda x: K.max(x, axis=3, keepdims=True))(fea)
att = Concatenate(axis=3)([avg_pool, max_pool])
att = Conv2D(filters=1, kernel_size=3, padding='same',
activation='sigmoid')(att)
fea = Multiply()([fea, att])
res1 = res(inputs, 64, (1, 3, 3), (3, 1, 1))
Pooling1 = layers.MaxPooling3D((1, 2, 2))(res1) # 16
res2 = res(Pooling1, 128, (1, 3, 3), (3, 1, 1))
Pooling2 = layers.MaxPooling3D((1, 2, 2))(res2) # 8
res3 = res(Pooling2, 256, (1, 3, 3), (3, 1, 1))
Pooling3 = layers.MaxPooling3D((1, 2, 2))(res3) # 4
res4 = res(Pooling3, 256, (1, 3, 3), (3, 1, 1))
Pooling4 = layers.MaxPooling3D((1, 2, 2))(res4) # 2
Conv3D_5 = layers.Conv3D(
256, (1, 1, 1), padding='same', activation='relu')(Pooling4) # 2
upres1 = upres(Conv3D_5, 256, (1, 4, 4), (1, 2, 2),
(1, 3, 3), (3, 1, 1), res4)
upres2 = upres(upres1, 256, (1, 4, 4), (1, 2, 2),
(1, 3, 3), (3, 1, 1), res3)
upres3 = upres(upres2, 128, (1, 4, 4), (1, 2, 2),
(1, 3, 3), (3, 1, 1), res2)
upres4 = upres(upres3, 64, (1, 4, 4), (1, 2, 2),
(1, 3, 3), (3, 1, 1), res1)
inter = layers.Conv3D(1, (7, 1, 1), activation='relu')(upres4) # 32
inter = Add()([att, inter])
x = Lambda(lambda x: x[:, 0, :, :, :])(inter)
res2d0 = res2d(x, 64, 1, 1, 1)
res2d1 = res2d(res2d0, 64, 3, 4, 2)
res2d2 = res2d(res2d1, 128, 3, 4, 2)
res2d3 = res2d(res2d2, 256, 3, 4, 2)
res2d4 = res2d(res2d3, 256, 3, 4, 2)
x = Conv2D(filters=512, kernel_size=(1, 1), strides=1,
padding='same', activation='relu')(res2d4) # 2
upres2d1 = upres2d(x, 256, 4, 2, res2d3, 3)
upres2d2 = upres2d(upres2d1, 256, 4, 2, res2d2, 3)
upres2d3 = upres2d(upres2d2, 128, 4, 2, res2d1, 3)
upres2d4 = upres2d(upres2d3, 64, 4, 2, res2d0, 3)
x = Conv2D(filters=1, kernel_size=1, strides=1)(upres2d4)
# x = Lambda(lambda x: K.mean(x, axis=3, keepdims=True))(upres2d4) # 32
model = Model(inputs, x)
return model
def get_model():
dvec_inp = Input(shape=[emb_dim],name="dvec")
input_spec = Input(shape=[T_dim,num_freq],name="input_spec")
x = Reshape((T_dim,num_freq,1))(input_spec)
# cnn
#cnn1
x = ZeroPadding2D(((0,0), (3,3)))(x)
x = Conv2D(filters=64, kernel_size=[1,7], dilation_rate=[1, 1])(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
#cnn2
x = ZeroPadding2D(((3,3), (0,0)))(x)
x = Conv2D(filters=64, kernel_size=[7,1], dilation_rate=[1, 1])(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
#cnn3
x = ZeroPadding2D(((2,2), (2,2)))(x)
x = Conv2D(filters=64, kernel_size=[5,5], dilation_rate=[1, 1])(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
#cnn4
x = ZeroPadding2D(((4,4), (2,2)))(x)
x = Conv2D(filters=64, kernel_size=[5,5], dilation_rate=[2, 1])(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
#cnn5
x = ZeroPadding2D(((8,8), (2,2)))(x)
x = Conv2D(filters=64, kernel_size=[5,5], dilation_rate=[4, 1])(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
#cnn6
x = ZeroPadding2D(((16,16), (2,2)))(x)
x = Conv2D(filters=64, kernel_size=[5,5], dilation_rate=[8, 1])(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
#cnn7
x = ZeroPadding2D(((32,32), (2,2)))(x)
x = Conv2D(filters=64, kernel_size=[5,5], dilation_rate=[16, 1])(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
#cnn8
x = Conv2D(filters=8, kernel_size=[1,1], dilation_rate=[1, 1])(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Reshape((x.shape[1],x.shape[2]*x.shape[3]))(x) #else use -1 as last arg
#x = tf.reshape(x, [x.shape[0],x.shape[1],-1])
dvec = Lambda(lambda a : tf.expand_dims(a,1))(dvec_inp)
dvec = Lambda(lambda a : tf.repeat(a,repeats =x.shape[1],axis =1))(dvec)
#dvec= tf.expand_dims(dvec_inp,1)
#dvec= tf.repeat(dvec,repeats =x.shape[1],axis =1)
x = concatenate([x,dvec],-1)
#x= tf.concat([x,dvec],-1)
#lstm
x = Bidirectional(LSTM(lstm_dim,return_sequences=True))(x)
#fc1
x = Dense(fc1_dim,activation ="relu")(x)
#fc2
mask = Dense(fc2_dim,activation ="sigmoid",name="mask")(x) #soft mask
#element-wise
output = Multiply()([input_spec,mask])
model = Model(inputs=[input_spec,dvec_inp], outputs=output)
return model
def call(self, x, mask=None):
return Multiply()([x, tf.cast(K.expand_dims(mask, -1), tf.float32)])
def attention_weights(input_uid, input_iid, x_u, x_i, user_num, item_num,
embed_id_dim, random_seed, attention_size,
l2_reg_lambda):
"""
Compute the weights generated from attention layers for each review
Args:
-------
input_uid: user's id
input_iid: item's id
x_u: semantics of user's reviews
x_i: semantics of item's reviews
other_args: model parameters
Outputs:
-------
: weights (usefulness) of user's reviews and item's reviews
"""
vec_uid = Embedding(user_num + 2,
embed_id_dim,
embeddings_initializer=RandomUniform(minval=-0.1,
maxval=0.1,
seed=random_seed),
name='user_id_embed')(input_uid)
vec_iid = Embedding(item_num + 2,
embed_id_dim,
embeddings_initializer=RandomUniform(minval=-0.1,
maxval=0.1,
seed=random_seed),
name='item_id_embed')(input_iid)
# Mapping user/item ID vectors and semantics of user/item's reviews to the attention space
vec_uid = Dense(attention_size,
activation=None,
use_bias=False,
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(l2_reg_lambda),
name='user_id_attention')(vec_uid)
vec_iid = Dense(attention_size,
activation=None,
use_bias=False,
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(l2_reg_lambda),
name='item_id_attention')(vec_iid)
vec_textu = Dense(attention_size,
activation=None,
use_bias=False,
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(l2_reg_lambda),
name='user_text_attention')(x_u)
vec_texti = Dense(attention_size,
activation=None,
use_bias=False,
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(l2_reg_lambda),
name='item_text_attention')(x_i)
# Interaction between the user and each item review to learn personalized review-usefulness
out_u = Multiply(name='usertext_itemid_interaction')([vec_textu, vec_iid])
out_i = Multiply(name='itemtext_userid_interaction')([vec_texti, vec_uid])
b_u = tf.keras.backend.random_uniform_variable([attention_size],
low=-0.1,
high=0.1,
seed=random_seed)
b_i = tf.keras.backend.random_uniform_variable([attention_size],
low=-0.1,
high=0.1,
seed=random_seed)
out_u = tf.keras.backend.bias_add(out_u, b_u)
out_i = tf.keras.backend.bias_add(out_i, b_i)
out_u = ReLU()(out_u)
out_i = ReLU()(out_i)
out_u = Dense(1,
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(out_u)
out_i = Dense(1,
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(out_i)
# Output the weight (usefulness) for each review
out_u = Softmax(axis=1, name='user_rev_weights')(out_u)
out_i = Softmax(axis=1, name='item_rev_weights')(out_i)
return out_u, out_i
def call(self, inputs):
sequence_length = inputs.shape[1]
## Get h_t, the current (target) hidden state ##
target_hidden_state = Lambda(function=lambda x: x[:, -1, :])(
inputs) # (B, H)
target_hidden_state_reshaped = Reshape(
target_shape=(1,
inputs.shape[2]))(target_hidden_state) # (B, 1, H)
## Get h_s, source hidden states through specified attention mechanism ##
if self.alignment_type == 'global': ## Global Approach ##
source_hidden_states = inputs # (B, S*=S, H)
elif 'local' in self.alignment_type: ## Local Approach ##
if self.window_width == None: ## Automatically set window width ##
self.window_width = sequence_length // 2
if self.alignment_type == 'local-m': ## Monotonic Alignment ##
aligned_position = sequence_length
left_border = aligned_position - self.window_width if aligned_position - self.window_width >= 0 else 0
source_hidden_states = Lambda(
function=lambda x: x[:, left_border:, :])(
inputs) # (B, S*=D, H)
elif self.alignment_type == 'local-p': ## Predictive Alignment ##
aligned_position = self.W_p(target_hidden_state) # (B, H)
aligned_position = Activation('tanh')(
aligned_position) # (B, H)
aligned_position = self.v_p(aligned_position) # (B, 1)
aligned_position = Activation('sigmoid')(
aligned_position) # (B, 1)
aligned_position = aligned_position * sequence_length # (B, 1)
source_hidden_states = inputs # (B, S, H)
elif self.alignment_type == 'local-p*': ## Completely Predictive Alignment ##
aligned_position = self.W_p(inputs) # (B, S, H)
aligned_position = Activation('tanh')(
aligned_position) # (B, S, H)
aligned_position = self.v_p(aligned_position) # (B, S, 1)
aligned_position = Activation('sigmoid')(
aligned_position) # (B, S, 1)
## Only keep top D values out of the sigmoid activation, and zero-out the rest ##
aligned_position = tf.squeeze(aligned_position,
axis=-1) # (B, S)
top_probabilities = tf.nn.top_k(
input=aligned_position, k=self.window_width,
sorted=False) # (values:(B, D), indices:(B, D))
onehot_vector = tf.one_hot(indices=top_probabilities.indices,
depth=sequence_length) # (B, D, S)
onehot_vector = tf.reduce_sum(onehot_vector, axis=1) # (B, S)
aligned_position = Multiply()(
[aligned_position, onehot_vector]) # (B, S)
aligned_position = tf.expand_dims(aligned_position,
axis=-1) # (B, S, 1)
source_hidden_states = Multiply()([inputs, aligned_position
]) # (B, S*=S(D), H)
## Scale back-to approximately original hidden state values ##
aligned_position += 1 # (B, S, 1)
source_hidden_states /= aligned_position # (B, S*=S(D), H)
## Compute alignment score through specified function ##
if 'dot' in self.score_function:
attention_score = Dot(axes=[2, 1])(
[source_hidden_states, target_hidden_state]) # (B, S*)
if self.score_function == 'scaled_dot':
attention_score = attention_score * (
1 / np.sqrt(float(inputs.shape[2]))) # (B, S*)
elif self.score_function == 'general':
weighted_hidden_states = self.W_a(
source_hidden_states) # (B, S*, H)
attention_score = Dot(axes=[2, 1])(
[weighted_hidden_states, target_hidden_state]) # (B, S*)
elif self.score_function == 'location':
weighted_target_state = self.W_a(target_hidden_state) # (B, H)
attention_score = Activation('softmax')(
weighted_target_state) # (B, H)
attention_score = RepeatVector(
n=inputs.shape[1] - 1 if self.seperate else inputs.shape[1])(
attention_score) # (B, S*, H)
attention_score = tf.reduce_sum(attention_score,
axis=-1) # (B, S*)
elif self.score_function == 'concat':
weighted_hidden_states = self.W_a(
source_hidden_states) # (B, S*, H)
weighted_target_state = self.U_a(
target_hidden_state_reshaped) # (B, 1, H)
weighted_sum = weighted_hidden_states + weighted_target_state # (B, S*, H)
weighted_sum = Activation('tanh')(weighted_sum) # (B, S*, H)
attention_score = self.v_a(weighted_sum) # (B, S*, 1)
attention_score = attention_score[:, :, 0] # (B, S*)
attention_weights = Activation('softmax')(attention_score) # (B, S*)
if self.alignment_type == 'local-p': ## Gaussian Distribution ##
gaussian_estimation = lambda s: tf.exp(-tf.square(
s - aligned_position) / (2 * tf.square(self.window_width / 2)))
gaussian_factor = gaussian_estimation(0)
for i in range(1, sequence_length):
gaussian_factor = Concatenate()(
[gaussian_factor, gaussian_estimation(i)])
# gaussian_factor: (B, S*)
attention_weights = attention_weights * gaussian_factor # (B, S*)
context_vector = Dot(axes=[1, 1])(
[source_hidden_states, attention_weights]) # (B, H)
combined_information = Concatenate()(
[context_vector, target_hidden_state]) # (B, 2*H)
attention_vector = self.attention_vector(
combined_information) # (B, self.size)
return attention_vector
def get_test_model_exhaustive():
"""Returns a exhaustive test model."""
input_shapes = [
(2, 3, 4, 5, 6),
(2, 3, 4, 5, 6),
(7, 8, 9, 10),
(7, 8, 9, 10),
(11, 12, 13),
(11, 12, 13),
(14, 15),
(14, 15),
(16, ),
(16, ),
(2, ),
(1, ),
(2, ),
(1, ),
(1, 3),
(1, 4),
(1, 1, 3),
(1, 1, 4),
(1, 1, 1, 3),
(1, 1, 1, 4),
(1, 1, 1, 1, 3),
(1, 1, 1, 1, 4),
(26, 28, 3),
(4, 4, 3),
(4, 4, 3),
(4, ),
(2, 3),
(1, ),
(1, ),
(1, ),
(2, 3),
]
inputs = [Input(shape=s) for s in input_shapes]
outputs = []
outputs.append(Conv1D(1, 3, padding='valid')(inputs[6]))
outputs.append(Conv1D(2, 1, padding='same')(inputs[6]))
outputs.append(Conv1D(3, 4, padding='causal', dilation_rate=2)(inputs[6]))
outputs.append(ZeroPadding1D(2)(inputs[6]))
outputs.append(Cropping1D((2, 3))(inputs[6]))
outputs.append(MaxPooling1D(2)(inputs[6]))
outputs.append(MaxPooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(AveragePooling1D(2)(inputs[6]))
outputs.append(AveragePooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(GlobalMaxPooling1D()(inputs[6]))
outputs.append(GlobalMaxPooling1D(data_format="channels_first")(inputs[6]))
outputs.append(GlobalAveragePooling1D()(inputs[6]))
outputs.append(
GlobalAveragePooling1D(data_format="channels_first")(inputs[6]))
outputs.append(Conv2D(4, (3, 3))(inputs[4]))
outputs.append(Conv2D(4, (3, 3), use_bias=False)(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4]))
outputs.append(SeparableConv2D(3, (3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((1, 2))(inputs[4]))
outputs.append(MaxPooling2D((2, 2))(inputs[4]))
outputs.append(
MaxPooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(AveragePooling2D((2, 2))(inputs[4]))
outputs.append(
AveragePooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(GlobalAveragePooling2D()(inputs[4]))
outputs.append(
GlobalAveragePooling2D(data_format="channels_first")(inputs[4]))
outputs.append(GlobalMaxPooling2D()(inputs[4]))
outputs.append(GlobalMaxPooling2D(data_format="channels_first")(inputs[4]))
outputs.append(BatchNormalization()(inputs[0]))
outputs.append(BatchNormalization(axis=1)(inputs[0]))
outputs.append(BatchNormalization(axis=2)(inputs[0]))
outputs.append(BatchNormalization(axis=3)(inputs[0]))
outputs.append(BatchNormalization(axis=4)(inputs[0]))
outputs.append(BatchNormalization(axis=5)(inputs[0]))
outputs.append(BatchNormalization()(inputs[2]))
outputs.append(BatchNormalization(axis=1)(inputs[2]))
outputs.append(BatchNormalization(axis=2)(inputs[2]))
outputs.append(BatchNormalization(axis=3)(inputs[2]))
outputs.append(BatchNormalization(axis=4)(inputs[2]))
outputs.append(BatchNormalization()(inputs[4]))
#outputs.append(BatchNormalization(axis=1)(inputs[4])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now.
outputs.append(BatchNormalization(axis=2)(inputs[4]))
outputs.append(BatchNormalization(axis=3)(inputs[4]))
outputs.append(BatchNormalization()(inputs[6]))
outputs.append(BatchNormalization(axis=1)(inputs[6]))
outputs.append(BatchNormalization(axis=2)(inputs[6]))
outputs.append(BatchNormalization()(inputs[8]))
outputs.append(BatchNormalization(axis=1)(inputs[8]))
outputs.append(Dropout(0.5)(inputs[4]))
outputs.append(ZeroPadding2D(2)(inputs[4]))
outputs.append(ZeroPadding2D((2, 3))(inputs[4]))
outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Cropping2D(2)(inputs[4]))
outputs.append(Cropping2D((2, 3))(inputs[4]))
outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Dense(3, use_bias=True)(inputs[13]))
outputs.append(Dense(3, use_bias=True)(inputs[14]))
outputs.append(Dense(4, use_bias=False)(inputs[16]))
outputs.append(Dense(4, use_bias=False, activation='tanh')(inputs[18]))
outputs.append(Dense(4, use_bias=False)(inputs[20]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='bilinear')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='bilinear')(inputs[4]))
for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]:
outputs.append(Concatenate(axis=axis)([inputs[0], inputs[1]]))
for axis in [-4, -3, -2, -1, 1, 2, 3, 4]:
outputs.append(Concatenate(axis=axis)([inputs[2], inputs[3]]))
for axis in [-3, -2, -1, 1, 2, 3]:
outputs.append(Concatenate(axis=axis)([inputs[4], inputs[5]]))
for axis in [-2, -1, 1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[6], inputs[7]]))
for axis in [-1, 1]:
outputs.append(Concatenate(axis=axis)([inputs[8], inputs[9]]))
for axis in [-1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]]))
for axis in [-1, 3]:
outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]]))
for axis in [-1, 4]:
outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]]))
for axis in [-1, 5]:
outputs.append(Concatenate(axis=axis)([inputs[20], inputs[21]]))
outputs.append(UpSampling1D(size=2)(inputs[6]))
outputs.append(Multiply()([inputs[10], inputs[11]]))
outputs.append(Multiply()([inputs[11], inputs[10]]))
outputs.append(Multiply()([inputs[11], inputs[13]]))
outputs.append(Multiply()([inputs[10], inputs[11], inputs[12]]))
outputs.append(Multiply()([inputs[11], inputs[12], inputs[13]]))
shared_conv = Conv2D(1, (1, 1),
padding='valid',
name='shared_conv',
activation='relu')
up_scale_2 = UpSampling2D((2, 2))
x1 = shared_conv(up_scale_2(inputs[23])) # (1, 8, 8)
x2 = shared_conv(up_scale_2(inputs[24])) # (1, 8, 8)
x3 = Conv2D(1, (1, 1),
padding='valid')(up_scale_2(inputs[24])) # (1, 8, 8)
x = Concatenate()([x1, x2, x3]) # (3, 8, 8)
outputs.append(x)
x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8)
outputs.append(x)
x = Dropout(0.5)(x)
outputs.append(x)
x = Concatenate()([MaxPooling2D((2, 2))(x),
AveragePooling2D((2, 2))(x)]) # (6, 4, 4)
outputs.append(x)
x = Flatten()(x) # (1, 1, 96)
x = Dense(4, use_bias=False)(x)
outputs.append(x)
x = Dense(3)(x) # (1, 1, 3)
outputs.append(x)
outputs.append(Add()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Subtract()([inputs[26], inputs[30]]))
outputs.append(Multiply()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Average()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Maximum()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Concatenate()([inputs[26], inputs[30], inputs[30]]))
intermediate_input_shape = (3, )
intermediate_in = Input(intermediate_input_shape)
intermediate_x = intermediate_in
intermediate_x = Dense(8)(intermediate_x)
intermediate_x = Dense(5)(intermediate_x)
intermediate_model = Model(inputs=[intermediate_in],
outputs=[intermediate_x],
name='intermediate_model')
intermediate_model.compile(loss='mse', optimizer='nadam')
x = intermediate_model(x) # (1, 1, 5)
intermediate_model_2 = Sequential()
intermediate_model_2.add(Dense(7, input_shape=(5, )))
intermediate_model_2.add(Dense(5))
intermediate_model_2.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_2(x) # (1, 1, 5)
x = Dense(3)(x) # (1, 1, 3)
shared_activation = Activation('tanh')
outputs = outputs + [
Activation('tanh')(inputs[25]),
Activation('hard_sigmoid')(inputs[25]),
Activation('selu')(inputs[25]),
Activation('sigmoid')(inputs[25]),
Activation('softplus')(inputs[25]),
Activation('softmax')(inputs[25]),
Activation('relu')(inputs[25]),
LeakyReLU()(inputs[25]),
ELU()(inputs[25]),
PReLU()(inputs[24]),
PReLU()(inputs[25]),
PReLU()(inputs[26]),
shared_activation(inputs[25]),
Activation('linear')(inputs[26]),
Activation('linear')(inputs[23]),
x,
shared_activation(x),
]
model = Model(inputs=inputs, outputs=outputs, name='test_model_exhaustive')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 1
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, epochs=10)
return model
def __init__(
self,
input_tensor,
cout=1,
ksize=3,
radix=4,
kpaths=4,
num_layers=6,
num_initial_filters=16,
output_mask_logit=False,
logging=False,
dropout=0.5,
training=False,
):
self.customlayers = lay.Layers()
conv_activation_layer = _get_conv_activation_layer({})
deconv_activation_layer = _get_deconv_activation_layer({})
kernel_initializer = he_uniform(seed=50)
conv2d_factory = partial(
Conv2D,
strides=(2, 2),
padding='same',
kernel_initializer=kernel_initializer,
)
conv2d_transpose_factory = partial(
Conv2DTranspose,
strides=(2, 2),
padding='same',
kernel_initializer=kernel_initializer,
)
enc_outputs = []
current_layer = input_tensor
for i in range(num_layers):
print(current_layer)
if i < num_layers - 1:
current_layer = self.residual_S(
current_layer,
ksize,
inchannel=int(current_layer.shape[-1]),
outchannel=num_initial_filters * (2**i),
radix=radix,
kpaths=kpaths,
name=f'residual_s_{i}',
logging=logging,
training=training,
)
enc_outputs.append(current_layer)
else:
current_layer = conv2d_factory(num_initial_filters * (2**i),
(5, 5))(current_layer)
if logging:
print(current_layer)
for i in range(num_layers - 1):
current_layer = conv2d_transpose_factory(
num_initial_filters * (2**(num_layers - i - 2)),
(5, 5))((current_layer))
current_layer = deconv_activation_layer(current_layer)
current_layer = BatchNormalization(axis=-1)(current_layer,
training=training)
if i < 3:
current_layer = Dropout(dropout)(current_layer,
training=training)
current_layer = Concatenate(axis=-1)(
[enc_outputs[-i - 1], current_layer])
if logging:
print(current_layer)
current_layer = conv2d_transpose_factory(1, (5, 5), strides=(2, 2))(
(current_layer))
current_layer = deconv_activation_layer(current_layer)
current_layer = BatchNormalization(axis=-1)(current_layer,
training=training)
if not output_mask_logit:
last = Conv2D(
cout,
(4, 4),
dilation_rate=(2, 2),
activation='sigmoid',
padding='same',
kernel_initializer=kernel_initializer,
)((current_layer))
output = Multiply()([last, input_tensor])
self.logits = output
else:
self.logits = Conv2D(
cout,
(4, 4),
dilation_rate=(2, 2),
padding='same',
kernel_initializer=kernel_initializer,
)((current_layer))
def create_model(input_size,
num_classes_1,
num_classes_2,
batch_size,
combining_weight,
weight_flag,
ratio=1.0,
percent_share=None):
from tensorflow.keras.layers import Input, Conv1D, Dense, MaxPooling1D, Flatten, Dropout, Activation, BatchNormalization
from tensorflow.keras.layers import Concatenate, Masking, Multiply, RepeatVector, Reshape, Permute, Lambda
inputs = Input(shape=(input_size, 4), name='inputs')
nn = Conv1D(filters=300, kernel_size=19, padding='same',
name='conv_1')(inputs)
nn = Activation('relu', name='relu_1')(nn)
nn = MaxPooling1D(pool_size=3, strides=3, padding='same',
name='maxpool_1')(nn)
nn = BatchNormalization(name='BN_1')(nn)
nn = Conv1D(filters=200, kernel_size=11, padding='same', name='conv_2')(nn)
nn = Activation('relu', name='relu_2')(nn)
nn = MaxPooling1D(pool_size=4, strides=4, padding='same',
name='maxpool_2')(nn)
nn = BatchNormalization(name='BN_2')(nn)
nn = Conv1D(filters=200, kernel_size=7, padding='same', name='conv_3')(nn)
nn = Activation('relu', name='relu_3')(nn)
nn = MaxPooling1D(pool_size=4, strides=4, padding='same',
name='maxpool_3')(nn)
nn = BatchNormalization(name='BN_3')(nn)
nn = Flatten(name='flatten')(nn)
nn = Dense(1000, name='dense_1')(nn)
nn = BatchNormalization(name='BN_4')(nn)
nn = Activation('relu', name='relu_4')(nn)
nn = Dropout(0.3, name='dropout_1')(nn)
nn = Dense(1000, name='dense_2')(nn)
nn = BatchNormalization(name='BN_5')(nn)
nn = Activation('relu', name='relu_5')(nn)
nn = Dropout(0.3, name='dropout_2')(nn)
## two head version:
labels_mask_1 = Input(shape=([num_classes_1]),
dtype='float32',
name='labels_mask_1')
mask_1 = Masking(mask_value=0.0, name='mask_1')(labels_mask_1)
labels_mask_2 = Input(shape=([num_classes_2]),
dtype='float32',
name='labels_mask_2')
mask_2 = Masking(mask_value=0.0, name='mask_2')(labels_mask_2)
result_1_ori = Dense(num_classes_1, name='dense_out_1_ori')(nn)
result_2_ori = Dense(num_classes_2, name='dense_out_2_ori')(nn)
result_1 = Multiply(name='dense_out_1')([result_1_ori, mask_1])
result_2 = Multiply(name='dense_out_2')([result_2_ori, mask_2])
model = tf.keras.models.Model(
inputs=[inputs, labels_mask_1, labels_mask_2],
outputs=[result_1, result_2])
loss_combined = {
'dense_out_1': losses.combine_loss_by_sample(0.01),
'dense_out_2': losses.combine_loss_by_sample(0.0052)
}
lossWeights = {'dense_out_1': ratio, 'dense_out_2': 1.0}
metric = {
'dense_out_1': ['mse', losses.pearson_loss, losses.r2_score],
'dense_out_2': ['mse', losses.pearson_loss, losses.r2_score]
}
opt = tf.keras.optimizers.Adam(lr=learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-8,
decay=0,
amsgrad=False) # decay=1e-6,
model.compile(optimizer=opt,
loss=loss_combined,
loss_weights=lossWeights,
metrics=metric)
return model
