def test_ProfilePredictorConv2D(self):
input_profile_names = ['a', 'b', 'c']
target_profile_names = ['a', 'b']
actuator_names = ['aa', 'bb', 'cc']
lookbacks = {'a': 1,
'b': 1,
'c': 1,
'aa': 5,
'bb': 5,
'cc': 5}
profile_lookback = 1
actuator_lookback = 5
lookahead = 4
profile_length = 33
std_activation = 'relu'
profile_inshape = (profile_lookback, profile_length)
past_actuator_inshape = (actuator_lookback,)
future_actuator_inshape = (lookahead,)
num_profiles = len(input_profile_names)
num_targets = len(target_profile_names)
num_actuators = len(actuator_names)
max_channels = 32
profile_inputs = []
profiles = []
for i in range(num_profiles):
profile_inputs.append(
Input(profile_inshape, name='input_' + input_profile_names[i]))
profiles.append(Reshape((profile_lookback, profile_length, 1))
(profile_inputs[i]))
profiles = Concatenate(axis=-1)(profiles)
# shape = (lookback, length, channels=num_profiles)
profiles = Conv2D(filters=int(num_profiles*max_channels/8),
kernel_size=(1, int(profile_length/12)),
strides=(1, 1), padding='same', activation=std_activation)(profiles)
profiles = Conv2D(filters=int(num_profiles*max_channels/4),
kernel_size=(1, int(profile_length/8)),
strides=(1, 1), padding='same', activation=std_activation)(profiles)
profiles = Conv2D(filters=int(num_profiles*max_channels/2),
kernel_size=(1, int(profile_length/6)),
strides=(1, 1), padding='same', activation=std_activation)(profiles)
profiles = Conv2D(filters=int(num_profiles*max_channels),
kernel_size=(1, int(profile_length/4)),
strides=(1, 1), padding='same', activation=std_activation)(profiles)
# shape = (lookback, length, channels)
if profile_lookback > 1:
profiles = Conv2D(filters=int(num_profiles*max_channels), kernel_size=(profile_lookback, 1),
strides=(1, 1), padding='valid', activation=std_activation)(profiles)
profiles = Reshape((profile_length, int(
num_profiles*max_channels)))(profiles)
# shape = (length, channels)
actuator_future_inputs = []
actuator_past_inputs = []
actuators = []
for i in range(num_actuators):
actuator_future_inputs.append(
Input(future_actuator_inshape, name='input_future_' + actuator_names[i]))
actuator_past_inputs.append(
Input(past_actuator_inshape, name='input_past_' + actuator_names[i]))
actuators.append(Concatenate(
axis=-1)([actuator_past_inputs[i], actuator_future_inputs[i]]))
actuators[i] = Reshape(
(actuator_lookback+lookahead, 1))(actuators[i])
actuators = Concatenate(axis=-1)(actuators)
# shaoe = (time, num_actuators)
actuators = Dense(units=int(num_profiles*max_channels/8),
activation=std_activation)(actuators)
# actuators = Conv1D(filters=int(num_profiles*max_channels/8), kernel_size=3, strides=1,
# padding='causal', activation=std_activation)(actuators)
actuators = Dense(units=int(num_profiles*max_channels/4),
activation=std_activation)(actuators)
# actuators = Conv1D(filters=int(num_profiles*max_channels/4), kernel_size=3, strides=1,
# padding='causal', activation=std_activation)(actuators)
actuators = Dense(units=int(num_profiles*max_channels/2),
activation=std_activation)(actuators)
actuators = LSTM(units=int(num_profiles*max_channels), activation=std_activation,
recurrent_activation='hard_sigmoid')(actuators)
actuators = Reshape((int(num_profiles*max_channels), 1))(actuators)
# shape = (channels, 1)
actuators = Dense(units=int(profile_length/4),
activation=std_activation)(actuators)
actuators = Dense(units=int(profile_length/2),
activation=std_activation)(actuators)
actuators = Dense(units=profile_length, activation=None)(actuators)
# shape = (channels, profile_length)
actuators = Permute(dims=(2, 1))(actuators)
# shape = (profile_length, channels)
merged = Add()([profiles, actuators])
merged = Reshape((1, profile_length, int(
num_profiles*max_channels)))(merged)
# shape = (1, length, channels)
prof_act = []
for i in range(num_targets):
prof_act.append(Conv2D(filters=max_channels, kernel_size=(1, int(profile_length/4)), strides=(1, 1),
padding='same', activation=std_activation)(merged))
# shape = (1,length,max_channels)
prof_act[i] = Conv2D(filters=int(max_channels/2), kernel_size=(1, int(profile_length/8)),
strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
prof_act[i] = Conv2D(filters=int(max_channels/4), kernel_size=(1, int(profile_length/6)),
strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
prof_act[i] = Conv2D(filters=int(max_channels/8), kernel_size=(1, int(profile_length/4)),
strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
prof_act[i] = Conv2D(filters=1, kernel_size=(1, int(profile_length/4)), strides=(1, 1),
padding='same', activation=None)(prof_act[i])
# shape = (1,length,1)
prof_act[i] = Reshape((profile_length,), name='target_' +
target_profile_names[i])(prof_act[i])
model = Model(inputs=profile_inputs + actuator_past_inputs +
actuator_future_inputs, outputs=prof_act)
name = 'test___ProfilePredictorConv2D' + str(int(time.time()))
keras2c_main.k2c(model, name)
rcode = build_and_run(name)
self.assertEqual(rcode, 0)
def u_net_module(p):
############################## Special Fun ###################################################
# embedding = tf.Variable(initial_value = np.load('../Embedding/embedd_w_5_ogt.npy').reshape(1,21,5))
############################## Transformation module #################################################
# Layers of stage 0 contraction
inp = Input(shape=(1024, ))
#w = Input(shape=(1024,))
#tct0_in = Dot(axes=(2,1))([inp,embedding])
#tct0_bn1 = BatchNormalization()(inp)#tct0_in)
#tf.keras.regularizers.L1L2(l1=0.0, l2=0.0)
x = Embedding(22, 5)(inp)
x = BatchNormalization()(x)
tct0_conv1 = Conv1D(int(p['n_fil_1']),
int(p['s_fil_1']),
activation='relu',
padding='same',
name='Convolution_Ct0_0')(x)
tct0_bn2 = BatchNormalization()(tct0_conv1)
tct0_conv2 = Conv1D(int(p['n_fil_1']),
int(p['s_fil_1']),
activation='relu',
padding='same',
name='Convolution_Ct0_1')(tct0_bn2)
tct0_bn3 = BatchNormalization()(tct0_conv2)
tct0_conv3 = Conv1D(int(p['n_fil_1']),
int(p['s_fil_1']),
activation='relu',
strides=int(p['stride_1']),
padding='same',
name='Convolution_Ct0_2')(tct0_bn3)
tct0_bn4 = BatchNormalization()(tct0_conv3)
#tct0_max = MaxPool1D(pool_size=2, strides=2)(tct0_bn2)
tct0_dp = Dropout(p['dOut_1'])(tct0_bn4)
# Layers of stage 1 contraction
tct1_conv1 = Conv1D(int(p['n_fil_2']),
int(p['s_fil_2']),
activation='relu',
padding='same',
name='Convolution_Ct1_0')(tct0_dp)
tct1_bn1 = BatchNormalization()(tct1_conv1)
tct1_conv2 = Conv1D(int(p['n_fil_2']),
int(p['s_fil_2']),
activation='relu',
strides=1,
padding='same',
name='Convolution_Ct1_1')(tct1_bn1)
tct1_bn2 = BatchNormalization()(tct1_conv2)
tct1_conv3 = Conv1D(int(p['n_fil_2']),
int(p['s_fil_2']),
activation='relu',
strides=int(p['stride_2']),
padding='same',
name='Convolution_Ct1_2')(tct1_bn2)
tct1_bn3 = BatchNormalization()(tct1_conv3)
#tct1_max = MaxPool1D(pool_size=2, strides=2)(tct1_bn2)
tct1_dp = Dropout(p['dOut_2'])(tct1_bn3)
# Layers of stage 2 contraction
tct2_conv1 = Conv1D(int(p['n_fil_3']),
int(p['s_fil_3']),
activation='relu',
padding='same',
name='Convolution_Ct2_0')(tct1_dp)
tct2_bn1 = BatchNormalization()(tct2_conv1)
tct2_conv2 = Conv1D(int(p['n_fil_3']),
int(p['s_fil_3']),
activation='relu',
strides=1,
padding='same',
name='Convolution_Ct2_1')(tct2_bn1)
tct2_bn2 = BatchNormalization()(tct2_conv2)
tct2_conv3 = Conv1D(int(p['n_fil_3']),
int(p['s_fil_3']),
activation='relu',
strides=int(p['stride_3']),
padding='same',
name='Convolution_Ct2_2')(tct2_bn2)
tct2_bn3 = BatchNormalization()(tct2_conv3)
#tct2_max = MaxPool1D(pool_size=2, strides=2)(tct2_bn2)
tct2_dp = Dropout(p['dOut_3'])(tct2_bn3)
# Layers of stage 3 contraction
tct3_conv1 = Conv1D(int(p['n_fil_4']),
int(p['s_fil_4']),
activation='relu',
padding='same',
name='Convolution_Ce3_0')(tct2_dp)
tct3_bn1 = BatchNormalization()(tct3_conv1)
tct3_conv2 = Conv1D(int(p['n_fil_4']),
int(p['s_fil_4']),
activation='relu',
padding='same',
name='Convolution_Ce3_1')(tct3_bn1)
tct3_bn2 = BatchNormalization()(tct3_conv2)
tct3_dp = Dropout(p['dOut_4'])(tct3_bn2)
# Layers of stage 1 expansion
tet1_Tconv = Conv1DTranspose(int(p['n_fil_3']),
int(p['s_fil_3']),
strides=int(p['stride_3']),
name='TransConv_Et1')(tct3_dp)
tet1_Concat = Concatenate(axis=2)([tet1_Tconv, tct2_conv1])
tet1_bn1 = BatchNormalization()(tet1_Concat)
tet1_conv1 = Conv1D(int(p['n_fil_3']),
int(p['s_fil_3']),
activation='relu',
padding='same',
name='Convolution_Et1_0')(tet1_bn1)
tet1_bn2 = BatchNormalization()(tet1_conv1)
tet1_conv2 = Conv1D(int(p['n_fil_3']),
int(p['s_fil_3']),
activation='relu',
padding='same',
name='Convolution_Et1_1')(tet1_bn2)
tet1_bn3 = BatchNormalization()(tet1_conv2)
tet1_dp = Dropout(p['dOut_3'])(tet1_bn3)
#Layers of stage 2 expansion
tet2_Tconv = Conv1DTranspose(int(p['n_fil_2']),
int(p['s_fil_2']),
strides=int(p['stride_2']),
name='TransConv_Et2')(tet1_dp)
tet2_Concat = Concatenate(axis=2)([tet2_Tconv, tct1_conv1])
tet2_bn1 = BatchNormalization()(tet2_Concat)
tet2_conv1 = Conv1D(int(p['n_fil_2']),
int(p['s_fil_2']),
activation='relu',
padding='same',
name='Convolution_Et2_0')(tet2_bn1)
tet2_bn2 = BatchNormalization()(tet2_conv1)
tet2_conv2 = Conv1D(int(p['n_fil_2']),
int(p['s_fil_2']),
activation='relu',
padding='same',
name='Convolution_Et2_1')(tet2_bn2)
tet2_bn3 = BatchNormalization()(tet2_conv2)
tet2_dp = Dropout(p['dOut_2'])(tet2_bn3)
#Layers of stage 3 expansion
tet3_Tconv = Conv1DTranspose(int(p['n_fil_1']),
int(p['s_fil_1']),
strides=int(p['stride_1']),
name='TransConv_Et3')(tet2_dp)
tet3_Concat = Concatenate(axis=2)([tet3_Tconv, tct0_conv1])
tet3_bn1 = BatchNormalization()(tet3_Concat)
tet3_conv1 = Conv1D(int(p['n_fil_1']),
int(p['s_fil_1']),
activation='relu',
padding='same',
name='Convolution_Et3_1')(tet3_bn1)
tet3_bn2 = BatchNormalization()(tet3_conv1)
tet3_conv2 = Conv1D(int(p['n_fil_1']),
int(p['s_fil_1']),
activation='relu',
padding='same',
name='Convolution_Et3_2')(tet3_bn2)
tet3_bn3 = BatchNormalization()(tet3_conv2)
tet3_dp = Dropout(p['dOut_5'])(tet3_bn3)
tet3_conv3 = Conv1D(3,
int(p['s_fil_5']),
activation='softmax',
padding='same',
name='Convolution_Et3_3')(tet3_dp)
cce = tf.keras.losses.CategoricalCrossentropy(
from_logits=True) # , sample_weight = w)
add = tf.keras.optimizers.Adam(learning_rate=1e-2, name='Adam')
model = Model(inputs=inp, outputs=tet3_conv3)
model.compile(optimizer=add,
loss=cce,
metrics=['accuracy'],
sample_weight_mode="temporal")
return model
def cnn_cycle_gan():
############################## Transformation module #################################################
# Layers of stage 0 contraction
InputLayer(input_shape=(in_, 21))
tct0_conv1 = Conv1D(16,
5,
activation='relu',
padding='same',
name='Convolution_Ct0_1')(tct0_in)
tct0_bn1 = BatchNormalization()(ct0_conv1)
tct0_conv2 = Conv1D(16,
5,
activation='relu',
padding='same',
name='Convolution_Ct0_2')(tct0_bn1)
tct0_bn2 = BatchNormalization()(tct0_conv2)
tct0_max = MaxPool1D(pool_size=2, strides=2)(tct0_bn2)
tct0_dp = Dropout(0.2)(tct0_max)
# Layers of stage 1 contraction
tct1_conv1 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Ct1_1')(tct0_dp)
tct1_bn1 = BatchNormalization()(tct1_conv1)
tct1_conv2 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Ct1_2')(tct1_bn1)
tct1_bn2 = BatchNormalization()(tct1_conv2)
tct1_max = MaxPool1D(pool_size=2, strides=2)(tct1_bn2)
tct1_dp = Dropout(0.2)(tct1_max)
# Layers of stage 2 contraction
tct2_conv1 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Ct2_1')(tct1_dp)
tct2_bn1 = BatchNormalization()(tct2_conv1)
tct2_conv2 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Ct2_2')(tct2_bn1)
tct2_bn2 = BatchNormalization()(tct2_conv2)
tct2_max = MaxPool1D(pool_size=2, strides=2)(tct2_bn2)
tct2_dp = Dropout(0.2)(tct2_max)
# Layers of stage 3 contraction
tct3_conv1 = Conv1D(128,
5,
activation='relu',
padding='same',
name='Convolution_Ce3_1')(tct2_dp)
tct3_bn1 = BatchNormalization()(tct3_conv1)
tct3_conv2 = Conv1D(128,
5,
activation='relu',
padding='same',
name='Convolution_Ce3_2')(tct3_bn1)
tct3_bn2 = BatchNormalization()(tct3_conv2)
tct3_max = MaxPool1D(pool_size=2, strides=2)(tct3_bn2)
tct3_dp = Dropout(0.2)(tct3_max)
# Layers of stage 1 expansion
tet1_Tconv = Conv1DTranspose(64,
5,
strides=2,
activation='relu',
padding='same',
name='TransConv_Et1')(tct3_dp)
tet1_Concat = Concatenate(axis=1)([tet1_Tconv, tct3_conv2])
tet1_bn1 = BatchNormalization()(et1_Concat)
tet1_conv1 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Et1_1')(tet1_bn1)
tet1_bn2 = BatchNormalization()(tet1_conv1)
tet1_conv2 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Et1_2')(tet1_bn2)
tet1_dp = Dropout(0.2)(tet1_conv2)
#Layers of stage 2 expansion
tet2_Tconv = Conv1DTranspose(32,
5,
strides=2,
activation='relu',
padding='same',
name='TransConv_Et2')(tet1_dp)
tet2_Concat = Concatenate(axis=1)([tet2_Tconv, tct1_conv2])
tet2_bn1 = BatchNormalization()(tet2_Concat)
tet2_conv1 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Et2_1')(tet2_bn1)
tet2_bn2 = BatchNormalization()(tet2_conv1)
tet2_conv2 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Et2_2')(tet2_bn2)
tet2_dp = Dropout(0.2)(tet2_conv2)
#Layers of stage 3 expansion
tet3_Tconv = Conv1DTranspose(16,
5,
strides=2,
activation='relu',
padding='same',
name='TransConv_Et3')(tet2_dp)
tet3_Concat = Concatenate(axis=1)([tet3_Tconv, tce0_conv2])
tet3_bn1 = BatchNormalization()(tet3_Concat)
tet3_conv1 = Conv1D(16,
5,
activation='relu',
padding='same',
name='Convolution_Et3_1')(tet3_bn1)
tet3_bn2 = BatchNormalization()(tet3_conv1)
tet3_conv2 = Conv1D(16,
5,
activation='relu',
padding='same',
name='Convolution_Et3_2')(tet3_bn2)
tet3_bn3 = BatchNormalization()(tet3_conv2)
tet3_dp = Dropout(0.1)(tet3_bn3)
tet3_conv3 = Conv1D(1,
5,
activation='sigmoid',
padding='same',
name='Convolution_Et3_3')(tet3_dp)
########################################## Reconstroction Module ######################################
# Layers of stage 0 contraction
rct0_conv1 = Conv1D(16,
5,
activation='relu',
padding='same',
name='Convolution_Ct0_1')(tet3_conv3)
rct0_bn1 = BatchNormalization()(rct0_conv1)
rct0_conv2 = Conv1D(16,
5,
activation='relu',
padding='same',
name='Convolution_Ct0_2')(rct0_bn1)
rct0_bn2 = BatchNormalization()(rct0_conv2)
rct0_max = MaxPool1D(pool_size=2, strides=2)(rct0_bn2)
rct0_dp = Dropout(0.2)(rct0_max)
# Layers of stage 1 contraction
rct1_conv1 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Ct1_1')(rct0_dp)
rct1_bn1 = BatchNormalization()(rct1_conv1)
rct1_conv2 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Ct1_2')(rct1_bn1)
rct1_bn2 = BatchNormalization(rct1_conv2)
rct1_max = MaxPool1D(pool_size=2, strides=2)(rct1_bn2)
rct1_dp = Dropout(0.2)(rct1_max)
# Layers of stage 2 contraction
rct2_conv1 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Ct2_1')(rct1_dp)
rct2_bn1 = BatchNormalization()(rct2_conv1)
rct2_conv2 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Ct2_2')(rct2_bn1)
rct2_bn2 = BatchNormalization()(rct2_conv2)
rct2_max = MaxPool1D(pool_size=2, strides=2)(rct2_bn2)
rct2_dp = Dropout(0.2)(rct2_max)
# Layers of stage 3 contraction
rct3_conv1 = Conv1D(128,
5,
activation='relu',
padding='same',
name='Convolution_Ce3_1')(rct2_dp)
rct3_bn1 = BatchNormalization()(rct3_conv1)
rct3_conv2 = Conv1D(128,
5,
activation='relu',
padding='same',
name='Convolution_Ce3_2')(rct3_bn1)
rct3_bn2 = BatchNormalization()(rct3_conv2)
rct3_max = MaxPool1D(pool_size=2, strides=2)(rct3_bn2)
rct3_dp = Dropout(0.2)(rct3_max)
# Layers of stage 1 expansion
ret1_Tconv = Conv1DTranspose(64,
5,
strides=2,
activation='relu',
padding='same',
name='TransConv_Et1')(rct3_dp)
ret1_Concat = Concatenate(axis=1)([ret1_Tconv, rct3_conv2])
ret1_bn1 = BatchNormalization()(ret1_Concat)
ret1_conv1 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Et1_1')(ret1_bn1)
ret1_bn2 = BatchNormalization()(ret1_conv1)
ret1_conv2 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Et1_2')(ret1_bn2)
ret1_dp = Dropout(0.2)(ret1_conv2)
#Layers of stage 2 expansion
ret2_Tconv = Conv1DTranspose(32,
5,
strides=2,
activation='relu',
padding='same',
name='TransConv_Et2')(ret1_dp)
ret2_Concat = Concatenate(axis=1)([ret2_Tconv, rct1_conv2])
ret2_bn1 = BatchNormalization()(ret2_Concat)
ret2_conv1 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Et2_1')(ret2_bn1)
ret2_bn2 = BatchNormalization()(ret2_conv1)
ret2_conv2 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Et2_2')(ret2_bn2)
ret2_dp = Dropout(0.2)(ret2_conv2)
#Layers of stage 3 expansion
ret3_Tconv = Conv1DTranspose(16,
5,
strides=2,
activation='relu',
padding='same',
name='TransConv_Et3')(ret2_dp)
ret3_Concat = Concatenate(axis=1)([ret3_Tconv, rce0_conv2])
ret3_bn1 = BatchNormalization()(ret3_Concat)
ret3_conv1 = Conv1D(16,
5,
activation='relu',
padding='same',
name='Convolution_Et3_1')(ret3_bn1)
ret3_bn2 = BatchNormalization()(ret3_conv1)
ret3_conv2 = Conv1D(16,
5,
activation='relu',
padding='same',
name='Convolution_Et3_2')(ret3_bn2)
ret3_bn3 = BatchNormalization()(ret3_conv2)
ret3_dp = Dropout(0.1)(ret3_bn3)
ret3_conv2 = Conv1D(1,
5,
activation='sigmoid',
padding='same',
name='Convolution_Et3_3')(ret3_dp)
######################################## Discriminator Module ###############################################
distrib_Y = Input(shape=(2000, 20))
def build(self):
'''
1. Build Code Representation Model
'''
logger.debug('Building Code Representation Model')
methname = Input(shape=(self.data_params['methname_len'], ),
dtype='int32',
name='methname')
apiseq = Input(shape=(self.data_params['apiseq_len'], ),
dtype='int32',
name='apiseq')
tokens = Input(shape=(self.data_params['tokens_len'], ),
dtype='int32',
name='tokens')
## method name representation ##
#1.embedding
init_emb_weights = np.load(
self.model_params['init_embed_weights_methname']
) if self.model_params[
'init_embed_weights_methname'] is not None else None
if init_emb_weights is not None: init_emb_weights = [init_emb_weights]
embedding = Embedding(
input_dim=self.data_params['n_words'],
output_dim=self.model_params.get('n_embed_dims', 100),
weights=init_emb_weights,
mask_zero=
False, #Whether 0 in the input is a special "padding" value that should be masked out.
#If True, all subsequent layers in the model must support masking, otherwise an exception will be raised.
name='embedding_methname')
methname_embedding = embedding(methname)
dropout = Dropout(0.25, name='dropout_methname_embed')
methname_dropout = dropout(methname_embedding)
#2.rnn
f_rnn = LSTM(self.model_params.get('n_lstm_dims', 128),
recurrent_dropout=0.2,
return_sequences=True,
name='lstm_methname_f')
b_rnn = LSTM(self.model_params.get('n_lstm_dims', 128),
return_sequences=True,
recurrent_dropout=0.2,
name='lstm_methname_b',
go_backwards=True)
methname_f_rnn = f_rnn(methname_dropout)
methname_b_rnn = b_rnn(methname_dropout)
dropout = Dropout(0.25, name='dropout_methname_rnn')
methname_f_dropout = dropout(methname_f_rnn)
methname_b_dropout = dropout(methname_b_rnn)
#3.maxpooling
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False),
output_shape=lambda x: (x[0], x[2]),
name='maxpool_methname')
methname_pool = Concatenate(name='concat_methname_lstms')(
[maxpool(methname_f_dropout),
maxpool(methname_b_dropout)])
activation = Activation('tanh', name='active_methname')
methname_repr = activation(methname_pool)
## API Sequence Representation ##
#1.embedding
embedding = Embedding(
input_dim=self.data_params['n_words'],
output_dim=self.model_params.get('n_embed_dims', 100),
#weights=weights,
mask_zero=
False, #Whether 0 in the input is a special "padding" value that should be masked out.
#If True, all subsequent layers must support masking, otherwise an exception will be raised.
name='embedding_apiseq')
apiseq_embedding = embedding(apiseq)
dropout = Dropout(0.25, name='dropout_apiseq_embed')
apiseq_dropout = dropout(apiseq_embedding)
#2.rnn
f_rnn = LSTM(self.model_params.get('n_lstm_dims', 100),
return_sequences=True,
recurrent_dropout=0.2,
name='lstm_apiseq_f')
b_rnn = LSTM(self.model_params.get('n_lstm_dims', 100),
return_sequences=True,
recurrent_dropout=0.2,
name='lstm_apiseq_b',
go_backwards=True)
apiseq_f_rnn = f_rnn(apiseq_dropout)
apiseq_b_rnn = b_rnn(apiseq_dropout)
dropout = Dropout(0.25, name='dropout_apiseq_rnn')
apiseq_f_dropout = dropout(apiseq_f_rnn)
apiseq_b_dropout = dropout(apiseq_b_rnn)
#3.maxpooling
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False),
output_shape=lambda x: (x[0], x[2]),
name='maxpool_apiseq')
apiseq_pool = Concatenate(name='concat_apiseq_lstms')(
[maxpool(apiseq_f_dropout),
maxpool(apiseq_b_dropout)])
activation = Activation('tanh', name='active_apiseq')
apiseq_repr = activation(apiseq_pool)
## Tokens Representation ##
#1.embedding
init_emb_weights = np.load(
self.model_params['init_embed_weights_tokens']
) if self.model_params[
'init_embed_weights_tokens'] is not None else None
if init_emb_weights is not None: init_emb_weights = [init_emb_weights]
embedding = Embedding(
input_dim=self.data_params['n_words'],
output_dim=self.model_params.get('n_embed_dims', 100),
weights=init_emb_weights,
#mask_zero=True,#Whether 0 in the input is a special "padding" value that should be masked out.
#If True, all subsequent layers must support masking, otherwise an exception will be raised.
name='embedding_tokens')
tokens_embedding = embedding(tokens)
dropout = Dropout(0.25, name='dropout_tokens_embed')
tokens_dropout = dropout(tokens_embedding)
#4.maxpooling
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False),
output_shape=lambda x: (x[0], x[2]),
name='maxpool_tokens')
tokens_pool = maxpool(tokens_dropout)
activation = Activation('tanh', name='active_tokens')
tokens_repr = activation(tokens_pool)
## concatenate the representation of code ##
merged_methname_api = Concatenate(name='merge_methname_api')(
[methname_repr, apiseq_repr])
merged_code_repr = Concatenate(name='merge_coderepr')(
[merged_methname_api, tokens_repr])
code_repr = Dense(self.model_params.get('n_hidden', 400),
activation='tanh',
name='dense_coderepr')(merged_code_repr)
self._code_repr_model = Model(inputs=[methname, apiseq, tokens],
outputs=[code_repr],
name='code_repr_model')
'''
2. Build Desc Representation Model
'''
## Desc Representation ##
logger.debug('Building Desc Representation Model')
desc = Input(shape=(self.data_params['desc_len'], ),
dtype='int32',
name='desc')
#1.embedding
init_emb_weights = np.load(
self.model_params['init_embed_weights_desc']
) if self.model_params['init_embed_weights_desc'] is not None else None
if init_emb_weights is not None: init_emb_weights = [init_emb_weights]
embedding = Embedding(
input_dim=self.data_params['n_words'],
output_dim=self.model_params.get('n_embed_dims', 100),
weights=init_emb_weights,
mask_zero=
True, #Whether 0 in the input is a special "padding" value that should be masked out.
#If True, all subsequent layers must support masking, otherwise an exception will be raised.
name='embedding_desc')
desc_embedding = embedding(desc)
dropout = Dropout(0.25, name='dropout_desc_embed')
desc_dropout = dropout(desc_embedding)
#2. rnn
f_rnn = LSTM(self.model_params.get('n_lstm_dims', 100),
return_sequences=True,
recurrent_dropout=0.2,
name='lstm_desc_f')
b_rnn = LSTM(self.model_params.get('n_lstm_dims', 100),
return_sequences=True,
recurrent_dropout=0.2,
name='lstm_desc_b',
go_backwards=True)
desc_f_rnn = f_rnn(desc_dropout)
desc_b_rnn = b_rnn(desc_dropout)
dropout = Dropout(0.25, name='dropout_desc_rnn')
desc_f_dropout = dropout(desc_f_rnn)
desc_b_dropout = dropout(desc_b_rnn)
#3. maxpooling
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False),
output_shape=lambda x: (x[0], x[2]),
name='maxpool_desc')
desc_pool = Concatenate(name='concat_desc_rnns')(
[maxpool(desc_f_dropout),
maxpool(desc_b_dropout)])
activation = Activation('tanh', name='active_desc')
desc_repr = activation(desc_pool)
self._desc_repr_model = Model(inputs=[desc],
outputs=[desc_repr],
name='desc_repr_model')
"""
3: calculate the cosine similarity between code and desc
"""
logger.debug('Building similarity model')
code_repr = self._code_repr_model([methname, apiseq, tokens])
desc_repr = self._desc_repr_model([desc])
cos_sim = Dot(axes=1, normalize=True,
name='cos_sim')([code_repr, desc_repr])
sim_model = Model(inputs=[methname, apiseq, tokens, desc],
outputs=[cos_sim],
name='sim_model')
self._sim_model = sim_model #for model evaluation
'''
4:Build training model
'''
good_sim = sim_model(
[self.methname, self.apiseq, self.tokens,
self.desc_good]) # similarity of good output
bad_sim = sim_model(
[self.methname, self.apiseq, self.tokens,
self.desc_bad]) #similarity of bad output
loss = Lambda(lambda x: K.maximum(
1e-6, self.model_params['margin'] - x[0] + x[1]),
output_shape=lambda x: x[0],
name='loss')([good_sim, bad_sim])
logger.debug('Building training model')
self._training_model = Model(inputs=[
self.methname, self.apiseq, self.tokens, self.desc_good,
self.desc_bad
],
outputs=[loss],
name='training_model')
# 모델2 input2 = Input(shape=(3, )) dense2 = Dense(10, activation='relu')(input2) dense2 = Dense(5, activation='relu')(dense2) dense2 = Dense(5, activation='relu')(dense2) dense2 = Dense(5, activation='relu')(dense2) # output2 = Dense(3)(dense2) # 모델 병합 / concatenate from tensorflow.keras.layers import Concatenate # from keras.layers.merge import concatenate, Concatenate # from keras.layers import concatenate, Concatenate # merge = 합치다 merge1 = Concatenate()([dense1, dense2]) # 제일 끝의 dense 변수명 넣기 middle1 = Dense(30)(merge1) middle1 = Dense(10)(middle1) middle1 = Dense(10)(middle1) # 모델 분기1 output1 = Dense(30)(middle1) output1 = Dense(7)(output1) output1 = Dense(3)(output1) # 모델 분기2 output2 = Dense(15)(middle1) output2 = Dense(7)(output2) output2 = Dense(7)(output2) output2 = Dense(3)(output2)
features_num = 10
# Users embedding features.
user_input = Input(shape=[1])
user_embedding = Embedding(len(dataset.user_id.unique()) + 1,
features_num)(user_input)
user_vector = Flatten()(user_embedding)
# Books embedding features.
book_input = Input(shape=[1])
book_embedding = Embedding(len(dataset.book_id.unique()) + 1,
features_num)(book_input)
book_vector = Flatten()(book_embedding)
# Concatenate features.
concatenated = Concatenate()([book_vector, user_vector])
# Create model.
layer1 = Dense(128, activation='relu')(concatenated)
dropout1 = Dropout(0.1)(layer1)
layer2 = Dense(64, activation='relu')(dropout1)
dropout2 = Dropout(0.1)(layer2)
output = Dense(1)(dropout2)
model = Model([user_input, book_input], output)
model.compile('adam', 'mean_squared_error')
# Train.
model.fit([train.user_id, train.book_id],
train.rating,
epochs=15,
batch_size=256)
def create_model(l2, num_classes, num_ctrl_classes):
##############
# BRANCH MODEL
##############
regul = regularizers.l2(l2)
# optim = Adam(lr=lr)
# kwargs = {'kernel_regularizer': regul}
base_model = efn.EfficientNetB2(input_shape=(network_shape[0],
network_shape[1], 3),
weights='imagenet',
include_top=False)
input_tensor = Input(shape=network_shape, dtype=K.floatx())
conv1 = Conv2D(32,
kernel_size=(3, 3),
strides=(2, 2),
activation='relu',
use_bias=False,
padding='same',
input_shape=(input_shape[0] + 6, input_shape[1] + 6,
input_shape[2]))
layers = []
layers.append(input_tensor)
layers.append(conv1)
layers[2:] = base_model.layers[2:]
new_model = copy_model_graph(layers, base_model, input_tensor)
weights = base_model.layers[1].get_weights()
weight0 = weights[0]
w = np.concatenate((weight0, weight0), axis=2)
w = w / 2.0
weights[0] = w
# weights.append(np.zeros((64),dtype='float32'))
new_model.layers[1].set_weights(weights)
inp = Input(shape=input_shape, dtype='uint8') # 384x384x6
x = Lambda(augment)(inp)
for layer in new_model.layers:
if type(layer) is Conv2D:
layer.kernel_regularizer = regul
x = new_model(x)
x = GlobalMaxPooling2D()(x)
x = BatchNormalization()(x)
x = Dropout(rate=0.5)(x)
x = Flatten()(x)
x = Dense(512, use_bias=False, kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
encoder_model = Model(inputs=inp, outputs=x)
# softmax model for training encoder
output_softmax = Dense(num_classes, use_bias=False,
activation='softmax')(x)
softmax_model = Model(inputs=inp, outputs=output_softmax)
#################
# COMPARE MODEL #
#################
mid = 32
xa_inp = Input(shape=encoder_model.output_shape[1:])
xb_inp = Input(shape=encoder_model.output_shape[1:])
x1 = Lambda(lambda x: x[0] * x[1])([xa_inp, xb_inp])
x2 = Lambda(lambda x: x[0] + x[1])([xa_inp, xb_inp])
x3 = Lambda(lambda x: x[0] - x[1])([xa_inp, xb_inp])
x4 = Lambda(lambda x: K.square(x))(x3)
head = Concatenate()([x1, x2, x3, x4])
head = Reshape((4, encoder_model.output_shape[1], 1),
name='reshape1')(head)
# Per feature NN with shared weight is implemented using CONV2D with appropriate stride.
head = Conv2D(mid, (4, 1), activation='relu', padding='valid')(head)
head = Reshape((encoder_model.output_shape[1], mid, 1))(head)
head = Conv2D(1, (1, mid), activation='linear', padding='valid')(head)
head = Flatten()(head)
compare_model = Model([xa_inp, xb_inp], head)
# process encoding from control
# compare the current features to all controls
features_controls = Input(
shape=[num_ctrl_classes, encoder_model.output_shape[1]])
fs = Lambda(lambda x: tf.unstack(x, axis=1))(features_controls)
# def create_mask(features_controls):
# # Use a function with a Keras Lambda layer wrapper to resolve a tensorflow issue.
# # https://stackoverflow.com/questions/50715928/valueerror-output-tensors-to-a-model-must-be-the-output-of-a-tensorflow-layer
# max_abs_features = K.max(K.abs(features_controls), axis=2)
# mask = tf.greater(max_abs_features, K.epsilon())
# mask = tf.expand_dims(tf.expand_dims(tf.dtypes.cast(mask, K.floatx()), axis=-1), axis=-1)
# return mask
# mask = Lambda(create_mask)(features_controls)
comps = []
for f in fs:
comp = compare_model([x, f])
comps.append(comp)
c = Concatenate()(comps)
c = Reshape((num_ctrl_classes, encoder_model.output_shape[1], 1))(c)
# c = Lambda(lambda x: tf.math.multiply(x[0], x[1]))([c, mask])
# compare = Lambda(compare_features)([x, features_controls])
# Per feature NN with shared weight is implemented using CONV2D with appropriate stride.
compare = Conv2D(mid, (num_ctrl_classes, 1),
activation='relu',
padding='valid')(c)
compare = Reshape((encoder_model.output_shape[1], mid, 1))(compare)
compare = Conv2D(1, (1, mid), activation='linear',
padding='valid')(compare)
compare = Flatten(name='flatten2')(compare)
feature_model = Model(inputs=[inp, features_controls], outputs=compare)
label = Input(shape=(num_classes, ))
output_arcface = ArcFace(num_classes, regularizer=regul)([compare, label])
arcface_model = Model([inp, features_controls, label], output_arcface)
output_cosface = CosFace(num_classes, regularizer=regul)([compare, label])
cosface_model = Model([inp, features_controls, label], output_cosface)
return encoder_model, softmax_model, feature_model, arcface_model, cosface_model
def _main(args):
config_path = os.path.expanduser(args.config_path)
weights_path = os.path.expanduser(args.weights_path)
assert config_path.endswith('.cfg'), '{} is not a .cfg file'.format(
config_path)
assert weights_path.endswith(
'.weights'), '{} is not a .weights file'.format(weights_path)
output_path = os.path.expanduser(args.output_path)
assert output_path.endswith(
'.h5'), 'output path {} is not a .h5 file'.format(output_path)
output_root = os.path.splitext(output_path)[0]
# Load weights and config.
print('Loading weights.')
weights_file = open(weights_path, 'rb')
major, minor, revision = np.ndarray(shape=(3, ),
dtype='int32',
buffer=weights_file.read(12))
if (major * 10 + minor) >= 2 and major < 1000 and minor < 1000:
seen = np.ndarray(shape=(1, ),
dtype='int64',
buffer=weights_file.read(8))
else:
seen = np.ndarray(shape=(1, ),
dtype='int32',
buffer=weights_file.read(4))
print('Weights Header: ', major, minor, revision, seen)
print('Parsing Darknet config.')
unique_config_file = unique_config_sections(config_path)
cfg_parser = configparser.ConfigParser()
cfg_parser.read_file(unique_config_file)
print('Creating Keras model.')
input_layer = Input(shape=(None, None, 3))
prev_layer = input_layer
all_layers = []
weight_decay = float(cfg_parser['net_0']['decay']
) if 'net_0' in cfg_parser.sections() else 5e-4
count = 0
out_index = []
for section in cfg_parser.sections():
print('Parsing section {}'.format(section))
if section.startswith('convolutional'):
filters = int(cfg_parser[section]['filters'])
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
pad = int(cfg_parser[section]['pad'])
activation = cfg_parser[section]['activation']
batch_normalize = 'batch_normalize' in cfg_parser[section]
padding = 'same' if pad == 1 and stride == 1 else 'valid'
# Setting weights.
# Darknet serializes convolutional weights as:
# [bias/beta, [gamma, mean, variance], conv_weights]
prev_layer_shape = K.int_shape(prev_layer)
weights_shape = (size, size, prev_layer_shape[-1], filters)
darknet_w_shape = (filters, weights_shape[2], size, size)
weights_size = np.product(weights_shape)
print('conv2d', 'bn' if batch_normalize else ' ', activation,
weights_shape)
conv_bias = np.ndarray(shape=(filters, ),
dtype='float32',
buffer=weights_file.read(filters * 4))
count += filters
if batch_normalize:
bn_weights = np.ndarray(shape=(3, filters),
dtype='float32',
buffer=weights_file.read(filters * 12))
count += 3 * filters
bn_weight_list = [
bn_weights[0], # scale gamma
conv_bias, # shift beta
bn_weights[1], # running mean
bn_weights[2] # running var
]
conv_weights = np.ndarray(shape=darknet_w_shape,
dtype='float32',
buffer=weights_file.read(weights_size *
4))
count += weights_size
# DarkNet conv_weights are serialized Caffe-style:
# (out_dim, in_dim, height, width)
# We would like to set these to Tensorflow order:
# (height, width, in_dim, out_dim)
conv_weights = np.transpose(conv_weights, [2, 3, 1, 0])
conv_weights = [conv_weights] if batch_normalize else [
conv_weights, conv_bias
]
# Handle activation.
act_fn = None
if activation == 'leaky':
pass # Add advanced activation later.
elif activation != 'linear':
raise ValueError(
'Unknown activation function `{}` in section {}'.format(
activation, section))
# Create Conv2D layer
if stride > 1:
# Darknet uses left and top padding instead of 'same' mode
prev_layer = ZeroPadding2D(((1, 0), (1, 0)))(prev_layer)
conv_layer = (Conv2D(filters, (size, size),
strides=(stride, stride),
kernel_regularizer=l2(weight_decay),
use_bias=not batch_normalize,
weights=conv_weights,
activation=act_fn,
padding=padding))(prev_layer)
if batch_normalize:
conv_layer = (BatchNormalization(
weights=bn_weight_list))(conv_layer)
prev_layer = conv_layer
if activation == 'linear':
all_layers.append(prev_layer)
elif activation == 'leaky':
act_layer = LeakyReLU(alpha=0.1)(prev_layer)
prev_layer = act_layer
all_layers.append(act_layer)
elif section.startswith('route'):
ids = [int(i) for i in cfg_parser[section]['layers'].split(',')]
layers = [all_layers[i] for i in ids]
if len(layers) > 1:
print('Concatenating route layers:', layers)
concatenate_layer = Concatenate()(layers)
all_layers.append(concatenate_layer)
prev_layer = concatenate_layer
else:
skip_layer = layers[0] # only one layer to route
all_layers.append(skip_layer)
prev_layer = skip_layer
elif section.startswith('maxpool'):
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
all_layers.append(
MaxPooling2D(pool_size=(size, size),
strides=(stride, stride),
padding='same')(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('shortcut'):
index = int(cfg_parser[section]['from'])
activation = cfg_parser[section]['activation']
assert activation == 'linear', 'Only linear activation supported.'
all_layers.append(Add()([all_layers[index], prev_layer]))
prev_layer = all_layers[-1]
elif section.startswith('upsample'):
stride = int(cfg_parser[section]['stride'])
assert stride == 2, 'Only stride=2 supported.'
all_layers.append(UpSampling2D(stride)(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('yolo'):
out_index.append(len(all_layers) - 1)
all_layers.append(None)
prev_layer = all_layers[-1]
elif section.startswith('net'):
pass
else:
raise ValueError(
'Unsupported section header type: {}'.format(section))
# Create and save model.
if len(out_index) == 0: out_index.append(len(all_layers) - 1)
model = Model(inputs=input_layer,
outputs=[all_layers[i] for i in out_index])
print(model.summary())
if args.weights_only:
model.save_weights('{}'.format(output_path))
print('Saved Keras weights to {}'.format(output_path))
else:
model.save('{}'.format(output_path))
print('Saved Keras model to {}'.format(output_path))
# Check to see if all weights have been read.
remaining_weights = len(weights_file.read()) / 4
weights_file.close()
print('Read {} of {} from Darknet weights.'.format(
count, count + remaining_weights))
if remaining_weights > 0:
print('Warning: {} unused weights'.format(remaining_weights))
if args.plot_model:
plot(model, to_file='{}.png'.format(output_root), show_shapes=True)
print('Saved model plot to {}.png'.format(output_root))
mu_model = Sequential()
mu_model.add(Flatten(input_shape=(1, ) + env.observation_space.shape))
mu_model.add(Dense(16))
mu_model.add(Activation('relu'))
mu_model.add(Dense(16))
mu_model.add(Activation('relu'))
mu_model.add(Dense(16))
mu_model.add(Activation('relu'))
mu_model.add(Dense(nb_actions))
mu_model.add(Activation('linear'))
print(mu_model.summary())
action_input = Input(shape=(nb_actions, ), name='action_input')
observation_input = Input(shape=(1, ) + env.observation_space.shape,
name='observation_input')
x = Concatenate()([action_input, Flatten()(observation_input)])
x = Dense(32)(x)
x = Activation('relu')(x)
x = Dense(32)(x)
x = Activation('relu')(x)
x = Dense(32)(x)
x = Activation('relu')(x)
x = Dense(((nb_actions * nb_actions + nb_actions) // 2))(x)
x = Activation('linear')(x)
L_model = Model(inputs=[action_input, observation_input], outputs=x)
print(L_model.summary())
# Finally, we configure and compile our agent. You can use every built-in tensorflow.keras optimizer and
# even the metrics!
processor = PendulumProcessor()
memory = SequentialMemory(limit=100000, window_length=1)
def building(self, x, skip):
"""
Given a TensorFlow layer, this functions continues adding more layers of a SkipCNN.
:param x: A layer from TensorFlow.
:param skip: Number of the layer it has to do the skip into. If the number is
greater than the number of layers, it will be calculated and reasigned a new value.
:return: The layer received from parameter with the SkipCNN concatenated to it.
"""
skip = (self.descriptor.number_hidden_layers % (skip - 2)) + 2
for lay_indx in range(self.descriptor.number_hidden_layers):
if lay_indx == 0:
skip_layer = x
skip_kernel_size = x.shape[1]
if skip == lay_indx:
actual_kernel_size = x.shape[1]
x = UpSampling2D((skip_kernel_size, skip_kernel_size))(x)
x = MaxPooling2D((actual_kernel_size, actual_kernel_size))(x)
x = Concatenate(axis=-1)([x, skip_layer])
if self.descriptor.layers[
lay_indx] == 2: # If the layer is convolutional
x = Conv2D(self.descriptor.filters[lay_indx][2], [
self.descriptor.filters[lay_indx][0],
self.descriptor.filters[lay_indx][1]
],
strides=[
self.descriptor.strides[lay_indx][0],
self.descriptor.strides[lay_indx][1]
],
padding="valid",
activation=self.descriptor.act_functions[lay_indx],
kernel_initializer=self.descriptor.
init_functions[lay_indx])(x)
elif self.descriptor.layers[
lay_indx] == 0: # If the layer is average pooling
x = AveragePooling2D(pool_size=[
self.descriptor.filters[lay_indx][0],
self.descriptor.filters[lay_indx][1]
],
strides=[
self.descriptor.strides[lay_indx][0],
self.descriptor.strides[lay_indx][1]
],
padding="valid")(x)
else:
x = MaxPooling2D(pool_size=[
self.descriptor.filters[lay_indx][0],
self.descriptor.filters[lay_indx][1]
],
strides=[
self.descriptor.strides[lay_indx][0],
self.descriptor.strides[lay_indx][1]
],
padding="valid")(x)
return x
def __init__(self):
super(VAE, self).__init__()
# VAE model = encoder + decoder
# [input_four + action] --> [predicted_{img + reward + done}]
# categorize input
self.inputs = InputLayer(input_shape=input_shape, name='encoder_input')
self.input_four = Lambda(
lambda x: x[:, :image_size[0] - 1, :, :image_size[2] - 1])
self.input_fifth = Lambda(
lambda x: x[:, :image_size[0] - 1, :, image_size[2] - 1])
# speed will not be counted in this version
self.action = Lambda(
lambda
x: # action is no longer two-value but a scalar, representing steering
x[:, image_size[0] - 1, 5:6, 0])
self.reward = Lambda(lambda x: x[:, image_size[0] - 1, 6:7, 0])
self.done = Lambda(
lambda x: # done is a binary value, i.e. either 0.0 or 1.0
x[:, image_size[0] - 1, 7:8, 0])
"""
Conv2D layer: Conv2D(filter_num, filter_size, activation, strides, padding)
padding = 'valid': H = ceil((H1 - filter_Height + 1) / stride)
padding = 'same': H = ceil(H1 / stride)
"""
# build encoder
self.conv1 = Conv2D(12, (5, 5),
activation='relu',
strides=(2, 2),
padding='valid') # (?, 58, 78, 12)
self.conv2 = Conv2D(24, (5, 5),
activation='relu',
strides=(2, 2),
padding='valid') # (?, 27, 37, 24)
self.conv3 = Conv2D(36, (5, 5),
activation='relu',
strides=(2, 2),
padding='valid') # (?, 12, 17, 36)
self.conv4 = Conv2D(48, (5, 5),
activation='relu',
strides=(1, 1),
padding='valid') # (?, 8, 13, 48)
self.conv5 = Conv2D(48, (3, 3),
activation='relu',
strides=(1, 1),
padding='valid') # (?, 6, 11, 48)
self.conv6 = Conv2D(64, (3, 3),
activation='relu',
strides=(1, 1),
padding='valid') # (?, 4, 9, 64)
# FC layer
self.flatten = Flatten()
self.d1 = Dense(1000, activation='relu') # (?, 1000)
self.d2 = Dense(100, activation='relu') # (?, 100)
# activation function before sampling process is special
# 'linear' for z_mean, clip at least from above in z_log_var in avoidance with NaN
self.dmean = Dense(latent_dim, activation='linear', name='z_mean')
self.dlogvar = Dense(latent_dim, activation='linear', name='z_log_var')
self.sampling = Lambda(sampling, output_shape=(latent_dim, ), name='z')
# merge classical latent space z along with action
self.merge = Concatenate()
# build decoder model
# 1. generate predicted_img
self.dmerge1 = Dense(100, activation='relu')
self.dmerge2 = Dense(1000, activation='relu')
self.shape = [4, 9, 64]
self.drecover = Dense(self.shape[0] * self.shape[1] * self.shape[2],
activation='relu')
self.reshape = Reshape(
(self.shape[0], self.shape[1], self.shape[2])) # (?, 4, 9, 64)
"""
Conv2DTranspose layer: (filter_num, filter_size, activation, strides, padding)
padding == 'valid': H = (H1 - 1) * stride + filter_Height
padding == 'same': H = H1 * stride
"""
self.deconv1 = Conv2DTranspose(48, (3, 3),
activation='relu',
strides=(1, 1),
padding='valid') # (?, 6, 11, 48)
self.deconv2 = Conv2DTranspose(48, (3, 3),
activation='relu',
strides=(1, 1),
padding='valid') # (?, 8, 13, 48)
self.deconv3 = Conv2DTranspose(36, (5, 5),
activation='relu',
strides=(1, 1),
padding='valid') # (?, 12, 17, 36)
self.deconv4 = Conv2DTranspose(24, (5, 5),
activation='relu',
strides=(2, 2),
padding='valid') # (?, 27, 37, 24)
self.deconv5 = Conv2DTranspose(12, (6, 6),
activation='relu',
strides=(2, 2),
padding='valid') # (?, 58, 78, 12)
# activation for last step in each part of decoder is special
# image will be clipped in [0, 1] as did for data set
# reward will use 'linear'
# done will use 'sigmoid'
self.deconv6 = Conv2DTranspose(1, (6, 6),
activation='sigmoid',
strides=(2, 2),
padding='valid') # (?, 120, 160, 1)
# predicted_img = Flatten()(predicted_img) # (?, 120 * 160 * 1) ??necessary?
# 2. generate predicted_reward
self.dr1 = Dense(100, activation='relu')
self.dr2 = Dense(1000, activation='relu')
self.dr3 = Dense(1000, activation='relu')
self.dr4 = Dense(100, activation='relu')
self.dreward = Dense(1, activation='linear') # (?, 1)
# 3. generate predicted_done
self.dd1 = Dense(100, activation='relu')
self.dd2 = Dense(1000, activation='relu')
self.dd3 = Dense(1000, activation='relu')
self.dd4 = Dense(100, activation='relu')
self.ddone = Dense(1, activation='sigmoid') # (?, 1)
self.input_var = None
self.input_four_var = None
self.input_fifth_var = None
self.action_var = None
self.reward_var = None
self.done_var = None
self.z_mean_var = None
self.z_log_var_var = None
self.predicted_img_var = None
self.predicted_reward_var = None
self.predicted_done_var = None
#######################ACTOR------END########################################
# Create the variables for the input to the critic network and flatten the observation input to match the shape of the network
action_input = Input(shape=(nb_actions, ), name='action_input')
observation_input = Input(shape=(1, ) + env.observation_space.shape,
name='observation_input')
flattened_observation = Flatten()(observation_input)
#######################CRITIC-------START####################################
# Create the critic network, ensure that the Reshape() comes from tensorflow, a workaround to a bug encountered during creation
# Print the summary of the network to the user in the command line interface
x = Input(batch_shape=(None, 16))
x = tf.keras.layers.Reshape((16, ))(x)
x = Concatenate()([action_input, flattened_observation])
x = Dense(16)(x)
x = Activation('tanh')(x)
x = Dense(32)(x)
x = Activation('tanh')(x)
x = Dropout(0.05)(x)
x = Dense(32)(x)
x = Activation('tanh')(x)
x = Dense(1)(x)
x = Activation('tanh')(x)
critic = Model(inputs=[action_input, observation_input], outputs=x)
print(critic.summary())
#######################CRITIC-------END######################################
# build model
n,m = x_train.shape[0], x_train.shape[1]
inputs=Input((m,))
embed=Embedding(input_dim=vocab_size, output_dim=embed_size, input_length=doc_length)
hidden=embed(inputs)
reshape=Reshape((m,embed_size,1), input_shape=(m,embed_size))(hidden)
pooled_output=[]
for fsize in filter_sizes:
conv=Conv2D(filters=filter_nums,
kernel_size=(fsize,embed_size),
activation='relu',
padding="valid")(reshape)
pool=MaxPool2D(pool_size=(m-fsize+1,1))(conv)
drop=Dropout(0.5)(pool)
reshaped=Reshape((filter_nums,), input_shape=(1,1,filter_nums))(drop)
pooled_output.append(reshaped)
hidden2=Concatenate()(pooled_output)
output=Dense(1, activation='sigmoid')(hidden2)
model = Model(inputs=inputs, outputs=output)
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
print(model.summary())
model.fit(x_train, y_train, epochs=10, verbose=1)
# evaluate the model
loss, accuracy = model.evaluate(x_dev, y_dev, verbose=0)
print('Accuracy: %f' % (accuracy))
def test_ProfilePredictorConv1D(self):
input_profile_names = ['a', 'b', 'c']
target_profile_names = ['a', 'b']
actuator_names = ['aa', 'bb', 'cc']
lookbacks = {'a': 1,
'b': 1,
'c': 1,
'aa': 5,
'bb': 5,
'cc': 5}
lookahead = 4
profile_length = 33
std_activation = 'relu'
rnn_layer = layers.LSTM
profile_inshape = (lookbacks[input_profile_names[0]], profile_length)
past_actuator_inshape = (lookbacks[actuator_names[0]],)
future_actuator_inshape = (lookahead,)
num_profiles = len(input_profile_names)
num_targets = len(target_profile_names)
num_actuators = len(actuator_names)
# input each profile sig one by one and then concat them together
profile_inputs = []
profiles = []
for i in range(num_profiles):
profile_inputs.append(
Input(profile_inshape, name='input_' + input_profile_names[i]))
profiles.append(Reshape((lookbacks[input_profile_names[i]], profile_length, 1))
(profile_inputs[i]))
current_profiles = Concatenate(axis=-1)(profiles)
current_profiles = Reshape(
(profile_length, num_profiles))(current_profiles)
# input previous and future actuators and concat each of them
actuator_past_inputs = []
actuator_future_inputs = []
previous_actuators = []
future_actuators = []
for i in range(num_actuators):
actuator_future_inputs.append(
Input(future_actuator_inshape,
name="input_future_{}".format(actuator_names[i]))
)
actuator_past_inputs.append(
Input(past_actuator_inshape,
name="input_past_{}".format(actuator_names[i]))
)
future_actuators.append(Reshape((lookahead, 1))
(actuator_future_inputs[i]))
previous_actuators.append(
Reshape((lookbacks[actuator_names[i]], 1))(actuator_past_inputs[i]))
future_actuators = Concatenate(axis=-1)(future_actuators)
previous_actuators = Concatenate(axis=-1)(previous_actuators)
print(future_actuators.shape)
print(previous_actuators.shape)
print(current_profiles.shape)
#######################################################################
actuator_effect = rnn_layer(
profile_length, activation=std_activation)(previous_actuators)
actuator_effect = layers.Reshape(
target_shape=(profile_length, 1))(actuator_effect)
future_actuator_effect = rnn_layer(
profile_length, activation=std_activation)(future_actuators)
future_actuator_effect = layers.Reshape(
target_shape=(profile_length, 1))(future_actuator_effect)
current_profiles_processed_0 = layers.Concatenate()(
[current_profiles, actuator_effect, future_actuator_effect])
prof_act = []
for i in range(num_targets):
current_profiles_processed_1 = layers.Conv1D(filters=8, kernel_size=2,
padding='same',
activation='relu')(current_profiles_processed_0)
current_profiles_processed_2 = layers.Conv1D(filters=8, kernel_size=4,
padding='same',
activation='relu')(current_profiles_processed_1)
current_profiles_processed_3 = layers.Conv1D(filters=8, kernel_size=8,
padding='same',
activation='relu')(current_profiles_processed_2)
final_output = layers.Concatenate()(
[current_profiles_processed_1, current_profiles_processed_2, current_profiles_processed_3])
final_output = layers.Conv1D(filters=10, kernel_size=4,
padding='same', activation='tanh')(final_output)
final_output = layers.Conv1D(filters=1, kernel_size=4,
padding='same', activation='linear')(final_output)
final_output = layers.Reshape(target_shape=(
profile_length,), name="target_"+target_profile_names[i])(final_output)
prof_act.append(final_output)
print(len(prof_act))
model = Model(inputs=profile_inputs + actuator_past_inputs +
actuator_future_inputs, outputs=prof_act)
name = 'test___ProfilePredictorConv1D' + str(int(time.time()))
keras2c_main.k2c(model, name)
rcode = build_and_run(name)
self.assertEqual(rcode, 0)
def projection_block(self, x, strides=(2, 2), **metaparameters):
""" Construct a ResNeXT block with projection shortcut
x : input to the block
strides : whether entry convolution is strided (i.e., (2, 2) vs (1, 1))
filters_in : number of filters (channels) at the input convolution
filters_out: number of filters (channels) at the output convolution
cardinality: width of group convolution
"""
filters_in = metaparameters['filters_in']
filters_out = metaparameters['filters_out']
if 'cardinality' in metaparameters:
cardinality = metaparameters['cardinality']
else:
cardinality = self.cardinality
# Construct the projection shortcut
# Increase filters by 2X to match shape when added to output of block
shortcut = self.Conv2D(x,
filters_out, (1, 1),
strides=strides,
padding='same',
**metaparameters)
shortcut = self.BatchNormalization(shortcut)
# Dimensionality Reduction
x = self.Conv2D(x,
filters_in, (1, 1),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters)
x = self.BatchNormalization(x)
x = self.ReLU(x)
# Cardinality (Wide) Layer (split-transform)
filters_card = filters_in // cardinality
groups = []
for i in range(cardinality):
group = Lambda(lambda z: z[:, :, :, i * filters_card:i *
filters_card + filters_card])(x)
groups.append(
self.Conv2D(group,
filters_card, (3, 3),
strides=strides,
padding='same',
use_bias=False,
**metaparameters))
# Concatenate the outputs of the cardinality layer together (merge)
x = Concatenate()(groups)
x = self.BatchNormalization(x)
x = self.ReLU(x)
# Dimensionality restoration
x = self.Conv2D(x,
filters_out, (1, 1),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters)
x = self.BatchNormalization(x)
# Identity Link: Add the shortcut (input) to the output of the block
x = Add()([shortcut, x])
x = self.ReLU(x)
return x
def SSD300(input_shape, num_classes):
input_tensor = Input(shape=input_shape)
img_size = (input_shape[1], input_shape[0]) # last channel
net = VGG16(input_tensor)
# 38x38x512
net['conv4_3_norm'] = Normalize(20, name='conv4_3_norm') (net['conv4_3'])
num_priors = 4
# 38x38x16 first featureMap
net['conv4_3_norm_mbox_loc'] = Conv2D(num_priors * 4, kernel_size=(3, 3), padding='same', name='conv4_3_norm_mbox_loc') (net['conv4_3_norm'])
net['conv4_3_norm_mbox_loc_flat'] = Flatten(name='conv4_3_norm_mbox_loc_flat') (net['conv4_3_norm_mbox_loc'])
# [38, 38, num_classes * num_priors] first featureMap
net['conv4_3_norm_mbox_conf'] = Conv2D(num_priors * num_classes, kernel_size=(3, 3), padding='same', name='conv4_3_norm_mbox_conf') (net['conv4_3_norm'])
net['conv4_3_norm_mbox_conf_flat'] = Flatten(name='onv4_3_norm_mbox_conf_flat') (net['conv4_3_norm_mbox_conf'])
priorbox = PriorBox(img_size, 30.0, max_size=60.0, aspect_ratios=[2], variances=[0.1, 0.1, 0.2, 0.2], name='conv4_3_norm_mbox_priorbox')
# 38 * 38 * 4 = 5576
net['conv4_3_norm_mbox_priorbox'] = priorbox.call(net['conv4_3_norm'])
num_priors = 6
# [19, 19, 24]
net['fc7_mbox_loc'] = Conv2D(num_priors * 4, kernel_size=(3, 3), padding='same', name='fc7_mbox_loc') (net['fc7'])
net['fc7_mbox_loc_flat'] = Flatten(name='fc7_mbox_loc_flat') (net['fc7_mbox_loc'])
# [19, 19, num_priors * num_classes] second featureMap
net['fc7_mbox_conf'] = Conv2D(num_priors * num_classes, kernel_size=(3, 3), padding='same', name='fc7_mbox_conf') (net['fc7'])
net['fc7_mbox_conf_flat'] = Flatten(name='fc7_mbox_conf_flat') (net['fc7_mbox_conf'])
priorbox = PriorBox(img_size, 60.0, max_size=111.0, aspect_ratios=[2, 3], variances=[0.1, 0.1, 0.2, 0.2], name='fc7_mbox_priorbox')
# 19 * 19 * 6 = 2166
net['fc7_mbox_priorbox'] = priorbox(net['fc7'])
num_priors = 6
# [10, 10, 24]
net['conv6_2_mbox_loc'] = Conv2D(num_priors * 4, kernel_size=(3, 3), padding='same', name='conv6_2_mbox_loc') (net['conv6_2'])
net['conv6_2_mbox_loc_flat'] = Flatten(name='conv6_2_mbox_loc_flat') (net['conv6_2_mbox_loc'])
net['conv6_2_mbox_conf'] = Conv2D(num_priors * num_classes, kernel_size=(3, 3), padding='same', name='conv6_2_mbox_conf') (net['conv6_2'])
net['conv6_2_mbox_conf_flat'] = Flatten(name='conv6_2_mbox_conf_flat') (net['conv6_2_mbox_conf'])
priorbox = PriorBox(img_size, 111.0, max_size=162.0, aspect_ratios=[2, 3], variances=[0.1, 0.1, 0.2, 0.2], name='conv6_2_mbox_priorbox')
# 10x10x6 = 600
net['conv6_2_mbox_priorbox'] = priorbox(net['conv6_2'])
num_priors = 6
# [5, 5, 24]
net['conv7_2_mbox_loc'] = Conv2D(num_priors * 4, kernel_size=(3, 3), padding='same', name='conv7_2_mbox_loc') (net['conv7_2'])
net['conv7_2_mbox_loc_flat'] = Flatten(name='conv7_2_mbox_loc_flat') (net['conv7_2_mbox_loc'])
net['conv7_2_mbox_conf'] = Conv2D(num_priors * num_classes, kernel_size=(3, 3), padding='same', name='conv7_2_mbox_conf') (net['conv7_2'])
net['conv7_2_mbox_conf_flat'] = Flatten(name='conv7_2_mbox_conf_flat') (net['conv7_2_mbox_conf'])
priorbox = PriorBox(img_size, 162.0, max_size=213.0, aspect_ratios=[2, 3], variances=[0.1, 0.1, 0.2, 0.2], name='conv7_2_mbox_priorbox')
# 5x5x6 = 150
net['conv7_2_mbox_priorbox'] = priorbox(net['conv7_2'])
num_priors = 4
net['conv8_2_mbox_loc'] = Conv2D(num_priors * 4, kernel_size=(3, 3), padding='same', name='conv8_2_mbox_loc') (net['conv8_2'])
net['conv8_2_mbox_loc_flat'] = Flatten(name='conv8_2_mbox_loc_flat') (net['conv8_2_mbox_loc'])
net['conv8_2_mbox_conf'] = Conv2D(num_priors * num_classes, kernel_size=(3, 3), padding='same', name='conv8_2_mbox_conf') (net['conv8_2'])
net['conv8_2_mbox_conf_flat'] = Flatten(name='conv8_2_mbox_conf_flat') (net['conv8_2_mbox_conf'])
# 3x3x4 = 36
priorbox = PriorBox(img_size, 213.0, max_size=264.0, aspect_ratios=[2], variances=[0.1, 0.1, 0.2, 0.2], name='conv8_2_mbox_priorbox')
net['conv8_2_mbox_priorbox'] = priorbox(net['conv8_2'])
num_priors = 4
net['conv9_2_mbox_loc'] = Conv2D(num_priors * 4, kernel_size=(3, 3), padding='same', name='conv9_2_mbox_loc') (net['conv9_2'])
net['conv9_2_mbox_loc_flat'] = Flatten(name='conv9_2_mbox_loc_flat') (net['conv9_2_mbox_loc'])
net['conv9_2_mbox_conf'] = Conv2D(num_priors * num_classes, kernel_size=(3, 3), padding='same', name='conv9_2_mbox_conf') (net['conv9_2'])
net['conv9_2_mbox_conf_flat'] = Flatten(name='conv9_2_mbox_conf_flat') (net['conv9_2_mbox_conf'])
# 1x1x4
priorbox = PriorBox(img_size, 264.0, max_size=315.0, aspect_ratios=[2], variances=[0.1, 0.1, 0.2, 0.2], name='conv9_2_mbox_priorbox')
net['conv9_2_mbox_priorbox'] = priorbox(net['conv9_2'])
# a = net['conv4_3_norm_mbox_loc_flat']
# b = net['fc7_mbox_loc_flat']
# d = net['conv6_2_mbox_loc_flat']
# e = net['conv7_2_mbox_loc_flat']
# f = net['conv8_2_mbox_loc_flat']
# g = net['conv9_2_mbox_loc_flat']
# v = net['mbox_loc']
net['mbox_loc'] = Concatenate(axis=1, name='mbox_loc') ([
net['conv4_3_norm_mbox_loc_flat'],
net['fc7_mbox_loc_flat'],
net['conv6_2_mbox_loc_flat'],
net['conv7_2_mbox_loc_flat'],
net['conv8_2_mbox_loc_flat'],
net['conv9_2_mbox_loc_flat']
])
net['mbox_conf'] = Concatenate(axis=1, name='mbox_conf') ([
net['conv4_3_norm_mbox_conf_flat'],
net['fc7_mbox_conf_flat'],
net['conv6_2_mbox_conf_flat'],
net['conv7_2_mbox_conf_flat'],
net['conv8_2_mbox_conf_flat'],
net['conv9_2_mbox_conf_flat']
])
net['mbox_priorbox'] = Concatenate(axis=1, name='mbox_priorbox') ([
net['conv4_3_norm_mbox_priorbox'],
net['fc7_mbox_priorbox'],
net['conv6_2_mbox_priorbox'],
net['conv7_2_mbox_priorbox'],
net['conv8_2_mbox_priorbox'],
net['conv9_2_mbox_priorbox']
])
# net['mbox_loc'] = tf.reshape((-1, 4), name='mbox_loc_final') (net['mbox_loc'])
# 最终8732个预测框, 回归问题
net['mbox_loc'] = tf.keras.layers.Reshape((-1, 4), name='mbox_loc_final') (net['mbox_loc'])
# finally [8732, num_classes]
net['mbox_conf'] = tf.keras.layers.Reshape((-1, num_classes), name='mbox_conf_logits') (net['mbox_conf'])
net['mbox_conf'] = tf.keras.layers.Activation('softmax', name='mbox_conf_final') (net['mbox_conf'])
net['predictions'] = Concatenate(axis=2, name='predictions') ([
net['mbox_loc'],
net['mbox_conf'],
net['mbox_priorbox']
])
model = tf.keras.Model(net['input'], net['predictions']) # 闭包
return model
def __init__(self):
super(MedModel, self).__init__()
# Optimizer
self.optimizer = tf.keras.optimizers.SGD(
learning_rate=hp.learning_rate, momentum=hp.momentum)
# Load instance of small model .h5 (get_appropriate layer and those weight)
small_model = SmallModel()
small_model(tf.keras.Input(shape=(8, 8, 3)))
print(os.getcwd())
small_model.load_weights("./models/small_weights.h5")
self.small_model = small_model
initializer = tf.keras.initializers.Ones()
# Define Model Layers
# First Conv Block
self.med_conv1 = Conv2D(filters=64,
kernel_size=3,
strides=1,
padding='SAME',
activation=None,
name="med_conv1")
self.upsamp_small_filters_conv1 = Conv2D(
filters=64,
kernel_size=3,
kernel_initializer=self.small_conv1_init,
padding='SAME',
name='upsamp_small_filters_conv1',
trainable=False)
self.comb_tensors1 = Concatenate(axis=3, name="med_concat1")
self.med_bn1 = BatchNormalization(name="med_bn1")
self.med_relu1 = ReLU(name="med_relu1")
# Second Conv Block
self.med_conv2 = Conv2D(filters=64,
kernel_size=3,
strides=1,
padding='SAME',
activation=None,
name="med_conv2")
self.down_med_relu1 = Conv2D(filters=64,
kernel_size=1,
padding='SAME',
activation=None,
kernel_initializer=initializer,
name="reduce_filters",
trainable=False)
self.upsamp_small_filters_conv2 = Conv2D(
filters=64,
kernel_size=3,
kernel_initializer=self.small_conv2_init,
padding='SAME',
name='upsamp_small_filters_conv2',
trainable=False)
self.comb_tensors2 = Concatenate(axis=3, name="med_concat2")
self.med_bn2 = BatchNormalization(name="med_bn2")
self.med_relu2 = ReLU(name="med_relu2")
# Third Conv Block
self.med_conv3 = Conv2D(filters=64,
kernel_size=3,
strides=1,
padding='SAME',
activation=None,
name="med_conv3")
self.med_bn3 = BatchNormalization(name="med_bn3")
self.med_relu3 = ReLU(name="med_relu3")
# Fourth Conv Block
self.med_conv4 = Conv2D(filters=64,
kernel_size=3,
strides=1,
padding='SAME',
activation=None,
name="med_conv4")
self.med_bn4 = BatchNormalization(name="med_bn4")
self.med_relu4 = ReLU(name="med_relu4")
# Classification Part
self.med_class_conv1 = Conv2D(filters=128,
kernel_size=3,
strides=2,
padding='same',
name="med_class_conv1")
self.med_class_conv2 = Conv2D(filters=128,
kernel_size=3,
strides=2,
padding='same',
name="med_class_conv2")
self.med_class_flatten = Flatten(name="med_class_flatten")
self.med_class_dense = Dense(units=10, activation='softmax')
def SqueezeNet(nb_classes, inputs=(3, 224, 224)):
""" Keras Implementation of SqueezeNet(arXiv 1602.07360)
@param nb_classes: total number of final categories
Arguments:
inputs -- shape of the input images (channel, cols, rows)
"""
input_img = Input(shape=inputs)
conv1 = Conv2D(96, (7, 7),
activation='relu',
kernel_initializer='glorot_uniform',
strides=(2, 2),
padding='same',
name='conv1',
data_format="channels_first")(input_img)
maxpool1 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
name='maxpool1',
data_format="channels_first")(conv1)
fire2_squeeze = Conv2D(16, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire2_squeeze',
data_format="channels_first")(maxpool1)
fire2_expand1 = Conv2D(64, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire2_expand1',
data_format="channels_first")(fire2_squeeze)
fire2_expand2 = Conv2D(64, (3, 3),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire2_expand2',
data_format="channels_first")(fire2_squeeze)
merge2 = Concatenate(axis=1)([fire2_expand1, fire2_expand2])
fire3_squeeze = Conv2D(16, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire3_squeeze',
data_format="channels_first")(merge2)
fire3_expand1 = Conv2D(64, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire3_expand1',
data_format="channels_first")(fire3_squeeze)
fire3_expand2 = Conv2D(64, (3, 3),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire3_expand2',
data_format="channels_first")(fire3_squeeze)
merge3 = Concatenate(axis=1)([fire3_expand1, fire3_expand2])
fire4_squeeze = Conv2D(32, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire4_squeeze',
data_format="channels_first")(merge3)
fire4_expand1 = Conv2D(128, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire4_expand1',
data_format="channels_first")(fire4_squeeze)
fire4_expand2 = Conv2D(128, (3, 3),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire4_expand2',
data_format="channels_first")(fire4_squeeze)
merge4 = Concatenate(axis=1)([fire4_expand1, fire4_expand2])
maxpool4 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
name='maxpool4',
data_format="channels_first")(merge4)
fire5_squeeze = Conv2D(32, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire5_squeeze',
data_format="channels_first")(maxpool4)
fire5_expand1 = Conv2D(128, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire5_expand1',
data_format="channels_first")(fire5_squeeze)
fire5_expand2 = Conv2D(128, (3, 3),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire5_expand2',
data_format="channels_first")(fire5_squeeze)
merge5 = Concatenate(axis=1)([fire5_expand1, fire5_expand2])
fire6_squeeze = Conv2D(48, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire6_squeeze',
data_format="channels_first")(merge5)
fire6_expand1 = Conv2D(192, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire6_expand1',
data_format="channels_first")(fire6_squeeze)
fire6_expand2 = Conv2D(192, (3, 3),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire6_expand2',
data_format="channels_first")(fire6_squeeze)
merge6 = Concatenate(axis=1)([fire6_expand1, fire6_expand2])
fire7_squeeze = Conv2D(48, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire7_squeeze',
data_format="channels_first")(merge6)
fire7_expand1 = Conv2D(192, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire7_expand1',
data_format="channels_first")(fire7_squeeze)
fire7_expand2 = Conv2D(192, (3, 3),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire7_expand2',
data_format="channels_first")(fire7_squeeze)
merge7 = Concatenate(axis=1)([fire7_expand1, fire7_expand2])
fire8_squeeze = Conv2D(64, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire8_squeeze',
data_format="channels_first")(merge7)
fire8_expand1 = Conv2D(256, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire8_expand1',
data_format="channels_first")(fire8_squeeze)
fire8_expand2 = Conv2D(256, (3, 3),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire8_expand2',
data_format="channels_first")(fire8_squeeze)
merge8 = Concatenate(axis=1)([fire8_expand1, fire8_expand2])
maxpool8 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
name='maxpool8',
data_format="channels_first")(merge8)
fire9_squeeze = Conv2D(64, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire9_squeeze',
data_format="channels_first")(maxpool8)
fire9_expand1 = Conv2D(256, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire9_expand1',
data_format="channels_first")(fire9_squeeze)
fire9_expand2 = Conv2D(256, (3, 3),
activation='relu',
kernel_initializer='glorot_uniform',
padding='same',
name='fire9_expand2',
data_format="channels_first")(fire9_squeeze)
merge9 = Concatenate(axis=1)([fire9_expand1, fire9_expand2])
fire9_dropout = Dropout(0.5, name='fire9_dropout')(merge9)
conv10 = Conv2D(nb_classes, (1, 1),
activation='relu',
kernel_initializer='glorot_uniform',
padding='valid',
name='conv10',
data_format="channels_first")(fire9_dropout)
global_avgpool10 = GlobalAveragePooling2D(
data_format='channels_first')(conv10)
softmax = Activation("softmax", name='softmax')(global_avgpool10)
return Model(inputs=input_img, outputs=softmax)
def model_unet(inp):
"""
Define the unet model. See relation for deeper explanation of this part of the code.
:param inp (tf.keras.layers.Layer): Input of the NN
:return: output (tf.keras.layers.Layer): last layer of the network (see relation)
"""
padding = 'same'
strides = (2, 2)
kernel_size = (3, 3)
conv1 = Conv2D(32, kernel_size, padding=padding)(inp)
conv1 = LeakyReLU(alpha=0.2)(conv1)
conv1 = Conv2D(32, kernel_size, padding=padding)(conv1)
conv1 = LeakyReLU(alpha=0.2)(conv1)
pool1 = MaxPooling2D(pool_size=(3, 3), strides=strides, padding=padding)(conv1)
conv2 = Conv2D(64, kernel_size, padding=padding)(pool1)
conv2 = LeakyReLU(alpha=0.2)(conv2)
conv2 = Conv2D(64, kernel_size, padding=padding)(conv2)
conv2 = LeakyReLU(alpha=0.2)(conv2)
pool2 = MaxPooling2D(pool_size=(3, 3), strides=strides, padding=padding)(conv2)
conv3 = Conv2D(128, kernel_size, padding=padding)(pool2)
conv3 = LeakyReLU(alpha=0.2)(conv3)
conv3 = Conv2D(128, kernel_size, padding=padding)(conv3)
conv3 = LeakyReLU(alpha=0.2)(conv3)
pool3 = MaxPooling2D(pool_size=kernel_size, strides=strides, padding=padding)(conv3)
conv4 = Conv2D(256, kernel_size, padding=padding)(pool3)
conv4 = LeakyReLU(alpha=0.2)(conv4)
conv4 = Conv2D(256, kernel_size, padding=padding)(conv4)
conv4 = LeakyReLU(alpha=0.2)(conv4)
pool4 = MaxPooling2D(pool_size=(3, 3), strides=strides, padding=padding)(conv4)
conv5 = Conv2D(512, kernel_size, padding=padding)(pool4)
conv5 = LeakyReLU(alpha=0.2)(conv5)
conv5 = Conv2D(512, kernel_size, padding=padding)(conv5)
conv5 = LeakyReLU(alpha=0.2)(conv5)
up6 = Conv2DTranspose(256, kernel_size, strides=strides, padding=padding)(
conv5)
up6 = Concatenate()([conv4, up6])
conv6 = Conv2D(256, kernel_size, padding=padding)(up6)
conv6 = LeakyReLU(alpha=0.2)(conv6)
conv6 = Conv2D(256, kernel_size, padding=padding)(conv6)
conv6 = LeakyReLU(alpha=0.2)(conv6)
up7 = Conv2DTranspose(128, kernel_size, strides=strides, padding=padding)(
conv6)
up7 = Concatenate()([conv3, up7])
conv7 = Conv2D(128, kernel_size, padding=padding)(up7)
conv7 = LeakyReLU(alpha=0.2)(conv7)
conv7 = Conv2D(128, kernel_size, padding=padding)(conv7)
conv7 = LeakyReLU(alpha=0.2)(conv7)
up8 = Conv2DTranspose(64, kernel_size, strides=strides, padding=padding)(
conv7)
up8 = Concatenate()([conv2, up8])
conv8 = Conv2D(64, kernel_size, padding=padding)(up8)
conv8 = LeakyReLU(alpha=0.2)(conv8)
conv8 = Conv2D(64, kernel_size, padding=padding)(conv8)
conv8 = LeakyReLU(alpha=0.2)(conv8)
up9 = Conv2DTranspose(32, kernel_size, strides=strides, padding=padding)(
conv8)
up9 = Concatenate()([conv1, up9])
conv9 = Conv2D(32, kernel_size, padding=padding)(up9)
conv9 = LeakyReLU(alpha=0.2)(conv9)
conv9 = Conv2D(32, kernel_size, padding=padding)(conv9)
conv9 = LeakyReLU(alpha=0.2)(conv9)
conv10 = Conv2D(12, kernel_size, padding=padding)(conv9)
conv10 = LeakyReLU(alpha=0.2)(conv10)
drop = Dropout(0.3)(conv10)
output = Conv2D(3, kernel_size, padding=padding, activation='sigmoid')(drop)
return output
def encoder(filter_size: int,
input_tensor: tf.Tensor,
output_scaled_down: bool = False,
do_upsample: bool = True,
inject_layers=[],
namescope: str = "encoder/"):
x = input_tensor
fms = []
if not output_scaled_down:
fms = [input_tensor]
filters = (np.array([1, 2, 4, 6, 8, 12, 16]) * filter_size)
# Downsample
# ----------------------------
x = Conv2D(filters[0],
5,
padding="same",
name=f"{namescope}initial_downsample",
strides=(2, 2))(x)
x = BatchNormalization(name=f"{namescope}initial_batchnorm")(x)
x = ReLU(6., name=f"{namescope}initial_acitvation")(x)
fms.append(x)
for layer in inject_layers:
x = Concatenate()([x, layer])
x = bottle_neck_block(f"{namescope}downsample_2/", x, filters[1])
x = bottle_neck_block(f"{namescope}downsample_3/",
x,
filters[1],
downsample=True)
fms.append(x)
x = bottle_neck_block(f"{namescope}downsample_4/", x, filters[2])
x = bottle_neck_block(f"{namescope}downsample_5/",
x,
filters[2],
downsample=True)
fms.append(x)
x = bottle_neck_block(f"{namescope}downsample_6/", x, filters[3])
x = bottle_neck_block(f"{namescope}downsample_7/",
x,
filters[3],
downsample=True)
fms.append(x)
x = bottle_neck_block(f"{namescope}downsample_8/", x, filters[4])
x = bottle_neck_block(f"{namescope}downsample_9/",
x,
filters[4],
downsample=True)
fms.append(x)
x = bottle_neck_block(f"{namescope}downsample_10/", x, filters[5])
x = bottle_neck_block(f"{namescope}downsample_11/",
x,
filters[5],
downsample=True)
fms.append(x)
x = bottle_neck_block(f"{namescope}downsample_12/", x, filters[6])
x = bottle_neck_block(f"{namescope}downsample_13/",
x,
filters[6],
downsample=True)
fms.append(x)
# Alternative to more downsampling, dilated layers:
# dilation_rates = [3, 6, 9, 12]
# concat_tensors = []
# x = Conv2D(filters, kernel_size=1, name=f"{namescope}start_dilation_1x1")(x)
# concat_tensors.append(x)
# for i, rate in enumerate(dilation_rates):
# x = bottle_neck_block(f"{namescope}dilation_{i}", x, filters)
# x = BatchNormalization(name=f"{namescope}dilation_batchnorm_{i}")(x)
# concat_tensors.append(x)
# x = Concatenate(name=f"{namescope}dilation_concat")(concat_tensors)
# fms.append(x)
# Upsample
# ----------------------------
if do_upsample:
for i in range(len(fms) - 2, -1, -1):
fms[i] = Conv2D(filters[i], (3, 3),
padding="same",
name=f"{namescope}conv2d_up_{i}",
kernel_regularizer=l2(l=0.0001))(fms[i])
fms[i] = BatchNormalization(name=f"{namescope}batchnorm_up_{i}")(
fms[i])
fms[i] = ReLU(6.0)(fms[i])
x = upsample_block(f"{namescope}upsample_{i}/", x, fms[i],
filters[i])
return x, fms
def yoloNano(anchors,input_size=416,num_classes = 1,expension = .75,decay=0.0005):
#f**k tensorflow 2.x
#backbone
input_0 = Input(shape=(input_size,input_size,3))
input_gt = [Input(shape=(input_size//{0:32, 1:16, 2:8}[l], input_size//{0:32, 1:16, 2:8}[l],len(anchors)//3, num_classes+5)) for l in range(3)]
x = Conv2D(filters=12,strides=(1,1),kernel_size=(3,3),use_bias=False,padding='same',kernel_regularizer=l2(l=decay))(input_0)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=24,strides=(2,2),kernel_size=(3,3),use_bias=False,padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_0 = LeakyReLU(alpha = 0.1)(x)
#PEP(7)(208x208x24)
x = Conv2D(filters=7,strides=(1,1),kernel_size=(1,1),use_bias=False,padding='same',kernel_regularizer=l2(l=decay))(x_0)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(24 * expension),strides=(1,1),kernel_size=(1,1),use_bias=False,padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1,1),kernel_size=(3,3),use_bias=False,padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=24,strides=(1,1),kernel_size=(1,1),use_bias=False,padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = Add()([x_0 ,x])
#EP(104x104x70)
x = Conv2D(filters=math.ceil(70 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(2, 2), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=70, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x_1 = BatchNormalization()(x)
#PEP(25)(104x104x70)
x = Conv2D(filters=25, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_1)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(70 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=70, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_2 = Add()([x_1, x])
# PEP(24)(104x104x70)
x = Conv2D(filters=24, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_2)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(70 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=70, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = Add()([x_2, x])
# EP(52x52x150)
x = Conv2D(filters=math.ceil(150 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(2, 2), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=150, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x_3 = BatchNormalization()(x)
# PEP(56)(52x52x150)
x = Conv2D(filters=56, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_3)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(150 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=150, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = Add()([x_3, x])
#Conv1x1
x = Conv2D(filters=150,kernel_size=(1,1),strides=(1,1),use_bias=False,padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_4 = LeakyReLU(alpha = 0.1)(x)
#FCA(8)
x = AvgPool2D(pool_size=(52,52))(x_4)
x = Dense(units=150 // 8,activation='relu',use_bias=False,kernel_regularizer=l2(l=decay))(x)
x = Dense(units=150, activation='sigmoid', use_bias=False,kernel_regularizer=l2(l=decay))(x)
x_5 = Multiply()([x_4,x])
#PEP(73)(52x52x150)
x = Conv2D(filters=73, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_5)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(150 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=150, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_6 = Add()([x_5, x])
# PEP(71)(52x52x150)
x = Conv2D(filters=71, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_6)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(150 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=150, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_7 = Add()([x_6, x])
# PEP(75)(52x52x150)
x = Conv2D(filters=75, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_7)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(150 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=150, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_8 = Add()([x_7, x]) #output 52x52x150
#EP(26x26x325)
x = Conv2D(filters=math.ceil(325 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_8)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(2, 2), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=325, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x_9 = BatchNormalization()(x)
# PEP(132)(26x26x325)
x = Conv2D(filters=132, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_9)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=325, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_10 = Add()([x_9, x])
# PEP(124)(26x26x325)
x = Conv2D(filters=124, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_10)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=325, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_11 = Add()([x_10, x])
# PEP(141)(26x26x325)
x = Conv2D(filters=141, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_11)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=325, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_12 = Add()([x_11, x])
# PEP(140)(26x26x325)
x = Conv2D(filters=140, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_12)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=325, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_13 = Add()([x_12, x])
# PEP(137)(26x26x325)
x = Conv2D(filters=137, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_13)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=325, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_14 = Add()([x_13, x])
# PEP(135)(26x26x325)
x = Conv2D(filters=135, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_14)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=325, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_15 = Add()([x_14, x])
# PEP(133)(26x26x325)
x = Conv2D(filters=133, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_15)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=325, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_16 = Add()([x_15, x])
# PEP(140)(26x26x325)
x = Conv2D(filters=140, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_16)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=325, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_17 = Add()([x_16, x]) #output 26x26x325
# EP(13x13x545)
x = Conv2D(filters=math.ceil(545 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_17)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(2, 2), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=545, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x_18 = BatchNormalization()(x)
# PEP(276)(13x13x545)
x = Conv2D(filters=276, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_18)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(545 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=545, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_19 = Add()([x_18, x])
#Conv1x1
x = Conv2D(filters=230, kernel_size=(1, 1), strides=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_19)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
# EP(13x13x489)
x = Conv2D(filters=math.ceil(489 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=489, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# PEP(213)(13x13x469)
x = Conv2D(filters=213, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(469 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=469, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# Conv1x1
x = Conv2D(filters=189, kernel_size=(1, 1), strides=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_20 = LeakyReLU(alpha = 0.1)(x) #output 13x13x189
# EP(13x13x462)
x = Conv2D(filters=math.ceil(462 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_20)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=462, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# feature 13x13x[(num_classes+5)x3]
feature_13x13 = Conv2D(filters=3 * (num_classes + 5), kernel_size=(1, 1), strides=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
# Conv1x1
x = Conv2D(filters=105, kernel_size=(1, 1), strides=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_20)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
# upsampling 26x26x105
x = UpSampling2D()(x)
# concatenate
x = Concatenate()([x,x_17])
# PEP(113)(26x26x325)
x = Conv2D(filters=113, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(325 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=325, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# PEP(99)(26x26x207)
x = Conv2D(filters=99, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(207 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=207, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# Conv1x1
x = Conv2D(filters=98, kernel_size=(1, 1), strides=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x_21 = LeakyReLU(alpha = 0.1)(x)
# EP(13x13x183)
x = Conv2D(filters=math.ceil(183 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_21)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=183, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# feature 26x26x[(num_classes+5)x3]
feature_26x26 = Conv2D(filters=3 * (num_classes + 5), kernel_size=(1, 1), strides=(1, 1), use_bias=False,padding='same',kernel_regularizer=l2(l=decay))(x)
# Conv1x1
x = Conv2D(filters=47, kernel_size=(1, 1), strides=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x_21)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
#upsampling
x = UpSampling2D()(x)
#concatenate
x = Concatenate()([x,x_8])
# PEP(58)(52x52x132)
x = Conv2D(filters=58, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(132 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=132, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# PEP(52)(52x52x87)
x = Conv2D(filters=52, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(87 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=87, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
# PEP(47)(52x52x93)
x = Conv2D(filters=47, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=math.ceil(93 * expension), strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = DepthwiseConv2D(strides=(1, 1), kernel_size=(3, 3), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha = 0.1)(x)
x = Conv2D(filters=93, strides=(1, 1), kernel_size=(1, 1), use_bias=False, padding='same',kernel_regularizer=l2(l=decay))(x)
x = BatchNormalization()(x)
feature_52x52 = Conv2D(filters=3 * (num_classes + 5), kernel_size=(1, 1), strides=(1, 1), use_bias=False,padding='same',kernel_regularizer=l2(l=decay))(x)
#loss layer
loss = Lambda(yolo_loss, output_shape=(1,), name='yolo_loss',arguments={'anchors': anchors, 'num_classes': num_classes, 'ignore_thresh': 0.5})([feature_13x13,feature_26x26,feature_52x52, *input_gt])
debug_model = tf.keras.Model(inputs=input_0,outputs=[feature_13x13,feature_26x26,feature_52x52])
train_model = tf.keras.Model(inputs=[input_0,*input_gt],outputs=loss)
return train_model,debug_model
# import numpy as np
# anchors = np.array([[6.,9.],[8.,13.],[11.,16.],[14.,22.],[17.,37.],[21.,26.],[29.,38.],[39.,62.],[79.,99.]],dtype='float32')
# model,_ = yoloNano(anchors,input_size=416,num_classes=1)
# model.summary()
def seq2seq_model(vocabulary_size, config_params, output_size, weights=None,
tokenizer=None, visualize=False, plot=False):
drop, rdrop = 0.2, 0.2
hidden_size = int(config_params['hidden_size'])
batch_size = int(config_params['batch_size'])
input_type = 'string' if tokenizer is not None else None
encoder_inputs = Input(shape=(None,), dtype=input_type,
batch_size=batch_size)
in_mask = Input(shape=(None, output_size),
batch_size=batch_size, name='Candidate_Synsets_Mask')
if tokenizer is not None:
encoder_embeddings = ElmoEmbeddingLayer()(encoder_inputs)
embedding_size = 1024
elif weights is not None:
embedding_size = weights.shape[1]
train = True # To fine-tune pretrained embeddings or not
encoder_embeddings = Embedding(input_dim=output_size, output_dim=embedding_size, weights=[
weights], trainable=train, mask_zero=True)(encoder_inputs)
else:
embedding_size = int(config_params['embedding_size'])
encoder_embeddings = Embedding(
input_dim=vocabulary_size, output_dim=embedding_size,
mask_zero=True, name="Embeddings")(encoder_inputs)
encoder_bilstm = Bidirectional(LSTM(hidden_size, dropout=drop,
recurrent_dropout=rdrop,
return_sequences=True,
return_state=True,
input_shape=(
None, None, embedding_size)
),
merge_mode='sum',
name='Encoder_BiLSTM_1')(encoder_embeddings)
encoder_bilstm2 = Bidirectional(LSTM(hidden_size, dropout=drop,
recurrent_dropout=rdrop,
return_sequences=True,
return_state=True,
input_shape=(
None, None, embedding_size)
),
merge_mode='sum', name='Encoder_BiLSTM_2')
(encoder_outputs, forward_h, forward_c, backward_h,
backward_c) = encoder_bilstm2(encoder_bilstm)
state_h = Concatenate()([forward_h, backward_h])
state_c = Concatenate()([forward_c, backward_c])
encoder_states = [state_h, state_c]
encoder_attention = SeqSelfAttention(
attention_activation='sigmoid', name='Attention')(encoder_outputs)
decoder_fwd_lstm, _, _ = LSTM(hidden_size, dropout=drop,
recurrent_dropout=rdrop,
return_sequences=True,
return_state=True,
input_shape=(None, None, embedding_size),
name='Decoder_FWD_LSTM')(encoder_attention,
initial_state=[forward_h, backward_h])
decoder_bck_lstm, _, _ = LSTM(hidden_size,
dropout=drop,
recurrent_dropout=rdrop,
return_sequences=True,
return_state=True,
input_shape=(None, None, embedding_size),
go_backwards=True,
name='Decoder_BWD_LSTM')(decoder_fwd_lstm)
decoder_bilstm = Concatenate()([decoder_fwd_lstm, decoder_bck_lstm])
decoder_output = TimeDistributed(Dense(output_size),
name='TimeDist_Dense')(decoder_bilstm)
logits_mask = Add()([decoder_output, in_mask])
decoder_outputs = Softmax()(logits_mask)
model = Model([encoder_inputs, in_mask],
outputs=decoder_outputs, name="SensEmbed_Seq2Seq_Attention")
model.compile(loss="sparse_categorical_crossentropy",
optimizer=Adadelta(), metrics=['acc'])
visualize_plot_mdl(visualize, plot, model)
return model
# Decoder Layer
decoder_layer = GRU(hidden_size,
return_sequences=True,
dropout=dropout,
recurrent_dropout=dropout / 2,
name='decode_layer')
decoder_outputs = decoder_layer(decoder_embedding,
initial_state=encoder_states)
# Attention Layer
attention_layer = Attention(name='attention_layer')
attention_outputs = attention_layer([decoder_outputs, encoder_outputs])
# Concatenate the Result of Attention and the Hidden States of Decoder
decoder_concat_inputs = Concatenate(
axis=-1, name='concatenate_layer')([decoder_outputs, attention_outputs])
# Output Layer
output_layer = Dense(num_of_morphemes, activation=softmax, name='output_layer')
outputs = output_layer(decoder_concat_inputs)
# Define Model
model = Model(inputs=[encoder_inputs, decoder_inputs],
outputs=outputs,
name='training_model')
# Compile
model.compile(optimizer=Adam(learning_rate=learning_late),
loss=sparse_categorical_crossentropy)
# Display Model Summary
def multitask_seq2seq_model(vocabulary_size, config_params,
output_size, pos_vocab_size,
lex_vocab_size, weights=None,
tokenizer=None, visualize=False, plot=False):
hidden_size = int(config_params['hidden_size'])
batch_size = int(config_params['batch_size'])
embedding_size = int(config_params['embedding_size'])
input_type = 'string' if tokenizer is not None else None
in_sentences = Input(shape=(None,), dtype=input_type,
batch_size=batch_size, name='Input')
if tokenizer is not None:
embeddings = ElmoEmbeddingLayer()(in_sentences)
embedding_size = 1024
elif weights is not None:
embedding_size = weights.shape[1]
train = True # To fine-tune pretrained embeddings or not
embeddings = Embedding(input_dim=output_size,
output_dim=embedding_size,
weights=[weights], trainable=train, mask_zero=False)(in_sentences)
else:
embeddings = Embedding(input_dim=vocabulary_size,
output_dim=embedding_size,
mask_zero=True,
name="Embeddings")(in_sentences)
bilstm, forward_h, _, backward_h, _ = Bidirectional(LSTM(hidden_size, return_sequences=True,
return_state=True, dropout=0.2, recurrent_dropout=0.2,
input_shape=(None, None, embedding_size)),
merge_mode='sum',
name='Encoder_BiLSTM')(embeddings)
state_h = Concatenate()([forward_h, backward_h])
context, _ = Attention(hidden_size)([bilstm, state_h])
concat = Concatenate()([bilstm, context])
decoder_fwd_lstm = LSTM(hidden_size, dropout=0.2,
recurrent_dropout=0.2,
return_sequences=True,
input_shape=(None, None, embedding_size),
name='Decoder_FWD_LSTM')(concat)
decoder_bck_lstm = LSTM(hidden_size,
dropout=0.2,
recurrent_dropout=0.2,
return_sequences=True,
input_shape=(None, None, embedding_size),
go_backwards=True,
name='Decoder_BWD_LSTM')(decoder_fwd_lstm)
decoder_bilstm = Concatenate()([decoder_fwd_lstm, decoder_bck_lstm])
logits = TimeDistributed(
Dense(output_size), name='WSD_logits')(decoder_bilstm)
in_mask = Input(shape=(None, output_size),
batch_size=batch_size, name='Candidate_Synsets_Mask')
logits_mask = Add(name="Masked_logits")([logits, in_mask])
pos_logits = TimeDistributed(Dense(pos_vocab_size),
name='POS_logits')(decoder_bilstm)
lex_logits = TimeDistributed(Dense(lex_vocab_size),
name='LEX_logits')(decoder_bilstm)
wsd_output = Softmax(name="WSD_output")(logits_mask)
pos_output = Softmax(name="POS_output")(pos_logits)
lex_output = Softmax(name="LEX_output")(lex_logits)
model = Model(inputs=[in_sentences, in_mask],
outputs=[wsd_output, pos_output, lex_output],
name='SensEmbed_Seq2Seq_Attention_MultiTask')
model.compile(loss="sparse_categorical_crossentropy",
optimizer=Adadelta(), metrics=['acc'])
visualize_plot_mdl(visualize, plot, model)
return model
def u_net_module_hp(hp):
############################## Special Fun ###################################################
# embedding = tf.Variable(initial_value = np.load('../Embedding/embedd_w_5_ogt.npy').reshape(1,21,5))
############################## Transformation module #################################################
p = {
'n_fil_1': hp.Choice('number_filters_conv1', [8, 16, 32, 64]),
's_fil_1': hp.Choice('size_filters_conv1', [3, 5, 7, 10]),
'stride_1': hp.Choice('stride_length_sampling1', [2, 4]),
'dOut_1': hp.Choice('Dropout_module1', [0.1, 0.2, 0.3, 0.4]),
'n_fil_2': hp.Choice('number_filters_conv2', [32, 64, 128]),
's_fil_2': hp.Choice('size_filters_conv2', [3, 5, 7, 10]),
'stride_2': hp.Choice('stride_length_sampling2', [2, 4]),
'dOut_2': hp.Choice('Dropout_module2', [0.1, 0.2, 0.3, 0.4]),
'n_fil_3': hp.Choice('number_filters_conv3', [64, 128, 256]),
's_fil_3': hp.Choice('size_filters_conv3', [3, 5, 7, 10]),
'stride_3': hp.Choice('stride_length_sampling3', [2, 4]),
'dOut_3': hp.Choice('Dropout_module3', [0.1, 0.2, 0.3, 0.4]),
'n_fil_4': hp.Choice('number_filters_conv4', [128, 256, 512]),
's_fil_4': hp.Choice('size_filters_conv4', [3, 5, 7, 10]),
'dOut_4': hp.Choice('Dropout_module4', [0.1, 0.2, 0.3, 0.4]),
'dOut_5': hp.Choice('Dropout_module5', [0.05, 0.1, 0.2, 0.3]),
's_fil_5': hp.Choice('size_filters_conv5', [3, 5, 7, 10])
}
# Layers of stage 0 contraction
inp = Input(shape=(1024, 21))
#tct0_in = Dot(axes=(2,1))([inp,embedding])
tct0_bn1 = BatchNormalization()(inp) #tct0_in)
#tf.keras.regularizers.L1L2(l1=0.0, l2=0.0)
tct0_conv1 = Conv1D(int(p['n_fil_1']),
int(p['s_fil_1']),
activation='relu',
padding='same',
name='Convolution_Ct0_0')(tct0_bn1)
tct0_bn2 = BatchNormalization()(tct0_conv1)
tct0_conv2 = Conv1D(int(p['n_fil_1']),
int(p['s_fil_1']),
activation='relu',
padding='same',
name='Convolution_Ct0_1')(tct0_bn2)
tct0_bn3 = BatchNormalization()(tct0_conv2)
tct0_conv3 = Conv1D(int(p['n_fil_1']),
int(p['s_fil_1']),
activation='relu',
strides=int(p['stride_1']),
padding='same',
name='Convolution_Ct0_2')(tct0_bn3)
tct0_bn4 = BatchNormalization()(tct0_conv3)
#tct0_max = MaxPool1D(pool_size=2, strides=2)(tct0_bn2)
tct0_dp = Dropout(p['dOut_1'])(tct0_bn4)
# Layers of stage 1 contraction
tct1_conv1 = Conv1D(int(p['n_fil_2']),
int(p['s_fil_2']),
activation='relu',
padding='same',
name='Convolution_Ct1_0')(tct0_dp)
tct1_bn1 = BatchNormalization()(tct1_conv1)
tct1_conv2 = Conv1D(int(p['n_fil_2']),
int(p['s_fil_2']),
activation='relu',
strides=1,
padding='same',
name='Convolution_Ct1_1')(tct1_bn1)
tct1_bn2 = BatchNormalization()(tct1_conv2)
tct1_conv3 = Conv1D(int(p['n_fil_2']),
int(p['s_fil_2']),
activation='relu',
strides=int(p['stride_2']),
padding='same',
name='Convolution_Ct1_2')(tct1_bn2)
tct1_bn3 = BatchNormalization()(tct1_conv3)
#tct1_max = MaxPool1D(pool_size=2, strides=2)(tct1_bn2)
tct1_dp = Dropout(p['dOut_2'])(tct1_bn3)
# Layers of stage 2 contraction
tct2_conv1 = Conv1D(int(p['n_fil_3']),
int(p['s_fil_3']),
activation='relu',
padding='same',
name='Convolution_Ct2_0')(tct1_dp)
tct2_bn1 = BatchNormalization()(tct2_conv1)
tct2_conv2 = Conv1D(int(p['n_fil_3']),
int(p['s_fil_3']),
activation='relu',
strides=1,
padding='same',
name='Convolution_Ct2_1')(tct2_bn1)
tct2_bn2 = BatchNormalization()(tct2_conv2)
tct2_conv3 = Conv1D(int(p['n_fil_3']),
int(p['s_fil_3']),
activation='relu',
strides=int(p['stride_3']),
padding='same',
name='Convolution_Ct2_2')(tct2_bn2)
tct2_bn3 = BatchNormalization()(tct2_conv3)
#tct2_max = MaxPool1D(pool_size=2, strides=2)(tct2_bn2)
tct2_dp = Dropout(p['dOut_3'])(tct2_bn3)
# Layers of stage 3 contraction
tct3_conv1 = Conv1D(int(p['n_fil_4']),
int(p['s_fil_4']),
activation='relu',
padding='same',
name='Convolution_Ce3_0')(tct2_dp)
tct3_bn1 = BatchNormalization()(tct3_conv1)
tct3_conv2 = Conv1D(int(p['n_fil_4']),
int(p['s_fil_4']),
activation='relu',
padding='same',
name='Convolution_Ce3_1')(tct3_bn1)
tct3_bn2 = BatchNormalization()(tct3_conv2)
tct3_dp = Dropout(p['dOut_4'])(tct3_bn2)
# Layers of stage 1 expansion
tet1_Tconv = Conv1DTranspose(int(p['n_fil_3']),
int(p['s_fil_3']),
strides=int(p['stride_3']),
activation='relu',
padding='same',
name='TransConv_Et1')(tct3_dp)
tet1_Concat = Concatenate(axis=2)([tet1_Tconv, tct2_conv1])
tet1_bn1 = BatchNormalization()(tet1_Concat)
tet1_conv1 = Conv1D(int(p['n_fil_3']),
int(p['s_fil_3']),
activation='relu',
padding='same',
name='Convolution_Et1_0')(tet1_bn1)
tet1_bn2 = BatchNormalization()(tet1_conv1)
tet1_conv2 = Conv1D(int(p['n_fil_3']),
int(p['s_fil_3']),
activation='relu',
padding='same',
name='Convolution_Et1_1')(tet1_bn2)
tet1_bn3 = BatchNormalization()(tet1_conv2)
tet1_dp = Dropout(p['dOut_3'])(tet1_bn3)
#Layers of stage 2 expansion
tet2_Tconv = Conv1DTranspose(int(p['n_fil_2']),
int(p['s_fil_2']),
strides=int(p['stride_2']),
activation='relu',
padding='same',
name='TransConv_Et2')(tet1_dp)
tet2_Concat = Concatenate(axis=2)([tet2_Tconv, tct1_conv1])
tet2_bn1 = BatchNormalization()(tet2_Concat)
tet2_conv1 = Conv1D(int(p['n_fil_2']),
int(p['s_fil_2']),
activation='relu',
padding='same',
name='Convolution_Et2_0')(tet2_bn1)
tet2_bn2 = BatchNormalization()(tet2_conv1)
tet2_conv2 = Conv1D(int(p['n_fil_2']),
int(p['s_fil_2']),
activation='relu',
padding='same',
name='Convolution_Et2_1')(tet2_bn2)
tet2_bn3 = BatchNormalization()(tet2_conv2)
tet2_dp = Dropout(p['dOut_2'])(tet2_bn3)
#Layers of stage 3 expansion
tet3_Tconv = Conv1DTranspose(int(p['n_fil_1']),
int(p['s_fil_1']),
strides=int(p['stride_1']),
activation='relu',
padding='same',
name='TransConv_Et3')(tet2_dp)
tet3_Concat = Concatenate(axis=2)([tet3_Tconv, tct0_conv1])
tet3_bn1 = BatchNormalization()(tet3_Concat)
tet3_conv1 = Conv1D(int(p['n_fil_1']),
int(p['s_fil_1']),
activation='relu',
padding='same',
name='Convolution_Et3_1')(tet3_bn1)
tet3_bn2 = BatchNormalization()(tet3_conv1)
tet3_conv2 = Conv1D(int(p['n_fil_1']),
int(p['s_fil_1']),
activation='relu',
padding='same',
name='Convolution_Et3_2')(tet3_bn2)
tet3_bn3 = BatchNormalization()(tet3_conv2)
tet3_dp = Dropout(p['dOut_5'])(tet3_bn3)
tet3_conv3 = Conv1D(3,
int(p['s_fil_5']),
activation='softmax',
padding='same',
name='Convolution_Et3_3')(tet3_dp)
model = Model(inputs=inp, outputs=tet3_conv3)
cce = tf.keras.losses.CategoricalCrossentropy(
from_logits=True) #, sample_weight = )
add = tf.keras.optimizers.Adam(learning_rate=hp.Choice(
'learning_rate', [1e-2, 1e-3, 1e-4]),
name='Adam')
model.compile(optimizer=add, loss=cce, metrics=['accuracy'])
return model
def InceptionResNetV2(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the Inception-ResNet v2 architecture.
Optionally loads weights pre-trained on ImageNet.
Note that when using TensorFlow, for best performance you should
set `"image_data_format": "channels_last"` in your Keras config
at `~/.keras/keras.json`.
The model and the weights are compatible with TensorFlow, Theano and
CNTK backends. The data format convention used by the model is
the one specified in your Keras config file.
Note that the default input image size for this model is 299x299, instead
of 224x224 as in the VGG16 and ResNet models. Also, the input preprocessing
function is different (i.e., do not use `imagenet_utils.preprocess_input()`
with this model. Use `preprocess_input()` defined in this module instead).
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is `False` (otherwise the input shape
has to be `(299, 299, 3)` (with `'channels_last'` data format)
or `(3, 299, 299)` (with `'channels_first'` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 139.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the last convolutional layer.
- `'avg'` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `'max'` means that global max pooling will be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is `True`, and
if no `weights` argument is specified.
# Returns
A Keras `Model` instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=299,
min_size=139,
data_format=K.image_data_format(),
require_flatten=False,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
# Stem block: 35 x 35 x 192
x = conv2d_bn(img_input, 32, 3, strides=2, padding='same')
x = conv2d_bn(x, 32, 3, padding='same')
x = conv2d_bn(x, 64, 3)
x = MaxPooling2D(3, strides=2, padding='same')(x)
x = conv2d_bn(x, 80, 1, padding='same')
x = conv2d_bn(x, 192, 3, padding='same')
x = MaxPooling2D(3, strides=2, padding='same')(x)
# Mixed 5b (Inception-A block): 35 x 35 x 320
branch_0 = conv2d_bn(x, 96, 1)
branch_1 = conv2d_bn(x, 48, 1)
branch_1 = conv2d_bn(branch_1, 64, 5)
branch_2 = conv2d_bn(x, 64, 1)
branch_2 = conv2d_bn(branch_2, 96, 3)
branch_2 = conv2d_bn(branch_2, 96, 3)
branch_pool = AveragePooling2D(3, strides=1, padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1)
branches = [branch_0, branch_1, branch_2, branch_pool]
channel_axis = 1 if K.image_data_format() == 'channels_first' else 3
x = Concatenate(axis=channel_axis, name='mixed_5b')(branches)
# 10x block35 (Inception-ResNet-A block): 35 x 35 x 320
for block_idx in range(1, 11):
x = inception_resnet_block(x,
scale=0.17,
block_type='block35',
block_idx=block_idx)
# Mixed 6a (Reduction-A block): 17 x 17 x 1088
branch_0 = conv2d_bn(x, 384, 3, strides=2, padding='same')
branch_1 = conv2d_bn(x, 256, 1)
branch_1 = conv2d_bn(branch_1, 256, 3)
branch_1 = conv2d_bn(branch_1, 384, 3, strides=2, padding='same')
branch_pool = MaxPooling2D(3, strides=2, padding='same')(x)
branches = [branch_0, branch_1, branch_pool]
x = Concatenate(axis=channel_axis, name='mixed_6a')(branches)
# 20x block17 (Inception-ResNet-B block): 17 x 17 x 1088
for block_idx in range(1, 21):
x = inception_resnet_block(x,
scale=0.1,
block_type='block17',
block_idx=block_idx)
# Mixed 7a (Reduction-B block): 8 x 8 x 2080
branch_0 = conv2d_bn(x, 256, 1)
branch_0 = conv2d_bn(branch_0, 384, 3, strides=2, padding='same')
branch_1 = conv2d_bn(x, 256, 1)
branch_1 = conv2d_bn(branch_1, 288, 3, strides=2, padding='same')
branch_2 = conv2d_bn(x, 256, 1)
branch_2 = conv2d_bn(branch_2, 288, 3)
branch_2 = conv2d_bn(branch_2, 320, 3, strides=2, padding='same')
branch_pool = MaxPooling2D(3, strides=2, padding='same')(x)
branches = [branch_0, branch_1, branch_2, branch_pool]
x = Concatenate(axis=channel_axis, name='mixed_7a')(branches)
# 10x block8 (Inception-ResNet-C block): 8 x 8 x 2080
for block_idx in range(1, 10):
x = inception_resnet_block(x,
scale=0.2,
block_type='block8',
block_idx=block_idx)
x = inception_resnet_block(x,
scale=1.,
activation=None,
block_type='block8',
block_idx=10)
# Final convolution block: 8 x 8 x 1536
x = conv2d_bn(x, 1536, 1, name='conv_7b')
if include_top:
# Classification block
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model
model = Model(inputs, x, name='inception_resnet_v2')
# Load weights
if weights == 'imagenet':
if K.image_data_format() == 'channels_first':
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
if include_top:
fname = 'inception_resnet_v2_weights_tf_dim_ordering_tf_kernels.h5'
weights_path = get_file(
fname,
BASE_WEIGHT_URL + fname,
cache_subdir='models',
file_hash='e693bd0210a403b3192acc6073ad2e96')
else:
fname = 'inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5'
weights_path = get_file(
fname,
BASE_WEIGHT_URL + fname,
cache_subdir='models',
file_hash='d19885ff4a710c122648d3b5c3b684e4')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
def u_net_module_2(in_):
############################## Special Fun ###################################################
embedding = tf.Variable(
initial_value=np.load('../Embedding/embedd_w_5.npy').reshape(1, 21, 5),
trainable=True)
############################## Transformation module #################################################
# Layers of stage 0 contraction
inp = InputLayer(input_shape=(in_, 21))
tct0_in = Dot(axes=(2, 1))([inp, embedding])
tct0_bn1 = BatchNormalization()(tct0_in)
tct0_conv1 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Ct0_0')(tct0_bn1)
tct0_bn2 = BatchNormalization()(tct0_conv1)
tct0_conv2 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Ct0_1')(tct0_bn2)
tct0_bn3 = BatchNormalization()(tct0_conv2)
tct0_conv3 = Conv1D(32,
5,
activation='relu',
strides=2,
padding='same',
name='Convolution_Ct0_2')(tct0_bn3)
tct0_bn4 = BatchNormalization()(tct0_conv3)
#tct0_max = MaxPool1D(pool_size=2, strides=2)(tct0_bn2)
tct0_dp = Dropout(0.2)(tct0_bn4)
# Layers of stage 1 contraction
tct1_conv1 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Ct1_0')(tct0_dp)
tct1_bn1 = BatchNormalization()(tct1_conv1)
tct1_conv2 = Conv1D(32,
5,
activation='relu',
strides=1,
padding='same',
name='Convolution_Ct1_1')(tct1_bn1)
tct1_bn2 = BatchNormalization()(tct1_conv2)
tct1_conv3 = Conv1D(32,
5,
activation='relu',
strides=2,
padding='same',
name='Convolution_Ct1_2')(tct1_bn2)
tct1_bn3 = BatchNormalization()(tct1_conv3)
#tct1_max = MaxPool1D(pool_size=2, strides=2)(tct1_bn2)
tct1_dp = Dropout(0.2)(tct1_bn3)
# Layers of stage 2 contraction
tct2_conv1 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Ct2_0')(tct1_dp)
tct2_bn1 = BatchNormalization()(tct2_conv1)
tct2_conv2 = Conv1D(64,
5,
activation='relu',
strides=1,
padding='same',
name='Convolution_Ct2_1')(tct2_bn1)
tct2_bn2 = BatchNormalization()(tct2_conv2)
tct2_conv3 = Conv1D(64,
5,
activation='relu',
strides=2,
padding='same',
name='Convolution_Ct2_2')(tct2_bn2)
tct2_bn3 = BatchNormalization()(tct2_conv3)
#tct2_max = MaxPool1D(pool_size=2, strides=2)(tct2_bn2)
tct2_dp = Dropout(0.2)(tct2_bn3)
# Layers of stage 3 contraction
tct3_conv1 = Conv1D(128,
5,
activation='relu',
padding='same',
name='Convolution_Ce3_0')(tct2_dp)
tct3_bn1 = BatchNormalization()(tct3_conv1)
tct3_conv2 = Conv1D(128,
5,
activation='relu',
padding='same',
name='Convolution_Ce3_1')(tct3_bn1)
tct3_bn2 = BatchNormalization()(tct3_conv2)
tct3_dp = Dropout(0.2)(tct3_bn2)
# Layers of stage 1 expansion
tet1_Tconv = Conv1DTranspose(64,
5,
strides=2,
activation='relu',
padding='same',
name='TransConv_Et1')(tct3_dp)
tet1_Concat = Concatenate(axis=2)([tet1_Tconv, tct2_conv1])
tet1_bn1 = BatchNormalization()(tet1_Concat)
tet1_conv1 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Et1_0')(tet1_bn1)
tet1_bn2 = BatchNormalization()(tet1_conv1)
tet1_conv2 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Et1_1')(tet1_bn2)
tet1_bn3 = BatchNormalization()(tet1_conv2)
tet1_dp = Dropout(0.2)(tet1_bn3)
#Layers of stage 2 expansion
tet2_Tconv = Conv1DTranspose(32,
5,
strides=2,
activation='relu',
padding='same',
name='TransConv_Et2')(tet1_dp)
tet2_Concat = Concatenate(axis=2)([tet2_Tconv, tct1_conv1])
tet2_bn1 = BatchNormalization()(tet2_Concat)
tet2_conv1 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Et2_0')(tet2_bn1)
tet2_bn2 = BatchNormalization()(tet2_conv1)
tet2_conv2 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Et2_1')(tet2_bn2)
tet2_bn3 = BatchNormalization()(tet2_conv2)
tet2_dp = Dropout(0.2)(tet2_bn3)
#Layers of stage 3 expansion
tet3_Tconv = Conv1DTranspose(32,
5,
strides=2,
activation='relu',
padding='same',
name='TransConv_Et3')(tet2_dp)
tet3_Concat = Concatenate(axis=2)([tet3_Tconv, tct0_conv1])
tet3_bn1 = BatchNormalization()(tet3_Concat)
tet3_conv1 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Et3_1')(tet3_bn1)
tet3_bn2 = BatchNormalization()(tet3_conv1)
tet3_conv2 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Et3_2')(tet3_bn2)
tet3_bn3 = BatchNormalization()(tet3_conv2)
tet3_dp = Dropout(0.1)(tet3_bn3)
tet3_conv3 = Conv1D(3,
5,
activation='softmax',
padding='same',
name='Convolution_Et3_3')(tet3_dp)
################################################ Stage 2 ######################################################
tet3_Concat2 = Concatenate(axis=2)([tct0_in, tet3_conv3])
tct0_bn12 = BatchNormalization()(tet3_Concat2)
tct0_conv12 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Ct0_02')(tct0_bn12)
tct0_bn22 = BatchNormalization()(tct0_conv12)
tct0_conv22 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Ct0_12')(tct0_bn22)
tct0_bn32 = BatchNormalization()(tct0_conv22)
tct0_conv32 = Conv1D(32,
5,
activation='relu',
strides=2,
padding='same',
name='Convolution_Ct0_22')(tct0_bn32)
tct0_bn42 = BatchNormalization()(tct0_conv32)
#tct0_max = MaxPool1D(pool_size=2, strides=2)(tct0_bn2)
tct0_dp2 = Dropout(0.2)(tct0_bn42)
# Layers of stage 1 contraction
tct1_conv12 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Ct1_02')(tct0_dp2)
tct1_bn12 = BatchNormalization()(tct1_conv12)
tct1_conv22 = Conv1D(32,
5,
activation='relu',
strides=1,
padding='same',
name='Convolution_Ct1_12')(tct1_bn12)
tct1_bn22 = BatchNormalization()(tct1_conv22)
tct1_conv32 = Conv1D(32,
5,
activation='relu',
strides=2,
padding='same',
name='Convolution_Ct1_22')(tct1_bn22)
tct1_bn32 = BatchNormalization()(tct1_conv32)
#tct1_max = MaxPool1D(pool_size=2, strides=2)(tct1_bn2)
tct1_dp2 = Dropout(0.2)(tct1_bn32)
# Layers of stage 2 contraction
tct2_conv12 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Ct2_02')(tct1_dp2)
tct2_bn12 = BatchNormalization()(tct2_conv12)
tct2_conv22 = Conv1D(64,
5,
activation='relu',
strides=1,
padding='same',
name='Convolution_Ct2_12')(tct2_bn12)
tct2_bn22 = BatchNormalization()(tct2_conv22)
tct2_conv32 = Conv1D(64,
5,
activation='relu',
strides=2,
padding='same',
name='Convolution_Ct2_22')(tct2_bn22)
tct2_bn32 = BatchNormalization()(tct2_conv32)
#tct2_max = MaxPool1D(pool_size=2, strides=2)(tct2_bn2)
tct2_dp2 = Dropout(0.2)(tct2_bn32)
# Layers of stage 3 contraction
tct3_conv12 = Conv1D(128,
5,
activation='relu',
padding='same',
name='Convolution_Ce3_02')(tct2_dp2)
tct3_bn12 = BatchNormalization()(tct3_conv12)
tct3_conv22 = Conv1D(128,
5,
activation='relu',
padding='same',
name='Convolution_Ce3_12')(tct3_bn12)
tct3_bn22 = BatchNormalization()(tct3_conv22)
tct3_dp2 = Dropout(0.2)(tct3_bn22)
# Layers of stage 1 expansion
tet1_Tconv2 = Conv1DTranspose(64,
5,
strides=2,
activation='relu',
padding='same',
name='TransConv_Et12')(tct3_dp2)
tet1_Concat2 = Concatenate(axis=2)([tet1_Tconv2, tct2_conv12])
tet1_bn12 = BatchNormalization()(tet1_Concat2)
tet1_conv12 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Et1_02')(tet1_bn12)
tet1_bn22 = BatchNormalization()(tet1_conv12)
tet1_conv22 = Conv1D(64,
5,
activation='relu',
padding='same',
name='Convolution_Et1_12')(tet1_bn22)
tet1_bn32 = BatchNormalization()(tet1_conv22)
tet1_dp2 = Dropout(0.2)(tet1_bn32)
#Layers of stage 2 expansion
tet2_Tconv2 = Conv1DTranspose(32,
5,
strides=2,
activation='relu',
padding='same',
name='TransConv_Et22')(tet1_dp2)
tet2_Concat2 = Concatenate(axis=2)([tet2_Tconv2, tct1_conv12])
tet2_bn12 = BatchNormalization()(tet2_Concat2)
tet2_conv12 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Et2_02')(tet2_bn12)
tet2_bn22 = BatchNormalization()(tet2_conv12)
tet2_conv22 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Et2_12')(tet2_bn22)
tet2_bn32 = BatchNormalization()(tet2_conv22)
tet2_dp2 = Dropout(0.2)(tet2_bn32)
#Layers of stage 3 expansion
tet3_Tconv2 = Conv1DTranspose(32,
5,
strides=2,
activation='relu',
padding='same',
name='TransConv_Et32')(tet2_dp2)
tet3_Concat2 = Concatenate(axis=2)([tet3_Tconv2, tct0_conv12])
tet3_bn12 = BatchNormalization()(tet3_Concat2)
tet3_conv12 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Et3_12')(tet3_bn12)
tet3_bn22 = BatchNormalization()(tet3_conv12)
tet3_conv22 = Conv1D(32,
5,
activation='relu',
padding='same',
name='Convolution_Et3_22')(tet3_bn22)
tet3_bn32 = BatchNormalization()(tet3_conv22)
tet3_dp2 = Dropout(0.1)(tet3_bn32)
tet3_conv32 = Conv1D(3,
5,
activation='softmax',
padding='same',
name='Convolution_Et3_32')(tet3_dp2)
model = Model(inputs=inp, outputs=[tet3_conv3, tet3_conv32])
return model
def inception_resnet_block(x, scale, block_type, block_idx, activation='relu'):
"""Adds a Inception-ResNet block.
This function builds 3 types of Inception-ResNet blocks mentioned
in the paper, controlled by the `block_type` argument (which is the
block name used in the official TF-slim implementation):
- Inception-ResNet-A: `block_type='block35'`
- Inception-ResNet-B: `block_type='block17'`
- Inception-ResNet-C: `block_type='block8'`
# Arguments
x: input tensor.
scale: scaling factor to scale the residuals (i.e., the output of
passing `x` through an inception module) before adding them
to the shortcut branch. Let `r` be the output from the residual branch,
the output of this block will be `x + scale * r`.
block_type: `'block35'`, `'block17'` or `'block8'`, determines
the network structure in the residual branch.
block_idx: an `int` used for generating layer names. The Inception-ResNet blocks
are repeated many times in this network. We use `block_idx` to identify
each of the repetitions. For example, the first Inception-ResNet-A block
will have `block_type='block35', block_idx=0`, ane the layer names will have
a common prefix `'block35_0'`.
activation: activation function to use at the end of the block
(see [activations](../activations.md)).
When `activation=None`, no activation is applied
(i.e., "linear" activation: `a(x) = x`).
# Returns
Output tensor for the block.
# Raises
ValueError: if `block_type` is not one of `'block35'`,
`'block17'` or `'block8'`.
"""
if block_type == 'block35':
branch_0 = conv2d_bn(x, 32, 1)
branch_1 = conv2d_bn(x, 32, 1)
branch_1 = conv2d_bn(branch_1, 32, 3)
branch_2 = conv2d_bn(x, 32, 1)
branch_2 = conv2d_bn(branch_2, 48, 3)
branch_2 = conv2d_bn(branch_2, 64, 3)
branches = [branch_0, branch_1, branch_2]
elif block_type == 'block17':
branch_0 = conv2d_bn(x, 192, 1)
branch_1 = conv2d_bn(x, 128, 1)
branch_1 = conv2d_bn(branch_1, 160, [1, 7])
branch_1 = conv2d_bn(branch_1, 192, [7, 1])
branches = [branch_0, branch_1]
elif block_type == 'block8':
branch_0 = conv2d_bn(x, 192, 1)
branch_1 = conv2d_bn(x, 192, 1)
branch_1 = conv2d_bn(branch_1, 224, [1, 3])
branch_1 = conv2d_bn(branch_1, 256, [3, 1])
branches = [branch_0, branch_1]
else:
raise ValueError('Unknown Inception-ResNet block type. '
'Expects "block35", "block17" or "block8", '
'but got: ' + str(block_type))
block_name = block_type + '_' + str(block_idx)
channel_axis = 1 if K.image_data_format() == 'channels_first' else 3
mixed = Concatenate(axis=channel_axis,
name=block_name + '_mixed')(branches)
up = conv2d_bn(mixed,
K.int_shape(x)[channel_axis],
1,
activation=None,
use_bias=True,
name=block_name + '_conv')
x = Lambda(lambda inputs, scale: inputs[0] + inputs[1] * scale,
output_shape=K.int_shape(x)[1:],
arguments={'scale': scale},
name=block_name)([x, up])
if activation is not None:
x = Activation(activation, name=block_name + '_ac')(x)
return x
def __init__(self):
super(Concatenate_, self).__init__()
self.concat = Concatenate(axis=-1)
def get_head_concat_model(DATA):
# shape：(sequence长度, )
# first input
input_creative_id = Input(shape=(None,), name='creative_id')
x1 = Embedding(input_dim=NUM_creative_id+1,
output_dim=128,
weights=[DATA['creative_id_emb']],
trainable=args.not_train_embedding,
input_length=LEN_creative_id,
mask_zero=True)(input_creative_id)
input_ad_id = Input(shape=(None,), name='ad_id')
x2 = Embedding(input_dim=NUM_ad_id+1,
output_dim=128,
weights=[DATA['ad_id_emb']],
trainable=args.not_train_embedding,
input_length=LEN_ad_id,
mask_zero=True)(input_ad_id)
input_product_id = Input(shape=(None,), name='product_id')
x3 = Embedding(input_dim=NUM_product_id+1,
output_dim=128,
weights=[DATA['product_id_emb']],
trainable=args.not_train_embedding,
input_length=LEN_product_id,
mask_zero=True)(input_product_id)
input_advertiser_id = Input(shape=(None,), name='advertiser_id')
x4 = Embedding(input_dim=NUM_advertiser_id+1,
output_dim=128,
weights=[DATA['advertiser_id_emb']],
trainable=args.not_train_embedding,
input_length=LEN_advertiser_id,
mask_zero=True)(input_advertiser_id)
input_industry = Input(shape=(None,), name='industry')
x5 = Embedding(input_dim=NUM_industry+1,
output_dim=128,
weights=[DATA['industry_emb']],
trainable=args.not_train_embedding,
input_length=LEN_industry,
mask_zero=True)(input_industry)
input_product_category = Input(shape=(None,), name='product_category')
x6 = Embedding(input_dim=NUM_product_category+1,
output_dim=128,
weights=[DATA['product_category_emb']],
trainable=args.not_train_embedding,
input_length=LEN_product_category,
mask_zero=True)(input_product_category)
x = Concatenate(axis=2)([x1, x2, x3, x4, x5, x6])
for _ in range(args.num_lstm):
x = Bidirectional(LSTM(128, return_sequences=True))(x)
x = layers.GlobalMaxPooling1D()(x)
# x = layers.GlobalAvaregePooling1D()(x)
output_gender = Dense(2, activation='softmax', name='gender')(x)
output_age = Dense(10, activation='softmax', name='age')(x)
model = Model(
[
input_creative_id,
input_ad_id,
input_product_id,
input_advertiser_id,
input_industry,
input_product_category
],
[
output_gender,
output_age
]
)
model.compile(
optimizer=optimizers.Adam(1e-4),
loss={'gender': losses.CategoricalCrossentropy(from_logits=False),
'age': losses.CategoricalCrossentropy(from_logits=False)},
loss_weights=[0.5, 0.5],
metrics=['accuracy'])
model.summary()
return model
