def __init__(self, latent_dim=49):
config = ConfigProto()
config.gpu_options.allow_growth = True
session = InteractiveSession(config=config)
# ENCODER
inp = Input((896, 896, 1))
e = Conv2D(32, (10, 10), activation='relu')(inp)
e = MaxPooling2D((10, 10))(e)
e = Conv2D(64, (6, 6), activation='relu')(e)
e = MaxPooling2D((10, 10))(e)
e = Conv2D(64, (3, 3), activation='relu')(e)
l = Flatten()(e)
l = Dense(49, activation='softmax')(l)
# DECODER
d = Reshape((7, 7, 1))(l)
d = Conv2DTranspose(64, (3, 3),
strides=8,
activation='relu',
padding='same')(d)
d = BatchNormalization()(d)
d = Conv2DTranspose(64, (3, 3),
strides=8,
activation='relu',
padding='same')(d)
d = BatchNormalization()(d)
d = Conv2DTranspose(64, (3, 3),
strides=2,
activation='relu',
padding='same')(d)
d = BatchNormalization()(d)
d = Conv2DTranspose(32, (3, 3), activation='relu', padding='same')(d)
decoded = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(d)
self.CAD = tf.keras.Model(inp, decoded)
opt = tf.keras.optimizers.RMSprop(lr=0.0001, decay=1e-6)
self.CAD.compile(loss="binary_crossentropy",
optimizer=opt,
metrics=["accuracy"])
self.Flow = tf.keras.Sequential([
tf.keras.layers.LSTM(32, input_shape=(3, 2),
return_sequences=True),
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Bidirectional(
tf.keras.layers.LSTM(32, return_sequences=True)),
tf.keras.layers.Dropout(0.4),
tf.keras.layers.TimeDistributed(
tf.keras.layers.Dense(10, activation='relu')),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(2, activation='relu')
])
opt = tf.keras.optimizers.RMSprop(lr=0.0001, decay=1e-6)
self.Flow.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
print(self.Flow.summary())
print(self.CAD.summary())
def CIFAR_CNY19(classes, input_shape, weights=None):
model = Sequential()
model.add(
Convolution2D(40, (5, 5), strides=(1, 1), input_shape=input_shape))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Dropout(0.25))
model.add(Convolution2D(20, (5, 5), strides=(1, 1)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(240, activation='relu'))
# model.add(Dropout(0.5))
model.add(Dense(84, activation='relu'))
# model.add(Dropout(0.5))
model.add(Dense(classes, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
return model
def _build_model(self, hl1_dims, hl2_dims, hl3_dims, input_layer_size, output_layer_size, optimizer, loss):
model = Sequential()
# The Layer Size is being set by testing multiple hyperparameter and Network sizes and comparing their
# performance
# Input dimension and first Hidden Layer
model.add(Dense(hl1_dims, input_dim=input_layer_size))
model.add(BatchNormalization())
model.add(Activation('relu'))
# Second Hidden Layer
model.add(Dense(hl2_dims))
model.add(BatchNormalization())
model.add(Activation('relu'))
# Third Hidden Layer
model.add(Dense(hl3_dims))
model.add(BatchNormalization())
model.add(Activation('relu'))
# Output Layer
model.add(Dense(output_layer_size))
model.add(Activation('linear'))
model.compile(optimizer=optimizer, loss=loss) # Use Huber Loss Function for DQN based on TensorBoard analysis
return model
def create_model():
units = 512
middle_units = 256
dropout_value = 0.3
activation_function = 'softmax'
loss_function = 'categorical_crossentropy'
optimizer = 'rmsprop'
model = Sequential()
model.add(
LSTM(units,
input_shape=(network_input.shape[1], network_input.shape[2]),
recurrent_dropout=dropout_value,
return_sequences=True))
model.add(
LSTM(
units,
return_sequences=True,
recurrent_dropout=dropout_value,
))
model.add(LSTM(units))
model.add(BatchNormalization())
model.add(Dropout(dropout_value))
model.add(Dense(middle_units))
model.add(Activation('relu'))
model.add(Dropout(dropout_value))
model.add(BatchNormalization())
model.add(Dropout(dropout_value))
model.add(Dense(vocab_size))
model.add(Activation(activation_function))
model.compile(loss=loss_function, optimizer=optimizer)
return model
def modelStandard(input_shape, parameter=None):
# define LSTM
model = Sequential()
model.add(
TimeDistributed(Conv2D(16, (2, 2), activation='relu'),
input_shape=input_shape))
model.add(Dropout(parameter['dropout']))
model.add(BatchNormalization())
model.add(TimeDistributed(MaxPooling2D(pool_size=(2, 2), strides=2)))
model.add(Dropout(parameter['dropout']))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(parameter['cell1']))
# model.add(Dropout(0.25))
model.add(BatchNormalization())
model.add(RepeatVector(8))
model.add(LSTM(parameter['cell2'], return_sequences=True))
# model.add(Dropout(0.25))
model.add(BatchNormalization())
model.add(TimeDistributed(Dense(5, activation='softmax')))
# Replicates `model` on 8 GPUs.
# This assumes that your machine has 8 available GPUs.
#parallel_model = multi_gpu_model(model, gpus=2)
#parallel_model.compile(loss='categorical_crossentropy',
# optimizer='adam', metrics=['accuracy'])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def modelC(row, col):
# define LSTM
model = Sequential()
model.add(
TimeDistributed(Conv2D(16, (2, 2), activation='relu'),
input_shape=(None, row, col, 1)))
model.add(Dropout(0.25))
model.add(BatchNormalization())
model.add(TimeDistributed(MaxPooling2D(pool_size=(2, 2))))
model.add(Dropout(0.25))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(75))
# model.add(Dropout(0.25))
model.add(BatchNormalization())
model.add(RepeatVector(4))
model.add(LSTM(50, return_sequences=True))
# model.add(Dropout(0.25))
model.add(BatchNormalization())
model.add(TimeDistributed(Dense(4, activation='softmax')))
# Replicates `model` on 8 GPUs.
# This assumes that your machine has 8 available GPUs.
# parallel_model = multi_gpu_model(model, gpus=[2])
# parallel_model.compile(loss='categorical_crossentropy',
# optimizer='adam', metrics=['accuracy'])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def conv_block(feat_maps_out, prev):
prev = BatchNormalization(axis=-1)(prev) # Specifying the axis and mode allows for later merging
prev = Activation('relu')(prev)
prev = Conv2D(filters=feat_maps_out, kernel_size=3, padding='same')(prev)
prev = BatchNormalization(axis=-1)(prev) # Specifying the axis and mode allows for later merging
prev = Activation('relu')(prev)
prev = Conv2D(filters=feat_maps_out, kernel_size=3, padding='same')(prev)
return prev
def modelB(row, col, parameter=None):
# define LSTM
input = Input(shape=(None, row, col, 1), name='main_input')
''' x = TimeDistributed(Conv2D(16, (2, 2)))(input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(0.25)(x)
'''
# tower_1 = TimeDistributed(Conv2D(16, (1, 1), padding='same', activation='relu'))(input)
# tower_1 = TimeDistributed(Conv2D(16, (3, 3), padding='same', activation='relu'))(tower_1)
tower_2 = TimeDistributed(Conv2D(16, (1, 1), padding='same'))(input)
x = BatchNormalization()(tower_2)
x = Activation('relu')(x)
x = Dropout(0.25)(x)
tower_2 = TimeDistributed(Conv2D(16, (5, 5), padding='same'))(x)
x = BatchNormalization()(tower_2)
x = Activation('relu')(x)
tower_2 = Dropout(0.25)(x)
tower_3 = TimeDistributed(
MaxPooling2D((3, 3), strides=(1, 1), padding='same'))(input)
tower_3 = TimeDistributed(Conv2D(16, (1, 1), padding='same'))(tower_3)
x = BatchNormalization()(tower_3)
x = Activation('relu')(x)
tower_3 = Dropout(0.25)(x)
concatenate_output = concatenate([tower_2, tower_3], axis=-1)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2),
strides=2))(concatenate_output)
x = Dropout(0.25)(x)
x = TimeDistributed(Flatten())(x)
# convLstm = ConvLSTM2D(filters=40, kernel_size=(3, 3),padding='same', return_sequences=False)(x)
lstm_output = LSTM(75)(x)
lstm_output = BatchNormalization()(lstm_output)
# lstm_output = BatchNormalization()(convLstm)
# auxiliary_output = Dense(1, activation='sigmoid', name='aux_output')(lstm_output)
# auxiliary_input = Input(shape=(4,), name='aux_input')
# x = concatenate([lstm_output, auxiliary_input])
x = RepeatVector(4)(lstm_output)
x = LSTM(50, return_sequences=True)(x)
# model.add(Dropout(0.25))
x = BatchNormalization()(x)
output = TimeDistributed(Dense(4, activation='softmax'),
name='main_output')(x)
model = Model(inputs=[input], outputs=[output])
model.compile(loss={'main_output': 'categorical_crossentropy'},
loss_weights={'main_output': 1.},
optimizer='adam',
metrics=['accuracy'])
return model
def SingleOutputCNN(
input_shape,
output_shape,
cnns_per_maxpool=1,
maxpool_layers=1,
dense_layers=1,
dense_units=64,
dropout=0.25,
regularization=False,
global_maxpool=False,
name='',
) -> Model:
function_name = cast(types.FrameType,
inspect.currentframe()).f_code.co_name
model_name = f"{function_name}-{name}" if name else function_name
# model_name = seq([ function_name, name ]).filter(lambda x: x).make_string("-") # remove dependency on pyfunctional - not in Kaggle repo without internet
inputs = Input(shape=input_shape)
x = inputs
for cnn1 in range(0, maxpool_layers):
for cnn2 in range(1, cnns_per_maxpool + 1):
x = Conv2D(32 * cnn2,
kernel_size=(3, 3),
padding='same',
activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = BatchNormalization()(x)
x = Dropout(dropout)(x)
if global_maxpool:
x = GlobalMaxPooling2D()(x)
x = Flatten()(x)
for nn1 in range(0, dense_layers):
if regularization:
x = Dense(dense_units,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(x)
else:
x = Dense(dense_units, activation='relu')(x)
x = BatchNormalization()(x)
x = Dropout(dropout)(x)
x = Dense(output_shape, activation='softmax')(x)
model = Model(inputs, x, name=model_name)
# plot_model(model, to_file=os.path.join(os.path.dirname(__file__), f"{name}.png"))
return model
def build_policy_network(shape, action_size, regularizer):
policy_input = Input(shape)
c1 = Conv2D(filters=1, kernel_size=1, padding='same', activation='linear',
kernel_regularizer=regularizer)(policy_input)
b1 = BatchNormalization(axis=-1)(c1)
l1 = LeakyReLU()(b1)
f1 = Flatten()(l1)
d1 = Dense(action_size, use_bias=False, activation='sigmoid', kernel_regularizer=regularizer)(f1)
policy_model = Model(inputs=policy_input, outputs=d1)
return policy_model
def modify_model(model: Model, class_index: int,
importance_type: ImportanceType) -> Model:
gamma_initializer: str = "zeros"
if importance_type & ImportanceType.GAMMA:
gamma_initializer = "ones"
gamma_regularizer = None
if importance_type & ImportanceType.L1 and not importance_type & ImportanceType.L2:
gamma_regularizer = l1()
if not importance_type & ImportanceType.L1 and importance_type & ImportanceType.L2:
gamma_regularizer = l2()
if importance_type & ImportanceType.L1 and importance_type & ImportanceType.L2:
gamma_regularizer = l1_l2()
max_layer: int = len(model.layers)
last_output: Input = None
network_input: Input = None
for i, layer in enumerate(model.layers):
if i == 0:
last_output = layer.output
network_input = layer.input
if 0 < i < max_layer:
new_layer: Union[BatchNormalization,
BatchNormalization] = BatchNormalization(
center=(importance_type
& ImportanceType.CENTERING),
gamma_initializer=gamma_initializer,
gamma_regularizer=gamma_regularizer)
last_output = new_layer(last_output)
if i == max_layer - 1:
new_end_layer: Dense = Dense(2,
activation="softmax",
name="binary_output_layer")
last_output = new_end_layer(last_output)
old_weights = layer.get_weights()
old_weights[0] = np.transpose(old_weights[0], (1, 0))
new_weights: List[np.array] = [
np.append(old_weights[0][class_index:class_index + 1],
np.subtract(
np.sum(old_weights[0], axis=0, keepdims=True),
old_weights[0][class_index:class_index + 1]),
axis=0),
np.append(old_weights[1][class_index:class_index + 1],
np.subtract(
np.sum(old_weights[1], axis=0, keepdims=True),
old_weights[1][class_index:class_index + 1]),
axis=0)
]
new_weights[0] = np.transpose(new_weights[0], (1, 0))
new_end_layer.set_weights(new_weights)
elif i > 0:
last_output = layer(last_output)
return Model(inputs=network_input, outputs=last_output)
def build_value_network(shape, value_support_size):
value_input = Input(shape)
c1 = Conv2D(filters=1, kernel_size=1, padding='same', activation='linear')(value_input)
b1 = BatchNormalization(axis=-1)(c1)
l1 = LeakyReLU()(b1)
f1 = Flatten()(l1)
d2 = Dense(20, use_bias=False, activation='linear')(f1)
l2 = LeakyReLU()(d2)
d2 = Dense(value_support_size, use_bias=False, activation='tanh')(l2)
value_model = Model(inputs=value_input, outputs=d2)
return value_model
def MultiOutputApplication(
input_shape,
output_shape: Union[List, Dict],
application='NASNetMobile',
weights=None, # None or 'imagenet'
pooling='avg', # None, 'avg', 'max',
dense_units=512, # != (1295+168+11+7),
dense_layers=2,
dropout=0.25) -> Model:
function_name = cast(types.FrameType,
inspect.currentframe()).f_code.co_name
model_name = f"{function_name}-{application}" if application else function_name
inputs = Input(shape=input_shape)
x = inputs
if application == 'NASNetMobile':
application_model = tf.keras.applications.nasnet.NASNetMobile(
input_shape=input_shape,
input_tensor=inputs,
include_top=False,
weights=weights,
pooling=pooling,
classes=1000,
)
else:
raise Exception(
f"MultiOutputApplication() - unknown application: {application}")
x = application_model(x)
x = Flatten(name='output')(x)
for nn1 in range(0, dense_layers):
x = Dense(dense_units, activation='relu')(x)
x = BatchNormalization()(x)
x = Dropout(dropout)(x)
if isinstance(output_shape, dict):
outputs = [
Dense(output_shape, activation='softmax', name=key)(x)
for key, output_shape in output_shape.items()
]
else:
outputs = [
Dense(output_shape, activation='softmax',
name=f'output_{index}')(x)
for index, output_shape in enumerate(output_shape)
]
model = Model(inputs, outputs, name=model_name)
# plot_model(model, to_file=os.path.join(os.path.dirname(__file__), f"{name}.png"))
return model
def _build_model(self, hl1_dims, hl2_dims, hl3_dims, input_layer_size, output_layer_size, optimizer, loss):
"""
:param hl1_dims: dimensions for first hidden layer
:param hl2_dims: dimensions for second hidden layer
:param hl3_dims: dimensions for third hidden layer
:param input_layer_size: dimensions for input layer
:param output_layer_size: dimensions for output layer
:param optimizer: optimizer that should be used
:param loss: loss function that should be used
:return: keras sequential with batch norm model
"""
model = Sequential()
# The Layer Size is being set by testing multiple hyperparameter and Network sizes and comparing their
# performance based on RE5 Model Settings
# Input dimension and first Hidden Layer
model.add(Dense(hl1_dims, input_dim=input_layer_size))
model.add(BatchNormalization())
model.add(Activation('relu'))
# Second Hidden Layer
model.add(Dense(hl2_dims))
model.add(BatchNormalization())
model.add(Activation('relu'))
# Third Hidden Layer
model.add(Dense(hl3_dims))
model.add(BatchNormalization())
model.add(Activation('relu'))
# Output Layer
model.add(Dense(output_layer_size))
model.add(Activation('linear'))
model.compile(optimizer=optimizer, loss=loss) # Use Huber Loss Function for DQN based on TensorBoard analysis
return model
def _build_model(self, hl1_dims, hl2_dims, hl3_dims, input_layer_size,
output_layer_size, optimizer, loss):
input_v = Input(shape=(1, input_layer_size))
branch_v = Flatten()(input_v)
branch_v = Dense(hl1_dims, activation='relu')(branch_v)
branch_v = BatchNormalization()(branch_v)
branch_v = Dense(hl2_dims, activation='relu')(branch_v)
branch_v = BatchNormalization()(branch_v)
out_v = Dense(hl3_dims, activation='relu')(branch_v)
out_v = BatchNormalization()(out_v)
out_v = Dense(output_layer_size, activation='linear')(out_v)
model = Model(inputs=input_v, outputs=out_v)
#model = Model(inputs=m_v.inputs, outputs=out_v)
# print(model.summary())
model.compile(
optimizer=optimizer, loss=loss
) # Use Huber Loss Function for DQN based on TensorBoard analysis
return model
def modelA(row, col):
# define LSTM
input = Input(shape=(None, row, col, 1), name='main_input')
x = TimeDistributed(Conv2D(16, (2, 2), activation='relu'))(input)
x = Dropout(0.25)(x)
x = BatchNormalization()(x)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2), strides=2))(x)
x = Dropout(0.25)(x)
x = TimeDistributed(Flatten())(x)
lstm_output = LSTM(75)(x)
lstm_output = BatchNormalization()(lstm_output)
auxiliary_output = Dense(1, activation='sigmoid',
name='aux_output')(lstm_output)
auxiliary_input = Input(shape=(4, ), name='aux_input')
x = concatenate([lstm_output, auxiliary_input])
x = RepeatVector(8)(x)
x = LSTM(50, return_sequences=True)(x)
# model.add(Dropout(0.25))
x = BatchNormalization()(x)
output = TimeDistributed(Dense(5, activation='softmax'),
name='main_output')(x)
model = Model(inputs=[input, auxiliary_input],
outputs=[output, auxiliary_output])
model.compile(loss={
'main_output': 'categorical_crossentropy',
'aux_output': 'binary_crossentropy'
},
loss_weights={
'main_output': 1.,
'aux_output': 0.2
},
optimizer='adam',
metrics=['accuracy'])
return model
def modelStandardB(row, col):
# define LSTM
input_img = Input(shape=(None, row, col, 1), name='input')
x = TimeDistributed(Conv2D(16, (2, 2), activation='relu'))(input_img)
x = Dropout(0.25)(x)
x = BatchNormalization()(x)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2), strides=2))(x)
x = Dropout(0.25)(x)
x = TimeDistributed(Flatten())(x)
x = LSTM(75)(x)
# model.add(Dropout(0.25))
x = BatchNormalization()(x)
x = RepeatVector(4)(x)
x = LSTM(50, return_sequences=True)(x)
# model.add(Dropout(0.25))
x = BatchNormalization()(x)
output = TimeDistributed(Dense(4, activation='softmax'))(x)
model = Model(input_img, output)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def _build_model(self, hl1_dims, hl2_dims, hl3_dims, input_layer_size,
output_layer_size, optimizer, loss):
"""
:param hl1_dims: dimensions for first hidden layer
:param hl2_dims: dimensions for second hidden layer
:param hl3_dims: dimensions for third hidden layer
:param input_layer_size: dimensions for input layer
:param output_layer_size: dimensions for output layer
:param optimizer: optimizer that should be used
:param loss: loss function that should be used
:return: keras sequential with batch norm model
"""
model = Sequential()
# Input dimension and first Hidden Layer
model.add(Dense(hl1_dims, input_dim=input_layer_size))
model.add(BatchNormalization())
model.add(Activation('relu'))
# Second Hidden Layer
model.add(Dense(hl2_dims))
model.add(BatchNormalization())
model.add(Activation('relu'))
# Third Hidden Layer
model.add(Dense(hl3_dims))
model.add(BatchNormalization())
model.add(Activation('relu'))
# Output Layer
model.add(Dense(output_layer_size))
model.add(Activation('linear'))
model.compile(optimizer=optimizer, loss=loss)
return model
def Train(self, input, target):
X_train, X_test, Y_train, Y_test = train_test_split(input, target, train_size=0.75)
Y_train = np.asarray(Y_train)
Y_test = np.array(Y_test)
X_train = np.reshape(X_train, [-1, X_train[0].shape[0], X_train[0].shape[1]])
X_test = np.reshape(X_test, [-1, X_train[0].shape[0], X_train[0].shape[1]])
model = Sequential()
model.add(Conv1D(16, 3, padding='same', input_shape=input[0].shape))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization())
model.add(GRU(16, return_sequences=True))
# model.add(Activation("sigmoid"))
# model.add(LSTM(lstm_out))
model.add(Flatten())
model.add(Dense(8, activity_regularizer=l2(0.001)))
# model.add(GRU(lstm_out, return_sequences=True))
# model.add(LSTM(lstm_out))
# model.add(Dense(20, activity_regularizer=l2(0.001)))
model.add(Activation("relu"))
model.add(Dense(2))
model.compile(loss=mean_absolute_error, optimizer='nadam',
metrics=[RootMeanSquaredError(), MAE])
print(model.summary())
batch_size = 12
epochs = 100
reduce_lr_acc = ReduceLROnPlateau(monitor='val_loss', factor=0.9, patience=epochs / 10, verbose=1, min_delta=1e-4, mode='max')
model.fit(X_train, Y_train,
epochs=epochs,
batch_size=batch_size, validation_data=(X_test, Y_test), callbacks=[reduce_lr_acc])
model.save("PositionEstimation.h5", overwrite=True)
# acc = model.evaluate(X_test,
# Y_test,
# batch_size=batch_size,
# verbose=0)
predicted = model.predict(X_test, batch_size=batch_size)
# predicted = out.ravel()
res = pd.DataFrame({"predicted_x": predicted[:, 0],
"predicted_y": predicted[:, 1],
"original_x": Y_test[:, 0],
"original_y": Y_test[:, 1]})
res.to_excel("res.xlsx")
def get_model(X, N_class, total_words=86627, EMBEDDING_DIM=100, maxlen=53):
embeddings_index = {}
f = open('glove.6B/glove.6B.100d.txt')
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
f.close()
embedding_matrix = np.zeros((total_words, EMBEDDING_DIM))
for word, i in X.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen, ), dtype='int32')
embedding = Embedding(total_words,
EMBEDDING_DIM,
embeddings_initializer=Constant(embedding_matrix),
input_length=maxlen,
trainable=False)(inp)
x = LSTM(300, dropout=0.25, recurrent_dropout=0.25,
return_sequences=True)(embedding)
x = Dropout(0.25)(x)
merged = Attention_COSTUM(maxlen)(x)
merged = Dense(256, activation='relu')(merged)
merged = Dropout(0.25)(merged)
merged = BatchNormalization()(merged)
outp = Dense(N_class, activation='softmax')(merged)
AttentionLSTM = Model(inputs=inp, outputs=outp)
AttentionLSTM.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['acc'])
return AttentionLSTM
def construct_keras_api_model(embedding_weights):
# input_no_time_no_repeat = Input(shape=max_len, dtype='int32')
# embedded_no_time_no_repeat = Embedding(
# creative_id_window,embedding_size,weights=[embedding_weights],trainable=False
# )(input_no_time_no_repeat)
# ==================================================================================
Input_fix_creative_id = Input(shape=(math.ceil(time_id_max / period_days) *
period_length),
dtype='int32',
name='input_fix_creative_id')
Embedded_fix_creative_id = Embedding(
creative_id_window,
embedding_size,
weights=[embedding_weights],
trainable=False)(Input_fix_creative_id)
# ==================================================================================
# input_no_time_with_repeat = Input(shape=max_len, dtype='int32')
# embedded_no_time_with_repeat = Embedding(creative_id_window,embedding_size,weights=[embedding_weights],trainable=False)(input_no_time_with_repeat)
# ----------------------------------------------------------------------
GM_x = keras.layers.GlobalMaxPooling1D()(Embedded_fix_creative_id)
GM_x = Dropout(0.5)(GM_x)
GM_x = Dense(embedding_size // 2, kernel_regularizer=l2(0.001))(GM_x)
GM_x = BatchNormalization()(GM_x)
GM_x = Activation('relu')(GM_x)
GM_x = Dropout(0.5)(GM_x)
GM_x = Dense(embedding_size // 4, kernel_regularizer=l2(0.001))(GM_x)
GM_x = BatchNormalization()(GM_x)
GM_x = Activation('relu')(GM_x)
GM_x = Dense(1, 'sigmoid')(GM_x)
# ----------------------------------------------------------------------
GA_x = GlobalAveragePooling1D()(Embedded_fix_creative_id)
GA_x = Dropout(0.5)(GA_x)
GA_x = Dense(embedding_size // 2, kernel_regularizer=l2(0.001))(GA_x)
GA_x = BatchNormalization()(GA_x)
GA_x = Activation('relu')(GA_x)
GA_x = Dropout(0.5)(GA_x)
GA_x = Dense(embedding_size // 4, kernel_regularizer=l2(0.001))(GA_x)
GA_x = BatchNormalization()(GA_x)
GA_x = Activation('relu')(GA_x)
GA_x = Dense(1, 'sigmoid')(GA_x)
# ==================================================================================
Conv_creative_id = Conv1D(embedding_size, 15, 5,
activation='relu')(Embedded_fix_creative_id)
# ----------------------------------------------------------------------
Conv_GM_x = MaxPooling1D(7)(Conv_creative_id)
Conv_GM_x = Conv1D(embedding_size, 2, 1, activation='relu')(Conv_GM_x)
Conv_GM_x = GlobalMaxPooling1D()(Conv_GM_x)
Conv_GM_x = Dropout(0.5)(Conv_GM_x)
Conv_GM_x = Dense(embedding_size // 2,
kernel_regularizer=l2(0.001))(Conv_GM_x)
Conv_GM_x = BatchNormalization()(Conv_GM_x)
Conv_GM_x = Activation('relu')(Conv_GM_x)
Conv_GM_x = Dropout(0.5)(Conv_GM_x)
Conv_GM_x = Dense(embedding_size // 4,
kernel_regularizer=l2(0.001))(Conv_GM_x)
Conv_GM_x = BatchNormalization()(Conv_GM_x)
Conv_GM_x = Activation('relu')(Conv_GM_x)
Conv_GM_x = Dense(1, 'sigmoid')(Conv_GM_x)
# ----------------------------------------------------------------------
Conv_GA_x = AveragePooling1D(7)(Conv_creative_id)
Conv_GA_x = Conv1D(embedding_size, 2, 1, activation='relu')(Conv_GA_x)
Conv_GA_x = GlobalAveragePooling1D()(Conv_GA_x)
Conv_GA_x = Dropout(0.5)(Conv_GA_x)
Conv_GA_x = Dense(embedding_size // 2,
kernel_regularizer=l2(0.001))(Conv_GA_x)
Conv_GA_x = BatchNormalization()(Conv_GA_x)
Conv_GA_x = Activation('relu')(Conv_GA_x)
Conv_GA_x = Dropout(0.5)(Conv_GA_x)
Conv_GA_x = Dense(embedding_size // 4,
kernel_regularizer=l2(0.001))(Conv_GA_x)
Conv_GA_x = BatchNormalization()(Conv_GA_x)
Conv_GA_x = Activation('relu')(Conv_GA_x)
Conv_GA_x = Dense(1, 'sigmoid')(Conv_GA_x)
# ----------------------------------------------------------------------
LSTM_x = Conv1D(embedding_size, 14, 7, activation='relu')(Conv_creative_id)
LSTM_x = LSTM(embedding_size, return_sequences=True)(LSTM_x)
LSTM_x = LSTM(embedding_size, return_sequences=True)(LSTM_x)
LSTM_x = LSTM(embedding_size)(LSTM_x)
LSTM_x = Dropout(0.5)(LSTM_x)
LSTM_x = Dense(embedding_size // 2, kernel_regularizer=l2(0.001))(LSTM_x)
LSTM_x = BatchNormalization()(LSTM_x)
LSTM_x = Activation('relu')(LSTM_x)
LSTM_x = Dropout(0.5)(LSTM_x)
LSTM_x = Dense(embedding_size // 4, kernel_regularizer=l2(0.001))(LSTM_x)
LSTM_x = BatchNormalization()(LSTM_x)
LSTM_x = Activation('relu')(LSTM_x)
LSTM_x = Dense(1, 'sigmoid')(LSTM_x)
# ----------------------------------------------------------------------
concatenated = concatenate([
GM_x,
GA_x,
Conv_GM_x,
Conv_GA_x,
LSTM_x,
],
axis=-1)
output_tensor = Dense(1, 'sigmoid')(concatenated)
keras_api_model = Model(
[
# input_no_time_no_repeat,
Input_fix_creative_id,
# input_no_time_with_repeat,
],
output_tensor)
keras_api_model.summary()
plot_model(keras_api_model, to_file='model/keras_api_word2vec_model.png')
print('-' * 5 + ' ' * 3 + "编译模型" + ' ' * 3 + '-' * 5)
keras_api_model.compile(optimizer=optimizers.RMSprop(lr=RMSProp_lr),
loss=losses.binary_crossentropy,
metrics=[metrics.binary_accuracy])
return keras_api_model
def build_model(self, model_name, query_dim, terms_dim, output_dim,
word_embedding):
self.model_name = model_name
query_input = Input(shape=(query_dim, ), name='query_input')
terms_input = Input(shape=(terms_dim, ), name='terms_input')
if model_name == 'lstm':
embedding_feature_block = Sequential(layers=[
Embedding(word_embedding.vocabulary_size,
word_embedding.dimensions,
weights=[word_embedding.embedding_matrix],
trainable=True,
mask_zero=False),
BatchNormalization(),
LSTM(64, return_sequences=True)
])
elif model_name == 'bilstm':
embedding_feature_block = Sequential(layers=[
Embedding(word_embedding.vocabulary_size,
word_embedding.dimensions,
weights=[word_embedding.embedding_matrix],
trainable=True,
mask_zero=False),
BatchNormalization(),
Bidirectional(LSTM(64, return_sequences=True))
])
else: # default cnn
embedding_feature_block = Sequential(layers=[
Embedding(word_embedding.vocabulary_size,
word_embedding.dimensions,
weights=[word_embedding.embedding_matrix],
trainable=True,
mask_zero=False),
BatchNormalization(),
Conv1D(filters=64, kernel_size=3, strides=1),
MaxPooling1D(pool_size=3)
])
# Features
query_feature = embedding_feature_block(query_input)
terms_feature = embedding_feature_block(terms_input)
# Query-Terms alignment
attention = Dot(axes=-1)([query_feature, terms_feature])
softmax_attention = Lambda(lambda x: softmax(x, axis=1),
output_shape=unchanged_shape)(attention)
terms_aligned = Dot(axes=1)([softmax_attention, terms_feature])
# Aligned features
if model_name == 'lstm':
flatten_layer = LSTM(128, return_sequences=False)(terms_aligned)
elif model_name == 'bilstm':
flatten_layer = Bidirectional(LSTM(
128, return_sequences=False))(terms_aligned)
else: # default cnn
merged_cnn = Conv1D(filters=128, kernel_size=3,
strides=1)(terms_aligned)
merged_cnn = MaxPooling1D(pool_size=3)(merged_cnn)
flatten_layer = Flatten()(merged_cnn)
# Output
dense = BatchNormalization()(flatten_layer)
dense = Dense(64, activation='sigmoid')(dense)
out = Dense(output_dim, activation='linear')(dense)
self.model = Model(inputs=[query_input, terms_input], outputs=out)
self.model.compile(optimizer='adam', loss=losses.mean_squared_error)
self.model.summary()
]
# Create CNN Model
NUM_EPOCHS = 25
LR = 0.001
BATCH_SIZE = 32
model = tf.keras.Sequential()
model.add(
Conv2D(28, (3, 3),
strides=2,
padding='same',
activation='relu',
input_shape=(28, 28, 1)))
model.add(BatchNormalization())
model.add(MaxPooling2D((2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D((2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dense(256, activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))