def simpleCNN(IMAGE_WIDTH, IMAGE_HEIGHT, IMAGE_CHANNELS):
"""First version of simple CNN
"""
from tf.keras.models import Sequential
from tf.keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense, Activation, BatchNormalization
model = Sequential()
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(
IMAGE_WIDTH, IMAGE_HEIGHT, IMAGE_CHANNELS)))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
# 2 because we have cat and dog classes
model.add(Dense(2, activation='softmax'))
return model
def mlp_layer(self, lstm):
fc = Dense(self.fc_args[0], activation='relu', kernel_initializer='truncated_normal')(lstm)
fc = Dropout(self.dropout_args)(fc)
fc = Dense(self.fc_args[1], activation='relu', kernel_initializer='truncated_normal')(fc)
fc = Dropout(self.dropout_args)(fc)
fc = Dense(self.fc_args[2], activation='relu', kernel_initializer='truncated_normal')(fc)
fc = Dropout(self.dropout_args)(fc)
fc = Dense(self.fc_args[3], activation='relu', kernel_initializer='truncated_normal')(fc)
fc = Dropout(self.dropout_args)(fc)
output = Dense(self.fc_args[4], activation='softmax', name='output')(fc)
return output
def _create_model_2(self, input_tensor, input_shape, num_classes,
dropout_rate):
#input_tensor = Input(shape=input_shape)
model = Sequential()
model.add(
Conv2D(32,
kernel_size=(5, 5),
padding='same',
activation='relu',
input_shape=input_shape[1:]))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(
Conv2D(64, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Flatten())
model.add(Dense(1024, activation='relu'))
model.add(Dropout(dropout_rate))
if 1 == num_classes:
model.add(Dense(1, activation='sigmoid'))
#model.add(Dense(1, activation='sigmoid', activity_regularizer=keras.regularizers.activity_l2(0.0001)))
elif num_classes >= 2:
model.add(Dense(num_classes, activation='softmax'))
#model.add(Dense(num_classes, activation='softmax', activity_regularizer=keras.regularizers.activity_l2(0.0001)))
else:
assert num_classes > 0, 'Invalid number of classes.'
# Display the model summary.
#model.summary()
return model(input_tensor)
def _create_model_1(self, input_tensor, num_classes, dropout_rate):
x = Conv2D(32, kernel_size=(5, 5), padding='same',
activation='relu')(input_tensor)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)
x = Conv2D(64, kernel_size=(3, 3), padding='same',
activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)
x = Flatten()(x)
x = Dense(1024, activation='relu')(x)
x = Dropout(dropout_rate)(x)
if 1 == num_classes:
x = Dense(1, activation='sigmoid')(x)
#x = Dense(1, activation='sigmoid', activity_regularizer=keras.regularizers.activity_l2(0.0001))(x)
elif num_classes >= 2:
x = Dense(num_classes, activation='softmax')(x)
#x = Dense(num_classes, activation='softmax', activity_regularizer=keras.regularizers.activity_l2(0.0001))(x)
else:
assert num_classes > 0, 'Invalid number of classes.'
#model = Model(inputs=input_tensor, outputs=x)
return x
def feature_extractor_network(self):
# input
in_image = Input(shape = in_shape)
# C1 Layer
nett = Conv2D(32,(5,5))(in_image)
nett = BatchNormalization()(nett)
nett = LeakyReLU(alpha = 0.2)(nett)
# M2 Layer
nett = MaxPooling2D(pool_size = (3,3))(nett)
# C3 Layer
nett = Conv2D(64,(3,3))
nett = BatchNormalization(pool_size = (3,3))(nett)
nett = LeakyReLU(alpha = 0.2)(nett)
# L4 Layer
nett = LocallyConnected2D(128,(3,3))(nett)
# L5 Layer
nett = LocallyConnected2D(256,(3,3))(nett)
# F6 Layer
nett = Dense(512,activation='relu')(nett)
nett = Dropout(0.2)(nett)
# F7 Layer
out_features = Dense(activation='tanh')(nett)
# output
model = Model(inputs = in_image, outputs = out_features)
return model
def build_discriminator(self):
model = Sequential()
model.add(Dense(78, activation="relu", input_dim=self.input_shape))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(56, activation="relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(32, activation="relu"))
model.add(Dropout(rate=0.3))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(28, activation="relu"))
model.add(Dropuout(rate=0.3))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(10, activation="relu"))
model.summary()
img = Input(shape=self.img_shape)
features = model(img)
valid = Dense(1, activation="sigmoid")(features)
label = Dense(self.num_classes + 1, activation="softmax")(features)
return Model(img, [valid, label])
def vanilla_rnn(num_words,
state,
lra,
dropout,
num_outputs=2,
emb_dim=50,
input_length=500):
model = Sequential()
model.add(
Embedding(input_dim=num_words + 1,
output_dim=emb_dim,
input_length=input_length,
trainable=False,
weights=[embed_matrix]))
model.add(
SimpleRNN(units=state,
input_shape=(num_words, 1),
return_sequences=False))
model.add(Dropout(dropout))
model.add(Dense(num_outputs, activation='sigmoid'))
RMS = optimizers.RMSprop(lr=lra)
model.compile(loss='binary_crossentropy',
optimizer=RMS,
metrics=['accuracy'])
return model
def init_model():
#K.clear_session()
tf.reset_default_graph()
model = Sequential()
model.add(Conv2D(16, (3, 3), input_shape=(28, 28, 1), padding = "SAME", activation = "relu"))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(MaxPool2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
for layer in model.layers:
print(layer.output_shape)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=keras.optimizers.Adadelta(lr=0.1),
metrics=['accuracy'])
return model
def discriminator_network(self):
# input
in_image = Input(shape=self.img_shape)
# C1 layer
nett = Conv2D(64,(5,5))(in_image)
nett = BatchNormalization()(nett)
nett = LeakyReLU(alpha = 0.2)(nett)
# C2 layer
nett = Conv2D(128,(5,5))(nett)
nett = BatchNormalization()(nett)
nett = LeakyReLU(alpha = 0.2)(nett)
nett = Dropout(0.2)(nett)
# C3 layer
nett = Conv2D(256,(5,5))(nett)
nett = BatchNormalization()(nett)
nett = LeakyReLU(alpha = 0.2)(nett)
nett = Dropout(0.2)(nett)
# F4 layer
nett = Flatten()(nett)
validity = Dense(1,alpha = 0.2)(nett)
#output
model = Model(inputs = in_image, outputs = validity)
return model
def simpleCNN_v2_AUC(input_shape):
"""Second version of simple CNN,
added with 2 CNN layers and 1 FC layer.
"""
from tf.keras.models import Sequential
from tf.keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense, Activation, BatchNormalization
model = Sequential()
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=input_shape))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
# 2 because we have cat and dog classes
model.add(Dense(2, activation='softmax'))
# AUC version
model.compile(loss='categorical_crossentropy',
optimizer='adam', metrics=['accuracy', auc])
return model
def target_word_hidden(inputs, target_role, n_word_vocab, n_role_vocab, emb_init,
n_factors_cls=512, n_hidden=256, using_dropout=False, dropout_rate=0.3):
"""Hidden layer of non-incremental model role-filler to predict target word given context
(input words, input roles, target role).
# Args:
inputs: output of context embedding from the last layer, shape is (batch_size, input_length)
target_role: place holder for target roles, shape is (batch_size, 1)
n_word_vocab: size of word vocabulary
n_role_vocab: size of role vocabulary
emb_init: initializer of embedding
n_hidden: number of hidden units
using_dropout: bool, using drop-out layer or not
dropout_rate: rate of drop-out layer
# Return:
(n_factors_cls, )
"""
# target role embedding; shape is (batch_size, 1, n_factors_cls)
target_role_embedding = Embedding(n_role_vocab, n_factors_cls,
embeddings_initializer=emb_init,
name='target_role_embedding')(target_role)
if using_dropout:
# Drop-out layer after embeddings
target_role_embedding = Dropout(dropout_rate)(target_role_embedding)
# reduce dimension of tensor from 3 to 2
target_role_embedding = Lambda(lambda x: K.sum(x, axis=1),
output_shape=(n_factors_cls,))(target_role_embedding)
# context_emb after linear projection
weighted_context_embedding = Dense(n_factors_cls, # QUESTION: what's the point of the linear transformation? (team1-change)
activation='linear',
use_bias=False,
input_shape=(n_hidden, ))(inputs)
# if using_dropout:
# # Drop-out layer after fully connected layer
# weighted_context_embedding = Dropout(0.5)(weighted_context_embedding)
# hidden units after combining 2 embeddings; shape is the same with embedding
hidden = Multiply()([weighted_context_embedding, target_role_embedding])
return hidden
def target_role_hidden(inputs, target_word, n_word_vocab, n_role_vocab, emb_init,
n_factors_cls=512, n_hidden=256, using_dropout=False, dropout_rate=0.3):
"""Hidden layer of multi-task non-incremental model role-filler to predict target role given context
(input words, input roles, target word).
# Args:
context_emb: output of context embedding from the last layer, shape is (batch_size, input_length)
target_word: place holder for target word, shape is (batch_size, 1)
n_word_vocab: size of word vocabulary
n_role_vocab: size of role vocabulary
emb_init: initializer of embedding
n_hidden: number of hidden units
# Return:
(n_factors_cls, )
"""
# target role embedding; shape is (batch_size, 1, n_factors_emb)
target_word_embedding = Embedding(n_word_vocab, n_factors_cls,
embeddings_initializer=emb_init,
name='target_word_embedding')(target_word)
if using_dropout:
target_word_embedding = Dropout(dropout_rate)(target_word_embedding)
# reduce dimension of tensor from 3 to 2
target_word_embedding = Lambda(lambda x: K.sum(x, axis=1),
output_shape=(n_factors_cls,))(target_word_embedding)
# context_emb after linear projection
weighted_context_embedding = Dense(n_factors_cls,
activation='linear',
use_bias=False,
input_shape=(n_hidden, ))(inputs)
# if using_dropout:
# weighted_context_embedding = Dropout(0.5)(weighted_context_embedding)
# hidden units after combining 2 embeddings; shape is the same with embedding
hidden = Multiply()([weighted_context_embedding, target_word_embedding])
return hidden
class Classifier: def __init__(self, dense_classifier, embedding_dim, layers, dropout): self.model = self.build_dense_classifier(embedding_dim, layers, dropout) if dense_classifier else self.build_recurrent_classifier(embedding_dim, layers, dropout) self.batch_size = batch_size self.epochs = epochs self.validation_split = validation_split def build_dense_classifier(self, embedding_dim, layers, dropout): input1 = Input(shape=(embedding_dim*2,)) h1 = Dense(layers[0], activation='relu')(input1) if dropout is not None: h1 = Dropout(rate=dropout)(h1) for layer in layers[1:]: h1 = Dense(layer, activation='relu')(h1) if dropout is not None: h1 = Dropout(rate=dropout)(h1) out = Dense(2, activation='softmax')(h1) model = Model(inputs=input1, outputs=out) return model
def factored_embedding(input_words,
input_roles,
n_word_vocab,
n_role_vocab,
emb_init,
missing_word_id,
input_length,
n_factors_emb=256,
n_hidden=256,
mask_zero=True,
using_dropout=False,
dropout_rate=0.3,
using_bias=False):
"""Role-based word embedding combining word and role embedding.
# Arguments:
input_words: place holder for input words, shape is (batch_size, input_length)
input_roles: place holder for input roles, shape is (batch_size, input_length)
n_word_vocab: size of word vocabulary
n_role_vocab: size of role vocabulary
emb_init: initializer of embedding
missing_word_id: the id used as place-holder for the role without a word appearing
n_factors_emb: tensor factorization number, default: 256
n_sample: number of samples, useful when there are negative samples
mask_zero: bool, zero out the weight of missing word
using_dropout: bool, using drop-out layer or not
dropout_rate: rate of drop-out layer
"""
# word embedding; shape is (batch_size, input_length, n_factors_emb)
word_embedding = Embedding(n_word_vocab,
n_factors_emb,
embeddings_initializer=emb_init,
name='org_word_embedding')(input_words)
if mask_zero:
# a hack zeros out the missing word inputs
weights = np.ones((n_word_vocab, n_factors_emb))
weights[missing_word_id] = 0
mask = Embedding(n_word_vocab,
n_factors_emb,
weights=[weights],
trainable=False,
name='mask_missing')(input_words)
# masked word embedding
word_embedding = Multiply(name='word_embedding')(
[word_embedding, mask])
# Alternative implementation, need missing_word_id == 0
# self.word_embedding = Masking(mask_value=0.,
# input_shape=(input_length, n_factors_emb)(word_embedding)
# role embedding; shape is (batch_size, input_length, n_factors_emb)
role_embedding = Embedding(n_role_vocab,
n_factors_emb,
embeddings_initializer=emb_init,
name='role_embedding')(input_roles)
if using_dropout:
# Drop-out layer after embeddings
word_embedding = Dropout(dropout_rate)(word_embedding)
role_embedding = Dropout(dropout_rate)(role_embedding)
# hidden units after combining 2 embeddings; shape is the same with embedding
hidden = Multiply(name='multiply_composition')([
word_embedding, role_embedding
]) # QUESTION: why multiply instead of concatenating? (team1-change)
# fully connected layer, output shape is (batch_size, input_length, n_hidden)
embedding = Dense(n_hidden,
activation='linear',
use_bias=using_bias,
input_shape=(n_factors_emb, ),
name='role_based_word_embedding')(hidden)
return embedding
model = Model(inputs=input1, outputs=out) return model def build_recurrent_classifier(self, embedding_dim, layers, dropout): input1 = Input(shape=(embedding_dim*2,)) h1 = LSTM(layers[0], activation='tanh')(input1) if dropout is not None: h1 = Dropout(rate=dropout)(h1) for layer in layers[1:]: h1 = LSTM(layer, activation='tanh')(h1) if args.dropout is not None: h1 = Dropout(rate=dropout)(h1) h1 = Dense(layers[-1], activation='relu')(h1) out = Dense(2, activation='softmax')(h1) model = Model(inputs=input1, outputs=out) return model def train(self, filepath, patience=10, validation_split=.2, batch_size=120, epochs=250, train_data=None, test_data=None): train_embeddings, train_labels = train_data checkpointing = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max') halting = EarlyStopping(monitor='val_loss', patience=patience, mode='auto', restore_best_weights=True) callbacks = [checkpointing, halting]
y_train = []
no_of_sample = len(training_set)
for i in range(60, no_of_sample):
X_train.append(training_set_scaled[i - 60:i, 0])
y_train.append(training_set_scaled[i, 0])
X_train, y_train = np.array(X_train), np.array(y_train)
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
# Xay dung model LSTM
regressor = Sequential()
regressor.add(
LSTM(units=50, return_sequences=True, input_shape=(X_train.shape[1], 1)))
regressor.add(Dropout(0.2))
regressor.add(LSTM(units=50, return_sequences=True))
regressor.add(Dropout(0.2))
regressor.add(LSTM(units=50, return_sequences=True))
regressor.add(Dropout(0.2))
regressor.add(LSTM(units=50))
regressor.add(Dropout(0.2))
regressor.add(Dense(units=1))
regressor.compile(optimizer='adam', loss='mean_squared_error')
# Neu ton tai file model thi load
if path.exists("mymodel.h5"):
regressor.load_weights("mymodel.h5")
else:
# Con khong thi train
regressor.fit(X_train, y_train, epochs=100, batch_size=32)
def build_dense_classifier(self, embedding_dim, layers, dropout): input1 = Input(shape=(embedding_dim*2,)) h1 = Dense(layers[0], activation='relu')(input1) if dropout is not None: h1 = Dropout(rate=dropout)(h1)
def build_recurrent_classifier(self, embedding_dim, layers, dropout): input1 = Input(shape=(embedding_dim*2,)) h1 = LSTM(layers[0], activation='tanh')(input1) if dropout is not None: h1 = Dropout(rate=dropout)(h1)