def create_movement_model(self):
if LOAD_MOVEMENT_MODEL != "":
print(f"loading {LOAD_MOVEMENT_MODEL}")
model = load_model(LOAD_MOVEMENT_MODEL)
print(f"Model {LOAD_MOVEMENT_MODEL} is now loaded!")
else:
if not USE_CONV_NET:
inputs = Input(shape=envs[0].OBSERVATION_SPACE_VALUES)
movement_branch = self.build_movement_branch(inputs)
model = Model(inputs=inputs,
outputs=[movement_branch],
name="movementnet")
model.compile(loss="mse",
optimizer=AdamKeras(lr=0.001),
metrics=['accuracy'])
else:
print(
"using a convolutional neural network to see if loss improves"
)
model = self.build_conv_branch()
return model
def SingleOutputCNN(
input_shape,
output_shape,
cnns_per_maxpool=1,
maxpool_layers=1,
dense_layers=1,
dense_units=64,
dropout=0.25,
regularization=False,
global_maxpool=False,
name='',
) -> Model:
function_name = cast(types.FrameType,
inspect.currentframe()).f_code.co_name
model_name = f"{function_name}-{name}" if name else function_name
# model_name = seq([ function_name, name ]).filter(lambda x: x).make_string("-") # remove dependency on pyfunctional - not in Kaggle repo without internet
inputs = Input(shape=input_shape)
x = inputs
for cnn1 in range(0, maxpool_layers):
for cnn2 in range(1, cnns_per_maxpool + 1):
x = Conv2D(32 * cnn2,
kernel_size=(3, 3),
padding='same',
activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = BatchNormalization()(x)
x = Dropout(dropout)(x)
if global_maxpool:
x = GlobalMaxPooling2D()(x)
x = Flatten()(x)
for nn1 in range(0, dense_layers):
if regularization:
x = Dense(dense_units,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(x)
else:
x = Dense(dense_units, activation='relu')(x)
x = BatchNormalization()(x)
x = Dropout(dropout)(x)
x = Dense(output_shape, activation='softmax')(x)
model = Model(inputs, x, name=model_name)
# plot_model(model, to_file=os.path.join(os.path.dirname(__file__), f"{name}.png"))
return model
def modelDemoStandardConvLSTM(row, col):
# define LSTM
input = Input(shape=(None, row, col, 1), name='main_input')
# x = TimeDistributed(Flatten())(x)
x = ConvLSTM2D(filters=75,
kernel_size=(3, 3),
padding='same',
return_sequences=False)(input)
x = (Flatten())(x)
x = RepeatVector(4)(x)
x = LSTM(50, return_sequences=True)(x)
output = TimeDistributed(Dense(4, activation='softmax'),
name='main_output')(x)
model = Model(inputs=[input], outputs=[output])
model.compile(loss={'main_output': 'categorical_crossentropy'},
loss_weights={'main_output': 1.},
optimizer='adam',
metrics=['accuracy'])
return model
def get_model(X, N_class, total_words=86627, EMBEDDING_DIM=100, maxlen=53):
embeddings_index = {}
f = open('glove.6B/glove.6B.100d.txt')
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
f.close()
embedding_matrix = np.zeros((total_words, EMBEDDING_DIM))
for word, i in X.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen, ), dtype='int32')
embedding = Embedding(total_words,
EMBEDDING_DIM,
embeddings_initializer=Constant(embedding_matrix),
input_length=maxlen,
trainable=False)(inp)
x = LSTM(300, dropout=0.25, recurrent_dropout=0.25,
return_sequences=True)(embedding)
x = Dropout(0.25)(x)
merged = Attention_COSTUM(maxlen)(x)
merged = Dense(256, activation='relu')(merged)
merged = Dropout(0.25)(merged)
merged = BatchNormalization()(merged)
outp = Dense(N_class, activation='softmax')(merged)
AttentionLSTM = Model(inputs=inp, outputs=outp)
AttentionLSTM.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['acc'])
return AttentionLSTM
def ModelSharedVision():
# First, define the vision modules
digit_input = Input(shape=(27, 27, 1))
x = Conv2D(64, (3, 3))(digit_input)
x = Conv2D(64, (3, 3))(x)
x = MaxPooling2D((2, 2))(x)
out = Flatten()(x)
vision_model = Model(digit_input, out)
# Then define the tell-digits-apart model
digit_a = Input(shape=(27, 27, 1))
digit_b = Input(shape=(27, 27, 1))
# The vision model will be shared, weights and all
out_a = vision_model(digit_a)
out_b = vision_model(digit_b)
concatenated = concatenate([out_a, out_b])
out = Dense(1, activation='sigmoid')(concatenated)
classification_model = Model([digit_a, digit_b], out)
return classification_model
def modelA(row, col):
# define LSTM
input = Input(shape=(None, row, col, 1), name='main_input')
x = TimeDistributed(Conv2D(16, (2, 2), activation='relu'))(input)
x = Dropout(0.25)(x)
x = BatchNormalization()(x)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2), strides=2))(x)
x = Dropout(0.25)(x)
x = TimeDistributed(Flatten())(x)
lstm_output = LSTM(75)(x)
lstm_output = BatchNormalization()(lstm_output)
auxiliary_output = Dense(1, activation='sigmoid',
name='aux_output')(lstm_output)
auxiliary_input = Input(shape=(4, ), name='aux_input')
x = concatenate([lstm_output, auxiliary_input])
x = RepeatVector(8)(x)
x = LSTM(50, return_sequences=True)(x)
# model.add(Dropout(0.25))
x = BatchNormalization()(x)
output = TimeDistributed(Dense(5, activation='softmax'),
name='main_output')(x)
model = Model(inputs=[input, auxiliary_input],
outputs=[output, auxiliary_output])
model.compile(loss={
'main_output': 'categorical_crossentropy',
'aux_output': 'binary_crossentropy'
},
loss_weights={
'main_output': 1.,
'aux_output': 0.2
},
optimizer='adam',
metrics=['accuracy'])
return model
def modelStandardB(row, col):
# define LSTM
input_img = Input(shape=(None, row, col, 1), name='input')
x = TimeDistributed(Conv2D(16, (2, 2), activation='relu'))(input_img)
x = Dropout(0.25)(x)
x = BatchNormalization()(x)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2), strides=2))(x)
x = Dropout(0.25)(x)
x = TimeDistributed(Flatten())(x)
x = LSTM(75)(x)
# model.add(Dropout(0.25))
x = BatchNormalization()(x)
x = RepeatVector(4)(x)
x = LSTM(50, return_sequences=True)(x)
# model.add(Dropout(0.25))
x = BatchNormalization()(x)
output = TimeDistributed(Dense(4, activation='softmax'))(x)
model = Model(input_img, output)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def create_model(step: Tensorflow2ModelStep) -> tf.keras.Model:
"""
Create a TensorFlow v2 sequence to sequence (seq2seq) encoder-decoder model.
:param step: The base Neuraxle step for TensorFlow v2 (Tensorflow2ModelStep)
:return: TensorFlow v2 Keras model
"""
# shape: (batch_size, seq_length, input_dim)
encoder_inputs = Input(shape=(None, step.hyperparams['input_dim']),
batch_size=None,
dtype=tf.dtypes.float32,
name='encoder_inputs')
last_encoder_outputs, last_encoders_states = _create_encoder(
step, encoder_inputs)
decoder_outputs = _create_decoder(step, last_encoder_outputs,
last_encoders_states)
return Model(encoder_inputs, decoder_outputs)
def ModelVisualQuestionAnswering():
# First, let's define a vision model using a Sequential model.
# This model will encode an image into a vector.
vision_model = Sequential()
vision_model.add(
Conv2D(64, (3, 3),
activation='relu',
padding='same',
input_shape=(224, 224, 3)))
vision_model.add(Conv2D(64, (3, 3), activation='relu'))
vision_model.add(MaxPooling2D((2, 2)))
vision_model.add(Conv2D(128, (3, 3), activation='relu', padding='same'))
vision_model.add(Conv2D(128, (3, 3), activation='relu'))
vision_model.add(MaxPooling2D((2, 2)))
vision_model.add(Conv2D(256, (3, 3), activation='relu', padding='same'))
vision_model.add(Conv2D(256, (3, 3), activation='relu'))
vision_model.add(Conv2D(256, (3, 3), activation='relu'))
vision_model.add(MaxPooling2D((2, 2)))
vision_model.add(Flatten())
# Now let's get a tensor with the output of our vision model:
image_input = Input(shape=(224, 224, 3))
encoded_image = vision_model(image_input)
# Next, let's define a language model to encode the question into a vector.
# Each question will be at most 100 words long,
# and we will index words as integers from 1 to 9999.
question_input = Input(shape=(100, ), dtype='int32')
embedded_question = Embedding(input_dim=10000,
output_dim=256,
input_length=100)(question_input)
encoded_question = LSTM(256)(embedded_question)
# Let's concatenate the question vector and the image vector:
merged = concatenate([encoded_question, encoded_image])
# And let's train a logistic regression over 1000 words on top:
output = Dense(1000, activation='softmax')(merged)
# This is our final model:
vqa_model = Model(inputs=[image_input, question_input], outputs=output)
return vqa_model
def plugmodel():
sbbox = tf.keras.layers.Input(shape=(52, 52, 768))
mbbox = tf.keras.layers.Input(shape=(26, 26, 1536))
lbbox = tf.keras.layers.Input(shape=(13, 13, 3072))
conv_spose = common.convolutional(sbbox, (1, 1, 256, 256))
conv_spose = common.convolutional(conv_spose, (1, 1, 256, 3 * (NUM_POSES)),
activate=False,
bn=False)
conv_mpose = common.convolutional(mbbox, (1, 1, 512, 512))
conv_mpose = common.convolutional(conv_mpose, (1, 1, 512, 3 * (NUM_POSES)),
activate=False,
bn=False)
conv_lpose = common.convolutional(lbbox, (1, 1, 255, 1024))
conv_lpose = common.convolutional(conv_lpose, (1, 1, 1024, 3 * NUM_POSES),
activate=False,
bn=False)
return Model(inputs=[sbbox, mbbox, lbbox],
outputs=[conv_spose, conv_mpose, conv_lpose])
def build(self):
input_img = Input(shape=(28, 28, 1))
cnn = Conv2D(32, (3, 3), activation='relu', padding='same')(input_img)
cnn = MaxPooling2D((2, 2), padding='same')(cnn)
cnn = Conv2D(32, (3, 3), activation='relu', padding='same')(cnn)
cnn = MaxPooling2D((2, 2), padding='same')(cnn)
cnn = Conv2D(32, (3, 3), activation='relu', padding='same')(cnn)
encoded = MaxPooling2D((2, 2), padding='same')(cnn)
cnn = Conv2D(32, (3, 3), activation='relu', padding='same')(encoded)
cnn = UpSampling2D((2, 2))(cnn)
cnn = Conv2D(32, (3, 3), activation='relu', padding='same')(cnn)
cnn = UpSampling2D((2, 2))(cnn)
cnn = Conv2D(32, (3, 3), activation='relu')(cnn)
cnn = UpSampling2D((2, 2))(cnn)
decoded = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(cnn)
cnn_autoencoder = Model(input_img, decoded)
cnn_autoencoder.compile(optimizer='adam', loss='binary_crossentropy')
x_train = self.x_train.reshape(-1, 28, 28, 1)
x_train_split, x_valid_split = train_test_split(x_train, test_size=self.train_test_split,
random_state=self.seed)
cnn_autoencoder.fit(x_train_split, x_train_split,
epochs=self.epochs,
batch_size=self.batch_size,
validation_data=(x_valid_split, x_valid_split),
verbose=self.verbosity)
x_train_pred = cnn_autoencoder.predict(x_train)
mse = np.mean(np.power(x_train - x_train_pred, 2), axis=1)
# Semi-supervised due to given threshold
self.threshold = np.quantile(mse, 0.9)
self.cnn_autoencoder = cnn_autoencoder
def build(self):
inputs = Input(shape=self.input_shape, name='encoder_input')
x = Dense(self.intermediate_dim,
activation=self.activation_fct)(inputs)
z_mean = Dense(self.latent_dim, name='z_mean')(x)
z_log_var = Dense(self.latent_dim, name='z_log_var')(x)
# use reparameterization trick to push the sampling out as input
# note that "output_shape" isn't necessary with the TensorFlow backend
z = Lambda(sampling, output_shape=(self.latent_dim, ),
name='z')([z_mean, z_log_var])
# instantiate encoder model
encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder')
# build decoder model
latent_inputs = Input(shape=(self.latent_dim, ), name='z_sampling')
x = Dense(self.intermediate_dim,
activation=self.activation_fct)(latent_inputs)
outputs = Dense(self.original_dim, activation='sigmoid')(x)
# instantiate decoder model
decoder = Model(latent_inputs, outputs, name='decoder')
# instantiate VAE model
outputs = decoder(encoder(inputs)[2])
vae = Model(inputs, outputs, name='vae_mlp')
# VAE Loss = mse_loss or xent_loss + kl_loss
reconstruction_loss = mse(inputs, outputs)
reconstruction_loss *= self.original_dim
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer=self.optimizer,
loss=self.loss,
metrics=['accuracy'])
x_train_split, x_valid_split = train_test_split(
self.x_train,
test_size=self.train_test_split,
random_state=self.seed)
vae.fit(x_train_split,
x_train_split,
batch_size=self.batch_size,
epochs=self.epochs,
verbose=self.verbosity,
shuffle=True,
validation_data=(x_valid_split, x_valid_split))
x_train_pred = vae.predict(self.x_train)
train_mse = np.mean(np.power(self.x_train - x_train_pred, 2), axis=1)
self.threshold = np.quantile(train_mse, 0.9)
self.vae = vae
def ModelVideoQuestionAnswering():
# First, let's define a vision model using a Sequential model.
# This model will encode an image into a vector.
vision_model = Sequential()
vision_model.add(
Conv2D(64, (3, 3),
activation='relu',
padding='same',
input_shape=(224, 224, 3)))
vision_model.add(Conv2D(64, (3, 3), activation='relu'))
vision_model.add(MaxPooling2D((2, 2)))
vision_model.add(Conv2D(128, (3, 3), activation='relu', padding='same'))
vision_model.add(Conv2D(128, (3, 3), activation='relu'))
vision_model.add(MaxPooling2D((2, 2)))
vision_model.add(Conv2D(256, (3, 3), activation='relu', padding='same'))
vision_model.add(Conv2D(256, (3, 3), activation='relu'))
vision_model.add(Conv2D(256, (3, 3), activation='relu'))
vision_model.add(MaxPooling2D((2, 2)))
vision_model.add(Flatten())
# Now let's get a tensor with the output of our vision model:
image_input = Input(shape=(224, 224, 3))
encoded_image = vision_model(image_input)
# Next, let's define a language model to encode the question into a vector.
# Each question will be at most 100 words long,
# and we will index words as integers from 1 to 9999.
question_input = Input(shape=(100, ), dtype='int32')
embedded_question = Embedding(input_dim=10000,
output_dim=256,
input_length=100)(question_input)
encoded_question = LSTM(256)(embedded_question)
# Let's concatenate the question vector and the image vector:
merged = concatenate([encoded_question, encoded_image])
# And let's train a logistic regression over 1000 words on top:
output = Dense(1000, activation='softmax')(merged)
# This is our final model:
# vqa_model = Model(inputs=[image_input, question_input], outputs=output)
video_input = Input(shape=(100, 224, 224, 3))
# This is our video encoded via the previously trained vision_model (weights are reused)
encoded_frame_sequence = TimeDistributed(vision_model)(
video_input) # the output will be a sequence of vectors
encoded_video = LSTM(256)(
encoded_frame_sequence) # the output will be a vector
# This is a model-level representation of the question encoder, reusing the same weights as before:
question_encoder = Model(inputs=question_input, outputs=encoded_question)
# Let's use it to encode the question:
video_question_input = Input(shape=(100, ), dtype='int32')
encoded_video_question = question_encoder(video_question_input)
# And this is our video question answering model:
merged = concatenate([encoded_video, encoded_video_question])
output = Dense(1000, activation='softmax')(merged)
video_qa_model = Model(inputs=[video_input, video_question_input],
outputs=output)
return video_qa_model
def construct_keras_api_model(embedding_weights):
# input_no_time_no_repeat = Input(shape=max_len, dtype='int32')
# embedded_no_time_no_repeat = Embedding(
# creative_id_window,embedding_size,weights=[embedding_weights],trainable=False
# )(input_no_time_no_repeat)
# ==================================================================================
Input_fix_creative_id = Input(shape=(math.ceil(time_id_max / period_days) *
period_length),
dtype='int32',
name='input_fix_creative_id')
Embedded_fix_creative_id = Embedding(
creative_id_window,
embedding_size,
weights=[embedding_weights],
trainable=False)(Input_fix_creative_id)
# ==================================================================================
# input_no_time_with_repeat = Input(shape=max_len, dtype='int32')
# embedded_no_time_with_repeat = Embedding(creative_id_window,embedding_size,weights=[embedding_weights],trainable=False)(input_no_time_with_repeat)
# ----------------------------------------------------------------------
GM_x = keras.layers.GlobalMaxPooling1D()(Embedded_fix_creative_id)
GM_x = Dropout(0.5)(GM_x)
GM_x = Dense(embedding_size // 2, kernel_regularizer=l2(0.001))(GM_x)
GM_x = BatchNormalization()(GM_x)
GM_x = Activation('relu')(GM_x)
GM_x = Dropout(0.5)(GM_x)
GM_x = Dense(embedding_size // 4, kernel_regularizer=l2(0.001))(GM_x)
GM_x = BatchNormalization()(GM_x)
GM_x = Activation('relu')(GM_x)
GM_x = Dense(1, 'sigmoid')(GM_x)
# ----------------------------------------------------------------------
GA_x = GlobalAveragePooling1D()(Embedded_fix_creative_id)
GA_x = Dropout(0.5)(GA_x)
GA_x = Dense(embedding_size // 2, kernel_regularizer=l2(0.001))(GA_x)
GA_x = BatchNormalization()(GA_x)
GA_x = Activation('relu')(GA_x)
GA_x = Dropout(0.5)(GA_x)
GA_x = Dense(embedding_size // 4, kernel_regularizer=l2(0.001))(GA_x)
GA_x = BatchNormalization()(GA_x)
GA_x = Activation('relu')(GA_x)
GA_x = Dense(1, 'sigmoid')(GA_x)
# ==================================================================================
Conv_creative_id = Conv1D(embedding_size, 15, 5,
activation='relu')(Embedded_fix_creative_id)
# ----------------------------------------------------------------------
Conv_GM_x = MaxPooling1D(7)(Conv_creative_id)
Conv_GM_x = Conv1D(embedding_size, 2, 1, activation='relu')(Conv_GM_x)
Conv_GM_x = GlobalMaxPooling1D()(Conv_GM_x)
Conv_GM_x = Dropout(0.5)(Conv_GM_x)
Conv_GM_x = Dense(embedding_size // 2,
kernel_regularizer=l2(0.001))(Conv_GM_x)
Conv_GM_x = BatchNormalization()(Conv_GM_x)
Conv_GM_x = Activation('relu')(Conv_GM_x)
Conv_GM_x = Dropout(0.5)(Conv_GM_x)
Conv_GM_x = Dense(embedding_size // 4,
kernel_regularizer=l2(0.001))(Conv_GM_x)
Conv_GM_x = BatchNormalization()(Conv_GM_x)
Conv_GM_x = Activation('relu')(Conv_GM_x)
Conv_GM_x = Dense(1, 'sigmoid')(Conv_GM_x)
# ----------------------------------------------------------------------
Conv_GA_x = AveragePooling1D(7)(Conv_creative_id)
Conv_GA_x = Conv1D(embedding_size, 2, 1, activation='relu')(Conv_GA_x)
Conv_GA_x = GlobalAveragePooling1D()(Conv_GA_x)
Conv_GA_x = Dropout(0.5)(Conv_GA_x)
Conv_GA_x = Dense(embedding_size // 2,
kernel_regularizer=l2(0.001))(Conv_GA_x)
Conv_GA_x = BatchNormalization()(Conv_GA_x)
Conv_GA_x = Activation('relu')(Conv_GA_x)
Conv_GA_x = Dropout(0.5)(Conv_GA_x)
Conv_GA_x = Dense(embedding_size // 4,
kernel_regularizer=l2(0.001))(Conv_GA_x)
Conv_GA_x = BatchNormalization()(Conv_GA_x)
Conv_GA_x = Activation('relu')(Conv_GA_x)
Conv_GA_x = Dense(1, 'sigmoid')(Conv_GA_x)
# ----------------------------------------------------------------------
LSTM_x = Conv1D(embedding_size, 14, 7, activation='relu')(Conv_creative_id)
LSTM_x = LSTM(embedding_size, return_sequences=True)(LSTM_x)
LSTM_x = LSTM(embedding_size, return_sequences=True)(LSTM_x)
LSTM_x = LSTM(embedding_size)(LSTM_x)
LSTM_x = Dropout(0.5)(LSTM_x)
LSTM_x = Dense(embedding_size // 2, kernel_regularizer=l2(0.001))(LSTM_x)
LSTM_x = BatchNormalization()(LSTM_x)
LSTM_x = Activation('relu')(LSTM_x)
LSTM_x = Dropout(0.5)(LSTM_x)
LSTM_x = Dense(embedding_size // 4, kernel_regularizer=l2(0.001))(LSTM_x)
LSTM_x = BatchNormalization()(LSTM_x)
LSTM_x = Activation('relu')(LSTM_x)
LSTM_x = Dense(1, 'sigmoid')(LSTM_x)
# ----------------------------------------------------------------------
concatenated = concatenate([
GM_x,
GA_x,
Conv_GM_x,
Conv_GA_x,
LSTM_x,
],
axis=-1)
output_tensor = Dense(1, 'sigmoid')(concatenated)
keras_api_model = Model(
[
# input_no_time_no_repeat,
Input_fix_creative_id,
# input_no_time_with_repeat,
],
output_tensor)
keras_api_model.summary()
plot_model(keras_api_model, to_file='model/keras_api_word2vec_model.png')
print('-' * 5 + ' ' * 3 + "编译模型" + ' ' * 3 + '-' * 5)
keras_api_model.compile(optimizer=optimizers.RMSprop(lr=RMSProp_lr),
loss=losses.binary_crossentropy,
metrics=[metrics.binary_accuracy])
return keras_api_model
class QueryReformulation:
def __init__(self, model_path=None, output_path=''):
self.model = None
self.model_name = None
self.model_output = output_path + '/qr_{name}_model_[e{epoch}]_[p{precision}]_' \
+ str(datetime.now().date()) + '.h5'
if model_path:
self.model = load_model(model_path)
self.model.summary()
def build_model(self, model_name, query_dim, terms_dim, output_dim,
word_embedding):
self.model_name = model_name
query_input = Input(shape=(query_dim, ), name='query_input')
terms_input = Input(shape=(terms_dim, ), name='terms_input')
if model_name == 'lstm':
embedding_feature_block = Sequential(layers=[
Embedding(word_embedding.vocabulary_size,
word_embedding.dimensions,
weights=[word_embedding.embedding_matrix],
trainable=True,
mask_zero=False),
BatchNormalization(),
LSTM(64, return_sequences=True)
])
elif model_name == 'bilstm':
embedding_feature_block = Sequential(layers=[
Embedding(word_embedding.vocabulary_size,
word_embedding.dimensions,
weights=[word_embedding.embedding_matrix],
trainable=True,
mask_zero=False),
BatchNormalization(),
Bidirectional(LSTM(64, return_sequences=True))
])
else: # default cnn
embedding_feature_block = Sequential(layers=[
Embedding(word_embedding.vocabulary_size,
word_embedding.dimensions,
weights=[word_embedding.embedding_matrix],
trainable=True,
mask_zero=False),
BatchNormalization(),
Conv1D(filters=64, kernel_size=3, strides=1),
MaxPooling1D(pool_size=3)
])
# Features
query_feature = embedding_feature_block(query_input)
terms_feature = embedding_feature_block(terms_input)
# Query-Terms alignment
attention = Dot(axes=-1)([query_feature, terms_feature])
softmax_attention = Lambda(lambda x: softmax(x, axis=1),
output_shape=unchanged_shape)(attention)
terms_aligned = Dot(axes=1)([softmax_attention, terms_feature])
# Aligned features
if model_name == 'lstm':
flatten_layer = LSTM(128, return_sequences=False)(terms_aligned)
elif model_name == 'bilstm':
flatten_layer = Bidirectional(LSTM(
128, return_sequences=False))(terms_aligned)
else: # default cnn
merged_cnn = Conv1D(filters=128, kernel_size=3,
strides=1)(terms_aligned)
merged_cnn = MaxPooling1D(pool_size=3)(merged_cnn)
flatten_layer = Flatten()(merged_cnn)
# Output
dense = BatchNormalization()(flatten_layer)
dense = Dense(64, activation='sigmoid')(dense)
out = Dense(output_dim, activation='linear')(dense)
self.model = Model(inputs=[query_input, terms_input], outputs=out)
self.model.compile(optimizer='adam', loss=losses.mean_squared_error)
self.model.summary()
def train_model(self,
query_objs,
query_sequence,
terms_sequence,
candidate_terms,
epochs=20,
batch_size=4):
best_precision = 0
pool = Pool(batch_size)
for e in range(epochs):
print('Epochs: %3d/%d' % (e + 1, epochs))
reward = np.zeros(shape=(len(query_objs)))
precision = np.zeros(shape=(len(query_objs)))
for i, query, q_seq, t_seq, terms in get_batch_data(
query_objs, query_sequence, terms_sequence,
candidate_terms, batch_size):
print(' [%4d-%-4d/%d]' % (i, i + batch_size, len(query_objs)))
weights = self.model.predict(x=[q_seq, t_seq])
batch_reward_precision = pool.map(evaluate_reward_precision,
zip(weights, terms, query))
batch_reward_precision = np.array(batch_reward_precision)
batch_reward = 0.8 * np.asarray(
batch_reward_precision[:,
0]) + 0.2 * reward[i:i + batch_size]
self.model.train_on_batch(x=[q_seq, t_seq],
y=weights,
sample_weight=batch_reward)
reward[i:i + batch_size] = batch_reward_precision[:, 0]
precision[i:i + batch_size] = batch_reward_precision[:, 1]
# Save model
avg_precision = precision.mean()
print(' Average precision %.5f on epoch %d, best precision %.5f' %
(avg_precision, e + 1, best_precision))
if avg_precision > best_precision:
best_precision = avg_precision
self.model.save(filepath=self.model_output.format(
name=self.model_name,
epoch=e + 1,
precision=round(avg_precision, 4)))
pool.close()
pool.join()
def test_model(self,
query_objs,
query_sequence,
terms_sequence,
candidate_terms,
batch_size=4):
pool = Pool(batch_size)
precision_recall = np.zeros(shape=(len(query_objs), 2))
for i, query, q_seq, t_seq, terms in get_batch_data(
query_objs, query_sequence, terms_sequence, candidate_terms,
batch_size):
print('[%4d-%-4d/%d]' % (i, i + batch_size, len(query_objs)))
weights = self.model.predict(x=[q_seq, t_seq])
batch_precision_recall = pool.map(evaluate_precision_recall,
zip(weights, terms, query))
precision_recall[i:i +
batch_size] = np.array(batch_precision_recall)
pool.close()
pool.join()
return precision_recall.mean(axis=0)
def reformulate_query(self,
query_sequence,
terms_sequence,
candidate_terms,
threshold=0.5):
weights = self.model.predict(x=[[query_sequence], [terms_sequence]])
reformulated_query = recreate_query(terms=candidate_terms,
weights=weights[0],
threshold=threshold)
return reformulated_query
def build_model(self, model_name, query_dim, terms_dim, output_dim,
word_embedding):
self.model_name = model_name
query_input = Input(shape=(query_dim, ), name='query_input')
terms_input = Input(shape=(terms_dim, ), name='terms_input')
if model_name == 'lstm':
embedding_feature_block = Sequential(layers=[
Embedding(word_embedding.vocabulary_size,
word_embedding.dimensions,
weights=[word_embedding.embedding_matrix],
trainable=True,
mask_zero=False),
BatchNormalization(),
LSTM(64, return_sequences=True)
])
elif model_name == 'bilstm':
embedding_feature_block = Sequential(layers=[
Embedding(word_embedding.vocabulary_size,
word_embedding.dimensions,
weights=[word_embedding.embedding_matrix],
trainable=True,
mask_zero=False),
BatchNormalization(),
Bidirectional(LSTM(64, return_sequences=True))
])
else: # default cnn
embedding_feature_block = Sequential(layers=[
Embedding(word_embedding.vocabulary_size,
word_embedding.dimensions,
weights=[word_embedding.embedding_matrix],
trainable=True,
mask_zero=False),
BatchNormalization(),
Conv1D(filters=64, kernel_size=3, strides=1),
MaxPooling1D(pool_size=3)
])
# Features
query_feature = embedding_feature_block(query_input)
terms_feature = embedding_feature_block(terms_input)
# Query-Terms alignment
attention = Dot(axes=-1)([query_feature, terms_feature])
softmax_attention = Lambda(lambda x: softmax(x, axis=1),
output_shape=unchanged_shape)(attention)
terms_aligned = Dot(axes=1)([softmax_attention, terms_feature])
# Aligned features
if model_name == 'lstm':
flatten_layer = LSTM(128, return_sequences=False)(terms_aligned)
elif model_name == 'bilstm':
flatten_layer = Bidirectional(LSTM(
128, return_sequences=False))(terms_aligned)
else: # default cnn
merged_cnn = Conv1D(filters=128, kernel_size=3,
strides=1)(terms_aligned)
merged_cnn = MaxPooling1D(pool_size=3)(merged_cnn)
flatten_layer = Flatten()(merged_cnn)
# Output
dense = BatchNormalization()(flatten_layer)
dense = Dense(64, activation='sigmoid')(dense)
out = Dense(output_dim, activation='linear')(dense)
self.model = Model(inputs=[query_input, terms_input], outputs=out)
self.model.compile(optimizer='adam', loss=losses.mean_squared_error)
self.model.summary()
class Autoencoder(object):
"""docstring for Autoencoder"""
# def __init__(self, sample_weights, sample_weight_mode):
def __init__(self, epochs, verbosity):
self.epochs = epochs
self.batch_size = 256
self.shuffle = True
self.validation_split = 0.05
self.optimizer = 'adadelta'
self.loss = 'mse'
self.verbosity = verbosity
self.code_layer_type = None
self.model = None
self.sample_weight_mode = None
self.sample_weights = None
self.y_true = None
self.y_pred = None
def model(self, code_layer_type, input_dim, code_dim):
self.code_layer_type = code_layer_type
assert len(code_dim) > 0
if self.code_layer_type == 'lstm':
assert len(input_dim) == 2
input_data = Input(shape=(input_dim[0], input_dim[1]))
if len(code_dim) == 1:
encoded = LSTM(code_dim[0])(input_data)
decoded = RepeatVector(input_dim[0])(encoded)
elif len(code_dim) > 1:
encoded = input_data
for i, units in enumerate(code_dim):
if i == len(code_dim) - 1:
encoded = LSTM(units)(encoded)
continue
encoded = LSTM(units, return_sequences=True)(encoded)
for i, units in enumerate(reversed(code_dim)):
if i == 1:
decoded = LSTM(units, return_sequences=True)(
RepeatVector(input_dim[0])(encoded))
elif i > 1:
decoded = LSTM(units, return_sequences=True)(decoded)
else:
raise ValueError("The codDim must be over 0.")
decoded = LSTM(input_dim[-1], return_sequences=True)(decoded)
self.model = Model(input_data, decoded)
elif self.code_layer_type == 'dense':
assert len(input_dim) == 1
input_data = Input(shape=(input_dim[0], ))
encoded = input_data
for i, units in enumerate(code_dim):
encoded = Dense(units, activation='relu')(encoded)
decoded = Dense(input_dim[-1], activation='sigmoid')(encoded)
self.model = Model(input_data, decoded)
elif self.code_layer_type == 'cov':
pass
def modelMasking(self, code_layer_type, input_dim, code_dim):
self.code_layer_type = code_layer_type
assert len(code_dim) > 0
if self.code_layer_type == 'lstm':
assert len(input_dim) == 2
input_data = Input(shape=(input_dim[0], input_dim[1]))
mask = Masking(mask_value=0.)(input_data)
if len(code_dim) == 1:
encoded = LSTM(code_dim[0])(mask)
decoded = RepeatVector(input_dim[0])(encoded)
elif len(code_dim) > 1:
encoded = mask
for i, units in enumerate(code_dim):
if i == len(code_dim) - 1:
encoded = LSTM(units)(encoded)
continue
encoded = LSTM(units, return_sequences=True)(encoded)
for i, units in enumerate(reversed(code_dim)):
if i == 1:
decoded = LSTM(units, return_sequences=True)(
RepeatVector(input_dim[0])(encoded))
elif i > 1:
decoded = LSTM(units, return_sequences=True)(decoded)
else:
raise ValueError("The codDim must be over 0.")
decoded = LSTM(input_dim[-1], return_sequences=True)(decoded)
self.model = Model(input_data, decoded)
elif self.code_layer_type == 'cov':
pass
elif self.code_layer_type == 'dense':
assert len(input_dim) == 1
input_data = Input(shape=(input_dim[0], ))
# encoded = input_data
# for i, units in enumerate(codeDim):
# encoded = Dense(units, activation='relu')(encoded)
# decoded = Dense(inputDim[-1], activation='sigmoid')(encoded)
# self.model = Model(input_data, decoded)
encoder = Dense(
code_dim[0],
activation="tanh",
activity_regularizer=regularizers.l1(10e-5))(input_data)
encoder = Dense(int(code_dim[0] / 2), activation="relu")(encoder)
decoder = Dense(int(code_dim[0] / 2), activation='tanh')(encoder)
decoder = Dense(input_dim[0], activation='relu')(decoder)
self.model = Model(input_data, decoder)
def compile(self, *args):
if len(args) == 0:
self.model.compile(optimizer=self.optimizer, loss=self.loss)
elif len(args) == 1:
if args[0] == 'temporal':
self.sample_weight_mode = args[0]
self.model.compile(optimizer=self.optimizer,
loss=self.loss,
sample_weight_mode=self.sample_weight_mode)
elif args[0] == 'customFunction':
self.model.compile(optimizer=self.optimizer,
loss=self.weighted_vector_mse)
else:
raise ValueError(
"Invalid maskType, please input 'sample_weights' or 'customFunction'"
)
else:
raise ValueError("argument # must be 0 or 1.")
def fit(self, *args):
# early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.01, patience=3, verbose=1, mode='auto')
if len(args) == 2:
if args[1] == 'nor':
self.model.fit(args[0],
args[0],
epochs=self.epochs,
batch_size=self.batch_size,
shuffle=self.shuffle,
validation_split=self.validation_split,
verbose=self.verbosity)
# callbacks = [early_stopping])
elif args[1] == 'rev':
self.model.fit(args[0],
np.flip(args[0], 1),
epochs=self.epochs,
batch_size=self.batch_size,
shuffle=self.shuffle,
validation_split=self.validation_split,
verbose=self.verbosity)
# callbacks=[early_stopping])
else:
raise ValueError(
"decoding sequence type: 'normal' or 'reverse'.")
elif len(args) == 3:
self.sample_weights = args[2]
if args[1] == 'nor':
self.model.fit(args[0],
args[0],
epochs=self.epochs,
batch_size=self.batch_size,
shuffle=self.shuffle,
validation_split=self.validation_split,
sample_weight=self.sample_weights,
verbose=self.verbosity)
# callbacks=[early_stopping])
elif args[1] == 'rev':
self.model.fit(args[0],
np.flip(args[0], 1),
epochs=self.epochs,
batch_size=self.batch_size,
shuffle=self.shuffle,
validation_split=self.validation_split,
sample_weight=self.sample_weights,
verbose=self.verbosity)
# callbacks=[early_stopping])
else:
raise ValueError(
"Please input, 'data', 'nor' or 'rev', 'sample_weights'")
def predict(self, data):
return self.model.predict(data)
def weighted_vector_mse(self, y_true, y_pred):
self.y_true = y_true
self.y_pred = y_pred
weight = T.ceil(self.y_true)
loss = T.square(weight * (self.y_true - self.y_pred))
# use appropriate relations for other objectives. E.g, for binary_crossentropy:
# loss = weights * (y_true * T.log(y_pred) + (1.0 - y_true) * T.log(1.0 - y_pred))
return T.mean(T.sum(loss, axis=-1))