def create_model():
# word embedding
text_input = Input(shape=(max_token_length, ),
dtype='int32') #输入行数为max_token_length的矩阵，数据类型是32位整数
x = Embedding(input_dim=vocab_size,
output_dim=embedding_size)(text_input) #embedding层
x = LSTM(256, return_sequences=True)(x) #输入LSTM模型,256个神经元
text_embedding = TimeDistributed(Dense(embedding_size))(
x) #使用包装器TimeDistributed包装Dense，以产生针对各个时间步信号的独立全连接
# image embedding
image_input = Input(shape=(2048, )) #输入行数为2048的图形数据矩阵
#全连接层dense，激活函数：relu
x = Dense(embedding_size, activation='relu',
name='image_embedding')(image_input)
# the image I is only input once RepeatVector层将输入只重复1次
image_embedding = RepeatVector(1)(x)
# language model
x = [image_embedding, text_embedding]
#按列合并
x = Concatenate(axis=1)(x)
#以一定概率丢弃神经元，uniform随机生成0-1内一个实数
x = Dropout({{uniform(0, 1)}})(x)
#输入LSTM模型,1024个神经元
x = LSTM(1024, return_sequences=True, name='language_lstm_1')(x)
x = Dropout({{uniform(0, 1)}})(x)
x = LSTM(1024, name='language_lstm_2')(x)
x = Dropout({{uniform(0, 1)}})(x)
# 全连接层dense，激活函数：softmax
output = Dense(vocab_size, activation='softmax', name='output')(x)
inputs = [image_input, text_input]
model = Model(inputs=inputs, outputs=output)
model_weights_path = os.path.join('models', best_model)
#加载权值
model.load_weights(model_weights_path)
#Adam 优化器
adam = keras.optimizers.Adam(lr={{loguniform(log(1e-6), log(1e-3))}})
#将优化器传递给 model.compile()，损失函数为多分类的对数损失函数
model.compile(loss='categorical_crossentropy',
metrics=['accuracy'],
optimizer=adam)
#逐个生成数据的batch并进行训练
model.fit_generator(DataGenSequence('train'),
steps_per_epoch=num_train_samples / batch_size // 10,
validation_data=DataGenSequence('valid'),
validation_steps=num_valid_samples / batch_size // 10)
score, acc = model.evaluate_generator(DataGenSequence('valid'),
verbose=0) #使用一个生成器作为数据源，来评估模型
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(train_generator, validation_generator):
l2_reg = regularizers.l2({{loguniform(log(1e-6), log(1e-2))}})
base_model = InceptionResNetV2(weights='imagenet', include_top=False)
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dropout({{uniform(0, 1)}})(x)
x = Dense(1024,
activation='relu',
kernel_regularizer=l2_reg,
activity_regularizer=l2_reg)(x)
x = Dropout({{uniform(0, 1)}})(x)
predictions = Dense(num_classes,
activation='softmax',
kernel_regularizer=l2_reg,
activity_regularizer=l2_reg)(x)
model = Model(inputs=base_model.input, outputs=predictions)
model_weights_path = os.path.join('models', best_model)
model.load_weights(model_weights_path)
for i in range(int(len(base_model.layers) * {{uniform(0, 1)}})):
layer = base_model.layers[i]
layer.trainable = False
adam = keras.optimizers.Adam(lr={{loguniform(log(1e-6), log(1e-3))}})
model.compile(loss='categorical_crossentropy',
metrics=['accuracy'],
optimizer=adam)
# print(model.summary())
model.fit_generator(train_generator,
steps_per_epoch=num_train_samples // batch_size,
validation_data=validation_generator,
validation_steps=num_valid_samples // batch_size)
score, acc = model.evaluate_generator(validation_generator)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(X_train, y_train, X_test, y_test, num_classes):
model = Sequential()
model.add(
Convolution2D(32,
3,
3,
border_mode='same',
input_shape=X_train.shape[1:]))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout({{uniform(0.2, 0.8)}}))
model.add(Convolution2D(32, 3, 3, border_mode='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout({{uniform(0.2, 0.8)}}))
model.add(Convolution2D(32, 3, 3, border_mode='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout({{uniform(0.2, 0.8)}}))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0.2, 0.8)}}))
model.add(Dense(num_classes, activation='softmax'))
# compile model
adam = Adam(lr={{loguniform(-10, -4)}})
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
epochs=20,
batch_size={{choice([50, 100, 200, 300, 400])}},
verbose=2)
scores, acc = model.evaluate(X_test, y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model():
# word embedding
text_input = Input(shape=(max_token_length, ), dtype='int32')
x = Embedding(input_dim=vocab_size, output_dim=embedding_size)(text_input)
x = CuDNNLSTM(256, return_sequences=True)(x)
text_embedding = TimeDistributed(Dense(embedding_size))(x)
# image embedding
image_input = Input(shape=(2048, ))
x = Dense(embedding_size, activation='relu',
name='image_embedding')(image_input)
# the image I is only input once
image_embedding = RepeatVector(1)(x)
# language model
x = [image_embedding, text_embedding]
x = Concatenate(axis=1)(x)
x = Dropout({{uniform(0, 1)}})(x)
x = CuDNNLSTM(1024, return_sequences=True, name='language_lstm_1')(x)
x = Dropout({{uniform(0, 1)}})(x)
x = CuDNNLSTM(1024, name='language_lstm_2')(x)
x = Dropout({{uniform(0, 1)}})(x)
output = Dense(vocab_size, activation='softmax', name='output')(x)
inputs = [image_input, text_input]
model = Model(inputs=inputs, outputs=output)
model_weights_path = os.path.join('models', best_model)
model.load_weights(model_weights_path)
adam = keras.optimizers.Adam(lr={{loguniform(log(1e-6), log(1e-3))}})
model.compile(loss='categorical_crossentropy',
metrics=['accuracy'],
optimizer=adam)
model.fit_generator(DataGenSequence('train'),
steps_per_epoch=num_train_samples / batch_size // 10,
validation_data=DataGenSequence('valid'),
validation_steps=num_valid_samples / batch_size // 10)
score, acc = model.evaluate_generator(DataGenSequence('valid'), verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(x_train, y_train, x_val, y_val, x_test, y_test):
if sys.argv[1] == 'german':
input_n = 24
elif sys.argv[1] == 'australian':
input_n = 15
batch_size = 32
epochs = 500
inits = [
'Zeros', 'Ones', 'RandomNormal', 'RandomUniform', 'TruncatedNormal',
'Orthogonal', 'lecun_uniform', 'lecun_normal', 'he_uniform',
'he_normal', 'glorot_uniform', 'glorot_normal'
]
acts = [
'tanh', 'softsign', 'sigmoid', 'hard_sigmoid', 'relu', 'softplus',
'LeakyReLU', 'PReLU', 'elu', 'selu'
]
init = inits[int({{quniform(0, 11, 1)}})]
act = acts[9]
neurons = int({{quniform(9, 180, 9)}})
layers = {{choice([1, 2, 4, 8])}}
norm = {{choice(['no', 'l1', 'l2'])}}
dropout = {{choice([0, 1])}}
earlystop = {{choice([0, 1])}}
k1 = None
k2 = None
p = None
if norm == 'no':
reg = None
elif norm == 'l1':
k1 = {{loguniform(-9.2, -2.3)}}
reg = regularizers.l1(k1)
elif norm == 'l2':
k2 = {{loguniform(-9.2, -2.3)}}
reg = regularizers.l2(k2)
X_input = Input((input_n, ))
X = X_input
for _ in range(layers):
X = Dense(
neurons,
kernel_initializer=init,
kernel_regularizer=reg,
)(X)
if act == 'LeakyReLU':
X = LeakyReLU()(X)
elif act == 'PReLU':
X = PReLU()(X)
else:
X = Activation(act)(X)
if dropout == 1:
p = {{uniform(0, 1)}}
X = Dropout(p)(X)
X = Dense(1, kernel_initializer=init, kernel_regularizer=reg)(X)
X_outputs = Activation('sigmoid')(X)
model = Model(inputs=X_input, outputs=X_outputs)
model.compile(
loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'],
)
patience = int({{quniform(1, 500, 1)}})
es = EarlyStopping(
monitor='val_loss',
patience=patience,
verbose=0,
mode='auto',
)
if earlystop == 1:
model.fit(
x_train,
y_train,
batch_size=batch_size,
verbose=0,
epochs=epochs,
validation_data=(x_val, y_val),
callbacks=[es],
)
else:
model.fit(
x_train,
y_train,
batch_size=batch_size,
verbose=0,
epochs=epochs,
validation_data=(x_val, y_val),
)
loss_t, score_t = model.evaluate(x_train, y_train, verbose=0)
loss_v, score_v = model.evaluate(x_val, y_val, verbose=0)
loss_te, score_te = model.evaluate(x_test, y_test, verbose=0)
print(init + '\t' + act + '\t' + str(neurons) + '\t' + str(layers) + '\t' +
str(norm) + '\t' + str(dropout) + '\t' + str(earlystop) +
'%-24s%-24s%-24s%s' % (str(k1), str(k2), str(p), str(patience)) +
' ' + str(score_v) + ' ' + str(loss_v) + ' ' + str(score_te) +
' ' + str(loss_te))
return {'loss': loss_v, 'status': STATUS_OK, 'model': model}
def Conv2DMultiTaskIn1(x_train, y_train, ddg_train, x_test, y_test, ddg_test,
class_weights_dict, obj):
K.clear_session()
summary = False
verbose = 0
# setHyperParams------------------------------------------------------------------------------------------------
batch_size = 64
epochs = {{choice([50, 100, 150, 200, 250])}}
lr = {{loguniform(np.log(1e-4), np.log(1e-2))}}
optimizer = {{choice(['adam', 'sgd', 'rmsprop'])}}
activator = {{choice(['elu', 'relu', 'tanh'])}}
basic_conv2D_layers = {{choice([1, 2])}}
basic_conv2D_filter_num = {{choice([16, 32])}}
loop_dilation2D_layers = {{choice([2, 4, 6])}}
loop_dilation2D_filter_num = {{choice([16, 32, 64])}} #used in the loop
loop_dilation2D_dropout_rate = {{uniform(0.001, 0.35)}}
dilation_lower = 2
dilation_upper = 16
ddg_reduce_layers = {{choice([3, 4, 5])}
} # conv 5 times: 120 => 60 => 30 => 15 => 8 => 4
y_reduce_layers = {{choice([3, 4, 5])}
} # conv 5 times: 120 => 60 => 30 => 15 => 8 => 4
ddg_reduce_conv2D_filter_num = {{choice([16, 32,
64])}} #used for reduce dimention
y_reduce_conv2D_filter_num = {{choice([8, 16,
32])}} #used for reduce dimention
reduce_conv2D_dropout_rate = {{uniform(0.001, 0.25)}}
ddg_residual_stride = 2
y_residual_stride = 2
ddg_dense1_num = {{choice([64, 128, 256])}}
ddg_dense2_num = {{choice([32, 64])}}
y_dense1_num = {{choice([32, 64, 128])}}
y_dense2_num = {{choice([16, 32])}}
drop_num = {{uniform(0.0001, 0.3)}}
kernel_size = (3, 3)
pool_size = (2, 2)
initializer = 'random_uniform'
padding_style = 'same'
loss_type = ['mse', 'binary_crossentropy']
loss_weights = [0.5, 10]
metrics = (['mae'], ['accuracy'])
my_callbacks = [
callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.8,
patience=10,
)
]
if lr > 0:
if optimizer == 'adam':
chosed_optimizer = optimizers.Adam(lr=lr)
elif optimizer == 'sgd':
chosed_optimizer = optimizers.SGD(lr=lr)
elif optimizer == 'rmsprop':
chosed_optimizer = optimizers.RMSprop(lr=lr)
# build --------------------------------------------------------------------------------------------------------
## basic Conv2D
input_layer = Input(shape=x_train.shape[1:])
y = layers.Conv2D(basic_conv2D_filter_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(input_layer)
y = layers.BatchNormalization(axis=-1)(y)
if basic_conv2D_layers == 2:
y = layers.Conv2D(basic_conv2D_filter_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
## loop with Conv2D with dilation (padding='same')
for _ in range(loop_dilation2D_layers):
y = layers.Conv2D(loop_dilation2D_filter_num,
kernel_size,
padding=padding_style,
dilation_rate=dilation_lower,
kernel_initializer=initializer,
activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(loop_dilation2D_dropout_rate)(y)
dilation_lower *= 2
if dilation_lower > dilation_upper:
dilation_lower = 2
## Conv2D with dilation (padding='valaid') and residual block to reduce dimention.
## for regressor branch
y_ddg = layers.Conv2D(ddg_reduce_conv2D_filter_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(y)
y_ddg = layers.BatchNormalization(axis=-1)(y_ddg)
y_ddg = layers.Dropout(reduce_conv2D_dropout_rate)(y_ddg)
y_ddg = layers.MaxPooling2D(pool_size, padding=padding_style)(y_ddg)
residual_ddg = layers.Conv2D(ddg_reduce_conv2D_filter_num,
1,
strides=ddg_residual_stride,
padding='same')(input_layer)
y_ddg = layers.add([y_ddg, residual_ddg])
ddg_residual_stride *= 2
for _ in range(ddg_reduce_layers - 1):
y_ddg = layers.Conv2D(ddg_reduce_conv2D_filter_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(y_ddg)
y_ddg = layers.BatchNormalization(axis=-1)(y_ddg)
y_ddg = layers.Dropout(reduce_conv2D_dropout_rate)(y_ddg)
y_ddg = layers.MaxPooling2D(pool_size, padding=padding_style)(y_ddg)
residual_ddg = layers.Conv2D(ddg_reduce_conv2D_filter_num,
1,
strides=ddg_residual_stride,
padding='same')(input_layer)
y_ddg = layers.add([y_ddg, residual_ddg])
ddg_residual_stride *= 2
## flat & dense
y_ddg = layers.Flatten()(y_ddg)
y_ddg = layers.Dense(ddg_dense1_num, activation=activator)(y_ddg)
y_ddg = layers.BatchNormalization(axis=-1)(y_ddg)
y_ddg = layers.Dropout(drop_num)(y_ddg)
y_ddg = layers.Dense(ddg_dense2_num, activation=activator)(y_ddg)
y_ddg = layers.BatchNormalization(axis=-1)(y_ddg)
y_ddg = layers.Dropout(drop_num)(y_ddg)
ddg_prediction = layers.Dense(1, name='ddg')(y_ddg)
# class_prediction = layers.Dense(len(np.unique(y_train)), activation='softmax', name='class')(y_ddg)
## for classifier branch
y_y = layers.Conv2D(y_reduce_conv2D_filter_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(y)
y_y = layers.BatchNormalization(axis=-1)(y_y)
y_y = layers.Dropout(reduce_conv2D_dropout_rate)(y_y)
y_y = layers.MaxPooling2D(pool_size, padding=padding_style)(y_y)
residual_y = layers.Conv2D(y_reduce_conv2D_filter_num,
1,
strides=y_residual_stride,
padding='same')(input_layer)
y_y = layers.add([y_y, residual_y])
y_residual_stride *= 2
for _ in range(y_reduce_layers - 1):
y_y = layers.Conv2D(y_reduce_conv2D_filter_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(y_y)
y_y = layers.BatchNormalization(axis=-1)(y_y)
y_y = layers.Dropout(reduce_conv2D_dropout_rate)(y_y)
y_y = layers.MaxPooling2D(pool_size, padding=padding_style)(y_y)
residual_y = layers.Conv2D(y_reduce_conv2D_filter_num,
1,
strides=y_residual_stride,
padding='same')(input_layer)
y_y = layers.add([y_y, residual_y])
y_residual_stride *= 2
## flat & dense
y_y = layers.Flatten()(y_y)
y_y = layers.Dense(y_dense1_num, activation=activator)(y_y)
y_y = layers.BatchNormalization(axis=-1)(y_y)
y_y = layers.Dropout(drop_num)(y_y)
y_y = layers.Dense(y_dense2_num, activation=activator)(y_y)
y_y = layers.BatchNormalization(axis=-1)(y_y)
y_y = layers.Dropout(drop_num)(y_y)
class_prediction = layers.Dense(len(np.unique(y_train)),
activation='softmax',
name='class')(y_y)
model = models.Model(inputs=input_layer,
outputs=[ddg_prediction, class_prediction])
if summary:
model.summary()
model.compile(optimizer=chosed_optimizer,
loss={
'ddg': loss_type[0],
'class': loss_type[1]
},
loss_weights={
'ddg': loss_weights[0],
'class': loss_weights[1]
},
metrics={
'ddg': metrics[0],
'class': metrics[1]
})
# K.set_session(tf.Session(graph=model.output.graph))
# init = K.tf.global_variables_initializer()
# K.get_session().run(init)
result = model.fit(
x=x_train,
y={
'ddg': ddg_train,
'class': y_train
},
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
callbacks=my_callbacks,
validation_data=(x_test, {
'ddg': ddg_test,
'class': y_test
}),
shuffle=True,
class_weight={
'ddg': None,
'class': class_weights_dict
},
)
# print('\n----------History:\n%s'%result.history)
if obj == 'test_report':
pearson_coeff, std, acc, mcc, recall_p, recall_n, precision_p, precision_n = test_report(
model, x_test, y_test, ddg_test)
print(
'\n----------Predict:'
'\npearson_coeff: %s, std: %s'
'\nacc: %s, mcc: %s, recall_p: %s, recall_n: %s, precision_p: %s, precision_n: %s'
% (pearson_coeff, std, acc, mcc, recall_p, recall_n, precision_p,
precision_n))
objective = pearson_coeff * 2 + std + acc + 5 * mcc + recall_p + recall_n + precision_p + precision_n
return {'loss': -objective, 'status': STATUS_OK}
elif obj == 'val':
validation_mae = np.amax(result.history['val_ddg_mean_absolute_error'])
validation_acc = np.amax(result.history['val_class_acc'])
print('Best validation of epoch, mae: %s, acc: %s:' %
(validation_mae, validation_acc))
return {
'loss': validation_mae - 2 * validation_acc,
'status': STATUS_OK
}
def Conv2DClassifierIn1(x_train, y_train, x_test, y_test, class_weights_dict,
obj):
K.clear_session()
summary = True
verbose = 0
# setHyperParams------------------------------------------------------------------------------------------------
batch_size = 64
epochs = {{choice([50, 100, 150, 200, 250])}}
lr = {{loguniform(np.log(1e-4), np.log(1e-2))}}
optimizer = {{choice(['adam', 'sgd', 'rmsprop'])}}
activator = {{choice(['elu', 'relu', 'tanh'])}}
basic_conv2D_layers = {{choice([1, 2])}}
basic_conv2D_filter_num = {{choice([16, 32])}}
loop_dilation2D_layers = {{choice([2, 4, 6])}}
loop_dilation2D_filter_num = {{choice([16, 32, 64])}} #used in the loop
loop_dilation2D_dropout_rate = {{uniform(0.001, 0.35)}}
dilation_lower = 2
dilation_upper = 16
reduce_layers = 5 # conv 3 times: 120 => 60 => 30 => 15
reduce_conv2D_filter_num = {{choice([8, 16,
32])}} #used for reduce dimention
reduce_conv2D_dropout_rate = {{uniform(0.001, 0.25)}}
residual_stride = 2
dense1_num = {{choice([64, 128, 256])}}
dense2_num = {{choice([32, 64])}}
drop_num = {{uniform(0.0001, 0.3)}}
kernel_size = (3, 3)
pool_size = (2, 2)
initializer = 'random_uniform'
padding_style = 'same'
loss_type = 'binary_crossentropy'
metrics = ('accuracy', )
my_callbacks = [
callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.8,
patience=10,
)
]
if lr > 0:
if optimizer == 'adam':
chosed_optimizer = optimizers.Adam(lr=lr)
elif optimizer == 'sgd':
chosed_optimizer = optimizers.SGD(lr=lr)
elif optimizer == 'rmsprop':
chosed_optimizer = optimizers.RMSprop(lr=lr)
# build --------------------------------------------------------------------------------------------------------
## basic Conv2D
input_layer = Input(shape=x_train.shape[1:])
y = layers.Conv2D(basic_conv2D_filter_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(input_layer)
y = layers.BatchNormalization(axis=-1)(y)
if basic_conv2D_layers == 2:
y = layers.Conv2D(basic_conv2D_filter_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
## loop with Conv2D with dilation (padding='same')
for _ in range(loop_dilation2D_layers):
y = layers.Conv2D(loop_dilation2D_filter_num,
kernel_size,
padding=padding_style,
dilation_rate=dilation_lower,
kernel_initializer=initializer,
activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(loop_dilation2D_dropout_rate)(y)
dilation_lower *= 2
if dilation_lower > dilation_upper:
dilation_lower = 2
## Conv2D with dilation (padding='valaid') and residual block to reduce dimention.
for _ in range(reduce_layers):
y = layers.Conv2D(reduce_conv2D_filter_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(reduce_conv2D_dropout_rate)(y)
y = layers.MaxPooling2D(pool_size, padding=padding_style)(y)
residual = layers.Conv2D(reduce_conv2D_filter_num,
1,
strides=residual_stride,
padding='same')(input_layer)
y = layers.add([y, residual])
residual_stride *= 2
## flat & dense
y = layers.Flatten()(y)
y = layers.Dense(dense1_num, activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(drop_num)(y)
y = layers.Dense(dense2_num, activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(drop_num)(y)
output_layer = layers.Dense(len(np.unique(y_train)),
activation='softmax')(y)
model = models.Model(inputs=input_layer, outputs=output_layer)
if summary:
model.summary()
model.compile(
optimizer=chosed_optimizer,
loss=loss_type,
metrics=list(metrics) # accuracy
)
K.set_session(tf.Session(graph=model.output.graph))
init = K.tf.global_variables_initializer()
K.get_session().run(init)
result = model.fit(x=x_train,
y=y_train,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
callbacks=my_callbacks,
validation_data=(x_test, y_test),
shuffle=True,
class_weight=class_weights_dict)
# print('\n----------History:\n%s'%result.history)
if obj == 'test_report_cla':
acc_test, mcc_test, recall_p_test, recall_n_test, precision_p_test, precision_n_test = test_report_cla(
model, x_test, y_test)
print(
'\n----------Predict:\nacc_test: %s, mcc_test: %s, recall_p_test: %s, recall_n_test: %s, precision_p_test: %s, precision_n_test: %s'
% (acc_test, mcc_test, recall_p_test, recall_n_test,
precision_p_test, precision_n_test))
objective = acc_test + 5 * mcc_test + recall_p_test + recall_n_test + precision_p_test + precision_n_test
return {'loss': -objective, 'status': STATUS_OK}
elif obj == 'val_acc':
validation_acc = np.amax(result.history['val_acc'])
print('Best validation acc of epoch:', validation_acc)
return {'loss': -validation_acc, 'status': STATUS_OK}
def model(train_x, train_y, dev_x, dev_y, test_x, test_y, overal_maxlen, qwks):
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, GlobalAveragePooling1D
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import LSTM
from keras.initializers import Constant
from keras import optimizers
import keras.backend as K
from deepats.my_layers import MeanOverTime
from deepats.rwa import RWA
import pickle as pk
import numpy as np
import string
import random
import os
from deepats.optimizers import get_optimizer
from deepats.ets_evaluator import Evaluator
import deepats.ets_reader as dataset
from deepats.ets_config import get_args
import GPUtil
def random_id(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def kappa_metric(t, x):
u = 0.5 * K.sum(K.square(x - t))
v = K.dot(K.transpose(x), t - K.mean(t))
return v / (v + u)
def kappa_loss(t, x):
u = K.sum(K.square(x - t))
v = K.dot(K.squeeze(x, 1), K.squeeze(t - K.mean(t), 1))
return u / (2 * v + u)
import time
ms = int(round(time.time() * 1000))
rand_seed = ms % (2**32 - 1)
random.seed(rand_seed)
args = get_args()
model_id = random_id()
abs_vocab_file = os.path.join(args.abs_out, 'vocab.pkl')
with open(abs_vocab_file, 'rb') as vocab_file:
vocab = pk.load(vocab_file)
vocab_size = len(vocab)
acts = ['tanh', 'relu', 'hard_sigmoid']
emb_dim = {{choice([50, 100, 200, 300])}}
rnn_dim = {{uniform(50, 500)}}
rnn_dim = int(rnn_dim)
rec_act = {{choice([0, 1, 2])}}
rec_act = acts[rec_act]
dropout = {{uniform(0.2, 0.95)}}
epochs = args.epochs
n_emb = vocab_size * emb_dim
n_rwa = (903 + 2 * rnn_dim) * rnn_dim
n_tot = n_emb + n_rwa + rnn_dim + 1
lr = {{lognormal(-3 * 2.3, .8)}}
lr = 1.5 * lr
rho = {{normal(.875, .04)}}
clipnorm = {{uniform(1, 15)}}
eps = {{loguniform(-8 * 2.3, -5 * 2.3)}}
opt = optimizers.RMSprop(lr=lr, rho=rho, clipnorm=clipnorm, epsilon=eps)
loss = kappa_loss
metric = kappa_metric
evl = Evaluator(dataset,
args.prompt_id,
args.abs_out,
dev_x,
test_x,
dev_df,
test_df,
model_id=model_id)
train_y_mean = train_y.mean(axis=0)
if train_y_mean.ndim == 0:
train_y_mean = np.expand_dims(train_y_mean, axis=1)
num_outputs = len(train_y_mean)
mask_zero = False
model = Sequential()
model.add(Embedding(vocab_size, emb_dim, mask_zero=mask_zero))
model.add(RWA(rnn_dim, recurrent_activation=rec_act))
model.add(Dropout(dropout))
bias_value = (np.log(train_y_mean) - np.log(1 - train_y_mean)).astype(
K.floatx())
model.add(Dense(num_outputs, bias_initializer=Constant(value=bias_value)))
model.add(Activation('tanh'))
model.emb_index = 0
from deepats.w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(args.emb_path, emb_dim)
emb_reader.load_embeddings(vocab)
emb_wts = emb_reader.get_emb_matrix_given_vocab(
vocab, model.layers[model.emb_index].get_weights()[0])
wts = model.layers[model.emb_index].get_weights()
wts[0] = emb_wts
model.layers[model.emb_index].set_weights(wts)
model.compile(loss=loss, optimizer=opt, metrics=[metric])
model_yaml = model.to_yaml()
import GPUtil
if GPUtil.avail_mem() < 0.1:
return {'loss': 1, 'status': STATUS_OK, 'model': '', 'weights': None}
print('model_id: %s' % (model_id))
print(model_yaml)
print('PARAMS\t\
%s\t\
lr= %.4f\t\
rho= %.4f\t\
clip= %.4f\t\
eps= %.4f\t\
embDim= %.4f\t\
rnnDim= %.4f\t\
drop= %.4f\t\
recAct= %s' % (model_id, lr, rho, clipnorm, np.log(eps) / 2.3, emb_dim,
rnn_dim, dropout, rec_act))
for i in range(epochs):
train_history = model.fit(train_x,
train_y,
batch_size=args.batch_size,
epochs=1,
verbose=0)
evl.evaluate(model, i)
evl.output_info()
p = evl.stats[3] / qwks[0]
if i > 10 and p < 0.9:
break
i = evl.comp_idx
j = i + 2
best_dev_kappa = evl.best_dev[i]
best_test_kappa = evl.best_dev[j]
print('Test kappa:', best_dev_kappa)
return {
'loss': 1 - best_dev_kappa,
'status': STATUS_OK,
'model': model.to_yaml(),
'weights': pk.dumps(model.get_weights())
}
def create_model(x_train, y_train, x_val, y_val):
batch_size = 64
epochs = 500
init = 'lecun_normal'
act = 'tanh'
neurons = int({{quniform(9, 180, 9)}})
layers = {{choice([1, 2, 4, 8])}}
norm = {{choice(['no', 'l1', 'l2'])}}
dropout = {{choice([0, 1])}}
earlystop = {{choice([0, 1])}}
k = None
p = None
patience = None
if norm == 'no':
reg = None
elif norm == 'l1':
k = {{loguniform(-9.2, -2.3)}}
reg = regularizers.l1(k)
elif norm == 'l2':
k = {{loguniform(-9.2, -2.3)}}
reg = regularizers.l2(k)
X_input = Input((24, ))
X = Reshape((-1, ))(X_input)
for _ in range(layers):
X = Dense(neurons, kernel_initializer=init, kernel_regularizer=reg)(X)
X = Activation(act)(X)
if dropout == 1:
p = {{uniform(0, 1)}}
X = Dropout(p)(X)
X = Dense(1, kernel_initializer=init, kernel_regularizer=reg)(X)
X_outputs = Activation('sigmoid')(X)
model = Model(inputs=X_input, outputs=X_outputs)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
if earlystop == 0:
model.fit(x_train,
y_train,
batch_size=batch_size,
verbose=0,
epochs=epochs,
validation_data=(x_val, y_val))
elif earlystop == 1:
patience = int({{quniform(1, 500, 1)}})
es = EarlyStopping(monitor='val_loss',
patience=patience,
verbose=0,
mode='auto')
model.fit(x_train,
y_train,
batch_size=batch_size,
verbose=0,
epochs=epochs,
validation_data=(x_val, y_val),
callbacks=[es])
loss_t, score_t = model.evaluate(x_train, y_train, verbose=0)
loss_v, score_v = model.evaluate(x_val, y_val, verbose=0)
print(
str(neurons) + '\t' + str(layers) + '\t' + str(norm) + '\t' +
str(dropout) + '\t' + str(earlystop) + '\t\t' + '%-24s%-24s%s' %
(str(k), str(p), str(patience)))
return {'loss': loss_v, 'status': STATUS_OK, 'model': model}
def model(train_data, test_data):
initializer = keras.initializers.RandomUniform(minval=0.0001, maxval=0.01)
activation = 'relu'
num_layers = {{quniform(1, 5, 1)}}
layer_size_five_hundreds = {{quniform(1, 10, 1)}} * 500
# # Single autoencoder
input = Input(shape=(len(train_data[0]), ))
output = input
for i in range(int(num_layers)):
output = Dense(int(layer_size_five_hundreds),
activation=activation)(output)
encoded_activation = {{choice(['relu', 'linear'])}}
output = Dense(
2,
activation=encoded_activation,
name="encoded",
kernel_initializer=initializer,
bias_initializer='zeros'
)(output) #maybe just try avoiding dead neurons in small encoding layer?
for i in range(int(num_layers)):
output = Dense(int(layer_size_five_hundreds),
activation=activation)(output)
output_activation = {{choice(['linear', 'sigmoid'])}}
output = Dense(len(train_data[0]), activation=output_activation)(
output
) #'linear',)(output) # First test seems to indicate no change on output with linear
autoencoder = Model(input, output)
learning_rate = {{loguniform(math.log(0.1), math.log(0.0001))}}
optimizer = keras.optimizers.Adam(lr=learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0)
autoencoder.compile(
optimizer=optimizer, #adamax and nadam worked well too
loss='mean_squared_error',
metrics=['mse'])
# from keras.callbacks import TensorBoard
# tensorboard = TensorBoard(log_dir="logs/{}".format(datetime.datetime.now().strftime("%I%M%p%B%d%Y")), write_graph=False)#,histogram_freq=2,)#, histogram_freq=1, write_graph=False)
early_stop = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='min')
batch_size_power_of_2 = 2**int({{quniform(6, 11, 1)}})
print('num_layers', num_layers)
print('layer_size_five_hundreds', layer_size_five_hundreds)
print('encoded_activation', encoded_activation)
print('output_activation', output_activation)
print('learning_rate', learning_rate)
print('batch_size_power_of_2', batch_size_power_of_2)
autoencoder.fit(train_data,
train_data,
epochs=10,
batch_size=int(batch_size_power_of_2),
shuffle=True,
callbacks=[early_stop],
validation_data=(test_data, test_data))
loss, mse = autoencoder.evaluate(test_data, test_data, verbose=0)
print('Test accuracy:', mse)
K.clear_session()
return {'loss': mse, 'status': STATUS_OK, 'model': autoencoder}
