def create_model(): # word embedding text_input = Input(shape=(max_token_length, ), dtype='int32') #输入行数为max_token_length的矩阵,数据类型是32位整数 x = Embedding(input_dim=vocab_size, output_dim=embedding_size)(text_input) #embedding层 x = LSTM(256, return_sequences=True)(x) #输入LSTM模型,256个神经元 text_embedding = TimeDistributed(Dense(embedding_size))( x) #使用包装器TimeDistributed包装Dense,以产生针对各个时间步信号的独立全连接 # image embedding image_input = Input(shape=(2048, )) #输入行数为2048的图形数据矩阵 #全连接层dense,激活函数:relu x = Dense(embedding_size, activation='relu', name='image_embedding')(image_input) # the image I is only input once RepeatVector层将输入只重复1次 image_embedding = RepeatVector(1)(x) # language model x = [image_embedding, text_embedding] #按列合并 x = Concatenate(axis=1)(x) #以一定概率丢弃神经元,uniform随机生成0-1内一个实数 x = Dropout({{uniform(0, 1)}})(x) #输入LSTM模型,1024个神经元 x = LSTM(1024, return_sequences=True, name='language_lstm_1')(x) x = Dropout({{uniform(0, 1)}})(x) x = LSTM(1024, name='language_lstm_2')(x) x = Dropout({{uniform(0, 1)}})(x) # 全连接层dense,激活函数:softmax output = Dense(vocab_size, activation='softmax', name='output')(x) inputs = [image_input, text_input] model = Model(inputs=inputs, outputs=output) model_weights_path = os.path.join('models', best_model) #加载权值 model.load_weights(model_weights_path) #Adam 优化器 adam = keras.optimizers.Adam(lr={{loguniform(log(1e-6), log(1e-3))}}) #将优化器传递给 model.compile(),损失函数为多分类的对数损失函数 model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=adam) #逐个生成数据的batch并进行训练 model.fit_generator(DataGenSequence('train'), steps_per_epoch=num_train_samples / batch_size // 10, validation_data=DataGenSequence('valid'), validation_steps=num_valid_samples / batch_size // 10) score, acc = model.evaluate_generator(DataGenSequence('valid'), verbose=0) #使用一个生成器作为数据源,来评估模型 print('Test accuracy:', acc) return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(train_generator, validation_generator): l2_reg = regularizers.l2({{loguniform(log(1e-6), log(1e-2))}}) base_model = InceptionResNetV2(weights='imagenet', include_top=False) x = base_model.output x = GlobalAveragePooling2D()(x) x = Dropout({{uniform(0, 1)}})(x) x = Dense(1024, activation='relu', kernel_regularizer=l2_reg, activity_regularizer=l2_reg)(x) x = Dropout({{uniform(0, 1)}})(x) predictions = Dense(num_classes, activation='softmax', kernel_regularizer=l2_reg, activity_regularizer=l2_reg)(x) model = Model(inputs=base_model.input, outputs=predictions) model_weights_path = os.path.join('models', best_model) model.load_weights(model_weights_path) for i in range(int(len(base_model.layers) * {{uniform(0, 1)}})): layer = base_model.layers[i] layer.trainable = False adam = keras.optimizers.Adam(lr={{loguniform(log(1e-6), log(1e-3))}}) model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=adam) # print(model.summary()) model.fit_generator(train_generator, steps_per_epoch=num_train_samples // batch_size, validation_data=validation_generator, validation_steps=num_valid_samples // batch_size) score, acc = model.evaluate_generator(validation_generator) print('Test accuracy:', acc) return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def model(X_train, y_train, X_test, y_test, num_classes): model = Sequential() model.add( Convolution2D(32, 3, 3, border_mode='same', input_shape=X_train.shape[1:])) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Dropout({{uniform(0.2, 0.8)}})) model.add(Convolution2D(32, 3, 3, border_mode='same')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Dropout({{uniform(0.2, 0.8)}})) model.add(Convolution2D(32, 3, 3, border_mode='same')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Dropout({{uniform(0.2, 0.8)}})) model.add(Flatten()) model.add(Dense(128)) model.add(Activation('relu')) model.add(Dropout({{uniform(0.2, 0.8)}})) model.add(Dense(num_classes, activation='softmax')) # compile model adam = Adam(lr={{loguniform(-10, -4)}}) model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=20, batch_size={{choice([50, 100, 200, 300, 400])}}, verbose=2) scores, acc = model.evaluate(X_test, y_test, verbose=0) print('Test accuracy:', acc) return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(): # word embedding text_input = Input(shape=(max_token_length, ), dtype='int32') x = Embedding(input_dim=vocab_size, output_dim=embedding_size)(text_input) x = CuDNNLSTM(256, return_sequences=True)(x) text_embedding = TimeDistributed(Dense(embedding_size))(x) # image embedding image_input = Input(shape=(2048, )) x = Dense(embedding_size, activation='relu', name='image_embedding')(image_input) # the image I is only input once image_embedding = RepeatVector(1)(x) # language model x = [image_embedding, text_embedding] x = Concatenate(axis=1)(x) x = Dropout({{uniform(0, 1)}})(x) x = CuDNNLSTM(1024, return_sequences=True, name='language_lstm_1')(x) x = Dropout({{uniform(0, 1)}})(x) x = CuDNNLSTM(1024, name='language_lstm_2')(x) x = Dropout({{uniform(0, 1)}})(x) output = Dense(vocab_size, activation='softmax', name='output')(x) inputs = [image_input, text_input] model = Model(inputs=inputs, outputs=output) model_weights_path = os.path.join('models', best_model) model.load_weights(model_weights_path) adam = keras.optimizers.Adam(lr={{loguniform(log(1e-6), log(1e-3))}}) model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=adam) model.fit_generator(DataGenSequence('train'), steps_per_epoch=num_train_samples / batch_size // 10, validation_data=DataGenSequence('valid'), validation_steps=num_valid_samples / batch_size // 10) score, acc = model.evaluate_generator(DataGenSequence('valid'), verbose=0) print('Test accuracy:', acc) return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def create_model(x_train, y_train, x_val, y_val, x_test, y_test): if sys.argv[1] == 'german': input_n = 24 elif sys.argv[1] == 'australian': input_n = 15 batch_size = 32 epochs = 500 inits = [ 'Zeros', 'Ones', 'RandomNormal', 'RandomUniform', 'TruncatedNormal', 'Orthogonal', 'lecun_uniform', 'lecun_normal', 'he_uniform', 'he_normal', 'glorot_uniform', 'glorot_normal' ] acts = [ 'tanh', 'softsign', 'sigmoid', 'hard_sigmoid', 'relu', 'softplus', 'LeakyReLU', 'PReLU', 'elu', 'selu' ] init = inits[int({{quniform(0, 11, 1)}})] act = acts[9] neurons = int({{quniform(9, 180, 9)}}) layers = {{choice([1, 2, 4, 8])}} norm = {{choice(['no', 'l1', 'l2'])}} dropout = {{choice([0, 1])}} earlystop = {{choice([0, 1])}} k1 = None k2 = None p = None if norm == 'no': reg = None elif norm == 'l1': k1 = {{loguniform(-9.2, -2.3)}} reg = regularizers.l1(k1) elif norm == 'l2': k2 = {{loguniform(-9.2, -2.3)}} reg = regularizers.l2(k2) X_input = Input((input_n, )) X = X_input for _ in range(layers): X = Dense( neurons, kernel_initializer=init, kernel_regularizer=reg, )(X) if act == 'LeakyReLU': X = LeakyReLU()(X) elif act == 'PReLU': X = PReLU()(X) else: X = Activation(act)(X) if dropout == 1: p = {{uniform(0, 1)}} X = Dropout(p)(X) X = Dense(1, kernel_initializer=init, kernel_regularizer=reg)(X) X_outputs = Activation('sigmoid')(X) model = Model(inputs=X_input, outputs=X_outputs) model.compile( loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'], ) patience = int({{quniform(1, 500, 1)}}) es = EarlyStopping( monitor='val_loss', patience=patience, verbose=0, mode='auto', ) if earlystop == 1: model.fit( x_train, y_train, batch_size=batch_size, verbose=0, epochs=epochs, validation_data=(x_val, y_val), callbacks=[es], ) else: model.fit( x_train, y_train, batch_size=batch_size, verbose=0, epochs=epochs, validation_data=(x_val, y_val), ) loss_t, score_t = model.evaluate(x_train, y_train, verbose=0) loss_v, score_v = model.evaluate(x_val, y_val, verbose=0) loss_te, score_te = model.evaluate(x_test, y_test, verbose=0) print(init + '\t' + act + '\t' + str(neurons) + '\t' + str(layers) + '\t' + str(norm) + '\t' + str(dropout) + '\t' + str(earlystop) + '%-24s%-24s%-24s%s' % (str(k1), str(k2), str(p), str(patience)) + ' ' + str(score_v) + ' ' + str(loss_v) + ' ' + str(score_te) + ' ' + str(loss_te)) return {'loss': loss_v, 'status': STATUS_OK, 'model': model}
def Conv2DMultiTaskIn1(x_train, y_train, ddg_train, x_test, y_test, ddg_test, class_weights_dict, obj): K.clear_session() summary = False verbose = 0 # setHyperParams------------------------------------------------------------------------------------------------ batch_size = 64 epochs = {{choice([50, 100, 150, 200, 250])}} lr = {{loguniform(np.log(1e-4), np.log(1e-2))}} optimizer = {{choice(['adam', 'sgd', 'rmsprop'])}} activator = {{choice(['elu', 'relu', 'tanh'])}} basic_conv2D_layers = {{choice([1, 2])}} basic_conv2D_filter_num = {{choice([16, 32])}} loop_dilation2D_layers = {{choice([2, 4, 6])}} loop_dilation2D_filter_num = {{choice([16, 32, 64])}} #used in the loop loop_dilation2D_dropout_rate = {{uniform(0.001, 0.35)}} dilation_lower = 2 dilation_upper = 16 ddg_reduce_layers = {{choice([3, 4, 5])} } # conv 5 times: 120 => 60 => 30 => 15 => 8 => 4 y_reduce_layers = {{choice([3, 4, 5])} } # conv 5 times: 120 => 60 => 30 => 15 => 8 => 4 ddg_reduce_conv2D_filter_num = {{choice([16, 32, 64])}} #used for reduce dimention y_reduce_conv2D_filter_num = {{choice([8, 16, 32])}} #used for reduce dimention reduce_conv2D_dropout_rate = {{uniform(0.001, 0.25)}} ddg_residual_stride = 2 y_residual_stride = 2 ddg_dense1_num = {{choice([64, 128, 256])}} ddg_dense2_num = {{choice([32, 64])}} y_dense1_num = {{choice([32, 64, 128])}} y_dense2_num = {{choice([16, 32])}} drop_num = {{uniform(0.0001, 0.3)}} kernel_size = (3, 3) pool_size = (2, 2) initializer = 'random_uniform' padding_style = 'same' loss_type = ['mse', 'binary_crossentropy'] loss_weights = [0.5, 10] metrics = (['mae'], ['accuracy']) my_callbacks = [ callbacks.ReduceLROnPlateau( monitor='val_loss', factor=0.8, patience=10, ) ] if lr > 0: if optimizer == 'adam': chosed_optimizer = optimizers.Adam(lr=lr) elif optimizer == 'sgd': chosed_optimizer = optimizers.SGD(lr=lr) elif optimizer == 'rmsprop': chosed_optimizer = optimizers.RMSprop(lr=lr) # build -------------------------------------------------------------------------------------------------------- ## basic Conv2D input_layer = Input(shape=x_train.shape[1:]) y = layers.Conv2D(basic_conv2D_filter_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(input_layer) y = layers.BatchNormalization(axis=-1)(y) if basic_conv2D_layers == 2: y = layers.Conv2D(basic_conv2D_filter_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(y) y = layers.BatchNormalization(axis=-1)(y) ## loop with Conv2D with dilation (padding='same') for _ in range(loop_dilation2D_layers): y = layers.Conv2D(loop_dilation2D_filter_num, kernel_size, padding=padding_style, dilation_rate=dilation_lower, kernel_initializer=initializer, activation=activator)(y) y = layers.BatchNormalization(axis=-1)(y) y = layers.Dropout(loop_dilation2D_dropout_rate)(y) dilation_lower *= 2 if dilation_lower > dilation_upper: dilation_lower = 2 ## Conv2D with dilation (padding='valaid') and residual block to reduce dimention. ## for regressor branch y_ddg = layers.Conv2D(ddg_reduce_conv2D_filter_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(y) y_ddg = layers.BatchNormalization(axis=-1)(y_ddg) y_ddg = layers.Dropout(reduce_conv2D_dropout_rate)(y_ddg) y_ddg = layers.MaxPooling2D(pool_size, padding=padding_style)(y_ddg) residual_ddg = layers.Conv2D(ddg_reduce_conv2D_filter_num, 1, strides=ddg_residual_stride, padding='same')(input_layer) y_ddg = layers.add([y_ddg, residual_ddg]) ddg_residual_stride *= 2 for _ in range(ddg_reduce_layers - 1): y_ddg = layers.Conv2D(ddg_reduce_conv2D_filter_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(y_ddg) y_ddg = layers.BatchNormalization(axis=-1)(y_ddg) y_ddg = layers.Dropout(reduce_conv2D_dropout_rate)(y_ddg) y_ddg = layers.MaxPooling2D(pool_size, padding=padding_style)(y_ddg) residual_ddg = layers.Conv2D(ddg_reduce_conv2D_filter_num, 1, strides=ddg_residual_stride, padding='same')(input_layer) y_ddg = layers.add([y_ddg, residual_ddg]) ddg_residual_stride *= 2 ## flat & dense y_ddg = layers.Flatten()(y_ddg) y_ddg = layers.Dense(ddg_dense1_num, activation=activator)(y_ddg) y_ddg = layers.BatchNormalization(axis=-1)(y_ddg) y_ddg = layers.Dropout(drop_num)(y_ddg) y_ddg = layers.Dense(ddg_dense2_num, activation=activator)(y_ddg) y_ddg = layers.BatchNormalization(axis=-1)(y_ddg) y_ddg = layers.Dropout(drop_num)(y_ddg) ddg_prediction = layers.Dense(1, name='ddg')(y_ddg) # class_prediction = layers.Dense(len(np.unique(y_train)), activation='softmax', name='class')(y_ddg) ## for classifier branch y_y = layers.Conv2D(y_reduce_conv2D_filter_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(y) y_y = layers.BatchNormalization(axis=-1)(y_y) y_y = layers.Dropout(reduce_conv2D_dropout_rate)(y_y) y_y = layers.MaxPooling2D(pool_size, padding=padding_style)(y_y) residual_y = layers.Conv2D(y_reduce_conv2D_filter_num, 1, strides=y_residual_stride, padding='same')(input_layer) y_y = layers.add([y_y, residual_y]) y_residual_stride *= 2 for _ in range(y_reduce_layers - 1): y_y = layers.Conv2D(y_reduce_conv2D_filter_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(y_y) y_y = layers.BatchNormalization(axis=-1)(y_y) y_y = layers.Dropout(reduce_conv2D_dropout_rate)(y_y) y_y = layers.MaxPooling2D(pool_size, padding=padding_style)(y_y) residual_y = layers.Conv2D(y_reduce_conv2D_filter_num, 1, strides=y_residual_stride, padding='same')(input_layer) y_y = layers.add([y_y, residual_y]) y_residual_stride *= 2 ## flat & dense y_y = layers.Flatten()(y_y) y_y = layers.Dense(y_dense1_num, activation=activator)(y_y) y_y = layers.BatchNormalization(axis=-1)(y_y) y_y = layers.Dropout(drop_num)(y_y) y_y = layers.Dense(y_dense2_num, activation=activator)(y_y) y_y = layers.BatchNormalization(axis=-1)(y_y) y_y = layers.Dropout(drop_num)(y_y) class_prediction = layers.Dense(len(np.unique(y_train)), activation='softmax', name='class')(y_y) model = models.Model(inputs=input_layer, outputs=[ddg_prediction, class_prediction]) if summary: model.summary() model.compile(optimizer=chosed_optimizer, loss={ 'ddg': loss_type[0], 'class': loss_type[1] }, loss_weights={ 'ddg': loss_weights[0], 'class': loss_weights[1] }, metrics={ 'ddg': metrics[0], 'class': metrics[1] }) # K.set_session(tf.Session(graph=model.output.graph)) # init = K.tf.global_variables_initializer() # K.get_session().run(init) result = model.fit( x=x_train, y={ 'ddg': ddg_train, 'class': y_train }, batch_size=batch_size, epochs=epochs, verbose=verbose, callbacks=my_callbacks, validation_data=(x_test, { 'ddg': ddg_test, 'class': y_test }), shuffle=True, class_weight={ 'ddg': None, 'class': class_weights_dict }, ) # print('\n----------History:\n%s'%result.history) if obj == 'test_report': pearson_coeff, std, acc, mcc, recall_p, recall_n, precision_p, precision_n = test_report( model, x_test, y_test, ddg_test) print( '\n----------Predict:' '\npearson_coeff: %s, std: %s' '\nacc: %s, mcc: %s, recall_p: %s, recall_n: %s, precision_p: %s, precision_n: %s' % (pearson_coeff, std, acc, mcc, recall_p, recall_n, precision_p, precision_n)) objective = pearson_coeff * 2 + std + acc + 5 * mcc + recall_p + recall_n + precision_p + precision_n return {'loss': -objective, 'status': STATUS_OK} elif obj == 'val': validation_mae = np.amax(result.history['val_ddg_mean_absolute_error']) validation_acc = np.amax(result.history['val_class_acc']) print('Best validation of epoch, mae: %s, acc: %s:' % (validation_mae, validation_acc)) return { 'loss': validation_mae - 2 * validation_acc, 'status': STATUS_OK }
def Conv2DClassifierIn1(x_train, y_train, x_test, y_test, class_weights_dict, obj): K.clear_session() summary = True verbose = 0 # setHyperParams------------------------------------------------------------------------------------------------ batch_size = 64 epochs = {{choice([50, 100, 150, 200, 250])}} lr = {{loguniform(np.log(1e-4), np.log(1e-2))}} optimizer = {{choice(['adam', 'sgd', 'rmsprop'])}} activator = {{choice(['elu', 'relu', 'tanh'])}} basic_conv2D_layers = {{choice([1, 2])}} basic_conv2D_filter_num = {{choice([16, 32])}} loop_dilation2D_layers = {{choice([2, 4, 6])}} loop_dilation2D_filter_num = {{choice([16, 32, 64])}} #used in the loop loop_dilation2D_dropout_rate = {{uniform(0.001, 0.35)}} dilation_lower = 2 dilation_upper = 16 reduce_layers = 5 # conv 3 times: 120 => 60 => 30 => 15 reduce_conv2D_filter_num = {{choice([8, 16, 32])}} #used for reduce dimention reduce_conv2D_dropout_rate = {{uniform(0.001, 0.25)}} residual_stride = 2 dense1_num = {{choice([64, 128, 256])}} dense2_num = {{choice([32, 64])}} drop_num = {{uniform(0.0001, 0.3)}} kernel_size = (3, 3) pool_size = (2, 2) initializer = 'random_uniform' padding_style = 'same' loss_type = 'binary_crossentropy' metrics = ('accuracy', ) my_callbacks = [ callbacks.ReduceLROnPlateau( monitor='val_loss', factor=0.8, patience=10, ) ] if lr > 0: if optimizer == 'adam': chosed_optimizer = optimizers.Adam(lr=lr) elif optimizer == 'sgd': chosed_optimizer = optimizers.SGD(lr=lr) elif optimizer == 'rmsprop': chosed_optimizer = optimizers.RMSprop(lr=lr) # build -------------------------------------------------------------------------------------------------------- ## basic Conv2D input_layer = Input(shape=x_train.shape[1:]) y = layers.Conv2D(basic_conv2D_filter_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(input_layer) y = layers.BatchNormalization(axis=-1)(y) if basic_conv2D_layers == 2: y = layers.Conv2D(basic_conv2D_filter_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(y) y = layers.BatchNormalization(axis=-1)(y) ## loop with Conv2D with dilation (padding='same') for _ in range(loop_dilation2D_layers): y = layers.Conv2D(loop_dilation2D_filter_num, kernel_size, padding=padding_style, dilation_rate=dilation_lower, kernel_initializer=initializer, activation=activator)(y) y = layers.BatchNormalization(axis=-1)(y) y = layers.Dropout(loop_dilation2D_dropout_rate)(y) dilation_lower *= 2 if dilation_lower > dilation_upper: dilation_lower = 2 ## Conv2D with dilation (padding='valaid') and residual block to reduce dimention. for _ in range(reduce_layers): y = layers.Conv2D(reduce_conv2D_filter_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(y) y = layers.BatchNormalization(axis=-1)(y) y = layers.Dropout(reduce_conv2D_dropout_rate)(y) y = layers.MaxPooling2D(pool_size, padding=padding_style)(y) residual = layers.Conv2D(reduce_conv2D_filter_num, 1, strides=residual_stride, padding='same')(input_layer) y = layers.add([y, residual]) residual_stride *= 2 ## flat & dense y = layers.Flatten()(y) y = layers.Dense(dense1_num, activation=activator)(y) y = layers.BatchNormalization(axis=-1)(y) y = layers.Dropout(drop_num)(y) y = layers.Dense(dense2_num, activation=activator)(y) y = layers.BatchNormalization(axis=-1)(y) y = layers.Dropout(drop_num)(y) output_layer = layers.Dense(len(np.unique(y_train)), activation='softmax')(y) model = models.Model(inputs=input_layer, outputs=output_layer) if summary: model.summary() model.compile( optimizer=chosed_optimizer, loss=loss_type, metrics=list(metrics) # accuracy ) K.set_session(tf.Session(graph=model.output.graph)) init = K.tf.global_variables_initializer() K.get_session().run(init) result = model.fit(x=x_train, y=y_train, batch_size=batch_size, epochs=epochs, verbose=verbose, callbacks=my_callbacks, validation_data=(x_test, y_test), shuffle=True, class_weight=class_weights_dict) # print('\n----------History:\n%s'%result.history) if obj == 'test_report_cla': acc_test, mcc_test, recall_p_test, recall_n_test, precision_p_test, precision_n_test = test_report_cla( model, x_test, y_test) print( '\n----------Predict:\nacc_test: %s, mcc_test: %s, recall_p_test: %s, recall_n_test: %s, precision_p_test: %s, precision_n_test: %s' % (acc_test, mcc_test, recall_p_test, recall_n_test, precision_p_test, precision_n_test)) objective = acc_test + 5 * mcc_test + recall_p_test + recall_n_test + precision_p_test + precision_n_test return {'loss': -objective, 'status': STATUS_OK} elif obj == 'val_acc': validation_acc = np.amax(result.history['val_acc']) print('Best validation acc of epoch:', validation_acc) return {'loss': -validation_acc, 'status': STATUS_OK}
def model(train_x, train_y, dev_x, dev_y, test_x, test_y, overal_maxlen, qwks): from keras.models import Sequential from keras.layers import Dense, Dropout, Activation, GlobalAveragePooling1D from keras.layers.embeddings import Embedding from keras.layers.recurrent import LSTM from keras.initializers import Constant from keras import optimizers import keras.backend as K from deepats.my_layers import MeanOverTime from deepats.rwa import RWA import pickle as pk import numpy as np import string import random import os from deepats.optimizers import get_optimizer from deepats.ets_evaluator import Evaluator import deepats.ets_reader as dataset from deepats.ets_config import get_args import GPUtil def random_id(size=6, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) def kappa_metric(t, x): u = 0.5 * K.sum(K.square(x - t)) v = K.dot(K.transpose(x), t - K.mean(t)) return v / (v + u) def kappa_loss(t, x): u = K.sum(K.square(x - t)) v = K.dot(K.squeeze(x, 1), K.squeeze(t - K.mean(t), 1)) return u / (2 * v + u) import time ms = int(round(time.time() * 1000)) rand_seed = ms % (2**32 - 1) random.seed(rand_seed) args = get_args() model_id = random_id() abs_vocab_file = os.path.join(args.abs_out, 'vocab.pkl') with open(abs_vocab_file, 'rb') as vocab_file: vocab = pk.load(vocab_file) vocab_size = len(vocab) acts = ['tanh', 'relu', 'hard_sigmoid'] emb_dim = {{choice([50, 100, 200, 300])}} rnn_dim = {{uniform(50, 500)}} rnn_dim = int(rnn_dim) rec_act = {{choice([0, 1, 2])}} rec_act = acts[rec_act] dropout = {{uniform(0.2, 0.95)}} epochs = args.epochs n_emb = vocab_size * emb_dim n_rwa = (903 + 2 * rnn_dim) * rnn_dim n_tot = n_emb + n_rwa + rnn_dim + 1 lr = {{lognormal(-3 * 2.3, .8)}} lr = 1.5 * lr rho = {{normal(.875, .04)}} clipnorm = {{uniform(1, 15)}} eps = {{loguniform(-8 * 2.3, -5 * 2.3)}} opt = optimizers.RMSprop(lr=lr, rho=rho, clipnorm=clipnorm, epsilon=eps) loss = kappa_loss metric = kappa_metric evl = Evaluator(dataset, args.prompt_id, args.abs_out, dev_x, test_x, dev_df, test_df, model_id=model_id) train_y_mean = train_y.mean(axis=0) if train_y_mean.ndim == 0: train_y_mean = np.expand_dims(train_y_mean, axis=1) num_outputs = len(train_y_mean) mask_zero = False model = Sequential() model.add(Embedding(vocab_size, emb_dim, mask_zero=mask_zero)) model.add(RWA(rnn_dim, recurrent_activation=rec_act)) model.add(Dropout(dropout)) bias_value = (np.log(train_y_mean) - np.log(1 - train_y_mean)).astype( K.floatx()) model.add(Dense(num_outputs, bias_initializer=Constant(value=bias_value))) model.add(Activation('tanh')) model.emb_index = 0 from deepats.w2vEmbReader import W2VEmbReader as EmbReader emb_reader = EmbReader(args.emb_path, emb_dim) emb_reader.load_embeddings(vocab) emb_wts = emb_reader.get_emb_matrix_given_vocab( vocab, model.layers[model.emb_index].get_weights()[0]) wts = model.layers[model.emb_index].get_weights() wts[0] = emb_wts model.layers[model.emb_index].set_weights(wts) model.compile(loss=loss, optimizer=opt, metrics=[metric]) model_yaml = model.to_yaml() import GPUtil if GPUtil.avail_mem() < 0.1: return {'loss': 1, 'status': STATUS_OK, 'model': '', 'weights': None} print('model_id: %s' % (model_id)) print(model_yaml) print('PARAMS\t\ %s\t\ lr= %.4f\t\ rho= %.4f\t\ clip= %.4f\t\ eps= %.4f\t\ embDim= %.4f\t\ rnnDim= %.4f\t\ drop= %.4f\t\ recAct= %s' % (model_id, lr, rho, clipnorm, np.log(eps) / 2.3, emb_dim, rnn_dim, dropout, rec_act)) for i in range(epochs): train_history = model.fit(train_x, train_y, batch_size=args.batch_size, epochs=1, verbose=0) evl.evaluate(model, i) evl.output_info() p = evl.stats[3] / qwks[0] if i > 10 and p < 0.9: break i = evl.comp_idx j = i + 2 best_dev_kappa = evl.best_dev[i] best_test_kappa = evl.best_dev[j] print('Test kappa:', best_dev_kappa) return { 'loss': 1 - best_dev_kappa, 'status': STATUS_OK, 'model': model.to_yaml(), 'weights': pk.dumps(model.get_weights()) }
def create_model(x_train, y_train, x_val, y_val): batch_size = 64 epochs = 500 init = 'lecun_normal' act = 'tanh' neurons = int({{quniform(9, 180, 9)}}) layers = {{choice([1, 2, 4, 8])}} norm = {{choice(['no', 'l1', 'l2'])}} dropout = {{choice([0, 1])}} earlystop = {{choice([0, 1])}} k = None p = None patience = None if norm == 'no': reg = None elif norm == 'l1': k = {{loguniform(-9.2, -2.3)}} reg = regularizers.l1(k) elif norm == 'l2': k = {{loguniform(-9.2, -2.3)}} reg = regularizers.l2(k) X_input = Input((24, )) X = Reshape((-1, ))(X_input) for _ in range(layers): X = Dense(neurons, kernel_initializer=init, kernel_regularizer=reg)(X) X = Activation(act)(X) if dropout == 1: p = {{uniform(0, 1)}} X = Dropout(p)(X) X = Dense(1, kernel_initializer=init, kernel_regularizer=reg)(X) X_outputs = Activation('sigmoid')(X) model = Model(inputs=X_input, outputs=X_outputs) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) if earlystop == 0: model.fit(x_train, y_train, batch_size=batch_size, verbose=0, epochs=epochs, validation_data=(x_val, y_val)) elif earlystop == 1: patience = int({{quniform(1, 500, 1)}}) es = EarlyStopping(monitor='val_loss', patience=patience, verbose=0, mode='auto') model.fit(x_train, y_train, batch_size=batch_size, verbose=0, epochs=epochs, validation_data=(x_val, y_val), callbacks=[es]) loss_t, score_t = model.evaluate(x_train, y_train, verbose=0) loss_v, score_v = model.evaluate(x_val, y_val, verbose=0) print( str(neurons) + '\t' + str(layers) + '\t' + str(norm) + '\t' + str(dropout) + '\t' + str(earlystop) + '\t\t' + '%-24s%-24s%s' % (str(k), str(p), str(patience))) return {'loss': loss_v, 'status': STATUS_OK, 'model': model}
def model(train_data, test_data): initializer = keras.initializers.RandomUniform(minval=0.0001, maxval=0.01) activation = 'relu' num_layers = {{quniform(1, 5, 1)}} layer_size_five_hundreds = {{quniform(1, 10, 1)}} * 500 # # Single autoencoder input = Input(shape=(len(train_data[0]), )) output = input for i in range(int(num_layers)): output = Dense(int(layer_size_five_hundreds), activation=activation)(output) encoded_activation = {{choice(['relu', 'linear'])}} output = Dense( 2, activation=encoded_activation, name="encoded", kernel_initializer=initializer, bias_initializer='zeros' )(output) #maybe just try avoiding dead neurons in small encoding layer? for i in range(int(num_layers)): output = Dense(int(layer_size_five_hundreds), activation=activation)(output) output_activation = {{choice(['linear', 'sigmoid'])}} output = Dense(len(train_data[0]), activation=output_activation)( output ) #'linear',)(output) # First test seems to indicate no change on output with linear autoencoder = Model(input, output) learning_rate = {{loguniform(math.log(0.1), math.log(0.0001))}} optimizer = keras.optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0) autoencoder.compile( optimizer=optimizer, #adamax and nadam worked well too loss='mean_squared_error', metrics=['mse']) # from keras.callbacks import TensorBoard # tensorboard = TensorBoard(log_dir="logs/{}".format(datetime.datetime.now().strftime("%I%M%p%B%d%Y")), write_graph=False)#,histogram_freq=2,)#, histogram_freq=1, write_graph=False) early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='min') batch_size_power_of_2 = 2**int({{quniform(6, 11, 1)}}) print('num_layers', num_layers) print('layer_size_five_hundreds', layer_size_five_hundreds) print('encoded_activation', encoded_activation) print('output_activation', output_activation) print('learning_rate', learning_rate) print('batch_size_power_of_2', batch_size_power_of_2) autoencoder.fit(train_data, train_data, epochs=10, batch_size=int(batch_size_power_of_2), shuffle=True, callbacks=[early_stop], validation_data=(test_data, test_data)) loss, mse = autoencoder.evaluate(test_data, test_data, verbose=0) print('Test accuracy:', mse) K.clear_session() return {'loss': mse, 'status': STATUS_OK, 'model': autoencoder}