from keras.callbacks import ModelCheckpoint from keras.callbacks import LearningRateScheduler # checkpoint filepath = "checkpointsTest/" + DataFile + "-{epoch:02d}-{accuracy:.2f}.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='accuracy', verbose=1, save_best_only=True, mode='max', save_weights_only=True) def lr_scheduler(epoch, lr): #if epoch == 1: # lr = 0.01 if epoch % 200 == 0 and epoch > 0: lr = lr * 0.1 return lr callbacks_list = [checkpoint, LearningRateScheduler(lr_scheduler, verbose=1)] model.fit(x=X, y=Y, shuffle=True, batch_size=batch_size, steps_per_epoch=step_size_train, epochs=200, callbacks=callbacks_list)
model.add(Conv1D(64, 2, activation='relu')) model.add(BatchNormalization()) model.add(GlobalAveragePooling1D()) model.add(Dense(256, activation='relu')) model.add(Dropout(0.6)) model.add(Dense(num_class, activation='softmax')) # model.summary() model.compile(optimizer=optimizers.Adam(learning_rate), loss='categorical_crossentropy', metrics=['accuracy']) model.fit(encode_pad_train, y_train, epochs=epochs, batch_size=1024, validation_data=(encode_pad_val, y_val), callbacks=[LearningRateScheduler(lr_adaptive)], verbose=2) scores = model.evaluate(encode_pad_test, y_test, verbose=2) print('Test loss: ', scores[0]) print('Test accuracy:', scores[1]) # I started with a simple model with two dense layers. It achieved a fairly # good result of about 86% validation accuracy. I was curious as to how # effective convolutional networks would be, and surprisingly, it did not # help that much. I tried a deeper network, upto depth 12, but going deeper # did not seem to help at all. I played around with the length of the # embedding vector, width and depth of conv-nets, different activations # but nothing really improved the result. The state of the art from the paper # is 92.7%. My final configuration gives results that are in the range of # validation accuracy of 88% to 91%.
#model = VGG19(rf, include_top=False, weights=None, input_shape=(64, 64, 3)) #model.compile(Adam(lr=lr), loss='categorical_crossentropy', metrics=['accuracy']) mixed_gen = mixed_generator(ratio) with tf.device('/gpu:0'): steps_per_epoch = 100 print("Starting experiment " + str(num_experiment)) print("IMAGE SIZE: ", IMG_SIZE) print("DATASET: ", gen_data_path) print("RATIO: ", ratio) model.fit_generator(mixed_gen, steps_per_epoch=steps_per_epoch, epochs=10, validation_data=valid, validation_steps=steps_per_epoch / 5, callbacks=[LearningRateScheduler(lr_schedule)]) print("Saved " + filename) num_experiment += 1 del model # # Experiment 3: # Train the baseline model on 20 epochs on real training dataset # Epoch 20/20 # 2000/2000 [==============================] - 206s - loss: 1.2179 - acc: 0.5625 - val_loss: 1.6750 - val_acc: 0.4242 # # # # Experiment 4: # # Finetune the baseline model with weights of #3 on 20 epochs on real training dataset # # Epoch 20/20:
output = keras.layers.Add()([output_1, output_2, output_3, output_3, output_2, output_3]) output = Activation('softmax')(output) model = Model([img_input_1, img_input_2, img_input_3], output) #model.load_weights('ckpt.h5') plot_model(model, show_shapes=True, to_file='model.png') print(model.summary()) # set optimizer sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) # set callback metrics = Metrics() tb_cb = TensorBoard(log_dir=args.save_dir, histogram_freq=0) change_lr = LearningRateScheduler(scheduler) ckpt = ModelCheckpoint(os.join(args.save_dir, './ckpt.h5'), save_best_only=False, mode='auto', period=10) cbks = [change_lr,tb_cb,ckpt,metrics] # set data augmentation print('Using real-time data augmentation.') data_generator = data_generator(x_train, y_train, batch_size) # start training model.fit_generator(data_generator, steps_per_epoch=iterations, epochs=epochs, callbacks=cbks,validation_data=([x_test[0], x_test[1], x_test[2]], y_test)) for i in range(epochs): print "epoch" + str(i) print(metrics.val_recalls_pos) print(metrics.val_recalls_neg) print(metrics.val_precisions) print(metrics.val_f1s)
def main(): args = get_args() input_path = args.input batch_size = args.batch_size nb_epochs = args.nb_epochs depth = args.depth k = args.width validation_split = args.validation_split use_augmentation = args.aug image_size = 224 os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" logging.debug("Loading data and Augmentor...") #sess = K.get_session() #sess = tf_debug.LocalCLIDebugWrapperSession(sess) #K.set_session(sess) #model = WideResNet(image_size, depth=22, k=k)() model = MyModel(image_size, trainable=False)() adam = Adam(lr=0.01, decay=0.001) sgd = SGD(lr=0.00001, momentum=0.9, nesterov=True, decay=0.0001) model.compile(optimizer=sgd, loss=["categorical_crossentropy", "MSE"], loss_weights=[0.5, 1.0], metrics=['accuracy']) logging.debug("Model summary...") model.count_params() model.summary() #model.load_weights(os.path.join("checkpoints", "weights.03-4.78.hdf5")) logging.debug("Saving model...") mk_dir("models") with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)), "w") as f: f.write(model.to_json()) mk_dir("checkpoints") train_datagen = ImageDataGenerator(rescale=1. / 255, shear_range=0.2, horizontal_flip=True, validation_split=0.2) train_generator = train_datagen.flow_from_directory( '../../dataset/imdb_crop/new_database/', target_size=(image_size, image_size), batch_size=batch_size, class_mode='categorical', subset='training', shuffle=True) print(next(train_generator)[1].shape) val_generator = train_datagen.flow_from_directory( '../../dataset/imdb_crop/new_database/', target_size=(image_size, image_size), batch_size=batch_size, class_mode='categorical', subset='validation', shuffle=True) callbacks = [ LearningRateScheduler(schedule=Schedule(38138 // batch_size)), ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5", monitor="val_loss", verbose=1, save_best_only=True, mode="auto") ] #class_weight = get_class_weights(train_generator.classes) #print(class_weight) h = model.fit_generator( generate_data_generator(train_generator), use_multiprocessing=True, epochs=10, validation_data=generate_data_generator(val_generator), workers=12, steps_per_epoch=len(train_generator.classes) / batch_size, validation_steps=len(val_generator.classes) / batch_size, #class_weight=class_weight ) logging.debug("Saving weights...") model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)), overwrite=True)
block3_output = merge([block3_w2, block2_output], mode='sum') calibMMDNet = Model(input=calibInput, output=block3_output) # learning rate schedule def step_decay(epoch): initial_lrate = 0.001 drop = 0.1 epochs_drop = 150.0 lrate = initial_lrate * math.pow(drop, math.floor( (1 + epoch) / epochs_drop)) return lrate lrate = LearningRateScheduler(step_decay) #train MMD net optimizer = keras.optimizers.rmsprop(lr=0.0) calibMMDNet.compile(optimizer=optimizer, loss=lambda y_true, y_pred: cf.MMD( block3_output, target, MMDTargetValidation_split=0.1). KerasCost(y_true, y_pred)) sourceLabels = np.zeros(source.shape[0]) calibMMDNet.fit(source, sourceLabels, nb_epoch=500, batch_size=1000, validation_split=0.1, verbose=1,
load_existent_model = False # load train and test data print('loading data ...') (x_train, y_train), (x_test, y_test) = Funcs.load_data(dirs, target_size, gray=gray, classes=classes) print('loading data done') # define callbacks for models to render model performance better checkpoint = ModelCheckpoint(model_save_path, monitor='val_acc', save_best_only=True, verbose=0) lrschedual = LearningRateScheduler( lambda epoch: Funcs.lr_schedual(epoch, epochs=epochs), verbose=0) # whether load existent model if load_existent_model and os.path.exists(model_load_path): model = load_model(model_load_path) else: model = model_design(x_train.shape[1:], classes) # train the model start_time = datetime.datetime.now() model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=(x_test, y_test), callbacks=[checkpoint, lrschedual],
model[j].add(Dropout(0.4)) model[j].add(Dense( 10, activation='softmax')) # softmax - 로지스틱스 회기분석과 비슷한 성질 / 강화학습을 일으키는 요소 # COMPILE WITH ADAM OPTIMIZER AND CROSS ENTROPY COST model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"]) #Adam method의 강점 #여기서 소개하는 Adam method는 Adagrad + RMSProp의 장점을 섞어 놓은 것으로 자세한 알고리즘은 추후에 다루기로 하겠다. # 저자가 말하는 Adam method의 의 주요 장점은 stepsize가 gradient의 rescaling에 영향 받지 않는다는 것이다. # gradient가 커져도 stepsize는 bound되어 있어서 어떠한 objective function을 사용한다 하더라도 안정적으로 최적화를 위한 하강이 가능하다. 게다가 stepsize를 과거의 gradient 크기를 참고하여 adapted시킬 수 있다. annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95**x) # TRAIN NETWORKS history = [0] * nets epochs = 45 for j in range(nets): X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size=0.1) history[j] = model[j].fit_generator(datagen.flow(X_train2, Y_train2, batch_size=64), epochs=epochs, steps_per_epoch=X_train2.shape[0] // 64, validation_data=(X_val2, Y_val2), callbacks=[annealer],
# print("x2p", x2Pred.shape) # return (tf.keras.losses.binary_crossentropy(x1True,x1Pred)) - lam*(tf.keras.losses.categorical_crossentropy(x2True,x2Pred, from_logits=False)) # Compile and fit the integrated model # loss before: {"fake_news_detector" : 'binary_crossentropy', "event_discriminator" : 'categorical_crossentropy'} f_model = Model(inputs=[sequence_input], outputs=[preds, ed_fc2_out]) f_model.compile(loss={ "fake_news_detector": 'binary_crossentropy', "event_discriminator": 'categorical_crossentropy' }, loss_weights=[-1, 1], optimizer=Adam(), metrics=['acc']) # Create learning rate scheduler lrate = LearningRateScheduler(decay_lr) callbacks_list = [lrate] # Print model summary print(f_model.summary()) # Fit model f_model.fit(data_train, { "fake_news_detector": labels_train, "event_discriminator": subjects_train }, batch_size=BATCH_SZ, epochs=EPOCHS, validation_data=(data_test, { "fake_news_detector": labels_test, "event_discriminator": subjects_test
def run(): config = {} config['MAX_NB_WORDS'] = 90000 config['EMBEDDING_DIM'] = 256 config['INPUT_LEN'] = 1100 config['SPLIT_SEED'] = 30 config['VALIDATION_SPLIT'] = 0.1 log(config) log('Get DataSet..') local_train_x, local_test_x, local_train_y, local_test_y, online_train_x, online_test_x, online_train_y, ID = getData( config) gc.collect() lr = LearningRateScheduler(get_lr) np.random.seed(30) #--get Local Test log('Local Validate Model...') # model = TextCNN_SoftMax(config, model_view = True) # model.fit(local_train_x,local_train_y, batch_size=100, epochs=20, # callbacks = [EarlyStopping(monitor='val_macro_f1',patience=1,mode = 'max'), lr], # validation_data=(local_test_x, local_test_y)) # del model log('Online Train Model(9/1 Data) SoftMax With Different Seed...') for i in range(10): log('Seed_%d' % i) config['SPLIT_SEED'] = i local_train_x, local_test_x, local_train_y, local_test_y, online_train_x, online_test_x, online_train_y, ID = getData( config) lr = LearningRateScheduler(get_lr) np.random.seed(i) model = TextCNN_SoftMax(config, model_view=False) model.fit(local_train_x, local_train_y, batch_size=100, epochs=8, callbacks=[lr]) model.save('../../model/yyt/money/textCNN(SoftMax)_money_9_seed.' + str(i) + '.hdf5') predict_online = model.predict(online_test_x, batch_size=100) pd.DataFrame(predict_online).to_csv( '../../result/yyt/money/textCNN(SoftMax)_9_%d' % (i), index=False) if i == 0: res = predict_online else: res += predict_online del model gc.collect() res = pd.DataFrame(res / 10) res = pd.concat([ID, res], axis=1) res.to_csv('../../result/yyt/money/textCNN(SoftMax)_9_prob_blend.csv', index=False, header=False, float_format='%.8f') log('Online Train Model(ALL Data) SigMoid With Different Seed...') for i in range(10): log('Seed_%d' % i) lr = LearningRateScheduler(get_lr) np.random.seed(i) model = TextCNN_SigMoid(config, model_view=False) model.fit(online_train_x, online_train_y, batch_size=100, epochs=8, callbacks=[lr]) model.save('../../model/yyt/money/textCNN(SigMoid)_money_all_seed.' + str(i) + '.hdf5') predict_online = model.predict(online_test_x, batch_size=100) pd.DataFrame(predict_online).to_csv( '../../result/yyt/money/textCNN(SigMoid)_all_%d' % (i), index=False) if i == 0: res = predict_online else: res += predict_online del model gc.collect() res = pd.DataFrame(res / 10) res = res.apply(lambda x: x / sum(x)) res = pd.concat([ID, res], axis=1) res.to_csv('../../result/yyt/money/textCNN(SigMoid)_all_prob_blend.csv', index=False, header=False, float_format='%.8f') #--get Online Result config['MAX_NB_WORDS'] = 150000 local_train_x, local_test_x, local_train_y, local_test_y, online_train_x, online_test_x, online_train_y, ID = getData( config) log('Online Train Model(ALL Data) SoftMax With Different Seed...') for i in range(10): log('Seed_%d' % i) lr = LearningRateScheduler(get_lr) np.random.seed(i) model = TextCNN_SoftMax(config, model_view=False) model.fit(online_train_x, online_train_y, batch_size=100, epochs=8, callbacks=[lr]) model.save('../../model/yyt/money/textCNN(SoftMax)_money_all_seed.' + str(i) + '.hdf5') predict_online = model.predict(online_test_x, batch_size=100) pd.DataFrame(predict_online).to_csv( '../../result/yyt/money/textCNN(SoftMax)_all_%d' % (i), index=False) if i == 0: res = predict_online else: res += predict_online del model gc.collect() res = pd.DataFrame(res / 10) res = pd.concat([ID, res], axis=1) res.to_csv('../../result/yyt/money/textCNN(SoftMax)_all_prob_blend.csv', index=False, header=False, float_format='%.8f')
2 * batch_size, 28) #2 times val because of val_aug nb_epochs = epochs[1] nb_cycles = 15 init_lr = 0.001 def _cosine_anneal_schedule(t): cos_inner = np.pi * (t % (nb_epochs // nb_cycles)) cos_inner /= nb_epochs // nb_cycles cos_out = np.cos(cos_inner) + 1 return float(init_lr / 2 * cos_out) lr_schedule = LearningRateScheduler(_cosine_anneal_schedule, verbose=True) callbacks_list = [lr_schedule, f1_metric, checkpoint, checkpoint2, tensorboard] # warm up model model = create_model(input_shape=(SIZE, SIZE, 3), n_out=28) POS_WEIGHT = 10 # multiplier for positive targets, needs to be tuned import tensorflow as tf import keras.backend.tensorflow_backend as tfb def weighted_binary_crossentropy(target, output): """ Weighted binary crossentropy between an output tensor and a target tensor. POS_WEIGHT is used as a multiplier
def main(args): input_shape = (32, 32, 3) num_classes = 10 batch_size = int(args.batch_size) epochs = int(args.epochs) # Loading cifar10 data (X_train, y_train), (X_test, y_test) = load_cifar10() # Define model model = MobileNetV2(input_shape=input_shape, nb_class=num_classes, include_top=True).build() MODEL_NAME = "mobilenetv2__" + datetime.now().strftime("%Y-%m%d-%H%M%S") # Path & Env. settings ------------------------------------------------------------- LOG_DIR = os.path.join("./log", MODEL_NAME) if not os.path.exists(LOG_DIR): os.makedirs(LOG_DIR) shutil.copyfile(os.path.join(os.getcwd(), 'training.py'), os.path.join(LOG_DIR, 'training.py')) shutil.copyfile(os.path.join(os.getcwd(), 'mobilenet.py'), os.path.join(LOG_DIR, 'mobilenet.py')) MODEL_WEIGHT_CKP_PATH = os.path.join(LOG_DIR, "best_weights.h5") MODEL_TRAIN_LOG_CSV_PATH = os.path.join(LOG_DIR, "train_log.csv") # ---------------------------------------------------------------------------------- # Compile model model.summary() model.compile( optimizer=keras.optimizers.SGD(lr=2e-2, momentum=0.9, decay=0.0, nesterov=False), loss='categorical_crossentropy', loss_weights=[ 1.0 ], # The loss weight for model output without regularization loss. Set 0.0 due to validate only regularization factor. metrics=['accuracy']) # Load initial weights from pre-trained model if args.trans_learn: model.load_weights(str(args.weights_path), by_name=False) print("Load model init weights from", MODEL_INIT_WEIGHTS_PATH) print("Produce training results in", LOG_DIR) # Set learning rate learning_rates = [] for i in range(5): learning_rates.append(2e-2) for i in range(50 - 5): learning_rates.append(1e-2) for i in range(100 - 50): learning_rates.append(8e-3) for i in range(150 - 100): learning_rates.append(4e-3) for i in range(200 - 150): learning_rates.append(2e-3) for i in range(300 - 200): learning_rates.append(1e-3) # Set model callbacks callbacks = [] callbacks.append( ModelCheckpoint(MODEL_WEIGHT_CKP_PATH, monitor='val_loss', save_best_only=True, save_weights_only=True)) callbacks.append(CSVLogger(MODEL_TRAIN_LOG_CSV_PATH)) callbacks.append( LearningRateScheduler(lambda epoch: float(learning_rates[epoch]))) # data generator with data augumatation datagen = keras.preprocessing.image.ImageDataGenerator( featurewise_center=False, featurewise_std_normalization=False, rotation_range=0.0, width_shift_range=0.2, height_shift_range=0.2, vertical_flip=False, horizontal_flip=True) datagen.fit(X_train) # Train model history = model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size), steps_per_epoch=len(X_train) / batch_size, epochs=epochs, verbose=1, callbacks=callbacks, validation_data=(X_test, y_test)) # Validation val_loss, val_acc = model.evaluate(X_test, y_test, verbose=1) print("--------------------------------------") print("model name : ", MODEL_NAME) print("validation loss : {:.5f}".format(val_loss)) print("validation accuracy : {:.5f}".format(val_acc)) # Save model as "instance" ins_name = 'model_instance' ins_path = os.path.join(LOG_DIR, ins_name) + '.h5' model.save(ins_path) # Save model as "architechture" arch_name = 'model_fin_architechture' arch_path = os.path.join(LOG_DIR, arch_name) + '.json' json_string = model.to_json() with open(arch_path, 'w') as f: f.write(json_string)
def train(self, epochs=5, batch_size=16): if self.debug_mode: epochs = 1 dataset = self.create_train_data() train_tokens = dataset["train_tokens"] train_label = dataset["train_label"] train_type_labels = dataset["train_type_labels"] valid_tokens = dataset["valid_tokens"] valid_label = dataset["valid_label"] valid_type_labels = dataset["valid_type_labels"] test_tokens = dataset["test_tokens"] tokenizer = dataset["tokenizer"] valid_identity_type_labels = dataset["valid_identity_type_labels"] train_identity_type_labels = dataset["train_identity_type_labels"] valid_identity_type_binary_lables = dataset[ "valid_identity_type_binary_lables"] train_identity_type_binary_lables = dataset[ "train_identity_type_binary_lables"] valid_identity_sum_label = dataset["valid_identity_sum_label"] train_identity_sum_label = dataset["train_identity_sum_label"] valid_identity_binary_label = dataset["valid_identity_binary_label"] train_identity_binary_label = dataset["train_identity_binary_label"] sample_weights = self.cal_sample_weights() word_embedding = self.create_emb_weights(tokenizer.word_index) model = self.build_model(word_embedding) previous_auc_score = 0 for epoch in range(epochs): # TODO:先不用test model.fit( x=train_tokens, y=[train_label, train_type_labels, train_identity_type_labels], batch_size=batch_size, epochs=1, verbose=2, validation_data=([valid_tokens], [ valid_label, valid_type_labels, valid_identity_type_labels ]), sample_weight=[ sample_weights, np.ones_like(sample_weights), np.ones_like(sample_weights) ], callbacks=[ LearningRateScheduler(lambda _: 1e-3 * (0.6**epoch)) ]) # 打分 y_pred = model.predict(valid_tokens)[0] auc_score = self.evaluator.get_final_metric( y_pred ) # y_pred 可以是 (n, 1) 也可以是 (n,) 不 squeeze 也没关系。y_true 必须要有正有负,否则无法计算 auc if auc_score < previous_auc_score: break else: previous_auc_score = auc_score print("auc_score:", auc_score) if not self.debug_mode and epoch > 0: model.save( os.path.join( self.data_dir, "model/model[%s]_%d_%.5f" % (self.model_name, epoch, auc_score))) # del 训练相关输入和模型,手动清除显存 training_history = [ dataset, train_tokens, train_label, train_type_labels, valid_tokens, valid_label, valid_type_labels, test_tokens, tokenizer, sample_weights, word_embedding, model, valid_identity_type_labels, train_identity_type_labels, valid_identity_type_binary_lables, train_identity_type_binary_lables, valid_identity_sum_label, train_identity_sum_label, valid_identity_binary_label, train_identity_binary_label ] for training_variable in training_history: del training_variable K.clear_session() gc.collect()
def main(): """Train ResNet on Cifar10 dataset """ # construct the argument parse and parse the arguments args = argparse.ArgumentParser() args.add_argument("-m", "--model", required=True, help="path to output model") args.add_argument("-o", "--output", required=True, help="path to output directory (logs, plots, etc.)") args = vars(args.parse_args()) # load the training and testing data, converting the images from # integers to floats print("[INFO] loading CIFAR-10 data...") ((train_x, train_y), (test_x, test_y)) = cifar10.load_data() train_x = train_x.astype("float") test_x = test_x.astype("float") # apply mean subtraction to the data mean = np.mean(train_x, axis=0) train_x -= mean test_x -= mean # convert the labels from integers to vectors label_binarizer = LabelBinarizer() train_y = label_binarizer.fit_transform(train_y) test_y = label_binarizer.transform(test_y) # construct the image generator for data augmentation aug = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True, fill_mode="nearest") # construct the set of callbacks fig_path = os.path.sep.join([args["output"], "{}.png".format(os.getpid())]) json_path = os.path.sep.join( [args["output"], "{}.json".format(os.getpid())]) callbacks = [ TrainingMonitor(fig_path, json_path=json_path), LearningRateScheduler(poly_decay) ] # initialize the optimizer and model (ResNet-56) print("[INFO] compiling model...") opt = SGD(lr=INIT_LR, momentum=0.9) model = ResNet.build(32, 32, 3, 10, (9, 9, 9), (64, 64, 128, 256), reg=0.0005) model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=["accuracy"]) # train the network print("[INFO] training network...") model.fit_generator(aug.flow(train_x, train_y, batch_size=128), validation_data=(test_x, test_y), steps_per_epoch=len(train_x) // 128, epochs=NUM_EPOCHS, callbacks=callbacks, verbose=1) # save the network to disk print("[INFO] serializing network...") model.save(args["model"])
if epoch + 1 <= 2: return K.get_value(train_model.optimizer.lr) else: lr = K.get_value(train_model.optimizer.lr) lr = lr * 0.5 if lr < 2e-6: return 2e-6 else: return lr #################################################################################################################### train_model, entity_model = build_model_from_config(config_path, checkpoint_path, seq_len=180) train_data, dev_data = split_data(_train_data) train_D = data_generator(train_data, 32) reduce_lr = LearningRateScheduler(scheduler, verbose=1) best_f1 = 0 for i in range(1, 2): train_model.fit_generator(train_D.__iter__(), steps_per_epoch=len(train_D), epochs=1, callbacks=[reduce_lr]) # if (i) % 2 == 0 : #两次对dev进行一次测评,并对dev结果进行保存 print('进入到这里了哟~') P, R, F = predict_test_batch('dev') if F > best_f1: train_model.save_weights(weight_name) print('当前第{}个epoch,准确度为{},召回为{},f1为:{}'.format(i, P, R, F)) predict_test_batch('test')
total_epochs = opts.epochs initial_lrate = opts.learning_rate if opts.cool: drop = 0.5 else: drop = 1.0 epochs_drop = 1 + int(np.floor(total_epochs / 3)) def step_decay(epoch): global initial_lrate, epochs_drop, drop lrate = initial_lrate * np.power(drop, np.floor((1 + epoch) / epochs_drop)) return lrate lr_scheduler = LearningRateScheduler(step_decay) #### Train the Model if opts.train_bool: history = callbacks.History() if opts.save_path != None: model_file = '%s/%s.hdf5' % (opts.save_path, memo) checkpointer = ModelCheckpoint(filepath=model_file, verbose=1) callbacks = [history, lr_scheduler, checkpointer] else: callbacks = [history, lr_scheduler] model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),\ samples_per_epoch=X_train.shape[0],nb_epoch=total_epochs,callbacks=callbacks,verbose=1) loss_data = {'train': history.history['loss']} if opts.save_path != None: loss_file = '%s/%s.pkl' % (opts.save_path, memo)
def main(): nb_epochs = config.MAXIMUM_EPOCHS batch_size = config.batch_size lr = 0.1 momentum = 0.9 model_name = 'ResNet50' image_size = config.IMAGE_SIZE output_dir = 'checkpoints' experiment_name = 'yu4u' early_stop_patience = config.EARLY_STOP_EPOCHS train_path = os.path.join(image_directory, 'train') validation_path = os.path.join(image_directory, 'validation') test_path = os.path.join(image_directory, 'test') PARAMS = { 'epoch_nr': nb_epochs, 'batch_size': batch_size, 'learning_rate': lr, 'momentum': momentum, # 'input_shape': (512, 32, 3), 'early_stop': early_stop_patience, 'image_size': image_size, 'network': model_name } neptune.init(project_qualified_name='4ND4/sandbox') neptune_tb.integrate_with_keras() result = neptune.create_experiment(name=experiment_name, params=PARAMS) name = result.id print(name) #train_gen = FaceGenerator(image_directory, batch_size=batch_size, image_size=image_size, number_classes=nb_classes) #val_gen = ValGenerator(image_directory, batch_size=batch_size, image_size=image_size, number_classes=nb_classes) train_gen, val_gen, test_gen = getdata(train_path, validation_path, test_path) model = get_model(model_name=model_name, image_size=image_size, number_classes=nb_classes) sgd = SGD(lr=lr, momentum=momentum, nesterov=True) model.compile(optimizer=sgd, loss="categorical_crossentropy", metrics=[age_mae]) model.summary() output_dir = Path(__file__).resolve().parent.joinpath(output_dir) if not output_dir.exists(): output_dir.mkdir(parents=True) if not os.path.exists('checkpoints/{}'.format(name)): os.mkdir('checkpoints/{}'.format(name)) callbacks = [ EarlyStopping(monitor='val_age_mae', mode='min', verbose=1, patience=early_stop_patience), LearningRateScheduler(schedule=Schedule(nb_epochs, initial_lr=lr)), ModelCheckpoint(os.path.join(output_dir, name) + "/weights.{epoch:03d}-{val_loss:.3f}-{" "val_age_mae:.3f}.hdf5", monitor="val_age_mae", verbose=1, save_best_only=True, mode="min") ] hist = model.fit_generator(generator=train_gen, epochs=nb_epochs, validation_data=val_gen, verbose=1, callbacks=callbacks) np.savez(str(output_dir.joinpath("history_{}.npz".format(name))), history=hist.history)
# print(data) def step_decay(epoch): # initialize the base initial learning rate, drop factor, and # epochs to drop every initAlpha = 0.0001 factor = 0.25 dropEvery = 10 # compute learning rate for the current epoch alpha = initAlpha * (factor ** np.floor((1 + epoch) / dropEvery)) # return the learning rate return float(alpha) callbacks = [LearningRateScheduler(step_decay)] def generator(data, target, lookback, delay, min_index, max_index, shuffle=False, batch_size=128, step=1): """ 将要用到的生成器 他生成了一个元组(samples, targets), 其中samples是输入数据的一个批量,targets是对应的目标温度数组。 :param data: 浮点数数组组成的原始数组,已经标准化。 :param target: 预测对象 0冷 1热 2电 :param lookback: 输入数据应该包括过去多少个时间步。 :param delay: 目标应该在未来多少个时间步之后。 :param min_index: data数组中的索引,用于界定需要抽取哪些时间步。 :param max_index: 保存一部分数据用于验证,另一部分用于测试。 :param shuffle: 打乱样本,还是按顺序抽取样本。 :param batch_size: 每个样本的批量数。 :param step: 数据采样的周期(单位:时间步)。我们将其设置为1,为的是每小时抽取一个数据点。 :return:
def main(): os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu_list FLAGS.checkpoint_path = osp.join(FLAGS.checkpoint_path, str(datetime.date.today())) # check if checkpoint path exists if not os.path.exists(FLAGS.checkpoint_path): os.makedirs(FLAGS.checkpoint_path) else: shutil.rmtree(FLAGS.checkpoint_path) os.makedirs(FLAGS.checkpoint_path) train_data_generator = data_processor.generator(FLAGS) train_samples_count = data_processor.count_samples(FLAGS) val_data = data_processor.load_data(FLAGS) if len(gpus) <= 1: print('Training with 1 GPU') east = EAST_model(FLAGS.input_size) parallel_model = east.model else: print('Training with %d GPUs' % len(gpus)) with tf.device("/cpu:0"): east = EAST_model(FLAGS.input_size) parallel_model = multi_gpu_model(east.model, gpus=len(gpus)) score_map_loss_weight = K.variable(0.01, name='score_map_loss_weight') small_text_weight = K.variable(0., name='small_text_weight') lr_scheduler = LearningRateScheduler(lr_decay) ckpt = CustomModelCheckpoint(model=east.model, path=FLAGS.checkpoint_path + '/model-{epoch:02d}.h5', period=FLAGS.save_checkpoint_epochs, save_weights_only=True) tb = CustomTensorBoard(log_dir=FLAGS.checkpoint_path + '/train', score_map_loss_weight=score_map_loss_weight, small_text_weight=small_text_weight, data_generator=train_data_generator, write_graph=True) small_text_weight_callback = SmallTextWeight(small_text_weight) validation_evaluator = ValidationEvaluator( val_data, validation_log_dir=FLAGS.checkpoint_path + '/val') callbacks = [ lr_scheduler, ckpt, tb, small_text_weight_callback, validation_evaluator ] opt = AdamW(FLAGS.init_learning_rate) parallel_model.compile(loss=[ dice_loss(east.overly_small_text_region_training_mask, east.text_region_boundary_training_mask, score_map_loss_weight, small_text_weight), rbox_loss(east.overly_small_text_region_training_mask, east.text_region_boundary_training_mask, small_text_weight, east.target_score_map) ], loss_weights=[1., 1.], optimizer=opt) east.model.summary() model_json = east.model.to_json() with open(FLAGS.checkpoint_path + '/model.json', 'w') as json_file: json_file.write(model_json) history = parallel_model.fit_generator( train_data_generator, epochs=FLAGS.max_epochs, steps_per_epoch=train_samples_count / FLAGS.batch_size, workers=FLAGS.nb_workers, use_multiprocessing=True, max_queue_size=10, callbacks=callbacks, verbose=1)
def init_callbacks(self) -> None: if self.config.trainer.use_lr_decay: # linear decay from the half of max_epochs def lr_scheduler(lr, epoch, max_epochs): return min(lr, 2 * lr * (1 - epoch / max_epochs)) self.model_callbacks['model'].append( LearningRateScheduler(schedule=lambda epoch: lr_scheduler( self.config.model.generator.lr, epoch, self.config.trainer. num_epochs))) # if horovod used, only worker 0 saves checkpoints is_master = True is_local_master = True if self.config.trainer.use_horovod: import horovod.keras as hvd is_master = hvd.rank() == 0 is_local_master = hvd.local_rank() == 0 # horovod callbacks if self.config.trainer.use_horovod: import horovod.keras as hvd self.model_callbacks["model"].append( hvd.callbacks.BroadcastGlobalVariablesCallback(0)) self.model_callbacks["model"].append( hvd.callbacks.MetricAverageCallback()) self.model_callbacks["model"].append( hvd.callbacks.LearningRateWarmupCallback(warmup_epochs=5, verbose=1)) if is_local_master: # model saver self.model_callbacks["serial_model"].append( ModelCheckpointWithKeepFreq(filepath=os.path.join( self.config.exp.checkpoints_dir, "{epoch:04d}-combined.hdf5"), keep_checkpoint_freq=self.config. trainer.keep_checkpoint_freq, save_checkpoint_freq=self.config. trainer.save_checkpoint_freq, save_best_only=False, save_weights_only=True, verbose=1)) # save optimizer weights for model_name in ['model']: self.model_callbacks[model_name].append( OptimizerSaver(self.config, model_name)) if is_master: # save individual models for model_name in ['model']: self.model_callbacks[model_name].append( ModelSaver(checkpoint_dir=self.config.exp.checkpoints_dir, keep_checkpoint_freq=self.config.trainer. keep_checkpoint_freq, model_name=model_name, num_epochs=self.config.trainer.num_epochs, verbose=1)) # send notification to telegram channel on train start and end self.model_callbacks["model"].append( TrainProgressAlertCallback( experiment_name=self.config.exp.name, total_epochs=self.config.trainer.num_epochs)) # tensorboard callback self.model_callbacks["model"].append( ScalarCollageTensorBoard( log_dir=self.config.exp.tensorboard_dir, batch_size=self.config.trainer.batch_size, write_images=True)) # initialize callbacks by setting model and params epochs = self.config.trainer.num_epochs steps_per_epoch = self.data_loader.get_train_data_size( ) // self.config.trainer.batch_size for model_name in self.model_callbacks: model = eval(f"self.{model_name}") callbacks = self.model_callbacks[model_name] for callback in callbacks: callback.set_model(model) callback.set_params({ "batch_size": self.config.trainer.batch_size, "epochs": epochs, "steps": steps_per_epoch, "samples": self.data_loader.get_train_data_size(), "verbose": True, "do_validation": False, "model_name": model_name, })
def main(): args = get_args() input_path = args.input batch_size = args.batch_size nb_epochs = args.nb_epochs lr = args.lr opt_name = args.opt depth = args.depth k = args.width validation_split = args.validation_split use_augmentation = args.aug output_path = Path(__file__).resolve().parent.joinpath(args.output_path) output_path.mkdir(parents=True, exist_ok=True) logging.debug("Loading data...") image, gender, age, _, image_size, _ = load_data(input_path) X_data = image y_data_g = np_utils.to_categorical(gender, 2) y_data_a = np_utils.to_categorical(age, 101) model = WideResNet(image_size, depth=depth, k=k)() opt = get_optimizer(opt_name, lr) model.compile( optimizer=opt, loss=["categorical_crossentropy", "categorical_crossentropy"], metrics=['accuracy']) logging.debug("Model summary...") model.count_params() model.summary() callbacks = [ LearningRateScheduler(schedule=Schedule(nb_epochs, lr)), ModelCheckpoint(str(output_path) + "/weights.{epoch:02d}-{val_loss:.2f}.hdf5", monitor="val_loss", verbose=1, save_best_only=True, mode="auto") ] logging.debug("Running training...") data_num = len(X_data) indexes = np.arange(data_num) np.random.shuffle(indexes) X_data = X_data[indexes] y_data_g = y_data_g[indexes] y_data_a = y_data_a[indexes] train_num = int(data_num * (1 - validation_split)) X_train = X_data[:train_num] X_test = X_data[train_num:] y_train_g = y_data_g[:train_num] y_test_g = y_data_g[train_num:] y_train_a = y_data_a[:train_num] y_test_a = y_data_a[train_num:] if use_augmentation: datagen = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True, preprocessing_function=get_random_eraser( v_l=0, v_h=255)) training_generator = MixupGenerator(X_train, [y_train_g, y_train_a], batch_size=batch_size, alpha=0.2, datagen=datagen)() hist = model.fit_generator(generator=training_generator, steps_per_epoch=train_num // batch_size, validation_data=(X_test, [y_test_g, y_test_a]), epochs=nb_epochs, verbose=1, callbacks=callbacks) else: hist = model.fit(X_train, [y_train_g, y_train_a], batch_size=batch_size, epochs=nb_epochs, callbacks=callbacks, validation_data=(X_test, [y_test_g, y_test_a])) logging.debug("Saving history...") pd.DataFrame(hist.history).to_hdf( output_path.joinpath("history_{}_{}.h5".format(depth, k)), "history")
model.compile(loss={ 'out_rec_img': combined_loss, 'out_pred_voxel': combined_voxel_loss }, loss_weights=[1.0, 1.0], optimizer=Adam(lr=5e-4, amsgrad=True), metrics={'out_rec_img': ['mse', 'mae']}) ##################################################### callbacks ######################################################## callback_list = [] if (config_file.decoder_tenosrboard_logs is not None): callback = TensorBoard(config_file.decoder_tenosrboard_logs) callback.set_model(model) callback_list.append(callback) reduce_lr = LearningRateScheduler(step_decay) callback_list.append(reduce_lr) if not os.path.exists(config_file.results): os.makedirs(config_file.results) callback_list.append( log_image_collage_callback(Y_test_avg, X_test_avg, decoder_model, dir=config_file.results + '/test_collge_ep/')) callback_list.append( log_image_collage_callback(Y[0:50], X[0:50], decoder_model, dir=config_file.results + '/train_collge_ep/')) ##################################################### generators #######################################################
reshape_layer = Reshape((dim1, ), input_shape=(1, dim1)) S_reshape = reshape_layer(S_embed) O_reshape = reshape_layer(O_embed) SO_merged = Multiply()([S_reshape, O_reshape]) SO_merged = Concatenate()([S_reshape, O_reshape, SO_merged]) SO_merged = Dropout(0.3)(SO_merged) SO_merged = Dense(int(dim2 * 1.5), activation="relu")(SO_merged) SO_merged = Dropout(0.1)(SO_merged) SO_merged = Dense(int(dim2 * 1.5 / 2), activation="relu")(SO_merged) SO_merged = Dropout(0.1)(SO_merged) P_pred = Dense(rel_voc, activation="sigmoid")(SO_merged) model = Model([S_input, O_input], P_pred) lr_cb = LearningRateScheduler(learning_rate) model.compile(optimizer="adam", loss=dynamic_weighted_binary_crossentropy(rel_voc, alpha=alpha), metrics=["binary_accuracy", P, R]) model.fit([s_train, o_train], p_train_comb, epochs=50, sample_weight=sample_weight, callbacks=[lr_cb]) model.save("kgml_" + data_path + ".h5")
def lr_schedule(epoch): # Learning Rate Schedule lr = 1e-3 if epoch > 180: lr *= 0.5e-3 elif epoch > 160: lr *= 1e-3 elif epoch > 120: lr *= 1e-2 elif epoch > 80: lr *= 1e-1 print('Learning rate: ', lr) return lr lr_callback = LearningRateScheduler(lr_schedule) # Define optimizer and compile model optimizer = optimizers.Adam(lr_schedule(0), beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) model = create_model(input_shape=input_shape, classes=n_classes, name=network, architecture=network) model.summary() parallel_model = multi_gpu_model(model, gpus=2) parallel_model.compile(optimizer=optimizer,
return model # Define Model #ResNet: model = resnet_v1(input_shape=input_shape, depth=depth) model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=lr_schedule(0)), metrics=['accuracy', 'top_k_categorical_accuracy']) model.summary() print(model_type) # Prepare model model saving directory. lr_scheduler = LearningRateScheduler(lr_schedule) lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6) from keras.callbacks import TensorBoard callbacks = [ lr_reducer, lr_scheduler, TensorBoard(log_dir=(work_path / 'TB_Log' / exp_name).__str__()) ] x_train_epoch = [] y_train_epoch = []
imgs_val_path + f for f in os.listdir(imgs_val_path) if f.endswith(('.jpg', '.jpeg', '.png')) ]) model = ASD_SA(img_cols=args.input_size[1], img_rows=args.input_size[0], DRE_Loss=args.dreloss, learning_rate=args.init_lr) model.summary() if args.model_path is not None: print("Load weights") weight_file = args.model_path model.load_weights(weight_file) print(weight_file) lr_sch = LearningRateScheduler(scheduler) checkpointdir = args.output_path print('save weights file at ' + checkpointdir) hist = model.fit_generator( generator(b_s=args.batch_size, root_path=args.train_set_path, args=args, output_size=output_size), steps_per_epoch=(nb_imgs_train // args.batch_size), validation_data=generator(b_s=args.batch_size, root_path=args.val_set_path, args=args, output_size=output_size), validation_steps=(nb_imgs_val // args.batch_size), epochs=args.epochs, verbose=1,
prior_box_manager = PriorBoxManager(prior_boxes, num_classes=num_classes, box_scale_factors=[.1, .1, .2, .2]) image_generator = ImageGenerator(ground_truth_data, prior_box_manager, batch_size, image_shape[0:2], train_keys, validation_keys, image_path_prefix, vertical_flip_probability=0.5, horizontal_flip_probability=0.5) model_names = ('../trained_models/model_checkpoints/COCO/' + 'SSD300_COCO_weights.{epoch:02d}-{val_loss:.2f}.hdf5') model_checkpoint = ModelCheckpoint(model_names, monitor='val_loss', verbose=1, save_best_only=False, save_weights_only=False) learning_rate_schedule = LearningRateScheduler(scheduler) model.fit_generator(image_generator.flow(mode='train'), steps_per_epoch=int(len(train_keys) / batch_size), epochs=num_epochs, verbose=1, callbacks=[model_checkpoint, learning_rate_schedule], validation_data=image_generator.flow(mode='val'), validation_steps=int(len(validation_keys) / batch_size))
def main(logging, params, trainops): # load model architectire and weights model = trainops.model(params) model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=params.learning_rate, momentum=params.momentum), metrics=['accuracy']) # print model structure model.summary() # get standard configured data generators train_generator, valid_generator, test_generator = i.create_generators( params.data_path) # get data number of samples for training num_training_images, num_validation_images, num_test_images = i.get_data_statistics( params.data_path) '''callbacks''' lr_scheduler = LearningRateScheduler(trainops.step_decay) csv_logger = CSVLogger(filename=logging.model_directory + '/history.csv', append=True, separator=",") print("save directory is", logging.model_directory) checkpoint = ModelCheckpoint(filepath=logging.model_directory + "/weights.hdf5", monitor='val_acc', save_best_only=True, verbose=1, save_weights_only=True) tb = TensorBoard(log_dir=logging.tensorboard_directory, histogram_freq=0, write_graph=True, write_images=True, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) rlr = ReduceLROnPlateau(monitor='val_acc', factor=0.1, patience=20, verbose=0, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0) # saving model config file to model output dir logging.save_dict_to_json(logging.model_directory + "/config.json") history = model.fit_generator( generator=train_generator, steps_per_epoch=int(num_training_images / (params.batch_size * 1)), epochs=params.num_epochs, validation_data=valid_generator, use_multiprocessing=False, workers=8, validation_steps=int(num_validation_images / (1)), callbacks=[checkpoint, lr_scheduler, tb, csv_logger, rlr]) pd.DataFrame(history.history).to_csv(logging.model_directory + "/loss_files.csv") print( "###################### inititing predictions and evaluations ######################" ) pred = model.predict_generator(generator=test_generator, steps=int(num_test_images / (1)), verbose=1, use_multiprocessing=False, workers=1) # get predictions and labels in list format preds = np.argmax(pred, axis=1).tolist() lbls = test_generator.labels.tolist()[:len(preds)] # instantiate the evaluation class evaluation = Evaluation(history=pd.DataFrame(history.history), labels=lbls, predictions=preds, softmax_output=pred, model_dir=logging.model_directory, filenames=test_generator.filenames, params=params) # get and save 5 example fundus images for each class in "./predictions and assemble to canvas" #evaluation.plot_examples() evaluation.write_plot_evaluation()
valid_split = 0.05 epc = 10 fname_model = 'model/dnn' checkpoint = ModelCheckpoint(fname_model, monitor='val_out_clf_acc', verbose=1, save_best_only=True, mode='max') early_stopping = EarlyStopping(monitor='val_out_clf_acc', min_delta=0, patience=2, verbose=1, mode='max') #reduce_lr = ReduceLROnPlateau(monitor='val_out_clf_loss', factor=0.1, patience=0, mode='auto') reduce_lr = LearningRateScheduler(scheduler) model = fcnmodel() model.fit(x=[ X_app_6, X_bhv_log, X_bsc1, X_bsc2, X_act, X_app, X_usage_time, X_usage_duration, X_usage_time_new, X_usage_duration_new, X_tfidf, X_constant, X_cv_max ], y=[Y, Y_1hot], batch_size=batch_sz, epochs=epc, validation_split=valid_split, callbacks=[checkpoint, early_stopping, reduce_lr], verbose=1) # , class_weight = ) #model.fit(x=[X_bhv_log, X_bsc1, X_bsc2, X_act, X_app, X_usage_time, X_usage_duration], y=[Y, Y_1hot], # batch_size=batch_sz, epochs=epc, validation_split=valid_split,
X_train = X_train.astype('float32') img_gen = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True, horizontal_flip=True) img_gen.fit(X_train) Y_train = np_utils.to_categorical(Y_train, nb_classes) # building and training net gates = collections.OrderedDict() model = resnet() set_decay_rate() model.compile(optimizer="rmsprop", loss="categorical_crossentropy", metrics=["accuracy"]) current_dir = os.path.dirname(os.path.realpath(__file__)) model_path = os.path.join(current_dir, "resnet_110.png") plot(model, to_file=model_path, show_shapes=True) for i in gates: print K.get_value(gates[i][1]), gates[i][0], i model.fit_generator( img_gen.flow(X_train, Y_train, batch_size=batch_size, shuffle=True), samples_per_epoch=len(X_train), nb_epoch=nb_epochs, callbacks=[Gates_Callback(), LearningRateScheduler(scheduler)]) model.save_weights('model_weight_ep400_110.hdf5')