def model(train_gen, valid_gen): n_classes = 196 steps_per_epoch = 204 validation_steps = 51 channels = 3 output = "label_bbox" # Tensorboard root_logdir = os.path.join(os.curdir, 'logs') run_id = time.strftime(f"run_%Y_%m_%d-%H_%M_%S") run_logdir = os.path.join(root_logdir, run_id) tensorboard = TensorBoard(run_logdir) # Early Stopping early_stopping = EarlyStopping(patience=15, restore_best_weights=True) # Model Checkpoints checkpoint = ModelCheckpoint( filepath=f'checkpoints/' + \ 'epoch.{epoch:02d}_val_loss.{val_loss:.6f}.h5', verbose=1, save_best_only=True) #Reduce LR on Plateau reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.3, patience=5, min_lr=1e-4, verbose=1) callbacks = [tensorboard, early_stopping, checkpoint, reduce_lr] DefaultConv2D = partial( Conv2D, kernel_size=3, kernel_initializer="he_normal", kernel_regularizer=l2({{uniform(1e-5, 1)}}), # activation="relu", padding="same") input = Input(shape=(224, 224, channels)) # ============================================================================= # conv_1a = DefaultConv2D(filters=64, # kernel_size=5, # strides=2)(input) # norm_1a = BatchNormalization()(conv_1a) # relu_1a = Activation(activation="relu")(norm_1a) # # max_pool_1 = MaxPooling2D(pool_size=2)(relu_1a) # # conv_1b = DefaultConv2D(filters=64)(max_pool_1) # norm_1b = BatchNormalization()(conv_1b) # relu_1b = Activation(activation="relu")(norm_1b) # conv_1c = DefaultConv2D(filters=64)(relu_1b) # norm_1c = BatchNormalization()(conv_1c) # relu_1c = Activation(activation="relu")(norm_1c) # # max_pool_2 = MaxPooling2D(pool_size=2)(relu_1c) # # conv_2a = DefaultConv2D(filters=128)(max_pool_2) # norm_2a = BatchNormalization()(conv_2a) # relu_2a = Activation(activation="relu")(norm_2a) # conv_2b = DefaultConv2D(filters=128)(relu_2a) # norm_2b = BatchNormalization()(conv_2b) # relu_2b = Activation(activation="relu")(norm_2b) # # max_pool_3 = MaxPooling2D(pool_size=2)(relu_2b) # # flatten = Flatten()(max_pool_3) # ============================================================================= choice_activation = {{choice(['relu', 'leaky_relu'])}} conv = DefaultConv2D(filters=64, strides=2)(input) norm = BatchNormalization()(conv) if choice_activation == 'relu': act = Activation(activation="relu")(norm) else: act = LeakyReLU(0.2)(norm) conv = DefaultConv2D(filters=64)(act) norm = BatchNormalization()(conv) if choice_activation == 'relu': act = Activation(activation="relu")(norm) else: act = LeakyReLU(0.2)(norm) choice_pool = {{choice(['max', 'avg'])}} if choice_pool == 'max': x = MaxPooling2D(pool_size=2)(act) else: x = AveragePooling2D(pool_size=2)(act) for filters in [64] * 3: x = Residual(filters, activation=choice_activation)(x) res_block_depth_1 = {{choice([2, 3])}} x = Residual(128, strides=2, activation=choice_activation)(x) for filters in [128] * res_block_depth_1: x = Residual(filters, activation=choice_activation)(x) res_block_depth_2 = {{choice([4, 5])}} x = Residual(256, strides=2, activation=choice_activation)(x) for filters in [256] * res_block_depth_2: x = Residual(filters, activation=choice_activation)(x) x = Residual(512, strides=2, activation=choice_activation)(x) for filters in [512] * 2: x = Residual(filters, activation=choice_activation)(x) x = GlobalAvgPool2D()(x) drop = Dropout({{uniform(0, .5)}})(x) dense = Dense(512)(drop) norm = BatchNormalization()(dense) if choice_activation == 'relu': act = Activation(activation="relu")(norm) else: act = LeakyReLU(0.2)(norm) # If we choose 'one', add an additional dense layer choice_dense = {{choice(['yes', 'no'])}} if choice_dense == 'yes': drop = Dropout({{uniform(0, .5)}})(act) dense = Dense(256)(drop) norm = BatchNormalization()(dense) if choice_activation == 'relu': act = Activation(activation="relu")(norm) else: act = LeakyReLU(0.2)(norm) drop = Dropout({{uniform(0, .5)}})(act) class_output = Dense(n_classes, activation="softmax", name="labels")(drop) bbox_output = Dense(units=4, name="bbox")(drop) # Optimizer sgd = SGD(lr={{uniform(.0001, .3)}}) adam = Adam(lr={{uniform(.0001, .3)}}) choice_opt = {{choice(['adam', 'sgd'])}} if choice_opt == 'adam': optimizer = adam else: optimizer = sgd model = None if output == "bbox": model = Model(inputs=input, outputs=bbox_output) model.compile(loss="msle", optimizer=optimizer, metrics=["accuracy"]) elif output == "label": model = Model(inputs=input, outputs=class_output) model.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=["accuracy"]) else: model = Model(inputs=input, outputs=[class_output, bbox_output]) model.compile(loss=["categorical_crossentropy", "msle"], loss_weights=[.8, .2], optimizer=optimizer, metrics=["accuracy"]) model.fit(train_gen, epochs=400, steps_per_epoch=steps_per_epoch, validation_data=valid_gen, validation_steps=validation_steps, callbacks=callbacks, verbose=1) # ============================================================================= # #print(model.metrics_names) # # ['loss', 'labels_loss', 'bbox_loss', 'labels_acc', 'bbox_acc'] # validation_loss = np.amin(result.history['loss']) # print('Best validation loss of epoch:', validation_loss) # ============================================================================= results = model.evaluate_generator(valid_gen, steps=validation_steps, verbose=0) validation_loss = results[0] print('val_labels_acc', results[3]) print('val_bbox_acc', results[4]) print('Validation Loss:', validation_loss) return {'loss': validation_loss, 'status': STATUS_OK, 'model': model}
regressor.add(Dropout(0.0)) regressor.add(LSTM(units=25, return_sequences=True)) regressor.add(Dropout(0.0)) regressor.add(LSTM(units=10)) regressor.add(Dropout(0.0)) regressor.add(Dense(units=1, activation='sigmoid')) regressor.compile(optimizer='RMSprop', loss='mean_squared_error', metrics=['mean_squared_error']) # es = EarlyStopping(monitor='loss', min_delta=1e-6, patience=10, verbose=1) rlr = ReduceLROnPlateau(monitor='loss', factor=0.01, patience=5, verbose=1) mcp = ModelCheckpoint(filepath='pesos.h5', monitor='loss', save_best_only=True, verbose=1) regressor.fit(previsores, preco_real, epochs=100, batch_size=32, callbacks=[es, rlr, mcp]) # preco_real_teste = base_teste.iloc[:, 3:4].values frames = [base_teste, base_treinamento] base_completa = pd.concat(frames) # entradas = base_completa[len(base_completa) - len(base_teste) - 90:].values
class_num = 50 batch_size = 16 x_train, y_train, x_val, y_val, x_test, y_test = utils.split_dataset() train_sequence = utils.Train_Sequence(x_train, y_train, batch_size) val_sequence = utils.Val_Sequence(x_val, y_val, batch_size) #test_sequence = utils.Val_Sequence(x_test, y_test, batch_size) model = build_model(class_num) lr_scheduler = LearningRateScheduler(lr_schedule) lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=3, min_lr=1e-6) earlystop = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=5, verbose=1, mode='auto') model.compile(loss='categorical_crossentropy', optimizer=optimizers.SGD(lr=lr_schedule(0), momentum=0.6, nesterov=True), metrics=["accuracy"]) model.summary() callbacks = [lr_scheduler, lr_reducer] history = model.fit_generator(train_sequence, steps_per_epoch=int(np.ceil(x_train.shape[0] / batch_size)), epochs=150,
# autoencoder = encoder + decoder # instantiate autoencoder model autoencoder = Model(inputs, decoder(encoder(inputs)), name='autoencoder') autoencoder.summary() # prepare model saving directory. save_dir = os.path.join(os.getcwd(), 'saved_models') model_name = 'colorized_ae_model.{epoch:03d}.h5' if not os.path.isdir(save_dir): os.makedirs(save_dir) filepath = os.path.join(save_dir, model_name) # reduce learning rate by sqrt(0.1) if the loss does not improve in 5 epochs lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, verbose=1, min_lr=0.5e-6) # save weights for future use (e.g. reload parameters w/o training) checkpoint = ModelCheckpoint(filepath=filepath, monitor='val_loss', verbose=1, save_best_only=True) # Mean Square Error (MSE) loss function, Adam optimizer autoencoder.compile(loss='mse', optimizer='adam') # called every epoch callbacks = [lr_reducer, checkpoint]
) model.summary() # model.trainable = False model.save('C:/nmb/nmb_data/h5/5s/speech_vgg/speechvgg_sgd_1.h5') # 컴파일, 훈련 op = SGD(lr=1e-3) batch_size = 16 es = EarlyStopping(monitor='val_loss', patience=20, restore_best_weights=True, verbose=1) lr = ReduceLROnPlateau(monitor='val_loss', vactor=0.5, patience=10, verbose=1) path = 'C:/nmb/nmb_data/h5/5s/speech_vgg/speechvgg_sgd_1.h5' mc = ModelCheckpoint(path, monitor='val_loss', verbose=1, save_best_only=True) tb = TensorBoard(log_dir='C:/study/graph/' + start_now.strftime("%Y%m%d-%H%M%S") + "/", histogram_freq=0, write_graph=True, write_images=True) model.compile(optimizer=op, loss="sparse_categorical_crossentropy", metrics=['acc']) history = model.fit(x_train, y_train, epochs=1000, batch_size=batch_size,
graph_scale=1) train_generator.get_config() model = StackedDenseNet(train_generator, n_stacks=2, growth_rate=32, pretrained=True) #model = DeepLabCut(train_generator, backbone="resnet50") #model = DeepLabCut(train_generator, backbone="mobilenetv2", alpha=0.35) # Increase alpha to improve accuracy #model = DeepLabCut(train_generator, backbone="densenet121") #model = LEAP(train_generator) #model = StackedHourglass(train_generator) model.get_config() logger = Logger(validation_batch_size=10) reduce_lr = ReduceLROnPlateau(monitor="val_loss", factor=0.2, verbose=1, patience=20) model_checkpoint = ModelCheckpoint('/n/home10/abahl/engert_storage_armin/maxwell_paper/deepposekit_training/my_best_model.h5', monitor="val_loss", # monitor="loss" # use if validation_split=0 verbose=1, save_best_only=True, ) early_stop = EarlyStopping( monitor="val_loss", # monitor="loss" # use if validation_split=0 min_delta=0.001, patience=100, verbose=1 )
def __init__(self, stop_patience=10, lr_factor=0.5, lr_patience=1, lr_epsilon=0.001, lr_cooldown=4, lr_minimum=1e-5, outputDir='', debug=1): self.nl_begin = newline_callbacks_begin(outputDir) self.nl_end = newline_callbacks_end() self.stopping = EarlyStopping(monitor='val_loss', patience=stop_patience, verbose=debug, mode='min') self.reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=lr_factor, patience=lr_patience, mode='min', verbose=debug, epsilon=lr_epsilon, cooldown=lr_cooldown, min_lr=lr_minimum) self.modelbestcheck = ModelCheckpoint(outputDir + "/KERAS_check_best_model.h5", monitor='val_loss', verbose=debug, save_best_only=True) self.modelbestcheckweights = ModelCheckpoint( outputDir + "/KERAS_check_best_model_weights.h5", monitor='val_loss', verbose=debug, save_best_only=True, save_weights_only=True) self.modelcheckperiod = ModelCheckpoint( outputDir + "/KERAS_check_model_epoch{epoch:02d}.h5", verbose=debug, period=10) self.modelcheck = ModelCheckpoint(outputDir + "/KERAS_check_model_last.h5", verbose=debug) self.modelcheckweights = ModelCheckpoint( outputDir + "/KERAS_check_model_last_weights.h5", verbose=debug, save_weights_only=True) self.tb = TensorBoard(log_dir=outputDir + '/logs') self.history = History() self.timer = Losstimer() # self.callbacks=[ # self.nl_begin, # self.modelbestcheck,self.modelbestcheckweights, self.modelcheck,self.modelcheckweights,self.modelcheckperiod, # self.reduce_lr, self.stopping, self.nl_end, self.tb, self.history, # self.timer # ] self.callbacks = [ self.nl_begin, self.modelbestcheck, self.modelbestcheckweights, self.reduce_lr, self.stopping, self.nl_end, self.history, self.timer ]
import tensorflow as tf from sklearn.datasets import load_breast_cancer from sklearn.model_selection import train_test_split from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau from tensorflow.keras.models import load_model x_train, x_test, y_train, y_test = train_test_split(load_breast_cancer().data, load_breast_cancer().target, train_size = 0.8, random_state = 77) print(x_train.shape) print(x_test.shape) print(y_train.shape) print(y_test.shape) model = ak.StructuredDataRegressor(overwrite = True, max_trials = 1, loss = 'mse', metrics = ['acc']) es = EarlyStopping(monitor = 'val_loss', mode = 'min', patience=6) lr = ReduceLROnPlateau(monitor = 'val_loss', patience=3, factr = 0.5, verbose = 2) model.fit(x_train, y_train, epochs = 1, callbacks = [es, lr], validation_split = 0.2) # SAVE Best Model # model = model.export_model() best_model = model.tuner.get_best_model() best_model.save('C:/data/h5/best_cancer.h5') # LOAD Best Model best_model = load_model('C:/data/h5/best_cancer.h5') results = best_model.evaluate(x_test, y_test) print('results: ', results) best_model.summary()
def main(_argv): # 使用GPU physical_devices = tf.config.experimental.list_physical_devices('GPU') for physical_device in physical_devices: tf.config.experimental.set_memory_growth(physical_device, True) if FLAGS.tiny: # tiny模型 model = YoloV3Tiny(FLAGS.size, training=True, classes=FLAGS.num_classes) anchors = yolo_tiny_anchors # 锚框 anchor_masks = yolo_tiny_anchor_masks # 锚框对应的索引值 if FLAGS.dataset: # 读取训练数据集 train_dataset = dataset.load_tfrecord_dataset(FLAGS.dataset, FLAGS.classes, FLAGS.size) # 用来打乱数据集中数据顺序,训练时非常常用,取所有数据的前buffer_size数据项 train_dataset = train_dataset.shuffle(buffer_size=512) # 设置批次,按照顺序取出batch_size行数据,最后一次输出可能小于batch train_dataset = train_dataset.batch(FLAGS.batch_size) # 训练数据,格式(x,y) train_dataset = train_dataset.map(lambda x, y: ( dataset.transform_images(x, FLAGS.size), dataset.transform_targets(y, anchors, anchor_masks, FLAGS.size))) # 预先载入buffer_size项 train_dataset = train_dataset.prefetch( buffer_size=tf.data.experimental.AUTOTUNE) if FLAGS.val_dataset: # 读取验证数据集 val_dataset = dataset.load_tfrecord_dataset(FLAGS.val_dataset, FLAGS.classes, FLAGS.size) # 设置批次,按照顺序取出batch_size行数据,最后一次输出可能小于batch val_dataset = val_dataset.batch(FLAGS.batch_size) # 验证数据,格式(x,y) val_dataset = val_dataset.map(lambda x, y: ( dataset.transform_images(x, FLAGS.size), dataset.transform_targets(y, anchors, anchor_masks, FLAGS.size))) # 优化器 optimizer = tf.keras.optimizers.Adam(lr=FLAGS.learning_rate) # 损失函数 loss = [ YoloLoss(anchors[mask], classes=FLAGS.num_classes) for mask in anchor_masks ] # 编译模型 model.compile(optimizer=optimizer, loss=loss, run_eagerly=False) # 回调函数列表 callbacks = [ ReduceLROnPlateau(verbose=1), EarlyStopping(patience=3, verbose=1), ModelCheckpoint('checkpoints/yolov3_tiny_train_{epoch}.tf', verbose=1, save_weights_only=True), TensorBoard(log_dir='logs') ] # 训练模型 history = model.fit(train_dataset, epochs=FLAGS.epochs, callbacks=callbacks, validation_data=val_dataset)
model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2))) #model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, activation="relu")) model.add(Dropout(0.5)) model.add(Dense(10, activation="softmax")) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) optimizer = tf.keras.optimizers.Adam() learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss', patience=2, verbose=1, factor=0.2, min_lr=0.00001) ckpt = ModelCheckpoint('cnn_model_adam.h5', verbose=1, save_weights_only=True, save_best_only=True) epochs = 50 batch_size = 86 model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=["accuracy"]) train_datagen = ImageDataGenerator(featurewise_center=False, featurewise_std_normalization=False,
Ghr_0 = G_0 * hr Ghr = np.reshape(Ghr_0, (total_num, -1)) train_dataset = np.concatenate( (np.real(Ghr), np.imag(Ghr), np.real(hd), np.imag(hd)), axis=-1) scaler = preprocessing.StandardScaler().fit(train_dataset) train_dataset = scaler.transform(train_dataset) best_model_path = './models/bigscale/best_%d_and_%d.h5' % (M, N) checkpointer = ModelCheckpoint( best_model_path, verbose=1, save_best_only=True, save_weights_only=True) #True for Lambda layer usage reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.333, patience=10, verbose=1, mode='auto', min_delta=0.0001, min_lr=0.00001) early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=20) model = beamformer_LIS(M, N, lr) model.fit(train_dataset,origin_dataset,epochs=epochs,batch_size=batch_size,verbose=1,shuffle=True,\ validation_split=0.2,callbacks=[checkpointer,early_stopping,reduce_lr]) #%% testing test_num = 10000 d0_test = np.random.uniform(0, 8, test_num)
def get_callbacks(): # return [EarlyStopping(patience=20), ReduceLROnPlateau()] return [ReduceLROnPlateau()]
model.add(Dropout(0.3)) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(BatchNormalization()) model.add(Dropout(0.3)) model.add(Dense(64, activation='relu')) model.add(BatchNormalization()) model.add(Dropout(0.3)) model.add(Dense(10, activation='softmax')) from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau early_stopping = EarlyStopping(monitor='val_loss', patience=10, mode='min') reduce_lr = ReduceLROnPlateau(monitor='val_loss', patience=10) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc']) model.fit(x_train, y_train, validation_split=0.2, batch_size=20, epochs=100, callbacks=[reduce_lr, early_stopping]) loss = model.evaluate(x_test, y_test) print('loss, acc : ', loss) # loss, acc : [0.5301111340522766, 0.8363000154495239]
from tensorflow.keras.layers import Embedding, Dense, LSTM, Conv2D, Flatten, BatchNormalization, Dropout model = Sequential() #model.add(Embedding(input_dim = 10000, output_dim = 230, input_length = 100)) model.add(LSTM(128, input_shape = (100, 1))) model.add(Dense(32, activation='relu')) model.add(BatchNormalization()) model.add(Dense(64, activation='relu')) model.add(BatchNormalization()) model.add(Dense(32, activation='relu')) model.add(Dense(1, activation='sigmoid')) model.summary() #3. 컴파일, 훈련 from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau es = EarlyStopping(monitor = 'val_loss', patience = 30, verbose= 1, mode = 'auto') rl = ReduceLROnPlateau(monitor='val_loss', patience = 20, factor = 0.3, verbose = 1, mode = 'auto') model.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['acc']) history = model.fit(x_train, y_train, epochs = 200, batch_size = 128, validation_data = (x_val, y_val), verbose = 1 ,callbacks = [es, rl]) #4.평가, 예측 loss, acc = model.evaluate(x_test, y_test) print("loss : ", loss) print("acc : ", acc) # loss : 1.9248653650283813 # acc : 0.8248800039291382 #시각화 epochs = range(1, len(history.history['acc']) + 1) plt.plot(epochs, history.history['loss'])
verbose=1, mode='min', restore_best_weights=True ) mcp_save = ModelCheckpoint( os.path.join(MODEL_DIR, 'fold_' + str(FOLD) + '_epoch_{epoch:02d}'), save_best_only=True, monitor='val_accuracy', mode='max', verbose=1 ) reduce_lr_loss = ReduceLROnPlateau( monitor='val_loss', factor=0.1, patience=7, verbose=1, epsilon=1e-4, mode='min' ) # To show balanced accuracy metrics = Metrics((testX, testY), model) # compile model print('Compiling model...') opt = Adam(lr=LR, decay=LR / EPOCHS) loss = ( tf.keras.losses.CategoricalCrossentropy() if not LOG_SOFTMAX else ( lambda labels, targets: tf.reduce_mean( tf.reduce_sum( -1 * tf.math.multiply(tf.cast(labels, tf.float32), targets), axis=1
Y_val = Y[val, ...] X_unused = X[unused_samples, ...] Y_unused = Y[unused_samples, ...] model = get_model() model_name = "model.h5" check = ModelCheckpoint(filepath=model_name, monitor="val_acc", save_best_only=True, verbose=1, save_weights_only=True) reduce = ReduceLROnPlateau(monitor="val_acc", patience=30, verbose=1, min_lr=1e-7) early = EarlyStopping(patience=40, monitor="val_acc") model.fit(X_used, Y_used, epochs=120, verbose=1, batch_size=32, validation_data=(X_val, Y_val), callbacks=[check, reduce, early]) model.load_weights(model_name) pred_test = model.predict(X_test, batch_size=1024)
parameters = param() model = KerasClassifier(build_fn=dnn_model, verbose=1) search = RandomizedSearchCV(model, parameters, cv=3) # search = GridSearchCV(model, parameters, cv=3) filepath = '../data/modelcheckpoint/k62_iris_{epoch:02d}-{val_loss:.4f}.hdf5' es = EarlyStopping(monitor='val_loss', patience=10, mode='auto') cp = ModelCheckpoint(filepath=filepath, monitor='val_loss', save_best_only=True, mode='auto') lr = ReduceLROnPlateau(monitor='val_loss', patience=3, factor=0.5, mode='auto') search.fit(x_train, y_train, validation_split=0.2, verbose=1, epochs=100, callbacks=[es, lr]) # cp print(search.best_params_) print(search.best_estimator_) print(search.best_score_) # 밑의 score 랑 다르다. acc = search.score(x_test, y_test) print('최종 acc :', acc) # {'optimizer': 'nadam', 'node': 256, 'drop': 0.2, 'batch_size': 20, 'activation': 'selu'} # search.best_score_ : 0.9435579180717468
model.add(GlobalMaxPooling1D()) model.add(Dropout(0.5)) model.add(Flatten()) model.add(Dropout(0.5)) model.add(Dense(10, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) epochs = 100 batch_size = 32 history = model.fit(train_padded, training_labels, shuffle=True , epochs=epochs, batch_size=batch_size, validation_split=0.2, callbacks=[ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=10, min_lr=0.0001), EarlyStopping(monitor='val_loss', mode='min', patience=10, verbose=1), EarlyStopping(monitor='val_accuracy', mode='max', patience=10, verbose=1)]) plt.title('Loss') plt.plot(history.history['loss'], label='train') plt.plot(history.history['val_loss'], label='test') plt.legend() plt.show(); plt.title('Accuracy') plt.plot(history.history['accuracy'], label='train') plt.plot(history.history['val_accuracy'], label='test') plt.legend() plt.show();
train_generator = train_datagen.flow_from_directory(trainingdir, target_size=(150, 150), batch_size=20, class_mode='binary') validation_generator = validation_datagen.flow_from_directory( validation_dir, target_size=(150, 150), batch_size=20, class_mode='binary') from keras.callbacks import ModelCheckpoint, EarlyStopping # Callbacks checkpoint = ModelCheckpoint(filepath='best_weights.hdf5', save_best_only=True, save_weights_only=True) lr_reduce = ReduceLROnPlateau(monitor='val_loss', factor=0.3, patience=2, verbose=2, mode='max') early_stop = EarlyStopping(monitor='val_loss', min_delta=0.1, patience=1, mode='min') history = model.fit_generator(train_generator, epochs=20, steps_per_epoch=5126 // 32, validation_steps=624 // 32, validation_data=validation_generator, verbose=1, callbacks=[checkpoint, early]) """**fitting the model2**"""
def train_survival(base_path: str, excel_path: str, create_csv: bool, training_model_folder: str, epochs: int, mask: bool, mixed_data: bool, dict_struct_data: dict = None, mixed_data_info: list = None) -> dict: """ Retrieve data (niftis/survival data) and trains the neural network for survival analysis with the given parameters Args: base_path (str): path to the directory containing the nifti files (CT and PT scans) excel_path (str): file's path to the excel containing the survival information and structured data create_csv (bool): file's path to save the data retrieved. training_model_folder (str): path to the directory where the training information will be saved epochs (int): number of epochs of the training mask (bool): True if mask is part of the input data (combined with PT and CT scans). mixed_data (bool): True if structured data added to image data. Default to False dict_struct_data (dict): dictionary associating criteria to the columns number of the excel file containg the criteria. Default to None. mixed_data_info (list): list containing the different criteria that can be used. Default to None. Returns: (dict): return a dictionnary containing the different loss and metrics calculated during the training """ now = datetime.now().strftime("%Y%m%d-%H:%M:%S") mixed_data_columns = [] #automatic filling for i in mixed_data_info: mixed_data_columns.append(dict_struct_data[i]) logdir = os.path.join( training_model_folder, 'logs') #path for logs of the trained model (tensorboard) if not os.path.exists(logdir): os.makedirs(logdir) ########################## PARAMETERS INPUT DATA ########################## reduced_size = 0.5 # needed if batch_size to big for GPU (divided images size by 2 => reduced_size == 0.5) => standard size: (256,128,128) (reduced_size == 0) val_size = 0.20 #0.2 => 20% of the data is for the validation last_layer = "softmax" #last laayer of the neural network ########################## IMAGE PROCESSING PARAMETERS ########################## modalities = ('pet_img', 'ct_img') #input neural network ct and pet image target_direction = (1, 0, 0, 0, 1, 0, 0, 0, 1) target_size = (128, 128, 256) target_spacing = (4.0, 4.0, 4.0) if reduced_size != 0: target_size = tuple(round(x * reduced_size) for x in target_size) target_spacing = tuple(round(x / reduced_size) for x in target_spacing) ########################## DATA MANAGER ########################## DM = DataManagerSurvival(base_path, excel_path, mask, mixed_data, mixed_data_columns, pod24=False) DM.get_data_survival(create_csv) train_dataset, val_dataset = DM.dataset_survival(modalities, mask, target_size, target_spacing, target_direction) batch_size_train = DM.batch_size_train num_train_data = DM.num_train_data batch_size_val = DM.batch_size_val num_val_data = DM.num_val_data time_horizon = DM.time_horizon #number of neurons on the last layer censure_val = DM.censure_val ########################## MODEL LOSS METRICS OPTIMIZER ########################## alpha = 0.6 #cross_entropy loss factor beta = 0.4 # ranking loss factor # definition of loss, optimizer and metrics optimizer = tfa.optimizers.AdamW(learning_rate=1e-4, weight_decay=1e-4) loss = get_loss_survival(time_horizon_dim=time_horizon, alpha=alpha, beta=beta) c_index = metric_cindex(time_horizon_dim=time_horizon) metrics = [c_index] #loss = get_loss_cox #metrics = [concordance_index_censored_cox] #c_index_weighted= metric_cindex_weighted(time_horizon_dim=time_horizon,batch_size=batch_size, y_val=y_val) #td_c_index = metric_td_c_index(time_horizon_dim=time_horizon, batch_size=batch_size) ########################## CALLBACKS PARAMETERS ########################## patience = 10 ReduceLROnPlateau1 = False EarlyStopping1 = False ModelCheckpoint1 = True TensorBoard1 = True callbacks = [] if ReduceLROnPlateau1 == True: # reduces learning rate if no improvement are seen learning_rate_reduction = ReduceLROnPlateau(monitor=loss, patience=patience, verbose=1, factor=0.5, min_lr=0.0000001) callbacks.append(learning_rate_reduction) if EarlyStopping1 == True: # stop training if no improvements are seen early_stop = EarlyStopping(monitor="val_loss", mode="min", patience=int(patience // 2), restore_best_weights=True) callbacks.append(early_stop) if ModelCheckpoint1 == True: # saves model weights to file # 'model_weights.{epoch:02d}-{val_loss:.2f}.hdf5' checkpoint = ModelCheckpoint( os.path.join(training_model_folder, 'model_weights.h5'), monitor='val_loss', #td_c_index ? verbose=1, save_best_only=True, mode='min', # max save_weights_only=False) callbacks.append(checkpoint) if TensorBoard1 == True: tensorboard_callback = TensorBoard(log_dir=logdir, histogram_freq=0, update_freq='epoch', write_graph=True, write_images=True) callbacks.append(tensorboard_callback) ########################## DEFINITION MODEL ########################## dim_mlp = len( mixed_data_columns) #dim input layer structured data neural network in_channels = len(modalities) if mask: in_channels += 1 out_channels = 1 channels_last = True keep_prob = 1 keep_prob_last_layer = 0.8 kernel_size = (5, 5, 5) num_channels = 12 num_levels = 4 num_convolutions = (1, 2, 3) num_levels = len(num_convolutions) bottom_convolutions = 1 activation = "relu" image_shape = (256, 128, 128) if reduced_size != 0: image_shape = tuple(round(x * reduced_size) for x in image_shape) if mixed_data: architecture = 'vnet_survival_mixed_data' else: architecture = 'vnet_survival' model = VnetSurvival(image_shape, in_channels, out_channels, time_horizon, channels_last, keep_prob, keep_prob_last_layer, kernel_size, num_channels, num_levels, num_convolutions, bottom_convolutions, activation, last_layer, activation, mixed_data, dim_mixed_data_input=dim_mlp).create_network() model.compile(loss=loss, optimizer=optimizer, metrics=metrics) print(model.summary()) model_json = model.to_json() with open( os.path.join( training_model_folder, 'architecture_{}_model_{}.json'.format(architecture, now)), "w") as json_file: json_file.write(model_json) ########################## TRAINING MODEL ########################## history = model.fit( train_dataset, steps_per_epoch=int(batch_size_train / num_train_data), validation_data=val_dataset, validation_steps=int(batch_size_val / num_val_data), epochs=epochs, callbacks=callbacks, # initial_epoch=0, verbose=1) ########################### RESULTS ########################## loss_and_metrics = {} censure_val = [] print(history.history.keys()) print(f'Loss :') index_loss_val = np.argmin(history.history['val_loss']) for i in history.history.keys(): loss_and_metrics[i] = [] if history.history[i][index_loss_val] != -1: loss_and_metrics[i].append(history.history[i][index_loss_val]) print(i + " : " + str(history.history[i][index_loss_val])) x_axis = np.arange(0, len(history.history['loss'])) fig, (ax1, ax2) = plt.subplots(1, 2) plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['validation', 'training']) fig.suptitle('Loss and accuracy plot') ax1.plot(x_axis, history.history['val_loss'], x_axis, history.history['loss']) ax2.plot(x_axis, history.history['val_cindex'], x_axis, history.history['cindex']) #plot when loss and metrics are based on cox #ax1.plot(x_axis, history.history['val_loss'],x_axis, history.history['loss']) #ax2.plot(x_axis, history.history['val_concordance_index_censored_cox'], x_axis, history.history['concordance_index_censored_cox']) #plt.plot(x_axis, history.history['val_loss'],x_axis, history.history['loss']) #plt.title('Loss and accuracy model no '+str(fold_no)) #fig.legend(['val_loss', 'loss', 'val_accuracy', 'accuracy']) for i in history.history.keys(): if history.history[i][index_loss_val] != -1: loss_and_metrics[i].append(history.history[i][index_loss_val]) print(i + " : " + str(history.history[i][index_loss_val])) create_info_file(loss_and_metrics, training_model_folder, mask, mixed_data, mixed_data_info, batch_size_train, batch_size_val, epochs, reduced_size, alpha, beta, censure_val, num_convolutions) plt.savefig(training_model_folder + '/loss.png') plt.show() return loss_and_metrics
start_time = datetime.now().strftime('%Y%m%d_%H%M%S') os.makedirs('models', exist_ok=True) model.fit(x=x_train, y=y_train, batch_size=256, epochs=10, callbacks=[ ModelCheckpoint('./models/%s.h5' % (start_time), monitor='val_acc', verbose=1, save_best_only=True, mode='auto'), ReduceLROnPlateau(monitor='val_acc', factor=0.2, patience=5, verbose=1, mode='auto') ], validation_data=(x_val, y_val), use_multiprocessing=True, workers=16) i = 0 for y in range(h): for x in range(w): print('%2d' % x_val[i][y, x], end='') print() y_pred = model.predict(np.expand_dims(x_val[i], axis=0)).squeeze()
def cross_validation_survival(K: int, base_path: str, excel_path: str, create_csv: bool, training_model_folder: str, epochs: int, mask: bool, mixed_data: bool, dict_struct_data: dict = None, mixed_data_info: list = None) -> dict: """ Retrieve data (niftis/survival data) and trains the neural network for survival analysis with the given parameters using cross validation : resampling procedure used to evaluate machine learning models on a limited data sample. Args: K (int): K refers to the number of groups that a given data sample is to be split into. base_path (str): path to the directory containing the nifti files (CT and PT scans) excel_path (str): file's path to the excel containing the survival information and structured data create_csv (bool): file's path to save the data retrieved. training_model_folder (str): path to the directory where the training information will be saved epochs (int): number of epochs of the training mask (bool): True if mask is part of the input data (combined with PT and CT scans). mixed_data (bool): True if structured data added to image data. Default to False dict_struct_data (dict): dictionary associating criteria to the columns number of the excel file containg the criteria. Default to None. mixed_data_info (list): list containing the different criteria that can be used. Default to None. Returns: (dict): return a dictionnary containing the different loss and metrics calculated during the training """ mixed_data_columns = [] #automatic filling for i in mixed_data_info: mixed_data_columns.append(dict_struct_data[i]) now = datetime.now().strftime("%Y%m%d-%H:%M:%S") logdir = os.path.join( training_model_folder, 'logs') #path for logs of the trained model (tensorboard) if not os.path.exists(logdir): os.makedirs(logdir) reduced_size = 0.5 # needed if batch_size to big for GPU (divided images size by 2 => reduced_size == 0.5) => standard size: (256,128,128) (reduced_size == 0 ########################## IMAGE PROCESSING PARAMETERS ########################## modalities = ('pet_img', 'ct_img') #input neural network ct and pet image target_direction = (1, 0, 0, 0, 1, 0, 0, 0, 1) target_size = (128, 128, 256) target_spacing = (4.0, 4.0, 4.0) if reduced_size != 0: target_size = tuple(round(x * reduced_size) for x in target_size) target_spacing = tuple(round(x / reduced_size) for x in target_spacing) ########################## DATA MANAGER ########################## DM = DataManagerSurvival(base_path, excel_path, mask, mixed_data, mixed_data_columns, pod24=False) DM.get_data_survival(create_csv) if mixed_data: struct_data = DM.struct_data else: struct_data = None x = DM.nifti y = DM.time loss_and_metrics = {} kfold = KFold(n_splits=K, shuffle=True) fold_no = 1 time_horizon = math.ceil(max(abs(y)) * 1.2) #number of neurons on the output layer last_layer = "softmax" patience = 10 ReduceLROnPlateau1 = True EarlyStopping1 = False ModelCheckpoint1 = True TensorBoard1 = True patience_ES = 30 censure_val = [] for train, val in kfold.split(x, y, struct_data): print('----------------------------------------------------') print(f'Training for fold {fold_no} ...') alpha = 0.4 #cross_entropy loss factor beta = 0.6 # ranking loss factor optimizer = tf.keras.optimizers.RMSprop(lr=0.0001) #optimizer = tfa.optimizers.AdamW(learning_rate=1e-4, weight_decay=1e-4) loss = get_loss_survival(time_horizon_dim=time_horizon, alpha=alpha, beta=beta) c_index = metric_cindex(time_horizon_dim=time_horizon) metrics = [c_index] #loss = get_loss_cox #metrics = [concordance_index_censored_cox] print('----------------------------------------------------') print(f'Training for fold {fold_no} ...') logdir_fold = logdir + '_fold_no_' + str(fold_no) if not os.path.exists(logdir_fold): os.makedirs(logdir_fold) callbacks = [] if ReduceLROnPlateau1 == True: # reduces learning rate if no improvement are seen learning_rate_reduction = ReduceLROnPlateau(monitor='loss', patience=patience, verbose=1, factor=0.5, min_lr=0.0000001) callbacks.append(learning_rate_reduction) if EarlyStopping1 == True: # stop training if no improvements are seen early_stop = EarlyStopping(monitor="val_loss", mode="min", patience=int(patience_ES), restore_best_weights=True) callbacks.append(early_stop) if ModelCheckpoint1 == True: # saves model weights to file weights_file = 'model_weights_' + str(fold_no) + '.h5' checkpoint = ModelCheckpoint( os.path.join(training_model_folder, weights_file), monitor='val_loss', verbose=1, save_best_only=True, mode='min', # max save_weights_only=False) callbacks.append(checkpoint) if TensorBoard1 == True: tensorboard_callback = TensorBoard(log_dir=logdir_fold, histogram_freq=0, update_freq='epoch', write_graph=True, write_images=True) callbacks.append(tensorboard_callback) #c_index_weighted= metric_cindex_weighted(time_horizon_dim=time_horizon,batch_size=batch_size, y_val=y_val) #td_c_index = metric_td_c_index(time_horizon_dim=time_horizon, batch_size=batch_size) #IMAGE PROCESSING #train_transforms = get_transform('train', modalities,target_size, target_spacing, target_direction, None, data_augmentation = True, mask_survival= mask) #val_transforms = get_transform('val', modalities, target_size, target_spacing, target_direction, None, data_augmentation = False, mask_survival= mask) dict_data = {} dict_data['x_train'] = [x[element] for element in train] dict_data['x_val'] = [x[element] for element in val] dict_data['y_train'] = [y[element] for element in train] dict_data['y_val'] = [y[element] for element in val] batch_size_train = len(dict_data['y_train']) batch_size_val = len(dict_data['y_val']) #censure_val.append(np.sum(np.where(np.array(y_val)<0, 1, 0))/len(y_val)*100) #print("Nombre patients non censurés fold no ", fold_no, " training : ", np.sum(np.where(np.array(y_train)<0, 0, 1))) #print("Censure fold no ", fold_no, " training :", np.sum(np.where(np.array(y_train)<0, 1, 0))/len(y_train), "%") #print("Censure fold no ", fold_no, " validation :", np.sum(np.where(np.array(y_val)<0, 0, 1))) #print("Censure fold no ", fold_no, " validation :", np.sum(np.where(np.array(y_val)<0, 1, 0))/len(y_val), "%") if mixed_data: dict_data['train_struct_data'] = [ struct_data[element] for element in train ] dict_data['val_struct_data'] = [ struct_data[element] for element in val ] train_dataset, val_dataset = DM.cross_val_dataset( dict_data, modalities, mask, target_size, target_spacing, target_direction, None) print(val_dataset) in_channels = len(modalities) if mask: in_channels += 1 out_channels = 1 channels_last = True keep_prob = 1 keep_prob_last_layer = 0.8 kernel_size = (5, 5, 5) num_channels = 12 num_levels = 4 num_convolutions = (1, 2, 1) num_levels = len(num_convolutions) bottom_convolutions = 1 activation = "relu" image_shape = (256, 128, 128) if reduced_size != 0: image_shape = tuple(round(x * reduced_size) for x in image_shape) if mixed_data: dim_mlp = len(mixed_data_columns ) #dim input layer structured data neural network else: dim_mlp = 0 if mixed_data: architecture = 'vnet_survival_mixed_data' else: architecture = 'vnet_survival_cross' model = VnetSurvival(image_shape, in_channels, out_channels, time_horizon, channels_last, keep_prob, keep_prob_last_layer, kernel_size, num_channels, num_levels, num_convolutions, bottom_convolutions, activation, last_layer, activation, mixed_data, dim_mixed_data_input=dim_mlp).create_network() model.compile(loss=loss, optimizer=optimizer, metrics=metrics) print(model.summary()) model_json = model.to_json() with open( os.path.join( training_model_folder, 'architecture_{}_model_{}.json'.format(architecture, now)), "w") as json_file: json_file.write(model_json) history = model.fit( train_dataset, steps_per_epoch=int(len(dict_data['y_train']) / batch_size_train), validation_data=val_dataset, validation_steps=int(len(dict_data['y_val']) / batch_size_val), epochs=epochs, callbacks=callbacks, # initial_epoch=0, verbose=1) print(history.history.keys()) print(f'Loss for fold {fold_no}:') index_loss_val = np.argmin(history.history['val_loss']) if fold_no == 1: for i in history.history.keys(): loss_and_metrics[i] = [] for i in history.history.keys(): if history.history[i][index_loss_val] != -1: loss_and_metrics[i].append(history.history[i][index_loss_val]) print(i + " : " + str(history.history[i][index_loss_val])) x_axis = np.arange(0, len(history.history['loss'])) fig, (ax1, ax2) = plt.subplots(1, 2) plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['validation', 'training']) fig.suptitle('Loss and accuracy plot') ax1.plot(x_axis, history.history['val_loss'], x_axis, history.history['loss']) ax2.plot(x_axis, history.history['val_cindex'], x_axis, history.history['cindex']) #ax1.plot(x_axis, history.history['val_loss'],x_axis, history.history['loss']) #ax2.plot(x_axis, history.history['val_concordance_index_censored_cox'], x_axis, history.history['concordance_index_censored_cox']) #plt.plot(x_axis, history.history['val_loss'],x_axis, history.history['loss']) #plt.title('Loss and accuracy model no '+str(fold_no)) #fig.legend(['val_loss', 'loss', 'val_accuracy', 'accuracy']) plt.savefig(training_model_folder + '/loss_' + str(fold_no) + '.png') plt.show() fold_no += 1 print('-----------------------------------------------------------------') print(f'Loss and metrics per fold') for i in loss_and_metrics.keys(): print( '------------------------------------------------------------------' ) for j in range(len(loss_and_metrics[i])): print(f'> {i}: {loss_and_metrics[i][j]}') print( '------------------------------------------------------------------' ) print(f'Loss and metrics per fold average and meean') print('------------------------------------------------------------------') for i in loss_and_metrics.keys(): print( f'> {i}: {np.mean(loss_and_metrics[i])}(+- {np.std(loss_and_metrics[i])})' ) print('------------------------------------------------------------------') for i in censure_val: print("censure validation fold : ", i) create_info_file(loss_and_metrics, training_model_folder, mask, mixed_data, mixed_data_info, batch_size_train, batch_size_val, epochs, reduced_size, alpha, beta, censure_val, num_convolutions) return 0
def main(_argv): physical_devices = tf.config.experimental.list_physical_devices('GPU') for physical_device in physical_devices: tf.config.experimental.set_memory_growth(physical_device, True) if FLAGS.tiny: model = YoloV3Tiny(FLAGS.size, training=True, classes=FLAGS.num_classes) anchors = yolo_tiny_anchors anchor_masks = yolo_tiny_anchor_masks else: model = YoloV3(FLAGS.size, training=True, classes=FLAGS.num_classes) anchors = yolo_anchors anchor_masks = yolo_anchor_masks train_dataset = dataset.load_fake_dataset() if FLAGS.dataset: train_dataset = dataset.load_tfrecord_dataset( FLAGS.dataset, FLAGS.classes, FLAGS.size) train_dataset = train_dataset.shuffle(buffer_size=512) train_dataset = train_dataset.batch(FLAGS.batch_size) train_dataset = train_dataset.map(lambda x, y: ( dataset.transform_images(x, FLAGS.size), dataset.transform_targets(y, anchors, anchor_masks, FLAGS.size))) train_dataset = train_dataset.prefetch( buffer_size=tf.data.experimental.AUTOTUNE) val_dataset = dataset.load_fake_dataset() if FLAGS.val_dataset: val_dataset = dataset.load_tfrecord_dataset( FLAGS.val_dataset, FLAGS.classes, FLAGS.size) val_dataset = val_dataset.batch(FLAGS.batch_size) val_dataset = val_dataset.map(lambda x, y: ( dataset.transform_images(x, FLAGS.size), dataset.transform_targets(y, anchors, anchor_masks, FLAGS.size))) # Configure the model for transfer learning if FLAGS.transfer == 'none': pass # Nothing to do elif FLAGS.transfer in ['darknet', 'no_output']: # Darknet transfer is a special case that works # with incompatible number of classes # reset top layers if FLAGS.tiny: model_pretrained = YoloV3Tiny( FLAGS.size, training=True, classes=FLAGS.weights_num_classes or FLAGS.num_classes) else: model_pretrained = YoloV3( FLAGS.size, training=True, classes=FLAGS.weights_num_classes or FLAGS.num_classes) model_pretrained.load_weights(FLAGS.weights) if FLAGS.transfer == 'darknet': model.get_layer('yolo_darknet').set_weights( model_pretrained.get_layer('yolo_darknet').get_weights()) freeze_all(model.get_layer('yolo_darknet')) elif FLAGS.transfer == 'no_output': for l in model.layers: if not l.name.startswith('yolo_output'): l.set_weights(model_pretrained.get_layer( l.name).get_weights()) freeze_all(l) else: # All other transfer require matching classes print('yes') model.load_weights(FLAGS.weights) if FLAGS.transfer == 'fine_tune': # freeze darknet and fine tune other layers darknet = model.get_layer('yolo_darknet') freeze_all(darknet) elif FLAGS.transfer == 'frozen': # freeze everything freeze_all(model) optimizer = tf.keras.optimizers.Adam(lr=FLAGS.learning_rate) loss = [YoloLoss(anchors[mask], classes=FLAGS.num_classes) for mask in anchor_masks] if FLAGS.mode == 'eager_tf': # Eager mode is great for debugging # Non eager graph mode is recommended for real training avg_loss = tf.keras.metrics.Mean('loss', dtype=tf.float32) avg_val_loss = tf.keras.metrics.Mean('val_loss', dtype=tf.float32) for epoch in range(1, FLAGS.epochs + 1): for batch, (images, labels) in enumerate(train_dataset): with tf.GradientTape() as tape: outputs = model(images, training=True) regularization_loss = tf.reduce_sum(model.losses) pred_loss = [] for output, label, loss_fn in zip(outputs, labels, loss): pred_loss.append(loss_fn(label, output)) total_loss = tf.reduce_sum(pred_loss) + regularization_loss grads = tape.gradient(total_loss, model.trainable_variables) optimizer.apply_gradients( zip(grads, model.trainable_variables)) logging.info("{}_train_{}, {}, {}".format( epoch, batch, total_loss.numpy(), list(map(lambda x: np.sum(x.numpy()), pred_loss)))) avg_loss.update_state(total_loss) for batch, (images, labels) in enumerate(val_dataset): outputs = model(images) regularization_loss = tf.reduce_sum(model.losses) pred_loss = [] for output, label, loss_fn in zip(outputs, labels, loss): pred_loss.append(loss_fn(label, output)) total_loss = tf.reduce_sum(pred_loss) + regularization_loss logging.info("{}_val_{}, {}, {}".format( epoch, batch, total_loss.numpy(), list(map(lambda x: np.sum(x.numpy()), pred_loss)))) avg_val_loss.update_state(total_loss) logging.info("{}, train: {}, val: {}".format( epoch, avg_loss.result().numpy(), avg_val_loss.result().numpy())) avg_loss.reset_states() avg_val_loss.reset_states() model.save_weights( 'checkpoints/yolov3_train_{}.tf'.format(epoch)) else: model.compile(optimizer=optimizer, loss=loss, run_eagerly=(FLAGS.mode == 'eager_fit')) callbacks = [ ReduceLROnPlateau(verbose=1), EarlyStopping(patience=3, verbose=1), ModelCheckpoint('checkpoints/yolov3_train_{epoch}.tf', verbose=1, save_weights_only=True), TensorBoard(log_dir='logs') ] history = model.fit(train_dataset, epochs=FLAGS.epochs, callbacks=callbacks, validation_data=val_dataset)
def train(cfg, args): # Basic configurations for GPU and CPU os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu) if args.gpu > -1: gpus = tf.config.list_physical_devices('GPU') for gpu in gpus: tf.config.experimental.set_memory_growth(gpu, True) # Reading data X, idx = read_genus_abu(args.input) Y = read_labels(args.labels, shuffle_idx=idx, dmax=get_dmax(args.labels)) print('Reordering labels and samples...') IDs = sorted( list(set(X.index.to_list()).intersection(Y[0].index.to_list()))) X = X.loc[IDs, :] Y = [y.loc[IDs, :] for y in Y] print('Total matched samples:', sum(X.index == Y[0].index)) # Reading basic configurations from config file pretrain_ep = cfg.getint('train', 'pretrain_ep') pretrain_lr = cfg.getfloat('train', 'pretrain_lr') lr = cfg.getfloat('train', 'lr') epochs = cfg.getint('train', 'epochs') reduce_patience = cfg.getint('train', 'reduce_patience') stop_patience = cfg.getint('train', 'stop_patience') batch_size = cfg.getint('train', 'batch_size') lrreducer = ReduceLROnPlateau(monitor='val_loss', patience=reduce_patience, min_lr=1e-5, verbose=5, factor=0.1) stopper = EarlyStopping(monitor='val_loss', patience=stop_patience, verbose=5, restore_best_weights=True) pretrain_callbacks = [] callbacks = [lrreducer, stopper] if args.log: pretrain_logger = CSVLogger(filename=args.log) logger = CSVLogger(filename=args.log) pretrain_callbacks.append(pretrain_logger) callbacks.append(logger) phylogeny = pd.read_csv(find_pkg_resource('resources/phylogeny.csv'), index_col=0) pretrain_opt = Adam(lr=pretrain_lr) optimizer = Adam(lr=lr) dropout_rate = args.dropout_rate # Calculate sample weight for each layer, assign 0 weight for sample with 0 labels #sample_weight = [zero_weight_unk(y=y, sample_weight=compute_sample_weight(class_weight='balanced', # y=y.to_numpy().argmax(axis=1))) # for i, y in enumerate(Y_train)] # Build the model ontology = load_otlg(args.otlg) _, layer_units = parse_otlg(ontology) model = Model(phylogeny=phylogeny, num_features=X.shape[1], ontology=ontology, dropout_rate=dropout_rate) # Feature encoding and standardization X = model.encoder.predict(X, batch_size=batch_size) X = X.reshape(X.shape[0], X.shape[1] * X.shape[2]) print('N. NaN in input features:', np.isnan(X).sum()) model.update_statistics(mean=X.mean(axis=0), std=X.std(axis=0)) X = model.standardize(X) #------------------------------- SELECTIVE LEARNING----------------------------------------------- # Sample weight "zero" to mask unknown samples' contribution to loss sample_weight = [ zero_weight_unk(y=y, sample_weight=np.ones(y.shape[0])) for i, y in enumerate(Y) ] Y = [y.drop(columns=['Unknown']) for y in Y] # Train EXPERT model print('Pre-training using Adam with lr={}...'.format(pretrain_lr)) model.nn.compile(optimizer=pretrain_opt, loss=BinaryCrossentropy(from_logits=True), loss_weights=(np.array(layer_units) / sum(layer_units)).tolist(), weighted_metrics=[BinaryAccuracy(name='acc')]) model.nn.fit(X, Y, validation_split=args.val_split, batch_size=batch_size, epochs=pretrain_ep, sample_weight=sample_weight, callbacks=pretrain_callbacks) model.nn.summary() print('Training using Adam with lr={}...'.format(lr)) model.nn.compile(optimizer=optimizer, loss=BinaryCrossentropy(from_logits=True), loss_weights=(np.array(layer_units) / sum(layer_units)).tolist(), weighted_metrics=[BinaryAccuracy(name='acc')]) model.nn.fit(X, Y, validation_split=args.val_split, batch_size=batch_size, initial_epoch=pretrain_ep, epochs=epochs + pretrain_ep, sample_weight=sample_weight, callbacks=callbacks) # Save the EXPERT model model.save_blocks(args.output)
warmup_steps = int(warmup_epoch * num_train / batch_size) # 学习率 reduce_lr = WarmUpCosineDecayScheduler( learning_rate_base=learning_rate_base, total_steps=total_steps, warmup_learning_rate=1e-4, warmup_steps=warmup_steps, hold_base_rate_steps=num_train, min_learn_rate=1e-6) model.compile(optimizer=Adam(), loss={ 'yolo_loss': lambda y_true, y_pred: y_pred }) else: reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, verbose=1) model.compile(optimizer=Adam(learning_rate_base), loss={ 'yolo_loss': lambda y_true, y_pred: y_pred }) print('Train on {} samples, val on {} samples, with batch size {}.'. format(num_train, num_val, batch_size)) model.fit( data_generator(lines[:num_train], batch_size, input_shape, anchors, num_classes, mosaic=mosaic),
from tensorflow.keras.losses import SparseCategoricalCrossentropy from tensorflow.keras.metrics import SparseCategoricalAccuracy from tensorflow.keras.optimizers import Adam # %load_ext tensorboard LEARNING_RATE = 1e-3 EARLY_STOPPING_CRITERIA = 3 NUM_EPOCHS = 50 # Compile model.compile(optimizer=Adam(lr=LEARNING_RATE), loss=SparseCategoricalCrossentropy(), metrics=[SparseCategoricalAccuracy()]) # Callbacks callbacks = [EarlyStopping(monitor='val_loss', patience=EARLY_STOPPING_CRITERIA, verbose=1, mode='min'), ReduceLROnPlateau(patience=1, factor=0.1, verbose=0), TensorBoard(log_dir='tensorboard', histogram_freq=1, update_freq='epoch')] # Training training_history = model.fit(x=X_train, y=y_train, epochs=NUM_EPOCHS, validation_data=(X_test, y_test), callbacks=callbacks, batch_size=64, verbose=1) """##Validation Data""" def get_performance(y_true, y_pred, classes):
return model #next step model = create_model(input_shape=(HEIGHT, WIDTH, COLORS), n_out=N_CLASSES) model.summary() metric_list = ["accuracy"] optimizer = optimizers.Adam(lr=WARMUP_LEARNING_RATE) model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=metric_list) rlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=RLROP_PATIENCE, factor=DECAY_DROP, min_lr=1e-7, verbose=1) callback_list = [rlrop] # warm up training phase, only two epochs STEP_SIZE_TRAIN = train_generator.n // train_generator.batch_size STEP_SIZE_VALID = val_generator.n // val_generator.batch_size history_warmup = model.fit_generator(generator=train_generator, steps_per_epoch=STEP_SIZE_TRAIN, validation_data=val_generator, validation_steps=STEP_SIZE_VALID, epochs=WARMUP_EPOCHS, verbose=1).history
# compile the model (should be done after setting layers to non-trainable) model.compile( optimizer=optimizer, loss="categorical_crossentropy", metrics=['accuracy'], ) # train the model # define callback function early_stopping = EarlyStopping( monitor='val_loss', patience=10, ) reduce_lr = ReduceLROnPlateau( monitor='val_loss', factor=0.1, patience=5, ) model.fit_generator(train_generator, steps_per_epoch=train_image_numbers // batch_size, epochs=epochs, validation_data=validation_generator, validation_steps=validation_steps, callbacks=[early_stopping, reduce_lr]) # test the model test_metrics = model.evaluate_generator( test_generator, steps=1, )
print('\033[32;1mLoading Model\033[0m') model.load_weights(model_path) if training: trainX, validX, trainY_SOS, validY_SOS, trainY, validY, line_len, valid_line_len = train_data_preprocessing( inputX, word2idx, max_seq_len) print( f'\033[32;1mtrainX: {trainX.shape}, validX: {validX.shape}, trainY: {trainY.shape}, validY: {validY.shape}, trainY_SOS: {trainY_SOS.shape}, validY_SOS: {validY_SOS.shape}\033[0m' ) checkpoint = ModelCheckpoint(model_path, 'val_word_out_loss', verbose=1, save_best_only=True, save_weights_only=True) reduce_lr = ReduceLROnPlateau('val_word_out_loss', 0.5, 3, verbose=1, min_lr=1e-6) logger = CSVLogger(model_path + '.csv', append=True) tensorboard = TensorBoard(model_path[:model_path.rfind('.')] + '_logs', histogram_freq=1, batch_size=1024, write_grads=True, write_images=True, update_freq=512) epochs = 10 for epoch in range(epochs): print(f'\033[32;1mepoch: {epoch+1}/{epochs}\033[0m') ith, ith_str, word = generate_word(trainY, line_len) #print(' '.join([idx2word[i] for i in trainX[8347]]).strip(), ith[8347, 0], idx2word[word[8347, 0]]) #print(' '.join([idx2word[i] for i in trainY[8347]]).strip())
def train_res_net(rotation_range_var=10, zoom_range_var=0.1, width_shift_range_var=0.1, height_shift_range_var=0.1, horizontal_flip_var=True, vertical_flip_var=True, batch_size_var=32, activation_func_var='relu', epoch_var=30, steps_per_epoch_var=256, validation_steps_var=256): train_dir = '../../dump/intel-image-classification/seg_train/seg_train/' test_dir = '../../dump/intel-image-classification/seg_test/seg_test/' data_gen = ImageDataGenerator(featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=False, rotation_range=rotation_range_var, zoom_range=zoom_range_var, width_shift_range=width_shift_range_var, height_shift_range=height_shift_range_var, horizontal_flip=horizontal_flip_var, vertical_flip=vertical_flip_var) train_gen = data_gen.flow_from_directory(train_dir, target_size=(150, 150), batch_size=32, class_mode='categorical') test_gen = data_gen.flow_from_directory(test_dir, target_size=(150, 150), batch_size=32, class_mode='categorical') es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10) learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', patience=3, verbose=1, factor=0.25, min_lr=0.000001) optimizer = Adam(learning_rate=0.5e-4, beta_1=0.9, beta_2=0.999, amsgrad=False) base_model = ResNet50(include_top=False, weights='imagenet', input_shape=(150, 150, 3), pooling='avg') base_model.trainable = False x = Dense(512, activation='relu')(base_model.output) x = Dropout(0.5)(x) x = Dense(6, activation='softmax')(x) transfer_model = Model(base_model.input, x) transfer_model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) history = transfer_model.fit_generator( train_gen, steps_per_epoch=256, validation_data=test_gen, validation_steps=256, epochs=1, verbose=1, callbacks=[es, learning_rate_reduction]) transfer_model.save("res_model.model") def get_images(directory): Images = [] Labels = [] label = 0 for labels in os.listdir(directory): if labels == 'glacier': label = 2 elif labels == 'sea': label = 4 elif labels == 'buildings': label = 0 elif labels == 'forest': label = 1 elif labels == 'street': label = 5 elif labels == 'mountain': label = 3 for image_file in os.listdir(directory + labels): image = cv2.imread(directory + labels + r'/' + image_file) image = cv2.resize(image, (150, 150)) Images.append(image) Labels.append(label) return Images, Labels test_images, test_labels = get_images( '../input/intel-image-classification/seg_test/seg_test/') test_images = np.array(test_images) test_labels = np.array(test_labels) test_labels = to_categorical(test_labels) return transfer_model.evaluate(test_images, test_labels)