def compile_models(lstm_model, hyperparams): """ Compile full training model with choosen hyperparameters. Parameters ---------- lstm_model -> Keras model : Full training model to be compiled hyperparams -> dict : Dictionary containing all hyperparamers for model compilation Returns ------- lstm_model (compiled) -> keras compiled model : A fully-compiled keras LSTM model using the specified hyperparameters. """ clip_select_flag = hyperparams["clip_select_flag"] learning_rate = hyperparams["learning_rate"] BETA_1 = hyperparams["BETA_1"] BETA_2 = hyperparams["BETA_2"] lr_decay = hyperparams["lr_decay"] slow_weights = hyperparams["slow_weights_lookahead"] sync_lookahead = hyperparams["sync_lookahead"] warmup_RAdam = hyperparams["warmup_RAdam"] min_lr_RAdam = hyperparams["min_lr_RAdam"] weight_decay_RAdam = hyperparams["weight_decay_RAdam"] total_steps_RAdam = hyperparams["total_steps_RAdam"] clip_norm_thresh = hyperparams["clip_norm_thresh"] clip_val_thresh = hyperparams["clip_val_thresh"] ams_grad_flag = hyperparams["ams_grad_flag"] loss_function = hyperparams["loss_fcn"] optimizer = hyperparams["optimizer"] epsilon = hyperparams["epsilon"] if(clip_select_flag == "norm" and optimizer == "adam"): opt_norm_clip = Lookahead(keras.optimizers.adam(lr = learning_rate, beta_1 = BETA_1, beta_2 = BETA_2, decay = lr_decay, amsgrad = ams_grad_flag), sync_period = sync_lookahead, slow_step = slow_weights) lstm_model.compile(optimizer = opt_norm_clip, loss = loss_function, metrics = [rmse]) return lstm_model if(clip_select_flag == "norm" and optimizer == "nadam"): opt_norm_clip = keras.optimizers.nadam(lr = learning_rate, beta_1 = BETA_1, beta_2 = BETA_2, epsilon = epsilon, schedule_decay = lr_decay) lstm_model.compile(optimizer = opt_norm_clip, loss = loss_function, metrics = [rmse]) return lstm_model elif(clip_select_flag == "value" and optimizer == "adam"): opt_val_clip = Lookahead(keras.optimizers.adam(lr = learning_rate, beta_1 = BETA_1, beta_2 = BETA_2, decay = lr_decay, clipvalue = clip_val_thresh, amsgrad = ams_grad_flag), sync_period = sync_lookahead, slow_step = slow_weights) lstm_model.compile(optimizer = opt_val_clip, loss = loss_function, metrics = [rmse]) return lstm_model elif(clip_select_flag == "value" and optimizer == "nadam"): opt_val_clip = keras.optimizers.nadam(lr = learning_rate, beta_1 = BETA_1, beta_2 = BETA_2, epsilon = epsilon, schedule_decay = lr_decay) lstm_model.compile(optimizer = opt_val_clip, loss = loss_function, metrics = [rmse]) return lstm_model elif(optimizer == "RAdam"): opt = RAdam(learning_rate = learning_rate, beta_1 = BETA_1, beta_2 = BETA_2, epsilon = epsilon, weight_decay = weight_decay_RAdam, amsgrad = ams_grad_flag, total_steps = total_steps_RAdam, warmup_proportion = warmup_RAdam, min_lr = min_lr_RAdam) lstm_model.compile(optimizer = opt, loss = loss_function, metrics = [rmse]) return lstm_model elif(optimizer == "Ranger"): opt = Lookahead(RAdam(learning_rate = learning_rate, beta_1 = BETA_1, beta_2 = BETA_2, epsilon = epsilon, weight_decay = weight_decay_RAdam, amsgrad = ams_grad_flag, total_steps = total_steps_RAdam, warmup_proportion = warmup_RAdam, min_lr = min_lr_RAdam), sync_period = sync_lookahead, slow_step = slow_weights) lstm_model.compile(optimizer = opt, loss = loss_function, metrics = [rmse]) return lstm_model else: print(" Clipping Method OR Optimizer Selected is not avalaible! Please enter a valid string for these parameter: \n Valid Clipping:['norm', 'value'] \n Valid Optimizers: ['adam', 'NAdam', 'RAdam', 'Ranger']") return lstm_model
def train_model(input_data: str, test_file=None, batch_size=64, nb_epoch=100, early_stop_patience=None, mod=None, max_x_length=50, scale_para=None, unit="s", out_dir="./", prefix="test", p_model=None, model=None, optimizer_name=None, use_radam=False, add_reverse=False, add_ReduceLROnPlateau=False, use_external_test_data=True): """ Used by AutoRT :param input_data: :param test_file: :param batch_size: :param nb_epoch: :param early_stop_patience: :param mod: :param max_x_length: :param min_rt: :param max_rt: :param unit: :param out_dir: :param prefix: :param p_model: :param model: :param optimizer_name: :param use_radam: :param use_external_test_data: :return: """ res_map = dict() if not os.path.exists(out_dir): os.makedirs(out_dir) print("Build deep learning model ...") X_train, Y_train, X_test, Y_test, scale_para = data_processing( input_data=input_data, test_file=test_file, mod=mod, max_x_length=max_x_length, scale_para=scale_para, unit=unit, out_dir=out_dir, add_reverse=add_reverse) if model is None: print("Use default model ...") model = build_default_model(X_train.shape[1:]) else: print("Use input model ...") model = clone_model(model) if p_model is not None: transfer_layer = 5 frozen = True # model_copy.set_weights(model.get_weights()) if use_radam == True: base_model = load_model(p_model, custom_objects={ "Lookahead": Lookahead, "RAdam": RAdam }) else: base_model = load_model(p_model) print("Perform transfer learning ...") n_layers = len(base_model.layers) print("The number of layers: %d" % (n_layers)) for l in range((n_layers - transfer_layer)): if l != 0: model.layers[l].set_weights(base_model.layers[l].get_weights()) if frozen is True: model.layers[l].trainable = False print("layer (frozen:True): %d %s" % (l, model.layers[l].name)) else: print("layer (frozen:False): %d %s" % (l, model.layers[l].name)) if model.optimizer is None: ## use default optimizer: Adam if optimizer_name is None: if use_radam == True: print("Use optimizer: %s" % ("rectified-adam")) model.compile( loss='mean_squared_error', # math.ceil(X_train.shape[0]/batch_size*nb_epoch) optimizer=Lookahead(RAdam(), sync_period=5, slow_step=0.5)) #optimizer=Lookahead(RAdam(total_steps=math.ceil(X_train.shape[0]/batch_size), warmup_proportion=0.1, min_lr=1e-5, lr=0.001),sync_period=5, slow_step=0.5), #metrics=['mean_squared_error']) else: print("Use default optimizer:Adam") model.compile(loss='mean_squared_error', optimizer="adam") #metrics=['mean_squared_error']) else: if use_radam == True: print("Use optimizer: %s" % ("rectified-adam")) model.compile(loss='mean_squared_error', optimizer=Lookahead(RAdam(), sync_period=5, slow_step=0.5)) #optimizer=Lookahead(RAdam(total_steps=math.ceil(X_train.shape[0]/batch_size), warmup_proportion=0.1, min_lr=1e-5, lr=0.001), # sync_period=5, slow_step=0.5), #metrics=['mean_squared_error']) else: print("Use optimizer provided by user: %s" % (optimizer_name)) model.compile(loss='mean_squared_error', optimizer=optimizer_name) # Implementation from https://github.com/GLambard/AdamW_Keras #optimizer=Adam(amsgrad=True)) #optimizer=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)) #optimizer=Lookahead(AdamW(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8, decay=0., weight_decay=1e-4, batch_size=batch_size, samples_per_epoch=X_train.shape[0], epochs=nb_epoch),sync_period=5, slow_step=0.5)) #metrics=['mean_squared_error']) else: if optimizer_name is None: if use_radam == True: print("Use optimizer: %s" % ("rectified-adam")) model.compile(loss='mean_squared_error', optimizer=Lookahead(RAdam(), sync_period=5, slow_step=0.5)) #optimizer=Lookahead(RAdam(total_steps=math.ceil(X_train.shape[0]/batch_size), warmup_proportion=0.1, min_lr=1e-5, lr=0.001), # sync_period=5, slow_step=0.5), #metrics=['mean_squared_error']) else: print("Use optimizer from the model.") model.compile( loss='mean_squared_error', ## In this case, we cannot change the learning rate. optimizer=model.optimizer) #metrics=['mean_squared_error']) else: if use_radam == True: print("Use optimizer: %s" % ("rectified-adam")) model.compile(loss='mean_squared_error', optimizer=Lookahead(RAdam(), sync_period=5, slow_step=0.5)) #optimizer=Lookahead(RAdam(total_steps=math.ceil(X_train.shape[0]/batch_size), warmup_proportion=0.1, min_lr=1e-5, lr=0.001), # sync_period=5, slow_step=0.5), #metrics=['mean_squared_error']) else: print("Use optimizer provided by user: %s" % (optimizer_name)) model.compile( loss='mean_squared_error', ## In this case, we cannot change the learning rate. optimizer=optimizer_name) #metrics=['mean_squared_error']) print("optimizer: %s" % (type(model.optimizer))) model.summary() # model = multi_gpu_model(model, gpus=3) my_callbacks = RegCallback(X_train, X_test, Y_train, Y_test, scale_para) # Save model model_chk_path = out_dir + "/best_model.hdf5" mcp = ModelCheckpoint(model_chk_path, save_best_only=True, save_weights_only=False, verbose=1, mode='min') all_callbacks = list() all_callbacks.append(my_callbacks) all_callbacks.append(mcp) if add_ReduceLROnPlateau is True: print("Use ReduceLROnPlateau!") all_callbacks.append( keras.callbacks.ReduceLROnPlateau(patience=5, factor=0.5, verbose=1, min_lr=0.000001, min_delta=0)) if early_stop_patience is not None: print("Use EarlyStopping: %d" % (early_stop_patience)) all_callbacks.append( EarlyStopping(patience=early_stop_patience, verbose=1)) ## monitor training information # tbCallBack = callbacks.TensorBoard(log_dir='./Graph', histogram_freq=0, write_graph=True, write_images=True) #model.fit(X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, validation_data=(X_test, Y_test), callbacks=[my_callbacks, mcp]) if use_external_test_data is True: model.fit( X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, validation_data=(X_test, Y_test), # callbacks=[my_callbacks, mcp]) callbacks=all_callbacks) else: model.fit( X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, validation_split=0.1, # callbacks=[my_callbacks, mcp]) callbacks=all_callbacks) ## get the best model if use_radam == True: model_best = load_model(model_chk_path, custom_objects={ "Lookahead": Lookahead, "RAdam": RAdam }) else: model_best = load_model(model_chk_path, custom_objects={ "Lookahead": Lookahead, "RAdam": RAdam }) y_pred = model_best.predict(X_test) y_pred_rev = scaling_y_rev(y_pred, scale_para) y_true = scaling_y_rev(Y_test, scale_para) #test_data['pred'] = y_pred_rev #test_data.to_csv("pred.csv") x = pd.DataFrame({ "y": y_true, "y_pred": y_pred_rev.reshape(y_pred_rev.shape[0]) }) out_file = out_dir + "/" + prefix + ".csv" print("Prediction result: %s" % (out_file)) x.to_csv(out_file) res_map['model'] = model_best return res_map
def build_model(): con_filters = 128 left_input_go = Input(shape=(bert_len + w2v_len, project_dim)) right_input_go = Input(shape=(bert_len + w2v_len, project_dim)) NUM_FILTERS = 32 FILTER_LENGTH1 = 5 FILTER_LENGTH2 = 5 left_x_go = Conv1D(filters=NUM_FILTERS, kernel_size=FILTER_LENGTH1, activation='relu', padding='valid', strides=1)(left_input_go) left_x_go = Conv1D(filters=NUM_FILTERS * 2, kernel_size=FILTER_LENGTH1, activation='relu', padding='valid', strides=1)(left_x_go) left_x_go = Conv1D(filters=NUM_FILTERS * 3, kernel_size=FILTER_LENGTH1, activation='relu', padding='valid', strides=1)(left_x_go) left_x_go_max = GlobalMaxPooling1D()(left_x_go) #pool_size=pool_length[i] left_x_go_avg = GlobalAveragePooling1D()( left_x_go) #pool_size=pool_length[i] right_x_go = Conv1D(filters=NUM_FILTERS, kernel_size=FILTER_LENGTH2, activation='relu', padding='valid', strides=1)(right_input_go) right_x_go = Conv1D(filters=NUM_FILTERS * 2, kernel_size=FILTER_LENGTH2, activation='relu', padding='valid', strides=1)(right_x_go) right_x_go = Conv1D(filters=NUM_FILTERS * 3, kernel_size=FILTER_LENGTH2, activation='relu', padding='valid', strides=1)(right_x_go) right_x_go_max = GlobalMaxPooling1D()(right_x_go) right_x_go_avg = GlobalAveragePooling1D()(right_x_go) x = Concatenate()( [left_x_go_avg, left_x_go_max, right_x_go_avg, right_x_go_max]) x = Dense(1024, activation='relu')(x) x = Dropout(0.1)(x) x = Dense(1024, activation='relu')(x) x = Dropout(0.1)(x) x = Dense(512, activation='relu')(x) x = Dense(1)(x) output = Activation('sigmoid')(x) # model = Model([left_input_go, right_input_go], output) optimizer = Lookahead(RAdam()) model = Model([left_input_go, right_input_go], output) # model = multi_gpu_model(model, gpus=2) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) return model
def ensemble_models(input_data: str, test_file=None, models_file=None, ga_file=None, ensemble_method="average", batch_size=64, nb_epoch=100, mod=None, max_x_length=50, scale_para=None, unit="s", out_dir="./", prefix="test", use_radam=False, early_stop_patience=None, add_reverse=False, add_ReduceLROnPlateau=False): """ This function is used to ensemble multiple deep learning models. It can be used for training and testing. """ # print("The number of models:", len(models)) # test data X_test = np.empty(1) Y_test = np.empty(1) y_pr = [] score = [] model_list = dict() if ga_file is not None: X_train, Y_train, X_test, Y_test, scale_para = data_processing( input_data=input_data, test_file=test_file, mod=mod, max_x_length=max_x_length, scale_para=scale_para, unit=unit, out_dir=out_dir, add_reverse=add_reverse) model_list['dp_model'] = dict() model_list['max_x_length'] = X_train.shape[1] model_list['aa'] = out_dir + "/aa.tsv" ## Useful for new data prediction model_list['min_rt'] = scale_para['rt_min'] model_list['max_rt'] = scale_para['rt_max'] if add_reverse is True: model_list['add_reverse'] = 1 model_list['max_x_length'] = model_list['max_x_length'] / 2 else: model_list['add_reverse'] = 0 print("max_x_length: %s" % (max_x_length)) # read models from genetic search result configure file optimizer_name = dict() with open(ga_file, "r") as read_file: ga_model_list = json.load(read_file) model_folder = os.path.dirname(ga_file) models = dict() for i in ga_model_list.keys(): m_file = model_folder + "/" + os.path.basename( ga_model_list[i]['model']) print("Model file: %s -> %s" % (str(i), m_file)) with open(m_file, "r") as json_read: models[i] = keras.models.model_from_json(json_read.read()) models[i]._layers[1].batch_input_shape = (None, X_train.shape[1], X_train.shape[2]) optimizer_name[i] = ga_model_list[i]['optimizer_name'] print("Training ...") # For each model, train the model for (name, model) in models.items(): print("Train model:", name) # perform sample specific training res_map = train_model(input_data=input_data, test_file=test_file, batch_size=batch_size, nb_epoch=nb_epoch, early_stop_patience=early_stop_patience, mod=mod, max_x_length=max_x_length, scale_para=scale_para, unit=unit, out_dir=out_dir, prefix=str(name), model=model, optimizer_name=optimizer_name[name], use_radam=use_radam, add_reverse=add_reverse, add_ReduceLROnPlateau=add_ReduceLROnPlateau) ## save the model to a file: model_file_name = "model_" + str(name) + ".h5" model_file_path = out_dir + "/" + model_file_name res_map["model"].save(model_file_path) model_list['dp_model'][name] = model_file_path del res_map gc.collect() K.clear_session() tf.reset_default_graph() else: print("Transfer learning ...") ## Transfer learning with open(models_file, "r") as read_file: model_list = json.load(read_file) model_folder = os.path.dirname(models_file) aa_file = os.path.basename(model_list['aa']) aa_file = model_folder + "/" + aa_file new_model_list = dict() new_model_list['dp_model'] = dict() model_i = 1 ## peptide length check peptide_max_length = get_max_length_from_input_data(input_data) if peptide_max_length > model_list['max_x_length']: print( "The max length (%d) in the training data should be <= the length supported by the model %d" % (peptide_max_length, model_list['max_x_length'])) sys.exit() for (name, dp_model_file) in model_list['dp_model'].items(): print("\nDeep learning model:", name) X_train, Y_train, X_test, Y_test, scale_para = data_processing( input_data=input_data, test_file=test_file, mod=mod, max_x_length=model_list['max_x_length'], scale_para=scale_para, unit=unit, out_dir=out_dir, aa_file=aa_file, add_reverse=add_reverse, random_seed=model_i) model_i = model_i + 1 # keras model evaluation: loss and accuracy # load model model_name = os.path.basename(dp_model_file) model_full_path = model_folder + "/" + model_name if use_radam == True: model = load_model(model_full_path, custom_objects={ "Lookahead": Lookahead, "RAdam": RAdam }) else: model = load_model(model_full_path) #new_model = change_model(model, X_train.shape[1:]) new_model = model print(get_peptide_length_from_model(new_model)) if "add_reverse" in model_list.keys(): if model_list['add_reverse'] == 1: if 2 * model_list[ 'max_x_length'] != get_peptide_length_from_model( new_model): print( "The max length (%d) in the training data should be less than the length supported by the model %d" % (model_list['max_x_length'], get_peptide_length_from_model(new_model))) else: if model_list[ 'max_x_length'] != get_peptide_length_from_model( new_model): print( "The max length (%d) in the training data should be less than the length supported by the model %d" % (model_list['max_x_length'], get_peptide_length_from_model(new_model))) else: if model_list['max_x_length'] != get_peptide_length_from_model( new_model): print( "The max length (%d) in the training data should be less than the length supported by the model %d" % (model_list['max_x_length'], get_peptide_length_from_model(new_model))) print("Perform transfer learning ...") n_layers = len(new_model.layers) print("The number of layers: %d" % (n_layers)) ''' for layer in new_model.layers: layer_name = str(layer.name) if layer_name.startswith("bidirectional_"): break else: layer.trainable = False print("layer (frozen:True): %s" % (layer_name)) ''' print(model.optimizer) if use_radam == True: print("Use optimizer: %s" % ("rectified-adam")) new_model.compile( loss='mean_squared_error', ## In this case, we cannot change the learning rate. optimizer=Lookahead(RAdam(), sync_period=5, slow_step=0.5)) #optimizer=RAdam(), #optimizer=Lookahead(RAdam(total_steps=1000, warmup_proportion=0.1, min_lr=1e-5, lr=0.001), # sync_period=5, slow_step=0.5)) # optimizer=Adam(lr=0.0001), # optimizer=SGD(lr=1e-3, decay=1e-4, momentum=0.9, nesterov=True), #metrics=['mean_squared_error']) else: print("Use optimizer: %s from saved model" % (model.optimizer.__class__.__name__)) new_model.compile( loss='mean_squared_error', ## In this case, we cannot change the learning rate. #optimizer=model.optimizer) optimizer=model.optimizer.__class__.__name__) #optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=0.01/nb_epoch)) #optimizer=Adam(lr=0.001)) #optimizer=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)) #metrics=['mean_squared_error']) print("Used optimizer:") print(model.optimizer) my_callbacks = RegCallback(X_train, X_test, Y_train, Y_test, scale_para) # Save model model_chk_path = out_dir + "/best_model.hdf5" mcp = ModelCheckpoint(model_chk_path, save_best_only=True, save_weights_only=False, verbose=1, mode='min') all_callbacks = list() all_callbacks.append(my_callbacks) all_callbacks.append(mcp) if add_ReduceLROnPlateau is True: print("Use ReduceLROnPlateau!") all_callbacks.append( keras.callbacks.ReduceLROnPlateau(patience=3, factor=0.5, verbose=1, min_lr=0.00001, min_delta=0)) if early_stop_patience is not None: print("Use EarlyStopping: %d" % (early_stop_patience)) all_callbacks.append( EarlyStopping(patience=early_stop_patience, verbose=1)) #all_callbacks.append(LearningRateScheduler(PolynomialDecay(maxEpochs=nb_epoch, initAlpha=0.001, power=5))) ## monitor training information # tbCallBack = callbacks.TensorBoard(log_dir='./Graph', histogram_freq=0, write_graph=True, write_images=True) new_model.fit( X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, validation_data=(X_test, Y_test), callbacks=all_callbacks ) #, keras.callbacks.ReduceLROnPlateau(patience=3, factor=0.5, verbose=1, min_lr=0.000001)]) ## get the best model if use_radam == True: best_model = load_model(model_chk_path, custom_objects={ "Lookahead": Lookahead, "RAdam": RAdam }) else: best_model = load_model(model_chk_path) ## save the model to a file: model_file_name = "model_" + str(name) + ".h5" model_file_path = out_dir + "/" + model_file_name best_model.save(model_file_path) new_model_list['dp_model'][name] = model_file_path gc.collect() K.clear_session() tf.reset_default_graph() new_model_list['max_x_length'] = model_list['max_x_length'] new_aa_file = out_dir + "/" + os.path.basename(model_list['aa']) copyfile(aa_file, new_aa_file) new_model_list['aa'] = new_aa_file ## Useful for new data prediction #new_model_list['min_rt'] = scale_para['rt_min'] #new_model_list['max_rt'] = scale_para['rt_max'] model_list = new_model_list # save model data #file_all_models = open(out_dir + "/all_models.obj", 'wb') #pickle.dump(models, file_all_models) #file_all_models.close() #################################################################################################################### print("Ensemble learning ...") para = dict() para['rt_min'] = scale_para['rt_min'] para['rt_max'] = scale_para['rt_max'] para['scaling_method'] = scale_para['scaling_method'] model_list['rt_min'] = scale_para['rt_min'] model_list['rt_max'] = scale_para['rt_max'] model_list['scaling_method'] = scale_para['scaling_method'] if scale_para['scaling_method'] == "mean_std": para['rt_mean'] = scale_para['rt_mean'] para['rt_std'] = scale_para['rt_std'] model_list['rt_mean'] = scale_para['rt_mean'] model_list['rt_std'] = scale_para['rt_std'] elif scale_para['scaling_method'] == "single_factor": para['scaling_factor'] = scale_para['scaling_factor'] model_list['scaling_factor'] = scale_para['scaling_factor'] if add_reverse is True: model_list['add_reverse'] = 1 para['add_reverse'] = 1 else: model_list['add_reverse'] = 0 para['add_reverse'] = 0 ## save result model_json = out_dir + "/model.json" with open(model_json, 'w') as f: json.dump(model_list, f) ## evaluation if test_file is not None: ensemble_predict(model_json, x=X_test, y=Y_test, para=para, batch_size=batch_size, method=ensemble_method, out_dir=out_dir, prefix="final_eval")
def run(model, backbone, batchsize, lr, epochs, data_path, data_name, fold, labels, partial, illustrations, ngpus, save_model): BACKBONE = backbone BATCH_SIZE = batchsize LR = lr EPOCHS = epochs DATA_DIR = Path(data_path) / (data_name + ("_FOLD%d" % fold)) x_train_dir = os.path.join(DATA_DIR, 'train_images') y_train_dir = os.path.join(DATA_DIR, 'train_labels') x_valid_dir = os.path.join(DATA_DIR, 'val_images') y_valid_dir = os.path.join(DATA_DIR, 'val_labels') x_test_dir = os.path.join(DATA_DIR, 'test_images') y_test_dir = os.path.join(DATA_DIR, 'test_labels') preprocess_input = sm.get_preprocessing(BACKBONE) all_classes = json.loads(open(DATA_DIR / "meta" / "codes.json").read()) all_weights = json.loads(open(DATA_DIR / "meta" / "weights.json").read()) if labels == "txt": CLASSES = ['text', 'tabelleninhalt'] if illustrations: CLASSES.append('illustration') elif labels == "sep": CLASSES = ['tab', 'h', 'v'] else: raise RuntimeError("unknown label type %s" % labels) background_w_inv = 0 for c, w in zip(all_classes, all_weights): if c not in CLASSES: background_w_inv += 1 / w WEIGHTS = [all_weights[all_classes.index(s)] for s in CLASSES] + [1 / background_w_inv] # reorder for background label. #CLASSES = CLASSES[1:] #WEIGHTS = WEIGHTS[1:] + [WEIGHTS[0]] # define network parameters n_classes = 1 if len(CLASSES) == 1 else ( len(CLASSES) + 1) # case for binary and multiclass segmentation activation = 'sigmoid' if n_classes == 1 else 'softmax' print("creating model...") #create model models = dict(unet=sm.Unet, pspnet=sm.PSPNet, linknet=sm.Linknet, fpn=sm.FPN) kwargs = dict() if model == "pspnet": kwargs['input_shape'] = (768, 512, 3) my_model = (models[model])(BACKBONE, classes=n_classes, activation=activation, **kwargs) # define optomizer #clr = tensorflow_addons.optimizers.TriangularCyclicalLearningRate( # initial_learning_rate=LR / 1000, maximal_learning_rate=LR, # step_size=2000) #optim = keras.optimizers.SGD(learning_rate=LR) #optim = NovoGrad(100) #optim = Lookahead(Adam(2 * 1e-3)) # was 2 * 1e-3 #optim = Lookahead(RAdam(2 * 1e-3, min_lr=1e-5)) #optim = Lookahead(SGD(2 * 1e-3)) # Segmentation models losses can be combined together by '+' and scaled by integer or float factor # set class weights for dice_loss (car: 1.; pedestrian: 2.; background: 0.5;) #dice_loss = sm.losses.DiceLoss(class_weights=np.array(WEIGHTS)) #focal_loss = sm.losses.BinaryFocalLoss() if n_classes == 1 else sm.losses.CategoricalFocalLoss() #total_loss = dice_loss + (args.focal * focal_loss) #total_loss = sm.losses.CategoricalCELoss(class_weights=np.array(WEIGHTS)) if labels == "txt": total_loss = sm.losses.CategoricalCELoss( class_weights=np.array(WEIGHTS)) elif labels == "sep": total_loss = sm.losses.DiceLoss(class_weights=np.array(WEIGHTS)) else: raise RuntimeError("unknown label type %s" % args.labels) # actulally total_loss can be imported directly from library, above example just show you how to manipulate with losses # total_loss = sm.losses.binary_focal_dice_loss # or sm.losses.categorical_focal_dice_loss normed_weights = np.array(WEIGHTS) normed_weights /= np.sum(normed_weights) metrics = [ sm.metrics.IOUScore(threshold=0.5, name="iou_score"), sm.metrics.Precision(threshold=0.5, name="precision"), sm.metrics.Recall(threshold=0.5, name="recall") ] # Dataset for train images train_dataset = Dataset(x_train_dir, y_train_dir, classes=CLASSES, augmentation=get_training_augmentation(), preprocessing=get_preprocessing(preprocess_input), partial=partial) # Dataset for validation images valid_dataset = Dataset( x_valid_dir, y_valid_dir, classes=CLASSES, augmentation=get_validation_augmentation(), preprocessing=get_preprocessing(preprocess_input), ) train_dataloader = Dataloader(train_dataset, batch_size=BATCH_SIZE, shuffle=True) valid_dataloader = Dataloader(valid_dataset, batch_size=1, shuffle=False) optim = Lookahead( AdamW( 2 * LR, # was: 5 * LR #beta_1=0.8, #eta_min=0.1 / 2, use_cosine_annealing=True, batch_size=batchsize, total_iterations=len(train_dataset) // batchsize)) #, slow_step=0.25) if ngpus > 1: #from keras.utils import multi_gpu_model #my_model = multi_gpu_model(my_model, gpus=ngpus) my_model = keras.utils.multi_gpu_model(my_model, gpus=ngpus) print("compiling model...") # compile keras model with defined optimozer, loss and metrics my_model.compile(optim, total_loss, metrics) if False: import matplotlib.pyplot as plt lr_finder = LRFinder(my_model) lr_finder.find(train_dataloader, num_iter=len(train_dataloader) * 1, start_lr=1e-3, end_lr=1e-1) lr_finder.plot_loss() with open("temp.csv", "w") as f: for a, b in zip(lr_finder.lrs, lr_finder.losses): f.write("%f;%f\n" % (a, b)) plt.savefig('lr.png') sys.exit(0) # check shapes for errors #assert train_dataloader[0][0].shape == (BATCH_SIZE, 768, 512, 3) #assert train_dataloader[0][1].shape == (BATCH_SIZE, 768, 512, n_classes) #keras.callbacks.ReduceLROnPlateau(mode='max', monitor='val_f0.5', patience=4, factor=0.5), #keras.callbacks.EarlyStopping(mode='max', monitor='val_f0.5', patience=25, min_delta=0.01), #OneCycleScheduler(3e-3, pct_start=0.9, start_div=3e-3 / 2e-3, end_div=1), save_model_path = None if save_model: save_model_path = get_tmp_model_path() # define callbacks for learning rate scheduling and best checkpoints saving callbacks = [ LogMetricsToSacred(), EvaluateMoreMetrics(valid_dataset, n_classes, save_model_path), keras.callbacks.TerminateOnNaN() #keras.callbacks.CSVLogger(str(model_path / "training.csv"), separator=',', append=False), ] if False: callbacks.append( keras.callbacks.ModelCheckpoint(str(model_path), save_weights_only=True, save_best_only=True, mode='max', monitor='val_matthews')) print("starting fit...") # train model history = my_model.fit_generator(train_dataloader, steps_per_epoch=len(train_dataloader), epochs=EPOCHS, callbacks=callbacks, validation_data=valid_dataloader, validation_steps=len(valid_dataloader)) if save_model_path: ex.add_artifact(str(save_model_path)) # evaluation. if False: lsd_name = ('pixel_accuracy', 'mean_accuracy', 'mean_IU', 'frequency_weighted_IU') lsd_value = defaultdict(list) for i in range(len(valid_dataset)): image, gt_mask = valid_dataset[i] image = np.expand_dims(image, axis=0) pr_mask = my_model.predict(image) metrics = lsd_metrics( np.argmax(pr_mask.squeeze(), axis=-1).astype(np.uint8), np.argmax(gt_mask.squeeze(), axis=-1).astype(np.uint8), n_classes) for k in lsd_name: lsd_value[k].append(float(getattr(metrics, k))) for k, v in lsd_value.items(): ex.log_scalar("val_" + k, np.mean(np.array(v)))
parser.add_argument('--batch', help='To do batching or not', required=True) # True/False args = vars(parser.parse_args()) from keras.utils import multi_gpu_model from keras.models import load_model # model = Inception_Inflated3d(num_frames = n_sequence, classes=3, input_shape=(32,168,64,3)) # p_model = multi_gpu_model(model, gpus=1) # p_model.load_weights(args['path']) # load multi-gpu model weights # old_model = p_model.layers[-2] #get single GPU model weights # # it's necessary to save the model before use this single GPU model # old_model.save("singlu_gpu.hdf5") # model.load("single_gpu.hdf5") # model = load_model(args['path']) model = load_model(args['path'], compile=False).layers[-2] model.compile(optimizer=Lookahead(RAdam()), loss='sparse_categorical_crossentropy', metrics=['accuracy']) # model = Inception_Inflated3d(num_frames=64, classes=3, input_shape=(64,168,64,1)) # model = model.load_weights(args['path']).layers[-2] print(args['mode'], "Model loaded") cap = cv2.VideoCapture(args['video']) print("Video loaded") w = int(cap.get(3)) h = int(cap.get(4)) fourcc = cv2.VideoWriter_fourcc(*'MJPG') print("creating writer") out = cv2.VideoWriter('output.avi', fourcc, 25.0, (w, h))
def compile_models(self): """ Compile full training model with chosen hyperparameters. """ if self.hyperparameters[ "clip_select_flag"] == "norm" and self.hyperparameters[ "optimizer"] == "adam": self.optimizer = Lookahead( keras.optimizers.adam( lr=self.hyperparameters["learning_rate"], beta_1=self.hyperparameters["BETA_1"], beta_2=self.hyperparameters["BETA_2"], decay=self.hyperparameters["lr_decay"], amsgrad=self.hyperparameters["ams_grad_flag"]), sync_period=self.hyperparameters["sync_lookahead"], slow_step=self.hyperparameters["slow_weights_lookahead"]) elif self.hyperparameters[ "clip_select_flag"] == "value" and self.hyperparameters[ "optimizer"] == "adam": self.optimizer = Lookahead( keras.optimizers.adam( lr=self.hyperparameters["learning_rate"], beta_1=self.hyperparameters["BETA_1"], beta_2=self.hyperparameters["BETA_2"], decay=self.hyperparameters["lr_decay"], clipvalue=self.hyperparameters["clip_val_thresh"], amsgrad=self.hyperparameters["ams_grad_flag"]), sync_period=self.hyperparameters["sync_lookahead"], slow_step=self.hyperparameters["slow_weights_lookahead"]) elif (self.hyperparameters["clip_select_flag"] == "norm" or self.hyperparameters["clip_select_flag"] == "value") and self.hyperparameters["optimizer"] == "nadam": self.optimizer = keras.optimizers.nadam( lr=self.hyperparameters["learning_rate"], beta_1=self.hyperparameters["BETA_1"], beta_2=self.hyperparameters["BETA_2"], epsilon=self.hyperparameters["epsilon"], schedule_decay=self.hyperparameters["lr_decay"]) elif self.hyperparameters["optimizer"] == "RAdam": self.optimizer = RAdam( learning_rate=self.hyperparameters["learning_rate"], beta_1=self.hyperparameters["BETA_1"], beta_2=self.hyperparameters["BETA_2"], epsilon=self.hyperparameters["epsilon"], weight_decay=self.hyperparameters["weight_decay_RAdam"], amsgrad=self.hyperparameters["ams_grad_flag"], total_steps=self.hyperparameters["total_steps_RAdam"], warmup_proportion=self.hyperparameters["warmup_RAdam"], min_lr=self.hyperparameters["min_lr_RAdam"]) elif self.hyperparameters["optimizer"] == "Ranger": self.optimizer = Lookahead( RAdam(learning_rate=self.hyperparameters["learning_rate"], beta_1=self.hyperparameters["BETA_1"], beta_2=self.hyperparameters["BETA_2"], epsilon=self.hyperparameters["epsilon"], weight_decay=self.hyperparameters["weight_decay_RAdam"], amsgrad=self.hyperparameters["ams_grad_flag"], total_steps=self.hyperparameters["total_steps_RAdam"], warmup_proportion=self.hyperparameters["warmup_RAdam"], min_lr=self.hyperparameters["min_lr_RAdam"]), sync_period=self.hyperparameters["sync_lookahead"], slow_step=self.hyperparameters["slow_weights_lookahead"]) else: print(" Clipping Method OR Optimizer Selected is not available! ") print( " Please enter a valid string for these parameter: \n Valid Clipping:['norm', 'value'] \n Valid Optimizers: ['adam', 'NAdam', 'RAdam', 'Ranger']" ) sys.exit(1) self.lstm_model.compile(optimizer=self.optimizer, loss=self.hyperparameters["loss_fcn"], metrics=[rmse])
print('[DEBUG] lb_classes_', lb.classes_) print('[DEBUG] trainY', trainY.shape) print('[DEBUG] testY', testY.shape) # save label classes to file essences.txt f = open(cfg.output_PATH + "essences.txt", "w+") f.write(str(lb.classes_)) f.close() # construct the image generator for data augmentation aug = ImageDataGenerator(width_shift_range=0.1, vertical_flip=True, rotation_range=180, height_shift_range=0.1, horizontal_flip=True) opt = Lookahead(RAdam(min_lr=cfg.INIT_LR_RADAM)) # construct the set of callbacks # radam I removed LearningRateScheduler(poly_decay) from the callbacks # figPath = cfg.output_PATH + "{}.png".format(os.getpid()) # jsonPath = cfg.output_PATH + "{}.png".format(os.getpid()) # callbacks = [TrainingMonitor(figPath, jsonPath=jsonPath), LearningRateScheduler(poly_decay)] # construct the set of callbacks callbacks = [ EpochCheckpoint(cfg.EPOCH_PATH, every=cfg.EPOCH_EVERY, startAt=cfg.EPOCH_START), TrainingMonitor(cfg.FIG_PATH + "{}.png".format(os.getpid()), jsonPath=cfg.JSON_PATH + "{}.png".format(os.getpid()), startAt=cfg.EPOCH_START) ]