def train_kfold(log_dir, hparams, model_name, k, state, X, Y, X_test=None, Y_test=None, X_train_add=None, Y_train_add=None, batch_size=20, epochs=200): ''' X: The training set X_test: The testing set X_train_add: The additional set for model training, used for the 2nd experiment in paper. ''' ''' Training ''' # Log timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f') model_dir = os.path.join(log_dir, timestamp, 'models') os.makedirs(model_dir) hist_dir = os.path.join(log_dir, timestamp, 'history') os.makedirs(hist_dir) eval_dir = os.path.join(log_dir, timestamp, 'evaluate') os.makedirs(eval_dir) params_savename = os.path.join(log_dir, timestamp, 'params.json') summary_savename = os.path.join(log_dir, timestamp, 'summary.json') test_hist_savename = os.path.join(log_dir, timestamp, 'test.json') # Start training kfold = StratifiedKFold(n_splits=k, shuffle=True, random_state=state) fold = 1 accs = [] for train_index, val_index in kfold.split(X, Y): model, params = compile_model(model_name, hparams) if fold == 1: print(model.summary()) print(params) print('\n' + '=' * 60 + ' Fold: ' + str(fold) + ' ' + '=' * 60 + '\n') # Callback functions model_savename = os.path.join(model_dir, 'model{0}.h5'.format(str(fold))) hist_savename = os.path.join(hist_dir, 'history{0}.json'.format(str(fold))) val_savename = os.path.join(eval_dir, 'evaluate{0}.json'.format(str(fold))) cb_list = [ callbacks.ModelCheckpoint(filepath=model_savename, monitor='val_acc', save_best_only=True), callbacks.EarlyStopping( monitor='acc', patience=6, ) ] # Add new training sets try: if X_train_add.any() and Y_train_add.any(): x_train = np.concatenate([X[train_index], X_train_add], axis=0) y_train = np.concatenate([Y[train_index], Y_train_add], axis=0) index = list(range(len(y_train))) random.seed(state + 1) random.shuffle(index) x_train = x_train[index] y_train = y_train[index] except AttributeError: x_train = X[train_index] y_train = Y[train_index] history = model.fit(x_train, y_train, validation_data=(X[val_index], Y[val_index]), batch_size=batch_size, epochs=epochs, callbacks=cb_list, verbose=2) # Log hist_dict = history.history m = models.load_model(model_savename, custom_objects={'tf': tf}) val_dict = evaluate_model(m, X[val_index], Y[val_index]) accs.append(val_dict['accuracy']) log_to_json(hist_dict, hist_savename) log_to_json(val_dict, val_savename) fold += 1 K.clear_session() print('Session cleared.') # Summary try: if X_test.any() and Y_test.any(): model_path = os.path.join( model_dir, 'model{0}.h5'.format(accs.index(max(accs)) + 1)) m = models.load_model(model_path, custom_objects={'tf': tf}) test_dict = evaluate_model(m, X_test, Y_test) log_to_json(test_dict, test_hist_savename) except AttributeError: pass log_to_json(hparams, params_savename) summary = summary_kfold(eval_dir) print(summary) log_to_json(summary, summary_savename)
def test(params): # 加载预训练模型的tokenizer tokenizer = BertTokenizer.from_pretrained(params["bert_pretrain_path"] + 'bert-base-chinese-vocab.txt') test_batch = batcher(tokenizer, params, params["test_dataset_path"]) df_test = pd.read_csv(params["test_dataset_path"], engine='python', encoding="utf-8") df_sub = df_test[['微博id']] if params["use_cross_valid"]: test_preds = [] for i in range(params["n_splits"]): cross_pred = [] K.clear_session() print("Building the model ...") model = bert_classify(params) print("Creating the checkpoint manager") checkpoint_dir = "{}/cross_valid_checkpoint/ckpt_{}/checkpoint".format( params["model_checkpoint_dir"], i) ckpt = tf.train.Checkpoint(BertModel=model) ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_dir, max_to_keep=5) ckpt.restore(ckpt_manager.latest_checkpoint) if ckpt_manager.latest_checkpoint: print("Restored from {}".format( ckpt_manager.latest_checkpoint)) else: print(checkpoint_dir) raise Exception("No checkpoint files.") for test_batcher in tqdm(test_batch): pred = model(test_batcher[0]) cross_pred.append(pred) # print(pred.shape) cross_pred = tf.concat(cross_pred, axis=0) test_preds.append(cross_pred) test_preds = tf.stack(test_preds, axis=0) sub = np.average(test_preds, axis=0) sub = np.argmax(sub, axis=1) df_sub['y'] = sub - 1 df_sub.columns = ['id', 'y'] df_sub.to_csv('test_sub.csv', index=False, encoding='utf-8') else: cross_pred = [] K.clear_session() print("Building the model ...") model = bert_classify(params) print("Creating the checkpoint manager") checkpoint_dir = "{}/split_valid_checkpoint/checkpoint".format( params["model_checkpoint_dir"]) ckpt = tf.train.Checkpoint(BertModel=model) ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_dir, max_to_keep=5) ckpt.restore(ckpt_manager.latest_checkpoint) if ckpt_manager.latest_checkpoint: print("Restored from {}".format(ckpt_manager.latest_checkpoint)) else: print(checkpoint_dir) raise Exception("No checkpoint files.") for test_batcher in tqdm(test_batch): pred = model(test_batcher[0]) cross_pred.append(pred) cross_pred = tf.concat(cross_pred, axis=0) sub = np.argmax(cross_pred, axis=1) df_sub['y'] = sub - 1 df_sub.columns = ['id', 'y'] df_sub.to_csv('test_sub.csv', index=False, encoding='utf-8')
def main(dataset_name): dataset = load_dataset() raw_data = np.asarray(dataset['raw']['data']) raw_label = np.asarray(dataset['raw']['label']) num_classes = len(np.unique(raw_label)) rskf = RepeatedStratifiedKFold(n_splits=k_folds, n_repeats=k_fold_reps, random_state=42) for fs_method, fs_range in fs_methods: print('FS-Method : ', fs_method.__name__) nfeats = [] accuracies = [] svc_accuracies = [] BAs = [] svc_BAs = [] mAPs = [] svc_mAPs = [] mus = [] name = dataset_name + '_mu_' + str(mu) print(name) for j, (train_index, test_index) in enumerate(rskf.split(raw_data, raw_label)): print('k_fold', j, 'of', k_folds * k_fold_reps) train_data, train_labels = raw_data[train_index].copy( ), raw_label[train_index].copy() test_data, test_labels = raw_data[test_index].copy( ), raw_label[test_index].copy() train_labels = to_categorical(train_labels, num_classes=num_classes) test_labels = to_categorical(test_labels, num_classes=num_classes) valid_features = np.where(np.abs(train_data).sum(axis=0) > 0)[0] if len(valid_features) < train_data.shape[1]: print('Removing', train_data.shape[1] - len(valid_features), 'zero features') train_data = train_data[:, valid_features] test_data = test_data[:, valid_features] model_kwargs = { # 'nclasses': num_classes, 'mu': mu / len(train_data), 'degree': 3 } print('mu :', model_kwargs['mu'], ', batch_size :', batch_size) svc_kwargs = {'C': 1.0, 'solver': 0.} print('Starting feature selection') best_fs = 0 best_value = None for fs_value in fs_range: fs_class = fs_method(10, fs_value, matlab_engine=matlab_engine) fs_class.fit(train_data, 2. * train_labels[:, -1] - 1.) svc_train_data = fs_class.transform(train_data) norm = normalization_func() svc_train_data_norm = norm.fit_transform(svc_train_data) for s in [0, 1, 2, 3]: for my_c in [ 0.001, 0.01, 0.1, 0.5, 1.0, 1.4, 1.5, 1.6, 2.0, 2.5, 5.0, 25.0, 50.0, 100.0 ]: cmd = '-v 5 -s ' + str(s) + ' -c ' + str(my_c) + ' -q' cv = liblinearutil.train( (2 * train_labels[:, -1] - 1).tolist(), svc_train_data_norm.tolist(), cmd) if cv > best_fs: best_fs = cv best_value = fs_value print('best fs_value: ', best_value) fs_class = fs_method(200, best_value, matlab_engine=matlab_engine) fs_class.fit(train_data, 2. * train_labels[:, -1] - 1.) print('Finishing feature selection') for i, n_features in enumerate([10, 50, 100, 150, 200]): n_accuracies = [] n_svc_accuracies = [] n_BAs = [] n_svc_BAs = [] n_mAPs = [] n_svc_mAPs = [] n_train_accuracies = [] print('n_features : ', n_features) fs_class.n_features_to_select = n_features svc_train_data = fs_class.transform(train_data) svc_test_data = fs_class.transform(test_data) norm = normalization_func() svc_train_data_norm = norm.fit_transform(svc_train_data) svc_test_data_norm = norm.transform(svc_test_data) bestcv = -1 bestc = None bestSolver = None for s in [0, 1, 2, 3]: for my_c in [ 0.001, 0.01, 0.1, 0.5, 1.0, 1.4, 1.5, 1.6, 2.0, 2.5, 5.0, 25.0, 50.0, 100.0 ]: cmd = '-v 5 -s ' + str(s) + ' -c ' + str(my_c) + ' -q' cv = liblinearutil.train( (2 * train_labels[:, -1] - 1).tolist(), svc_train_data_norm.tolist(), cmd) if cv > bestcv: bestcv = cv bestc = my_c bestSolver = s svc_kwargs['C'] = bestc svc_kwargs['solver'] = bestSolver print('Best -> C:', bestc, ', s:', bestSolver, ', acc:', bestcv) for r in range(reps): model = train_SVC(svc_train_data_norm, train_labels, svc_kwargs) _, accuracy, test_pred = liblinearutil.predict( (2 * test_labels[:, -1] - 1).tolist(), svc_test_data_norm.tolist(), model, '-q') test_pred = np.asarray(test_pred) n_svc_accuracies.append(accuracy[0]) n_svc_BAs.append(balance_accuracy(test_labels, test_pred)) n_svc_mAPs.append( average_precision_score(test_labels[:, -1], test_pred)) del model model = train_Keras(svc_train_data, train_labels, svc_test_data, test_labels, model_kwargs) train_data_norm = model.normalization.transform( svc_train_data) test_data_norm = model.normalization.transform( svc_test_data) test_pred = model.predict(test_data_norm) n_BAs.append(balance_accuracy(test_labels, test_pred)) n_mAPs.append( average_precision_score(test_labels[:, -1], test_pred)) n_accuracies.append( model.evaluate(test_data_norm, test_labels, verbose=0)[-1]) n_train_accuracies.append( model.evaluate(train_data_norm, train_labels, verbose=0)[-1]) del model K.clear_session() print( 'n_features : ', n_features, ', acc : ', n_accuracies[-1], ', BA : ', n_BAs[-1], ', mAP : ', n_mAPs[-1], ', train_acc : ', n_train_accuracies[-1], ', svc_acc : ', n_svc_accuracies[-1], ', svc_BA : ', n_svc_BAs[-1], ', svc_mAP : ', n_svc_mAPs[-1], ) if i >= len(accuracies): accuracies.append(n_accuracies) svc_accuracies.append(n_svc_accuracies) BAs.append(n_BAs) mAPs.append(n_mAPs) svc_BAs.append(n_svc_BAs) svc_mAPs.append(n_svc_mAPs) nfeats.append(n_features) mus.append(model_kwargs['mu']) else: accuracies[i] += n_accuracies svc_accuracies[i] += n_svc_accuracies BAs[i] += n_BAs mAPs[i] += n_mAPs svc_BAs[i] += n_svc_BAs svc_mAPs[i] += n_svc_mAPs output_filename = directory + 'LinearSVC_' + fs_method.__name__ + '.json' if not os.path.isdir(directory): os.makedirs(directory) info_data = { 'reps': reps, 'classification': { 'mus': mus, 'n_features': nfeats, 'accuracy': accuracies, 'mean_accuracy': np.array(accuracies).mean(axis=1).tolist(), 'svc_accuracy': svc_accuracies, 'mean_svc_accuracy': np.array(svc_accuracies).mean(axis=1).tolist(), 'BA': BAs, 'mean_BA': np.array(BAs).mean(axis=1).tolist(), 'mAP': mAPs, 'mean_mAP': np.array(mAPs).mean(axis=1).tolist(), 'svc_BA': svc_BAs, 'svc_mean_BA': np.array(svc_BAs).mean(axis=1).tolist(), 'svc_mAP': svc_mAPs, 'svc_mean_mAP': np.array(svc_mAPs).mean(axis=1).tolist(), } } for k, v in info_data['classification'].items(): if 'mean' in k: print(k, v) with open(output_filename, 'w') as outfile: json.dump(info_data, outfile)
def prediction(constants, model_name, dataset_folder, dataset_name, frame, repeat_index, img_folder, save_path): img_path = dataset_folder + dataset_name + img_folder if constants.self_training_type is None: save_path = save_path + '{}/frame{}_{}_repeat{}/'.format( dataset_name, str(frame), model_name, str(repeat_index)) else: save_path = save_path + '{}_{}/frame{}_repeat{}/'.format( model_name, dataset_name, str(frame), str(repeat_index)) print('save_path:', save_path) if os.path.isdir(save_path) == 0: os.makedirs(save_path) # ------------------- Data loading ------------------- a_strategy = constants.strategy_type if 'TIRF' in dataset_name and 'specialist' in constants.strategy_type: a_strategy = constants.strategy_type + '_normalize' prediction_data_generator = DataGenerator(img_path, frame, 128, 68, a_strategy, img_format=constants.img_format) imgs_val, namelist, image_cols, image_rows, orig_cols, orig_rows = prediction_data_generator.get_expanded_whole_frames( ) print('img size:', image_rows, image_cols) print('orig img size:', orig_rows, orig_cols) print('imgs_val: ', imgs_val.dtype, imgs_val.shape) # ------------------- Load trained Model ------------------- weights_path = constants.get_trained_weights_path(str(frame), model_name, str(repeat_index)) # print(debugger.check_loaded_weights(weights_path)) if "Res50V2" == str(constants.strategy_type): model = ResNet50V2Keras(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "Dense201" == str(constants.strategy_type): model = DenseNet201Keras(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "InceptionResV2" == str(constants.strategy_type): model = InceptionResV2(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "deeplabv3" == str(constants.strategy_type): K.set_image_data_format('channels_last') imgs_val = np.moveaxis(imgs_val, 1, -1) # first channel to last channel print(imgs_val.dtype, imgs_val.shape) model = Deeplabv3(input_shape=(image_rows, image_cols, 3), output_shape=(orig_rows, orig_cols)) model.load_weights(weights_path, by_name=True) elif "VGG16_dropout" == str(constants.strategy_type): model = VGG16_dropout(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "VGG16_batchnorm" == str(constants.strategy_type): model = VGG16_batchnorm(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "VGG16_instancenorm" == str(constants.strategy_type): model = VGG16_instancenorm(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "movie3" in str(constants.strategy_type): model = VGG16_movie(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "VGG16_dac_input256" == constants.strategy_type: model = VGG16_dac(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "VGG16_spp_input256" == constants.strategy_type: model = VGG16_spp(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "VGG16" in str(constants.strategy_type): model = VGG16(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "VGG19_dropout_dac" in str(constants.strategy_type): model = VGG19_dropout_dac(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "VGG19_dropout_feature_extractor" in str(constants.strategy_type): model = VGG19_dropout_feature_extractor(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "VGG19_batchnorm_dropout" == str(constants.strategy_type): model = VGG19_batchnorm_dropout(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "VGG19_batchnorm" == str(constants.strategy_type): model = VGG19_batchnorm(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "VGG19_dropout" in str(constants.strategy_type): model = VGG19_dropout(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "VGG19" in str(constants.strategy_type): model = VGG19(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path, encoder_weights=None) elif "EFF_B7" == str( constants.strategy_type) or "EFF_B7_no_preprocessing" == str( constants.strategy_type): K.set_image_data_format('channels_last') imgs_val = np.moveaxis(imgs_val, 1, -1) # first channel to last channel print(imgs_val.dtype, imgs_val.shape) model = EFF_B7(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "unet_feature_extractor" in str(constants.strategy_type): model = UNet_feature_extractor(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) elif "unet" in str(constants.strategy_type): model = UNet(image_rows, image_cols, 0, image_cols - orig_cols, image_rows - orig_rows, weights_path=weights_path) print('model layers: ', len(model.layers)) plot_model(model, to_file='model_plots/model_round{}_{}_predict.png'.format( constants.round_num, constants.strategy_type), show_shapes=True, show_layer_names=True, dpi=144) # ------------------- predict segmented images and save them ------------------- if "feature_extractor" in str(constants.strategy_type): segmented_output, style_output = model.predict(imgs_val, batch_size=1, verbose=1) np.save(save_path + 'style_feature_vector.npy', style_output) else: segmented_output = model.predict(imgs_val, batch_size=1, verbose=1) segmented_output = 255 * segmented_output # 0=black color and 255=white color if "deeplabv3" == str(constants.strategy_type) or "EFF_B7" == str( constants.strategy_type) or "EFF_B7_no_preprocessing" == str( constants.strategy_type): # move last channel to first channel segmented_output = np.moveaxis(segmented_output, -1, 1) print(segmented_output.shape) for f in range(len(namelist)): if constants.strategy_type == 'movie3' or constants.strategy_type == 'movie3_loss': out = segmented_output[f, 1, :, :] else: out = segmented_output[f, 0, :, :] cv2.imwrite(save_path + namelist[f], out) K.clear_session()
sample_weight=None, multioutput='uniform_average') with open(SAVE_DIR + '/results_model_baseline_' + MODE + '.csv', 'a') as out_stream: out_stream.write( str(SEED) + ', ' + str(DATA_FILE[0:-4]) + ', ' + str(i) + ', ' + str(early_stopping_epoch) + ', ' + str(all_trainable_count) + ', ' + str(acc_fold) + ', ' + str(MAE) + ', ' + str(recall_fold) + ', ' + str(f1_fold) + '\n') print('Accuracy[{:.4f}] Recall[{:.4f}] F1[{:.4f}] at fold[{}]'. format(acc_fold, recall_fold, f1_fold, i)) print('______________________________________________________') K.clear_session() ic_acc = st.t.interval(0.9, len(avg_acc) - 1, loc=np.mean(avg_acc), scale=st.sem(avg_acc)) ic_recall = st.t.interval(0.9, len(avg_recall) - 1, loc=np.mean(avg_recall), scale=st.sem(avg_recall)) ic_f1 = st.t.interval(0.9, len(avg_f1) - 1, loc=np.mean(avg_f1), scale=st.sem(avg_f1)) print('Mean Accuracy[{:.4f}] IC [{:.4f}, {:.4f}]'.format(
def setUp(self): tf.reset_default_graph() K.clear_session() self.previous_logging_level = logging.getLogger().level logging.getLogger().setLevel(logging.ERROR)
def lav_test_case( test: unittest.TestCase, scenario: Type[ScenarioBase], debug=False, delta=1e-5, batch_size_test=True, ignore_binary_metric=False, ignore_array_metric=False, instantiate_graph=False, args: AdditionalTrainerArgs = AdditionalTrainerArgs(), ): with tempfile.TemporaryDirectory() as tmp_dir: trainer_params = scenario.default_trainer_params() trainer_params.output_dir = tmp_dir trainer_params.epochs = 1 trainer_params.samples_per_epoch = 3 trainer_params.skip_model_load_test = True # not required in this test trainer_params.force_eager = debug trainer_params.export_best = True trainer_params.export_final = True trainer_params.write_checkpoints = False trainer_params.random_seed = 324 trainer_params.scenario.data.pre_proc.run_parallel = False # Deterministic results! args.apply(trainer_params) batch_and_limit = 5 trainer = scenario.create_trainer(trainer_params) trainer.train() json_path = os.path.join(tmp_dir, "trainer_params.json") with open(json_path) as f: trainer_params_dict = json.load(f) trainer_params_dict["epochs"] = 2 lav_params = scenario.lav_cls().params_cls()() lav_params.print_samples = True lav_params.model_path = os.path.join(trainer_params.output_dir, "export") lav_params.pipeline = trainer_params.gen.setup.val clear_session() scenario_params = scenario.params_from_path(lav_params.model_path) lav = scenario.create_lav(lav_params, scenario_params) lav.run([trainer_params.gen.val_gen()], instantiate_graph=instantiate_graph) clear_session() set_global_random_seed(trainer_params.random_seed) lav_params.model_path = os.path.join(trainer_params.output_dir, "best") scenario_params = scenario.params_from_path(lav_params.model_path) lav_params.pipeline.batch_size = 1 lav_params.pipeline.limit = batch_and_limit lav = scenario.create_lav(lav_params, scenario_params) bs1_results = next( iter(lav.run([trainer_params.gen.val_gen()], run_eagerly=debug, instantiate_graph=instantiate_graph)) ) lav.benchmark_results.pretty_print() if batch_size_test: clear_session() set_global_random_seed(trainer_params.random_seed) lav_params.model_path = os.path.join(trainer_params.output_dir, "best") scenario_params = scenario.params_from_path(lav_params.model_path) lav_params.pipeline.batch_size = batch_and_limit lav_params.pipeline.limit = batch_and_limit lav = scenario.create_lav(lav_params, scenario_params) bs5_results = next( iter(lav.run([trainer_params.gen.val_gen()], run_eagerly=debug, instantiate_graph=instantiate_graph)) ) lav.benchmark_results.pretty_print() assert_equality_dict(test, bs1_results, bs5_results, delta, ignore_binary_metric, ignore_array_metric)
def run(params): args = Struct(**params) set_seed(args.rng_seed) ext = extension_from_parameters(args) verify_path(args.save) prefix = args.save + ext logfile = args.logfile if args.logfile else prefix + '.log' set_up_logger(logfile, args.verbose) logger.info('Params: {}'.format(params)) loader = ComboDataLoader(seed=args.rng_seed, val_split=args.validation_split, cell_features=args.cell_features, drug_features=args.drug_features, response_url=args.response_url, use_landmark_genes=args.use_landmark_genes, preprocess_rnaseq=args.preprocess_rnaseq, exclude_cells=args.exclude_cells, exclude_drugs=args.exclude_drugs, use_combo_score=args.use_combo_score, cv_partition=args.cv_partition, cv=args.cv) # test_loader(loader) # test_generator(loader) train_gen = ComboDataGenerator(loader, batch_size=args.batch_size).flow() val_gen = ComboDataGenerator(loader, partition='val', batch_size=args.batch_size).flow() train_steps = int(loader.n_train / args.batch_size) val_steps = int(loader.n_val / args.batch_size) model = build_model(loader, args, verbose=True) print('Creating model PNG') from keras.utils import plot_model plot_model(model, 'model_global_combo.png', show_shapes=True) print('Model PNG has been created successfuly!') model.summary() # plot_model(model, to_file=prefix+'.model.png', show_shapes=True) if args.cp: model_json = model.to_json() with open(prefix + '.model.json', 'w') as f: print(model_json, file=f) def warmup_scheduler(epoch): lr = args.learning_rate or base_lr * args.batch_size / 100 if epoch <= 5: K.set_value(model.optimizer.lr, (base_lr * (5 - epoch) + lr * epoch) / 5) logger.debug('Epoch {}: lr={}'.format(epoch, K.get_value(model.optimizer.lr))) return K.get_value(model.optimizer.lr) df_pred_list = [] cv_ext = '' cv = args.cv if args.cv > 1 else 1 fold = 0 while fold < cv: if args.cv > 1: logger.info('Cross validation fold {}/{}:'.format(fold + 1, cv)) cv_ext = '.cv{}'.format(fold + 1) model = build_model(loader, args) optimizer = optimizers.deserialize({ 'class_name': args.optimizer, 'config': {} }) base_lr = args.base_lr or K.get_value(optimizer.lr) if args.learning_rate: K.set_value(optimizer.lr, args.learning_rate) model.compile(loss=args.loss, optimizer=optimizer, metrics=[mae, r2]) # calculate trainable and non-trainable params params.update(candle.compute_trainable_params(model)) candle_monitor = candle.CandleRemoteMonitor(params=params) timeout_monitor = candle.TerminateOnTimeOut(params['timeout']) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=0.00001) warmup_lr = LearningRateScheduler(warmup_scheduler) checkpointer = ModelCheckpoint(prefix + cv_ext + '.weights.h5', save_best_only=True, save_weights_only=True) tensorboard = TensorBoard(log_dir="tb/tb{}{}".format(ext, cv_ext)) history_logger = LoggingCallback(logger.debug) model_recorder = ModelRecorder() # callbacks = [history_logger, model_recorder] callbacks = [ candle_monitor, timeout_monitor, history_logger, model_recorder ] if args.reduce_lr: callbacks.append(reduce_lr) if args.warmup_lr: callbacks.append(warmup_lr) if args.cp: callbacks.append(checkpointer) if args.tb: callbacks.append(tensorboard) if args.gen: history = model.fit_generator(train_gen, train_steps, epochs=args.epochs, callbacks=callbacks, validation_data=val_gen, validation_steps=val_steps) fold += 1 else: if args.cv > 1: x_train_list, y_train, x_val_list, y_val, df_train, df_val = loader.load_data_cv( fold) else: x_train_list, y_train, x_val_list, y_val, df_train, df_val = loader.load_data( ) y_shuf = np.random.permutation(y_val) log_evaluation(evaluate_prediction(y_val, y_shuf), description='Between random pairs in y_val:') history = model.fit(x_train_list, y_train, batch_size=args.batch_size, shuffle=args.shuffle, epochs=args.epochs, callbacks=callbacks, validation_data=(x_val_list, y_val)) if args.cp: model.load_weights(prefix + cv_ext + '.weights.h5') if not args.gen: y_val_pred = model.predict(x_val_list, batch_size=args.batch_size).flatten() scores = evaluate_prediction(y_val, y_val_pred) if args.cv > 1 and scores[args.loss] > args.max_val_loss: logger.warn( 'Best val_loss {} is greater than {}; retrain the model...' .format(scores[args.loss], args.max_val_loss)) continue else: fold += 1 log_evaluation(scores) df_val.is_copy = False df_val['GROWTH_PRED'] = y_val_pred df_val['GROWTH_ERROR'] = y_val_pred - y_val df_pred_list.append(df_val) if args.cp: # model.save(prefix+'.model.h5') model_recorder.best_model.save(prefix + '.model.h5') # test reloadded model prediction # new_model = keras.models.load_model(prefix+'.model.h5') # new_model.load_weights(prefix+cv_ext+'.weights.h5') # new_pred = new_model.predict(x_val_list, batch_size=args.batch_size).flatten() # print('y_val:', y_val[:10]) # print('old_pred:', y_val_pred[:10]) # print('new_pred:', new_pred[:10]) plot_history(prefix, history, 'loss') plot_history(prefix, history, 'r2') if K.backend() == 'tensorflow': K.clear_session() if not args.gen: pred_fname = prefix + '.predicted.growth.tsv' if args.use_combo_score: pred_fname = prefix + '.predicted.score.tsv' df_pred = pd.concat(df_pred_list) df_pred.to_csv(pred_fname, sep='\t', index=False, float_format='%.4g') logger.handlers = [] return history
def main(_): train_dir = os.path.join(FLAGS.image_dir, "train") val_dir = os.path.join(FLAGS.image_dir, "val") test_dir = os.path.join(FLAGS.image_dir, "test") checkpoint_dir = os.path.join(os.path.dirname(FLAGS.model_dir), "checkpoints") checkpoint_path = os.path.join(checkpoint_dir, "checkpoint.pb") os.makedirs(checkpoint_dir, exist_ok=True) timestamp = time.strftime("%Y%m%d-%H%M%S") tb_dir = os.path.join(FLAGS.summaries_dir, timestamp) logger.info("\n===\tSetting up data loaders for train, val and test data.") train_gen, val_gen, test_gen = create_data_feeders( train_dir, val_dir, test_dir, batch_size=FLAGS.train_batch_size, image_size=FLAGS.image_size) logger.info("\n===\tSaving key file with label names <> index conversion.") save_labels_to_file(train_gen, FLAGS.output_labels) logger.info( "\n===\tCreating a classification model based on pre-trained weights.") num_train_images = 2055 num_labels = 5 model = create_model(input_shape=(FLAGS.image_size, FLAGS.image_size, 3), num_labels=num_labels, learning_rate=FLAGS.learning_rate, optimizer=FLAGS.optimizer_name) logger.info("\n===\tInitial accuracy (before retraining):") untrained_path = tf.contrib.saved_model.save_keras_model( model, checkpoint_dir).decode("utf-8") evaluate_model(untrained_path, test_gen) logger.info("\n===\tPreparing Tensorboard callback, to monitor training.") callbacks = create_callbacks(output_model_path=checkpoint_path, summary_dir=tb_dir) logger.info("\n===\tRetraining downloaded model.") steps_per_epoch = num_train_images // FLAGS.train_batch_size model.fit_generator(train_gen, steps_per_epoch=steps_per_epoch, epochs=FLAGS.how_many_training_steps // steps_per_epoch, validation_data=val_gen, validation_steps=5, callbacks=callbacks) logger.info( "\n===\tExporting the model so it can be served (TF Serving, TF Lite, etc.) to: {}" .format(FLAGS.model_dir)) output_path = tf.contrib.saved_model.save_keras_model( model, checkpoint_dir).decode("utf-8") copy_tree(output_path, FLAGS.model_dir) logger.info("\n===\tReporting final model accuracy:") evaluate_model(FLAGS.model_dir, test_gen) K.clear_session()
def _test_model(self, model, input_shape): batch_size = 1000 epochs = 1 steps_per_epoch = 125 k_lip_data = 2.0 # clear session to avoid side effects from previous train K.clear_session() # create the keras model, defin opt, and compile it optimizer = Adam(lr=0.001) model.compile( optimizer=optimizer, loss="mean_squared_error", metrics=[metrics.mse] ) # model.summary() # create the synthetic data generator output_shape = model.compute_output_shape((batch_size,) + input_shape)[1:] kernel = build_kernel(input_shape, output_shape, k_lip_data) # define logging features logdir = os.path.join( "logs", "lip_layers", "condense_test" ) # , datetime.now().strftime("%Y_%m_%d-%H_%M_%S"))) callback_list = [callbacks.TensorBoard(logdir)] # train model model.__getattribute__(FIT)( linear_generator(batch_size, input_shape, kernel), steps_per_epoch=steps_per_epoch, epochs=epochs, verbose=0, callbacks=callback_list, ) # the seed is set to compare all models with the same data np.random.seed(42) # get original results loss, mse = model.__getattribute__(EVALUATE)( linear_generator(batch_size, input_shape, kernel), steps=10, verbose=0, ) # generate vanilla vanilla_model = model.vanilla_export() vanilla_model.compile( optimizer=optimizer, loss="mean_squared_error", metrics=[metrics.mse] ) np.random.seed(42) # evaluate vanilla loss2, mse2 = model.__getattribute__(EVALUATE)( linear_generator(batch_size, input_shape, kernel), steps=10, verbose=0, ) np.random.seed(42) # check if original has changed vanilla_loss, vanilla_mse = vanilla_model.__getattribute__(EVALUATE)( linear_generator(batch_size, input_shape, kernel), steps=10, verbose=0, ) model.summary() vanilla_model.summary() self.assertAlmostEqual( mse, vanilla_mse, 4, "the exported vanilla model must have same behaviour as original", ) self.assertAlmostEqual( mse, mse2, 4, "exporting a model must not change original model" ) # add one epoch to orginal model.__getattribute__(FIT)( linear_generator(batch_size, input_shape, kernel), steps_per_epoch=steps_per_epoch, epochs=1, verbose=0, callbacks=callback_list, ) np.random.seed(42) loss3, mse3 = model.__getattribute__(EVALUATE)( linear_generator(batch_size, input_shape, kernel), steps=10, verbose=0, ) # check if vanilla has changed np.random.seed(42) vanilla_loss2, vanilla_mse2 = vanilla_model.__getattribute__(EVALUATE)( linear_generator(batch_size, input_shape, kernel), steps=10, verbose=0, ) self.assertAlmostEqual( vanilla_mse, vanilla_mse2, 4, "exported model must be completely independent from original", ) self.assertNotAlmostEqual( mse, mse3, 4, "all tests passe but integrity check failed: the test cannot conclude that " + "vanilla_export create a distinct model", )
def main(): parser = argparse.ArgumentParser(description='Evaluate genesis and cam') parser.add_argument('--json', type=str, default="../data/cross_val.json") parser.add_argument('--genesis_weights', type=str, default='video_genesis_lr1e4') parser.add_argument('--frame_model_weights', type=str, default='trained_models_cam') parser.add_argument('--frame_model_id', type=str, default='vgg_cam') parser.add_argument('--videos', type=str, default='../data/pocus_videos/convex') parser.add_argument('--output_dir', type=str, default='evaluation_outputs.dat') args = parser.parse_args() K.clear_session() print("---------------------------------") print("WARNING: THIS SCRIPT MUST BE RUN ON A GPU") print("---------------------------------") with open(args.json, "r") as infile: cross_val_split = json.load(infile) VIDEO_DIR = args.videos all_genesis_preds = [] all_frame_preds = [] for i in range(5): gen_eval = GenesisEvaluator(weights_dir=args.genesis_weights, ensemble=False, split=i) K.set_image_data_format("channels_last") normal_eval = VideoEvaluator(weights_dir=args.frame_model_weights, ensemble=False, split=i, model_id=args.frame_model_id, num_classes=4) files = cross_val_split[str(i)]["test"][0] # print(files) for f in files: print("evaluate", f) # TEST if the video is working vid3d = Videoto3D("", 64, 64, 5, 5) vid3d.max_vid = {"cov": 20, "pne": 20, "reg": 20} X_test, _, fn = vid3d.video3d([os.path.join(VIDEO_DIR, f)], ["cov"]) if len(np.unique(fn)) != 1: print("ERROR: WRONG FILE!") print(fn) print(X_test.shape) continue # run genesis model K.set_image_data_format("channels_first") preds = gen_eval(os.path.join(VIDEO_DIR, f)) vid_pred_genesis = np.argmax(np.mean(preds, axis=(0, 1))) all_genesis_preds.append(preds) # run cam model K.set_image_data_format("channels_last") preds_framebased = normal_eval(os.path.join(VIDEO_DIR, f)) frame_pred = np.argmax(np.mean(preds_framebased, axis=0)) all_frame_preds.append(preds_framebased) print(preds.shape, preds_framebased.shape) print("genesis pred", vid_pred_genesis, "frame based pred", frame_pred) print("-------------") with open(args.output_dir, "wb") as outfile: pickle.dump((all_genesis_preds, all_frame_preds), outfile)
def save(self, path): self.model.save(path) K.clear_session()
def modify_search_args(args: Namespace): """ Modifies and validates searching arguments in place. :param args: Arguments. """ # TODO : Take glob_mod_name as input argument glob_mod_name = 'g1' local_mod_no = glob_mod_name[-1] #Load Global Model print("--------< LOADING GLOBAL MODEL >----------") global_model = load_model(os.path.join(args.global_model_path + '/')) print('Global model loaded successfully!') print("-" * 62) # initial list to collect local model weights after scalling scaled_local_weight_list = list() print("--------< LOADING CLIENT MODELS >----------") for subdir, dirs, files in os.walk(args.client_dir): if str(subdir)[-2:] == 'l{}'.format(local_mod_no): # TODO - sampleNo/Total sample - write code below, using metadataof each client # for file in files: # if 'metadata' in str(file): # with open(os.path.join(subdir, file)) as f: # m_data = json.load(f) # print(os.path.join(subdir, file)) # scale the model weights and add to list #TODO scaling factor is hard coded at the moment # scaling_factor = weight_scalling_factor(clients_batched, client) #Loading local model local_model = load_model(os.path.join(subdir + '/')) # scale the model weights and add to list scaling_factor = 0.2 scaled_weights = scale_model_weights(local_model.get_weights(), scaling_factor) scaled_local_weight_list.append(scaled_weights) # clear session to free memory after each communication round K.clear_session() print("--------< RUNNING FEDERATED AVERAGING ALGORITHM... >----------") # to get the average over all the local model, we simply take the sum of the scaled weights average_weights = sum_scaled_weights(scaled_local_weight_list) # update global model global_model.set_weights(average_weights) #Load test data test_batched = load_data(args.test_dataset_path) #NOTE: no. of keys in global model metadata file = rounds done till now try: with open(args.metadata_path + '/metrics.json') as f: glob_mod_metadata = json.load(f) comm_round = len(glob_mod_metadata.keys()) + 1 except: glob_mod_metadata = dict() comm_round = 1 #test global model and print out metrics after each communications round for (X_test, Y_test) in test_batched: global_acc, global_loss = test_model(X_test, Y_test, global_model, comm_round) #Save metrics in dict glob_mod_metadata['round_{}_results'.format(comm_round)] = { 'global_acc': global_acc, 'global_loss': global_loss } #Save metrics in metadata with open(args.metadata_path + '/metrics.json', 'w') as f: json.dump(glob_mod_metadata, f, indent=4) #Save hyperparameter / config in dict glob_mod_config = dict() glob_mod_config['round_{}_config'.format(comm_round)] = { 'global_acc': global_model.to_json() } with open(args.global_model_path + '/global_mod_config.json', 'w') as f: json.dump(glob_mod_config, f, indent=4)
# log setup current_script_name = os.path.basename(__file__).split('.')[0] log_path_filename = ''.join([local_submodule_settings['log_path'], current_script_name, '.log']) logging.basicConfig(filename=log_path_filename, level=logging.DEBUG, format='%(asctime)s %(levelname)s %(name)s %(message)s') logger = logging.getLogger(__name__) logHandler = handlers.RotatingFileHandler(log_path_filename, maxBytes=10485760, backupCount=5) logger.addHandler(logHandler) # set random seed for reproducibility --> done in _2_train.py module np.random.seed(42) tf.random.set_seed(42) # clear session kb.clear_session() # functions definitions def general_mean_scaler(local_array): if len(local_array) == 0: return "argument length 0" mean_local_array = np.mean(local_array, axis=1) mean_scaling = np.divide(local_array, 1 + mean_local_array) return mean_scaling, mean_local_array def window_based_normalizer(local_window_array): if len(local_window_array) == 0: return "argument length 0"
def main(args): myseed = None if args.seed == 0 else args.seed p1 = args.p1 lr = args.lr W = 1 batchsize = args.batchsize maxepochs = args.maxepochs initializer = args.initializer activation = args.activation masktype = args.masktype train_weights = str2bool(args.trainweights) train_masks = str2bool(args.trainmasks) alpha = args.alpha trainingtype = "" + "TrainWeights" * train_weights + "TrainMasks" * train_masks outputpath = args.outputpath + "/" + trainingtype data = None network = None if "Conv" in args.nettype: csize = int(args.nettype[-1]) # Pre-calculated W scaling factor (depends on the architecture) # See https://arxiv.org/abs/2006.16627 for how to calculate W if initializer == "binary": if csize == 6: W = 8.344820201940066e-12 if csize == 4: W = 4.806616356300754e-09 if csize == 2: W = 1.384305440187043e-06 data = utils.SetMyData("CIFAR", W) outputpath += "/Conv" + str(csize) network = PrepareConvolutional(csize, data, myseed, initializer, activation, masktype, train_weights, train_masks, p1, alpha) if args.nettype == "LeNet": if initializer == "binary": W = 0.0005942073791483592 data = utils.SetMyData("MNIST", W) outputpath += "/LeNet" network = PrepareMaskedMLP(data, myseed, initializer, activation, masktype, train_weights, train_masks, p1, alpha) outputpath += "/P1_" + str(p1) + "/alpha_" + str(alpha) outputpath += "/" + masktype + "_" + activation + "_" + initializer + "_LR" + str( lr) outputpath += "/" + "bs_" + str(batchsize) + "/" network.compile(loss='categorical_crossentropy', optimizer=tf.keras.optimizers.Adam(lr=lr), metrics=['accuracy']) network.summary() NetworkTrainer(network, data, outputpath, batchsize, maxepochs) kb.clear_session() return
def test_tuners(tuner, data, y, args): import os from hypermodel_utils import load_partition, get_callbacks, get_results_table, allocate_stats, mlflow_logs import tensorflow.keras.callbacks as C MODEL_TYPE = args.model_type TIMESTAMP = args.timestamp samples = args.samples X = [x for x in data.X] FIT_MAX_EPOCHS = args.epochs # Get the best hyperparameters from the search best_tuner_mcc = -1000.0 best_tuner_idx = 0 tuners = tuner.oracle.get_best_trials(num_trials=1) results = list() for idx, t in enumerate(tuners): # loss = t.metrics.get_best_value('loss') # acc = t.metrics.get_best_value('accuracy') # print('- ' * 30) # print('{}\t{}'.format(loss, acc)) # print(t.hyperparameters.values) # Build the model using the best hyperparameters # params = tuner.get_best_hyperparameters()[idx] params = t.hyperparameters print('PARAMS:', params) model = tuner.hypermodel.build(params) model.summary() kf = StratifiedShuffleSplit(n_splits=args.cv, random_state=args.samples[0]) kf.get_n_splits(X[0], y) cv_partition = 0 results.append(get_results_table()) best_weights = [None for _ in range(args.cv)] best_stats = [None for _ in range(args.cv)] # Cross validation loop for cv_train_index, cv_test_index in kf.split(X[0], y): cv_partition += 1 (x_train, y_train), (x_test, y_test) = data.load_partition( cv_train_index, cv_test_index) callbacks = get_callbacks(args, cv_partition) best_cv_mcc = -10000.0 best_cv_stats = None # Validation samples loop for s, seed in enumerate(samples): print('TUNER {} - CV {} - {} Train on SEED {}'.format( idx, cv_partition, s, seed)) weight_file_name = '{}-{}-partition_{}-seed_{}'.format( MODEL_TYPE, TIMESTAMP, cv_partition, s) weight_file_name += '-epoch_{epoch:02d}.hdf5' weight_path = os.path.join(args.weights_dir, weight_file_name) callbacks[0] = C.ModelCheckpoint(weight_path, save_best_only=True, save_weights_only=True, verbose=1) kf = StratifiedShuffleSplit(n_splits=1, random_state=seed, test_size=0.05) kf.get_n_splits(x_train, y_train) for t_index, v_index in kf.split(x_train[0], y_train): (xx_train, yy_train), val_data = data.load_partition( t_index, v_index) model.fit(x=xx_train, y=yy_train, batch_size=args.batch_size, epochs=FIT_MAX_EPOCHS, validation_data=val_data, callbacks=callbacks, class_weight={ 0: .2, 1: .8 }, verbose=0) # # Data Augmentation ================================================= # train_datagen = AugmentedGeneratorMultipleInputs(xx_train, yy_train, args.batch_size) # _steps_per_epoch = int(len(yy_train) / args.batch_size) # print("_steps_per_epoch", _steps_per_epoch) # # model.fit_generator(train_datagen.flow(xx_train, yy_train, batch_size=args.batch_size), # model.fit_generator(train_datagen, # validation_data=val_data, # steps_per_epoch=_steps_per_epoch, # epochs=FIT_MAX_EPOCHS, # callbacks=callbacks, # verbose=1) # # ================================================================== K.clear_session() # # Test model fitted using seed validation set # stats, y_pred = get_test_stats(model, x_test, y_test) # print(stats.Mcc) # print(stats.F1) cv_idx = cv_partition - 1 best_stats[cv_idx], best_weights[cv_idx] = do_partition_test( model, cv_partition, args, test_data=(x_test, y_test)) results[idx] = allocate_stats(results[idx], best_stats[cv_idx], cv_partition) # Persist weights files cv_mean_mcc = np.mean(results[idx]['mcc']) tuner_folder_name = '{}-{}-tuner_{}-mcc_{:.4f}'.format( MODEL_TYPE, TIMESTAMP, idx, cv_mean_mcc) folder_name = os.path.join(args.best_weights_dir, tuner_folder_name) if not os.path.exists(folder_name): os.makedirs(folder_name) for cv in range(args.cv): weight_name = '{}-{}-partition_{}-mcc_{:.4f}.hdf5'.format( MODEL_TYPE, TIMESTAMP, cv, results[idx]['mcc'][cv]) weight_path = os.path.join(folder_name, weight_name) model.set_weights(best_weights[cv]) model.save_weights(weight_path) for idx, t in enumerate(tuners): print('EVAL TUNER {} - {}'.format(idx, np.mean(results[idx]['mcc']))) model = tuner.hypermodel.build(t.hyperparameters) mlflow_logs(args, t.hyperparameters.values, results[idx], model, idx)
if __name__ == '__main__': from itertools import product import numpy as np import random from time import time t0 = time() expr_out_dir = "/media/wheatley/38882E5E882E1AC0/Deep_Face_Verifier/experiments" logs_out_dir = "/media/wheatley/38882E5E882E1AC0/Deep_Face_Verifier/tensorboard_logs" extracted_dir = "extracted" models_types = ["eucl", "cos"] extract_layers = np.arange(3) combination_funcs = np.arange(4) n_neurons = [16, 1024] n_layers = np.arange(2) combs = list(product(models_types, extract_layers, combination_funcs, n_neurons, n_layers)) random.shuffle(combs) for m_type, ext_lay, comb_f, n_neur, n_lay in combs: n_neur = 0 if (n_lay == 0) else n_neur exp_name = "{}_outLay:{}_combFun:{}_nNeur:{}_nLay:{}".format(m_type, ext_lay, comb_f, n_neur, n_lay) if not os.path.isdir(os.path.join(expr_out_dir, exp_name)): clear_session() input_size = 2622 if ext_lay == 2 else 4096 model = build_model.build_eucl_model(input_size, comb_f, n_neur, n_lay, True) if m_type == "eucl" else build_model.build_cos_model(input_size, comb_f, n_neur, n_lay, True) run_experiment(model, os.path.join(extracted_dir, "layer_{}".format(ext_lay)), exp_name, experiments_dir=expr_out_dir, tensorboard_logs_dir=logs_out_dir) print("Total Time:", time() - t0)
def train_test(model, data, y, args): import os import tensorflow.keras.backend as bk import tensorflow as tf from hypermodel_utils import load_partition, get_callbacks, get_results_table, allocate_stats, mlflow_logs MODEL_TYPE = args.model_type TIMESTAMP = args.timestamp samples = args.samples X = [x for x in data.X] FIT_MAX_EPOCHS = args.epochs results = list() model.summary() kf = StratifiedShuffleSplit(n_splits=args.cv, random_state=args.samples[0]) kf.get_n_splits(X[0], y) results.append(get_results_table()) best_weights = [None for _ in range(args.cv)] best_stats = [None for _ in range(args.cv)] metrics_stats = { m: {'p{:02d}'.format(x): [] for x in range(args.cv)} for m in ('Prec', 'Sn', 'Sp', 'Acc', 'F1', 'Mcc') } weights_sufix = '.hdf5' # Cross validation loop for idx, (cv_train_index, cv_test_index) in enumerate(kf.split(X[0], y)): # TODO split nucleotides strings, not matrix (x_train, y_train), (x_test, y_test) = data.load_partition( cv_train_index, cv_test_index) callbacks = get_callbacks(args, idx) best_cv_mcc = -10000.0 # === Perform Training Phase === # Iterate over samples to create different samples on same partition based on different train-val splits for s, seed in enumerate(samples): print('CV PARTITION {} - CV SEED {} | Train on SEED {}'.format( idx, s, seed)) weight_file_prefix = '{}-{}-partition_{:02d}'.format( MODEL_TYPE, TIMESTAMP, idx) weight_file_name = weight_file_prefix + '-sample_{:02d}'.format(s) weight_file_name += '-epoch_{epoch:02d}.hdf5' weight_path = os.path.join(args.weights_dir, weight_file_name) callbacks[0] = tf.keras.callbacks.ModelCheckpoint( weight_path, save_best_only=True, save_weights_only=True, verbose=1) kf = StratifiedShuffleSplit(n_splits=1, random_state=seed, test_size=0.05) kf.get_n_splits(x_train, y_train) histories = [] for t_index, v_index in kf.split(x_train[0], y_train): (xx_train, yy_train), val_data = data.load_partition(t_index, v_index) history = model.fit(x=xx_train, y=yy_train, batch_size=args.batch_size, epochs=FIT_MAX_EPOCHS, validation_data=val_data, callbacks=callbacks, class_weight={ 0: .2, 1: .8 }, verbose=0) # PLOT HISTORY _plot_history(args, 'accuracy', history, idx, s) _plot_history(args, 'loss', history, idx, s) # # Data Augmentation ================================================= # train_datagen = AugmentedGeneratorMultipleInputs(xx_train, yy_train, args.batch_size) # _steps_per_epoch = int(len(yy_train) / args.batch_size) # print("_steps_per_epoch", _steps_per_epoch) # # model.fit_generator(train_datagen.flow(xx_train, yy_train, batch_size=args.batch_size), # model.fit_generator(train_datagen, # validation_data=val_data, # steps_per_epoch=_steps_per_epoch, # epochs=FIT_MAX_EPOCHS, # callbacks=callbacks, # verbose=1) # # ================================================================== # # Test model fitted using seed validation set # stats, y_pred = get_test_stats(model, x_test, y_test) # print(stats.Mcc) # print(stats.F1) # === Perform Test Phase === sample_weights = [ x for x in os.listdir(args.weights_dir + os.path.sep) if x.startswith(weight_file_prefix) and x.endswith(weights_sufix) ] # Iterate over all weights saved on checkpoints for this sample of this partition for i, f in enumerate(sample_weights): best_sample_mcc = -10000.0 best_sample_stats = None bk.clear_session() name = os.path.join(args.weights_dir, f) # print(name) model.load_weights(name) stats, y_pred = get_test_stats(model=model, X=x_test, y=y_test) # Select best sample weights if best_sample_mcc < stats.Mcc: best_sample_mcc = stats.Mcc best_sample_stats = stats # Select best weigths for this partition if best_cv_mcc < stats.Mcc: best_cv_mcc = stats.Mcc selected_weight = f best_stats[idx] = stats # print(stats) for metric in best_sample_stats.get_stats_types(): sample_stats = best_sample_stats.to_dict() metrics_stats[metric]['p{:02}'.format(idx)].append( sample_stats[metric]) # Delete temporary weights for i, f in enumerate(sample_weights): if f != selected_weight: # print('Deleting weight: {}'.format(f)) path = args.weights_dir + '/' + f os.remove(path) bk.clear_session() print(best_sample_stats) # Persist best weights of this partition on logs args.logs.set_metrics(**metrics_stats) # Log metrics args.logs.set_artifacts() # Log artifacts model.load_weights(os.path.join(args.weights_dir, selected_weight)) persist_model_path = os.path.join( args.best_weights_dir, 'model_{}_{}_p{:02d}'.format(MODEL_TYPE, TIMESTAMP, idx)) # mlflow_keras.save_model(model, persist_model_path) # mlflow_keras.log_model(model, args.logs.get_model_path()) mlflow_keras.save_model(model, args.logs.get_model_path(idx)) mlflow_keras.log_model(model, 'models') # print('Deleting weight: {}'.format(selected_weight)) path = args.weights_dir + '/' + selected_weight os.remove(path)
def reset(): K.clear_session()
def fit(self, trainX, trainy, timey, valX=None, valy=None, use_pretrained=False): """ Fit the model to training data """ start_time = time.time() # clear memory K.clear_session() # allocate lists for the ensemble self.ensemble = [] self.history = [] self.val_loss = [] self.segment_len = trainX.shape[0] // self.hyperparameters['n_segments'] if self.hyperparameters['n_segments'] == 1 and (valX is not None or valy is not None): warnings.warn( "Validation and test data set are the same if n_segements is 1!" ) i = 0 while i < self.hyperparameters['n_members_segment']: j = 0 while j < self.hyperparameters['n_segments']: print("build") ensemble_member = self.build_model(trainX.shape[1]) n_ens_sel = len(self.ensemble) small_print_header( f"Train member Nr {n_ens_sel+1}/{self.hyperparameters['n_members']}" ) if use_pretrained: ensemble_member.load_weights(self.pretrained_weights) print("compile") ensemble_member.compile(loss=self.loss, optimizer=self.optimizer, metrics=[self.loss]) # validate on the spare segment if self.hyperparameters['n_segments'] != 1: if valX is not None or valy is not None: warnings.warn( "Validation data set will be one of the segments. The provided validation data set is not used!" ) start_ind = j * self.segment_len end_ind = (j + 1) * self.segment_len trainXens = np.delete(trainX, np.s_[start_ind:end_ind], axis=0) trainyens = np.delete(trainy, np.s_[start_ind:end_ind]) # upsample el nino like periods if False: timeyens = np.delete(timey, np.s_[start_ind:end_ind]) elninolike = (timeyens >= f'1982-01-01') & ( timeyens <= f'2001-12-01') laninalike = np.invert(elninolike) trainXens_nino, trainyens_nino = trainXens[ elninolike], trainyens[elninolike] trainXens_nina, trainyens_nina = trainXens[ laninalike], trainyens[laninalike] nino_choices = np.random.choice(np.arange( len(trainyens_nino)), size=1000) nina_choices = np.random.choice(np.arange( len(trainyens_nina)), size=1000) trainXens = np.concatenate( (trainXens_nino[nino_choices], trainXens_nina[nina_choices])) trainyens = np.concatenate( (trainyens_nino[nino_choices], trainyens_nina[nina_choices])) valXens = trainX[start_ind:end_ind] valyens = trainy[start_ind:end_ind] if False: valXens, trainXens = trainXens, valXens valyens, trainyens = trainyens, valyens # validate on test data set elif self.hyperparameters['n_segments'] == 1: if valX is None or valy is None: raise MissingArgumentError( "When segments length is 1, a validation data set must be provided." ) trainXens = trainX trainyens = trainy valXens = valX valyens = valy history = ensemble_member.fit( trainXens, trainyens, epochs=self.hyperparameters['epochs'], batch_size=self.hyperparameters['batch_size'], verbose=self.hyperparameters['verbose'], shuffle=True, callbacks=[self.es], validation_data=(valXens, valyens)) self.history.append(history) self.val_loss.append( ensemble_member.evaluate(valXens, valyens)[1]) self.ensemble.append(ensemble_member) j += 1 i += 1 self.mean_val_loss = np.mean(self.val_loss) print(f'Loss: {self.mean_val_loss}') # print computation time end_time = time.time() passed_time = np.round(end_time - start_time, decimals=1) print(f'Computation time: {passed_time}s')
def make_image_classifier(tfhub_module, image_dir, hparams, requested_image_size=None, saveModelDir=False): """Builds and trains a TensorFLow model for image classification. Args: tfhub_module: A Python string with the handle of the Hub module. image_dir: A Python string naming a directory with subdirectories of images, one per class. hparams: A HParams object with hyperparameters controlling the training. requested_image_size: A Python integer controlling the size of images to feed into the Hub module. If the module has a fixed input size, this must be omitted or set to that same value. """ print("Using hparams:") for key, value in hparams._asdict().items(): print("\t{0} : {1}".format(key, value)) module_layer = hub.KerasLayer(tfhub_module, trainable=hparams.do_fine_tuning) image_size = _image_size_for_module(module_layer, requested_image_size) print("Using module {} with image size {}".format(tfhub_module, image_size)) train_data_and_size, valid_data_and_size, labels = _get_data_with_keras( image_dir, image_size, hparams.batch_size, hparams.validation_split, hparams.do_data_augmentation) print("Found", len(labels), "classes:", ", ".join(labels)) model = build_model(module_layer, hparams, image_size, len(labels)) # If we are fine-tuning, check and see if weights # already exists at the output directory. This way, a user # can simply run two consecutive training sessions. One without # fine-tuning, followed by another with. if hparams.do_fine_tuning: if saveModelDir is not None: existingWeightsPath = os.path.join(saveModelDir, "saved_model_weights.h5") if os.path.exists(existingWeightsPath): print("Loading existing weights for fine-tuning") model.load_weights(existingWeightsPath) train_result = train_model(model, hparams, train_data_and_size, valid_data_and_size) # Tear down model, set training to 0 and then re-create. # 1 - Save model weights as Keras H5. tempDir = tempfile.gettempdir() tempModelWeightsFile = os.path.join(tempDir, "weights.h5") model.save_weights(tempModelWeightsFile) # 2 - Set training to 0 K.clear_session() K.set_learning_phase(0) # 3 - Create model again model = build_model(module_layer, hparams, image_size, len(labels)) # 4 - Load model weights. model.load_weights(tempModelWeightsFile) # Clean up temp weights file os.remove(tempModelWeightsFile) # 5 - Pass model to lib.model_to_frozen_graph. frozen_inference_graph = model_to_frozen_graph(model) return model, labels, train_result, frozen_inference_graph
def __del__(self): K.clear_session()
def predict_traffic_file(): #K.clear_session() #BELOW docstring lines are required to support swagger documentation """ Endpoint returning traffic sign image prediction --- parameters: - name: input_file in: formData type: file required: true """ # Get the input file from the http request #read image file string data filestr = request.files['input_file'].read() #convert string data to numpy array npimg = np.fromstring(filestr, np.uint8) # convert numpy array to image img = cv2.imdecode(npimg, cv2.IMREAD_UNCHANGED) in_image = cv2.resize(img, (32, 32), interpolation = cv2.INTER_AREA) in_image = np.expand_dims(in_image, axis = 0) # Load the saved traffic sign keras model model_filename = "model.h5" # Load model from file - read mode traffic_model = load_model(model_filename, compile=False) # Make prediction using the input image file ## check dimensions before and after of the numpy array to see 2D, 3D, 4D result = traffic_model.predict(in_image) def sign(i): switcher={ 0:'Maximum speed limit (10 km/h)', 1:'Maximum speed limit (30 km/h)', 2:'Maximum speed limit (50 km/h)', 3:'Maximum speed limit (60 km/h)', 4:'Maximum speed limit (70 km/h)', 5:'Maximum speed limit (80 km/h)', 6:'End 80 km/h speed limit', 7:'Maximum speed limit (100 km/h)', 8:'Maximum speed limit (120 km/h)', 9:'No Overtaking', 10:'No overtaking by trucks/heavy goods vehicles', 11:'Crossroads ahead with a minor road', 12:'Priority road', 13:'Give way', 14:'Stop', 15:'Road closed to all traffic', 16:'No trucks/heavy goods vehicles', 17:'No Entry', 18:'Attention: Other Dangers!', 19:'Curve to the left', 20:'Curve to the right', 21:'Series of curves, first to the left', 22:'Uneven road surface', 23:'Slippery road surface', 24:'Road narrows on the right', 25:'Roadworks', 26:'Traffic signals ahead', 27:'Pedestrian crossing ahead', 28:'Watch for children', 29:'Watch for cyclists', 30:'Risk of ice', 31:'Watch for wild animals', 32:'End all previously signed restrictions or prohibitions', 33:'Turn right only', 34:'Turn left only', 35:'Proceed straight ahead only', 36:'Proceed straight or turn right', 37:'Proceed straight or turn left', 38:'Keep right', 39:'Keep left', 40:'Roundabout', 41:'End overtaking prohibition', 42:'End overtaking prohibition for trucks/heavy goods vehicles', } return switcher.get(i,"Unknown") print(sign(np.argmax(result))) K.clear_session() # Send the prediction as response return str(sign(np.argmax(result)))
val_labels) = (train_data / 255, train_labels / 255), (val_data / 255, val_labels / 255) epochs = [500] csv_file = "../historique_tests/best_mlp_without_regularisation.csv" structs = generateStructsFromCSV(csv_file) struct = structs[5] for j in range(1): struct.use_dropout = True struct.dropout_indexes = [0] # dropout_indexes_number = randint(1, struct.nb_hidden_layers) struct.dropout_value = 0.3 # for j in range(dropout_indexes_number): # dropout_indexes.append(randint(1, struct.nb_hidden_layers)) model = [create_custom_mlp(struct)] desc = [getMlpStructAsString(struct)] print(desc[0]) test_models('mlp_best_models_with_dropout_1', model, desc, train_data, train_labels, val_data, val_labels, epochs_p=epochs, batch_size_p=8192) clear_session() #clear tf GPU memory
def parameterSearch(self, paramSets, X, Y, numSplits=2, valSplit=0.0, epochs=1, batchSize=None, saveModel=False, visualize=False, saveLoc=''): # create CV dat LOOV #numSplits = 2 Kf = StratifiedKFold(n_splits=numSplits) callBacks = [ EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True) ] if (visualize): callBacks.append( TensorBoard(log_dir='./logs', histogram_freq=3, write_graph=False, write_images=False, update_freq='epoch', profile_batch=2, embeddings_freq=0, embeddings_metadata=None)) #for each parameter set # make a model # #X = [0,1,2,3,4,5,6,7,8,9] modelFile = open(self.outputPath + "fileModel.csv", 'w') resultFile = open(self.outputPath + "fileResult.csv", 'w') resultFile.write( "modelNum|True REM|False REM|False NonREM|True NonREM|Acc|Sens|Spec|Recall|Precision|f1score|finalLoss\n" ) modelNum = 0 for paramSet in paramSets: modelFile.write(str(modelNum) + "|") json.dump(paramSet, modelFile) modelFile.write("\n") print("\n\n=================\nTesting Model " + str(modelNum) + "\n=================\n") #print(paramSet, flush=True) try: model = self.convModel(paramSet) print(model.summary()) #model.save_weights('temp_weights.h5') j = 0 for trainInd, testInd in Kf.split(X, np.argmax(Y, axis=1)): fitHistory = model.fit(X[trainInd], Y[trainInd], batch_size=batchSize, verbose=0, validation_split=valSplit, epochs=epochs, callbacks=callBacks) if (saveModel): modelWeightFile = saveLoc + f'{modelNum}.{j}.weights.h5' model.save_weights(modelWeightFile) #model.save(modelWeightFile) Ypred = np.zeros((testInd.shape[0], Y.shape[1])) Yi = 0 for pred in np.argmax(model.predict(X[testInd], batch_size=None), axis=1): Ypred[Yi][pred] = 1 Yi += 1 #NOTE: #confusionMatrix = multilabel_confusion_matrix(Y[testInd], Ypred)[0] ##print(confusionMatrix) ##confusionMatrix = confusion_matrix(np.argmax(Y[testInd], axis=1), np.argmax(Ypred, axis=1)) ##print(confusionMatrix) ##print('f1_score:',f1_score(Y[testInd], Ypred, average='macro')) #resultFile.write(str(modelNum) + "|") ##for row in confusionMatrix: ## for el in row: ## resultFile.write(str(el) + "|") ##"modelNum|True REM|False NonREM|False REM|True NonREM|Acc|Sens|Spec|Recall|Precision|f1score\n" # #tn = confusionMatrix[0][0] #fn = confusionMatrix[1][0] #tp = confusionMatrix[1][1] #fp = confusionMatrix[0][1] tp = tn = fn = fp = 0 Yi = 0 for y in Y[testInd]: tp += Ypred[Yi][0] * y[0] fp += max(Ypred[Yi][0] - y[0], 0) tn += Ypred[Yi][1] * y[1] fn += max(Ypred[Yi][1] - y[1], 0) Yi += 1 acc = sens = spec = prec = rec = f1 = 0 acc = (tp + tn) / (tp + tn + fp + fn) if (tp + fn > 0): sens = tp / (tp + fn) if (tn + fp > 0): spec = tn / (tn + fp) if (tp + fp > 0): prec = tp / (tp + fp) if (tp + fn > 0): rec = tp / (tp + fn) if (prec + rec > 0): f1 = 2 * ((prec * rec) / (prec + rec)) resultFile.write( f"{modelNum}|{tp:.3f}|{fp:.3f}|{fn:.3f}|{tn:.3f}|{acc:.3f}|{sens:.3f}|{spec:.3f}|{rec:.3f}|{prec:.3f}|{f1:.3f}|{fitHistory.history['loss'][-1]:10.3f}\n" ) print( f"{'Validate':10s}|{'modelNum':10s}|{'tp':10s}|{'fp':10s}|{'fn':10s}|{'tn':10s}|{'acc':10s}|{'sens':10s}|{'spec':10s}|{'rec':10s}|{'prec':10s}|{'f1':10s}|{'loss':10s}\n" ) print( f"{j:10d}|{modelNum:10d}|{tp:10.3f}|{fp:10.3f}|{fn:10.3f}|{tn:10.3f}|{acc:10.3f}|{sens:10.3f}|{spec:10.3f}|{rec:10.3f}|{prec:10.3f}|{f1:10.3f}|{fitHistory.history['loss'][-1]:10.3f}\n", flush=True) #resultFile.write(str(f1_score(Y[testInd], Ypred, average='macro')) + "|\n") #model.load_weights('temp_weights.h5') self.reset_weights(model) j += 1 except Exception as e: resultFile.write("error\n") print(str(e)) K.clear_session() modelNum += 1 if self.killer.kill_now: resultFile.write("killed\n") print("killed") break modelFile.close() resultFile.close()
def run_whole_procedure(fig_prefix, model_type, cp_family, att_type): """ execute the whole procedure :param fig_prefix: :return: """ global ins2vec_model, word_index, x_test, y_test # clear existing tf graph at the start of each iteration kb.clear_session() # re-setup sess_setup() # step 1: load pre-trained ins2vec model print('loading pre-trained ins2vec model...') ins2vec_model = Word2Vec.load(dic_file_path) # step 2: construct ins2vec embeddings print('constructing pre-trained ins2vec embeddings...') word_index, embeddings_matrix = construct_ins_embedding() # step 3: construct the textcnn model print('setting up the neural model...') rnn_models = RNNModels(MAX_SEQUENCE_LENGTH, len(embeddings_matrix), EMBEDDING_DIM, CLASS_NUMBER, 'softmax', att_type=att_type) if model_type == 'cudnn_ver': model, fn = rnn_models.get_cudnn_version_model(embeddings_matrix, DROP_OUT) elif model_type == 'cnn_rnn': model, fn = rnn_models.get_cnn_rnn_model(embeddings_matrix, DROP_OUT) elif model_type == 'bidirectional_cudnn': model, fn = rnn_models.get_bidirectional_cudnn_model( embeddings_matrix, DROP_OUT) elif model_type == 'naive': model, fn = rnn_models.get_naive_version_model(embeddings_matrix, DROP_OUT) elif model_type == 'bidirectional': model = rnn_models.get_bidirectional_model(embeddings_matrix, DROP_OUT) elif model_type == 'multilayer_stateful': model = rnn_models.get_multilayer_stateful_model( embeddings_matrix, DROP_OUT) elif model_type == 'multilayer': model = rnn_models.get_multilayer_model(embeddings_matrix, DROP_OUT) else: print('!!!!!!!!!!NO SUCH MODEL: ' + model_type) # 搭建神经网络完成后,这一步相当于编译它 opti = optimizers.Adam() model.compile(opti, 'categorical_crossentropy', metrics=['accuracy']) # 多分类 # step 4: train the model with available dataset print('---------------------Step 4---------------------------') if unique_fun_stg == 'Intact': texts, labels = prepare_data(corpus_path, path_stg_suffix, cp_family) else: texts, labels = prepare_data_with_unique(corpus_path, path_stg_suffix, unique_fun_path, cp_family) print('train-test-data split and shuffle...') x_train, x_test, y_train, y_test = train_test_split( texts, labels, test_size=dataset_split_ratio, random_state=SEED, shuffle=True) print(len(x_train), 'train sequences') print(len(x_test), 'test sequences') # train the neural network print('training...') start = time.clock() model = neural_net_train(model, x_train=x_train, y_train=y_train, fig_prefix=fig_prefix) end = time.clock() print('neural-net training takes: %s seconds' % (end - start)) # model is got and stored # step 5: evaluate the model on test data model_evaluaiton(model, fig_prefix)
def main(dataset_name): dataset = load_dataset() raw_data = np.asarray(dataset['raw']['data']) raw_label = np.asarray(dataset['raw']['label']) num_classes = len(np.unique(raw_label)) rskf = RepeatedStratifiedKFold(n_splits=k_folds, n_repeats=k_fold_reps, random_state=42) for e2efs_class in e2efs_classes: print('E2EFS-Method : ', e2efs_class.__name__) cont_seed = 0 nfeats = [] accuracies = [] model_accuracies = [] svc_accuracies = [] fs_time = [] BAs = [] svc_BAs = [] model_BAs = [] mAPs = [] svc_mAPs = [] model_mAPs = [] mus = [] name = dataset_name + '_' + kernel + '_mu_' + str(mu) print(name) for j, (train_index, test_index) in enumerate(rskf.split(raw_data, raw_label)): print('k_fold', j, 'of', k_folds * k_fold_reps) train_data, train_labels = raw_data[train_index], raw_label[ train_index] test_data, test_labels = raw_data[test_index], raw_label[ test_index] train_labels = to_categorical(train_labels, num_classes=num_classes) test_labels = to_categorical(test_labels, num_classes=num_classes) valid_features = np.where(np.abs(train_data).sum(axis=0) > 0)[0] if len(valid_features) < train_data.shape[1]: print('Removing', train_data.shape[1] - len(valid_features), 'zero features') train_data = train_data[:, valid_features] test_data = test_data[:, valid_features] model_kwargs = { 'mu': mu / len(train_data), 'kernel': kernel, 'degree': 3 } svc_kwargs = {'C': 1.0, 'solver': 0.} for i, n_features in enumerate([10, 50, 100, 150, 200]): n_accuracies = [] n_svc_accuracies = [] n_model_accuracies = [] n_BAs = [] n_svc_BAs = [] n_model_BAs = [] n_mAPs = [] n_svc_mAPs = [] n_model_mAPs = [] n_train_accuracies = [] n_time = [] print('n_features : ', n_features) heatmaps = [] weight = train_labels[:, -1].mean() for r in range(reps): np.random.seed(cont_seed) K.tf.set_random_seed(cont_seed) cont_seed += 1 model = train_Keras( train_data, train_labels, test_data, test_labels, model_kwargs, e2efs_class=e2efs_class, n_features=n_features, ) heatmaps.append(K.eval(model.heatmap)) n_time.append(model.fs_time) test_data_norm = model.normalization.transform(test_data) train_data_norm = model.normalization.transform(train_data) test_pred = model.predict(test_data_norm) n_model_accuracies.append( model.evaluate(test_data_norm, test_labels, verbose=0)[-1]) n_model_BAs.append(balance_accuracy( test_labels, test_pred)) n_model_mAPs.append( average_precision_score(test_labels[:, -1], test_pred)) train_acc = model.evaluate(train_data_norm, train_labels, verbose=0)[-1] print('n_features : ', n_features, ', accuracy : ', n_model_accuracies[-1], ', BA : ', n_model_BAs[-1], ', mAP : ', n_model_mAPs[-1], ', train_accuracy : ', train_acc, ', time : ', n_time[-1], 's') del model K.clear_session() heatmap = np.mean(heatmaps, axis=0) best_features = np.argsort(heatmap)[::-1][:n_features] svc_train_data = train_data[:, best_features] svc_test_data = test_data[:, best_features] norm = normalization_func() svc_train_data_norm = norm.fit_transform(svc_train_data) svc_test_data_norm = norm.transform(svc_test_data) bestcv = -1 bestc = None bestSolver = None for s in [0, 1, 2, 3]: for my_c in [ 0.001, 0.1, 0.5, 1.0, 1.4, 1.5, 1.6, 2.0, 2.5, 5.0, 100.0 ]: cmd = '-v 5 -s ' + str(s) + ' -c ' + str(my_c) + ' -q' cv = liblinearutil.train( (2 * train_labels[:, -1] - 1).tolist(), svc_train_data_norm.tolist(), cmd) if cv > bestcv: # print('Best -> C:', my_c, ', s:', s, ', acc:', cv) bestcv = cv bestc = my_c bestSolver = s svc_kwargs['C'] = bestc svc_kwargs['solver'] = bestSolver print('Best -> C:', bestc, ', s:', bestSolver, ', acc:', bestcv) for r in range(reps): np.random.seed(cont_seed) K.tf.set_random_seed(cont_seed) cont_seed += 1 model = train_SVC(svc_train_data_norm, train_labels, svc_kwargs) _, accuracy, test_pred = liblinearutil.predict( (2 * test_labels[:, -1] - 1).tolist(), svc_test_data_norm.tolist(), model, '-q') test_pred = np.asarray(test_pred) n_svc_accuracies.append(accuracy[0]) n_svc_BAs.append(balance_accuracy(test_labels, test_pred)) n_svc_mAPs.append( average_precision_score(test_labels[:, -1], test_pred)) del model model = train_Keras(svc_train_data, train_labels, svc_test_data, test_labels, model_kwargs) train_data_norm = model.normalization.transform( svc_train_data) test_data_norm = model.normalization.transform( svc_test_data) test_pred = model.predict(test_data_norm) n_BAs.append(balance_accuracy(test_labels, test_pred)) n_mAPs.append( average_precision_score(test_labels[:, -1], test_pred)) n_accuracies.append( model.evaluate(test_data_norm, test_labels, verbose=0)[-1]) n_train_accuracies.append( model.evaluate(train_data_norm, train_labels, verbose=0)[-1]) del model K.clear_session() print( 'n_features : ', n_features, ', acc : ', n_accuracies[-1], ', BA : ', n_BAs[-1], ', mAP : ', n_mAPs[-1], ', train_acc : ', n_train_accuracies[-1], ', svc_acc : ', n_svc_accuracies[-1], ', svc_BA : ', n_svc_BAs[-1], ', svc_mAP : ', n_svc_mAPs[-1], ) if i >= len(accuracies): accuracies.append(n_accuracies) svc_accuracies.append(n_svc_accuracies) model_accuracies.append(n_model_accuracies) BAs.append(n_BAs) mAPs.append(n_mAPs) fs_time.append(n_time) svc_BAs.append(n_svc_BAs) svc_mAPs.append(n_svc_mAPs) model_BAs.append(n_model_BAs) model_mAPs.append(n_model_mAPs) nfeats.append(n_features) mus.append(model_kwargs['mu']) else: accuracies[i] += n_accuracies svc_accuracies[i] += n_svc_accuracies model_accuracies[i] += n_model_accuracies fs_time[i] += n_time BAs[i] += n_BAs mAPs[i] += n_mAPs svc_BAs[i] += n_svc_BAs svc_mAPs[i] += n_svc_mAPs model_BAs[i] += n_model_BAs model_mAPs[i] += n_model_mAPs output_filename = directory + 'LinearSVC_' + kernel + '_' + e2efs_class.__name__ + '.json' if not os.path.isdir(directory): os.makedirs(directory) info_data = { 'kernel': kernel, 'reps': reps, 'classification': { 'mus': mus, 'n_features': nfeats, 'accuracy': accuracies, 'mean_accuracy': np.array(accuracies).mean(axis=1).tolist(), 'svc_accuracy': svc_accuracies, 'mean_svc_accuracy': np.array(svc_accuracies).mean(axis=1).tolist(), 'model_accuracy': model_accuracies, 'mean_model_accuracy': np.array(model_accuracies).mean(axis=1).tolist(), 'BA': BAs, 'mean_BA': np.array(BAs).mean(axis=1).tolist(), 'mAP': mAPs, 'mean_mAP': np.array(mAPs).mean(axis=1).tolist(), 'svc_BA': svc_BAs, 'svc_mean_BA': np.array(svc_BAs).mean(axis=1).tolist(), 'svc_mAP': svc_mAPs, 'svc_mean_mAP': np.array(svc_mAPs).mean(axis=1).tolist(), 'model_BA': model_BAs, 'model_mean_BA': np.array(model_BAs).mean(axis=1).tolist(), 'model_mAP': model_mAPs, 'model_mean_mAP': np.array(model_mAPs).mean(axis=1).tolist(), 'fs_time': fs_time } } for k, v in info_data['classification'].items(): if 'mean' in k: print(k, v) with open(output_filename, 'w') as outfile: json.dump(info_data, outfile)
def explain_with_saliency_map(fig_prefix, model_type, cp_family, att_type, input_files=None, func_list=None, top_k=10): """ get the prediction result explained with saliency map analysis :param model_path: :param input_files: list of files (each corresponds to an executable) :param func_list: list of functions (names) to be analyzed :param top_k: :return: """ global ins2vec_model, word_index, x_test, y_test, func_names # clear existing tf graph at the start of each iteration kb.clear_session() # re-setup sess_setup() # step 1: load pre-trained ins2vec model print('loading pre-trained ins2vec model...') ins2vec_model = Word2Vec.load(dic_file_path) # step 2: construct ins2vec embeddings print('constructing pre-trained ins2vec embeddings...') word_index, embeddings_matrix = construct_ins_embedding() # step 3: construct the textcnn model print('setting up the neural model...') rnn_models = RNNModels(MAX_SEQUENCE_LENGTH, len(embeddings_matrix), EMBEDDING_DIM, CLASS_NUMBER, 'softmax', att_type=att_type) if model_type == 'cudnn_ver': model, fn = rnn_models.get_cudnn_version_model(embeddings_matrix, DROP_OUT) elif model_type == 'cnn_rnn': model, fn = rnn_models.get_cnn_rnn_model(embeddings_matrix, DROP_OUT) elif model_type == 'bidirectional_cudnn': model, fn = rnn_models.get_bidirectional_cudnn_model( embeddings_matrix, DROP_OUT) elif model_type == 'naive': model, fn = rnn_models.get_naive_version_model(embeddings_matrix, DROP_OUT) elif model_type == 'bidirectional': model = rnn_models.get_bidirectional_model(embeddings_matrix, DROP_OUT) elif model_type == 'multilayer_stateful': model = rnn_models.get_multilayer_stateful_model( embeddings_matrix, DROP_OUT) elif model_type == 'multilayer': model = rnn_models.get_multilayer_model(embeddings_matrix, DROP_OUT) else: print('!!!!!!!!!!NO SUCH MODEL: ' + model_type) if unique_fun_stg == 'Intact': x_test, y_test, func_names = prepare_data(corpus_path, path_stg_suffix, cp_family, True) else: x_test, y_test, func_names = prepare_data_with_unique( corpus_path, path_stg_suffix, unique_fun_path, cp_family, True) print(len(x_test), 'test sequences') # load the trained model print('loading...' + model_store_path + '\n') # step 5: evaluate the model on test data model_evaluaiton_for_explain(model, fig_prefix, fn, top_k)
def main(args): train_df = data.load_data.load_custom_text_as_pd(args.train_data,sep='\t',header=True, \ text_column=['Text'],target_column=['Label']) val_df = data.load_data.load_custom_text_as_pd(args.val_data,sep='\t', header=False, \ text_column=['Text'],target_column=['Label']) train_df = pd.DataFrame(train_df, copy=False) val_df = pd.DataFrame(val_df, copy=False) val_df.columns = train_df.columns model_save_dir = args.model_save_path try: os.makedirs(model_save_dir) except OSError: pass nli_train_df = pd.DataFrame() nli_val_df = pd.DataFrame() mnli = data.data_utils.mnli_data(train_df.labels.unique()) df_ = pd.DataFrame() for i in tqdm(range(train_df.shape[0])): df_['Id'], df_['words'], df_['text2'], df_['labels'], df_['orig_label'] = mnli.convert_to_mnli_format(\ train_df.Id.iloc[i], train_df.words.iloc[i], train_df.labels.iloc[i]) nli_train_df = pd.concat([nli_train_df, df_]) for i in tqdm(range(val_df.shape[0])): df_['Id'], df_['words'], df_['text2'], df_['labels'], df_['orig_label'] = mnli.convert_to_mnli_format(\ val_df.Id.iloc[i], val_df.words.iloc[i], val_df.labels.iloc[i]) nli_val_df = pd.concat([nli_val_df, df_]) nli_train_df.labels, label2idx = data.data_utils.convert_categorical_label_to_int(nli_train_df.labels, \ save_path=os.path.join(model_save_dir,'label2idx.pkl')) nli_val_df.labels, _ = data.data_utils.convert_categorical_label_to_int(nli_val_df.labels, \ save_path=os.path.join(model_save_dir,'label2idx.pkl')) #print (nli_train_df.head(5)) #print (nli_val_df.head(5)) print("Tokenization") if 'bertweet' in args.transformer_model_name.lower(): bertweettokenizer = True else: bertweettokenizer = False if bertweettokenizer == True: tokenizer = data.custom_tokenizers.BERTweetTokenizer( args.transformer_model_name) else: tokenizer = AutoTokenizer.from_pretrained(args.transformer_model_name) trainX = data.data_utils.compute_transformer_input_arrays( nli_train_df, 'words', tokenizer, args.max_text_len, 'text2', bertweettokenizer) valX = data.data_utils.compute_transformer_input_arrays( nli_val_df, 'words', tokenizer, args.max_text_len, 'text2', bertweettokenizer) outputs = data.data_utils.compute_output_arrays(nli_train_df, 'labels') val_outputs = data.data_utils.compute_output_arrays(nli_val_df, 'labels') outputs = outputs[:, np.newaxis] val_outputs = val_outputs[:, np.newaxis] print("Modelling") model = models.tf_models.transformer_base_model_cls_token( args.transformer_model_name, args.max_text_len, dropout=args.dropout) print(model.summary()) optimizer = tf.keras.optimizers.Adam(learning_rate=args.lr) #SGD model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics='accuracy') #binary_crossentropy early = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=5, \ verbose=1, mode='auto', restore_best_weights=True) lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_accuracy', factor=0.7, \ patience=3, verbose=1, mode='auto', min_lr=0.000001) f1callback = models.tf_utils.F1Callback(model, valX, val_outputs, filename=os.path.join( model_save_dir, 'model.h5'), patience=5) snapshot = models.tf_utils.Snapshot(model, valX, val_outputs) config = { 'text_max_len': args.max_text_len, 'epochs': args.epochs, "learning_rate": args.lr, "batch_size": args.train_batch_size, "dropout": args.dropout, "model_description": args.transformer_model_name + ' with NLI' } with open(os.path.join(model_save_dir, 'config.pkl'), 'wb') as handle: pickle.dump(config, handle, protocol=pickle.HIGHEST_PROTOCOL) K.clear_session() if os.path.exists(os.path.join(model_save_dir, 'model.h5')) == False: if _has_wandb and args.wandb_logging: wandb.init(project='wnut-task2', config=config) model.fit(trainX, outputs, validation_data=(valX, val_outputs), epochs=args.epochs,\ batch_size=args.train_batch_size, callbacks=[early, lr, f1callback, WandbCallback(), snapshot], verbose=1) else: model.fit(trainX, outputs, validation_data=(valX, val_outputs), epochs=args.epochs,\ batch_size=args.train_batch_size, callbacks=[early,lr, f1callback, snapshot], verbose=1) model.load_weights(os.path.join(model_save_dir, 'model.h5')) #model_json = model.to_json() #with open(os.path.join(model_save_dir,"model.json"), "w") as json_file: # json_file.write(model_json) val_pred = np.round(model.predict(valX))[:, 0] print("NLI Evaluation") f1 = f1_score(nli_val_df.labels, val_pred) precision = precision_score(nli_val_df.labels, val_pred) recall = recall_score(nli_val_df.labels, val_pred) print("NLI scores: \nF1 {}, Precision {} and Recall {}".format( f1, precision, recall)) print("Original Evaluation") nli_val_df['pred_proba'] = model.predict(valX)[:, 0] nli_val_df_ = nli_val_df.sort_values(['Id', 'pred_proba'], ascending=[True, True ]).reset_index(drop=True) nli_val_df_ = nli_val_df_.drop_duplicates(subset=['Id']).reset_index( drop=True) val_df = val_df.drop_duplicates(subset=['Id']).reset_index(drop=True) assert nli_val_df_.shape[0] == val_df.shape[0] val_df = pd.merge(val_df, nli_val_df_[['Id', 'orig_label']], how='inner') #train_df.labels, _ = data.data_utils.convert_categorical_label_to_int(train_df.labels, \ # save_path=os.path.join(model_save_dir,'label2idx_orig.pkl')) print(val_df.iloc[0]) val_df.labels, _ = data.data_utils.convert_categorical_label_to_int(val_df.labels, \ save_path=os.path.join(model_save_dir,'label2idx_orig.pkl')) val_df.orig_label, _ = data.data_utils.convert_categorical_label_to_int(val_df.orig_label, \ save_path=os.path.join(model_save_dir,'label2idx_orig.pkl')) print(val_df.iloc[0]) f1 = f1_score(val_df.labels, val_df.orig_label) precision = precision_score(val_df.labels, val_df.orig_label) recall = recall_score(val_df.labels, val_df.orig_label) ''' snapshot_val_pred = np.round(np.concatenate(snapshot.best_scoring_snapshots, axis=-1).mean(-1)) print ("Snapshot Evaluation") f1_snapshot = f1_score(val_df.labels, snapshot_val_pred) precision_snapshot = precision_score(val_df.labels, snapshot_val_pred) recall_snapshot = recall_score(val_df.labels, snapshot_val_pred) ''' results_ = pd.DataFrame() results_['description'] = [args.transformer_model_name + ' with NLI'] results_['f1'] = [f1] results_['precision'] = [precision] results_['recall'] = [recall] print(results_.iloc[0]) #print ("Snapshot scores: \nF1 {}, Precision {} and Recall {}".format(f1_snapshot, precision_snapshot, recall_snapshot)) if os.path.exists('../results/result.csv'): results = pd.read_csv('../results/result.csv') results = pd.concat([results, results_], axis=0) results.to_csv('../results/result.csv', index=False) else: results_.to_csv('../results/result.csv', index=False)
def scan_round(self): '''The main operational function that manages the experiment on the level of execution of each round.''' # determine the parameters for the particular execution self.round_params = round_params(self) # print round params if self.print_params is True: print(self.round_params) # set start time round_start = strftime('%H%M%S') start = time() # fit the model try: _hr_out, self.keras_model = ingest_model(self) except TypeError as err: if err.args[0] == "unsupported operand type(s) for +: 'int' and 'numpy.str_'": raise TalosTypeError("Activation should be as object and not string in params") else: raise TalosReturnError("Make sure that input model returns 'out, model' where out is history object from model.fit()") # set end time and log round_end = strftime('%H%M%S') round_seconds = time() - start self.round_times.append([round_start, round_end, round_seconds]) # create log and other stats try: self.epoch_entropy.append(epoch_entropy(_hr_out)) except (TypeError, AttributeError): raise TalosReturnError("Make sure that input model returns in the order 'out, model'") if self.round_counter == 0: _for_header = create_header(self, _hr_out) self.result.append(_for_header) save_result(self) _hr_out = run_round_results(self, _hr_out) self.result.append(_hr_out) save_result(self) # apply reduction if self.reduction_method is not None: if (self.round_counter + 1) % self.reduction_interval == 0: len_before_reduce = len(self.param_log) self = reduce_run(self) total_reduced = len_before_reduce - len(self.param_log) # update the progress bar self.pbar.update(total_reduced) # save model and weights self.saved_models.append(self.keras_model.to_json()) self.saved_weights.append(self.keras_model.get_weights()) # clear tensorflow sessions (maybe) if self.clear_tf_session is True: K.clear_session() self.round_counter += 1 return self