def evaluate_model(model_weights_path, test_list): # load model weights if 'baseline' in model_weights_path: model = LFI_conv3D.build_model() model.load_weights(model_weights_path) elif 'CLSTM' in model_weights_path: model = LFV_conv3D_STCLSTM.build_model() model.load_weights(model_weights_path) else: raise Exception(f'model weights path error.') print(model.summary()) # path settings save_dir = Path(model_weights_path).parent / 'evaluated' save_dir.mkdir(parents=True, exist_ok=True) now = datetime.datetime.now().strftime("%Y-%m-%d_%H%M") csv_path = save_dir / f'_metrics_{now}.csv' with open(csv_path, 'a', newline='') as f: writer = csv.writer(f) metrics_dict = calc_metrics(pred=None, true=None) writer.writerow(['scene_name', *[metric for metric in metrics_dict.keys()]]) # evaluate test_gen = test_generator(test_list) preds = [] gts = [] scene = {'name':'', 'frame_n':0} for inputs, gt, scenes in test_gen.flow_from_directory(): pred = model.predict(inputs) for i, scene_name in enumerate(scenes): preds.append(pred[i]) gts.append(gt[i]) if len(preds) == x_patch_n*y_patch_n: fullmap_pred = create_fullmap(preds) fullmap_gt = create_fullmap(gts) # npz_save if not scene['name'] == scene_name: scene['frame_n'] = 0 scene['name'] = scene_name save_name = save_dir / f"{scene['name']}_{scene['frame_n']:03}.npz" np.savez_compressed(save_name, pred=fullmap_pred, gt=fullmap_gt) print('saved:', save_name) scene['frame_n'] += 1 preds = [] gts = [] # metrics output metrics_dict = calc_metrics(fullmap_pred, fullmap_gt) with open(csv_path, 'a', newline='') as f: writer = csv.writer(f) writer.writerow([scene_name, *metrics_dict.values()])
def main(): # set GPU ID os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu cudnn.benchmark = True # check save path save_path = args.save_path if not os.path.exists(save_path): os.makedirs(save_path) # make dataloader train_loader, test_loader, \ test_onehot, test_label = dataset.get_loader(args.data, args.data_path, args.batch_size) # set num_class if args.data == 'cifar100': num_class = 100 else: num_class = 10 # set num_classes model_dict = { "num_classes": num_class, } # set network if args.model == 'res': model = resnet.resnet110(**model_dict).cuda() elif args.model == 'dense': model = densenet_BC.DenseNet3(depth=100, num_classes=num_class, growth_rate=12, reduction=0.5, bottleneck=True, dropRate=0.0).cuda() elif args.model == 'vgg': model = vgg.vgg16(**model_dict).cuda() # set criterion cls_criterion = nn.CrossEntropyLoss().cuda() # make logger result_logger = utils.Logger(os.path.join(save_path, 'result.log')) # load pretrained model model_state_dict = torch.load(os.path.join(args.save_path, '{0}.pth'.format(args.file_name))) model.load_state_dict(model_state_dict) # calc measure acc, aurc, eaurc, aupr, fpr, ece, nll, brier = metrics.calc_metrics(test_loader, test_label, test_onehot, model, cls_criterion) # result write result_logger.write([acc,aurc*1000,eaurc*1000,aupr*100,fpr*100,ece*100,nll*10,brier*100])
def main(): # set GPU ID os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu cudnn.benchmark = True # check save path save_path = args.save_path if not os.path.exists(save_path): os.makedirs(save_path) # make dataloader train_loader, test_loader, \ test_onehot, test_label = dataset.get_loader(args.data, args.data_path, args.batch_size) # set num_class if args.data == 'cifar100': num_class = 100 else: num_class = 10 # set num_classes model_dict = { "num_classes": num_class, } # set model if args.model == 'res': model = resnet.resnet110(**model_dict).cuda() elif args.model == 'dense': model = densenet_BC.DenseNet3(depth=100, num_classes=num_class, growth_rate=12, reduction=0.5, bottleneck=True, dropRate=0.0).cuda() elif args.model == 'vgg': model = vgg.vgg16(**model_dict).cuda() # set criterion cls_criterion = nn.CrossEntropyLoss().cuda() ranking_criterion = nn.MarginRankingLoss(margin=0.0).cuda() # set optimizer (default:sgd) optimizer = optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=False) # set scheduler scheduler = MultiStepLR(optimizer, milestones=[150, 250], gamma=0.1) # make logger train_logger = utils_orig.Logger(os.path.join(save_path, 'train.log')) result_logger = utils_orig.Logger(os.path.join(save_path, 'result.log')) # make History Class correctness_history = crl_utils.History(len(train_loader.dataset)) # start Train for epoch in range(1, args.epochs + 1): scheduler.step() train.train(train_loader, model, cls_criterion, ranking_criterion, optimizer, epoch, correctness_history, train_logger, args) # save model if epoch == args.epochs: torch.save(model.state_dict(), os.path.join(save_path, 'model.pth')) # finish train # calc measure acc, aurc, eaurc, aupr, fpr, ece, nll, brier = metrics.calc_metrics( test_loader, test_label, test_onehot, model, cls_criterion) # result write result_logger.write([ acc, aurc * 1000, eaurc * 1000, aupr * 100, fpr * 100, ece * 100, nll * 10, brier * 100 ])
def main(): parser = argparse.ArgumentParser() # Experiment parser.add_argument("--experiment_name", default="default") parser.add_argument("--idd_name", default="skeletal-age") parser.add_argument( "--mode", type=str, default='devries', choices=['baseline', 'devries', 'devries_odin', 'energy', 'oe']) parser.add_argument("--outlier_name", type=str, nargs='+', default=['mura', 'mimic-crx']) parser.add_argument("--ood_name", type=str, nargs='+', default=['retina']) parser.add_argument('--use_xent', '-x', action='store_true', help='Use cross entropy scoring instead of the MSP.') parser.add_argument("--network", type=str, default="resnet") # Hyper params parser.add_argument("--num_epochs", type=int, default=300) parser.add_argument("--hint_rate", type=float, default=0.5) parser.add_argument("--beta", type=float, default=0.3) parser.add_argument("--lmbda", type=float, default=0.1) parser.add_argument("--batch_size", type=int, default=64) parser.add_argument( '--m_in', type=float, default=-25., help='margin for in-distribution; above this value will be penalized') parser.add_argument( '--m_out', type=float, default=-7., help='margin for out-distribution; below this value will be penalized') # Training params parser.add_argument("--use_scheduler", type=bool, default=False) parser.add_argument("--lr", type=int, default=1e-3) parser.add_argument('--losses', nargs='+', default=["boneage_mad", "accuracy"]) parser.add_argument('--early_stop_metric', type=str, default="fpr_at_95_tpr") parser.add_argument('--early_stop', type=int, default=10) parser.add_argument('--eval_start', type=int, default=1) # Misc parser.add_argument("--checkpoint", default="") parser.add_argument("--load_memory", type=bool, default=False, help="Load images into CPU") args = parser.parse_args() args = utils.compute_args(args) # Create dataloader according to experiments loader_args = { 'name': args.idd_name, 'mode': 'idd', 'root_dir': args.root[args.idd_name], 'csv_file': 'train.csv', 'load_memory': args.load_memory } train_loader = DataLoader(LarsonDataset(**loader_args), batch_size=args.batch_size, shuffle=True, num_workers=4) test_true_loader = DataLoader( LarsonDataset(**dict(loader_args, **{'csv_file': 'test.csv'})), batch_size=16, shuffle=False, num_workers=4) outlier_set = ConcatDataset([ LarsonDataset( **{ 'name': outlier_name, 'mode': 'idd', 'root_dir': args.root[outlier_name], 'csv_file': 'train.csv', 'load_memory': False }) for outlier_name in args.outlier_name ]) assert len(outlier_set) >= len(train_loader.dataset) test_false_loaders = {} for ood_name in args.ood_name: test_false_loaders[ood_name] = DataLoader(LarsonDataset( **{ 'name': ood_name, 'mode': 'ood', 'root_dir': args.root[ood_name], 'csv_file': 'test.csv', 'load_memory': False }), batch_size=16, shuffle=False, num_workers=4) model_config = DeVriesLarsonModelConfig(args=args, hint_rate=args.hint_rate, lmbda=args.lmbda, beta=args.beta) net = model_config.net loss_plots = Plot(idd_name=args.idd_name, early_stop_metric=args.early_stop_metric) if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") net = nn.DataParallel(net) print( "Network", args.network, 'mode', args.mode, "\nLambda", args.lmbda, "beta", args.beta, "hint_rate", args.hint_rate, "\nTotal number of parameters : " + str(sum([p.numel() for p in net.parameters()]) / 1e6) + "M") init_epoch = 0 checkpoints_folder = f"checkpoints/{args.experiment_name}" early_stop = 0 best_early_stop_value = args.early_stop_start # -inf or +inf for epoch in range(init_epoch, init_epoch + args.num_epochs): train_start = time.time() # Train phases net.train() # Shuffling outlier set outlier_loader = DataLoader(outlier_set, batch_size=16, shuffle=True, num_workers=4) for train_iter, (in_set, out_set) in enumerate( zip(train_loader, outlier_loader)): data = torch.cat((in_set[0], out_set[0]), dim=0) target = in_set[1] data, target = data.cuda(), target.cuda() # forward pred, _ = net(data) # backward model_config.optimizer.zero_grad() task_loss = F.cross_entropy(pred[:len(in_set[0])], target) # cross-entropy from softmax distribution to uniform distribution if args.mode == 'energy': Ec_out = -torch.logsumexp(pred[len(in_set[0]):], dim=1) Ec_in = -torch.logsumexp(pred[:len(in_set[0])], dim=1) oe_loss = 0.1 * ( torch.pow(F.relu(Ec_in - args.m_in), 2).mean() + torch.pow(F.relu(args.m_out - Ec_out), 2).mean()) elif args.mode == 'oe': oe_loss = args.beta * -(pred[len(in_set[0]):].mean( 1) - torch.logsumexp(pred[len(in_set[0]):], dim=1)).mean() else: raise NotImplementedError total_loss = task_loss + oe_loss total_loss.backward() model_config.optimizer.step() print( "\r[Epoch {}][Step {}/{}] Loss: {:.2f} [Task: {:.2f}, Energy: {:.2f}], Lr: {:.2e}, ES: {}, {:.2f} m remaining" .format( epoch + 1, train_iter, int(len(train_loader.dataset) / args.batch_size), total_loss.cpu().data.numpy(), task_loss.cpu().data.numpy(), oe_loss.cpu().data.numpy(), *[ group['lr'] for group in model_config.optimizer.param_groups ], early_stop, ((time.time() - train_start) / (train_iter + 1)) * ((len(train_loader.dataset) / args.batch_size) - train_iter) / 60, ), end=' ') # Eval phase if epoch + 1 >= args.eval_start: net.eval() def evaluate(data_loader, mode="confidence", idd=False): confidences = [] idd_metrics = defaultdict(list) for test_iter, sample in enumerate(data_loader, 0): images, labels, _ = sample # Reassigns inputs_batch and label_batch to cuda pred, confidence = net(images.cuda()) labels = labels.data.cpu().numpy() # manage in domain metric if idd: loss_dict = {} task_predictions = torch.argmax( pred, dim=-1).data.cpu().numpy() if "accuracy" in args.losses: loss_dict['accuracy'] = list( task_predictions == labels) elif "boneage_mad" in args.losses: loss_dict['boneage_mad'] = list( abs(task_predictions - labels)) else: raise NotImplementedError for k, v in loss_dict.items(): idd_metrics[k].extend(v) # Get confidence in prediction confidences.extend( get_confidence(net, images, pred, confidence, args)) confidences = np.array(confidences) if idd: # Plot accuracy if 'accuracy' in idd_metrics: plot_classification( idd_metrics['accuracy'], confidences, checkpoints_folder, name=str(args.idd_name) + ' - ' + str( round(float(np.mean(idd_metrics['accuracy'])), 4)) + ' - ' + 'Epoch ' + str(epoch + 1) + '.jpg') # Average metric over valset for k, v in idd_metrics.items(): idd_metrics[k] = round(float(np.mean(v)), 4) return confidences, idd_metrics # In domain evaluation ind_confs, ind_metrics = evaluate(test_true_loader, idd=True) ind_labels = np.ones(ind_confs.shape[0]) ind_metrics["IDD name"] = args.idd_name print(str(ind_metrics)) # Out of domain evaluation early_stop_metric_value = 0 ood_metric_dicts = [] for ood_name, test_false_loader in test_false_loaders.items(): ood_confs, _ = evaluate(test_false_loader, idd=False) ood_labels = np.zeros(ood_confs.shape[0]) labels = np.concatenate([ind_labels, ood_labels]) scores = np.concatenate([ind_confs, ood_confs]) ood_metrics = calc_metrics(scores, labels) ood_metrics['OOD Name'] = ood_name print(str(ood_metrics)) ood_metric_dicts.append(ood_metrics) # Plot metrics plot_metrics(scores, labels, ind_confs, ood_confs, checkpoints_folder, name=str(ood_name) + ' - ' + 'Epoch ' + str(epoch + 1)) # fetch early stop value (might be a iid metric) early_stop_metric_value += { **ind_metrics, **ood_metrics }[args.early_stop_metric] early_stop_metric_value = early_stop_metric_value / len( test_false_loaders) early_stop += 1 loss_plots.update(epoch + 1, ind_metrics, ood_metric_dicts) # Save model + early stop # Early_stop_operator is min or max if args.early_stop_operator( early_stop_metric_value, best_early_stop_value) != best_early_stop_value: early_stop = 0 best_early_stop_value = early_stop_metric_value utils.save_checkpoint(checkpoints_folder, { "init_epoch": epoch + 1, "net": net.state_dict(), "optimizer": model_config.optimizer.state_dict(), "scheduler": model_config.scheduler.state_dict() if args.use_scheduler else None, "ood_metrics": ood_metric_dicts, "ind_metrics": ind_metrics, "best_early_stop_value": best_early_stop_value, "args": args, }, keep_n_best=1) print('Early stop metric ' + str(args.early_stop_metric) + ' beaten. Now ' + str(best_early_stop_value)) if args.use_scheduler: model_config.scheduler.step(ind_metrics['accuracy']) if early_stop == args.early_stop: loss_plots.draw(checkpoints_folder) print("early_stop reached") break loss_plots.draw(checkpoints_folder) print('Done') return
def main(*kargs, **kwargs): get_kwargs(kwargs) train_fname = kwargs['train'] test_fname = kwargs['test'] result_fname = kwargs['output'] embeds_fname = kwargs['embeds'] logger_fname = kwargs['logger'] swear_words_fname = kwargs['swear_words'] wrong_words_fname = kwargs['wrong_words'] warm_start = kwargs['warm_start'] format_embeds = kwargs['format_embeds'] cnn_model_file = 'data/cnn.h5' lstm_model_file = 'data/lstm.h5' concat_model_file = 'data/concat.h5' cnn_model_file = 'data/cnn.h5' lr_model_file = 'data/{}_logreg.bin' meta_catboost_model_file = 'data/{}_meta_catboost.bin' # ====Create logger==== logger = Logger(logging.getLogger(), logger_fname) # ====Load data==== logger.info('Loading data...') train_df = load_data(train_fname) test_df = load_data(test_fname) target_labels = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'] num_classes = len(target_labels) # ====Load additional data==== logger.info('Loading additional data...') swear_words = load_data(swear_words_fname, func=lambda x: set(x.T[0]), header=None) wrong_words_dict = load_data(wrong_words_fname, func=lambda x: {val[0] : val[1] for val in x}) tokinizer = RegexpTokenizer(r'\w+') regexps = [re.compile("([a-zA-Z]+)([0-9]+)"), re.compile("([0-9]+)([a-zA-Z]+)")] # ====Load word vectors==== logger.info('Loading embeddings...') embed_dim = 300 embeds = Embeds(embeds_fname, 'fasttext', format=format_embeds) # ====Clean texts==== logger.info('Cleaning text...') if warm_start: logger.info('Use warm start...') else: train_df['comment_text_clear'] = clean_text(train_df['comment_text'], tokinizer, wrong_words_dict, swear_words, regexps) test_df['comment_text_clear'] = clean_text(test_df['comment_text'], tokinizer, wrong_words_dict, swear_words, regexps) train_df.to_csv(train_clear, index=False) test_df.to_csv(test_clear, index=False) # ====Calculate maximum seq length==== logger.info('Calc text length...') train_df.fillna('unknown', inplace=True) test_df.fillna('unknown', inplace=True) train_df['text_len'] = train_df['comment_text_clear'].apply(lambda words: len(words.split())) test_df['text_len'] = test_df['comment_text_clear'].apply(lambda words: len(words.split())) max_seq_len = np.round(train_df['text_len'].mean() + 3*train_df['text_len'].std()).astype(int) logger.debug('Max seq length = {}'.format(max_seq_len)) # ====Prepare data to NN==== logger.info('Converting texts to sequences...') max_words = 100000 train_df['comment_seq'], test_df['comment_seq'], word_index = convert_text2seq(train_df['comment_text_clear'].tolist(), test_df['comment_text_clear'].tolist(), max_words, max_seq_len, lower=True, char_level=False) logger.debug('Dictionary size = {}'.format(len(word_index))) logger.info('Preparing embedding matrix...') embedding_matrix, words_not_found = get_embedding_matrix(embed_dim, embeds, max_words, word_index) logger.debug('Embedding matrix shape = {}'.format(np.shape(embedding_matrix))) logger.debug('Number of null word embeddings = {}'.format(np.sum(np.sum(embedding_matrix, axis=1) == 0))) # ====Train/test split data==== x = np.array(train_df['comment_seq'].tolist()) y = np.array(train_df[target_labels].values) x_train_nn, x_test_nn, y_train_nn, y_test_nn, train_idxs, test_idxs = split_data(x, y, test_size=0.2, shuffle=True, random_state=42) test_df_seq = np.array(test_df['comment_seq'].tolist()) # ====Train models==== # CNN logger.info("training CNN ...") cnn = get_cnn(embedding_matrix, num_classes, embed_dim, max_seq_len, num_filters=64, l2_weight_decay=0.0001, dropout_val=0.5, dense_dim=32, add_sigmoid=True) cnn_hist = train(x_train_nn, y_train_nn, cnn, batch_size=256, num_epochs=100, learning_rate=0.005, early_stopping_delta=0.0001, early_stopping_epochs=3, use_lr_stratagy=True, lr_drop_koef=0.66, epochs_to_drop=2, logger=logger) y_cnn = cnn.predict(x_test_nn) save_predictions(test_df, cnn.predict(test_df_seq), target_labels, 'cnn') metrics_cnn = get_metrics(y_test_nn, y_cnn, target_labels, hist=cnn_hist, plot=False) logger.debug('CNN metrics:\n{}'.format(metrics_cnn)) cnn.save(cnn_model_file) # LSTM logger.info("training LSTM ...") lstm = get_lstm(embedding_matrix, num_classes, embed_dim, max_seq_len, l2_weight_decay=0.0001, lstm_dim=50, dropout_val=0.3, dense_dim=32, add_sigmoid=True) lstm_hist = train(x_train_nn, y_train_nn, lstm, batch_size=256, num_epochs=100, learning_rate=0.005, early_stopping_delta=0.0001, early_stopping_epochs=3, use_lr_stratagy=True, lr_drop_koef=0.66, epochs_to_drop=2, logger=logger) y_lstm = lstm.predict(x_test_nn) save_predictions(test_df, lstm.predict(test_df_seq), target_labels, 'lstm') metrics_lstm = get_metrics(y_test_nn, y_lstm, target_labels, hist=lstm_hist, plot=False) logger.debug('LSTM metrics:\n{}'.format(metrics_lstm)) lstm.save(lstm_model_file) # CONCAT logger.info("training Concat NN (LSTM + CNN) ...") concat = get_concat_model(embedding_matrix, num_classes, embed_dim, max_seq_len, num_filters=64, l2_weight_decay=0.0001, lstm_dim=50, dropout_val=0.5, dense_dim=32, add_sigmoid=True) concat_hist = train([x_train_nn, x_train_nn], y_train_nn, concat, batch_size=256, num_epochs=100, learning_rate=0.005, early_stopping_delta=0.0001, early_stopping_epochs=4, use_lr_stratagy=True, lr_drop_koef=0.66, epochs_to_drop=3, logger=logger) y_concat = concat.predict([x_test_nn, x_test_nn]) save_predictions(test_df, concat.predict([test_df_seq, test_df_seq]), target_labels, 'concat') metrics_concat = get_metrics(y_test_nn, y_concat, target_labels, hist=concat_hist, plot=False) logger.debug('Concat_NN metrics:\n{}'.format(metrics_concat)) concat.save(concat_model_file) # TFIDF + LogReg logger.info('training LogReg over tfidf...') train_tfidf, val_tfidf, test_tfidf, word_tfidf, char_tfidf = get_tfidf(train_df['comment_text_clear'].values[train_idxs], train_df['comment_text_clear'].values[test_idxs], test_df['comment_text_clear'].values) models_lr = [] metrics_lr = {} y_tfidf = [] for i, label in enumerate(target_labels): model = NbSvmClassifier(C=4.0, solver='sag', max_iter=1000) model.fit(train_tfidf, y_train_nn[:, i]) y_tfidf.append(model.predict_proba(val_tfidf)[:,1]) test_df['tfidf_{}'.format(label)] = model.predict_proba(test_tfidf)[:,1] metrics_lr[label] = calc_metrics(y_test_nn[:, i], y_tfidf[-1]) models_lr.append(model) joblib.dump(model, lr_model_file.format(label)) metrics_lr['Avg logloss'] = np.mean([metric[0] for label, metric in metrics_lr.items()]) logger.debug('LogReg(TFIDF) metrics:\n{}'.format(metrics_lr)) # Bow for catboost top_pos_words = [] top_neg_words = [] for i in range(num_classes): top_pos_words.append([]) top_neg_words.append([]) top_pos_words[-1], top_neg_words[-1] = get_most_informative_features([word_tfidf, char_tfidf], models_lr[i], n=100) top_pos_words = list(set(np.concatenate([[val for score, val in top] for top in top_pos_words]))) top_neg_words = list(set(np.concatenate([[val for score, val in top] for top in top_neg_words]))) top = list(set(np.concatenate([top_pos_words, top_neg_words]))) train_bow = get_bow(train_df['comment_text_clear'].values[train_idxs], top) val_bow = get_bow(train_df['comment_text_clear'].values[test_idxs], top) test_bow = get_bow(test_df['comment_text_clear'].values, top) logger.debug('Count bow words = {}'.format(len(top))) # Meta catboost logger.info('training catboost as metamodel...') train_df['text_unique_len'] = train_df['comment_text_clear'].apply(calc_text_uniq_words) test_df['text_unique_len'] = test_df['comment_text_clear'].apply(calc_text_uniq_words) train_df['text_unique_koef'] = train_df['text_unique_len'] / train_df['text_len'] test_df['text_unique_koef'] = test_df['text_unique_len'] / test_df['text_len'] text_len_features = train_df[['text_len', 'text_unique_len', 'text_unique_koef']].values[test_idxs] x_train_catboost = [] y_train_catboost = y_test_nn for len_f, y_hat_cnn, y_hat_lstm, y_hat_concat, y_hat_tfidf, bow in zip(text_len_features, y_cnn, y_lstm, y_concat, np.array(y_tfidf).T, val_bow): x_train_catboost.append(np.concatenate([len_f, y_hat_cnn, y_hat_lstm, y_hat_concat, y_hat_tfidf, bow])) models_cb = [] metrics_cb = {} x_train_cb, x_val_cb, y_train_cb, y_val_cb = train_test_split(x_train_catboost, y_train_catboost, test_size=0.20, random_state=42) for i, label in enumerate(target_labels): model = CatBoostClassifier(loss_function='Logloss', iterations=1000, depth=6, rsm=1, learning_rate=0.01) model.fit(x_train_cb, y_train_cb[:, i], plot=True, eval_set=(x_val_cb, y_val_cb[:, i]), use_best_model=True) y_hat_cb = model.predict_proba(x_val_cb) metrics_cb[label] = calc_metrics(y_val_cb[:, i], y_hat_cb[:, 1]) models_cb.append(model) joblib.dump(model, meta_catboost_model_file.format(label)) metrics_cb['Avg logloss'] = np.mean([metric[0] for label,metric in metrics_cb.items()]) logger.debug('CatBoost metrics:\n{}'.format(metrics_cb)) # ====Predict==== logger.info('Applying models...') text_len_features = test_df[['text_len', 'text_unique_len', 'text_unique_koef']].values y_cnn_test = test_df[['cnn_{}'.format(label) for label in target_labels]].values y_lstm_test = test_df[['lstm_{}'.format(label) for label in target_labels]].values y_concat_test = test_df[['concat_{}'.format(label) for label in target_labels]].values y_tfidf_test = test_df[['tfidf_{}'.format(label) for label in target_labels]].values x_test_cb = [] for len_f, y_hat_cnn, y_hat_lstm, y_hat_concat, y_hat_tfidf, bow in tqdm(zip(text_len_features, y_cnn_test, y_lstm_test, y_concat_test, y_tfidf_test, test_bow)): x_test_cb.append(np.concatenate([len_f, y_hat_cnn, y_hat_lstm, y_hat_concat, y_hat_tfidf, bow])) for label, model in zip(target_labels, models_cb): pred = model.predict_proba(x_test_cb) test_df[label] = np.array(list(pred))[:, 1] # ====Save results==== logger.info('Saving results...') test_df[['id', 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].to_csv(result_fname, index=False, header=True)
X_test[interv] = np.array(X_test[interv]) y_train[interv] = np.array(y_train[interv]) y_test[interv] = np.array(y_test[interv]) size = len(y_test[interv]) print "\t", "Model compute for size = " + str(size) print "\t", "computing model..." start = datetime.datetime.now() text_clf = model.fit(X_train[interv], y_train[interv]) stop = datetime.datetime.now() final_time = (stop-start) print "\t", "Running = ", final_time print "\t", "predicting..." predicted = text_clf.predict(y_test[interv]) parameters = { "filename_model_results":"results/RandomForest" + "_" + interv + ".txt", "filename_confusion_matrix":None, "model_interval" : [interval[interv][0],interval[interv][1]], "model_type":"RandomForest Model", "model":text_clf, "expected":y_test[interv], "predicted":predicted, "target_names":target_names} metrics.calc_metrics(parameters, final_time) else: print "\t","No data for this interval"
def calculate_alg(singletons, net, uw, ud, g_type, alg, metric): communities = "/home/amaury/communities_hashmap/" + str( g_type) + "/" + str(alg) + "/" + str(singletons) + "/" + str(net) + "/" output = str(output_dir) + str(metric) + "/" + str(g_type) + "/" + str( alg) + "/" + str(singletons) + "/" + str(net) + "/" graphs = "/home/amaury/graphs_hashmap/" + str(net) + "/" + str( g_type) + "/" if not os.path.exists(communities): print("Diretório com as comunidades não encontrado: " + str(communities) + "\n") else: print print( "######################################################################" ) print("Os arquivos serão armazenados em: " + str(output)) print( "######################################################################" ) for threshold in os.listdir(communities): if not os.path.isdir(str(communities) + str(threshold) + "/"): print("Threshold para a rede " + str(net) + " não encontrado: " + str(threshold)) else: print("Salvando dados em: " + str(output) + str(threshold) + ".json") if not os.path.exists(output): os.makedirs(output) if os.path.exists(str(output) + str(threshold) + ".json"): print("Arquivo de destino já existe: " + str(output) + str(threshold) + ".json") else: print( "######################################################################" ) result = [] i = 0 #Ponteiro para o ego for file in os.listdir( str(communities) + str(threshold) + "/"): if os.path.isfile( str(communities) + str(threshold) + "/" + file): ego_id = file.split(".txt") ego_id = long(ego_id[0]) i += 1 if not os.path.isfile( str(graphs) + str(ego_id) + ".edge_list"): print( "ERROR - EGO: " + str(i) + " - Arquivo com lista de arestas não encontrado:" + str(graphs) + str(ego_id) + ".edge_list") else: with open( str(communities) + str(threshold) + "/" + file, 'r') as community_file: if ud is False: G = snap.LoadEdgeList( snap.PNGraph, str(graphs) + str(ego_id) + ".edge_list", 0, 1 ) # load from a text file - pode exigir um separador.: snap.LoadEdgeList(snap.PNGraph, file, 0, 1, '\t') else: G = snap.LoadEdgeList( snap.PUNGraph, str(graphs) + str(ego_id) + ".edge_list", 0, 1 ) # load from a text file - pode exigir um separador.: snap.LoadEdgeList(snap.PNGraph, file, 0, 1, '\t') print( str(g_type) + " - " + str(alg) + " - " + str(singletons) + " - Rede: " + str(net) + " - THRESHOLD: " + str(threshold) + " - ego(" + str(i) + "): " + str(file)) communities_dict = prepare_communities( community_file ) #Função para devolver um dicionário com as comunidades avg = metrics.calc_metrics( communities_dict, G, ud, metric) # Calcular as métricas result.append( avg['media']) # Salvar Métrica print metric, result[i - 1] print print( "######################################################################" ) with open(str(output) + str(threshold) + ".json", "w") as f: f.write( json.dumps(result, separators=(',', ':')) + "\n") print( "######################################################################" )
def on_epoch_end(self, epoch, logs=None): output_path_name = os.path.join(self.output_path, f'epoch_{epoch+1}') os.makedirs(output_path_name, exist_ok=True) metrics_log_name = os.path.join(output_path_name, "metrics.txt") if os.path.exists(metrics_log_name): os.remove(metrics_log_name) metrics_log = open(metrics_log_name, "a+") self.lrs.append(K.eval(self.model.optimizer.lr)) predicted = [] all_metrics = ['iou'] metrics_values = {i: 0 for i in all_metrics} tmp_metrics_per_cls_res = {i: [] for i in range(self.n_classes)} t1 = time() ii = 0 for image, mask in zip(self.images, self.masks): s = f'Testing on {ii+1} of {self.images.shape[0]}' print(s) metrics_log.write('\n' + s + '\n') pred = self.predict_image(image) predicted.append(pred) metrics = calc_metrics( mask[self.offset:-self.offset, self.offset:-self.offset, ...], pred, all_metrics, self.n_classes) for metric in metrics: s = 'Metrics for each class:' print(s) metrics_log.write(s + '\n') i = 0 for value in metric[1]: s = f"{metric[0]} for {classes_mask[i]} : {value}" print(s) metrics_log.write(s + '\n') tmp_metrics_per_cls_res[i].append(value) i += 1 metrics_values[metric[0]] += sum(metric[1]) / len(metric[1]) ii += 1 for i in range(self.n_classes): self.metrics_per_cls_res[i].append(mean( tmp_metrics_per_cls_res[i])) s = 'Average metrics values:' print(s) metrics_log.write('\n' + s + '\n') for metrics_name in metrics_values.keys(): self.metrics_results[metrics_name][ epoch + 1] = metrics_values[metrics_name] / self.images.shape[0] s = f'{metrics_name} : {(metrics_values[metrics_name] / self.images.shape[0])}' print(s) metrics_log.write(s + '\n') t2 = time() print(f'Prediction completed in {t2-t1} seconds') print('Processing visualization:') visualize_segmentation_result(np.array([ i[self.offset:-self.offset, self.offset:-self.offset, ...] for i in self.images ]), [ np.argmax(i[self.offset:-self.offset, self.offset:-self.offset, ...], axis=2) for i in self.masks ], [np.argmax(i, axis=2) for i in predicted], names=self.names, n_classes=self.n_classes, output_path=self.output_path, epoch=epoch) visualize_pred_heatmaps(predicted, self.n_classes, self.output_path, epoch) plot_metrics_history(self.metrics_results, self.output_path) plot_per_class_history(self.metrics_per_cls_res, self.output_path) plot_lrs(self.lrs, self.output_path) t3 = time() print(f'Visualization completed in {t3-t2} seconds')
# #### evalute the model # In[12]: model, _ = get_model(params) model.load_state_dict(torch.load(f"./models/{model.params.name}.pth")) model.eval().to(device) # In[13]: model.eval() _, all_preds, all_labels = validate(model, valid_dl, custom_loss) # In[14]: calc_metrics(all_preds, all_labels) # #### plot results # In[15]: # for convience, we can pass an integer instead of the full string int2key = { 0: 'red_light', 1: 'hazard_stop', 2: 'speed_sign', 3: 'relative_angle', 4: 'center_distance', 5: 'veh_distance' }
type=int, help='How many times to repeat an experiment') metrics_parser = subparsers.add_parser( 'metric', help= 'Metric calculation based on the snapshots produced by a blocking aggression minimization simulation experiment' ) metrics_parser.add_argument( 'aggression_threshold', type=float, help= "The threshold in [0,1] that determines when a user becomes aggressive. An aggression score larger than the threshold means that the user is aggressive. Eg. = 0.4" ) metrics_parser.add_argument( 'metric_type', type=str, choices=['similarity', 'aggression'], help= "The type of metric to calculate, similarity metrics or aggression. 'aggression' is suggested for blocking aggression minimization experiment" ) args = parser.parse_args() print() print(args) if args.mode == 'simulation': experiment(args, False) elif args.mode == 'metric': calc_metrics(args, 'blocking')
def main(): parser = argparse.ArgumentParser() # Experiment parser.add_argument("--experiment_name", default="default") parser.add_argument("--idd_name", default="skeletal-age") parser.add_argument( "--mode", type=str, default='devries', choices=['baseline', 'devries', 'devries_odin', 'energy', 'oe']) parser.add_argument("--ood_name", type=str, nargs='+', default=['retina', 'mura', 'mimic-crx']) parser.add_argument("--network", type=str, default="resnet") # Hyper params parser.add_argument("--num_epochs", type=int, default=300) parser.add_argument("--beta", type=float, default=0.3) parser.add_argument("--lmbda", type=float, default=0.1) parser.add_argument("--batch_size", type=int, default=64) parser.add_argument("--hint", type=bool, default=False) parser.add_argument("--hint_rate", type=float, default=0.5) parser.add_argument("--use_budget", type=bool, default=False) # Training params parser.add_argument("--use_scheduler", type=bool, default=False) parser.add_argument("--lr", type=int, default=1e-3) parser.add_argument('--losses', nargs='+', default=["boneage_mad", "accuracy"]) parser.add_argument('--early_stop_metric', type=str, default="fpr_at_95_tpr") parser.add_argument('--early_stop', type=int, default=10) parser.add_argument('--eval_start', type=int, default=1) # Misc parser.add_argument("--checkpoint", default="") parser.add_argument("--load_memory", type=bool, default=False, help="Load images into CPU") args = parser.parse_args() args = utils.compute_args(args) # Create dataloader according to experiments loader_args = { 'name': args.idd_name, 'mode': 'idd', 'root_dir': args.root[args.idd_name], 'csv_file': 'train.csv', 'load_memory': args.load_memory } import torchvision.transforms as trn train_data = dset.MNIST( 'MNIST', train=True, transform=trn.ToTensor(), download=True, ) test_data = dset.MNIST( 'MNIST', train=False, transform=trn.ToTensor(), download=True, ) num_classes = 10 train_loader = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, num_workers=4, pin_memory=True) test_true_loader = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, num_workers=4, pin_memory=True) print(len(train_loader)) print(len(test_true_loader)) test_false_loaders = {} # /////////////// gaussian Noise /////////////// dummy_targets = torch.ones(5000) ood_data = torch.from_numpy( np.clip( np.random.normal(size=(5000, 1, 28, 28), loc=0.5, scale=0.5).astype(np.float32), 0, 1)) ood_data = torch.utils.data.TensorDataset(ood_data, dummy_targets) ood_loader = torch.utils.data.DataLoader(ood_data, batch_size=16, shuffle=True) test_false_loaders["gaussian"] = ood_loader # /////////////// Bernoulli Noise /////////////// dummy_targets = torch.ones(5000) ood_data = torch.from_numpy( np.random.binomial(n=1, p=0.5, size=(5000, 1, 28, 28)).astype(np.float32)) ood_data = torch.utils.data.TensorDataset(ood_data, dummy_targets) ood_loader = torch.utils.data.DataLoader(ood_data, batch_size=16, shuffle=True) test_false_loaders["Bernoulli"] = ood_loader # /////////////// CIFAR data /////////////// ood_data = dset.CIFAR10( 'cifar', train=False, download=True, transform=trn.Compose([ trn.Resize(28), trn.Lambda(lambda x: x.convert('L', (0.2989, 0.5870, 0.1140, 0))), trn.ToTensor() ])) ood_loader = torch.utils.data.DataLoader(ood_data, batch_size=16, shuffle=True, num_workers=4, pin_memory=True) test_false_loaders["CIFAR"] = ood_loader model_config = DeVriesLarsonModelConfig(args=args, hint_rate=args.hint_rate, lmbda=args.lmbda, beta=args.beta) def gelu(x): return torch.sigmoid(1.702 * x) * x # return 0.5 * x * (1 + torch.tanh(x * 0.7978845608 * (1 + 0.044715 * x * x))) class ConvNet(nn.Module): def __init__(self): super(ConvNet, self).__init__() self.conv1 = nn.Conv2d(1, 10, kernel_size=5) self.conv2 = nn.Conv2d(10, 20, kernel_size=5) self.conv2_drop = nn.Dropout2d() self.fc1 = nn.Linear(320, 50) self.fc2 = nn.Linear(50, 10) self.fc3 = nn.Linear(50, 1) def forward(self, x): import torch.nn.functional as F x = gelu(F.max_pool2d(self.conv1(x), 2)) x = gelu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2)) x = x.view(-1, 320) x = gelu(self.fc1(x)) # x = F.dropout(x) return self.fc2(x), self.fc3(x) net = ConvNet().cuda() import torch.optim as optim optimizer = optim.Adam(net.parameters(), lr=1e-3) loss_plots = Plot(idd_name=args.idd_name, early_stop_metric=args.early_stop_metric) if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") net = nn.DataParallel(net) print( "Network", args.network, 'mode', args.mode, "\nLambda", args.lmbda, "beta", args.beta, "hint_rate", args.hint_rate, "\nTotal number of parameters : " + str(sum([p.numel() for p in net.parameters()]) / 1e6) + "M") init_epoch = 0 checkpoints_folder = f"checkpoints/{args.experiment_name}" early_stop = 0 best_early_stop_value = args.early_stop_start # -inf or +inf for epoch in range(init_epoch, init_epoch + args.num_epochs): train_start = time.time() # Train phases net.train() for train_iter, (data, labels) in enumerate(train_loader, 0): optimizer.zero_grad() data, labels = data.cuda(), labels.cuda() output_batches = net(data.cuda()) total_loss, task_loss, confidence_loss = model_config.criterion( output_batches, labels.cuda()) total_loss.backward() optimizer.step() print( "\r[Epoch {}][Step {}/{}] Loss: {:.2f} [Task: {:.2f}, Confidence: {:.2f}, lambda: {:.2f}], Lr: {:.2e}, ES: {}, {:.2f} m remaining" .format( epoch + 1, train_iter, int(len(train_loader.dataset) / args.batch_size), total_loss.cpu().data.numpy(), task_loss.cpu().data.numpy(), confidence_loss.cpu().data.numpy(), model_config.criterion.lmbda, *[group['lr'] for group in optimizer.param_groups], early_stop, ((time.time() - train_start) / (train_iter + 1)) * ((len(train_loader.dataset) / args.batch_size) - train_iter) / 60, ), end=' ') # Eval phase if epoch + 1 >= args.eval_start: net.eval() def evaluate(data_loader, mode="confidence", idd=False): confidences = [] idd_metrics = defaultdict(list) for test_iter, (data, labels) in enumerate(data_loader, 0): data = data.view(-1, 1, 28, 28).cuda() # Reassigns inputs_batch and label_batch to cuda pred, confidence = net(data.cuda()) labels = labels.data.cpu().numpy() # manage in domain metric if idd: loss_dict = {} task_predictions = torch.argmax( pred, dim=-1).data.cpu().numpy() if "accuracy" in args.losses: loss_dict['accuracy'] = list( task_predictions == labels) elif "boneage_mad" in args.losses: loss_dict['boneage_mad'] = list( abs(task_predictions - labels)) else: raise NotImplementedError for k, v in loss_dict.items(): idd_metrics[k].extend(v) # Get confidence in prediction confidences.extend( get_confidence(net, data, pred, confidence, args)) confidences = np.array(confidences) if idd: # Plot accuracy if 'accuracy' in idd_metrics: plot_classification( idd_metrics['accuracy'], confidences, checkpoints_folder, name=str(args.idd_name) + ' - ' + str( round(float(np.mean(idd_metrics['accuracy'])), 4)) + ' - ' + 'Epoch ' + str(epoch + 1) + '.jpg') # Average metric over valset for k, v in idd_metrics.items(): idd_metrics[k] = round(float(np.mean(v)), 4) return confidences, idd_metrics # In domain evaluation ind_confs, ind_metrics = evaluate(test_true_loader, idd=True) ind_labels = np.ones(ind_confs.shape[0]) ind_metrics["IDD name"] = args.idd_name print(str(ind_metrics)) # Out of domain evaluation early_stop_metric_value = 0 ood_metric_dicts = [] for ood_name, test_false_loader in test_false_loaders.items(): ood_confs, _ = evaluate(test_false_loader, idd=False) ood_labels = np.zeros(ood_confs.shape[0]) labels = np.concatenate([ind_labels, ood_labels]) scores = np.concatenate([ind_confs, ood_confs]) ood_metrics = calc_metrics(scores, labels) ood_metrics['OOD Name'] = ood_name print(str(ood_metrics)) ood_metric_dicts.append(ood_metrics) # Plot metrics plot_metrics(scores, labels, ind_confs, ood_confs, checkpoints_folder, name=str(ood_name) + ' - ' + 'Epoch ' + str(epoch + 1)) # fetch early stop value (might be a iid metric) early_stop_metric_value += { **ind_metrics, **ood_metrics }[args.early_stop_metric] early_stop_metric_value = early_stop_metric_value / len( test_false_loaders) early_stop += 1 loss_plots.update(epoch + 1, ind_metrics, ood_metric_dicts) # Save model + early stop # Early_stop_operator is min or max if args.early_stop_operator( early_stop_metric_value, best_early_stop_value) != best_early_stop_value: early_stop = 0 best_early_stop_value = early_stop_metric_value utils.save_checkpoint(checkpoints_folder, { "init_epoch": epoch + 1, "net": net.state_dict(), "optimizer": optimizer.state_dict(), "scheduler": model_config.scheduler.state_dict() if args.use_scheduler else None, "ood_metrics": ood_metric_dicts, "ind_metrics": ind_metrics, "best_early_stop_value": best_early_stop_value, "args": args, }, keep_n_best=1) print('Early stop metric ' + str(args.early_stop_metric) + ' beaten. Now ' + str(best_early_stop_value)) if args.use_scheduler: model_config.scheduler.step(ind_metrics['accuracy']) if early_stop == args.early_stop: loss_plots.draw(checkpoints_folder) print("early_stop reached") break loss_plots.draw(checkpoints_folder) print('Done') return
def run(results_dir, model_dir, base_dir, file_names, data_type, use_multi_scale=False, exported_disc_path=None, use_3dconv=False, compute_metrics=False, min_distance=3., max_distance=150., show_result=False): sess = tf.Session() in_image = tf.placeholder(tf.float32, [None, None, None, 3]) gt_image = tf.placeholder(tf.float32, [None, None, None, 1]) gt_mask = None if data_type == 'real': gt_mask = tf.placeholder(tf.float32, [None, None, None, 1]) model = lsgan.build_model(in_image, gt_image, data_type=data_type, gt_mask=gt_mask, smooth_weight=0.1, adv_weight=0.0001, discriminator_ckpt=exported_disc_path, use_multi_scale=use_multi_scale, use_3dconv=use_3dconv) min_eval_distance = min_distance max_eval_distance = 80. else: model = lsgan.build_model(in_image, gt_image, data_type=data_type, gt_mask=gt_mask, smooth_weight=1e-4, adv_weight=0.0001, use_multi_scale=use_multi_scale, use_3dconv=use_3dconv) min_eval_distance = min_distance max_eval_distance = max_distance out_image = model['out_image'] saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) saver.restore(sess, model_dir) per_image_metrics = [] #mae = [] if not os.path.isdir(results_dir): os.makedirs(results_dir) results_folder = ['gated2depth', 'gated2depth_img', 'all'] for result_folder in results_folder: if not os.path.exists(os.path.join(results_dir, result_folder)): os.makedirs(os.path.join(results_dir, result_folder)) for ind in range(len(file_names)): # get the path from image id train_fn = file_names[ind] if data_type == 'real': img_id = train_fn gta_pass = '' else: img_id = train_fn gta_pass = '' in_img = dsutil.read_gated_image(base_dir, gta_pass, img_id, data_type) input_patch = in_img output = sess.run(out_image, feed_dict={in_image: input_patch}) output = np.clip(output * max_distance, min_distance, max_distance) gt_patch, _ = dsutil.read_gt_image(base_dir, gta_pass, img_id, data_type, raw_values_only=True, min_distance=min_distance, max_distance=max_distance) if compute_metrics: #if data_type != 'real': #curr_mae = np.mean(np.abs(output - gt_patch), dtype=np.float64) curr_metrics = calc_metrics(output[0, :, :, 0], gt_patch, min_distance=min_eval_distance, max_distance=max_eval_distance) per_image_metrics.append(curr_metrics) #mae.append(curr_mae) # else: #depth_lidar1, _ = dsutil.read_gt_image(base_dir, gta_pass, img_id, data_type, raw_values_only=True, min_distance=min_distance, max_distance=max_distance) #curr_metrics = calc_metrics(output[0, :, :, 0], gt_patch, min_distance=min_eval_distance, # max_distance=max_eval_distance) #per_image_metrics.append(curr_metrics) #mae.append(curr_metrics) np.savez_compressed( os.path.join(results_dir, 'gated2depth', '{}'.format(img_id)), output) #depth_lidar1, _ = dsutil.read_gt_image(base_dir, gta_pass, img_id, data_type, raw_values_only=True, min_distance=min_distance, max_distance=max_distance) if data_type != 'real': #print(depth_lidar1.shape) depth_lidar1_color = visualize2D.colorize_depth( gt_patch, min_distance=min_eval_distance, max_distance=max_eval_distance) else: #print(depth_lidar1.shape) depth_lidar1_color = visualize2D.colorize_pointcloud( gt_patch, min_distance=min_eval_distance, max_distance=max_eval_distance, radius=3) depth_map_color = visualize2D.colorize_depth( output[0, :, :, 0], min_distance=min_eval_distance, max_distance=max_eval_distance) in_out_shape = (int(depth_map_color.shape[0] + depth_map_color.shape[0] / 3. + gt_patch.shape[0]), depth_map_color.shape[1], 3) input_output = np.zeros(shape=in_out_shape) scaled_input = cv2.resize(input_patch[0, :, :, :], dsize=(int(input_patch.shape[2] / 3), int(input_patch.shape[1] / 3)), interpolation=cv2.INTER_AREA) * 255 for i in range(3): input_output[:scaled_input.shape[0], :scaled_input.shape[1], i] = scaled_input[:, :, 0] input_output[:scaled_input.shape[0], scaled_input.shape[1]:2 * scaled_input.shape[1], i] = scaled_input[:, :, 1] input_output[:scaled_input.shape[0], scaled_input.shape[1] * 2:scaled_input.shape[1] * 3, i] = scaled_input[:, :, 2] input_output[scaled_input.shape[0]:scaled_input.shape[0] + depth_map_color.shape[0], :, :] = depth_map_color input_output[scaled_input.shape[0] + depth_map_color.shape[0]:, :, :] = depth_lidar1_color cv2.imwrite( os.path.join(results_dir, 'gated2depth_img', '{}.jpg'.format(img_id)), depth_map_color.astype(np.uint8)) cv2.imwrite(os.path.join(results_dir, 'all', '{}.jpg'.format(img_id)), input_output.astype(np.uint8)) if show_result: import matplotlib.pyplot as plt plt.imshow( cv2.cvtColor(input_output.astype(np.uint8), cv2.COLOR_BGR2RGB)) plt.show() if compute_metrics: res = np.mean(per_image_metrics, axis=0) res_str = '' for i in range(res.shape[0]): res_str += '{}={:.2f} \n'.format(metric_str[i], res[i]) print(res_str) with open(os.path.join(results_dir, 'results.txt'), 'w') as f: f.write(res_str) with open(os.path.join(results_dir, 'results.tex'), 'w') as f: f.write(' & '.join(metric_str) + '\n') f.write(' & '.join(['{:.2f}'.format(r) for r in res]))
def main(): # set GPU ID os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu cudnn.benchmark = True # check save path save_path = args.save_path if not os.path.exists(save_path): os.makedirs(save_path) # set num_class if args.data == 'cifar100': num_class = 100 else: num_class = 10 # set num_classes model_dict = { "num_classes": num_class, } _, test_loader, \ test_onehot, test_label = dataset.get_loader(args.data, args.data_path, args.batch_size) train_set = dataset.get_dataset(args.data, args.data_path, mode='train') unlabeled_pool = dataset.get_dataset(args.data, args.data_path, mode='unlabeled') num_train = len(train_set) indices = list(range(num_train)) random.shuffle(indices) labeled_set = indices[:args.initial_budget] unlabeled_set = indices[args.initial_budget:] labeled_dataloader = DataLoader(train_set, sampler=SubsetRandomSampler(labeled_set), batch_size=args.batch_size, drop_last=True) now = datetime.datetime.now() formatedDate = now.strftime('%Y%m%d_%H_%M_') result_logger = utils.Logger( os.path.join(args.save_path, formatedDate + 'result.log')) arguments = [] for key, val in (args.__dict__.items()): arguments.append("{} : {}\n".format(key, val)) result_logger.write(arguments) result_logger = utils.Logger( os.path.join(args.save_path, formatedDate + 'result.log')) # make logger train_logger = utils.Logger( os.path.join(save_path, formatedDate + 'train.log')) test_epoch_logger = utils.Logger( os.path.join(save_path, formatedDate + 'test_epoch.log')) current_train = len(labeled_set) while (current_train < args.max_budget + 1): # set model if args.model == 'res': model = resnet.ResNet152(**model_dict).cuda() elif args.model == 'dense': model = densenet_BC.DenseNet3(depth=100, num_classes=num_class, growth_rate=12, reduction=0.5, bottleneck=True, dropRate=0.0).cuda() elif args.model == 'vgg': model = vgg.vgg16(**model_dict).cuda() # set criterion cls_criterion = nn.CrossEntropyLoss().cuda() ranking_criterion = nn.MarginRankingLoss(margin=0.0).cuda() # set optimizer (default:sgd) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=0.0005, nesterov=False) # set scheduler scheduler = MultiStepLR(optimizer, milestones=[120, 160], gamma=0.1) # make History Class correctness_history = crl_utils.History(len( labeled_dataloader.dataset)) # start Train for epoch in range(1, args.epochs + 1): train.train(labeled_dataloader, model, cls_criterion, ranking_criterion, optimizer, epoch, correctness_history, train_logger, args) test_acc, test_loss = metrics.evaluate(test_loader, model, cls_criterion, args.budget, epoch, test_epoch_logger) scheduler.step() # save model if epoch == args.epochs: torch.save(model.state_dict(), os.path.join(save_path, 'model.pth')) # finish train # calc measure acc, aurc, eaurc, aupr, fpr, ece, nll, brier = metrics.calc_metrics( test_loader, test_label, test_onehot, model, cls_criterion) # result write result_logger.write([ current_train, test_acc, aurc * 1000, eaurc * 1000, aupr * 100, fpr * 100, ece * 100, nll * 10, brier * 100 ]) random.shuffle(unlabeled_set) subset = unlabeled_set[:args.subset] unlabeled_poolloader = DataLoader( unlabeled_pool, sampler=SubsetSequentialSampler(subset), batch_size=args.batch_size, drop_last=False) all_confidence = get_confidence(model, unlabeled_poolloader) print(len(all_confidence)) arg = np.argsort(all_confidence) labeled_set = list( set(labeled_set) | set(np.array(unlabeled_set)[arg][:args.budget])) unlabeled_set = list(set(unlabeled_set) - set(labeled_set)) current_train = len(labeled_set) #unlabeled_set = list(torch.tensor(unlabeled_set)[arg][args.budget:].numpy()) \ # + unlabeled_set[args.subset:] print("after acquistiion") print('current labeled :', len(labeled_set)) print('current unlabeled :', len(unlabeled_set)) labeled_dataloader = DataLoader( train_set, sampler=SubsetRandomSampler(labeled_set), batch_size=args.batch_size, drop_last=True)
def calculate_alg(singletons, net, uw, ud, g_type, alg): communities = "/home/amaury/communities_hashmap/" + str( g_type) + "/" + str(alg) + "/" + str(singletons) + "/" + str(net) + "/" graphs = "/home/amaury/graphs_hashmap/" + str(net) + "/" + str( g_type) + "/" out_ad = str(output_dir) + "average_degree/" + str(g_type) + "/" + str( alg) + "/" + str(singletons) + "/" + str(net) + "/" out_c = str(output_dir) + "conductance/" + str(g_type) + "/" + str( alg) + "/" + str(singletons) + "/" + str(net) + "/" out_cut_r = str(output_dir) + "cut_ratio/" + str(g_type) + "/" + str( alg) + "/" + str(singletons) + "/" + str(net) + "/" out_d = str(output_dir) + "density/" + str(g_type) + "/" + str( alg) + "/" + str(singletons) + "/" + str(net) + "/" out_e = str(output_dir) + "expansion/" + str(g_type) + "/" + str( alg) + "/" + str(singletons) + "/" + str(net) + "/" out_normal_cut = str(output_dir) + "normalized_cut/" + str( g_type) + "/" + str(alg) + "/" + str(singletons) + "/" + str(net) + "/" out_s = str(output_dir) + "separability/" + str(g_type) + "/" + str( alg) + "/" + str(singletons) + "/" + str(net) + "/" if not os.path.exists(communities): print("Diretório com as comunidades não encontrado: " + str(communities) + "\n") else: print( "\n######################################################################" ) for threshold in os.listdir(communities): if not os.path.isdir(str(communities) + str(threshold) + "/"): print("Threshold para a rede " + str(net) + " não encontrado: " + str(threshold)) else: create_dirs(out_ad, out_c, out_cut_r, out_d, out_e, out_normal_cut, out_s) if os.path.exists( str(out_ad) + str(threshold) + ".json") and os.path.exists( str(out_c) + str(threshold) + ".json") and os.path.exists( str(out_cut_r) + str(threshold) + ".json") and os.path.exists( str(out_d) + str(threshold) + ".json") and os.path.exists( str(out_e) + str(threshold) + ".json") and os.path.exists( str(out_normal_cut) + str(threshold) + ".json") and os.path.exists( str(out_s) + str(threshold) + ".json"): print("Arquivo de destino já existe: " + str(threshold) + ".json") else: print( "######################################################################" ) result_ad = [] result_c = [] result_cut_r = [] result_d = [] result_e = [] result_normal_cut = [] result_s = [] i = 0 #Ponteiro para o ego for file in os.listdir( str(communities) + str(threshold) + "/"): if os.path.isfile( str(communities) + str(threshold) + "/" + file): ego_id = file.split(".txt") ego_id = long(ego_id[0]) i += 1 if not os.path.isfile( str(graphs) + str(ego_id) + ".edge_list"): print( "ERROR - EGO: " + str(i) + " - Arquivo com lista de arestas não encontrado:" + str(graphs) + str(ego_id) + ".edge_list") else: with open( str(communities) + str(threshold) + "/" + file, 'r') as community_file: if ud is False: G = snap.LoadEdgeList( snap.PNGraph, str(graphs) + str(ego_id) + ".edge_list", 0, 1 ) # load from a text file - pode exigir um separador.: snap.LoadEdgeList(snap.PNGraph, file, 0, 1, '\t') else: G = snap.LoadEdgeList( snap.PUNGraph, str(graphs) + str(ego_id) + ".edge_list", 0, 1 ) # load from a text file - pode exigir um separador.: snap.LoadEdgeList(snap.PNGraph, file, 0, 1, '\t') print( str(g_type) + " - " + str(alg) + " - " + str(singletons) + " - Rede: " + str(net) + " - THRESHOLD: " + str(threshold) + " - ego(" + str(i) + "): " + str(file)) communities_dict = prepare_communities( community_file ) #Função para devolver um dicionário com as comunidades avg_ad, avg_c, avg_cut_r, avg_d, avg_e, avg_normal_cut, avg_s = metrics.calc_metrics( communities_dict, G, ud) # Calcular as métricas result_ad.append( avg_ad['media']) # Salvar Métrica result_c.append( avg_c['media']) # Salvar Métrica result_cut_r.append( avg_cut_r['media']) # Salvar Métrica result_d.append( avg_d['media']) # Salvar Métrica result_e.append( avg_e['media']) # Salvar Métrica result_normal_cut.append( avg_normal_cut['media'] ) # Salvar Métrica result_s.append( avg_s['media']) # Salvar Métrica print("Average Degree: " + str(result_ad[i - 1]) + " - Conductance: " + str(result_c[i - 1]) + " - Cut Ratio: " + str(result_cut_r[i - 1]) + " - Density: " + str(result_d[i - 1])) print("Expansion: " + str(result_e[i - 1]) + " - Normalized Cut: " + str(result_normal_cut[i - 1]) + " - Separability: " + str(result_s[i - 1])) print print( "######################################################################" ) with open(str(out_ad) + str(threshold) + ".json", "w") as f: f.write( json.dumps(result_ad, separators=(',', ':')) + "\n") with open(str(out_c) + str(threshold) + ".json", "w") as f: f.write( json.dumps(result_c, separators=(',', ':')) + "\n") with open(str(out_cut_r) + str(threshold) + ".json", "w") as f: f.write( json.dumps(result_cut_r, separators=(',', ':')) + "\n") with open(str(out_d) + str(threshold) + ".json", "w") as f: f.write( json.dumps(result_d, separators=(',', ':')) + "\n") with open(str(out_e) + str(threshold) + ".json", "w") as f: f.write( json.dumps(result_e, separators=(',', ':')) + "\n") with open( str(out_normal_cut) + str(threshold) + ".json", "w") as f: f.write( json.dumps(result_normal_cut, separators=(',', ':')) + "\n") with open(str(out_s) + str(threshold) + ".json", "w") as f: f.write( json.dumps(result_s, separators=(',', ':')) + "\n") print( "######################################################################" )
def main(*kargs, **kwargs): get_kwargs(kwargs) train_fname = kwargs['train'] test_fname = kwargs['test'] result_fname = kwargs['output'] embeds_fname = kwargs['embeds'] logger_fname = kwargs['logger'] swear_words_fname = kwargs['swear_words'] wrong_words_fname = kwargs['wrong_words'] warm_start = kwargs['warm_start'] format_embeds = kwargs['format_embeds'] config = kwargs['config'] train_clear = kwargs['train_clear'] test_clear = kwargs['test_clear'] output_dir = kwargs['output_dir'] norm_prob = kwargs['norm_prob'] norm_prob_koef = kwargs['norm_prob_koef'] gpus = kwargs['gpus'] model_file = { 'dense': os.path.join(output_dir, 'dense.h5'), 'cnn': os.path.join(output_dir, 'cnn.h5'), 'lstm': os.path.join(output_dir, 'lstm.h5'), 'concat': os.path.join(output_dir, 'concat.h5'), 'lr': os.path.join(output_dir, '{}_logreg.bin'), 'catboost': os.path.join(output_dir, '{}_catboost.bin') } # ====Create logger==== logger = Logger(logging.getLogger(), logger_fname) # ====Detect GPUs==== logger.debug(device_lib.list_local_devices()) # ====Load data==== logger.info('Loading data...') train_df = load_data(train_fname) test_df = load_data(test_fname) target_labels = [ 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate' ] num_classes = len(target_labels) # ====Load additional data==== logger.info('Loading additional data...') swear_words = load_data(swear_words_fname, func=lambda x: set(x.T[0]), header=None) wrong_words_dict = load_data(wrong_words_fname, func=lambda x: {val[0]: val[1] for val in x}) tokinizer = RegexpTokenizer(r'\w+') regexps = [ re.compile("([a-zA-Z]+)([0-9]+)"), re.compile("([0-9]+)([a-zA-Z]+)") ] # ====Load word vectors==== logger.info('Loading embeddings...') embed_dim = 300 embeds = Embeds(embeds_fname, 'fasttext', format=format_embeds) # ====Clean texts==== logger.info('Cleaning text...') if warm_start: logger.info('Use warm start...') else: train_df['comment_text_clear'] = clean_text(train_df['comment_text'], tokinizer, wrong_words_dict, swear_words, regexps) test_df['comment_text_clear'] = clean_text(test_df['comment_text'], tokinizer, wrong_words_dict, swear_words, regexps) train_df.to_csv(train_clear, index=False) test_df.to_csv(test_clear, index=False) # ====Calculate maximum seq length==== logger.info('Calc text length...') train_df.fillna('unknown', inplace=True) test_df.fillna('unknown', inplace=True) train_df['text_len'] = train_df['comment_text_clear'].apply( lambda words: len(words.split())) test_df['text_len'] = test_df['comment_text_clear'].apply( lambda words: len(words.split())) max_seq_len = np.round(train_df['text_len'].mean() + 3 * train_df['text_len'].std()).astype(int) logger.debug('Max seq length = {}'.format(max_seq_len)) # ====Prepare data to NN==== logger.info('Converting texts to sequences...') max_words = 100000 train_df['comment_seq'], test_df[ 'comment_seq'], word_index = convert_text2seq( train_df['comment_text_clear'].tolist(), test_df['comment_text_clear'].tolist(), max_words, max_seq_len, lower=True, char_level=False, uniq=True) logger.debug('Dictionary size = {}'.format(len(word_index))) logger.info('Preparing embedding matrix...') embedding_matrix, words_not_found = get_embedding_matrix( embed_dim, embeds, max_words, word_index) logger.debug('Embedding matrix shape = {}'.format( np.shape(embedding_matrix))) logger.debug('Number of null word embeddings = {}'.format( np.sum(np.sum(embedding_matrix, axis=1) == 0))) logger.info('Deleting unknown words from seq...') train_df['comment_seq'] = clean_seq(train_df['comment_seq'], embedding_matrix, max_seq_len) test_df['comment_seq'] = clean_seq(test_df['comment_seq'], embedding_matrix, max_seq_len) # ====Train/test split data==== x = np.array(train_df['comment_seq'].tolist()) y = np.array(train_df[target_labels].values) x_train_nn, x_test_nn, y_train_nn, y_test_nn, train_idxs, test_idxs = split_data( x, y, test_size=0.2, shuffle=True, random_state=42) test_df_seq = np.array(test_df['comment_seq'].tolist()) y_nn = [] logger.debug('X shape = {}'.format(np.shape(x_train_nn))) # ====Train models==== params = Params(config) cnn = get_cnn(embedding_matrix, num_classes, max_seq_len, num_filters=params.get('cnn').get('num_filters'), l2_weight_decay=params.get('cnn').get('l2_weight_decay'), dropout_val=params.get('cnn').get('dropout_val'), dense_dim=params.get('cnn').get('dense_dim'), add_sigmoid=True, train_embeds=params.get('cnn').get('train_embeds'), gpus=gpus) lstm = get_lstm(embedding_matrix, num_classes, max_seq_len, l2_weight_decay=params.get('lstm').get('l2_weight_decay'), lstm_dim=params.get('lstm').get('lstm_dim'), dropout_val=params.get('lstm').get('dropout_val'), dense_dim=params.get('lstm').get('dense_dim'), add_sigmoid=True, train_embeds=params.get('lstm').get('train_embeds'), gpus=gpus) concat = get_concat_model( embedding_matrix, num_classes, max_seq_len, n_layers=params.get('concat').get('n_layers'), concat=params.get('concat').get('concat'), pool=params.get('concat').get('pool'), num_filters=params.get('concat').get('num_filters'), l2_weight_decay=params.get('concat').get('l2_weight_decay'), lstm_dim=params.get('concat').get('lstm_dim'), dropout_val=params.get('concat').get('dropout_val'), dense_dim=params.get('concat').get('dense_dim'), add_sigmoid=True, train_embeds=params.get('concat').get('train_embeds'), gpus=gpus) models = [] for model_label in params.get('models'): if model_label == 'cnn': models.append([model_label, cnn]) elif model_label == 'dense': models.append([model_label, dense]) elif model_label == 'lstm': models.append([model_label, lstm]) elif model_label == 'concat': models.append([model_label, concat]) else: raise ValueError( 'Invalid model {}. Model hasn`t defined.'.format(model_label)) for i in range(len(models)): model_label, model = models[i] logger.info("training {} ...".format(model_label)) if params.get(model_label).get('warm_start') and os.path.exists( params.get(model_label).get('model_file')): logger.info('{} warm starting...'.format(model_label)) model = load_model(params.get(model_label).get('model_file')) models[i][1] = model else: hist = train( x_train_nn, y_train_nn, model, batch_size=params.get(model_label).get('batch_size'), num_epochs=params.get(model_label).get('num_epochs'), learning_rate=params.get(model_label).get('learning_rate'), early_stopping_delta=params.get(model_label).get( 'early_stopping_delta'), early_stopping_epochs=params.get(model_label).get( 'early_stopping_epochs'), use_lr_strategy=params.get(model_label).get('use_lr_strategy'), lr_drop_koef=params.get(model_label).get('lr_drop_koef'), epochs_to_drop=params.get(model_label).get('epochs_to_drop'), logger=logger) y_nn.append(model.predict(x_test_nn)) save_predictions(test_df, model.predict(test_df_seq), target_labels, model_label) metrics = get_metrics(y_test_nn, y_nn[-1], target_labels) logger.debug('{} metrics:\n{}'.format(model_label, print_metrics(metrics))) logger.debug('Model path = {}'.format(model_file[model_label])) model.save(model_file[model_label]) # TFIDF + LogReg logger.info('training LogReg over tfidf...') train_tfidf, val_tfidf, test_tfidf, word_tfidf, char_tfidf = get_tfidf( train_df['comment_text_clear'].values[train_idxs], train_df['comment_text_clear'].values[test_idxs], test_df['comment_text_clear'].values) models_lr = [] metrics_lr = {} y_tfidf = [] for i, label in enumerate(target_labels): model = LogisticRegression(C=4.0, solver='sag', max_iter=1000, n_jobs=16) model.fit(train_tfidf, y_train_nn[:, i]) y_tfidf.append(model.predict_proba(val_tfidf)[:, 1]) test_df['tfidf_{}'.format(label)] = model.predict_proba(test_tfidf)[:, 1] metrics_lr[label] = calc_metrics(y_test_nn[:, i], y_tfidf[-1]) models_lr.append(model) joblib.dump(model, model_file['lr'].format(label)) metrics_lr['Avg'] = { 'Logloss': np.mean([metric['Logloss'] for label, metric in metrics_lr.items()]) } logger.debug('LogReg(TFIDF) metrics:\n{}'.format( print_metrics(metrics_lr))) # Bow for catboost if params.get('catboost').get('add_bow'): top_pos_words = [] top_neg_words = [] for i in range(num_classes): top_pos_words.append([]) top_neg_words.append([]) top_pos_words[-1], top_neg_words[ -1] = get_most_informative_features( [word_tfidf, char_tfidf], models_lr[i], n=params.get('catboost').get('bow_top')) top_pos_words = list( set( np.concatenate([[val for score, val in top] for top in top_pos_words]))) top_neg_words = list( set( np.concatenate([[val for score, val in top] for top in top_neg_words]))) top = list(set(np.concatenate([top_pos_words, top_neg_words]))) train_bow = get_bow(train_df['comment_text_clear'].values[train_idxs], top) val_bow = get_bow(train_df['comment_text_clear'].values[test_idxs], top) test_bow = get_bow(test_df['comment_text_clear'].values, top) logger.debug('Count bow words = {}'.format(len(top))) # Meta catboost logger.info('training catboost as metamodel...') train_df['text_unique_len'] = train_df['comment_text_clear'].apply( calc_text_uniq_words) test_df['text_unique_len'] = test_df['comment_text_clear'].apply( calc_text_uniq_words) train_df['text_unique_koef'] = train_df['text_unique_len'] / train_df[ 'text_len'] test_df[ 'text_unique_koef'] = test_df['text_unique_len'] / test_df['text_len'] text_len_features = train_df[[ 'text_len', 'text_unique_len', 'text_unique_koef' ]].values[test_idxs] x_train_catboost = [] y_train_catboost = y_test_nn features = y_nn features.extend([text_len_features, np.array(y_tfidf).T]) if params.get('catboost').get('add_bow'): features.append(val_bow) for feature in zip(*features): x_train_catboost.append(np.concatenate(feature)) models_cb = [] metrics_cb = {} x_train_cb, x_val_cb, y_train_cb, y_val_cb = train_test_split( x_train_catboost, y_train_catboost, test_size=0.20, random_state=42) for i, label in enumerate(target_labels): model = CatBoostClassifier( loss_function='Logloss', iterations=params.get('catboost').get('iterations'), depth=params.get('catboost').get('depth'), rsm=params.get('catboost').get('rsm'), learning_rate=params.get('catboost').get('learning_rate'), device_config=params.get('catboost').get('device_config')) model.fit(x_train_cb, y_train_cb[:, i], eval_set=(x_val_cb, y_val_cb[:, i]), use_best_model=True) y_hat_cb = model.predict_proba(x_val_cb) metrics_cb[label] = calc_metrics(y_val_cb[:, i], y_hat_cb[:, 1]) models_cb.append(model) joblib.dump(model, model_file['catboost'].format(label)) metrics_cb['Avg'] = { 'Logloss': np.mean([metric['Logloss'] for label, metric in metrics_cb.items()]) } logger.debug('CatBoost metrics:\n{}'.format(print_metrics(metrics_cb))) # ====Predict==== logger.info('Applying models...') text_len_features = test_df[[ 'text_len', 'text_unique_len', 'text_unique_koef' ]].values y_tfidf_test = test_df[[ 'tfidf_{}'.format(label) for label in target_labels ]].values x_test_cb = [] features = [] for model_label, _ in models: features.append(test_df[[ '{}_{}'.format(model_label, label) for label in target_labels ]].values) features.extend([text_len_features, y_tfidf_test]) if params.get('catboost').get('add_bow'): features.append(test_bow) for feature in tqdm(zip(*features)): x_test_cb.append(np.concatenate(feature)) for label, model in zip(target_labels, models_cb): pred = model.predict_proba(x_test_cb) test_df[label] = np.array(list(pred))[:, 1] # ====Normalize probabilities==== if norm_prob: for label in target_labels: test_df[label] = norm_prob_koef * test_df[label] # ====Save results==== logger.info('Saving results...') test_df[[ 'id', 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate' ]].to_csv(result_fname, index=False, header=True)
" An aggression score larger than the threshold means that the user is aggressive. Eg. = 0.4" ) metrics_parser.add_argument( '--metric_type', default='similarity', type=str, choices=['similarity', 'aggression'], help= "The type of metric to calculate, similarity metrics or aggression. " "'similarity' is suggested for aggression modeling experiment") metrics_parser.add_argument( "--seedsize", default=None, type=int, help= "Seed set size. It is the number of initial infected nodes. 5594 is the total number of aggressive users" ) metrics_parser.add_argument( "--configuration", default=None, type=str, help="Specifies a single experiment to run metrics for") args = parser.parse_args() print() print(args) if args.mode == 'simulation': experiment(args, True) elif args.mode == 'metric': calc_metrics(args, 'modeling')
type=int, help='How many times to repeat an experiment') metrics_parser = subparsers.add_parser( 'metric', help= 'Metric calculation based on the snapshots produced by a competitive aggression minimization simulation experiment' ) metrics_parser.add_argument( 'aggression_threshold', type=float, help= "The threshold in [0,1] that determines when a user becomes aggressive. An aggression score larger than the threshold means that the user is aggressive. Eg. = 0.4" ) metrics_parser.add_argument( 'metric_type', type=str, choices=['similarity', 'aggression'], help= "The type of metric to calculate, similarity metrics or aggression. 'aggression' is suggested for competitive aggression minimization experiment" ) args = parser.parse_args() print() print(args) if args.mode == 'simulation': experiment(args, True) elif args.mode == 'metric': calc_metrics(args, 'minimization')
print "\t", "Converting to numpy array..." X_all[interv] = np.array(X_all[interv]) X_all_test[interv] = np.array(X_all_test[interv]) y_all[interv] = np.array(y_all[interv]) y_all_test[interv] = np.array(y_all_test[interv]) size = len(y_all[interv]) print "\t", "Model compute for size = " + str(size) print "\t", "computing model..." model.fit(X_all[interv], y_all[interv]) print "\t", "predicting..." expected = y_all_test[interv] predicted = model.predict(X_all_test[interv]) parameters = { "filename_model_results": "results/SVM" + "_" + interv + ".txt", "filename_confusion_matrix": "results/SVM" + "_" + interv + "_Confusion_Matrix.csv", "model_interval": [interval[interv][0], interval[interv][1]], "model_type": "SVM Model", "model": model, "expected": y_all_test[interv], "predicted": predicted, "target_names": target_names } metrics.calc_metrics(parameters)
def main(): file_name = "./flood_graph/150_250/128/500/ji_sort/1_conf/sample-wised/default/{}/".format( args.b) start = time.time() # set GPU ID os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu cudnn.benchmark = True # check save path save_path = file_name # save_path = args.save_path if not os.path.exists(save_path): os.makedirs(save_path) # make dataloader if args.valid == True: train_loader, valid_loader, test_loader, test_onehot, test_label = dataset.get_valid_loader( args.data, args.data_path, args.batch_size) else: train_loader, train_onehot, train_label, test_loader, test_onehot, test_label = dataset.get_loader( args.data, args.data_path, args.batch_size) # set num_class if args.data == 'cifar100': num_class = 100 else: num_class = 10 # set num_classes model_dict = { "num_classes": num_class, } # set model if args.model == 'res': model = resnet.resnet110(**model_dict).cuda() elif args.model == 'dense': model = densenet_BC.DenseNet3(depth=100, num_classes=num_class, growth_rate=12, reduction=0.5, bottleneck=True, dropRate=0.0).cuda() elif args.model == 'vgg': model = vgg.vgg16(**model_dict).cuda() # set criterion if args.loss == 'MS': cls_criterion = losses.MultiSimilarityLoss().cuda() elif args.loss == 'Contrastive': cls_criterion = losses.ContrastiveLoss().cuda() elif args.loss == 'Triplet': cls_criterion = losses.TripletLoss().cuda() elif args.loss == 'NPair': cls_criterion = losses.NPairLoss().cuda() elif args.loss == 'Focal': cls_criterion = losses.FocalLoss(gamma=3.0).cuda() else: if args.mode == 0: cls_criterion = nn.CrossEntropyLoss().cuda() else: cls_criterion = nn.CrossEntropyLoss(reduction="none").cuda() ranking_criterion = nn.MarginRankingLoss(margin=0.0).cuda() # set optimizer (default:sgd) optimizer = optim.SGD( model.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4, # weight_decay=0.0001, nesterov=False) # optimizer = optim.SGD(model.parameters(), # lr=float(args.lr), # momentum=0.9, # weight_decay=args.weight_decay, # nesterov=False) # set scheduler # scheduler = MultiStepLR(optimizer, # milestones=[500, 750], # gamma=0.1) scheduler = MultiStepLR(optimizer, milestones=[150, 250], gamma=0.1) # scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=args.lr_decay_step, gamma=args.lr_decay_gamma) # make logger train_logger = utils.Logger(os.path.join(save_path, 'train.log')) result_logger = utils.Logger(os.path.join(save_path, 'result.log')) # make History Class correctness_history = crl_utils.History(len(train_loader.dataset)) ## define matrix if args.data == 'cifar': matrix_idx_confidence = [[_] for _ in range(50000)] matrix_idx_iscorrect = [[_] for _ in range(50000)] else: matrix_idx_confidence = [[_] for _ in range(73257)] matrix_idx_iscorrect = [[_] for _ in range(73257)] # write csv #''' import csv f = open('{}/logs_{}_{}.txt'.format(file_name, args.b, args.epochs), 'w', newline='') f.write("location = {}\n\n".format(file_name) + str(args)) f0 = open('{}/Test_confidence_{}_{}.csv'.format(file_name, args.b, args.epochs), 'w', newline='') # f0 = open('./baseline_graph/150_250/128/500/Test_confidence_{}_{}.csv'.format(args.b, args.epochs), 'w', newline='') # f0 = open('./CRL_graph/150_250/Test_confidence_{}_{}.csv'.format(args.b, args.epochs), 'w', newline='') wr_conf_test = csv.writer(f0) header = [_ for _ in range(args.epochs + 1)] header[0] = 'Epoch' wr_conf_test.writerows([header]) f1 = open('{}/Train_confidence_{}_{}.csv'.format(file_name, args.b, args.epochs), 'w', newline='') # f1 = open('./baseline_graph/150_250/128/500/Train_confidence_{}_{}.csv'.format(args.b, args.epochs), 'w', newline='') # f1 = open('./CRL_graph/150_250/Train_confidence_{}_{}.csv'.format(args.b, args.epochs), 'w', newline='') wr = csv.writer(f1) header = [_ for _ in range(args.epochs + 1)] header[0] = 'Epoch' wr.writerows([header]) f2 = open('{}/Train_Flood_{}_{}_{}.csv'.format(file_name, args.data, args.b, args.epochs), 'w', newline='') # f2 = open('./baseline_graph/150_250/128/500/Train_Base_{}_{}_{}.csv'.format(args.data, args.b, args.epochs), 'w', newline='') # f2 = open('./CRL_graph/150_250/Train_Flood_{}_{}_{}.csv'.format(args.data, args.b, args.epochs), 'w', newline='') wr_train = csv.writer(f2) header = [_ for _ in range(args.epochs + 1)] header[0] = 'Epoch' wr_train.writerows([header]) f3 = open('{}/Test_Flood_{}_{}_{}.csv'.format(file_name, args.data, args.b, args.epochs), 'w', newline='') # f3 = open('./baseline_graph/150_250/128/500/Test_Base_{}_{}_{}.csv'.format(args.data, args.b, args.epochs), 'w', newline='') # f3 = open('./CRL_graph/150_250/Test_Flood_{}_{}_{}.csv'.format(args.data, args.b, args.epochs), 'w', newline='') wr_test = csv.writer(f3) header = [_ for _ in range(args.epochs + 1)] header[0] = 'Epoch' wr_test.writerows([header]) #''' # start Train best_valid_acc = 0 test_ece_report = [] test_acc_report = [] test_nll_report = [] test_over_con99_report = [] test_e99_report = [] test_cls_loss_report = [] train_ece_report = [] train_acc_report = [] train_nll_report = [] train_over_con99_report = [] train_e99_report = [] train_cls_loss_report = [] train_rank_loss_report = [] train_total_loss_report = [] for epoch in range(1, args.epochs + 1): scheduler.step() matrix_idx_confidence, matrix_idx_iscorrect, idx, iscorrect, confidence, target, cls_loss_tr, rank_loss_tr, batch_correctness, total_confidence, total_correctness = \ train.train(matrix_idx_confidence, matrix_idx_iscorrect, train_loader, model, wr, cls_criterion, ranking_criterion, optimizer, epoch, correctness_history, train_logger, args) if args.rank_weight != 0.0: print("RANK ", rank_loss_tr) total_loss_tr = cls_loss_tr + rank_loss_tr if args.valid == True: idx, iscorrect, confidence, target, cls_loss_val, acc = train.valid( valid_loader, model, cls_criterion, ranking_criterion, optimizer, epoch, correctness_history, train_logger, args) if acc > best_valid_acc: best_valid_acc = acc print("*** Update Best Acc ***") # save model if epoch == args.epochs: torch.save(model.state_dict(), os.path.join(save_path, 'model.pth')) print("########### Train ###########") acc_tr, aurc_tr, eaurc_tr, aupr_tr, fpr_tr, ece_tr, nll_tr, brier_tr, E99_tr, over_99_tr, cls_loss_tr = metrics.calc_metrics( train_loader, train_label, train_onehot, model, cls_criterion, args) if args.sort == True and epoch == 260: #if args.sort == True: train_loader = dataset.sort_get_loader( args.data, args.data_path, args.batch_size, idx, np.array(target), iscorrect, batch_correctness, total_confidence, total_correctness, np.array(confidence), epoch, args) train_acc_report.append(acc_tr) train_nll_report.append(nll_tr * 10) train_ece_report.append(ece_tr) train_over_con99_report.append(over_99_tr) train_e99_report.append(E99_tr) train_cls_loss_report.append(cls_loss_tr) if args.rank_weight != 0.0: train_total_loss_report.append(total_loss_tr) train_rank_loss_report.append(rank_loss_tr) print("CLS ", cls_loss_tr) # finish train print("########### Test ###########") # calc measure acc_te, aurc_te, eaurc_te, aupr_te, fpr_te, ece_te, nll_te, brier_te, E99_te, over_99_te, cls_loss_te = metrics.calc_metrics( test_loader, test_label, test_onehot, model, cls_criterion, args) test_ece_report.append(ece_te) test_acc_report.append(acc_te) test_nll_report.append(nll_te * 10) test_over_con99_report.append(over_99_te) test_e99_report.append(E99_te) test_cls_loss_report.append(cls_loss_te) print("CLS ", cls_loss_te) print("############################") # for idx in matrix_idx_confidence: # wr.writerow(idx) #''' # draw graph df = pd.DataFrame() df['epoch'] = [i for i in range(1, args.epochs + 1)] df['test_ece'] = test_ece_report df['train_ece'] = train_ece_report fig_loss = plt.figure(figsize=(35, 35)) fig_loss.set_facecolor('white') ax = fig_loss.add_subplot() ax.plot(df['epoch'], df['test_ece'], df['epoch'], df['train_ece'], linewidth=10) ax.legend(['Test', 'Train'], loc=2, prop={'size': 60}) plt.title('[FL] ECE per epoch', fontsize=80) # plt.title('[BASE] ECE per epoch', fontsize=80) # plt.title('[CRL] ECE per epoch', fontsize=80) plt.xlabel('Epoch', fontsize=70) plt.ylabel('ECE', fontsize=70) plt.ylim([0, 1]) plt.setp(ax.get_xticklabels(), fontsize=30) plt.setp(ax.get_yticklabels(), fontsize=30) plt.savefig('{}/{}_{}_ECE_lr_{}.png'.format(file_name, args.model, args.b, args.epochs)) # plt.savefig('./baseline_graph/150_250/128/500/{}_{}_ECE_lr_{}.png'.format(args.model, args.b, args.epochs)) # plt.savefig('./CRL_graph/150_250/{}_{}_ECE_lr_{}.png'.format(args.model, args.b, args.epochs)) df2 = pd.DataFrame() df2['epoch'] = [i for i in range(1, args.epochs + 1)] df2['test_acc'] = test_acc_report df2['train_acc'] = train_acc_report fig_acc = plt.figure(figsize=(35, 35)) fig_acc.set_facecolor('white') ax = fig_acc.add_subplot() ax.plot(df2['epoch'], df2['test_acc'], df2['epoch'], df2['train_acc'], linewidth=10) ax.legend(['Test', 'Train'], loc=2, prop={'size': 60}) plt.title('[FL] Accuracy per epoch', fontsize=80) # plt.title('[BASE] Accuracy per epoch', fontsize=80) # plt.title('[CRL] Accuracy per epoch', fontsize=80) plt.xlabel('Epoch', fontsize=70) plt.ylabel('Accuracy', fontsize=70) plt.ylim([0, 100]) plt.setp(ax.get_xticklabels(), fontsize=30) plt.setp(ax.get_yticklabels(), fontsize=30) plt.savefig('{}/{}_{}_acc_lr_{}.png'.format(file_name, args.model, args.b, args.epochs)) # plt.savefig('./baseline_graph/150_250/128/500/{}_{}_acc_lr_{}.png'.format(args.model, args.b, args.epochs)) # plt.savefig('./CRL_graph/150_250/{}_{}_acc_lr_{}.png'.format(args.model, args.b, args.epochs)) df3 = pd.DataFrame() df3['epoch'] = [i for i in range(1, args.epochs + 1)] df3['test_nll'] = test_nll_report df3['train_nll'] = train_nll_report fig_acc = plt.figure(figsize=(35, 35)) fig_acc.set_facecolor('white') ax = fig_acc.add_subplot() ax.plot(df3['epoch'], df3['test_nll'], df3['epoch'], df3['train_nll'], linewidth=10) ax.legend(['Test', 'Train'], loc=2, prop={'size': 60}) plt.title('[FL] NLL per epoch', fontsize=80) # plt.title('[BASE] NLL per epoch', fontsize=80) # plt.title('[CRL] NLL per epoch', fontsize=80) plt.xlabel('Epoch', fontsize=70) plt.ylabel('NLL', fontsize=70) plt.ylim([0, 45]) plt.setp(ax.get_xticklabels(), fontsize=30) plt.setp(ax.get_yticklabels(), fontsize=30) plt.savefig('{}/{}_{}_nll_lr_{}.png'.format(file_name, args.model, args.b, args.epochs)) # plt.savefig('./baseline_graph/150_250/128/500/{}_{}_nll_lr_{}.png'.format(args.model, args.b, args.epochs)) # plt.savefig('./CRL_graph/150_250/{}_{}_nll_lr_{}.png'.format(args.model, args.b, args.epochs)) df4 = pd.DataFrame() df4['epoch'] = [i for i in range(1, args.epochs + 1)] df4['test_over_con99'] = test_over_con99_report df4['train_over_con99'] = train_over_con99_report fig_acc = plt.figure(figsize=(35, 35)) fig_acc.set_facecolor('white') ax = fig_acc.add_subplot() ax.plot(df4['epoch'], df4['test_over_con99'], df4['epoch'], df4['train_over_con99'], linewidth=10) ax.legend(['Test', 'Train'], loc=2, prop={'size': 60}) plt.title('[FL] Over conf99 per epoch', fontsize=80) # plt.title('[BASE] Over conf99 per epoch', fontsize=80) # plt.title('[CRL] Over conf99 per epoch', fontsize=80) plt.xlabel('Epoch', fontsize=70) plt.ylabel('Over con99', fontsize=70) if args.data == 'cifar10' or args.data == 'cifar100': plt.ylim([0, 50000]) else: plt.ylim([0, 73257]) plt.setp(ax.get_xticklabels(), fontsize=30) plt.setp(ax.get_yticklabels(), fontsize=30) plt.savefig('{}/{}_{}_over_conf99_lr_{}.png'.format( file_name, args.model, args.b, args.epochs)) # plt.savefig('./baseline_graph/150_250/128/500/{}_{}_over_conf99_lr_{}.png'.format(args.model, args.b, args.epochs)) # plt.savefig('./CRL_graph/150_250/{}_{}_over_conf99_lr_{}.png'.format(args.model, args.b, args.epochs)) df5 = pd.DataFrame() df5['epoch'] = [i for i in range(1, args.epochs + 1)] df5['test_e99'] = test_e99_report df5['train_e99'] = train_e99_report fig_acc = plt.figure(figsize=(35, 35)) fig_acc.set_facecolor('white') ax = fig_acc.add_subplot() ax.plot(df5['epoch'], df5['test_e99'], df5['epoch'], df5['train_e99'], linewidth=10) ax.legend(['Test', 'Train'], loc=2, prop={'size': 60}) plt.title('[FL] E99 per epoch', fontsize=80) # plt.title('[BASE] E99 per epoch', fontsize=80) # plt.title('[CRL] E99 per epoch', fontsize=80) plt.xlabel('Epoch', fontsize=70) plt.ylabel('E99', fontsize=70) plt.ylim([0, 0.2]) plt.setp(ax.get_xticklabels(), fontsize=30) plt.setp(ax.get_yticklabels(), fontsize=30) plt.savefig('{}/{}_{}_E99_flood_lr_{}.png'.format(file_name, args.model, args.b, args.epochs)) # plt.savefig('./baseline_graph/150_250/128/500/{}_{}_E99_flood_lr_{}.png'.format(args.model, args.b, args.epochs)) # plt.savefig('./CRL_graph/150_250/{}_{}_E99_flood_lr_{}.png'.format(args.model, args.b, args.epochs)) df5 = pd.DataFrame() df5['epoch'] = [i for i in range(1, args.epochs + 1)] df5['test_cls_loss'] = test_cls_loss_report df5['train_cls_loss'] = train_cls_loss_report fig_acc = plt.figure(figsize=(35, 35)) fig_acc.set_facecolor('white') ax = fig_acc.add_subplot() ax.plot(df5['epoch'], df5['test_cls_loss'], df5['epoch'], df5['train_cls_loss'], linewidth=10) ax.legend(['Test', 'Train'], loc=2, prop={'size': 60}) plt.title('[FL] CLS_loss per epoch', fontsize=80) # plt.title('[BASE] CLS_loss per epoch', fontsize=80) # plt.title('[CRL] CLS_loss per epoch', fontsize=80) plt.xlabel('Epoch', fontsize=70) plt.ylabel('Loss', fontsize=70) plt.ylim([0, 5]) plt.setp(ax.get_xticklabels(), fontsize=30) plt.setp(ax.get_yticklabels(), fontsize=30) plt.savefig('{}/{}_{}_cls_loss_flood_lr_{}.png'.format( file_name, args.model, args.b, args.epochs)) # plt.savefig('./baseline_graph/150_250/128/500/{}_{}_cls_loss_flood_lr_{}.png'.format(args.model, args.b, args.epochs)) # plt.savefig('./CRL_graph/150_250/{}_{}_cls_loss_flood_lr_{}.png'.format(args.model, args.b, args.epochs)) if args.rank_weight != 0.0: df6 = pd.DataFrame() df6['epoch'] = [i for i in range(1, args.epochs + 1)] df6['train_cls_loss'] = train_cls_loss_report df6['train_rank_loss'] = train_rank_loss_report df6['train_total_loss'] = train_total_loss_report fig_acc = plt.figure(figsize=(35, 35)) fig_acc.set_facecolor('white') ax = fig_acc.add_subplot() ax.plot(df6['epoch'], df6['train_cls_loss'], df6['epoch'], df6['train_rank_loss'], df6['epoch'], df6['train_total_loss'], linewidth=10) ax.legend(['CLS', 'Rank', 'Total'], loc=2, prop={'size': 60}) plt.title('[FL] CLS_loss per epoch', fontsize=80) plt.xlabel('Epoch', fontsize=70) plt.ylabel('Loss', fontsize=70) # plt.ylim([0, 5]) plt.setp(ax.get_xticklabels(), fontsize=30) plt.setp(ax.get_yticklabels(), fontsize=30) plt.savefig( './CRL_graph/150_250/{}_{}_cls_loss_flood_lr_{}.png'.format( args.model, args.b, args.epochs)) test_acc_report.insert(0, 'ACC') test_ece_report.insert(0, 'ECE') test_nll_report.insert(0, 'NLL') test_over_con99_report.insert(0, 'Over_conf99') test_e99_report.insert(0, 'E99') test_cls_loss_report.insert(0, 'CLS') wr_test.writerow(test_acc_report) wr_test.writerow(test_ece_report) wr_test.writerow(test_nll_report) wr_test.writerow(test_over_con99_report) wr_test.writerow(test_e99_report) wr_test.writerow(test_cls_loss_report) train_acc_report.insert(0, 'ACC') train_ece_report.insert(0, 'ECE') train_nll_report.insert(0, 'NLL') train_over_con99_report.insert(0, 'Over_conf99') train_e99_report.insert(0, 'E99') train_cls_loss_report.insert(0, 'CLS') wr_train.writerow(train_acc_report) wr_train.writerow(train_ece_report) wr_train.writerow(train_nll_report) wr_train.writerow(train_over_con99_report) wr_train.writerow(train_e99_report) wr_train.writerow(train_cls_loss_report) if args.rank_weight != 0.0: train_rank_loss_report.insert(0, 'Rank') train_total_loss_report.insert(0, 'Total') wr_train.writerow(train_rank_loss_report) wr_train.writerow(train_total_loss_report) #''' # result write result_logger.write([ acc_te, aurc_te * 1000, eaurc_te * 1000, aupr_te * 100, fpr_te * 100, ece_te * 100, nll_te * 10, brier_te * 100, E99_te * 100 ]) if args.valid == True: print("Best Valid Acc : {}".format(acc)) print("Flood Level: {}".format(args.b)) print("Sort : {}".format(args.sort)) print("Sort Mode : {}".format(args.sort_mode)) print("TIME : ", time.time() - start)