best_prec1 = 0.
for epoch in range(args.start_epoch, args.epochs):
if epoch in [args.epochs * 0.5, args.epochs * 0.75]:
for param_group in optimizer.param_groups:
param_group['lr'] *= 0.1
avg_loss, train_acc = train(model,
optimizer,
epoch=epoch,
device=device,
train_loader=train_loader,
valid=args.valid,
valid_size=valid_size,
log_interval=args.log_interval)
if args.valid:
prec1 = valid(model, device, valid_loader, valid_size=valid_size)
test(model, device, test_loader)
else:
prec1 = test(model, device, test_loader)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
'cfg': model.cfg
},
is_best,
filepath=args.save)
linear_lr=0.001)
optimizer = optim.SGD(optim_set, momentum=args.momentum)
for epoch in range(args.start_epoch, args.epochs):
avg_loss, train_acc = train(newmodel,
optimizer,
epoch=epoch,
device=device,
train_loader=train_loader,
valid=args.valid,
valid_len=valid_len,
log_interval=args.log_interval)
# test(newmodel, device, test_loader)
if args.valid:
prec = valid(newmodel,
device,
valid_loader,
valid_len=valid_len)
else:
prec = train_acc
if prec > args.pruneT:
break
prec_list.append(prec)
drop_threshold = args.dropT
print('(last_prec1 - prec): %.3f' % (last_prec1 - prec))
if prec > args.pruneT and (last_prec1 - prec) <= drop_threshold:
last_prec1 = prec
success_flag_list.append(1)
del model
model = newmodel
prune_success = True
linear_lr=0.001)
optimizer = optim.SGD(optim_set, momentum=args.momentum)
for epoch in range(args.start_epoch, args.epochs):
avg_loss, train_acc = train(newmodel,
optimizer,
epoch=epoch,
device=device,
train_loader=train_loader,
valid=args.valid,
valid_size=valid_size,
log_interval=args.log_interval)
# test(newmodel, device, test_loader)
if args.valid:
prec = valid(newmodel,
device,
valid_loader,
valid_size=valid_size)
else:
prec = train_acc
if prec > args.pruneT:
break
prec_list.append(prec)
drop_threshold = args.dropT
print('(last_prec1 - prec): %.3f' % (last_prec1 - prec))
if prec > args.pruneT and (last_prec1 - prec) <= drop_threshold:
last_prec1 = prec
success_flag_list.append(1)
del model
model = newmodel
prune_success = True
help="Select dataset to train. expected value: [cifar10, imagenet, lsun, celeba]")
parser.add_argument("--batch_size", type=int, default=64, \
help="Number of data to learn at a time")
parser.add_argument("--excute_mode", type=str, default="valid", \
help="Select execute mode. expected value: [train, valid, sampling]")
parser.add_argument("--recall_iter", type=int, default=100000, \
help="Select the number of recall iter. 0 to start new iter.")
arguments = parser.parse_args()
max_iter = arguments.recall_iter
min_loss = float('inf')
loss_list = [0, 0, 0]
right_valid_loss = train.valid(arguments)
loss_list[2] = arguments.recall_iter
arguments.recall_iter = 0
left_valid_loss = train.valid(arguments)
loss_list[0] = arguments.recall_iter
arguments.recall_iter = max_iter//2
mid_valid_loss = train.valid(arguments)
loss_list[1] = arguments.recall_iter
while 1:
min_loss = min(loss_list)
min_iter = loss_list.index(min_loss)
load_backboe_weight=False)
# load weight
model_file = args.weight_path
chkpt = torch.load(model_file, map_location='cpu') # load checkpoint
model.load_state_dict(chkpt['model'])
print('load weights from ' + model_file)
model = model.to(device)
if args.distributed:
model = nn.parallel.DistributedDataParallel(
model,
device_ids=[args.local_rank],
output_device=args.local_rank,
broadcast_buffers=False,
)
valid_loader = DataLoader(
valid_set,
batch_size=args.batch,
sampler=data_sampler(valid_set,
shuffle=False,
distributed=args.distributed),
num_workers=args.num_workers,
collate_fn=collate_fn(args),
)
predictions = valid(args, 0, valid_loader, valid_set, model, device)
#save_predictions_to_images(valid_set, predictions)
best_prec1 = 0.
for epoch in range(args.start_epoch, args.epochs):
if epoch in [args.epochs * 0.5, args.epochs * 0.75]:
for param_group in optimizer.param_groups:
param_group['lr'] *= 0.1
avg_loss, train_acc = train(model,
optimizer,
epoch=epoch,
device=device,
train_loader=train_loader,
valid=args.valid,
valid_len=valid_len,
log_interval=args.log_interval)
if args.valid:
prec1 = valid(model, device, valid_loader, valid_len=valid_len)
test(model, device, test_loader)
else:
prec1 = test(model, device, test_loader)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
'cfg': model.cfg
},
is_best,
filepath=args.save)
def main():
save_dir = os.path.join('output',
"{model}").format(model=str(FLAGS.model_num))
file_exists = os.path.isfile(
_get_model_file(save_dir, str(FLAGS.model_num)))
results_path = os.path.join(save_dir, 'results')
if file_exists and not FLAGS.overwrite:
print("Model file of \"%s\" already exists. Skipping training..." %
FLAGS.model_num)
results = aggregate_json_results(results_path)
return results
else:
if file_exists:
print("Model file exists, but will be overwritten...")
transform_aug_train = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize(FLAGS.resize_val),
#transforms.RandomCrop(FLAGS.resize_val, padding=4),
transforms.RandomHorizontalFlip(p=0.5),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
transform_aug_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
transform_train = transform_aug_train
transform_test = transform_aug_test
cif10 = CIFAR10_loader(root='./data/cifar-10-batches-py/',
transform=transform_train,
train=True)
train_idx, val_idx = train_test_split(np.arange(cif10.labels.shape[1]),
train_size=0.8,
shuffle=True,
stratify=cif10.labels.reshape(
(-1, 1)))
cif10_train = Subset(cif10, train_idx)
cif10_val = Subset(cif10, val_idx)
mytrainloader = DataLoader(cif10_train,
batch_size=FLAGS.batch_size,
shuffle=True,
num_workers=0)
myvalloader = DataLoader(cif10_val,
batch_size=FLAGS.batch_size,
shuffle=False)
cif10_test = CIFAR10_loader(root='./data/cifar-10-batches-py/',
transform=transform_test,
train=False)
mytestloader = DataLoader(cif10_test,
batch_size=FLAGS.batch_size,
shuffle=False,
num_workers=0)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
convnet = ConvNet3()
experiment_timer = time.time()
results = custom_train(convnet,
mytrainloader,
valloader=myvalloader,
testloader=mytestloader,
dir=save_dir,
model_num=FLAGS.model_num,
epochs=FLAGS.epochs,
device=device,
lr=FLAGS.learning_rate)
results['final_time'] = time.time() - experiment_timer
flags_dict = vars(FLAGS)
save_dict(os.path.join(save_dir, 'flags.json'), flags_dict)
train_results = os.path.join(results_path, "train.json")
save_dict(train_results, results)
dict1 = valid(convnet, mytestloader, device=device)
dict2 = valid_class(convnet,
mytestloader, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
device=device)
final_dict = {**dict1, **dict2}
acc_results = os.path.join(results_path, "test.json")
save_dict(acc_results, final_dict)
plt.plot(results['train_losses'])
plt.title('Losses per epoch')
plt.savefig(os.path.join(results_path, 'loss.png'))
plt.close()
plt.plot(results['train_scores'])
plt.title('Accuracy per epoch')
plt.savefig(os.path.join(results_path, 'acc.png'))
plt.close()
def main(**kwargs):
opt = Config()
for k_, v_ in kwargs.items():
setattr(opt, k_, v_)
if opt.vis:
vis = Visualizer(opt.env)
else:
vis = None
init_loss_file(opt)
if opt.data_source == "statics":
opt.fold_dataset = True
train_path, valid_path, test_path = init_file_path(opt)
print(opt.fold_dataset)
# random_state = random.randint(1, 50)
# print("random_state:", random_state)
train_dataset = KTData(train_path, fold_dataset=opt.fold_dataset, q_numbers=opt.output_dim, opt='None')
valid_dataset = KTData(valid_path, fold_dataset=opt.fold_dataset, q_numbers=opt.output_dim, opt='None')
test_dataset = KTData(test_path, fold_dataset=opt.fold_dataset, q_numbers=opt.output_dim, opt='None')
# print(train_path, valid_path, test_path)
print(len(train_dataset), len(valid_dataset), len(test_dataset))
train_loader = DataLoader(train_dataset, batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers,
drop_last=True, collate_fn=myutils.collate_fn)
valid_loader = DataLoader(valid_dataset, batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers,
drop_last=True, collate_fn=myutils.collate_fn)
test_loader = DataLoader(test_dataset, batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers,
drop_last=True, collate_fn=myutils.collate_fn)
if opt.model_name == "CNN":
model = CNN(opt.input_dim, opt.embed_dim, opt.hidden_dim, opt.num_layers, opt.output_dim, opt.batch_size, opt.device)
elif opt.model_name == "CNN_3D":
model = CNN_3D(opt.k_frames, opt.input_dim, opt.embed_dim, opt.hidden_dim, opt.num_layers, opt.output_dim, opt.batch_size, opt.device)
else:
model = RNN_DKT(opt.input_dim, opt.embed_dim, opt.hidden_dim, opt.num_layers, opt.output_dim, opt.batch_size, opt.device)
lr = opt.lr
last_epoch = -1
optimizer = torch.optim.Adam(
params=model.parameters(),
lr=lr,
weight_decay=opt.weight_decay,
betas=(0.9, 0.99)
)
if opt.model_path:
map_location = lambda storage, loc: storage
checkpoint = torch.load(opt.model_path, map_location=map_location)
model.load_state_dict(checkpoint["model"])
last_epoch = checkpoint["epoch"]
lr = checkpoint["lr"]
optimizer.load_state_dict(checkpoint["optimizer"])
model = model.to(opt.device)
loss_result = {}
auc_resilt = {}
best_test_auc = 0
# START TRAIN
for epoch in range(opt.max_epoch):
if epoch < last_epoch:
continue
if opt.model_name == "CNN_3D":
train_loss_meter, train_auc_meter, train_loss_list = train.train_3d(opt, vis, model, train_loader, epoch, lr,
optimizer)
val_loss_meter, val_auc_meter, val_loss_list = train.valid_3d(opt, vis, model, valid_loader, epoch)
test_loss_meter, test_auc_meter, test_loss_list = test.test_3d(opt, vis, model, test_loader, epoch)
else:
train_loss_meter, train_auc_meter, train_loss_list = train.train(opt, vis, model, train_loader, epoch, lr, optimizer)
val_loss_meter, val_auc_meter, val_loss_list = train.valid(opt, vis, model, valid_loader, epoch)
test_loss_meter, test_auc_meter, test_loss_list = test.test(opt, vis, model, test_loader, epoch)
loss_result["train_loss"] = train_loss_meter.value()[0]
auc_resilt["train_auc"] = train_auc_meter.value()[0]
loss_result["val_loss"] = val_loss_meter.value()[0]
auc_resilt["val_auc"] = val_auc_meter.value()[0]
loss_result["test_loss"] = test_loss_meter.value()[0]
auc_resilt["test_auc"] = test_auc_meter.value()[0]
for k, v in loss_result.items():
print("epoch:{epoch}, {k}:{v:.5f}".format(epoch=epoch, k=k, v=v))
if opt.vis:
vis.line(X=np.array([epoch]), Y=np.array([v]),
win="loss",
opts=dict(title="loss", showlegend=True),
name = k,
update='append')
for k, v in auc_resilt.items():
print("epoch:{epoch}, {k}:{v:.5f}".format(epoch=epoch, k=k, v=v))
if opt.vis:
vis.line(X=np.array([epoch]), Y=np.array([v]),
win="auc",
opts=dict(title="auc", showlegend=True),
name = k,
update='append')
best_test_auc = max(best_test_auc, test_auc_meter.value()[0], val_auc_meter.value()[0])
print("best_test_auc is: ", best_test_auc)
# TODO 每个epoch结束后把loss写入文件
myutils.save_loss_file(opt, epoch, train_loss_list, val_loss_list, test_loss_list)
# TODO 每save_every个epoch结束后保存模型参数+optimizer参数
if epoch % opt.save_every == 0:
myutils.save_model_weight(opt, model, optimizer, epoch, lr)
# TODO 做lr_decay
lr = myutils.adjust_lr(opt, optimizer, epoch, train_loss_meter.value()[0])
# TODO 结束的时候保存final模型参数
myutils.save_model_weight(opt, model, optimizer, epoch, lr, is_final=True)
def main():
file_name = "./flood_graph/150_250/128/500/ji_sort/1_conf/sample-wised/default/{}/".format(
args.b)
start = time.time()
# set GPU ID
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
cudnn.benchmark = True
# check save path
save_path = file_name
# save_path = args.save_path
if not os.path.exists(save_path):
os.makedirs(save_path)
# make dataloader
if args.valid == True:
train_loader, valid_loader, test_loader, test_onehot, test_label = dataset.get_valid_loader(
args.data, args.data_path, args.batch_size)
else:
train_loader, train_onehot, train_label, test_loader, test_onehot, test_label = dataset.get_loader(
args.data, args.data_path, args.batch_size)
# set num_class
if args.data == 'cifar100':
num_class = 100
else:
num_class = 10
# set num_classes
model_dict = {
"num_classes": num_class,
}
# set model
if args.model == 'res':
model = resnet.resnet110(**model_dict).cuda()
elif args.model == 'dense':
model = densenet_BC.DenseNet3(depth=100,
num_classes=num_class,
growth_rate=12,
reduction=0.5,
bottleneck=True,
dropRate=0.0).cuda()
elif args.model == 'vgg':
model = vgg.vgg16(**model_dict).cuda()
# set criterion
if args.loss == 'MS':
cls_criterion = losses.MultiSimilarityLoss().cuda()
elif args.loss == 'Contrastive':
cls_criterion = losses.ContrastiveLoss().cuda()
elif args.loss == 'Triplet':
cls_criterion = losses.TripletLoss().cuda()
elif args.loss == 'NPair':
cls_criterion = losses.NPairLoss().cuda()
elif args.loss == 'Focal':
cls_criterion = losses.FocalLoss(gamma=3.0).cuda()
else:
if args.mode == 0:
cls_criterion = nn.CrossEntropyLoss().cuda()
else:
cls_criterion = nn.CrossEntropyLoss(reduction="none").cuda()
ranking_criterion = nn.MarginRankingLoss(margin=0.0).cuda()
# set optimizer (default:sgd)
optimizer = optim.SGD(
model.parameters(),
lr=0.1,
momentum=0.9,
weight_decay=5e-4,
# weight_decay=0.0001,
nesterov=False)
# optimizer = optim.SGD(model.parameters(),
# lr=float(args.lr),
# momentum=0.9,
# weight_decay=args.weight_decay,
# nesterov=False)
# set scheduler
# scheduler = MultiStepLR(optimizer,
# milestones=[500, 750],
# gamma=0.1)
scheduler = MultiStepLR(optimizer, milestones=[150, 250], gamma=0.1)
# scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=args.lr_decay_step, gamma=args.lr_decay_gamma)
# make logger
train_logger = utils.Logger(os.path.join(save_path, 'train.log'))
result_logger = utils.Logger(os.path.join(save_path, 'result.log'))
# make History Class
correctness_history = crl_utils.History(len(train_loader.dataset))
## define matrix
if args.data == 'cifar':
matrix_idx_confidence = [[_] for _ in range(50000)]
matrix_idx_iscorrect = [[_] for _ in range(50000)]
else:
matrix_idx_confidence = [[_] for _ in range(73257)]
matrix_idx_iscorrect = [[_] for _ in range(73257)]
# write csv
#'''
import csv
f = open('{}/logs_{}_{}.txt'.format(file_name, args.b, args.epochs),
'w',
newline='')
f.write("location = {}\n\n".format(file_name) + str(args))
f0 = open('{}/Test_confidence_{}_{}.csv'.format(file_name, args.b,
args.epochs),
'w',
newline='')
# f0 = open('./baseline_graph/150_250/128/500/Test_confidence_{}_{}.csv'.format(args.b, args.epochs), 'w', newline='')
# f0 = open('./CRL_graph/150_250/Test_confidence_{}_{}.csv'.format(args.b, args.epochs), 'w', newline='')
wr_conf_test = csv.writer(f0)
header = [_ for _ in range(args.epochs + 1)]
header[0] = 'Epoch'
wr_conf_test.writerows([header])
f1 = open('{}/Train_confidence_{}_{}.csv'.format(file_name, args.b,
args.epochs),
'w',
newline='')
# f1 = open('./baseline_graph/150_250/128/500/Train_confidence_{}_{}.csv'.format(args.b, args.epochs), 'w', newline='')
# f1 = open('./CRL_graph/150_250/Train_confidence_{}_{}.csv'.format(args.b, args.epochs), 'w', newline='')
wr = csv.writer(f1)
header = [_ for _ in range(args.epochs + 1)]
header[0] = 'Epoch'
wr.writerows([header])
f2 = open('{}/Train_Flood_{}_{}_{}.csv'.format(file_name, args.data,
args.b, args.epochs),
'w',
newline='')
# f2 = open('./baseline_graph/150_250/128/500/Train_Base_{}_{}_{}.csv'.format(args.data, args.b, args.epochs), 'w', newline='')
# f2 = open('./CRL_graph/150_250/Train_Flood_{}_{}_{}.csv'.format(args.data, args.b, args.epochs), 'w', newline='')
wr_train = csv.writer(f2)
header = [_ for _ in range(args.epochs + 1)]
header[0] = 'Epoch'
wr_train.writerows([header])
f3 = open('{}/Test_Flood_{}_{}_{}.csv'.format(file_name, args.data, args.b,
args.epochs),
'w',
newline='')
# f3 = open('./baseline_graph/150_250/128/500/Test_Base_{}_{}_{}.csv'.format(args.data, args.b, args.epochs), 'w', newline='')
# f3 = open('./CRL_graph/150_250/Test_Flood_{}_{}_{}.csv'.format(args.data, args.b, args.epochs), 'w', newline='')
wr_test = csv.writer(f3)
header = [_ for _ in range(args.epochs + 1)]
header[0] = 'Epoch'
wr_test.writerows([header])
#'''
# start Train
best_valid_acc = 0
test_ece_report = []
test_acc_report = []
test_nll_report = []
test_over_con99_report = []
test_e99_report = []
test_cls_loss_report = []
train_ece_report = []
train_acc_report = []
train_nll_report = []
train_over_con99_report = []
train_e99_report = []
train_cls_loss_report = []
train_rank_loss_report = []
train_total_loss_report = []
for epoch in range(1, args.epochs + 1):
scheduler.step()
matrix_idx_confidence, matrix_idx_iscorrect, idx, iscorrect, confidence, target, cls_loss_tr, rank_loss_tr, batch_correctness, total_confidence, total_correctness = \
train.train(matrix_idx_confidence, matrix_idx_iscorrect, train_loader,
model,
wr,
cls_criterion,
ranking_criterion,
optimizer,
epoch,
correctness_history,
train_logger,
args)
if args.rank_weight != 0.0:
print("RANK ", rank_loss_tr)
total_loss_tr = cls_loss_tr + rank_loss_tr
if args.valid == True:
idx, iscorrect, confidence, target, cls_loss_val, acc = train.valid(
valid_loader, model, cls_criterion, ranking_criterion,
optimizer, epoch, correctness_history, train_logger, args)
if acc > best_valid_acc:
best_valid_acc = acc
print("*** Update Best Acc ***")
# save model
if epoch == args.epochs:
torch.save(model.state_dict(),
os.path.join(save_path, 'model.pth'))
print("########### Train ###########")
acc_tr, aurc_tr, eaurc_tr, aupr_tr, fpr_tr, ece_tr, nll_tr, brier_tr, E99_tr, over_99_tr, cls_loss_tr = metrics.calc_metrics(
train_loader, train_label, train_onehot, model, cls_criterion,
args)
if args.sort == True and epoch == 260:
#if args.sort == True:
train_loader = dataset.sort_get_loader(
args.data, args.data_path, args.batch_size, idx,
np.array(target), iscorrect,
batch_correctness, total_confidence, total_correctness,
np.array(confidence), epoch, args)
train_acc_report.append(acc_tr)
train_nll_report.append(nll_tr * 10)
train_ece_report.append(ece_tr)
train_over_con99_report.append(over_99_tr)
train_e99_report.append(E99_tr)
train_cls_loss_report.append(cls_loss_tr)
if args.rank_weight != 0.0:
train_total_loss_report.append(total_loss_tr)
train_rank_loss_report.append(rank_loss_tr)
print("CLS ", cls_loss_tr)
# finish train
print("########### Test ###########")
# calc measure
acc_te, aurc_te, eaurc_te, aupr_te, fpr_te, ece_te, nll_te, brier_te, E99_te, over_99_te, cls_loss_te = metrics.calc_metrics(
test_loader, test_label, test_onehot, model, cls_criterion, args)
test_ece_report.append(ece_te)
test_acc_report.append(acc_te)
test_nll_report.append(nll_te * 10)
test_over_con99_report.append(over_99_te)
test_e99_report.append(E99_te)
test_cls_loss_report.append(cls_loss_te)
print("CLS ", cls_loss_te)
print("############################")
# for idx in matrix_idx_confidence:
# wr.writerow(idx)
#'''
# draw graph
df = pd.DataFrame()
df['epoch'] = [i for i in range(1, args.epochs + 1)]
df['test_ece'] = test_ece_report
df['train_ece'] = train_ece_report
fig_loss = plt.figure(figsize=(35, 35))
fig_loss.set_facecolor('white')
ax = fig_loss.add_subplot()
ax.plot(df['epoch'],
df['test_ece'],
df['epoch'],
df['train_ece'],
linewidth=10)
ax.legend(['Test', 'Train'], loc=2, prop={'size': 60})
plt.title('[FL] ECE per epoch', fontsize=80)
# plt.title('[BASE] ECE per epoch', fontsize=80)
# plt.title('[CRL] ECE per epoch', fontsize=80)
plt.xlabel('Epoch', fontsize=70)
plt.ylabel('ECE', fontsize=70)
plt.ylim([0, 1])
plt.setp(ax.get_xticklabels(), fontsize=30)
plt.setp(ax.get_yticklabels(), fontsize=30)
plt.savefig('{}/{}_{}_ECE_lr_{}.png'.format(file_name, args.model, args.b,
args.epochs))
# plt.savefig('./baseline_graph/150_250/128/500/{}_{}_ECE_lr_{}.png'.format(args.model, args.b, args.epochs))
# plt.savefig('./CRL_graph/150_250/{}_{}_ECE_lr_{}.png'.format(args.model, args.b, args.epochs))
df2 = pd.DataFrame()
df2['epoch'] = [i for i in range(1, args.epochs + 1)]
df2['test_acc'] = test_acc_report
df2['train_acc'] = train_acc_report
fig_acc = plt.figure(figsize=(35, 35))
fig_acc.set_facecolor('white')
ax = fig_acc.add_subplot()
ax.plot(df2['epoch'],
df2['test_acc'],
df2['epoch'],
df2['train_acc'],
linewidth=10)
ax.legend(['Test', 'Train'], loc=2, prop={'size': 60})
plt.title('[FL] Accuracy per epoch', fontsize=80)
# plt.title('[BASE] Accuracy per epoch', fontsize=80)
# plt.title('[CRL] Accuracy per epoch', fontsize=80)
plt.xlabel('Epoch', fontsize=70)
plt.ylabel('Accuracy', fontsize=70)
plt.ylim([0, 100])
plt.setp(ax.get_xticklabels(), fontsize=30)
plt.setp(ax.get_yticklabels(), fontsize=30)
plt.savefig('{}/{}_{}_acc_lr_{}.png'.format(file_name, args.model, args.b,
args.epochs))
# plt.savefig('./baseline_graph/150_250/128/500/{}_{}_acc_lr_{}.png'.format(args.model, args.b, args.epochs))
# plt.savefig('./CRL_graph/150_250/{}_{}_acc_lr_{}.png'.format(args.model, args.b, args.epochs))
df3 = pd.DataFrame()
df3['epoch'] = [i for i in range(1, args.epochs + 1)]
df3['test_nll'] = test_nll_report
df3['train_nll'] = train_nll_report
fig_acc = plt.figure(figsize=(35, 35))
fig_acc.set_facecolor('white')
ax = fig_acc.add_subplot()
ax.plot(df3['epoch'],
df3['test_nll'],
df3['epoch'],
df3['train_nll'],
linewidth=10)
ax.legend(['Test', 'Train'], loc=2, prop={'size': 60})
plt.title('[FL] NLL per epoch', fontsize=80)
# plt.title('[BASE] NLL per epoch', fontsize=80)
# plt.title('[CRL] NLL per epoch', fontsize=80)
plt.xlabel('Epoch', fontsize=70)
plt.ylabel('NLL', fontsize=70)
plt.ylim([0, 45])
plt.setp(ax.get_xticklabels(), fontsize=30)
plt.setp(ax.get_yticklabels(), fontsize=30)
plt.savefig('{}/{}_{}_nll_lr_{}.png'.format(file_name, args.model, args.b,
args.epochs))
# plt.savefig('./baseline_graph/150_250/128/500/{}_{}_nll_lr_{}.png'.format(args.model, args.b, args.epochs))
# plt.savefig('./CRL_graph/150_250/{}_{}_nll_lr_{}.png'.format(args.model, args.b, args.epochs))
df4 = pd.DataFrame()
df4['epoch'] = [i for i in range(1, args.epochs + 1)]
df4['test_over_con99'] = test_over_con99_report
df4['train_over_con99'] = train_over_con99_report
fig_acc = plt.figure(figsize=(35, 35))
fig_acc.set_facecolor('white')
ax = fig_acc.add_subplot()
ax.plot(df4['epoch'],
df4['test_over_con99'],
df4['epoch'],
df4['train_over_con99'],
linewidth=10)
ax.legend(['Test', 'Train'], loc=2, prop={'size': 60})
plt.title('[FL] Over conf99 per epoch', fontsize=80)
# plt.title('[BASE] Over conf99 per epoch', fontsize=80)
# plt.title('[CRL] Over conf99 per epoch', fontsize=80)
plt.xlabel('Epoch', fontsize=70)
plt.ylabel('Over con99', fontsize=70)
if args.data == 'cifar10' or args.data == 'cifar100':
plt.ylim([0, 50000])
else:
plt.ylim([0, 73257])
plt.setp(ax.get_xticklabels(), fontsize=30)
plt.setp(ax.get_yticklabels(), fontsize=30)
plt.savefig('{}/{}_{}_over_conf99_lr_{}.png'.format(
file_name, args.model, args.b, args.epochs))
# plt.savefig('./baseline_graph/150_250/128/500/{}_{}_over_conf99_lr_{}.png'.format(args.model, args.b, args.epochs))
# plt.savefig('./CRL_graph/150_250/{}_{}_over_conf99_lr_{}.png'.format(args.model, args.b, args.epochs))
df5 = pd.DataFrame()
df5['epoch'] = [i for i in range(1, args.epochs + 1)]
df5['test_e99'] = test_e99_report
df5['train_e99'] = train_e99_report
fig_acc = plt.figure(figsize=(35, 35))
fig_acc.set_facecolor('white')
ax = fig_acc.add_subplot()
ax.plot(df5['epoch'],
df5['test_e99'],
df5['epoch'],
df5['train_e99'],
linewidth=10)
ax.legend(['Test', 'Train'], loc=2, prop={'size': 60})
plt.title('[FL] E99 per epoch', fontsize=80)
# plt.title('[BASE] E99 per epoch', fontsize=80)
# plt.title('[CRL] E99 per epoch', fontsize=80)
plt.xlabel('Epoch', fontsize=70)
plt.ylabel('E99', fontsize=70)
plt.ylim([0, 0.2])
plt.setp(ax.get_xticklabels(), fontsize=30)
plt.setp(ax.get_yticklabels(), fontsize=30)
plt.savefig('{}/{}_{}_E99_flood_lr_{}.png'.format(file_name, args.model,
args.b, args.epochs))
# plt.savefig('./baseline_graph/150_250/128/500/{}_{}_E99_flood_lr_{}.png'.format(args.model, args.b, args.epochs))
# plt.savefig('./CRL_graph/150_250/{}_{}_E99_flood_lr_{}.png'.format(args.model, args.b, args.epochs))
df5 = pd.DataFrame()
df5['epoch'] = [i for i in range(1, args.epochs + 1)]
df5['test_cls_loss'] = test_cls_loss_report
df5['train_cls_loss'] = train_cls_loss_report
fig_acc = plt.figure(figsize=(35, 35))
fig_acc.set_facecolor('white')
ax = fig_acc.add_subplot()
ax.plot(df5['epoch'],
df5['test_cls_loss'],
df5['epoch'],
df5['train_cls_loss'],
linewidth=10)
ax.legend(['Test', 'Train'], loc=2, prop={'size': 60})
plt.title('[FL] CLS_loss per epoch', fontsize=80)
# plt.title('[BASE] CLS_loss per epoch', fontsize=80)
# plt.title('[CRL] CLS_loss per epoch', fontsize=80)
plt.xlabel('Epoch', fontsize=70)
plt.ylabel('Loss', fontsize=70)
plt.ylim([0, 5])
plt.setp(ax.get_xticklabels(), fontsize=30)
plt.setp(ax.get_yticklabels(), fontsize=30)
plt.savefig('{}/{}_{}_cls_loss_flood_lr_{}.png'.format(
file_name, args.model, args.b, args.epochs))
# plt.savefig('./baseline_graph/150_250/128/500/{}_{}_cls_loss_flood_lr_{}.png'.format(args.model, args.b, args.epochs))
# plt.savefig('./CRL_graph/150_250/{}_{}_cls_loss_flood_lr_{}.png'.format(args.model, args.b, args.epochs))
if args.rank_weight != 0.0:
df6 = pd.DataFrame()
df6['epoch'] = [i for i in range(1, args.epochs + 1)]
df6['train_cls_loss'] = train_cls_loss_report
df6['train_rank_loss'] = train_rank_loss_report
df6['train_total_loss'] = train_total_loss_report
fig_acc = plt.figure(figsize=(35, 35))
fig_acc.set_facecolor('white')
ax = fig_acc.add_subplot()
ax.plot(df6['epoch'],
df6['train_cls_loss'],
df6['epoch'],
df6['train_rank_loss'],
df6['epoch'],
df6['train_total_loss'],
linewidth=10)
ax.legend(['CLS', 'Rank', 'Total'], loc=2, prop={'size': 60})
plt.title('[FL] CLS_loss per epoch', fontsize=80)
plt.xlabel('Epoch', fontsize=70)
plt.ylabel('Loss', fontsize=70)
# plt.ylim([0, 5])
plt.setp(ax.get_xticklabels(), fontsize=30)
plt.setp(ax.get_yticklabels(), fontsize=30)
plt.savefig(
'./CRL_graph/150_250/{}_{}_cls_loss_flood_lr_{}.png'.format(
args.model, args.b, args.epochs))
test_acc_report.insert(0, 'ACC')
test_ece_report.insert(0, 'ECE')
test_nll_report.insert(0, 'NLL')
test_over_con99_report.insert(0, 'Over_conf99')
test_e99_report.insert(0, 'E99')
test_cls_loss_report.insert(0, 'CLS')
wr_test.writerow(test_acc_report)
wr_test.writerow(test_ece_report)
wr_test.writerow(test_nll_report)
wr_test.writerow(test_over_con99_report)
wr_test.writerow(test_e99_report)
wr_test.writerow(test_cls_loss_report)
train_acc_report.insert(0, 'ACC')
train_ece_report.insert(0, 'ECE')
train_nll_report.insert(0, 'NLL')
train_over_con99_report.insert(0, 'Over_conf99')
train_e99_report.insert(0, 'E99')
train_cls_loss_report.insert(0, 'CLS')
wr_train.writerow(train_acc_report)
wr_train.writerow(train_ece_report)
wr_train.writerow(train_nll_report)
wr_train.writerow(train_over_con99_report)
wr_train.writerow(train_e99_report)
wr_train.writerow(train_cls_loss_report)
if args.rank_weight != 0.0:
train_rank_loss_report.insert(0, 'Rank')
train_total_loss_report.insert(0, 'Total')
wr_train.writerow(train_rank_loss_report)
wr_train.writerow(train_total_loss_report)
#'''
# result write
result_logger.write([
acc_te, aurc_te * 1000, eaurc_te * 1000, aupr_te * 100, fpr_te * 100,
ece_te * 100, nll_te * 10, brier_te * 100, E99_te * 100
])
if args.valid == True:
print("Best Valid Acc : {}".format(acc))
print("Flood Level: {}".format(args.b))
print("Sort : {}".format(args.sort))
print("Sort Mode : {}".format(args.sort_mode))
print("TIME : ", time.time() - start)
