def test(test_loader, net, criterion, gpu):
print('\nTest_net...')
start = time.time()
net.eval()
losses = AverageMeter()
total_correct = 0
mtr = meter.AUCMeter()
with torch.no_grad():
for i, (imgs, marks, labels) in enumerate(test_loader):
if gpu:
imgs = imgs.cuda()
labels = labels.cuda()
outputs = net(imgs) # 网络输出
# _, preds = torch.max(outputs, 1)
loss = criterion(outputs, labels)
losses.update(loss.item(), imgs.size(0))
smax_probs = nn.Softmax(dim=1)(outputs) # 每个类的概率
mtr.add(smax_probs.data[:, 1], labels.cpu()) # 计算AUC
pred = torch.max(smax_probs, 1)[1] # 预测类别标签
correct = pred.eq(labels.view_as(pred)).sum()
total_correct += correct
accuracy = 100. * float(total_correct) / float(len(test_loader.dataset))
mtr_values = mtr.value()
end = time.time()
print(
f'Time:{int((end - start)//60)}:{int((end - start)%60)}\nTest set: Average loss: {losses.avg:.4f}, Accuracy: {total_correct}/{len(test_loader.dataset)} ({accuracy:.2f}%), AUC: {mtr_values[0]:.4f}\n'
)
return losses.avg, accuracy, mtr_values
def metrics(self, testloader, accuracy = True, auc = False, conf_matrix = False):
am = meter.AUCMeter()
cm = meter.ConfusionMeter(2)
correct = 0
total = 0
for data in testloader:
x,y = data
y_ = self.model(Variable(x))
_, predicted = torch.max(y_.data, 1)
cm.add(y_.data, y)
am.add(y_.data[:,1].clone(),y)
total += y.size(0)
correct += (predicted == y).sum()
print (correct, total)
if accuracy:
print("Accuracy for the model is", round(correct/float(total)*100, 4), correct, "/", total)
if auc:
print("Area under ROC curve for the given model is", round(am.value()[0],4))
if conf_matrix:
print ("Confusion Matrix for the given model is\n", cm.value())
def val(model, dataloader):
model.eval()
confusion_matrix = meter.ConfusionMeter(opt.num_class)
auc_meter = meter.AUCMeter()
for ii, data in enumerate(dataloader):
input, label = data
val_input = Variable(input)
val_label = Variable(label.long())
if opt.use_gpu:
val_input = val_input.cuda()
val_label = val_label.cuda()
with t.no_grad():
score = model(val_input)
confusion_matrix.add(score.data.squeeze(), label)
if opt.num_class == 2:
auc_meter.add(score.data[:, 1], label)
model.train()
cm_value = confusion_matrix.value()
accuracy = 1. * np.trace(cm_value) / (cm_value.sum())
if opt.num_class == 2:
return confusion_matrix, accuracy, auc_meter.value()[0]
else:
return confusion_matrix, accuracy
def test(model, dataloader, num_workers, batch_size, resultpath):
print("num test = {}".format(len(dataloader.dataset)))
"""
测试指标：
1、 准确率(Accuracy): 模型预测正确样本数占总样本数的比例。test_acc
2、 各个类的精度: 模型对各个类别的预测准确率。
3、 AUC
4、 混淆矩阵: 用于计算各种指标（包括灵敏性，特异性等）
"""
# 整个测试数据集的准确率
test_acc = meter.ClassErrorMeter(topk=[1], accuracy=True)
# 每一类的精度
test_ap = meter.APMeter()
# AUC指标，AUC要求输入样本预测为正例的概率
"""根据我的数据集文件命名，0表示阴性，1表示阳性（即1表示正例）"""
test_auc = meter.AUCMeter()
# 混淆矩阵
test_conf = meter.ConfusionMeter(k=2, normalized=False)
result_writer = ResultsWriter(str(resultpath), overwrite=False)
with torch.no_grad():
for inputs, labels in tqdm(dataloader, desc="Test"):
# inputs[B,C,H,W]
inputs = inputs.cuda() if torch.cuda.is_available() else inputs
# labes[B,numclasses]
labels = labels.cuda() if torch.cuda.is_available() else labels
# outputs[B,numclasses]
outputs = model(inputs)
# 计算指标
pred_proc = F.softmax(outputs.detach(), dim=1)
test_acc.add(pred_proc, labels.detach())
test_ap.add(pred_proc, labels.detach())
# 取出output第1列的数，正例即1（患病）的概率
test.auc.add(pred_proc[:1], labels.detach())
test_conf.add(pred_proc, labels.detach())
# 记录保存, 便于evaluate.py计算和画图一些结果
result_writer.update(
"test", {
"acc": test_acc.value(),
"ap": test_ap.value(),
"test_auc": test_auc.value()[0],
"test_tpr": test_auc.value()[1],
"test_fpr": test_auc.value()[2],
"test_conf": test_conf.value()
})
return test_acc, test_ap, test_auc
def __init__(self, name=None, n_classes=2):
self.name = name
self.n_classes = n_classes
self.path = os.path.join('log', name)
self.conf_mtr = meter.ConfusionMeter(n_classes)
self.auc_mtr = meter.AUCMeter()
self.err_mtr = meter.ClassErrorMeter(topk=[1], accuracy=True)
saveMkdir(self.path)
self.fp = open(os.path.join(self.path, 'res.log'), 'w')
self.y_scores = np.array([], dtype=np.float32).reshape(0, 1)
self.y_true = np.array([], dtype=np.float32).reshape(0, 1)
def batch_handler(mode, epoch_metrics, dict_, model, criterion, optimizer,
logger, args):
if len(epoch_metrics) == 0: # first time metric for epoch, initialization
epoch_metrics = {
"loss": meter.AverageValueMeter(),
"auc": meter.AUCMeter()
}
target = dict_.pop("target")
bs = len(target)
if torch.cuda.is_available():
input_var = {
key: torch.autograd.Variable(value.cuda(async=True),
requires_grad=False)
for key, value in dict_.items()
}
else:
input_var = {
key: torch.autograd.Variable(value, requires_grad=False)
for key, value in dict_.items()
}
if torch.cuda.is_available():
target = target.cuda(async=True)
target_var = torch.autograd.Variable(target, requires_grad=False)
# compute output
output = model(input_var)
# @TODO: BCE loss issue
output = output.squeeze(1)
loss = criterion(output, target_var)
loss_ = float(loss.data.cpu().squeeze().numpy()[0])
epoch_metrics["loss"].add(loss_)
logger.add_scalar("loss", loss_, args.step)
epoch_metrics["auc"].add(output.data, target)
logger.add_scalar("auc", epoch_metrics["auc"].value()[0], args.step)
if mode == "train":
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
epoch_metrics["batch_size"] = bs
return epoch_metrics
def metrics_val(self, testloader):
am = meter.AUCMeter()
cm = meter.ConfusionMeter(2)
correct = 0
total = 0
for data in testloader:
x,y = data
y_ = self.model(Variable(x))
_, predicted = torch.max(y_.data, 1)
cm.add(y_.data, y)
am.add(y_.data[:,1].clone(),y)
total += y.size(0)
correct += (predicted == y).sum()
cor_tot = str(correct) + "/" + str(total)
return round(correct/float(total)*100, 4), cor_tot, round(am.value()[0],4), cm.value()
def testAUCMeter(self):
mtr = meter.AUCMeter()
test_size = 1000
mtr.add(torch.rand(test_size), torch.zeros(test_size))
mtr.add(torch.rand(test_size), torch.Tensor(test_size).fill_(1))
val, tpr, fpr = mtr.value()
self.assertTrue(math.fabs(val - 0.5) < 0.1, msg="AUC Meter fails")
mtr.reset()
mtr.add(torch.Tensor(test_size).fill_(0), torch.zeros(test_size))
mtr.add(torch.Tensor(test_size).fill_(0.1), torch.zeros(test_size))
mtr.add(torch.Tensor(test_size).fill_(0.2), torch.zeros(test_size))
mtr.add(torch.Tensor(test_size).fill_(0.3), torch.zeros(test_size))
mtr.add(torch.Tensor(test_size).fill_(0.4), torch.zeros(test_size))
mtr.add(torch.Tensor(test_size).fill_(1), torch.Tensor(test_size).fill_(1))
val, tpr, fpr = mtr.value()
self.assertEqual(val, 1.0, msg="AUC Meter fails")
def train(train_loader, net, criterion, optimizer, gpu, epoch):
print('\nTrain_net...')
start = time.time()
net.train()
losses = AverageMeter()
total_correct = 0
mtr = meter.AUCMeter()
for i, (imgs, marks, labels) in enumerate(train_loader):
if gpu:
imgs = imgs.cuda()
labels = labels.cuda()
outputs = net(imgs) # 网络输出
# _, preds = torch.max(outputs, 1)
loss = criterion(outputs, labels)
losses.update(loss.item(), imgs.size(0))
smax_probs = nn.Softmax(dim=1)(outputs) # 每个类的概率
mtr.add(smax_probs.data[:, 1], labels.cpu()) # 计算AUC
pred = torch.max(smax_probs, 1)[1] # 预测类别标签
correct = pred.eq(labels.view_as(pred)).sum()
total_correct += correct
optimizer.zero_grad()
loss.backward()
optimizer.step()
print(
f'Train Epoch: {epoch + 1} [{(i + 1) * len(imgs)}/{len(train_loader.dataset)} ({100 * (i + 1) / len(train_loader):.0f}%)]\tLoss: {loss.item():.6f}'
)
end = time.time()
accuracy = 100. * float(total_correct) / float(len(train_loader.dataset))
mtr_values = mtr.value()
print(
f'Time:{int((end - start)//60)}:{int((end - start)%60)}\nTrain set: Average loss: {losses.avg:.4f}, Accuracy: {total_correct}/{len(train_loader.dataset)} ({accuracy:.2f}%), AUC: {mtr_values[0]:.4f}'
)
return losses.avg, accuracy, mtr_values
batch_size=30)
a = custom_model(model, loss_fn)
a.model.apply(init_weights)
a.train(trainloader, testloader, validloader, optimizer, 30, plot=True)
cross_val_models.append(a.model.state_dict())
accuracy, ct, auc, cm = a.metrics_val(testloader)
if accuracy > best_accuracy:
best_accuracy_model = a.model.state_dict()
best_accuracy = accuracy
cross_val_accu.append(accuracy)
print("Accuracy:", accuracy, ct)
print("Average Accuracy:", sum(cross_val_accu) / len(cross_val_accu))
#cross_val_models = pkl.load(open("cross_models_best", "rb"))
am = meter.AUCMeter()
cm = meter.ConfusionMeter(2)
correct = 0
total = 0
Y_ = []
a = custom_model(model, loss_fn)
for data in testloader:
Y_ = []
x, y = data
# a.model.load_state_dict(cross_val_models)
# y_ = a.model(Variable(x))
for mod in cross_val_models:
a.model.load_state_dict(mod)
Y_.append(a.model(Variable(x)))
y_ = Y_[0]
def test_2class(**kwargs):
config.parse(kwargs)
# ============================================= Prepare Data =============================================
test_data = ContextVB_Dataset(config.test_paths, phase='test', num_classes=config.num_classes, useRGB=config.useRGB,
usetrans=config.usetrans, padding=config.padding, balance=config.data_balance)
test_dataloader = DataLoader(test_data, batch_size=config.batch_size, shuffle=False, num_workers=config.num_workers)
test_dist = test_data.dist()
print('Test Image:', test_data.__len__())
# ============================================= Prepare Model ============================================
model = ContextNet(num_classes=config.num_classes)
print(model)
if config.load_model_path:
model.load(config.load_model_path)
print('Model has been loaded!')
else:
print("Don't load model")
if config.use_gpu:
model.cuda()
if config.parallel:
model = torch.nn.DataParallel(model, device_ids=list(range(config.num_of_gpu)))
model.eval()
# =========================================== Prepare Metrics =====================================
test_cm = meter.ConfusionMeter(config.num_classes)
test_AUC = meter.AUCMeter()
softmax = functional.softmax
results = []
y_true, y_scores = [], []
# =========================================== Test ============================================
for i, (image, label, image_path) in tqdm(enumerate(test_dataloader)):
# ******************* prepare input and go through the model *******************
if config.use_gpu:
last_image, cur_image, next_image = image[0].cuda(), image[1].cuda(), image[2].cuda()
last_label, cur_label, next_label = label[0].cuda(), label[1].cuda(), label[2].cuda()
else:
last_image, cur_image, next_image = image[0], image[1], image[2]
last_label, cur_label, next_label = label[0], label[1], label[2]
last_image.requires_grad = False
cur_image.requires_grad = False
next_image.requires_grad = False
last_label.requires_grad = False
cur_label.requires_grad = False
next_label.requires_grad = False
# score = model(last_image, cur_image, next_image)
score, diff1, diff2 = model(last_image, cur_image, next_image)
# *************************** confusion matrix and AUC *************************
test_cm.add(softmax(score, dim=1).data, cur_label.data)
positive_score = np.array([item[1] for item in softmax(score, dim=1).data.cpu().numpy().tolist()])
test_AUC.add(positive_score, cur_label.data) # torchnet计算AUC和ROC
y_true.extend(cur_label.data.cpu().numpy().tolist()) # 用于sklearn计算AUC和ROC
y_scores.extend(positive_score.tolist())
# ******************************** record prediction results ******************************
for l, p, ip in zip(cur_label.detach(), softmax(score, dim=1).detach(), image_path):
if p[1] < 0.5:
results.append((ip, int(l), 0, round(float(p[0]), 4), round(float(p[1]), 4)))
else:
results.append((ip, int(l), 1, round(float(p[0]), 4), round(float(p[1]), 4)))
# ************************** TPR, FPR, AUC ******************************
SKL_FPR, SKL_TPR, SKL_Thresholds = roc_curve(y_true, y_scores)
SKL_AUC = roc_auc_score(np.array(y_true), np.array(y_scores), average='weighted')
TNet_AUC, TNet_TPR, TNet_FPR = test_AUC.value()
# ******************** Best SE, SP, Thresh, Matrix ***********************
best_index = np.argmax(SKL_TPR - SKL_FPR, axis=0)
best_SE, best_SP, best_T = SKL_TPR[best_index], 1 - SKL_FPR[best_index], SKL_Thresholds[best_index]
best_confusion_matrix = [[int(round(test_dist['0'] * best_SP)), int(round(test_dist['0'] * (1 - best_SP)))],
[int(round(test_dist['1'] * (1 - best_SE))), int(round(test_dist['1'] * best_SE))]]
# *********************** accuracy and sensitivity ***********************
test_accuracy = 100. * sum([test_cm.value()[c][c] for c in range(config.num_classes)]) / np.sum(test_cm.value())
test_se = [100. * test_cm.value()[i][i] / np.sum(test_cm.value()[i]) for i in range(config.num_classes)]
# ================================ Save and Print Prediction Results ===========================
if config.result_file:
write_csv(os.path.join('results', config.result_file), tag=['path', 'label', 'predict', 'p1', 'p2'], content=results)
draw_ROC(tpr=SKL_TPR, fpr=SKL_FPR, best_index=best_index, tangent=True, save_path=os.path.join('results', config.load_model_path.split('/')[-1][:-4] + "_ROC.png"))
print('test_acc:', test_accuracy)
print('test_avgse:', round(np.average(test_se), 4), 'train_se0:', round(test_se[0], 4), 'train_se1:', round(test_se[1], 4))
print('SKL_AUC:', SKL_AUC, 'TNet_AUC:', TNet_AUC)
print('Best_SE:', best_SE, 'Best_SP:', best_SP, 'Best_Threshold:', best_T)
print('test_cm:')
print(best_confusion_matrix)
def train(**kwargs):
config.parse(kwargs)
vis = Visualizer(port=2333, env=config.env)
vis.log('Use config:')
for k, v in config.__class__.__dict__.items():
if not k.startswith('__'):
vis.log(f"{k}: {getattr(config, k)}")
# prepare data
train_data = VB_Dataset(config.train_paths,
phase='train',
useRGB=config.useRGB,
usetrans=config.usetrans,
padding=config.padding,
balance=config.data_balance)
val_data = VB_Dataset(config.test_paths,
phase='val',
useRGB=config.useRGB,
usetrans=config.usetrans,
padding=config.padding,
balance=False)
print('Training Images:', train_data.__len__(), 'Validation Images:',
val_data.__len__())
dist = train_data.dist()
print('Train Data Distribution:', dist, 'Val Data Distribution:',
val_data.dist())
train_dataloader = DataLoader(train_data,
batch_size=config.batch_size,
shuffle=True,
num_workers=config.num_workers)
val_dataloader = DataLoader(val_data,
batch_size=config.batch_size,
shuffle=False,
num_workers=config.num_workers)
# prepare model
# model = ResNet18(num_classes=config.num_classes)
# model = Vgg16(num_classes=config.num_classes)
# model = densenet_collapse(num_classes=config.num_classes)
model = ShallowVgg(num_classes=config.num_classes)
print(model)
if config.load_model_path:
model.load(config.load_model_path)
if config.use_gpu:
model.cuda()
if config.parallel:
model = torch.nn.DataParallel(
model, device_ids=[x for x in range(config.num_of_gpu)])
# criterion and optimizer
# weight = torch.Tensor([1/dist['0'], 1/dist['1'], 1/dist['2'], 1/dist['3']])
# weight = torch.Tensor([1/dist['0'], 1/dist['1']])
# weight = torch.Tensor([dist['1'], dist['0']])
# weight = torch.Tensor([1, 10])
# vis.log(f'loss weight: {weight}')
# print('loss weight:', weight)
# weight = weight.cuda()
# criterion = torch.nn.CrossEntropyLoss()
criterion = LabelSmoothing(size=config.num_classes, smoothing=0.1)
# criterion = torch.nn.CrossEntropyLoss(weight=weight)
# criterion = FocalLoss(gamma=4, alpha=None)
lr = config.lr
optimizer = torch.optim.Adam(model.parameters(),
lr=lr,
weight_decay=config.weight_decay)
# metric
softmax = functional.softmax
log_softmax = functional.log_softmax
loss_meter = meter.AverageValueMeter()
epoch_loss = meter.AverageValueMeter()
train_cm = meter.ConfusionMeter(config.num_classes)
train_AUC = meter.AUCMeter()
previous_avgse = 0
# previous_AUC = 0
if config.parallel:
save_model_dir = config.save_model_dir if config.save_model_dir else model.module.model_name
save_model_name = config.save_model_name if config.save_model_name else model.module.model_name + '_best_model.pth'
else:
save_model_dir = config.save_model_dir if config.save_model_dir else model.model_name
save_model_name = config.save_model_name if config.save_model_name else model.model_name + '_best_model.pth'
save_epoch = 1 # 用于记录验证集上效果最好模型对应的epoch
# process_record = {'epoch_loss': [], # 用于记录实验过程中的曲线，便于画曲线图
# 'train_avgse': [], 'train_se0': [], 'train_se1': [], 'train_se2': [], 'train_se3': [],
# 'val_avgse': [], 'val_se0': [], 'val_se1': [], 'val_se2': [], 'val_se3': []}
process_record = {
'epoch_loss': [], # 用于记录实验过程中的曲线，便于画曲线图
'train_avgse': [],
'train_se0': [],
'train_se1': [],
'val_avgse': [],
'val_se0': [],
'val_se1': [],
'train_AUC': [],
'val_AUC': []
}
# train
for epoch in range(config.max_epoch):
print(
f"epoch: [{epoch+1}/{config.max_epoch}] {config.save_model_name[:-4]} =================================="
)
epoch_loss.reset()
train_cm.reset()
train_AUC.reset()
# train
model.train()
for i, (image, label, image_path) in tqdm(enumerate(train_dataloader)):
loss_meter.reset()
# prepare input
if config.use_gpu:
image = image.cuda()
label = label.cuda()
# go through the model
score = model(image)
# backpropagate
optimizer.zero_grad()
# loss = criterion(score, label)
loss = criterion(log_softmax(score, dim=1), label)
loss.backward()
optimizer.step()
loss_meter.add(loss.item())
epoch_loss.add(loss.item())
train_cm.add(softmax(score, dim=1).data, label.data)
positive_score = np.array([
item[1]
for item in softmax(score, dim=1).data.cpu().numpy().tolist()
])
train_AUC.add(positive_score, label.data)
if (i + 1) % config.print_freq == 0:
vis.plot('loss', loss_meter.value()[0])
# print result
# train_se = [100. * train_cm.value()[0][0] / (train_cm.value()[0][0] + train_cm.value()[0][1] + train_cm.value()[0][2] + train_cm.value()[0][3]),
# 100. * train_cm.value()[1][1] / (train_cm.value()[1][0] + train_cm.value()[1][1] + train_cm.value()[1][2] + train_cm.value()[1][3]),
# 100. * train_cm.value()[2][2] / (train_cm.value()[2][0] + train_cm.value()[2][1] + train_cm.value()[2][2] + train_cm.value()[2][3]),
# 100. * train_cm.value()[3][3] / (train_cm.value()[3][0] + train_cm.value()[3][1] + train_cm.value()[3][2] + train_cm.value()[3][3])]
train_se = [
100. * train_cm.value()[0][0] /
(train_cm.value()[0][0] + train_cm.value()[0][1]),
100. * train_cm.value()[1][1] /
(train_cm.value()[1][0] + train_cm.value()[1][1])
]
# validate
model.eval()
if (epoch + 1) % 1 == 0:
val_cm, val_se, val_accuracy, val_AUC = val_2class(
model, val_dataloader)
if np.average(
val_se) > previous_avgse: # 当测试集上的平均sensitivity升高时保存模型
# if val_AUC.value()[0] > previous_AUC: # 当测试集上的AUC升高时保存模型
if config.parallel:
if not os.path.exists(
os.path.join('checkpoints', save_model_dir,
save_model_name.split('.')[0])):
os.makedirs(
os.path.join('checkpoints', save_model_dir,
save_model_name.split('.')[0]))
model.module.save(
os.path.join('checkpoints', save_model_dir,
save_model_name.split('.')[0],
save_model_name))
else:
if not os.path.exists(
os.path.join('checkpoints', save_model_dir,
save_model_name.split('.')[0])):
os.makedirs(
os.path.join('checkpoints', save_model_dir,
save_model_name.split('.')[0]))
model.save(
os.path.join('checkpoints', save_model_dir,
save_model_name.split('.')[0],
save_model_name))
previous_avgse = np.average(val_se)
# previous_AUC = val_AUC.value()[0]
save_epoch = epoch + 1
process_record['epoch_loss'].append(epoch_loss.value()[0])
process_record['train_avgse'].append(np.average(train_se))
process_record['train_se0'].append(train_se[0])
process_record['train_se1'].append(train_se[1])
# process_record['train_se2'].append(train_se[2])
# process_record['train_se3'].append(train_se[3])
process_record['train_AUC'].append(train_AUC.value()[0])
process_record['val_avgse'].append(np.average(val_se))
process_record['val_se0'].append(val_se[0])
process_record['val_se1'].append(val_se[1])
# process_record['val_se2'].append(val_se[2])
# process_record['val_se3'].append(val_se[3])
process_record['val_AUC'].append(val_AUC.value()[0])
# vis.plot_many({'epoch_loss': epoch_loss.value()[0],
# 'train_avgse': np.average(train_se), 'train_se0': train_se[0], 'train_se1': train_se[1], 'train_se2': train_se[2], 'train_se3': train_se[3],
# 'val_avgse': np.average(val_se), 'val_se0': val_se[0], 'val_se1': val_se[1], 'val_se2': val_se[2], 'val_se3': val_se[3]})
# vis.log(f"epoch: [{epoch+1}/{config.max_epoch}] =========================================")
# vis.log(f"lr: {optimizer.param_groups[0]['lr']}, loss: {round(loss_meter.value()[0], 5)}")
# vis.log(f"train_avgse: {round(np.average(train_se), 4)}, train_se0: {round(train_se[0], 4)}, train_se1: {round(train_se[1], 4)}, train_se2: {round(train_se[2], 4)}, train_se3: {round(train_se[3], 4)},")
# vis.log(f"val_avgse: {round(np.average(val_se), 4)}, val_se0: {round(val_se[0], 4)}, val_se1: {round(val_se[1], 4)}, val_se2: {round(val_se[2], 4)}, val_se3: {round(val_se[3], 4)}")
# vis.log(f'train_cm: {train_cm.value()}')
# vis.log(f'val_cm: {val_cm.value()}')
# print("lr:", optimizer.param_groups[0]['lr'], "loss:", round(epoch_loss.value()[0], 5))
# print('train_avgse:', round(np.average(train_se), 4), 'train_se0:', round(train_se[0], 4), 'train_se1:', round(train_se[1], 4), 'train_se2:', round(train_se[2], 4), 'train_se3:', round(train_se[3], 4))
# print('val_avgse:', round(np.average(val_se), 4), 'val_se0:', round(val_se[0], 4), 'val_se1:', round(val_se[1], 4), 'val_se2:', round(val_se[2], 4), 'val_se3:', round(val_se[3], 4))
# print('train_cm:')
# print(train_cm.value())
# print('val_cm:')
# print(val_cm.value())
vis.plot_many({
'epoch_loss': epoch_loss.value()[0],
'train_avgse': np.average(train_se),
'train_se0': train_se[0],
'train_se1': train_se[1],
'val_avgse': np.average(val_se),
'val_se0': val_se[0],
'val_se1': val_se[1],
'train_AUC': train_AUC.value()[0],
'val_AUC': val_AUC.value()[0]
})
vis.log(
f"epoch: [{epoch + 1}/{config.max_epoch}] ========================================="
)
vis.log(
f"lr: {optimizer.param_groups[0]['lr']}, loss: {round(loss_meter.value()[0], 5)}"
)
vis.log(
f"train_avgse: {round(np.average(train_se), 4)}, train_se0: {round(train_se[0], 4)}, train_se1: {round(train_se[1], 4)}"
)
vis.log(
f"val_avgse: {round(np.average(val_se), 4)}, val_se0: {round(val_se[0], 4)}, val_se1: {round(val_se[1], 4)}"
)
vis.log(f'train_AUC: {train_AUC.value()[0]}')
vis.log(f'val_AUC: {val_AUC.value()[0]}')
vis.log(f'train_cm: {train_cm.value()}')
vis.log(f'val_cm: {val_cm.value()}')
print("lr:", optimizer.param_groups[0]['lr'], "loss:",
round(epoch_loss.value()[0], 5))
print('train_avgse:', round(np.average(train_se), 4), 'train_se0:',
round(train_se[0], 4), 'train_se1:', round(train_se[1], 4))
print('val_avgse:', round(np.average(val_se), 4), 'val_se0:',
round(val_se[0], 4), 'val_se1:', round(val_se[1], 4))
print('train_AUC:',
train_AUC.value()[0], 'val_AUC:',
val_AUC.value()[0])
print('train_cm:')
print(train_cm.value())
print('val_cm:')
print(val_cm.value())
if os.path.exists(
os.path.join('checkpoints', save_model_dir,
save_model_name.split('.')[0])):
write_json(file=os.path.join('checkpoints', save_model_dir,
save_model_name.split('.')[0],
'process_record.json'),
content=process_record)
# if (epoch+1) % 5 == 0:
# lr = lr * config.lr_decay
# for param_group in optimizer.param_groups:
# param_group['lr'] = lr
vis.log(f"Best Epoch: {save_epoch}")
print("Best Epoch:", save_epoch)
def train(**kwargs):
# torch.manual_seed(100) # 10, 100, 666,
opt.parse(kwargs)
vis = Visualizer(opt.env)
# step1: configure model
model = getattr(models, opt.model)()
if opt.load_model_path:
model.load(opt.load_model_path)
if opt.use_gpu:
model.cuda()
# step2: load data
if os.path.isfile(opt.train_features_path) and\
os.path.isfile(opt.train_targets_path):
print "load train dataset from file"
features = torch.load(opt.train_features_path)
# features[features == float('Inf')] = 0 # for errors
targets = torch.load(opt.train_targets_path) * 5 # !!!!!!!!!!!!!!!!!
train_data = torch.utils.data.TensorDataset(features, targets)
train_dataloader = DataLoader(train_data, opt.batch_size,
shuffle=True,
num_workers=opt.num_workers)
else:
train_data = STSDataset(opt.train_data_path, opt)
train_dataloader = DataLoader(train_data, opt.batch_size,
shuffle=True,
num_workers=opt.num_workers)
torch.save(train_data.X, opt.train_features_path)
torch.save(train_data.y, opt.train_targets_path)
if os.path.isfile(opt.test_features_path) and\
os.path.isfile(opt.test_targets_path):
print "load test dataset from file"
features = torch.load(opt.test_features_path)
# features[features == float('Inf')] = 0 # for errors
targets = torch.load(opt.test_targets_path) * 5 # !!!!!!!!!!!!!!!!!
test_data = torch.utils.data.TensorDataset(features, targets)
test_dataloader = DataLoader(test_data, opt.batch_size,
shuffle=False,
num_workers=opt.num_workers)
else:
test_data = STSDataset(opt.test_data_path, opt)
test_dataloader = DataLoader(test_data, opt.batch_size,
shuffle=True,
num_workers=opt.num_workers)
torch.save(test_data.X, opt.test_features_path)
torch.save(test_data.y, opt.test_targets_path)
# step3: set criterion and optimizer
# criterion = torch.nn.MarginRankingLoss()
criterion = torch.nn.BCELoss()
lr = opt.lr
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
# weight_decay=opt.weight_decay)
# optimizer = torch.optim.RMSprop(model.parameters(), lr=lr)
# weight_decay=opt.weight_decay)
# optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9)
# step4: set meters
loss_meter = meter.AUCMeter()
# loss_meter = meter.ClassErrorMeter()
previous_loss = 1e100
# train
testf1 = []
test_f_a_l = []
for epoch in range(opt.max_epoch):
loss_meter.reset()
for ii, (data, label) in enumerate(train_dataloader):
# train model on a batch data
input = Variable(data)
target = Variable(torch.FloatTensor(label.numpy()))
if opt.use_gpu:
input = input.cuda()
target = target.cuda()
optimizer.zero_grad()
score = model(input)
loss = criterion(score, target)
# print score
loss.backward()
optimizer.step()
# update meters and visualize
loss_meter.add(score.data, target.data)
# if ii % opt.print_freq == opt.print_freq - 1:
# vis.plot('loss', loss_meter.value()[0])
# save model for each epoch
# model.save()
# validate and visualize
train_ce, train_acc, train_f1 = val(model, train_dataloader)
test_ce, test_acc, test_f1 = val(model, test_dataloader)
testf1.append(test_acc)
test_f_a_l.append([test_f1, test_acc, test_ce])
print('epoch: %d' %epoch)
print('test acc, f1: '+str(test_acc)+' , '+str(test_f1))
# vis.plot_many({"train_ce": train_ce,
# "test_ce": test_ce}) # !!!!!!!!!!!!!!!!!
# vis.plot_many({"train_acc": train_acc,
# "test_acc": test_acc})
# vis.plot_many({"train_f1": train_f1,
# "test_f1": test_f1})
# vis.log("epoch:{epoch}, lr:{lr}, loss:{loss}, \
# train_ce:{train_ce}, train_acc:{train_acc}, \
# test_ce:{test_ce}, test_acc:{test_acc}".format(
# epoch=epoch, lr=lr, loss=loss_meter.value(),
# train_ce=str(train_ce), train_acc=str(train_acc),
# test_ce=str(test_ce), test_acc=str(test_acc)))
# update learning rate
if loss_meter.value() > previous_loss:
lr = lr * opt.lr_decay
for param_group in optimizer.param_groups:
param_group['lr'] = lr
previous_loss = loss_meter.value()
x1 = max(testf1)
ind = testf1.index(x1)
return test_f_a_l[ind], ind
def train(**kwargs):
opt.parse(kwargs)
vis = Visualizer(opt.env)
model = getattr(models, opt.models)(opt.num_class)
if opt.load_model_path:
model.load(opt.load_model_path)
if opt.use_gpu:
model.cuda()
train_data = Echocardiography(opt.train_data_root, train=True)
val_data = Echocardiography(opt.train_data_root, train=False)
train_dataloader = DataLoader(train_data,
opt.batch_size,
shuffle=True,
num_workers=opt.num_workers)
val_dataloader = DataLoader(val_data,
opt.batch_size,
shuffle=True,
num_workers=opt.num_workers)
criterion = nn.CrossEntropyLoss()
lr = opt.lr
optimizer = t.optim.Adam(model.parameters(),
lr=lr,
weight_decay=opt.weight_decay)
loss_meter = meter.AverageValueMeter()
confusion_matrix = meter.ConfusionMeter(opt.num_class)
auc_meter = meter.AUCMeter()
previous_loss = 100
for epoch in range(opt.max_epoch):
loss_meter.reset()
confusion_matrix.reset()
if opt.num_class == 2:
auc_meter.reset()
for ii, (data, label) in enumerate(train_dataloader):
input = Variable(data)
target = Variable(label.long())
if opt.use_gpu:
input = input.cuda()
target = target.cuda()
optimizer.zero_grad()
score = model(input)
loss = criterion(score, target)
loss.backward()
optimizer.step()
loss_meter.add(loss.data)
confusion_matrix.add(score.data, target.data)
if opt.num_class == 2:
auc_meter.add(score.data[:, 1], target.data)
if ii % opt.print_freq == opt.print_freq - 1:
vis.plot('loss', loss_meter.value()[0])
if opt.num_class == 2:
vis.plot('auc', auc_meter.value()[0])
val_cm, val_accuracy = val(model, val_dataloader)[:2]
if opt.num_class == 2:
val_auc = val(model, val_dataloader)[2]
vis.plot('val_auc', val_auc)
vis.plot('val_accuracy', val_accuracy)
vis.log(
"epoch:{epoch}, lr:{lr}, loss:{loss}, train_cm:{train_cm}, val_cm:{val_cm}"
.format(epoch=epoch,
lr=lr,
loss=loss_meter.value()[0],
train_cm=str(confusion_matrix.value()),
val_cm=str(val_cm.value())))
model.save()
if loss_meter.value()[0] > previous_loss:
lr = lr * opt.lr_decay
for param_group in optimizer.param_groups:
param_group['lr'] = lr
previous_loss = loss_meter.value()[0]
