def evaluate(data_loader, model, criterion, args):
with torch.no_grad():
model.eval()
meter = VOCMApMetric(class_names=("normal", "smoke", "call"))
total_loss, correct, total, tp, fp, tn, fn = 0, 0, 0, 0, 0, 0, 0
for data in data_loader:
image = data["image"].cuda()
label = data["label"].cuda()
if args.model == "wsdan":
y_pred_raw, feature_matrix, attention_map = model(image)
with torch.no_grad():
crop_images = batch_augment(image,
attention_map[:, :1, :, :],
mode='crop',
theta=(0.4, 0.6),
padding_ratio=0.1)
y_pred_crop, _, _ = model(crop_images)
loss = (criterion(y_pred_raw, label) +
criterion(y_pred_crop, label)) / 2
predict = (y_pred_raw + y_pred_crop) / 2
else:
predict = model(image)
loss = criterion(predict, label)
predict = softmax(predict, dim=-1)
pred_labels = torch.argmax(predict, dim=-1)
pred_bboxes = torch.ones((pred_labels.size(0), 1, 4))
gt_bboxes = torch.ones((pred_labels.size(0), 1, 4))
meter.update(pred_bboxes.cpu().numpy(),
pred_labels.cpu().numpy(),
predict.cpu().numpy(),
gt_bboxes.cpu().numpy(),
label.cpu().numpy())
_, predict = predict.max(1)
total_loss += loss
total += label.size(0)
correct += predict.eq(label).sum().item()
tp += torch.sum(predict & label)
fp += torch.sum(predict & (1 - label))
tn += torch.sum((1 - predict) & (1 - label))
fn += torch.sum((1 - predict) & label)
loss = total_loss / len(data_loader)
acc = float(correct) / float(total)
precision = (float(tp) + 1e-6) / (float(tp + fp) + 1e-3)
recall = (float(tp) + 1e-6) / (float(tp + fn) + 1e-3)
m_ap_normal, m_ap_smoke, m_ap_calling, m_ap = meter.get()[1]
print("mAP = %.3f, mAP_n = %.3f, mAP_s = %.3f, mAP_c = %.3f, "
"loss = %.3f, acc = %.3f, p = %.3f, r = %.3f"
"" % (m_ap, m_ap_normal, m_ap_smoke, m_ap_calling, loss, acc,
precision, recall))
return m_ap, acc, precision, recall, loss
def test(data_loader, model, ckp_path, args):
with torch.no_grad():
optimizer = torch.optim.Adagrad(model.parameters())
model = load_model(model, optimizer, ckp_path)
model.eval()
outputs = []
idx_to_name = ['normal', 'smoking', 'calling']
for i, data in tqdm(enumerate(data_loader), total=len(data_loader)):
image = data["image"].cuda()
name = data["name"]
if args.model == "wsdan":
y_pred_raw, _, attention_map = model(image)
with torch.no_grad():
crop_images = batch_augment(image,
attention_map[:, :1, :, :],
mode='crop',
theta=(0.4, 0.6),
padding_ratio=0.1)
y_pred_crop, _, _ = model(crop_images)
output = (y_pred_raw + y_pred_crop) / 2
else:
output = model(image)
output = softmax(output, dim=-1)
for j in range(image.size(0)):
idx = torch.argmax(output[j, :])
category = idx_to_name[idx]
score = output[j, idx]
outputs.append({
"category": category,
"image_name": name[j],
"score": round(float(score), 5)
})
outputs.sort(key=lambda x: int(x['image_name'].split('.')[0]))
with open("./log/result.json", "w+") as f:
json.dump(outputs, f, indent=4)
print("Done.")
return 0
def evaluate(test_loader_, epoch_, model_):
model_.eval()
test_loss, correct, total, tp, fp, tn, fn = 0, 0, 0, 0, 0, 0, 0
criterion = torch.nn.CrossEntropyLoss()
# print(len(test_loader))
for data_ in tqdm(test_loader_):
try:
image_ = data_["image"].cuda()
label_ = data_["label"].cuda()
except OSError:
# print("OSError of image. ")
continue
y_pred_raw, _, attention_map = model_(image_)
crop_images = batch_augment(image_,
attention_map,
mode='crop',
theta=0.1,
padding_ratio=0.05)
y_pred_crop, _, _ = model_(crop_images)
y_pred = (y_pred_raw + y_pred_crop) / 2.
loss_ = criterion(y_pred, label_)
test_loss += loss_.item()
_, predict = y_pred.max(1)
total += label_.size(0)
correct += predict.eq(label_).sum().item()
tp += torch.sum(predict & label_)
fp += torch.sum(predict & (1 - label_))
tn += torch.sum((1 - predict) & (1 - label_))
fn += torch.sum((1 - predict) & label_)
acc = 100. * correct / total
precision = 100.0 * tp / float(tp + fp)
recall = 100.0 * tp / float(tp + fn)
print(
"==> [evaluate] epoch {}, loss = {}, acc = {}, precision = {}, recall = {}"
.format(epoch_, test_loss, acc, precision, recall))
return acc, precision, recall
def evl(model, train_loader, eval_loader, args):
with torch.no_grad():
optimizer = torch.optim.Adagrad(model.parameters())
model = load_model(model, optimizer, args.ckp_path)
model.eval()
criterion = torch.nn.CrossEntropyLoss()
meter = VOCMApMetric(class_names=("normal", "smoke", "call"))
total_loss, correct, total, tp, fp, tn, fn = 0, 0, 0, 0, 0, 0, 0
bad_true = []
bad_false = []
for data_loader in [train_loader, eval_loader]:
for data in data_loader:
image = data["image"].cuda()
label = data["label"].cuda()
name = data["name"]
if args.model == "wsdan":
y_pred_raw, feature_matrix, attention_map = model(image)
with torch.no_grad():
crop_images = batch_augment(image,
attention_map[:, :1, :, :],
mode='crop',
theta=(0.4, 0.6),
padding_ratio=0.1)
y_pred_crop, _, _ = model(crop_images)
loss = (criterion(y_pred_raw, label) +
criterion(y_pred_crop, label)) / 2
predict = (y_pred_raw + y_pred_crop) / 2
else:
predict = model(image)
loss = criterion(predict, label)
predict = softmax(predict, dim=-1)
pred_labels = torch.argmax(predict, dim=-1)
pred_bboxes = torch.ones((pred_labels.size(0), 1, 4))
gt_bboxes = torch.ones((pred_labels.size(0), 1, 4))
meter.update(pred_bboxes.cpu().numpy(),
pred_labels.cpu().numpy(),
predict.cpu().numpy(),
gt_bboxes.cpu().numpy(),
label.cpu().numpy())
score = predict
_, predict = predict.max(1)
for j in range(predict.size(0)):
if predict[j] == label[j]:
bad_true.append((name[j], score[j,
predict[j]], label[j]))
else:
bad_false.append(
(name[j], score[j, predict[j]], label[j]))
total_loss += loss
total += label.size(0)
correct += predict.eq(label).sum().item()
tp += torch.sum(predict & label)
fp += torch.sum(predict & (1 - label))
tn += torch.sum((1 - predict) & (1 - label))
fn += torch.sum((1 - predict) & label)
loss = total_loss / len(data_loader)
acc = float(correct) / float(total)
precision = (float(tp) + 1e-6) / (float(tp + fp) + 1e-3)
recall = (float(tp) + 1e-6) / (float(tp + fn) + 1e-3)
m_ap_normal, m_ap_smoke, m_ap_calling, m_ap = meter.get()[1]
print("mAP = %.3f, mAP_n = %.3f, mAP_s = %.3f, mAP_c = %.3f, "
"loss = %.3f, acc = %.3f, p = %.3f, r = %.3f"
"" % (m_ap, m_ap_normal, m_ap_smoke, m_ap_calling, loss, acc,
precision, recall))
return m_ap, acc, precision, recall, loss
def train(model, train_loader, eval_loader, args):
model.train()
print("Start training")
writer = SummaryWriter(log_dir=args.save_path)
criterion = nn.CrossEntropyLoss()
# criterion = LabelSmoothCELoss()
# criterion = WeightedLabelSmoothCELoss(1978, 2168, 1227)
fc_params = list(map(id, model.fc.parameters()))
# fc_params += list(map(id, model.ca.parameters()))
# fc_params += list(map(id, model.sa.parameters()))
base_params = filter(lambda p: id(p) not in fc_params, model.parameters())
if args.optim == 'Adam':
optimizer = optim.Adam([
{'params': base_params, 'lr': args.lr / 10},
{'params': model.fc.parameters()},
# {'params': model.ca.parameters()},
# {'params': model.sa.parameters()}
# {'params': model.parameters()}
], lr=args.lr)
elif args.optim == 'SGD':
optimizer = optim.SGD([{'params': base_params, 'lr': args.lr / 10},
{'params': model.fc.parameters()}], lr=args.lr, momentum=0.9)
else:
raise ValueError
if args.sche == 'reduce':
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'max', factor=args.factor, patience=args.patience)
elif args.sche == 'cos':
scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=args.t0, T_mult=args.tm)
elif args.sche == 'None':
scheduler = None
else:
raise ValueError
global_step, best_map, loss, t_remain, best_loss = 0, 0, 0, 0, 999.0
for epoch in range(0, args.num_epochs, 1):
running_loss = 0.0
t = time.time()
for i, data in enumerate(tqdm(train_loader)):
image = data["image"].cuda()
label = data["label"].cuda()
optimizer.zero_grad()
if args.model == "wsdan":
y_pred_raw, feature_matrix, attention_map = model(image)
with torch.no_grad():
crop_images = batch_augment(image, attention_map[:, :1, :, :], mode='crop', theta=(0.4, 0.6),
padding_ratio=0.1)
y_pred_crop, _, _ = model(crop_images)
loss = (criterion(y_pred_raw, label) + criterion(y_pred_crop, label)) / 2
else:
predict = model(image)
loss = criterion(predict, label)
running_loss += loss.item()
loss.backward()
optimizer.step()
if args.sche == 'cos':
scheduler.step(epoch + i / len(train_loader))
if i % args.print_interval == 0 and i != 0:
batch_time = (time.time() - t) / args.print_interval / args.batch_size
running_loss = running_loss / args.print_interval
print("==> [train] epoch = %2d, batch = %4d, global_step = %4d, loss = %.2f, "
"time per picture = %.2fs" % (epoch, i, global_step, running_loss, batch_time))
writer.add_scalar("scalar/loss", running_loss, global_step, time.time())
running_loss = 0.0
t = time.time()
global_step += 1
print("[train] epoch = %2d, loss = %.4f, lr = %.1e, time per picture = %.2fs, remaining time = %s"
% (epoch + 1, running_loss / len(train_loader), optimizer.state_dict()['param_groups'][0]['lr'],
(time.time() - t) / len(train_loader) / args.batch_size,
sec2time((time.time() - t_remain) * (args.num_epochs - epoch - 1)) if t_remain != 0 else '-1'))
t_remain = time.time()
print("[eval train] ", end='')
map_on_train, acc_on_train, precision_on_train, recall_on_train, loss_on_train = evaluate(
train_loader, model, criterion, args)
print("[eval valid] ", end='')
map_on_valid, acc_on_valid, precision_on_valid, recall_on_valid, loss_on_valid = evaluate(
eval_loader, model, criterion, args)
if args.sche == 'reduce':
# scheduler.step(loss_on_valid) # ReduceLR
scheduler.step(map_on_valid) # ReduceLR
writer.add_scalar("scalar/loss_on_train", loss_on_train, global_step, time.time())
writer.add_scalar("scalar/loss_on_valid", loss_on_valid, global_step, time.time())
writer.add_scalar("scalar/mAP_on_train", map_on_train, global_step, time.time())
writer.add_scalar("scalar/mAP_on_valid", map_on_valid, global_step, time.time())
writer.add_scalar("scalar/accuracy_on_train", acc_on_train, global_step, time.time())
writer.add_scalar("scalar/accuracy_on_valid", acc_on_valid, global_step, time.time())
writer.add_scalar("scalar/precision_on_train", precision_on_train, global_step, time.time())
writer.add_scalar("scalar/precision_on_valid", precision_on_valid, global_step, time.time())
writer.add_scalar("scalar/recall_on_train", recall_on_train, global_step, time.time())
writer.add_scalar("scalar/recall_on_valid", recall_on_valid, global_step, time.time())
if loss_on_valid < best_loss:
best_loss = loss_on_valid
print("==> [best] loss: %.5f" % best_loss)
if float(loss_on_valid) < 0.16:
torch.save({
"model_state_dict": model.state_dict(),
}, os.path.join(args.save_path, args.model + "_loss_%.5f" % best_loss + ".tar"))
if map_on_valid > best_map:
best_map = map_on_valid
print("==> [best] mAP: %.5f" % best_map)
if float(map_on_valid) > 0.93:
torch.save({
"model_state_dict": model.state_dict(),
}, os.path.join(args.save_path, args.model + "_mAP_%.5f" % best_map + ".tar"))
writer.close()
print("Done.")
def train(model, train_loader, eval_loader, cfg):
model.train()
print("Start training")
writer = SummaryWriter(log_dir=cfg.log_path)
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=0.0001,
weight_decay=0.0005,
amsgrad=True)
current_epoch = 0
global_step = 0
loss = 0
if cfg.load_ckp:
model, optimizer, current_epoch, global_step, loss = load_model(
model, optimizer, cfg.ckp_path)
# feature_center = torch.zeros(2, cfg_.num_attentions * model_.num_features)
# center_loss = CenterLoss()
for epoch in range(current_epoch, cfg.NUM_EPOCHS, 1):
running_loss = 0.0
t = time()
for i, data in enumerate(tqdm(train_loader)):
if i == len(train_loader) - 1:
break
try:
image = data["image"].cuda()
label = data["label"].cuda()
except OSError:
print("OSError of image. ")
continue
optimizer.zero_grad()
y_pred_raw, feature_matrix, attention_map = model(image)
'''
# Update Feature Center
feature_center_batch = torch.nn.functional.normalize(feature_center[label], dim=-1)
print(feature_center[label].shape, feature_matrix.detach().shape, feature_center_batch.shape)
feature_center_batch[label] += cfg.beta * (feature_matrix.detach() - feature_center_batch)
'''
# Attention Cropping
with torch.no_grad():
crop_images = batch_augment(image,
attention_map[:, :1, :, :],
mode='crop',
theta=(0.4, 0.6),
padding_ratio=0.1)
# crop images forward
y_pred_crop, _, _ = model(crop_images)
'''
# Attention Dropping
with torch.no_grad():
drop_images = batch_augment(image, attention_map[:, 1:, :, :], mode='drop', theta=(0.2, 0.5))
# drop images forward
y_pred_drop, _, _ = model(drop_images)
'''
loss = criterion(y_pred_raw, label) / 3. + \
criterion(y_pred_crop, label) / 3
# criterion(y_pred_drop, label) / 3. + \
# 0 center_loss(feature_matrix, feature_center_batch)
# print(loss)
loss.backward()
optimizer.step()
running_loss += loss.item()
if i % cfg.print_interval == 0:
batch_time = time() - t
print(
"==> [train] epoch {}, batch {}, global_step {}. loss for 10 batches: {}, "
"time for 10 batches: {}s".format(epoch, i, global_step,
running_loss,
batch_time))
writer.add_scalar("scalar/loss", running_loss, global_step,
time())
running_loss = 0.0
t = time()
global_step += 1
# TODO add save condition eg. acc
if epoch % cfg.evaluate_epoch == 0:
torch.save(
{
"epoch": epoch,
"model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
"global_step": global_step,
'loss': loss,
},
os.path.join(cfg.save_path,
"train_epoch_" + str(epoch) + ".tar"))
print("==> [eval] on train dataset")
acc_on_train, precision_on_train, recall_on_train = evaluate(
train_loader, epoch, model)
print("==> [eval] on valid dataset")
acc_on_valid, precision_on_valid, recall_on_valid = evaluate(
eval_loader, epoch, model)
writer.add_scalar("scalar/accuracy_on_train", acc_on_train,
global_step, time())
writer.add_scalar("scalar/accuracy_on_valid", acc_on_valid,
global_step, time())
writer.add_scalar("scalar/precision_on_train", precision_on_train,
global_step, time())
writer.add_scalar("scalar/precision_on_valid", precision_on_valid,
global_step, time())
writer.add_scalar("scalar/recall_on_train", recall_on_train,
global_step, time())
writer.add_scalar("scalar/recall_on_valid", recall_on_valid,
global_step, time())
writer.close()
print("Finish training.")