def run_test(cfg, model, local_rank, distributed):
logger = logging.getLogger("FADA.tester")
if local_rank==0:
logger.info('>>>>>>>>>>>>>>>> Start Testing >>>>>>>>>>>>>>>>')
batch_time = AverageMeter()
data_time = AverageMeter()
loss_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
target_meter = AverageMeter()
feature_extractor, classifier = model
if distributed:
feature_extractor, classifier = feature_extractor.module, classifier.module
torch.cuda.empty_cache() # TODO check if it helps
dataset_name = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name)
mkdir(output_folder)
test_data = build_dataset(cfg, mode='test', is_source=False)
if distributed:
test_sampler = torch.utils.data.distributed.DistributedSampler(test_data)
else:
test_sampler = None
test_loader = torch.utils.data.DataLoader(test_data, batch_size=cfg.TEST.BATCH_SIZE, shuffle=False, num_workers=4, pin_memory=True, sampler=test_sampler)
feature_extractor.eval()
classifier.eval()
end = time.time()
with torch.no_grad():
for i, (x, y, _) in enumerate(test_loader):
x = x.cuda(non_blocking=True)
y = y.cuda(non_blocking=True).long()
size = y.shape[-2:]
pred = classifier(feature_extractor(x))
pred = F.interpolate(pred, size=size, mode='bilinear', align_corners=True)
output = pred.max(1)[1]
intersection, union, target = intersectionAndUnionGPU(output, y, cfg.MODEL.NUM_CLASSES, cfg.INPUT.IGNORE_LABEL)
if distributed:
torch.distributed.all_reduce(intersection), torch.distributed.all_reduce(union), torch.distributed.all_reduce(target)
intersection, union, target = intersection.cpu().numpy(), union.cpu().numpy(), target.cpu().numpy()
intersection_meter.update(intersection), union_meter.update(union), target_meter.update(target)
accuracy = sum(intersection_meter.val) / (sum(target_meter.val) + 1e-10)
batch_time.update(time.time() - end)
end = time.time()
iou_class = intersection_meter.sum / (union_meter.sum + 1e-10)
accuracy_class = intersection_meter.sum / (target_meter.sum + 1e-10)
mIoU = np.mean(iou_class)
mAcc = np.mean(accuracy_class)
allAcc = sum(intersection_meter.sum) / (sum(target_meter.sum) + 1e-10)
if local_rank==0:
logger.info('Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(mIoU, mAcc, allAcc))
for i in range(cfg.MODEL.NUM_CLASSES):
logger.info('Class_{} Result: iou/accuracy {:.4f}/{:.4f}.'.format(i, iou_class[i], accuracy_class[i]))
def test(cfg, thres_const):
logger = logging.getLogger("pseudo_label.tester")
logger.info("Start testing")
device = torch.device(cfg.MODEL.DEVICE)
feature_extractor = build_feature_extractor(cfg)
feature_extractor.to(device)
classifier = build_classifier(cfg)
classifier.to(device)
if cfg.resume:
logger.info("Loading checkpoint from {}".format(cfg.resume))
checkpoint = torch.load(cfg.resume, map_location=torch.device('cpu'))
feature_extractor_weights = strip_prefix_if_present(checkpoint['feature_extractor'], 'module.')
feature_extractor.load_state_dict(feature_extractor_weights)
classifier_weights = strip_prefix_if_present(checkpoint['classifier'], 'module.')
classifier.load_state_dict(classifier_weights)
feature_extractor.eval()
classifier.eval()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
target_meter = AverageMeter()
torch.cuda.empty_cache()
assert cfg.DATASETS.TEST == 'cityscapes_train'
dataset_name = cfg.DATASETS.TEST
output_folder = '.'
if cfg.OUTPUT_DIR:
output_folder = os.path.join(cfg.OUTPUT_DIR, "soft_labels", dataset_name)
mkdir(output_folder)
test_data = build_dataset(cfg, mode='test', is_source=False)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=cfg.SOLVER.BATCH_SIZE_VAL,
shuffle=False,
num_workers=4,
pin_memory=True,
sampler=None)
for index, batch in enumerate(test_loader):
if index % 100 == 0:
logger.info("{} processed".format(index))
x, y, name = batch
x = x.cuda(non_blocking=True)
y = y.cuda(non_blocking=True).long()
pred = inference(feature_extractor, classifier, x, y, flip=False)
output = pred.max(1)[1]
intersection, union, target = intersectionAndUnionGPU(output, y, cfg.MODEL.NUM_CLASSES, cfg.INPUT.IGNORE_LABEL)
intersection, union, target = intersection.cpu().numpy(), union.cpu().numpy(), target.cpu().numpy()
intersection_meter.update(intersection), union_meter.update(union), target_meter.update(target)
accuracy = sum(intersection_meter.val) / (sum(target_meter.val) + 1e-10)
# save the pseudo label
pred = pred.cpu().numpy().squeeze()
pred_max = np.max(pred, 0)
pred_label = pred.argmax(0)
for i in range(cfg.MODEL.NUM_CLASSES):
pred_label[(pred_max < thres_const[i]) * (pred_label == i)] = 255
mask = get_color_pallete(pred_label, "city")
mask_filename = name[0] if len(name[0].split("/")) < 2 else name[0].split("/")[1]
mask.save(os.path.join(output_folder, mask_filename))
iou_class = intersection_meter.sum / (union_meter.sum + 1e-10)
accuracy_class = intersection_meter.sum / (target_meter.sum + 1e-10)
mIoU = np.mean(iou_class)
mAcc = np.mean(accuracy_class)
allAcc = sum(intersection_meter.sum) / (sum(target_meter.sum) + 1e-10)
logger.info('Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(mIoU, mAcc, allAcc))
for i in range(cfg.MODEL.NUM_CLASSES):
logger.info(
'{} {} iou/accuracy: {:.4f}/{:.4f}.'.format(i, test_data.trainid2name[i], iou_class[i], accuracy_class[i]))
def test_all(cfg, saveres):
logger = logging.getLogger("BCDM.tester")
logger.info("Start testing")
device = torch.device(cfg.MODEL.DEVICE)
feature_extractor = build_feature_extractor(cfg)
feature_extractor.to(device)
classifier = build_classifier(cfg)
classifier.to(device)
classifier_2 = build_classifier(cfg)
classifier_2.to(device)
test_data = build_dataset(cfg, mode='test', is_source=False)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=cfg.TEST.BATCH_SIZE,
shuffle=False,
num_workers=4,
pin_memory=True,
sampler=None)
test_stats = []
best_iter = 0
best_miou = 0
for fname in sorted(os.listdir(cfg.resume)):
if not fname.endswith('.pth'):
continue
logger.info("Loading checkpoint from {}".format(cfg.resume + '/' +
fname))
checkpoint = torch.load(cfg.resume + '/' + fname)
feature_extractor_weights = strip_prefix_if_present(
checkpoint['feature_extractor'], 'module.')
feature_extractor.load_state_dict(feature_extractor_weights)
classifier_weights = strip_prefix_if_present(checkpoint['classifier'],
'module.')
classifier.load_state_dict(classifier_weights)
classifier_weights_2 = strip_prefix_if_present(
checkpoint['classifier_2'], 'module.')
classifier_2.load_state_dict(classifier_weights_2)
feature_extractor.eval()
classifier.eval()
classifier_2.eval()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
target_meter = AverageMeter()
torch.cuda.empty_cache()
dataset_name = cfg.DATASETS.TEST
output_folder = '.'
if cfg.OUTPUT_DIR:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference",
dataset_name)
mkdir(output_folder)
if saveres:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference",
dataset_name,
fname.replace('.pth', ''))
mkdir(output_folder)
for batch in tqdm(test_loader):
x, y, name = batch
x = x.cuda(non_blocking=True)
y = y.cuda(non_blocking=True).long()
pred = inference(feature_extractor,
classifier,
classifier_2,
x,
y,
flip=False)
output = pred.max(1)[1]
intersection, union, target = intersectionAndUnionGPU(
output, y, cfg.MODEL.NUM_CLASSES, cfg.INPUT.IGNORE_LABEL)
intersection, union, target = intersection.cpu().numpy(
), union.cpu().numpy(), target.cpu().numpy()
intersection_meter.update(intersection), union_meter.update(
union), target_meter.update(target)
accuracy = sum(
intersection_meter.val) / (sum(target_meter.val) + 1e-10)
if saveres:
pred = pred.cpu().numpy().squeeze().argmax(0)
mask = get_color_pallete(pred, "city")
mask_filename = name[0].split("/")[1]
mask.save(os.path.join(output_folder, mask_filename))
iou_class = intersection_meter.sum / (union_meter.sum + 1e-10)
accuracy_class = intersection_meter.sum / (target_meter.sum + 1e-10)
mIoU = np.mean(iou_class)
mAcc = np.mean(accuracy_class)
allAcc = sum(intersection_meter.sum) / (sum(target_meter.sum) + 1e-10)
iter_num = int(re.findall(r'\d+', fname)[0])
rec = {'iters': iter_num, 'mIoU': mIoU}
logger.info(
'Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(
mIoU, mAcc, allAcc))
for i in range(cfg.MODEL.NUM_CLASSES):
rec[test_data.trainid2name[i]] = iou_class[i]
logger.info('{} {} iou/accuracy: {:.4f}/{:.4f}.'.format(
i, test_data.trainid2name[i], iou_class[i], accuracy_class[i]))
test_stats.append(rec)
if mIoU > best_miou:
best_iter = iter_num
best_miou = mIoU
logger.info('Best result is got at iters {} with mIoU {:.4f}.'.format(
best_iter, best_miou))
with open(os.path.join(output_folder, 'test_results.csv'), 'w') as handle:
for i, rec in enumerate(test_stats):
if i == 0:
handle.write(','.join(list(rec.keys())) + '\n')
line = [str(rec[key]) for key in rec.keys()]
handle.write(','.join(line) + '\n')
def test(cfg, saveres):
logger = logging.getLogger("FADA.tester")
logger.info("Start testing")
device = torch.device(cfg.MODEL.DEVICE)
feature_extractor = build_feature_extractor(cfg)
feature_extractor.to(device)
classifier = build_classifier(cfg)
classifier.to(device)
if cfg.resume:
logger.info("Loading checkpoint from {}".format(cfg.resume))
checkpoint = torch.load(cfg.resume, map_location=torch.device('cpu'))
feature_extractor_weights = strip_prefix_if_present(checkpoint['feature_extractor'], 'module.')
feature_extractor.load_state_dict(feature_extractor_weights)
classifier_weights = strip_prefix_if_present(checkpoint['classifier'], 'module.')
classifier.load_state_dict(classifier_weights)
feature_extractor.eval()
classifier.eval()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
target_meter = AverageMeter()
torch.cuda.empty_cache() # TODO check if it helps
dataset_name = cfg.DATASETS.TEST
output_folder = '.'
if cfg.OUTPUT_DIR:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name)
mkdir(output_folder)
test_data = build_dataset(cfg, mode='test', is_source=False)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=cfg.TEST.BATCH_SIZE, shuffle=False, num_workers=4, pin_memory=True, sampler=None)
for batch in tqdm(test_loader):
x, y, name = batch
x = x.cuda(non_blocking=True)
y = y.cuda(non_blocking=True).long()
pred = inference(feature_extractor, classifier, x, y, flip=False)
# pred = multi_scale_inference(feature_extractor, classifier, x, y, flip=True)
output = pred.max(1)[1]
intersection, union, target = intersectionAndUnionGPU(output, y, cfg.MODEL.NUM_CLASSES, cfg.INPUT.IGNORE_LABEL)
intersection, union, target = intersection.cpu().numpy(), union.cpu().numpy(), target.cpu().numpy()
intersection_meter.update(intersection), union_meter.update(union), target_meter.update(target)
accuracy = sum(intersection_meter.val) / (sum(target_meter.val) + 1e-10)
if saveres:
pred = pred.cpu().numpy().squeeze()
pred_max = np.max(pred, 0)
pred = pred.argmax(0)
# uncomment the following line when visualizing SYNTHIA->Cityscapes
# pred = transform_color(pred)
mask = get_color_pallete(pred, "city")
mask_filename = name[0] if len(name[0].split("/"))<2 else name[0].split("/")[1]
mask.save(os.path.join(output_folder, mask_filename))
iou_class = intersection_meter.sum / (union_meter.sum + 1e-10)
accuracy_class = intersection_meter.sum / (target_meter.sum + 1e-10)
mIoU = np.mean(iou_class)
mAcc = np.mean(accuracy_class)
allAcc = sum(intersection_meter.sum) / (sum(target_meter.sum) + 1e-10)
logger.info('Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(mIoU, mAcc, allAcc))
for i in range(cfg.MODEL.NUM_CLASSES):
logger.info('{} {} iou/accuracy: {:.4f}/{:.4f}.'.format(i, test_data.trainid2name[i], iou_class[i], accuracy_class[i]))