def test_TestSectionLoader_should_load_data_from_path_override_data():
with tempfile.TemporaryDirectory() as data_dir:
os.makedirs(os.path.join(data_dir, "volume_name"))
os.makedirs(os.path.join(data_dir, "splits"))
seimic = np.zeros([IL, XL, D])
generate_npy_files(
os.path.join(data_dir, "volume_name", "seismic.npy"), seimic)
labels = np.ones([IL, XL, D])
generate_npy_files(os.path.join(data_dir, "volume_name", "labels.npy"),
labels)
txt_path = os.path.join(data_dir, "splits", "section_volume_name.txt")
open(txt_path, "a").close()
TestSectionLoader = get_test_loader(config)
config.merge_from_list(["DATASET.ROOT", data_dir])
test_set = TestSectionLoader(
config,
split="volume_name",
is_transform=True,
augmentations=None,
seismic_path=os.path.join(data_dir, "volume_name", "seismic.npy"),
label_path=os.path.join(data_dir, "volume_name", "labels.npy"),
)
assert_dimensions(test_set)
def _evaluate_split(split,
section_aug,
model,
device,
running_metrics_overall,
config,
debug=False):
logger = logging.getLogger(__name__)
TestSectionLoader = get_test_loader(config)
test_set = TestSectionLoader(
data_dir=config.DATASET.ROOT,
split=split,
is_transform=True,
augmentations=section_aug,
)
n_classes = test_set.n_classes
test_loader = data.DataLoader(test_set,
batch_size=1,
num_workers=config.WORKERS,
shuffle=False)
if debug:
logger.info("Running in Debug/Test mode")
test_loader = take(1, test_loader)
running_metrics_split = runningScore(n_classes)
# testing mode:
with torch.no_grad(): # operations inside don't track history
model.eval()
total_iteration = 0
for i, (images, labels) in enumerate(test_loader):
logger.info(f"split: {split}, section: {i}")
total_iteration = total_iteration + 1
outputs = model(images.to(device))
pred = outputs.detach().max(1)[1].cpu().numpy()
gt = labels.numpy()
running_metrics_split.update(gt, pred)
running_metrics_overall.update(gt, pred)
# get scores
score, class_iou = running_metrics_split.get_scores()
# Log split results
logger.info(f'Pixel Acc: {score["Pixel Acc: "]:.3f}')
for cdx, class_name in enumerate(_CLASS_NAMES):
logger.info(
f' {class_name}_accuracy {score["Class Accuracy: "][cdx]:.3f}')
logger.info(f'Mean Class Acc: {score["Mean Class Acc: "]:.3f}')
logger.info(f'Freq Weighted IoU: {score["Freq Weighted IoU: "]:.3f}')
logger.info(f'Mean IoU: {score["Mean IoU: "]:0.3f}')
running_metrics_split.reset()
def test_TestSectionLoader_should_load_data_from_test2_set():
with tempfile.TemporaryDirectory() as data_dir:
os.makedirs(os.path.join(data_dir, "test_once"))
os.makedirs(os.path.join(data_dir, "splits"))
seimic = np.zeros([IL, XL, D])
generate_npy_files(
os.path.join(data_dir, "test_once", "test2_seismic.npy"), seimic)
A = np.load(os.path.join(data_dir, "test_once", "test2_seismic.npy"))
labels = np.ones([IL, XL, D])
generate_npy_files(
os.path.join(data_dir, "test_once", "test2_labels.npy"), labels)
txt_path = os.path.join(data_dir, "splits", "section_test2.txt")
open(txt_path, "a").close()
TestSectionLoader = get_test_loader(config)
config.merge_from_list(["DATASET.ROOT", data_dir])
test_set = TestSectionLoader(config, split="test2")
assert_dimensions(test_set)
def _evaluate_split(
split,
section_aug,
model,
pre_processing,
output_processing,
device,
running_metrics_overall,
config,
debug=False,
):
logger = logging.getLogger(__name__)
TestSectionLoader = get_test_loader(config)
test_set = TestSectionLoader(
config.DATASET.ROOT,
split=split,
is_transform=True,
augmentations=section_aug,
)
n_classes = test_set.n_classes
test_loader = data.DataLoader(test_set,
batch_size=1,
num_workers=config.WORKERS,
shuffle=False)
if debug:
logger.info("Running in Debug/Test mode")
test_loader = take(1, test_loader)
try:
output_dir = generate_path(
config.OUTPUT_DIR + "_test",
git_branch(),
git_hash(),
config.MODEL.NAME,
current_datetime(),
)
except TypeError:
output_dir = generate_path(
config.OUTPUT_DIR + "_test",
config.MODEL.NAME,
current_datetime(),
)
running_metrics_split = runningScore(n_classes)
# testing mode:
with torch.no_grad(): # operations inside don't track history
model.eval()
total_iteration = 0
for i, (images, labels) in enumerate(test_loader):
logger.info(f"split: {split}, section: {i}")
total_iteration = total_iteration + 1
outputs = _patch_label_2d(
model,
images,
pre_processing,
output_processing,
config.TRAIN.PATCH_SIZE,
config.TEST.TEST_STRIDE,
config.VALIDATION.BATCH_SIZE_PER_GPU,
device,
n_classes,
)
pred = outputs.detach().max(1)[1].numpy()
gt = labels.numpy()
running_metrics_split.update(gt, pred)
running_metrics_overall.update(gt, pred)
# dump images to disk for review
mask_to_disk(pred.squeeze(),
os.path.join(output_dir, f"{i}_pred.png"))
mask_to_disk(gt.squeeze(), os.path.join(output_dir, f"{i}_gt.png"))
# get scores
score, class_iou = running_metrics_split.get_scores()
# Log split results
logger.info(f'Pixel Acc: {score["Pixel Acc: "]:.3f}')
for cdx, class_name in enumerate(_CLASS_NAMES):
logger.info(
f' {class_name}_accuracy {score["Class Accuracy: "][cdx]:.3f}')
logger.info(f'Mean Class Acc: {score["Mean Class Acc: "]:.3f}')
logger.info(f'Freq Weighted IoU: {score["Freq Weighted IoU: "]:.3f}')
logger.info(f'Mean IoU: {score["Mean IoU: "]:0.3f}')
running_metrics_split.reset()
def _evaluate_split(
split,
section_aug,
model,
pre_processing,
output_processing,
device,
running_metrics_overall,
config,
data_flow,
debug=False,
):
logger = logging.getLogger(__name__)
TestSectionLoader = get_test_loader(config)
test_set = TestSectionLoader(
config,
split=split,
is_transform=True,
augmentations=section_aug,
debug=debug,
)
n_classes = test_set.n_classes
if debug:
data_flow[split] = dict()
data_flow[split]["test_section_loader_length"] = len(test_set)
data_flow[split]["test_input_shape"] = test_set.seismic.shape
data_flow[split]["test_label_shape"] = test_set.labels.shape
data_flow[split]["n_classes"] = n_classes
test_loader = data.DataLoader(test_set,
batch_size=1,
num_workers=config.WORKERS,
shuffle=False)
if debug:
data_flow[split]["test_loader_length"] = len(test_loader)
logger.info("Running in Debug/Test mode")
take_n = 2
test_loader = take(take_n, test_loader)
data_flow[split]["take_n_sections"] = take_n
pred_list, gt_list, img_list = [], [], []
try:
output_dir = generate_path(
f"{config.OUTPUT_DIR}/test/{split}",
git_branch(),
git_hash(),
config.MODEL.NAME,
current_datetime(),
)
except:
output_dir = generate_path(
f"{config.OUTPUT_DIR}/test/{split}",
config.MODEL.NAME,
current_datetime(),
)
running_metrics_split = runningScore(n_classes)
# evaluation mode:
with torch.no_grad(): # operations inside don't track history
model.eval()
for i, (images, labels) in enumerate(test_loader):
logger.info(f"split: {split}, section: {i}")
outputs = _patch_label_2d(
model,
images,
pre_processing,
output_processing,
config.TRAIN.PATCH_SIZE,
config.TEST.TEST_STRIDE,
config.VALIDATION.BATCH_SIZE_PER_GPU,
device,
n_classes,
split,
debug,
config.DATASET.MIN,
config.DATASET.MAX,
)
pred = outputs.detach().max(1)[1].numpy()
gt = labels.numpy()
if debug:
pred_list.append((pred.shape, len(np.unique(pred))))
gt_list.append((gt.shape, len(np.unique(gt))))
img_list.append(images.numpy().shape)
running_metrics_split.update(gt, pred)
running_metrics_overall.update(gt, pred)
# dump images to disk for review
mask_to_disk(pred.squeeze(),
os.path.join(output_dir, f"{i}_pred.png"), n_classes)
mask_to_disk(gt.squeeze(), os.path.join(output_dir, f"{i}_gt.png"),
n_classes)
if debug:
data_flow[split]["pred_shape"] = pred_list
data_flow[split]["gt_shape"] = gt_list
data_flow[split]["img_shape"] = img_list
# get scores
score, class_iou = running_metrics_split.get_scores()
# Log split results
logger.info(f'Pixel Acc: {score["Pixel Acc: "]:.3f}')
if debug:
for cdx in range(n_classes):
logger.info(
f' Class_{cdx}_accuracy {score["Class Accuracy: "][cdx]:.3f}')
else:
for cdx, class_name in enumerate(_CLASS_NAMES):
logger.info(
f' {class_name}_accuracy {score["Class Accuracy: "][cdx]:.3f}'
)
logger.info(f'Mean Class Acc: {score["Mean Class Acc: "]:.3f}')
logger.info(f'Freq Weighted IoU: {score["Freq Weighted IoU: "]:.3f}')
logger.info(f'Mean IoU: {score["Mean IoU: "]:0.3f}')
running_metrics_split.reset()