def setup_logging(name, cfg):
# Create argparser and logging object to send to train()
parser = argparse.ArgumentParser()
# Pass name to the argparser
parser.add_argument("--name",
nargs="?",
type=str,
default=name,
help="argparse.SUPPRESS")
# train() needs this arg but we wont be using it here
parser.add_argument('--evaluate',
action='store_true',
help='argparse.SUPPRESS')
args = parser.parse_args()
# Define the experiment data directory
logdir = os.path.join('runs', cfg['training']['loss']['name'], args.name)
writer = SummaryWriter(log_dir=logdir)
print('RUNDIR: {}'.format(logdir))
shutil.copy(config, logdir)
logger_old = get_logger(logdir)
logger_old.info('Let the games begin')
return args, writer, logger_old
break
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="config")
parser.add_argument(
"--config",
nargs="?",
type=str,
default="configs/fcn8s_pascal.yml",
help="Configuration file to use",
)
args = parser.parse_args()
with open(args.config) as fp:
cfg = yaml.load(fp)
run_id = random.randint(1, 100000)
logdir = os.path.join("runs",
os.path.basename(args.config)[:-4], str(run_id))
writer = SummaryWriter(log_dir=logdir)
print("RUNDIR: {}".format(logdir))
shutil.copy(args.config, logdir)
logger = get_logger(logdir)
logger.info("Let the games begin")
train(cfg, writer, logger)
with open(args.config) as fp:
cfg = yaml.load(fp)
# Set logdir
if cfg["logging"]["log_name"] == "id":
run_id = random.randint(1, 100000)
log_name = str(run_id)
elif cfg["logging"]["log_name"] == "timestamp":
log_name = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
else:
log_name = cfg["logging"]["log_name"]
logdir = os.path.join(cfg["logging"]["log_dir"],
os.path.basename(args.config)[:-4], log_name)
writer = SummaryWriter(log_dir=logdir)
# Setup logger
log_lvls = {
"debug": logging.DEBUG,
"info": logging.INFO,
"warning": logging.WARNING
}
logger = get_logger(logdir, lvl=log_lvls[cfg["logging"]["log_level"]])
logger.info("Set logging level to " + str(logger.level))
logger.info("Saving logs and checkpoints to {}".format(logdir))
shutil.copy(args.config, logdir)
# Start training
logger.info('Starting training')
train(cfg, writer, logger)
path_ori += '-ori.png'
path_denoise += '-denoise.png'
path_clean += '-clean.png'
cv2.imwrite(path_ori, pauli_ori)
cv2.imwrite(path_denoise, pauli_denoise)
cv2.imwrite(path_clean, pauli_clean)
if cfg.data.simulate:
logger.info(f'overall psnr: {test_psnr_meter.avg}, ssim: {test_ssim_meter.avg}')
logger.info(f'\ndone')
if __name__=='__main__':
cfg = args.get_argparser('configs/hoekman_unetpp4_simulate_step.yml')
# choose deterministic algorithms, and disable benchmark for variable size input
utils.set_random_seed(0)
run_id = utils.get_work_dir(osp.join(cfg.test.out_path, osp.split(osp.split(cfg.test.pth)[0])[1]))
shutil.copy(cfg.config_file, run_id)
# logger
logger = get_logger(run_id)
logger.info(f'RUN DIR: {run_id}')
test(cfg, logger, run_id)
logger.info(f'RUN DIR: {run_id}')
type=str,
default=str(run_id),
help="name to give the experiment output directory")
parser.add_argument(
'-e',
'--evaluate',
action='store_true',
help='causes prediction/image pairs to be saved for later evaluation')
args = parser.parse_args()
with open(args.config) as fp:
cfg = yaml.load(fp)
logdir = os.path.join('runs', cfg['training']['loss']['name'], args.name)
writer = SummaryWriter(log_dir=logdir)
if cfg['training']['optimizer']['lr'] and args.learning_rate:
cfg['training']['optimizer']['lr'] = args.learning_rate[0]
if cfg['training']['optimizer']['weight_decay'] and args.weight_decay:
cfg['training']['optimizer']['weight_decay'] = args.weight_decay[0]
if args.superpixels:
cfg['training']['loss']['superpixels'] = args.superpixels[0]
print('RUNDIR: {}'.format(logdir))
shutil.copy(args.config, logdir)
logger_old = get_logger(logdir)
logger_old.info('Let the games begin')
_ = train(cfg, writer, logger_old, args)
def validate(cfg, args, roi_only=False):
# Make sure this flag is on.
torch.backends.cudnn.benchmark = True
logger = get_logger(
cfg['logdir'],
'eval',
level=logging.WARN if args.prefix == 'benchmark' else logging.INFO)
results = {}
results['valid'] = {}
results['test'] = {}
result_tags = ['valid', 'test']
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = torch.device(args.device)
# Setup Dataloader
data_loader = get_loader(cfg['data']['dataset'])
data_path = cfg['data']['path']
is_void_class = get_void_class(cfg['data']['dataset'])
logger.info("{} is void? {}".format(cfg['data'], is_void_class))
if cfg['data']['dataset'] not in ['cityscapes']:
update_raw = True
img_norm = True
else:
update_raw = False
img_norm = False
loader = data_loader(data_path,
split=cfg['data']['val_split'],
is_transform=True,
img_size=(cfg['data']['img_rows'],
cfg['data']['img_cols']),
img_norm=img_norm)
test_loader = data_loader(data_path,
is_transform=True,
split=cfg['data']['test_split'],
img_size=(cfg['data']['img_rows'],
cfg['data']['img_cols']),
img_norm=img_norm)
IPython.embed()
n_classes = loader.n_classes
if roi_only:
# assert n_classes > 2
assert cfg['data']['void_class'] > 0
assert loader.void_classes == cfg['data']['void_class']
validate_batch_size = cfg['training'].get(
'validate_batch_size') or cfg['training']['batch_size']
valloader = data.DataLoader(loader,
batch_size=cfg['training']['batch_size'],
num_workers=8)
testloader = data.DataLoader(test_loader,
batch_size=validate_batch_size,
num_workers=8)
if cfg['training']['loss']['name'] in [
'multi_step_cross_entropy'
] and cfg['model']['arch'] not in ['pspnet']:
running_metrics = None
else:
running_metrics = runningScore(n_classes, void=is_void_class) if not roi_only else \
runningScore(n_classes + 1, roi_only)
# Setup Model
model, model_path = load_model_and_preprocess(cfg, args, n_classes, device)
logger.info("Loading model {} from {}".format(cfg['model']['arch'],
model_path))
with torch.no_grad():
if cfg['training']['loss']['name'] in [
'multi_step_cross_entropy'
] and cfg['model']['arch'] not in ['pspnet']:
for loader_type, myloader in enumerate([valloader, testloader]):
computation_time = 0
img_no = 0
if args.benchmark and loader_type == 0:
continue
# For all the images in this loader.
for i, (images, labels) in enumerate(myloader):
if args.benchmark:
if i > 100:
break
start_time = timeit.default_timer()
images = images.to(device)
if args.eval_flip:
# Flip images in numpy (not support in tensor)
flipped_images = np.copy(
images.data.cpu().numpy()[:, :, :, ::-1])
flipped_images = torch.from_numpy(
flipped_images).float().to(device)
if cfg['model']['arch'] in ['reclast']:
h0 = torch.ones([
images.shape[0], args.hidden_size,
images.shape[2], images.shape[3]
],
dtype=torch.float32,
device=device)
outputs = model(images, h0)
outputs_flipped = model(flipped_images, h0)
elif cfg['model']['arch'] in ['recmid']:
W, H = images.shape[2], images.shape[3]
w = int(
np.floor(np.floor(np.floor(W / 2) / 2) / 2) /
2)
h = int(
np.floor(np.floor(np.floor(H / 2) / 2) / 2) /
2)
h0 = torch.ones(
[images.shape[0], args.hidden_size, w, h],
dtype=torch.float32,
device=device)
outputs = model(images, h0)
outputs_flipped = model(flipped_images, h0)
elif cfg['model']['arch'] in ['dru', 'sru']:
W, H = images.shape[2], images.shape[3]
w = int(
np.floor(np.floor(np.floor(W / 2) / 2) / 2) /
2)
h = int(
np.floor(np.floor(np.floor(H / 2) / 2) / 2) /
2)
h0 = torch.ones(
[images.shape[0], args.hidden_size, w, h],
dtype=torch.float32,
device=device)
s0 = torch.ones([images.shape[0], n_classes, W, H],
dtype=torch.float32,
device=device)
outputs = model(images, h0, s0)
outputs_flipped = model(flipped_images, h0, s0)
elif cfg['model']['arch'] in [
'druvgg16', 'druresnet50',
'druresnet50syncedbn'
]:
W, H = images.shape[2], images.shape[3]
w, h = int(W / 2**4), int(H / 2**4)
if cfg['model']['arch'] in [
'druresnet50', 'druresnet50syncedbn'
]:
w, h = int(W / 2**5), int(H / 2**5)
h0 = torch.ones(
[images.shape[0], args.hidden_size, w, h],
dtype=torch.float32,
device=device)
s0 = torch.zeros(
[images.shape[0], n_classes, W, H],
dtype=torch.float32,
device=device)
outputs = model(images, h0, s0)
outputs_flipped = model(flipped_images, h0, s0)
else:
outputs = model(images)
outputs_flipped = model(flipped_images)
outputs_list = [
output.data.cpu().numpy() for output in outputs
]
outputs_flipped_list = [
output_flipped.data.cpu().numpy()
for output_flipped in outputs_flipped
]
outputs_list = [
(outputs + outputs_flipped[:, :, :, ::-1]) / 2.0
for outputs, outputs_flipped in zip(
outputs_list, outputs_flipped_list)
]
else:
if cfg['model']['arch'] in ['reclast']:
h0 = torch.ones([
images.shape[0], args.hidden_size,
images.shape[2], images.shape[3]
],
dtype=torch.float32,
device=device)
outputs = model(images, h0)
elif cfg['model']['arch'] in ['recmid']:
W, H = images.shape[2], images.shape[3]
w = int(
np.floor(np.floor(np.floor(W / 2) / 2) / 2) /
2)
h = int(
np.floor(np.floor(np.floor(H / 2) / 2) / 2) /
2)
h0 = torch.ones(
[images.shape[0], args.hidden_size, w, h],
dtype=torch.float32,
device=device)
outputs = model(images, h0)
elif cfg['model']['arch'] in ['dru', 'sru']:
W, H = images.shape[2], images.shape[3]
w = int(
np.floor(np.floor(np.floor(W / 2) / 2) / 2) /
2)
h = int(
np.floor(np.floor(np.floor(H / 2) / 2) / 2) /
2)
h0 = torch.ones(
[images.shape[0], args.hidden_size, w, h],
dtype=torch.float32,
device=device)
s0 = torch.ones([images.shape[0], n_classes, W, H],
dtype=torch.float32,
device=device)
outputs = model(images, h0, s0)
elif cfg['model']['arch'] in [
'druvgg16', 'druresnet50',
'druresnet50syncedbn'
]:
W, H = images.shape[2], images.shape[3]
w, h = int(W / 2**4), int(H / 2**4)
if cfg['model']['arch'] in [
'druresnet50', 'druresnet50syncedbn'
]:
w, h = int(W / 2**5), int(H / 2**5)
h0 = torch.ones(
[images.shape[0], args.hidden_size, w, h],
dtype=torch.float32,
device=device)
s0 = torch.zeros(
[images.shape[0], n_classes, W, H],
dtype=torch.float32,
device=device)
outputs = model(images, h0, s0)
else:
outputs = model(images)
outputs_list = [
output.data.cpu().numpy() for output in outputs
]
pred = [
np.argmax(outputs, axis=1) for outputs in outputs_list
]
gt = labels.numpy()
if roi_only:
""" Process for ROI, basically, mask the Pred based on GT"""
# IPython.embed()
for k in range(len(pred)):
pred[k] = np.where(gt == loader.void_classes,
loader.void_classes, pred[k])
if args.measure_time:
elapsed_time = timeit.default_timer() - start_time
computation_time += elapsed_time
img_no += pred[-1].shape[0]
if (i + 1) % 5 == 0:
logger.warning("Inference time \
(iter {0:5d}): {1:3.5f} fps".format(
i + 1, pred[-1].shape[0] / elapsed_time))
for k in range(len(pred)):
if running_metrics is None:
running_metrics = []
for _ in range(len(pred)):
running_metrics = running_metrics + [
runningScore(n_classes, void=is_void_class)
if not roi_only else runningScore(
n_classes + 1, roi_only)
]
else:
running_metrics[k].update(gt, pred[k], step=k)
if update_raw and k == len(pred) - 1:
running_metrics[k].update_raw(gt,
outputs_list[k],
step=k)
if args.measure_time:
logger.warning(f'{computation_time}, {img_no}')
logger.warning("Overall Inference time {} fps".format(
img_no * 1. / computation_time))
if loader_type == 0:
logger.info('validation set performance :')
elif loader_type == 1:
logger.info('test set performance')
results[result_tags[loader_type]] = {}
for j in range(len(running_metrics)):
score, class_iou, class_f1 = running_metrics[j].get_scores(
)
logger.info('RNN step {}'.format(j + 1))
results[result_tags[loader_type]][j] = {}
for k, v in score.items():
v = float(v)
logger.info(wrap_str(k, v))
results[result_tags[loader_type]][j][k] = v
for i in range(n_classes -
1 if is_void_class else n_classes):
logger.info(
wrap_str("iou class {}: \t".format(i),
class_iou[i]))
results[result_tags[loader_type]][j]['iou{}'.format(
i)] = float(class_iou[i])
for i in range(n_classes -
1 if is_void_class else n_classes):
logger.info(
wrap_str("f1 class {}: \t".format(i), class_f1[i]))
results[result_tags[loader_type]][j]['f1{}'.format(
i)] = float(class_f1[i])
# if j == 2:
# p, r, thresh = running_metrics[j].compute_break_even()
# logger.info(f"P/R Break even at {(p + r) / 2}.")
# running_metrics = None
else:
for loader_type, myloader in enumerate([valloader, testloader]):
start = timeit.default_timer()
computation_time = 0
img_no = 0
if args.benchmark and loader_type == 0:
continue
for i, (images, labels) in enumerate(myloader):
if args.benchmark:
if i > 100:
break
start_time = timeit.default_timer()
images = images.to(device)
if args.eval_flip:
outputs = model(images)
# Flip images in numpy (not support in tensor)
outputs = outputs.data.cpu().numpy()
flipped_images = np.copy(
images.data.cpu().numpy()[:, :, :, ::-1])
flipped_images = torch.from_numpy(
flipped_images).float().to(device)
outputs_flipped = model(flipped_images)
outputs_flipped = outputs_flipped.data.cpu().numpy()
outputs = (outputs +
outputs_flipped[:, :, :, ::-1]) / 2.0
pred = np.argmax(outputs, axis=1)
else:
outputs = model(images)
outputs = outputs.data.cpu().numpy()
pred = np.argmax(outputs, axis=1)
gt = labels.numpy()
if roi_only:
pred = np.where(gt == loader.void_classes,
loader.void_classes, pred)
if args.measure_time:
elapsed_time = timeit.default_timer() - start_time
computation_time += elapsed_time
img_no += pred.shape[0]
if (i + 1) % 5 == 0:
logging.warning("Inference time \
(iter {0:5d}): {1:3.5f} fps".format(
i + 1, pred.shape[0] / elapsed_time))
running_metrics.update(gt, pred)
if update_raw:
running_metrics.update_raw(gt, outputs)
if args.measure_time:
logging.warning("{computation_time}, {img_no}")
logging.warning("Overall Inference time {} fps".format(
img_no * 1. / computation_time))
logging.warning(
"Inference time with data loading {} fps".format(
img_no * 1. / (timeit.default_timer() - start)))
# running_metrics = None
score, class_iou, class_f1 = running_metrics.get_scores()
if update_raw:
p, r, thres = running_metrics.compute_break_even()
logger.warning("P/R Break even at {}.".format((p + r) / 2))
results[result_tags[loader_type]] = {}
if loader_type == 0:
logger.warning('validation set performance :')
elif loader_type == 1:
logger.warning('test set performance')
for k, v in score.items():
logger.info(wrap_str(k, v))
results[result_tags[loader_type]][k] = v
for i in range(n_classes - 1 if is_void_class else n_classes):
logger.info(
wrap_str("iou class {}: \t".format(i), class_iou[i]))
results[result_tags[loader_type]]['iou{}'.format(
i)] = class_iou[i]
for i in range(n_classes - 1 if is_void_class else n_classes):
logger.info(
wrap_str("f1 class {}: \t".format(i), class_f1[i]))
results[result_tags[loader_type]]['f1{}'.format(
i)] = class_f1[i]
result_path = result_root(cfg, create=True) + '.yml'
with open(result_path, 'w') as f:
yaml.dump(results, f, default_flow_style=False)
print(model_path)
# IPython.embed()
clean_logger(logger)
return results
def test_with_cfg(cfg, args):
logger = get_logger(cfg['logdir'], 'test')
device = torch.device(args.device)
out_path = cfg['eval_out_path']
if not os.path.exists(out_path):
os.makedirs(out_path)
# Setup image
valid_images = [".jpg", ".gif", ".png", ".tga", ".tif", ".tiff"]
data_loader = get_loader(cfg['data']['dataset'])
data_path = cfg['data']['path']
print("Read Input Image from : {}".format(data_path))
print(f"Save the output to : {cfg['eval_out_path']}")
loader = data_loader(
data_path,
is_transform=True, # Return the original image without any augmentation.
split=cfg['data']['test_split'],
img_size=(cfg['data']['img_rows'],
cfg['data']['img_cols']),
)
im_loader = data_loader(
data_path,
is_transform=False, # Return the original image without any augmentation.
split=cfg['data']['test_split'],
img_size=(cfg['data']['img_rows'],
cfg['data']['img_cols']),
)
testloader = data.DataLoader(
loader,
batch_size=1,
num_workers=2
)
roi_only = 'roi' in cfg['data']['dataset']
n_classes = loader.n_classes
# Setup Model
model, model_path = load_model_and_preprocess(cfg, args, n_classes, device)
logger.info(f"Loading model {cfg['model']['arch']} from {model_path}")
# model_file_name = os.path.split(args.model_path)[1]
# model_name = cfg['model']['arch']
# flag_subf = False
# Replace the entire loader, like doing the validation.
# IPython.embed()
with torch.no_grad():
# For all the images in this loader.
for i, (images, labels) in enumerate(testloader):
# TODO DEBUGING here.
# if i > 2:
# break
(orig_img, org_lab) = im_loader[i]
img_name = loader.files[loader.split][i]
if type(img_name) is list:
img_name = img_name[0]
start_time = timeit.default_timer()
images = images.to(device)
n_classes = loader.n_classes
pred = _evaluate_from_model(model, images, args, cfg, n_classes, device)
gt = labels.numpy()
# CHeck the org_lab == labels
# IPython.embed()
if roi_only:
""" Process for ROI, basically, mask the Pred based on GT"""
# IPython.embed()
# if args.is_recurrent:
if type(pred) is list:
for k in range(len(pred)):
pred[k] = np.where(gt == loader.void_classes, loader.void_classes, pred[k])
if cfg['data']['dataset'] == 'drive':
pred[k] = pred[k] + 1
pred[k][gt == 250] = 2
pred[k][pred[k] == 2] = 0
else:
pred = np.where(gt == loader.void_classes, loader.void_classes, pred)
if cfg['data']['dataset'] == 'drive':
pred = pred + 1
pred[gt == 250] = 2
pred[pred == 2] = 0
if type(pred) is list:
for k in range(len(pred)):
pred[k] = np.where(gt == loader.void_classes, loader.void_classes, pred[k])
if cfg['data']['dataset'] == 'drive':
pred[k] = pred[k] + 1
pred[k][gt == 250] = 2
pred[k][pred[k] == 2] = 0
else:
pred = np.where(gt == loader.void_classes, loader.void_classes, pred)
if cfg['data']['dataset'] == 'drive':
pred = pred + 1
pred[gt == 250] = 2
pred[pred == 2] = 0
output_masks_to_files(pred, loader, orig_img, img_name, out_path, args, cfg)
# Other unrelated stuff
if args.measure_time:
elapsed_time = timeit.default_timer() - start_time
if (i + 1) % 50 == 0:
print(
"Inference time \
(iter {0:5d}): {1:3.5f} fps".format(
i + 1, pred[-1].shape[0] / elapsed_time
)
)
cfg = load_cfg_with_overwrite(args)
run_id = cfg['run_id']
logdir = cfg['logdir']
writer = SummaryWriter(log_dir=logdir)
with open(os.path.join(logdir, 'config.yaml'), 'w') as fp:
yaml.dump(cfg, fp, default_flow_style=False)
print('RUNDIR: {}'.format(logdir))
# Write the config file to logdir
# shutil.copy(args.config, logdir)
logger = get_logger(logdir, level=logging.WARN if args.prefix == 'benchmark' else logging.INFO)
logger.info('Let the games begin')
try:
train(cfg, writer, logger, args)
# except (RuntimeError, KeyboardInterrupt) as e:
except (KeyboardInterrupt) as e:
logger.error(e)
logger.info("\nValidate the training result...")
valid_parser = validate_parser(parser)
valid_args = valid_parser.parse_args()
# set the model path.
# valid_args.steps = 3
if args.prefix == 'benchmark':
valid_args.benchmark = True
