def validate(cfgs):
utils.set_device(cfgs.get("val", "device"),
cfgs.getint("val", "device_id"))
loader = get_imagelist_dataloader(cfgs, "val")
model_str = cfgs.get("val", "params")
if model_str:
model_list = glob(model_str)
else:
path = os.path.join(cfgs.get("train", "snapshot_prefix"),
cfgs.get("model", "net"))
model_list = [os.path.join(path, p) for p in os.listdir(path) if ".pt" in p]
model_list = sorted(sorted(model_list), key=len)
for model_path in model_list:
val_list = cfgs.get('val','imagelist').split('/')[-1].split('.')[0]
prefix = model_path.replace('.pt', '_%s'%val_list)
res_file = glob(prefix+'*.npy')
if len(res_file) != 0:
logging.info("Test Result %s already existed!"%res_file[0])
continue
logging.info("Loading Model %s"%model_path)
net = get_net(cfgs.get("model", "net"),
cfgs.getint("model", "classes"),
model_path)
val_net(net, loader, model_path, cfgs)
def train(cfgs):
utils.set_device(cfgs.get("train", "device"),
cfgs.getint("train", "device_id"))
trainloader = get_imagelist_dataloader(cfgs, "train")
net = get_net(cfgs.get("model", "net"), cfgs.getint("model", "classes"),
cfgs.get("train", "params"))
criterion = nn.CrossEntropyLoss()
#criterion = LSoftmaxLinear()
optimizer = get_optimizer(net, cfgs)
scheduler = get_scheduler(optimizer, cfgs)
train_net(net, criterion, trainloader, optimizer, scheduler, cfgs)
def __init__(self,
init_lr,
net_params,
n_epochs,
loss,
optimizer=None,
scheduler=None):
super().__init__()
self.model = get_net(**net_params)
self.init_lr = init_lr
self.n_epochs = n_epochs
self.loss = loss
if args.snap_mix:
self.snapmix_criterion = SnapMixLoss()
def test(cfgs):
utils.set_device(cfgs.get("test", "device"),
cfgs.getint("test", "device_id"))
model_str = cfgs.get("test", "params")
if model_str:
model_list = glob(model_str)
else:
logging.info("Please specify the model!")
sys.exit(1)
for model_path in model_list:
logging.info("Loading model %s" % model_path)
net = get_net(cfgs.get("model", "net"),
cfgs.getint("model", "classes"), model_path)
test_net(net, model_path, cfgs)
def main(args: APNamespace):
root_path = Path(args.root).expanduser()
config_path = root_path / Path(args.config).expanduser()
data_path = root_path / Path(args.data).expanduser()
output_path = root_path / Path(args.output).expanduser()
global checkpoint_path
checkpoint_path = root_path / Path(args.checkpoint).expanduser()
if not config_path.exists():
# logging.critical(f"AdaS: Config path {config_path} does not exist")
print(f"AdaS: Config path {config_path} does not exist")
raise ValueError
if not data_path.exists():
print(f"AdaS: Data dir {data_path} does not exists, building")
data_path.mkdir(exist_ok=True, parents=True)
if not output_path.exists():
print(f"AdaS: Output dir {output_path} does not exists, building")
output_path.mkdir(exist_ok=True, parents=True)
if not checkpoint_path.exists():
if args.resume:
print(f"AdaS: Cannot resume from checkpoint without specifying " +
"checkpoint dir")
raise ValueError
if checkpoint_path.is_dir():
print(f"AdaS: Checkpoint dir {checkpoint_path} does not exists, " +
"building")
checkpoint_path.mkdir(exist_ok=True, parents=True)
else:
print(f"AdaS: Checkpoint path {checkpoint_path} doesn't exist " +
"building directory to store checkpoints: .adas-checkpoint")
checkpoint_path.cwd().mkdir(exist_ok=True, parents=True)
with config_path.open() as f:
config = yaml.load(f)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
global best_acc
best_acc = 0 # best test accuracy
start_epoch = 0 # start from epoch 0 or last checkpoint epoch
print("Adas: Argument Parser Options")
print("-"*45)
print(f" {'config':<20}: {args.config:<20}")
print(f" {'data':<20}: {args.data:<20}")
print(f" {'output':<20}: {args.output:<20}")
print(f" {'checkpoint':<20}: {args.checkpoint:<20}")
print(f" {'resume':<20}: {args.resume:<20}")
print("\nAdas: Train: Config")
print(f" {'Key':<20} {'Value':<20}")
print("-"*45)
for k, v in config.items():
print(f" {k:<20} {v:<20}")
for trial in range(config['n_trials']):
device
# Data
# logging.info("Adas: Preparing Data")
train_loader, test_loader = get_data(
root=data_path,
dataset=config['dataset'],
mini_batch_size=config['mini_batch_size'])
global performance_statistics, net, metrics, adas
performance_statistics = {}
# logging.info("AdaS: Building Model")
net = get_net(config['network'], num_classes=10 if config['dataset'] ==
'CIFAR10' else 100 if config['dataset'] == 'CIFAR100'
else 1000 if config['dataset'] == 'ImageNet' else 10)
metrics = Metrics(list(net.parameters()),
p=config['p'])
if config['lr_scheduler'] == 'AdaS':
adas = AdaS(parameters=list(net.parameters()),
beta=config['beta'],
zeta=config['zeta'],
init_lr=float(config['init_lr']),
min_lr=float(config['min_lr']),
p=config['p'])
net = net.to(device)
global criterion
criterion = get_loss(config['loss'])
# TODO config
optimizer, scheduler = get_optimizer_scheduler(
init_lr=float(config['init_lr']),
optim_method=config['optim_method'],
lr_scheduler=config['lr_scheduler'])
if device == 'cuda':
net = torch.nn.DataParallel(net)
cudnn.benchmark = True
if args.resume:
# Load checkpoint.
print("Adas: Resuming from checkpoint...")
if checkpoint_path.is_dir():
checkpoint = torch.load(str(checkpoint_path / 'ckpt.pth'))
else:
checkpoint = torch.load(str(checkpoint_path))
net.load_state_dict(checkpoint['net'])
best_acc = checkpoint['acc']
start_epoch = checkpoint['epoch']
if adas is not None:
adas.historical_io_metrics = \
checkpoint['historical_io_metrics']
# model_parameters = filter(lambda p: p.requires_grad,
# net.parameters())
# params = sum([np.prod(p.size()) for p in model_parameters])
# print(params)
epochs = range(start_epoch, start_epoch + config['max_epoch'])
for epoch in epochs:
start_time = time.time()
print(f"AdaS: Epoch {epoch} Started.")
train_loss, train_accuracy = epoch_iteration(
train_loader, epoch, device, optimizer)
end_time = time.time()
if config['lr_scheduler'] == 'StepLR':
scheduler.step()
test_loss, test_accuracy = test_main(test_loader, epoch, device)
total_time = time.time()
print(
f"AdaS: Epoch {epoch}/{epochs[-1]} Ended | " +
"Total Time: {:.3f}s | ".format(total_time - start_time) +
"Epoch Time: {:.3f}s | ".format(end_time - start_time) +
"Est. Time Remaining: {:.3f}s | ".format(
(total_time - start_time) * (epochs[-1] - epoch)),
"Train Loss: {:.4f}% | Train Acc. {:.4f}% | ".format(
train_loss,
train_accuracy) +
"Test Loss: {:.4f}% | Test Acc. {:.4f}%".format(test_loss,
test_accuracy))
df = pd.DataFrame(data=performance_statistics)
if config['lr_scheduler'] == 'AdaS':
xlsx_name = \
f"config['optim_method']_AdaS_trial={trial}_" +\
f"beta={config['beta']}_initlr=config['init_lr']_" +\
f"net={config['network']}_dataset={config['dataset']}.xlsx"
else:
xlsx_name = \
f"config['optim_method']_config['lr_scheduler']_" +\
f"trial={trial}_initlr=config['init_lr']" +\
f"net={config['network']}_dataset={config['dataset']}.xlsx"
df.to_excel(str(output_path / xlsx_name))
def main():
batch_size = config.batch_size
img_size = (config.img_size, config.img_size)
num_classes = config.num_classes
if config.png:
img_type = '.png'
else:
img_type = '.jpg'
dataset = HumanDataset(
config.data,
target_file='/content/gdrive/My Drive/atlas/sample_submission.csv',
train=False,
multi_label=config.multi_label,
tags_type='all',
img_type=img_type,
img_size=img_size,
test_aug=config.tta,
)
tags = get_tags()
output_col = ['Id'] + tags
submission_col = ['Id', 'Predicted']
loader = data.DataLoader(dataset,
batch_size=batch_size,
shuffle=False,
num_workers=config.num_processes)
model = get_net(config.model,
num_classes=config.num_classes,
drop_rate=0.5,
channels=config.channels)
if not config.no_cuda:
if config.num_gpu > 1:
model = torch.nn.DataParallel(model,
device_ids=list(range(
config.num_gpu))).cuda()
else:
model.cuda()
if config.resume is not None:
assert os.path.isfile(config.resume), '%s not found' % config.resume
checkpoint = torch.load(config.resume)
print('Restoring model with %s architecture...' % checkpoint['arch'])
model.load_state_dict(checkpoint['state_dict'])
if 'threshold' in checkpoint:
threshold = checkpoint['threshold']
threshold = torch.FloatTensor(threshold)
print('Using thresholds:', threshold)
if not config.no_cuda:
threshold = threshold.cuda()
else:
threshold = 0.2
threshold = 0.2
csplit = os.path.normpath(config.resume).split(sep=os.path.sep)
if len(csplit) > 1:
exp_name = csplit[-2] + '-' + csplit[-1].split('.')[0]
else:
exp_name = ''
print('Model restored from file: %s' % config.resume)
else:
assert False and "No checkpoint specified"
if config.output:
output_base = config.output
else:
output_base = os.path.join('/content/gdrive/My Drive/', 'output')
if not exp_name:
exp_name = '-'.join([
config.model,
str(config.img_size), 'f' + str(config.fold),
'png' if config.png else 'jpg'
])
output_dir = get_outdir(output_base, 'predictions', exp_name)
model.eval()
batch_time_m = AverageMeter()
data_time_m = AverageMeter()
results_raw = []
results_thr = []
results_sub = []
try:
end = time.time()
for batch_idx, (input, target, index) in enumerate(loader):
data_time_m.update(time.time() - end)
if not config.no_cuda:
input = input.cuda()
input_var = autograd.Variable(input, volatile=True)
output = model(input_var)
# augmentation reduction
reduce_factor = loader.dataset.get_aug_factor()
if reduce_factor > 1:
output.data = output.data.unfold(
0, reduce_factor, reduce_factor).mean(dim=2).squeeze(dim=2)
index = index[0:index.size(0):reduce_factor]
# output non-linearity and thresholding
output = torch.sigmoid(output)
if isinstance(threshold, torch.FloatTensor) or isinstance(
threshold, torch.cuda.FloatTensor):
threshold_m = torch.unsqueeze(threshold,
0).expand_as(output.data)
output_thr = (output.data > threshold_m).byte()
else:
output_thr = (output.data > threshold).byte()
# move data to CPU and collect
output = output.cpu().data.numpy()
output_thr = output_thr.cpu().numpy()
index = index.cpu().numpy().flatten()
for i, o, ot in zip(index, output, output_thr):
#print(dataset.inputs[i], o, ot)
image_name = os.path.splitext(
os.path.basename(dataset.inputs[i]))[0]
results_raw.append([image_name] + list(o))
results_thr.append([image_name] + list(ot))
results_sub.append([image_name] + [vector_to_tags(ot, tags)])
# end iterating through batch
batch_time_m.update(time.time() - end)
if batch_idx % config.log_interval == 0:
print('Inference: [{}/{} ({:.0f}%)] '
'Time: {batch_time.val:.3f}s, {rate:.3f}/s '
'({batch_time.avg:.3f}s, {rate_avg:.3f}/s) '
'Data: {data_time.val:.3f} ({data_time.avg:.3f})'.format(
batch_idx * len(input),
len(loader.sampler),
100. * batch_idx / len(loader),
batch_time=batch_time_m,
rate=input_var.size(0) / batch_time_m.val,
rate_avg=input_var.size(0) / batch_time_m.avg,
data_time=data_time_m))
end = time.time()
#end iterating through dataset
except KeyboardInterrupt:
pass
results_raw_df = pd.DataFrame(results_raw, columns=output_col)
results_raw_df.to_csv(os.path.join(output_dir, 'results_raw.csv'),
index=False)
results_thr_df = pd.DataFrame(results_thr, columns=output_col)
results_thr_df.to_csv(os.path.join(output_dir, 'results_thr.csv'),
index=False)
results_sub_df = pd.DataFrame(results_sub, columns=submission_col)
results_sub_df.to_csv(os.path.join(output_dir, 'submission.csv'),
index=False)
def main(img: int = 0,
num_iter: int = 40000,
lr: float = 3e-4,
gpu: int = 0,
seed: int = 42,
save: bool = True):
np.random.seed(seed)
torch.manual_seed(seed)
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
dtype = torch.cuda.FloatTensor if torch.cuda.is_available(
) else torch.FloatTensor
global i, out_avg, psnr_noisy_last, last_net, net_input, losses, psnrs, ssims, average_dropout_rate, no_layers, \
img_mean, sample_count, recons, uncerts, uncerts_ale, loss_last, roll_back
imsize = (256, 256)
PLOT = True
timestamp = int(time.time())
save_path = '/media/fastdata/laves/unsure'
os.mkdir(f'{save_path}/{timestamp}')
# denoising
if img == 0:
fname = '../NORMAL-4951060-8.jpeg'
imsize = (256, 256)
elif img == 1:
fname = '../BACTERIA-1351146-0006.png'
imsize = (256, 256)
elif img == 2:
fname = '../081_HC.png'
imsize = (256, 256)
elif img == 3:
fname = '../CNV-9997680-30.png'
imsize = (256, 256)
else:
assert False
if fname == '../NORMAL-4951060-8.jpeg':
# Add Gaussian noise to simulate speckle
img_pil = crop_image(get_image(fname, imsize)[0], d=32)
img_np = pil_to_np(img_pil)
print(img_np.shape)
p_sigma = 0.1
img_noisy_pil, img_noisy_np = get_noisy_image_gaussian(img_np, p_sigma)
elif fname == '../BACTERIA-1351146-0006.png':
# Add Poisson noise to simulate low dose X-ray
img_pil = crop_image(get_image(fname, imsize)[0], d=32)
img_np = pil_to_np(img_pil)
print(img_np.shape)
#p_lambda = 50.0
#img_noisy_pil, img_noisy_np = get_noisy_image_poisson(img_np, p_lambda)
# for lam > 20, poisson can be approximated with Gaussian
p_sigma = 0.1
img_noisy_pil, img_noisy_np = get_noisy_image_gaussian(img_np, p_sigma)
elif fname == '../081_HC.png':
# Add Gaussian noise to simulate speckle
img_pil = crop_image(get_image(fname, imsize)[0], d=32)
img_np = pil_to_np(img_pil)
print(img_np.shape)
p_sigma = 0.1
img_noisy_pil, img_noisy_np = get_noisy_image_gaussian(img_np, p_sigma)
elif fname == '../CNV-9997680-30.png':
# Add Gaussian noise to simulate speckle
img_pil = crop_image(get_image(fname, imsize)[0], d=32)
img_np = pil_to_np(img_pil)
print(img_np.shape)
p_sigma = 0.1
img_noisy_pil, img_noisy_np = get_noisy_image_gaussian(img_np, p_sigma)
else:
assert False
if PLOT:
q = plot_image_grid([img_np, img_noisy_np], 4, 6)
out_pil = np_to_pil(q)
out_pil.save(f'{save_path}/{timestamp}/input.png', 'PNG')
INPUT = 'noise'
pad = 'reflection'
OPT_OVER = 'net' # 'net,input'
reg_noise_std = 1. / 10.
LR = lr
roll_back = False # To solve the oscillation of model training
input_depth = 32
show_every = 100
exp_weight = 0.99
mse = torch.nn.MSELoss()
img_noisy_torch = np_to_torch(img_noisy_np).type(dtype)
LOSSES = {}
RECONS = {}
UNCERTS = {}
UNCERTS_ALE = {}
PSNRS = {}
SSIMS = {}
# # SGD
OPTIMIZER = 'adamw'
weight_decay = 0
LOSS = 'mse'
figsize = 4
NET_TYPE = 'skip'
skip_n33d = 128
skip_n33u = 128
skip_n11 = 4
num_scales = 5
upsample_mode = 'bilinear'
dropout_mode_down = 'None'
dropout_p_down = 0.0
dropout_mode_up = 'None'
dropout_p_up = dropout_p_down
dropout_mode_skip = 'None'
dropout_p_skip = dropout_p_down
dropout_mode_output = 'None'
dropout_p_output = dropout_p_down
net_input = get_noise(
input_depth, INPUT,
(img_pil.size[1], img_pil.size[0])).type(dtype).detach()
net_input_saved = net_input.detach().clone()
noise = net_input.detach().clone()
out_avg = None
last_net = None
mc_iter = 1
def closure_dip():
global i, out_avg, psnr_noisy_last, last_net, net_input, losses, psnrs, ssims, average_dropout_rate, no_layers,\
img_mean, sample_count, recons, uncerts, loss_last
if reg_noise_std > 0:
net_input = net_input_saved + (noise.normal_() * reg_noise_std)
out = net(net_input)
out[:, :1] = out[:, :1].sigmoid()
_loss = mse(out[:, :1], img_noisy_torch)
_loss.backward()
# Smoothing
if out_avg is None:
out_avg = out.detach()
else:
out_avg = out_avg * exp_weight + out.detach() * (1 - exp_weight)
losses.append(mse(out_avg[:, :1], img_noisy_torch).item())
_out = out.detach().cpu().numpy()[0, :1]
_out_avg = out_avg.detach().cpu().numpy()[0, :1]
psnr_noisy = compare_psnr(img_noisy_np, _out)
psnr_gt = compare_psnr(img_np, _out)
psnr_gt_sm = compare_psnr(img_np, _out_avg)
ssim_noisy = compare_ssim(img_noisy_np[0], _out[0])
ssim_gt = compare_ssim(img_np[0], _out[0])
ssim_gt_sm = compare_ssim(img_np[0], _out_avg[0])
psnrs.append([psnr_noisy, psnr_gt, psnr_gt_sm])
ssims.append([ssim_noisy, ssim_gt, ssim_gt_sm])
if PLOT and i % show_every == 0:
print(
f'Iteration: {i} Loss: {_loss.item():.4f} PSNR_noisy: {psnr_noisy:.4f} PSRN_gt: {psnr_gt:.4f} PSNR_gt_sm: {psnr_gt_sm:.4f}'
)
out_np = _out
psnr_noisy = compare_psnr(img_noisy_np, out_np)
psnr_gt = compare_psnr(img_np, out_np)
if sample_count != 0:
psnr_mean = compare_psnr(img_np, img_mean / sample_count)
else:
psnr_mean = 0
print('###################')
recons.append(out_np)
i += 1
return _loss
if '../NORMAL-4951060-8.jpeg':
net = get_net(input_depth,
NET_TYPE,
pad,
skip_n33d=skip_n33d,
skip_n33u=skip_n33u,
skip_n11=skip_n11,
num_scales=num_scales,
n_channels=1,
upsample_mode=upsample_mode,
dropout_mode_down=dropout_mode_down,
dropout_p_down=dropout_p_down,
dropout_mode_up=dropout_mode_up,
dropout_p_up=dropout_p_up,
dropout_mode_skip=dropout_mode_skip,
dropout_p_skip=dropout_p_skip,
dropout_mode_output=dropout_mode_output,
dropout_p_output=dropout_p_output).type(dtype)
else:
assert False
net.apply(init_normal)
losses = []
recons = []
uncerts = []
uncerts_ale = []
psnrs = []
ssims = []
img_mean = 0
sample_count = 0
i = 0
psnr_noisy_last = 0
loss_last = 1e16
parameters = get_params(OPT_OVER, net, net_input)
out_avg = None
optimizer = torch.optim.AdamW(parameters, lr=LR, weight_decay=weight_decay)
optimize(optimizer, closure_dip, num_iter)
LOSSES['dip'] = losses
RECONS['dip'] = recons
UNCERTS['dip'] = uncerts
UNCERTS_ALE['dip'] = uncerts_ale
PSNRS['dip'] = psnrs
SSIMS['dip'] = ssims
to_plot = [img_np] + [np.clip(img, 0, 1) for img in RECONS['dip']]
q = plot_image_grid(to_plot, factor=13)
out_pil = np_to_pil(q)
out_pil.save(f'{save_path}/{timestamp}/dip_recons.png', 'PNG')
## SGLD
weight_decay = 1e-4
LOSS = 'mse'
input_depth = 32
param_noise_sigma = 2
NET_TYPE = 'skip'
skip_n33d = 128
skip_n33u = 128
skip_n11 = 4
num_scales = 5
upsample_mode = 'bilinear'
dropout_mode_down = 'None'
dropout_p_down = 0.0
dropout_mode_up = 'None'
dropout_p_up = dropout_p_down
dropout_mode_skip = 'None'
dropout_p_skip = dropout_p_down
dropout_mode_output = 'None'
dropout_p_output = dropout_p_down
net_input = get_noise(
input_depth, INPUT,
(img_pil.size[1], img_pil.size[0])).type(dtype).detach()
net_input_saved = net_input.detach().clone()
noise = net_input.detach().clone()
mc_iter = 25
def add_noise(model):
for n in [x for x in model.parameters() if len(x.size()) == 4]:
noise = torch.randn(n.size()) * param_noise_sigma * LR
noise = noise.type(dtype)
n.data = n.data + noise
def closure_sgld():
global i, out_avg, psnr_noisy_last, last_net, net_input, losses, psnrs, ssims, average_dropout_rate, no_layers, img_mean, sample_count, recons, uncerts, loss_last
add_noise(net)
if reg_noise_std > 0:
net_input = net_input_saved + (noise.normal_() * reg_noise_std)
out = net(net_input)
out[:, :1] = out[:, :1].sigmoid()
_loss = mse(out[:, :1], img_noisy_torch)
_loss.backward()
# Smoothing
if out_avg is None:
out_avg = out.detach()
else:
out_avg = out_avg * exp_weight + out.detach() * (1 - exp_weight)
losses.append(mse(out_avg[:, :1], img_noisy_torch).item())
_out = out.detach().cpu().numpy()[0, :1]
_out_avg = out_avg.detach().cpu().numpy()[0, :1]
psnr_noisy = compare_psnr(img_noisy_np, _out)
psnr_gt = compare_psnr(img_np, _out)
psnr_gt_sm = compare_psnr(img_np, _out_avg)
ssim_noisy = compare_ssim(img_noisy_np[0], _out[0])
ssim_gt = compare_ssim(img_np[0], _out[0])
ssim_gt_sm = compare_ssim(img_np[0], _out_avg[0])
psnrs.append([psnr_noisy, psnr_gt, psnr_gt_sm])
ssims.append([ssim_noisy, ssim_gt, ssim_gt_sm])
if PLOT and i % show_every == 0:
print(
f'Iteration: {i} Loss: {_loss.item():.4f} PSNR_noisy: {psnr_noisy:.4f} PSRN_gt: {psnr_gt:.4f} PSNR_gt_sm: {psnr_gt_sm:.4f}'
)
out_np = _out
recons.append(out_np)
out_np_var = np.var(np.array(recons[-mc_iter:]), axis=0)[:1]
print('mean epi', out_np_var.mean())
print('###################')
uncerts.append(out_np_var)
i += 1
return _loss
if '../NORMAL-4951060-8.jpeg':
net = get_net(input_depth,
NET_TYPE,
pad,
skip_n33d=skip_n33d,
skip_n33u=skip_n33u,
skip_n11=skip_n11,
num_scales=num_scales,
n_channels=1,
upsample_mode=upsample_mode,
dropout_mode_down=dropout_mode_down,
dropout_p_down=dropout_p_down,
dropout_mode_up=dropout_mode_up,
dropout_p_up=dropout_p_up,
dropout_mode_skip=dropout_mode_skip,
dropout_p_skip=dropout_p_skip,
dropout_mode_output=dropout_mode_output,
dropout_p_output=dropout_p_output).type(dtype)
else:
assert False
net.apply(init_normal)
losses = []
recons = []
uncerts = []
uncerts_ale = []
psnrs = []
ssims = []
img_mean = 0
sample_count = 0
i = 0
psnr_noisy_last = 0
loss_last = 1e10
out_avg = None
last_net = None
parameters = get_params(OPT_OVER, net, net_input)
optimizer = torch.optim.AdamW(parameters, lr=LR, weight_decay=weight_decay)
optimize(optimizer, closure_sgld, num_iter)
LOSSES['sgld'] = losses
RECONS['sgld'] = recons
UNCERTS['sgld'] = uncerts
UNCERTS_ALE['sgld'] = uncerts_ale
PSNRS['sgld'] = psnrs
SSIMS['sgld'] = ssims
to_plot = [img_np] + [np.clip(img, 0, 1) for img in RECONS['sgld']]
q = plot_image_grid(to_plot, factor=13)
out_pil = np_to_pil(q)
out_pil.save(f'{save_path}/{timestamp}/sgld_recons.png', 'PNG')
errs = img_noisy_torch.cpu() - torch.tensor(RECONS['sgld'][-1])
uncerts_epi = torch.tensor(UNCERTS['sgld'][-1]).unsqueeze(0)
uncerts = uncerts_epi
uce, err, uncert, freq = uceloss(errs**2, uncerts, n_bins=21)
fig, ax = plot_uncert(err, uncert, freq, outlier_freq=0.001)
ax.set_title(
f'U = {uncerts.mean().sqrt().item():.4f}, UCE = {uce.item()*100:.3f}')
plt.tight_layout()
fig.savefig(f'{save_path}/{timestamp}/sgld_calib.png')
## SGLD + NLL
LOSS = 'nll'
net_input = get_noise(
input_depth, INPUT,
(img_pil.size[1], img_pil.size[0])).type(dtype).detach()
net_input_saved = net_input.detach().clone()
noise = net_input.detach().clone()
def closure_sgldnll():
global i, out_avg, psnr_noisy_last, last_net, net_input, losses, psnrs, ssims, average_dropout_rate, no_layers,\
img_mean, sample_count, recons, uncerts, uncerts_ale, loss_last
add_noise(net)
if reg_noise_std > 0:
net_input = net_input_saved + (noise.normal_() * reg_noise_std)
out = net(net_input)
out[:, :1] = out[:, :1].sigmoid()
_loss = gaussian_nll(out[:, :1], out[:, 1:], img_noisy_torch)
_loss.backward()
out[:, 1:] = torch.exp(-out[:, 1:]) # aleatoric uncertainty
# Smoothing
if out_avg is None:
out_avg = out.detach()
else:
out_avg = out_avg * exp_weight + out.detach() * (1 - exp_weight)
with torch.no_grad():
mse_loss = mse(out_avg[:, :1], img_noisy_torch).item()
losses.append(mse_loss)
_out = out.detach().cpu().numpy()[0, :1]
_out_avg = out_avg.detach().cpu().numpy()[0, :1]
psnr_noisy = compare_psnr(img_noisy_np, _out)
psnr_gt = compare_psnr(img_np, _out)
psnr_gt_sm = compare_psnr(img_np, _out_avg)
ssim_noisy = compare_ssim(img_noisy_np[0], _out[0])
ssim_gt = compare_ssim(img_np[0], _out[0])
ssim_gt_sm = compare_ssim(img_np[0], _out_avg[0])
psnrs.append([psnr_noisy, psnr_gt, psnr_gt_sm])
ssims.append([ssim_noisy, ssim_gt, ssim_gt_sm])
if PLOT and i % show_every == 0:
print(
f'Iteration: {i} Loss: {_loss.item():.4f} PSNR_noisy: {psnr_noisy:.4f} PSRN_gt: {psnr_gt:.4f} PSNR_gt_sm: {psnr_gt_sm:.4f}'
)
out_np = _out
recons.append(out_np)
out_np_ale = out.detach().cpu().numpy()[0, 1:]
out_np_var = np.var(np.array(recons[-mc_iter:]), axis=0)[:1]
print('mean epi', out_np_var.mean())
print('mean ale', out_np_ale.mean())
print('###################')
uncerts.append(out_np_var)
uncerts_ale.append(out_np_ale)
i += 1
return _loss
if '../NORMAL-4951060-8.jpeg':
net = get_net(input_depth,
NET_TYPE,
pad,
skip_n33d=skip_n33d,
skip_n33u=skip_n33u,
skip_n11=skip_n11,
num_scales=num_scales,
n_channels=2,
upsample_mode=upsample_mode,
dropout_mode_down=dropout_mode_down,
dropout_p_down=dropout_p_down,
dropout_mode_up=dropout_mode_up,
dropout_p_up=dropout_p_up,
dropout_mode_skip=dropout_mode_skip,
dropout_p_skip=dropout_p_skip,
dropout_mode_output=dropout_mode_output,
dropout_p_output=dropout_p_output).type(dtype)
else:
assert False
net.apply(init_normal)
losses = []
recons = []
uncerts = []
uncerts_ale = []
psnrs = []
ssims = []
img_mean = 0
sample_count = 0
i = 0
psnr_noisy_last = 0
loss_last = 1e6
out_avg = None
last_net = None
parameters = get_params(OPT_OVER, net, net_input)
optimizer = torch.optim.AdamW(parameters, lr=LR, weight_decay=weight_decay)
optimize(optimizer, closure_sgldnll, num_iter)
LOSSES['sgldnll'] = losses
RECONS['sgldnll'] = recons
UNCERTS['sgldnll'] = uncerts
UNCERTS_ALE['sgldnll'] = uncerts_ale
PSNRS['sgldnll'] = psnrs
SSIMS['sgldnll'] = ssims
to_plot = [img_np] + [np.clip(img, 0, 1) for img in RECONS['sgldnll']]
q = plot_image_grid(to_plot, factor=13)
out_pil = np_to_pil(q)
out_pil.save(f'{save_path}/{timestamp}/sgldnll_recons.png', 'PNG')
errs = img_noisy_torch.cpu() - torch.tensor(RECONS['sgldnll'][-1])
uncerts_epi = torch.tensor(UNCERTS['sgldnll'][-1]).unsqueeze(0)
uncerts_ale = torch.tensor(UNCERTS_ALE['sgldnll'][-1]).unsqueeze(0)
uncerts = uncerts_epi + uncerts_ale
uce, err, uncert, freq = uceloss(errs**2, uncerts, n_bins=21)
fig, ax = plot_uncert(err, uncert, freq, outlier_freq=0.001)
ax.set_title(
f'U = {uncerts.mean().sqrt().item():.4f}, UCE = {uce.item()*100:.3f}')
plt.tight_layout()
fig.savefig(f'{save_path}/{timestamp}/sgldnll_calib.png')
errs = torch.tensor(img_np).unsqueeze(0) - torch.tensor(
RECONS['sgldnll'][-1])
uncerts_epi = torch.tensor(UNCERTS['sgldnll'][-1]).unsqueeze(0)
uncerts_ale = torch.tensor(UNCERTS_ALE['sgldnll'][-1]).unsqueeze(0)
uncerts = uncerts_epi + uncerts_ale
uce, err, uncert, freq = uceloss(errs**2, uncerts, n_bins=21)
fig, ax = plot_uncert(err, uncert, freq, outlier_freq=0.001)
ax.set_title(
f'U = {uncerts.mean().sqrt().item():.4f}, UCE = {uce.item()*100:.3f}')
plt.tight_layout()
fig.savefig(f'{save_path}/{timestamp}/sgldnll_calib2.png')
## MCDIP
OPTIMIZER = 'adamw'
weight_decay = 1e-4
LOSS = 'nll'
input_depth = 32
figsize = 4
NET_TYPE = 'skip'
skip_n33d = 128
skip_n33u = 128
skip_n11 = 4
num_scales = 5
upsample_mode = 'bilinear'
dropout_mode_down = '2d'
dropout_p_down = 0.3
dropout_mode_up = '2d'
dropout_p_up = dropout_p_down
dropout_mode_skip = 'None'
dropout_p_skip = dropout_p_down
dropout_mode_output = 'None'
dropout_p_output = dropout_p_down
net_input = get_noise(
input_depth, INPUT,
(img_pil.size[1], img_pil.size[0])).type(dtype).detach()
net_input_saved = net_input.detach().clone()
noise = net_input.detach().clone()
mc_iter = 25
def closure_mcdip():
global i, out_avg, psnr_noisy_last, last_net, net_input, losses, psnrs, ssims, average_dropout_rate, no_layers,\
img_mean, sample_count, recons, uncerts, uncerts_ale, loss_last
if reg_noise_std > 0:
net_input = net_input_saved + (noise.normal_() * reg_noise_std)
out = net(net_input)
out[:, :1] = out[:, :1].sigmoid()
_loss = gaussian_nll(out[:, :1], out[:, 1:], img_noisy_torch)
_loss.backward()
out[:, 1:] = torch.exp(-out[:, 1:]) # aleatoric uncertainty
# Smoothing
if out_avg is None:
out_avg = out.detach()
else:
out_avg = out_avg * exp_weight + out.detach() * (1 - exp_weight)
losses.append(mse(out_avg[:, :1], img_noisy_torch).item())
_out = out.detach().cpu().numpy()[0, :1]
_out_avg = out_avg.detach().cpu().numpy()[0, :1]
psnr_noisy = compare_psnr(img_noisy_np, _out)
psnr_gt = compare_psnr(img_np, _out)
psnr_gt_sm = compare_psnr(img_np, _out_avg)
ssim_noisy = compare_ssim(img_noisy_np[0], _out[0])
ssim_gt = compare_ssim(img_np[0], _out[0])
ssim_gt_sm = compare_ssim(img_np[0], _out_avg[0])
psnrs.append([psnr_noisy, psnr_gt, psnr_gt_sm])
ssims.append([ssim_noisy, ssim_gt, ssim_gt_sm])
if PLOT and i % show_every == 0:
print(
f'Iteration: {i} Loss: {_loss.item():.4f} PSNR_noisy: {psnr_noisy:.4f} PSRN_gt: {psnr_gt:.4f} PSNR_gt_sm: {psnr_gt_sm:.4f}'
)
img_list = []
aleatoric_list = []
with torch.no_grad():
net_input = net_input_saved + (noise.normal_() * reg_noise_std)
for _ in range(mc_iter):
img = net(net_input)
img[:, :1] = img[:, :1].sigmoid()
img[:, 1:] = torch.exp(-img[:, 1:])
img_list.append(torch_to_np(img[:1]))
aleatoric_list.append(torch_to_np(img[:, 1:]))
img_list_np = np.array(img_list)
out_np = np.mean(img_list_np, axis=0)[:1]
out_np_ale = np.mean(aleatoric_list, axis=0)[:1]
out_np_var = np.var(img_list_np, axis=0)[:1]
psnr_noisy = compare_psnr(img_noisy_np, out_np)
psnr_gt = compare_psnr(img_np, out_np)
print('mean epi', out_np_var.mean())
print('mean ale', out_np_ale.mean())
print('###################')
recons.append(out_np)
uncerts.append(out_np_var)
uncerts_ale.append(out_np_ale)
i += 1
return _loss
if '../NORMAL-4951060-8.jpeg':
net = get_net(input_depth,
NET_TYPE,
pad,
skip_n33d=skip_n33d,
skip_n33u=skip_n33u,
skip_n11=skip_n11,
num_scales=num_scales,
n_channels=2,
upsample_mode=upsample_mode,
dropout_mode_down=dropout_mode_down,
dropout_p_down=dropout_p_down,
dropout_mode_up=dropout_mode_up,
dropout_p_up=dropout_p_up,
dropout_mode_skip=dropout_mode_skip,
dropout_p_skip=dropout_p_skip,
dropout_mode_output=dropout_mode_output,
dropout_p_output=dropout_p_output).type(dtype)
else:
assert False
net.apply(init_normal)
losses = []
recons = []
uncerts = []
uncerts_ale = []
psnrs = []
ssims = []
img_mean = 0
sample_count = 0
i = 0
psnr_noisy_last = 0
loss_last = 1e16
out_avg = None
last_net = None
parameters = get_params(OPT_OVER, net, net_input)
optimizer = torch.optim.AdamW(parameters, lr=LR, weight_decay=weight_decay)
optimize(optimizer, closure_mcdip, num_iter)
LOSSES['mcdip'] = losses
RECONS['mcdip'] = recons
UNCERTS['mcdip'] = uncerts
UNCERTS_ALE['mcdip'] = uncerts_ale
PSNRS['mcdip'] = psnrs
SSIMS['mcdip'] = ssims
# In[75]:
to_plot = [img_np] + [np.clip(img, 0, 1) for img in RECONS['mcdip']]
q = plot_image_grid(to_plot, factor=13)
out_pil = np_to_pil(q)
out_pil.save(f'{save_path}/{timestamp}/mcdip_recons.png', 'PNG')
# In[85]:
errs = img_noisy_torch.cpu() - torch.tensor(RECONS['mcdip'][-1])
uncerts_epi = torch.tensor(UNCERTS['mcdip'][-1]).unsqueeze(0)
uncerts_ale = torch.tensor(UNCERTS_ALE['mcdip'][-1]).unsqueeze(0)
uncerts = uncerts_epi + uncerts_ale
uce, err, uncert, freq = uceloss(errs**2, uncerts, n_bins=21)
fig, ax = plot_uncert(err, uncert, freq, outlier_freq=0.001)
ax.set_title(
f'U = {uncerts.mean().sqrt().item():.4f}, UCE = {uce.item()*100:.3f}')
plt.tight_layout()
fig.savefig(f'{save_path}/{timestamp}/mcdip_calib.png')
# In[86]:
errs = torch.tensor(img_np).unsqueeze(0) - torch.tensor(
RECONS['mcdip'][-1])
uncerts_epi = torch.tensor(UNCERTS['mcdip'][-1]).unsqueeze(0)
uncerts_ale = torch.tensor(UNCERTS_ALE['mcdip'][-1]).unsqueeze(0)
uncerts = uncerts_epi + uncerts_ale
uce, err, uncert, freq = uceloss(errs**2, uncerts, n_bins=21)
fig, ax = plot_uncert(err, uncert, freq, outlier_freq=0.001)
ax.set_title(
f'U = {uncerts.mean().sqrt().item():.4f}, UCE = {uce.item()*100:.3f}')
plt.tight_layout()
fig.savefig(f'{save_path}/{timestamp}/mcdip_calib2.png')
fig, ax0 = plt.subplots(1, 1)
for key, loss in LOSSES.items():
ax0.plot(range(len(loss)), loss, label=key)
ax0.set_title('MSE')
ax0.set_xlabel('iteration')
ax0.set_ylabel('mse loss')
ax0.set_ylim(0, 0.03)
ax0.grid(True)
ax0.legend()
plt.tight_layout()
plt.savefig(f'{save_path}/{timestamp}/losses.png')
plt.show()
fig, axs = plt.subplots(1, 3, constrained_layout=True)
labels = ["psnr_noisy", "psnr_gt", "psnr_gt_sm"]
for key, psnr in PSNRS.items():
psnr = np.array(psnr)
for i in range(psnr.shape[1]):
axs[i].plot(range(psnr.shape[0]), psnr[:, i], label=key)
axs[i].set_title(labels[i])
axs[i].set_xlabel('iteration')
axs[i].set_ylabel('psnr')
axs[i].legend()
plt.savefig(f'{save_path}/{timestamp}/psnrs.png')
plt.show()
fig, axs = plt.subplots(1, 3, constrained_layout=True)
labels = ["ssim_noisy", "ssim_gt", "ssim_gt_sm"]
for key, ssim in SSIMS.items():
ssim = np.array(ssim)
for i in range(ssim.shape[1]):
axs[i].plot(range(ssim.shape[0]), ssim[:, i], label=key)
axs[i].set_title(labels[i])
axs[i].set_xlabel('iteration')
axs[i].legend()
axs[i].set_ylabel('ssim')
plt.savefig(f'{save_path}/{timestamp}/ssims.png')
plt.show()
# save stuff for plotting
if save:
np.savez(f"{save_path}/{timestamp}/save.npz",
noisy_img=img_noisy_np,
losses=LOSSES,
recons=RECONS,
uncerts=UNCERTS,
uncerts_ale=UNCERTS_ALE,
psnrs=PSNRS,
ssims=SSIMS)
def main(args: APNamespace):
root_path = Path(args.root).expanduser()
config_path = Path(args.config).expanduser()
data_path = root_path / Path(args.data).expanduser()
output_path = root_path / Path(args.output).expanduser()
global checkpoint_path, config
checkpoint_path = root_path / Path(args.checkpoint).expanduser()
if not config_path.exists():
# logging.critical(f"AdaS: Config path {config_path} does not exist")
print(f"AdaS: Config path {config_path} does not exist")
raise ValueError
if not data_path.exists():
print(f"AdaS: Data dir {data_path} does not exist, building")
data_path.mkdir(exist_ok=True, parents=True)
if not output_path.exists():
print(f"AdaS: Output dir {output_path} does not exist, building")
output_path.mkdir(exist_ok=True, parents=True)
if not checkpoint_path.exists():
if args.resume:
print(f"AdaS: Cannot resume from checkpoint without specifying " +
"checkpoint dir")
raise ValueError
checkpoint_path.mkdir(exist_ok=True, parents=True)
with config_path.open() as f:
config = yaml.load(f)
print("Adas: Argument Parser Options")
print("-" * 45)
print(f" {'config':<20}: {args.config:<40}")
print(f" {'data':<20}: {str(Path(args.root) / args.data):<40}")
print(f" {'output':<20}: {str(Path(args.root) / args.output):<40}")
print(f" {'checkpoint':<20}: " +
f"{str(Path(args.root) / args.checkpoint):<40}")
print(f" {'root':<20}: {args.root:<40}")
print(f" {'resume':<20}: {'True' if args.resume else 'False':<20}")
print("\nAdas: Train: Config")
print(f" {'Key':<20} {'Value':<20}")
print("-" * 45)
for k, v in config.items():
print(f" {k:<20} {v:<20}")
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print(f"AdaS: Pytorch device is set to {device}")
global best_acc
best_acc = 0 # best test accuracy
start_epoch = 0 # start from epoch 0 or last checkpoint epoch
if np.less(float(config['early_stop_threshold']), 0):
print("AdaS: Notice: early stop will not be used as it was set to " +
f"{config['early_stop_threshold']}, training till completion.")
for trial in range(config['n_trials']):
if config['lr_scheduler'] == 'AdaS':
filename = \
f"stats_{config['optim_method']}_AdaS_trial={trial}_" +\
f"beta={config['beta']}_initlr={config['init_lr']}_" +\
f"net={config['network']}_dataset={config['dataset']}.csv"
else:
filename = \
f"stats_{config['optim_method']}_{config['lr_scheduler']}_" +\
f"trial={trial}_initlr={config['init_lr']}" +\
f"net={config['network']}_dataset={config['dataset']}.csv"
Profiler.filename = output_path / filename
device
# Data
# logging.info("Adas: Preparing Data")
train_loader, test_loader = get_data(
root=data_path,
dataset=config['dataset'],
mini_batch_size=config['mini_batch_size'])
global performance_statistics, net, metrics, adas
performance_statistics = {}
# logging.info("AdaS: Building Model")
net = get_net(config['network'],
num_classes=10 if config['dataset'] == 'CIFAR10' else
100 if config['dataset'] == 'CIFAR100' else
1000 if config['dataset'] == 'ImageNet' else 10)
metrics = Metrics(list(net.parameters()), p=config['p'])
if config['lr_scheduler'] == 'AdaS':
adas = AdaS(parameters=list(net.parameters()),
beta=config['beta'],
zeta=config['zeta'],
init_lr=float(config['init_lr']),
min_lr=float(config['min_lr']),
p=config['p'])
net = net.to(device)
global criterion
criterion = get_loss(config['loss'])
optimizer, scheduler = get_optimizer_scheduler(
net_parameters=net.parameters(),
init_lr=float(config['init_lr']),
optim_method=config['optim_method'],
lr_scheduler=config['lr_scheduler'],
train_loader_len=len(train_loader),
max_epochs=int(config['max_epoch']))
early_stop = EarlyStop(patience=int(config['early_stop_patience']),
threshold=float(config['early_stop_threshold']))
if device == 'cuda':
net = torch.nn.DataParallel(net)
cudnn.benchmark = True
if args.resume:
# Load checkpoint.
print("Adas: Resuming from checkpoint...")
checkpoint = torch.load(str(checkpoint_path / 'ckpt.pth'))
# if checkpoint_path.is_dir():
# checkpoint = torch.load(str(checkpoint_path / 'ckpt.pth'))
# else:
# checkpoint = torch.load(str(checkpoint_path))
net.load_state_dict(checkpoint['net'])
best_acc = checkpoint['acc']
start_epoch = checkpoint['epoch']
if adas is not None:
metrics.historical_metrics = \
checkpoint['historical_io_metrics']
# model_parameters = filter(lambda p: p.requires_grad,
# net.parameters())
# params = sum([np.prod(p.size()) for p in model_parameters])
# print(params)
epochs = range(start_epoch, start_epoch + config['max_epoch'])
for epoch in epochs:
start_time = time.time()
# print(f"AdaS: Epoch {epoch}/{epochs[-1]} Started.")
train_loss, train_accuracy, test_loss, test_accuracy = epoch_iteration(
train_loader, test_loader, epoch, device, optimizer, scheduler)
end_time = time.time()
if config['lr_scheduler'] == 'StepLR':
scheduler.step()
total_time = time.time()
print(
f"AdaS: Trial {trial}/{config['n_trials'] - 1} | " +
f"Epoch {epoch}/{epochs[-1]} Ended | " +
"Total Time: {:.3f}s | ".format(total_time - start_time) +
"Epoch Time: {:.3f}s | ".format(end_time - start_time) +
"~Time Left: {:.3f}s | ".format(
(total_time - start_time) * (epochs[-1] - epoch)),
"Train Loss: {:.4f}% | Train Acc. {:.4f}% | ".format(
train_loss, train_accuracy) +
"Test Loss: {:.4f}% | Test Acc. {:.4f}%".format(
test_loss, test_accuracy))
df = pd.DataFrame(data=performance_statistics)
if config['lr_scheduler'] == 'AdaS':
xlsx_name = \
f"{config['optim_method']}_AdaS_trial={trial}_" +\
f"beta={config['beta']}_initlr={config['init_lr']}_" +\
f"net={config['network']}_dataset={config['dataset']}.xlsx"
else:
xlsx_name = \
f"{config['optim_method']}_{config['lr_scheduler']}_" +\
f"trial={trial}_initlr={config['init_lr']}" +\
f"net={config['network']}_dataset={config['dataset']}.xlsx"
df.to_excel(str(output_path / xlsx_name))
if early_stop(train_loss):
print("AdaS: Early stop activated.")
break
return
def main():
config = DefaultConfigs()
train_input_root = os.path.join(config.data)
train_labels_file = 'labels.csv'
if config.output:
if not os.path.exists(config.output):
os.makedirs(config.output)
output_base = config.output
else:
if not os.path.exists(config.output):
os.makedirs(config.output)
output_base = config.output
exp_name = '-'.join([
datetime.now().strftime("%Y%m%d-%H%M%S"), config.model,
str(config.img_size), 'f' + str(config.fold)
])
mask_exp_name = '-'.join(
[config.model,
str(config.img_size), 'f' + str(config.fold)])
mask_exp_name = glob.glob(
os.path.join(output_base, 'train', '*' + mask_exp_name))
if config.resume and mask_exp_name:
output_dir = mask_exp_name
else:
output_dir = get_outdir(output_base, 'train', exp_name)
batch_size = config.batch_size
test_batch_size = config.test_batch_size
num_epochs = config.epochs
img_type = config.image_type
img_size = (config.img_size, config.img_size)
num_classes = get_tags_size(config.labels)
torch.manual_seed(config.seed)
dataset_train = HumanDataset(
train_input_root,
train_labels_file,
train=True,
multi_label=config.multi_label,
img_type=img_type,
img_size=img_size,
fold=config.fold,
)
#sampler = WeightedRandomOverSampler(dataset_train.get_sample_weights())
loader_train = data.DataLoader(
dataset_train,
batch_size=batch_size,
shuffle=True,
#sampler=sampler,
num_workers=config.num_processes)
dataset_eval = HumanDataset(
train_input_root,
train_labels_file,
train=False,
multi_label=config.multi_label,
img_type=img_type,
img_size=img_size,
test_aug=config.tta,
fold=config.fold,
)
loader_eval = data.DataLoader(dataset_eval,
batch_size=test_batch_size,
shuffle=False,
num_workers=config.num_processes)
# model = model_factory.create_model(
# config.model,
# pretrained=True,
# num_classes=num_classes,
# drop_rate=config.drop,
# global_pool=config.gp)
model = get_net(config.model, num_classes, config.drop, config.channels)
if not config.no_cuda:
if config.num_gpu > 1:
model = torch.nn.DataParallel(model,
device_ids=list(range(
config.num_gpu))).cuda()
else:
model.cuda()
if config.opt.lower() == 'sgd':
optimizer = optim.SGD(model.parameters(),
lr=config.lr,
momentum=config.momentum,
weight_decay=config.weight_decay)
elif config.opt.lower() == 'adam':
optimizer = optim.Adam(model.parameters(),
lr=config.lr,
weight_decay=config.weight_decay)
elif config.opt.lower() == 'adadelta':
optimizer = optim.Adadelta(model.parameters(),
lr=config.lr,
weight_decay=config.weight_decay)
elif config.opt.lower() == 'rmsprop':
optimizer = optim.RMSprop(model.parameters(),
lr=config.lr,
alpha=0.9,
momentum=config.momentum,
weight_decay=config.weight_decay)
elif config.opt.lower() == 'yellowfin':
optimizer = YFOptimizer(model.parameters(),
lr=config.lr,
weight_decay=config.weight_decay,
clip_thresh=2)
else:
assert False and "Invalid optimizer"
if not config.decay_epochs:
lr_scheduler = ReduceLROnPlateau(optimizer, patience=8)
else:
lr_scheduler = None
if config.class_weights:
class_weights = torch.from_numpy(
dataset_train.get_class_weights()).float()
class_weights_norm = class_weights / class_weights.sum()
if not config.no_cuda:
class_weights = class_weights.cuda()
class_weights_norm = class_weights_norm.cuda()
else:
class_weights = None
class_weights_norm = None
if config.loss.lower() == 'nll':
#assert not args.multi_label and 'Cannot use crossentropy with multi-label target.'
loss_fn = torch.nn.CrossEntropyLoss(weight=class_weights)
elif config.loss.lower() == 'mlsm':
assert config.multi_label
loss_fn = torch.nn.MultiLabelSoftMarginLoss(weight=class_weights)
else:
assert config and "Invalid loss function"
if not config.no_cuda:
loss_fn = loss_fn.cuda()
# optionally resume from a checkpoint
start_epoch = 1
if config.resume:
if os.path.isfile(config.resume):
print("=> loading checkpoint '{}'".format(config.resume))
checkpoint = torch.load(config.resume)
config.start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
config.resume, checkpoint['epoch']))
start_epoch = checkpoint['epoch']
else:
print("=> no checkpoint found at '{}'".format(config.resume))
exit(-1)
use_tensorboard = not config.no_tb and CrayonClient is not None
if use_tensorboard:
hostname = '127.0.0.1'
port = 8889
host_port = config.tbh.split(':')[:2]
if len(host_port) == 1:
hostname = host_port[0]
elif len(host_port) >= 2:
hostname, port = host_port[:2]
try:
cc = CrayonClient(hostname=hostname, port=port)
try:
cc.remove_experiment(exp_name)
except ValueError:
pass
exp = cc.create_experiment(exp_name)
except Exception as e:
exp = None
print(
"Error (%s) connecting to Tensoboard/Crayon server. Giving up..."
% str(e))
else:
exp = None
# Optional fine-tune of only the final classifier weights for specified number of epochs (or part of)
if not config.resume and config.ft_epochs > 0.:
if config.opt.lower() == 'adam':
finetune_optimizer = optim.Adam(model.get_fc().parameters(),
lr=config.ft_lr,
weight_decay=config.weight_decay)
else:
finetune_optimizer = optim.SGD(model.get_fc().parameters(),
lr=config.ft_lr,
momentum=config.momentum,
weight_decay=config.weight_decay)
finetune_epochs_int = int(np.ceil(config.ft_epochs))
finetune_final_batches = int(
np.ceil((1 - (finetune_epochs_int - config.ft_epochs)) *
len(loader_train)))
print(finetune_epochs_int, finetune_final_batches)
for fepoch in range(1, finetune_epochs_int + 1):
if fepoch == finetune_epochs_int and finetune_final_batches:
batch_limit = finetune_final_batches
else:
batch_limit = 0
train_epoch(fepoch,
model,
loader_train,
finetune_optimizer,
loss_fn,
config,
class_weights_norm,
output_dir,
batch_limit=batch_limit)
step = fepoch * len(loader_train)
score, _ = validate(step, model, loader_eval, loss_fn, config, 0.3,
output_dir)
score_metric = 'f2'
best_loss = None
best_f2 = None
threshold = 0.2
try:
for epoch in range(start_epoch, num_epochs + 1):
if config.decay_epochs:
adjust_learning_rate(optimizer,
epoch,
initial_lr=config.lr,
decay_epochs=config.decay_epochs)
train_metrics = train_epoch(epoch,
model,
loader_train,
optimizer,
loss_fn,
config,
class_weights_norm,
output_dir,
exp=exp)
step = epoch * len(loader_train)
eval_metrics, latest_threshold = validate(step,
model,
loader_eval,
loss_fn,
config,
threshold,
output_dir,
exp=exp)
if lr_scheduler is not None:
lr_scheduler.step(eval_metrics['eval_loss'])
rowd = OrderedDict(epoch=epoch)
rowd.update(train_metrics)
rowd.update(eval_metrics)
with open(os.path.join(output_dir, 'summary.csv'), mode='a') as cf:
dw = csv.DictWriter(cf, fieldnames=rowd.keys())
if best_loss is None: # first iteration (epoch == 1 can't be used)
dw.writeheader()
dw.writerow(rowd)
best = False
if best_loss is None or eval_metrics['eval_loss'] < best_loss[1]:
best_loss = (epoch, eval_metrics['eval_loss'])
if score_metric == 'loss':
best = True
if best_f2 is None or eval_metrics['eval_f2'] > best_f2[1]:
best_f2 = (epoch, eval_metrics['eval_f2'])
if score_metric == 'f2':
best = True
save_checkpoint(
{
'epoch': epoch + 1,
'arch': config.model,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'threshold': latest_threshold,
'config': config
},
is_best=best,
filename=os.path.join(config.checkpoint_path,
'checkpoint-%d.pth.tar' % epoch),
output_dir=output_dir)
except KeyboardInterrupt:
pass
print('*** Best loss: {0} (epoch {1})'.format(best_loss[1], best_loss[0]))
print('*** Best f2: {0} (epoch {1})'.format(best_f2[1], best_f2[0]))
print("\nload model relu for second run")
train_result_model_batch = train_and_evaluate_model(
model_batch_norm, batch_size2, epochs3, BaseX_train, BaseY_train,
X_test, Y_test, BaseX_val, BaseY_val)
# plot_train_stat(train_result_model_batch)
"""Part 2"""
input_shape = (32, 32, 1)
learn_rate = 1e-5
decay = 1e-03
batch_size = 64
epochs = 25
drop = True
dropRate = 0.3
reg = 1e-2
filters1 = [64, 128, 128, 256, 256]
NNet1 = get_net(filters1, input_shape, drop, dropRate, reg)
# Defining the optimizar parameters:
AdamOpt = Adam(lr=learn_rate, decay=decay)
# Compile the network:
NNet1.compile(optimizer=AdamOpt,
metrics=['acc'],
loss='categorical_crossentropy')
# Saving checkpoints during training:
# Checkpath = os.getcwd()
h1 = NNet1.fit(x=BaseX_train,
y=BaseY_train,
batch_size=batch_size,
"raccoons. baseline accuracy", print_statement)
print("x_test: ", len(x_test))
print("y_test: ", y_test.shape[0])
unique, counts = np.unique(y_test, return_counts=True)
if counts[0] >= counts[1]:
baseline_acc = counts[0] / (counts[0] + counts[1])
print_statement = str(round(baseline_acc, 2)) + " (always coyote)."
else:
baseline_acc = counts[0] / (counts[0] + counts[1])
print_statement = str(round(baseline_acc, 2)) + " (always coyote)."
print("test class distribution: ", counts[0], "coyotes and", counts[1],
"raccoons. baseline accuracy", print_statement)
# get model and data handler
net = get_net(dataset_name)
handler = get_handler(dataset_name)
# GPU enabled?
print("Using GPU - {}".format(torch.cuda.is_available()))
final_train_accs = []
final_test_accs = []
train_process = Train(envs, x_test, y_test, net, handler, args)
print()
if args['optimizer_args']['penalty_weight'] > 1.0:
model_name = model_name + "IRM"
print(
"========================================IRM========================================"
)
def denoise(fname, plot=False):
"""Add AWGN with sigma=25 to the given image and denoise it.
Args:
fname: Path to the image.
mode: Stopping mode to use. either "AMNS", "SMNS", or "static".
Returns:
A tuple with the denoised image in numpy format as the first element,
and a history of the PSNR in the second element.
"""
dtype = torch.cuda.FloatTensor
sigma = 25
sigma_ = sigma/255.
imsize = -1
np.random.seed(7)
img_pil = crop_image(get_image(fname, imsize)[0], d=32)
img_np = pil_to_np(img_pil)
img_noisy_pil, img_noisy_np = get_noisy_image(img_np, sigma_)
if plot:
plot_image_grid([img_np, img_noisy_np], 4, 6)
INPUT = 'noise' # 'meshgrid'
pad = 'reflection'
OPT_OVER = 'net' # 'net,input'
reg_noise_std = 1./30. # set to 1./20. for sigma=50
LR = 0.01
exp_weight = 0.99 # Exponential averaging coefficient
OPTIMIZER = 'adam' # 'LBFGS'
show_every = 500
num_iter = 1800
input_depth = 32
figsize = 4
net = get_net(input_depth, 'skip', pad,
skip_n33d=128,
skip_n33u=128,
skip_n11=4,
num_scales=5,
upsample_mode='bilinear').type(dtype)
net_input = get_noise(input_depth, INPUT, (img_np.shape[1], img_np.shape[2])).type(dtype).detach()
# Compute number of parameters
s = sum([np.prod(list(p.size())) for p in net.parameters()])
print('Number of params: %d' % s)
# Loss
mse = torch.nn.MSELoss().type(dtype)
img_noisy_torch = np_to_torch(img_noisy_np).type(dtype)
net_input_saved = net_input.detach().clone()
noise = net_input.detach().clone()
out_avg = None
last_net = None
psrn_noisy_last = 0
i = 0
psnr_history = []
def closure():
nonlocal i, out_avg, psrn_noisy_last, last_net, psnr_history
if reg_noise_std > 0:
net_input = net_input_saved + (noise.normal_() * reg_noise_std)
out = net(net_input)
# Smoothing
if exp_weight is not None:
if out_avg is None:
out_avg = out.detach()
else:
out_avg = out_avg * exp_weight + out.detach() * (1 - exp_weight)
total_loss = mse(out, img_noisy_torch)
total_loss.backward()
psrn_noisy = compare_psnr(img_noisy_np, out.detach().cpu().numpy()[0])
psrn_gt = compare_psnr(img_np, out.detach().cpu().numpy()[0])
psrn_gt_sm = compare_psnr(img_np, out_avg.detach().cpu().numpy()[0])
psnr_history.append(psrn_gt_sm)
print('Iteration %05d Loss %f PSNR_noisy: %f PSRN_gt: %f PSNR_gt_sm: %f' % (i, total_loss.item(), psrn_noisy, psrn_gt, psrn_gt_sm), '\r', end='')
if plot and i % show_every == 0:
out_np = torch_to_np(out)
plot_image_grid([np.clip(out_np, 0, 1), np.clip(torch_to_np(out_avg), 0, 1)], factor=figsize, nrow=1)
# Backtracking
if i % show_every:
if psrn_noisy - psrn_noisy_last < -5:
print('Falling back to previous checkpoint.')
for new_param, net_param in zip(last_net, net.parameters()):
net_param.data.copy_(new_param.cuda())
return total_loss*0
else:
last_net = [x.data.cpu() for x in net.parameters()]
psrn_noisy_last = psrn_noisy
i += 1
return total_loss
p = get_params(OPT_OVER, net, net_input)
try:
optimize(OPTIMIZER, p, closure, LR, num_iter)
except StopIteration:
pass
return out_avg, psnr_history