def train():
# Init data
train_dataset, val_dataset = prepare_datasets()
train_loader = DataLoader(train_dataset, batch_size=10, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=10, shuffle=True)
loaders = dict(train=train_loader, val=val_loader)
# Init Model
model = UNet().cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3, amsgrad=True)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer,
gamma=0.984)
loss_fn = nn.BCELoss()
epochs = 500
for epoch in range(epochs):
for phase in 'train val'.split():
if phase == 'train':
model = model.train()
torch.set_grad_enabled(True)
else:
model = model.eval()
torch.set_grad_enabled(False)
loader = loaders[phase]
epoch_losses = dict(train=[], val=[])
running_loss = []
for batch in loader:
imgs, masks = batch
imgs = imgs.cuda()
masks = masks.cuda()
outputs = model(imgs)
loss = loss_fn(outputs, masks)
running_loss.append(loss.item())
if phase == 'train':
optimizer.zero_grad()
loss.backward()
optimizer.step()
# End of Epoch
print(f'{epoch}) {phase} loss: {np.mean(running_loss)}')
visualize_results(loader, model, epoch, phase)
epoch_losses[phase].append(np.mean(running_loss))
tensorboard(epoch_losses[phase], phase)
if phase == 'train':
scheduler.step()
partition = 'train'
unet_train = HistologyData(ROOT_DIR, partition, True)
unet_loader = torch.utils.data.DataLoader(
unet_train,
batch_size=1,
shuffle=True,
)
# Create model
model = ShapeUNet((15, 512, 512))
unet = UNet((3, 512, 512))
model.to(device)
unet.to(device)
mask_values = (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)
# here not RGB but BGR because of OPENCV.
real_colors = ((0, 0, 0), (255, 0, 0), (0, 255, 0), (0, 0, 255), (85, 0, 0),
(0, 170, 0), (255, 0, 127), (0, 255, 255), (0, 85, 0),
(255, 0, 255), (255, 85, 0), (255, 165, 0), (255, 255, 0),
(128, 130, 128), (128, 190, 190))
lr = 1e-4
optimizer = Adam(model.parameters(), lr=lr)
NUM_OF_EPOCHS = 40
lr1 = 1e-4
unet_optim = Adam(unet.parameters(), lr=lr1)
train_network_on_top_of_other(model, train_loader, val_loader, optimizer, unet,
unet_loader, unet_optim, NUM_OF_EPOCHS,
'weights/')
def train(frame_num,
layer_nums,
input_channels,
output_channels,
discriminator_num_filters,
bn=False,
pretrain=False,
generator_pretrain_path=None,
discriminator_pretrain_path=None):
generator = UNet(n_channels=input_channels,
layer_nums=layer_nums,
output_channel=output_channels,
bn=bn)
discriminator = PixelDiscriminator(output_channels,
discriminator_num_filters,
use_norm=False)
generator = generator.cuda()
discriminator = discriminator.cuda()
flow_network = Network()
flow_network.load_state_dict(torch.load(lite_flow_model_path))
flow_network.cuda().eval()
adversarial_loss = Adversarial_Loss().cuda()
discriminate_loss = Discriminate_Loss().cuda()
gd_loss = Gradient_Loss(alpha, num_channels).cuda()
op_loss = Flow_Loss().cuda()
int_loss = Intensity_Loss(l_num).cuda()
step = 0
if not pretrain:
generator.apply(weights_init_normal)
discriminator.apply(weights_init_normal)
else:
assert (generator_pretrain_path != None
and discriminator_pretrain_path != None)
generator.load_state_dict(torch.load(generator_pretrain_path))
discriminator.load_state_dict(torch.load(discriminator_pretrain_path))
step = int(generator_pretrain_path.split('-')[-1])
print('pretrained model loaded!')
print('initializing the model with Generator-Unet {} layers,'
'PixelDiscriminator with filters {} '.format(
layer_nums, discriminator_num_filters))
optimizer_G = torch.optim.Adam(generator.parameters(), lr=g_lr)
optimizer_D = torch.optim.Adam(discriminator.parameters(), lr=d_lr)
writer = SummaryWriter(writer_path)
dataset = img_dataset.ano_pred_Dataset(training_data_folder, frame_num)
dataset_loader = DataLoader(dataset=dataset,
batch_size=batch_size,
shuffle=True,
num_workers=1,
drop_last=True)
test_dataset = img_dataset.ano_pred_Dataset(testing_data_folder, frame_num)
test_dataloader = DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=1,
drop_last=True)
for epoch in range(epochs):
for (input, _), (test_input, _) in zip(dataset_loader,
test_dataloader):
# generator = generator.train()
# discriminator = discriminator.train()
target = input[:, -1, :, :, :].cuda()
input = input[:, :-1, ]
input_last = input[:, -1, ].cuda()
input = input.view(input.shape[0], -1, input.shape[-2],
input.shape[-1]).cuda()
test_target = test_input[:, -1, ].cuda()
test_input = test_input[:, :-1].view(test_input.shape[0], -1,
test_input.shape[-2],
test_input.shape[-1]).cuda()
#------- update optim_G --------------
G_output = generator(input)
pred_flow_esti_tensor = torch.cat([input_last, G_output], 1)
gt_flow_esti_tensor = torch.cat([input_last, target], 1)
flow_gt = batch_estimate(gt_flow_esti_tensor, flow_network)
flow_pred = batch_estimate(pred_flow_esti_tensor, flow_network)
g_adv_loss = adversarial_loss(discriminator(G_output))
g_op_loss = op_loss(flow_pred, flow_gt)
g_int_loss = int_loss(G_output, target)
g_gd_loss = gd_loss(G_output, target)
g_loss = lam_adv * g_adv_loss + lam_gd * g_gd_loss + lam_op * g_op_loss + lam_int * g_int_loss
optimizer_G.zero_grad()
g_loss.backward()
optimizer_G.step()
train_psnr = psnr_error(G_output, target)
#----------- update optim_D -------
optimizer_D.zero_grad()
d_loss = discriminate_loss(discriminator(target),
discriminator(G_output.detach()))
#d_loss.requires_grad=True
d_loss.backward()
optimizer_D.step()
#----------- cal psnr --------------
test_generator = generator.eval()
test_output = test_generator(test_input)
test_psnr = psnr_error(test_output, test_target).cuda()
if step % 10 == 0:
print("[{}/{}]: g_loss: {} d_loss {}".format(
step, epoch, g_loss, d_loss))
print('\t gd_loss {}, op_loss {}, int_loss {} ,'.format(
g_gd_loss, g_op_loss, g_int_loss))
print('\t train psnr{}，test_psnr {}'.format(
train_psnr, test_psnr))
writer.add_scalar('psnr/train_psnr',
train_psnr,
global_step=step)
writer.add_scalar('psnr/test_psnr',
test_psnr,
global_step=step)
writer.add_scalar('total_loss/g_loss',
g_loss,
global_step=step)
writer.add_scalar('total_loss/d_loss',
d_loss,
global_step=step)
writer.add_scalar('g_loss/adv_loss',
g_adv_loss,
global_step=step)
writer.add_scalar('g_loss/op_loss',
g_op_loss,
global_step=step)
writer.add_scalar('g_loss/int_loss',
g_int_loss,
global_step=step)
writer.add_scalar('g_loss/gd_loss',
g_gd_loss,
global_step=step)
writer.add_image('image/train_target',
target[0],
global_step=step)
writer.add_image('image/train_output',
G_output[0],
global_step=step)
writer.add_image('image/test_target',
test_target[0],
global_step=step)
writer.add_image('image/test_output',
test_output[0],
global_step=step)
step += 1
if step % 500 == 0:
utils.saver(generator.state_dict(),
model_generator_save_path,
step,
max_to_save=10)
utils.saver(discriminator.state_dict(),
model_discriminator_save_path,
step,
max_to_save=10)
if step >= 2000:
print('==== begin evaluate the model of {} ===='.format(
model_generator_save_path + '-' + str(step)))
auc = evaluate(frame_num=5,
layer_nums=4,
input_channels=12,
output_channels=3,
model_path=model_generator_save_path + '-' +
str(step),
evaluate_name='compute_auc')
writer.add_scalar('results/auc', auc, global_step=step)
normalize=False)
tf_predict = T.ToTensor()
print(args.results_folder)
print(args.train)
print(args.val)
print(args.test)
if args.model_path is not None:
net = torch.load(args.model_path)
else:
net = UNet(3, 1)
loss = JaccardLoss()
lr_milestones = [int(p * args.epochs) for p in [0.5, 0.7, 0.9]]
optimizer = optim.Adam(net.parameters(), lr=1e-3)
scheduler = optim.lr_scheduler.MultiStepLR(optimizer, lr_milestones)
train_dataset = JointlyTransformedDataset(args.train,
transform=tf_train,
sigma=args.sigma)
val_dataset = JointlyTransformedDataset(args.val,
transform=tf_train,
sigma=args.sigma)
stage1_test_dataset = TestDataset(args.test, transform=tf_predict)
model = Model(
model=net,
loss=loss,
optimizer=optimizer,
scheduler=scheduler,
input_ = torch.randn((bs, 3, height, width)).cuda()
schedule = Schedule(graph, solver_info)
schedule.init_schedule(solution, mode)
torch.cuda.synchronize()
start_event_monet = torch.cuda.Event(enable_timing=True)
end_event_monet = torch.cuda.Event(enable_timing=True)
for iterid in range(120):
if iterid == 100:
torch.cuda.reset_max_memory_allocated()
start_event_monet.record()
x1 = schedule.forward(
input_, *list(model.state_dict(keep_vars=True).values()))
schedule.backward(-torch.ones_like(x1))
for v in model.parameters():
v.grad = None
end_event_monet.record()
torch.cuda.synchronize()
del x1
monet_maxmem = torch.cuda.max_memory_allocated() / 2**20
print("monet: %f ms avg, %8.2f MB" %
(start_event_monet.elapsed_time(end_event_monet) / 20,
monet_maxmem))
exit()
if args.check_diff:
graph = Graph.create(model, input_shape=(3, height, width))
model.cuda()
input_ = torch.randn((bs, 3, height, width)).cuda()
from models.track_net import TrackNet
from utils.get_dataloader import get_dataloader
from env import post_slack
from utils.detector import judge
from models.unet import UNet
def write_log(path, context, mode="a"):
with open(path, mode=mode) as f:
f.writelines(context + "\n")
cuda0 = torch.device('cuda:0')
net = UNet(27).to(cuda0)
criterion = nn.MSELoss().to(cuda0)
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
train_data_laoder, test_data_loader = get_dataloader(batch_size=4)
write_log("weight/train.log", str(datetime.datetime.now()), "w")
write_log("weight/train.log", "train start")
print(net)
for epoch in range(300):
# train phase
running_loss = 0.0
net.train()
for i, batch in enumerate(train_data_laoder):
inputs = batch['image'].to(cuda0)
target = batch['target'].to(cuda0)
optimizer.zero_grad()
outputs = net(inputs)
shuffle=False,
num_workers=5,
pin_memory=True)
# net and optimizer
ds_unet = UNet(1, 1, domain_specific=True)
ds_unet.cuda()
labeller = UNet(1, 1)
# import weights here...
labeller_path = './results/unet_sobel_eadan_in/net'
labeller.load_state_dict(
torch.load(os.path.join(labeller_path),
map_location=lambda storage, loc: storage))
labeller.cuda()
optimiser = optim.Adam(ds_unet.parameters(), lr=learning_rate)
print('Project name ', project_name)
train_dices = []
train_losses = []
val_a_dices = []
val_a_losses = []
val_b_dices = []
val_b_losses = []
for i in range(epochs):
ds_unet.set_domain(DOMAIN_A)
train_dice, train_loss = train_segmentation_net(ds_unet,
train_a_loader,
(-x0).sum().backward()
KEEP_FWDOP = True
x1 = schedule.forward(input_,
*list(model.state_dict(keep_vars=True).values()))
print(
'Forward mean absolute difference',
abs(x0[0] -
x1).mean() if 'googlenet' in args.model else abs(x0 -
x1).mean())
print('original output', x0)
print('ours output', x1)
print('Gradient of normal model', [
'{:.5f} {}'.format(float(v.grad.mean()), v.shape)
for v in model.parameters() if v.grad is not None
])
if args.check_runtime:
FORWARD_EMPTY_CACHE = False
if len(args.solution_file) > 0:
solver_info, solution = load_solution(args.solution_file)
else:
input_ = torch.randn((bs, 3, height, width)).cuda()
if args.pipeline:
solver_info = PipelinedSolverInfo(bs=bs,
model_name=model_name,
mode=mode)
else:
solver_info = SolverInfo(bs=bs,
def train_eval_model(opts):
# parse model configuration
num_epochs = opts["num_epochs"]
train_batch_size = opts["train_batch_size"]
val_batch_size = opts["eval_batch_size"]
dataset_type = opts["dataset_type"]
opti_mode = opts["optimizer"]
loss_criterion = opts["loss_criterion"]
lr = opts["lr"]
lr_decay = opts["lr_decay"]
wd = opts["weight_decay"]
gpus = opts["gpu_list"].split(',')
os.environ['CUDA_VISIBLE_DEVICE'] = opts["gpu_list"]
train_dir = opts["log_dir"]
train_data_dir = opts["train_data_dir"]
eval_data_dir = opts["eval_data_dir"]
pretrained = opts["pretrained_model"]
resume = opts["resume"]
display_iter = opts["display_iter"]
save_epoch = opts["save_every_epoch"]
show = opts["vis"]
# backup train configs
log_file = os.path.join(train_dir, "log_file.txt")
os.makedirs(train_dir, exist_ok=True)
model_dir = os.path.join(train_dir, "code_backup")
os.makedirs(model_dir, exist_ok=True)
if resume is None and os.path.exists(log_file): os.remove(log_file)
shutil.copy("./models/unet.py", os.path.join(model_dir, "unet.py"))
shutil.copy("./trainer_unet.py", os.path.join(model_dir,
"trainer_unet.py"))
shutil.copy("./datasets/dataset.py", os.path.join(model_dir, "dataset.py"))
ckt_dir = os.path.join(train_dir, "checkpoints")
os.makedirs(ckt_dir, exist_ok=True)
# format printing configs
print("*" * 50)
table_key = []
table_value = []
n = 0
for key, value in opts.items():
table_key.append(key)
table_value.append(str(value))
n += 1
print_table([table_key, ["="] * n, table_value])
# format gpu list
gpu_list = []
for str_id in gpus:
id = int(str_id)
gpu_list.append(id)
# dataloader
print("==> Create dataloader")
dataloaders_dict = {
"train":
er_data_loader(train_data_dir,
train_batch_size,
dataset_type,
is_train=True),
"eval":
er_data_loader(eval_data_dir,
val_batch_size,
dataset_type,
is_train=False)
}
# define parameters of two networks
print("==> Create network")
num_channels = 1
num_classes = 1
model = UNet(num_channels, num_classes)
init_weights(model)
# loss layer
criterion = create_criterion(criterion=loss_criterion)
best_acc = 0.0
start_epoch = 0
# load pretrained model
if pretrained is not None and os.path.isfile(pretrained):
print("==> Train from model '{}'".format(pretrained))
checkpoint_gan = torch.load(pretrained)
model.load_state_dict(checkpoint_gan['model_state_dict'])
print("==> Loaded checkpoint '{}')".format(pretrained))
for param in model.parameters():
param.requires_grad = False
# resume training
elif resume is not None and os.path.isfile(resume):
print("==> Resume from checkpoint '{}'".format(resume))
checkpoint = torch.load(resume)
start_epoch = checkpoint['epoch'] + 1
best_acc = checkpoint['best_acc']
model_dict = model.state_dict()
pretrained_dict = {
k: v
for k, v in checkpoint['model_state_dict'].items()
if k in model_dict and v.size() == model_dict[k].size()
}
model_dict.update(pretrained_dict)
model.load_state_dict(pretrained_dict)
print("==> Loaded checkpoint '{}' (epoch {})".format(
resume, checkpoint['epoch'] + 1))
# train from scratch
else:
print("==> Train from initial or random state.")
# define mutiple-gpu mode
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.cuda()
model = nn.DataParallel(model)
# print learnable parameters
print("==> List learnable parameters")
for name, param in model.named_parameters():
if param.requires_grad == True:
print("\t{}, size {}".format(name, param.size()))
params_to_update = [{'params': model.parameters()}]
# define optimizer
print("==> Create optimizer")
optimizer = create_optimizer(params_to_update,
opti_mode,
lr=lr,
momentum=0.9,
wd=wd)
if resume is not None and os.path.isfile(resume):
optimizer.load_state_dict(checkpoint['optimizer'])
# start training
since = time.time()
# Each epoch has a training and validation phase
print("==> Start training")
total_steps = 0
for epoch in range(start_epoch, num_epochs):
print('-' * 50)
print("==> Epoch {}/{}".format(epoch + 1, num_epochs))
total_steps = train_one_epoch(epoch, total_steps,
dataloaders_dict['train'], model, device,
criterion, optimizer, lr, lr_decay,
display_iter, log_file, show)
epoch_acc, epoch_iou, epoch_f1 = eval_one_epoch(
epoch, dataloaders_dict['eval'], model, device, log_file)
if best_acc < epoch_acc and epoch >= 5:
best_acc = epoch_acc
torch.save(
{
'epoch': epoch,
'model_state_dict': model.module.state_dict(),
'optimizer': optimizer.state_dict(),
'best_acc': best_acc
}, os.path.join(ckt_dir, "best.pth"))
if (epoch + 1) % save_epoch == 0 and (epoch + 1) >= 20:
torch.save(
{
'epoch': epoch,
'model_state_dict': model.module.state_dict(),
'optimizer': optimizer.state_dict(),
'best_iou': epoch_iou
},
os.path.join(ckt_dir,
"checkpoints_" + str(epoch + 1) + ".pth"))
time_elapsed = time.time() - since
time_message = 'Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60)
print(time_message)
with open(log_file, "a+") as fid:
fid.write('%s\n' % time_message)
print('==> Best val Acc: {:4f}'.format(best_acc))
def denoising(noise_im, clean_im, LR=1e-2, sigma=3, rho=1, eta=0.5, total_step=30,
prob1_iter=500, noise_level=None, result_root=None, f=None):
input_depth = 3
latent_dim = 3
en_net = UNet(input_depth, latent_dim).to(device)
de_net = UNet(latent_dim, input_depth).to(device)
parameters = [p for p in en_net.parameters()] + [p for p in de_net.parameters()]
optimizer = torch.optim.Adam(parameters, lr=LR)
l2_loss = torch.nn.MSELoss().cuda()
i0 = np_to_torch(noise_im).to(device)
noise_im_torch = np_to_torch(noise_im).to(device)
i0_til_torch = np_to_torch(noise_im).to(device)
Y = torch.zeros_like(noise_im_torch).to(device)
diff_original_np = noise_im.astype(np.float32) - clean_im.astype(np.float32)
diff_original_name = 'Original_dis.png'
save_hist(diff_original_np, result_root+diff_original_name)
best_psnr = 0
for i in range(total_step):
################################# sub-problem 1 ###############################
for i_1 in range(prob1_iter):
optimizer.zero_grad()
mean = en_net(noise_im_torch)
z = sample_z(mean)
out = de_net(z)
total_loss = 0.5 * l2_loss(out, noise_im_torch)
total_loss += 0.5 * (1/sigma**2)*l2_loss(mean, i0)
total_loss += (rho/2) * l2_loss(i0 + Y, i0_til_torch)
total_loss.backward()
optimizer.step()
with torch.no_grad():
i0 = ((1/sigma**2)*mean.detach() + rho*(i0_til_torch - Y)) / ((1/sigma**2) + rho)
with torch.no_grad():
################################# sub-problem 2 ###############################
i0_np = torch_to_np(i0)
Y_np = torch_to_np(Y)
sig = eval_sigma(i+1, noise_level)
i0_til_np = bm3d.bm3d_rgb(i0_np.transpose(1, 2, 0) + Y_np.transpose(1, 2, 0), sig).transpose(2, 0, 1)
i0_til_torch = np_to_torch(i0_til_np).to(device)
################################# sub-problem 3 ###############################
Y = Y + eta * (i0 - i0_til_torch)
###############################################################################
Y_name = 'Y_{:04d}'.format(i) + '.png'
i0_name = 'i0_num_epoch_{:04d}'.format(i) + '.png'
mean_name = 'Latent_im_num_epoch_{:04d}'.format(i) + '.png'
out_name = 'res_of_dec_num_epoch_{:04d}'.format(i) + '.png'
diff_name = 'Latent_dis_num_epoch_{:04d}'.format(i) + '.png'
Y_np = torch_to_np(Y)
Y_norm_np = np.sqrt((Y_np*Y_np).sum(0))
save_heatmap(Y_norm_np, result_root + Y_name)
save_torch(mean, result_root + mean_name)
save_torch(out, result_root + out_name)
save_torch(i0, result_root + i0_name)
mean_np = torch_to_np(mean)
diff_np = mean_np - clean_im
save_hist(diff_np, result_root + diff_name)
i0_til_np = torch_to_np(i0_til_torch).clip(0, 255)
psnr = compare_psnr(clean_im.transpose(1, 2, 0), i0_til_np.transpose(1, 2, 0), 255)
ssim = compare_ssim(clean_im.transpose(1, 2, 0), i0_til_np.transpose(1, 2, 0), multichannel=True, data_range=255)
i0_til_pil = np_to_pil(i0_til_np)
i0_til_pil.save(os.path.join(result_root, '{}'.format(i) + '.png'))
print('Iteration: {:02d}, VAE Loss: {:f}, PSNR: {:f}, SSIM: {:f}'.format(i, total_loss.item(), psnr, ssim), file=f, flush=True)
if best_psnr < psnr:
best_psnr = psnr
best_ssim = ssim
else:
break
return i0_til_np, best_psnr, best_ssim
test_dataset = TestFromFolder(os.path.join(all_datasets,
'stage1_test/loc.csv'),
transform=T.ToTensor(),
remove_alpha=True)
"""
-----------------
----- Model -----
-----------------
"""
generator = UNet(3, 1)
discriminator = Discriminator(4, 1)
generator.cuda()
discriminator.cuda()
# lr = 0.001 seems to work WITHOUT PRETRAINING
g_optim = optim.Adam(generator.parameters(), lr=0.001)
d_optim = optim.Adam(discriminator.parameters(), lr=0.001)
#g_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(g_optim, factor=0.1, verbose=True, patience=5)
#d_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(d_optim, factor=0.1, verbose=True, patience=5)
gan = GAN(
g=generator,
d=discriminator,
g_optim=g_optim,
d_optim=d_optim,
g_loss=nn.MSELoss().cuda(),
d_loss=nn.MSELoss().cuda(),
#g_scheduler=g_scheduler, d_scheduler=d_scheduler
)
# pretraining generator
batch_size=1,
shuffle=False,
num_workers=5,
pin_memory=True)
refine_net = UNet(1,
1,
norm_layer=nn.BatchNorm2d,
affine=True,
track_running_stats=True)
refine_net.cuda()
# list optimisers here...
# single optimiser variant 1
optimiser_ref = optim.Adam(refine_net.parameters(), lr=learning_rate)
print('Project name ', project_name)
train_dices = []
train_losses = []
val_dices = []
val_losses = []
for i in range(epochs):
train_dice, train_loss = train_segmentation_net(refine_net,
train_loader,
'A',
3,
1,
criterion_seg,
test_loader = DataLoader(scan_dataset_test,
batch_size=1,
shuffle=False,
num_workers=5,
pin_memory=True)
pseudo = UNet(1, 1)
net = UNet(1, 1, domain_specific=True)
pseudo.cuda()
net.cuda()
# list optimisers here...
# single optimiser variant 1
optimiser_ps = optim.Adam(pseudo.parameters(), lr=learning_rate)
optimiser_net = optim.Adam(net.parameters(), lr=learning_rate)
print('Project name ', project_name)
print('Learning rate ', learning_rate)
print('Epochs ', epochs)
train_loss = []
train_loss_rec = []
train_loss_seg = []
train_loss_seg_a = []
train_loss_seg_b = []
train_dice = []
val_loss_a = []
def train_UNet():
cfg = UnetConfig()
train_transform = transforms.Compose([
GrayscaleNormalization(mean=0.5, std=0.5),
RandomRotation(),
RandomFlip(),
ToTensor(),
])
val_transform = transforms.Compose([
GrayscaleNormalization(mean=0.5, std=0.5),
ToTensor(),
])
# Set Dataset
train_dataset = Dataset(imgs_dir=TRAIN_IMGS_DIR,
labels_dir=TRAIN_LABELS_DIR,
transform=train_transform)
train_loader = DataLoader(train_dataset,
batch_size=cfg.BATCH_SIZE,
shuffle=True,
num_workers=0)
val_dataset = Dataset(imgs_dir=VAL_IMGS_DIR,
labels_dir=VAL_LABELS_DIR,
transform=val_transform)
val_loader = DataLoader(val_dataset,
batch_size=cfg.BATCH_SIZE,
shuffle=False,
num_workers=0)
train_data_num = len(train_dataset)
val_data_num = len(val_dataset)
train_batch_num = int(np.ceil(train_data_num / cfg.BATCH_SIZE)) # np.ceil
val_batch_num = int(np.ceil(val_data_num / cfg.BATCH_SIZE))
# Network
net = UNet().to(device)
print(count_parameters(net))
# Loss Function
loss_fn = nn.BCEWithLogitsLoss().to(device)
# Optimizer
optim = torch.optim.Adam(params=net.parameters(), lr=cfg.LEARNING_RATE)
# Tensorboard
# train_writer = SummaryWriter(log_dir=TRAIN_LOG_DIR)
# val_writer = SummaryWriter(log_dir=VAL_LOG_DIR)
# Training
start_epoch = 0
# Load Checkpoint File
if os.listdir(os.path.join(CKPT_DIR, 'unet')):
net, optim, start_epoch = load_net(ckpt_dir=os.path.join(
CKPT_DIR, 'unet'),
net=net,
optim=optim)
else:
print('* Training from scratch')
num_epochs = cfg.NUM_EPOCHS
for epoch in range(start_epoch + 1, num_epochs + 1):
net.train()
train_loss_arr = list()
for batch_idx, data in enumerate(train_loader, 1):
# Forward Propagation
img = data['img'].to(device)
label = data['label'].to(device)
output = net(img)
# Backward Propagation
optim.zero_grad()
loss = loss_fn(output, label)
loss.backward()
optim.step()
# Calc Loss Function
train_loss_arr.append(loss.item())
print_form = '[Train] | Epoch: {:0>4d} / {:0>4d} | Batch: {:0>4d} / {:0>4d} | Loss: {:.4f}'
print(
print_form.format(epoch, num_epochs, batch_idx,
train_batch_num, train_loss_arr[-1]))
train_loss_avg = np.mean(train_loss_arr)
# train_writer.add_scalar(tag='loss', scalar_value=train_loss_avg, global_step=epoch)
# Validation (No Back Propagation)
with torch.no_grad():
net.eval() # Evaluation Mode
val_loss_arr = list()
for batch_idx, data in enumerate(val_loader, 1):
# Forward Propagation
img = data['img'].to(device)
label = data['label'].to(device)
output = net(img)
# Calc Loss Function
loss = loss_fn(output, label)
val_loss_arr.append(loss.item())
print_form = '[Validation] | Epoch: {:0>4d} / {:0>4d} | Batch: {:0>4d} / {:0>4d} | Loss: {:.4f}'
print(
print_form.format(epoch, num_epochs, batch_idx,
val_batch_num, val_loss_arr[-1]))
val_loss_avg = np.mean(val_loss_arr)
# val_writer.add_scalar(tag='loss', scalar_value=val_loss_avg, global_step=epoch)
print_form = '[Epoch {:0>4d}] Training Avg Loss: {:.4f} | Validation Avg Loss: {:.4f}'
print(print_form.format(epoch, train_loss_avg, val_loss_avg))
if epoch % 10 == 0:
save_net(ckpt_dir=os.path.join(CKPT_DIR, 'unet'),
net=net,
optim=optim,
epoch=epoch)
def test_UNet():
cfg = UnetConfig()
transform = transforms.Compose([
GrayscaleNormalization(mean=0.5, std=0.5),
ToTensor(),
])
RESULTS_DIR = os.path.join(ROOT_DIR, 'test_results/unet')
if not os.path.exists(RESULTS_DIR):
os.makedirs(RESULTS_DIR)
label_save_path = os.path.join(RESULTS_DIR, 'label')
output_save_path = os.path.join(RESULTS_DIR, 'output')
if not os.path.exists(label_save_path):
os.makedirs(label_save_path, exist_ok=True)
if not os.path.exists(output_save_path):
os.makedirs(output_save_path, exist_ok=True)
test_dataset = Dataset(imgs_dir=TEST_IMGS_DIR,
labels_dir=TEST_LABELS_DIR,
transform=transform)
test_loader = DataLoader(test_dataset,
batch_size=cfg.BATCH_SIZE,
shuffle=False,
num_workers=0)
test_data_num = len(test_dataset)
test_batch_num = int(np.ceil(test_data_num / cfg.BATCH_SIZE))
# Network
net = UNet().to(device)
# Loss Function
loss_fn = nn.BCEWithLogitsLoss().to(device)
# Optimizer
optim = torch.optim.Adam(params=net.parameters(), lr=cfg.LEARNING_RATE)
start_epoch = 0
# Load Checkpoint File
if os.listdir(CKPT_DIR):
net, optim, _ = load_net(ckpt_dir=os.path.join(CKPT_DIR, 'unet'),
net=net,
optim=optim)
# Evaluation
with torch.no_grad():
net.eval()
loss_arr = list()
for batch_idx, data in enumerate(test_loader, 1):
# Forward Propagation
img = data['img'].to(device)
label = data['label'].to(device)
output = net(img)
# Calc Loss Function
loss = loss_fn(output, label)
loss_arr.append(loss.item())
print_form = '[Test] | Batch: {:0>4d} / {:0>4d} | Loss: {:.4f}'
print(print_form.format(batch_idx, test_batch_num, loss_arr[-1]))
label = to_numpy(label)
output = to_numpy(classify_class(output))
for j in range(label.shape[0]):
crt_id = int(test_batch_num * (batch_idx - 1) + j)
plt.imsave(os.path.join(label_save_path, f'{crt_id:04}.png'),
label[j].squeeze(),
cmap='gray')
plt.imsave(os.path.join(output_save_path, f'{crt_id:04}.png'),
output[j].squeeze(),
cmap='gray')
unet_acc(output_save_path, label_save_path)
param_noise = False
else:
assert False
else:
assert False
net = net.type(dtype)
net_input = get_noise(input_depth, INPUT, img_np.shape[1:]).type(dtype)
# In[ ]:
# Compute number of parameters
s = sum(np.prod(list(p.size())) for p in net.parameters())
print ('Number of params: %d' % s)
# Loss
mse = torch.nn.MSELoss().type(dtype)
img_var = np_to_torch(img_np).type(dtype)
mask_var = np_to_torch(img_mask_np).type(dtype)
# # Main loop
# In[ ]:
i = 0
class Noise2Noise(object):
"""Implementation of Noise2Noise from Lehtinen et al. (2018)."""
def __init__(self, params, trainable):
"""Initializes model."""
self.p = params
self.trainable = trainable
self._compile() #初始化模型
def _compile(self):
"""
Compiles model (architecture, loss function, optimizers, etc.).
初始化 网络、损失函数、优化器等
"""
print('Noise2Noise: Learning Image Restoration without Clean Data (Lethinen et al., 2018)')
# Model (3x3=9 channels for Monte Carlo since it uses 3 HDR buffers) 已删除蒙特卡洛相关代码
if self.p.noise_type == 'mc':
self.is_mc = True
self.model = UNet(in_channels=9)
else:
self.is_mc = False
self.model = UNet(in_channels=3)
# Set optimizer and loss, if in training mode
# 如果 为训练，则初始化优化器和损失
if self.trainable:
self.optim = Adam(self.model.parameters(),
lr=self.p.learning_rate,
betas=self.p.adam[:2],
eps=self.p.adam[2])
# Learning rate adjustment
self.scheduler = lr_scheduler.ReduceLROnPlateau(self.optim,
patience=self.p.nb_epochs/4, factor=0.5, verbose=True)
# Loss function
if self.p.loss == 'hdr':
assert self.is_mc, 'Using HDR loss on non Monte Carlo images'
self.loss = HDRLoss()
elif self.p.loss == 'l2':
self.loss = nn.MSELoss()
else:
self.loss = nn.L1Loss()
# CUDA support
self.use_cuda = torch.cuda.is_available() and self.p.cuda
if self.use_cuda:
self.model = self.model.cuda()
if self.trainable:
self.loss = self.loss.cuda()
def _print_params(self):
"""Formats parameters to print when training."""
print('Training parameters: ')
self.p.cuda = self.use_cuda
param_dict = vars(self.p)
pretty = lambda x: x.replace('_', ' ').capitalize()
print('\n'.join(' {} = {}'.format(pretty(k), str(v)) for k, v in param_dict.items()))
print()
def save_model(self, epoch, stats, first=False):
"""Saves model to files; can be overwritten at every epoch to save disk space."""
# Create directory for model checkpoints, if nonexistent
if first:
if self.p.clean_targets:
ckpt_dir_name = f'{datetime.now():{self.p.noise_type}-clean-%H%M}'
else:
ckpt_dir_name = f'{datetime.now():{self.p.noise_type}-%H%M}'
if self.p.ckpt_overwrite:
if self.p.clean_targets:
ckpt_dir_name = f'{self.p.noise_type}-clean'
else:
ckpt_dir_name = self.p.noise_type
self.ckpt_dir = os.path.join(self.p.ckpt_save_path, ckpt_dir_name)
if not os.path.isdir(self.p.ckpt_save_path):
os.mkdir(self.p.ckpt_save_path)
if not os.path.isdir(self.ckpt_dir):
os.mkdir(self.ckpt_dir)
# Save checkpoint dictionary
if self.p.ckpt_overwrite:
fname_unet = '{}/n2n-{}.pt'.format(self.ckpt_dir, self.p.noise_type)
else:
valid_loss = stats['valid_loss'][epoch]
fname_unet = '{}/n2n-epoch{}-{:>1.5f}.pt'.format(self.ckpt_dir, epoch + 1, valid_loss)
print('Saving checkpoint to: {}\n'.format(fname_unet))
torch.save(self.model.state_dict(), fname_unet)
# Save stats to JSON
fname_dict = '{}/n2n-stats.json'.format(self.ckpt_dir)
with open(fname_dict, 'w') as fp:
json.dump(stats, fp, indent=2)
def load_model(self, ckpt_fname):
"""Loads model from checkpoint file."""
print('Loading checkpoint from: {}'.format(ckpt_fname))
if self.use_cuda:
self.model.load_state_dict(torch.load(ckpt_fname))
else:
self.model.load_state_dict(torch.load(ckpt_fname, map_location='cpu'))
def _on_epoch_end(self, stats, train_loss, epoch, epoch_start, valid_loader):
"""Tracks and saves starts after each epoch."""
# Evaluate model on validation set
print('\rTesting model on validation set... ', end='')
epoch_time = time_elapsed_since(epoch_start)[0]
valid_loss, valid_time, valid_psnr = self.eval(valid_loader)
show_on_epoch_end(epoch_time, valid_time, valid_loss, valid_psnr)
# Decrease learning rate if plateau
self.scheduler.step(valid_loss)
# Save checkpoint
stats['train_loss'].append(train_loss)
stats['valid_loss'].append(valid_loss)
stats['valid_psnr'].append(valid_psnr)
self.save_model(epoch, stats, epoch == 0)
def test(self, test_loader, show=1):
"""Evaluates denoiser on test set."""
self.model.train(False)
source_imgs = []
denoised_imgs = []
clean_imgs = []
# Create directory for denoised images
denoised_dir = os.path.dirname(self.p.data)
save_path = os.path.join(denoised_dir, 'denoised')
if not os.path.isdir(save_path):
os.mkdir(save_path)
for batch_idx, (source, target) in enumerate(test_loader):
# Only do first <show> images
if show == 0 or batch_idx >= show:
break
source_imgs.append(source)
clean_imgs.append(target)
if self.use_cuda:
source = source.cuda()
# Denoise
denoised_img = self.model(source).detach()
denoised_imgs.append(denoised_img)
# Squeeze tensors
source_imgs = [t.squeeze(0) for t in source_imgs]
denoised_imgs = [t.squeeze(0) for t in denoised_imgs]
clean_imgs = [t.squeeze(0) for t in clean_imgs]
# Create montage and save images
print('Saving images and montages to: {}'.format(save_path))
for i in range(len(source_imgs)):
img_name = test_loader.dataset.imgs[i]
create_montage(img_name, self.p.noise_type, save_path, source_imgs[i], denoised_imgs[i], clean_imgs[i], show)
def eval(self, valid_loader):
"""Evaluates denoiser on validation set."""
self.model.train(False)
valid_start = datetime.now()
loss_meter = AvgMeter()
psnr_meter = AvgMeter()
for batch_idx, (source, target) in enumerate(valid_loader):
if self.use_cuda:
source = source.cuda()
target = target.cuda()
# Denoise
source_denoised = self.model(source)
# Update loss
loss = self.loss(source_denoised, target)
loss_meter.update(loss.item())
# Compute PSRN
if self.is_mc:
source_denoised = reinhard_tonemap(source_denoised)
# TODO: Find a way to offload to GPU, and deal with uneven batch sizes
for i in range(self.p.batch_size):
source_denoised = source_denoised.cpu()
target = target.cpu()
psnr_meter.update(psnr(source_denoised[i], target[i]).item())
valid_loss = loss_meter.avg
valid_time = time_elapsed_since(valid_start)[0]
psnr_avg = psnr_meter.avg
return valid_loss, valid_time, psnr_avg
def train(self, train_loader, valid_loader):
"""Trains denoiser on training set."""
self.model.train(True)
self._print_params()
num_batches = len(train_loader)
assert num_batches % self.p.report_interval == 0, 'Report interval must divide total number of batches'
# Dictionaries of tracked stats
stats = {'noise_type': self.p.noise_type,
'noise_param': self.p.noise_param,
'train_loss': [],
'valid_loss': [],
'valid_psnr': []}
# Main training loop
train_start = datetime.now()
for epoch in range(self.p.nb_epochs):
print('EPOCH {:d} / {:d}'.format(epoch + 1, self.p.nb_epochs))
# Some stats trackers
epoch_start = datetime.now()
train_loss_meter = AvgMeter()
loss_meter = AvgMeter()
time_meter = AvgMeter()
# Minibatch SGD
for batch_idx, (source, target) in enumerate(train_loader):
batch_start = datetime.now()
progress_bar(batch_idx, num_batches, self.p.report_interval, loss_meter.val)
if self.use_cuda:
source = source.cuda()
target = target.cuda()
# Denoise image
source_denoised = self.model(source)
loss = self.loss(source_denoised, target)
loss_meter.update(loss.item())
# Zero gradients, perform a backward pass, and update the weights
self.optim.zero_grad()
loss.backward()
self.optim.step()
# Report/update statistics
time_meter.update(time_elapsed_since(batch_start)[1])
if (batch_idx + 1) % self.p.report_interval == 0 and batch_idx:
show_on_report(batch_idx, num_batches, loss_meter.avg, time_meter.avg)
train_loss_meter.update(loss_meter.avg)
loss_meter.reset()
time_meter.reset()
# Epoch end, save and reset tracker
self._on_epoch_end(stats, train_loss_meter.avg, epoch, epoch_start, valid_loader)
train_loss_meter.reset()
train_elapsed = time_elapsed_since(train_start)[0]
print('Training done! Total elapsed time: {}\n'.format(train_elapsed))
class Trainer:
def __init__(self, seq_length, color_channels, unet_path="pretrained/unet.mdl",
discrim_path="pretrained/dicrim.mdl",
facenet_path="pretrained/facenet.mdl",
vgg_path="",
embedding_size=1000,
unet_depth=3,
unet_filts=32,
facenet_filts=32,
resnet=18):
self.color_channels = color_channels
self.margin = 0.5
self.writer = SummaryWriter(log_dir="logs")
self.unet_path = unet_path
self.discrim_path = discrim_path
self.facenet_path = facenet_path
self.unet = UNet(in_channels=color_channels, out_channels=color_channels,
depth=unet_depth,
start_filts=unet_filts,
up_mode="upsample",
merge_mode='concat').to(device)
self.discrim = FaceNetModel(embedding_size=embedding_size, start_filts=facenet_filts,
in_channels=color_channels, resnet=resnet,
pretrained=False).to(device)
self.facenet = FaceNetModel(embedding_size=embedding_size, start_filts=facenet_filts,
in_channels=color_channels, resnet=resnet,
pretrained=False).to(device)
if os.path.isfile(unet_path):
self.unet.load_state_dict(torch.load(unet_path))
print("unet loaded")
if os.path.isfile(discrim_path):
self.discrim.load_state_dict(torch.load(discrim_path))
print("discrim loaded")
if os.path.isfile(facenet_path):
self.facenet.load_state_dict(torch.load(facenet_path))
print("facenet loaded")
if os.path.isfile(vgg_path):
self.vgg_loss_network = LossNetwork(vgg_face_dag(vgg_path)).to(device)
self.vgg_loss_network.eval()
print("vgg loaded")
self.mse_loss_function = nn.MSELoss().to(device)
self.discrim_loss_function = nn.BCELoss().to(device)
self.triplet_loss_function = TripletLoss(margin=self.margin)
self.unet_optimizer = torch.optim.Adam(self.unet.parameters(), betas=(0.9, 0.999))
self.discrim_optimizer = torch.optim.Adam(self.discrim.parameters(), betas=(0.9, 0.999))
self.facenet_optimizer = torch.optim.Adam(self.facenet.parameters(), betas=(0.9, 0.999))
def test(self, test_loader, epoch=0):
X, y = next(iter(test_loader))
B, D, C, W, H = X.shape
# X = X.view(B, C * D, W, H)
self.unet.eval()
self.facenet.eval()
self.discrim.eval()
with torch.no_grad():
y_ = self.unet(X.to(device))
mse = self.mse_loss_function(y_, y.to(device))
loss_G = self.loss_GAN_generator(btch_X=X.to(device))
loss_D = self.loss_GAN_discrimator(btch_X=X.to(device), btch_y=y.to(device))
loss_facenet, _, n_bad = self.loss_facenet(X.to(device), y.to(device))
plt.title(f"epoch {epoch} mse={mse.item():.4} facenet={loss_facenet.item():.4} bad={n_bad / B ** 2}")
i = np.random.randint(0, B)
a = np.hstack((y[i].transpose(0, 1).transpose(1, 2), y_[i].transpose(0, 1).transpose(1, 2).to(cpu)))
b = np.hstack((X[i][0].transpose(0, 1).transpose(1, 2),
X[i][-1].transpose(0, 1).transpose(1, 2)))
plt.imshow(np.vstack((a, b)))
plt.axis('off')
plt.show()
self.writer.add_scalar("test bad_percent", n_bad / B ** 2, global_step=epoch)
self.writer.add_scalar("test loss", mse.item(), global_step=epoch)
# self.writer.add_scalars("test GAN", {"discrim": loss_D.item(),
# "gen": loss_G.item()}, global_step=epoch)
with torch.no_grad():
n_for_show = 10
y_show_ = y_.to(device)
y_show = y.to(device)
embeddings_anc, _ = self.facenet(y_show_)
embeddings_pos, _ = self.facenet(y_show)
embeds = torch.cat((embeddings_anc[:n_for_show], embeddings_pos[:n_for_show]))
imgs = torch.cat((y_show_[:n_for_show], y_show[:n_for_show]))
names = list(range(n_for_show)) * 2
# print(embeds.shape, imgs.shape, len(names))
# self.writer.add_embedding(mat=embeds, metadata=names, label_img=imgs, tag="embeddings", global_step=epoch)
trshs, fprs, tprs = roc_curve(embeddings_anc.detach().to(cpu), embeddings_pos.detach().to(cpu))
rnk1 = rank1(embeddings_anc.detach().to(cpu), embeddings_pos.detach().to(cpu))
plt.step(fprs, tprs)
# plt.xlim((1e-4, 1))
plt.yticks(np.arange(0, 1, 0.05))
plt.xticks(np.arange(min(fprs), max(fprs), 10))
plt.xscale('log')
plt.title(f"ROC auc={auc(fprs, tprs)} rnk1={rnk1}")
self.writer.add_figure("ROC test", plt.gcf(), global_step=epoch)
self.writer.add_scalar("auc", auc(fprs, tprs), global_step=epoch)
self.writer.add_scalar("rank1", rnk1, global_step=epoch)
print(f"\n###### {epoch} TEST mse={mse.item():.4} GAN(G/D)={loss_G.item():.4}/{loss_D.item():.4} "
f"facenet={loss_facenet.item():.4} bad={n_bad / B ** 2:.4} auc={auc(fprs, tprs)} rank1={rnk1} #######")
def test_test(self, test_loader):
X, ys = next(iter(test_loader))
true_idx = 0
x = X[true_idx]
D, C, W, H = x.shape
# x = x.view(C * D, W, H)
dists = list()
with torch.no_grad():
y_ = self.unet(x.to(device))
embedding_anc, _ = self.facenet(y_)
embeddings_pos, _ = self.facenet(ys)
for emb_pos_item in embeddings_pos:
dist = l2_dist.forward(embedding_anc, emb_pos_item)
dists.append(dist)
a_sorted = np.argsort(dists)
a = np.hstack((ys[true_idx].transpose(0, 1).transpose(1, 2),
y_.transpose(0, 1).transpose(1, 2).to(cpu).numpy(),
ys[a_sorted[0]].transpose(0, 1).transpose(1, 2)))
b = np.hstack((x[0:3].transpose(0, 1).transpose(1, 2),
x[D // 2 * C:D // 2 * C + 3].transpose(0, 1).transpose(1, 2),
x[-3:].transpose(0, 1).transpose(1, 2)))
b_ = b - np.min(b)
b_ = b_ / np.max(b)
b_ = equalize_func([(b_ * 255).astype(np.uint8)], use_clahe=True)[0]
b = b_.astype(np.float32) / 255
plt.imshow(cv2.cvtColor(np.vstack((a, b)), cv2.COLOR_BGR2RGB))
plt.axis('off')
plt.show()
def loss_facenet(self, X, y, is_detached=False):
B, D, C, W, H = X.shape
y_ = self.unet(X)
embeddings_anc, D_fake = self.facenet(y_ if not is_detached else y_.detach())
embeddings_pos, D_real = self.facenet(y)
target_real = torch.full_like(D_fake, 1)
loss_gen = self.discrim_loss_function(D_fake, target_real)
pos_dist = l2_dist.forward(embeddings_anc, embeddings_pos)
bad_triplets_loss = None
n_bad = 0
for shift in range(1, B):
embeddings_neg = torch.roll(embeddings_pos, shift, 0)
neg_dist = l2_dist.forward(embeddings_anc, embeddings_neg)
bad_triplets_idxs = np.where((neg_dist - pos_dist < self.margin).cpu().numpy().flatten())[0]
if shift == 1:
bad_triplets_loss = self.triplet_loss_function.forward(embeddings_anc[bad_triplets_idxs],
embeddings_pos[bad_triplets_idxs],
embeddings_neg[bad_triplets_idxs]).to(
device)
else:
bad_triplets_loss += self.triplet_loss_function.forward(embeddings_anc[bad_triplets_idxs],
embeddings_pos[bad_triplets_idxs],
embeddings_neg[bad_triplets_idxs]).to(device)
n_bad += len(bad_triplets_idxs)
bad_triplets_loss /= B
return bad_triplets_loss, torch.mean(loss_gen), n_bad
# def loss_mse(self, btch_X, btch_y):
# btch_y_ = self.unet(btch_X)
# loss_unet = self.mse_loss_function(btch_y_, btch_y)
#
# features_target = self.facenet.forward_mse(btch_y)
# features = self.facenet.forward_mse(btch_y_)
#
# loss_first_layer = self.mse_loss_function(features, features_target)
# return loss_unet + loss_first_layer
def loss_mse_vgg(self, btch_X, btch_y, k_mse, k_vgg):
btch_y_ = self.unet(btch_X)
# print(btch_y_.shape,btch_y.shape)
perceptual_btch_y_ = self.vgg_loss_network(btch_y_)
perceptual_btch_y = self.vgg_loss_network(btch_y)
perceptual_loss = 0.0
for a, b in zip(perceptual_btch_y_, perceptual_btch_y):
perceptual_loss += self.mse_loss_function(a, b)
return k_vgg * perceptual_loss + k_mse * self.mse_loss_function(btch_y_, btch_y)
def loss_GAN_discrimator(self, btch_X, btch_y):
btch_y_ = self.unet(btch_X)
_, y_D_fake_ = self.discrim(btch_y_.detach())
_, y_D_real_ = self.discrim(btch_y)
target_fake = torch.full_like(y_D_fake_, 0)
target_real = torch.full_like(y_D_real_, 1)
loss_D_fake_ = self.discrim_loss_function(y_D_fake_, target_fake)
loss_D_real_ = self.discrim_loss_function(y_D_real_, target_real)
loss_discrim = (loss_D_real_ + loss_D_fake_)
return loss_discrim
def loss_GAN_generator(self, btch_X):
btch_y_ = self.unet(btch_X)
_, y_D_fake_ = self.discrim(btch_y_)
target_real = torch.full_like(y_D_fake_, 1)
loss_gen = self.discrim_loss_function(y_D_fake_, target_real)
return loss_gen
def relax_discriminator(self, btch_X, btch_y):
self.discrim.zero_grad()
# train with real
y_discrim_real_ = self.discrim(btch_y)
y_discrim_real_ = y_discrim_real_.mean()
y_discrim_real_.backward(self.mone)
# train with fake
btch_y_ = self.unet(btch_X)
y_discrim_fake_detached_ = self.discrim(btch_y_.detach())
y_discrim_fake_detached_ = y_discrim_fake_detached_.mean()
y_discrim_fake_detached_.backward(self.one)
# gradient_penalty
gradient_penalty = self.discrim_gradient_penalty(btch_y, btch_y_)
gradient_penalty.backward()
self.discrim_optimizer.step()
def relax_generator(self, btch_X):
self.unet.zero_grad()
btch_y_ = self.unet(btch_X)
y_discrim_fake_ = self.discrim(btch_y_)
y_discrim_fake_ = y_discrim_fake_.mean()
y_discrim_fake_.backward(self.mone)
self.unet_optimizer.step()
def discrim_gradient_penalty(self, real_y, fake_y):
lambd = 10
btch_size = real_y.shape[0]
alpha = torch.rand(btch_size, 1, 1, 1).to(device)
# print(alpha.shape, real_y.shape)
alpha = alpha.expand_as(real_y)
interpolates = alpha * real_y + (1 - alpha) * fake_y
interpolates = interpolates.to(device)
interpolates = autograd.Variable(interpolates, requires_grad=True)
interpolates_out = self.discrim(interpolates)
gradients = autograd.grad(outputs=interpolates_out, inputs=interpolates,
grad_outputs=torch.ones(interpolates_out.size()).to(device),
create_graph=True, retain_graph=True, only_inputs=True)[0]
gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * lambd
return gradient_penalty
def train(self, train_loader, test_loader, batch_size=2, epochs=30,
k_gen=1, k_discrim=1, k_mse=1, k_facenet=1, k_facenet_back=1, k_vgg=1):
"""
:param X: np.array shape=(n_videos, n_frames, h, w)
:param y: np.array shape=(n_videos, h, w)
:param epochs: int
"""
print("\nSTART TRAINING\n")
for epoch in range(epochs):
self.test(test_loader, epoch)
self.unet.train()
self.facenet.train()
self.discrim.train()
# train by batches
for idx, (btch_X, btch_y) in enumerate(train_loader):
B, D, C, W, H = btch_X.shape
# btch_X = btch_X.view(B, C * D, W, H)
btch_X = btch_X.to(device)
btch_y = btch_y.to(device)
# Mse loss
self.unet.zero_grad()
mse = self.loss_mse_vgg(btch_X, btch_y, k_mse, k_vgg)
mse.backward()
self.unet_optimizer.step()
# facenet_backup = deepcopy(self.facenet.state_dict())
# for i in range(unrolled_iterations):
self.discrim.zero_grad()
loss_D = self.loss_GAN_discrimator(btch_X, btch_y)
loss_D = k_discrim * loss_D
loss_D.backward()
self.discrim_optimizer.step()
self.discrim.zero_grad()
self.unet.zero_grad()
loss_G = self.loss_GAN_generator(btch_X)
loss_G = k_gen * loss_G
loss_G.backward()
self.unet_optimizer.step()
# Facenet
self.unet.zero_grad()
self.facenet.zero_grad()
facenet_loss, _, n_bad = self.loss_facenet(btch_X, btch_y)
facenet_loss = k_facenet * facenet_loss
facenet_loss.backward()
self.facenet_optimizer.step()
self.unet.zero_grad()
self.facenet.zero_grad()
facenet_back_loss, _, n_bad = self.loss_facenet(btch_X, btch_y)
facenet_back_loss = k_facenet_back * facenet_back_loss
facenet_back_loss.backward()
self.unet_optimizer.step()
print(f"btch {idx * batch_size} mse={mse.item():.4} GAN(G/D)={loss_G.item():.4}/{loss_D.item():.4} "
f"facenet={facenet_loss.item():.4} bad={n_bad / B ** 2:.4}")
global_step = epoch * len(train_loader.dataset) // batch_size + idx
self.writer.add_scalar("train bad_percent", n_bad / B ** 2, global_step=global_step)
self.writer.add_scalar("train loss", mse.item(), global_step=global_step)
# self.writer.add_scalars("train GAN", {"discrim": loss_D.item(),
# "gen": loss_G.item()}, global_step=global_step)
torch.save(self.unet.state_dict(), self.unet_path)
torch.save(self.discrim.state_dict(), self.discrim_path)
torch.save(self.facenet.state_dict(), self.facenet_path)
shuffle=False,
num_workers=4)
if args.model == "unet":
model = UNet(input_channels=NUM_INPUT_CHANNELS,
output_channels=NUM_OUTPUT_CHANNELS)
elif args.model == "segnet":
model = SegNet(input_channels=NUM_INPUT_CHANNELS,
output_hannels=NUM_OUTPUT_CHANNELS)
else:
model = PSPNet(layers=50, bins=(1, 2, 3, 6), dropout=0.1, classes=NUM_OUTPUT_CHANNELS, use_ppm=True, pretrained=True)
class_weights = 1.0/train_dataset.get_class_probability()
criterion = torch.nn.CrossEntropyLoss(weight=class_weights)
if CUDA:
model = model.cuda(GPU_ID)
class_weights = class_weights.cuda(GPU_ID)
criterion = criterion.cuda(GPU_ID)
if args.checkpoint:
model.load_state_dict(torch.load(args.checkpoint))
optimizer = torch.optim.Adam(model.parameters(),
lr=LEARNING_RATE)
train()
def train():
startTime = time.time()
args = parameters.parse_arguments()
logging.basicConfig(filename=args.logfile, level=logging.INFO)
logging.critical("\n\n" + args.log_header)
logging.info(args)
device = ("cuda" if torch.cuda.is_available() else "cpu")
logging.info(f"TIME: {time.time() - startTime}s Using device {device}")
logging.info(f"TIME: {time.time()-startTime}s Loading dataset")
try:
with open(os.path.join(args.datadir, "data.pkl"), "rb") as f:
data = pickle.load(f)
except:
data = DataLoader(args.datadir,
int(args.batchsize),
shuffle=int(args.shuffle))
with open(os.path.join(args.datadir, "data.pkl"), "wb") as f:
pickle.dump(data, f)
data.batchSize = int(args.batchsize)
logging.info(f"TIME: {time.time()-startTime}s Dataset Loaded")
random.seed(args.seed)
indices = list(range(len(data)))
random.shuffle(
indices
) # 0:floor((1-validationFrac)*len(data)) will be training data, rest will be validation data
trainEndIndex = math.floor((1 - args.validation_frac) * (len(data)))
model = UNet(in_channels=1,
num_classes=2,
start_filts=int(args.conv_filters),
up_mode=args.mode,
depth=int(args.depth),
batchnorm=args.batchnorm)
model.reset_params()
model = model.to(device)
optimizer = None
if args.optimizer == 'adam':
optimizer = optim.Adam(model.parameters(), lr=args.lrstart)
logging.info(f"TIME: {time.time()-startTime}s Optimizer: adam")
elif args.optimizer == 'sgd':
optimizer = optim.SGD(model.parameters(),
lr=args.lrstart,
momentum=args.momentum)
logging.info(f"TIME: {time.time()-startTime}s Optimizer: SGD")
elif args.optimizer == 'rmsprop':
optimizer = optim.RMSprop(model.parameters(), lr=args.lrstart)
logging.info(f"TIME: {time.time()-startTime}s Optimizer: RMSProp")
else:
logging.error(
f"TIME: {time.time()-startTime}s Incorrect optimizer given")
scheduler = []
if args.lrscheduler == "steplr":
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=args.decay)
logging.info(f"TIME: {time.time()-startTime}s LRScheduler: StepLR")
elif args.lrscheduler == "exponentiallr":
scheduler = optim.lr_scheduler.ExponentialLR(optimizer,
gamma=args.decay)
logging.info(
f"TIME: {time.time()-startTime}s LRScheduler: exponentialLR")
else:
scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=int(args.epochs))
logging.info(
f"TIME: {time.time()-startTime}s LRScheduler: lr shouldn't change with epochs"
)
criteria = CombinedLoss(args.lambda_loss, args.loss_type)
diceCoeff = DiceLoss()
TL = []
VL = []
if not os.path.exists(os.path.join(os.getcwd(), "loss_files")):
os.makedirs(os.path.join(os.getcwd(), "loss_files"))
lossFile = open(os.path.join("loss_files", args.log_header + ".csv"), "w+")
lossFile.write("Epoch,TrainLoss,ValidationLoss,Dice Coefficient\n")
for epoch in tqdm(range(1, int(args.epochs) + 1), desc="Training model"):
trainLoss = 0
valLoss = 0
trainingSample = 0
testSample = 0
netCoeff = 0
for i in range(len(data)):
images, masks = data[i]
images = torch.tensor(images.astype(np.float32))
masks = torch.tensor(masks.astype(np.float32))
images = images.to(device)
masks = masks.to(device)
images = torch.transpose(images, 1, 3)
masks = torch.transpose(masks, 1, 3)
if i in indices[:trainEndIndex]:
trainingSample += images.shape[0]
networkPred = model(images)
if args.regularization == 'l1':
reg = L1_regularization(model, args.reg_lamda1)
loss = criteria(masks, networkPred) + reg
elif args.regularization == 'l1l2':
reg = L1L2_regularization(model, args.reg_lamda1,
args.reg_lamda2)
loss = criteria(masks, networkPred) + reg
else:
loss = criteria(masks, networkPred)
loss.backward()
trainLoss += loss.item()
optimizer.step()
model.zero_grad()
else:
with torch.no_grad():
testSample += images.shape[0]
prediction = model(images)
if (epoch % args.save_epochs
== 0) or (epoch == 1) or (epoch == args.epochs):
imgPath = os.path.join("validation_sample",
args.log_header,
f"epoch {epoch}")
if not os.path.exists(imgPath):
os.makedirs(imgPath)
hrt = images[0, 0, :, :].to("cpu")
plt.imshow(np.array(hrt), cmap='gray')
plt.title("Heart Image")
plt.savefig(os.path.join(imgPath, "heart.png"))
plt.clf()
# ax = figure.add_subplot(232, title="Mask 1 Predicted")
msk1 = prediction[0, 0, :, :].to("cpu")
plt.imshow(np.array(msk1), cmap='gray')
plt.title("Predicted Mask 1")
plt.savefig(os.path.join(imgPath, "pred-mask1.png"))
plt.clf()
# ax = figure.add_subplot(231, title="Mask 2 Predicted")
msk2 = prediction[0, 1, :, :].to("cpu")
plt.imshow(np.array(msk2), cmap='gray')
plt.title("Predicted Mask 2")
plt.savefig(os.path.join(imgPath, "pred-mask2.png"))
plt.clf()
msk = np.zeros((192, 192, 3))
msk[:, :, 0] = np.array(msk1)
msk[:, :, 1] = np.array(msk2)
plt.imshow(np.array(hrt), cmap='gray')
plt.imshow(msk, cmap='jet', alpha=0.4)
plt.title("predicted-RV")
plt.savefig(os.path.join(imgPath, "pred-RV.png"))
plt.clf()
# ax = figure.add_subplot(231, title="Mask 2 Real")
msk1 = masks[0, 0, :, :].to("cpu")
plt.imshow(np.array(msk1), cmap='gray')
plt.title("Actual Mask 1")
plt.savefig(os.path.join(imgPath, "actual-mask1.png"))
plt.clf()
# ax = figure.add_subplot(231, title="Mask 2 Real")
msk2 = masks[0, 1, :, :].to("cpu")
plt.imshow(np.array(msk2), cmap='gray')
plt.title("Actual Mask 2")
plt.savefig(os.path.join(imgPath, "actual-mask2.png"))
plt.clf()
# plt.savefig(os.path.join("validation_sample", f"{args.log_header}-epoch {epoch}.png"))
msk = np.zeros((192, 192, 3))
msk[:, :, 0] = np.array(msk1)
msk[:, :, 1] = np.array(msk2)
plt.imshow(np.array(hrt), cmap='gray')
plt.imshow(msk, cmap='jet', alpha=0.4)
plt.title("actual-RV")
plt.savefig(os.path.join(imgPath, "actual-RV.png"))
plt.clf()
if args.regularization == 'l1':
reg = L1_regularization(model, args.reg_lamda1)
loss = criteria(masks, prediction) + reg
elif args.regularization == 'l1l2':
reg = L1L2_regularization(model, args.reg_lamda1,
args.reg_lamda2)
loss = criteria(masks, prediction) + reg
else:
loss = criteria(masks, prediction)
valLoss += loss.item()
coeff = diceCoeff(masks, prediction)
netCoeff += torch.sum(1 - coeff).item()
if (epoch % int(args.save_epochs) == 0) or (epoch == int(args.epochs)):
if not os.path.exists(args.model_save_dir):
os.makedirs(args.model_save_dir)
# save model
torch.save(
{
"epoch": epoch,
"model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
},
os.path.join(args.model_save_dir,
f"model-epoch({epoch}).hdf5"))
logging.info(
f"TIME: {time.time()-startTime}s Model state saved for epoch: {epoch}"
)
logging.info(
f"TIME: {time.time()-startTime}s TRAINING: Epoch: {epoch}, lr: {scheduler.get_last_lr()}, loss: {trainLoss/(2*trainingSample)}"
)
logging.info(
f"TIME: {time.time()-startTime}s VALIDATION: Epoch: {epoch}, lr: {scheduler.get_last_lr()}, loss: {valLoss/(2*testSample)}"
)
TL.append(trainLoss / (2 * trainingSample))
VL.append(valLoss / (2 * testSample))
lossFile.write(
f"{epoch},{trainLoss/(2*trainingSample)},{valLoss/(2*testSample)},{netCoeff/(2*testSample)}\n"
)
scheduler.step(
) # https://www.deeplearningwizard.com/deep_learning/boosting_models_pytorch/lr_scheduling/
plt.plot(list(range(1, int(args.epochs) + 1)), TL, label="Training loss")
plt.plot(list(range(1, int(args.epochs) + 1)), VL, label="Validation loss")
plt.xlabel("Epoch")
plt.ylabel("Loss")
plt.legend(loc="best")
if not os.path.exists(os.path.join(os.getcwd(), "plots")):
os.makedirs(os.path.join(os.getcwd(), "plots"))
plt.savefig(os.path.join("plots", args.log_header + ".png"))
def train(input_data_type,
grade,
seg_type,
num_classes,
batch_size,
epochs,
use_gpu,
learning_rate,
w_decay,
pre_trained=False):
logger.info('Start training using {} modal.'.format(input_data_type))
model = UNet(4, 4, residual=True, expansion=2)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(params=model.parameters(),
lr=learning_rate,
weight_decay=w_decay)
if pre_trained:
checkpoint = torch.load(pre_trained_path, map_location=device)
model.load_state_dict(checkpoint['model_state_dict'])
if use_gpu:
ts = time.time()
model.to(device)
print("Finish cuda loading, time elapsed {}".format(time.time() - ts))
scheduler = lr_scheduler.StepLR(
optimizer, step_size=step_size,
gamma=gamma) # decay LR by a factor of 0.5 every 5 epochs
data_set, data_loader = get_dataset_dataloader(input_data_type,
seg_type,
batch_size,
grade=grade)
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
best_iou = 0.0
epoch_loss = np.zeros((2, epochs))
epoch_acc = np.zeros((2, epochs))
epoch_class_acc = np.zeros((2, epochs))
epoch_mean_iou = np.zeros((2, epochs))
evaluator = Evaluator(num_classes)
def term_int_handler(signal_num, frame):
np.save(os.path.join(score_dir, 'epoch_accuracy'), epoch_acc)
np.save(os.path.join(score_dir, 'epoch_mean_iou'), epoch_mean_iou)
np.save(os.path.join(score_dir, 'epoch_loss'), epoch_loss)
model.load_state_dict(best_model_wts)
logger.info('Got terminated and saved model.state_dict')
torch.save(model.state_dict(),
os.path.join(score_dir, 'terminated_model.pt'))
torch.save(
{
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, os.path.join(score_dir, 'terminated_model.tar'))
quit()
signal.signal(signal.SIGINT, term_int_handler)
signal.signal(signal.SIGTERM, term_int_handler)
for epoch in range(epochs):
logger.info('Epoch {}/{}'.format(epoch + 1, epochs))
logger.info('-' * 28)
for phase_ind, phase in enumerate(['train', 'val']):
if phase == 'train':
model.train()
logger.info(phase)
else:
model.eval()
logger.info(phase)
evaluator.reset()
running_loss = 0.0
running_dice = 0.0
for batch_ind, batch in enumerate(data_loader[phase]):
imgs, targets = batch
imgs = imgs.to(device)
targets = targets.to(device)
# zero the learnable parameters gradients
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = model(imgs)
loss = criterion(outputs, targets)
if phase == 'train':
loss.backward()
optimizer.step()
preds = torch.argmax(F.softmax(outputs, dim=1),
dim=1,
keepdim=True)
running_loss += loss * imgs.size(0)
logger.debug('Batch {} running loss: {:.4f}'.format(batch_ind,\
running_loss))
# test the iou and pixelwise accuracy using evaluator
preds = torch.squeeze(preds, dim=1)
preds = preds.cpu().numpy()
targets = targets.cpu().numpy()
evaluator.add_batch(targets, preds)
epoch_loss[phase_ind, epoch] = running_loss / len(data_set[phase])
epoch_acc[phase_ind, epoch] = evaluator.Pixel_Accuracy()
epoch_class_acc[phase_ind,
epoch] = evaluator.Pixel_Accuracy_Class()
epoch_mean_iou[phase_ind,
epoch] = evaluator.Mean_Intersection_over_Union()
logger.info('{} loss: {:.4f}, acc: {:.4f}, class acc: {:.4f}, mean iou: {:.6f}'.format(phase,\
epoch_loss[phase_ind, epoch],\
epoch_acc[phase_ind, epoch],\
epoch_class_acc[phase_ind, epoch],\
epoch_mean_iou[phase_ind, epoch]))
if phase == 'val' and epoch_mean_iou[phase_ind, epoch] > best_iou:
best_iou = epoch_mean_iou[phase_ind, epoch]
best_model_wts = copy.deepcopy(model.state_dict())
if phase == 'val' and (epoch + 1) % 10 == 0:
logger.info('Saved model.state_dict in epoch {}'.format(epoch +
1))
torch.save(
model.state_dict(),
os.path.join(score_dir,
'epoch{}_model.pt'.format(epoch + 1)))
print()
time_elapsed = time.time() - since
logger.info('Training completed in {}m {}s'.format(int(time_elapsed / 60),\
int(time_elapsed) % 60))
# load best model weights
model.load_state_dict(best_model_wts)
# save numpy results
np.save(os.path.join(score_dir, 'epoch_accuracy'), epoch_acc)
np.save(os.path.join(score_dir, 'epoch_mean_iou'), epoch_mean_iou)
np.save(os.path.join(score_dir, 'epoch_loss'), epoch_loss)
return model, optimizer
def denoising_fixd_noise_level(noise_im,
clean_im,
LR=1e-2,
sigma=5,
rho=1,
eta=0.5,
alpha=0.5,
total_step=19,
prob1_iter=1000,
noise_level=None,
result_root=None,
fo=None):
sig = noise_level
r_nlm = denoise_nl_means(noise_im.transpose(1, 2, 0),
h=0.8 * sig,
sigma=sig,
fast_mode=False,
**patch_kw)
r_nlm = np.clip(r_nlm, 0, 255)
psnr_nlm = compare_psnr(clean_im.transpose(1, 2, 0), r_nlm, 255)
ssim_nlm = compare_ssim(r_nlm,
clean_im.transpose(1, 2, 0),
multichannel=True,
data_range=255)
print('noise level {} '.format(noise_level), file=fo, flush=True)
print('PSNR_NLM: {}, SSIM_NLM: {}'.format(psnr_nlm, ssim_nlm),
file=fo,
flush=True)
r_nlm = Image.fromarray(r_nlm.astype(np.uint8))
r_nlm.save(result_root + 'nlm_result.png')
input_depth = 3
latent_dim = 3
en_net = UNet(input_depth, latent_dim, need_sigmoid=True).cuda()
de_net = UNet(latent_dim, input_depth, need_sigmoid=True).cuda()
en_optimizer = torch.optim.Adam(en_net.parameters(), lr=LR)
de_optimizer = torch.optim.Adam(de_net.parameters(), lr=LR)
l2_loss = torch.nn.MSELoss().cuda()
noise_im = noise_im / 255.0
i0 = torch.Tensor(noise_im[None, ...]).cuda()
noise_im_torch = torch.Tensor(noise_im)[None, ...].cuda()
Y = torch.zeros_like(noise_im_torch).cuda()
i0_til_torch = torch.Tensor(noise_im[None, ...]).cuda()
output = None
for i in range(total_step):
############################### sub-problem 1 #################################
for i_1 in range(prob1_iter):
mean_i = en_net(noise_im_torch)
eps = mean_i.clone().normal_()
out = de_net(mean_i + eps)
total_loss = 0.5 * l2_loss(out, noise_im_torch)
total_loss += 1 / (2 * sigma**2) * l2_loss(mean_i, i0)
en_optimizer.zero_grad()
de_optimizer.zero_grad()
total_loss.backward()
en_optimizer.step()
de_optimizer.step()
with torch.no_grad():
i0 = ((1 / sigma**2) * mean_i + rho *
(i0_til_torch - Y) + alpha * noise_im_torch) / (
(1 / sigma**2) + rho + alpha)
with torch.no_grad():
############################### sub-problem 2 #################################
i0_np = i0.cpu().squeeze().detach().numpy()
Y_np = Y.cpu().squeeze().detach().numpy()
tmp = i0_np.transpose(1, 2, 0) + Y_np.transpose(1, 2, 0)
tmp = np.clip(tmp, 0, 1)
sig = noise_level
i0_til_np = denoise_nl_means(tmp * 255,
h=0.8 * sig,
sigma=sig,
fast_mode=False,
**patch_kw)
i0_til_np = i0_til_np / 255
i0_til_torch = torch.Tensor(
i0_til_np.transpose(2, 0, 1)[None, ...]).cuda()
############################### sub-problem 3 #################################
Y = Y + eta * (i0 - i0_til_torch)
###############################################################################
i0_til_np = i0_til_torch.cpu().squeeze().detach().numpy()
i0_til_np = np.clip(i0_til_np, 0, 1)
output = i0_til_np
psnr_gt = compare_psnr(clean_im.transpose(1, 2, 0),
255 * i0_til_np.transpose(1, 2, 0), 255)
ssim_gt = compare_ssim(255 * i0_til_np.transpose(1, 2, 0),
clean_im.transpose(1, 2, 0),
multichannel=True,
data_range=255)
if not i % 5:
denoise_obj_pil = Image.fromarray((tmp * 255).astype(np.uint8))
Y_np = Y.cpu().squeeze().detach().numpy()
i0_np = np.clip(i0_np, 0, 1)
i0_pil = Image.fromarray(
np.uint8(255 * i0_np.transpose(1, 2, 0)))
i0_til_np = i0_til_np.transpose(1, 2, 0)
i0_til_pil = Image.fromarray(
(255 * i0_til_np).astype(np.uint8))
mean_i_np = mean_i.cpu().squeeze().detach().numpy().clip(0, 1)
mean_i_pil = Image.fromarray(
(255 * mean_i_np.transpose(1, 2, 0)).astype(np.uint8))
out_np = out.cpu().squeeze().detach().numpy().clip(0, 1)
out_pil = Image.fromarray(
(255 * out_np.transpose(1, 2, 0)).astype(np.uint8))
denoise_obj_name = 'denoise_obj_{:04d}'.format(i) + '.png'
i0_name = 'i0_num_epoch_{:04d}'.format(i) + '.png'
result_name = 'num_epoch_{:04d}'.format(i) + '.png'
mean_i_name = 'Latent_im_num_epoch_{:04d}'.format(i) + '.png'
out_name = 'res_of_dec_num_epoch_{:04d}'.format(i) + '.png'
denoise_obj_pil.save(result_root + denoise_obj_name)
i0_pil.save(result_root + i0_name)
i0_til_pil.save(result_root + result_name)
mean_i_pil.save(result_root + mean_i_name)
out_pil.save(result_root + out_name)
print('Iteration %05d Loss %f PSNR_gt: %f SSIM_gt: %f' %
(i, total_loss.item(), psnr_gt, ssim_gt),
file=fo,
flush=True)
psnr = psnr_gt
ssim = ssim_gt
output_pil = Image.fromarray(
(255 * output.transpose(1, 2, 0)).astype(np.uint8))
output_pil.save(result_root + 'ours_result.png')
return psnr, ssim, psnr_nlm, ssim_nlm
model = UNet(input_channels=NUM_INPUT_CHANNELS,
output_channels=NUM_OUTPUT_CHANNELS)
elif args.model == "segnet":
model = SegNet(input_channels=NUM_INPUT_CHANNELS,
output_hannels=NUM_OUTPUT_CHANNELS)
else:
model = PSPNet(
layers=50,
bins=(1, 2, 3, 6),
dropout=0.1,
classes=NUM_OUTPUT_CHANNELS,
use_ppm=True,
pretrained=True,
)
# class_weights = 1.0 / train_dataset.get_class_probability()
# criterion = torch.nn.CrossEntropyLoss(weight=class_weights)
criterion = torch.nn.CrossEntropyLoss()
if CUDA:
model = model.cuda(device=GPU_ID)
# class_weights = class_weights.cuda(GPU_ID)
criterion = criterion.cuda(device=GPU_ID)
if args.checkpoint:
model.load_state_dict(torch.load(args.checkpoint))
optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)
train()
val_size = len(datasets) - train_size
train_dataset, val_dataset = torch.utils.data.random_split(
datasets, [train_size, val_size])
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=4)
val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=4)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = UNet(n_channels=3, n_classes=21).to(device)
criterion = BCEDiceLoss().to(device)
model_fcn8 = fcn8s().to(device)
criterion_fcn8 = BCEDiceLoss().to(device)
model_segnet = segnet().to(device)
criterion_segnet = BCEDiceLoss().to(device)
optimizer = optim.Adam(model.parameters(), lr=0.001)
optimizer_fcn8 = optim.Adam(model_fcn8.parameters(), lr=0.001)
optimizer_segnet = optim.Adam(model_segnet.parameters(), lr=0.001)
# colabは相対パスがいいみたい
# logdir = "logs"
# logdir_path = os.path.join(base_path, logdir)
logdir_path = "./logs"
if not os.path.isdir(logdir_path):
os.mkdir(logdir_path)
dt = datetime.datetime.now()
# log writer for unet
model_id = len(
glob.glob(
os.path.join(logdir_path, "{}{}{}*".format(dt.year, dt.month,
def main():
parser = argparse.ArgumentParser(description="Train the model")
parser.add_argument('-trainf', "--train-filepath", type=str, default=None, required=True,
help="training dataset filepath.")
parser.add_argument('-validf', "--val-filepath", type=str, default=None,
help="validation dataset filepath.")
parser.add_argument("--shuffle", action="store_true", default=False,
help="Shuffle the dataset")
parser.add_argument("--load-weights", type=str, default=None,
help="load pretrained weights")
parser.add_argument("--load-model", type=str, default=None,
help="load pretrained model, entire model (filepath, default: None)")
parser.add_argument("--debug", action="store_true", default=False)
parser.add_argument('--epochs', type=int, default=30,
help='number of epochs to train (default: 30)')
parser.add_argument("--batch-size", type=int, default=32,
help="Batch size")
parser.add_argument('--img-shape', type=str, default="(1,512,512)",
help='Image shape (default "(1,512,512)"')
parser.add_argument("--num-cpu", type=int, default=10,
help="Number of CPUs to use in parallel for dataloader.")
parser.add_argument('--cuda', type=int, default=0,
help='CUDA visible device (use CPU if -1, default: 0)')
parser.add_argument('--cuda-non-deterministic', action='store_true', default=False,
help="sets flags for non-determinism when using CUDA (potentially fast)")
parser.add_argument('-lr', type=float, default=0.0005,
help='Learning rate')
parser.add_argument('--seed', type=int, default=0,
help='Seed (numpy and cuda if GPU is used.).')
parser.add_argument('--log-dir', type=str, default=None,
help='Save the results/model weights/logs under the directory.')
args = parser.parse_args()
# TODO: support image reshape
img_shape = tuple(map(int, args.img_shape.strip()[1:-1].split(",")))
if args.log_dir:
os.makedirs(args.log_dir, exist_ok=True)
best_model_path = os.path.join(args.log_dir, "model_weights.pth")
else:
best_model_path = None
if args.seed is not None:
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda >= 0:
if args.cuda_non_deterministic:
printBlue("Warning: using CUDA non-deterministc. Could be faster but results might not be reproducible.")
else:
printBlue("Using CUDA deterministc. Use --cuda-non-deterministic might accelerate the training a bit.")
# Make CuDNN Determinist
torch.backends.cudnn.deterministic = not args.cuda_non_deterministic
# torch.cuda.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
# TODO [OPT] enable multi-GPUs ?
# https://pytorch.org/tutorials/beginner/former_torchies/parallelism_tutorial.html
device = torch.device("cuda:{}".format(args.cuda) if torch.cuda.is_available()
and (args.cuda >= 0) else "cpu")
# ================= Build dataloader =================
# DataLoader
# transform_normalize = transforms.Normalize(mean=[0.5, 0.5, 0.5],
# std=[0.5, 0.5, 0.5])
transform_normalize = transforms.Normalize(mean=[0.5],
std=[0.5])
# Warning: DO NOT use geometry transform (do it in the dataloader instead)
data_transform = transforms.Compose([
# transforms.ToPILImage(mode='F'), # mode='F' for one-channel image
# transforms.Resize((256, 256)) # NO
# transforms.RandomResizedCrop(256), # NO
# transforms.RandomHorizontalFlip(p=0.5), # NO
# WARNING, ISSUE: transforms.ColorJitter doesn't work with ToPILImage(mode='F').
# Need custom data augmentation functions: TODO: DONE.
# transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),
# Use OpenCVRotation, OpenCVXXX, ... (our implementation)
# OpenCVRotation((-10, 10)), # angles (in degree)
transforms.ToTensor(), # already done in the dataloader
transform_normalize
])
geo_transform = GeoCompose([
OpenCVRotation(angles=(-10, 10),
scales=(0.9, 1.1),
centers=(-0.05, 0.05)),
# TODO add more data augmentation here
])
def worker_init_fn(worker_id):
# WARNING spawn start method is used,
# worker_init_fn cannot be an unpicklable object, e.g., a lambda function.
# A work-around for issue #5059: https://github.com/pytorch/pytorch/issues/5059
np.random.seed()
data_loader_train = {'batch_size': args.batch_size,
'shuffle': args.shuffle,
'num_workers': args.num_cpu,
# 'sampler': balanced_sampler,
'drop_last': True, # for GAN-like
'pin_memory': False,
'worker_init_fn': worker_init_fn,
}
data_loader_valid = {'batch_size': args.batch_size,
'shuffle': False,
'num_workers': args.num_cpu,
'drop_last': False,
'pin_memory': False,
}
train_set = LiTSDataset(args.train_filepath,
dtype=np.float32,
geometry_transform=geo_transform, # TODO enable data augmentation
pixelwise_transform=data_transform,
)
valid_set = LiTSDataset(args.val_filepath,
dtype=np.float32,
pixelwise_transform=data_transform,
)
dataloader_train = torch.utils.data.DataLoader(train_set, **data_loader_train)
dataloader_valid = torch.utils.data.DataLoader(valid_set, **data_loader_valid)
# =================== Build model ===================
# TODO: control the model by bash command
if args.load_weights:
model = UNet(in_ch=1,
out_ch=3, # there are 3 classes: 0: background, 1: liver, 2: tumor
depth=4,
start_ch=32, # 64
inc_rate=2,
kernel_size=5, # 3
padding=True,
batch_norm=True,
spec_norm=False,
dropout=0.5,
up_mode='upconv',
include_top=True,
include_last_act=False,
)
printYellow(f"Loading pretrained weights from: {args.load_weights}...")
model.load_state_dict(torch.load(args.load_weights))
printYellow("+ Done.")
elif args.load_model:
# load entire model
model = torch.load(args.load_model)
printYellow("Successfully loaded pretrained model.")
model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, betas=(0.9, 0.95)) # TODO
best_valid_loss = float('inf')
# TODO TODO: add learning decay
for epoch in range(args.epochs):
for valid_mode, dataloader in enumerate([dataloader_train, dataloader_valid]):
n_batch_per_epoch = len(dataloader)
if args.debug:
n_batch_per_epoch = 1
# infinite dataloader allows several update per iteration (for special models e.g. GAN)
dataloader = infinite_dataloader(dataloader)
if valid_mode:
printYellow("Switch to validation mode.")
model.eval()
prev_grad_mode = torch.is_grad_enabled()
torch.set_grad_enabled(False)
else:
model.train()
st = time.time()
cum_loss = 0
for iter_ind in range(n_batch_per_epoch):
supplement_logs = ""
# reset cumulated losses at the begining of each batch
# loss_manager.reset_losses() # TODO: use torch.utils.tensorboard !!
optimizer.zero_grad()
img, msk = next(dataloader)
img, msk = img.to(device), msk.to(device)
# TODO this is ugly: convert dtype and convert the shape from (N, 1, 512, 512) to (N, 512, 512)
msk = msk.to(torch.long).squeeze(1)
msk_pred = model(img) # shape (N, 3, 512, 512)
# label_weights is determined according the liver_ratio & tumor_ratio
# loss = CrossEntropyLoss(msk_pred, msk, label_weights=[1., 10., 100.], device=device)
loss = DiceLoss(msk_pred, msk, label_weights=[1., 20., 50.], device=device)
# loss = DiceLoss(msk_pred, msk, label_weights=[1., 20., 500.], device=device)
if valid_mode:
pass
else:
loss.backward()
optimizer.step()
loss = loss.item() # release
cum_loss += loss
if valid_mode:
print("\r--------(valid) {:.2%} Loss: {:.3f} (time: {:.1f}s) |supp: {}".format(
(iter_ind+1)/n_batch_per_epoch, cum_loss/(iter_ind+1), time.time()-st, supplement_logs), end="")
else:
print("\rEpoch: {:3}/{} {:.2%} Loss: {:.3f} (time: {:.1f}s) |supp: {}".format(
(epoch+1), args.epochs, (iter_ind+1)/n_batch_per_epoch, cum_loss/(iter_ind+1), time.time()-st, supplement_logs), end="")
print()
if valid_mode:
torch.set_grad_enabled(prev_grad_mode)
valid_mean_loss = cum_loss/(iter_ind+1) # validation (mean) loss of the current epoch
if best_model_path and (valid_mean_loss < best_valid_loss):
printGreen("Valid loss decreases from {:.5f} to {:.5f}, saving best model.".format(
best_valid_loss, valid_mean_loss))
best_valid_loss = valid_mean_loss
# Only need to save the weights
# torch.save(model.state_dict(), best_model_path)
# save the entire model
torch.save(model, best_model_path)
return best_valid_loss
transform=T.ToTensor(),
remove_alpha=True)
"""
-----------------
----- Model -----
-----------------
"""
s = UNet(3, 2)
t = UNet(3, 3)
if use_gpu:
s.cuda()
t.cuda()
# lr = 0.001 seems to work WITHOUT PRETRAINING
s_optim = optim.Adam(s.parameters(), lr=0.1)
t_optim = optim.Adam(t.parameters(), lr=0.1)
s_scheduler = torch.optim.lr_scheduler.StepLR(s_optim, step_size=10)
t_scheduler = torch.optim.lr_scheduler.StepLR(t_optim, step_size=10)
gan = GANv2(s=s,
s_optim=s_optim,
s_loss=CrossEntropyLoss2d().cuda(),
s_scheduler=s_scheduler,
g=t,
g_optim=t_optim)
gan.train_segmenter(train_dataset,
n_epochs=20,
n_batch=4,
use_gpu=use_gpu,
val_size = len(datasets) - train_size
train_dataset, val_dataset = torch.utils.data.random_split(
datasets, [train_size, val_size])
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=4)
val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=4)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = UNet(n_channels=3, n_classes=21).to(device)
criterion = BCEDiceLoss().to(device)
model_fcn8 = fcn8s().to(device)
criterion_fcn8 = BCEDiceLoss().to(device)
model_segnet = segnet().to(device)
criterion_segnet = BCEDiceLoss().to(device)
optimizer = optim.Adam(model.parameters(), lr=0.001)
optimizer_fcn8 = optim.Adam(model_fcn8.parameters(), lr=0.001)
optimizer_segnet = optim.Adam(model_segnet.parameters(), lr=0.001)
# colabは相対パスがいいみたい
# logdir = "logs"
# logdir_path = os.path.join(base_path, logdir)
logdir_path = "./logs"
if not os.path.isdir(logdir_path):
os.mkdir(logdir_path)
dt = datetime.datetime.now()
# log writer for unet
model_id = len(
glob.glob(
os.path.join(logdir_path, "{}{}{}*".format(dt.year, dt.month,
# Число эпох
N_EPOCHS = 10
# tensorboard
writer = SummaryWriter(log_dir='./{}'.format(MODEL_NAME), comment=MODEL_NAME)
# Задаем модель
model = UNet(3, NUM_PTS)
model.to(device)
with torch.no_grad():
# writer.add_graph(model, next(iter(val_dataloader))['image'].to(device))
summary(model, next(iter(train_dataloader))['image'].shape[1:])
# Задаем параметры оптимизации
optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE, amsgrad=True)
# criterion = F.mse_loss
criterion = AdaptiveWingLoss()
# Временные параметры для выбора наилучшего результата
best_val_loss, best_model_state_dict = np.inf, {}
CURRENT_EPOCH = 0
for epoch in range(CURRENT_EPOCH, N_EPOCHS):
train_loss = train_iter(MODEL_NAME,
470,
epoch,
model,
train_dataloader,
criterion,
def denoising(noise_im,
clean_im,
LR=1e-2,
sigma=5,
rho=1,
eta=0.5,
alpha=1,
total_step=20,
prob1_iter=500,
noise_level=None,
result_root=None,
f=None):
input_depth = 1
latent_dim = 1
en_net = UNet(input_depth, latent_dim, need_sigmoid=False).cuda()
de_net = UNet(latent_dim, input_depth, need_sigmoid=False).cuda()
parameters = [p for p in en_net.parameters()
] + [p for p in de_net.parameters()]
optimizer = torch.optim.Adam(parameters, lr=LR)
l2_loss = torch.nn.MSELoss().cuda()
i0 = torch.Tensor(noise_im)[None, None, ...].cuda()
noise_im_torch = torch.Tensor(noise_im)[None, None, ...].cuda()
Y = torch.zeros_like(noise_im_torch).cuda()
i0_til_torch = torch.Tensor(noise_im)[None, None, ...].cuda()
diff_original_np = noise_im.astype(np.float32) - clean_im.astype(
np.float32)
diff_original_name = 'Original_dis.png'
save_hist(diff_original_np, result_root + diff_original_name)
best_psnr = 0
best_ssim = 0
for i in range(total_step):
################################# sub-problem 1 ###############################
for i_1 in range(prob1_iter):
mean_i = en_net(noise_im_torch)
eps = mean_i.clone().normal_()
out = de_net(mean_i + eps)
total_loss = 0.5 * l2_loss(out, noise_im_torch)
total_loss += 1 / (2 * sigma**2) * l2_loss(mean_i, i0)
total_loss += (rho / 2) * l2_loss(i0 + Y, i0_til_torch)
optimizer.zero_grad()
total_loss.backward()
optimizer.step()
with torch.no_grad():
i0 = ((1 / sigma**2) * mean_i + rho *
(i0_til_torch - Y) + alpha * noise_im_torch) / (
(1 / sigma**2) + rho + alpha)
with torch.no_grad():
################################# sub-problem 2 ###############################
i0_np = i0.cpu().squeeze().detach().numpy()
Y_np = Y.cpu().squeeze().detach().numpy()
sig = noise_level
i0_til_np = denoise_nl_means(i0_np + Y_np,
h=20 * sig,
sigma=sig,
fast_mode=False,
**patch_kw)
i0_til_torch = torch.Tensor(i0_til_np[None, None, ...]).cuda()
################################# sub-problem 3 ###############################
Y = Y + eta * (i0 - i0_til_torch)
###############################################################################
denoise_obj_pil = Image.fromarray(
(i0_np + Y_np).clip(0, 255).astype(np.uint8))
Y_np = Y.cpu().squeeze().detach().numpy()
Y_norm_np = np.abs(Y_np)
i0_pil = Image.fromarray(np.uint8(i0_np.clip(0, 255)))
mean_i_np = mean_i.cpu().squeeze().detach().numpy().clip(0, 255)
mean_i_pil = Image.fromarray(mean_i_np.astype(np.uint8))
out_np = out.cpu().squeeze().detach().numpy().clip(0, 255)
out_pil = Image.fromarray(out_np.astype(np.uint8))
diff_np = mean_i_np - clean_im
denoise_obj_name = 'denoise_obj_{:04d}'.format(i) + '.png'
Y_name = 'Y_{:04d}'.format(i) + '.png'
i0_name = 'i0_num_epoch_{:04d}'.format(i) + '.png'
mean_i_name = 'Latent_im_num_epoch_{:04d}'.format(i) + '.png'
out_name = 'res_of_dec_num_epoch_{:04d}'.format(i) + '.png'
diff_name = 'Latent_dis_num_epoch_{:04d}'.format(i) + '.png'
denoise_obj_pil.save(result_root + denoise_obj_name)
save_heatmap(Y_norm_np, result_root + Y_name)
i0_pil.save(result_root + i0_name)
mean_i_pil.save(result_root + mean_i_name)
out_pil.save(result_root + out_name)
save_hist(diff_np, result_root + diff_name)
i0_til_np = i0_til_torch.cpu().squeeze().detach().numpy()
i0_til_np = np.clip(i0_til_np, 0, 255)
psnr = compare_psnr(clean_im, i0_til_np, data_range=255)
ssim = compare_ssim(clean_im, i0_til_np, data_range=255)
i0_til_pil = Image.fromarray(i0_til_np.astype(np.uint8))
i0_til_pil.save(os.path.join(result_root, '{}'.format(i) + '.png'))
print('Iteration: {:02d}, VAE Loss: {:f}, PSNR: {:f}, SSIM: {:f}'.
format(i, total_loss.item(), psnr, ssim),
file=f,
flush=True)
if best_psnr < psnr:
best_psnr = psnr
best_ssim = ssim
else:
break
return i0_til_np, best_psnr, best_ssim
def train(args):
'''
-------------------------Hyperparameters--------------------------
'''
EPOCHS = args.epochs
START = 0 # could enter a checkpoint start epoch
ITER = args.iterations # per epoch
LR = args.lr
MOM = args.momentum
# LOGInterval = args.log_interval
BATCHSIZE = args.batch_size
TEST_BATCHSIZE = args.test_batch_size
NUMBER_OF_WORKERS = args.workers
DATA_FOLDER = args.data
TESTSET_FOLDER = args.testset
ROOT = args.run
WEIGHT_DIR = os.path.join(ROOT, "weights")
CUSTOM_LOG_DIR = os.path.join(ROOT, "additionalLOGS")
CHECKPOINT = os.path.join(WEIGHT_DIR,
str(args.model) + str(args.name) + ".pt")
useTensorboard = args.tb
# check existance of data
if not os.path.isdir(DATA_FOLDER):
print("data folder not existant or in wrong layout.\n\t", DATA_FOLDER)
exit(0)
# check existance of testset
if TESTSET_FOLDER is not None and not os.path.isdir(TESTSET_FOLDER):
print("testset folder not existant or in wrong layout.\n\t",
DATA_FOLDER)
exit(0)
'''
---------------------------preparations---------------------------
'''
# CUDA for PyTorch
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
print("using device: ", str(device))
# loading the validation samples to make online evaluations
path_to_valX = args.valX
path_to_valY = args.valY
valX = None
valY = None
if path_to_valX is not None and path_to_valY is not None \
and os.path.exists(path_to_valX) and os.path.exists(path_to_valY) \
and os.path.isfile(path_to_valX) and os.path.isfile(path_to_valY):
with torch.no_grad():
valX, valY = torch.load(path_to_valX, map_location='cpu'), \
torch.load(path_to_valY, map_location='cpu')
'''
---------------------------loading dataset and normalizing---------------------------
'''
# Dataloader Parameters
train_params = {
'batch_size': BATCHSIZE,
'shuffle': True,
'num_workers': NUMBER_OF_WORKERS
}
test_params = {
'batch_size': TEST_BATCHSIZE,
'shuffle': False,
'num_workers': NUMBER_OF_WORKERS
}
# create a folder for the weights and custom logs
if not os.path.isdir(WEIGHT_DIR):
os.makedirs(WEIGHT_DIR)
if not os.path.isdir(CUSTOM_LOG_DIR):
os.makedirs(CUSTOM_LOG_DIR)
labelsNorm = None
# NORMLABEL
# normalizing on a trainingset wide mean and std
mean = None
std = None
if args.norm:
print('computing mean and std over trainingset')
# computes mean and std over all ground truths in dataset to tackle the problem of numerical insignificance
mean, std = computeMeanStdOverDataset('CONRADataset', DATA_FOLDER,
train_params, device)
print('\niodine (mean/std): {}\t{}'.format(mean[0], std[0]))
print('water (mean/std): {}\t{}\n'.format(mean[1], std[1]))
labelsNorm = transforms.Normalize(mean=[0, 0], std=std)
m2, s2 = computeMeanStdOverDataset('CONRADataset',
DATA_FOLDER,
train_params,
device,
transform=labelsNorm)
print("new mean and std are:")
print('\nnew iodine (mean/std): {}\t{}'.format(m2[0], s2[0]))
print('new water (mean/std): {}\t{}\n'.format(m2[1], s2[1]))
traindata = CONRADataset(DATA_FOLDER,
True,
device=device,
precompute=True,
transform=labelsNorm)
testdata = None
if TESTSET_FOLDER is not None:
testdata = CONRADataset(TESTSET_FOLDER,
False,
device=device,
precompute=True,
transform=labelsNorm)
else:
testdata = CONRADataset(DATA_FOLDER,
False,
device=device,
precompute=True,
transform=labelsNorm)
trainingset = DataLoader(traindata, **train_params)
testset = DataLoader(testdata, **test_params)
'''
----------------loading model and checkpoints---------------------
'''
if args.model == "unet":
m = UNet(2, 2).to(device)
print(
"using the U-Net architecture with {} trainable params; Good Luck!"
.format(count_trainables(m)))
else:
m = simpleConvNet(2, 2).to(device)
o = optim.SGD(m.parameters(), lr=LR, momentum=MOM)
loss_fn = nn.MSELoss()
test_loss = None
train_loss = None
if len(os.listdir(WEIGHT_DIR)) != 0:
checkpoints = os.listdir(WEIGHT_DIR)
checkDir = {}
latestCheckpoint = 0
for i, checkpoint in enumerate(checkpoints):
stepOfCheckpoint = int(
checkpoint.split(str(args.model) +
str(args.name))[-1].split('.pt')[0])
checkDir[stepOfCheckpoint] = checkpoint
latestCheckpoint = max(latestCheckpoint, stepOfCheckpoint)
print("[{}] {}".format(stepOfCheckpoint, checkpoint))
# if on development machine, prompt for input, else just take the most recent one
if 'faui' in os.uname()[1]:
toUse = int(input("select checkpoint to use: "))
else:
toUse = latestCheckpoint
checkpoint = torch.load(os.path.join(WEIGHT_DIR, checkDir[toUse]))
m.load_state_dict(checkpoint['model_state_dict'])
m.to(device) # pushing weights to gpu
o.load_state_dict(checkpoint['optimizer_state_dict'])
train_loss = checkpoint['train_loss']
test_loss = checkpoint['test_loss']
START = checkpoint['epoch']
print("using checkpoint {}:\n\tloss(train/test): {}/{}".format(
toUse, train_loss, test_loss))
else:
print("starting from scratch")
'''
-----------------------------training-----------------------------
'''
global_step = 0
# calculating initial loss
if test_loss is None or train_loss is None:
print("calculating initial loss")
m.eval()
print("testset...")
test_loss = calculate_loss(set=testset,
loss_fn=loss_fn,
length_set=len(testdata),
dev=device,
model=m)
print("trainset...")
train_loss = calculate_loss(set=trainingset,
loss_fn=loss_fn,
length_set=len(traindata),
dev=device,
model=m)
## SSIM and R value
R = []
SSIM = []
performanceFLE = os.path.join(CUSTOM_LOG_DIR, "performance.csv")
with open(performanceFLE, 'w+') as f:
f.write(
"step, SSIMiodine, SSIMwater, Riodine, Rwater, train_loss, test_loss\n"
)
print("computing ssim and r coefficents to: {}".format(performanceFLE))
# printing runtime information
print(
"starting training at {} for {} epochs {} iterations each\n\t{} total".
format(START, EPOCHS, ITER, EPOCHS * ITER))
print("\tbatchsize: {}\n\tloss: {}\n\twill save results to \"{}\"".format(
BATCHSIZE, train_loss, CHECKPOINT))
print(
"\tmodel: {}\n\tlearningrate: {}\n\tmomentum: {}\n\tnorming output space: {}"
.format(args.model, LR, MOM, args.norm))
#start actual training loops
for e in range(START, START + EPOCHS):
# iterations will not be interupted with validation and metrics
for i in range(ITER):
global_step = (e * ITER) + i
# training
m.train()
iteration_loss = 0
for x, y in tqdm(trainingset):
x, y = x.to(device=device,
dtype=torch.float), y.to(device=device,
dtype=torch.float)
pred = m(x)
loss = loss_fn(pred, y)
iteration_loss += loss.item()
o.zero_grad()
loss.backward()
o.step()
print("\niteration {}: --accumulated loss {}".format(
global_step, iteration_loss))
# validation, saving and logging
print("\nvalidating")
m.eval() # disable dropout batchnorm etc
print("testset...")
test_loss = calculate_loss(set=testset,
loss_fn=loss_fn,
length_set=len(testdata),
dev=device,
model=m)
print("trainset...")
train_loss = calculate_loss(set=trainingset,
loss_fn=loss_fn,
length_set=len(traindata),
dev=device,
model=m)
print("calculating SSIM and R coefficients")
currSSIM, currR = performance(set=testset,
dev=device,
model=m,
bs=TEST_BATCHSIZE)
print("SSIM (iod/water): {}/{}\nR (iod/water): {}/{}".format(
currSSIM[0], currSSIM[1], currR[0], currR[1]))
with open(performanceFLE, 'a') as f:
newCSVline = "{}, {}, {}, {}, {}, {}, {}\n".format(
global_step, currSSIM[0], currSSIM[1], currR[0], currR[1],
train_loss, test_loss)
f.write(newCSVline)
print("wrote new line to csv:\n\t{}".format(newCSVline))
'''
if valX and valY were set in preparations, use them to perform analytics.
if not, use the first sample from the testset to perform analytics
'''
with torch.no_grad():
truth, pred = None, None
IMAGE_LOG_DIR = os.path.join(CUSTOM_LOG_DIR, str(global_step))
if not os.path.isdir(IMAGE_LOG_DIR):
os.makedirs(IMAGE_LOG_DIR)
if valX is not None and valY is not None:
batched = np.zeros((BATCHSIZE, *valX.numpy().shape))
batched[0] = valX.numpy()
batched = torch.from_numpy(batched).to(device=device,
dtype=torch.float)
pred = m(batched)
pred = pred.cpu().numpy()[0]
truth = valY.numpy() # still on cpu
assert pred.shape == truth.shape
else:
for x, y in testset:
# x, y in shape[2,2,480,620] [b,c,h,w]
x, y = x.to(device=device,
dtype=torch.float), y.to(device=device,
dtype=torch.float)
pred = m(x)
pred = pred.cpu().numpy()[
0] # taking only the first sample of batch
truth = y.cpu().numpy()[
0] # first projection for evaluation
advanvedMetrics(truth, pred, mean, std, global_step, args.norm,
IMAGE_LOG_DIR)
print("logging")
CHECKPOINT = os.path.join(
WEIGHT_DIR,
str(args.model) + str(args.name) + str(global_step) + ".pt")
torch.save(
{
'epoch': e + 1, # end of this epoch; so resume at next.
'model_state_dict': m.state_dict(),
'optimizer_state_dict': o.state_dict(),
'train_loss': train_loss,
'test_loss': test_loss
},
CHECKPOINT)
print('\tsaved weigths to: ', CHECKPOINT)
if logger is not None and train_loss is not None:
logger.add_scalar('test_loss', test_loss, global_step=global_step)
logger.add_scalar('train_loss',
train_loss,
global_step=global_step)
logger.add_image("iodine-prediction",
pred[0].reshape(1, 480, 620),
global_step=global_step)
logger.add_image("water-prediction",
pred[1].reshape(1, 480, 620),
global_step=global_step)
# logger.add_image("water-prediction", wat)
print(
"\ttensorboard updated with test/train loss and a sample image"
)
elif train_loss is not None:
print("\tloss of global-step {}: {}".format(
global_step, train_loss))
elif not useTensorboard:
print("\t(tb-logging disabled) test/train loss: {}/{} ".format(
test_loss, train_loss))
else:
print("\tno loss accumulated yet")
# saving final results
print("saving upon exit")
torch.save(
{
'epoch': EPOCHS,
'model_state_dict': m.state_dict(),
'optimizer_state_dict': o.state_dict(),
'train_loss': train_loss,
'test_loss': test_loss
}, CHECKPOINT)
print('\tsaved progress to: ', CHECKPOINT)
if logger is not None and train_loss is not None:
logger.add_scalar('test_loss', test_loss, global_step=global_step)
logger.add_scalar('train_loss', train_loss, global_step=global_step)
