pred, loss = model(img, mask)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 100)
opt_dec.step()
opt_en.step()
writer.add_scalar('loss', float(loss), global_step=iterate)
#if iterate % args.display_freq == 0:
# for masked in pred:
# writer.add_image('{}'.format(masked.size()[2]), masked, global_step=iterate)
# writer.add_image('GT', mask, iterate)
# writer.add_image('Image', img, iterate)
if iterate % 200 == 0:
if i != 0:
torch.save(
model.state_dict(),
os.path.join(weight_save_dir, '{}epo_{}step.ckpt'.format(epo, iterate))
)
if iterate % 1000 == 0 and i != 0:
for file in weight_save_dir:
if '00' in file and '000' not in file:
os.remove(os.path.join(weight_save_dir, file))
if i + epo * len(dataloader) % decay_step == 0 and i != 0:
learning_rate *= lr_decay
opt_en = torch.optim.SGD(model.encoder.parameters(), lr=learning_rate, momentum=0.9,
weight_decay=0.0005)
opt_dec = torch.optim.SGD(model.decoder.parameters(), lr=learning_rate * 10, momentum=0.9,
weight_decay=0.0005)
iterate += args.batch_size
del loss
start_iter = 0
opt_dec.zero_grad()
opt_en.zero_grad()
img = batch['image'].cuda()
mask = batch['mask'].cuda()
pred, loss = model(img, mask)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 100)
opt_dec.step()
opt_en.step()
writer.add_scalar('loss', float(loss), global_step=iterate)
if iterate % config['display_freq'] == 0:
for masked in pred:
writer.add_image('{}'.format(masked.size()[2]), masked[0], global_step=iterate)
writer.add_image('GT', mask[0], iterate)
writer.add_image('Image', img[0], iterate)
if iterate % 200 == 0:
if i != 0:
torch.save(model.state_dict(), os.path.join(weight_save_dir, '{}epo_{}step.ckpt'.format(epo, iterate)))
if iterate % 1000 == 0 and i != 0:
for file in weight_save_dir:
if '00' in file and '000' not in file:
os.remove(os.path.join(weight_save_dir, file))
if i + epo * len(dataloader) % config['decay_step'] == 0 and i != 0:
learning_rate *= config[lr_decay]
opt_en = torch.optim.SGD(model.encoder.parameters(), lr=learning_rate, momentum=0.9, weight_decay=0.0005)
opt_dec = torch.optim.SGD(model.decoder.parameters(), lr=learning_rate * 10, momentum=0.9, weight_decay=0.0005)
iterate += config['batch_size']
del loss
start_iter = 0