pred, loss = model(img, mask) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 100) opt_dec.step() opt_en.step() writer.add_scalar('loss', float(loss), global_step=iterate) #if iterate % args.display_freq == 0: # for masked in pred: # writer.add_image('{}'.format(masked.size()[2]), masked, global_step=iterate) # writer.add_image('GT', mask, iterate) # writer.add_image('Image', img, iterate) if iterate % 200 == 0: if i != 0: torch.save( model.state_dict(), os.path.join(weight_save_dir, '{}epo_{}step.ckpt'.format(epo, iterate)) ) if iterate % 1000 == 0 and i != 0: for file in weight_save_dir: if '00' in file and '000' not in file: os.remove(os.path.join(weight_save_dir, file)) if i + epo * len(dataloader) % decay_step == 0 and i != 0: learning_rate *= lr_decay opt_en = torch.optim.SGD(model.encoder.parameters(), lr=learning_rate, momentum=0.9, weight_decay=0.0005) opt_dec = torch.optim.SGD(model.decoder.parameters(), lr=learning_rate * 10, momentum=0.9, weight_decay=0.0005) iterate += args.batch_size del loss start_iter = 0
opt_dec.zero_grad() opt_en.zero_grad() img = batch['image'].cuda() mask = batch['mask'].cuda() pred, loss = model(img, mask) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 100) opt_dec.step() opt_en.step() writer.add_scalar('loss', float(loss), global_step=iterate) if iterate % config['display_freq'] == 0: for masked in pred: writer.add_image('{}'.format(masked.size()[2]), masked[0], global_step=iterate) writer.add_image('GT', mask[0], iterate) writer.add_image('Image', img[0], iterate) if iterate % 200 == 0: if i != 0: torch.save(model.state_dict(), os.path.join(weight_save_dir, '{}epo_{}step.ckpt'.format(epo, iterate))) if iterate % 1000 == 0 and i != 0: for file in weight_save_dir: if '00' in file and '000' not in file: os.remove(os.path.join(weight_save_dir, file)) if i + epo * len(dataloader) % config['decay_step'] == 0 and i != 0: learning_rate *= config[lr_decay] opt_en = torch.optim.SGD(model.encoder.parameters(), lr=learning_rate, momentum=0.9, weight_decay=0.0005) opt_dec = torch.optim.SGD(model.decoder.parameters(), lr=learning_rate * 10, momentum=0.9, weight_decay=0.0005) iterate += config['batch_size'] del loss start_iter = 0