def stariy_hm_loss(pred, target, coef=1.0): pred_mes = UniformMeasure2DFactory.from_heatmap(pred) target_mes = UniformMeasure2DFactory.from_heatmap(target) return Loss(nn.BCELoss()(pred, target) * coef + nn.MSELoss()(pred_mes.coord, target_mes.coord) * (0.001 * coef) + nn.L1Loss()(pred_mes.coord, target_mes.coord) * (0.001 * coef))
def verka(encoder: nn.Module): res = [] for i, (image, lm) in enumerate(LazyLoader.celeba_test(64)): content = encoder(image.cuda()) mes = UniformMeasure2D01(lm.cuda()) pred_measures: UniformMeasure2D01 = UniformMeasure2DFactory.from_heatmap(content) res.append(Samples_Loss(p=1)(mes, pred_measures).item() * image.shape[0]) return np.mean(res)/len(LazyLoader.celeba_test(1).dataset)
def forward(self, image: Tensor): img_content = self.gh(image) pred_measures: UniformMeasure2D01 = UniformMeasure2DFactory.from_heatmap( img_content) sparce_hm = self.heatmaper.forward(pred_measures.coord * 63) return pred_measures, sparce_hm
def hm_svoego_roda_loss(pred, target, coef=1.0, l1_coef=0.0): pred_mes = UniformMeasure2DFactory.from_heatmap(pred) target_mes = UniformMeasure2DFactory.from_heatmap(target) # pred = pred.relu() + 1e-15 # target[target < 1e-7] = 0 # target[target > 1 - 1e-7] = 1 if torch.isnan(pred).any() or torch.isnan(target).any(): print("nan in hm") return Loss.ZERO() bce = nn.BCELoss()(pred, target) if torch.isnan(bce).any(): print("nan in bce") return Loss.ZERO() return Loss(bce * coef + nn.MSELoss()(pred_mes.coord, target_mes.coord) * (0.0005 * coef) + nn.L1Loss()(pred_mes.coord, target_mes.coord) * l1_coef)
def liuboff(encoder: nn.Module): sum_loss = 0 for i, batch in enumerate(LazyLoader.w300().test_loader): data = batch['data'].to(device) landmarks = batch["meta"]["keypts_normalized"].cuda() landmarks[landmarks > 1] = 0.99999 # content = heatmap_to_measure(encoder(data))[0] pred_measure = UniformMeasure2DFactory.from_heatmap(encoder(data)) target = UniformMeasure2D01(torch.clamp(landmarks, max=1)) eye_dist = landmarks[:, 45] - landmarks[:, 36] eye_dist = eye_dist.pow(2).sum(dim=1).sqrt() sum_loss += (handmadew1(pred_measure, target) / eye_dist).sum().item() return sum_loss / len(LazyLoader.w300().test_dataset)
def train_content(cont_opt, R_b, R_t, real_img, heatmaper, g_transforms): requires_grad(encoder_HG, True) coefs = json.load(open("../parameters/content_loss_sup.json")) # pred_measures, sparse_hm = encoder_HG(real_img) content = encoder_HG(real_img) pred_measures: UniformMeasure2D01 = UniformMeasure2DFactory.from_heatmap(content) sparse_hm = heatmaper.forward(pred_measures.coord * 63) ll = ( writable("R_b", R_b.__call__)(real_img, pred_measures) * coefs["R_b"] + writable("Sparse", hm_loss_bes_xy)(content, sparse_hm.detach()) * coefs["Sparse"] + writable("R_t", R_t.__call__)(real_img, sparse_hm) * coefs["R_t"] ) ll.minimize_step(cont_opt)
def liuboff(encoder: nn.Module): sum_loss = 0 for i, batch in enumerate(LazyLoader.mafl().test_loader): data = batch['data'].to(device) landmarks = batch["meta"]["keypts_normalized"].cuda().type(dtype=torch.float32) landmarks[landmarks > 1] = 0.99999 # content = heatmap_to_measure(encoder(data))[0] pred_measure = UniformMeasure2DFactory.from_heatmap(encoder(data)) target = UniformMeasure2D01(torch.clamp(landmarks, max=1)) eye_dist = landmarks[:, 1] - landmarks[:, 0] eye_dist = eye_dist.pow(2).sum(dim=1).sqrt() # w1_loss = (handmadew1(pred_measure, target) / eye_dist).sum().item() # l1_loss = ((pred_measure.coord - target.coord).pow(2).sum(dim=2).sqrt().mean(dim=1) / eye_dist).sum().item() # print(w1_loss, l1_loss) sum_loss += ((pred_measure.coord - target.coord).pow(2).sum(dim=2).sqrt().mean(dim=1) / eye_dist).sum().item() return sum_loss / len(LazyLoader.mafl().test_dataset)
def train_content(cont_opt, tuner_verka, heatmaper, encoder_HG, R_b, R_t, real_img): requires_grad(encoder_HG, True) img_content = encoder_HG(real_img) pred_measures: UniformMeasure2D01 = UniformMeasure2DFactory.from_heatmap( img_content) # sparce_hm = blur( # heatmaper.forward(pred_measures.probability, pred_measures.coord * 63).detach() # ) sparce_hm = heatmaper.forward(pred_measures.coord * 63).detach() # sparce_hm = sparse_heatmap(img_content).detach() ll = tuner_verka.sum_losses([ writable("R_b", R_b.__call__)(real_img, pred_measures) * 0.5, writable("Sparse", hm_svoego_roda_loss)(img_content, sparce_hm, 500) * 0.5, writable("R_t", R_t.__call__)(real_img, sparce_hm) * 2.5 ]) # print(i, ll.item()) ll.minimize_step(cont_opt)
def coord_hm_loss(pred_coord: Tensor, target_hm: Tensor, coef=1.0): target_hm = target_hm / target_hm.sum(dim=[2, 3], keepdim=True) target_hm = target_hm.detach() heatmapper = ToGaussHeatMap(256, 4) target_coord = UniformMeasure2DFactory.from_heatmap( target_hm).coord.detach() # sk = CoordToGaussSkeleton(target_hm.shape[-1], 1) # pred_sk = sk.forward(pred_coord).sum(dim=1, keepdim=True) # target_sk = sk.forward(target_coord).sum(dim=1, keepdim=True).detach() pred_hm = heatmapper.forward(pred_coord).sum(dim=1, keepdim=True) pred_hm = pred_hm / pred_hm.sum(dim=[2, 3], keepdim=True).detach() target_hm = heatmapper.forward(target_coord).sum(dim=1, keepdim=True).detach() target_hm = target_hm / target_hm.sum(dim=[2, 3], keepdim=True).detach() return Loss(nn.BCELoss()(pred_hm, target_hm) * coef * 1.5 + # noviy_hm_loss(pred_sk, target_sk, coef).to_tensor() * 0.5 + nn.MSELoss()(pred_coord, target_coord) * (0.001 * coef) + nn.L1Loss()(pred_coord, target_coord) * (0.001 * coef))
def do_train(real_img): batch_size = real_img.shape[0] requires_grad(encoder_HG, True) requires_grad(generator, False) requires_grad(model.loss.discriminator, False) # pred_measures, sparse_hm = encoder_HG(real_img) img_content = encoder_HG(real_img) pred_measures: UniformMeasure2D01 = UniformMeasure2DFactory.from_heatmap(img_content) sparse_hm = heatmaper.forward(pred_measures.coord * 63).detach() # restored = decoder(sparse_hm, style_encoder(real_img)) trans = g_transforms(image=real_img, mask=img_content) trans_content, trans_image = trans["mask"], trans["image"] restored = decoder(img_content, style_encoder(trans_image)) noise1 = mixing_noise(batch_size, latent_size, 0.9, device) fake1, _ = generator(trans_content, noise1) trans_fake_content = encoder_HG(fake1.detach()) coefs = json.load(open("../parameters/content_loss.json")) tuner.sum_losses([ writable("Fake-content D", model.loss.generator_loss)( real=None, fake=[fake1, img_content.detach()]) * coefs["Fake-content D"], # 50 000 writable("Real-content D", model.loss.discriminator_loss_as_is)( [real_img, img_content], [fake1.detach(), img_content]) * coefs["Real-content D"], # 3000 writable("R_b", R_b.__call__)(real_img, pred_measures) * coefs["R_b"], # 3000 writable("Sparse", hm_loss_bes_xy)(img_content, sparse_hm) * coefs["Sparse"], # 1.5 writable("R_t", R_t.__call__)(real_img, sparse_hm) * coefs["R_t"], # 3 L1("L1 image")(restored, real_img) * coefs["L1 image"], # 10 writable("fake_content loss", stariy_hm_loss)( trans_fake_content, trans_content ) * coefs["fake_content loss"] # 6477 ]).minimize_step( cont_opt )
def do_train(real_img): batch_size = real_img.shape[0] requires_grad(encoder_HG, True) img_content = encoder_HG(real_img) pred_measures: UniformMeasure2D01 = UniformMeasure2DFactory.from_heatmap( img_content) sparce_hm = heatmaper.forward(pred_measures.coord * 63) restored = decoder(sparce_hm, style_encoder(real_img)) noise1 = mixing_noise(batch_size, latent_size, 0.9, device) fake1, _ = generator(img_content, noise1) cont_fake1 = encoder_HG(fake1.detach()) coefs = json.load(open("../parameters/content_loss_sup.json")) tuner.sum_losses([ writable("Fake-content D", model.loss.generator_loss)( real=None, fake=[fake1, img_content.detach()]) * coefs["Fake-content D"], # 800 writable("Real-content D", model.loss.generator_loss)( real=None, fake=[real_img, img_content]) * coefs["Real-content D"], # 5 writable("R_b", R_b.__call__)(real_img, pred_measures) * coefs["R_b"], # 3000 writable("Sparse", hm_svoego_roda_loss)( img_content, sparce_hm.detach(), 500) * coefs["Sparse"], # 500 writable("R_t", R_t.__call__)(real_img, img_content) * coefs["R_t"], # 16155 L1("L1 image")(restored, real_img) * coefs["L1 image"], # 10 writable("fake_content loss", hm_svoego_roda_loss)( cont_fake1, img_content.detach(), 500, 0.1) * coefs["fake_content loss"] # 6477 ]).minimize_step(cont_opt)
def train(generator, decoder, discriminator, encoder_HG, style_encoder, device, starting_model_number): latent_size = 512 batch_size = 12 sample_z = torch.randn(8, latent_size, device=device) MAFL.batch_size = batch_size MAFL.test_batch_size = 64 Celeba.batch_size = batch_size test_img = next(LazyLoader.mafl().loader_train_inf)["data"][:8].cuda() loss_st: StyleGANLoss = StyleGANLoss(discriminator) model = CondStyleGanModel(generator, loss_st, (0.001, 0.0015)) style_opt = optim.Adam(style_encoder.parameters(), lr=5e-4, betas=(0.9, 0.99)) cont_opt = optim.Adam(encoder_HG.parameters(), lr=2e-5, betas=(0.5, 0.97)) g_transforms: albumentations.DualTransform = albumentations.Compose([ ToNumpy(), NumpyBatch(albumentations.Compose([ ResizeMask(h=256, w=256), albumentations.ElasticTransform(p=0.7, alpha=150, alpha_affine=1, sigma=10), albumentations.ShiftScaleRotate(p=0.7, rotate_limit=15), ResizeMask(h=64, w=64), NormalizeMask(dim=(0, 1, 2)) ])), ToTensor(device), ]) R_t = DualTransformRegularizer.__call__( g_transforms, lambda trans_dict, img: # rt_loss(encoder_HG(trans_dict['image']), trans_dict['mask']) stariy_hm_loss(encoder_HG(trans_dict['image']), trans_dict['mask']) ) R_s = UnoTransformRegularizer.__call__( g_transforms, lambda trans_dict, img, ltnt: L1("R_s")(ltnt, style_encoder(trans_dict['image'])) ) barycenter: UniformMeasure2D01 = UniformMeasure2DFactory.load( f"{Paths.default.models()}/face_barycenter_5").cuda().batch_repeat(batch_size) R_b = BarycenterRegularizer.__call__(barycenter, 1.0, 2.0, 4.0) tuner = GoldTuner([0.37, 1.55, 0.9393, 0.1264, 1.7687, 0.8648, 1.8609], device=device, rule_eps=0.01 / 2, radius=0.1, active=True) heatmaper = ToGaussHeatMap(64, 1.0) sparse_bc = heatmaper.forward(barycenter.coord * 63) sparse_bc = nn.Upsample(scale_factor=4)(sparse_bc).sum(dim=1, keepdim=True).repeat(1, 3, 1, 1) * \ torch.tensor([1.0, 1.0, 0.0], device=device).view(1, 3, 1, 1) sparse_bc = (sparse_bc - sparse_bc.min()) / sparse_bc.max() send_images_to_tensorboard(writer, sparse_bc, "BC", 0, normalize=False, range=(0, 1)) trainer_gan = gan_trainer(model, generator, decoder, encoder_HG, style_encoder, R_s, style_opt, heatmaper, g_transforms) content_trainer = content_trainer_with_gan(cont_opt, tuner, heatmaper, encoder_HG, R_b, R_t, model, generator, g_transforms) supervise_trainer = content_trainer_supervised(cont_opt, encoder_HG, LazyLoader.mafl().loader_train_inf) for i in range(100000): counter.update(i) requires_grad(encoder_HG, False) # REMOVE BEFORE TRAINING real_img = next(LazyLoader.mafl().loader_train_inf)["data"].to(device) \ if i % 5 == 0 else next(LazyLoader.celeba().loader).to(device) img_content = encoder_HG(real_img) pred_measures: UniformMeasure2D01 = UniformMeasure2DFactory.from_heatmap(img_content) sparse_hm = heatmaper.forward(pred_measures.coord * 63).detach() trainer_gan(i, real_img, pred_measures.detach(), sparse_hm.detach(), apply_g=False) supervise_trainer() if i % 4 == 0: # real_img = next(LazyLoader.mafl().loader_train_inf)["data"].to(device) trainer_gan(i, real_img, pred_measures.detach(), sparse_hm.detach(), apply_g=True) content_trainer(real_img) if i % 100 == 0: coefs = json.load(open("../parameters/content_loss.json")) print(i, coefs) with torch.no_grad(): # pred_measures_test, sparse_hm_test = encoder_HG(test_img) content_test = encoder_HG(test_img) pred_measures_test: UniformMeasure2D01 = UniformMeasure2DFactory.from_heatmap(content_test) heatmaper_256 = ToGaussHeatMap(256, 2.0) sparse_hm_test = heatmaper.forward(pred_measures_test.coord * 63) sparse_hm_test_1 = heatmaper_256.forward(pred_measures_test.coord * 255) latent_test = style_encoder(test_img) sparce_mask = sparse_hm_test_1.sum(dim=1, keepdim=True) sparce_mask[sparce_mask < 0.0003] = 0 iwm = imgs_with_mask(test_img, sparce_mask) send_images_to_tensorboard(writer, iwm, "REAL", i) fake_img, _ = generator(sparse_hm_test, [sample_z]) iwm = imgs_with_mask(fake_img, pred_measures_test.toImage(256)) send_images_to_tensorboard(writer, iwm, "FAKE", i) restored = decoder(sparse_hm_test, latent_test) iwm = imgs_with_mask(restored, pred_measures_test.toImage(256)) send_images_to_tensorboard(writer, iwm, "RESTORED", i) content_test_256 = nn.Upsample(scale_factor=4)(sparse_hm_test).sum(dim=1, keepdim=True).repeat(1, 3, 1, 1) * \ torch.tensor([1.0, 1.0, 0.0], device=device).view(1, 3, 1, 1) content_test_256 = (content_test_256 - content_test_256.min()) / content_test_256.max() send_images_to_tensorboard(writer, content_test_256, "HM", i, normalize=False, range=(0, 1)) if i % 50 == 0 and i >= 0: test_loss = liuboff(encoder_HG) # test_loss = nadbka(encoder_HG) tuner.update(test_loss) writer.add_scalar("liuboff", test_loss, i) if i % 10000 == 0 and i > 0: torch.save( { 'g': generator.module.state_dict(), 'd': discriminator.module.state_dict(), 'c': encoder_HG.module.state_dict(), "s": style_encoder.state_dict() }, f'{Paths.default.models()}/stylegan2_MAFL_{str(i + starting_model_number).zfill(6)}.pt', )
def train(generator, decoder, discriminator, encoder_HG, style_encoder, device, starting_model_number): latent_size = 512 batch_size = 8 sample_z = torch.randn(8, latent_size, device=device) Celeba.batch_size = batch_size W300DatasetLoader.batch_size = batch_size W300DatasetLoader.test_batch_size = 16 test_img = next(LazyLoader.w300().loader_train_inf)["data"][:8].cuda() model = CondStyleGanModel(generator, StyleGANLoss(discriminator), (0.001 / 4, 0.0015 / 4)) style_opt = optim.Adam(style_encoder.parameters(), lr=5e-4, betas=(0.9, 0.99)) cont_opt = optim.Adam(encoder_HG.parameters(), lr=3e-5, betas=(0.5, 0.97)) g_transforms: albumentations.DualTransform = albumentations.Compose([ ToNumpy(), NumpyBatch( albumentations.Compose([ albumentations.ElasticTransform(p=0.7, alpha=150, alpha_affine=1, sigma=10), albumentations.ShiftScaleRotate(p=0.9, rotate_limit=15), ])), ToTensor(device), ]) g_transforms_without_norm: albumentations.DualTransform = albumentations.Compose( [ ToNumpy(), NumpyBatch( albumentations.Compose([ albumentations.ElasticTransform(p=0.3, alpha=150, alpha_affine=1, sigma=10), albumentations.ShiftScaleRotate(p=0.7, rotate_limit=15), ])), ToTensor(device), ]) R_t = DualTransformRegularizer.__call__( g_transforms, lambda trans_dict, img: coord_hm_loss( encoder_HG(trans_dict['image'])["coords"], trans_dict['mask'])) R_s = UnoTransformRegularizer.__call__( g_transforms, lambda trans_dict, img, ltnt: WR.L1("R_s") (ltnt, style_encoder(trans_dict['image']))) barycenter: UniformMeasure2D01 = UniformMeasure2DFactory.load( f"{Paths.default.models()}/face_barycenter_68").cuda().batch_repeat( batch_size) R_b = BarycenterRegularizer.__call__(barycenter, 1.0, 2.0, 4.0) tuner = GoldTuner([0.37, 2.78, 0.58, 1.43, 3.23], device=device, rule_eps=0.001, radius=0.3, active=False) trainer_gan = gan_trainer(model, generator, decoder, encoder_HG, style_encoder, R_s, style_opt, g_transforms) content_trainer = content_trainer_with_gan(cont_opt, tuner, encoder_HG, R_b, R_t, model, generator, g_transforms, decoder, style_encoder) supervise_trainer = content_trainer_supervised( cont_opt, encoder_HG, LazyLoader.w300().loader_train_inf) for i in range(11000): WR.counter.update(i) requires_grad(encoder_HG, False) real_img = next(LazyLoader.celeba().loader).to(device) encoded = encoder_HG(real_img) internal_content = encoded["skeleton"].detach() trainer_gan(i, real_img, internal_content) # content_trainer(real_img) train_content(cont_opt, R_b, R_t, real_img, model, encoder_HG, decoder, generator, style_encoder) supervise_trainer() encoder_ema.accumulate(encoder_HG.module, i, 0.98) if i % 50 == 0 and i > 0: encoder_ema.write_to(encoder_HG.module) if i % 100 == 0: coefs = json.load(open("../parameters/content_loss.json")) print(i, coefs) with torch.no_grad(): # pred_measures_test, sparse_hm_test = encoder_HG(test_img) encoded_test = encoder_HG(test_img) pred_measures_test: UniformMeasure2D01 = UniformMeasure2D01( encoded_test["coords"]) heatmaper_256 = ToGaussHeatMap(256, 1.0) sparse_hm_test_1 = heatmaper_256.forward( pred_measures_test.coord) latent_test = style_encoder(test_img) sparce_mask = sparse_hm_test_1.sum(dim=1, keepdim=True) sparce_mask[sparce_mask < 0.0003] = 0 iwm = imgs_with_mask(test_img, sparce_mask) send_images_to_tensorboard(WR.writer, iwm, "REAL", i) fake_img, _ = generator(encoded_test["skeleton"], [sample_z]) iwm = imgs_with_mask(fake_img, pred_measures_test.toImage(256)) send_images_to_tensorboard(WR.writer, iwm, "FAKE", i) restored = decoder(encoded_test["skeleton"], latent_test) iwm = imgs_with_mask(restored, pred_measures_test.toImage(256)) send_images_to_tensorboard(WR.writer, iwm, "RESTORED", i) content_test_256 = (encoded_test["skeleton"]).repeat(1, 3, 1, 1) * \ torch.tensor([1.0, 1.0, 0.0], device=device).view(1, 3, 1, 1) content_test_256 = (content_test_256 - content_test_256.min() ) / content_test_256.max() send_images_to_tensorboard(WR.writer, content_test_256, "HM", i, normalize=False, range=(0, 1)) if i % 50 == 0 and i >= 0: test_loss = liuboff(encoder_HG) print("liuboff", test_loss) # test_loss = nadbka(encoder_HG) tuner.update(test_loss) WR.writer.add_scalar("liuboff", test_loss, i) if i % 10000 == 0 and i > 0: torch.save( { 'g': generator.module.state_dict(), 'd': discriminator.module.state_dict(), 'c': encoder_HG.module.state_dict(), "s": style_encoder.state_dict(), "e": encoder_ema.storage_model.state_dict() }, f'{Paths.default.models()}/stylegan2_new_{str(i + starting_model_number).zfill(6)}.pt', )
NumpyBatch( albumentations.Compose([ albumentations.ElasticTransform(p=0.7, alpha=150, alpha_affine=1, sigma=10), albumentations.ShiftScaleRotate(p=0.9, rotate_limit=15), ])), ToTensor(device), ]) R_t = DualTransformRegularizer.__call__( g_transforms_2, lambda trans_dict, img: coord_hm_loss( hg(trans_dict['image'])["mes"].coord, trans_dict['mask'])) barycenter: UniformMeasure2D01 = UniformMeasure2DFactory.load( f"{Paths.default.models()}/cardio_200").cuda().batch_repeat(batch_size) R_b = BarycenterRegularizer.__call__(barycenter, 0.5, 1.0, 5.0) def train_content(cont_opt, R_b, R_t, real_img, encoder_HG): # heatmapper = ToGaussHeatMap(256, 4) requires_grad(encoder_HG, True) coefs = json.load( open(os.path.join(sys.path[0], "../parameters/content_loss.json"))) encoded = encoder_HG(real_img) pred_measures: UniformMeasure2D01 = encoded["mes"] heatmap_content = encoded["hm"]
def train(generator, decoder, discriminator, discriminatorHG, encoder_HG, style_encoder, device, starting_model_number): latent_size = 512 batch_size = 12 sample_z = torch.randn(8, latent_size, device=device) Celeba.batch_size = batch_size W300DatasetLoader.batch_size = batch_size W300DatasetLoader.test_batch_size = 64 encoder_HG_supervised = HG_softmax2020(num_classes=68, heatmap_size=64) encoder_HG_supervised.load_state_dict(torch.load(f'{Paths.default.models()}/hg2_e29.pt', map_location="cpu")) encoder_HG_supervised = encoder_HG_supervised.cuda() requires_grad(encoder_HG_supervised, False) style_opt = optim.Adam(style_encoder.parameters(), lr=5e-4, betas=(0.9, 0.99)) test_img = next(LazyLoader.celeba().loader)[:8].cuda() loss_st: StyleGANLoss = StyleGANLoss(discriminator) model = CondStyleGanModel(generator, loss_st, (0.001, 0.0015)) loss_st2: StyleGANLoss = StyleGANLoss(discriminatorHG) model2 = CondStyleGanModel(encoder_HG, loss_st2, (2e-5, 0.0005)) heatmaper = ToGaussHeatMap(64, 1.0) # wide_heatmaper = HeatMapToGaussHeatMap(64, 5.0) tuda_trainer = gan_tuda_trainer(model, generator) obratno_trainer = gan_obratno_trainer(model2, encoder_HG) tuda_obratno_trainer = gan_tuda_obratno_trainer(generator, encoder_HG, decoder, style_encoder, style_opt) for i in range(100000): counter.update(i) # requires_grad(encoder_HG, False) # REMOVE BEFORE TRAINING real_img = next(LazyLoader.celeba().loader).to(device) heatmap = encoder_HG_supervised(real_img).detach() # heatmap = wide_heatmaper.forward(encoder_HG_supervised(real_img)).detach() # heatmap = encoder_HG_supervised(next(LazyLoader.celeba().loader).to(device)).detach() # tuda_trainer(real_img, heatmap) obratno_trainer(real_img, heatmap) # tuda_obratno_trainer(real_img, heatmap) # pred_hm = encoder_HG(real_img) # stariy_hm_loss(pred_hm, heatmap).minimize_step(model2.optimizer.opt_min) if i % 100 == 0: with torch.no_grad(): content_test = encoder_HG(test_img) pred_measures_test: UniformMeasure2D01 = UniformMeasure2DFactory.from_heatmap(content_test) heatmaper_256 = ToGaussHeatMap(256, 2.0) sparse_hm_test = heatmaper.forward(pred_measures_test.coord * 63) sparse_hm_test_1 = heatmaper_256.forward(pred_measures_test.coord * 255) latent_test = style_encoder(test_img) sparce_mask = sparse_hm_test_1.sum(dim=1, keepdim=True) sparce_mask[sparce_mask < 0.0003] = 0 iwm = imgs_with_mask(test_img, sparce_mask) send_images_to_tensorboard(writer, iwm, "REAL", i) fake_img, _ = generator(sparse_hm_test, [sample_z]) iwm = imgs_with_mask(fake_img, pred_measures_test.toImage(256)) send_images_to_tensorboard(writer, iwm, "FAKE", i) restored = decoder(sparse_hm_test, latent_test) iwm = imgs_with_mask(restored, pred_measures_test.toImage(256)) send_images_to_tensorboard(writer, iwm, "RESTORED", i) content_test_256 = nn.Upsample(scale_factor=4)(sparse_hm_test).sum(dim=1, keepdim=True).repeat(1, 3, 1, 1) * \ torch.tensor([1.0, 1.0, 0.0], device=device).view(1, 3, 1, 1) content_test_256 = (content_test_256 - content_test_256.min()) / content_test_256.max() send_images_to_tensorboard(writer, content_test_256, "HM", i, normalize=False, range=(0, 1)) if i % 50 == 0 and i >= 0: test_loss = liuboff(encoder_HG) print("liuboff", test_loss) # test_loss = nadbka(encoder_HG) # tuner.update(test_loss) writer.add_scalar("liuboff", test_loss, i) if i % 10000 == 0 and i > 0: torch.save( { 'g': generator.module.state_dict(), 'd': discriminator.module.state_dict(), 'c': encoder_HG.module.state_dict(), "s": style_encoder.state_dict(), "d2": discriminatorHG.state_dict() }, f'{Paths.default.models()}/cyclegan_{str(i + starting_model_number).zfill(6)}.pt', )
ResizeMask(h=256, w=256), albumentations.ElasticTransform(p=1, alpha=150, alpha_affine=1, sigma=10), # albumentations.ShiftScaleRotate(p=0.7, rotate_limit=15), ResizeMask(h=64, w=64), NormalizeMask(dim=(0, 1, 2)) ])), ToTensor(device), ]) with torch.no_grad(): from matplotlib import pyplot as plt content_test = encoder_HG(test_img) pred_measures = UniformMeasure2DFactory.from_heatmap(content_test) sparce_hm = heatmaper.forward(pred_measures.coord * 63).detach() fake_img, _ = generator(sparce_hm, noise) iwm = imgs_with_mask(fake_img, pred_measures.toImage(256)) plt.imshow(((iwm[0].permute(1, 2, 0) + 1) / 2).cpu().numpy()) plt.show() trans_dict = g_transforms(image=test_img, mask=sparce_hm) sparce_hm = trans_dict["mask"] pred_measures = UniformMeasure2DFactory.from_heatmap(sparce_hm) #iwm = imgs_with_mask(test_img, pred_measures.toImage(256)) fake_img, _ = generator(sparce_hm, noise) iwm = imgs_with_mask(fake_img, pred_measures.toImage(256))
def train(generator, decoder, discriminator, encoder_HG, style_encoder, device, starting_model_number): latent_size = 512 batch_size = 24 sample_z = torch.randn(8, latent_size, device=device) Celeba.batch_size = batch_size test_img = next(LazyLoader.celeba().loader)[:8].cuda() loss_st: StyleGANLoss = StyleGANLoss(discriminator) model = CondStyleGanModel(generator, loss_st, (0.001, 0.0015)) style_opt = optim.Adam(style_encoder.parameters(), lr=5e-4, betas=(0.9, 0.99)) cont_opt = optim.Adam(encoder_HG.parameters(), lr=4e-5, betas=(0.5, 0.97)) g_transforms: albumentations.DualTransform = albumentations.Compose([ ToNumpy(), NumpyBatch( albumentations.Compose([ ResizeMask(h=256, w=256), albumentations.ElasticTransform(p=0.7, alpha=150, alpha_affine=1, sigma=10), albumentations.ShiftScaleRotate(p=0.7, rotate_limit=15), ResizeMask(h=64, w=64), NormalizeMask(dim=(0, 1, 2)) ])), ToTensor(device), ]) R_t = DualTransformRegularizer.__call__( g_transforms, lambda trans_dict, img: hm_svoego_roda_loss( encoder_HG(trans_dict['image']), trans_dict['mask'], 1000, 0.3)) R_s = UnoTransformRegularizer.__call__( g_transforms, lambda trans_dict, img, ltnt: L1("R_s") (ltnt, style_encoder(trans_dict['image']))) barycenter: UniformMeasure2D01 = UniformMeasure2DFactory.load( f"{Paths.default.models()}/face_barycenter_68").cuda().batch_repeat( batch_size) # plt.imshow(barycenter.toImage(256)[0][0].detach().cpu().numpy()) # plt.show() R_b = BarycenterRegularizer.__call__(barycenter, 1.0, 2.0, 3.0) # 4.5, 1.2, 1.12, 1.4, 0.07, 2.2 # 1.27, 3.55, 5.88, 3.83, 2.17, 0.22, 1.72 tuner = GoldTuner([2.2112, 2.3467, 3.8438, 3.2202, 2.0494, 0.0260, 5.8378], device=device, rule_eps=0.03, radius=1, active=True) # tuner_verka = GoldTuner([3.0, 1.2, 2.0], device=device, rule_eps=0.05, radius=1, active=True) best_igor = 100 heatmaper = ToGaussHeatMap(64, 1.5) trainer_gan = gan_trainer(model, generator, decoder, encoder_HG, style_encoder, R_s, style_opt, heatmaper, g_transforms) content_trainer = content_trainer_with_gan(cont_opt, tuner, heatmaper, encoder_HG, R_b, R_t, model, generator) for i in range(100000): counter.update(i) requires_grad(encoder_HG, False) # REMOVE BEFORE TRAINING real_img = next(LazyLoader.celeba().loader).to(device) img_content = encoder_HG(real_img).detach() pred_measures: UniformMeasure2D01 = UniformMeasure2DFactory.from_heatmap( img_content) sparce_hm = heatmaper.forward(pred_measures.coord * 63).detach() trainer_gan(i, real_img, img_content, sparce_hm) if i % 3 == 0: real_img = next(LazyLoader.celeba().loader).to(device) content_trainer(real_img) if i % 100 == 0: coefs = json.load(open("../parameters/content_loss_sup.json")) print(i, coefs) with torch.no_grad(): content_test = encoder_HG(test_img) latent_test = style_encoder(test_img) pred_measures = UniformMeasure2DFactory.from_heatmap( content_test) sparce_hm = heatmaper.forward(pred_measures.coord * 63).detach() iwm = imgs_with_mask(test_img, pred_measures.toImage(256)) send_images_to_tensorboard(writer, iwm, "REAL", i) fake_img, _ = generator(sparce_hm, [sample_z]) iwm = imgs_with_mask(fake_img, pred_measures.toImage(256)) send_images_to_tensorboard(writer, iwm, "FAKE", i) restored = decoder(sparce_hm, latent_test) iwm = imgs_with_mask(restored, pred_measures.toImage(256)) send_images_to_tensorboard(writer, iwm, "RESTORED", i) content_test_256 = nn.Upsample( scale_factor=4)(content_test).sum(dim=1, keepdim=True) content_test_256 = content_test_256 / content_test_256.max() send_images_to_tensorboard(writer, content_test_256, "HM", i) if i % 50 == 0 and i > 0: test_loss = verka(encoder_HG) tuner.update(test_loss) writer.add_scalar("verka", test_loss, i) if i % 10000 == 0 and i > 0: torch.save( { 'g': generator.module.state_dict(), 'd': discriminator.module.state_dict(), 'c': encoder_HG.module.state_dict(), "s": style_encoder.state_dict() }, f'{Paths.default.models()}/stylegan2_new_{str(i + starting_model_number).zfill(6)}.pt', )
def forward(self, coord: Tensor): p0, p1 = self.find_pairs(coord) assert not torch.any(torch.isnan(p0)) assert not torch.any(torch.isnan(p1)) assert p0.max().item() <= 1 + 1e-5 assert p1.max().item() <= 1 + 1e-5 return super().forward(p0 * (self.size - 1), p1 * (self.size - 1)) if __name__ == "__main__": from matplotlib import pyplot as plt from dataset.probmeasure import UniformMeasure2DFactory, UniformMeasure2D01 N = 10 B = 1 size = 256 barycenter: UniformMeasure2D01 = UniformMeasure2DFactory.load( f"{Paths.default.models()}/face_barycenter_68").batch_repeat(10) barycenter.coord = barycenter.coord[:, torch.randperm(68)] # hm = CoordToGaussSkeleton(size, 5).forward(barycenter.coord) # plt.imshow(hm[0].sum(0)) # # plt.show()