def SinGAN_generate(Gs,Zs,reals,NoiseAmp,opt,in_s=None,scale_v=1,scale_h=1,n=0,gen_start_scale=0,num_samples=50):
#if torch.is_tensor(in_s) == False:
if in_s is None:
in_s = torch.full(reals[0].shape, 0, device=opt.device)
images_cur = []
for G,Z_opt,noise_amp in zip(Gs,Zs,NoiseAmp):
pad1 = ((opt.ker_size-1)*opt.num_layer)/2
m = nn.ZeroPad2d(int(pad1))
nzx = (Z_opt.shape[2]-pad1*2)*scale_v
nzy = (Z_opt.shape[3]-pad1*2)*scale_h
images_prev = images_cur
images_cur = []
for i in range(0,num_samples,1):
if n == 0:
z_curr = functions.generate_noise([1,nzx,nzy], device=opt.device)
z_curr = z_curr.expand(1,3,z_curr.shape[2],z_curr.shape[3])
z_curr = m(z_curr)
else:
z_curr = functions.generate_noise([opt.nc_z,nzx,nzy], device=opt.device)
z_curr = m(z_curr)
if images_prev == []:
I_prev = m(in_s)
#I_prev = m(I_prev)
#I_prev = I_prev[:,:,0:z_curr.shape[2],0:z_curr.shape[3]]
#I_prev = functions.upsampling(I_prev,z_curr.shape[2],z_curr.shape[3])
else:
I_prev = images_prev[i]
I_prev = imresize(I_prev,1/opt.scale_factor, opt)
if opt.mode != "SR":
I_prev = I_prev[:, :, 0:round(scale_v * reals[n].shape[2]), 0:round(scale_h * reals[n].shape[3])]
I_prev = m(I_prev)
I_prev = I_prev[:,:,0:z_curr.shape[2],0:z_curr.shape[3]]
I_prev = functions.upsampling(I_prev,z_curr.shape[2],z_curr.shape[3])
else:
I_prev = m(I_prev)
if n < gen_start_scale:
z_curr = Z_opt
z_in = noise_amp*(z_curr)+I_prev
I_curr = G(z_in.detach(),I_prev)
if opt.mode == 'train':
dir2save = '%s/RandomSamples/%s/gen_start_scale=%d' % (opt.out, opt.input_name[:-4], gen_start_scale)
else:
dir2save = functions.generate_dir2save(opt)
try:
os.makedirs(dir2save)
except OSError:
pass
if (opt.mode != "harmonization") & (opt.mode != "editing") & (opt.mode != "SR") & (opt.mode != "paint2image"):
plt.imsave('%s/%d.png' % (dir2save, i), functions.convert_image_np(I_curr.detach()), vmin=0,vmax=1)
#plt.imsave('%s/%d_%d.png' % (dir2save,i,n),functions.convert_image_np(I_curr.detach()), vmin=0, vmax=1)
#plt.imsave('%s/in_s.png' % (dir2save), functions.convert_image_np(in_s), vmin=0,vmax=1)
images_cur.append(I_curr)
n+=1
return I_curr.detach()
def SinGAN_generate(Gs,
Zs,
reals1,
reals2,
NoiseAmp,
opt,
in_s1=None,
in_s2=None,
scale_v=1,
scale_h=1,
n=0,
gen_start_scale=0,
num_samples=100):
#if torch.is_tensor(in_s) == False:
# if in_s is None:
in_s = torch.full(reals1[0].shape, 0, device=opt.device)
images_cur = []
# assert len(reals1) == len(Gs)
def random_noise_mode():
prob = torch.rand(1)
if prob < 0.5:
noise_mode = NoiseMode.Z1
else:
noise_mode = NoiseMode.Z2
return noise_mode
noise_modes = [random_noise_mode() for _ in range(num_samples)]
for G, (Z_opt1, Z_opt2), (noise_amp1, noise_amp2) in zip(Gs, Zs, NoiseAmp):
pad1 = ((opt.ker_size - 1) * opt.num_layer) / 2
m = nn.ZeroPad2d(int(pad1))
# assumption: same size
nzx = (Z_opt1.shape[2] - pad1 * 2) * scale_v
nzy = (Z_opt1.shape[3] - pad1 * 2) * scale_h
images_prev = images_cur
images_cur = []
for i in range(0, num_samples, 1):
z_curr = _generate_noise_for_sampling(m, n, nzx, nzy, opt,
noise_modes[i])
if images_prev == []:
I_prev = m(in_s)
#I_prev = m(I_prev)
#I_prev = I_prev[:,:,0:z_curr.shape[2],0:z_curr.shape[3]]
#I_prev = functions.upsampling(I_prev,z_curr.shape[2],z_curr.shape[3])
else:
I_prev = images_prev[i]
I_prev = imresize(I_prev, 1 / opt.scale_factor, opt)
I_prev = I_prev[:, :, 0:round(scale_v * reals1[n].shape[2]),
0:round(scale_h * reals1[n].shape[3])]
I_prev = m(I_prev)
I_prev = I_prev[:, :, 0:z_curr.shape[2], 0:z_curr.shape[3]]
I_prev = functions.upsampling(I_prev, z_curr.shape[2],
z_curr.shape[3])
if n < gen_start_scale:
zero = torch.zeros(Z_opt1.shape)
if noise_modes[i] == NoiseMode.Z1:
z_curr = functions.merge_noise_vectors(
Z_opt1, zero, opt.noise_vectors_merge_method)
elif noise_modes[i] == NoiseMode.Z2:
z_curr = functions.merge_noise_vectors(
zero, Z_opt2, opt.noise_vectors_merge_method)
else:
z_curr = functions.merge_noise_vectors(
Z_opt1, Z_opt2, opt.noise_vectors_merge_method)
noise_amp = noise_amp1 if noise_modes[
i] == NoiseMode.Z1 else noise_amp2
z_in = noise_amp * (z_curr) + I_prev
I_curr = G(z_in.detach(), I_prev)[0]
if n == len(reals1) - 1:
if opt.mode == 'train':
dir2save = '%s/RandomSamples/%s/gen_start_scale=%d' % (
opt.out, opt.exp_name, gen_start_scale)
else:
dir2save = functions.generate_dir2save(opt)
try:
os.makedirs(dir2save)
except OSError:
pass
if (opt.mode != "harmonization") & (opt.mode != "editing") & (
opt.mode != "SR") & (opt.mode != "paint2image"):
print(f"Saving image: {i}")
plt.imsave(f'%s/%d_{noise_modes[i].name}.png' %
(dir2save, i),
functions.convert_image_np(I_curr.detach()),
vmin=0,
vmax=1)
#plt.imsave('%s/%d_%d.png' % (dir2save,i,n),functions.convert_image_np(I_curr.detach()), vmin=0, vmax=1)
#plt.imsave('%s/in_s.png' % (dir2save), functions.convert_image_np(in_s), vmin=0,vmax=1)
images_cur.append(I_curr)
print(f"Done Generating level: {n}")
n += 1
return I_curr.detach()
def SinGAN_denoise(Gs,
Zs,
reals,
NoiseAmp,
opt,
in_s=None,
scale_v=1,
scale_h=1,
n=0,
gen_start_scale=0,
num_samples=1):
#if torch.is_tensor(in_s) == False:
if in_s is None:
in_s = torch.full(reals[0].shape, 0, device=opt.device)
images_cur = []
for G, Z_opt, noise_amp in zip(Gs, Zs, NoiseAmp):
pad1 = ((opt.ker_size - 1) * opt.num_layer) / 2
m = nn.ZeroPad2d(int(pad1))
nzx = (Z_opt.shape[2] - pad1 * 2) * scale_v
nzy = (Z_opt.shape[3] - pad1 * 2) * scale_h
images_prev = images_cur
images_cur = []
for i in range(0, num_samples, 1):
if n == 0:
z_curr = functions.generate_noise([1, nzx, nzy],
device=opt.device)
z_curr = z_curr.expand(1, 3, z_curr.shape[2], z_curr.shape[3])
z_curr = m(z_curr)
else:
z_curr = functions.generate_noise([opt.nc_z, nzx, nzy],
device=opt.device)
z_curr = m(z_curr)
if images_prev == []:
I_prev = m(in_s)
#I_prev = m(I_prev)
#I_prev = I_prev[:,:,0:z_curr.shape[2],0:z_curr.shape[3]]
#I_prev = functions.upsampling(I_prev,z_curr.shape[2],z_curr.shape[3])
else:
I_prev = images_prev[i]
I_prev = imresize(I_prev, 1 / opt.scale_factor, opt)
if opt.mode != "SR":
I_prev = I_prev[:, :, 0:round(scale_v * reals[n].shape[2]),
0:round(scale_h * reals[n].shape[3])]
I_prev = m(I_prev)
I_prev = I_prev[:, :, 0:z_curr.shape[2], 0:z_curr.shape[3]]
I_prev = functions.upsampling(I_prev, z_curr.shape[2],
z_curr.shape[3])
else:
I_prev = m(I_prev)
if n < gen_start_scale:
z_curr = Z_opt
z_curr = Z_opt
z_in = noise_amp * (z_curr) + I_prev
I_curr = G(z_in.detach(), I_prev)
images_cur.append(I_curr)
n += 1
return I_curr.detach()
def SinGAN_anchor_generate(Gs,
Zs,
reals,
NoiseAmp,
opt,
in_s=None,
scale_v=1,
scale_h=1,
n=0,
gen_start_scale=0,
num_samples=1,
anchor_image=None,
direction=None,
transfer=None,
noise_solutions=None,
factor=None,
base=None,
insert_limit=0):
#### Loading in Anchor if Needed #####
anchor = anchor_image
if anchor is not None:
anchors = []
anchor = functions.np2torch(anchor_image, opt)
anchor_ = imresize(anchor, opt.scale1, opt)
anchors = functions.creat_reals_pyramid(anchor_, anchors,
opt) #high key hacky code
if direction is not None:
directions = []
direction = functions.np2torch(direction, opt)
direction_ = imresize(direction, opt.scale1, opt)
directions = functions.creat_reals_pyramid(direction_, directions,
opt) #high key hacky code
if base is not None:
bases = []
base = functions.np2torch(base, opt)
base_ = imresize(base, opt.scale1, opt)
bases = functions.creat_reals_pyramid(base_, bases,
opt) #high key hacky code
#### MY CODE ####
#if torch.is_tensor(in_s) == False:
if in_s is None:
in_s = torch.full(reals[0].shape, 0, device=opt.device)
images_cur = []
for G, Z_opt, noise_amp in zip(Gs, Zs, NoiseAmp):
pad1 = ((opt.ker_size - 1) * opt.num_layer) / 2
m = nn.ZeroPad2d(int(pad1))
nzx = (Z_opt.shape[2] - pad1 * 2) * scale_v
nzy = (Z_opt.shape[3] - pad1 * 2) * scale_h
images_prev = images_cur
images_cur = []
for i in range(0, num_samples, 1):
if n == 0: #COARSEST SCALE
z_curr = functions.generate_noise([1, nzx, nzy],
device=opt.device)
z_curr = z_curr.expand(1, 3, z_curr.shape[2], z_curr.shape[3])
z_curr = m(z_curr)
else:
z_curr = functions.generate_noise([opt.nc_z, nzx, nzy],
device=opt.device)
z_curr = m(z_curr)
z_orig = z_curr
if images_prev == []: #FIRST GENERATION IN COARSEST SCALE
I_prev = m(in_s)
else: #NOT FIRST GENERATION, BUT AT COARSEST SCALE
I_prev = images_prev[i]
I_prev = imresize(I_prev, 1 / opt.scale_factor, opt) #upscale
#print(n)
if opt.mode != "SR":
I_prev = I_prev[:, :, 0:round(scale_v * reals[n].shape[2]),
0:round(scale_h * reals[n].shape[3])]
I_prev = m(I_prev)
I_prev = I_prev[:, :, 0:z_curr.shape[2], 0:z_curr.shape[3]]
I_prev = functions.upsampling(
I_prev, z_curr.shape[2],
z_curr.shape[3]) #make it fit padded noise
else:
#prev_before = I_prev #MY ADDITION
I_prev = m(I_prev)
if n < gen_start_scale: #anything less than final
z_curr = Z_opt #Z_opt comes from trained pyramid....
z_in = noise_amp * (z_curr) + I_prev
if noise_solutions is not None:
z_curr = noise_solutions[n]
z_in = (1 - factor) * noise_amp * (
z_curr
) + I_prev + factor * noise_amp * z_orig #adds in previous image to z_opt'''
I_curr = G(z_in.detach(), I_prev)
if base is not None:
if n == insert_limit:
I_curr = bases[n] * factor + I_curr * (1 - factor)
if anchor is not None and direction is not None:
anchor_curr = anchors[n]
I_curr = reinforcement(anchor_curr, I_curr, directions[n])
#I_curr = reinforcement_sigmoid(anchor_curr, I_curr, direction, n)
###### ENFORCE LH = ANCHOR FOR IMAGE #######
if n == opt.stop_scale: #hacky code
if anchor is not None and direction is not None:
anchor_curr = anchors[n]
I_curr = reinforcement(anchor_curr, I_curr, direction)
#I_curr = reinforcement_sigmoid(anchor_curr, I_curr, direction, n)
array = functions.convert_image_np(I_curr.detach())
images_cur.append(I_curr)
n += 1
return array
def invert_model(test_image,
model_name,
scales2invert=None,
penalty=1e-3,
show=True):
'''test_image is an array, model_name is a name'''
Noise_Solutions = []
parser = get_arguments()
parser.add_argument('--input_dir',
help='input image dir',
default='Input/Images')
parser.add_argument('--mode', default='RandomSamples')
opt = parser.parse_args("")
opt.input_name = model_name
opt.reg = penalty
if model_name == 'islands2_basis_2.jpg': #HARDCODED
opt.scale_factor = 0.6
opt = functions.post_config(opt)
### Loading in Generators
Gs, Zs, reals, NoiseAmp = functions.load_trained_pyramid(opt)
for G in Gs:
G = functions.reset_grads(G, False)
G.eval()
### Loading in Ground Truth Test Images
reals = [] #deleting old real images
real = functions.np2torch(test_image, opt)
functions.adjust_scales2image(real, opt)
real_ = functions.np2torch(test_image, opt)
real = imresize(real_, opt.scale1, opt)
reals = functions.creat_reals_pyramid(real, reals, opt)
### General Padding
pad_noise = int(((opt.ker_size - 1) * opt.num_layer) / 2)
m_noise = nn.ZeroPad2d(int(pad_noise))
pad_image = int(((opt.ker_size - 1) * opt.num_layer) / 2)
m_image = nn.ZeroPad2d(int(pad_image))
I_prev = None
REC_ERROR = 0
if scales2invert is None:
scales2invert = opt.stop_scale + 1
for scale in range(scales2invert):
#for scale in range(3):
#Get X, G
X = reals[scale]
G = Gs[scale]
noise_amp = NoiseAmp[scale]
#Defining Dimensions
opt.nc_z = X.shape[1]
opt.nzx = X.shape[2]
opt.nzy = X.shape[3]
#getting parameters for prior distribution penalty
pdf = torch.distributions.Normal(0, 1)
alpha = opt.reg
#alpha = 1e-2
#Defining Z
if scale == 0:
z_init = functions.generate_noise(
[1, opt.nzx, opt.nzy], device=opt.device) #only 1D noise
else:
z_init = functions.generate_noise(
[3, opt.nzx, opt.nzy],
device=opt.device) #otherwise move up to 3d noise
z_init = Variable(z_init.cuda(),
requires_grad=True) #variable to optimize
#Building I_prev
if I_prev == None: #first scale scenario
in_s = torch.full(reals[0].shape, 0, device=opt.device) #all zeros
I_prev = in_s
I_prev = m_image(I_prev) #padding
else: #otherwise take the output from the previous scale and upsample
I_prev = imresize(I_prev, 1 / opt.scale_factor, opt) #upsamples
I_prev = m_image(I_prev)
I_prev = I_prev[:, :, 0:X.shape[2] + 10, 0:X.shape[
3] + 10] #making sure that precision errors don't mess anything up
I_prev = functions.upsampling(I_prev, X.shape[2] + 10, X.shape[3] +
10) #seems to be redundant
LR = [2e-3, 2e-2, 2e-1, 2e-1, 2e-1, 2e-1, 2e-1, 2e-1, 2e-1, 2e-1, 2e-1]
Zoptimizer = torch.optim.RMSprop([z_init],
lr=LR[scale]) #Defining Optimizer
x_loss = [] #for plotting
epochs = [] #for plotting
niter = [
200, 400, 400, 400, 400, 400, 400, 400, 400, 400, 400, 400, 400,
400, 400
]
for epoch in range(niter[scale]): #Gradient Descent on Z
if scale == 0:
noise_input = m_noise(z_init.expand(1, 3, opt.nzx,
opt.nzy)) #expand and padd
else:
noise_input = m_noise(z_init) #padding
z_in = noise_amp * noise_input + I_prev
G_z = G(z_in, I_prev)
x_recLoss = F.mse_loss(G_z, X) #MSE loss
logProb = pdf.log_prob(z_init).mean() #Gaussian loss
loss = x_recLoss - (alpha * logProb.mean())
Zoptimizer.zero_grad()
loss.backward()
Zoptimizer.step()
#losses['rec'].append(x_recLoss.data[0])
#print('Image loss: [%d] loss: %0.5f' % (epoch, x_recLoss.item()))
#print('Noise loss: [%d] loss: %0.5f' % (epoch, z_recLoss.item()))
x_loss.append(loss.item())
epochs.append(epoch)
REC_ERROR = x_recLoss
if show:
plt.plot(epochs, x_loss, label='x_loss')
plt.legend()
plt.show()
I_prev = G_z.detach(
) #take final output, maybe need to edit this line something's very very fishy
_ = show_image(X, show, 'target')
reconstructed_image = show_image(I_prev, show, 'output')
_ = show_image(noise_input.detach().cpu(), show, 'noise')
Noise_Solutions.append(noise_input.detach())
return Noise_Solutions, reconstructed_image, REC_ERROR
def SinGAN_generate(Gs,Zs,reals,NoiseAmp,opt,in_s=None,scale_v=1,scale_h=1,n=0,gen_start_scale=0,num_samples=50):
#if torch.is_tensor(in_s) == False:
passes = 0
if in_s is None:
in_s = torch.full(reals[0].shape, 0, device=opt.device)
images_cur = []
for G,Z_opt,noise_amp in zip(Gs,Zs,NoiseAmp):
pad1 = ((opt.ker_size-1)*opt.num_layer)/2
m = nn.ZeroPad2d(int(pad1))
nzx = (Z_opt.shape[2]-pad1*2)*scale_v
nzy = (Z_opt.shape[3]-pad1*2)*scale_h
images_prev = images_cur
images_cur = []
for i in range(0,num_samples,1):
if n == 0:
z_curr = functions.generate_noise([1,nzx,nzy], device=opt.device)
z_curr = z_curr.expand(1,3,z_curr.shape[2],z_curr.shape[3])
z_curr = m(z_curr)
else:
z_curr = functions.generate_noise([opt.nc_z,nzx,nzy], device=opt.device)
z_curr = m(z_curr)
if images_prev == []:
print("in_s shape before padding with m", in_s.shape)
I_prev = m(in_s)
print("in_s shape after padding with m now I_prev", in_s.shape)
# I_prev = m(I_prev)
# I_prev = I_prev[:,:,0:z_curr.shape[2],0:z_curr.shape[3]]
I_prev = functions.upsampling(I_prev,z_curr.shape[2],z_curr.shape[3])
print("I_prev shape after upsampling using noise shape", I_prev.shape)
else:
I_prev = images_prev[i]
I_prev = imresize(I_prev,1/opt.scale_factor, opt)
if opt.mode != "SR":
I_prev = I_prev[:, :, 0:round(scale_v * reals[n].shape[2]), 0:round(scale_h * reals[n].shape[3])]
I_prev = m(I_prev)
I_prev = I_prev[:,:,0:z_curr.shape[2],0:z_curr.shape[3]]
I_prev = functions.upsampling(I_prev,z_curr.shape[2],z_curr.shape[3])
else:
I_prev = m(I_prev)
if n < gen_start_scale:
z_curr = Z_opt
# real = img.imread("D:\MVA\CompVision\Project\SinGAN-master\Input\Images/Salt_and_Pepper_Golden_Bridge_by_night (2).png")
real = img.imread("D:\MVA\CompVision\Project\SinGAN-master\Input\Images/Noisy_Golden_Bridge_by_night.jpg")
# real = img.imread("D:\MVA\CompVision\Project\SinGAN-master\Output\RandomSamples\Golden_Bridge_by_night/1.png")
real = real[:,:,:,None]
real = real.transpose((3,2,0,1))/255
real = torch.from_numpy(real)
real = move_to_gpu(real)
real = real.type(torch.cuda.FloatTensor)
real = ((real - 0.5)*2).clamp(-1,1)
real = real[:,0:3,:,:]
# real = imresize(real,1/opt.scale_factor, opt)
# real = real[:, :, 0:round(scale_v * reals[n].shape[2]), 0:round(scale_h * reals[n].shape[3])]
real = m(real)
# real = real[:,:,0:z_curr.shape[2],0:z_curr.shape[3]]
I_prev = functions.upsampling(real,z_curr.shape[2],z_curr.shape[3])
print("I_prev",I_prev.shape)
print("z_curr",z_curr.shape)
print('---')
# z_in = noise_amp*(z_curr)+I_prev
z_in = 0*(z_curr)+I_prev
I_curr = G(z_in.detach(),I_prev)
if n == len(reals)-1:
if opt.mode == 'train':
dir2save = '%s/RandomSamples/%s/gen_start_scale=%d' % (opt.out, opt.input_name[:-4], gen_start_scale)
else:
dir2save = functions.generate_dir2save(opt)
try:
os.makedirs(dir2save)
except OSError:
pass
if (opt.mode != "harmonization") & (opt.mode != "editing") & (opt.mode != "SR") & (opt.mode != "paint2image"):
plt.imsave('%s/%d.png' % (dir2save, i), functions.convert_image_np(I_curr.detach()), vmin=0,vmax=1)
# plt.imsave('%s/%d.png' % (dir2save, passes), functions.convert_image_np(I_curr.detach()), vmin=0,vmax=1)
# passes +=1
#plt.imsave('%s/%d_%d.png' % (dir2save,i,n),functions.convert_image_np(I_curr.detach()), vmin=0, vmax=1)
#plt.imsave('%s/in_s.png' % (dir2save), functions.convert_image_np(in_s), vmin=0,vmax=1)
images_cur.append(I_curr)
n+=1
# plt.imsave('D:\MVA\CompVision\Project\SinGAN-master\Output\RandomSamples\Golden_Bridge_by_night\gen_start_scale=0\Denoised.png', functions.convert_image_np(I_curr.detach()), vmin=0,vmax=1)
return I_curr.detach()
def SinGAN_generate(Gs, Zs, reals, NoiseAmp, opt, modification=None, in_s=None, scale_v=1, scale_h=1, n=0,
gen_start_scale=0, num_samples=10):
# start_scale = here we manipulate the image
# func stylize
# if torch.is_tensor(in_s) == False:
if in_s is None:
in_s = torch.full(reals[0].shape, 0, device=opt.device)
images_cur = []
for G, Z_opt, noise_amp in zip(Gs, Zs, NoiseAmp):
pad1 = ((opt.ker_size - 1) * opt.num_layer) / 2
m = nn.ZeroPad2d(int(pad1))
nzx = (Z_opt.shape[2] - pad1 * 2) * scale_v
nzy = (Z_opt.shape[3] - pad1 * 2) * scale_h
images_prev = images_cur
images_cur = []
for i in range(0, num_samples, 1):
if n == 0:
z_curr = functions.generate_noise([1, nzx, nzy], device=opt.device)
z_curr = z_curr.expand(1, 3, z_curr.shape[2], z_curr.shape[3])
z_curr = m(z_curr)
else:
z_curr = functions.generate_noise([opt.nc_z, nzx, nzy], device=opt.device)
z_curr = m(z_curr)
if images_prev == []:
I_prev = m(in_s)
# I_prev = m(I_prev)
# I_prev = I_prev[:,:,0:z_curr.shape[2],0:z_curr.shape[3]]
# I_prev = functions.upsampling(I_prev,z_curr.shape[2],z_curr.shape[3])
else:
I_prev = images_prev[i]
I_prev = imresize(I_prev, 1 / opt.scale_factor, opt)
if opt.mode != "SR":
I_prev = I_prev[:, :, 0:round(scale_v * reals[n].shape[2]), 0:round(scale_h * reals[n].shape[3])]
I_prev = m(I_prev)
I_prev = I_prev[:, :, 0:z_curr.shape[2], 0:z_curr.shape[3]]
I_prev = functions.upsampling(I_prev, z_curr.shape[2], z_curr.shape[3])
else:
I_prev = m(I_prev)
if n < gen_start_scale:
z_curr = Z_opt
z_in = noise_amp * (z_curr) + I_prev
dir2save = '%s/RandomSamples/%s/%s/gen_start_scale=%d' % (opt.out, opt.input_name[:-4], modification, gen_start_scale)
try:
os.makedirs(dir2save)
except OSError:
pass
if n==gen_start_scale:
plt.imsave('%s/%d_before_modification.png' % (dir2save, i), functions.convert_image_np(z_in.detach()), vmin=0,vmax=1)
# ##################################### Image modification #################################################
#TODO if you want the modification to happen only once, change the >= into ==
#TODO at the moment, modification happens at every scale from the gen_start_scale and above, unless no
#TODO modification is specificed
#TODO The modified image is saved only at the generation scale
#TODO when using blending, consider trying different blending options and opcity. These can be modified
#TODO within the modify_input_to_generator function below
if (n >= gen_start_scale) & (modification is not None):
shape = z_in.shape
cont_in = preprocess_content_image(opt, reals,n)
z_in = modify_input_to_generator(z_in, cont_in, modification, opacity=1)
assert shape == z_in.shape
if n==gen_start_scale:
plt.imsave('%s/%d_after_modification.png' % (dir2save, i), functions.convert_image_np(z_in.detach()), vmin=0,vmax=1)
# ################################## End of image modification #############################################
I_curr = G(z_in.detach(), I_prev)
if n == len(reals) - 1:
if opt.mode == 'train':
dir2save = '%s/RandomSamples/%s/%s/gen_start_scale=%d' % (
opt.out, opt.input_name[:-4], modification, gen_start_scale)
else:
#dir2save = functions.generate_dir2save(opt)
dir2save = '%s/RandomSamples/%s/%s/gen_start_scale=%d' % (
opt.out, opt.input_name[:-4], modification, gen_start_scale)
try:
os.makedirs(dir2save)
except OSError:
pass
if (opt.mode != "harmonization") & (opt.mode != "editing") & (opt.mode != "SR") & (
opt.mode != "paint2image"):
plt.imsave('%s/%d.png' % (dir2save, i), functions.convert_image_np(I_curr.detach()), vmin=0, vmax=1)
# plt.imsave('%s/%d_%d.png' % (dir2save,i,n),functions.convert_image_np(I_curr.detach()), vmin=0, vmax=1)
# plt.imsave('%s/in_s.png' % (dir2save), functions.convert_image_np(in_s), vmin=0,vmax=1)
images_cur.append(I_curr)
n += 1
return I_curr.detach()
def SinGAN_generate(Gs,
Zs,
reals,
NoiseAmp,
opt,
in_s=None,
scale_v=1,
scale_h=1,
n=0,
gen_start_scale=0,
num_samples=50,
output_image=False):
#if torch.is_tensor(in_s) == False:
if in_s is None:
# make in_s a 0 tensor with reals[0] shape
in_s = torch.full(reals[0].shape, 0, device=opt.device)
images_cur = []
#for each layers
for G, Z_opt, noise_amp in zip(Gs, Zs, NoiseAmp):
#generate a pad class with width ((ker_size-1)*num_layer)/2
pad1 = ((opt.ker_size - 1) * opt.num_layer) / 2
m = nn.ZeroPad2d(int(pad1))
#the shape inside padding * scale
nzx = (Z_opt.shape[2] - pad1 * 2) * scale_v
nzy = (Z_opt.shape[3] - pad1 * 2) * scale_h
#get all the previsous image
images_prev = images_cur
images_cur = []
output_list = []
#for the number of samples
for i in range(0, num_samples, 1):
if n == 0:
#generate the noise
z_curr = functions.generate_noise([1, nzx, nzy],
device=opt.device)
#broadcast to the correct shape
z_curr = z_curr.expand(1, 3, z_curr.shape[2], z_curr.shape[3])
#padding it
z_curr = m(z_curr)
else:
#generate noise with defined shape
z_curr = functions.generate_noise([opt.nc_z, nzx, nzy],
device=opt.device)
#padding
z_curr = m(z_curr)
#if it's the first scale
if images_prev == []:
#use in_s as the first one
I_prev = m(in_s)
#I_prev = m(I_prev)
#I_prev = I_prev[:,:,0:z_curr.shape[2],0:z_curr.shape[3]]
#I_prev = functions.upsampling(I_prev,z_curr.shape[2],z_curr.shape[3])
else:
#get the last image
I_prev = images_prev[i]
#resize it by 1/scale_factor
I_prev = imresize(I_prev, 1 / opt.scale_factor, opt)
# cut a piece of shape (round(scale_v * reals[n].shape[2] * round(scale_h * reals[n].shape[3]))
I_prev = I_prev[:, :, 0:round(scale_v * reals[n].shape[2]),
0:round(scale_h * reals[n].shape[3])]
#padding
I_prev = m(I_prev)
#cut a piece of shape (z_curr.shape[2], z_curr.shape[3])
I_prev = I_prev[:, :, 0:z_curr.shape[2], 0:z_curr.shape[3]]
#upsample this piece to original shape, with bilinear policy
I_prev = functions.upsampling(I_prev, z_curr.shape[2],
z_curr.shape[3])
# amplify the z by the param, add the previous graph
z_in = noise_amp * (z_curr) + I_prev
# pass this value and previous graph to generator, get the value
I_curr = G(z_in.detach(), I_prev)
#for the last loop
if n == len(reals) - 1:
#generate the directory
dir2save = functions.generate_dir2save(opt) #modified
try:
os.makedirs(dir2save)
except OSError:
pass
# new variable
if (output_image):
#save the new generated image
plt.imsave(f'{dir2save}/{i}.png',
functions.convert_image_np(I_curr.detach()),
vmin=0,
vmax=1)
# have the generated image into the list
output_list.append(functions.convert_image_np(I_curr.detach()))
images_cur.append(I_curr)
n += 1
return I_curr.detach(), output_list #newly added
def SinGAN_generate(Gs,
Zs,
reals,
NoiseAmp,
opt,
in_s=None,
scale_v=1,
scale_h=1,
n=0,
gen_start_scale=0,
num_samples=50):
real = functions.read_image(opt)
real = real.numpy()
real = resize(real, reals[-1].shape)
real = torch.from_numpy(real)
new_reals = creat_reals_pyramid(real, [], opt)
buffer = []
for new_real, real in zip(new_reals, reals):
ele = new_real.numpy()
ele = resize(ele, real.shape)
ele = torch.from_numpy(ele)
buffer.append(ele)
reals = buffer
for i, real_img in enumerate(reals):
dir2save = '%s/RandomSamples/%s/gen_start_scale=%d' % (
opt.out, opt.input_name[:-4], gen_start_scale)
plt.imsave('%s/%s_%d.png' % (dir2save, "real", i),
functions.convert_image_np(real_img.detach()),
vmin=0,
vmax=1)
if in_s is None:
in_s = torch.full(reals[0].shape, 0, device=opt.device)
images_cur = []
for G, Z_opt, noise_amp in zip(Gs, Zs, NoiseAmp):
pad1 = ((opt.ker_size - 1) * opt.num_layer) / 2
m = nn.ZeroPad2d(int(pad1))
nzx = (Z_opt.shape[2] - pad1 * 2) * scale_v
nzy = (Z_opt.shape[3] - pad1 * 2) * scale_h
# For Section IV
# if n == 0:
# images_prev = images_cur
# else:
# new_img_prev = []
# for img in images_cur:
# ele = reals[n].numpy()
# ele = resize(ele, img.shape)
# ele = torch.from_numpy(ele)
# new_img_prev.append(ele)
# images_prev = new_img_prev
images_prev = images_cur
# if n != 0:
# dir2save = '%s/RandomSamples/%s/gen_start_scale=%d' % (opt.out, opt.input_name[:-4], gen_start_scale)
# plt.imsave('%s/%s_%d.png' % (dir2save, "img_cur", n), functions.convert_image_np(images_prev[0].detach()), vmin=0,vmax=1)
# plt.imsave('%s/%s_%d.png' % (dir2save, "img_prev", n), functions.convert_image_np(images_cur[0].detach()), vmin=0,vmax=1)
images_cur = []
for i in range(0, num_samples, 1):
if n == 0:
z_curr = functions.generate_noise([1, nzx, nzy],
device=opt.device)
z_curr = z_curr.expand(1, 3, z_curr.shape[2], z_curr.shape[3])
z_curr = m(z_curr)
else:
z_curr = functions.generate_noise([opt.nc_z, nzx, nzy],
device=opt.device)
z_curr = m(z_curr)
if images_prev == []:
I_prev = m(in_s)
else:
I_prev = images_prev[i]
I_prev = imresize(I_prev, 1 / opt.scale_factor, opt)
if opt.mode != "SR":
I_prev = I_prev[:, :, 0:round(scale_v * reals[n].shape[2]),
0:round(scale_h * reals[n].shape[3])]
I_prev = m(I_prev)
I_prev = I_prev[:, :, 0:z_curr.shape[2], 0:z_curr.shape[3]]
I_prev = functions.upsampling(I_prev, z_curr.shape[2],
z_curr.shape[3])
else:
I_prev = m(I_prev)
if n < gen_start_scale:
z_curr = Z_opt
z_in = noise_amp * (z_curr) + I_prev
if opt.skip != '' and int(opt.skip) == n:
I_curr = I_prev
else:
I_curr = G(z_in.detach(), I_prev)
if n == len(reals) - 1:
if opt.mode == 'train':
dir2save = '%s/RandomSamples/%s/gen_start_scale=%d' % (
opt.out, opt.input_name[:-4], gen_start_scale)
else:
dir2save = functions.generate_dir2save(opt)
try:
os.makedirs(dir2save)
except OSError:
pass
if (opt.mode != "harmonization") & (opt.mode != "editing") & (
opt.mode != "SR") & (opt.mode != "paint2image"):
plt.imsave('%s/%d.png' % (dir2save, i),
functions.convert_image_np(I_curr.detach()),
vmin=0,
vmax=1)
# plt.imsave('%s/%d_%d.png' % (dir2save, i, n), functions.convert_image_np(I_curr.detach()), vmin=0,vmax=1)
# For Section VI
# if opt.mode == 'train':
# dir2save = '%s/RandomSamples/%s/gen_start_scale=%d' % (opt.out, opt.input_name[:-4], gen_start_scale)
# else:
# dir2save = functions.generate_dir2save(opt)
# try:
# os.makedirs(dir2save)
# except OSError:
# pass
# if (opt.mode != "harmonization") & (opt.mode != "editing") & (opt.mode != "SR") & (opt.mode != "paint2image"):
# plt.imsave('%s/%d_%d.png' % (dir2save, i, n), functions.convert_image_np(I_curr.detach()), vmin=0,vmax=1)
images_cur.append(I_curr)
n += 1
return I_curr.detach()
def SinGAN_generate(Gs,
Zs,
reals,
NoiseAmp,
opt,
in_s=None,
scale_v=1,
scale_h=1,
n=0,
gen_start_scale=0,
num_samples=50):
if in_s is None: # torch NCWH tf NWHC
in_s = tf.zeros_like(reals[0])
images_cur = []
for G, Z_opt, noise_amp in zip(Gs, Zs, NoiseAmp):
pad1 = ((opt.ker_size - 1) * opt.num_layer) / 2
m = tf.keras.layers.ZeroPadding2D(padding=(int(pad1), int(pad1)))
nzx = (Z_opt.shape[1] - pad1 * 2) * scale_v
nzy = (Z_opt.shape[2] - pad1 * 2) * scale_h
images_prev = images_cur
images_cur = []
for i in range(0, num_samples, 1):
if n == 0:
z_curr = functions.generate_noise([1, nzx, nzy])
z_curr = tf.tile(z_curr, multiples=(1, 1, 1, 3))
z_curr = m(z_curr)
else:
z_curr = functions.generate_noise([opt.nc_z, nzx, nzy])
z_curr = m(z_curr)
if images_prev == []:
I_prev = m(in_s)
else:
I_prev = images_prev[i]
I_prev = imresize(I_prev, 1 / opt.scale_factor, opt)
if opt.mode != "SR":
I_prev = I_prev[:, 0:round(scale_v * reals[n].shape[1]),
0:round(scale_h * reals[n].shape[2]), :]
I_prev = m(I_prev)
I_prev = I_prev[:, 0:z_curr.shape[1], 0:z_curr.shape[2], :]
I_prev = functions.upsampling(I_prev, z_curr.shape[1],
z_curr.shape[2])
else:
I_prev = m(I_prev)
if n < gen_start_scale:
z_curr = Z_opt
z_in = noise_amp * (z_curr) + I_prev
I_curr = G(z_in, I_prev, train=True)
if n == len(reals) - 1:
if opt.mode == 'train':
dir2save = '%s/RandomSamples/%s/gen_start_scale=%d' % (
opt.out, opt.input_name[:-4], gen_start_scale)
else:
dir2save = functions.generate_dir2save(opt)
try:
os.makedirs(dir2save)
except OSError:
pass
if (opt.mode != "harmonization") & (opt.mode != "editing") & (
opt.mode != "SR") & (opt.mode != "paint2image"):
plt.imsave('%s/%d.png' % (dir2save, i),
functions.convert_image_np(I_curr))
images_cur.append(I_curr)
n += 1
return I_curr
