def analogy(img_A, img_BP, config):
start_time_0 = time.time()
weights = config['weights']
sizes = config['sizes']
rangee = config['rangee']
params = config['params']
lr = config['lr']
if config['use_cuda']:
device = torch.device('cuda:0')
else:
raise NotImplementedError('cpu mode is not supported yet')
# preparing data
img_A_tensor = torch.FloatTensor(img_A.transpose(2, 0, 1))
img_BP_tensor = torch.FloatTensor(img_BP.transpose(2, 0, 1))
img_A_tensor, img_BP_tensor = img_A_tensor.to(device), img_BP_tensor.to(
device)
img_A_tensor = img_A_tensor.unsqueeze(0)
img_BP_tensor = img_BP_tensor.unsqueeze(0)
# compute 5 feature maps
model = VGG19(device=device)
data_A, data_A_size = model.get_features(img_tensor=img_A_tensor.clone(),
layers=params['layers'])
data_AP = copy.deepcopy(data_A)
data_BP, data_B_size = model.get_features(img_tensor=img_BP_tensor.clone(),
layers=params['layers'])
data_B = copy.deepcopy(data_BP)
print("Features extracted!")
for curr_layer in range(5):
print("\n### current stage: %d - start ###" % (5 - curr_layer))
start_time_1 = time.time()
if curr_layer == 0:
ann_AB = init_nnf(data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
ann_BA = init_nnf(data_B_size[curr_layer][2:],
data_A_size[curr_layer][2:])
else:
ann_AB = upSample_nnf(ann_AB, data_A_size[curr_layer][2:])
ann_BA = upSample_nnf(ann_BA, data_B_size[curr_layer][2:])
# blend feature
Ndata_A, response_A = normalize(data_A[curr_layer])
Ndata_BP, response_BP = normalize(data_BP[curr_layer])
data_AP[curr_layer] = blend(response_A, data_A[curr_layer],
data_AP[curr_layer], weights[curr_layer])
data_B[curr_layer] = blend(response_BP, data_BP[curr_layer],
data_B[curr_layer], weights[curr_layer])
Ndata_AP, _ = normalize(data_AP[curr_layer])
Ndata_B, _ = normalize(data_B[curr_layer])
# NNF search
print("- NNF search for ann_AB")
start_time_2 = time.time()
ann_AB, _ = propagate(ann_AB, ts2np(Ndata_A), ts2np(Ndata_AP),
ts2np(Ndata_B), ts2np(Ndata_BP),
sizes[curr_layer], params['iter'],
rangee[curr_layer])
print("\tElapse: " +
str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])
print("- NNF search for ann_BA")
start_time_2 = time.time()
ann_BA, _ = propagate(ann_BA, ts2np(Ndata_BP), ts2np(Ndata_B),
ts2np(Ndata_AP), ts2np(Ndata_A),
sizes[curr_layer], params['iter'],
rangee[curr_layer])
print("\tElapse: " +
str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])
if curr_layer >= 4:
print("### current stage: %d - end | " % (5 - curr_layer) +
"Elapse: " +
str(datetime.timedelta(seconds=time.time() -
start_time_1))[:-7] + ' ###')
break
# using backpropagation to approximate feature
next_layer = curr_layer + 2
ann_AB_upnnf2 = upSample_nnf(ann_AB, data_A_size[next_layer][2:])
ann_BA_upnnf2 = upSample_nnf(ann_BA, data_B_size[next_layer][2:])
data_AP_np = avg_vote(ann_AB_upnnf2, ts2np(data_BP[next_layer]),
sizes[next_layer], data_A_size[next_layer][2:],
data_B_size[next_layer][2:])
data_B_np = avg_vote(ann_BA_upnnf2, ts2np(data_A[next_layer]),
sizes[next_layer], data_B_size[next_layer][2:],
data_A_size[next_layer][2:])
data_AP[next_layer] = np2ts(data_AP_np, device)
data_B[next_layer] = np2ts(data_B_np, device)
target_BP_np = avg_vote(ann_AB, ts2np(data_BP[curr_layer]),
sizes[curr_layer], data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
target_A_np = avg_vote(ann_BA, ts2np(data_A[curr_layer]),
sizes[curr_layer], data_B_size[curr_layer][2:],
data_A_size[curr_layer][2:])
target_BP = np2ts(target_BP_np, device)
target_A = np2ts(target_A_np, device)
print('- deconvolution for feat A\'')
start_time_2 = time.time()
data_AP[curr_layer + 1] = model.get_deconvoluted_feat(
target_BP,
curr_layer,
data_AP[next_layer],
lr=lr[curr_layer],
iters=400,
display=False)
print("\tElapse: " +
str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])
print('- deconvolution for feat B')
start_time_2 = time.time()
data_B[curr_layer + 1] = model.get_deconvoluted_feat(
target_A,
curr_layer,
data_B[next_layer],
lr=lr[curr_layer],
iters=400,
display=False)
print("\tElapse: " +
str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])
# in case of data type inconsistency
if data_B[curr_layer + 1].type() == torch.cuda.DoubleTensor:
data_B[curr_layer + 1] = data_B[curr_layer + 1].type(
torch.cuda.FloatTensor)
data_AP[curr_layer + 1] = data_AP[curr_layer + 1].type(
torch.cuda.FloatTensor)
print("### current stage: %d - end | " % (5 - curr_layer) +
"Elapse: " +
str(datetime.timedelta(seconds=time.time() -
start_time_1))[:-7] + ' ###')
print('\n- reconstruct images A\' and B')
img_AP = reconstruct_avg(ann_AB, img_BP, sizes[curr_layer],
data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
img_B = reconstruct_avg(ann_BA, img_A, sizes[curr_layer],
data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
img_AP = np.clip(img_AP, 0, 255)
img_B = np.clip(img_B, 0, 255)
return img_AP, img_B, str(
datetime.timedelta(seconds=time.time() - start_time_0))[:-7]
def analogy(img_A, img_BP, config):
weights = config['weights']
sizes = config['sizes']
rangee = config['rangee']
use_cuda = config['use_cuda']
params = config['params']
lr = config['lr']
# assert use_cuda==True, "cpu version is not implemented yet. You can modify VGG19.py to make it support CPU if you like."
# preparing data
img_A_tensor = torch.FloatTensor(img_A.transpose(2, 0, 1))
img_BP_tensor = torch.FloatTensor(img_BP.transpose(2, 0, 1))
if use_cuda:
img_A_tensor, img_BP_tensor = img_A_tensor.cuda(), img_BP_tensor.cuda()
img_A_tensor = img_A_tensor.unsqueeze(0)
img_BP_tensor = img_BP_tensor.unsqueeze(0)
# compute 5 feature maps
model = VGG19(use_cuda=use_cuda)
data_A, data_A_size = model.get_features(
img_tensor=img_A_tensor.clone(), layers=params['layers'])
data_AP = copy.deepcopy(data_A)
data_BP, data_B_size = model.get_features(
img_tensor=img_BP_tensor.clone(), layers=params['layers'])
data_B = copy.deepcopy(data_BP)
for curr_layer in range(5):
if curr_layer == 0:
ann_AB = init_nnf(
data_A_size[curr_layer][2:], data_B_size[curr_layer][2:])
ann_BA = init_nnf(
data_B_size[curr_layer][2:], data_A_size[curr_layer][2:])
else:
ann_AB = upSample_nnf(ann_AB, data_A_size[curr_layer][2:])
ann_BA = upSample_nnf(ann_BA, data_B_size[curr_layer][2:])
# blend feature
Ndata_A, response_A = normalize(data_A[curr_layer])
Ndata_BP, response_BP = normalize(data_BP[curr_layer])
data_AP[curr_layer] = blend(
response_A, data_A[curr_layer], data_AP[curr_layer], weights[curr_layer])
data_B[curr_layer] = blend(
response_BP, data_BP[curr_layer], data_B[curr_layer], weights[curr_layer])
Ndata_AP, _ = normalize(data_AP[curr_layer])
Ndata_B, _ = normalize(data_B[curr_layer])
# NNF search
ann_AB, _ = propagate(ann_AB, ts2np(Ndata_A), ts2np(Ndata_AP), ts2np(Ndata_B), ts2np(Ndata_BP), sizes[curr_layer],
params['iter'], rangee[curr_layer])
ann_BA, _ = propagate(ann_BA, ts2np(Ndata_BP), ts2np(Ndata_B), ts2np(Ndata_AP), ts2np(Ndata_A), sizes[curr_layer],
params['iter'], rangee[curr_layer])
if curr_layer >= 4:
break
# using backpropagation to approximate feature
next_layer = curr_layer + 2
ann_AB_upnnf2 = upSample_nnf(ann_AB, data_A_size[next_layer][2:])
ann_BA_upnnf2 = upSample_nnf(ann_BA, data_B_size[next_layer][2:])
data_AP_np = avg_vote(ann_AB_upnnf2, ts2np(data_BP[next_layer]), sizes[next_layer], data_A_size[next_layer][2:],
data_B_size[next_layer][2:])
data_B_np = avg_vote(ann_BA_upnnf2, ts2np(data_A[next_layer]), sizes[next_layer], data_B_size[next_layer][2:],
data_A_size[next_layer][2:])
data_AP[next_layer] = np2ts(data_AP_np)
data_B[next_layer] = np2ts(data_B_np)
target_BP_np = avg_vote(ann_AB, ts2np(data_BP[curr_layer]), sizes[curr_layer], data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
target_A_np = avg_vote(ann_BA, ts2np(data_A[curr_layer]), sizes[curr_layer], data_B_size[curr_layer][2:],
data_A_size[curr_layer][2:])
target_BP = np2ts(target_BP_np)
target_A = np2ts(target_A_np)
data_AP[curr_layer+1] = model.get_deconvoluted_feat(target_BP, curr_layer, data_AP[next_layer], lr=lr[curr_layer],
iters=400, display=False)
data_B[curr_layer+1] = model.get_deconvoluted_feat(target_A, curr_layer, data_B[next_layer], lr=lr[curr_layer],
iters=400, display=False)
if type(data_B[curr_layer + 1]) == torch.DoubleTensor:
data_B[curr_layer + 1] = data_B[curr_layer +
1].type(torch.FloatTensor)
data_AP[curr_layer + 1] = data_AP[curr_layer +
1].type(torch.FloatTensor)
elif type(data_B[curr_layer + 1]) == torch.cuda.DoubleTensor:
data_B[curr_layer + 1] = data_B[curr_layer +
1].type(torch.cuda.FloatTensor)
data_AP[curr_layer + 1] = data_AP[curr_layer +
1].type(torch.cuda.FloatTensor)
img_AP = reconstruct_avg(
ann_AB, img_BP, sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:])
img_B = reconstruct_avg(
ann_BA, img_A, sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:])
img_AP = np.clip(img_AP/255.0, 0, 1)[:, :, ::-1]
img_B = np.clip(img_B/255.0, 0, 1)[:, :, ::-1]
return img_AP, img_B
def deep_image_analogy(A, BP, config):
alphas = config['alpha']
nnf_patch_size = config['nnf_patch_size']
radii = config['radii']
params = config['params']
lr = config['lr']
# preparing data
img_A_tensor = torch.FloatTensor(A.transpose(2, 0, 1)).cuda()
img_BP_tensor = torch.FloatTensor(BP.transpose(2, 0, 1)).cuda()
# fake a batch dimension
img_A_tensor = img_A_tensor.unsqueeze(0)
img_BP_tensor = img_BP_tensor.unsqueeze(0)
# 4.1 Preprocessing Step
model = VGG19()
F_A, F_A_size = model.get_features(img_tensor=img_A_tensor.clone(),
layers=params['layers'])
F_BP, F_B_size = model.get_features(img_tensor=img_BP_tensor.clone(),
layers=params['layers'])
# Init AP&B 's feature maps with F_A&F_BP
F_AP = copy.deepcopy(F_A)
F_B = copy.deepcopy(F_BP)
#Note that the feature_maps now is in the order of [5,4,3,2,1,input]
for curr_layer in range(5):
#ANN init step, coarsest layer is initialized randomly,
#Other layers is initialized using upsample technique described in the paper
if curr_layer == 0:
ann_AB = init_nnf(F_A_size[curr_layer][2:],
F_B_size[curr_layer][2:])
ann_BA = init_nnf(F_B_size[curr_layer][2:],
F_A_size[curr_layer][2:])
else:
ann_AB = upSample_nnf(ann_AB, F_A_size[curr_layer][2:])
ann_BA = upSample_nnf(ann_BA, F_B_size[curr_layer][2:])
# According to Equotion(2), we need to normalize F_A and F_BP
# response denotes the M in Equotion(6)
F_A_BAR, response_A = normalize(F_A[curr_layer])
F_BP_BAR, response_BP = normalize(F_BP[curr_layer])
# F_AP&F_B is reconstructed according to Equotion(4)
# Note that we reuse the varibale F_AP here,
# it denotes the RBprime as is stated in the Equotion(4) which is calculated
# at the end of the previous iteration
F_AP[curr_layer] = blend(response_A, F_A[curr_layer], F_AP[curr_layer],
alphas[curr_layer])
F_B[curr_layer] = blend(response_BP, F_BP[curr_layer], F_B[curr_layer],
alphas[curr_layer])
# Normalize F_AP&F_B as well
F_AP_BAR, _ = normalize(F_AP[curr_layer])
F_B_BAR, _ = normalize(F_B[curr_layer])
# Run PatchMatch algorithm to get mapping AB and BA
ann_AB, _ = propagate(ann_AB, ts2np(F_A_BAR), ts2np(F_AP_BAR),
ts2np(F_B_BAR), ts2np(F_BP_BAR),
nnf_patch_size[curr_layer], params['iter'],
radii[curr_layer])
ann_BA, _ = propagate(ann_BA, ts2np(F_BP_BAR), ts2np(F_B_BAR),
ts2np(F_AP_BAR), ts2np(F_A_BAR),
nnf_patch_size[curr_layer], params['iter'],
radii[curr_layer])
if curr_layer >= 4:
break
# The code below is used to initialize the F_AP&F_B in the next layer,
# it generates the R_B' and R_A as is stated in Equotion(4)
# R_B' is stored in F_AP, R_A is stored in F_B
# using backpropagation to approximate feature
# About why we add 2 here:
# https://github.com/msracver/Deep-Image-Analogy/issues/30
next_layer = curr_layer + 2
ann_AB_upnnf2 = upSample_nnf(ann_AB, F_A_size[next_layer][2:])
ann_BA_upnnf2 = upSample_nnf(ann_BA, F_B_size[next_layer][2:])
F_AP_np = avg_vote(ann_AB_upnnf2, ts2np(F_BP[next_layer]),
nnf_patch_size[next_layer],
F_A_size[next_layer][2:], F_B_size[next_layer][2:])
F_B_np = avg_vote(ann_BA_upnnf2, ts2np(F_A[next_layer]),
nnf_patch_size[next_layer], F_B_size[next_layer][2:],
F_A_size[next_layer][2:])
# Initialize R_B' and R_A
F_AP[next_layer] = np2ts(F_AP_np)
F_B[next_layer] = np2ts(F_B_np)
# Warp F_BP using ann_AB, Warp F_A using ann_BA
target_BP_np = avg_vote(ann_AB, ts2np(F_BP[curr_layer]),
nnf_patch_size[curr_layer],
F_A_size[curr_layer][2:],
F_B_size[curr_layer][2:])
target_A_np = avg_vote(ann_BA, ts2np(F_A[curr_layer]),
nnf_patch_size[curr_layer],
F_B_size[curr_layer][2:],
F_A_size[curr_layer][2:])
target_BP = np2ts(target_BP_np)
target_A = np2ts(target_A_np)
#LBFGS algorithm to approximate R_B' and R_A
F_AP[curr_layer + 1] = model.get_deconvoluted_feat(
target_BP,
curr_layer,
F_AP[next_layer],
lr=lr[curr_layer],
blob_layers=params['layers'])
F_B[curr_layer + 1] = model.get_deconvoluted_feat(
target_A,
curr_layer,
F_B[next_layer],
lr=lr[curr_layer],
blob_layers=params['layers'])
if type(F_B[curr_layer + 1]) == torch.DoubleTensor:
F_B[curr_layer + 1] = F_B[curr_layer + 1].type(torch.FloatTensor)
F_AP[curr_layer + 1] = F_AP[curr_layer + 1].type(torch.FloatTensor)
elif type(F_B[curr_layer + 1]) == torch.cuda.DoubleTensor:
F_B[curr_layer + 1] = F_B[curr_layer + 1].type(
torch.cuda.FloatTensor)
F_AP[curr_layer + 1] = F_AP[curr_layer + 1].type(
torch.cuda.FloatTensor)
# Obtain the output according to 4.5
img_AP = reconstruct_avg(ann_AB, BP, nnf_patch_size[curr_layer],
F_A_size[curr_layer][2:],
F_B_size[curr_layer][2:])
img_B = reconstruct_avg(ann_BA, A, nnf_patch_size[curr_layer],
F_A_size[curr_layer][2:], F_B_size[curr_layer][2:])
img_AP = np.clip(img_AP / 255.0, 0, 1)[:, :, ::-1]
img_B = np.clip(img_B / 255.0, 0, 1)[:, :, ::-1]
return img_AP, img_B