def loss_net(x_in, trux_x_in, width, height, style_image_path, content_weight,
style_weight):
# Append the initial input to the FastNet input to the VGG inputs
x = concatenate([x_in, trux_x_in], axis=0)
# Normalize the inputs via custom VGG Normalization layer
x = VGGNormalize(name="vgg_normalize")(x)
# Using pretrained VGG 19 model
vgg = VGG19(include_top=False, input_tensor=x)
vgg_output_dict = dict([(layer.name, layer.output)
for layer in vgg.layers[-21:]])
vgg_layers = dict([(layer.name, layer) for layer in vgg.layers[-21:]])
if style_weight > 0:
add_style_loss(vgg, style_image_path, vgg_layers, vgg_output_dict,
width, height, style_weight)
if content_weight > 0:
add_content_loss(vgg_layers, vgg_output_dict, content_weight)
# Freeze all VGG layers
for layer in vgg.layers[-22:]:
layer.trainable = False
return vgg
def __init__(self, content_layers, style_layers, content_image,
style_image, loss_ratio, num_iter, init_image=None):
#content_layers and style_layers should be dicts: {name: weight}
self.net = VGG19('./imagenet-vgg-verydeep-19.mat')
self.content_layers = content_layers
self.style_layers = style_layers
self.content = np.float32(content_image)
self.style = np.float32(style_image)
self.alpha = loss_ratio
self.beta = 1
self.iteration = num_iter
self.mean = (np.mean(content_image, axis=(0, 1)) + np.mean(style_image, axis=(0, 1))) / (2 * 255)
self.p = tf.constant(content_image - self.mean, name='content')
self.a = tf.constant(style_image - self.mean, name='style')
if init_image is None:
self.img = tf.Variable(tf.random_normal(self.content.shape), trainable=True, dtype=tf.float32)
else:
self.img = tf.Variable(init_image - self.mean, trainable=True, dtype=tf.float32)
# self.sess = tf.Session()
# self.sess.run(tf.global_variables_initializer())
self._built_net()
return loss
# def style_transfer(content_img, style_img, content_layer_id,
# style_layer_id, alpha = 1.5, beta = 10, num_iter = 100, step_size = 10):
content_img = preprocess(content_img_path)
style_img = preprocess(style_img_path)
mixed_img = preprocess(mixed_img_path)
tf.reset_default_graph()
img = tf.placeholder(tf.float32, (1, height, width, num_channels),
name='my_original_image')
vgg19 = VGG19(image_shape=(1, height, width, num_channels), input_tensor=img)
vgg19.summary()
output = tf.identity(vgg19['block5_pool'], name='my_output')
# show_graph(tf.get_default_graph().as_graph_def())
with tf.Session() as sess:
vgg19.load_weights()
fd = {img: style_img}
output_val = sess.run(output, fd)
print(output_val.shape, output_val.mean())
# vgg19 = VGG19(weights='imagenet', include_top=False, pooling='avg')
# model = Model(inputs=vgg19.input, outputs=vgg19.get_layer('block5_pool').output)
def analogy(img_A, img_BP, config):
weights = config['weights']
sizes = config['sizes']
rangee = config['rangee']
use_cuda = config['use_cuda']
params = config['params']
lr = config['lr']
# assert use_cuda==True, "cpu version is not implemented yet. You can modify VGG19.py to make it support CPU if you like."
# preparing data
img_A_tensor = torch.FloatTensor(img_A.transpose(2, 0, 1))
img_BP_tensor = torch.FloatTensor(img_BP.transpose(2, 0, 1))
if use_cuda:
img_A_tensor, img_BP_tensor = img_A_tensor.cuda(), img_BP_tensor.cuda()
img_A_tensor = img_A_tensor.unsqueeze(0)
img_BP_tensor = img_BP_tensor.unsqueeze(0)
# compute 5 feature maps
model = VGG19(use_cuda=use_cuda)
data_A, data_A_size = model.get_features(
img_tensor=img_A_tensor.clone(), layers=params['layers'])
data_AP = copy.deepcopy(data_A)
data_BP, data_B_size = model.get_features(
img_tensor=img_BP_tensor.clone(), layers=params['layers'])
data_B = copy.deepcopy(data_BP)
for curr_layer in range(5):
if curr_layer == 0:
ann_AB = init_nnf(
data_A_size[curr_layer][2:], data_B_size[curr_layer][2:])
ann_BA = init_nnf(
data_B_size[curr_layer][2:], data_A_size[curr_layer][2:])
else:
ann_AB = upSample_nnf(ann_AB, data_A_size[curr_layer][2:])
ann_BA = upSample_nnf(ann_BA, data_B_size[curr_layer][2:])
# blend feature
Ndata_A, response_A = normalize(data_A[curr_layer])
Ndata_BP, response_BP = normalize(data_BP[curr_layer])
data_AP[curr_layer] = blend(
response_A, data_A[curr_layer], data_AP[curr_layer], weights[curr_layer])
data_B[curr_layer] = blend(
response_BP, data_BP[curr_layer], data_B[curr_layer], weights[curr_layer])
Ndata_AP, _ = normalize(data_AP[curr_layer])
Ndata_B, _ = normalize(data_B[curr_layer])
# NNF search
ann_AB, _ = propagate(ann_AB, ts2np(Ndata_A), ts2np(Ndata_AP), ts2np(Ndata_B), ts2np(Ndata_BP), sizes[curr_layer],
params['iter'], rangee[curr_layer])
ann_BA, _ = propagate(ann_BA, ts2np(Ndata_BP), ts2np(Ndata_B), ts2np(Ndata_AP), ts2np(Ndata_A), sizes[curr_layer],
params['iter'], rangee[curr_layer])
if curr_layer >= 4:
break
# using backpropagation to approximate feature
next_layer = curr_layer + 2
ann_AB_upnnf2 = upSample_nnf(ann_AB, data_A_size[next_layer][2:])
ann_BA_upnnf2 = upSample_nnf(ann_BA, data_B_size[next_layer][2:])
data_AP_np = avg_vote(ann_AB_upnnf2, ts2np(data_BP[next_layer]), sizes[next_layer], data_A_size[next_layer][2:],
data_B_size[next_layer][2:])
data_B_np = avg_vote(ann_BA_upnnf2, ts2np(data_A[next_layer]), sizes[next_layer], data_B_size[next_layer][2:],
data_A_size[next_layer][2:])
data_AP[next_layer] = np2ts(data_AP_np)
data_B[next_layer] = np2ts(data_B_np)
target_BP_np = avg_vote(ann_AB, ts2np(data_BP[curr_layer]), sizes[curr_layer], data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
target_A_np = avg_vote(ann_BA, ts2np(data_A[curr_layer]), sizes[curr_layer], data_B_size[curr_layer][2:],
data_A_size[curr_layer][2:])
target_BP = np2ts(target_BP_np)
target_A = np2ts(target_A_np)
data_AP[curr_layer+1] = model.get_deconvoluted_feat(target_BP, curr_layer, data_AP[next_layer], lr=lr[curr_layer],
iters=400, display=False)
data_B[curr_layer+1] = model.get_deconvoluted_feat(target_A, curr_layer, data_B[next_layer], lr=lr[curr_layer],
iters=400, display=False)
if type(data_B[curr_layer + 1]) == torch.DoubleTensor:
data_B[curr_layer + 1] = data_B[curr_layer +
1].type(torch.FloatTensor)
data_AP[curr_layer + 1] = data_AP[curr_layer +
1].type(torch.FloatTensor)
elif type(data_B[curr_layer + 1]) == torch.cuda.DoubleTensor:
data_B[curr_layer + 1] = data_B[curr_layer +
1].type(torch.cuda.FloatTensor)
data_AP[curr_layer + 1] = data_AP[curr_layer +
1].type(torch.cuda.FloatTensor)
img_AP = reconstruct_avg(
ann_AB, img_BP, sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:])
img_B = reconstruct_avg(
ann_BA, img_A, sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:])
img_AP = np.clip(img_AP/255.0, 0, 1)[:, :, ::-1]
img_B = np.clip(img_B/255.0, 0, 1)[:, :, ::-1]
return img_AP, img_B
def analogy(img_A, img_BP, config):
start_time_0 = time.time()
weights = config['weights']
sizes = config['sizes']
rangee = config['rangee']
params = config['params']
lr = config['lr']
if config['use_cuda']:
device = torch.device('cuda:0')
else:
raise NotImplementedError('cpu mode is not supported yet')
# preparing data
img_A_tensor = torch.FloatTensor(img_A.transpose(2, 0, 1))
img_BP_tensor = torch.FloatTensor(img_BP.transpose(2, 0, 1))
img_A_tensor, img_BP_tensor = img_A_tensor.to(device), img_BP_tensor.to(
device)
img_A_tensor = img_A_tensor.unsqueeze(0)
img_BP_tensor = img_BP_tensor.unsqueeze(0)
# compute 5 feature maps
model = VGG19(device=device)
data_A, data_A_size = model.get_features(img_tensor=img_A_tensor.clone(),
layers=params['layers'])
data_AP = copy.deepcopy(data_A)
data_BP, data_B_size = model.get_features(img_tensor=img_BP_tensor.clone(),
layers=params['layers'])
data_B = copy.deepcopy(data_BP)
print("Features extracted!")
for curr_layer in range(5):
print("\n### current stage: %d - start ###" % (5 - curr_layer))
start_time_1 = time.time()
if curr_layer == 0:
ann_AB = init_nnf(data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
ann_BA = init_nnf(data_B_size[curr_layer][2:],
data_A_size[curr_layer][2:])
else:
ann_AB = upSample_nnf(ann_AB, data_A_size[curr_layer][2:])
ann_BA = upSample_nnf(ann_BA, data_B_size[curr_layer][2:])
# blend feature
Ndata_A, response_A = normalize(data_A[curr_layer])
Ndata_BP, response_BP = normalize(data_BP[curr_layer])
data_AP[curr_layer] = blend(response_A, data_A[curr_layer],
data_AP[curr_layer], weights[curr_layer])
data_B[curr_layer] = blend(response_BP, data_BP[curr_layer],
data_B[curr_layer], weights[curr_layer])
Ndata_AP, _ = normalize(data_AP[curr_layer])
Ndata_B, _ = normalize(data_B[curr_layer])
# NNF search
print("- NNF search for ann_AB")
start_time_2 = time.time()
ann_AB, _ = propagate(ann_AB, ts2np(Ndata_A), ts2np(Ndata_AP),
ts2np(Ndata_B), ts2np(Ndata_BP),
sizes[curr_layer], params['iter'],
rangee[curr_layer])
print("\tElapse: " +
str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])
print("- NNF search for ann_BA")
start_time_2 = time.time()
ann_BA, _ = propagate(ann_BA, ts2np(Ndata_BP), ts2np(Ndata_B),
ts2np(Ndata_AP), ts2np(Ndata_A),
sizes[curr_layer], params['iter'],
rangee[curr_layer])
print("\tElapse: " +
str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])
if curr_layer >= 4:
print("### current stage: %d - end | " % (5 - curr_layer) +
"Elapse: " +
str(datetime.timedelta(seconds=time.time() -
start_time_1))[:-7] + ' ###')
break
# using backpropagation to approximate feature
next_layer = curr_layer + 2
ann_AB_upnnf2 = upSample_nnf(ann_AB, data_A_size[next_layer][2:])
ann_BA_upnnf2 = upSample_nnf(ann_BA, data_B_size[next_layer][2:])
data_AP_np = avg_vote(ann_AB_upnnf2, ts2np(data_BP[next_layer]),
sizes[next_layer], data_A_size[next_layer][2:],
data_B_size[next_layer][2:])
data_B_np = avg_vote(ann_BA_upnnf2, ts2np(data_A[next_layer]),
sizes[next_layer], data_B_size[next_layer][2:],
data_A_size[next_layer][2:])
data_AP[next_layer] = np2ts(data_AP_np, device)
data_B[next_layer] = np2ts(data_B_np, device)
target_BP_np = avg_vote(ann_AB, ts2np(data_BP[curr_layer]),
sizes[curr_layer], data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
target_A_np = avg_vote(ann_BA, ts2np(data_A[curr_layer]),
sizes[curr_layer], data_B_size[curr_layer][2:],
data_A_size[curr_layer][2:])
target_BP = np2ts(target_BP_np, device)
target_A = np2ts(target_A_np, device)
print('- deconvolution for feat A\'')
start_time_2 = time.time()
data_AP[curr_layer + 1] = model.get_deconvoluted_feat(
target_BP,
curr_layer,
data_AP[next_layer],
lr=lr[curr_layer],
iters=400,
display=False)
print("\tElapse: " +
str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])
print('- deconvolution for feat B')
start_time_2 = time.time()
data_B[curr_layer + 1] = model.get_deconvoluted_feat(
target_A,
curr_layer,
data_B[next_layer],
lr=lr[curr_layer],
iters=400,
display=False)
print("\tElapse: " +
str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])
# in case of data type inconsistency
if data_B[curr_layer + 1].type() == torch.cuda.DoubleTensor:
data_B[curr_layer + 1] = data_B[curr_layer + 1].type(
torch.cuda.FloatTensor)
data_AP[curr_layer + 1] = data_AP[curr_layer + 1].type(
torch.cuda.FloatTensor)
print("### current stage: %d - end | " % (5 - curr_layer) +
"Elapse: " +
str(datetime.timedelta(seconds=time.time() -
start_time_1))[:-7] + ' ###')
print('\n- reconstruct images A\' and B')
img_AP = reconstruct_avg(ann_AB, img_BP, sizes[curr_layer],
data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
img_B = reconstruct_avg(ann_BA, img_A, sizes[curr_layer],
data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
img_AP = np.clip(img_AP, 0, 255)
img_B = np.clip(img_B, 0, 255)
return img_AP, img_B, str(
datetime.timedelta(seconds=time.time() - start_time_0))[:-7]
if (device.type == 'cuda') and (ngpu > 1): # Handle multi-gpu if desired
netG = nn.DataParallel(netG, list(range(ngpu)))
#print(netG) # Print the model
netG.train()
# DISCRIMINATOR
netD = Discriminator(in_nc_d, out_nc_d, ndf).to(device)
if (device.type == 'cuda') and (ngpu > 1): # Handle multi-gpu if desired
netD = nn.DataParallel(netD, list(range(ngpu)))
#print(netD) # Print the model
netD.train()
# VGG19
vgg = VGG19(init_weights=vggroot, feature_mode=True).to(device)
# Initialize BCELoss, L1Loss function
bce_loss = nn.BCELoss().to(device)
l1_loss = nn.L1Loss().to(device)
# Setup Adam optimizers for both G and D
optimizerD = optim.Adam(netD.parameters(), lr=lr, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters(), lr=lr, betas=(beta1, 0.999))
G_scheduler = optim.lr_scheduler.MultiStepLR(
optimizer=optimizerG,
milestones=[num_epochs // 2, num_epochs // 4 * 3],
gamma=0.1)
D_scheduler = optim.lr_scheduler.MultiStepLR(
optimizer=optimizerD,
def choose_model():
# switch models
print(args.model)
print(args.layerlist)
if args.model == 'lenet':
cnn = LeNet(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
batch_norm=args.batchnorm,
if_dropout=args.dropout)
elif args.model == 'resnet':
cnn = ResNet50(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'resnet18':
cnn = ResNet18(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg16':
cnn = VGG16(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg11':
cnn = VGG11(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg13':
cnn = VGG13(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg19':
cnn = VGG19(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'densenet':
cnn = DenseNet()
cnn.cuda()
if args.enable_lat:
cnn.choose_layer()
return cnn
def analogy(img_A, img_BP, config):
# set basic param
weights = config['weights']
sizes = config['sizes']
radius = config['radius']
layers = config['layers']
iters = config['iters']
lr = config['lr']
show_step = config['show_step']
# compute 5 feature maps
model = VGG19()
data_A, data_A_size = model.get_features(img_tensor=img_A, layers=layers)
data_AP = copy.deepcopy(data_A)
data_BP, data_B_size = model.get_features(img_tensor=img_BP, layers=layers)
data_B = copy.deepcopy(data_BP)
for idx, layer_size in enumerate(data_A_size):
print("layer_{}_size:".format(idx), layer_size)
for curr_layer in range(5):
if curr_layer == 0:
ann_AB = init_nnf(data_A_size[curr_layer][2:])
ann_BA = init_nnf(data_B_size[curr_layer][2:])
else:
ann_AB = pmAB.upsample_nnf(data_A_size[curr_layer][2])
ann_BA = pmBA.upsample_nnf(data_B_size[curr_layer][2])
# blend feature
Ndata_A, response_A = normalize(data_A[curr_layer])
Ndata_BP, response_BP = normalize(data_BP[curr_layer])
data_AP[curr_layer] = blend(response_A, data_A[curr_layer],
data_AP[curr_layer], weights[curr_layer])
data_B[curr_layer] = blend(response_BP, data_BP[curr_layer],
data_B[curr_layer], weights[curr_layer])
Ndata_AP, _ = normalize(data_AP[curr_layer])
Ndata_B, _ = normalize(data_B[curr_layer])
# NNF search
print("propagate_for_{}".format(curr_layer))
pmAB = propagate(ann_AB,
nd2np(Ndata_A), nd2np(Ndata_AP), nd2np(Ndata_B),
nd2np(Ndata_BP), sizes[curr_layer], iters[curr_layer],
radius[curr_layer])
pmBA = propagate(ann_BA, nd2np(Ndata_BP), nd2np(Ndata_B),
nd2np(Ndata_AP), nd2np(Ndata_A), sizes[curr_layer],
iters[curr_layer], radius[curr_layer])
if show_step:
img_1 = pmAB.reconstruct_image(img_BP)
img_2 = pmBA.reconstruct_image(img_A)
output_img(post_process(img_1), post_process(img_2))
if curr_layer < 4:
# using backpropagation to approximate feature
next_layer = curr_layer + 2
data_AP_np = pmAB.reconstruct_image(nd2np(data_BP[next_layer]))
data_B_np = pmBA.reconstruct_image(nd2np(data_A[next_layer]))
target_BP_np = pmAB.reconstruct_image(nd2np(data_BP[curr_layer]))
target_A_np = pmBA.reconstruct_image(nd2np(data_A[curr_layer]))
print("deconvolution_for_{}".format(curr_layer))
data_AP[curr_layer + 1] = model.get_deconvoluted_feat(
np2nd(target_BP_np),
curr_layer,
np2nd(data_AP_np),
lr=lr[curr_layer],
iters=3000)
data_B[curr_layer + 1] = model.get_deconvoluted_feat(
np2nd(target_A_np),
curr_layer,
np2nd(data_B_np),
lr=lr[curr_layer],
iters=3000)
print("reconstruction image")
img_AP = pmAB.reconstruct_avg(img_BP, 5) # size 5 is in paper
img_B = pmBA.reconstruct_avg(img_A, 5)
img_AP = post_process(img_AP)
img_B = post_process(img_B)
return img_AP, img_B
def analogy(img_A_L, img_BP_L, config):
img_A_L, img_A_Lab = prepare_image(img_A_L)
img_BP_L, img_BP_Lab = prepare_image(img_BP_L)
start_time_0 = time.time()
weights = config['weights']
sizes = config['sizes']
rangee = config['rangee']
deconv_iters = config['deconv_iters']
params = config['params']
lr = config['lr']
model_name = config['model']
device = torch.device("cuda" if USE_CUDA else "cpu")
# preparing data
img_A_tensor = torch.FloatTensor(img_A_L.transpose(2, 0, 1))
img_BP_tensor = torch.FloatTensor(img_BP_L.transpose(2, 0, 1))
img_A_tensor, img_BP_tensor = img_A_tensor.to(device), img_BP_tensor.to(
device)
img_A_tensor = img_A_tensor.unsqueeze(0)
img_BP_tensor = img_BP_tensor.unsqueeze(0)
# compute 5 feature maps
if model_name == "VGG19":
model = VGG19(device=device)
elif model_name == "VGG19Gray":
model = VGG19Gray(device=device)
data_A, data_A_size = model.get_features(img_tensor=img_A_tensor.clone(),
layers=params['layers'])
data_AP = copy.deepcopy(data_A)
data_BP, data_B_size = model.get_features(img_tensor=img_BP_tensor.clone(),
layers=params['layers'])
data_B = copy.deepcopy(data_BP)
print("Features extracted!")
# usually 5 layers
n_layers = len(params['layers'])
for curr_layer in range(n_layers):
print("\n### current stage: %d - start ###" % (5 - curr_layer))
start_time_1 = time.time()
if curr_layer == 0:
ann_AB = pm.init_nnf(data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
ann_BA = pm.init_nnf(data_B_size[curr_layer][2:],
data_A_size[curr_layer][2:])
else:
ann_AB = pm.upSample_nnf(ann_AB, data_A_size[curr_layer][2:])
ann_BA = pm.upSample_nnf(ann_BA, data_B_size[curr_layer][2:])
# blend feature
Ndata_A, response_A = normalize(data_A[curr_layer])
Ndata_BP, response_BP = normalize(data_BP[curr_layer])
data_AP[curr_layer] = blend(response_A, data_A[curr_layer],
data_AP[curr_layer], weights[curr_layer])
data_B[curr_layer] = blend(response_BP, data_BP[curr_layer],
data_B[curr_layer], weights[curr_layer])
Ndata_AP, _ = normalize(data_AP[curr_layer])
Ndata_B, _ = normalize(data_B[curr_layer])
# NNF search
print("- NNF search for ann_AB")
start_time_2 = time.time()
ann_AB, _ = pm.propagate(ann_AB, ts2np(Ndata_A), ts2np(Ndata_AP),
ts2np(Ndata_B), ts2np(Ndata_BP),
sizes[curr_layer], params['propagate_iter'],
rangee[curr_layer])
print("\tElapse: " +
str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])
print("- NNF search for ann_BA")
start_time_2 = time.time()
ann_BA, _ = pm.propagate(ann_BA, ts2np(Ndata_BP), ts2np(Ndata_B),
ts2np(Ndata_AP), ts2np(Ndata_A),
sizes[curr_layer], params['propagate_iter'],
rangee[curr_layer])
print("\tElapse: " +
str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])
if curr_layer >= 4:
print("### current stage: %d - end | " % (5 - curr_layer) +
"Elapse: " +
str(datetime.timedelta(seconds=time.time() -
start_time_1))[:-7] + ' ###')
break
# using backpropagation to approximate feature
next_layer = curr_layer + 2
ann_AB_upnnf2 = pm.upSample_nnf(ann_AB, data_A_size[next_layer][2:])
ann_BA_upnnf2 = pm.upSample_nnf(ann_BA, data_B_size[next_layer][2:])
data_AP_np = pm.avg_vote(ann_AB_upnnf2, ts2np(data_BP[next_layer]),
sizes[next_layer],
data_A_size[next_layer][2:],
data_B_size[next_layer][2:])
data_B_np = pm.avg_vote(ann_BA_upnnf2, ts2np(data_A[next_layer]),
sizes[next_layer], data_B_size[next_layer][2:],
data_A_size[next_layer][2:])
data_AP[next_layer] = np2ts(data_AP_np, device)
data_B[next_layer] = np2ts(data_B_np, device)
target_BP_np = pm.avg_vote(ann_AB, ts2np(data_BP[curr_layer]),
sizes[curr_layer],
data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
target_A_np = pm.avg_vote(ann_BA, ts2np(data_A[curr_layer]),
sizes[curr_layer],
data_B_size[curr_layer][2:],
data_A_size[curr_layer][2:])
target_BP = np2ts(target_BP_np, device)
target_A = np2ts(target_A_np, device)
print('- deconvolution for feat A\'')
start_time_2 = time.time()
data_AP[curr_layer + 1] = model.get_deconvoluted_feat(
target_BP,
curr_layer,
data_AP[next_layer],
lr=lr[curr_layer],
iters=deconv_iters,
display=False)
print("\tElapse: " +
str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])
print('- deconvolution for feat B')
start_time_2 = time.time()
data_B[curr_layer + 1] = model.get_deconvoluted_feat(
target_A,
curr_layer,
data_B[next_layer],
lr=lr[curr_layer],
iters=deconv_iters,
display=False)
print("\tElapse: " +
str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])
if USE_CUDA:
# in case of data type inconsistency
if data_B[curr_layer + 1].type() == torch.cuda.DoubleTensor:
data_B[curr_layer + 1] = data_B[curr_layer + 1].type(
torch.cuda.FloatTensor)
data_AP[curr_layer + 1] = data_AP[curr_layer + 1].type(
torch.cuda.FloatTensor)
else:
if data_B[curr_layer + 1].type() == torch.DoubleTensor:
data_B[curr_layer + 1] = data_B[curr_layer + 1].type(
torch.FloatTensor)
data_AP[curr_layer + 1] = data_AP[curr_layer + 1].type(
torch.FloatTensor)
print("### current stage: %d - end | " % (5 - curr_layer) +
"Elapse: " +
str(datetime.timedelta(seconds=time.time() -
start_time_1))[:-7] + ' ###')
print('\n- reconstruct images A\' and B')
img_AP_Lab = pm.reconstruct_avg(ann_AB, img_BP_Lab, sizes[curr_layer],
data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
img_B_Lab = pm.reconstruct_avg(ann_BA, img_A_Lab, sizes[curr_layer],
data_A_size[curr_layer][2:],
data_B_size[curr_layer][2:])
img_AP_Lab = np.clip(img_AP_Lab, 0, 255).astype("uint8")
img_B_Lab = np.clip(img_B_Lab, 0, 255).astype("uint8")
# img_AP_L = cv2.split(img_AP_Lab)[0]
# img_B_L = cv2.split(img_B_Lab)[0]
img_AP_bgr = cv2.cvtColor(img_AP_Lab, cv2.COLOR_LAB2BGR)
img_B_bgr = cv2.cvtColor(img_B_Lab, cv2.COLOR_LAB2BGR)
return img_AP_bgr, img_B_bgr, str(
datetime.timedelta(seconds=time.time() - start_time_0))[:-7]
def deep_image_analogy(A, BP, config):
alphas = config['alpha']
nnf_patch_size = config['nnf_patch_size']
radii = config['radii']
params = config['params']
lr = config['lr']
# preparing data
img_A_tensor = torch.FloatTensor(A.transpose(2, 0, 1)).cuda()
img_BP_tensor = torch.FloatTensor(BP.transpose(2, 0, 1)).cuda()
# fake a batch dimension
img_A_tensor = img_A_tensor.unsqueeze(0)
img_BP_tensor = img_BP_tensor.unsqueeze(0)
# 4.1 Preprocessing Step
model = VGG19()
F_A, F_A_size = model.get_features(img_tensor=img_A_tensor.clone(),
layers=params['layers'])
F_BP, F_B_size = model.get_features(img_tensor=img_BP_tensor.clone(),
layers=params['layers'])
# Init AP&B 's feature maps with F_A&F_BP
F_AP = copy.deepcopy(F_A)
F_B = copy.deepcopy(F_BP)
#Note that the feature_maps now is in the order of [5,4,3,2,1,input]
for curr_layer in range(5):
#ANN init step, coarsest layer is initialized randomly,
#Other layers is initialized using upsample technique described in the paper
if curr_layer == 0:
ann_AB = init_nnf(F_A_size[curr_layer][2:],
F_B_size[curr_layer][2:])
ann_BA = init_nnf(F_B_size[curr_layer][2:],
F_A_size[curr_layer][2:])
else:
ann_AB = upSample_nnf(ann_AB, F_A_size[curr_layer][2:])
ann_BA = upSample_nnf(ann_BA, F_B_size[curr_layer][2:])
# According to Equotion(2), we need to normalize F_A and F_BP
# response denotes the M in Equotion(6)
F_A_BAR, response_A = normalize(F_A[curr_layer])
F_BP_BAR, response_BP = normalize(F_BP[curr_layer])
# F_AP&F_B is reconstructed according to Equotion(4)
# Note that we reuse the varibale F_AP here,
# it denotes the RBprime as is stated in the Equotion(4) which is calculated
# at the end of the previous iteration
F_AP[curr_layer] = blend(response_A, F_A[curr_layer], F_AP[curr_layer],
alphas[curr_layer])
F_B[curr_layer] = blend(response_BP, F_BP[curr_layer], F_B[curr_layer],
alphas[curr_layer])
# Normalize F_AP&F_B as well
F_AP_BAR, _ = normalize(F_AP[curr_layer])
F_B_BAR, _ = normalize(F_B[curr_layer])
# Run PatchMatch algorithm to get mapping AB and BA
ann_AB, _ = propagate(ann_AB, ts2np(F_A_BAR), ts2np(F_AP_BAR),
ts2np(F_B_BAR), ts2np(F_BP_BAR),
nnf_patch_size[curr_layer], params['iter'],
radii[curr_layer])
ann_BA, _ = propagate(ann_BA, ts2np(F_BP_BAR), ts2np(F_B_BAR),
ts2np(F_AP_BAR), ts2np(F_A_BAR),
nnf_patch_size[curr_layer], params['iter'],
radii[curr_layer])
if curr_layer >= 4:
break
# The code below is used to initialize the F_AP&F_B in the next layer,
# it generates the R_B' and R_A as is stated in Equotion(4)
# R_B' is stored in F_AP, R_A is stored in F_B
# using backpropagation to approximate feature
# About why we add 2 here:
# https://github.com/msracver/Deep-Image-Analogy/issues/30
next_layer = curr_layer + 2
ann_AB_upnnf2 = upSample_nnf(ann_AB, F_A_size[next_layer][2:])
ann_BA_upnnf2 = upSample_nnf(ann_BA, F_B_size[next_layer][2:])
F_AP_np = avg_vote(ann_AB_upnnf2, ts2np(F_BP[next_layer]),
nnf_patch_size[next_layer],
F_A_size[next_layer][2:], F_B_size[next_layer][2:])
F_B_np = avg_vote(ann_BA_upnnf2, ts2np(F_A[next_layer]),
nnf_patch_size[next_layer], F_B_size[next_layer][2:],
F_A_size[next_layer][2:])
# Initialize R_B' and R_A
F_AP[next_layer] = np2ts(F_AP_np)
F_B[next_layer] = np2ts(F_B_np)
# Warp F_BP using ann_AB, Warp F_A using ann_BA
target_BP_np = avg_vote(ann_AB, ts2np(F_BP[curr_layer]),
nnf_patch_size[curr_layer],
F_A_size[curr_layer][2:],
F_B_size[curr_layer][2:])
target_A_np = avg_vote(ann_BA, ts2np(F_A[curr_layer]),
nnf_patch_size[curr_layer],
F_B_size[curr_layer][2:],
F_A_size[curr_layer][2:])
target_BP = np2ts(target_BP_np)
target_A = np2ts(target_A_np)
#LBFGS algorithm to approximate R_B' and R_A
F_AP[curr_layer + 1] = model.get_deconvoluted_feat(
target_BP,
curr_layer,
F_AP[next_layer],
lr=lr[curr_layer],
blob_layers=params['layers'])
F_B[curr_layer + 1] = model.get_deconvoluted_feat(
target_A,
curr_layer,
F_B[next_layer],
lr=lr[curr_layer],
blob_layers=params['layers'])
if type(F_B[curr_layer + 1]) == torch.DoubleTensor:
F_B[curr_layer + 1] = F_B[curr_layer + 1].type(torch.FloatTensor)
F_AP[curr_layer + 1] = F_AP[curr_layer + 1].type(torch.FloatTensor)
elif type(F_B[curr_layer + 1]) == torch.cuda.DoubleTensor:
F_B[curr_layer + 1] = F_B[curr_layer + 1].type(
torch.cuda.FloatTensor)
F_AP[curr_layer + 1] = F_AP[curr_layer + 1].type(
torch.cuda.FloatTensor)
# Obtain the output according to 4.5
img_AP = reconstruct_avg(ann_AB, BP, nnf_patch_size[curr_layer],
F_A_size[curr_layer][2:],
F_B_size[curr_layer][2:])
img_B = reconstruct_avg(ann_BA, A, nnf_patch_size[curr_layer],
F_A_size[curr_layer][2:], F_B_size[curr_layer][2:])
img_AP = np.clip(img_AP / 255.0, 0, 1)[:, :, ::-1]
img_B = np.clip(img_B / 255.0, 0, 1)[:, :, ::-1]
return img_AP, img_B
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Read Image
img = cv2.imread(mri_img)
print(img.shape)
assert (img.shape[2] == 3)
# ************************************OPTION 1************************************
img_tensor = torch.FloatTensor(img.transpose(2, 0, 1))
img_tensor = img_tensor.to(device)
img_tensor = img_tensor.unsqueeze(0)
# compute 5 feature maps
model = VGG19(device=device)
# data, data_size = model.get_features(img_tensor=img_tensor.clone(), layers=[29])
data, data_size = model.get_features(img_tensor=img_tensor.clone(),
layers=[34])
features, = copy.deepcopy(data[:-1])
features_size = data_size[:-1]
print(features, features_size)
print(torch.max(features), torch.min(features))
# ************************************************************************
# ************************************OPTION 1************************************
# img_tensor = torch.FloatTensor(img_mri.transpose(2, 0, 1))
# img_tensor = img_tensor.to(device)
#