def loss_net(x_in, trux_x_in, width, height, style_image_path, content_weight, style_weight): # Append the initial input to the FastNet input to the VGG inputs x = concatenate([x_in, trux_x_in], axis=0) # Normalize the inputs via custom VGG Normalization layer x = VGGNormalize(name="vgg_normalize")(x) # Using pretrained VGG 19 model vgg = VGG19(include_top=False, input_tensor=x) vgg_output_dict = dict([(layer.name, layer.output) for layer in vgg.layers[-21:]]) vgg_layers = dict([(layer.name, layer) for layer in vgg.layers[-21:]]) if style_weight > 0: add_style_loss(vgg, style_image_path, vgg_layers, vgg_output_dict, width, height, style_weight) if content_weight > 0: add_content_loss(vgg_layers, vgg_output_dict, content_weight) # Freeze all VGG layers for layer in vgg.layers[-22:]: layer.trainable = False return vgg
def __init__(self, content_layers, style_layers, content_image, style_image, loss_ratio, num_iter, init_image=None): #content_layers and style_layers should be dicts: {name: weight} self.net = VGG19('./imagenet-vgg-verydeep-19.mat') self.content_layers = content_layers self.style_layers = style_layers self.content = np.float32(content_image) self.style = np.float32(style_image) self.alpha = loss_ratio self.beta = 1 self.iteration = num_iter self.mean = (np.mean(content_image, axis=(0, 1)) + np.mean(style_image, axis=(0, 1))) / (2 * 255) self.p = tf.constant(content_image - self.mean, name='content') self.a = tf.constant(style_image - self.mean, name='style') if init_image is None: self.img = tf.Variable(tf.random_normal(self.content.shape), trainable=True, dtype=tf.float32) else: self.img = tf.Variable(init_image - self.mean, trainable=True, dtype=tf.float32) # self.sess = tf.Session() # self.sess.run(tf.global_variables_initializer()) self._built_net()
return loss # def style_transfer(content_img, style_img, content_layer_id, # style_layer_id, alpha = 1.5, beta = 10, num_iter = 100, step_size = 10): content_img = preprocess(content_img_path) style_img = preprocess(style_img_path) mixed_img = preprocess(mixed_img_path) tf.reset_default_graph() img = tf.placeholder(tf.float32, (1, height, width, num_channels), name='my_original_image') vgg19 = VGG19(image_shape=(1, height, width, num_channels), input_tensor=img) vgg19.summary() output = tf.identity(vgg19['block5_pool'], name='my_output') # show_graph(tf.get_default_graph().as_graph_def()) with tf.Session() as sess: vgg19.load_weights() fd = {img: style_img} output_val = sess.run(output, fd) print(output_val.shape, output_val.mean()) # vgg19 = VGG19(weights='imagenet', include_top=False, pooling='avg') # model = Model(inputs=vgg19.input, outputs=vgg19.get_layer('block5_pool').output)
def analogy(img_A, img_BP, config): weights = config['weights'] sizes = config['sizes'] rangee = config['rangee'] use_cuda = config['use_cuda'] params = config['params'] lr = config['lr'] # assert use_cuda==True, "cpu version is not implemented yet. You can modify VGG19.py to make it support CPU if you like." # preparing data img_A_tensor = torch.FloatTensor(img_A.transpose(2, 0, 1)) img_BP_tensor = torch.FloatTensor(img_BP.transpose(2, 0, 1)) if use_cuda: img_A_tensor, img_BP_tensor = img_A_tensor.cuda(), img_BP_tensor.cuda() img_A_tensor = img_A_tensor.unsqueeze(0) img_BP_tensor = img_BP_tensor.unsqueeze(0) # compute 5 feature maps model = VGG19(use_cuda=use_cuda) data_A, data_A_size = model.get_features( img_tensor=img_A_tensor.clone(), layers=params['layers']) data_AP = copy.deepcopy(data_A) data_BP, data_B_size = model.get_features( img_tensor=img_BP_tensor.clone(), layers=params['layers']) data_B = copy.deepcopy(data_BP) for curr_layer in range(5): if curr_layer == 0: ann_AB = init_nnf( data_A_size[curr_layer][2:], data_B_size[curr_layer][2:]) ann_BA = init_nnf( data_B_size[curr_layer][2:], data_A_size[curr_layer][2:]) else: ann_AB = upSample_nnf(ann_AB, data_A_size[curr_layer][2:]) ann_BA = upSample_nnf(ann_BA, data_B_size[curr_layer][2:]) # blend feature Ndata_A, response_A = normalize(data_A[curr_layer]) Ndata_BP, response_BP = normalize(data_BP[curr_layer]) data_AP[curr_layer] = blend( response_A, data_A[curr_layer], data_AP[curr_layer], weights[curr_layer]) data_B[curr_layer] = blend( response_BP, data_BP[curr_layer], data_B[curr_layer], weights[curr_layer]) Ndata_AP, _ = normalize(data_AP[curr_layer]) Ndata_B, _ = normalize(data_B[curr_layer]) # NNF search ann_AB, _ = propagate(ann_AB, ts2np(Ndata_A), ts2np(Ndata_AP), ts2np(Ndata_B), ts2np(Ndata_BP), sizes[curr_layer], params['iter'], rangee[curr_layer]) ann_BA, _ = propagate(ann_BA, ts2np(Ndata_BP), ts2np(Ndata_B), ts2np(Ndata_AP), ts2np(Ndata_A), sizes[curr_layer], params['iter'], rangee[curr_layer]) if curr_layer >= 4: break # using backpropagation to approximate feature next_layer = curr_layer + 2 ann_AB_upnnf2 = upSample_nnf(ann_AB, data_A_size[next_layer][2:]) ann_BA_upnnf2 = upSample_nnf(ann_BA, data_B_size[next_layer][2:]) data_AP_np = avg_vote(ann_AB_upnnf2, ts2np(data_BP[next_layer]), sizes[next_layer], data_A_size[next_layer][2:], data_B_size[next_layer][2:]) data_B_np = avg_vote(ann_BA_upnnf2, ts2np(data_A[next_layer]), sizes[next_layer], data_B_size[next_layer][2:], data_A_size[next_layer][2:]) data_AP[next_layer] = np2ts(data_AP_np) data_B[next_layer] = np2ts(data_B_np) target_BP_np = avg_vote(ann_AB, ts2np(data_BP[curr_layer]), sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:]) target_A_np = avg_vote(ann_BA, ts2np(data_A[curr_layer]), sizes[curr_layer], data_B_size[curr_layer][2:], data_A_size[curr_layer][2:]) target_BP = np2ts(target_BP_np) target_A = np2ts(target_A_np) data_AP[curr_layer+1] = model.get_deconvoluted_feat(target_BP, curr_layer, data_AP[next_layer], lr=lr[curr_layer], iters=400, display=False) data_B[curr_layer+1] = model.get_deconvoluted_feat(target_A, curr_layer, data_B[next_layer], lr=lr[curr_layer], iters=400, display=False) if type(data_B[curr_layer + 1]) == torch.DoubleTensor: data_B[curr_layer + 1] = data_B[curr_layer + 1].type(torch.FloatTensor) data_AP[curr_layer + 1] = data_AP[curr_layer + 1].type(torch.FloatTensor) elif type(data_B[curr_layer + 1]) == torch.cuda.DoubleTensor: data_B[curr_layer + 1] = data_B[curr_layer + 1].type(torch.cuda.FloatTensor) data_AP[curr_layer + 1] = data_AP[curr_layer + 1].type(torch.cuda.FloatTensor) img_AP = reconstruct_avg( ann_AB, img_BP, sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:]) img_B = reconstruct_avg( ann_BA, img_A, sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:]) img_AP = np.clip(img_AP/255.0, 0, 1)[:, :, ::-1] img_B = np.clip(img_B/255.0, 0, 1)[:, :, ::-1] return img_AP, img_B
def analogy(img_A, img_BP, config): start_time_0 = time.time() weights = config['weights'] sizes = config['sizes'] rangee = config['rangee'] params = config['params'] lr = config['lr'] if config['use_cuda']: device = torch.device('cuda:0') else: raise NotImplementedError('cpu mode is not supported yet') # preparing data img_A_tensor = torch.FloatTensor(img_A.transpose(2, 0, 1)) img_BP_tensor = torch.FloatTensor(img_BP.transpose(2, 0, 1)) img_A_tensor, img_BP_tensor = img_A_tensor.to(device), img_BP_tensor.to( device) img_A_tensor = img_A_tensor.unsqueeze(0) img_BP_tensor = img_BP_tensor.unsqueeze(0) # compute 5 feature maps model = VGG19(device=device) data_A, data_A_size = model.get_features(img_tensor=img_A_tensor.clone(), layers=params['layers']) data_AP = copy.deepcopy(data_A) data_BP, data_B_size = model.get_features(img_tensor=img_BP_tensor.clone(), layers=params['layers']) data_B = copy.deepcopy(data_BP) print("Features extracted!") for curr_layer in range(5): print("\n### current stage: %d - start ###" % (5 - curr_layer)) start_time_1 = time.time() if curr_layer == 0: ann_AB = init_nnf(data_A_size[curr_layer][2:], data_B_size[curr_layer][2:]) ann_BA = init_nnf(data_B_size[curr_layer][2:], data_A_size[curr_layer][2:]) else: ann_AB = upSample_nnf(ann_AB, data_A_size[curr_layer][2:]) ann_BA = upSample_nnf(ann_BA, data_B_size[curr_layer][2:]) # blend feature Ndata_A, response_A = normalize(data_A[curr_layer]) Ndata_BP, response_BP = normalize(data_BP[curr_layer]) data_AP[curr_layer] = blend(response_A, data_A[curr_layer], data_AP[curr_layer], weights[curr_layer]) data_B[curr_layer] = blend(response_BP, data_BP[curr_layer], data_B[curr_layer], weights[curr_layer]) Ndata_AP, _ = normalize(data_AP[curr_layer]) Ndata_B, _ = normalize(data_B[curr_layer]) # NNF search print("- NNF search for ann_AB") start_time_2 = time.time() ann_AB, _ = propagate(ann_AB, ts2np(Ndata_A), ts2np(Ndata_AP), ts2np(Ndata_B), ts2np(Ndata_BP), sizes[curr_layer], params['iter'], rangee[curr_layer]) print("\tElapse: " + str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7]) print("- NNF search for ann_BA") start_time_2 = time.time() ann_BA, _ = propagate(ann_BA, ts2np(Ndata_BP), ts2np(Ndata_B), ts2np(Ndata_AP), ts2np(Ndata_A), sizes[curr_layer], params['iter'], rangee[curr_layer]) print("\tElapse: " + str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7]) if curr_layer >= 4: print("### current stage: %d - end | " % (5 - curr_layer) + "Elapse: " + str(datetime.timedelta(seconds=time.time() - start_time_1))[:-7] + ' ###') break # using backpropagation to approximate feature next_layer = curr_layer + 2 ann_AB_upnnf2 = upSample_nnf(ann_AB, data_A_size[next_layer][2:]) ann_BA_upnnf2 = upSample_nnf(ann_BA, data_B_size[next_layer][2:]) data_AP_np = avg_vote(ann_AB_upnnf2, ts2np(data_BP[next_layer]), sizes[next_layer], data_A_size[next_layer][2:], data_B_size[next_layer][2:]) data_B_np = avg_vote(ann_BA_upnnf2, ts2np(data_A[next_layer]), sizes[next_layer], data_B_size[next_layer][2:], data_A_size[next_layer][2:]) data_AP[next_layer] = np2ts(data_AP_np, device) data_B[next_layer] = np2ts(data_B_np, device) target_BP_np = avg_vote(ann_AB, ts2np(data_BP[curr_layer]), sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:]) target_A_np = avg_vote(ann_BA, ts2np(data_A[curr_layer]), sizes[curr_layer], data_B_size[curr_layer][2:], data_A_size[curr_layer][2:]) target_BP = np2ts(target_BP_np, device) target_A = np2ts(target_A_np, device) print('- deconvolution for feat A\'') start_time_2 = time.time() data_AP[curr_layer + 1] = model.get_deconvoluted_feat( target_BP, curr_layer, data_AP[next_layer], lr=lr[curr_layer], iters=400, display=False) print("\tElapse: " + str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7]) print('- deconvolution for feat B') start_time_2 = time.time() data_B[curr_layer + 1] = model.get_deconvoluted_feat( target_A, curr_layer, data_B[next_layer], lr=lr[curr_layer], iters=400, display=False) print("\tElapse: " + str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7]) # in case of data type inconsistency if data_B[curr_layer + 1].type() == torch.cuda.DoubleTensor: data_B[curr_layer + 1] = data_B[curr_layer + 1].type( torch.cuda.FloatTensor) data_AP[curr_layer + 1] = data_AP[curr_layer + 1].type( torch.cuda.FloatTensor) print("### current stage: %d - end | " % (5 - curr_layer) + "Elapse: " + str(datetime.timedelta(seconds=time.time() - start_time_1))[:-7] + ' ###') print('\n- reconstruct images A\' and B') img_AP = reconstruct_avg(ann_AB, img_BP, sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:]) img_B = reconstruct_avg(ann_BA, img_A, sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:]) img_AP = np.clip(img_AP, 0, 255) img_B = np.clip(img_B, 0, 255) return img_AP, img_B, str( datetime.timedelta(seconds=time.time() - start_time_0))[:-7]
if (device.type == 'cuda') and (ngpu > 1): # Handle multi-gpu if desired netG = nn.DataParallel(netG, list(range(ngpu))) #print(netG) # Print the model netG.train() # DISCRIMINATOR netD = Discriminator(in_nc_d, out_nc_d, ndf).to(device) if (device.type == 'cuda') and (ngpu > 1): # Handle multi-gpu if desired netD = nn.DataParallel(netD, list(range(ngpu))) #print(netD) # Print the model netD.train() # VGG19 vgg = VGG19(init_weights=vggroot, feature_mode=True).to(device) # Initialize BCELoss, L1Loss function bce_loss = nn.BCELoss().to(device) l1_loss = nn.L1Loss().to(device) # Setup Adam optimizers for both G and D optimizerD = optim.Adam(netD.parameters(), lr=lr, betas=(beta1, 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=lr, betas=(beta1, 0.999)) G_scheduler = optim.lr_scheduler.MultiStepLR( optimizer=optimizerG, milestones=[num_epochs // 2, num_epochs // 4 * 3], gamma=0.1) D_scheduler = optim.lr_scheduler.MultiStepLR( optimizer=optimizerD,
def choose_model(): # switch models print(args.model) print(args.layerlist) if args.model == 'lenet': cnn = LeNet(enable_lat=args.enable_lat, layerlist=args.layerlist, epsilon=args.epsilon, alpha=args.alpha, pro_num=args.pro_num, batch_size=args.batchsize, batch_norm=args.batchnorm, if_dropout=args.dropout) elif args.model == 'resnet': cnn = ResNet50(enable_lat=args.enable_lat, layerlist=args.layerlist, epsilon=args.epsilon, alpha=args.alpha, pro_num=args.pro_num, batch_size=args.batchsize, if_dropout=args.dropout) elif args.model == 'resnet18': cnn = ResNet18(enable_lat=args.enable_lat, layerlist=args.layerlist, epsilon=args.epsilon, alpha=args.alpha, pro_num=args.pro_num, batch_size=args.batchsize, if_dropout=args.dropout) elif args.model == 'vgg16': cnn = VGG16(enable_lat=args.enable_lat, layerlist=args.layerlist, epsilon=args.epsilon, alpha=args.alpha, pro_num=args.pro_num, batch_size=args.batchsize, if_dropout=args.dropout) elif args.model == 'vgg11': cnn = VGG11(enable_lat=args.enable_lat, layerlist=args.layerlist, epsilon=args.epsilon, alpha=args.alpha, pro_num=args.pro_num, batch_size=args.batchsize, if_dropout=args.dropout) elif args.model == 'vgg13': cnn = VGG13(enable_lat=args.enable_lat, layerlist=args.layerlist, epsilon=args.epsilon, alpha=args.alpha, pro_num=args.pro_num, batch_size=args.batchsize, if_dropout=args.dropout) elif args.model == 'vgg19': cnn = VGG19(enable_lat=args.enable_lat, layerlist=args.layerlist, epsilon=args.epsilon, alpha=args.alpha, pro_num=args.pro_num, batch_size=args.batchsize, if_dropout=args.dropout) elif args.model == 'densenet': cnn = DenseNet() cnn.cuda() if args.enable_lat: cnn.choose_layer() return cnn
def analogy(img_A, img_BP, config): # set basic param weights = config['weights'] sizes = config['sizes'] radius = config['radius'] layers = config['layers'] iters = config['iters'] lr = config['lr'] show_step = config['show_step'] # compute 5 feature maps model = VGG19() data_A, data_A_size = model.get_features(img_tensor=img_A, layers=layers) data_AP = copy.deepcopy(data_A) data_BP, data_B_size = model.get_features(img_tensor=img_BP, layers=layers) data_B = copy.deepcopy(data_BP) for idx, layer_size in enumerate(data_A_size): print("layer_{}_size:".format(idx), layer_size) for curr_layer in range(5): if curr_layer == 0: ann_AB = init_nnf(data_A_size[curr_layer][2:]) ann_BA = init_nnf(data_B_size[curr_layer][2:]) else: ann_AB = pmAB.upsample_nnf(data_A_size[curr_layer][2]) ann_BA = pmBA.upsample_nnf(data_B_size[curr_layer][2]) # blend feature Ndata_A, response_A = normalize(data_A[curr_layer]) Ndata_BP, response_BP = normalize(data_BP[curr_layer]) data_AP[curr_layer] = blend(response_A, data_A[curr_layer], data_AP[curr_layer], weights[curr_layer]) data_B[curr_layer] = blend(response_BP, data_BP[curr_layer], data_B[curr_layer], weights[curr_layer]) Ndata_AP, _ = normalize(data_AP[curr_layer]) Ndata_B, _ = normalize(data_B[curr_layer]) # NNF search print("propagate_for_{}".format(curr_layer)) pmAB = propagate(ann_AB, nd2np(Ndata_A), nd2np(Ndata_AP), nd2np(Ndata_B), nd2np(Ndata_BP), sizes[curr_layer], iters[curr_layer], radius[curr_layer]) pmBA = propagate(ann_BA, nd2np(Ndata_BP), nd2np(Ndata_B), nd2np(Ndata_AP), nd2np(Ndata_A), sizes[curr_layer], iters[curr_layer], radius[curr_layer]) if show_step: img_1 = pmAB.reconstruct_image(img_BP) img_2 = pmBA.reconstruct_image(img_A) output_img(post_process(img_1), post_process(img_2)) if curr_layer < 4: # using backpropagation to approximate feature next_layer = curr_layer + 2 data_AP_np = pmAB.reconstruct_image(nd2np(data_BP[next_layer])) data_B_np = pmBA.reconstruct_image(nd2np(data_A[next_layer])) target_BP_np = pmAB.reconstruct_image(nd2np(data_BP[curr_layer])) target_A_np = pmBA.reconstruct_image(nd2np(data_A[curr_layer])) print("deconvolution_for_{}".format(curr_layer)) data_AP[curr_layer + 1] = model.get_deconvoluted_feat( np2nd(target_BP_np), curr_layer, np2nd(data_AP_np), lr=lr[curr_layer], iters=3000) data_B[curr_layer + 1] = model.get_deconvoluted_feat( np2nd(target_A_np), curr_layer, np2nd(data_B_np), lr=lr[curr_layer], iters=3000) print("reconstruction image") img_AP = pmAB.reconstruct_avg(img_BP, 5) # size 5 is in paper img_B = pmBA.reconstruct_avg(img_A, 5) img_AP = post_process(img_AP) img_B = post_process(img_B) return img_AP, img_B
def analogy(img_A_L, img_BP_L, config): img_A_L, img_A_Lab = prepare_image(img_A_L) img_BP_L, img_BP_Lab = prepare_image(img_BP_L) start_time_0 = time.time() weights = config['weights'] sizes = config['sizes'] rangee = config['rangee'] deconv_iters = config['deconv_iters'] params = config['params'] lr = config['lr'] model_name = config['model'] device = torch.device("cuda" if USE_CUDA else "cpu") # preparing data img_A_tensor = torch.FloatTensor(img_A_L.transpose(2, 0, 1)) img_BP_tensor = torch.FloatTensor(img_BP_L.transpose(2, 0, 1)) img_A_tensor, img_BP_tensor = img_A_tensor.to(device), img_BP_tensor.to( device) img_A_tensor = img_A_tensor.unsqueeze(0) img_BP_tensor = img_BP_tensor.unsqueeze(0) # compute 5 feature maps if model_name == "VGG19": model = VGG19(device=device) elif model_name == "VGG19Gray": model = VGG19Gray(device=device) data_A, data_A_size = model.get_features(img_tensor=img_A_tensor.clone(), layers=params['layers']) data_AP = copy.deepcopy(data_A) data_BP, data_B_size = model.get_features(img_tensor=img_BP_tensor.clone(), layers=params['layers']) data_B = copy.deepcopy(data_BP) print("Features extracted!") # usually 5 layers n_layers = len(params['layers']) for curr_layer in range(n_layers): print("\n### current stage: %d - start ###" % (5 - curr_layer)) start_time_1 = time.time() if curr_layer == 0: ann_AB = pm.init_nnf(data_A_size[curr_layer][2:], data_B_size[curr_layer][2:]) ann_BA = pm.init_nnf(data_B_size[curr_layer][2:], data_A_size[curr_layer][2:]) else: ann_AB = pm.upSample_nnf(ann_AB, data_A_size[curr_layer][2:]) ann_BA = pm.upSample_nnf(ann_BA, data_B_size[curr_layer][2:]) # blend feature Ndata_A, response_A = normalize(data_A[curr_layer]) Ndata_BP, response_BP = normalize(data_BP[curr_layer]) data_AP[curr_layer] = blend(response_A, data_A[curr_layer], data_AP[curr_layer], weights[curr_layer]) data_B[curr_layer] = blend(response_BP, data_BP[curr_layer], data_B[curr_layer], weights[curr_layer]) Ndata_AP, _ = normalize(data_AP[curr_layer]) Ndata_B, _ = normalize(data_B[curr_layer]) # NNF search print("- NNF search for ann_AB") start_time_2 = time.time() ann_AB, _ = pm.propagate(ann_AB, ts2np(Ndata_A), ts2np(Ndata_AP), ts2np(Ndata_B), ts2np(Ndata_BP), sizes[curr_layer], params['propagate_iter'], rangee[curr_layer]) print("\tElapse: " + str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7]) print("- NNF search for ann_BA") start_time_2 = time.time() ann_BA, _ = pm.propagate(ann_BA, ts2np(Ndata_BP), ts2np(Ndata_B), ts2np(Ndata_AP), ts2np(Ndata_A), sizes[curr_layer], params['propagate_iter'], rangee[curr_layer]) print("\tElapse: " + str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7]) if curr_layer >= 4: print("### current stage: %d - end | " % (5 - curr_layer) + "Elapse: " + str(datetime.timedelta(seconds=time.time() - start_time_1))[:-7] + ' ###') break # using backpropagation to approximate feature next_layer = curr_layer + 2 ann_AB_upnnf2 = pm.upSample_nnf(ann_AB, data_A_size[next_layer][2:]) ann_BA_upnnf2 = pm.upSample_nnf(ann_BA, data_B_size[next_layer][2:]) data_AP_np = pm.avg_vote(ann_AB_upnnf2, ts2np(data_BP[next_layer]), sizes[next_layer], data_A_size[next_layer][2:], data_B_size[next_layer][2:]) data_B_np = pm.avg_vote(ann_BA_upnnf2, ts2np(data_A[next_layer]), sizes[next_layer], data_B_size[next_layer][2:], data_A_size[next_layer][2:]) data_AP[next_layer] = np2ts(data_AP_np, device) data_B[next_layer] = np2ts(data_B_np, device) target_BP_np = pm.avg_vote(ann_AB, ts2np(data_BP[curr_layer]), sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:]) target_A_np = pm.avg_vote(ann_BA, ts2np(data_A[curr_layer]), sizes[curr_layer], data_B_size[curr_layer][2:], data_A_size[curr_layer][2:]) target_BP = np2ts(target_BP_np, device) target_A = np2ts(target_A_np, device) print('- deconvolution for feat A\'') start_time_2 = time.time() data_AP[curr_layer + 1] = model.get_deconvoluted_feat( target_BP, curr_layer, data_AP[next_layer], lr=lr[curr_layer], iters=deconv_iters, display=False) print("\tElapse: " + str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7]) print('- deconvolution for feat B') start_time_2 = time.time() data_B[curr_layer + 1] = model.get_deconvoluted_feat( target_A, curr_layer, data_B[next_layer], lr=lr[curr_layer], iters=deconv_iters, display=False) print("\tElapse: " + str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7]) if USE_CUDA: # in case of data type inconsistency if data_B[curr_layer + 1].type() == torch.cuda.DoubleTensor: data_B[curr_layer + 1] = data_B[curr_layer + 1].type( torch.cuda.FloatTensor) data_AP[curr_layer + 1] = data_AP[curr_layer + 1].type( torch.cuda.FloatTensor) else: if data_B[curr_layer + 1].type() == torch.DoubleTensor: data_B[curr_layer + 1] = data_B[curr_layer + 1].type( torch.FloatTensor) data_AP[curr_layer + 1] = data_AP[curr_layer + 1].type( torch.FloatTensor) print("### current stage: %d - end | " % (5 - curr_layer) + "Elapse: " + str(datetime.timedelta(seconds=time.time() - start_time_1))[:-7] + ' ###') print('\n- reconstruct images A\' and B') img_AP_Lab = pm.reconstruct_avg(ann_AB, img_BP_Lab, sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:]) img_B_Lab = pm.reconstruct_avg(ann_BA, img_A_Lab, sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:]) img_AP_Lab = np.clip(img_AP_Lab, 0, 255).astype("uint8") img_B_Lab = np.clip(img_B_Lab, 0, 255).astype("uint8") # img_AP_L = cv2.split(img_AP_Lab)[0] # img_B_L = cv2.split(img_B_Lab)[0] img_AP_bgr = cv2.cvtColor(img_AP_Lab, cv2.COLOR_LAB2BGR) img_B_bgr = cv2.cvtColor(img_B_Lab, cv2.COLOR_LAB2BGR) return img_AP_bgr, img_B_bgr, str( datetime.timedelta(seconds=time.time() - start_time_0))[:-7]
def deep_image_analogy(A, BP, config): alphas = config['alpha'] nnf_patch_size = config['nnf_patch_size'] radii = config['radii'] params = config['params'] lr = config['lr'] # preparing data img_A_tensor = torch.FloatTensor(A.transpose(2, 0, 1)).cuda() img_BP_tensor = torch.FloatTensor(BP.transpose(2, 0, 1)).cuda() # fake a batch dimension img_A_tensor = img_A_tensor.unsqueeze(0) img_BP_tensor = img_BP_tensor.unsqueeze(0) # 4.1 Preprocessing Step model = VGG19() F_A, F_A_size = model.get_features(img_tensor=img_A_tensor.clone(), layers=params['layers']) F_BP, F_B_size = model.get_features(img_tensor=img_BP_tensor.clone(), layers=params['layers']) # Init AP&B 's feature maps with F_A&F_BP F_AP = copy.deepcopy(F_A) F_B = copy.deepcopy(F_BP) #Note that the feature_maps now is in the order of [5,4,3,2,1,input] for curr_layer in range(5): #ANN init step, coarsest layer is initialized randomly, #Other layers is initialized using upsample technique described in the paper if curr_layer == 0: ann_AB = init_nnf(F_A_size[curr_layer][2:], F_B_size[curr_layer][2:]) ann_BA = init_nnf(F_B_size[curr_layer][2:], F_A_size[curr_layer][2:]) else: ann_AB = upSample_nnf(ann_AB, F_A_size[curr_layer][2:]) ann_BA = upSample_nnf(ann_BA, F_B_size[curr_layer][2:]) # According to Equotion(2), we need to normalize F_A and F_BP # response denotes the M in Equotion(6) F_A_BAR, response_A = normalize(F_A[curr_layer]) F_BP_BAR, response_BP = normalize(F_BP[curr_layer]) # F_AP&F_B is reconstructed according to Equotion(4) # Note that we reuse the varibale F_AP here, # it denotes the RBprime as is stated in the Equotion(4) which is calculated # at the end of the previous iteration F_AP[curr_layer] = blend(response_A, F_A[curr_layer], F_AP[curr_layer], alphas[curr_layer]) F_B[curr_layer] = blend(response_BP, F_BP[curr_layer], F_B[curr_layer], alphas[curr_layer]) # Normalize F_AP&F_B as well F_AP_BAR, _ = normalize(F_AP[curr_layer]) F_B_BAR, _ = normalize(F_B[curr_layer]) # Run PatchMatch algorithm to get mapping AB and BA ann_AB, _ = propagate(ann_AB, ts2np(F_A_BAR), ts2np(F_AP_BAR), ts2np(F_B_BAR), ts2np(F_BP_BAR), nnf_patch_size[curr_layer], params['iter'], radii[curr_layer]) ann_BA, _ = propagate(ann_BA, ts2np(F_BP_BAR), ts2np(F_B_BAR), ts2np(F_AP_BAR), ts2np(F_A_BAR), nnf_patch_size[curr_layer], params['iter'], radii[curr_layer]) if curr_layer >= 4: break # The code below is used to initialize the F_AP&F_B in the next layer, # it generates the R_B' and R_A as is stated in Equotion(4) # R_B' is stored in F_AP, R_A is stored in F_B # using backpropagation to approximate feature # About why we add 2 here: # https://github.com/msracver/Deep-Image-Analogy/issues/30 next_layer = curr_layer + 2 ann_AB_upnnf2 = upSample_nnf(ann_AB, F_A_size[next_layer][2:]) ann_BA_upnnf2 = upSample_nnf(ann_BA, F_B_size[next_layer][2:]) F_AP_np = avg_vote(ann_AB_upnnf2, ts2np(F_BP[next_layer]), nnf_patch_size[next_layer], F_A_size[next_layer][2:], F_B_size[next_layer][2:]) F_B_np = avg_vote(ann_BA_upnnf2, ts2np(F_A[next_layer]), nnf_patch_size[next_layer], F_B_size[next_layer][2:], F_A_size[next_layer][2:]) # Initialize R_B' and R_A F_AP[next_layer] = np2ts(F_AP_np) F_B[next_layer] = np2ts(F_B_np) # Warp F_BP using ann_AB, Warp F_A using ann_BA target_BP_np = avg_vote(ann_AB, ts2np(F_BP[curr_layer]), nnf_patch_size[curr_layer], F_A_size[curr_layer][2:], F_B_size[curr_layer][2:]) target_A_np = avg_vote(ann_BA, ts2np(F_A[curr_layer]), nnf_patch_size[curr_layer], F_B_size[curr_layer][2:], F_A_size[curr_layer][2:]) target_BP = np2ts(target_BP_np) target_A = np2ts(target_A_np) #LBFGS algorithm to approximate R_B' and R_A F_AP[curr_layer + 1] = model.get_deconvoluted_feat( target_BP, curr_layer, F_AP[next_layer], lr=lr[curr_layer], blob_layers=params['layers']) F_B[curr_layer + 1] = model.get_deconvoluted_feat( target_A, curr_layer, F_B[next_layer], lr=lr[curr_layer], blob_layers=params['layers']) if type(F_B[curr_layer + 1]) == torch.DoubleTensor: F_B[curr_layer + 1] = F_B[curr_layer + 1].type(torch.FloatTensor) F_AP[curr_layer + 1] = F_AP[curr_layer + 1].type(torch.FloatTensor) elif type(F_B[curr_layer + 1]) == torch.cuda.DoubleTensor: F_B[curr_layer + 1] = F_B[curr_layer + 1].type( torch.cuda.FloatTensor) F_AP[curr_layer + 1] = F_AP[curr_layer + 1].type( torch.cuda.FloatTensor) # Obtain the output according to 4.5 img_AP = reconstruct_avg(ann_AB, BP, nnf_patch_size[curr_layer], F_A_size[curr_layer][2:], F_B_size[curr_layer][2:]) img_B = reconstruct_avg(ann_BA, A, nnf_patch_size[curr_layer], F_A_size[curr_layer][2:], F_B_size[curr_layer][2:]) img_AP = np.clip(img_AP / 255.0, 0, 1)[:, :, ::-1] img_B = np.clip(img_B / 255.0, 0, 1)[:, :, ::-1] return img_AP, img_B
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Read Image img = cv2.imread(mri_img) print(img.shape) assert (img.shape[2] == 3) # ************************************OPTION 1************************************ img_tensor = torch.FloatTensor(img.transpose(2, 0, 1)) img_tensor = img_tensor.to(device) img_tensor = img_tensor.unsqueeze(0) # compute 5 feature maps model = VGG19(device=device) # data, data_size = model.get_features(img_tensor=img_tensor.clone(), layers=[29]) data, data_size = model.get_features(img_tensor=img_tensor.clone(), layers=[34]) features, = copy.deepcopy(data[:-1]) features_size = data_size[:-1] print(features, features_size) print(torch.max(features), torch.min(features)) # ************************************************************************ # ************************************OPTION 1************************************ # img_tensor = torch.FloatTensor(img_mri.transpose(2, 0, 1)) # img_tensor = img_tensor.to(device) #