示例#1
0
def analogy(img_A, img_BP, config):
    start_time_0 = time.time()

    weights = config['weights']
    sizes = config['sizes']
    rangee = config['rangee']
    params = config['params']
    lr = config['lr']
    if config['use_cuda']:
        device = torch.device('cuda:0')
    else:
        raise NotImplementedError('cpu mode is not supported yet')

    # preparing data
    img_A_tensor = torch.FloatTensor(img_A.transpose(2, 0, 1))
    img_BP_tensor = torch.FloatTensor(img_BP.transpose(2, 0, 1))
    img_A_tensor, img_BP_tensor = img_A_tensor.to(device), img_BP_tensor.to(
        device)

    img_A_tensor = img_A_tensor.unsqueeze(0)
    img_BP_tensor = img_BP_tensor.unsqueeze(0)

    # compute 5 feature maps
    model = VGG19(device=device)
    data_A, data_A_size = model.get_features(img_tensor=img_A_tensor.clone(),
                                             layers=params['layers'])
    data_AP = copy.deepcopy(data_A)
    data_BP, data_B_size = model.get_features(img_tensor=img_BP_tensor.clone(),
                                              layers=params['layers'])
    data_B = copy.deepcopy(data_BP)
    print("Features extracted!")

    for curr_layer in range(5):
        print("\n### current stage: %d - start ###" % (5 - curr_layer))
        start_time_1 = time.time()

        if curr_layer == 0:
            ann_AB = init_nnf(data_A_size[curr_layer][2:],
                              data_B_size[curr_layer][2:])
            ann_BA = init_nnf(data_B_size[curr_layer][2:],
                              data_A_size[curr_layer][2:])
        else:
            ann_AB = upSample_nnf(ann_AB, data_A_size[curr_layer][2:])
            ann_BA = upSample_nnf(ann_BA, data_B_size[curr_layer][2:])

        # blend feature
        Ndata_A, response_A = normalize(data_A[curr_layer])
        Ndata_BP, response_BP = normalize(data_BP[curr_layer])

        data_AP[curr_layer] = blend(response_A, data_A[curr_layer],
                                    data_AP[curr_layer], weights[curr_layer])
        data_B[curr_layer] = blend(response_BP, data_BP[curr_layer],
                                   data_B[curr_layer], weights[curr_layer])

        Ndata_AP, _ = normalize(data_AP[curr_layer])
        Ndata_B, _ = normalize(data_B[curr_layer])

        # NNF search
        print("- NNF search for ann_AB")
        start_time_2 = time.time()
        ann_AB, _ = propagate(ann_AB, ts2np(Ndata_A), ts2np(Ndata_AP),
                              ts2np(Ndata_B), ts2np(Ndata_BP),
                              sizes[curr_layer], params['iter'],
                              rangee[curr_layer])
        print("\tElapse: " +
              str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])

        print("- NNF search for ann_BA")
        start_time_2 = time.time()
        ann_BA, _ = propagate(ann_BA, ts2np(Ndata_BP), ts2np(Ndata_B),
                              ts2np(Ndata_AP), ts2np(Ndata_A),
                              sizes[curr_layer], params['iter'],
                              rangee[curr_layer])
        print("\tElapse: " +
              str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])

        if curr_layer >= 4:
            print("### current stage: %d - end | " % (5 - curr_layer) +
                  "Elapse: " +
                  str(datetime.timedelta(seconds=time.time() -
                                         start_time_1))[:-7] + ' ###')
            break

        # using backpropagation to approximate feature
        next_layer = curr_layer + 2

        ann_AB_upnnf2 = upSample_nnf(ann_AB, data_A_size[next_layer][2:])
        ann_BA_upnnf2 = upSample_nnf(ann_BA, data_B_size[next_layer][2:])

        data_AP_np = avg_vote(ann_AB_upnnf2, ts2np(data_BP[next_layer]),
                              sizes[next_layer], data_A_size[next_layer][2:],
                              data_B_size[next_layer][2:])
        data_B_np = avg_vote(ann_BA_upnnf2, ts2np(data_A[next_layer]),
                             sizes[next_layer], data_B_size[next_layer][2:],
                             data_A_size[next_layer][2:])

        data_AP[next_layer] = np2ts(data_AP_np, device)
        data_B[next_layer] = np2ts(data_B_np, device)

        target_BP_np = avg_vote(ann_AB, ts2np(data_BP[curr_layer]),
                                sizes[curr_layer], data_A_size[curr_layer][2:],
                                data_B_size[curr_layer][2:])
        target_A_np = avg_vote(ann_BA, ts2np(data_A[curr_layer]),
                               sizes[curr_layer], data_B_size[curr_layer][2:],
                               data_A_size[curr_layer][2:])

        target_BP = np2ts(target_BP_np, device)
        target_A = np2ts(target_A_np, device)

        print('- deconvolution for feat A\'')
        start_time_2 = time.time()
        data_AP[curr_layer + 1] = model.get_deconvoluted_feat(
            target_BP,
            curr_layer,
            data_AP[next_layer],
            lr=lr[curr_layer],
            iters=400,
            display=False)
        print("\tElapse: " +
              str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])

        print('- deconvolution for feat B')
        start_time_2 = time.time()
        data_B[curr_layer + 1] = model.get_deconvoluted_feat(
            target_A,
            curr_layer,
            data_B[next_layer],
            lr=lr[curr_layer],
            iters=400,
            display=False)
        print("\tElapse: " +
              str(datetime.timedelta(seconds=time.time() - start_time_2))[:-7])

        # in case of data type inconsistency
        if data_B[curr_layer + 1].type() == torch.cuda.DoubleTensor:
            data_B[curr_layer + 1] = data_B[curr_layer + 1].type(
                torch.cuda.FloatTensor)
            data_AP[curr_layer + 1] = data_AP[curr_layer + 1].type(
                torch.cuda.FloatTensor)

        print("### current stage: %d - end | " % (5 - curr_layer) +
              "Elapse: " +
              str(datetime.timedelta(seconds=time.time() -
                                     start_time_1))[:-7] + ' ###')

    print('\n- reconstruct images A\' and B')
    img_AP = reconstruct_avg(ann_AB, img_BP, sizes[curr_layer],
                             data_A_size[curr_layer][2:],
                             data_B_size[curr_layer][2:])
    img_B = reconstruct_avg(ann_BA, img_A, sizes[curr_layer],
                            data_A_size[curr_layer][2:],
                            data_B_size[curr_layer][2:])

    img_AP = np.clip(img_AP, 0, 255)
    img_B = np.clip(img_B, 0, 255)

    return img_AP, img_B, str(
        datetime.timedelta(seconds=time.time() - start_time_0))[:-7]
示例#2
0
def analogy(img_A, img_BP, config):
    weights = config['weights']
    sizes = config['sizes']
    rangee = config['rangee']
    use_cuda = config['use_cuda']
    params = config['params']
    lr = config['lr']
    # assert use_cuda==True, "cpu version is not implemented yet. You can modify VGG19.py to make it support CPU if you like."

    # preparing data
    img_A_tensor = torch.FloatTensor(img_A.transpose(2, 0, 1))
    img_BP_tensor = torch.FloatTensor(img_BP.transpose(2, 0, 1))
    if use_cuda:
        img_A_tensor, img_BP_tensor = img_A_tensor.cuda(), img_BP_tensor.cuda()

    img_A_tensor = img_A_tensor.unsqueeze(0)
    img_BP_tensor = img_BP_tensor.unsqueeze(0)

    # compute 5 feature maps
    model = VGG19(use_cuda=use_cuda)
    data_A, data_A_size = model.get_features(
        img_tensor=img_A_tensor.clone(), layers=params['layers'])
    data_AP = copy.deepcopy(data_A)
    data_BP, data_B_size = model.get_features(
        img_tensor=img_BP_tensor.clone(), layers=params['layers'])
    data_B = copy.deepcopy(data_BP)

    for curr_layer in range(5):

        if curr_layer == 0:
            ann_AB = init_nnf(
                data_A_size[curr_layer][2:], data_B_size[curr_layer][2:])
            ann_BA = init_nnf(
                data_B_size[curr_layer][2:], data_A_size[curr_layer][2:])
        else:
            ann_AB = upSample_nnf(ann_AB, data_A_size[curr_layer][2:])
            ann_BA = upSample_nnf(ann_BA, data_B_size[curr_layer][2:])

        # blend feature
        Ndata_A, response_A = normalize(data_A[curr_layer])
        Ndata_BP, response_BP = normalize(data_BP[curr_layer])

        data_AP[curr_layer] = blend(
            response_A, data_A[curr_layer], data_AP[curr_layer], weights[curr_layer])
        data_B[curr_layer] = blend(
            response_BP, data_BP[curr_layer], data_B[curr_layer], weights[curr_layer])

        Ndata_AP, _ = normalize(data_AP[curr_layer])
        Ndata_B, _ = normalize(data_B[curr_layer])

        # NNF search
        ann_AB, _ = propagate(ann_AB, ts2np(Ndata_A), ts2np(Ndata_AP), ts2np(Ndata_B), ts2np(Ndata_BP), sizes[curr_layer],
                              params['iter'], rangee[curr_layer])
        ann_BA, _ = propagate(ann_BA, ts2np(Ndata_BP), ts2np(Ndata_B), ts2np(Ndata_AP), ts2np(Ndata_A), sizes[curr_layer],
                              params['iter'], rangee[curr_layer])

        if curr_layer >= 4:
            break

        # using backpropagation to approximate feature
        next_layer = curr_layer + 2

        ann_AB_upnnf2 = upSample_nnf(ann_AB, data_A_size[next_layer][2:])
        ann_BA_upnnf2 = upSample_nnf(ann_BA, data_B_size[next_layer][2:])

        data_AP_np = avg_vote(ann_AB_upnnf2, ts2np(data_BP[next_layer]), sizes[next_layer], data_A_size[next_layer][2:],
                              data_B_size[next_layer][2:])
        data_B_np = avg_vote(ann_BA_upnnf2, ts2np(data_A[next_layer]), sizes[next_layer], data_B_size[next_layer][2:],
                             data_A_size[next_layer][2:])

        data_AP[next_layer] = np2ts(data_AP_np)
        data_B[next_layer] = np2ts(data_B_np)

        target_BP_np = avg_vote(ann_AB, ts2np(data_BP[curr_layer]), sizes[curr_layer], data_A_size[curr_layer][2:],
                                data_B_size[curr_layer][2:])
        target_A_np = avg_vote(ann_BA, ts2np(data_A[curr_layer]), sizes[curr_layer], data_B_size[curr_layer][2:],
                               data_A_size[curr_layer][2:])

        target_BP = np2ts(target_BP_np)
        target_A = np2ts(target_A_np)

        data_AP[curr_layer+1] = model.get_deconvoluted_feat(target_BP, curr_layer, data_AP[next_layer], lr=lr[curr_layer],
                                                            iters=400, display=False)
        data_B[curr_layer+1] = model.get_deconvoluted_feat(target_A, curr_layer, data_B[next_layer], lr=lr[curr_layer],
                                                           iters=400, display=False)

        if type(data_B[curr_layer + 1]) == torch.DoubleTensor:
            data_B[curr_layer + 1] = data_B[curr_layer +
                                            1].type(torch.FloatTensor)
            data_AP[curr_layer + 1] = data_AP[curr_layer +
                                              1].type(torch.FloatTensor)
        elif type(data_B[curr_layer + 1]) == torch.cuda.DoubleTensor:
            data_B[curr_layer + 1] = data_B[curr_layer +
                                            1].type(torch.cuda.FloatTensor)
            data_AP[curr_layer + 1] = data_AP[curr_layer +
                                              1].type(torch.cuda.FloatTensor)

    img_AP = reconstruct_avg(
        ann_AB, img_BP, sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:])
    img_B = reconstruct_avg(
        ann_BA, img_A, sizes[curr_layer], data_A_size[curr_layer][2:], data_B_size[curr_layer][2:])

    img_AP = np.clip(img_AP/255.0, 0, 1)[:, :, ::-1]
    img_B = np.clip(img_B/255.0, 0, 1)[:, :, ::-1]

    return img_AP, img_B
def deep_image_analogy(A, BP, config):
    alphas = config['alpha']
    nnf_patch_size = config['nnf_patch_size']
    radii = config['radii']
    params = config['params']
    lr = config['lr']

    # preparing data
    img_A_tensor = torch.FloatTensor(A.transpose(2, 0, 1)).cuda()
    img_BP_tensor = torch.FloatTensor(BP.transpose(2, 0, 1)).cuda()
    # fake a batch dimension
    img_A_tensor = img_A_tensor.unsqueeze(0)
    img_BP_tensor = img_BP_tensor.unsqueeze(0)

    # 4.1 Preprocessing Step
    model = VGG19()
    F_A, F_A_size = model.get_features(img_tensor=img_A_tensor.clone(),
                                       layers=params['layers'])
    F_BP, F_B_size = model.get_features(img_tensor=img_BP_tensor.clone(),
                                        layers=params['layers'])

    # Init AP&B 's feature maps with F_A&F_BP
    F_AP = copy.deepcopy(F_A)
    F_B = copy.deepcopy(F_BP)

    #Note that the feature_maps now is in the order of [5,4,3,2,1,input]
    for curr_layer in range(5):

        #ANN init step, coarsest layer is initialized randomly,
        #Other layers is initialized using upsample technique described in the paper
        if curr_layer == 0:
            ann_AB = init_nnf(F_A_size[curr_layer][2:],
                              F_B_size[curr_layer][2:])
            ann_BA = init_nnf(F_B_size[curr_layer][2:],
                              F_A_size[curr_layer][2:])
        else:
            ann_AB = upSample_nnf(ann_AB, F_A_size[curr_layer][2:])
            ann_BA = upSample_nnf(ann_BA, F_B_size[curr_layer][2:])

        # According to Equotion(2), we need to normalize F_A and F_BP
        # response denotes the M in Equotion(6)
        F_A_BAR, response_A = normalize(F_A[curr_layer])
        F_BP_BAR, response_BP = normalize(F_BP[curr_layer])

        # F_AP&F_B is reconstructed according to Equotion(4)
        # Note that we reuse the varibale F_AP here,
        # it denotes the RBprime as is stated in the  Equotion(4) which is calculated
        # at the end of the previous iteration
        F_AP[curr_layer] = blend(response_A, F_A[curr_layer], F_AP[curr_layer],
                                 alphas[curr_layer])
        F_B[curr_layer] = blend(response_BP, F_BP[curr_layer], F_B[curr_layer],
                                alphas[curr_layer])

        # Normalize F_AP&F_B as well
        F_AP_BAR, _ = normalize(F_AP[curr_layer])
        F_B_BAR, _ = normalize(F_B[curr_layer])

        # Run PatchMatch algorithm to get mapping AB and BA
        ann_AB, _ = propagate(ann_AB, ts2np(F_A_BAR), ts2np(F_AP_BAR),
                              ts2np(F_B_BAR), ts2np(F_BP_BAR),
                              nnf_patch_size[curr_layer], params['iter'],
                              radii[curr_layer])
        ann_BA, _ = propagate(ann_BA, ts2np(F_BP_BAR), ts2np(F_B_BAR),
                              ts2np(F_AP_BAR), ts2np(F_A_BAR),
                              nnf_patch_size[curr_layer], params['iter'],
                              radii[curr_layer])

        if curr_layer >= 4:
            break

        # The code below is used to initialize the F_AP&F_B in the next layer,
        # it generates the R_B' and R_A as is stated in Equotion(4)
        # R_B' is stored in F_AP, R_A is stored in F_B

        # using backpropagation to approximate feature

        # About why we add 2 here:
        # https://github.com/msracver/Deep-Image-Analogy/issues/30
        next_layer = curr_layer + 2
        ann_AB_upnnf2 = upSample_nnf(ann_AB, F_A_size[next_layer][2:])
        ann_BA_upnnf2 = upSample_nnf(ann_BA, F_B_size[next_layer][2:])
        F_AP_np = avg_vote(ann_AB_upnnf2, ts2np(F_BP[next_layer]),
                           nnf_patch_size[next_layer],
                           F_A_size[next_layer][2:], F_B_size[next_layer][2:])
        F_B_np = avg_vote(ann_BA_upnnf2, ts2np(F_A[next_layer]),
                          nnf_patch_size[next_layer], F_B_size[next_layer][2:],
                          F_A_size[next_layer][2:])

        # Initialize  R_B' and R_A
        F_AP[next_layer] = np2ts(F_AP_np)
        F_B[next_layer] = np2ts(F_B_np)

        # Warp F_BP using ann_AB, Warp F_A using ann_BA
        target_BP_np = avg_vote(ann_AB, ts2np(F_BP[curr_layer]),
                                nnf_patch_size[curr_layer],
                                F_A_size[curr_layer][2:],
                                F_B_size[curr_layer][2:])
        target_A_np = avg_vote(ann_BA, ts2np(F_A[curr_layer]),
                               nnf_patch_size[curr_layer],
                               F_B_size[curr_layer][2:],
                               F_A_size[curr_layer][2:])

        target_BP = np2ts(target_BP_np)
        target_A = np2ts(target_A_np)

        #LBFGS algorithm to approximate R_B' and R_A
        F_AP[curr_layer + 1] = model.get_deconvoluted_feat(
            target_BP,
            curr_layer,
            F_AP[next_layer],
            lr=lr[curr_layer],
            blob_layers=params['layers'])
        F_B[curr_layer + 1] = model.get_deconvoluted_feat(
            target_A,
            curr_layer,
            F_B[next_layer],
            lr=lr[curr_layer],
            blob_layers=params['layers'])

        if type(F_B[curr_layer + 1]) == torch.DoubleTensor:
            F_B[curr_layer + 1] = F_B[curr_layer + 1].type(torch.FloatTensor)
            F_AP[curr_layer + 1] = F_AP[curr_layer + 1].type(torch.FloatTensor)
        elif type(F_B[curr_layer + 1]) == torch.cuda.DoubleTensor:
            F_B[curr_layer + 1] = F_B[curr_layer + 1].type(
                torch.cuda.FloatTensor)
            F_AP[curr_layer + 1] = F_AP[curr_layer + 1].type(
                torch.cuda.FloatTensor)

    # Obtain the output according to 4.5
    img_AP = reconstruct_avg(ann_AB, BP, nnf_patch_size[curr_layer],
                             F_A_size[curr_layer][2:],
                             F_B_size[curr_layer][2:])
    img_B = reconstruct_avg(ann_BA, A, nnf_patch_size[curr_layer],
                            F_A_size[curr_layer][2:], F_B_size[curr_layer][2:])

    img_AP = np.clip(img_AP / 255.0, 0, 1)[:, :, ::-1]
    img_B = np.clip(img_B / 255.0, 0, 1)[:, :, ::-1]
    return img_AP, img_B