Ejemplo n.º 1
0
    def __getitem__(self, index):
        """Return a data point and its metadata information.

        Parameters:
            index - - a random integer for data indexing

        Returns a dictionary that contains A and A_paths
            A(tensor) - - an image in one domain
            A_paths(str) - - the path of the image
        """
        A_path = self.A_paths[index]
        gtlx_path = os.path.splitext(A_path)[0] + '_lux.csv'
        print('img:', A_path)
        print('gxlx:', gtlx_path)
        srgb_img = Image.open(A_path).convert('RGB')
        gtlx_np = np.loadtxt(gtlx_path, delimiter=',')
        gtlx_pil = Image.fromarray(gtlx_np)
        srgb_img = self.transform(srgb_img)
        gt_SH = self.transform_lux(gtlx_pil)
        gt_SH = gt_SH / gt_SH.max()
        mask = torch.ones_like(gt_SH)

        gt_BA, brightest_20, gt_BP, gt_BC\
             = util.calc_brightest(
                 gt_SH, mask,
                 nr_tap=self.opt.bp_nr_tap,
                 nr_sigma=self.opt.bp_nr_sigma,
                 spread_tap=self.opt.bp_tap,
                 spread_sigma=self.opt.bp_sigma
                 )

        gt_SH = normalize(grayscale=True)(gt_SH)
        mask = normalize(grayscale=True)(mask)
        gt_BA = normalize(grayscale=True)(gt_BA)
        gt_BP = normalize(grayscale=True)(gt_BP)
        gt_BC = torch.Tensor(list(gt_BC))

        srgb_img = torch.unsqueeze(srgb_img, 0)  # [1, 3, 256, 256]
        gt_AL = torch.zeros_like(srgb_img)
        gt_SH = torch.unsqueeze(gt_SH, 0)
        mask = torch.unsqueeze(mask, 0)
        gt_BA = torch.unsqueeze(gt_BA, 0)
        gt_BP = torch.unsqueeze(gt_BP, 0)

        return {
            'A': srgb_img,
            'gt_AL': gt_AL,
            'gt_SH': gt_SH,
            'mask': mask,
            'gt_BA': gt_BA,
            'gt_BP': gt_BP,
            'gt_BC': gt_BC,
            'A_paths': A_path
        }
Ejemplo n.º 2
0
    def __getitem__(self, index):        
        ### input A (label maps)
        A_path = self.A_paths[index]              
        A = Image.open(A_path)        
        params = get_params(self.opt, A.size)
        transform_A = get_transform(self.opt, params)
        A_tensor = transform_A(A.convert('RGB'))

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]   
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params)      
            B_tensor = transform_B(B)

        ### if using instance maps        
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]            
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))                            

        input_dict = {'label': A_tensor, 'inst': inst_tensor, 'image': B_tensor, 
                      'feat': feat_tensor, 'path': A_path}

        return input_dict
Ejemplo n.º 3
0
    def __getitem__(self, index):
        image_tensor = ds_tensor = inst_tensor = feat_tensor = 0
        ### real images
        #if self.opt.isTrain:
        image_path = self.image_paths[index]
        image = Image.open(image_path).convert('RGB')
        params = get_params(self.opt, image.size)
        transform_image = get_transform(self.opt, params)
        image_tensor = transform_image(image)

        label_tensor = 0
        transform_label = 0
        if not self.opt.no_seg:
            ### label maps
            label_path = self.label_paths[index]
            label = Image.open(label_path)

            if self.opt.label_nc == 0:
                transform_label = get_transform(self.opt, params)
                label_tensor = transform_label(label.convert('RGB'))
            else:
                transform_label = get_transform(self.opt,
                                                params,
                                                method=Image.NEAREST,
                                                normalize=False)
                label_tensor = transform_label(label) * 255.0

        image2 = Image.fromarray(util.tensor2im(image_tensor))

        ds = 0
        if self.opt.comp_type == 'ds':
            ds = image2.resize((image2.size[0] // self.opt.alpha,
                                image2.size[1] // self.opt.alpha),
                               Image.LANCZOS)
            ds = ds.resize((image2.size[0], image2.size[1]), Image.ANTIALIAS)
            ds_tensor = transform_image(ds)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_label(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_label(feat))

        input_dict = {
            'label': label_tensor,
            'inst': inst_tensor,
            'image': image_tensor,
            'ds': ds_tensor,
            'feat': feat_tensor,
            'path': image_path
        }

        return input_dict
Ejemplo n.º 4
0
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path).convert('RGB')
        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            # if self.opt.isTrain:
            # A = trans.rotate(A,angle=params['degree'],fill=255)
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('L'))

            # BP_path = self.BP_paths[index]
            # BP = torch.from_numpy(np.load(BP_path))
            # BP = F.interpolate(BP.unsqueeze(0), size=([self.opt.loadSize, self.opt.loadSize]), mode='nearest').squeeze(0)  ### resize
            # if self.opt.isTrain:
            #     BP = BP[:, params['crop_pos'][1]:params['crop_pos'][1] + self.opt.fineSize,
            #          params['crop_pos'][0]:params['crop_pos'][0] + self.opt.fineSize]  ###crop
            #     if params['flip']:
            #         BP = par_flip(BP, 2)
            #
            # A_tensor = torch.cat([A_tensor, BP], 0)

        else:
            transform_A = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=False)
            A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            # B = trans.rotate(B, angle=params['degree'], fill=255)
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B.convert('L'))

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': A_path
        }

        return input_dict
    def __getitem__(self, index):
        # To avoid using first and last image of the video
        if index == 0:
            index += 1

        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path)
        params = get_params(self.opt, A.size)
        transform_A = get_transform(self.opt, params, method=Image.NEAREST,
                                    normalize=False)
        A_tensor = transform_A(A) * 255.0

        ### input A_previous (previous label map)
        A_previous_path = self.A_paths[index-1]

        A_previous = Image.open(A_previous_path)
        params = get_params(self.opt, A_previous.size)
       
        transform_A_previous = get_transform(self.opt, params, method=Image.NEAREST,
                                    normalize=False)
        A_tensor_previous = transform_A_previous(A_previous) * 255.0

        B_tensor_previous = B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain:
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B)

        ### input B_previous (previous real images)
        if self.opt.isTrain:
            B_previous_path = self.B_paths[index-1]
            B_previous = Image.open(B_previous_path).convert('RGB')
            transform_B_previous = get_transform(self.opt, params)
            B_tensor_previous = transform_B_previous(B_previous)


        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        input_dict = {'label': A_tensor, 'previous_label': A_tensor_previous,
                      'inst': inst_tensor, 'image': B_tensor,
                      'previous_image': B_tensor_previous,
                      'feat': feat_tensor, 'path': A_path}

        return input_dict
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path)
        A = A.resize((640, 480), Image.ANTIALIAS)
        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('RGB'))
        else:
            transform_A = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=False)
            A_tensor = transform_A(A) * 255.0

        # B_tensor = inst_tensor = feat_tensor = 0
        B_tensor = C_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            B = B.resize((640, 480), Image.ANTIALIAS)
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B)

            C_path = self.C_paths[index]
            C = Image.open(C_path).convert('RGB')
            C = C.resize((640, 480), Image.ANTIALIAS)
            transform_C = get_transform(self.opt, params)
            C_tensor = transform_C(C)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        # input_dict = {'label': A_tensor, 'inst': inst_tensor, 'image': B_tensor,
        #            'feat': feat_tensor, 'path': A_path}
        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'fixpts': C_tensor,
            'feat': feat_tensor,
            'path': A_path
        }

        return input_dict
    def __getitem__(self, index):
        ### input A (label maps)
        if index > self.dataset_size - self.clip_length:
            index = 0  # it's a rare chance and won't be effecting training dynamics

        A_path = self.A_paths[index:index + self.clip_length]
        A = [Image.open(path) for path in A_path]
        params = get_params(self.opt, A[0].size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = [transform_A(item.convert('RGB')) for item in A]
            A_tensor = torch.stack(A_tensor, dim=0)
        else:
            transform_A = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=False)
            A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain:
            B_path = self.B_paths[index:index + self.clip_length]
            B = [Image.open(path).convert('RGB') for path in B_path]
            transform_B = get_transform(self.opt, params)
            B_tensor = [transform_B(item) for item in B]
            B_tensor = torch.stack(B_tensor, dim=0)
        else:  # only retain first frame for testing
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B)

        ### if using instance maps (which is never supposed to)
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index:index + self.clip_length]
            inst = [Image.open(path) for path in inst_path]
            inst_tensor = [transform_A(item) for item in inst]
            inst_tensor = torch.stack(inst_tensor, dim=0)

            if self.opt.load_features:
                feat_path = self.feat_paths[index:index + self.clip_length]
                feat = [Image.open(path).convert('RGB') for path in feat_path]
                norm = normalize()
                feat_tensor = [norm(transform_A(item)) for item in feat]
                feat_tensor = torch.stack(feat_tensor, dim=0)

        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': A_path
        }

        return input_dict
Ejemplo n.º 8
0
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path)
        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert("RGB"))
        else:
            transform_A = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=False)
            # transforms.ToTenosrは[0, 255]の値を[0, 1]にrescaleするた,255をかける
            A_tensor = transform_A(A) * 255.0
            print("label_map====================")
            print(A_tensor.shape)
            print(torch.unique(A_tensor))

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert("RGB")
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B)
            print("real_map=======================")
            print(B_tensor.shape)
            print(torch.unique(B_tensor))

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)

            inst_tensor = transform_A(inst)
            print("inst_tensor:", torch.unique(inst_tenosr))

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert("RGB")
                norm = normalize()
                feat_tensor = norm(transform_A(feat))
            print("inst_map=======================")
            print(inst_tensor.shape)
            print(torch.unique(inst_tensor))

        input_dict = {
            "label": A_tensor,
            "inst": inst_tensor,
            "image": B_tensor,
            "feat": feat_tensor,
            "path": A_path,
        }

        return input_dict
Ejemplo n.º 9
0
    def __getitem__(self, index):

        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path)
        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('RGB'))
        else:
            transform_A = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=False)
            A_tensor = transform_A(A) * 255.0  # resize to 1024 * 512

        B_tensor = inst_tensor = feat_tensor = labelTrain = 0
        ### input B (real images)
        if self.opt.isTrain:
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B)

        ### labelTrain semantic maps
        if self.opt.isTrain:
            labelTrain_path = self.labelTrain_paths[index]
            labelTrain = Image.open(labelTrain_path).convert('P')
            labelTrain = MyCoTransform(labelTrain)
            # print(np.array(labelTrain_tensor).shape) # (1, 512, 1024)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': A_path,
            'labelTrain': labelTrain
        }

        return input_dict
Ejemplo n.º 10
0
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path)
        params = get_params(self.opt, A.size)
        '''if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('RGB'))
        #else:'''
        transform_A = get_transform(self.opt,
                                    params,
                                    method=Image.NEAREST,
                                    normalize=False)
        A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B)

        C_path = "/scratch/train_numbered/train_B/" + self.find_canonical_image(
            B_path)
        C = Image.open(C_path).convert('RGB')
        transform_C = get_transform(self.opt, params)
        C_tensor = transform_C(C)

        D_Tensor = torch.cat((A_tensor, C_tensor), 0)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        input_dict = {
            'label': D_Tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': A_path
        }

        return input_dict
Ejemplo n.º 11
0
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path)

        file_name = os.path.split(A_path)
        file_name = os.path.splitext(file_name)[0]
        add_label = int(file_name.split('_')[-1])
        add_label_tensor = torch.tensor([add_label], dtype=torch.long)

        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('RGB'))
        else:
            transform_A = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=False)
            A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        # if self.opt.isTrain or self.opt.use_encoded_image:
        B_path = self.B_paths[index]
        B = Image.open(B_path).convert('RGB')
        transform_B = get_transform(self.opt, params)
        B_tensor = transform_B(B)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': A_path,
            'add_label': add_label_tensor
        }

        return input_dict
Ejemplo n.º 12
0
def getitem(A_path, B_path, inst_path, feat_path):
    ### input A (label maps)
    A = Image.open(A_path)
    params = get_params(opt, A.size)
    if opt.label_nc == 0:
        transform_A = get_transform(opt, params)
        A_tensor = transform_A(A.convert('RGB'))
    else:
        transform_A = get_transform(opt,
                                    params,
                                    method=Image.NEAREST,
                                    normalize=False)
        A_tensor = transform_A(A) * 255.0

    B_tensor = inst_tensor = feat_tensor = 0
    ### input B (real images)
    B = Image.open(B_path).convert('RGB')
    transform_B = get_transform(opt, params)
    B_tensor = transform_B(B)

    ### if using instance maps
    inst = Image.open(inst_path)
    inst_tensor = transform_A(inst)

    #get feat
    netE = networks.define_G(opt.output_nc,
                             opt.feat_num,
                             opt.nef,
                             'encoder',
                             opt.n_downsample_E,
                             norm=opt.norm,
                             gpu_ids=opt.gpu_ids)
    feat_map = netE.forward(
        Variable(B_tensor[np.newaxis, :].cuda(), volatile=True),
        inst_tensor[np.newaxis, :].cuda())
    feat_map = nn.Upsample(scale_factor=2, mode='nearest')(feat_map)
    image_numpy = util.tensor2im(feat_map.data[0])
    util.save_image(image_numpy, feat_path)

    feat = Image.open(feat_path).convert('RGB')
    norm = normalize()
    feat_tensor = norm(transform_A(feat))
    input_dict = {
        'label': A_tensor,
        'inst': inst_tensor,
        'image': B_tensor,
        'feat': feat_tensor,
        'path': A_path
    }

    return get_features(input_dict['inst'], input_dict['feat'])
Ejemplo n.º 13
0
    def __getitem__(self, index):
        seed = time.time()
        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path)
        params = get_params(self.opt, A.size)
        p = random.random()
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params, p=p)
            random.seed(seed)  # apply this seed to img tranfsorms
            torch.manual_seed(seed)  # needed for torchvision 0.7
            A_tensor = transform_A(A.convert('RGB'))
        else:
            transform_A = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=False,
                                        p=p)
            A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params, p=p)
            random.seed(seed)  # apply this seed to img tranfsorms
            torch.manual_seed(seed)  # needed for torchvision 0.7
            B_tensor = transform_B(B)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': A_path
        }

        return input_dict
Ejemplo n.º 14
0
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A_tensor = torch.load(A_path).permute((2, 0, 1))
        # A = Image.open(A_path)
        # params = get_params(self.opt, A.size)
        # if self.opt.label_nc == 0:
        #     transform_A = get_transform(self.opt, params)
        #     A_tensor = transform_A(A.convert('RGB'))
        # else:
        #     transform_A = get_transform(self.opt, params, method=Image.NEAREST, normalize=False)
        #     A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain:
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            # transform_B = get_transform(self.opt, params)
            # B_tensor = transform_B(B)
            B = np.array(B, dtype=float) / 255.
            B_tensor = torch.tensor(B)[:, :, :3].permute((2, 0, 1)).float()

            # fig = plt.figure(1)
            # ax = fig.add_subplot(111)
            # ax.imshow(B_tensor[:,:1024,:].permute((1,2,0)))
            # plt.show()

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': A_path
        }

        return input_dict
Ejemplo n.º 15
0
    def __getitem__(self, index):
        geometry_tensor = 0
        norm = normalize()
        #### Before get_transform: convert to png
        A_name = self.names_label[index]

        ### input A (label maps)
        A_path = self.A_paths[index]
        exr = cv2.imread(A_path, -1)
        img = np.clip(exr, 0, 1)
        img = img.astype(np.float32)
        A_tensor = norm(toTensor(np.transpose(img, (2, 0, 1))))

        if not self.opt.no_geometry:
            geo_path = os.path.join(
                self.dir_geo,
                re.sub(r'diffuse_albedo(_\d\d)*', 'pointcloud', A_name))
            geometry = cv2.imread(geo_path, cv2.IMREAD_UNCHANGED)
            geo = rescale(np.copy(geometry), type=0)
            geometry_tensor = norm(geo)

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = os.path.join(
                self.dir_specular,
                re.sub(r'diffuse_albedo(_\d\d)*', 'specular_unlit', A_name))
            exr = cv2.imread(B_path, cv2.IMREAD_UNCHANGED)
            specular = rescale(np.copy(exr), type=1)
            B_tensor = norm(specular)

        if not self.opt.isTrain and self.opt.loadGT:
            B_path = os.path.join(
                self.dir_specular,
                re.sub(r'diffuse_albedo(_\d\d)*', 'specular_unlit', A_name))
            exr = cv2.imread(B_path, cv2.IMREAD_UNCHANGED)
            specular = rescale(np.copy(exr), type=1)
            B_tensor = norm(specular)

        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': A_path,
            'geometry': geometry_tensor
        }

        return input_dict
Ejemplo n.º 16
0
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path)
        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('RGB'))
        else:
            transform_A = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=False)
            A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        # <<<<<<< HEAD
        #         if self.opt.isTrain:
        #             B_path = self.B_paths[index]
        # =======
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]
            # >>>>>>> 5a2c87201c5957e2bf51d79b8acddb9cc1920b26
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': A_path
        }

        return input_dict
Ejemplo n.º 17
0
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path)
        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('L'))
        else:
            transform_A = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=False)
            A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain:
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=True)
            B_tensor = transform_B(B)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))
        w, h = A.size
        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': A_path,
            'size': [w, h]
        }

        return input_dict
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path)
        params = get_params(self.opt, A.size)
        '''if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('RGB'))
        #else:'''
        transform_A = get_transform(self.opt, params, method=Image.NEAREST, normalize=False)
        A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain:
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B)

        import glob
        canon_img_list = []
        concat_img_list = []
        for filename in glob.glob('/data/jl5/canon_frames/*.png'):
            C = Image.open(filename).convert('RGB')
            transform_C = get_transform(self.opt, params)
            C_tensor = transform_C(C)
            canon_img_list.append(C_tensor)
            D_Tensor = torch.cat((A_tensor, C_tensor), 0)
            concat_img_list.append(D_Tensor)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        input_dict = {'label': concat_img_list, 'inst': inst_tensor, 'image': B_tensor,
                      'feat': feat_tensor, 'path': A_path, 'canon': canon_img_list}

        return input_dict
Ejemplo n.º 19
0
    def __getitem__(self, index):
        ### label maps
        label_path = self.label_paths[index]
        #label = Image.open(label_path)
        label = numpy.load(label_path)
        label_size = (label.shape[0], label.shape[1])
        params = get_params(self.opt, label_size)
        if self.opt.label_nc == 0:
            transform_label = get_transform(self.opt, params)
            #label_tensor = transform_label(label.convert('RGB'))
            label_tensor = transform_label(label)
        else:
            transform_label = get_transform(self.opt,
                                            params,
                                            method=Image.NEAREST,
                                            normalize=False)
            label_tensor = transform_label(label) * 255.0

        image_tensor = inst_tensor = feat_tensor = 0
        ### real images
        if self.opt.isTrain:
            image_path = self.image_paths[index]
            image = Image.open(image_path).convert('RGB')
            transform_image = get_transform(self.opt, params)
            image_tensor = transform_image(image)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_label(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_label(feat))

        input_dict = {
            'label': label_tensor,
            'inst': inst_tensor,
            'image': image_tensor,
            'feat': feat_tensor,
            'path': label_path
        }

        return input_dict
Ejemplo n.º 20
0
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A_tensor = np.load(A_path)["arr_0"][0].transpose(2, 0, 1)
        #A_tensor=self.trans(A_tensor)

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain:
            B_path = self.B_paths[index]
            B = Image.open(B_path)
            B = B.convert("RGB")
            print(B.getextrema())
            """
            transform_B = get_transform(self.opt, params)      
            B_tensor = transform_B(B)
            """
            B = self.trans(B)
            print(B.max())
            B = self.normalize(B)
            print(B.max())
            #B=self.trans(self.normalize(B))
            B_tensor = B

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': A_path
        }

        return input_dict
Ejemplo n.º 21
0
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path)
        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('RGB'))
        else:
            transform_A = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=False)
            A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst).int(
            )  # sometimes inst_tensor is float and it crashes the dataloader

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': A_path
        }

        return input_dict
Ejemplo n.º 22
0
    def __getitem__(self, index):   # The specific operations in getitem are the key operations of this class
 ### input A (label maps) # Read label map      
        ### input A (label maps)
        A_path = self.A_paths[index]   # Get image path
         # A = Image.open(self.dir_A +'/' + A_path) # First read an image

        A = Image.open(A_path)        
        params = get_params(self.opt, A.size) # According to the input opt ​​and size, return random parameters
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            tmp = A.convert('RGB')
            A_tensor = transform_A(tmp)
        else:
            transform_A = get_transform(self.opt, params, method=Image.NEAREST, normalize=False)
            A_tensor = transform_A(A) * 255.0 # To preprocess the data, after to_tensor operation, multiply by 255


        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images) ### input B (real images) # Then read in the real image B
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]  # B = Image.open(self.dir_B + '/' + B_path).convert('RGB') 
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params)      
            B_tensor = transform_B(B)

        ### if using instance maps    ### if using instance maps # Then read in the instance, and then it will be processed into an edge map, which is consistent with the description in the paper.
      
        if not self.opt.no_instance: # no_instance default value is true
            inst_path = self.inst_paths[index] # inst = Image.open(self.dir_inst + '/' + inst_path)
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst) # Same as semantic processing 0-1

            if self.opt.load_features: # Note that the role of self.opt.load_features is to read the pre-calculated features of each category. There are 10 categories in the paper, which are formed by clustering. But the default is not implemented. I personally read the thesis and knew nothing about this part. I will study it later if there is demand.

                feat_path = self.feat_paths[index]            
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))                            

        input_dict = {'label': A_tensor, 'inst': inst_tensor, 'image': B_tensor, 
                      'feat': feat_tensor, 'path': A_path}

        return input_dict # Return a dictionary recording the above read and processed data set.
Ejemplo n.º 23
0
    def __getitem__(self, index):
        ### input A (label maps)
        edges_path, edges_im = None, None
        A_path = self.A_paths[index]
        A = Image.open(A_path)
        params = get_params(self.opt, A.size, A)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_img = A.convert('RGB')
            A_tensor = transform_A(A_img)
            if self.opt.apply_binary_threshold == 1:
                ones = torch.ones_like(A_tensor)
                minus_ones = torch.ones_like(A_tensor)*-1
                A_tensor = torch.where(A_tensor>=self.opt.edge_threshold, ones,minus_ones)
            
        else:
            transform_A = get_transform(self.opt, params, method=Image.NEAREST, normalize=False)
            A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]   
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B)

        ### if using instance maps        
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]            
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))                            

        input_dict = {'label': A_tensor, 'inst': inst_tensor, 'image': B_tensor, 
                      'feat': feat_tensor, 'path': A_path}

        return input_dict
Ejemplo n.º 24
0
    def __getitem__(self, index):        
        ### input A (label maps)
        A_path = self.A_paths[0]              
        A = Image.open(A_path)        
        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('RGB'))
        else:
            transform_A = get_transform(self.opt, params, method=Image.NEAREST, normalize=False)
            A_tensor = transform_A(A) * 255.0

        #A_tensor = torch.ones(A_tensor.shape)

        Lighting_path = self.LIGHTING_paths[index]
        Lighting_image = Image.open(Lighting_path)  
        Lighting_image = cast_PIL_to_tensor(transforms.Resize([512, 512])(Lighting_image))

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]   
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params)      
            B_tensor = transform_B(B)

        ### if using instance maps        
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]            
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))                            

        input_dict = {'label': A_tensor, 'lighting': Lighting_image, 'inst': inst_tensor, 'image': B_tensor, 
                      'feat': feat_tensor, 'path': Lighting_path}

        return input_dict
Ejemplo n.º 25
0
    def __getitem__(self, index):        
        ### input A (label maps)
        A_path = self.A_paths[index]              
        A = Image.open(A_path)        
        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('RGB'))
        else:
            transform_A = get_transform(self.opt, params, method=Image.NEAREST, normalize=False)
            A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain:
            B_path = self.B_paths[index]   
            B = Image.open(B_path).convert('RGB')
            transform_B = get_transform(self.opt, params)      
            B_tensor = transform_B(B)

        ### if using instance maps        
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]            
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))                            

        input_dict = {'label': A_tensor, 'inst': inst_tensor, 'image': B_tensor, 
                      'feat': feat_tensor, 'path': A_path}

        return input_dict
Ejemplo n.º 26
0
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path)
        A = A.resize((640, 512), Image.BILINEAR)  # for SALICON only
        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('RGB'))
        else:
            transform_A = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=False)
            A_tensor = transform_A(A) * 255.0

        # B_tensor = inst_tensor = feat_tensor = 0
        B_tensor = C_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            # B = B.resize((80, 60), Image.BILINEAR) # for SALICON only
            B = B.resize(
                (160, 128), Image.BILINEAR
            )  # for SALICON only , the small size should be 64 so that the smallest dimention of feature is even number, for better feature alignment
            # B = B.resize((320, 256), Image.BILINEAR) # for SALICON only , the small size should be 64 so that the smallest dimention of feature is even number, for better feature alignment
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B)

            C_path = self.C_paths[index]
            C = Image.open(C_path).convert('RGB')
            # C = C.resize((80, 60), Image.BILINEAR) # for SALICON only
            C = C.resize((160, 128), Image.BILINEAR)  # for SALICON only
            # C = C.resize((320, 256), Image.BILINEAR) # for SALICON only
            transform_C = get_transform(self.opt, params)
            C_tensor = transform_C(C)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        # print("A_tensor :", A_tensor.size())
        # print("B_tensor :", B_tensor.size())
        # print("C_tensor :", C_tensor.size())

        # input_dict = {'label': A_tensor, 'inst': inst_tensor, 'image': B_tensor,
        #            'feat': feat_tensor, 'path': A_path}
        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'fixpts': C_tensor,
            'feat': feat_tensor,
            'path': A_path
        }

        return input_dict
Ejemplo n.º 27
0
    def __getitem__(self, index):
        ### input A (label maps)
        lpath = self.label_paths[index]
        A_tensor_0 = torch.load(lpath).permute((2, 0, 1)).float()

        idx_ = lpath.split('/')[-1][:12]
        spath = self.opt.input_image_root + '%s_synthesized_image.jpg' % idx_
        A = Image.open(spath).convert('RGB')
        A = np.array(A, dtype=float) / 255.
        A = A[:, :, :3]
        idx = lpath.split('/')[-1].split('.')[0]

        minx, maxx, miny, maxy = list(self.crop_coor[int(idx), :])
        A = A[minx:maxx + 1, miny:maxy + 1, :]
        A = cv2.resize(A, (128, 128))
        A_tensor_1 = torch.tensor(A).permute((2, 0, 1)).float()
        A_tensor = torch.cat((A_tensor_0, A_tensor_1), dim=0)

        B_tensor = inst_tensor = feat_tensor = 0

        lidx = lpath.split('/')[-1][:12]
        sidx = spath.split('/')[-1][:12]
        ### input B (real images)
        if self.opt.isTrain:
            B_path = self.rimage_paths[index]
            B = Image.open(B_path).convert('RGB')
            B = np.array(B, dtype=float) / 255.
            B_tensor = torch.tensor(B)[:, :, :3].permute((2, 0, 1)).float()

            # fig = plt.figure(1)
            # ax = fig.add_subplot(111)
            # ax.imshow(B_tensor[:,:1024,:].permute((1,2,0)))
            # plt.show()
            ridx = B_path.split('/')[-1][:12]
            assert lidx == ridx, "Wrong match"

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        # print(lpath, spath, B_path)
        # print(lidx, sidx )
        assert lidx == sidx, "Wrong match"
        # fig = plt.figure(1)
        # ax = fig.add_subplot(111)
        # ax.imshow(A)
        # plt.show()

        input_dict = {
            'label': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'feat': feat_tensor,
            'path': lpath
        }

        return input_dict
Ejemplo n.º 28
0
    def __getitem__(self, index):        
        ### input A (label maps)s
        A_path = self.A_paths[index]              
        A = Image.open(A_path)

        if DEBUG:
            print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
            arr = np.array(A)
            arr2txt(arr, 'src_A.txt')
            print("A.size: {}, arr.shape: {}".format(A.size, arr.shape))
            print("min,max,mean: ", arr.min(), arr.max(), arr.mean())
            tmp_A = A.resize((64, 32), Image.NEAREST)
            tmp_arr = np.array(tmp_A)
            arr2txt(tmp_arr, 'resized_A.txt')
            print("tmp_A.size: {}, tmp_arr.shape: {}".format(tmp_A.size, tmp_arr.shape))
            print("min,max,mean: ", tmp_arr.min(), tmp_arr.max(), tmp_arr.mean())
            print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
            # cv2.imshow('show', tmp_arr)
            # cv2.waitKey()
            # cv2.waitKey()

        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A.convert('RGB'))
        else:
            transform_A = get_transform(self.opt, params, method=Image.NEAREST, normalize=False)
            A_tensor = transform_A(A) * 255.0

        B_tensor = inst_tensor = feat_tensor = 0
        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:
            B_path = self.B_paths[index]   
            B = Image.open(B_path).convert('RGB')

            if DEBUG:
                print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
                arr = np.array(B)
                arr2txt(cv2.cvtColor(arr, cv2.COLOR_RGB2GRAY), 'src_B.txt')
                print("B.size: {}, arr.shape: {}".format(B.size, arr.shape))
                print("min,max,mean: ", arr.min(), arr.max(), arr.mean())
                tmp_B = B.resize((64, 32), Image.NEAREST)
                tmp_arr = np.array(tmp_B)
                arr2txt(cv2.cvtColor(tmp_arr, cv2.COLOR_RGB2GRAY), 'resized_B.txt')
                print("tmp_B.size: {}, tmp_arr.shape: {}".format(tmp_B.size, tmp_arr.shape))
                print("min,max,mean: ", tmp_arr.min(), tmp_arr.max(), tmp_arr.mean())
                print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
                # cv2.imshow('show', tmp_arr)
                # cv2.waitKey()

            transform_B = get_transform(self.opt, params)      
            B_tensor = transform_B(B)

        ### if using instance maps        
        if not self.opt.no_instance:  # TODO: cityscapes 数据集的 train_inst 图片是使用 16位格式存的png图,Image、OpenCV 无法读这类图,可使用png读16位图
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)

            if DEBUG:
                print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
                arr = np.array(inst)
                arr2txt(arr, 'src_inst.txt')
                print("inst.size: {}, arr.shape: {}".format(inst.size, arr.shape))
                print("min,max,mean: ", arr.min(), arr.max(), arr.mean())
                tmp_inst = inst.resize((64, 32), Image.NEAREST)
                tmp_arr = np.array(tmp_inst)
                arr2txt(tmp_arr, 'resized_inst.txt')
                print("tmp_inst.size: {}, tmp_arr.shape: {}".format(tmp_inst.size, tmp_arr.shape))
                print("min,max,mean: ", tmp_arr.min(), tmp_arr.max(), tmp_arr.mean())
                print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
                # cv2.imshow('show', tmp_arr)
                # cv2.waitKey()

            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]            
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))                            

        input_dict = {'label': A_tensor, 'inst': inst_tensor, 'image': B_tensor, 
                      'feat': feat_tensor, 'path': A_path}

        return input_dict
    def __getitem__(self, index):
        """Return a data point and its metadata information.

        Parameters:
            index (int)      -- a random integer for data indexing

        Returns a dictionary that contains A, B, A_paths and B_paths
            A (tensor)       -- an image in the input domain
            B (tensor)       -- its corresponding image in the target domain
            A_paths (str)    -- image paths
            B_paths (str)    -- image paths
        """
        # read a image given a random integer index
        img_path = self.dataroot + "/intrinsics_final/images/" + self.img_paths[
            index]
        srgb_img = Image.open(img_path).convert('RGB')
        file_name = self.img_paths[index].split('/')

        R_path = self.dataroot + "/intrinsics_final/images/" + file_name[
            0] + "/" + file_name[1][:-4] + "_albedo.png"
        gt_AL = Image.open(R_path).convert('RGB')

        mask_path = self.dataroot + "/intrinsics_final/images/" + file_name[
            0] + "/" + file_name[1][:-4] + "_mask.png"
        mask = Image.open(mask_path).convert('RGB')

        irradiance = self.stat_dict[self.img_paths[index][:-4] + '.rgbe']

        # apply the same transform to both A and B
        transform_params = get_params(self.opt, srgb_img.size)
        srgb_img_transform = get_transform(self.opt,
                                           transform_params,
                                           grayscale=False,
                                           convert=False)
        gt_AL_transform = get_transform(self.opt,
                                        transform_params,
                                        grayscale=False,
                                        convert=False)
        mask_transform = get_transform(self.opt,
                                       transform_params,
                                       grayscale=True,
                                       convert=False)

        srgb_img = srgb_img_transform(srgb_img)
        gt_AL = gt_AL_transform(gt_AL)
        mask = mask_transform(mask)

        rgb_img = srgb_img**2.2
        gt_SH = rgb_img / torch.clamp(gt_AL, min=1e-6)

        # if albedo is too low, replace shading as radiance
        sh_mask = torch.zeros_like(gt_SH)
        sh_mask[gt_AL < 1e-6] = 1
        sh_mask_inv = 1 - sh_mask
        gt_SH = gt_SH * sh_mask_inv + rgb_img * sh_mask

        srgb_img_gray = torch.mean(srgb_img, 0, keepdim=True)
        rgb_img_gray = torch.mean(rgb_img, 0, keepdim=True)
        gt_AL_gray = torch.mean(gt_AL, 0, keepdim=True)
        gt_SH_gray = torch.mean(gt_SH, 0, keepdim=True)

        gt_AL[gt_AL < 1e-6] = 1e-6
        # gt_SH[gt_SH_gray.expand(gt_SH.size()) > 20] = 20
        gt_SH[gt_SH_gray.expand(gt_SH.size()) > 1] = 1
        gt_SH[gt_SH_gray.expand(gt_SH.size()) < 1e-4] = 1e-4

        if not self.opt.no_mask:
            mask[srgb_img_gray < 1e-6] = 0
            mask[gt_AL_gray < 1e-6] = 0
            mask[gt_SH_gray < 1e-4] = 0
            mask[gt_SH_gray > 10] = 0
            mask = 1.0 - util.erosion(1.0 - mask)

        gt_BA, brightest_20, gt_BP, gt_BC\
             = util.calc_brightest(
                 gt_SH_gray, mask,
                 nr_tap=self.opt.bp_nr_tap,
                 nr_sigma=self.opt.bp_nr_sigma,
                 spread_tap=self.opt.bp_tap,
                 spread_sigma=self.opt.bp_sigma
                 )

        if self.opt.shading_norm:
            gt_SH = gt_SH / brightest_20

        if irradiance < 0.25:
            srgb_img = srgb_img.cpu().numpy()
            gt_SH = gt_SH.cpu().numpy()
            srgb_img = denoise_tv_chambolle(srgb_img,
                                            weight=0.05,
                                            multichannel=True)
            gt_SH = denoise_tv_chambolle(gt_SH, weight=0.1, multichannel=True)
            srgb_img = torch.from_numpy(srgb_img)
            gt_SH = torch.from_numpy(gt_SH)

        srgb_img = normalize()(srgb_img)
        gt_AL = normalize()(gt_AL)
        gt_SH = normalize()(gt_SH)
        mask = normalize(grayscale=True)(mask)
        gt_BA = normalize(grayscale=True)(gt_BA)
        gt_BP = normalize(grayscale=True)(gt_BP)
        gt_BC = torch.Tensor(list(gt_BC))

        srgb_img = torch.unsqueeze(srgb_img, 0)  # [1, 3, 256, 256]
        gt_AL = torch.unsqueeze(gt_AL, 0)
        gt_SH = torch.unsqueeze(gt_SH, 0)
        mask = torch.unsqueeze(mask, 0)
        gt_BA = torch.unsqueeze(gt_BA, 0)
        gt_BP = torch.unsqueeze(gt_BP, 0)
        # radiantest = torch.unsqueeze(radiantest, 0)

        return {
            'A': srgb_img,
            'gt_AL': gt_AL,
            'gt_SH': gt_SH,
            'mask': mask,
            'gt_BA': gt_BA,
            'gt_BP': gt_BP,
            'gt_BC': gt_BC,
            'A_paths': img_path
        }
Ejemplo n.º 30
0
    def __getitem__(self, index):
        ### input A (label maps)
        A_path = self.A_paths[index]
        A = Image.open(A_path).convert('RGB')
        A = A.resize((512, 512), Image.BICUBIC)
        a = float(np.random.random_integers(-1, 1)) / float(10)
        #A = F.adjust_hue(A, a)
        #A = F.adjust_saturation(A,2)
        #w, h = COLOR.size
        #w2 = int(w/3)
        #A = COLOR.crop([0,0,w2,h])
        params = get_params(self.opt, A.size)
        if self.opt.label_nc == 0:
            transform_A = get_transform(self.opt, params)
            A_tensor = transform_A(A)
        else:
            transform_A = get_transform(self.opt,
                                        params,
                                        method=Image.NEAREST,
                                        normalize=False)
            A_tensor = transform_A(A) * 255.0

        ### input H (heat images)
        H_path = self.H_paths[index]
        H = Image.open(H_path).convert('RGB')
        #H = COLOR.crop([w2,0,w2,h])
        #H = F.adjust_hue(H, a)
        #H = F.adjust_saturation(H,2)
        #H = F.adjust_brightness(H, -0.2)
        transform_H = get_transform(self.opt, params)
        H_tensor = transform_H(H)

        B_tensor = inst_tensor = feat_tensor = 0

        ### input B (real images)
        if self.opt.isTrain or self.opt.use_encoded_image:

            B_path = self.B_paths[index]
            B = Image.open(B_path).convert('RGB')
            #B = COLOR.crop([w2*2,0,w2,h])
            B = F.adjust_hue(B, a)
            transform_B = get_transform(self.opt, params)
            B_tensor = transform_B(B)

        ### if using instance maps
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))

        input_dict = {
            'A': A_tensor,
            'inst': inst_tensor,
            'image': B_tensor,
            'H': H_tensor,
            'feat': feat_tensor,
            'path': A_path
        }  #

        return input_dict
Ejemplo n.º 31
0
    def __getitem__(self, index): 

        ### MyTest for diffuse
        if self.MyTest=='Diff':
            ## load images
            A_path = self.A_paths[index]              
            AB = Image.open(A_path).convert('RGB')
            w, h = AB.size 
            ## crop to obtain imageA(input) and imageB(diff)             
            w5 = int(w / 5)
            A = AB.crop((0, 0, w5, h))
            Nor = AB.crop((w5, 0, 2*w5, h))    
            Dif = AB.crop((2*w5, 0, 3*w5, h))    
            Rou = AB.crop((3*w5, 0, 4*w5, h))    
            Spe = AB.crop((4*w5, 0, 5*w5, h))

            ## gamma correction for both A and B
            gamma_A=255.0*(np.array(A)/255.0)**(1/2.2)
            # gamma_B=255.0*(np.array(B)/255.0)**(1/2.2)
            A=Image.fromarray(np.uint8(gamma_A))

            if self.augment_color:
                gamma = np.random.rand() + 0.5
                gamma_Dif=255.0*(np.array(Dif)/255.0)**gamma
                Dif=Image.fromarray(np.uint8(gamma_Dif))

                gamma_Spe=255.0*(np.array(Spe)/255.0)**gamma
                Spe=Image.fromarray(np.uint8(gamma_Spe))    

            ## transform A and B in the same way
            B_tensor = inst_tensor = feat_tensor = 0

            params = get_params(self.opt, A.size)
            transform = get_transform(self.opt, params)
            A_tensor = transform(A)
            Nor_tensor = transform(Nor)
            Dif_tensor = transform(Dif)
            Rou_tensor = transform(Rou)
            Spe_tensor = transform(Spe)

            B_tensor=torch.cat((Nor_tensor,Dif_tensor,Rou_tensor,Spe_tensor), 0)
        elif self.MyTest=='Normal':

            A_path = self.A_paths[index]              
            AB = Image.open(A_path).convert('RGB') 
            w, h = AB.size 
            ## crop to obtain imageA(input) and imageB(normal)             
            w5 = int(w / 5)
            A = AB.crop((0, 0, w5, h))
            B = AB.crop((w5, 0, 2*w5, h))    
            ## gamma correction for only A
            gamma_A=255.0*(np.array(A)/255.0)**(1/2.2)
            A=Image.fromarray(np.uint8(gamma_A))
            ## transform A and B in the same way
            B_tensor = inst_tensor = feat_tensor = 0

            params = get_params(self.opt, A.size)
            transform = get_transform(self.opt, params)
            A_tensor = transform(A)
            B_tensor = transform(B)
        elif self.MyTest=='Spec':

            A_path = self.A_paths[index]              
            AB = Image.open(A_path).convert('RGB') 
            w, h = AB.size 
            ## crop to obtain imageA(input) and imageB(normal)             
            w5 = int(w / 5)
            A = AB.crop((0, 0, w5, h))
            B = AB.crop((4*w5, 0, 5*w5, h))    
            ## gamma correction for only A
            gamma_A=255.0*(np.array(A)/255.0)**(1/2.2)
            A=Image.fromarray(np.uint8(gamma_A))
            ## transform A and B in the same way
            B_tensor = inst_tensor = feat_tensor = 0

            params = get_params(self.opt, A.size)
            transform = get_transform(self.opt, params)
            A_tensor = transform(A)
            B_tensor = transform(B)
        elif self.MyTest=='Rough':

            A_path = self.A_paths[index]              
            AB = Image.open(A_path).convert('RGB') 
            w, h = AB.size 
            ## crop to obtain imageA(input) and imageB(normal)             
            w5 = int(w / 5)
            A = AB.crop((0, 0, w5, h))
            B = AB.crop((3*w5, 0, 4*w5, h))    
            ## gamma correction for only A
            gamma_A=255.0*(np.array(A)/255.0)**(1/2.2)
            A=Image.fromarray(np.uint8(gamma_A))
            ## transform A and B in the same way
            B_tensor = inst_tensor = feat_tensor = 0

            params = get_params(self.opt, A.size)
            transform = get_transform(self.opt, params)
            A_tensor = transform(A)
            B_tensor = transform(B)
        elif self.MyTest=='NA':
            ### input A (label maps)
            A_path = self.A_paths[index]              
            A = Image.open(A_path)        
            params = get_params(self.opt, A.size)
            if self.opt.label_nc == 0:
                transform_A = get_transform(self.opt, params)
                A_tensor = transform_A(A.convert('RGB'))
            else:
                transform_A = get_transform(self.opt, params, method=Image.NEAREST, normalize=False)
                A_tensor = transform_A(A) * 255.0

            B_tensor = inst_tensor = feat_tensor = 0
            ### input B (real images)
            if self.opt.isTrain or self.opt.use_encoded_image:
                B_path = self.B_paths[index]   
                B = Image.open(B_path).convert('RGB')
                transform_B = get_transform(self.opt, params)      
                B_tensor = transform_B(B)
        # for all_4D and all_5D cases
        else:
            if self.mode=='Syn':
                A_path = self.A_paths[index]    
                # print('syn index', index)          
                AB = Image.open(A_path).convert('RGB') 
                w, h = AB.size 
                ## crop to obtain imageA(input) and imageB(normal)             
                w5 = int(w / 5)
                A = AB.crop((0, 0, w5, h))
                Nor = AB.crop((w5, 0, 2*w5, h))    
                Dif = AB.crop((2*w5, 0, 3*w5, h))    
                Rou = AB.crop((3*w5, 0, 4*w5, h))    
                Spe = AB.crop((4*w5, 0, 5*w5, h))    

                ## gamma correction for only A
                ## 2.2
                gamma_A=255.0*(np.array(A)/255.0)**(1/2.2)
                ## log scale
                # gamma_A=255.0*logTensor(np.array(A)/255.0)
                A=Image.fromarray(np.uint8(gamma_A))

                if self.augment_color:
                    gamma = np.random.rand() + 0.5
                    gamma_Dif=255.0*(np.array(Dif)/255.0)**gamma
                    Dif=Image.fromarray(np.uint8(gamma_Dif))

                    gamma_Spe=255.0*(np.array(Spe)/255.0)**gamma
                    Spe=Image.fromarray(np.uint8(gamma_Spe))

                ## transform A and B in the same way
                B_tensor = inst_tensor = feat_tensor = 0

                params = get_params(self.opt, A.size)
                transform = get_transform(self.opt, params)
                A_tensor = transform(A)
                Nor_tensor = transform(Nor)
                Dif_tensor = transform(Dif)
                Rou_tensor = transform(Rou)
                Spe_tensor = transform(Spe)

                B_tensor=torch.cat((Nor_tensor,Dif_tensor,Rou_tensor,Spe_tensor), 0)
                # print('norm: ',B_tensor.shape)
            
            ## this is for real images testing
            elif self.mode=='Real':

                A_path = self.A_paths[index]              
                A = Image.open(A_path).convert('RGB') 

                ## first gamma correction to linear
                # gamma_A=(np.array(A)/255.0)**(2.2)*255
                # ## then linear to log domain
                # gamma_A=255.0*lognp(gamma_A)
                # A=Image.fromarray(np.uint8(gamma_A))

                ## transform A and B in the same way
                B_tensor = inst_tensor = feat_tensor = 0

                params = get_params(self.opt, A.size)
                transform = get_transform(self.opt, params,normalize=False)
                A_tensor = transform(A)


        ### if using instance maps        
        if not self.opt.no_instance:
            inst_path = self.inst_paths[index]
            inst = Image.open(inst_path)
            inst_tensor = transform_A(inst)

            if self.opt.load_features:
                feat_path = self.feat_paths[index]            
                feat = Image.open(feat_path).convert('RGB')
                norm = normalize()
                feat_tensor = norm(transform_A(feat))                            

        input_dict = {'label': A_tensor, 'inst': inst_tensor, 'image': B_tensor, 
                      'feat': feat_tensor, 'path': A_path}

        return input_dict