def initialize(self, opt): self.opt = opt label_paths, image_paths, instance_paths, label1_paths, instance1_paths = self.get_paths( opt) util.natural_sort(label_paths) util.natural_sort(image_paths) util.natural_sort(label1_paths) if not opt.no_instance: util.natural_sort(instance_paths) util.natural_sort(instance1_paths) label_paths = label_paths[:opt.max_dataset_size] image_paths = image_paths[:opt.max_dataset_size] instance_paths = instance_paths[:opt.max_dataset_size] label1_paths = label1_paths[:opt.max_dataset_size] instance1_paths = instance1_paths[:opt.max_dataset_size] if not opt.no_pairing_check: for path1, path2 in zip(label_paths, image_paths): assert self.paths_match(path1, path2), \ "The label-image pair (%s, %s) do not look like the right pair because the filenames are quite different. Are you sure about the pairing? Please see data/pix2pix_dataset.py to see what is going on, and use --no_pairing_check to bypass this." % (path1, path2) self.label_paths = label_paths self.image_paths = image_paths self.instance_paths = instance_paths self.label1_paths = label1_paths self.instance1_paths = instance1_paths size = len(self.label_paths) self.dataset_size = size
def initialize(self, opt): self.opt = opt label_paths, image_paths = self.get_paths(opt) if opt.dataset_mode != 'celebahq' and opt.dataset_mode != 'deepfashion': util.natural_sort(label_paths) util.natural_sort(image_paths) label_paths = label_paths[:opt.max_dataset_size] image_paths = image_paths[:opt.max_dataset_size] if not opt.no_pairing_check: for path1, path2 in zip(label_paths, image_paths): assert self.paths_match(path1, path2), \ "The label-image pair (%s, %s) do not look like the right pair because the filenames are quite different. Are you sure about the pairing? Please see data/pix2pix_dataset.py to see what is going on, and use --no_pairing_check to bypass this." % (path1, path2) self.label_paths = label_paths self.image_paths = image_paths size = len(self.label_paths) self.dataset_size = size self.real_reference_probability = 1 if opt.phase == 'test' else opt.real_reference_probability self.hard_reference_probability = 0 if opt.phase == 'test' else opt.hard_reference_probability self.ref_dict, self.train_test_folder = self.get_ref(opt)
def initialize(self, opt): self.opt = opt self.downsampling_method = Image.BICUBIC if self.opt.downsampling_method == "bilinear": self.downsampling_method = Image.BILINEAR label_paths, image_paths = self.get_paths(opt) util.natural_sort(label_paths) util.natural_sort(image_paths) label_paths = label_paths[:opt.max_dataset_size] image_paths = image_paths[:opt.max_dataset_size] if not opt.no_pairing_check: for path1, path2 in zip(label_paths, image_paths): assert self.paths_match(path1, path2), \ "The label-image pair (%s, %s) do not look like the right pair because the filenames are quite different. Are you sure about the pairing? Please see data/base_dataset.py to see what is going on, and use --no_pairing_check to bypass this." % (path1, path2) self.label_paths = label_paths self.image_paths = image_paths size = len(self.label_paths) self.dataset_size = size
def initialize(self, opt): self.opt = opt image_paths = make_dataset(opt.results_dir, recursive=False, read_cache=True) util.natural_sort(image_paths) self.image_paths = image_paths self.dataset_size = len(image_paths)
def initialize(self, opt): self.opt = opt image_paths = self.get_paths(opt) util.natural_sort(image_paths) image_paths = image_paths[:opt.max_dataset_size] self.image_paths = image_paths size = len(self.image_paths) self.dataset_size = size
def get_paths_by_label_length(self, opt, phase, label_len): root = opt.dataroot phase = 'val' if phase == 'test' else 'train' # find all images image_dir = os.path.join(root, 'img_deploy') all_image_paths = make_dataset(image_dir, recursive=True) # find all seg_maps label_dir = os.path.join(root, 'seg_deploy') all_label_paths = make_dataset(label_dir, recursive=True) util.natural_sort(all_image_paths) util.natural_sort(all_label_paths) image_paths = [] label_paths = [] transform_label = self.get_transform(self.opt, method=Image.NEAREST, normalize=False, flip_prob=0) for (img_f, label_f) in tqdm(zip(all_image_paths, all_label_paths), total=len(all_label_paths)): assert self.paths_match(label_f, img_f) # need to know the label_length label = Image.open(label_f) label_tensor = transform_label(label) * 255. l = len(label_tensor.unique()) - 1 if l == label_len: image_paths.append(img_f) label_paths.append(label_f) util.natural_sort(image_paths) util.natural_sort(label_paths) # split them split_ratio = 0.8 n_imgs = len(image_paths) all_idx = [x for x in range(n_imgs)] train_idx = np.random.choice(n_imgs, size=int(n_imgs * split_ratio), replace=False) test_idx = np.array([x for x in all_idx if x not in train_idx]) if phase == 'train': label_paths = [label_paths[ix] for ix in train_idx] image_paths = [image_paths[ix] for ix in train_idx] else: label_paths = [label_paths[ix] for ix in test_idx] image_paths = [image_paths[ix] for ix in test_idx] assert len(set(train_idx).intersection(set(test_idx))) == 0 instance_paths = None return label_paths, image_paths, instance_paths
def initialize(self, opt): self.opt = opt name = 'train' if opt.isTrain else 'test' self.root = Path(opt.dataset_root) / name paths_dict = self.get_paths(opt) for k, v in paths_dict.items(): util.natural_sort(v) for k, v in paths_dict.items(): setattr(self, f"{k}_paths", v) self.dataset_size = len(v) # note that the last element is used. # commonly used since the evaluation is done in this size self.basic_transform = transforms.Compose( [transforms.Resize(size=(480, 640)), transforms.ToTensor()])
def initialize(self, opt): self.opt = opt label_paths, instance_paths, image_paths, empty_image_paths = self.get_paths( opt) util.natural_sort(label_paths) util.natural_sort(instance_paths) util.natural_sort(image_paths) util.natural_sort(empty_image_paths) label_paths = label_paths[:opt.max_dataset_size] instance_paths = instance_paths[:opt.max_dataset_size] image_paths = image_paths[:opt.max_dataset_size] empty_image_paths = empty_image_paths[:opt.max_dataset_size] self.label_paths = label_paths self.instance_paths = instance_paths self.image_paths = image_paths self.empty_image_paths = empty_image_paths size = len(self.label_paths) self.dataset_size = size classes = np.genfromtxt('./struct3d_classes.csv', dtype='|U', delimiter=',') self.class_ids = {c[0]: i for (i, c) in enumerate(classes)}
def get_paths_by_label_length(self, opt, phase, label_len): root = opt.dataroot phase = 'val' if phase == 'test' else 'train' # find all images image_dir = os.path.join(root, 'img') label_dir = os.path.join(root, 'segmentation_map') image_paths = [] label_paths = [] all_image_paths = make_dataset(image_dir, recursive=True) all_label_paths = [ os.path.join(label_dir, f) for f in os.listdir(label_dir) ] util.natural_sort(all_image_paths) util.natural_sort(all_label_paths) for (img_f, label_f) in zip(all_image_paths, all_label_paths): assert self.paths_match(label_f, img_f) # get label-set and calculate length parts_path = os.listdir(label_f) if '0.png' in parts_path: import pdb pdb.set_trace() l = len(parts_path) if l == label_len: image_paths.append(img_f) label_paths.append(label_f) util.natural_sort(image_paths) util.natural_sort(label_paths) # split them split_ratio = 0.8 n_imgs = len(image_paths) all_idx = [x for x in range(n_imgs)] train_idx = np.random.choice(n_imgs, size=int(n_imgs * split_ratio), replace=False) test_idx = np.array([x for x in all_idx if x not in train_idx]) if phase == 'train': label_paths = [label_paths[ix] for ix in train_idx] image_paths = [image_paths[ix] for ix in train_idx] else: label_paths = [label_paths[ix] for ix in test_idx] image_paths = [image_paths[ix] for ix in test_idx] assert len(set(train_idx).intersection(set(test_idx))) == 0 instance_paths = None return label_paths, image_paths, instance_paths
def get_paths(self, opt, phase): """ for celeba, we split train test on the fly. """ # root: 'server'_data/opt.dataset root = opt.dataroot phase = 'val' if phase == 'test' else 'train' # find all images image_dir = os.path.join(root, 'img') image_paths = make_dataset(image_dir, recursive=True) # find all seg_maps label_dir = os.path.join(root, 'segmentation_map') label_paths = [ os.path.join(label_dir, f) for f in os.listdir(label_dir) ] util.natural_sort(image_paths) util.natural_sort(label_paths) # split them split_ratio = 0.8 n_imgs = len(image_paths) all_idx = [x for x in range(n_imgs)] train_idx = np.random.choice(n_imgs, size=int(n_imgs * split_ratio), replace=False) test_idx = np.array([x for x in all_idx if x not in train_idx]) if phase == 'train': label_paths = [label_paths[ix] for ix in train_idx] image_paths = [image_paths[ix] for ix in train_idx] else: label_paths = [label_paths[ix] for ix in test_idx] image_paths = [image_paths[ix] for ix in test_idx] assert len(set(train_idx).intersection(set(test_idx))) == 0 instance_paths = None return label_paths, image_paths, instance_paths
def initialize(self, opt): image_src_paths, image_rec_paths, label_paths, pred_paths = self.get_paths( opt) util.natural_sort(image_src_paths) util.natural_sort(image_rec_paths) util.natural_sort(label_paths) util.natural_sort(pred_paths) self.image_src_paths = image_src_paths self.image_rec_paths = image_rec_paths self.label_paths = label_paths self.pred_paths = pred_paths print(len(label_paths)) self.transform = torchvision.transforms.Compose([ torchvision.transforms.ToTensor(), torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ])
def initialize(self, opt): self.opt = opt self.debug_mode = False self.attribute_hook = None self.class_hook = None self.threshold = 0.2 self.manual_caption_id = 0 label_paths, image_paths, precomputed_captions_paths = self.get_paths( opt) util.natural_sort(label_paths) util.natural_sort(image_paths) label_paths = label_paths[:opt.max_dataset_size] image_paths = image_paths[:opt.max_dataset_size] if opt.embed_captions and opt.use_precomputed_captions: util.natural_sort(precomputed_captions_paths) precomputed_captions_paths = precomputed_captions_paths[:opt. max_dataset_size] else: assert precomputed_captions_paths is None if not opt.no_pairing_check: for path1, path2 in zip(label_paths, image_paths): assert self.paths_match(path1[:-4], path2), \ "The label-image pair (%s, %s) do not look like the right pair because the filenames are quite different. Are you sure about the pairing? Please see data/pix2pix_dataset.py to see what is going on, and use --no_pairing_check to bypass this." % (path1, path2) if opt.embed_captions and opt.use_precomputed_captions: for path1, path3 in zip(label_paths, precomputed_captions_paths): assert self.paths_match(path1[:-4], path3, cast_to_int=True), \ "The label-image pair (%s, %s) do not look like the right pair because the filenames are quite different. Are you sure about the pairing? Please see data/pix2pix_dataset.py to see what is going on, and use --no_pairing_check to bypass this." % (path1, path3) self.label_paths = label_paths self.image_paths = image_paths self.precomputed_captions_paths = precomputed_captions_paths size = len(self.label_paths) self.dataset_size = size self.initialize_metadata(opt) if opt.embed_captions and not opt.use_precomputed_captions: # Import huggingface tokenizer print( 'Precomputed captions are not being used. Importing huggingface tokenizer library...' ) from transformers import BertTokenizer self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
def initialize(self, opt): self.opt = opt label_paths, image_paths, instance_paths = self.get_paths(opt) util.natural_sort(label_paths) util.natural_sort(image_paths) if not opt.no_instance: util.natural_sort(instance_paths) label_paths = label_paths[:opt.max_dataset_size] image_paths = image_paths[:opt.max_dataset_size] instance_paths = instance_paths[:opt.max_dataset_size] self.label_paths = label_paths self.image_paths = image_paths self.instance_paths = instance_paths size = len(self.label_paths) self.dataset_size = size print(self.dataset_size)
def initialize(self, opt): self.opt = opt label_paths, image_paths, instance_paths = self.get_paths(opt) n_fold = opt.n_fold fold = opt.fold L = len(image_paths) if n_fold == 0: leaveout_indices = [] else: leaveout_indices = np.arange(int(fold * L / n_fold), int((fold + 1) * L / n_fold)) util.natural_sort(label_paths) util.natural_sort(image_paths) if not opt.no_instance: util.natural_sort(instance_paths) if n_fold == 0 or opt.cross_validation_mode == 'train': instance_paths = [ instance_paths[i] for i in range(L) if i not in leaveout_indices ] else: instance_paths = [ instance_paths[i] for i in range(L) if i in leaveout_indices ] if n_fold == 0 or opt.cross_validation_mode == 'train': label_paths = [ label_paths[i] for i in range(L) if i not in leaveout_indices ] image_paths = [ image_paths[i] for i in range(L) if i not in leaveout_indices ] else: label_paths = [ label_paths[i] for i in range(L) if i in leaveout_indices ] image_paths = [ image_paths[i] for i in range(L) if i in leaveout_indices ] label_paths = label_paths[:opt.max_dataset_size] image_paths = image_paths[:opt.max_dataset_size] instance_paths = instance_paths[:opt.max_dataset_size] if not opt.no_pairing_check: for path1, path2 in zip(label_paths, image_paths): assert self.paths_match(path1, path2), \ "The label-image pair (%s, %s) do not look like the right pair because the filenames are quite different. Are you sure about the pairing? Please see data/pix2pix_dataset.py to see what is going on, and use --no_pairing_check to bypass this." % (path1, path2) self.label_paths = label_paths self.image_paths = image_paths self.instance_paths = instance_paths self.id_to_trainid = { 7: 0, 8: 1, 11: 2, 12: 3, 13: 4, 17: 5, 19: 6, 20: 7, 21: 8, 22: 9, 23: 10, 24: 11, 25: 12, 26: 13, 27: 14, 28: 15, 31: 16, 32: 17, 33: 18 } self.trainid_to_id = {v: k for k, v in self.id_to_trainid.items()} self.ignore_label = 255 size = len(self.label_paths) self.dataset_size = size
def initialize(self, opt): self.with_conf_map = False image_src_paths, image_rec_paths, label_paths, pred_paths, entropy_paths, conf_map_paths = self.get_paths( opt) util.natural_sort(image_src_paths) util.natural_sort(image_rec_paths) util.natural_sort(label_paths) util.natural_sort(pred_paths) util.natural_sort(entropy_paths) if opt.with_conf_map: self.with_conf_map = opt.with_conf_map util.natural_sort(conf_map_paths) self.conf_map_paths = conf_map_paths self.image_src_paths = image_src_paths self.image_rec_paths = image_rec_paths self.label_paths = label_paths self.entropy_paths = entropy_paths self.pred_paths = pred_paths print(len(label_paths)) self.transform = torchvision.transforms.Compose([ torchvision.transforms.ToTensor(), torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ])
def initialize(self, opt): self.opt = opt label_paths = self.get_paths(opt) #TODO modify util.natural_sort(label_paths) self.label_paths = label_paths[:opt.max_dataset_size] #TODO modify self.dataset_size = len(self.label_paths)
def initialize(self, opt): self.opt = opt if self.opt.retrival: label_paths, image_paths, instance_paths, label_paths_train, image_paths_train = self.get_paths( opt) util.natural_sort(label_paths_train) util.natural_sort(image_paths_train) label_paths_train = label_paths_train[:opt.max_dataset_size] image_paths_train = image_paths_train[:opt.max_dataset_size] self.label_paths_train = label_paths_train self.image_paths_train = image_paths_train elif self.opt.retrival_memory: label_paths, image_paths, instance_paths, label_paths_train, image_paths_train = self.get_paths( opt) util.natural_sort(label_paths_train) util.natural_sort(image_paths_train) label_paths_train = label_paths_train[:opt.max_dataset_size] image_paths_train = image_paths_train[:opt.max_dataset_size] self.label_paths_train = label_paths_train self.image_paths_train = image_paths_train else: label_paths, image_paths, instance_paths, label_paths_train, image_paths_train = self.get_paths( opt) util.natural_sort(label_paths) util.natural_sort(image_paths) if not opt.no_instance: util.natural_sort(instance_paths) util.natural_sort(instance_paths) label_paths = label_paths[:opt.max_dataset_size] image_paths = image_paths[:opt.max_dataset_size] instance_paths = instance_paths[:opt.max_dataset_size] perm = list(range(len(label_paths))) shuffle(perm) label_paths_fine = [label_paths[index] for index in perm] image_paths_fine = [image_paths[index] for index in perm] if not opt.no_instance: instance_paths_fine = [instance_paths[index] for index in perm] if not opt.no_pairing_check: for path1, path2 in zip(label_paths, image_paths): assert self.paths_match(path1, path2), \ "The label-image pair (%s, %s) do not look like the right pair because the filenames are quite different. Are you sure about the pairing? Please see data/pix2pix_dataset.py to see what is going on, and use --no_pairing_check to bypass this." % (path1, path2) for path1, path2 in zip(label_paths_fine, image_paths_fine): assert self.paths_match(path1, path2), \ "The label-image pair (%s, %s) do not look like the right pair because the filenames are quite different. Are you sure about the pairing? Please see data/pix2pix_dataset.py to see what is going on, and use --no_pairing_check to bypass this." % (path1, path2) self.label_paths = label_paths self.image_paths = image_paths self.instance_paths = instance_paths self.label_paths_fine = label_paths_fine self.image_paths_fine = image_paths_fine if not opt.no_instance: self.instance_paths_fine = instance_paths_fine size = len(self.label_paths) self.dataset_size = size
def initialize(self, opt): self.opt = opt label_paths, image_paths, instance_paths = self.get_paths(opt) util.natural_sort(label_paths) # modification: check if the dataset mode is brats if opt.dataset_mode == 'brats': util.natural_sort(image_paths['t1ce']) util.natural_sort(image_paths['flair']) util.natural_sort(image_paths['t2']) util.natural_sort(image_paths['t1']) else: util.natural_sort(image_paths) if not opt.no_instance: util.natural_sort(instance_paths) label_paths = label_paths[:opt.max_dataset_size] if opt.dataset_mode == 'brats': image_paths['t1ce'] = image_paths['t1ce'][:opt.max_dataset_size] image_paths['flair'] = image_paths['flair'][:opt.max_dataset_size] image_paths['t2'] = image_paths['t2'][:opt.max_dataset_size] image_paths['t1'] = image_paths['t1'][:opt.max_dataset_size] else: image_paths = image_paths[:opt.max_dataset_size] instance_paths = instance_paths[:opt.max_dataset_size] if not opt.no_pairing_check: for path1, path2 in zip(label_paths, image_paths): assert self.paths_match(path1, path2), \ "The label-image pair (%s, %s) do not look like the right pair because the filenames are " \ "quite different. Are you sure about the pairing? Please see data/pix2pix_dataset.py to see " \ "what is going on, and use --no_pairing_check to bypass this." % (path1, path2) # check if the the number of scanner classes given is the same as initialized in base_dataset file if opt.dataset_mode == "brats": assert len(self.scanner_classes) == opt.classes_nc, "The number of scanner classes mismatch" elif opt.dataset_mode == "isic": assert len(self.lesion_classes) == opt.classes_nc, "The number of lesion classes mismatch" self.label_paths = label_paths if opt.dataset_mode == 'brats': self.image_paths = dict() self.image_paths['t1ce'] = image_paths['t1ce'] self.image_paths['flair'] = image_paths['flair'] self.image_paths['t2'] = image_paths['t2'] self.image_paths['t1'] = image_paths['t1'] else: self.image_paths = image_paths self.instance_paths = instance_paths size = len(self.label_paths) self.dataset_size = size if opt.dataset_mode == 'isic': meta_file = "/home/qasima/segmentation_models.pytorch/code/meta_data.json" self.lesion_cls = dict() with open(meta_file, 'r') as f: meta_data = json.load(f) for meta in meta_data: diag = meta["meta"]["clinical"]["diagnosis"] self.lesion_cls[meta["name"]] = diag