def get_voc2007_negatives(path): "voc2007 root path -> list of paths to all non-person image files" # list all images images_path = os.path.join(path, "JPEGImages") images = [os.path.join(images_path, f) for f in os.listdir(images_path) \ if is_image_file(f)] # load mapping between filename and has/doesn't have person person_table_path = os.path.join(path, "ImageSets/Main/person_trainval.txt") person_table = np.fromfile(person_table_path, sep=' ').reshape(-1, 2) # get set of ints where person_trainval == -1 (no person in the image) no_person = {int(a) for a, b in person_table if b == -1} images_no_person = [] for image_name in os.listdir(images_path): if not is_image_file(image_name): continue # only add files to the list if they have no person in them base_name = os.path.splitext(image_name)[0] if int(base_name) in no_person: images_no_person.append(os.path.join(images_path, image_name)) return images_no_person
def write_outlier(image_dir, ratio=0.1): type_list = [ i for i in os.listdir(image_dir) if os.path.isdir(image_dir + i + '/images/') ] for type in type_list: if os.path.isdir(image_dir + type + '/images/'): images = [ i for i in os.listdir(image_dir + type + '/images/') if is_image_file(i) ] size_list = len(images) outlier_num = int(size_list * ratio) outlier_index = np.random.choice(size_list, outlier_num, replace=False) outlier_label = np.random.choice( [i for i in type_list if i != type], outlier_num, replace=True) with open( image_dir + type + '/' + type + '_outlier_' + str(float(ratio)) + '.csv', 'wb') as csvfile: spamwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) count = 0 for i, im in enumerate(images): if i not in outlier_index: spamwriter.writerow([im, 0, type, type]) else: spamwriter.writerow( [im, 1, outlier_label[count], type]) count += 1
def __init__( self, root: str, dataset: str, output_size: Tuple[int, int], transform=None, target_transform=None, ): super(GeoSetFromFolder, self).__init__(root, transform=transform, target_transform=target_transform) assert dataset.lower() in {'train', 'test'}, 'unknown dataset' assert os.path.isdir(root), 'Root folder not found.' img_root = os.path.join(root, dataset) labelfile = os.path.join(root, 'train_anno.json') assert os.path.isdir(img_root), 'Image folder not found.' assert os.path.isfile(labelfile), 'Annotations file not found' self.dataset = dataset.lower() self.images = [] self.output_size = output_size for root, _, files in os.walk(img_root): for x in files: if is_image_file(x): self.images.append(os.path.join(root, x)) self.labels = read_annotation_file( labelfile, img_root) if dataset == 'train' else {}
def __init__(self, root, transform=None, loader=default_loader): self.root = root + "/" self.transform = transform self.loader = loader self.files = sorted( [file for file in os.listdir(self.root) if is_image_file(file)])
def _make_dataset(dir, slc): images = [] masks = [] for root, _, fnames in sorted(os.walk(dir)): for fname in fnames: if is_image_file(fname): path = os.path.join(root, fname) item = path # split path by / and look if it's "sat" or "map". Then use either extension or sat/map to decide if os.path.split(root)[1] == "sat": images.append(item) elif os.path.split(root)[1] == "map": masks.append(item) else: ext = os.path.splitext(path)[1] if ext == ".jpg": images.append(item) elif ext == ".png": masks.append(item) else: raise FileNotFoundError # if self.split != "valid": # assert(len(images) == len(masks)) # sloppy sanity check # print("len(images)",len(images)) # print("len(masks) ",len(masks)) # todo: debug: print color mode # print("color mode:", images[0].mode) return sorted(images)[slc], sorted(masks)[slc]
def get_image_files(path: str) -> tp.List[str]: samples = [] for dirpath, _, filenames in os.walk(path): for filename in sorted(filenames): full_path = os.path.join(dirpath, filename) if is_image_file(full_path): samples.append(full_path) return samples
def _make_dataset(dir): images = [] for root, _, fnames in sorted(os.walk(dir)): for fname in fnames: if is_image_file(fname): path = os.path.join(root, fname) item = path images.append(item) return images
def __init__(self, root: str, transform=None, target_transform=None, image_mode: str = 'RGB'): super(ImagesFromFolder, self).__init__(root, transform=transform, target_transform=target_transform) assert image_mode in _PIL_IMAGE_MODES_, 'Unknown PIL image mode.' assert os.path.isdir(root), 'Image folder not found.' self.images = [os.path.join(root, x) for x in os.listdir(root) if is_image_file(x)] self.mode = image_mode
def __init__(self, root, transform): self.root = os.path.abspath(os.path.expanduser(root)) self.transform = transform images = [] for dir, _, names in sorted(os.walk(self.root)): for name in sorted(names): path = os.path.join(root, name) if is_image_file(path): images.append(path) self.images = images
def _make_dataset(dir): images = [] for root, _, fnames in sorted(os.walk(dir)): for fname in fnames: # exclude .png files if is_image_file(fname) and not fname.endswith('.png'): path = os.path.join(root, fname) item = path images.append(item) return images
def find_images(directory): """Recursively search a directory for images, returning a list of paths""" paths = [] for root, _, fnames in sorted(os.walk(directory)): for fname in sorted(fnames): if is_image_file(fname): path = os.path.join(root, fname) paths.append(path) return paths
def make_dataset(dir): images = [] assert os.path.isdir(dir), '%s is not a valid directory' % dir for root, _, fnames in sorted(os.walk(dir)): for fname in fnames: if is_image_file(fname): path = os.path.join(root, fname) images.append(path) return images
def make_dataset(_dir, max_data_size=float('inf')): images = [] assert os.path.isdir(_dir), '{} is not a valid' \ 'directory'.format(_dir) for dirpath, _, file_names in os.walk(_dir): for file_name in file_names: if is_image_file(file_name): pth = os.path.join(dirpath, file_name) images.append(pth) return images[:min(max_data_size, len(images))]
def _make_image_namelist(dir): images = [] namelist = [] for root, _, fnames in sorted(os.walk(dir)): for fname in fnames: if is_image_file(fname): item_name = fname namelist.append(item_name) item_path = os.path.join(root, fname) images.append(item_path) return images, namelist
def _make_dataset(dir): images = [] for root, _, fnames in sorted(os.walk(dir)): # generate the file names in directory tree # root = 'SegNet-Tutorial/CamVid/train' # fnames: all the filenames in this folder for fname in fnames: if is_image_file(fname): path = os.path.join(root, fname) # 'SegNet-Tutorial/CamVid/train/0006R0_f02430.png' item = path images.append(item) return images
def make_resize_dataset(_dir, _transform, new_dir_pth): assert os.path.isdir(_dir), '{} is not a valid' \ 'directory'.format(_dir) if not os.path.exists(new_dir_pth): os.makedirs(new_dir_pth) num = 0 for dir_path, _, file_names in os.walk(_dir): for file_name in file_names: if is_image_file(file_name): pth = os.path.join(dir_path, file_name) img = Image.open(pth).convert('RGB') trans = _transform(img) save_image(trans, os.path.join(new_dir_pth, '{:05d}.{}'.format(num, func_format(file_name.split('.')[-1])))) num += 1
def _collect_image_files(self, depth: Optional[int]) -> Tuple[str, ...]: image_files = tuple( os.path.join(root, file) for root, _, files in walkupto(self.root, depth=depth) for file in files if is_image_file(file)) if not image_files: msg = f"The directory {self.root} does not contain any image files" if depth is not None: msg += f" up to a depth of {depth}" msg += "." raise RuntimeError(msg) return image_files
def _collect_guides(file: str) -> Optional["LocalImageCollection"]: dir = path.splitext(file)[0] if not path.isdir(dir): return None image_files = [file for file in os.listdir(dir) if is_image_file(file)] if not image_files: return None guides: Dict[str, "LocalImage"] = {} for file in image_files: region = path.splitext(path.basename(file))[0] guides[region] = LocalImage(path.join(dir, file), collect_local_guides=False) return LocalImageCollection(guides)
def make_dataset(self): for sample_dir in sorted(self.dataset_path.iterdir()): if sample_dir.is_dir(): sample_dict = {} for filepath in sorted(sample_dir.iterdir()): if filepath.is_file() and is_image_file(str(filepath)): filename = filepath.stem if filename == "height": sample_dict[ChannelEnum.GT_DEM] = filepath elif filename == "occlusion": sample_dict[ChannelEnum.OCC_MASK] = filepath else: continue self.samples.append(sample_dict)
def _parse_train_images( self, train_path: str, classes: List[str], class_to_idx: Dict[str, int]) -> List[Tuple[str, int]]: instances = [] for target_class in classes: target_dir = os.path.join(train_path, target_class, "images") class_index = class_to_idx[target_class] if not os.path.isdir(target_dir): continue for root, _, fnames in sorted(os.walk(target_dir, followlinks=True)): for fname in sorted(fnames): path = os.path.join(root, fname) if is_image_file(path): item = path, class_index instances.append(item) return instances
def get_frames(dirname): """ return: list of image files """ from torchvision.datasets.folder import is_image_file def get_numeric(file): """ sorting frames by frame number """ return int(re.findall('\d+', os.path.split(file)[-1])[0]) return sorted([ os.path.join(dirname, file) for file in os.listdir(dirname) if is_image_file(file) ], key=get_numeric)
def _parse_val_images( self, val_path: str, classes: List[str], class_to_idx: Dict[str, int]) -> List[Tuple[str, int]]: val_annotations_filepath = os.path.join(val_path, "val_annotations.txt") with open(val_annotations_filepath, "r") as f: context = f.read() context: List[str] = context.strip().split("\n") instances = [] for line in context: annotation = line.strip().split("\t") img_filepath = os.path.join(val_path, "images", annotation[0]) wid = annotation[1] class_index = class_to_idx[wid] if is_image_file(img_filepath): item = img_filepath, class_index instances.append(item) return instances
def make_dataset(dir, class_to_idx, target_indices): images = [] dir = os.path.expanduser(dir) for target in sorted(os.listdir(dir)): if not os.path.isdir(os.path.join(dir, target)): continue # Only get data for target classes if not class_to_idx[target] in target_indices: continue d = os.path.join(dir, target) if not os.path.isdir(d): continue for root, _, fnames in sorted(os.walk(d)): for fname in sorted(fnames): if is_image_file(fname): path = os.path.join(root, fname) item = (path, class_to_idx[target], fname) images.append(item) return images
def make_dataset(dir, class_to_idx): actions = [] images = [] dir = os.path.expanduser(dir) for target in sorted(os.listdir(dir)): d = os.path.join(dir, target) if not os.path.isdir(d): continue for root, _, fnames in sorted(os.walk(d)): # if root[-2:] not in ['66', '67', '68']: for fname in sorted(fnames): if is_image_file(fname): path = os.path.join(root, fname) item = (path, class_to_idx[target]) images.append(item) if fname == 'actions.npy': path = os.path.join(root, fname) actions.append(np.load(path)) actions[-1][-1, 4] = 0.0 return images, np.concatenate(actions, axis=0)
def _make_dataset(dir, slc): images = [] masks = [] for root, _, fnames in sorted(os.walk(dir)): for fname in fnames: if is_image_file(fname): path = os.path.join(root, fname) if os.path.split(path)[0][-3:] == "sat": images.append(path) elif os.path.split(path)[0][-3:] == "map": masks.append(path) else: raise FileNotFoundError # if self.split != "valid": # assert(len(images) == len(masks)) # sloppy sanity check # print("len(images)",len(images)) # print("len(masks) ",len(masks)) return sorted(images)[slc], sorted(masks)[slc]
def get_frames(dirname): from torchvision.datasets.folder import is_image_file return sorted([os.path.join(dirname, file) for file in os.listdir(dirname) if is_image_file(file)])
def search_images(self, path): path = pathlib.Path(path) if path.is_dir(): return [e for e in path.glob('*') if is_image_file(str(e))] else: return [path]
def is_valid_file(filename: str) -> bool: is_valid_file.calls += 1 return is_image_file( filename) and is_valid_file.calls % world_size() == get_rank()
import os import shutil from torchvision.datasets.folder import is_image_file p_folder = './tmp/' out_folder = './tmp_out/' sub_folders = os.listdir(p_folder) for sub_folder in sub_folders: sub_path = './tmp/' + sub_folder for root, _, fnames in sorted(os.walk(sub_path)): for fname in fnames: if is_image_file(fname): if "rgb" in fname: old_path = os.path.join(root, fname) new_name = fname.replace("rgb", "") new_path = out_folder + "train/" + new_name shutil.move(old_path, new_path) else: old_path = os.path.join(root, fname) new_name = fname.replace("depth", "") new_path = out_folder + "train_dep/" + new_name shutil.move(old_path, new_path)
def __init__(self, root=None, image_dir=None, target_path=None, transform=None, target_transform=None, loader=default_loader, target_loader=load): super(ImageFoldersTargetFileDataset, self).__init__(root=root, transforms=None, transform=transform, target_transform=target_transform) self.loader = loader self.target_loader = target_loader # By default the structure is the following # root/ # ├── imgs # └── tgts/tgts.pkl # if either image_dir or target_dir is None then we use this structure # Otherwise, we look for the following structure # root/ # ├── $image_dir # └── $target_path <- a path to a file # Finally, if root is None, we expect two independent paths for the images and targets if root is None: self.image_dir = os.path.abspath(image_dir) self.target_path = os.path.abspath(target_path) else: if image_dir is None: self.image_dir = os.path.abspath(os.path.join(root, "imgs")) else: self.image_dir = os.path.abspath(os.path.join(root, image_dir)) if target_path is None: self.target_path = os.path.abspath( os.path.join(root, "tgts", "tgts.pkl")) else: self.target_path = os.path.abspath( os.path.join(root, target_path)) assert os.path.isdir( self.image_dir), "This is not a directory:\n" + str(self.image_dir) assert os.path.isfile( self.target_path), "This is not a file:\n" + str(self.target_path) self.targets = target_loader(self.target_path) target_accessed = [False for _ in range(len(self.targets))] images = ["" for _ in range(len(self.targets))] for im in os.listdir(self.image_dir): impath = os.path.join(self.image_dir, im) if os.path.isfile(impath) and is_image_file(impath): # Found an image, let's check that there is a matching target im_index = int(os.path.splitext(im)[0]) assert not target_accessed[ im_index], "Two images share the index " + str(im_index) target_accessed[im_index] = True images[im_index] = impath assert all( target_accessed ), "Image numbering incorrect. Please name your images from 0 to N without gaps." self.images = images