def get_voc2007_negatives(path):
"voc2007 root path -> list of paths to all non-person image files"
# list all images
images_path = os.path.join(path, "JPEGImages")
images = [os.path.join(images_path, f) for f in os.listdir(images_path) \
if is_image_file(f)]
# load mapping between filename and has/doesn't have person
person_table_path = os.path.join(path,
"ImageSets/Main/person_trainval.txt")
person_table = np.fromfile(person_table_path, sep=' ').reshape(-1, 2)
# get set of ints where person_trainval == -1 (no person in the image)
no_person = {int(a) for a, b in person_table if b == -1}
images_no_person = []
for image_name in os.listdir(images_path):
if not is_image_file(image_name):
continue
# only add files to the list if they have no person in them
base_name = os.path.splitext(image_name)[0]
if int(base_name) in no_person:
images_no_person.append(os.path.join(images_path, image_name))
return images_no_person
def write_outlier(image_dir, ratio=0.1):
type_list = [
i for i in os.listdir(image_dir)
if os.path.isdir(image_dir + i + '/images/')
]
for type in type_list:
if os.path.isdir(image_dir + type + '/images/'):
images = [
i for i in os.listdir(image_dir + type + '/images/')
if is_image_file(i)
]
size_list = len(images)
outlier_num = int(size_list * ratio)
outlier_index = np.random.choice(size_list,
outlier_num,
replace=False)
outlier_label = np.random.choice(
[i for i in type_list if i != type], outlier_num, replace=True)
with open(
image_dir + type + '/' + type + '_outlier_' +
str(float(ratio)) + '.csv', 'wb') as csvfile:
spamwriter = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
count = 0
for i, im in enumerate(images):
if i not in outlier_index:
spamwriter.writerow([im, 0, type, type])
else:
spamwriter.writerow(
[im, 1, outlier_label[count], type])
count += 1
def __init__(
self,
root: str,
dataset: str,
output_size: Tuple[int, int],
transform=None,
target_transform=None,
):
super(GeoSetFromFolder,
self).__init__(root,
transform=transform,
target_transform=target_transform)
assert dataset.lower() in {'train', 'test'}, 'unknown dataset'
assert os.path.isdir(root), 'Root folder not found.'
img_root = os.path.join(root, dataset)
labelfile = os.path.join(root, 'train_anno.json')
assert os.path.isdir(img_root), 'Image folder not found.'
assert os.path.isfile(labelfile), 'Annotations file not found'
self.dataset = dataset.lower()
self.images = []
self.output_size = output_size
for root, _, files in os.walk(img_root):
for x in files:
if is_image_file(x):
self.images.append(os.path.join(root, x))
self.labels = read_annotation_file(
labelfile, img_root) if dataset == 'train' else {}
def __init__(self, root, transform=None, loader=default_loader):
self.root = root + "/"
self.transform = transform
self.loader = loader
self.files = sorted(
[file for file in os.listdir(self.root) if is_image_file(file)])
def _make_dataset(dir, slc):
images = []
masks = []
for root, _, fnames in sorted(os.walk(dir)):
for fname in fnames:
if is_image_file(fname):
path = os.path.join(root, fname)
item = path
# split path by / and look if it's "sat" or "map". Then use either extension or sat/map to decide
if os.path.split(root)[1] == "sat":
images.append(item)
elif os.path.split(root)[1] == "map":
masks.append(item)
else:
ext = os.path.splitext(path)[1]
if ext == ".jpg":
images.append(item)
elif ext == ".png":
masks.append(item)
else:
raise FileNotFoundError
# if self.split != "valid":
# assert(len(images) == len(masks)) # sloppy sanity check
# print("len(images)",len(images))
# print("len(masks) ",len(masks))
# todo: debug: print color mode
# print("color mode:", images[0].mode)
return sorted(images)[slc], sorted(masks)[slc]
def get_image_files(path: str) -> tp.List[str]:
samples = []
for dirpath, _, filenames in os.walk(path):
for filename in sorted(filenames):
full_path = os.path.join(dirpath, filename)
if is_image_file(full_path):
samples.append(full_path)
return samples
def _make_dataset(dir):
images = []
for root, _, fnames in sorted(os.walk(dir)):
for fname in fnames:
if is_image_file(fname):
path = os.path.join(root, fname)
item = path
images.append(item)
return images
def __init__(self, root: str, transform=None, target_transform=None, image_mode: str = 'RGB'):
super(ImagesFromFolder, self).__init__(root,
transform=transform,
target_transform=target_transform)
assert image_mode in _PIL_IMAGE_MODES_, 'Unknown PIL image mode.'
assert os.path.isdir(root), 'Image folder not found.'
self.images = [os.path.join(root, x) for x in os.listdir(root) if is_image_file(x)]
self.mode = image_mode
def __init__(self, root, transform):
self.root = os.path.abspath(os.path.expanduser(root))
self.transform = transform
images = []
for dir, _, names in sorted(os.walk(self.root)):
for name in sorted(names):
path = os.path.join(root, name)
if is_image_file(path):
images.append(path)
self.images = images
def _make_dataset(dir):
images = []
for root, _, fnames in sorted(os.walk(dir)):
for fname in fnames:
# exclude .png files
if is_image_file(fname) and not fname.endswith('.png'):
path = os.path.join(root, fname)
item = path
images.append(item)
return images
def find_images(directory):
"""Recursively search a directory for images, returning a list of paths"""
paths = []
for root, _, fnames in sorted(os.walk(directory)):
for fname in sorted(fnames):
if is_image_file(fname):
path = os.path.join(root, fname)
paths.append(path)
return paths
def make_dataset(dir):
images = []
assert os.path.isdir(dir), '%s is not a valid directory' % dir
for root, _, fnames in sorted(os.walk(dir)):
for fname in fnames:
if is_image_file(fname):
path = os.path.join(root, fname)
images.append(path)
return images
def make_dataset(_dir, max_data_size=float('inf')):
images = []
assert os.path.isdir(_dir), '{} is not a valid' \
'directory'.format(_dir)
for dirpath, _, file_names in os.walk(_dir):
for file_name in file_names:
if is_image_file(file_name):
pth = os.path.join(dirpath, file_name)
images.append(pth)
return images[:min(max_data_size, len(images))]
def _make_image_namelist(dir):
images = []
namelist = []
for root, _, fnames in sorted(os.walk(dir)):
for fname in fnames:
if is_image_file(fname):
item_name = fname
namelist.append(item_name)
item_path = os.path.join(root, fname)
images.append(item_path)
return images, namelist
def _make_dataset(dir):
images = []
for root, _, fnames in sorted(os.walk(dir)):
# generate the file names in directory tree
# root = 'SegNet-Tutorial/CamVid/train'
# fnames: all the filenames in this folder
for fname in fnames:
if is_image_file(fname):
path = os.path.join(root, fname)
# 'SegNet-Tutorial/CamVid/train/0006R0_f02430.png'
item = path
images.append(item)
return images
def make_resize_dataset(_dir, _transform, new_dir_pth):
assert os.path.isdir(_dir), '{} is not a valid' \
'directory'.format(_dir)
if not os.path.exists(new_dir_pth):
os.makedirs(new_dir_pth)
num = 0
for dir_path, _, file_names in os.walk(_dir):
for file_name in file_names:
if is_image_file(file_name):
pth = os.path.join(dir_path, file_name)
img = Image.open(pth).convert('RGB')
trans = _transform(img)
save_image(trans, os.path.join(new_dir_pth, '{:05d}.{}'.format(num, func_format(file_name.split('.')[-1]))))
num += 1
def _collect_image_files(self, depth: Optional[int]) -> Tuple[str, ...]:
image_files = tuple(
os.path.join(root, file)
for root, _, files in walkupto(self.root, depth=depth)
for file in files if is_image_file(file))
if not image_files:
msg = f"The directory {self.root} does not contain any image files"
if depth is not None:
msg += f" up to a depth of {depth}"
msg += "."
raise RuntimeError(msg)
return image_files
def _collect_guides(file: str) -> Optional["LocalImageCollection"]:
dir = path.splitext(file)[0]
if not path.isdir(dir):
return None
image_files = [file for file in os.listdir(dir) if is_image_file(file)]
if not image_files:
return None
guides: Dict[str, "LocalImage"] = {}
for file in image_files:
region = path.splitext(path.basename(file))[0]
guides[region] = LocalImage(path.join(dir, file),
collect_local_guides=False)
return LocalImageCollection(guides)
def make_dataset(self):
for sample_dir in sorted(self.dataset_path.iterdir()):
if sample_dir.is_dir():
sample_dict = {}
for filepath in sorted(sample_dir.iterdir()):
if filepath.is_file() and is_image_file(str(filepath)):
filename = filepath.stem
if filename == "height":
sample_dict[ChannelEnum.GT_DEM] = filepath
elif filename == "occlusion":
sample_dict[ChannelEnum.OCC_MASK] = filepath
else:
continue
self.samples.append(sample_dict)
def _parse_train_images(
self, train_path: str, classes: List[str],
class_to_idx: Dict[str, int]) -> List[Tuple[str, int]]:
instances = []
for target_class in classes:
target_dir = os.path.join(train_path, target_class, "images")
class_index = class_to_idx[target_class]
if not os.path.isdir(target_dir):
continue
for root, _, fnames in sorted(os.walk(target_dir,
followlinks=True)):
for fname in sorted(fnames):
path = os.path.join(root, fname)
if is_image_file(path):
item = path, class_index
instances.append(item)
return instances
def get_frames(dirname):
"""
return: list of image files
"""
from torchvision.datasets.folder import is_image_file
def get_numeric(file):
"""
sorting frames by frame number
"""
return int(re.findall('\d+', os.path.split(file)[-1])[0])
return sorted([
os.path.join(dirname, file)
for file in os.listdir(dirname) if is_image_file(file)
],
key=get_numeric)
def _parse_val_images(
self, val_path: str, classes: List[str],
class_to_idx: Dict[str, int]) -> List[Tuple[str, int]]:
val_annotations_filepath = os.path.join(val_path,
"val_annotations.txt")
with open(val_annotations_filepath, "r") as f:
context = f.read()
context: List[str] = context.strip().split("\n")
instances = []
for line in context:
annotation = line.strip().split("\t")
img_filepath = os.path.join(val_path, "images", annotation[0])
wid = annotation[1]
class_index = class_to_idx[wid]
if is_image_file(img_filepath):
item = img_filepath, class_index
instances.append(item)
return instances
def make_dataset(dir, class_to_idx, target_indices):
images = []
dir = os.path.expanduser(dir)
for target in sorted(os.listdir(dir)):
if not os.path.isdir(os.path.join(dir, target)):
continue
# Only get data for target classes
if not class_to_idx[target] in target_indices:
continue
d = os.path.join(dir, target)
if not os.path.isdir(d):
continue
for root, _, fnames in sorted(os.walk(d)):
for fname in sorted(fnames):
if is_image_file(fname):
path = os.path.join(root, fname)
item = (path, class_to_idx[target], fname)
images.append(item)
return images
def make_dataset(dir, class_to_idx):
actions = []
images = []
dir = os.path.expanduser(dir)
for target in sorted(os.listdir(dir)):
d = os.path.join(dir, target)
if not os.path.isdir(d):
continue
for root, _, fnames in sorted(os.walk(d)):
# if root[-2:] not in ['66', '67', '68']:
for fname in sorted(fnames):
if is_image_file(fname):
path = os.path.join(root, fname)
item = (path, class_to_idx[target])
images.append(item)
if fname == 'actions.npy':
path = os.path.join(root, fname)
actions.append(np.load(path))
actions[-1][-1, 4] = 0.0
return images, np.concatenate(actions, axis=0)
def _make_dataset(dir, slc):
images = []
masks = []
for root, _, fnames in sorted(os.walk(dir)):
for fname in fnames:
if is_image_file(fname):
path = os.path.join(root, fname)
if os.path.split(path)[0][-3:] == "sat":
images.append(path)
elif os.path.split(path)[0][-3:] == "map":
masks.append(path)
else:
raise FileNotFoundError
# if self.split != "valid":
# assert(len(images) == len(masks)) # sloppy sanity check
# print("len(images)",len(images))
# print("len(masks) ",len(masks))
return sorted(images)[slc], sorted(masks)[slc]
def get_frames(dirname):
from torchvision.datasets.folder import is_image_file
return sorted([os.path.join(dirname, file)
for file in os.listdir(dirname)
if is_image_file(file)])
def search_images(self, path):
path = pathlib.Path(path)
if path.is_dir():
return [e for e in path.glob('*') if is_image_file(str(e))]
else:
return [path]
def is_valid_file(filename: str) -> bool:
is_valid_file.calls += 1
return is_image_file(
filename) and is_valid_file.calls % world_size() == get_rank()
import os
import shutil
from torchvision.datasets.folder import is_image_file
p_folder = './tmp/'
out_folder = './tmp_out/'
sub_folders = os.listdir(p_folder)
for sub_folder in sub_folders:
sub_path = './tmp/' + sub_folder
for root, _, fnames in sorted(os.walk(sub_path)):
for fname in fnames:
if is_image_file(fname):
if "rgb" in fname:
old_path = os.path.join(root, fname)
new_name = fname.replace("rgb", "")
new_path = out_folder + "train/" + new_name
shutil.move(old_path, new_path)
else:
old_path = os.path.join(root, fname)
new_name = fname.replace("depth", "")
new_path = out_folder + "train_dep/" + new_name
shutil.move(old_path, new_path)
def __init__(self,
root=None,
image_dir=None,
target_path=None,
transform=None,
target_transform=None,
loader=default_loader,
target_loader=load):
super(ImageFoldersTargetFileDataset,
self).__init__(root=root,
transforms=None,
transform=transform,
target_transform=target_transform)
self.loader = loader
self.target_loader = target_loader
# By default the structure is the following
# root/
# ├── imgs
# └── tgts/tgts.pkl
# if either image_dir or target_dir is None then we use this structure
# Otherwise, we look for the following structure
# root/
# ├── $image_dir
# └── $target_path <- a path to a file
# Finally, if root is None, we expect two independent paths for the images and targets
if root is None:
self.image_dir = os.path.abspath(image_dir)
self.target_path = os.path.abspath(target_path)
else:
if image_dir is None:
self.image_dir = os.path.abspath(os.path.join(root, "imgs"))
else:
self.image_dir = os.path.abspath(os.path.join(root, image_dir))
if target_path is None:
self.target_path = os.path.abspath(
os.path.join(root, "tgts", "tgts.pkl"))
else:
self.target_path = os.path.abspath(
os.path.join(root, target_path))
assert os.path.isdir(
self.image_dir), "This is not a directory:\n" + str(self.image_dir)
assert os.path.isfile(
self.target_path), "This is not a file:\n" + str(self.target_path)
self.targets = target_loader(self.target_path)
target_accessed = [False for _ in range(len(self.targets))]
images = ["" for _ in range(len(self.targets))]
for im in os.listdir(self.image_dir):
impath = os.path.join(self.image_dir, im)
if os.path.isfile(impath) and is_image_file(impath):
# Found an image, let's check that there is a matching target
im_index = int(os.path.splitext(im)[0])
assert not target_accessed[
im_index], "Two images share the index " + str(im_index)
target_accessed[im_index] = True
images[im_index] = impath
assert all(
target_accessed
), "Image numbering incorrect. Please name your images from 0 to N without gaps."
self.images = images
