def ls(path: str):
"""
List the contents of the directory at the provided URI.
Args:
path (str): A URI supported by this PathHandler
Returns:
List[str]: list of contents in given path
"""
return megfile.smart_listdir(path)
def get_all_checkpoint_files(self):
"""
Returns:
list: All available checkpoint files (.pth files) in target
directory.
"""
all_model_checkpoints = [
os.path.join(self.save_dir, file)
for file in megfile.smart_listdir(self.save_dir)
if megfile.smart_isfile(os.path.join(self.save_dir, file))
and file.endswith(".pth")
]
return all_model_checkpoints
def _load_annotations(self):
timer = Timer()
"""Constructs the imdb."""
# Compile the split data path
logger.info('{} data path: {}'.format(self.name, self.label_file))
# Images are stored per class in subdirs (format: n<number>)
class_ids = [k for k, v in IMAGENET_CATEGORIES.items()]
class_id_cont_id = {k: v[0] - 1 for k, v in IMAGENET_CATEGORIES.items()}
# class_ids = sorted([
# f for f in os.listdir(split_path) if re.match(r'^n[0-9]+$', f)
# ])
# # Map ImageNet class ids to contiguous ids
# class_id_cont_id = {v: i for i, v in enumerate(class_ids)}
# Construct the image db
imdb = []
for class_id in class_ids:
cont_id = class_id_cont_id[class_id]
im_dir = megfile.smart_path_join(self.label_file, class_id)
for im_name in megfile.smart_listdir(im_dir):
imdb.append({
'im_path': os.path.join(im_dir, im_name),
'class': cont_id,
})
logger.info("Loading {} takes {:.2f} seconds.".format(self.label_file, timer.seconds()))
dataset_dicts = []
for i, item in enumerate(imdb):
dataset_dicts.append({
"image_id": i,
"category_id": item["class"],
"file_name": item["im_path"],
})
return dataset_dicts
def load_sem_seg(gt_root, image_root, gt_ext="png", image_ext="jpg"):
"""
Load semantic segmentation datasets. All files under "gt_root" with "gt_ext" extension are
treated as ground truth annotations and all files under "image_root" with "image_ext" extension
as input images. Ground truth and input images are matched using file paths relative to
"gt_root" and "image_root" respectively without taking into account file extensions.
This works for COCO as well as some other datasets.
Args:
gt_root (str): full path to ground truth semantic segmentation files. Semantic segmentation
annotations are stored as images with integer values in pixels that represent
corresponding semantic labels.
image_root (str): the directory where the input images are.
gt_ext (str): file extension for ground truth annotations.
image_ext (str): file extension for input images.
Returns:
list[dict]:
a list of dicts in cvpods standard format without instance-level
annotation.
Notes:
1. This function does not read the image and ground truth files.
The results do not have the "image" and "sem_seg" fields.
"""
# We match input images with ground truth based on their relative filepaths (without file
# extensions) starting from 'image_root' and 'gt_root' respectively.
def file2id(folder_path, file_path):
# extract relative path starting from `folder_path`
image_id = os.path.normpath(
os.path.relpath(file_path, start=folder_path))
# remove file extension
image_id = os.path.splitext(image_id)[0]
return image_id
input_files = sorted(
(os.path.join(image_root, f)
for f in megfile.smart_listdir(image_root) if f.endswith(image_ext)),
key=lambda file_path: file2id(image_root, file_path),
)
gt_files = sorted(
(os.path.join(gt_root, f)
for f in megfile.smart_listdir(gt_root) if f.endswith(gt_ext)),
key=lambda file_path: file2id(gt_root, file_path),
)
assert len(gt_files) > 0, "No annotations found in {}.".format(gt_root)
# Use the intersection, so that val2017_100 annotations can run smoothly with val2017 images
if len(input_files) != len(gt_files):
logger.warning(
"Directory {} and {} has {} and {} files, respectively.".format(
image_root, gt_root, len(input_files), len(gt_files)))
input_basenames = [
os.path.basename(f)[:-len(image_ext)] for f in input_files
]
gt_basenames = [os.path.basename(f)[:-len(gt_ext)] for f in gt_files]
intersect = list(set(input_basenames) & set(gt_basenames))
# sort, otherwise each worker may obtain a list[dict] in different order
intersect = sorted(intersect)
logger.warning("Will use their intersection of {} files.".format(
len(intersect)))
input_files = [
os.path.join(image_root, f + image_ext) for f in intersect
]
gt_files = [os.path.join(gt_root, f + gt_ext) for f in intersect]
logger.info("Loaded {} images with semantic segmentation from {}".format(
len(input_files), image_root))
dataset_dicts = []
for (img_path, gt_path) in zip(input_files, gt_files):
record = {}
record["file_name"] = img_path
record["sem_seg_file_name"] = gt_path
dataset_dicts.append(record)
return dataset_dicts