def __init__(self,
root,
split="training",
img_size=(640, 1280),
is_transform=True,
augmentations=None):
self.root = root # root is path not tree, /home/brm512/datasets/vistas_v1.1/
self.split = split
self.is_transform = is_transform
self.augmentations = augmentations
self.n_classes = 12
self.img_size = img_size if isinstance(img_size, tuple) else (img_size,
img_size)
#self.mean = np.array([80.5423, 91.3162, 81.4312])
self.files = {}
self.images_base = os.path.join(self.root, self.split, 'images')
self.annotations_base = os.path.join(self.root, self.split, 'labels')
self.files[split] = recursive_glob(rootdir=self.images_base,
suffix='.jpg')
self.number_of_images = len(self.files[split])
# self.class_ids, self.class_names, self.class_colors = self.parse_config()
self.ignore_id = 250
if not self.files[split]:
raise Exception("No files for split=[%s] found in %s" %
(split, self.images_base))
print("Found %d %s images" % (len(self.files[split]), split))
def merge_results(output_path):
search_strings = ['p_arrivals*', 's_arrivals*']
output_fns = ['p_combined.txt', 's_combined.txt']
for ss, ofn in zip(search_strings, output_fns):
files = recursive_glob(output_path, ss)
ofn = open('%s/%s' % (output_path, ofn), 'w+')
data = set()
for i, fn in enumerate(files):
lines = open(fn, 'r').readlines()
if (i == 0):
ofn.write(lines[0])
# end if
for j in range(1, len(lines)):
data.add(lines[j])
# end for
os.system('rm %s' % (fn))
# end for
for l in data:
ofn.write(l)
# end for
ofn.close()
def get_drive_filename_triples(image_path, lbl_path, mask_path, mode):
imgs_full_path = recursive_glob(image_path, suffix=".tif")
triples = []
for img_full_path in imgs_full_path:
lbl_full_path, mask_full_path = get_lbl_mask_path(os.path.basename(img_full_path), lbl_path, mask_path, mode)
triples.append((img_full_path, lbl_full_path, mask_full_path))
return triples
def __init__(
self,
root,
split="train",
index=1, # class for binary classification
is_transform=False,
img_size=(320, 320),
channels=1,
augmentations=None,
img_norm=True,
test_mode=False,
suffix=".png"):
"""__init__
:param root:
:param split:
:param is_transform:
:param img_size:
:param augmentations
"""
self.root = root
self.split = split
self.index = index
self.is_transform = is_transform
self.augmentations = augmentations
self.img_norm = img_norm
self.n_classes = len(label_map)
self.n_channels = channels
self.img_size = img_size if isinstance(img_size, tuple) else (img_size,
img_size,
channels)
self.files = {}
self.images_base = os.path.join(
self.root,
self.split,
"images",
)
self.annotations_base = os.path.join(self.root, self.split, "labels")
self.files[split] = recursive_glob(rootdir=self.images_base,
suffix=".png")
self.void_classes = np.arange(len(label_map))
self.void_classes = self.void_classes[self.void_classes != self.index]
self.valid_classes = [self.index]
self.class_names = label_map.keys()
self.ignore_index = 255
self.label_colours = dict(
zip(range(len(label_map)), list(label_map.values())))
if not self.files[split]:
raise Exception("No files for split=[%s] found in %s" %
(split, self.images_base))
print("Found %d %s images in dataset %s" %
(len(self.files[split]), split, root.split('/')[-1]))
def _analyze_data(self):
json_files = recursive_glob(os.path.join(self.root, 'gtFine',
self.split),
suffix='json')
category = parse_cityscapes(json_files)
self.category = list(category.keys())[:-2]
table = pretty_table(category)
return table
def eval(cfg):
# Setup seeds
torch.manual_seed(cfg.get("seed", 1337))
torch.cuda.manual_seed(cfg.get("seed", 1337))
np.random.seed(cfg.get("seed", 1337))
random.seed(cfg.get("seed", 1337))
# Setup device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Setup evaluation data
data_eval_images = utils.recursive_glob(
os.path.join(cfg["data"]["path"], 'images'))
# data_eval_labels = utils.recursive_glob(os.path.join(cfg["data"]["path"], 'labels'))
# Setup model
model = DRCN(cfg).to(device)
checkpoint = torch.load(cfg["training"]["checkpoint"])
model.load_state_dict(checkpoint["model_state"])
# Setup Metrics and visualizer
running_metrics_val = runningScore(cfg["data"]["n_classes"])
# Start training
utils.mkdirs(cfg["training"]["checkpoint"])
s = cfg["data"]["img_rows"]
for img_name in tqdm.tqdm(data_eval_images):
img = np.array(Image.open(img_name))
lbl = np.array(Image.open(img_name.replace('images', 'labels')))
w, h, _ = img.shape
out = np.zeros((6, w, h))
for x in range(0, w - s, 200):
for y in range(0, h - s, 200):
img_input, lbl_input = threeCityLoader.transform(
img[x:x + s, y:y + s, :], lbl[x:x + s, y:y + s])
model.set_input(img_input.unsqueeze(0), lbl_input.unsqueeze(0))
model.inference()
out[:, x:x + s,
y:y + s] += model.out1.cpu().detach().numpy().squeeze()
max_x = (w - s) // 200 * 200
max_y = (h - s) // 200 * 200
pred = out[:, :max_x, :max_y]
pred = pred.argmax(0).squeeze()
gt = lbl[:max_x, :max_y]
running_metrics_val.update(gt, pred)
score, class_iou = running_metrics_val.get_scores()
for k, v in score.items():
print(k, v)
for k, v in class_iou.items():
print("{}: {}".format(k, v))
running_metrics_val.reset()
def __init__(self, base_dir='../data'):
self._base_dir = base_dir
self._dir = os.path.join(base_dir, 'PKLotSegmented')
self.X = np.array(utils.recursive_glob(self._dir, "*.jpg"))
self.y = np.array(["Occupied" in s for s in self.X]).astype(np.float32)
self.variant = np.array([generate_variant(s) for s in self.X])
self._test_indices = None
self._train_indices = None
def main(settings):
"""."""
files = recursive_glob(settings['source_dir'], settings['filter'])
patterns = extract_patterns(get_lines_from_file(settings['patterns']),
settings['sep'])
for filename in files:
lines = get_lines_from_file(filename)
for search, replace in patterns:
lines = search_and_replace_lines(search, replace, lines)
write_lines_to_file(filename, lines)
def __init__(self, base_dir='../data'):
self._base_dir = base_dir
self._dir = os.path.join(base_dir, 'PKLot')
_cleanup(self._dir)
self.X = np.array(utils.recursive_glob(self._dir, "*.jpg"))
self.y = np.array([s[:-3] + 'xml' for s in self.X])
shuffled_indices = np.arange(len(self.X))
np.random.shuffle(shuffled_indices)
self.X = self.X[shuffled_indices]
self.y = self.y[shuffled_indices]
self.variant = np.array([generate_variant(s) for s in self.X])
self._test_indices = None
self._train_indices = None
def _set_files(self):
assert (self.mode == 'fine' and self.split in ['train', 'val']) or \
(self.mode == 'coarse' and self.split in ['train', 'train_extra', 'val'])
img_list, label_list = None, None
if self.mode == 'coarse':
img_dir_name = 'leftImg8bit_trainextra' if self.split == 'train_extra' else 'leftImg8bit_trainvaltest'
label_path = os.path.join(self.root, 'gtCoarse', 'gtCoarse',
self.split)
else:
img_list = recursive_glob(os.path.join(self.root, 'leftImg8bit',
self.split),
suffix='.png')
# label_list = os.path.join(self.root, 'gtFine', self.split, img_list)
label_list = [
os.path.join(
self.root, 'gtFine', self.split,
label_name.split('/')[-1].split('_')[0],
label_name.split('/')[-1].replace('leftImg8bit',
'gtFine_labelIds'))
for label_name in img_list
]
self.files = list(zip(img_list, label_list))
def __init__(self,
root_path,
name="train",
ratio=0.5,
transformation=False,
augmentation=None):
self.root = root_path
self.name = name
assert transformation is not None, 'transformation must be provided, give None'
self.transformation = transformation
self.augmentation = augmentation
self.n_classes = 20
self.ratio = ratio
self.files = {}
assert name in (
'label', 'unlabel', 'val', 'test'
), 'dataset name should be restricted in "label", "unlabel", "test" and "val", given %s' % name
if self.name != 'test':
self.images_base = os.path.join(self.root, "leftImg8bit")
self.annotations_base = os.path.join(self.root, "gtFine",
'trainval')
else:
self.images_base = os.path.join(self.root, "leftImg8bit", 'test')
# self.annotations_base = os.path.join(
# self.root, "gtFine", 'test'
# )
np.random.seed(1)
if self.name != 'test':
train_imgs = recursive_glob(rootdir=os.path.join(
self.images_base, 'train'),
suffix=".png")
train_imgs = np.array(train_imgs)
val_imgs = recursive_glob(rootdir=os.path.join(
self.images_base, 'val'),
suffix=".png")
labeled_imgs = np.random.choice(train_imgs,
size=int(self.ratio *
train_imgs.__len__()),
replace=False)
labeled_imgs = list(labeled_imgs)
unlabeled_imgs = [x for x in train_imgs if x not in labeled_imgs]
### Now here we equalize the lengths of labelled and unlabelled imgs by just repeating up some images
if self.ratio > 0.5:
new_ratio = round((self.ratio / (1 - self.ratio + 1e-6)), 1)
excess_ratio = new_ratio - 1
new_list_1 = unlabeled_imgs * int(excess_ratio)
new_list_2 = list(
np.random.choice(np.array(unlabeled_imgs),
size=int(
(excess_ratio - int(excess_ratio)) *
unlabeled_imgs.__len__()),
replace=False))
unlabeled_imgs += (new_list_1 + new_list_2)
elif self.ratio < 0.5:
new_ratio = round(((1 - self.ratio) / (self.ratio + 1e-6)), 1)
excess_ratio = new_ratio - 1
new_list_1 = labeled_imgs * int(excess_ratio)
new_list_2 = list(
np.random.choice(np.array(labeled_imgs),
size=int(
(excess_ratio - int(excess_ratio)) *
labeled_imgs.__len__()),
replace=False))
labeled_imgs += (new_list_1 + new_list_2)
else:
test_imgs = recursive_glob(rootdir=self.images_base, suffix=".png")
if (self.name == 'label'):
self.files[name] = list(labeled_imgs)
elif (self.name == 'unlabel'):
self.files[name] = list(unlabeled_imgs)
elif (self.name == 'val'):
self.files[name] = list(val_imgs)
elif (self.name == 'test'):
self.files[name] = list(test_imgs)
'''
This pattern for the various classes has been borrowed from the official repo for Cityscapes dataset
You can see it here: https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/helpers/labels.py
'''
self.void_classes = [
0, 1, 2, 3, 4, 5, 6, 9, 10, 14, 15, 16, 18, 29, 30, -1
]
self.valid_classes = [
7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31,
32, 33
]
self.class_names = [
"road", "sidewalk", "building", "wall", "fence", "pole",
"traffic_light", "traffic_sign", "vegetation", "terrain", "sky",
"person", "rider", "car", "truck", "bus", "train", "motorcycle",
"bicycle", "unlabelled"
]
self.ignore_index = 250
self.class_map = dict(zip(self.valid_classes, range(19)))
if not self.files[name]:
raise Exception("No files for name=[%s] found in %s" %
(name, self.images_base))
print("Found %d %s images" % (len(self.files[name]), name))
def __init__(self,
root,
split="train",
is_transform=False,
img_size=(512, 1024),
augmentations=None,
train_transform=Compose([RandomHorizontallyFlip(0.5)]),
scale_transform=Compose([Resize([224, 224])]),
version="cityscapes",
out_dir=""):
"""__init__
:param root:
:param split:
:param is_transform:
:param img_size:
:param augmentations
"""
self.root = root
self.split = split
self.is_transform = is_transform
self.augmentations = augmentations
self.train_transform = train_transform
self.scale_transform = scale_transform
self.n_classes = 8 #19
self.img_size = (img_size if isinstance(img_size, tuple) else
(img_size, img_size))
self.images_base = os.path.join(self.root, "leftImg8bit", self.split)
self.annotations_base = os.path.join(self.root, "gtFine", self.split)
img_paths = recursive_glob(rootdir=self.images_base, suffix=".png")
self.void_classes = [
0,
1,
2,
3,
4,
5,
6,
9,
10,
14,
15,
16,
18,
29,
30,
-1,
7,
8,
11,
12,
13,
17,
19,
20,
21,
22,
23,
]
self.valid_classes = [
24,
25,
26,
27,
28,
31,
32,
33,
]
self.class_names = [
"person",
"rider",
"car",
"truck",
"bus",
"train",
"motorcycle",
"bicycle",
]
self.ignore_index = 250
self.class_map = dict(zip(self.valid_classes, range(8)))
self.img_paths, self.labels_coords, self.img_index_of_label, self.ins_ids = self._prepare_labels(
img_paths, out_dir)
if not self.img_paths:
raise Exception("No files for split=[%s] found in %s" %
(split, self.images_base))
print("Found %d %s images" % (len(self.img_paths), split))
def __init__(
self,
root,
split="train",
is_transform=False,
img_size=(512, 1024),
max_depth=250,
augmentations=None,
img_norm=True,
mean=[0, 0, 0],
test_mode=False,
):
"""__init__
:param root:
:param split:
:param is_transform:
:param img_size:
:param augmentations
"""
self.root = root
self.split = split
self.is_transform = is_transform
self.augmentations = augmentations
self.img_norm = img_norm
self.n_classes = 19
self.img_size = img_size
self.max_depth = max_depth
self.mean = np.array(mean)
self.files = {}
self.images_base = os.path.join(self.root, "leftImg8bit", self.split)
self.annotations_base = os.path.join(self.root, "gtFine", self.split)
self.disparity_base = os.path.join(self.root, "disparity", self.split)
self.files[split] = recursive_glob(rootdir=self.images_base, suffix=".png")
self.void_classes = [0, 1, 2, 3, 4, 5, 6, 9, 10, 14, 15, 16, 18, 29, 30, -1]
self.valid_classes = [7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33]
self.class_names = [
"unlabelled",
"road",
"sidewalk",
"building",
"wall",
"fence",
"pole",
"traffic_light",
"traffic_sign",
"vegetation",
"terrain",
"sky",
"person",
"rider",
"car",
"truck",
"bus",
"train",
"motorcycle",
"bicycle",
]
self.ignore_index = 250
self.class_map = dict(zip(self.valid_classes, range(19)))
if not self.files[split]:
raise Exception("No files for split=[%s] found in %s" % (split, self.images_base))
print("Found %d %s images" % (len(self.files[split]), split))