def __init__(self, root, split="training", img_size=(640, 1280), is_transform=True, augmentations=None): self.root = root # root is path not tree, /home/brm512/datasets/vistas_v1.1/ self.split = split self.is_transform = is_transform self.augmentations = augmentations self.n_classes = 12 self.img_size = img_size if isinstance(img_size, tuple) else (img_size, img_size) #self.mean = np.array([80.5423, 91.3162, 81.4312]) self.files = {} self.images_base = os.path.join(self.root, self.split, 'images') self.annotations_base = os.path.join(self.root, self.split, 'labels') self.files[split] = recursive_glob(rootdir=self.images_base, suffix='.jpg') self.number_of_images = len(self.files[split]) # self.class_ids, self.class_names, self.class_colors = self.parse_config() self.ignore_id = 250 if not self.files[split]: raise Exception("No files for split=[%s] found in %s" % (split, self.images_base)) print("Found %d %s images" % (len(self.files[split]), split))
def merge_results(output_path): search_strings = ['p_arrivals*', 's_arrivals*'] output_fns = ['p_combined.txt', 's_combined.txt'] for ss, ofn in zip(search_strings, output_fns): files = recursive_glob(output_path, ss) ofn = open('%s/%s' % (output_path, ofn), 'w+') data = set() for i, fn in enumerate(files): lines = open(fn, 'r').readlines() if (i == 0): ofn.write(lines[0]) # end if for j in range(1, len(lines)): data.add(lines[j]) # end for os.system('rm %s' % (fn)) # end for for l in data: ofn.write(l) # end for ofn.close()
def get_drive_filename_triples(image_path, lbl_path, mask_path, mode): imgs_full_path = recursive_glob(image_path, suffix=".tif") triples = [] for img_full_path in imgs_full_path: lbl_full_path, mask_full_path = get_lbl_mask_path(os.path.basename(img_full_path), lbl_path, mask_path, mode) triples.append((img_full_path, lbl_full_path, mask_full_path)) return triples
def __init__( self, root, split="train", index=1, # class for binary classification is_transform=False, img_size=(320, 320), channels=1, augmentations=None, img_norm=True, test_mode=False, suffix=".png"): """__init__ :param root: :param split: :param is_transform: :param img_size: :param augmentations """ self.root = root self.split = split self.index = index self.is_transform = is_transform self.augmentations = augmentations self.img_norm = img_norm self.n_classes = len(label_map) self.n_channels = channels self.img_size = img_size if isinstance(img_size, tuple) else (img_size, img_size, channels) self.files = {} self.images_base = os.path.join( self.root, self.split, "images", ) self.annotations_base = os.path.join(self.root, self.split, "labels") self.files[split] = recursive_glob(rootdir=self.images_base, suffix=".png") self.void_classes = np.arange(len(label_map)) self.void_classes = self.void_classes[self.void_classes != self.index] self.valid_classes = [self.index] self.class_names = label_map.keys() self.ignore_index = 255 self.label_colours = dict( zip(range(len(label_map)), list(label_map.values()))) if not self.files[split]: raise Exception("No files for split=[%s] found in %s" % (split, self.images_base)) print("Found %d %s images in dataset %s" % (len(self.files[split]), split, root.split('/')[-1]))
def _analyze_data(self): json_files = recursive_glob(os.path.join(self.root, 'gtFine', self.split), suffix='json') category = parse_cityscapes(json_files) self.category = list(category.keys())[:-2] table = pretty_table(category) return table
def eval(cfg): # Setup seeds torch.manual_seed(cfg.get("seed", 1337)) torch.cuda.manual_seed(cfg.get("seed", 1337)) np.random.seed(cfg.get("seed", 1337)) random.seed(cfg.get("seed", 1337)) # Setup device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Setup evaluation data data_eval_images = utils.recursive_glob( os.path.join(cfg["data"]["path"], 'images')) # data_eval_labels = utils.recursive_glob(os.path.join(cfg["data"]["path"], 'labels')) # Setup model model = DRCN(cfg).to(device) checkpoint = torch.load(cfg["training"]["checkpoint"]) model.load_state_dict(checkpoint["model_state"]) # Setup Metrics and visualizer running_metrics_val = runningScore(cfg["data"]["n_classes"]) # Start training utils.mkdirs(cfg["training"]["checkpoint"]) s = cfg["data"]["img_rows"] for img_name in tqdm.tqdm(data_eval_images): img = np.array(Image.open(img_name)) lbl = np.array(Image.open(img_name.replace('images', 'labels'))) w, h, _ = img.shape out = np.zeros((6, w, h)) for x in range(0, w - s, 200): for y in range(0, h - s, 200): img_input, lbl_input = threeCityLoader.transform( img[x:x + s, y:y + s, :], lbl[x:x + s, y:y + s]) model.set_input(img_input.unsqueeze(0), lbl_input.unsqueeze(0)) model.inference() out[:, x:x + s, y:y + s] += model.out1.cpu().detach().numpy().squeeze() max_x = (w - s) // 200 * 200 max_y = (h - s) // 200 * 200 pred = out[:, :max_x, :max_y] pred = pred.argmax(0).squeeze() gt = lbl[:max_x, :max_y] running_metrics_val.update(gt, pred) score, class_iou = running_metrics_val.get_scores() for k, v in score.items(): print(k, v) for k, v in class_iou.items(): print("{}: {}".format(k, v)) running_metrics_val.reset()
def __init__(self, base_dir='../data'): self._base_dir = base_dir self._dir = os.path.join(base_dir, 'PKLotSegmented') self.X = np.array(utils.recursive_glob(self._dir, "*.jpg")) self.y = np.array(["Occupied" in s for s in self.X]).astype(np.float32) self.variant = np.array([generate_variant(s) for s in self.X]) self._test_indices = None self._train_indices = None
def main(settings): """.""" files = recursive_glob(settings['source_dir'], settings['filter']) patterns = extract_patterns(get_lines_from_file(settings['patterns']), settings['sep']) for filename in files: lines = get_lines_from_file(filename) for search, replace in patterns: lines = search_and_replace_lines(search, replace, lines) write_lines_to_file(filename, lines)
def __init__(self, base_dir='../data'): self._base_dir = base_dir self._dir = os.path.join(base_dir, 'PKLot') _cleanup(self._dir) self.X = np.array(utils.recursive_glob(self._dir, "*.jpg")) self.y = np.array([s[:-3] + 'xml' for s in self.X]) shuffled_indices = np.arange(len(self.X)) np.random.shuffle(shuffled_indices) self.X = self.X[shuffled_indices] self.y = self.y[shuffled_indices] self.variant = np.array([generate_variant(s) for s in self.X]) self._test_indices = None self._train_indices = None
def _set_files(self): assert (self.mode == 'fine' and self.split in ['train', 'val']) or \ (self.mode == 'coarse' and self.split in ['train', 'train_extra', 'val']) img_list, label_list = None, None if self.mode == 'coarse': img_dir_name = 'leftImg8bit_trainextra' if self.split == 'train_extra' else 'leftImg8bit_trainvaltest' label_path = os.path.join(self.root, 'gtCoarse', 'gtCoarse', self.split) else: img_list = recursive_glob(os.path.join(self.root, 'leftImg8bit', self.split), suffix='.png') # label_list = os.path.join(self.root, 'gtFine', self.split, img_list) label_list = [ os.path.join( self.root, 'gtFine', self.split, label_name.split('/')[-1].split('_')[0], label_name.split('/')[-1].replace('leftImg8bit', 'gtFine_labelIds')) for label_name in img_list ] self.files = list(zip(img_list, label_list))
def __init__(self, root_path, name="train", ratio=0.5, transformation=False, augmentation=None): self.root = root_path self.name = name assert transformation is not None, 'transformation must be provided, give None' self.transformation = transformation self.augmentation = augmentation self.n_classes = 20 self.ratio = ratio self.files = {} assert name in ( 'label', 'unlabel', 'val', 'test' ), 'dataset name should be restricted in "label", "unlabel", "test" and "val", given %s' % name if self.name != 'test': self.images_base = os.path.join(self.root, "leftImg8bit") self.annotations_base = os.path.join(self.root, "gtFine", 'trainval') else: self.images_base = os.path.join(self.root, "leftImg8bit", 'test') # self.annotations_base = os.path.join( # self.root, "gtFine", 'test' # ) np.random.seed(1) if self.name != 'test': train_imgs = recursive_glob(rootdir=os.path.join( self.images_base, 'train'), suffix=".png") train_imgs = np.array(train_imgs) val_imgs = recursive_glob(rootdir=os.path.join( self.images_base, 'val'), suffix=".png") labeled_imgs = np.random.choice(train_imgs, size=int(self.ratio * train_imgs.__len__()), replace=False) labeled_imgs = list(labeled_imgs) unlabeled_imgs = [x for x in train_imgs if x not in labeled_imgs] ### Now here we equalize the lengths of labelled and unlabelled imgs by just repeating up some images if self.ratio > 0.5: new_ratio = round((self.ratio / (1 - self.ratio + 1e-6)), 1) excess_ratio = new_ratio - 1 new_list_1 = unlabeled_imgs * int(excess_ratio) new_list_2 = list( np.random.choice(np.array(unlabeled_imgs), size=int( (excess_ratio - int(excess_ratio)) * unlabeled_imgs.__len__()), replace=False)) unlabeled_imgs += (new_list_1 + new_list_2) elif self.ratio < 0.5: new_ratio = round(((1 - self.ratio) / (self.ratio + 1e-6)), 1) excess_ratio = new_ratio - 1 new_list_1 = labeled_imgs * int(excess_ratio) new_list_2 = list( np.random.choice(np.array(labeled_imgs), size=int( (excess_ratio - int(excess_ratio)) * labeled_imgs.__len__()), replace=False)) labeled_imgs += (new_list_1 + new_list_2) else: test_imgs = recursive_glob(rootdir=self.images_base, suffix=".png") if (self.name == 'label'): self.files[name] = list(labeled_imgs) elif (self.name == 'unlabel'): self.files[name] = list(unlabeled_imgs) elif (self.name == 'val'): self.files[name] = list(val_imgs) elif (self.name == 'test'): self.files[name] = list(test_imgs) ''' This pattern for the various classes has been borrowed from the official repo for Cityscapes dataset You can see it here: https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/helpers/labels.py ''' self.void_classes = [ 0, 1, 2, 3, 4, 5, 6, 9, 10, 14, 15, 16, 18, 29, 30, -1 ] self.valid_classes = [ 7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33 ] self.class_names = [ "road", "sidewalk", "building", "wall", "fence", "pole", "traffic_light", "traffic_sign", "vegetation", "terrain", "sky", "person", "rider", "car", "truck", "bus", "train", "motorcycle", "bicycle", "unlabelled" ] self.ignore_index = 250 self.class_map = dict(zip(self.valid_classes, range(19))) if not self.files[name]: raise Exception("No files for name=[%s] found in %s" % (name, self.images_base)) print("Found %d %s images" % (len(self.files[name]), name))
def __init__(self, root, split="train", is_transform=False, img_size=(512, 1024), augmentations=None, train_transform=Compose([RandomHorizontallyFlip(0.5)]), scale_transform=Compose([Resize([224, 224])]), version="cityscapes", out_dir=""): """__init__ :param root: :param split: :param is_transform: :param img_size: :param augmentations """ self.root = root self.split = split self.is_transform = is_transform self.augmentations = augmentations self.train_transform = train_transform self.scale_transform = scale_transform self.n_classes = 8 #19 self.img_size = (img_size if isinstance(img_size, tuple) else (img_size, img_size)) self.images_base = os.path.join(self.root, "leftImg8bit", self.split) self.annotations_base = os.path.join(self.root, "gtFine", self.split) img_paths = recursive_glob(rootdir=self.images_base, suffix=".png") self.void_classes = [ 0, 1, 2, 3, 4, 5, 6, 9, 10, 14, 15, 16, 18, 29, 30, -1, 7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, ] self.valid_classes = [ 24, 25, 26, 27, 28, 31, 32, 33, ] self.class_names = [ "person", "rider", "car", "truck", "bus", "train", "motorcycle", "bicycle", ] self.ignore_index = 250 self.class_map = dict(zip(self.valid_classes, range(8))) self.img_paths, self.labels_coords, self.img_index_of_label, self.ins_ids = self._prepare_labels( img_paths, out_dir) if not self.img_paths: raise Exception("No files for split=[%s] found in %s" % (split, self.images_base)) print("Found %d %s images" % (len(self.img_paths), split))
def __init__( self, root, split="train", is_transform=False, img_size=(512, 1024), max_depth=250, augmentations=None, img_norm=True, mean=[0, 0, 0], test_mode=False, ): """__init__ :param root: :param split: :param is_transform: :param img_size: :param augmentations """ self.root = root self.split = split self.is_transform = is_transform self.augmentations = augmentations self.img_norm = img_norm self.n_classes = 19 self.img_size = img_size self.max_depth = max_depth self.mean = np.array(mean) self.files = {} self.images_base = os.path.join(self.root, "leftImg8bit", self.split) self.annotations_base = os.path.join(self.root, "gtFine", self.split) self.disparity_base = os.path.join(self.root, "disparity", self.split) self.files[split] = recursive_glob(rootdir=self.images_base, suffix=".png") self.void_classes = [0, 1, 2, 3, 4, 5, 6, 9, 10, 14, 15, 16, 18, 29, 30, -1] self.valid_classes = [7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33] self.class_names = [ "unlabelled", "road", "sidewalk", "building", "wall", "fence", "pole", "traffic_light", "traffic_sign", "vegetation", "terrain", "sky", "person", "rider", "car", "truck", "bus", "train", "motorcycle", "bicycle", ] self.ignore_index = 250 self.class_map = dict(zip(self.valid_classes, range(19))) if not self.files[split]: raise Exception("No files for split=[%s] found in %s" % (split, self.images_base)) print("Found %d %s images" % (len(self.files[split]), split))