def __init__(self, dataset_root=None, transforms=None, mode='train', edge=False): self.dataset_root = dataset_root self.transforms = Compose(transforms) mode = mode.lower() self.mode = mode self.file_list = list() self.num_classes = self.NUM_CLASSES self.ignore_index = 255 self.edge = edge if mode not in ['train', 'val', 'test']: raise ValueError( "`mode` should be 'train', 'val' or 'test', but got {}.".format( mode)) if self.transforms is None: raise ValueError("`transforms` is necessary, but it is None.") if self.dataset_root is None: self.dataset_root = download_file_and_uncompress( url=URL, savepath=seg_env.DATA_HOME, extrapath=seg_env.DATA_HOME) elif not os.path.exists(self.dataset_root): self.dataset_root = os.path.normpath(self.dataset_root) savepath, extraname = self.dataset_root.rsplit( sep=os.path.sep, maxsplit=1) self.dataset_root = download_file_and_uncompress( url=URL, savepath=savepath, extrapath=savepath, extraname=extraname) if mode == 'train': file_path = os.path.join(self.dataset_root, 'train_list.txt') elif mode == 'val': file_path = os.path.join(self.dataset_root, 'val_list.txt') else: file_path = os.path.join(self.dataset_root, 'test_list.txt') with open(file_path, 'r') as f: for line in f: items = line.strip().split() if len(items) != 2: if mode == 'train' or mode == 'val': raise Exception( "File list format incorrect! It should be" " image_name label_name\\n") image_path = os.path.join(self.dataset_root, items[0]) grt_path = None else: image_path = os.path.join(self.dataset_root, items[0]) grt_path = os.path.join(self.dataset_root, items[1]) self.file_list.append([image_path, grt_path])
def __init__(self, common_transforms, transforms1, transforms2, dataset_root=None, mode='train', edge=False): self.dataset_root = dataset_root self.common_transforms = Compose(common_transforms) self.transforms = self.common_transforms if transforms1 is not None: self.transforms1 = Compose(transforms1, to_rgb=False) if transforms2 is not None: self.transforms2 = Compose(transforms2, to_rgb=False) mode = mode.lower() self.ignore_index = 255 self.mode = mode self.num_classes = self.NUM_CLASSES self.input_width = 224 self.input_height = 224 if self.dataset_root is None: self.dataset_root = download_file_and_uncompress( url=URL, savepath=seg_env.DATA_HOME, extrapath=seg_env.DATA_HOME) elif not os.path.exists(self.dataset_root): self.dataset_root = os.path.normpath(self.dataset_root) savepath, extraname = self.dataset_root.rsplit(sep=os.path.sep, maxsplit=1) self.dataset_root = download_file_and_uncompress( url=URL, savepath=savepath, extrapath=savepath, extraname=extraname) if mode == 'train': path = os.path.join(dataset_root, 'eg1800_train.txt') else: path = os.path.join(dataset_root, 'eg1800_test.txt') with open(path, 'r') as f: files = f.readlines() img_files = [ os.path.join(dataset_root, 'Images', file).strip() for file in files ] label_files = [ os.path.join(dataset_root, 'Labels', file).strip() for file in files ] self.file_list = [[ img_path, label_path ] for img_path, label_path in zip(img_files, label_files)] pass
def download_data(savepath, extrapath): url = "https://paddleseg.bj.bcebos.com/humanseg/data/mini_supervisely.zip" download_file_and_uncompress(url=url, savepath=savepath, extrapath=extrapath) url = "https://paddleseg.bj.bcebos.com/humanseg/data/video_test.zip" download_file_and_uncompress(url=url, savepath=savepath, extrapath=extrapath, extraname='video_test.mp4')
def __init__(self, transforms, dataset_root=None, mode='train', edge=False): self.dataset_root = dataset_root self.transforms = Compose(transforms) mode = mode.lower() self.mode = mode self.file_list = list() self.num_classes = 150 self.ignore_index = 255 self.edge = edge if mode not in ['train', 'val']: raise ValueError( "`mode` should be one of ('train', 'val') in ADE20K dataset, but got {}." .format(mode)) if self.transforms is None: raise ValueError("`transforms` is necessary, but it is None.") if self.dataset_root is None: self.dataset_root = download_file_and_uncompress( url=URL, savepath=seg_env.DATA_HOME, extrapath=seg_env.DATA_HOME, extraname='ADEChallengeData2016') elif not os.path.exists(self.dataset_root): self.dataset_root = os.path.normpath(self.dataset_root) savepath, extraname = self.dataset_root.rsplit(sep=os.path.sep, maxsplit=1) self.dataset_root = download_file_and_uncompress( url=URL, savepath=savepath, extrapath=savepath, extraname=extraname) if mode == 'train': img_dir = os.path.join(self.dataset_root, 'images/training') label_dir = os.path.join(self.dataset_root, 'annotations/training') elif mode == 'val': img_dir = os.path.join(self.dataset_root, 'images/validation') label_dir = os.path.join(self.dataset_root, 'annotations/validation') img_files = os.listdir(img_dir) label_files = [i.replace('.jpg', '.png') for i in img_files] for i in range(len(img_files)): img_path = os.path.join(img_dir, img_files[i]) label_path = os.path.join(label_dir, label_files[i]) self.file_list.append([img_path, label_path])
def download_pretrained_model(pretrained_model): """ Download pretrained model from url. Args: pretrained_model (str): the url of pretrained weight Returns: str: the path of pretrained weight """ assert urlparse(pretrained_model).netloc, "The url is not valid." pretrained_model = unquote(pretrained_model) savename = pretrained_model.split('/')[-1] if not savename.endswith(('tgz', 'tar.gz', 'tar', 'zip')): savename = pretrained_model.split('/')[-2] else: savename = savename.split('.')[0] with generate_tempdir() as _dir: with filelock.FileLock(os.path.join(seg_env.TMP_HOME, savename)): pretrained_model = download_file_and_uncompress( pretrained_model, savepath=_dir, extrapath=seg_env.PRETRAINED_MODEL_HOME, extraname=savename) pretrained_model = os.path.join(pretrained_model, 'model.pdparams') return pretrained_model
def load_pretrained_model(model, pretrained_model): if pretrained_model is not None: logger.info( 'Loading pretrained model from {}'.format(pretrained_model)) # download pretrained model from url if urlparse(pretrained_model).netloc: pretrained_model = unquote(pretrained_model) savename = pretrained_model.split('/')[-1] if not savename.endswith(('tgz', 'tar.gz', 'tar', 'zip')): savename = pretrained_model.split('/')[-2] else: savename = savename.split('.')[0] with generate_tempdir() as _dir: with filelock.FileLock(os.path.join(seg_env.TMP_HOME, savename)): pretrained_model = download_file_and_uncompress( pretrained_model, savepath=_dir, extrapath=seg_env.PRETRAINED_MODEL_HOME, extraname=savename) pretrained_model = os.path.join(pretrained_model, 'model.pdparams') if os.path.exists(pretrained_model): para_state_dict = paddle.load(pretrained_model) model_state_dict = model.state_dict() keys = model_state_dict.keys() num_params_loaded = 0 for k in keys: if k not in para_state_dict: logger.warning("{} is not in pretrained model".format(k)) elif list(para_state_dict[k].shape) != list( model_state_dict[k].shape): logger.warning( "[SKIP] Shape of pretrained params {} doesn't match.(Pretrained: {}, Actual: {})" .format(k, para_state_dict[k].shape, model_state_dict[k].shape)) else: model_state_dict[k] = para_state_dict[k] num_params_loaded += 1 model.set_dict(model_state_dict) logger.info("There are {}/{} variables loaded into {}.".format( num_params_loaded, len(model_state_dict), model.__class__.__name__)) else: raise ValueError( 'The pretrained model directory is not Found: {}'.format( pretrained_model)) else: logger.info( 'No pretrained model to load, {} will be trained from scratch.'. format(model.__class__.__name__))
def main(): args = parse_args() sbd_path = download_file_and_uncompress(url=URL, savepath=DATA_HOME, extrapath=DATA_HOME, extraname='benchmark_RELEASE') with open(os.path.join(sbd_path, 'dataset/train.txt'), 'r') as f: sbd_file_list = [line.strip() for line in f] with open(os.path.join(sbd_path, 'dataset/val.txt'), 'r') as f: sbd_file_list += [line.strip() for line in f] if not os.path.exists(args.voc_path): raise FileNotFoundError( 'There is no voc_path: {}. Please ensure that the Pascal VOC dataset has been downloaded correctly' ) with open( os.path.join(args.voc_path, 'VOC2012/ImageSets/Segmentation/trainval.txt'), 'r') as f: voc_file_list = [line.strip() for line in f] aug_file_list = list(set(sbd_file_list) - set(voc_file_list)) with open( os.path.join(args.voc_path, 'VOC2012/ImageSets/Segmentation/aug.txt'), 'w') as f: f.writelines(''.join([line, '\n']) for line in aug_file_list) sbd_cls_dir = os.path.join(sbd_path, 'dataset/cls') save_dir = os.path.join(args.voc_path, 'VOC2012/SegmentationClassAug') if not os.path.exists(save_dir): os.mkdir(save_dir) mat_file_list = os.listdir(sbd_cls_dir) p = Pool(args.num_workers) for f in tqdm.tqdm(mat_file_list): p.apply_async(mat_to_png, args=(f, sbd_cls_dir, save_dir)) p.close() p.join()
# limitations under the License. import sys import os LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) TEST_PATH = os.path.join(LOCAL_PATH, "../../../", "test") sys.path.append(TEST_PATH) from paddleseg.utils.download import download_file_and_uncompress model_urls = { "ppseg_lite_portrait_398x224": "https://paddleseg.bj.bcebos.com/dygraph/ppseg/ppseg_lite_portrait_398x224.tar.gz", "deeplabv3p_resnet50_os8_humanseg_512x512_100k": "https://paddleseg.bj.bcebos.com/dygraph/humanseg/train/deeplabv3p_resnet50_os8_humanseg_512x512_100k.zip", "fcn_hrnetw18_small_v1_humanseg_192x192": "https://paddleseg.bj.bcebos.com/dygraph/humanseg/train/fcn_hrnetw18_small_v1_humanseg_192x192.zip", "shufflenetv2_humanseg_192x192": "https://paddleseg.bj.bcebos.com/dygraph/humanseg/train/shufflenetv2_humanseg_192x192.zip", } if __name__ == "__main__": for model_name, url in model_urls.items(): download_file_and_uncompress(url=url, savepath=LOCAL_PATH, extrapath=LOCAL_PATH, extraname=model_name) print("Pretrained model download success!")
def __init__(self, transforms, dataset_root=None, mode='train', edge=False): self.dataset_root = dataset_root self.transforms = Compose(transforms) mode = mode.lower() self.mode = mode self.file_list = list() self.num_classes = self.NUM_CLASSES self.ignore_index = 255 self.edge = edge if mode not in ['train', 'trainval', 'trainaug', 'val']: raise ValueError( "`mode` should be one of ('train', 'trainval', 'trainaug', 'val') in PascalVOC dataset, but got {}." .format(mode)) if self.transforms is None: raise ValueError("`transforms` is necessary, but it is None.") if self.dataset_root is None: self.dataset_root = download_file_and_uncompress( url=URL, savepath=seg_env.DATA_HOME, extrapath=seg_env.DATA_HOME, extraname='VOCdevkit') elif not os.path.exists(self.dataset_root): self.dataset_root = os.path.normpath(self.dataset_root) savepath, extraname = self.dataset_root.rsplit( sep=os.path.sep, maxsplit=1) self.dataset_root = download_file_and_uncompress( url=URL, savepath=savepath, extrapath=savepath, extraname=extraname) image_set_dir = os.path.join(self.dataset_root, 'VOC2012', 'ImageSets', 'Segmentation') if mode == 'train': file_path = os.path.join(image_set_dir, 'train.txt') elif mode == 'val': file_path = os.path.join(image_set_dir, 'val.txt') elif mode == 'trainval': file_path = os.path.join(image_set_dir, 'trainval.txt') elif mode == 'trainaug': file_path = os.path.join(image_set_dir, 'train.txt') file_path_aug = os.path.join(image_set_dir, 'aug.txt') if not os.path.exists(file_path_aug): raise RuntimeError( "When `mode` is 'trainaug', Pascal Voc dataset should be augmented, " "Please make sure voc_augment.py has been properly run when using this mode." ) img_dir = os.path.join(self.dataset_root, 'VOC2012', 'JPEGImages') label_dir = os.path.join(self.dataset_root, 'VOC2012', 'SegmentationClass') label_dir_aug = os.path.join(self.dataset_root, 'VOC2012', 'SegmentationClassAug') with open(file_path, 'r') as f: for line in f: line = line.strip() image_path = os.path.join(img_dir, ''.join([line, '.jpg'])) label_path = os.path.join(label_dir, ''.join([line, '.png'])) self.file_list.append([image_path, label_path]) if mode == 'trainaug': with open(file_path_aug, 'r') as f: for line in f: line = line.strip() image_path = os.path.join(img_dir, ''.join([line, '.jpg'])) label_path = os.path.join(label_dir_aug, ''.join([line, '.png'])) self.file_list.append([image_path, label_path])