def test_serdes_celeba_dataset(remove_json_files=True): """ Test serdes on Celeba dataset pipeline. """ DATA_DIR = "../data/dataset/testCelebAData/" data1 = ds.CelebADataset(DATA_DIR, decode=True, num_shards=1, shard_id=0) # define map operations data1 = data1.repeat(2) center_crop = vision.CenterCrop((80, 80)) pad_op = vision.Pad(20, fill_value=(20, 20, 20)) data1 = data1.map(operations=[center_crop, pad_op], input_columns=["image"], num_parallel_workers=8) data2 = util_check_serialize_deserialize_file(data1, "celeba_dataset_pipeline", remove_json_files) num_samples = 0 # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) for item1, item2 in zip( data1.create_dict_iterator(num_epochs=1, output_numpy=True), data2.create_dict_iterator(num_epochs=1, output_numpy=True)): np.testing.assert_array_equal(item1['image'], item2['image']) num_samples += 1 assert num_samples == 8
def test_crop_grayscale(height=375, width=375): """ Test that centercrop works with pad and grayscale images """ # Note: image.transpose performs channel swap to allow py transforms to # work with c transforms transforms = [ py_vision.Decode(), py_vision.Grayscale(1), py_vision.ToTensor(), (lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8)) ] transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) data1 = data1.map(operations=transform, input_columns=["image"]) # If input is grayscale, the output dimensions should be single channel crop_gray = vision.CenterCrop([height, width]) data1 = data1.map(operations=crop_gray, input_columns=["image"]) for item1 in data1.create_dict_iterator(num_epochs=1, output_numpy=True): c_image = item1["image"] # Check that the image is grayscale assert (c_image.ndim == 3 and c_image.shape[2] == 1)
def test_center_crop_op(height=375, width=375, plot=False): """ Test CenterCrop """ logger.info("Test CenterCrop") # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) decode_op = vision.Decode() # 3 images [375, 500] [600, 500] [512, 512] center_crop_op = vision.CenterCrop([height, width]) data1 = data1.map(operations=decode_op, input_columns=["image"]) data1 = data1.map(operations=center_crop_op, input_columns=["image"]) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) data2 = data2.map(operations=decode_op, input_columns=["image"]) image_cropped = [] image = [] for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True), data2.create_dict_iterator(num_epochs=1, output_numpy=True)): image_cropped.append(item1["image"].copy()) image.append(item2["image"].copy()) if plot: visualize_list(image, image_cropped)
def test_center_crop_comp(height=375, width=375, plot=False): """ Test CenterCrop between python and c image augmentation """ logger.info("Test CenterCrop") # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) decode_op = vision.Decode() center_crop_op = vision.CenterCrop([height, width]) data1 = data1.map(operations=decode_op, input_columns=["image"]) data1 = data1.map(operations=center_crop_op, input_columns=["image"]) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) transforms = [ py_vision.Decode(), py_vision.CenterCrop([height, width]), py_vision.ToTensor() ] transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data2 = data2.map(operations=transform, input_columns=["image"]) image_c_cropped = [] image_py_cropped = [] for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True), data2.create_dict_iterator(num_epochs=1, output_numpy=True)): c_image = item1["image"] py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) # Note: The images aren't exactly the same due to rounding error assert diff_mse(py_image, c_image) < 0.001 image_c_cropped.append(c_image.copy()) image_py_cropped.append(py_image.copy()) if plot: visualize_list(image_c_cropped, image_py_cropped, visualize_mode=2)
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="GPU", dtype="fp16", device_num=1): ds.config.set_numa_enable(True) if device_num == 1: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=4, shuffle=True) else: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=4, shuffle=True, num_shards=device_num, shard_id=get_rank()) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] # define map operations normalize_op = C.Normalize(mean=mean, std=std) if dtype == "fp16": if args_opt.eval: x_dtype = "float32" else: x_dtype = "float16" normalize_op = C.NormalizePad(mean=mean, std=std, dtype=x_dtype) if do_train: trans = [ C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), normalize_op, ] else: trans = [ C.Decode(), C.Resize(256), C.CenterCrop(image_size), normalize_op, ] if dtype == "fp32": trans.append(C.HWC2CHW()) data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=8) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) # apply dataset repeat operation if repeat_num > 1: data_set = data_set.repeat(repeat_num) return data_set
def create_dataset(dataset_path, config, do_train, repeat_num=1): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. config(dict): config of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1. Returns: dataset """ rank = config.rank group_size = config.group_size if group_size == 1: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=True) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=True, num_shards=group_size, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(config.image_size), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode ] else: trans = [ C.Decode(), C.Resize(int(config.image_size / 0.875)), C.CenterCrop(config.image_size) ] trans += [ C.Rescale(1.0 / 255.0, 0.0), C.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=config.work_nums) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=config.work_nums) # apply batch operations ds = ds.batch(config.batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): """ Create a train or eval dataset. Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 Returns: dataset """ device_num = int(os.getenv("RANK_SIZE")) rank_id = int(os.getenv("RANK_ID")) if device_num == 1: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] # define map operations if do_train: trans = [ C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] else: trans = [ C.Decode(), C.Resize((256, 256)), C.CenterCrop(image_size), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=8) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset4(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): """ create a train or eval imagenet2012 dataset for se-resnet50 Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend Returns: dataset """ if target == "Ascend": device_num, rank_id = _get_rank_info() if device_num == 1: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True, num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [123.68, 116.78, 103.94] std = [1.0, 1.0, 1.0] # define map operations if do_train: trans = [ C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] else: trans = [ C.Decode(), C.Resize(292), C.CenterCrop(256), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=12) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=12) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, batch_size=16, device_num=1, rank=0): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. batch_size(int): the batch size of dataset. Default: 16. device_num (int): Number of shards that the dataset should be divided into (default=1). rank (int): The shard ID within num_shards (default=0). Returns: dataset """ if device_num == 1: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(299), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) ] else: trans = [C.Decode(), C.Resize(320), C.CenterCrop(299)] trans += [ C.Normalize(mean=[127.5, 127.5, 127.5], std=[127.5, 127.5, 127.5]), C.HWC2CHW(), C2.TypeCast(mstype.float32) ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8) ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): """ Create a train or eval dataset. Args: dataset_path (str): The path of dataset. do_train (bool): Whether dataset is used for train or eval. repeat_num (int): The repeat times of dataset. Default: 1. batch_size (int): The batch size of dataset. Default: 32. Returns: Dataset. """ do_shuffle = bool(do_train) if device_num == 1 or not do_train: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=do_shuffle) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=do_shuffle, num_shards=device_num, shard_id=device_id) image_length = 299 if do_train: trans = [ C.RandomCropDecodeResize(image_length, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) ] else: trans = [ C.Decode(), C.Resize(image_length), C.CenterCrop(image_length) ] trans += [ C.Rescale(1.0 / 255.0, 0.0), C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=config.work_nums) ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=config.work_nums) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. rank (int): The shard ID within num_shards (default=None). group_size (int): Number of shards that the dataset should be divided into (default=None). repeat_num(int): the repeat times of dataset. Default: 1. Returns: dataset """ if group_size == 1: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True, num_shards=group_size, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(299, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) ] else: trans = [C.Decode(), C.Resize(299), C.CenterCrop(299)] trans += [ C.Rescale(1.0 / 255.0, 0.0), C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=cfg.work_nums) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=cfg.work_nums) # apply batch operations ds = ds.batch(cfg.batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, device_num=1, rank=0): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. rank (int): The shard ID within num_shards (default=None). group_size (int): Number of shards that the dataset should be divided into (default=None). repeat_num(int): the repeat times of dataset. Default: 1. Returns: dataset """ if device_num == 1: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(224), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) ] else: trans = [C.Decode(), C.Resize(239), C.CenterCrop(224)] trans += [ C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]), C.HWC2CHW(), ] type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(input_columns="image", operations=trans, num_parallel_workers=8) data_set = data_set.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) # apply batch operations data_set = data_set.batch(config.batch_size, drop_remainder=True) return data_set
def create_dataset_val(batch_size=128, val_data_url='', workers=8, distributed=False): if not os.path.exists(val_data_url): raise ValueError('Path not exists') rank_id = get_rank() if distributed else 0 rank_size = get_group_size() if distributed else 1 dataset = ds.ImageFolderDataset(val_data_url, num_parallel_workers=workers, num_shards=rank_size, shard_id=rank_id, shuffle=False) scale_size = None interpolation = str2MsInter(inter_method) if isinstance(img_size, tuple): assert len(img_size) == 2 if img_size[-1] == img_size[-2]: scale_size = int(math.floor(img_size[0] / crop_pct)) else: scale_size = tuple([int(x / crop_pct) for x in img_size]) else: scale_size = int(math.floor(img_size / crop_pct)) type_cast_op = C2.TypeCast(mstype.int32) decode_op = C.Decode() resize_op = C.Resize(size=scale_size, interpolation=interpolation) center_crop = C.CenterCrop(size=224) rescale_op = C.Rescale(rescale, shift) normalize_op = C.Normalize(IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD) changeswap_op = C.HWC2CHW() ctrans = [ decode_op, resize_op, center_crop, rescale_op, normalize_op, changeswap_op ] dataset = dataset.map(input_columns=["label"], operations=type_cast_op, num_parallel_workers=workers) dataset = dataset.map(input_columns=["image"], operations=ctrans, num_parallel_workers=workers) dataset = dataset.batch(batch_size, drop_remainder=True, num_parallel_workers=workers) dataset = dataset.repeat(1) return dataset
def create_dataset(data_path, is_train=True, batch_size=32): # import import mindspore.common.dtype as mstype import mindspore.dataset.engine as de import mindspore.dataset.transforms.c_transforms as C2 import mindspore.dataset.vision.c_transforms as C from mindspore.common import set_seed set_seed(1) # shard num_shards = shard_id = None rand_size = os.getenv("RANK_SIZE") rand_id = os.getenv("RANK_ID") if rand_size is not None and rand_id is not None: num_shards = int(rand_size) shard_id = int(rand_id) # define dataset data_path = os.path.join(data_path, "train" if is_train else "val") ds = de.ImageFolderDataset(data_path, shuffle=True, num_parallel_workers=8, num_shards=num_shards, shard_id=shard_id, num_samples=None) # define ops comps_ops = list() # train or val if is_train: comps_ops.append(C.RandomCropDecodeResize(224, scale=(0.08, 1.0), ratio=(0.75, 1.333))) comps_ops.append(C.RandomHorizontalFlip(prob=0.5)) comps_ops.append(C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)) else: comps_ops.append(C.Decode()) comps_ops.append(C.Resize(224)) comps_ops.append(C.CenterCrop(224)) comps_ops.append(C.Rescale(1 / 255.0, 0.)) comps_ops.append(C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])) comps_ops.append(C.HWC2CHW()) # map ops ds = ds.map(input_columns=["image"], operations=comps_ops, num_parallel_workers=8) ds = ds.map(input_columns=["label"], operations=C2.TypeCast(mstype.int32), num_parallel_workers=8) # batch & repeat ds = ds.batch(batch_size=batch_size, drop_remainder=is_train) ds = ds.repeat(count=1) return ds
def test_center_crop_md5(height=375, width=375): """ Test CenterCrop """ logger.info("Test CenterCrop") # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) decode_op = vision.Decode() # 3 images [375, 500] [600, 500] [512, 512] center_crop_op = vision.CenterCrop([height, width]) data1 = data1.map(operations=decode_op, input_columns=["image"]) data1 = data1.map(operations=center_crop_op, input_columns=["image"]) # Compare with expected md5 from images filename = "center_crop_01_result.npz" save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN)
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="GPU", dtype="fp16"): ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=4, shuffle=True) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] # define map operations if do_train: trans = [ C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.Normalize(mean=mean, std=std), ] else: trans = [ C.Decode(), C.Resize(256), C.CenterCrop(image_size), C.Normalize(mean=mean, std=std), ] if dtype == "fp32": trans.append(C.HWC2CHW()) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=4) if dtype == "fp16": ds = ds.map(operations=pad, input_columns="image", num_parallel_workers=4) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation if repeat_num > 1: ds = ds.repeat(repeat_num) return ds
def test_celeba_dataset_op(): data = ds.CelebADataset(DATA_DIR, decode=True, num_shards=1, shard_id=0) crop_size = (80, 80) resize_size = (24, 24) # define map operations data = data.repeat(2) center_crop = vision.CenterCrop(crop_size) resize_op = vision.Resize(resize_size, Inter.LINEAR) # Bilinear mode data = data.map(operations=center_crop, input_columns=["image"]) data = data.map(operations=resize_op, input_columns=["image"]) count = 0 for item in data.create_dict_iterator(num_epochs=1): logger.info("----------image--------") logger.info(item["image"]) count = count + 1 assert count == 4
def test_get_column_name_map(): data = ds.Cifar10Dataset(CIFAR10_DIR) center_crop_op = vision.CenterCrop(10) data = data.map(operations=center_crop_op, input_columns=["image"]) assert data.get_col_names() == ["image", "label"] data = ds.Cifar10Dataset(CIFAR10_DIR) data = data.map(operations=center_crop_op, input_columns=["image"], output_columns=["image"]) assert data.get_col_names() == ["image", "label"] data = ds.Cifar10Dataset(CIFAR10_DIR) data = data.map(operations=center_crop_op, input_columns=["image"], output_columns=["col1"]) assert data.get_col_names() == ["col1", "label"] data = ds.Cifar10Dataset(CIFAR10_DIR) data = data.map(operations=center_crop_op, input_columns=["image"], output_columns=["col1", "col2"], column_order=["col2", "col1"]) assert data.get_col_names() == ["col2", "col1"]
def create_dataset4(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend", distribute=False, enable_cache=False, cache_session_id=None): """ create a train or eval imagenet2012 dataset for se-resnet50 Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend distribute(bool): data for distribute or not. Default: False enable_cache(bool): whether tensor caching service is used for eval. Default: False cache_session_id(int): If enable_cache, cache session_id need to be provided. Default: None Returns: dataset """ if target == "Ascend": device_num, rank_id = _get_rank_info() else: if distribute: init() rank_id = get_rank() device_num = get_group_size() else: device_num = 1 if device_num == 1: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True) else: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True, num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [123.68, 116.78, 103.94] std = [1.0, 1.0, 1.0] # define map operations if do_train: trans = [ C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] else: trans = [ C.Decode(), C.Resize(292), C.CenterCrop(256), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=12) # only enable cache for eval if do_train: enable_cache = False if enable_cache: if not cache_session_id: raise ValueError( "A cache session_id must be provided to use cache.") eval_cache = ds.DatasetCache(session_id=int(cache_session_id), size=0) data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=12, cache=eval_cache) else: data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=12) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set
def create_dataset(dataset_path, config, buffer_size=1000): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. config(struct): the config of train and eval in diffirent platform. repeat_num(int): the repeat times of dataset. Default: 1. Returns: train_dataset, val_dataset """ ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=4) train_ds, eval_ds = ds.split(config.data_split, randomize=True) resize_height = config.image_height resize_width = config.image_width # define operations mapping to each sample normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) change_swap_op = C.HWC2CHW() type_cast_op = C2.TypeCast(mstype.int32) # operations for training crop_decode_resize = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) color_adjust = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) # operations for inference decode_op = C.Decode() resize_op = C.Resize((256, 256)) center_crop = C.CenterCrop(resize_width) train_trans = [ crop_decode_resize, horizontal_flip_op, color_adjust, normalize_op, change_swap_op ] train_ds = train_ds.map(input_columns="image", operations=train_trans, num_parallel_workers=4) train_ds = train_ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=4) eval_trans = [ decode_op, resize_op, center_crop, normalize_op, change_swap_op ] eval_ds = eval_ds.map(input_columns="image", operations=eval_trans, num_parallel_workers=4) eval_ds = eval_ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=4) # apply shuffle operations train_ds = train_ds.shuffle(buffer_size=buffer_size) # apply batch operations train_ds = train_ds.batch(config.batch_size, drop_remainder=True) eval_ds = eval_ds.batch(config.eval_batch_size, drop_remainder=True) return train_ds, eval_ds
def create_dataset(dataset_path, do_train, config, repeat_num=1): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. config(struct): the config of train and eval in diffirent platform. repeat_num(int): the repeat times of dataset. Default: 1. Returns: dataset """ if config.platform == "Ascend": rank_size = int(os.getenv("RANK_SIZE", '1')) rank_id = int(os.getenv("RANK_ID", '0')) if rank_size == 1: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=rank_size, shard_id=rank_id) elif config.platform == "GPU": if do_train: if config.run_distribute: from mindspore.communication.management import get_rank, get_group_size ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=get_group_size(), shard_id=get_rank()) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) elif config.platform == "CPU": ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) resize_height = config.image_height resize_width = config.image_width buffer_size = 1000 # define map operations decode_op = C.Decode() resize_crop_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) resize_op = C.Resize((256, 256)) center_crop = C.CenterCrop(resize_width) rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) change_swap_op = C.HWC2CHW() if do_train: trans = [ resize_crop_op, horizontal_flip_op, rescale_op, normalize_op, change_swap_op ] else: trans = [ decode_op, resize_op, center_crop, normalize_op, change_swap_op ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=8) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) # apply shuffle operations ds = ds.shuffle(buffer_size=buffer_size) # apply batch operations ds = ds.batch(config.batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1. batch_size(int): the batch size of dataset. Default: 32. Returns: dataset """ if device_target == "Ascend": rank_size = int(os.getenv("RANK_SIZE")) rank_id = int(os.getenv("RANK_ID")) columns_list = ['image', 'label'] if config.data_load_mode == "mindrecord": load_func = partial(de.MindDataset, dataset_path, columns_list) else: load_func = partial(de.ImageFolderDataset, dataset_path) if do_train: if rank_size == 1: ds = load_func(num_parallel_workers=8, shuffle=True) else: ds = load_func(num_parallel_workers=8, shuffle=True, num_shards=rank_size, shard_id=rank_id) else: ds = load_func(num_parallel_workers=8, shuffle=False) elif device_target == "GPU": if do_train: from mindspore.communication.management import get_rank, get_group_size ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=get_group_size(), shard_id=get_rank()) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: raise ValueError("Unsupported device_target.") resize_height = config.image_height if do_train: buffer_size = 20480 # apply shuffle operations ds = ds.shuffle(buffer_size=buffer_size) # define map operations decode_op = C.Decode() resize_crop_decode_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) resize_op = C.Resize(256) center_crop = C.CenterCrop(resize_height) normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) change_swap_op = C.HWC2CHW() if do_train: trans = [ resize_crop_decode_op, horizontal_flip_op, normalize_op, change_swap_op ] else: trans = [ decode_op, resize_op, center_crop, normalize_op, change_swap_op ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=16) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32, run_distribute=False): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 Returns: dataset """ if device_target == "GPU": if do_train: if run_distribute: from mindspore.communication.management import get_rank, get_group_size data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=get_group_size(), shard_id=get_rank()) else: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: raise ValueError("Unsupported device_target.") resize_height = config.image_height resize_width = config.image_width buffer_size = 1000 # define map operations decode_op = C.Decode() resize_crop_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) resize_op = C.Resize(256) center_crop = C.CenterCrop(resize_width) rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) change_swap_op = C.HWC2CHW() if do_train: trans = [ resize_crop_op, horizontal_flip_op, rescale_op, normalize_op, change_swap_op ] else: trans = [ decode_op, resize_op, center_crop, normalize_op, change_swap_op ] type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=8) data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) # apply shuffle operations data_set = data_set.shuffle(buffer_size=buffer_size) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set
def create_dataset(data_dir, repeat=400, train_batch_size=16, augment=False, cross_val_ind=1, run_distribute=False, do_crop=None, img_size=None): images = _load_multipage_tiff(os.path.join(data_dir, 'train-volume.tif')) masks = _load_multipage_tiff(os.path.join(data_dir, 'train-labels.tif')) train_indices, val_indices = _get_val_train_indices( len(images), cross_val_ind) train_images = images[train_indices] train_masks = masks[train_indices] train_images = np.repeat(train_images, repeat, axis=0) train_masks = np.repeat(train_masks, repeat, axis=0) val_images = images[val_indices] val_masks = masks[val_indices] train_image_data = {"image": train_images} train_mask_data = {"mask": train_masks} valid_image_data = {"image": val_images} valid_mask_data = {"mask": val_masks} ds_train_images = ds.NumpySlicesDataset(data=train_image_data, sampler=None, shuffle=False) ds_train_masks = ds.NumpySlicesDataset(data=train_mask_data, sampler=None, shuffle=False) if run_distribute: rank_id = get_rank() rank_size = get_group_size() ds_train_images = ds.NumpySlicesDataset(data=train_image_data, sampler=None, shuffle=False, num_shards=rank_size, shard_id=rank_id) ds_train_masks = ds.NumpySlicesDataset(data=train_mask_data, sampler=None, shuffle=False, num_shards=rank_size, shard_id=rank_id) ds_valid_images = ds.NumpySlicesDataset(data=valid_image_data, sampler=None, shuffle=False) ds_valid_masks = ds.NumpySlicesDataset(data=valid_mask_data, sampler=None, shuffle=False) if do_crop: resize_size = [ int(img_size[x] * do_crop[x]) for x in range(len(img_size)) ] else: resize_size = img_size c_resize_op = c_vision.Resize(size=(resize_size[0], resize_size[1]), interpolation=Inter.BILINEAR) c_pad = c_vision.Pad(padding=(img_size[0] - resize_size[0]) // 2) c_rescale_image = c_vision.Rescale(1.0 / 127.5, -1) c_rescale_mask = c_vision.Rescale(1.0 / 255.0, 0) c_trans_normalize_img = [c_rescale_image, c_resize_op, c_pad] c_trans_normalize_mask = [c_rescale_mask, c_resize_op, c_pad] c_center_crop = c_vision.CenterCrop(size=388) train_image_ds = ds_train_images.map(input_columns="image", operations=c_trans_normalize_img) train_mask_ds = ds_train_masks.map(input_columns="mask", operations=c_trans_normalize_mask) train_ds = ds.zip((train_image_ds, train_mask_ds)) train_ds = train_ds.project(columns=["image", "mask"]) if augment: augment_process = train_data_augmentation c_resize_op = c_vision.Resize(size=(img_size[0], img_size[1]), interpolation=Inter.BILINEAR) train_ds = train_ds.map(input_columns=["image", "mask"], operations=augment_process) train_ds = train_ds.map(input_columns="image", operations=c_resize_op) train_ds = train_ds.map(input_columns="mask", operations=c_resize_op) if do_crop: train_ds = train_ds.map(input_columns="mask", operations=c_center_crop) post_process = data_post_process train_ds = train_ds.map(input_columns=["image", "mask"], operations=post_process) train_ds = train_ds.shuffle(repeat * 24) train_ds = train_ds.batch(batch_size=train_batch_size, drop_remainder=True) valid_image_ds = ds_valid_images.map(input_columns="image", operations=c_trans_normalize_img) valid_mask_ds = ds_valid_masks.map(input_columns="mask", operations=c_trans_normalize_mask) valid_ds = ds.zip((valid_image_ds, valid_mask_ds)) valid_ds = valid_ds.project(columns=["image", "mask"]) if do_crop: valid_ds = valid_ds.map(input_columns="mask", operations=c_center_crop) post_process = data_post_process valid_ds = valid_ds.map(input_columns=["image", "mask"], operations=post_process) valid_ds = valid_ds.batch(batch_size=1, drop_remainder=True) return train_ds, valid_ds
def create_dataset2(dataset_path, do_train=True, repeat_num=1, batch_size=32, target="gpu", rank=0, size=1): """ create a train or eval imagenet2012 dataset for resnet50 Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend Returns: dataset """ if target == "Ascend": device_num, rank_id = _get_rank_info() else: init() rank_id = rank device_num = size file_list = [os.path.join(dataset_path, f'train-{num:05d}-of-01024') for num in range(1024)] if device_num == 1: ds = msds.MindDataset(dataset_file=file_list, num_parallel_workers=8, shuffle=True) else: ds = msds.MindDataset(dataset_file=file_list, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] # define map operations if do_train: trans = [ C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.Normalize(mean=mean, std=std), C.HWC2CHW(), C2.TypeCast(mstype.float16) ] else: trans = [ C.Decode(), C.Resize(256), C.CenterCrop(image_size), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=8) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend Returns: dataset """ if target == "Ascend": device_num = int(os.getenv("RANK_SIZE")) rank_id = int(os.getenv("RANK_ID")) else: init() rank_id = get_rank() device_num = get_group_size() columns_list = ['image', 'label'] if config.data_load_mode == "mindrecord": load_func = partial(ds.MindDataset, dataset_path, columns_list) else: load_func = partial(ds.ImageFolderDataset, dataset_path) if device_num == 1: data_set = load_func(num_parallel_workers=8, shuffle=True) else: data_set = load_func(num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] # define map operations if do_train: trans = [ C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] else: trans = [ C.Decode(), C.Resize(256), C.CenterCrop(image_size), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=8) data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set
def classification_dataset(data_dir, image_size, per_batch_size, max_epoch, rank, group_size, mode='train', input_mode='folder', root='', num_parallel_workers=None, shuffle=None, sampler=None, class_indexing=None, drop_remainder=True, transform=None, target_transform=None): """ A function that returns a dataset for classification. The mode of input dataset could be "folder" or "txt". If it is "folder", all images within one folder have the same label. If it is "txt", all paths of images are written into a textfile. Args: data_dir (str): Path to the root directory that contains the dataset for "input_mode="folder"". Or path of the textfile that contains every image's path of the dataset. image_size (Union(int, sequence)): Size of the input images. per_batch_size (int): the batch size of evey step during training. max_epoch (int): the number of epochs. rank (int): The shard ID within num_shards (default=None). group_size (int): Number of shards that the dataset should be divided into (default=None). mode (str): "train" or others. Default: " train". input_mode (str): The form of the input dataset. "folder" or "txt". Default: "folder". root (str): the images path for "input_mode="txt"". Default: " ". num_parallel_workers (int): Number of workers to read the data. Default: None. shuffle (bool): Whether or not to perform shuffle on the dataset (default=None, performs shuffle). sampler (Sampler): Object used to choose samples from the dataset. Default: None. class_indexing (dict): A str-to-int mapping from folder name to index (default=None, the folder names will be sorted alphabetically and each class will be given a unique index starting from 0). Examples: >>> from src.dataset import classification_dataset >>> # path to imagefolder directory. This directory needs to contain sub-directories which contain the images >>> data_dir = "/path/to/imagefolder_directory" >>> de_dataset = classification_dataset(data_dir, image_size=[224, 244], >>> per_batch_size=64, max_epoch=100, >>> rank=0, group_size=4) >>> # Path of the textfile that contains every image's path of the dataset. >>> data_dir = "/path/to/dataset/images/train.txt" >>> images_dir = "/path/to/dataset/images" >>> de_dataset = classification_dataset(data_dir, image_size=[224, 244], >>> per_batch_size=64, max_epoch=100, >>> rank=0, group_size=4, >>> input_mode="txt", root=images_dir) """ mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] if transform is None: if mode == 'train': transform_img = [ V_C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), V_C.RandomHorizontalFlip(prob=0.5), V_C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4), V_C.Normalize(mean=mean, std=std), V_C.HWC2CHW() ] else: transform_img = [ V_C.Decode(), V_C.Resize((256, 256)), V_C.CenterCrop(image_size), V_C.Normalize(mean=mean, std=std), V_C.HWC2CHW() ] else: transform_img = transform if target_transform is None: transform_label = [C.TypeCast(mstype.int32)] else: transform_label = target_transform if input_mode == 'folder': de_dataset = de.ImageFolderDataset( data_dir, num_parallel_workers=num_parallel_workers, shuffle=shuffle, sampler=sampler, class_indexing=class_indexing, num_shards=group_size, shard_id=rank) else: dataset = TxtDataset(root, data_dir) sampler = DistributedSampler(dataset, rank, group_size, shuffle=shuffle) de_dataset = de.GeneratorDataset(dataset, ["image", "label"], sampler=sampler) de_dataset = de_dataset.map(operations=transform_img, input_columns="image", num_parallel_workers=num_parallel_workers) de_dataset = de_dataset.map(operations=transform_label, input_columns="label", num_parallel_workers=num_parallel_workers) columns_to_project = ["image", "label"] de_dataset = de_dataset.project(columns=columns_to_project) de_dataset = de_dataset.batch(per_batch_size, drop_remainder=drop_remainder) de_dataset = de_dataset.repeat(max_epoch) return de_dataset
def create_dataset_imagenet(dataset_path, batch_size=32, repeat_num=1, training=True, num_parallel_workers=None, shuffle=None, sampler=None, class_indexing=None): """ create a train or eval imagenet2012 dataset for resnet50 Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend Returns: dataset """ device_num, rank_id = _get_rank_info() cfg = alexnet_imagenet_cfg num_parallel_workers = 16 if device_num == 1: num_parallel_workers = 48 ds.config.set_prefetch_size(8) else: ds.config.set_numa_enable(True) data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=4, shuffle=shuffle, sampler=sampler, class_indexing=class_indexing, num_shards=device_num, shard_id=rank_id) assert cfg.image_height == cfg.image_width, "imagenet_cfg.image_height not equal imagenet_cfg.image_width" image_size = cfg.image_height mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] # define map operations if training: transform_img = [ CV.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), CV.RandomHorizontalFlip(prob=0.5), CV.Normalize(mean=mean, std=std), CV.HWC2CHW() ] else: transform_img = [ CV.Decode(), CV.Resize((256, 256)), CV.CenterCrop(image_size), CV.Normalize(mean=mean, std=std), CV.HWC2CHW() ] data_set = data_set.map(input_columns="image", num_parallel_workers=num_parallel_workers, operations=transform_img) data_set = data_set.batch(batch_size, drop_remainder=True) # apply dataset repeat operation if repeat_num > 1: data_set = data_set.repeat(repeat_num) return data_set
def create_dataset2(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend", distribute=False): """ create a train or eval imagenet2012 dataset for resnet50 Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend distribute(bool): data for distribute or not. Default: False Returns: dataset """ if target == "Ascend": device_num, rank_id = _get_rank_info() else: if distribute: init() rank_id = get_rank() device_num = get_group_size() else: device_num = 1 if device_num == 1: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] # define map operations if do_train: trans = [ C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] else: trans = [ C.Decode(), C.Resize(256), C.CenterCrop(image_size), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=8) data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set
def create_dataset_imagenet(dataset_path, repeat_num=1, training=True, num_parallel_workers=None, shuffle=None): """ create a train or eval imagenet2012 dataset for resnet50 Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend Returns: dataset """ device_num, rank_id = _get_rank_info() if device_num == 1: data_set = ds.ImageFolderDataset( dataset_path, num_parallel_workers=num_parallel_workers, shuffle=shuffle) else: data_set = ds.ImageFolderDataset( dataset_path, num_parallel_workers=num_parallel_workers, shuffle=shuffle, num_shards=device_num, shard_id=rank_id) assert imagenet_cfg.image_height == imagenet_cfg.image_width, "image_height not equal image_width" image_size = imagenet_cfg.image_height mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] # define map operations if training: transform_img = [ vision.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), vision.RandomHorizontalFlip(prob=0.5), vision.RandomColorAdjust(0.4, 0.4, 0.4, 0.1), vision.Normalize(mean=mean, std=std), vision.HWC2CHW() ] else: transform_img = [ vision.Decode(), vision.Resize(256), vision.CenterCrop(image_size), vision.Normalize(mean=mean, std=std), vision.HWC2CHW() ] transform_label = [C.TypeCast(mstype.int32)] data_set = data_set.map(input_columns="image", num_parallel_workers=12, operations=transform_img) data_set = data_set.map(input_columns="label", num_parallel_workers=4, operations=transform_label) # apply batch operations data_set = data_set.batch(imagenet_cfg.batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set