def util_test_random_color_adjust_error(brightness=(1, 1), contrast=(1, 1), saturation=(1, 1), hue=(0, 0)): """ Util function that tests the error message in case of grayscale images """ transforms = [ py_vision.Decode(), py_vision.Grayscale(1), py_vision.ToTensor(), (lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8)) ] transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) data1 = data1.map(operations=transform, input_columns=["image"]) # if input is grayscale, the output dimensions should be single channel, the following should fail random_adjust_op = c_vision.RandomColorAdjust(brightness=brightness, contrast=contrast, saturation=saturation, hue=hue) with pytest.raises(RuntimeError) as info: data1 = data1.map(operations=random_adjust_op, input_columns=["image"]) dataset_shape_1 = [] for item1 in data1.create_dict_iterator(num_epochs=1): c_image = item1["image"] dataset_shape_1.append(c_image.shape) error_msg = "image shape is not <H,W,C>" assert error_msg in str(info.value)
def create_ssd_dataset(mindrecord_file, batch_size=32, repeat_num=10, device_num=1, rank=0, is_training=True, num_parallel_workers=4, use_multiprocessing=True): """Create SSD dataset with MindDataset.""" ds = de.MindDataset(mindrecord_file, columns_list=["img_id", "image", "annotation"], num_shards=device_num, shard_id=rank, num_parallel_workers=num_parallel_workers, shuffle=is_training) decode = C.Decode() ds = ds.map(operations=decode, input_columns=["image"]) change_swap_op = C.HWC2CHW() normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) color_adjust_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) compose_map_func = (lambda img_id, image, annotation: preprocess_fn(img_id, image, annotation, is_training)) if is_training: output_columns = ["image", "box", "label", "num_match"] trans = [color_adjust_op, normalize_op, change_swap_op] else: output_columns = ["img_id", "image", "image_shape"] trans = [normalize_op, change_swap_op] ds = ds.map(operations=compose_map_func, input_columns=["img_id", "image", "annotation"], output_columns=output_columns, column_order=output_columns, python_multiprocessing=use_multiprocessing, num_parallel_workers=num_parallel_workers) ds = ds.map(operations=trans, input_columns=["image"], python_multiprocessing=use_multiprocessing, num_parallel_workers=num_parallel_workers) ds = ds.batch(batch_size, drop_remainder=True) ds = ds.repeat(repeat_num) return ds
def test_random_color_adjust_md5(): """ Test RandomColorAdjust with md5 check """ logger.info("Test RandomColorAdjust with md5 check") original_seed = config_get_set_seed(10) original_num_parallel_workers = config_get_set_num_parallel_workers(1) # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) decode_op = c_vision.Decode() random_adjust_op = c_vision.RandomColorAdjust(0.4, 0.4, 0.4, 0.1) data1 = data1.map(operations=decode_op, input_columns=["image"]) data1 = data1.map(operations=random_adjust_op, input_columns=["image"]) # Second dataset transforms = [ py_vision.Decode(), py_vision.RandomColorAdjust(0.4, 0.4, 0.4, 0.1), py_vision.ToTensor() ] transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) data2 = data2.map(operations=transform, input_columns=["image"]) # Compare with expected md5 from images filename = "random_color_adjust_01_c_result.npz" save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN) filename = "random_color_adjust_01_py_result.npz" save_and_check_md5(data2, filename, generate_golden=GENERATE_GOLDEN) # Restore configuration ds.config.set_seed(original_seed) ds.config.set_num_parallel_workers(original_num_parallel_workers)
def get_de_dataset(args): '''Get de_dataset.''' transform_label = [C.TypeCast(mstype.int32)] transform_img = [ VC.Decode(), VC.Resize((96, 64)), VC.RandomColorAdjust(brightness=0.3, contrast=0.3, saturation=0.3, hue=0), VC.RandomHorizontalFlip(), VC.Normalize((127.5, 127.5, 127.5), (127.5, 127.5, 127.5)), VC.HWC2CHW() ] de_dataset = de.ImageFolderDataset(dataset_dir=args.data_dir, num_shards=args.world_size, shard_id=args.local_rank, shuffle=True) de_dataset = de_dataset.map(input_columns="image", operations=transform_img) de_dataset = de_dataset.map(input_columns="label", operations=transform_label) de_dataset = de_dataset.project(columns=["image", "label"]) de_dataset = de_dataset.batch(args.per_batch_size, drop_remainder=True) num_iter_per_gpu = de_dataset.get_dataset_size() de_dataset = de_dataset.repeat(args.max_epoch) num_classes = de_dataset.num_classes() return de_dataset, num_iter_per_gpu, num_classes
def test_cpp_uniform_augment(plot=False, num_ops=2): """ Test UniformAugment """ logger.info("Test CPP UniformAugment") # Original Images data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) transforms_original = [C.Decode(), C.Resize(size=[224, 224]), F.ToTensor()] ds_original = data_set.map(operations=transforms_original, input_columns="image") ds_original = ds_original.batch(512) for idx, (image, _) in enumerate(ds_original): if idx == 0: images_original = np.transpose(image.asnumpy(), (0, 2, 3, 1)) else: images_original = np.append(images_original, np.transpose(image.asnumpy(), (0, 2, 3, 1)), axis=0) # UniformAugment Images data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) transforms_ua = [C.RandomCrop(size=[224, 224], padding=[32, 32, 32, 32]), C.RandomHorizontalFlip(), C.RandomVerticalFlip(), C.RandomColorAdjust(), C.RandomRotation(degrees=45)] uni_aug = C.UniformAugment(transforms=transforms_ua, num_ops=num_ops) transforms_all = [C.Decode(), C.Resize(size=[224, 224]), uni_aug, F.ToTensor()] ds_ua = data_set.map(operations=transforms_all, input_columns="image", num_parallel_workers=1) ds_ua = ds_ua.batch(512) for idx, (image, _) in enumerate(ds_ua): if idx == 0: images_ua = np.transpose(image.asnumpy(), (0, 2, 3, 1)) else: images_ua = np.append(images_ua, np.transpose(image.asnumpy(), (0, 2, 3, 1)), axis=0) if plot: visualize_list(images_original, images_ua) num_samples = images_original.shape[0] mse = np.zeros(num_samples) for i in range(num_samples): mse[i] = diff_mse(images_ua[i], images_original[i]) logger.info("MSE= {}".format(str(np.mean(mse))))
def create_dataset(dataset_path, config, do_train, repeat_num=1): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. config(dict): config of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1. Returns: dataset """ rank = config.rank group_size = config.group_size if group_size == 1: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=True) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=True, num_shards=group_size, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(config.image_size), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode ] else: trans = [ C.Decode(), C.Resize(int(config.image_size / 0.875)), C.CenterCrop(config.image_size) ] trans += [ C.Rescale(1.0 / 255.0, 0.0), C.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=config.work_nums) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=config.work_nums) # apply batch operations ds = ds.batch(config.batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, batch_size=16, device_num=1, rank=0): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. batch_size(int): the batch size of dataset. Default: 16. device_num (int): Number of shards that the dataset should be divided into (default=1). rank (int): The shard ID within num_shards (default=0). Returns: dataset """ if device_num == 1: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(299), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) ] else: trans = [C.Decode(), C.Resize(320), C.CenterCrop(299)] trans += [ C.Normalize(mean=[127.5, 127.5, 127.5], std=[127.5, 127.5, 127.5]), C.HWC2CHW(), C2.TypeCast(mstype.float32) ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8) ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): """ Create a train or eval dataset. Args: dataset_path (str): The path of dataset. do_train (bool): Whether dataset is used for train or eval. repeat_num (int): The repeat times of dataset. Default: 1. batch_size (int): The batch size of dataset. Default: 32. Returns: Dataset. """ do_shuffle = bool(do_train) if device_num == 1 or not do_train: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=do_shuffle) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=do_shuffle, num_shards=device_num, shard_id=device_id) image_length = 299 if do_train: trans = [ C.RandomCropDecodeResize(image_length, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) ] else: trans = [ C.Decode(), C.Resize(image_length), C.CenterCrop(image_length) ] trans += [ C.Rescale(1.0 / 255.0, 0.0), C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=config.work_nums) ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=config.work_nums) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. rank (int): The shard ID within num_shards (default=None). group_size (int): Number of shards that the dataset should be divided into (default=None). repeat_num(int): the repeat times of dataset. Default: 1. Returns: dataset """ if group_size == 1: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True, num_shards=group_size, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(299, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) ] else: trans = [C.Decode(), C.Resize(299), C.CenterCrop(299)] trans += [ C.Rescale(1.0 / 255.0, 0.0), C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=cfg.work_nums) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=cfg.work_nums) # apply batch operations ds = ds.batch(cfg.batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def util_test_random_color_adjust_op(brightness=(1, 1), contrast=(1, 1), saturation=(1, 1), hue=(0, 0), plot=False): """ Util function that tests RandomColorAdjust for a specific argument """ # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) decode_op = c_vision.Decode() random_adjust_op = c_vision.RandomColorAdjust(brightness=brightness, contrast=contrast, saturation=saturation, hue=hue) ctrans = [decode_op, random_adjust_op, ] data1 = data1.map(operations=ctrans, input_columns=["image"]) # Second dataset transforms = [ py_vision.Decode(), py_vision.RandomColorAdjust(brightness=brightness, contrast=contrast, saturation=saturation, hue=hue), py_vision.ToTensor() ] transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) data2 = data2.map(operations=transform, input_columns=["image"]) num_iter = 0 for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True), data2.create_dict_iterator(num_epochs=1, output_numpy=True)): num_iter += 1 c_image = item1["image"] py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) logger.info("shape of c_image: {}".format(c_image.shape)) logger.info("shape of py_image: {}".format(py_image.shape)) logger.info("dtype of c_image: {}".format(c_image.dtype)) logger.info("dtype of py_image: {}".format(py_image.dtype)) mse = diff_mse(c_image, py_image) logger.info("mse is {}".format(mse)) logger.info("random_rotation_op_{}, mse: {}".format(num_iter + 1, mse)) assert mse < 0.01 if plot: visualize_image(c_image, py_image, mse)
def create_dataset(dataset_path, do_train, device_num=1, rank=0): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. rank (int): The shard ID within num_shards (default=None). group_size (int): Number of shards that the dataset should be divided into (default=None). repeat_num(int): the repeat times of dataset. Default: 1. Returns: dataset """ if device_num == 1: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(224), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) ] else: trans = [C.Decode(), C.Resize(239), C.CenterCrop(224)] trans += [ C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]), C.HWC2CHW(), ] type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(input_columns="image", operations=trans, num_parallel_workers=8) data_set = data_set.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) # apply batch operations data_set = data_set.batch(config.batch_size, drop_remainder=True) return data_set
def create_dataset(data_path, is_train=True, batch_size=32): # import import mindspore.common.dtype as mstype import mindspore.dataset.engine as de import mindspore.dataset.transforms.c_transforms as C2 import mindspore.dataset.vision.c_transforms as C from mindspore.common import set_seed set_seed(1) # shard num_shards = shard_id = None rand_size = os.getenv("RANK_SIZE") rand_id = os.getenv("RANK_ID") if rand_size is not None and rand_id is not None: num_shards = int(rand_size) shard_id = int(rand_id) # define dataset data_path = os.path.join(data_path, "train" if is_train else "val") ds = de.ImageFolderDataset(data_path, shuffle=True, num_parallel_workers=8, num_shards=num_shards, shard_id=shard_id, num_samples=None) # define ops comps_ops = list() # train or val if is_train: comps_ops.append(C.RandomCropDecodeResize(224, scale=(0.08, 1.0), ratio=(0.75, 1.333))) comps_ops.append(C.RandomHorizontalFlip(prob=0.5)) comps_ops.append(C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)) else: comps_ops.append(C.Decode()) comps_ops.append(C.Resize(224)) comps_ops.append(C.CenterCrop(224)) comps_ops.append(C.Rescale(1 / 255.0, 0.)) comps_ops.append(C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])) comps_ops.append(C.HWC2CHW()) # map ops ds = ds.map(input_columns=["image"], operations=comps_ops, num_parallel_workers=8) ds = ds.map(input_columns=["label"], operations=C2.TypeCast(mstype.int32), num_parallel_workers=8) # batch & repeat ds = ds.batch(batch_size=batch_size, drop_remainder=is_train) ds = ds.repeat(count=1) return ds
def test_cpp_uniform_augment_exception_float_numops(num_ops=2.5): """ Test UniformAugment invalid float number of ops """ logger.info("Test CPP UniformAugment invalid float num_ops exception") transforms_ua = [C.RandomCrop(size=[224, 224], padding=[32, 32, 32, 32]), C.RandomHorizontalFlip(), C.RandomVerticalFlip(), C.RandomColorAdjust(), C.RandomRotation(degrees=45)] try: _ = C.UniformAugment(transforms=transforms_ua, num_ops=num_ops) except Exception as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "Argument num_ops with value 2.5 is not of type (<class 'int'>,)" in str(e)
def test_cpp_uniform_augment_exception_nonpositive_numops(num_ops=0): """ Test UniformAugment invalid non-positive number of ops """ logger.info("Test CPP UniformAugment invalid non-positive num_ops exception") transforms_ua = [C.RandomCrop(size=[224, 224], padding=[32, 32, 32, 32]), C.RandomHorizontalFlip(), C.RandomVerticalFlip(), C.RandomColorAdjust(), C.RandomRotation(degrees=45)] try: _ = C.UniformAugment(transforms=transforms_ua, num_ops=num_ops) except Exception as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "Input num_ops must be greater than 0" in str(e)
def test_cpp_uniform_augment_exception_pyops(num_ops=2): """ Test UniformAugment invalid op in operations """ logger.info("Test CPP UniformAugment invalid OP exception") transforms_ua = [C.RandomCrop(size=[224, 224], padding=[32, 32, 32, 32]), C.RandomHorizontalFlip(), C.RandomVerticalFlip(), C.RandomColorAdjust(), C.RandomRotation(degrees=45), F.Invert()] with pytest.raises(TypeError) as e: C.UniformAugment(transforms=transforms_ua, num_ops=num_ops) logger.info("Got an exception in DE: {}".format(str(e))) assert "Type of Transforms[5] must be c_transform" in str(e.value)
def test_serdes_voc_dataset(remove_json_files=True): """ Test serdes on VOC dataset pipeline. """ data_dir = "../data/dataset/testVOC2012" original_seed = config_get_set_seed(1) original_num_parallel_workers = config_get_set_num_parallel_workers(1) # define map operations random_color_adjust_op = vision.RandomColorAdjust(brightness=(0.5, 0.5)) random_rotation_op = vision.RandomRotation((0, 90), expand=True, resample=Inter.BILINEAR, center=(50, 50), fill_value=150) data1 = ds.VOCDataset(data_dir, task="Detection", usage="train", decode=True) data1 = data1.map(operations=random_color_adjust_op, input_columns=["image"]) data1 = data1.map(operations=random_rotation_op, input_columns=["image"]) data1 = data1.skip(2) data2 = util_check_serialize_deserialize_file(data1, "voc_dataset_pipeline", remove_json_files) num_samples = 0 # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) for item1, item2 in zip( data1.create_dict_iterator(num_epochs=1, output_numpy=True), data2.create_dict_iterator(num_epochs=1, output_numpy=True)): np.testing.assert_array_equal(item1['image'], item2['image']) num_samples += 1 assert num_samples == 7 # Restore configuration num_parallel_workers ds.config.set_seed(original_seed) ds.config.set_num_parallel_workers(original_num_parallel_workers)
def test_cpp_uniform_augment_exception_pyops(num_ops=2): """ Test UniformAugment invalid op in operations """ logger.info("Test CPP UniformAugment invalid OP exception") transforms_ua = [C.RandomCrop(size=[224, 224], padding=[32, 32, 32, 32]), C.RandomHorizontalFlip(), C.RandomVerticalFlip(), C.RandomColorAdjust(), C.RandomRotation(degrees=45), F.Invert()] with pytest.raises(TypeError) as e: C.UniformAugment(transforms=transforms_ua, num_ops=num_ops) logger.info("Got an exception in DE: {}".format(str(e))) assert "Argument transforms[5] with value" \ " <mindspore.dataset.vision.py_transforms.Invert" in str(e.value) assert "is not of type (<class 'mindspore._c_dataengine.TensorOp'>,"\ " <class 'mindspore._c_dataengine.TensorOperation'>)" in str(e.value)
def create_dataset_cifar(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): """ create a train or evaluate cifar10 dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend Returns: dataset """ if target == "Ascend": device_num, rank_id = _get_rank_info() elif target == "CPU": device_num = 1 else: init() rank_id = get_rank() device_num = get_group_size() if device_num == 1: data_set = ds.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) else: data_set = ds.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) # define map operations if do_train: trans = [ C.RandomCrop((32, 32), (4, 4, 4, 4)), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4), C.Resize((227, 227)), C.Rescale(1.0 / 255.0, 0.0), C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), C.CutOut(112), C.HWC2CHW() ] else: trans = [ C.Resize((227, 227)), C.Rescale(1.0 / 255.0, 0.0), C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=8) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set
def create_dataset(dataset_path, config, buffer_size=1000): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. config(struct): the config of train and eval in diffirent platform. repeat_num(int): the repeat times of dataset. Default: 1. Returns: train_dataset, val_dataset """ ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=4) train_ds, eval_ds = ds.split(config.data_split, randomize=True) resize_height = config.image_height resize_width = config.image_width # define operations mapping to each sample normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) change_swap_op = C.HWC2CHW() type_cast_op = C2.TypeCast(mstype.int32) # operations for training crop_decode_resize = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) color_adjust = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) # operations for inference decode_op = C.Decode() resize_op = C.Resize((256, 256)) center_crop = C.CenterCrop(resize_width) train_trans = [ crop_decode_resize, horizontal_flip_op, color_adjust, normalize_op, change_swap_op ] train_ds = train_ds.map(input_columns="image", operations=train_trans, num_parallel_workers=4) train_ds = train_ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=4) eval_trans = [ decode_op, resize_op, center_crop, normalize_op, change_swap_op ] eval_ds = eval_ds.map(input_columns="image", operations=eval_trans, num_parallel_workers=4) eval_ds = eval_ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=4) # apply shuffle operations train_ds = train_ds.shuffle(buffer_size=buffer_size) # apply batch operations train_ds = train_ds.batch(config.batch_size, drop_remainder=True) eval_ds = eval_ds.batch(config.eval_batch_size, drop_remainder=True) return train_ds, eval_ds
def create_dataset_imagenet(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): """ create a train or eval imagenet dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend Returns: dataset """ if target == "Ascend": device_num, rank_id = _get_rank_info() else: init() rank_id = get_rank() device_num = get_group_size() if device_num == 1: data_set = ds.ImageFolderDataset(dataset_path, shuffle=True) else: data_set = ds.ImageFolderDataset(dataset_path, shuffle=True, num_shards=device_num, shard_id=rank_id) image_size = 227 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] # define map operations if do_train: trans = [ C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4), C.Normalize(mean=mean, std=std), C.CutOut(112), C.HWC2CHW() ] else: trans = [ C.Decode(), C.Resize((256, 256)), C.CenterCrop(image_size), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(operations=type_cast_op, input_columns="label") data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=10) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set
def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32, run_distribute=False): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 Returns: dataset """ if device_target == "GPU": if do_train: if run_distribute: from mindspore.communication.management import get_rank, get_group_size data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=get_group_size(), shard_id=get_rank()) else: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: raise ValueError("Unsupported device_target.") resize_height = config.image_height resize_width = config.image_width buffer_size = 1000 # define map operations decode_op = C.Decode() resize_crop_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) resize_op = C.Resize(256) center_crop = C.CenterCrop(resize_width) rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) change_swap_op = C.HWC2CHW() if do_train: trans = [ resize_crop_op, horizontal_flip_op, rescale_op, normalize_op, change_swap_op ] else: trans = [ decode_op, resize_op, center_crop, normalize_op, change_swap_op ] type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=8) data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) # apply shuffle operations data_set = data_set.shuffle(buffer_size=buffer_size) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set
def create_dataset(dataset_path, do_train, config, repeat_num=1): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. config(struct): the config of train and eval in diffirent platform. repeat_num(int): the repeat times of dataset. Default: 1. Returns: dataset """ if config.platform == "Ascend": rank_size = int(os.getenv("RANK_SIZE", '1')) rank_id = int(os.getenv("RANK_ID", '0')) if rank_size == 1: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=rank_size, shard_id=rank_id) elif config.platform == "GPU": if do_train: if config.run_distribute: from mindspore.communication.management import get_rank, get_group_size ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=get_group_size(), shard_id=get_rank()) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) elif config.platform == "CPU": ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) resize_height = config.image_height resize_width = config.image_width buffer_size = 1000 # define map operations decode_op = C.Decode() resize_crop_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) resize_op = C.Resize((256, 256)) center_crop = C.CenterCrop(resize_width) rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) change_swap_op = C.HWC2CHW() if do_train: trans = [ resize_crop_op, horizontal_flip_op, rescale_op, normalize_op, change_swap_op ] else: trans = [ decode_op, resize_op, center_crop, normalize_op, change_swap_op ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=8) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) # apply shuffle operations ds = ds.shuffle(buffer_size=buffer_size) # apply batch operations ds = ds.batch(config.batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(args, dataset_mode, repeat_num=1): """ create a train or evaluate cifar10 dataset for SimCLR """ if args.dataset_name != "cifar10": raise ValueError("Unsupported dataset.") if dataset_mode in ("train_endcoder", "train_classifier"): dataset_path = args.train_dataset_path else: dataset_path = args.eval_dataset_path if args.run_distribute and args.device_target == "Ascend": data_set = ds.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=args.device_num, shard_id=args.device_id) else: data_set = ds.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) # define map operations trans = [] if dataset_mode == "train_endcoder": if args.use_crop: trans += [C.Resize(256, interpolation=Inter.BICUBIC)] trans += [ C.RandomResizedCrop(size=(32, 32), scale=(0.31, 1), interpolation=Inter.BICUBIC, max_attempts=100) ] if args.use_flip: trans += [C.RandomHorizontalFlip(prob=0.5)] if args.use_color_jitter: scale = 0.6 color_jitter = C.RandomColorAdjust(0.8 * scale, 0.8 * scale, 0.8 * scale, 0.2 * scale) trans += [C2.RandomApply([color_jitter], prob=0.8)] if args.use_color_gray: trans += [ py_vision.ToPIL(), py_vision.RandomGrayscale(prob=0.2), np.array ] # need to convert PIL image to a NumPy array to pass it to C++ operation if args.use_blur: trans += [C2.RandomApply([gaussian_blur], prob=0.8)] if args.use_norm: trans += [ C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ] trans += [C2.TypeCast(mstype.float32), C.HWC2CHW()] else: trans += [C.Resize(32)] trans += [C2.TypeCast(mstype.float32)] if args.use_norm: trans += [ C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ] trans += [C.HWC2CHW()] type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) data_set = data_set.map(operations=copy_column, input_columns=["image", "label"], output_columns=["image1", "image2", "label"], column_order=["image1", "image2", "label"], num_parallel_workers=8) data_set = data_set.map(operations=trans, input_columns=["image1"], num_parallel_workers=8) data_set = data_set.map(operations=trans, input_columns=["image2"], num_parallel_workers=8) # apply batch operations data_set = data_set.batch(args.batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set
def classification_dataset(data_dir, image_size, per_batch_size, max_epoch, rank, group_size, mode='train', input_mode='folder', root='', num_parallel_workers=None, shuffle=None, sampler=None, class_indexing=None, drop_remainder=True, transform=None, target_transform=None): """ A function that returns a dataset for classification. The mode of input dataset could be "folder" or "txt". If it is "folder", all images within one folder have the same label. If it is "txt", all paths of images are written into a textfile. Args: data_dir (str): Path to the root directory that contains the dataset for "input_mode="folder"". Or path of the textfile that contains every image's path of the dataset. image_size (Union(int, sequence)): Size of the input images. per_batch_size (int): the batch size of evey step during training. max_epoch (int): the number of epochs. rank (int): The shard ID within num_shards (default=None). group_size (int): Number of shards that the dataset should be divided into (default=None). mode (str): "train" or others. Default: " train". input_mode (str): The form of the input dataset. "folder" or "txt". Default: "folder". root (str): the images path for "input_mode="txt"". Default: " ". num_parallel_workers (int): Number of workers to read the data. Default: None. shuffle (bool): Whether or not to perform shuffle on the dataset (default=None, performs shuffle). sampler (Sampler): Object used to choose samples from the dataset. Default: None. class_indexing (dict): A str-to-int mapping from folder name to index (default=None, the folder names will be sorted alphabetically and each class will be given a unique index starting from 0). Examples: >>> from src.dataset import classification_dataset >>> # path to imagefolder directory. This directory needs to contain sub-directories which contain the images >>> data_dir = "/path/to/imagefolder_directory" >>> de_dataset = classification_dataset(data_dir, image_size=[224, 244], >>> per_batch_size=64, max_epoch=100, >>> rank=0, group_size=4) >>> # Path of the textfile that contains every image's path of the dataset. >>> data_dir = "/path/to/dataset/images/train.txt" >>> images_dir = "/path/to/dataset/images" >>> de_dataset = classification_dataset(data_dir, image_size=[224, 244], >>> per_batch_size=64, max_epoch=100, >>> rank=0, group_size=4, >>> input_mode="txt", root=images_dir) """ mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] if transform is None: if mode == 'train': transform_img = [ V_C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), V_C.RandomHorizontalFlip(prob=0.5), V_C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4), V_C.Normalize(mean=mean, std=std), V_C.HWC2CHW() ] else: transform_img = [ V_C.Decode(), V_C.Resize((256, 256)), V_C.CenterCrop(image_size), V_C.Normalize(mean=mean, std=std), V_C.HWC2CHW() ] else: transform_img = transform if target_transform is None: transform_label = [C.TypeCast(mstype.int32)] else: transform_label = target_transform if input_mode == 'folder': de_dataset = de.ImageFolderDataset( data_dir, num_parallel_workers=num_parallel_workers, shuffle=shuffle, sampler=sampler, class_indexing=class_indexing, num_shards=group_size, shard_id=rank) else: dataset = TxtDataset(root, data_dir) sampler = DistributedSampler(dataset, rank, group_size, shuffle=shuffle) de_dataset = de.GeneratorDataset(dataset, ["image", "label"], sampler=sampler) de_dataset = de_dataset.map(operations=transform_img, input_columns="image", num_parallel_workers=num_parallel_workers) de_dataset = de_dataset.map(operations=transform_label, input_columns="label", num_parallel_workers=num_parallel_workers) columns_to_project = ["image", "label"] de_dataset = de_dataset.project(columns=columns_to_project) de_dataset = de_dataset.batch(per_batch_size, drop_remainder=drop_remainder) de_dataset = de_dataset.repeat(max_epoch) return de_dataset
def classification_dataset_cifar10(data_dir, image_size, per_batch_size, max_epoch, rank, group_size, mode='train', num_parallel_workers=None, shuffle=None, sampler=None, drop_remainder=True, transform=None, target_transform=None): """ A function that returns cifar10 dataset for classification. Args: data_dir (str): Path to the root directory that contains the dataset's bin files. image_size (Union(int, sequence)): Size of the input images. per_batch_size (int): the batch size of evey step during training. max_epoch (int): the number of epochs. rank (int): The shard ID within num_shards (default=None). group_size (int): Number of shards that the dataset should be divided into (default=None). mode (str): "train" or others. Default: " train". input_mode (str): The form of the input dataset. "folder" or "txt". Default: "folder". root (str): the images path for "input_mode="txt"". Default: " ". num_parallel_workers (int): Number of workers to read the data. Default: None. shuffle (bool): Whether or not to perform shuffle on the dataset (default=None, performs shuffle). sampler (Sampler): Object used to choose samples from the dataset. Default: None. Examples: >>> from src.datasets.classification import classification_dataset_cifar10 >>> # path to imagefolder directory. This directory needs to contain bin files of data. >>> data_dir = "/path/to/datafolder_directory" >>> de_dataset = classification_dataset_cifar10(data_dir, image_size=[32, 32], >>> per_batch_size=64, max_epoch=100, >>> rank=0, group_size=1) """ mean = [0.5 * 255, 0.5 * 255, 0.5 * 255] std = [0.5 * 255, 0.5 * 255, 0.5 * 255] if transform is None: if mode == 'train': transform_img = [ vision_C.RandomCrop(image_size, padding=4), vision_C.RandomHorizontalFlip(prob=0.5), vision_C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4), vision_C.Normalize(mean=mean, std=std), vision_C.HWC2CHW() ] else: transform_img = [ vision_C.Normalize(mean=mean, std=std), vision_C.HWC2CHW() ] else: transform_img = transform if target_transform is None: transform_label = [normal_C.TypeCast(mstype.int32)] else: transform_label = target_transform de_dataset = de.Cifar10Dataset(data_dir, num_parallel_workers=num_parallel_workers, shuffle=shuffle, sampler=sampler, num_shards=group_size, shard_id=rank) de_dataset = de_dataset.map(input_columns="image", num_parallel_workers=8, operations=transform_img) de_dataset = de_dataset.map(input_columns="label", num_parallel_workers=8, operations=transform_label) columns_to_project = ["image", "label"] de_dataset = de_dataset.project(columns=columns_to_project) de_dataset = de_dataset.batch(per_batch_size, drop_remainder=drop_remainder) de_dataset = de_dataset.repeat(1) return de_dataset
def create_dataset_imagenet(dataset_path, repeat_num=1, training=True, num_parallel_workers=None, shuffle=None): """ create a train or eval imagenet2012 dataset for resnet50 Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend Returns: dataset """ device_num, rank_id = _get_rank_info() if device_num == 1: data_set = ds.ImageFolderDataset( dataset_path, num_parallel_workers=num_parallel_workers, shuffle=shuffle) else: data_set = ds.ImageFolderDataset( dataset_path, num_parallel_workers=num_parallel_workers, shuffle=shuffle, num_shards=device_num, shard_id=rank_id) assert imagenet_cfg.image_height == imagenet_cfg.image_width, "image_height not equal image_width" image_size = imagenet_cfg.image_height mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] # define map operations if training: transform_img = [ vision.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), vision.RandomHorizontalFlip(prob=0.5), vision.RandomColorAdjust(0.4, 0.4, 0.4, 0.1), vision.Normalize(mean=mean, std=std), vision.HWC2CHW() ] else: transform_img = [ vision.Decode(), vision.Resize(256), vision.CenterCrop(image_size), vision.Normalize(mean=mean, std=std), vision.HWC2CHW() ] transform_label = [C.TypeCast(mstype.int32)] data_set = data_set.map(input_columns="image", num_parallel_workers=12, operations=transform_img) data_set = data_set.map(input_columns="label", num_parallel_workers=4, operations=transform_label) # apply batch operations data_set = data_set.batch(imagenet_cfg.batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set
def create_dataset(dataset_path, do_train, config, platform, repeat_num=1, batch_size=100): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 Returns: dataset """ if platform == "Ascend": rank_size = int(os.getenv("RANK_SIZE")) rank_id = int(os.getenv("RANK_ID")) if rank_size == 1: data_set = ds.MindDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: data_set = ds.MindDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=rank_size, shard_id=rank_id) elif platform == "GPU": if do_train: from mindspore.communication.management import get_rank, get_group_size data_set = ds.MindDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=get_group_size(), shard_id=get_rank()) else: data_set = ds.MindDataset(dataset_path, num_parallel_workers=8, shuffle=False) else: raise ValueError("Unsupported platform.") resize_height = config.image_height buffer_size = 1000 # define map operations resize_crop_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) color_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) rescale_op = C.Rescale(1 / 255.0, 0) normalize_op = C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) change_swap_op = C.HWC2CHW() # define python operations decode_p = P.Decode() resize_p = P.Resize(256, interpolation=Inter.BILINEAR) center_crop_p = P.CenterCrop(224) totensor = P.ToTensor() normalize_p = P.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) composeop = P2.Compose( [decode_p, resize_p, center_crop_p, totensor, normalize_p]) if do_train: trans = [ resize_crop_op, horizontal_flip_op, color_op, rescale_op, normalize_op, change_swap_op ] else: trans = composeop type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(input_columns="image", operations=trans, num_parallel_workers=8) data_set = data_set.map(input_columns="label_list", operations=type_cast_op, num_parallel_workers=8) # apply shuffle operations data_set = data_set.shuffle(buffer_size=buffer_size) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set