def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): """ create a train or eval imagenet2012 dataset for resnet50 Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend Returns: dataset """ if target == "Ascend": device_num, rank_id = _get_rank_info() else: init("nccl") rank_id = get_rank() device_num = get_group_size() if device_num == 1: ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) else: ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] # define map operations if do_train: trans = [ C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] else: trans = [ C.Decode(), C.Resize(256), C.CenterCrop(image_size), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 Returns: dataset """ device_num = int(os.getenv("RANK_SIZE")) rank_id = int(os.getenv("RANK_ID")) if device_num == 1: ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False) else: ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] if do_train: transform_img = [ V_C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), V_C.RandomHorizontalFlip(prob=0.5), V_C.Normalize(mean=mean, std=std), V_C.HWC2CHW() ] else: transform_img = [ V_C.Decode(), V_C.Resize((256, 256)), V_C.CenterCrop(image_size), V_C.Normalize(mean=mean, std=std), V_C.HWC2CHW() ] # type_cast_op = C2.TypeCast(mstype.float16) type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="image", operations=transform_img, num_parallel_workers=8) ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) # apply shuffle operations # ds = ds.shuffle(buffer_size=config.buffer_size) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_icdar_train_dataset(mindrecord_file='icdar_train.mindrecord', batch_size=32, repeat_num=10, is_training=True, num_parallel_workers=1, length=512, scale=0.25): dataset = ds.MindDataset(mindrecord_file, columns_list=['image', 'annotation', 'label'], num_parallel_workers=num_parallel_workers, shuffle=False) dataset.set_dataset_size(1000) decode = C.Decode() dataset = dataset.map(input_columns=["image"], operations=decode, python_multiprocessing=is_training, num_parallel_workers=num_parallel_workers) change_swap_op = C.HWC2CHW() normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255]) color_adjust_op = C.RandomColorAdjust(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.25) compose_map_func = (lambda image, annotation, label: preprocess(image, annotation, label, length, scale)) output_columns = ["image", "score_map", "geo_map", "ignored_map"] dataset = dataset.map(input_columns=["image", "annotation", "label"], output_columns=output_columns, columns_order=output_columns, operations=compose_map_func, python_multiprocessing=is_training, num_parallel_workers=num_parallel_workers) trans = [normalize_op, change_swap_op] dataset = dataset.map(input_columns=["image"], operations=trans, python_multiprocessing=is_training, num_parallel_workers=num_parallel_workers) dataset = dataset.batch(batch_size, drop_remainder=True) dataset = dataset.repeat(repeat_num) return dataset
def test_normalize_op_c(plot=False): """ Test Normalize in cpp transformations """ logger.info("Test Normalize in cpp") mean = [121.0, 115.0, 100.0] std = [70.0, 68.0, 71.0] # define map operations decode_op = c_vision.Decode() normalize_op = c_vision.Normalize(mean, std) # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) data1 = data1.map(input_columns=["image"], operations=decode_op) data1 = data1.map(input_columns=["image"], operations=normalize_op) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) data2 = data2.map(input_columns=["image"], operations=decode_op) num_iter = 0 for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): image_de_normalized = item1["image"] image_original = item2["image"] image_np_normalized = normalize_np(image_original, mean, std) mse = diff_mse(image_de_normalized, image_np_normalized) logger.info("image_{}, mse: {}".format(num_iter + 1, mse)) assert mse < 0.01 if plot: visualize_mse(image_de_normalized, image_np_normalized, mse, image_original) num_iter += 1
def test_decode_normalize_op(): logger.info("Test [Decode, Normalize] in one Map") data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image", "label"], num_parallel_workers=1, shuffle=False) # define map operations decode_op = vision.Decode() normalize_op = vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0]) # apply map operations on images data1 = data1.map(input_columns=["image"], operations=[decode_op, normalize_op]) num_iter = 0 image = None for item in data1.create_dict_iterator(): logger.info("Looping inside iterator {}".format(num_iter)) image = item["image"] # plt.subplot(131) # plt.imshow(image) # plt.title("DE image") # plt.show() num_iter += 1
def create_dataset(data_path, batch_size=32, repeat_size=1, mode="train"): """ create dataset for train or test """ cifar_ds = ds.Cifar10Dataset(data_path) rescale = 1.0 / 255.0 shift = 0.0 resize_op = CV.Resize((cfg.image_height, cfg.image_width)) rescale_op = CV.Rescale(rescale, shift) normalize_op = CV.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) if mode == "train": random_crop_op = CV.RandomCrop([32, 32], [4, 4, 4, 4]) random_horizontal_op = CV.RandomHorizontalFlip() channel_swap_op = CV.HWC2CHW() typecast_op = C.TypeCast(mstype.int32) cifar_ds = cifar_ds.map(input_columns="label", operations=typecast_op) if mode == "train": cifar_ds = cifar_ds.map(input_columns="image", operations=random_crop_op) cifar_ds = cifar_ds.map(input_columns="image", operations=random_horizontal_op) cifar_ds = cifar_ds.map(input_columns="image", operations=resize_op) cifar_ds = cifar_ds.map(input_columns="image", operations=rescale_op) cifar_ds = cifar_ds.map(input_columns="image", operations=normalize_op) cifar_ds = cifar_ds.map(input_columns="image", operations=channel_swap_op) cifar_ds = cifar_ds.shuffle(buffer_size=cfg.buffer_size) cifar_ds = cifar_ds.batch(batch_size, drop_remainder=True) cifar_ds = cifar_ds.repeat(repeat_size) return cifar_ds
def create_ssd_dataset(mindrecord_file, batch_size=32, repeat_num=10, device_num=1, rank=0, is_training=True, num_parallel_workers=4): """Creatr SSD dataset with MindDataset.""" ds = de.MindDataset(mindrecord_file, columns_list=["img_id", "image", "annotation"], num_shards=device_num, shard_id=rank, num_parallel_workers=num_parallel_workers, shuffle=is_training) decode = C.Decode() ds = ds.map(input_columns=["image"], operations=decode) change_swap_op = C.HWC2CHW() normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255]) color_adjust_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) compose_map_func = (lambda img_id, image, annotation: preprocess_fn(img_id, image, annotation, is_training)) if is_training: output_columns = ["image", "box", "label", "num_match"] trans = [color_adjust_op, normalize_op, change_swap_op] else: output_columns = ["img_id", "image", "image_shape"] trans = [normalize_op, change_swap_op] ds = ds.map(input_columns=["img_id", "image", "annotation"], output_columns=output_columns, columns_order=output_columns, operations=compose_map_func, python_multiprocessing=is_training, num_parallel_workers=num_parallel_workers) ds = ds.map(input_columns=["image"], operations=trans, python_multiprocessing=is_training, num_parallel_workers=num_parallel_workers) ds = ds.batch(batch_size, drop_remainder=True) ds = ds.repeat(repeat_num) return ds
def test_decode_normalize_op(): """ Test Decode op followed by Normalize op """ logger.info("Test [Decode, Normalize] in one Map") data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image", "label"], num_parallel_workers=1, shuffle=False) # define map operations decode_op = c_vision.Decode() normalize_op = c_vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0]) # apply map operations on images data1 = data1.map(input_columns=["image"], operations=[decode_op, normalize_op]) num_iter = 0 for item in data1.create_dict_iterator(): logger.info("Looping inside iterator {}".format(num_iter)) _ = item["image"] num_iter += 1
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): """ Create a train or eval dataset. Args: dataset_path (str): The path of dataset. do_train (bool): Whether dataset is used for train or eval. repeat_num (int): The repeat times of dataset. Default: 1. batch_size (int): The batch size of dataset. Default: 32. Returns: Dataset. """ if do_train: dataset_path = os.path.join(dataset_path, 'train') do_shuffle = True else: dataset_path = os.path.join(dataset_path, 'eval') do_shuffle = False if device_num == 1 or not do_train: ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=do_shuffle) else: ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=do_shuffle, num_shards=device_num, shard_id=device_id) resize_height = 224 resize_width = 224 buffer_size = 100 rescale = 1.0 / 255.0 shift = 0.0 # define map operations random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4)) random_horizontal_flip_op = C.RandomHorizontalFlip(device_id / (device_id + 1)) resize_op = C.Resize((resize_height, resize_width)) rescale_op = C.Rescale(rescale, shift) normalize_op = C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) change_swap_op = C.HWC2CHW() trans = [] if do_train: trans += [random_crop_op, random_horizontal_flip_op] trans += [resize_op, rescale_op, normalize_op, change_swap_op] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def util_test_normalize(mean, std, op_type): """ Utility function for testing Normalize. Input arguments are given by other tests """ if op_type == "cpp": # define map operations decode_op = c_vision.Decode() normalize_op = c_vision.Normalize(mean, std) # Generate dataset data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) data = data.map(input_columns=["image"], operations=decode_op) data = data.map(input_columns=["image"], operations=normalize_op) elif op_type == "python": # define map operations transforms = [ py_vision.Decode(), py_vision.ToTensor(), py_vision.Normalize(mean, std) ] transform = py_vision.ComposeOp(transforms) # Generate dataset data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) data = data.map(input_columns=["image"], operations=transform()) else: raise ValueError("Wrong parameter value") return data
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 Returns: dataset """ device_num = int(os.getenv("DEVICE_NUM")) rank_id = int(os.getenv("RANK_ID")) if device_num == 1: ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) else: ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) # define map operations trans = [] if do_train: trans += [ C.RandomCrop((32, 32), (4, 4, 4, 4)), C.RandomHorizontalFlip(prob=0.5) ] trans += [ C.Resize((config.image_height, config.image_width)), C.Rescale(1.0 / 255.0, 0.0), C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 Returns: dataset """ if device_target == "GPU": if do_train: from mindspore.communication.management import get_rank, get_group_size ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=get_group_size(), shard_id=get_rank()) else: ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) else: raise ValueError("Unsupported device_target.") resize_height = config.image_height resize_width = config.image_width buffer_size = 1000 # define map operations decode_op = C.Decode() resize_crop_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) resize_op = C.Resize(256) center_crop = C.CenterCrop(resize_width) rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255]) change_swap_op = C.HWC2CHW() if do_train: trans = [resize_crop_op, horizontal_flip_op, rescale_op, normalize_op, change_swap_op] else: trans = [decode_op, resize_op, center_crop, normalize_op, change_swap_op] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8) ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) # apply shuffle operations ds = ds.shuffle(buffer_size=buffer_size) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 Returns: dataset """ rank_size = int(os.getenv("RANK_SIZE")) rank_id = int(os.getenv("RANK_ID")) if rank_size == 1: ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) else: ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=rank_size, shard_id=rank_id) resize_height = config.image_height resize_width = config.image_width buffer_size = 1000 # define map operations decode_op = C.Decode() resize_crop_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) resize_op = C.Resize((256, 256)) center_crop = C.CenterCrop(resize_width) rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255]) change_swap_op = C.HWC2CHW() if do_train: trans = [resize_crop_op, horizontal_flip_op, rescale_op, normalize_op, change_swap_op] else: trans = [decode_op, resize_op, center_crop, rescale_op, normalize_op, change_swap_op] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8) ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) # apply shuffle operations ds = ds.shuffle(buffer_size=buffer_size) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(data_home, repeat_num=1, training=True): """Data operations.""" ds.config.set_seed(1) rank_size = int( os.environ.get("RANK_SIZE")) if os.environ.get("RANK_SIZE") else None device_num = int(os.getenv("RANK_SIZE")) rank_id = int(os.getenv("DEVICE_ID")) if device_num == 1: data_set = ds.Cifar10Dataset(data_home, num_parallel_workers=8, shuffle=True) else: data_set = ds.Cifar10Dataset(data_home, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) resize_height = cfg.image_height resize_width = cfg.image_width rescale = 1.0 / 255.0 shift = 0.0 # define map operations random_crop_op = vision.RandomCrop( (32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_horizontal_op = vision.RandomHorizontalFlip() resize_op = vision.Resize( (resize_height, resize_width)) # interpolation default BILINEAR rescale_op = vision.Rescale(rescale, shift) normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) changeswap_op = vision.HWC2CHW() type_cast_op = C.TypeCast(mstype.int32) c_trans = [] if training: c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images data_set = data_set.map(input_columns="label", operations=type_cast_op) data_set = data_set.map(input_columns="image", operations=c_trans) # apply repeat operations data_set = data_set.repeat(repeat_num) # apply shuffle operations data_set = data_set.shuffle(buffer_size=10) # apply batch operations data_set = data_set.batch(batch_size=cfg.batch_size, drop_remainder=True) return data_set
def create_dataset(dataset_path, config, do_train, repeat_num=1): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. config(dict): config of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1. Returns: dataset """ rank = config.rank group_size = config.group_size if group_size == 1: ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=config.work_nums, shuffle=True) else: ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=config.work_nums, shuffle=True, num_shards=group_size, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(config.image_size), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode #C.RandomColorAdjust(brightness=0.4, contrast=0.5, saturation=0.5, hue=0.2) ] else: trans = [ C.Decode(), C.Resize(int(config.image_size / 0.875)), C.CenterCrop(config.image_size) ] trans += [ C.Rescale(1.0 / 255.0, 0.0), C.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=config.work_nums) ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=config.work_nums) # apply batch operations ds = ds.batch(config.batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def test_normalize_exception_unequal_size_c(): """ Test Normalize in c transformation: len(mean) != len(std) expected to raise ValueError """ logger.info("test_normalize_exception_unequal_size_c") try: _ = c_vision.Normalize([100, 250, 125], [50, 50, 75, 75]) except ValueError as e: logger.info("Got an exception in DE: {}".format(str(e))) assert str(e) == "Length of mean and std must be equal"
def create_dataset(repeat_num=1, training=True, batch_size=32, rank_id=0, rank_size=1, enable_hccl=False): data_dir = data_home + "/cifar-10-batches-bin" if not training: data_dir = data_home + "/cifar-10-verify-bin" data_set = ds.Cifar10Dataset(data_dir) if enable_hccl: rank_id = rank_id rank_size = rank_size data_set = ds.Cifar10Dataset(data_dir, num_shards=rank_size, shard_id=rank_id) resize_height = 224 resize_width = 224 rescale = 1.0 / 255.0 shift = 0.0 # define map operations random_crop_op = vision.RandomCrop( (32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_horizontal_op = vision.RandomHorizontalFlip() # interpolation default BILINEAR resize_op = vision.Resize((resize_height, resize_width)) rescale_op = vision.Rescale(rescale, shift) normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) changeswap_op = vision.HWC2CHW() type_cast_op = C.TypeCast(mstype.int32) c_trans = [] if training: c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images data_set = data_set.map(input_columns="label", operations=type_cast_op) data_set = data_set.map(input_columns="image", operations=c_trans) # apply shuffle operations data_set = data_set.shuffle(buffer_size=1000) # apply batch operations data_set = data_set.batch(batch_size=batch_size, drop_remainder=True) # apply repeat operations data_set = data_set.repeat(repeat_num) return data_set
def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. rank (int): The shard ID within num_shards (default=None). group_size (int): Number of shards that the dataset should be divided into (default=None). repeat_num(int): the repeat times of dataset. Default: 1. Returns: dataset """ if group_size == 1: ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True) else: ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True, num_shards=group_size, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(299, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) ] else: trans = [C.Decode(), C.Resize(299), C.CenterCrop(299)] trans += [ C.Rescale(1.0 / 255.0, 0.0), C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=cfg.work_nums) ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=cfg.work_nums) # apply batch operations ds = ds.batch(cfg.batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(repeat_num=1, training=True): """ create data for next use such as training or infering """ cifar_ds = ds.Cifar10Dataset(data_home) if args_opt.run_distribute: rank_id = int(os.getenv('RANK_ID')) rank_size = int(os.getenv('RANK_SIZE')) cifar_ds = ds.Cifar10Dataset(data_home, num_shards=rank_size, shard_id=rank_id) resize_height = 224 resize_width = 224 rescale = 1.0 / 255.0 shift = 0.0 # define map operations random_crop_op = C.RandomCrop( (32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_horizontal_op = C.RandomHorizontalFlip() resize_op = C.Resize( (resize_height, resize_width)) # interpolation default BILINEAR rescale_op = C.Rescale(rescale, shift) normalize_op = C.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) changeswap_op = C.HWC2CHW() type_cast_op = C2.TypeCast(mstype.int32) c_trans = [] if training: c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images cifar_ds = cifar_ds.map(input_columns="label", operations=type_cast_op) cifar_ds = cifar_ds.map(input_columns="image", operations=c_trans) # apply repeat operations cifar_ds = cifar_ds.repeat(repeat_num) # apply shuffle operations cifar_ds = cifar_ds.shuffle(buffer_size=10) # apply batch operations cifar_ds = cifar_ds.batch(batch_size=args_opt.batch_size, drop_remainder=True) return cifar_ds
def create_dataset(data_home, repeat_num=1, training=True): """Data operations.""" ds.config.set_seed(1) data_dir = os.path.join(data_home, "cifar-10-batches-bin") if not training: data_dir = os.path.join(data_home, "cifar-10-verify-bin") rank_size, rank_id = _get_rank_info() if training: data_set = ds.Cifar10Dataset(data_dir, num_shards=rank_size, shard_id=rank_id, shuffle=True) else: data_set = ds.Cifar10Dataset(data_dir, num_shards=rank_size, shard_id=rank_id, shuffle=False) resize_height = cfg.image_height resize_width = cfg.image_width # define map operations random_crop_op = vision.RandomCrop( (32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_horizontal_op = vision.RandomHorizontalFlip() resize_op = vision.Resize( (resize_height, resize_width)) # interpolation default BILINEAR rescale_op = vision.Rescale(1.0 / 255.0, 0.0) normalize_op = vision.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) changeswap_op = vision.HWC2CHW() type_cast_op = C.TypeCast(mstype.int32) c_trans = [] if training: c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images data_set = data_set.map(input_columns="label", operations=type_cast_op) data_set = data_set.map(input_columns="image", operations=c_trans) # apply batch operations data_set = data_set.batch(batch_size=cfg.batch_size, drop_remainder=True) # apply repeat operations data_set = data_set.repeat(repeat_num) return data_set
def vgg_create_dataset(data_home, image_size, batch_size, rank_id=0, rank_size=1, repeat_num=1, training=True): """Data operations.""" de.config.set_seed(1) data_dir = os.path.join(data_home, "cifar-10-batches-bin") if not training: data_dir = os.path.join(data_home, "cifar-10-verify-bin") data_set = de.Cifar10Dataset(data_dir, num_shards=rank_size, shard_id=rank_id) rescale = 1.0 / 255.0 shift = 0.0 # define map operations random_crop_op = vision.RandomCrop( (32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_horizontal_op = vision.RandomHorizontalFlip() resize_op = vision.Resize(image_size) # interpolation default BILINEAR rescale_op = vision.Rescale(rescale, shift) normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) changeswap_op = vision.HWC2CHW() type_cast_op = C.TypeCast(mstype.int32) c_trans = [] if training: c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images data_set = data_set.map(input_columns="label", operations=type_cast_op) data_set = data_set.map(input_columns="image", operations=c_trans) # apply repeat operations data_set = data_set.repeat(repeat_num) # apply shuffle operations data_set = data_set.shuffle(buffer_size=10) # apply batch operations data_set = data_set.batch(batch_size=batch_size, drop_remainder=True) return data_set
def test_normalize_op(): """ Test Normalize """ logger.info("Test Normalize") # define map operations decode_op = vision.Decode() normalize_op = vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0]) # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) data1 = data1.map(input_columns=["image"], operations=decode_op) data1 = data1.map(input_columns=["image"], operations=normalize_op) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) data2 = data2.map(input_columns=["image"], operations=decode_op) num_iter = 0 for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): image_de_normalized = item1["image"] image_np_normalized = normalize_np(item2["image"]) diff = image_de_normalized - image_np_normalized mse = np.sum(np.power(diff, 2)) logger.info("image_{}, mse: {}".format(num_iter + 1, mse)) assert mse < 0.01 # Uncomment these blocks to see visual results # plt.subplot(131) # plt.imshow(image_de_normalized) # plt.title("DE normalize image") # # plt.subplot(132) # plt.imshow(image_np_normalized) # plt.title("Numpy normalized image") # # plt.subplot(133) # plt.imshow(diff) # plt.title("Difference image, mse : {}".format(mse)) # # plt.show() num_iter += 1
def create_cifar10_dataset(cifar_dir, num_parallel_workers=1): """ Creat the cifar10 dataset. """ ds = de.Cifar10Dataset(cifar_dir) training = True resize_height = 224 resize_width = 224 rescale = 1.0 / 255.0 shift = 0.0 repeat_num = 10 batch_size = 32 # define map operations random_crop_op = vision.RandomCrop( (32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_horizontal_op = vision.RandomHorizontalFlip() resize_op = vision.Resize( (resize_height, resize_width)) # interpolation default BILINEAR rescale_op = vision.Rescale(rescale, shift) normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) changeswap_op = vision.HWC2CHW() type_cast_op = C.TypeCast(mstype.int32) if training: c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images ds = ds.map(input_columns="label", operations=type_cast_op) ds = ds.map(input_columns="image", operations=c_trans) # apply repeat operations ds = ds.repeat(repeat_num) # apply shuffle operations ds = ds.shuffle(buffer_size=10) # apply batch operations ds = ds.batch(batch_size=batch_size, drop_remainder=True) return ds
def create_dataset(data_path, repeat_num=1, batch_size=32, rank_id=0, rank_size=1): resize_height = 224 resize_width = 224 rescale = 1.0 / 255.0 shift = 0.0 # get rank_id and rank_size rank_id = get_rank() rank_size = get_group_size() data_set = ds.Cifar10Dataset(data_path, num_shards=rank_size, shard_id=rank_id) # define map operations random_crop_op = vision.RandomCrop((32, 32), (4, 4, 4, 4)) random_horizontal_op = vision.RandomHorizontalFlip() resize_op = vision.Resize((resize_height, resize_width)) rescale_op = vision.Rescale(rescale, shift) normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) changeswap_op = vision.HWC2CHW() type_cast_op = C.TypeCast(mstype.int32) c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images data_set = data_set.map(input_columns="label", operations=type_cast_op) data_set = data_set.map(input_columns="image", operations=c_trans) # apply shuffle operations data_set = data_set.shuffle(buffer_size=10) # apply batch operations data_set = data_set.batch(batch_size=batch_size, drop_remainder=True) # apply repeat operations data_set = data_set.repeat(repeat_num) return data_set
def test_apply_imagefolder_case(): # apply dataset map operations data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3) data2 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3) decode_op = vision.Decode() normalize_op = vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0]) def dataset_fn(ds): ds = ds.map(operations = decode_op) ds = ds.map(operations = normalize_op) ds = ds.repeat(2) return ds data1 = data1.apply(dataset_fn) data2 = data2.map(operations = decode_op) data2 = data2.map(operations = normalize_op) data2 = data2.repeat(2) for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): assert np.array_equal(item1["image"], item2["image"])
def create_dataset(dataset_path, batch_size=1, num_shards=1, shard_id=0, device_target='Ascend'): """ create train or evaluation dataset for warpctc Args: dataset_path(int): dataset path batch_size(int): batch size of generated dataset, default is 1 num_shards(int): number of devices shard_id(int): rank id device_target(str): platform of training, support Ascend and GPU """ dataset = _CaptchaDataset(dataset_path, cf.max_captcha_digits, device_target) ds = de.GeneratorDataset(dataset, ["image", "label"], shuffle=True, num_shards=num_shards, shard_id=shard_id) ds.set_dataset_size(m.ceil(len(dataset) / num_shards)) image_trans = [ vc.Rescale(1.0 / 255.0, 0.0), vc.Normalize([0.9010, 0.9049, 0.9025], std=[0.1521, 0.1347, 0.1458]), vc.Resize((m.ceil(cf.captcha_height / 16) * 16, cf.captcha_width)), vc.HWC2CHW() ] label_trans = [c.TypeCast(mstype.int32)] ds = ds.map(input_columns=["image"], num_parallel_workers=8, operations=image_trans) ds = ds.map(input_columns=["label"], num_parallel_workers=8, operations=label_trans) ds = ds.batch(batch_size, drop_remainder=True) return ds
def create_dataset(dataset_path, repeat_num=1, batch_size=1): """ create train or evaluation dataset for warpctc Args: dataset_path(int): dataset path repeat_num(int): dataset repetition num, default is 1 batch_size(int): batch size of generated dataset, default is 1 """ rank_size = int( os.environ.get("RANK_SIZE")) if os.environ.get("RANK_SIZE") else 1 rank_id = int( os.environ.get("RANK_ID")) if os.environ.get("RANK_ID") else 0 dataset = _CaptchaDataset(dataset_path, cf.max_captcha_digits) ds = de.GeneratorDataset(dataset, ["image", "label"], shuffle=True, num_shards=rank_size, shard_id=rank_id) ds.set_dataset_size(m.ceil(len(dataset) / rank_size)) image_trans = [ vc.Rescale(1.0 / 255.0, 0.0), vc.Normalize([0.9010, 0.9049, 0.9025], std=[0.1521, 0.1347, 0.1458]), vc.Resize((m.ceil(cf.captcha_height / 16) * 16, cf.captcha_width)), vc.HWC2CHW() ] label_trans = [c.TypeCast(mstype.int32)] ds = ds.map(input_columns=["image"], num_parallel_workers=8, operations=image_trans) ds = ds.map(input_columns=["label"], num_parallel_workers=8, operations=label_trans) ds = ds.batch(batch_size) ds = ds.repeat(repeat_num) return ds
def create_dataset(data_path, batch_size=32, repeat_size=1, status="train"): """ create dataset for train or test """ rank_id = int(os.getenv('DEVICE_ID')) device_num = int(os.getenv('RANK_SIZE')) if device_num == 1: cifar_ds = ds.Cifar10Dataset(data_path, num_parallel_workers=8, shuffle=True) else: cifar_ds = ds.Cifar10Dataset(data_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) rescale = 1.0 / 255.0 shift = 0.0 resize_op = CV.Resize((cfg.image_height, cfg.image_width)) rescale_op = CV.Rescale(rescale, shift) normalize_op = CV.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) if status == "train": random_crop_op = CV.RandomCrop([32, 32], [4, 4, 4, 4]) random_horizontal_op = CV.RandomHorizontalFlip() channel_swap_op = CV.HWC2CHW() typecast_op = C.TypeCast(mstype.int32) cifar_ds = cifar_ds.map(input_columns="label", operations=typecast_op) if status == "train": cifar_ds = cifar_ds.map(input_columns="image", operations=random_crop_op) cifar_ds = cifar_ds.map(input_columns="image", operations=random_horizontal_op) cifar_ds = cifar_ds.map(input_columns="image", operations=resize_op) cifar_ds = cifar_ds.map(input_columns="image", operations=rescale_op) cifar_ds = cifar_ds.map(input_columns="image", operations=normalize_op) cifar_ds = cifar_ds.map(input_columns="image", operations=channel_swap_op) cifar_ds = cifar_ds.shuffle(buffer_size=cfg.buffer_size) cifar_ds = cifar_ds.batch(batch_size, drop_remainder=True) cifar_ds = cifar_ds.repeat(repeat_size) return cifar_ds
def create_fasterrcnn_dataset(mindrecord_file, batch_size=2, repeat_num=12, device_num=1, rank_id=0, is_training=True, num_parallel_workers=8): """Creatr FasterRcnn dataset with MindDataset.""" ds = de.MindDataset(mindrecord_file, columns_list=["image", "annotation"], num_shards=device_num, shard_id=rank_id, num_parallel_workers=num_parallel_workers, shuffle=is_training) decode = C.Decode() ds = ds.map(input_columns=["image"], operations=decode) compose_map_func = (lambda image, annotation: preprocess_fn( image, annotation, is_training)) hwc_to_chw = C.HWC2CHW() normalize_op = C.Normalize((123.675, 116.28, 103.53), (58.395, 57.12, 57.375)) horizontally_op = C.RandomHorizontalFlip(1) type_cast0 = CC.TypeCast(mstype.float32) type_cast1 = CC.TypeCast(mstype.float16) type_cast2 = CC.TypeCast(mstype.int32) type_cast3 = CC.TypeCast(mstype.bool_) if is_training: ds = ds.map(input_columns=["image", "annotation"], output_columns=[ "image", "image_shape", "box", "label", "valid_num" ], columns_order=[ "image", "image_shape", "box", "label", "valid_num" ], operations=compose_map_func, num_parallel_workers=4) ds = ds.map(input_columns=["image"], operations=[normalize_op, type_cast0], num_parallel_workers=num_parallel_workers) flip = (np.random.rand() < config.flip_ratio) if flip: ds = ds.map(input_columns=["image"], operations=[horizontally_op], num_parallel_workers=num_parallel_workers) ds = ds.map(input_columns=[ "image", "image_shape", "box", "label", "valid_num" ], operations=flipped_generation, num_parallel_workers=4) # transpose_column from python to c ds = ds.map(input_columns=["image"], operations=[hwc_to_chw, type_cast1]) ds = ds.map(input_columns=["image_shape"], operations=[type_cast1]) ds = ds.map(input_columns=["box"], operations=[type_cast1]) ds = ds.map(input_columns=["label"], operations=[type_cast2]) ds = ds.map(input_columns=["valid_num"], operations=[type_cast3]) ds = ds.batch(batch_size, drop_remainder=True) ds = ds.repeat(repeat_num) else: ds = ds.map(input_columns=["image", "annotation"], output_columns=[ "image", "image_shape", "box", "label", "valid_num" ], columns_order=[ "image", "image_shape", "box", "label", "valid_num" ], operations=compose_map_func, num_parallel_workers=num_parallel_workers) # transpose_column from python to c ds = ds.map(input_columns=["image"], operations=[hwc_to_chw, type_cast1]) ds = ds.map(input_columns=["image_shape"], operations=[type_cast1]) ds = ds.map(input_columns=["box"], operations=[type_cast1]) ds = ds.map(input_columns=["label"], operations=[type_cast2]) ds = ds.map(input_columns=["valid_num"], operations=[type_cast3]) ds = ds.batch(batch_size, drop_remainder=True) ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 Returns: dataset """ device_num = int(os.getenv("RANK_SIZE")) try: # global_rank_id = int(os.getenv('RANK_ID').split("-")[1].split("custom")[1]) global_rank_id = int(os.getenv('RANK_ID').split("-")[-1]) except: global_rank_id = 0 rank_id = int(os.getenv('DEVICE_ID')) + global_rank_id * 8 columns_list = ["data", "label"] if do_train: ds = de.MindDataset(dataset_path + '/imagenet_train.mindrecord00', columns_list, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) print("train dataset size", ds.get_dataset_size()) else: padded_sample = {} white_io = BytesIO() Image.new('RGB', (224, 224), (255, 255, 255)).save(white_io, 'JPEG') padded_sample['data'] = white_io.getvalue() padded_sample['label'] = -1 batch_per_step = batch_size * device_num print("eval batch per step:", batch_per_step) if batch_per_step < 50000: if 50000 % batch_per_step == 0: num_padded = 0 else: num_padded = batch_per_step - (50000 % batch_per_step) else: num_padded = batch_per_step - 50000 print("Padded samples:", num_padded) ds = de.MindDataset(dataset_path + '/imagenet_eval.mindrecord0', columns_list, num_parallel_workers=8, shuffle=False, num_shards=device_num, shard_id=rank_id, padded_sample=padded_sample, num_padded=num_padded) print("eval dataset size", ds.get_dataset_size()) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] std = [0.229 * 255, 0.224 * 255, 0.225 * 255] # define map operations if do_train: trans = [ C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] else: trans = [ C.Decode(), C.Resize((365, 365)), C.CenterCrop(320), C.Normalize(mean=mean, std=std), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) # ds = ds.shuffle(buffer_size=100000) ds = ds.map(input_columns="data", num_parallel_workers=8, operations=trans) ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds