def create_dataset(args, data_path, batch_size): if args.mode == 'train' and args.use_kungfu: rank = kfops.kungfu_current_rank() size = kfops.kungfu_current_cluster_size() ds = de.Cifar10Dataset( data_path, num_parallel_workers=8, shuffle=False, num_shards=size, shard_id=rank, ) print('using shard %d of %d' % (rank, size)) else: ds = de.Cifar10Dataset( data_path, num_parallel_workers=8, shuffle=False, ) # define map operations trans = [] # if do_train: # trans += [ # # C.RandomCrop((32, 32), (4, 4, 4, 4)), # # C.RandomHorizontalFlip(prob=0.5) # ] trans += [ C.Rescale(1.0 / 255.0, 0.0), C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=8) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) return ds
def create_dataset(repeat_num=1, training=True, batch_size=32, num_samples=1600): data_home = "/home/workspace/mindspore_dataset" data_dir = data_home + "/cifar-10-batches-bin" if not training: data_dir = data_home + "/cifar-10-verify-bin" data_set = ds.Cifar10Dataset(data_dir, num_samples=num_samples) resize_height = 224 resize_width = 224 rescale = 1.0 / 255.0 shift = 0.0 # define map operations random_crop_op = vision.RandomCrop( (32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_horizontal_op = vision.RandomHorizontalFlip() # interpolation default BILINEAR resize_op = vision.Resize((resize_height, resize_width)) rescale_op = vision.Rescale(rescale, shift) normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) changeswap_op = vision.HWC2CHW() type_cast_op = C.TypeCast(mstype.int32) c_trans = [] if training: c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images data_set = data_set.map(operations=type_cast_op, input_columns="label") data_set = data_set.map(operations=c_trans, input_columns="image") # apply shuffle operations data_set = data_set.shuffle(buffer_size=1000) # apply batch operations data_set = data_set.batch(batch_size=batch_size, drop_remainder=True) # apply repeat operations data_set = data_set.repeat(repeat_num) return data_set
def create_dataset(repeat_num=1, training=True): """ create data for next use such as training or infering """ cifar_ds = ds.Cifar10Dataset(data_home) if args_opt.run_distribute: rank_id = int(os.getenv('RANK_ID')) rank_size = int(os.getenv('RANK_SIZE')) cifar_ds = ds.Cifar10Dataset(data_home, num_shards=rank_size, shard_id=rank_id) resize_height = 224 resize_width = 224 rescale = 1.0 / 255.0 shift = 0.0 # define map operations random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_horizontal_op = C.RandomHorizontalFlip() resize_op = C.Resize((resize_height, resize_width)) # interpolation default BILINEAR rescale_op = C.Rescale(rescale, shift) normalize_op = C.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) changeswap_op = C.HWC2CHW() type_cast_op = C2.TypeCast(mstype.int32) c_trans = [] if training: c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images cifar_ds = cifar_ds.map(operations=type_cast_op, input_columns="label") cifar_ds = cifar_ds.map(operations=c_trans, input_columns="image") # apply shuffle operations cifar_ds = cifar_ds.shuffle(buffer_size=10) # apply batch operations cifar_ds = cifar_ds.batch(batch_size=args_opt.batch_size, drop_remainder=True) # apply repeat operations cifar_ds = cifar_ds.repeat(repeat_num) return cifar_ds
def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. rank (int): The shard ID within num_shards (default=None). group_size (int): Number of shards that the dataset should be divided into (default=None). repeat_num(int): the repeat times of dataset. Default: 1. Returns: dataset """ if group_size == 1: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True) else: data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True, num_shards=group_size, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(299, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) ] else: trans = [ C.Decode(), C.Resize(299), C.CenterCrop(299) ] trans += [ C.Rescale(1.0 / 255.0, 0.0), C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=cfg.work_nums) data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=cfg.work_nums) # apply batch operations data_set = data_set.batch(cfg.batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set
def dataset_cifar(dataset_path=None, batch_size=32, repeat_num=1, num_rows=9600, distribution_num=None, shard_id=None, drop_remainder=True, usage=None, shuffle=False, num_workers=8, resize_size=32, pad_info=None): if dataset_path is None: dataset_path = DATA_DIR ds = de.Cifar10Dataset(dataset_path, num_samples=num_rows, num_shards=distribution_num, shard_id=shard_id, shuffle=shuffle, usage=usage, num_parallel_workers=num_workers) typecast_op = c_trans.TypeCast(mstype.int32) ds = ds.map(input_columns="label", operations=typecast_op, num_parallel_workers=num_workers) image_op_list = [ c_vision.Resize(resize_size), c_vision.Rescale(1.0 / 255.0, 0.0), c_vision.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), c_vision.HWC2CHW() ] ds = ds.map(input_columns="image", operations=image_op_list, num_parallel_workers=num_workers) ds = ds.batch(batch_size, drop_remainder=drop_remainder, num_parallel_workers=num_workers, pad_info=pad_info) ds = ds.repeat(repeat_num) return ds
def create_dataset(data_path, repeat_num=1, batch_size=32, rank_id=0, rank_size=1): """create dataset""" resize_height = 224 resize_width = 224 rescale = 1.0 / 255.0 shift = 0.0 # get rank_id and rank_size rank_id = get_rank() rank_size = get_group_size() data_set = ds.Cifar10Dataset(data_path, num_shards=rank_size, shard_id=rank_id) # define map operations random_crop_op = vision.RandomCrop((32, 32), (4, 4, 4, 4)) random_horizontal_op = vision.RandomHorizontalFlip() resize_op = vision.Resize((resize_height, resize_width)) rescale_op = vision.Rescale(rescale, shift) normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) changeswap_op = vision.HWC2CHW() type_cast_op = C.TypeCast(mstype.int32) c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images data_set = data_set.map(operations=type_cast_op, input_columns="label") data_set = data_set.map(operations=c_trans, input_columns="image") # apply shuffle operations data_set = data_set.shuffle(buffer_size=10) # apply batch operations data_set = data_set.batch(batch_size=batch_size, drop_remainder=True) # apply repeat operations data_set = data_set.repeat(repeat_num) return data_set
def create_dataset(dataset_path, config, repeat_num=1, batch_size=32): """ create a train dataset Args: dataset_path(string): the path of dataset. config(EasyDict):the basic config for training repeat_num(int): the repeat times of dataset. Default: 1. batch_size(int): the batch size of dataset. Default: 32. Returns: dataset """ load_func = partial(ds.Cifar10Dataset, dataset_path) cifar_ds = load_func(num_parallel_workers=8, shuffle=False) resize_height = config.image_height resize_width = config.image_width rescale = 1.0 / 255.0 shift = 0.0 # define map operations # interpolation default BILINEAR resize_op = C.Resize((resize_height, resize_width)) rescale_op = C.Rescale(rescale, shift) normalize_op = C.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) changeswap_op = C.HWC2CHW() type_cast_op = C2.TypeCast(mstype.int32) c_trans = [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images cifar_ds = cifar_ds.map(input_columns="label", operations=type_cast_op) cifar_ds = cifar_ds.map(input_columns="image", operations=c_trans) # apply batch operations cifar_ds = cifar_ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation cifar_ds = cifar_ds.repeat(repeat_num) return cifar_ds
def test_cutmix_batch_success2(plot=False): """ Test CutMixBatch op with default values for alpha and prob on a batch of rescaled HWC images """ logger.info("test_cutmix_batch_success2") # Original Images ds_original = ds.Cifar10Dataset(DATA_DIR, num_samples=10, shuffle=False) ds_original = ds_original.batch(5, drop_remainder=True) images_original = None for idx, (image, _) in enumerate(ds_original): if idx == 0: images_original = image.asnumpy() else: images_original = np.append(images_original, image.asnumpy(), axis=0) # CutMix Images data1 = ds.Cifar10Dataset(DATA_DIR, num_samples=10, shuffle=False) one_hot_op = data_trans.OneHot(num_classes=10) data1 = data1.map(operations=one_hot_op, input_columns=["label"]) rescale_op = vision.Rescale((1.0 / 255.0), 0.0) data1 = data1.map(operations=rescale_op, input_columns=["image"]) cutmix_batch_op = vision.CutMixBatch(mode.ImageBatchFormat.NHWC) data1 = data1.batch(5, drop_remainder=True) data1 = data1.map(operations=cutmix_batch_op, input_columns=["image", "label"]) images_cutmix = None for idx, (image, _) in enumerate(data1): if idx == 0: images_cutmix = image.asnumpy() else: images_cutmix = np.append(images_cutmix, image.asnumpy(), axis=0) if plot: visualize_list(images_original, images_cutmix) num_samples = images_original.shape[0] mse = np.zeros(num_samples) for i in range(num_samples): mse[i] = diff_mse(images_cutmix[i], images_original[i]) logger.info("MSE= {}".format(str(np.mean(mse))))
def create_cifar10_dataset(cifar_dir, num_parallel_workers=1): """ Creat the cifar10 dataset. """ ds = de.Cifar10Dataset(cifar_dir) training = True resize_height = 224 resize_width = 224 rescale = 1.0 / 255.0 shift = 0.0 repeat_num = 10 batch_size = 32 # define map operations random_crop_op = vision.RandomCrop( (32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_horizontal_op = vision.RandomHorizontalFlip() resize_op = vision.Resize( (resize_height, resize_width)) # interpolation default BILINEAR rescale_op = vision.Rescale(rescale, shift) normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) changeswap_op = vision.HWC2CHW() type_cast_op = C.TypeCast(mstype.int32) if training: c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images ds = ds.map(operations=type_cast_op, input_columns="label") ds = ds.map(operations=c_trans, input_columns="image") # apply repeat operations ds = ds.repeat(repeat_num) # apply shuffle operations ds = ds.shuffle(buffer_size=10) # apply batch operations ds = ds.batch(batch_size=batch_size, drop_remainder=True) return ds
def test_one_hot_post_aug(): """ Test One Hot Encoding after Multiple Data Augmentation Operators """ logger.info("test_one_hot_post_aug") data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) # Define data augmentation parameters rescale = 1.0 / 255.0 shift = 0.0 resize_height, resize_width = 224, 224 # Define map operations decode_op = c_vision.Decode() rescale_op = c_vision.Rescale(rescale, shift) resize_op = c_vision.Resize((resize_height, resize_width)) # Apply map operations on images data1 = data1.map(operations=decode_op, input_columns=["image"]) data1 = data1.map(operations=rescale_op, input_columns=["image"]) data1 = data1.map(operations=resize_op, input_columns=["image"]) # Apply one-hot encoding on labels depth = 4 one_hot_encode = data_trans.OneHot(depth) data1 = data1.map(operations=one_hot_encode, input_columns=["label"]) # Apply datasets ops buffer_size = 100 seed = 10 batch_size = 2 ds.config.set_seed(seed) data1 = data1.shuffle(buffer_size=buffer_size) data1 = data1.batch(batch_size, drop_remainder=True) num_iter = 0 for item in data1.create_dict_iterator(num_epochs=1): logger.info("image is: {}".format(item["image"])) logger.info("label is: {}".format(item["label"])) num_iter += 1 assert num_iter == 1
def generate_mnist_dataset(data_path, batch_size=32, repeat_size=1, num_samples=None, num_parallel_workers=1, sparse=True): """ create dataset for training or testing """ # define dataset ds1 = ds.MnistDataset(data_path, num_samples=num_samples) # define operation parameters resize_height, resize_width = 32, 32 rescale = 1.0 / 255.0 shift = 0.0 # define map operations resize_op = CV.Resize((resize_height, resize_width), interpolation=Inter.LINEAR) rescale_op = CV.Rescale(rescale, shift) hwc2chw_op = CV.HWC2CHW() type_cast_op = C.TypeCast(mstype.int32) # apply map operations on images if not sparse: one_hot_enco = C.OneHot(10) ds1 = ds1.map(input_columns="label", operations=one_hot_enco, num_parallel_workers=num_parallel_workers) type_cast_op = C.TypeCast(mstype.float32) ds1 = ds1.map(input_columns="label", operations=type_cast_op, num_parallel_workers=num_parallel_workers) ds1 = ds1.map(input_columns="image", operations=resize_op, num_parallel_workers=num_parallel_workers) ds1 = ds1.map(input_columns="image", operations=rescale_op, num_parallel_workers=num_parallel_workers) ds1 = ds1.map(input_columns="image", operations=hwc2chw_op, num_parallel_workers=num_parallel_workers) # apply DatasetOps buffer_size = 10000 ds1 = ds1.shuffle(buffer_size=buffer_size) ds1 = ds1.batch(batch_size, drop_remainder=True) ds1 = ds1.repeat(repeat_size) return ds1
def test_rescale_with_random_posterize(): """ Test RandomPosterize: only support CV_8S/CV_8U """ logger.info("test_rescale_with_random_posterize") DATA_DIR_10 = "../data/dataset/testCifar10Data" dataset = ds.Cifar10Dataset(DATA_DIR_10) rescale_op = c_vision.Rescale((1.0 / 255.0), 0.0) dataset = dataset.map(operations=rescale_op, input_columns=["image"]) random_posterize_op = c_vision.RandomPosterize((4, 8)) dataset = dataset.map(operations=random_posterize_op, input_columns=["image"], num_parallel_workers=1) try: _ = dataset.output_shapes() except RuntimeError as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "input image data type can not be float" in str(e)
def create_dataset(data_dir, p=16, k=8): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. p(int): randomly choose p classes from all classes. k(int): randomly choose k images from each of the chosen p classes. p * k is the batchsize. Returns: dataset """ dataset = MGDataset(data_dir) sampler = DistributedPKSampler(dataset, p=p, k=k) de_dataset = de.GeneratorDataset(dataset, ["image", "label1", "label2"], sampler=sampler) resize_height = config.image_height resize_width = config.image_width rescale = 1.0 / 255.0 shift = 0.0 resize_op = CV.Resize((resize_height, resize_width)) rescale_op = CV.Rescale(rescale, shift) normalize_op = CV.Normalize([0.486, 0.459, 0.408], [0.229, 0.224, 0.225]) change_swap_op = CV.HWC2CHW() trans = [] trans += [resize_op, rescale_op, normalize_op, change_swap_op] type_cast_op_label1 = C.TypeCast(mstype.int32) type_cast_op_label2 = C.TypeCast(mstype.float32) de_dataset = de_dataset.map(input_columns="label1", operations=type_cast_op_label1) de_dataset = de_dataset.map(input_columns="label2", operations=type_cast_op_label2) de_dataset = de_dataset.map(input_columns="image", operations=trans) de_dataset = de_dataset.batch(p*k, drop_remainder=True) return de_dataset
def test_serdes_cifar10_dataset(remove_json_files=True): """ Test serdes on Cifar10 dataset pipeline """ data_dir = "../data/dataset/testCifar10Data" original_seed = config_get_set_seed(1) original_num_parallel_workers = config_get_set_num_parallel_workers(1) data1 = ds.Cifar10Dataset(data_dir, num_samples=10, shuffle=False) data1 = data1.take(6) trans = [ vision.RandomCrop((32, 32), (4, 4, 4, 4)), vision.Resize((224, 224)), vision.Rescale(1.0 / 255.0, 0.0), vision.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), vision.HWC2CHW() ] type_cast_op = c.TypeCast(mstype.int32) data1 = data1.map(operations=type_cast_op, input_columns="label") data1 = data1.map(operations=trans, input_columns="image") data1 = data1.batch(3, drop_remainder=True) data1 = data1.repeat(1) data2 = util_check_serialize_deserialize_file(data1, "cifar10_dataset_pipeline", remove_json_files) num_samples = 0 # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) for item1, item2 in zip( data1.create_dict_iterator(num_epochs=1, output_numpy=True), data2.create_dict_iterator(num_epochs=1, output_numpy=True)): np.testing.assert_array_equal(item1['image'], item2['image']) num_samples += 1 assert num_samples == 2 # Restore configuration num_parallel_workers ds.config.set_seed(original_seed) ds.config.set_num_parallel_workers(original_num_parallel_workers)
def create_dataset(dataset_path, batch_size=1, num_shards=1, shard_id=0, device_target='Ascend'): """ create train or evaluation dataset for warpctc Args: dataset_path(str): dataset path batch_size(int): batch size of generated dataset, default is 1 num_shards(int): number of devices shard_id(int): rank id device_target(str): platform of training, support Ascend and GPU """ dataset = _CaptchaDataset(dataset_path, cf.max_captcha_digits, device_target) ds = de.GeneratorDataset(dataset, ["image", "label"], shuffle=True, num_shards=num_shards, shard_id=shard_id) image_trans = [ vc.Rescale(1.0 / 255.0, 0.0), vc.Normalize([0.9010, 0.9049, 0.9025], std=[0.1521, 0.1347, 0.1458]), vc.Resize((m.ceil(cf.captcha_height / 16) * 16, cf.captcha_width)), c.TypeCast(mstype.float16) ] label_trans = [c.TypeCast(mstype.int32)] ds = ds.map(operations=image_trans, input_columns=["image"], num_parallel_workers=8) ds = ds.map(operations=transpose_hwc2whc, input_columns=["image"], num_parallel_workers=8) ds = ds.map(operations=label_trans, input_columns=["label"], num_parallel_workers=8) ds = ds.batch(batch_size, drop_remainder=True) return ds
def test_rescale_md5(): """ Test Rescale with md5 comparison """ logger.info("Test Rescale with md5 comparison") # generate dataset data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) decode_op = vision.Decode() rescale_op = vision.Rescale(1.0 / 255.0, -1.0) # apply map operations on images data = data.map(operations=decode_op, input_columns=["image"]) data = data.map(operations=rescale_op, input_columns=["image"]) # check results with md5 comparison filename = "rescale_01_result.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
def create_dataset_cifar10(data_home, repeat_num=1, training=True, cifar_cfg=None): """Data operations.""" data_dir = os.path.join(data_home, "cifar-10-batches-bin") if not training: data_dir = os.path.join(data_home, "cifar-10-verify-bin") rank_size, rank_id = _get_rank_info() if training: data_set = ds.Cifar10Dataset(data_dir, num_shards=rank_size, shard_id=rank_id, shuffle=True) else: data_set = ds.Cifar10Dataset(data_dir, num_shards=rank_size, shard_id=rank_id, shuffle=False) resize_height = cifar_cfg.image_height resize_width = cifar_cfg.image_width # define map operations random_crop_op = vision.RandomCrop((32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_horizontal_op = vision.RandomHorizontalFlip() resize_op = vision.Resize((resize_height, resize_width)) # interpolation default BILINEAR rescale_op = vision.Rescale(1.0 / 255.0, 0.0) normalize_op = vision.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) changeswap_op = vision.HWC2CHW() type_cast_op = c_transforms.TypeCast(mstype.int32) c_trans = [] if training: c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images data_set = data_set.map(operations=type_cast_op, input_columns="label") data_set = data_set.map(operations=c_trans, input_columns="image") # apply batch operations data_set = data_set.batch(batch_size=cifar_cfg.batch_size, drop_remainder=True) # apply repeat operations data_set = data_set.repeat(repeat_num) return data_set
def create_dataset(data_path, batch_size=32, repeat_size=1, num_parallel_workers=1): """Create dataset for train or test.""" # define dataset mnist_ds = dataset.MnistDataset(data_path) resize_height, resize_width = 32, 32 rescale = 1.0 / 255.0 shift = 0.0 # define map operations resize_op = transforms.Resize((resize_height, resize_width), interpolation=Inter.LINEAR) # Bilinear mode rescale_op = transforms.Rescale(rescale, shift) hwc2chw_op = transforms.HWC2CHW() type_cast_op = C.TypeCast(mstype.int32) # apply map operations on images mnist_ds = mnist_ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=num_parallel_workers) mnist_ds = mnist_ds.map(operations=resize_op, input_columns="image", num_parallel_workers=num_parallel_workers) mnist_ds = mnist_ds.map(operations=rescale_op, input_columns="image", num_parallel_workers=num_parallel_workers) mnist_ds = mnist_ds.map(operations=hwc2chw_op, input_columns="image", num_parallel_workers=num_parallel_workers) # apply DatasetOps buffer_size = 10000 mnist_ds = mnist_ds.shuffle(buffer_size=buffer_size) mnist_ds = mnist_ds.batch(batch_size, drop_remainder=True) mnist_ds = mnist_ds.repeat(repeat_size) return mnist_ds
def create_dataset(indices, repeat_num=1, training=True): """ create data for next use such as training or infering """ cifar_ds = ds.Cifar10Dataset(data_home, sampler=FixedIndeciesSampler(indices)) resize_height = 224 resize_width = 224 rescale = 1.0 / 255.0 shift = 0.0 # define map operations random_crop_op = C.RandomCrop( (32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_horizontal_op = C.RandomHorizontalFlip() resize_op = C.Resize( (resize_height, resize_width)) # interpolation default BILINEAR rescale_op = C.Rescale(rescale, shift) normalize_op = C.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) changeswap_op = C.HWC2CHW() type_cast_op = C2.TypeCast(mstype.int32) c_trans = [] # if training: # c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images cifar_ds = cifar_ds.map(operations=type_cast_op, input_columns="label") cifar_ds = cifar_ds.map(operations=c_trans, input_columns="image") # apply batch operations cifar_ds = cifar_ds.batch(batch_size=batch_size, drop_remainder=True) # apply repeat operations cifar_ds = cifar_ds.repeat(repeat_num) return cifar_ds
def create_dataset(data_dir, training=True, batch_size=32, resize=(32, 32), rescale=1 / (255 * 0.3081), shift=-0.1307 / 0.3081, buffer_size=64): data_train = os.path.join(data_dir, 'train') # train set data_test = os.path.join(data_dir, 'test') # test set ds = ms.dataset.MnistDataset(data_train if training else data_test) ds = ds.map(input_columns=["image"], operations=[ CV.Resize(resize), CV.Rescale(rescale, shift), CV.HWC2CHW() ]) ds = ds.map(input_columns=["label"], operations=C.TypeCast(ms.int32)) ds = ds.shuffle(buffer_size=buffer_size).batch(batch_size, drop_remainder=True) return ds
def create_dataset1(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend", distribute=False): """ create a train or evaluate cifar10 dataset for resnet50 Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend distribute(bool): data for distribute or not. Default: False Returns: dataset """ if target == "Ascend": device_num, rank_id = _get_rank_info() else: if distribute: init() rank_id = get_rank() device_num = get_group_size() else: device_num = 1 if device_num == 1: data_set = ds.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) else: data_set = ds.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) # define map operations trans = [] if do_train: trans += [ C.RandomCrop((32, 32), (4, 4, 4, 4)), C.RandomHorizontalFlip(prob=0.5) ] trans += [ C.Resize((224, 224)), C.Rescale(1.0 / 255.0, 0.0), C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=8) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set
def test_serdes_imagefolder_dataset(remove_json_files=True): """ Test simulating resnet50 dataset pipeline. """ data_dir = "../data/dataset/testPK/data" ds.config.set_seed(1) # define data augmentation parameters rescale = 1.0 / 255.0 shift = 0.0 resize_height, resize_width = 224, 224 weights = [ 1.0, 0.1, 0.02, 0.3, 0.4, 0.05, 1.2, 0.13, 0.14, 0.015, 0.16, 1.1 ] # Constructing DE pipeline sampler = ds.WeightedRandomSampler(weights, 11) child_sampler = ds.SequentialSampler() sampler.add_child(child_sampler) data1 = ds.ImageFolderDataset(data_dir, sampler=sampler) data1 = data1.repeat(1) data1 = data1.map(operations=[vision.Decode(True)], input_columns=["image"]) rescale_op = vision.Rescale(rescale, shift) resize_op = vision.Resize((resize_height, resize_width), Inter.LINEAR) data1 = data1.map(operations=[rescale_op, resize_op], input_columns=["image"]) data1 = data1.batch(2) # Serialize the dataset pre-processing pipeline. # data1 should still work after saving. ds.serialize(data1, "imagenet_dataset_pipeline.json") ds1_dict = ds.serialize(data1) assert validate_jsonfile("imagenet_dataset_pipeline.json") is True # Print the serialized pipeline to stdout ds.show(data1) # Deserialize the serialized json file data2 = ds.deserialize(json_filepath="imagenet_dataset_pipeline.json") # Serialize the pipeline we just deserialized. # The content of the json file should be the same to the previous serialize. ds.serialize(data2, "imagenet_dataset_pipeline_1.json") assert validate_jsonfile("imagenet_dataset_pipeline_1.json") is True assert filecmp.cmp('imagenet_dataset_pipeline.json', 'imagenet_dataset_pipeline_1.json') # Deserialize the latest json file again data3 = ds.deserialize(json_filepath="imagenet_dataset_pipeline_1.json") data4 = ds.deserialize(input_dict=ds1_dict) num_samples = 0 # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) for item1, item2, item3, item4 in zip( data1.create_dict_iterator(num_epochs=1, output_numpy=True), data2.create_dict_iterator(num_epochs=1, output_numpy=True), data3.create_dict_iterator(num_epochs=1, output_numpy=True), data4.create_dict_iterator(num_epochs=1, output_numpy=True)): np.testing.assert_array_equal(item1['image'], item2['image']) np.testing.assert_array_equal(item1['image'], item3['image']) np.testing.assert_array_equal(item1['label'], item2['label']) np.testing.assert_array_equal(item1['label'], item3['label']) np.testing.assert_array_equal(item3['image'], item4['image']) np.testing.assert_array_equal(item3['label'], item4['label']) num_samples += 1 logger.info("Number of data in data1: {}".format(num_samples)) assert num_samples == 6 # Remove the generated json file if remove_json_files: delete_json_files()
def create_dataset(dataset_path, do_train, config, repeat_num=1): """ Create a train or eval dataset. Args: dataset_path (string): The path of dataset. do_train (bool): Whether dataset is used for train or eval. config: configuration repeat_num (int): The repeat times of dataset. Default: 1. Returns: Dataset. """ if do_train: dataset_path = os.path.join(dataset_path, 'train') do_shuffle = True else: dataset_path = os.path.join(dataset_path, 'eval') do_shuffle = False device_id = 0 device_num = 1 if config.platform == "GPU": if config.run_distribute: from mindspore.communication.management import get_rank, get_group_size device_id = get_rank() device_num = get_group_size() elif config.platform == "Ascend": device_id = int(os.getenv('DEVICE_ID')) device_num = int(os.getenv('RANK_SIZE')) if device_num == 1 or not do_train: ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=4, shuffle=do_shuffle) else: ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=4, shuffle=do_shuffle, num_shards=device_num, shard_id=device_id) resize_height = config.image_height resize_width = config.image_width buffer_size = 100 rescale = 1.0 / 255.0 shift = 0.0 # define map operations random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4)) random_horizontal_flip_op = C.RandomHorizontalFlip(device_id / (device_id + 1)) resize_op = C.Resize((resize_height, resize_width)) rescale_op = C.Rescale(rescale, shift) normalize_op = C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) change_swap_op = C.HWC2CHW() trans = [] if do_train: trans += [random_crop_op, random_horizontal_flip_op] trans += [resize_op, rescale_op, normalize_op, change_swap_op] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="label", operations=type_cast_op) ds = ds.map(input_columns="image", operations=trans) # apply shuffle operations ds = ds.shuffle(buffer_size=buffer_size) # apply batch operations ds = ds.batch(config.batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset_cifar(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): """ create a train or evaluate cifar10 dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 target(str): the device target. Default: Ascend Returns: dataset """ if target == "Ascend": device_num, rank_id = _get_rank_info() else: init() rank_id = get_rank() device_num = get_group_size() if device_num == 1: ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) else: ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=device_num, shard_id=rank_id) # define map operations if do_train: trans = [ C.RandomCrop((32, 32), (4, 4, 4, 4)), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4), C.Resize((227, 227)), C.Rescale(1.0 / 255.0, 0.0), C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), C.CutOut(112), C.HWC2CHW() ] else: trans = [ C.Resize((227, 227)), C.Rescale(1.0 / 255.0, 0.0), C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=8) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, shard_id=0): """ Create a train or eval dataset. Args: dataset_path (str): The path of dataset. do_train (bool): Whether dataset is used for train or eval. repeat_num (int): The repeat times of dataset. Default: 1. batch_size (int): The batch size of dataset. Default: 32. Returns: Dataset. """ do_shuffle = bool(do_train) if device_num == 1 or not do_train: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=do_shuffle) else: ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=do_shuffle, num_shards=device_num, shard_id=shard_id) image_length = 299 if do_train: trans = [ C.RandomCropDecodeResize(image_length, scale=(0.08, 1.0), ratio=(0.75, 1.333)), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) ] else: trans = [ C.Decode(), C.Resize(image_length), C.CenterCrop(image_length) ] trans += [ C.Rescale(1.0 / 255.0, 0.0), C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), C.HWC2CHW() ] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=config.work_nums) ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=config.work_nums) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): """ Create a train or eval dataset. Args: dataset_path (str): The path of dataset. do_train (bool): Whether dataset is used for train or eval. repeat_num (int): The repeat times of dataset. Default: 1. batch_size (int): The batch size of dataset. Default: 32. Returns: Dataset. """ if do_train: dataset_path = os.path.join(dataset_path, 'train') do_shuffle = True else: dataset_path = os.path.join(dataset_path, 'eval') do_shuffle = False if device_num == 1 or not do_train: ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=do_shuffle) else: ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=do_shuffle, num_shards=device_num, shard_id=device_id) resize_height = 224 resize_width = 224 buffer_size = 100 rescale = 1.0 / 255.0 shift = 0.0 # define map operations random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4)) random_horizontal_flip_op = C.RandomHorizontalFlip(device_id / (device_id + 1)) resize_op = C.Resize((resize_height, resize_width)) rescale_op = C.Rescale(rescale, shift) normalize_op = C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) change_swap_op = C.HWC2CHW() trans = [] if do_train: trans += [random_crop_op, random_horizontal_flip_op] trans += [resize_op, rescale_op, normalize_op, change_swap_op] type_cast_op = C2.TypeCast(mstype.int32) ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=8) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) # apply dataset repeat operation ds = ds.repeat(repeat_num) return ds
def create_dataset(dataset_path, do_train, config, platform, repeat_num=1, batch_size=100): """ create a train or eval dataset Args: dataset_path(string): the path of dataset. do_train(bool): whether dataset is used for train or eval. repeat_num(int): the repeat times of dataset. Default: 1 batch_size(int): the batch size of dataset. Default: 32 Returns: dataset """ if platform == "Ascend": rank_size = int(os.getenv("RANK_SIZE")) rank_id = int(os.getenv("RANK_ID")) if rank_size == 1: data_set = ds.MindDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: data_set = ds.MindDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=rank_size, shard_id=rank_id) elif platform == "GPU": if do_train: from mindspore.communication.management import get_rank, get_group_size data_set = ds.MindDataset(dataset_path, num_parallel_workers=8, shuffle=True, num_shards=get_group_size(), shard_id=get_rank()) else: data_set = ds.MindDataset(dataset_path, num_parallel_workers=8, shuffle=False) else: raise ValueError("Unsupported platform.") resize_height = config.image_height buffer_size = 1000 # define map operations resize_crop_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) color_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) rescale_op = C.Rescale(1 / 255.0, 0) normalize_op = C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) change_swap_op = C.HWC2CHW() # define python operations decode_p = P.Decode() resize_p = P.Resize(256, interpolation=Inter.BILINEAR) center_crop_p = P.CenterCrop(224) totensor = P.ToTensor() normalize_p = P.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) composeop = P2.Compose( [decode_p, resize_p, center_crop_p, totensor, normalize_p]) if do_train: trans = [ resize_crop_op, horizontal_flip_op, color_op, rescale_op, normalize_op, change_swap_op ] else: trans = composeop type_cast_op = C2.TypeCast(mstype.int32) data_set = data_set.map(input_columns="image", operations=trans, num_parallel_workers=8) data_set = data_set.map(input_columns="label_list", operations=type_cast_op, num_parallel_workers=8) # apply shuffle operations data_set = data_set.shuffle(buffer_size=buffer_size) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) # apply dataset repeat operation data_set = data_set.repeat(repeat_num) return data_set
def keypoint_dataset(config, ann_file=None, image_path=None, bbox_file=None, rank=0, group_size=1, train_mode=True, num_parallel_workers=8, transform=None, shuffle=None): """ A function that returns an imagenet dataset for classification. The mode of input dataset should be "folder" . Args: rank (int): The shard ID within num_shards (default=None). group_size (int): Number of shards that the dataset should be divided into (default=None). mode (str): "train" or others. Default: " train". num_parallel_workers (int): Number of workers to read the data. Default: None. """ # config per_batch_size = config.TRAIN.BATCH_SIZE if train_mode else config.TEST.BATCH_SIZE image_path = image_path if image_path else os.path.join( config.DATASET.ROOT, config.DATASET.TRAIN_SET if train_mode else config.DATASET.TEST_SET) print('loading dataset from {}'.format(image_path)) ann_file = ann_file if ann_file else os.path.join( config.DATASET.ROOT, 'annotations/person_keypoints_{}2017.json'.format( 'train' if train_mode else 'val')) shuffle = shuffle if shuffle is not None else train_mode # gen dataset db dataset_generator = KeypointDatasetGenerator(config, is_train=train_mode) if not train_mode and not config.TEST.USE_GT_BBOX: print('loading bbox file from {}'.format(bbox_file)) dataset_generator.load_detect_dataset(image_path, ann_file, bbox_file) else: dataset_generator.load_gt_dataset(image_path, ann_file) # construct dataset de_dataset = de.GeneratorDataset(dataset_generator, column_names=[ "image", "target", "weight", "scale", "center", "score", "id" ], num_parallel_workers=num_parallel_workers, num_shards=group_size, shard_id=rank, shuffle=shuffle) # inputs map functions if transform is None: transform_img = [ V_C.Rescale(1.0 / 255.0, 0.0), V_C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), V_C.HWC2CHW() ] else: transform_img = transform de_dataset = de_dataset.map(input_columns="image", num_parallel_workers=num_parallel_workers, operations=transform_img) # batch de_dataset = de_dataset.batch(per_batch_size, drop_remainder=train_mode) return de_dataset, dataset_generator
def vgg_create_dataset100(data_home, image_size, batch_size, rank_id=0, rank_size=1, repeat_num=1, training=True, num_samples=None, shuffle=True): """Data operations.""" ds.config.set_seed(1) data_dir = os.path.join(data_home, "train") if not training: data_dir = os.path.join(data_home, "test") if num_samples is not None: data_set = ds.Cifar100Dataset(data_dir, num_shards=rank_size, shard_id=rank_id, num_samples=num_samples, shuffle=shuffle) else: data_set = ds.Cifar100Dataset(data_dir, num_shards=rank_size, shard_id=rank_id) input_columns = ["fine_label"] output_columns = ["label"] data_set = data_set.rename(input_columns=input_columns, output_columns=output_columns) data_set = data_set.project(["image", "label"]) rescale = 1.0 / 255.0 shift = 0.0 # define map operations random_crop_op = CV.RandomCrop( (32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_horizontal_op = CV.RandomHorizontalFlip() resize_op = CV.Resize(image_size) # interpolation default BILINEAR rescale_op = CV.Rescale(rescale, shift) normalize_op = CV.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) changeswap_op = CV.HWC2CHW() type_cast_op = C.TypeCast(mstype.int32) c_trans = [] if training: c_trans = [random_crop_op, random_horizontal_op] c_trans += [resize_op, rescale_op, normalize_op, changeswap_op] # apply map operations on images data_set = data_set.map(input_columns="label", operations=type_cast_op) data_set = data_set.map(input_columns="image", operations=c_trans) # apply repeat operations data_set = data_set.repeat(repeat_num) # apply shuffle operations # data_set = data_set.shuffle(buffer_size=1000) # apply batch operations data_set = data_set.batch(batch_size=batch_size, drop_remainder=True) return data_set
def create_dataset(data_dir, repeat=400, train_batch_size=16, augment=False, cross_val_ind=1, run_distribute=False, do_crop=None, img_size=None): images = _load_multipage_tiff(os.path.join(data_dir, 'train-volume.tif')) masks = _load_multipage_tiff(os.path.join(data_dir, 'train-labels.tif')) train_indices, val_indices = _get_val_train_indices( len(images), cross_val_ind) train_images = images[train_indices] train_masks = masks[train_indices] train_images = np.repeat(train_images, repeat, axis=0) train_masks = np.repeat(train_masks, repeat, axis=0) val_images = images[val_indices] val_masks = masks[val_indices] train_image_data = {"image": train_images} train_mask_data = {"mask": train_masks} valid_image_data = {"image": val_images} valid_mask_data = {"mask": val_masks} ds_train_images = ds.NumpySlicesDataset(data=train_image_data, sampler=None, shuffle=False) ds_train_masks = ds.NumpySlicesDataset(data=train_mask_data, sampler=None, shuffle=False) if run_distribute: rank_id = get_rank() rank_size = get_group_size() ds_train_images = ds.NumpySlicesDataset(data=train_image_data, sampler=None, shuffle=False, num_shards=rank_size, shard_id=rank_id) ds_train_masks = ds.NumpySlicesDataset(data=train_mask_data, sampler=None, shuffle=False, num_shards=rank_size, shard_id=rank_id) ds_valid_images = ds.NumpySlicesDataset(data=valid_image_data, sampler=None, shuffle=False) ds_valid_masks = ds.NumpySlicesDataset(data=valid_mask_data, sampler=None, shuffle=False) if do_crop: resize_size = [ int(img_size[x] * do_crop[x]) for x in range(len(img_size)) ] else: resize_size = img_size c_resize_op = c_vision.Resize(size=(resize_size[0], resize_size[1]), interpolation=Inter.BILINEAR) c_pad = c_vision.Pad(padding=(img_size[0] - resize_size[0]) // 2) c_rescale_image = c_vision.Rescale(1.0 / 127.5, -1) c_rescale_mask = c_vision.Rescale(1.0 / 255.0, 0) c_trans_normalize_img = [c_rescale_image, c_resize_op, c_pad] c_trans_normalize_mask = [c_rescale_mask, c_resize_op, c_pad] c_center_crop = c_vision.CenterCrop(size=388) train_image_ds = ds_train_images.map(input_columns="image", operations=c_trans_normalize_img) train_mask_ds = ds_train_masks.map(input_columns="mask", operations=c_trans_normalize_mask) train_ds = ds.zip((train_image_ds, train_mask_ds)) train_ds = train_ds.project(columns=["image", "mask"]) if augment: augment_process = train_data_augmentation c_resize_op = c_vision.Resize(size=(img_size[0], img_size[1]), interpolation=Inter.BILINEAR) train_ds = train_ds.map(input_columns=["image", "mask"], operations=augment_process) train_ds = train_ds.map(input_columns="image", operations=c_resize_op) train_ds = train_ds.map(input_columns="mask", operations=c_resize_op) if do_crop: train_ds = train_ds.map(input_columns="mask", operations=c_center_crop) post_process = data_post_process train_ds = train_ds.map(input_columns=["image", "mask"], operations=post_process) train_ds = train_ds.shuffle(repeat * 24) train_ds = train_ds.batch(batch_size=train_batch_size, drop_remainder=True) valid_image_ds = ds_valid_images.map(input_columns="image", operations=c_trans_normalize_img) valid_mask_ds = ds_valid_masks.map(input_columns="mask", operations=c_trans_normalize_mask) valid_ds = ds.zip((valid_image_ds, valid_mask_ds)) valid_ds = valid_ds.project(columns=["image", "mask"]) if do_crop: valid_ds = valid_ds.map(input_columns="mask", operations=c_center_crop) post_process = data_post_process valid_ds = valid_ds.map(input_columns=["image", "mask"], operations=post_process) valid_ds = valid_ds.batch(batch_size=1, drop_remainder=True) return train_ds, valid_ds