def pipeline(): sampler = ds.SubsetSampler(indices, num_samples) data = ds.NumpySlicesDataset(list(range(0, 10)), sampler=sampler) dataset_size = data.get_dataset_size() return [ d[0] for d in data.create_tuple_iterator(num_epochs=1, output_numpy=True) ], dataset_size
def test_config(num_samples, start_index, subset_size): sampler = ds.SubsetSampler(start_index, subset_size) d = ds.ManifestDataset(manifest_file, sampler=sampler) res = [] for item in d.create_dict_iterator(): res.append(map_[(item["image"].shape[0], item["label"].item())]) return res
def test_cv_minddataset_subset_random_sample_negative(add_and_remove_cv_file): columns_list = ["data", "file_name", "label"] num_readers = 4 indices = [1, 2, 4, -1, -2] samplers = ds.SubsetRandomSampler(indices), ds.SubsetSampler(indices) for sampler in samplers: data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers, sampler=sampler) assert data_set.get_dataset_size() == 5 num_iter = 0 for item in data_set.create_dict_iterator(num_epochs=1, output_numpy=True): logger.info( "-------------- cv reader basic: {} ------------------------".format(num_iter)) logger.info( "-------------- item[data]: {} -----------------------------".format(item["data"])) logger.info( "-------------- item[file_name]: {} ------------------------".format(item["file_name"])) logger.info( "-------------- item[label]: {} ----------------------------".format(item["label"])) num_iter += 1 assert num_iter == 5