def util_check_serialize_deserialize_file(data_orig, filename, remove_json_files): """ Utility function for testing serdes files. It is to check if a json file is indeed created with correct name after serializing and if it remains the same after repeatedly saving and loading. :param data_orig: original data pipeline to be serialized :param filename: filename to be saved as json format :param remove_json_files: whether to remove the json file after testing :return: The data pipeline after serializing and deserializing using the original pipeline """ file1 = filename + ".json" file2 = filename + "_1.json" ds.serialize(data_orig, file1) assert validate_jsonfile(file1) is True assert validate_jsonfile("wrong_name.json") is False data_changed = ds.deserialize(json_filepath=file1) ds.serialize(data_changed, file2) assert validate_jsonfile(file2) is True assert filecmp.cmp(file1, file2) # Remove the generated json file if remove_json_files: delete_json_files() return data_changed
def test_serdes_exception(): """ Test exception case in serdes """ data_dir = [ "../data/dataset/test_tf_file_3_images/train-0000-of-0001.data" ] schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" data1 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) data1 = data1.filter(input_columns=["image", "label"], predicate=lambda data: data < 11, num_parallel_workers=4) data1_json = ds.serialize(data1) with pytest.raises(RuntimeError) as msg: ds.deserialize(input_dict=data1_json) assert "Filter is not yet supported by ds.engine.deserialize" in str(msg)
def test_serdes_zip_dataset(remove_json_files=True): """ Test serdes on zip dataset pipeline. """ files = ["../data/dataset/testTFTestAllTypes/test.data"] schema_file = "../data/dataset/testTFTestAllTypes/datasetSchema.json" ds.config.set_seed(1) ds0 = ds.TFRecordDataset(files, schema=schema_file, shuffle=ds.Shuffle.GLOBAL) data1 = ds.TFRecordDataset(files, schema=schema_file, shuffle=ds.Shuffle.GLOBAL) data2 = ds.TFRecordDataset(files, schema=schema_file, shuffle=ds.Shuffle.FILES) data2 = data2.shuffle(10000) data2 = data2.rename(input_columns=[ "col_sint16", "col_sint32", "col_sint64", "col_float", "col_1d", "col_2d", "col_3d", "col_binary" ], output_columns=[ "column_sint16", "column_sint32", "column_sint64", "column_float", "column_1d", "column_2d", "column_3d", "column_binary" ]) data3 = ds.zip((data1, data2)) ds.serialize(data3, "zip_dataset_pipeline.json") assert validate_jsonfile("zip_dataset_pipeline.json") is True assert validate_jsonfile("zip_dataset_pipeline_typo.json") is False data4 = ds.deserialize(json_filepath="zip_dataset_pipeline.json") ds.serialize(data4, "zip_dataset_pipeline_1.json") assert validate_jsonfile("zip_dataset_pipeline_1.json") is True assert filecmp.cmp('zip_dataset_pipeline.json', 'zip_dataset_pipeline_1.json') rows = 0 for d0, d3, d4 in zip(ds0.create_tuple_iterator(output_numpy=True), data3.create_tuple_iterator(output_numpy=True), data4.create_tuple_iterator(output_numpy=True)): num_cols = len(d0) offset = 0 for t1 in d0: np.testing.assert_array_equal(t1, d3[offset]) np.testing.assert_array_equal(t1, d3[offset + num_cols]) np.testing.assert_array_equal(t1, d4[offset]) np.testing.assert_array_equal(t1, d4[offset + num_cols]) offset += 1 rows += 1 assert rows == 12 if remove_json_files: delete_json_files()
def test_serdes_mnist_dataset(remove_json_files=True): """ Test serdes on mnist dataset pipeline. """ data_dir = "../data/dataset/testMnistData" ds.config.set_seed(1) data1 = ds.MnistDataset(data_dir, num_samples=100) one_hot_encode = c.OneHot(10) # num_classes is input argument data1 = data1.map(operations=one_hot_encode, input_columns="label") # batch_size is input argument data1 = data1.batch(batch_size=10, drop_remainder=True) ds.serialize(data1, "mnist_dataset_pipeline.json") assert validate_jsonfile("mnist_dataset_pipeline.json") is True data2 = ds.deserialize(json_filepath="mnist_dataset_pipeline.json") ds.serialize(data2, "mnist_dataset_pipeline_1.json") assert validate_jsonfile("mnist_dataset_pipeline_1.json") is True assert filecmp.cmp('mnist_dataset_pipeline.json', 'mnist_dataset_pipeline_1.json') data3 = ds.deserialize(json_filepath="mnist_dataset_pipeline_1.json") num = 0 for data1, data2, data3 in zip( data1.create_dict_iterator(num_epochs=1, output_numpy=True), data2.create_dict_iterator(num_epochs=1, output_numpy=True), data3.create_dict_iterator(num_epochs=1, output_numpy=True)): np.testing.assert_array_equal(data1['image'], data2['image']) np.testing.assert_array_equal(data1['image'], data3['image']) np.testing.assert_array_equal(data1['label'], data2['label']) np.testing.assert_array_equal(data1['label'], data3['label']) num += 1 logger.info("mnist total num samples is {}".format(str(num))) assert num == 10 if remove_json_files: delete_json_files()
def test_mnist_dataset(remove_json_files=True): data_dir = "../data/dataset/testMnistData" ds.config.set_seed(1) data1 = ds.MnistDataset(data_dir, 100) one_hot_encode = c.OneHot(10) # num_classes is input argument data1 = data1.map(input_columns="label", operations=one_hot_encode) # batch_size is input argument data1 = data1.batch(batch_size=10, drop_remainder=True) ds.serialize(data1, "mnist_dataset_pipeline.json") assert (validate_jsonfile("mnist_dataset_pipeline.json") is True) data2 = ds.deserialize(json_filepath="mnist_dataset_pipeline.json") ds.serialize(data2, "mnist_dataset_pipeline_1.json") assert (validate_jsonfile("mnist_dataset_pipeline_1.json") is True) assert (filecmp.cmp('mnist_dataset_pipeline.json', 'mnist_dataset_pipeline_1.json')) data3 = ds.deserialize(json_filepath="mnist_dataset_pipeline_1.json") num = 0 for data1, data2, data3 in zip(data1.create_dict_iterator(), data2.create_dict_iterator(), data3.create_dict_iterator()): assert (np.array_equal(data1['image'], data2['image'])) assert (np.array_equal(data1['image'], data3['image'])) assert (np.array_equal(data1['label'], data2['label'])) assert (np.array_equal(data1['label'], data3['label'])) num += 1 logger.info("mnist total num samples is {}".format(str(num))) assert (num == 10) if remove_json_files: delete_json_files()
def test_serdes_random_crop(): """ Test serdes on RandomCrop pipeline. """ logger.info("test_random_crop") DATA_DIR = [ "../data/dataset/test_tf_file_3_images/train-0000-of-0001.data" ] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" original_seed = config_get_set_seed(1) original_num_parallel_workers = config_get_set_num_parallel_workers(1) # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) decode_op = vision.Decode() random_crop_op = vision.RandomCrop([512, 512], [200, 200, 200, 200]) data1 = data1.map(operations=decode_op, input_columns="image") data1 = data1.map(operations=random_crop_op, input_columns="image") # Serializing into python dictionary ds1_dict = ds.serialize(data1) # Serializing into json object _ = json.dumps(ds1_dict, indent=2) # Reconstruct dataset pipeline from its serialized form data1_1 = ds.deserialize(input_dict=ds1_dict) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) data2 = data2.map(operations=decode_op, input_columns="image") for item1, item1_1, item2 in zip( data1.create_dict_iterator(num_epochs=1, output_numpy=True), data1_1.create_dict_iterator(num_epochs=1, output_numpy=True), data2.create_dict_iterator(num_epochs=1, output_numpy=True)): np.testing.assert_array_equal(item1['image'], item1_1['image']) _ = item2["image"] # Restore configuration num_parallel_workers ds.config.set_seed(original_seed) ds.config.set_num_parallel_workers(original_num_parallel_workers)
def test_serdes_imagefolder_dataset(remove_json_files=True): """ Test simulating resnet50 dataset pipeline. """ data_dir = "../data/dataset/testPK/data" ds.config.set_seed(1) # define data augmentation parameters rescale = 1.0 / 255.0 shift = 0.0 resize_height, resize_width = 224, 224 weights = [ 1.0, 0.1, 0.02, 0.3, 0.4, 0.05, 1.2, 0.13, 0.14, 0.015, 0.16, 1.1 ] # Constructing DE pipeline sampler = ds.WeightedRandomSampler(weights, 11) child_sampler = ds.SequentialSampler() sampler.add_child(child_sampler) data1 = ds.ImageFolderDataset(data_dir, sampler=sampler) data1 = data1.repeat(1) data1 = data1.map(operations=[vision.Decode(True)], input_columns=["image"]) rescale_op = vision.Rescale(rescale, shift) resize_op = vision.Resize((resize_height, resize_width), Inter.LINEAR) data1 = data1.map(operations=[rescale_op, resize_op], input_columns=["image"]) data1 = data1.batch(2) # Serialize the dataset pre-processing pipeline. # data1 should still work after saving. ds.serialize(data1, "imagenet_dataset_pipeline.json") ds1_dict = ds.serialize(data1) assert validate_jsonfile("imagenet_dataset_pipeline.json") is True # Print the serialized pipeline to stdout ds.show(data1) # Deserialize the serialized json file data2 = ds.deserialize(json_filepath="imagenet_dataset_pipeline.json") # Serialize the pipeline we just deserialized. # The content of the json file should be the same to the previous serialize. ds.serialize(data2, "imagenet_dataset_pipeline_1.json") assert validate_jsonfile("imagenet_dataset_pipeline_1.json") is True assert filecmp.cmp('imagenet_dataset_pipeline.json', 'imagenet_dataset_pipeline_1.json') # Deserialize the latest json file again data3 = ds.deserialize(json_filepath="imagenet_dataset_pipeline_1.json") data4 = ds.deserialize(input_dict=ds1_dict) num_samples = 0 # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) for item1, item2, item3, item4 in zip( data1.create_dict_iterator(num_epochs=1, output_numpy=True), data2.create_dict_iterator(num_epochs=1, output_numpy=True), data3.create_dict_iterator(num_epochs=1, output_numpy=True), data4.create_dict_iterator(num_epochs=1, output_numpy=True)): np.testing.assert_array_equal(item1['image'], item2['image']) np.testing.assert_array_equal(item1['image'], item3['image']) np.testing.assert_array_equal(item1['label'], item2['label']) np.testing.assert_array_equal(item1['label'], item3['label']) np.testing.assert_array_equal(item3['image'], item4['image']) np.testing.assert_array_equal(item3['label'], item4['label']) num_samples += 1 logger.info("Number of data in data1: {}".format(num_samples)) assert num_samples == 6 # Remove the generated json file if remove_json_files: delete_json_files()