예제 #1
0
def test_add_sampler_invalid_input():
    manifest_file = "../data/dataset/testManifestData/test5trainimgs.json"
    _ = {
        (172876, 0): 0,
        (54214, 0): 1,
        (54214, 1): 2,
        (173673, 0): 3,
        (64631, 1): 4
    }
    data1 = ds.ManifestDataset(manifest_file)

    with pytest.raises(TypeError) as info:
        data1.use_sampler(1)
    assert "not an instance of a sampler" in str(info.value)

    with pytest.raises(TypeError) as info:
        data1.use_sampler("sampler")
    assert "not an instance of a sampler" in str(info.value)

    sampler = ds.SequentialSampler()
    with pytest.raises(ValueError) as info:
        data2 = ds.ManifestDataset(manifest_file,
                                   sampler=sampler,
                                   num_samples=20)
    assert "Conflicting arguments during sampler assignments" in str(
        info.value)
예제 #2
0
def test_add_sampler_invalid_input():
    manifest_file = "../data/dataset/testManifestData/test5trainimgs.json"
    _ = {
        (172876, 0): 0,
        (54214, 0): 1,
        (54214, 1): 2,
        (173673, 0): 3,
        (64631, 1): 4
    }
    data1 = ds.ManifestDataset(manifest_file)

    with pytest.raises(TypeError) as info:
        data1.use_sampler(1)
    assert "not an instance of a sampler" in str(info.value)

    with pytest.raises(TypeError) as info:
        data1.use_sampler("sampler")
    assert "not an instance of a sampler" in str(info.value)

    sampler = ds.SequentialSampler()
    with pytest.raises(RuntimeError) as info:
        data2 = ds.ManifestDataset(manifest_file,
                                   sampler=sampler,
                                   num_samples=20)
    assert "sampler and num_samples cannot be specified at the same time" in str(
        info.value)
예제 #3
0
def test_manifest_dataset_exception():
    def exception_func(item):
        raise Exception("Error occur!")

    try:
        data = ds.ManifestDataset(DATA_FILE)
        data = data.map(operations=exception_func,
                        input_columns=["image"],
                        num_parallel_workers=1)
        for _ in data.__iter__():
            pass
        assert False
    except RuntimeError as e:
        assert "map operation: [PyFunc] failed. The corresponding data files" in str(
            e)

    try:
        data = ds.ManifestDataset(DATA_FILE)
        data = data.map(operations=vision.Decode(),
                        input_columns=["image"],
                        num_parallel_workers=1)
        data = data.map(operations=exception_func,
                        input_columns=["image"],
                        num_parallel_workers=1)
        for _ in data.__iter__():
            pass
        assert False
    except RuntimeError as e:
        assert "map operation: [PyFunc] failed. The corresponding data files" in str(
            e)

    try:
        data = ds.ManifestDataset(DATA_FILE)
        data = data.map(operations=exception_func,
                        input_columns=["label"],
                        num_parallel_workers=1)
        for _ in data.__iter__():
            pass
        assert False
    except RuntimeError as e:
        assert "map operation: [PyFunc] failed. The corresponding data files" in str(
            e)

    NO_SOURCE_DATA_FILE = "../data/dataset/testManifestData/invalidNoSource.manifest"
    try:
        data = ds.ManifestDataset(NO_SOURCE_DATA_FILE)
        for _ in data.__iter__():
            pass
        assert False
    except RuntimeError as e:
        assert "Invalid data, source is not found in Manifest file" in str(e)

    NO_USAGE_DATA_FILE = "../data/dataset/testManifestData/invalidNoUsage.manifest"
    try:
        data = ds.ManifestDataset(NO_USAGE_DATA_FILE)
        for _ in data.__iter__():
            pass
        assert False
    except RuntimeError as e:
        assert "Invalid data, usage is not found in Manifest file" in str(e)
예제 #4
0
def test_mappable_invalid_input():
    d = ds.ManifestDataset(manifest_file)
    split_with_invalid_inputs(d)

    d = ds.ManifestDataset(manifest_file, num_shards=2, shard_id=0)
    with pytest.raises(RuntimeError) as info:
        _, _ = d.split([4, 1])
    assert "Dataset should not be sharded before split" in str(info.value)
예제 #5
0
def test_manifest_dataset_size():
    ds_total = ds.ManifestDataset(MANIFEST_DATA_FILE)
    assert ds_total.get_dataset_size() == 4

    ds_shard_1_0 = ds.ManifestDataset(MANIFEST_DATA_FILE, num_shards=1, shard_id=0)
    assert ds_shard_1_0.get_dataset_size() == 4

    ds_shard_2_0 = ds.ManifestDataset(MANIFEST_DATA_FILE, num_shards=2, shard_id=0)
    assert ds_shard_2_0.get_dataset_size() == 2

    ds_shard_3_0 = ds.ManifestDataset(MANIFEST_DATA_FILE, num_shards=3, shard_id=0)
    assert ds_shard_3_0.get_dataset_size() == 2
예제 #6
0
def test_manifest():
    data = ds.ManifestDataset("../data/dataset/testManifestData/test.manifest")
    assert data.get_dataset_size() == 4
    assert data.num_classes() == 3

    data = data.shuffle(100)
    assert data.num_classes() == 3
예제 #7
0
def test_rounding():
    d = ds.ManifestDataset(manifest_file, shuffle=False)

    # under rounding
    s1, s2 = d.split([0.5, 0.5], randomize=False)

    s1_output = []
    for item in s1.create_dict_iterator():
        s1_output.append(manifest_map[(item["image"].shape[0], item["label"].item())])

    s2_output = []
    for item in s2.create_dict_iterator():
        s2_output.append(manifest_map[(item["image"].shape[0], item["label"].item())])

    assert s1_output == [0, 1, 2]
    assert s2_output == [3, 4]

    # over rounding
    s1, s2, s3 = d.split([0.15, 0.55, 0.3], randomize=False)

    s1_output = []
    for item in s1.create_dict_iterator():
        s1_output.append(manifest_map[(item["image"].shape[0], item["label"].item())])

    s2_output = []
    for item in s2.create_dict_iterator():
        s2_output.append(manifest_map[(item["image"].shape[0], item["label"].item())])

    s3_output = []
    for item in s3.create_dict_iterator():
        s3_output.append(manifest_map[(item["image"].shape[0], item["label"].item())])

    assert s1_output == [0]
    assert s2_output == [1, 2]
    assert s3_output == [3, 4]
예제 #8
0
def test_mappable_get_dataset_size():
    d = ds.ManifestDataset(manifest_file, shuffle=False)
    s1, s2 = d.split([4, 1])

    assert d.get_dataset_size() == 5
    assert s1.get_dataset_size() == 4
    assert s2.get_dataset_size() == 1
예제 #9
0
def test_manifest_sampler_chain():
    """
    Test Manifest sampler chain
    """
    logger.info("test_manifest_sampler_chain")

    sampler = ds.RandomSampler(replacement=True, num_samples=2)
    child_sampler = ds.DistributedSampler(num_shards=1,
                                          shard_id=0,
                                          shuffle=False,
                                          num_samples=3,
                                          offset=1)
    sampler.add_child(child_sampler)
    data1 = ds.ManifestDataset(MANIFEST_DATA_FILE, sampler=sampler)

    # Verify dataset size
    data1_size = data1.get_dataset_size()
    logger.info("dataset size is: {}".format(data1_size))
    assert data1_size == 2
    # Verify number of rows
    assert sum([1 for _ in data1]) == 2

    # Verify dataset contents
    res = []
    for item in data1.create_tuple_iterator(num_epochs=1, output_numpy=True):
        logger.info("item: {}".format(item))
        res.append(item)
    logger.info("dataset: {}".format(res))
예제 #10
0
def test_manifest_sampler_chain_repeat():
    """
    Test ManifestDataset sampler chain DistributedSampler->SequentialSampler, with repeat
    """
    logger.info("test_manifest_sampler_chain_batch")
    manifest_file = "../data/dataset/testManifestData/test5trainimgs.json"

    # Create sampler chain DistributedSampler->SequentialSampler
    sampler = ds.DistributedSampler(num_shards=1,
                                    shard_id=0,
                                    shuffle=False,
                                    num_samples=5)
    child_sampler = ds.SequentialSampler()
    sampler.add_child(child_sampler)

    # Create ManifestDataset with sampler chain
    data1 = ds.ManifestDataset(manifest_file, sampler=sampler)
    data1 = data1.repeat(count=2)

    # Verify dataset size
    data1_size = data1.get_dataset_size()
    logger.info("dataset size is: {}".format(data1_size))
    assert data1_size == 10

    # Verify number of rows
    assert sum([1 for _ in data1]) == 10

    # Verify dataset contents
    filename = "sampler_chain_manifest_repeat_result.npz"
    save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN)
예제 #11
0
def test_manifest_sampler_chain_batch_repeat():
    """
    Test ManifestDataset sampler chain DistributedSampler->SequentialSampler, with batch then repeat
    """
    logger.info("test_manifest_sampler_chain_batch_repeat")
    manifest_file = "../data/dataset/testManifestData/test5trainimgs.json"

    # Create sampler chain DistributedSampler->SequentialSampler
    sampler = ds.DistributedSampler(num_shards=1,
                                    shard_id=0,
                                    shuffle=False,
                                    num_samples=5)
    child_sampler = ds.SequentialSampler()
    sampler.add_child(child_sampler)

    # Create ManifestDataset with sampler chain
    data1 = ds.ManifestDataset(manifest_file, decode=True, sampler=sampler)
    one_hot_encode = c_transforms.OneHot(3)
    data1 = data1.map(operations=one_hot_encode, input_columns=["label"])
    data1 = data1.batch(batch_size=5, drop_remainder=False)
    data1 = data1.repeat(count=2)

    # Verify dataset size
    data1_size = data1.get_dataset_size()
    logger.info("dataset size is: {}".format(data1_size))
    assert data1_size == 2
예제 #12
0
    def test_config(start_index, num_samples):
        sampler = ds.SequentialSampler(start_index, num_samples)
        d = ds.ManifestDataset(manifest_file, sampler=sampler)

        res = []
        for item in d.create_dict_iterator(num_epochs=1, output_numpy=True):
            res.append(map_[(item["image"].shape[0], item["label"].item())])

        return res
예제 #13
0
    def test_config(num_samples, start_index, subset_size):
        sampler = ds.SubsetSampler(start_index, subset_size)
        d = ds.ManifestDataset(manifest_file, sampler=sampler)

        res = []
        for item in d.create_dict_iterator():
            res.append(map_[(item["image"].shape[0], item["label"].item())])

        return res
예제 #14
0
def test_manifest_dataset_multi_label():
    data = ds.ManifestDataset(DATA_FILE, decode=True, shuffle=False)
    count = 0
    expect_label = [1, 0, 0, [0, 2]]
    for item in data.create_dict_iterator(num_epochs=1, output_numpy=True):
        assert item["label"].tolist() == expect_label[count]
        logger.info("item[image] is {}".format(item["image"]))
        count = count + 1
    assert count == 4
예제 #15
0
def test_manifest_dataset_eval():
    data = ds.ManifestDataset(DATA_FILE, "eval", decode=True)
    count = 0
    for item in data.create_dict_iterator(num_epochs=1, output_numpy=True):
        logger.info("item[image] is {}".format(item["image"]))
        count = count + 1
        if item["label"] != 0 and item["label"] != 1:
            assert 0
    assert count == 2
예제 #16
0
 def test_config(replacement, num_samples, num_repeats):
     sampler = ds.RandomSampler(replacement=replacement, num_samples=num_samples)
     data1 = ds.ManifestDataset(manifest_file, sampler=sampler)
     data1 = data1.repeat(num_repeats)
     res = []
     for item in data1.create_dict_iterator():
         res.append(map_[(item["image"].shape[0], item["label"].item())])
     if print_res:
         logger.info("image.shapes and labels: {}".format(res))
     return res
예제 #17
0
def test_manifest_dataset_exception():
    def exception_func(item):
        raise Exception("Error occur!")

    try:
        data = ds.ManifestDataset(DATA_FILE)
        data = data.map(operations=exception_func,
                        input_columns=["image"],
                        num_parallel_workers=1)
        for _ in data.__iter__():
            pass
        assert False
    except RuntimeError as e:
        assert "map operation: [PyFunc] failed. The corresponding data files" in str(
            e)

    try:
        data = ds.ManifestDataset(DATA_FILE)
        data = data.map(operations=vision.Decode(),
                        input_columns=["image"],
                        num_parallel_workers=1)
        data = data.map(operations=exception_func,
                        input_columns=["image"],
                        num_parallel_workers=1)
        for _ in data.__iter__():
            pass
        assert False
    except RuntimeError as e:
        assert "map operation: [PyFunc] failed. The corresponding data files" in str(
            e)

    try:
        data = ds.ManifestDataset(DATA_FILE)
        data = data.map(operations=exception_func,
                        input_columns=["label"],
                        num_parallel_workers=1)
        for _ in data.__iter__():
            pass
        assert False
    except RuntimeError as e:
        assert "map operation: [PyFunc] failed. The corresponding data files" in str(
            e)
예제 #18
0
 def test_config(num_samples, num_repeats, sampler):
     data1 = ds.ManifestDataset(manifest_file, num_samples=num_samples, sampler=sampler)
     if num_repeats is not None:
         data1 = data1.repeat(num_repeats)
     res = []
     for item in data1.create_dict_iterator():
         logger.info("item[image].shape[0]: {}, item[label].item(): {}"
                     .format(item["image"].shape[0], item["label"].item()))
         res.append(map_[(item["image"].shape[0], item["label"].item())])
     # print(res)
     return res
예제 #19
0
def test_num_samples():
    manifest_file = "../data/dataset/testManifestData/test5trainimgs.json"
    num_samples = 1
    # sampler = ds.DistributedSampler(num_shards=1, shard_id=0, shuffle=False, num_samples=3, offset=1)
    data1 = ds.ManifestDataset(
        manifest_file, num_samples=num_samples, num_shards=3, shard_id=1
    )
    row_count = 0
    for _ in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
        row_count += 1
    assert row_count == 1
예제 #20
0
def test_manifest_dataset_multi_label_onehot():
    data = ds.ManifestDataset(DATA_FILE, decode=True, shuffle=False)
    expect_label = [[[0, 1, 0], [1, 0, 0]], [[1, 0, 0], [1, 0, 1]]]
    one_hot_encode = data_trans.OneHot(3)
    data = data.map(operations=one_hot_encode, input_columns=["label"])
    data = data.map(operations=multi_label_hot, input_columns=["label"])
    data = data.batch(2)
    count = 0
    for item in data.create_dict_iterator(num_epochs=1, output_numpy=True):
        assert item["label"].tolist() == expect_label[count]
        logger.info("item[image] is {}".format(item["image"]))
        count = count + 1
예제 #21
0
def test_add_sampler_invalid_input():
    manifest_file = "../data/dataset/testManifestData/test5trainimgs.json"
    _ = {(172876, 0): 0, (54214, 0): 1, (54214, 1): 2, (173673, 0): 3, (64631, 1): 4}
    data1 = ds.ManifestDataset(manifest_file)

    with pytest.raises(TypeError) as info:
        data1.use_sampler(1)
    assert "not an instance of a sampler" in str(info.value)

    with pytest.raises(TypeError) as info:
        data1.use_sampler("sampler")
    assert "not an instance of a sampler" in str(info.value)
예제 #22
0
def test_manifest_dataset_get_class_index():
    data = ds.ManifestDataset(DATA_FILE, decode=True)
    class_indexing = data.get_class_indexing()
    assert class_indexing == {'cat': 0, 'dog': 1, 'flower': 2}
    data = data.shuffle(4)
    class_indexing = data.get_class_indexing()
    assert class_indexing == {'cat': 0, 'dog': 1, 'flower': 2}
    count = 0
    for item in data.create_dict_iterator(num_epochs=1):
        logger.info("item[image] is {}".format(item["image"]))
        count = count + 1
    assert count == 4
예제 #23
0
 def test_config(num_samples, num_repeats=None):
     sampler = ds.SequentialSampler(num_samples=num_samples)
     data1 = ds.ManifestDataset(manifest_file, sampler=sampler)
     if num_repeats is not None:
         data1 = data1.repeat(num_repeats)
     res = []
     for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
         logger.info("item[image].shape[0]: {}, item[label].item(): {}"
                     .format(item["image"].shape[0], item["label"].item()))
         res.append(map_[(item["image"].shape[0], item["label"].item())])
     if print_res:
         logger.info("image.shapes and labels: {}".format(res))
     return res
예제 #24
0
 def test_config(replacement, num_samples, num_repeats, validate):
     sampler = ds.RandomSampler(replacement=replacement, num_samples=num_samples)
     data1 = ds.ManifestDataset(manifest_file, sampler=sampler)
     while num_repeats > 0:
         res = []
         for item in data1.create_dict_iterator():
             res.append(map_[(item["image"].shape[0], item["label"].item())])
         if print_res:
             logger.info("image.shapes and labels: {}".format(res))
         if validate != sorted(res):
             break
         num_repeats -= 1
     assert num_repeats > 0
예제 #25
0
    def test_config(num_shards, shard_id):
        sampler = ds.DistributedSampler(num_shards, shard_id, False)
        child_sampler = ds.SequentialSampler()
        sampler.add_child(child_sampler)

        data1 = ds.ManifestDataset(manifest_file, num_samples=5, sampler=sampler)

        res = []
        for item in data1.create_dict_iterator():
            logger.info("item[image].shape[0]: {}, item[label].item(): {}"
                        .format(item["image"].shape[0], item["label"].item()))
            res.append(map_[(item["image"].shape[0], item["label"].item())])
        return res
예제 #26
0
def test_mappable_split_general():
    d = ds.ManifestDataset(manifest_file, shuffle=False)
    d = d.take(5)

    # absolute rows
    s1, s2 = d.split([4, 1], randomize=False)

    s1_output = []
    for item in s1.create_dict_iterator():
        s1_output.append(manifest_map[(item["image"].shape[0],
                                       item["label"].item())])

    s2_output = []
    for item in s2.create_dict_iterator():
        s2_output.append(manifest_map[(item["image"].shape[0],
                                       item["label"].item())])

    assert s1_output == [0, 1, 2, 3]
    assert s2_output == [4]

    # exact percentages
    s1, s2 = d.split([0.8, 0.2], randomize=False)

    s1_output = []
    for item in s1.create_dict_iterator():
        s1_output.append(manifest_map[(item["image"].shape[0],
                                       item["label"].item())])

    s2_output = []
    for item in s2.create_dict_iterator():
        s2_output.append(manifest_map[(item["image"].shape[0],
                                       item["label"].item())])

    assert s1_output == [0, 1, 2, 3]
    assert s2_output == [4]

    # fuzzy percentages
    s1, s2 = d.split([0.33, 0.67], randomize=False)

    s1_output = []
    for item in s1.create_dict_iterator():
        s1_output.append(manifest_map[(item["image"].shape[0],
                                       item["label"].item())])

    s2_output = []
    for item in s2.create_dict_iterator():
        s2_output.append(manifest_map[(item["image"].shape[0],
                                       item["label"].item())])

    assert s1_output == [0, 1]
    assert s2_output == [2, 3, 4]
예제 #27
0
def test_manifest_dataset_class_index():
    class_indexing = {"dog": 11}
    data = ds.ManifestDataset(DATA_FILE,
                              decode=True,
                              class_indexing=class_indexing)
    out_class_indexing = data.get_class_indexing()
    assert out_class_indexing == {"dog": 11}
    count = 0
    for item in data.create_dict_iterator(num_epochs=1, output_numpy=True):
        logger.info("item[image] is {}".format(item["image"]))
        count = count + 1
        if item["label"] != 11:
            assert 0
    assert count == 1
예제 #28
0
def test_manifest_dataset_train():
    data = ds.ManifestDataset(DATA_FILE, decode=True)
    count = 0
    cat_count = 0
    dog_count = 0
    for item in data.create_dict_iterator(num_epochs=1, output_numpy=True):
        logger.info("item[image] is {}".format(item["image"]))
        count = count + 1
        if item["label"].size == 1 and item["label"] == 0:
            cat_count = cat_count + 1
        elif item["label"].size == 1 and item["label"] == 1:
            dog_count = dog_count + 1
    assert cat_count == 2
    assert dog_count == 1
    assert count == 4
예제 #29
0
def test_manifest_dataset_get_num_class():
    data = ds.ManifestDataset(DATA_FILE, decode=True, shuffle=False)
    assert data.num_classes() == 3

    padded_samples = [{
        'image': np.zeros(1, np.uint8),
        'label': np.array(1, np.int32)
    }]
    padded_ds = ds.PaddedDataset(padded_samples)

    data = data.repeat(2)
    padded_ds = padded_ds.repeat(2)

    data1 = data + padded_ds
    assert data1.num_classes() == 3
예제 #30
0
 def sharding_config(num_shards,
                     shard_id,
                     num_samples,
                     shuffle,
                     repeat_cnt=1):
     data1 = ds.ManifestDataset(manifest_file,
                                num_samples=num_samples,
                                num_shards=num_shards,
                                shard_id=shard_id,
                                shuffle=shuffle,
                                decode=True)
     data1 = data1.repeat(repeat_cnt)
     res = []
     for item in data1.create_dict_iterator():  # each data is a dictionary
         res.append(item["label"].item())
     if print_res:
         logger.info("labels of dataset: {}".format(res))
     return res