예제 #1
0
 def testMultipleComponents(self):
     dataset = dataset_ops.Dataset.from_tensor_slices(([1, 2], [3,
                                                                4], [5, 6]))
     self.assertEqual((1, 3, 5), self.evaluate(random_access.at(dataset,
                                                                0)))
     self.assertEqual((2, 4, 6), self.evaluate(random_access.at(dataset,
                                                                1)))
예제 #2
0
    def testBasic(self, count, batch_size, drop_remainder):
        """Tests the batch dataset logic for various input configurations.

    Args:
      count: the number of input elements
      batch_size: the batch size
      drop_remainder: whether a smaller batch size should be produced if batch
        size does not divide number of inputs evenly
    """
        dataset = dataset_ops.Dataset.from_tensor_slices(list(
            range(count))).batch(batch_size=batch_size,
                                 drop_remainder=drop_remainder)
        num_full_batches = count // batch_size
        for i in range(num_full_batches):
            expected_batch = np.arange(i * batch_size,
                                       (i * batch_size + batch_size),
                                       1,
                                       dtype=np.int32)
            self.assertAllEqual(expected_batch,
                                self.evaluate(random_access.at(dataset, i)))
        has_remainder = (not drop_remainder) and (count % batch_size != 0)
        if has_remainder:
            expected_batch = np.arange(batch_size * num_full_batches, count, 1)
            self.assertAllEqual(
                expected_batch,
                self.evaluate(random_access.at(dataset, num_full_batches)))
        with self.assertRaises(errors.OutOfRangeError):
            self.evaluate(
                random_access.at(dataset,
                                 index=num_full_batches +
                                 (1 if has_remainder else 0)))
예제 #3
0
 def testInvalidIndex(self):
   dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3
                                                    ]).shuffle(buffer_size=100)
   with self.assertRaises(errors.OutOfRangeError):
     self.evaluate(random_access.at(dataset, -1))
   with self.assertRaises(errors.OutOfRangeError):
     self.evaluate(random_access.at(dataset, 4))
예제 #4
0
    def testRandomAccessBatchWithShuffle(self):
        dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6, 7])
        shuffle_dataset = dataset.shuffle(buffer_size=10, seed=2)
        batch_dataset = shuffle_dataset.batch(2)

        expected_output = [
            np.array([5, 2], dtype=np.int32),
            np.array([4, 7], dtype=np.int32),
            np.array([1, 3], dtype=np.int32),
            np.array([6], dtype=np.int32)
        ]
        for i in range(4):
            self.assertAllEqual(
                expected_output[i],
                self.evaluate(random_access.at(batch_dataset, i)))

        # Checks the order is consistent with shuffle dataset.
        for i in range(3):
            self.assertAllEqual(
                expected_output[i][0],
                self.evaluate(random_access.at(shuffle_dataset, i * 2)))
            self.assertAllEqual(
                expected_output[i][1],
                self.evaluate(random_access.at(shuffle_dataset, (i * 2) + 1)))

        # Checks the remainder is the last element in shuffled dataset.
        self.assertAllEqual(
            expected_output[3][0],
            self.evaluate(random_access.at(shuffle_dataset, 6)))
예제 #5
0
 def testBasic(self, count):
   dataset = dataset_ops.Dataset.range(10).take(count)
   num_output = min(count, 10) if count != -1 else 10
   for i in range(num_output):
     self.assertEqual(
         self.evaluate(random_access.at(dataset, index=i)), i)
   with self.assertRaises(errors.OutOfRangeError):
     self.evaluate(random_access.at(dataset, index=num_output))
 def testConcatenateTwoNonEmptyDatasets(self):
     input_dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2])
     concatenate_dataset = dataset_ops.Dataset.from_tensor_slices([3, 4])
     concatenated = input_dataset.concatenate(concatenate_dataset)
     for i in range(5):
         self.assertAllEqual(random_access.at(concatenated, index=i), i)
     with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(random_access.at(concatenated, index=5))
예제 #7
0
 def testNumpyArray(self):
     components = (np.array(1), np.array([1, 2, 3]), np.array(37.0))
     dataset = dataset_ops.Dataset.from_tensors(components)
     result = self.evaluate(random_access.at(dataset, 0))
     for i in range(3):
         self.assertAllEqual(result[i], components[i])
     with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(random_access.at(dataset, 1))
 def testConcatenateAnEmptyDataset(self):
     input_dataset = dataset_ops.Dataset.from_tensor_slices([1.0])
     concatenate_dataset = dataset_ops.Dataset.from_tensor_slices([])
     concatenated = input_dataset.concatenate(concatenate_dataset)
     self.assertAllEqual(
         self.evaluate(random_access.at(concatenated, index=0)), 1.0)
     with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(random_access.at(concatenated, index=1))
예제 #9
0
 def testZipUnequal(self):
     components = [[1, 2, 3, 4], [1, 2, 3, 4, 5], [1.0, 2.0]]
     dataset = _dataset_factory(components)
     for i in range(2):
         results = self.evaluate(random_access.at(dataset, index=i))
         for component, result_component in zip(components, results):
             self.assertAllEqual(component[i], result_component)
     with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(random_access.at(dataset, index=2))
예제 #10
0
 def testMultipleCombinations(self, elements, buffer_size):
     dataset = dataset_ops.Dataset.range(elements).prefetch(
         buffer_size=buffer_size)
     len_dataset = self.evaluate(dataset.cardinality())
     expected_output = np.arange(elements)
     for i in range(len_dataset):
         self.assertEqual(self.evaluate(random_access.at(dataset, index=i)),
                          expected_output[i])
     with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(random_access.at(dataset, index=len_dataset))
예제 #11
0
 def testDictionary(self):
     dataset = from_list.from_list([{"a": 1, "b": 3}, {"a": 2, "b": 4}])
     self.assertEqual({
         "a": 1,
         "b": 3
     }, self.evaluate(random_access.at(dataset, 0)))
     self.assertEqual({
         "a": 2,
         "b": 4
     }, self.evaluate(random_access.at(dataset, 1)))
예제 #12
0
 def testDictionary(self):
   dataset = dataset_ops.Dataset.from_tensor_slices({"a": [1, 2], "b": [3, 4]})
   self.assertEqual({
       "a": 1,
       "b": 3
   }, self.evaluate(random_access.at(dataset, 0)))
   self.assertEqual({
       "a": 2,
       "b": 4
   }, self.evaluate(random_access.at(dataset, 1)))
예제 #13
0
 def testMultipleCombinations(self, elements, num_shards, index):
   components = range(elements)
   dataset = dataset_ops.Dataset.range(elements).shard(
       num_shards=num_shards, index=index)
   len_dataset = self.evaluate(dataset.cardinality())
   for i in range(self.evaluate(dataset.cardinality())):
     self.assertAllEqual(components[index + (num_shards * i)],
                         self.evaluate(random_access.at(dataset, i)))
   with self.assertRaises(errors.OutOfRangeError):
     self.evaluate(random_access.at(dataset, index=len_dataset))
예제 #14
0
    def testWithOptions(self):
        dataset = dataset_ops.Dataset.from_tensors(range(4))
        options = options_lib.Options()
        options.experimental_optimization.map_and_batch_fusion = True
        dataset = dataset.with_options(options)

        self.assertAllEqual(self.evaluate(random_access.at(dataset, 0)),
                            range(4))
        with self.assertRaises(errors.OutOfRangeError):
            self.evaluate(random_access.at(dataset, 1))
예제 #15
0
 def testNamedTuple(self):
     Foo = collections.namedtuple("Foo", ["x", "y"])
     x = Foo(x=dataset_ops.Dataset.range(3),
             y=dataset_ops.Dataset.range(3, 6))
     dataset = dataset_ops.Dataset.zip(x)
     expected = [Foo(x=0, y=3), Foo(x=1, y=4), Foo(x=2, y=5)]
     for i in range(3):
         self.assertAllEqual(
             self.evaluate(random_access.at(dataset, index=i)), expected[i])
     with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(random_access.at(dataset, index=4))
예제 #16
0
  def testBasicWithoutSeedEager(self):
    dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3, 4, 5])
    shuffled_dataset = dataset.shuffle(buffer_size=100)

    dataset_array = []
    shuffled_dataset_array = []

    for i in range(5):
      shuffled_dataset_array.append(
          self.evaluate(random_access.at(shuffled_dataset, i)))
      dataset_array.append(self.evaluate(random_access.at(dataset, i)))
    self.assertAllEqual(sorted(dataset_array), sorted(shuffled_dataset_array))
예제 #17
0
 def testZipEqual(self):
     components = [
         np.tile(np.array([[1], [2], [3], [4]]), 20),
         np.tile(np.array([[12], [13], [14], [15]]), 22),
         np.array([37.0, 38.0, 39.0, 40.0])
     ]
     dataset = _dataset_factory(components)
     for i in range(4):
         results = self.evaluate(random_access.at(dataset, index=i))
         for component, result_component in zip(components, results):
             self.assertAllEqual(component[i], result_component)
     with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(random_access.at(dataset, index=4))
예제 #18
0
  def testSameSeedReturnsSameSequence(self):
    dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3, 4, 5])
    shuffled_dataset = dataset.shuffle(buffer_size=100, seed=5)
    shuffled_dataset_2 = dataset.shuffle(buffer_size=100, seed=5)

    shuffled_dataset_array = []
    shuffled_dataset_array_2 = []

    for i in range(5):
      shuffled_dataset_array.append(
          self.evaluate(random_access.at(shuffled_dataset, i)))
      shuffled_dataset_array_2.append(
          self.evaluate(random_access.at(shuffled_dataset_2, i)))
    self.assertAllEqual(shuffled_dataset_array, shuffled_dataset_array_2)
예제 #19
0
  def verifyRandomAccessInfiniteCardinality(self, dataset, expected):
    """Tests randomly accessing elements of a dataset."""
    # Tests accessing the elements in a shuffled order with repeats.
    len_expected = len(expected)
    indices = list(range(len_expected)) * 2
    random.shuffle(indices)
    for i in indices:
      self.assertAllEqual(expected[i],
                          self.evaluate(random_access.at(dataset, i)))

    # Tests accessing the elements in order.
    indices = set(sorted(indices))
    for i in indices:
      self.assertAllEqual(expected[i],
                          self.evaluate(random_access.at(dataset, i)))
예제 #20
0
 def testMultipleCombinations(self, start, stop, step, output_type):
     dataset = dataset_ops.Dataset.range(start,
                                         stop,
                                         step,
                                         output_type=output_type)
     expected_output = np.arange(start,
                                 stop,
                                 step,
                                 dtype=output_type.as_numpy_dtype)
     len_dataset = self.evaluate(dataset.cardinality())
     for i in range(len_dataset):
         self.assertEqual(self.evaluate(random_access.at(dataset, index=i)),
                          expected_output[i])
     with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(random_access.at(dataset, index=len_dataset))
예제 #21
0
 def testMultipleCombinations(self, elements, num_shards, index):
     components = range(elements)
     dataset = dataset_ops.Dataset.from_tensor_slices(components).shard(
         num_shards=num_shards, index=index)
     for i in range(self.evaluate(dataset.cardinality())):
         self.assertAllEqual(components[index + (num_shards * i)],
                             self.evaluate(random_access.at(dataset, i)))
예제 #22
0
 def testZipBasic(self):
     dataset = dataset_ops.Dataset.zip(
         (dataset_ops.Dataset.range(1, 4), dataset_ops.Dataset.range(4, 7)))
     expected_dataset = [(1, 4), (2, 5), (3, 6)]
     for i in range(3):
         self.assertEqual(self.evaluate(random_access.at(dataset, index=i)),
                          expected_dataset[i])
예제 #23
0
  def testDifferentSeedDifferentSequence(self):
    components = list(range(1000))
    dataset = dataset_ops.Dataset.from_tensor_slices(components)
    shuffled_dataset = dataset.shuffle(buffer_size=1000, seed=124)
    shuffled_dataset_2 = dataset.shuffle(buffer_size=1000, seed=51)

    shuffled_dataset_array = []
    shuffled_dataset_array_2 = []

    for i in range(1000):
      shuffled_dataset_array.append(
          self.evaluate(random_access.at(shuffled_dataset, i)))
      shuffled_dataset_array_2.append(
          self.evaluate(random_access.at(shuffled_dataset_2, i)))
    self.assertNotEqual(shuffled_dataset_array, shuffled_dataset_array_2)
    self.assertAllEqual(
        sorted(shuffled_dataset_array), sorted(shuffled_dataset_array_2))
예제 #24
0
    def testAttrs(self):
        if attr is None:
            self.skipTest("attr module is not available.")

        @attr.s
        class Foo:
            x = attr.ib()
            y = attr.ib()

        x = Foo(x=dataset_ops.Dataset.range(3),
                y=dataset_ops.Dataset.range(3, 6))
        dataset = dataset_ops.Dataset.zip(x)
        expected = [Foo(x=0, y=3), Foo(x=1, y=4), Foo(x=2, y=5)]
        for i in range(3):
            self.assertAllEqual(
                self.evaluate(random_access.at(dataset, index=i)), expected[i])
        with self.assertRaises(errors.OutOfRangeError):
            self.evaluate(random_access.at(dataset, index=4))
예제 #25
0
 def testFiniteRepeat(self, elements, count):
     dataset = dataset_ops.Dataset.range(elements).repeat(count)
     expected_dataset = np.tile(
         np.arange(start=0,
                   stop=elements,
                   step=1,
                   dtype=dtypes.int64.as_numpy_dtype), count)
     for i in range(elements * count):
         self.assertEqual(self.evaluate(random_access.at(dataset, index=i)),
                          expected_dataset[i])
예제 #26
0
 def testNumpy(self):
     elements = [
         np.tile(np.array([[0], [1]], dtype=np.uint64), 2),
         np.tile(np.array([[2], [256]], dtype=np.uint64), 2),
         np.tile(np.array([[4], [65536]], dtype=np.uint64), 2),
         np.tile(np.array([[8], [4294967296]], dtype=np.uint64), 2),
     ]
     dataset = from_list.from_list(elements)
     for i in range(len(elements)):
         result = self.evaluate(random_access.at(dataset, i))
         self.assertAllEqual(elements[i], result)
예제 #27
0
 def testNested(self):
     components = [
         np.tile(np.array([[1], [2], [3], [4]]), 20),
         np.tile(np.array([[12], [13], [14], [15]]), 22),
         np.array([37.0, 38.0, 39.0, 40.0])
     ]
     datasets = [
         dataset_ops.Dataset.from_tensor_slices(component)
         for component in components
     ]
     dataset = dataset_ops.Dataset.zip(
         (datasets[0], (datasets[1], datasets[2])))
     for i in range(4):
         result1, (result2, result3) = self.evaluate(
             random_access.at(dataset, index=i))
         self.assertAllEqual(components[0][i], result1)
         self.assertAllEqual(components[1][i], result2)
         self.assertAllEqual(components[2][i], result3)
     with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(random_access.at(dataset, index=4))
예제 #28
0
    def testNumpy(self):
        components = (
            np.tile(np.array([[0], [1]], dtype=np.uint8), 2),
            np.tile(np.array([[2], [256]], dtype=np.uint16), 2),
            np.tile(np.array([[4], [65536]], dtype=np.uint32), 2),
            np.tile(np.array([[8], [4294967296]], dtype=np.uint64), 2),
        )
        expected_output = [tuple([c[i] for c in components]) for i in range(2)]

        dataset = dataset_ops.Dataset.from_tensor_slices(components)
        for i in range(2):
            result = self.evaluate(random_access.at(dataset, i))
            self.assertAllEqual(expected_output[i], result)
예제 #29
0
 def testEmptyDataset(self):
   dataset = dataset_ops.Dataset.from_tensor_slices(
       []).shuffle(buffer_size=100)
   with self.assertRaises(errors.OutOfRangeError):
     self.evaluate(random_access.at(dataset, 0))
예제 #30
0
 def testEmptyDataset(self, index):
     dataset = dataset_ops.Dataset.from_tensor_slices(
         []).prefetch(buffer_size=5)
     with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(random_access.at(dataset, index=index))