def test_standard_iterator_unbuffered(self):
dci = DataChunkIterator(data=range(10), buffer_size=1)
self.assertEqual(dci.dtype, np.dtype(int))
self.assertTupleEqual(dci.maxshape, (10,))
self.assertTupleEqual(dci.recommended_data_shape(), (10,)) # Test before and after iteration
count = 0
for chunk in dci:
self.assertTupleEqual(chunk.data.shape, (1,))
count += 1
self.assertEqual(count, 10)
self.assertTupleEqual(dci.recommended_data_shape(), (10,)) # Test before and after iteration
self.assertIsNone(dci.recommended_chunk_shape())
def test_numpy_iter_unbuffered_last_axis(self):
"""Test DataChunkIterator with numpy data, no buffering, and iterating on the last dimension.
"""
a = np.arange(30).reshape(5, 2, 3)
dci = DataChunkIterator(data=a, buffer_size=1, iter_axis=2)
count = 0
for chunk in dci:
self.assertTupleEqual(chunk.shape, (5, 2, 1))
count += 1
self.assertEqual(count, 3)
self.assertTupleEqual(dci.recommended_data_shape(), a.shape)
self.assertIsNone(dci.recommended_chunk_shape())
def test_multidimensional_list(self):
a = np.arange(30).reshape(5, 2, 3).tolist()
dci = DataChunkIterator(a)
self.assertTupleEqual(dci.maxshape, (5, 2, 3))
self.assertEqual(dci.dtype, np.dtype(int))
count = 0
for chunk in dci:
self.assertTupleEqual(chunk.data.shape, (1, 2, 3))
count += 1
self.assertEqual(count, 5)
self.assertTupleEqual(dci.recommended_data_shape(), (5, 2, 3))
self.assertIsNone(dci.recommended_chunk_shape())
def test_none_iter(self):
dci = DataChunkIterator(None)
self.assertIsNone(dci.maxshape)
self.assertIsNone(dci.dtype)
count = 0
for chunk in dci:
self.assertEqual(chunk.data, None)
self.assertEqual(chunk.selection, None)
count += 1
self.assertEqual(count, 0)
self.assertIsNone(dci.recommended_data_shape())
self.assertIsNone(dci.recommended_chunk_shape())
def test_list_none_dtype(self):
"""Test that DataChunkIterator has the passed-in dtype and no chunks when given a list of None.
"""
a = [None, None, None]
dci = DataChunkIterator(a, dtype=np.dtype('int'))
self.assertTupleEqual(dci.maxshape, (3,))
self.assertEqual(dci.dtype, np.dtype('int'))
count = 0
for chunk in dci:
pass
self.assertEqual(count, 0)
self.assertTupleEqual(dci.recommended_data_shape(), (3,))
self.assertIsNone(dci.recommended_chunk_shape())
def test_set_maxshape(self):
a = np.array([3])
dci = DataChunkIterator(a, maxshape=(5, 2, 3), buffer_size=2)
self.assertTupleEqual(dci.maxshape, (5, 2, 3))
self.assertEqual(dci.dtype, np.dtype(int))
count = 0
for chunk in dci:
self.assertListEqual(chunk.data.tolist(), [3])
self.assertTupleEqual(chunk.selection, (slice(0, 1), slice(None), slice(None)))
count += 1
self.assertEqual(count, 1)
self.assertTupleEqual(dci.recommended_data_shape(), (5, 2, 3))
self.assertIsNone(dci.recommended_chunk_shape())
def test_list_numpy_scalar(self):
a = np.array([3])
dci = DataChunkIterator(a, buffer_size=2)
self.assertTupleEqual(dci.maxshape, (1,))
self.assertEqual(dci.dtype, np.dtype(int))
count = 0
for chunk in dci:
self.assertListEqual(chunk.data.tolist(), [3])
self.assertEqual(len(chunk.selection), 1)
self.assertEqual(chunk.selection[0], slice(0, 1))
count += 1
self.assertEqual(count, 1)
self.assertTupleEqual(dci.recommended_data_shape(), (1,))
self.assertIsNone(dci.recommended_chunk_shape())
def test_start_with_none(self):
a = [None, None, 3]
dci = DataChunkIterator(a, buffer_size=2)
self.assertTupleEqual(dci.maxshape, (3,))
self.assertEqual(dci.dtype, np.dtype(int))
count = 0
for chunk in dci:
self.assertListEqual(chunk.data.tolist(), [3])
self.assertEqual(len(chunk.selection), 1)
self.assertEqual(chunk.selection[0], slice(2, 3))
count += 1
self.assertEqual(count, 1)
self.assertTupleEqual(dci.recommended_data_shape(), (3,))
self.assertIsNone(dci.recommended_chunk_shape())
def test_sparse_data_buffer_aligned(self):
a = [1, 2, 3, 4, None, None, 7, 8, None, None]
dci = DataChunkIterator(a, buffer_size=2)
self.assertTupleEqual(dci.maxshape, (10,))
self.assertEqual(dci.dtype, np.dtype(int))
count = 0
for chunk in dci:
self.assertTupleEqual(chunk.data.shape, (2,))
self.assertEqual(len(chunk.selection), 1)
self.assertEqual(chunk.selection[0], slice(chunk.data[0] - 1, chunk.data[1]))
count += 1
self.assertEqual(count, 3)
self.assertTupleEqual(dci.recommended_data_shape(), (10,))
self.assertIsNone(dci.recommended_chunk_shape())
def test_multidimensional_list_first_axis(self):
"""Test DataChunkIterator with multidimensional list data, no buffering, and iterating on the first dimension.
"""
a = np.arange(30).reshape(5, 2, 3).tolist()
dci = DataChunkIterator(a)
self.assertTupleEqual(dci.maxshape, (5, 2, 3))
self.assertEqual(dci.dtype, np.dtype(int))
count = 0
for chunk in dci:
self.assertTupleEqual(chunk.data.shape, (1, 2, 3))
count += 1
self.assertEqual(count, 5)
self.assertTupleEqual(dci.recommended_data_shape(), (5, 2, 3))
self.assertIsNone(dci.recommended_chunk_shape())
def test_none_iter(self):
"""Test that DataChunkIterator __init__ sets defaults correctly and all chunks and recommended shapes are None.
"""
dci = DataChunkIterator(dtype=np.dtype('int'))
self.assertIsNone(dci.maxshape)
self.assertEqual(dci.dtype, np.dtype('int'))
self.assertEqual(dci.buffer_size, 1)
self.assertEqual(dci.iter_axis, 0)
count = 0
for chunk in dci:
pass
self.assertEqual(count, 0)
self.assertIsNone(dci.recommended_data_shape())
self.assertIsNone(dci.recommended_chunk_shape())
def test_numpy_iter_unmatched_buffer_size(self):
a = np.arange(10)
dci = DataChunkIterator(data=a, buffer_size=3)
self.assertTupleEqual(dci.maxshape, a.shape)
self.assertEqual(dci.dtype, a.dtype)
count = 0
for chunk in dci:
if count < 3:
self.assertTupleEqual(chunk.data.shape, (3,))
else:
self.assertTupleEqual(chunk.data.shape, (1,))
count += 1
self.assertEqual(count, 4)
self.assertTupleEqual(dci.recommended_data_shape(), a.shape)
self.assertIsNone(dci.recommended_chunk_shape())
def test_numpy_iter_buffered_middle_axis(self):
"""Test DataChunkIterator with numpy data, buffering, and iterating on a middle dimension.
"""
a = np.arange(45).reshape(5, 3, 3)
dci = DataChunkIterator(data=a, buffer_size=2, iter_axis=1)
count = 0
for chunk in dci:
if count < 1:
self.assertTupleEqual(chunk.shape, (5, 2, 3))
else:
self.assertTupleEqual(chunk.shape, (5, 1, 3))
count += 1
self.assertEqual(count, 2)
self.assertTupleEqual(dci.recommended_data_shape(), a.shape)
self.assertIsNone(dci.recommended_chunk_shape())
def test_multidimensional_list_last_axis(self):
"""Test DataChunkIterator with multidimensional list data, no buffering, and iterating on the last dimension.
"""
a = np.arange(30).reshape(5, 2, 3).tolist()
warn_msg = ('Iterating over an axis other than the first dimension of list or tuple data '
'involves converting the data object to a numpy ndarray, which may incur a computational '
'cost.')
with self.assertWarnsWith(UserWarning, warn_msg):
dci = DataChunkIterator(a, iter_axis=2)
self.assertTupleEqual(dci.maxshape, (5, 2, 3))
self.assertEqual(dci.dtype, np.dtype(int))
count = 0
for chunk in dci:
self.assertTupleEqual(chunk.data.shape, (5, 2, 1))
count += 1
self.assertEqual(count, 3)
self.assertTupleEqual(dci.recommended_data_shape(), (5, 2, 3))
self.assertIsNone(dci.recommended_chunk_shape())
def test_sparse_data_buffer_notaligned(self):
a = [1, 2, 3, None, None, None, None, 8, 9, 10]
dci = DataChunkIterator(a, buffer_size=2)
self.assertTupleEqual(dci.maxshape, (10,))
self.assertEqual(dci.dtype, np.dtype(int))
count = 0
for chunk in dci:
self.assertEqual(len(chunk.selection), 1)
if count == 0: # [1, 2]
self.assertListEqual(chunk.data.tolist(), [1, 2])
self.assertEqual(chunk.selection[0], slice(chunk.data[0] - 1, chunk.data[1]))
elif count == 1: # [3, None]
self.assertListEqual(chunk.data.tolist(), [3, ])
self.assertEqual(chunk.selection[0], slice(chunk.data[0] - 1, chunk.data[0]))
elif count == 2: # [8, 9]
self.assertListEqual(chunk.data.tolist(), [8, 9])
self.assertEqual(chunk.selection[0], slice(chunk.data[0] - 1, chunk.data[1]))
else: # count == 3, [10]
self.assertListEqual(chunk.data.tolist(), [10, ])
self.assertEqual(chunk.selection[0], slice(chunk.data[0] - 1, chunk.data[0]))
count += 1
self.assertEqual(count, 4)
self.assertTupleEqual(dci.recommended_data_shape(), (10,))
self.assertIsNone(dci.recommended_chunk_shape())
####################
# Step 3: Write the data as usual
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
#
# Here we use our wrapped generator to create the data for a synthetic time series.
write_test_file(filename='basic_iterwrite_example.nwb', data=data)
####################
# Discussion
# ^^^^^^^^^^
# Note, we here actually do not know how long our timeseries will be.
print(
"maxshape=%s, recommended_data_shape=%s, dtype=%s" %
(str(data.maxshape), str(data.recommended_data_shape()), str(data.dtype)))
####################
# ``[Out]:``
#
# .. code-block:: python
#
# maxshape=(None, 10), recommended_data_shape=(1, 10), dtype=float64
#
# As we can see :py:class:`~hdmf.data_utils.DataChunkIterator` automatically recommends
# in its ``maxshape`` that the first dimensions of our array should be unlimited (``None``) and the second
# dimension be ``10`` (i.e., the length of our chunk. Since :py:class:`~hdmf.data_utils.DataChunkIterator`
# has no way of knowing the minimum size of the array it automatically recommends the size of the first
# chunk as the minimum size (i.e, ``(1, 10)``) and also infers the data type automatically from the first chunk.
# To further customize this behavior we may also define the ``maxshape``, ``dtype``, and ``buffer_size`` when
# we create the :py:class:`~hdmf.data_utils.DataChunkIterator`.
