def test_high_dimension_memmap_array_reducing(tmpdir):
assert_array_equal = np.testing.assert_array_equal
filename = tmpdir.join('test.mmap').strpath
# Create a high dimensional memmap
a = np.memmap(filename,
dtype=np.float64,
shape=(100, 15, 15, 3),
mode='w+')
a[:] = np.arange(100 * 15 * 15 * 3).reshape(a.shape)
# Create some slices/indices at various dimensions
b = a[0:10]
c = a[:, 5:10]
d = a[:, :, :, 0]
e = a[1:3:4]
# Array reducer with auto dumping disabled
reducer = ArrayMemmapForwardReducer(None, tmpdir.strpath, 'c', True)
def reconstruct_array_or_memmap(x):
cons, args = reducer(x)
return cons(*args)
a_reconstructed = reconstruct_array_or_memmap(a)
assert has_shareable_memory(a_reconstructed)
assert isinstance(a_reconstructed, np.memmap)
assert_array_equal(a_reconstructed, a)
b_reconstructed = reconstruct_array_or_memmap(b)
assert has_shareable_memory(b_reconstructed)
assert_array_equal(b_reconstructed, b)
c_reconstructed = reconstruct_array_or_memmap(c)
assert has_shareable_memory(c_reconstructed)
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_array_or_memmap(d)
assert has_shareable_memory(d_reconstructed)
assert_array_equal(d_reconstructed, d)
e_reconstructed = reconstruct_array_or_memmap(e)
assert has_shareable_memory(e_reconstructed)
assert_array_equal(e_reconstructed, e)
def test_memmap_based_array_reducing(tmpdir):
"""Check that it is possible to reduce a memmap backed array"""
assert_array_equal = np.testing.assert_array_equal
filename = tmpdir.join('test.mmap').strpath
# Create a file larger than what will be used by a
buffer = np.memmap(filename, dtype=np.float64, shape=500, mode='w+')
# Fill the original buffer with negative markers to detect over of
# underflow in case of test failures
buffer[:] = -1.0 * np.arange(buffer.shape[0], dtype=buffer.dtype)
buffer.flush()
# Memmap a 2D fortran array on a offseted subsection of the previous
# buffer
a = np.memmap(filename,
dtype=np.float64,
shape=(3, 5, 4),
mode='r+',
order='F',
offset=4)
a[:] = np.arange(60).reshape(a.shape)
# Build various views that share the buffer with the original memmap
# b is an memmap sliced view on an memmap instance
b = a[1:-1, 2:-1, 2:4]
# c and d are array views
c = np.asarray(b)
d = c.T
# Array reducer with auto dumping disabled
reducer = ArrayMemmapForwardReducer(None, tmpdir.strpath, 'c', True)
def reconstruct_array_or_memmap(x):
cons, args = reducer(x)
return cons(*args)
# Reconstruct original memmap
a_reconstructed = reconstruct_array_or_memmap(a)
assert has_shareable_memory(a_reconstructed)
assert isinstance(a_reconstructed, np.memmap)
assert_array_equal(a_reconstructed, a)
# Reconstruct strided memmap view
b_reconstructed = reconstruct_array_or_memmap(b)
assert has_shareable_memory(b_reconstructed)
assert_array_equal(b_reconstructed, b)
# Reconstruct arrays views on memmap base
c_reconstructed = reconstruct_array_or_memmap(c)
assert not isinstance(c_reconstructed, np.memmap)
assert has_shareable_memory(c_reconstructed)
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_array_or_memmap(d)
assert not isinstance(d_reconstructed, np.memmap)
assert has_shareable_memory(d_reconstructed)
assert_array_equal(d_reconstructed, d)
# Test graceful degradation on fake memmap instances with in-memory
# buffers
a3 = a * 3
assert not has_shareable_memory(a3)
a3_reconstructed = reconstruct_array_or_memmap(a3)
assert not has_shareable_memory(a3_reconstructed)
assert not isinstance(a3_reconstructed, np.memmap)
assert_array_equal(a3_reconstructed, a * 3)
# Test graceful degradation on arrays derived from fake memmap instances
b3 = np.asarray(a3)
assert not has_shareable_memory(b3)
b3_reconstructed = reconstruct_array_or_memmap(b3)
assert isinstance(b3_reconstructed, np.ndarray)
assert not has_shareable_memory(b3_reconstructed)
assert_array_equal(b3_reconstructed, b3)