def test_pool_with_memmap_array_view(tmpdir):
"""Check that subprocess can access and update shared memory array"""
assert_array_equal = np.testing.assert_array_equal
# Fork the subprocess before allocating the objects to be passed
pool_temp_folder = tmpdir.mkdir('pool').strpath
p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
try:
filename = tmpdir.join('test.mmap').strpath
a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
a.fill(1.0)
# Create an ndarray view on the memmap instance
a_view = np.asarray(a)
assert not isinstance(a_view, np.memmap)
assert has_shareable_memory(a_view)
p.map(inplace_double, [(a_view, (i, j), 1.0)
for i in range(a.shape[0])
for j in range(a.shape[1])])
# Both a and the a_view have been updated
assert_array_equal(a, 2 * np.ones(a.shape))
assert_array_equal(a_view, 2 * np.ones(a.shape))
# Passing memmap array view to the pool should not trigger the
# creation of new files on the FS
assert os.listdir(pool_temp_folder) == []
finally:
p.terminate()
del p
def test_pool_with_memmap_array_view():
"""Check that subprocess can access and update shared memory array"""
assert_array_equal = np.testing.assert_array_equal
# Fork the subprocess before allocating the objects to be passed
pool_temp_folder = os.path.join(TEMP_FOLDER, 'pool')
os.makedirs(pool_temp_folder)
p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
try:
filename = os.path.join(TEMP_FOLDER, 'test.mmap')
a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
a.fill(1.0)
# Create an ndarray view on the memmap instance
a_view = np.asarray(a)
assert_false(isinstance(a_view, np.memmap))
assert_true(has_shareable_memory(a_view))
p.map(inplace_double, [(a_view, (i, j), 1.0) for i in range(a.shape[0])
for j in range(a.shape[1])])
# Both a and the a_view have been updated
assert_array_equal(a, 2 * np.ones(a.shape))
assert_array_equal(a_view, 2 * np.ones(a.shape))
# Passing memmap array view to the pool should not trigger the
# creation of new files on the FS
assert_equal(os.listdir(pool_temp_folder), [])
finally:
p.terminate()
del p
def test_high_dimension_memmap_array_reducing():
assert_array_equal = np.testing.assert_array_equal
filename = os.path.join(TEMP_FOLDER, 'test.mmap')
# Create a high dimensional memmap
a = np.memmap(filename,
dtype=np.float64,
shape=(100, 15, 15, 3),
mode='w+')
a[:] = np.arange(100 * 15 * 15 * 3).reshape(a.shape)
# Create some slices/indices at various dimensions
b = a[0:10]
c = a[:, 5:10]
d = a[:, :, :, 0]
e = a[1:3:4]
def reconstruct_memmap(x):
cons, args = reduce_memmap(x)
res = cons(*args)
return res
a_reconstructed = reconstruct_memmap(a)
assert_true(has_shareable_memory(a_reconstructed))
assert_true(isinstance(a_reconstructed, np.memmap))
assert_array_equal(a_reconstructed, a)
b_reconstructed = reconstruct_memmap(b)
assert_true(has_shareable_memory(b_reconstructed))
assert_array_equal(b_reconstructed, b)
c_reconstructed = reconstruct_memmap(c)
assert_true(has_shareable_memory(c_reconstructed))
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_memmap(d)
assert_true(has_shareable_memory(d_reconstructed))
assert_array_equal(d_reconstructed, d)
e_reconstructed = reconstruct_memmap(e)
assert_true(has_shareable_memory(e_reconstructed))
assert_array_equal(e_reconstructed, e)
def test_high_dimension_memmap_array_reducing(tmpdir):
assert_array_equal = np.testing.assert_array_equal
filename = tmpdir.join('test.mmap').strpath
# Create a high dimensional memmap
a = np.memmap(filename, dtype=np.float64, shape=(100, 15, 15, 3),
mode='w+')
a[:] = np.arange(100 * 15 * 15 * 3).reshape(a.shape)
# Create some slices/indices at various dimensions
b = a[0:10]
c = a[:, 5:10]
d = a[:, :, :, 0]
e = a[1:3:4]
def reconstruct_memmap(x):
cons, args = reduce_memmap(x)
res = cons(*args)
return res
a_reconstructed = reconstruct_memmap(a)
assert has_shareable_memory(a_reconstructed)
assert isinstance(a_reconstructed, np.memmap)
assert_array_equal(a_reconstructed, a)
b_reconstructed = reconstruct_memmap(b)
assert has_shareable_memory(b_reconstructed)
assert_array_equal(b_reconstructed, b)
c_reconstructed = reconstruct_memmap(c)
assert has_shareable_memory(c_reconstructed)
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_memmap(d)
assert has_shareable_memory(d_reconstructed)
assert_array_equal(d_reconstructed, d)
e_reconstructed = reconstruct_memmap(e)
assert has_shareable_memory(e_reconstructed)
assert_array_equal(e_reconstructed, e)
def test_memmaping_pool_for_large_arrays_in_return(tmpdir):
"""Check that large arrays are not copied in memory in return"""
assert_array_equal = np.testing.assert_array_equal
# Build an array reducers that automaticaly dump large array content
# but check that the returned datastructure are regular arrays to avoid
# passing a memmap array pointing to a pool controlled temp folder that
# might be confusing to the user
# The MemmapingPool user can always return numpy.memmap object explicitly
# to avoid memory copy
p = MemmapingPool(3, max_nbytes=10, temp_folder=tmpdir.strpath)
try:
res = p.apply_async(np.ones, args=(1000,))
large = res.get()
assert not has_shareable_memory(large)
assert_array_equal(large, np.ones(1000))
finally:
p.terminate()
del p
def test_memmaping_pool_for_large_arrays_in_return():
"""Check that large arrays are not copied in memory in return"""
assert_array_equal = np.testing.assert_array_equal
# Build an array reducers that automaticaly dump large array content
# but check that the returned datastructure are regular arrays to avoid
# passing a memmap array pointing to a pool controlled temp folder that
# might be confusing to the user
# The MemmapingPool user can always return numpy.memmap object explicitly
# to avoid memory copy
p = MemmapingPool(3, max_nbytes=10, temp_folder=TEMP_FOLDER)
try:
res = p.apply_async(np.ones, args=(1000, ))
large = res.get()
assert_false(has_shareable_memory(large))
assert_array_equal(large, np.ones(1000))
finally:
p.terminate()
del p
def test_workaround_against_bad_memmap_with_copied_buffers(tmpdir):
"""Check that memmaps with a bad buffer are returned as regular arrays
Unary operations and ufuncs on memmap instances return a new memmap
instance with an in-memory buffer (probably a numpy bug).
"""
assert_array_equal = np.testing.assert_array_equal
p = MemmapingPool(3, max_nbytes=10, temp_folder=tmpdir.strpath)
try:
# Send a complex, large-ish view on a array that will be converted to
# a memmap in the worker process
a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),
order='F')[:, :1, :]
# Call a non-inplace multiply operation on the worker and memmap and
# send it back to the parent.
b = p.apply_async(_worker_multiply, args=(a, 3)).get()
assert not has_shareable_memory(b)
assert_array_equal(b, 3 * a)
finally:
p.terminate()
del p
def test_workaround_against_bad_memmap_with_copied_buffers():
"""Check that memmaps with a bad buffer are returned as regular arrays
Unary operations and ufuncs on memmap instances return a new memmap
instance with an in-memory buffer (probably a numpy bug).
"""
assert_array_equal = np.testing.assert_array_equal
p = MemmapingPool(3, max_nbytes=10, temp_folder=TEMP_FOLDER)
try:
# Send a complex, large-ish view on a array that will be converted to
# a memmap in the worker process
a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),
order='F')[:, :1, :]
# Call a non-inplace multiply operation on the worker and memmap and
# send it back to the parent.
b = p.apply_async(_worker_multiply, args=(a, 3)).get()
assert_false(has_shareable_memory(b))
assert_array_equal(b, 3 * a)
finally:
p.terminate()
del p
def test_memmap_based_array_reducing(tmpdir):
"""Check that it is possible to reduce a memmap backed array"""
assert_array_equal = np.testing.assert_array_equal
filename = tmpdir.join('test.mmap').strpath
# Create a file larger than what will be used by a
buffer = np.memmap(filename, dtype=np.float64, shape=500, mode='w+')
# Fill the original buffer with negative markers to detect over of
# underflow in case of test failures
buffer[:] = - 1.0 * np.arange(buffer.shape[0], dtype=buffer.dtype)
buffer.flush()
# Memmap a 2D fortran array on a offseted subsection of the previous
# buffer
a = np.memmap(filename, dtype=np.float64, shape=(3, 5, 4),
mode='r+', order='F', offset=4)
a[:] = np.arange(60).reshape(a.shape)
# Build various views that share the buffer with the original memmap
# b is an memmap sliced view on an memmap instance
b = a[1:-1, 2:-1, 2:4]
# c and d are array views
c = np.asarray(b)
d = c.T
# Array reducer with auto dumping disabled
reducer = ArrayMemmapReducer(None, tmpdir.strpath, 'c')
def reconstruct_array(x):
cons, args = reducer(x)
return cons(*args)
def reconstruct_memmap(x):
cons, args = reduce_memmap(x)
return cons(*args)
# Reconstruct original memmap
a_reconstructed = reconstruct_memmap(a)
assert has_shareable_memory(a_reconstructed)
assert isinstance(a_reconstructed, np.memmap)
assert_array_equal(a_reconstructed, a)
# Reconstruct strided memmap view
b_reconstructed = reconstruct_memmap(b)
assert has_shareable_memory(b_reconstructed)
assert_array_equal(b_reconstructed, b)
# Reconstruct arrays views on memmap base
c_reconstructed = reconstruct_array(c)
assert not isinstance(c_reconstructed, np.memmap)
assert has_shareable_memory(c_reconstructed)
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_array(d)
assert not isinstance(d_reconstructed, np.memmap)
assert has_shareable_memory(d_reconstructed)
assert_array_equal(d_reconstructed, d)
# Test graceful degradation on fake memmap instances with in-memory
# buffers
a3 = a * 3
assert not has_shareable_memory(a3)
a3_reconstructed = reconstruct_memmap(a3)
assert not has_shareable_memory(a3_reconstructed)
assert not isinstance(a3_reconstructed, np.memmap)
assert_array_equal(a3_reconstructed, a * 3)
# Test graceful degradation on arrays derived from fake memmap instances
b3 = np.asarray(a3)
assert not has_shareable_memory(b3)
b3_reconstructed = reconstruct_array(b3)
assert isinstance(b3_reconstructed, np.ndarray)
assert not has_shareable_memory(b3_reconstructed)
assert_array_equal(b3_reconstructed, b3)
def _worker_multiply(a, n_times):
"""Multiplication function to be executed by subprocess"""
assert has_shareable_memory(a)
return a * n_times
def test_memmap_based_array_reducing():
"""Check that it is possible to reduce a memmap backed array"""
assert_array_equal = np.testing.assert_array_equal
filename = os.path.join(TEMP_FOLDER, 'test.mmap')
# Create a file larger than what will be used by a
buffer = np.memmap(filename, dtype=np.float64, shape=500, mode='w+')
# Fill the original buffer with negative markers to detect over of
# underflow in case of test failures
buffer[:] = -1.0 * np.arange(buffer.shape[0], dtype=buffer.dtype)
buffer.flush()
# Memmap a 2D fortran array on a offseted subsection of the previous
# buffer
a = np.memmap(filename,
dtype=np.float64,
shape=(3, 5, 4),
mode='r+',
order='F',
offset=4)
a[:] = np.arange(60).reshape(a.shape)
# Build various views that share the buffer with the original memmap
# b is an memmap sliced view on an memmap instance
b = a[1:-1, 2:-1, 2:4]
# c and d are array views
c = np.asarray(b)
d = c.T
# Array reducer with auto dumping disabled
reducer = ArrayMemmapReducer(None, TEMP_FOLDER, 'c')
def reconstruct_array(x):
cons, args = reducer(x)
return cons(*args)
def reconstruct_memmap(x):
cons, args = reduce_memmap(x)
return cons(*args)
# Reconstruct original memmap
a_reconstructed = reconstruct_memmap(a)
assert_true(has_shareable_memory(a_reconstructed))
assert_true(isinstance(a_reconstructed, np.memmap))
assert_array_equal(a_reconstructed, a)
# Reconstruct strided memmap view
b_reconstructed = reconstruct_memmap(b)
assert_true(has_shareable_memory(b_reconstructed))
assert_array_equal(b_reconstructed, b)
# Reconstruct arrays views on memmap base
c_reconstructed = reconstruct_array(c)
assert_false(isinstance(c_reconstructed, np.memmap))
assert_true(has_shareable_memory(c_reconstructed))
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_array(d)
assert_false(isinstance(d_reconstructed, np.memmap))
assert_true(has_shareable_memory(d_reconstructed))
assert_array_equal(d_reconstructed, d)
# Test graceful degradation on fake memmap instances with in-memory
# buffers
a3 = a * 3
assert_false(has_shareable_memory(a3))
a3_reconstructed = reconstruct_memmap(a3)
assert_false(has_shareable_memory(a3_reconstructed))
assert_false(isinstance(a3_reconstructed, np.memmap))
assert_array_equal(a3_reconstructed, a * 3)
# Test graceful degradation on arrays derived from fake memmap instances
b3 = np.asarray(a3)
assert_false(has_shareable_memory(b3))
b3_reconstructed = reconstruct_array(b3)
assert_true(isinstance(b3_reconstructed, np.ndarray))
assert_false(has_shareable_memory(b3_reconstructed))
assert_array_equal(b3_reconstructed, b3)
def _worker_multiply(a, n_times):
"""Multiplication function to be executed by subprocess"""
assert_true(has_shareable_memory(a))
return a * n_times
