def test_pool_memmap_with_big_offset(tmpdir):
# Test that numpy memmap offset is set correctly if greater than
# mmap.ALLOCATIONGRANULARITY, see
# https://github.com/joblib/joblib/issues/451 and
# https://github.com/numpy/numpy/pull/8443 for more details.
fname = tmpdir.join('test.mmap').strpath
size = 5 * mmap.ALLOCATIONGRANULARITY
offset = mmap.ALLOCATIONGRANULARITY + 1
obj = make_memmap(fname,
mode='w+',
shape=size,
dtype='uint8',
offset=offset)
p = MemmapingPool(2, temp_folder=tmpdir.strpath)
result = p.apply_async(identity, args=(obj, )).get()
assert isinstance(result, np.memmap)
assert result.offset == offset
np.testing.assert_array_equal(obj, result)
def test_memmaping_pool_for_large_arrays_in_return(tmpdir):
"""Check that large arrays are not copied in memory in return"""
assert_array_equal = np.testing.assert_array_equal
# Build an array reducers that automaticaly dump large array content
# but check that the returned datastructure are regular arrays to avoid
# passing a memmap array pointing to a pool controlled temp folder that
# might be confusing to the user
# The MemmapingPool user can always return numpy.memmap object explicitly
# to avoid memory copy
p = MemmapingPool(3, max_nbytes=10, temp_folder=tmpdir.strpath)
try:
res = p.apply_async(np.ones, args=(1000,))
large = res.get()
assert not has_shareable_memory(large)
assert_array_equal(large, np.ones(1000))
finally:
p.terminate()
del p
def test_memmaping_pool_for_large_arrays_in_return():
"""Check that large arrays are not copied in memory in return"""
assert_array_equal = np.testing.assert_array_equal
# Build an array reducers that automaticaly dump large array content
# but check that the returned datastructure are regular arrays to avoid
# passing a memmap array pointing to a pool controlled temp folder that
# might be confusing to the user
# The MemmapingPool user can always return numpy.memmap object explicitly
# to avoid memory copy
p = MemmapingPool(3, max_nbytes=10, temp_folder=TEMP_FOLDER)
try:
res = p.apply_async(np.ones, args=(1000, ))
large = res.get()
assert_false(has_shareable_memory(large))
assert_array_equal(large, np.ones(1000))
finally:
p.terminate()
del p
def test_workaround_against_bad_memmap_with_copied_buffers(tmpdir):
"""Check that memmaps with a bad buffer are returned as regular arrays
Unary operations and ufuncs on memmap instances return a new memmap
instance with an in-memory buffer (probably a numpy bug).
"""
assert_array_equal = np.testing.assert_array_equal
p = MemmapingPool(3, max_nbytes=10, temp_folder=tmpdir.strpath)
try:
# Send a complex, large-ish view on a array that will be converted to
# a memmap in the worker process
a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),
order='F')[:, :1, :]
# Call a non-inplace multiply operation on the worker and memmap and
# send it back to the parent.
b = p.apply_async(_worker_multiply, args=(a, 3)).get()
assert not has_shareable_memory(b)
assert_array_equal(b, 3 * a)
finally:
p.terminate()
del p
def test_workaround_against_bad_memmap_with_copied_buffers():
"""Check that memmaps with a bad buffer are returned as regular arrays
Unary operations and ufuncs on memmap instances return a new memmap
instance with an in-memory buffer (probably a numpy bug).
"""
assert_array_equal = np.testing.assert_array_equal
p = MemmapingPool(3, max_nbytes=10, temp_folder=TEMP_FOLDER)
try:
# Send a complex, large-ish view on a array that will be converted to
# a memmap in the worker process
a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),
order='F')[:, :1, :]
# Call a non-inplace multiply operation on the worker and memmap and
# send it back to the parent.
b = p.apply_async(_worker_multiply, args=(a, 3)).get()
assert_false(has_shareable_memory(b))
assert_array_equal(b, 3 * a)
finally:
p.terminate()
del p
