def test_memmaping_on_dev_shm():
"""Check that MemmapingPool uses /dev/shm when possible"""
p = MemmapingPool(3, max_nbytes=10)
try:
# Check that the pool has correctly detected the presence of the
# shared memory filesystem.
pool_temp_folder = p._temp_folder
folder_prefix = '/dev/shm/joblib_memmaping_pool_'
assert_true(pool_temp_folder.startswith(folder_prefix))
assert_true(os.path.exists(pool_temp_folder))
# Try with a file larger than the memmap threshold of 10 bytes
a = np.ones(100, dtype=np.float64)
assert_equal(a.nbytes, 800)
p.map(id, [a] * 10)
# a should have been memmaped to the pool temp folder: the joblib
# pickling procedure generate a .pkl and a .npy file:
assert_equal(len(os.listdir(pool_temp_folder)), 2)
# create a new array with content that is different from 'a' so that
# it is mapped to a different file in the temporary folder of the
# pool.
b = np.ones(100, dtype=np.float64) * 2
assert_equal(b.nbytes, 800)
p.map(id, [b] * 10)
# A copy of both a and b are now stored in the shared memory folder
assert_equal(len(os.listdir(pool_temp_folder)), 4)
finally:
# Cleanup open file descriptors
p.terminate()
del p
# The temp folder is cleaned up upon pool termination
assert_false(os.path.exists(pool_temp_folder))
def test_pool_with_memmap_array_view():
"""Check that subprocess can access and update shared memory array"""
assert_array_equal = np.testing.assert_array_equal
# Fork the subprocess before allocating the objects to be passed
pool_temp_folder = os.path.join(TEMP_FOLDER, 'pool')
os.makedirs(pool_temp_folder)
p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
try:
filename = os.path.join(TEMP_FOLDER, 'test.mmap')
a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
a.fill(1.0)
# Create an ndarray view on the memmap instance
a_view = np.asarray(a)
assert_false(isinstance(a_view, np.memmap))
assert_true(has_shareable_memory(a_view))
p.map(inplace_double, [(a_view, (i, j), 1.0) for i in range(a.shape[0])
for j in range(a.shape[1])])
# Both a and the a_view have been updated
assert_array_equal(a, 2 * np.ones(a.shape))
assert_array_equal(a_view, 2 * np.ones(a.shape))
# Passing memmap array view to the pool should not trigger the
# creation of new files on the FS
assert_equal(os.listdir(pool_temp_folder), [])
finally:
p.terminate()
del p
def test_pool_with_memmap():
"""Check that subprocess can access and update shared memory memmap"""
assert_array_equal = np.testing.assert_array_equal
# Fork the subprocess before allocating the objects to be passed
pool_temp_folder = os.path.join(TEMP_FOLDER, 'pool')
os.makedirs(pool_temp_folder)
p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
try:
filename = os.path.join(TEMP_FOLDER, 'test.mmap')
a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
a.fill(1.0)
p.map(inplace_double, [(a, (i, j), 1.0) for i in range(a.shape[0])
for j in range(a.shape[1])])
assert_array_equal(a, 2 * np.ones(a.shape))
# Open a copy-on-write view on the previous data
b = np.memmap(filename, dtype=np.float32, shape=(5, 3), mode='c')
p.map(inplace_double, [(b, (i, j), 2.0) for i in range(b.shape[0])
for j in range(b.shape[1])])
# Passing memmap instances to the pool should not trigger the creation
# of new files on the FS
assert_equal(os.listdir(pool_temp_folder), [])
# the original data is untouched
assert_array_equal(a, 2 * np.ones(a.shape))
assert_array_equal(b, 2 * np.ones(b.shape))
# readonly maps can be read but not updated
c = np.memmap(filename,
dtype=np.float32,
shape=(10, ),
mode='r',
offset=5 * 4)
assert_raises(AssertionError, p.map, check_array,
[(c, i, 3.0) for i in range(c.shape[0])])
# depending on the version of numpy one can either get a RuntimeError
# or a ValueError
assert_raises((RuntimeError, ValueError), p.map, inplace_double,
[(c, i, 2.0) for i in range(c.shape[0])])
finally:
# Clean all filehandlers held by the pool
p.terminate()
del p
def initialize(self, n_parallel):
self.n_parallel = n_parallel
if self.pool is not None:
print("Warning: terminating existing pool")
self.pool.terminate()
self.queue.close()
self.worker_queue.close()
self.G = SharedGlobal()
if n_parallel > 1:
self.queue = mp.Queue()
self.worker_queue = mp.Queue()
self.pool = MemmapingPool(
self.n_parallel,
temp_folder="/tmp",
)
def test_memmaping_pool_for_large_arrays():
"""Check that large arrays are not copied in memory"""
assert_array_equal = np.testing.assert_array_equal
# Check that the tempfolder is empty
assert_equal(os.listdir(TEMP_FOLDER), [])
# Build an array reducers that automaticaly dump large array content
# to filesystem backed memmap instances to avoid memory explosion
p = MemmapingPool(3, max_nbytes=40, temp_folder=TEMP_FOLDER)
try:
# The tempory folder for the pool is not provisioned in advance
assert_equal(os.listdir(TEMP_FOLDER), [])
assert_false(os.path.exists(p._temp_folder))
small = np.ones(5, dtype=np.float32)
assert_equal(small.nbytes, 20)
p.map(check_array, [(small, i, 1.0) for i in range(small.shape[0])])
# Memory has been copied, the pool filesystem folder is unused
assert_equal(os.listdir(TEMP_FOLDER), [])
# Try with a file larger than the memmap threshold of 40 bytes
large = np.ones(100, dtype=np.float64)
assert_equal(large.nbytes, 800)
p.map(check_array, [(large, i, 1.0) for i in range(large.shape[0])])
# The data has been dumped in a temp folder for subprocess to share it
# without per-child memory copies
assert_true(os.path.isdir(p._temp_folder))
dumped_filenames = os.listdir(p._temp_folder)
assert_equal(len(dumped_filenames), 2)
# Check that memmory mapping is not triggered for arrays with
# dtype='object'
objects = np.array(['abc'] * 100, dtype='object')
results = p.map(has_shareable_memory, [objects])
assert_false(results[0])
finally:
# check FS garbage upon pool termination
p.terminate()
assert_false(os.path.exists(p._temp_folder))
del p
def test_pool_memmap_with_big_offset(tmpdir):
# Test that numpy memmap offset is set correctly if greater than
# mmap.ALLOCATIONGRANULARITY, see
# https://github.com/joblib/joblib/issues/451 and
# https://github.com/numpy/numpy/pull/8443 for more details.
fname = tmpdir.join('test.mmap').strpath
size = 5 * mmap.ALLOCATIONGRANULARITY
offset = mmap.ALLOCATIONGRANULARITY + 1
obj = make_memmap(fname,
mode='w+',
shape=size,
dtype='uint8',
offset=offset)
p = MemmapingPool(2, temp_folder=tmpdir.strpath)
result = p.apply_async(identity, args=(obj, )).get()
assert isinstance(result, np.memmap)
assert result.offset == offset
np.testing.assert_array_equal(obj, result)
def test_memmaping_pool_for_large_arrays_in_return():
"""Check that large arrays are not copied in memory in return"""
assert_array_equal = np.testing.assert_array_equal
# Build an array reducers that automaticaly dump large array content
# but check that the returned datastructure are regular arrays to avoid
# passing a memmap array pointing to a pool controlled temp folder that
# might be confusing to the user
# The MemmapingPool user can always return numpy.memmap object explicitly
# to avoid memory copy
p = MemmapingPool(3, max_nbytes=10, temp_folder=TEMP_FOLDER)
try:
res = p.apply_async(np.ones, args=(1000, ))
large = res.get()
assert_false(has_shareable_memory(large))
assert_array_equal(large, np.ones(1000))
finally:
p.terminate()
del p
def test_memmaping_pool_for_large_arrays_disabled():
"""Check that large arrays memmaping can be disabled"""
# Set max_nbytes to None to disable the auto memmaping feature
p = MemmapingPool(3, max_nbytes=None, temp_folder=TEMP_FOLDER)
try:
# Check that the tempfolder is empty
assert_equal(os.listdir(TEMP_FOLDER), [])
# Try with a file largish than the memmap threshold of 40 bytes
large = np.ones(100, dtype=np.float64)
assert_equal(large.nbytes, 800)
p.map(check_array, [(large, i, 1.0) for i in range(large.shape[0])])
# Check that the tempfolder is still empty
assert_equal(os.listdir(TEMP_FOLDER), [])
finally:
# Cleanup open file descriptors
p.terminate()
del p
def test_workaround_against_bad_memmap_with_copied_buffers():
"""Check that memmaps with a bad buffer are returned as regular arrays
Unary operations and ufuncs on memmap instances return a new memmap
instance with an in-memory buffer (probably a numpy bug).
"""
assert_array_equal = np.testing.assert_array_equal
p = MemmapingPool(3, max_nbytes=10, temp_folder=TEMP_FOLDER)
try:
# Send a complex, large-ish view on a array that will be converted to
# a memmap in the worker process
a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),
order='F')[:, :1, :]
# Call a non-inplace multiply operation on the worker and memmap and
# send it back to the parent.
b = p.apply_async(_worker_multiply, args=(a, 3)).get()
assert_false(has_shareable_memory(b))
assert_array_equal(b, 3 * a)
finally:
p.terminate()
del p
