def test__strided_from_memmap(tmpdir):
fname = tmpdir.join('test.mmap').strpath
size = 5 * mmap.ALLOCATIONGRANULARITY
offset = mmap.ALLOCATIONGRANULARITY + 1
# This line creates the mmap file that is reused later
memmap_obj = np.memmap(fname, mode='w+', shape=size + offset)
# filename, dtype, mode, offset, order, shape, strides, total_buffer_len
memmap_obj = _strided_from_memmap(fname,
dtype='uint8',
mode='r',
offset=offset,
order='C',
shape=size,
strides=None,
total_buffer_len=None)
assert isinstance(memmap_obj, np.memmap)
assert memmap_obj.offset == offset
memmap_backed_obj = _strided_from_memmap(fname,
dtype='uint8',
mode='r',
offset=offset,
order='C',
shape=(size // 2, ),
strides=(2, ),
total_buffer_len=size)
assert _get_backing_memmap(memmap_backed_obj).offset == offset
def test_memmap_returned_as_regular_array(backend):
data = np.ones(int(1e3))
# Check that child processes send temporary memmaps back as numpy arrays.
[result
] = Parallel(n_jobs=2, backend=backend,
max_nbytes=100)(delayed(check_memmap_and_send_back)(data)
for _ in range(1))
assert _get_backing_memmap(result) is None
def test__strided_from_memmap(tmpdir):
fname = tmpdir.join('test.mmap').strpath
size = 5 * mmap.ALLOCATIONGRANULARITY
offset = mmap.ALLOCATIONGRANULARITY + 1
# This line creates the mmap file that is reused later
memmap_obj = np.memmap(fname, mode='w+', shape=size + offset)
# filename, dtype, mode, offset, order, shape, strides, total_buffer_len
memmap_obj = _strided_from_memmap(fname, dtype='uint8', mode='r',
offset=offset, order='C', shape=size,
strides=None, total_buffer_len=None)
assert isinstance(memmap_obj, np.memmap)
assert memmap_obj.offset == offset
memmap_backed_obj = _strided_from_memmap(fname, dtype='uint8', mode='r',
offset=offset, order='C',
shape=(size // 2,), strides=(2,),
total_buffer_len=size)
assert _get_backing_memmap(memmap_backed_obj).offset == offset
def check_memmap_and_send_back(array):
assert _get_backing_memmap(array) is not None
return array
def print_filename_and_raise(arr):
from joblib._memmapping_reducer import _get_backing_memmap
print(_get_backing_memmap(arr).filename)
raise ValueError