def test_memory_usage(tmpdir, compress):
# Verify memory stays within expected bounds.
filename = tmpdir.join('test.pkl').strpath
small_array = np.ones((10, 10))
big_array = np.ones(shape=100 * int(1e6), dtype=np.uint8)
small_matrix = np.matrix(small_array)
big_matrix = np.matrix(big_array)
for obj in (small_array, big_array, small_matrix, big_matrix):
size = obj.nbytes / 1e6
obj_filename = filename + str(np.random.randint(0, 1000))
mem_used = memory_used(numpy_pickle.dump,
obj,
obj_filename,
compress=compress)
# The memory used to dump the object shouldn't exceed the buffer
# size used to write array chunks (16MB).
write_buf_size = _IO_BUFFER_SIZE + 16 * 1024**2 / 1e6
assert mem_used <= write_buf_size
mem_used = memory_used(numpy_pickle.load, obj_filename)
# memory used should be less than array size + buffer size used to
# read the array chunk by chunk.
read_buf_size = 32 + _IO_BUFFER_SIZE # MiB
assert mem_used < size + read_buf_size
def test_memory_usage():
# Verify memory stays within expected bounds.
filename = env['filename']
small_array = np.ones((10, 10))
big_array = np.ones(shape=100 * int(1e6), dtype=np.uint8)
small_matrix = np.matrix(small_array)
big_matrix = np.matrix(big_array)
for compress in (True, False):
for obj in (small_array, big_array, small_matrix, big_matrix):
size = obj.nbytes / 1e6
obj_filename = filename + str(np.random.randint(0, 1000))
mem_used = memory_used(numpy_pickle.dump,
obj, obj_filename, compress=compress)
# The memory used to dump the object shouldn't exceed the buffer
# size used to write array chunks (16MB).
write_buf_size = _IO_BUFFER_SIZE + 16 * 1024 ** 2 / 1e6
assert mem_used <= write_buf_size
mem_used = memory_used(numpy_pickle.load, obj_filename)
# memory used should be less than array size + buffer size used to
# read the array chunk by chunk.
read_buf_size = 32 + _IO_BUFFER_SIZE # MiB
assert mem_used < size + read_buf_size