def test_memory_leak():
import psutil
import tempfile
import gc
from . import reader
from bob.io.base import save
my_video = numpy.random.random_integers(0, 255, (10, 3, 256 * 4, 256 * 4))
with tempfile.NamedTemporaryFile(suffix='.avi') as f:
save(my_video.astype('uint8'), f.name)
del my_video
gc.collect()
for i in range(20):
vd = reader(f.name)
for frame in vd:
pass
del frame
del vd
gc.collect()
if i == 0:
first = psutil.virtual_memory().used
last = psutil.virtual_memory().used
assert (
last - first
) / first < 0.1, f"Looks like we have a memory leak! first:{first} last:{last}"
def read_write_check(data, numpy_assert=True):
"""Testing loading and save different file types"""
with tempfile.NamedTemporaryFile(prefix="bobtest_", suffix=".hdf5") as f:
save(data, f.name)
data2 = load(f.name)
if numpy_assert:
assert np.allclose(data, data2, atol=10e-5, rtol=10e-5)
else:
assert data == data2
def test_open_file_leak():
import psutil
import tempfile
import gc
from . import reader
from bob.io.base import save
my_video = numpy.random.random_integers(0, 255, (10, 3, 256 * 4, 256 * 4))
process = psutil.Process()
def open_files_count(path):
return len([f for f in process.open_files() if f.path == path])
with tempfile.NamedTemporaryFile(suffix='.avi') as f:
path = f.name
save(my_video.astype('uint8'), path)
# Get a count before doing anything. This following count is 1 because
# NamedTemporaryFile keeps a pointer to this file. We will check if the
# count goes to 0 in the end!
last_count = open_files_count(path)
# load the video at once and see if the count is constant
load(path)
gc.collect()
new_count = open_files_count(path)
assert new_count == last_count, \
('We did not close all the files when loading the video at once. '
'old count: {}, new count: {}'.format(last_count, new_count))
# iterate over all frames and see if the count is constant
for frame in reader(path):
pass
gc.collect()
new_count = open_files_count(path)
assert new_count == last_count, \
('We did not close all the files when iterating over all frames. '
'old count: {}, new count: {}'.format(last_count, new_count))
# iterate over one frame and see if the count is constant
for frame in reader(path):
break
gc.collect()
new_count = open_files_count(path)
assert new_count == last_count, \
('We did not close all the files when iterating over one frame. '
'old count: {}, new count: {}'.format(last_count, new_count))
count = open_files_count(path)
assert count == 0, 'The temporary file was not closed! {}'.format(count)
def save_one(self, id, obj, directory, extension):
"""Saves a single object supporting the bob save() protocol.
.. deprecated:: 1.1.0
This function is *deprecated*, use :py:meth:`.File.save()` instead.
This method will call save() on the the given object using the correct
database filename stem for the given id.
Keyword Parameters:
id
The id of the object in the database table "file".
obj
The object that needs to be saved, respecting the bob save() protocol.
directory
This is the base directory to which you want to save the data. The
directory is tested for existence and created if it is not there with
os.makedirs()
extension
The extension determines the way each of the arrays will be saved.
"""
import warnings
warnings.warn(
"The method Database.save_one() is deprecated, use the File object directly as returned by Database.objects() for more powerful object manipulation.",
DeprecationWarning)
self.assert_validity()
fobj = self.session.query(File).filter_by(id=id).one()
fullpath = os.path.join(directory, str(fobj.path) + extension)
fulldir = os.path.dirname(fullpath)
utils.makedirs_safe(fulldir)
from bob.io.base import save
save(obj, fullpath)
def save_one(self, id, obj, directory, extension):
"""Saves a single object supporting the bob save() protocol.
.. deprecated:: 1.1.0
This function is *deprecated*, use :py:meth:`.File.save()` instead.
This method will call save() on the the given object using the correct
database filename stem for the given id.
Keyword Parameters:
id
The id of the object in the database table "file".
obj
The object that needs to be saved, respecting the bob save() protocol.
directory
This is the base directory to which you want to save the data. The
directory is tested for existence and created if it is not there with
os.makedirs()
extension
The extension determines the way each of the arrays will be saved.
"""
import warnings
warnings.warn(
"The method Database.save_one() is deprecated, use the File object directly as returned by Database.objects() for more powerful object manipulation.",
DeprecationWarning)
self.assert_validity()
fobj = self.session.query(File).filter_by(id=id).one()
fullpath = os.path.join(directory, str(fobj.path) + extension)
fulldir = os.path.dirname(fullpath)
utils.makedirs_safe(fulldir)
from bob.io.base import save
save(obj, fullpath)
def save_by_filename(self, filename, obj, directory, extension):
"""Saves a single object supporting the bob save() protocol.
.. deprecated:: 1.1.0
This function is *deprecated*, use :py:meth:`.File.save()` instead.
This method will call save() on the the given object using the correct
database filename stem for the given filename
Keyword Parameters:
filename
The unique filename under which the object will be saved. Before calling this method, the method files() should be called (with no directory and extension arguments passed) in order to obtain the unique filenames for each of the files to be saved.
obj
The object that needs to be saved, respecting the bob save() protocol.
directory
This is the base directory to which you want to save the data. The
directory is tested for existence and created if it is not there with
os.makedirs()
extension
The extension determines the way each of the arrays will be saved.
"""
import warnings
warnings.warn(
"The method Database.save() is deprecated, use the File object directly as returned by Database.objects() for more powerful object manipulation.",
DeprecationWarning)
from bob.io.base import save
fullpath = os.path.join(directory, filename + extension)
fulldir = os.path.dirname(fullpath)
utils.makedirs_safe(fulldir)
save(obj, fullpath)
def save_by_filename(self, filename, obj, directory, extension):
"""Saves a single object supporting the bob save() protocol.
.. deprecated:: 1.1.0
This function is *deprecated*, use :py:meth:`.File.save()` instead.
This method will call save() on the the given object using the correct
database filename stem for the given filename
Keyword Parameters:
filename
The unique filename under which the object will be saved. Before calling this method, the method files() should be called (with no directory and extension arguments passed) in order to obtain the unique filenames for each of the files to be saved.
obj
The object that needs to be saved, respecting the bob save() protocol.
directory
This is the base directory to which you want to save the data. The
directory is tested for existence and created if it is not there with
os.makedirs()
extension
The extension determines the way each of the arrays will be saved.
"""
import warnings
warnings.warn("The method Database.save() is deprecated, use the File object directly as returned by Database.objects() for more powerful object manipulation.", DeprecationWarning)
from bob.io.base import save
fullpath = os.path.join(directory, filename + extension)
fulldir = os.path.dirname(fullpath)
utils.makedirs_safe(fulldir)
save(obj, fullpath)
def test_io_vstack():
paths = [1, 2, 3, 4, 5]
def oracle(reader, paths):
return np.vstack([reader(p) for p in paths])
def reader_same_size_C(path):
return np.arange(10).reshape(5, 2)
def reader_different_size_C(path):
return np.arange(2 * path).reshape(path, 2)
def reader_same_size_F(path):
return np.asfortranarray(np.arange(10).reshape(5, 2))
def reader_different_size_F(path):
return np.asfortranarray(np.arange(2 * path).reshape(path, 2))
def reader_same_size_C2(path):
return np.arange(30).reshape(5, 2, 3)
def reader_different_size_C2(path):
return np.arange(6 * path).reshape(path, 2, 3)
def reader_same_size_F2(path):
return np.asfortranarray(np.arange(30).reshape(5, 2, 3))
def reader_different_size_F2(path):
return np.asfortranarray(np.arange(6 * path).reshape(path, 2, 3))
def reader_wrong_size(path):
return np.arange(2 * path).reshape(2, path)
# when same_size is False
for reader in [
reader_different_size_C,
reader_different_size_F,
reader_same_size_C,
reader_same_size_F,
reader_different_size_C2,
reader_different_size_F2,
reader_same_size_C2,
reader_same_size_F2,
]:
np.all(vstack_features(reader, paths) == oracle(reader, paths))
# when same_size is True
for reader in [
reader_same_size_C,
reader_same_size_F,
reader_same_size_C2,
reader_same_size_F2,
]:
np.all(vstack_features(reader, paths, True) == oracle(reader, paths))
with pytest.raises(AssertionError):
vstack_features(reader_wrong_size, paths)
# test actual files
paths = [temporary_filename(), temporary_filename(), temporary_filename()]
try:
# try different readers:
for reader in [
reader_different_size_C,
reader_different_size_F,
reader_same_size_C,
reader_same_size_F,
reader_different_size_C2,
reader_different_size_F2,
reader_same_size_C2,
reader_same_size_F2,
]:
# save some data in files
for i, path in enumerate(paths):
save(reader(i + 1), path)
# test when all data is present
reference = oracle(load, paths)
np.all(vstack_features(load, paths) == reference)
os.remove(paths[0])
# Check if RuntimeError is raised when one of the files is missing
with pytest.raises(RuntimeError):
vstack_features(load, paths)
finally:
try:
for path in paths:
os.remove(path)
except Exception:
pass