def test_compress_string_argument(tmpdir, compress_string):
# Verify the string is correctly taken into account.
filename = tmpdir.join('test.pkl').strpath
numpy_pickle.dump("dummy", filename, compress=compress_string)
# Verify the file contains the right magic number
with open(filename, 'rb') as f:
assert _detect_compressor(f) == compress_string
def test_joblib_compression_formats(tmpdir, compress, cmethod):
filename = tmpdir.join('test.pkl').strpath
objects = (np.ones(shape=(100, 100), dtype='f8'), range(10), {
'a': 1,
2: 'b'
}, [], (), {}, 0, 1.0)
if cmethod in ("lzma", "xz") and lzma is None:
pytest.skip("lzma is support not available")
elif cmethod == 'lz4' and with_lz4.args[0]:
# Skip the test if lz4 is not installed. We here use the with_lz4
# skipif fixture whose argument is True when lz4 is not installed
pytest.skip("lz4 is not installed.")
dump_filename = filename + "." + cmethod
for obj in objects:
numpy_pickle.dump(obj, dump_filename, compress=(cmethod, compress))
# Verify the file contains the right magic number
with open(dump_filename, 'rb') as f:
assert _detect_compressor(f) == cmethod
# Verify the reloaded object is correct
obj_reloaded = numpy_pickle.load(dump_filename)
assert isinstance(obj_reloaded, type(obj))
if isinstance(obj, np.ndarray):
np.testing.assert_array_equal(obj_reloaded, obj)
else:
assert obj_reloaded == obj
def test_joblib_compression_formats(tmpdir, compress, cmethod):
filename = tmpdir.join('test.pkl').strpath
objects = (np.ones(shape=(100, 100), dtype='f8'), range(10), {
'a': 1,
2: 'b'
}, [], (), {}, 0, 1.0)
dump_filename = filename + "." + cmethod
for obj in objects:
if not PY3_OR_LATER and cmethod in ('lzma', 'xz', 'lz4'):
# Lzma module only available for python >= 3.3
msg = "{} compression is only available".format(cmethod)
error = NotImplementedError
if cmethod == 'lz4':
error = ValueError
with raises(error) as excinfo:
numpy_pickle.dump(obj,
dump_filename,
compress=(cmethod, compress))
excinfo.match(msg)
else:
numpy_pickle.dump(obj, dump_filename, compress=(cmethod, compress))
# Verify the file contains the right magic number
with open(dump_filename, 'rb') as f:
assert _detect_compressor(f) == cmethod
# Verify the reloaded object is correct
obj_reloaded = numpy_pickle.load(dump_filename)
assert isinstance(obj_reloaded, type(obj))
if isinstance(obj, np.ndarray):
np.testing.assert_array_equal(obj_reloaded, obj)
else:
assert obj_reloaded == obj
def test_compression_using_file_extension():
# test that compression method corresponds to the given filename extension.
extensions_dict = {
# valid compressor extentions
'.z': 'zlib',
'.gz': 'gzip',
'.bz2': 'bz2',
'.lzma': 'lzma',
'.xz': 'xz',
# invalid compressor extensions
'.pkl': 'not-compressed',
'': 'not-compressed'
}
filename = env['filename'] + str(random.randint(0, 1000))
obj = "object to dump"
for ext, cmethod in extensions_dict.items():
dump_fname = filename + ext
if not PY3_OR_LATER and cmethod in ('xz', 'lzma'):
# Lzma module only available for python >= 3.3
msg = "{0} compression is only available".format(cmethod)
assert_raises_regex(NotImplementedError, msg, numpy_pickle.dump,
obj, dump_fname)
else:
numpy_pickle.dump(obj, dump_fname)
# Verify the file contains the right magic number
with open(dump_fname, 'rb') as f:
nose.tools.assert_equal(_detect_compressor(f), cmethod)
# Verify the reloaded object is correct
obj_reloaded = numpy_pickle.load(dump_fname)
nose.tools.assert_true(isinstance(obj_reloaded, type(obj)))
nose.tools.assert_equal(obj_reloaded, obj)
os.remove(dump_fname)
def test_compression_using_file_extension():
# test that compression method corresponds to the given filename extension.
extensions_dict = {
# valid compressor extentions
'.z': 'zlib',
'.gz': 'gzip',
'.bz2': 'bz2',
'.lzma': 'lzma',
'.xz': 'xz',
# invalid compressor extensions
'.pkl': 'not-compressed',
'': 'not-compressed'
}
filename = env['filename'] + str(random.randint(0, 1000))
obj = "object to dump"
for ext, cmethod in extensions_dict.items():
dump_fname = filename + ext
if not PY3_OR_LATER and cmethod in ('xz', 'lzma'):
# Lzma module only available for python >= 3.3
msg = "{} compression is only available".format(cmethod)
assert_raises_regex(NotImplementedError, msg,
numpy_pickle.dump, obj, dump_fname)
else:
numpy_pickle.dump(obj, dump_fname)
# Verify the file contains the right magic number
with open(dump_fname, 'rb') as f:
assert _detect_compressor(f) == cmethod
# Verify the reloaded object is correct
obj_reloaded = numpy_pickle.load(dump_fname)
assert isinstance(obj_reloaded, type(obj))
assert obj_reloaded == obj
os.remove(dump_fname)
def test_joblib_compression_formats():
compresslevels = (1, 3, 6)
filename = env['filename'] + str(random.randint(0, 1000))
objects = (np.ones(shape=(100, 100), dtype='f8'),
range(10),
{'a': 1, 2: 'b'}, [], (), {}, 0, 1.0)
for compress in compresslevels:
for cmethod in _COMPRESSORS:
dump_filename = filename + "." + cmethod
for obj in objects:
if not PY3_OR_LATER and cmethod in ('xz', 'lzma'):
# Lzma module only available for python >= 3.3
msg = "{} compression is only available".format(cmethod)
assert_raises_regex(NotImplementedError, msg,
numpy_pickle.dump, obj, dump_filename,
compress=(cmethod, compress))
else:
numpy_pickle.dump(obj, dump_filename,
compress=(cmethod, compress))
# Verify the file contains the right magic number
with open(dump_filename, 'rb') as f:
assert _detect_compressor(f) == cmethod
# Verify the reloaded object is correct
obj_reloaded = numpy_pickle.load(dump_filename)
assert isinstance(obj_reloaded, type(obj))
if isinstance(obj, np.ndarray):
np.testing.assert_array_equal(obj_reloaded, obj)
else:
assert obj_reloaded == obj
os.remove(dump_filename)
def test_compress_tuple_argument():
compress_tuples = (('zlib', 3),
('gzip', 3))
# Verify the tuple is correctly taken into account.
filename = env['filename'] + str(random.randint(0, 1000))
for compress in compress_tuples:
numpy_pickle.dump("dummy", filename,
compress=compress)
# Verify the file contains the right magic number
with open(filename, 'rb') as f:
assert _detect_compressor(f) == compress[0]
# Verify setting a wrong compress tuple raises a ValueError.
assert_raises_regex(ValueError,
'Compress argument tuple should contain exactly '
'2 elements',
numpy_pickle.dump, "dummy", filename,
compress=('zlib', 3, 'extra'))
# Verify a tuple with a wrong compress method raises a ValueError.
msg = 'Non valid compression method given: "{}"'.format('wrong')
assert_raises_regex(ValueError, msg,
numpy_pickle.dump, "dummy", filename,
compress=('wrong', 3))
# Verify a tuple with a wrong compress level raises a ValueError.
msg = 'Non valid compress level given: "{}"'.format('wrong')
assert_raises_regex(ValueError, msg,
numpy_pickle.dump, "dummy", filename,
compress=('zlib', 'wrong'))
def test_compression_using_file_extension(tmpdir, extension, cmethod):
# test that compression method corresponds to the given filename extension.
filename = tmpdir.join('test.pkl').strpath
obj = "object to dump"
dump_fname = filename + extension
numpy_pickle.dump(obj, dump_fname)
# Verify the file contains the right magic number
with open(dump_fname, 'rb') as f:
assert _detect_compressor(f) == cmethod
# Verify the reloaded object is correct
obj_reloaded = numpy_pickle.load(dump_fname)
assert isinstance(obj_reloaded, type(obj))
assert obj_reloaded == obj
def test_compression_using_file_extension(tmpdir, extension, cmethod):
# test that compression method corresponds to the given filename extension.
filename = tmpdir.join('test.pkl').strpath
obj = "object to dump"
dump_fname = filename + extension
if not PY3_OR_LATER and cmethod in ('xz', 'lzma'):
# Lzma module only available for python >= 3.3
msg = "{} compression is only available".format(cmethod)
with raises(NotImplementedError) as excinfo:
numpy_pickle.dump(obj, dump_fname)
excinfo.match(msg)
else:
numpy_pickle.dump(obj, dump_fname)
# Verify the file contains the right magic number
with open(dump_fname, 'rb') as f:
assert _detect_compressor(f) == cmethod
# Verify the reloaded object is correct
obj_reloaded = numpy_pickle.load(dump_fname)
assert isinstance(obj_reloaded, type(obj))
assert obj_reloaded == obj
def test_compress_tuple_argument():
compress_tuples = (('zlib', 3), ('gzip', 3))
# Verify the tuple is correctly taken into account.
filename = env['filename'] + str(random.randint(0, 1000))
for compress in compress_tuples:
numpy_pickle.dump("dummy", filename, compress=compress)
# Verify the file contains the right magic number
with open(filename, 'rb') as f:
nose.tools.assert_equal(_detect_compressor(f), compress[0])
# Verify setting a wrong compress tuple raises a ValueError.
assert_raises_regex(ValueError,
'Compress argument tuple should contain exactly '
'2 elements',
numpy_pickle.dump,
"dummy",
filename,
compress=('zlib', 3, 'extra'))
# Verify a tuple with a wrong compress method raises a ValueError.
msg = 'Non valid compression method given: "{0}"'.format('wrong')
assert_raises_regex(ValueError,
msg,
numpy_pickle.dump,
"dummy",
filename,
compress=('wrong', 3))
# Verify a tuple with a wrong compress level raises a ValueError.
msg = 'Non valid compress level given: "{0}"'.format('wrong')
assert_raises_regex(ValueError,
msg,
numpy_pickle.dump,
"dummy",
filename,
compress=('zlib', 'wrong'))
def test_joblib_compression_formats():
compresslevels = (1, 3, 6)
filename = env['filename'] + str(random.randint(0, 1000))
objects = (np.ones(shape=(100, 100), dtype='f8'), range(10), {
'a': 1,
2: 'b'
}, [], (), {}, 0, 1.0)
for compress in compresslevels:
for cmethod in _COMPRESSORS:
dump_filename = filename + "." + cmethod
for obj in objects:
if not PY3_OR_LATER and cmethod in ('xz', 'lzma'):
# Lzma module only available for python >= 3.3
msg = "{0} compression is only available".format(cmethod)
assert_raises_regex(NotImplementedError,
msg,
numpy_pickle.dump,
obj,
dump_filename,
compress=(cmethod, compress))
else:
numpy_pickle.dump(obj,
dump_filename,
compress=(cmethod, compress))
# Verify the file contains the right magic number
with open(dump_filename, 'rb') as f:
nose.tools.assert_equal(_detect_compressor(f), cmethod)
# Verify the reloaded object is correct
obj_reloaded = numpy_pickle.load(dump_filename)
nose.tools.assert_true(isinstance(obj_reloaded, type(obj)))
if isinstance(obj, np.ndarray):
np.testing.assert_array_equal(obj_reloaded, obj)
else:
nose.tools.assert_equal(obj_reloaded, obj)
os.remove(dump_filename)
