def test_compression_using_file_extension():
# test that compression method corresponds to the given filename extension.
extensions_dict = {
# valid compressor extentions
'.z': 'zlib',
'.gz': 'gzip',
'.bz2': 'bz2',
'.lzma': 'lzma',
'.xz': 'xz',
# invalid compressor extensions
'.pkl': 'not-compressed',
'': 'not-compressed'
}
filename = env['filename'] + str(random.randint(0, 1000))
obj = "object to dump"
for ext, cmethod in extensions_dict.items():
dump_fname = filename + ext
if not PY3_OR_LATER and cmethod in ('xz', 'lzma'):
# Lzma module only available for python >= 3.3
msg = "{0} compression is only available".format(cmethod)
assert_raises_regex(NotImplementedError, msg, numpy_pickle.dump,
obj, dump_fname)
else:
numpy_pickle.dump(obj, dump_fname)
# Verify the file contains the right magic number
with open(dump_fname, 'rb') as f:
nose.tools.assert_equal(_detect_compressor(f), cmethod)
# Verify the reloaded object is correct
obj_reloaded = numpy_pickle.load(dump_fname)
nose.tools.assert_true(isinstance(obj_reloaded, type(obj)))
nose.tools.assert_equal(obj_reloaded, obj)
os.remove(dump_fname)
def test_filter_args_python_3():
nose.tools.assert_equal(
filter_args(func_with_kwonly_args, [], (1, 2), {
'kw1': 3,
'kw2': 4
}), {
'a': 1,
'b': 2,
'kw1': 3,
'kw2': 4
})
# filter_args doesn't care about keyword-only arguments so you
# can pass 'kw1' into *args without any problem
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
filter_args, func_with_kwonly_args, [], (1, 2, 3), {'kw2': 2})
nose.tools.assert_equal(
filter_args(func_with_kwonly_args, ['b', 'kw2'], (1, 2), {
'kw1': 3,
'kw2': 4
}), {
'a': 1,
'kw1': 3
})
nose.tools.assert_equal(
filter_args(func_with_signature, ['b'], (1, 2)), {'a': 1})
def test_memory_func_with_kwonly_args():
mem = Memory(cachedir=env['dir'], verbose=0)
func_cached = mem.cache(func_with_kwonly_args)
nose.tools.assert_equal(func_cached(1, 2, kw1=3), (1, 2, 3, 'kw2'))
# Making sure that providing a keyword-only argument by
# position raises an exception
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
func_cached,
1, 2, 3, {'kw2': 4})
# Keyword-only parameter passed by position with cached call
# should still raise ValueError
func_cached(1, 2, kw1=3, kw2=4)
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
func_cached,
1, 2, 3, {'kw2': 4})
# Test 'ignore' parameter
func_cached = mem.cache(func_with_kwonly_args, ignore=['kw2'])
nose.tools.assert_equal(func_cached(1, 2, kw1=3, kw2=4), (1, 2, 3, 4))
nose.tools.assert_equal(func_cached(1, 2, kw1=3, kw2='ignored'), (1, 2, 3, 4))
def test_joblib_compression_formats():
compresslevels = (1, 3, 6)
filename = env['filename'] + str(random.randint(0, 1000))
objects = (np.ones(shape=(100, 100), dtype='f8'),
range(10),
{'a': 1, 2: 'b'}, [], (), {}, 0, 1.0)
for compress in compresslevels:
for cmethod in _COMPRESSORS:
dump_filename = filename + "." + cmethod
for obj in objects:
if not PY3_OR_LATER and cmethod in ('xz', 'lzma'):
# Lzma module only available for python >= 3.3
msg = "{} compression is only available".format(cmethod)
assert_raises_regex(NotImplementedError, msg,
numpy_pickle.dump, obj, dump_filename,
compress=(cmethod, compress))
else:
numpy_pickle.dump(obj, dump_filename,
compress=(cmethod, compress))
# Verify the file contains the right magic number
with open(dump_filename, 'rb') as f:
assert _detect_compressor(f) == cmethod
# Verify the reloaded object is correct
obj_reloaded = numpy_pickle.load(dump_filename)
assert isinstance(obj_reloaded, type(obj))
if isinstance(obj, np.ndarray):
np.testing.assert_array_equal(obj_reloaded, obj)
else:
assert obj_reloaded == obj
os.remove(dump_filename)
def test_hashing_pickling_error():
def non_picklable():
return 42
assert_raises_regex(pickle.PicklingError,
'PicklingError while hashing',
hash, non_picklable)
def test_hashing_pickling_error():
try:
raise Exception
except Exception:
ex_type, ex, non_picklable = sys.exc_info()
assert_raises_regex((pickle.PicklingError, TypeError),
'PicklingError while hashing', hash, non_picklable)
def test_compression_using_file_extension():
# test that compression method corresponds to the given filename extension.
extensions_dict = {
# valid compressor extentions
'.z': 'zlib',
'.gz': 'gzip',
'.bz2': 'bz2',
'.lzma': 'lzma',
'.xz': 'xz',
# invalid compressor extensions
'.pkl': 'not-compressed',
'': 'not-compressed'
}
filename = env['filename'] + str(random.randint(0, 1000))
obj = "object to dump"
for ext, cmethod in extensions_dict.items():
dump_fname = filename + ext
if not PY3_OR_LATER and cmethod in ('xz', 'lzma'):
# Lzma module only available for python >= 3.3
msg = "{} compression is only available".format(cmethod)
assert_raises_regex(NotImplementedError, msg,
numpy_pickle.dump, obj, dump_fname)
else:
numpy_pickle.dump(obj, dump_fname)
# Verify the file contains the right magic number
with open(dump_fname, 'rb') as f:
assert _detect_compressor(f) == cmethod
# Verify the reloaded object is correct
obj_reloaded = numpy_pickle.load(dump_fname)
assert isinstance(obj_reloaded, type(obj))
assert obj_reloaded == obj
os.remove(dump_fname)
def test_memory_func_with_kwonly_args():
mem = Memory(cachedir=env['dir'], verbose=0)
func_cached = mem.cache(func_with_kwonly_args)
nose.tools.assert_equal(func_cached(1, 2, kw1=3), (1, 2, 3, 'kw2'))
# Making sure that providing a keyword-only argument by
# position raises an exception
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
func_cached, 1, 2, 3, {'kw2': 4})
# Keyword-only parameter passed by position with cached call
# should still raise ValueError
func_cached(1, 2, kw1=3, kw2=4)
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
func_cached, 1, 2, 3, {'kw2': 4})
# Test 'ignore' parameter
func_cached = mem.cache(func_with_kwonly_args, ignore=['kw2'])
nose.tools.assert_equal(func_cached(1, 2, kw1=3, kw2=4), (1, 2, 3, 4))
nose.tools.assert_equal(func_cached(1, 2, kw1=3, kw2='ignored'),
(1, 2, 3, 4))
def test_check_subprocess_call_non_matching_regex():
code = '42'
non_matching_pattern = '_no_way_this_matches_anything_'
assert_raises_regex(ValueError,
'Unexpected stdout.+{0}'.format(non_matching_pattern),
check_subprocess_call, [sys.executable, '-c', code],
stdout_regex=non_matching_pattern)
def test_hashing_pickling_error():
def non_picklable():
return 42
assert_raises_regex(pickle.PicklingError,
'PicklingError while hashing',
hash, non_picklable)
def test_check_subprocess_call_non_matching_regex():
code = '42'
non_matching_pattern = '_no_way_this_matches_anything_'
assert_raises_regex(ValueError,
'Unexpected output.+{0}'.format(non_matching_pattern),
check_subprocess_call,
[sys.executable, '-c', code],
stdout_regex=non_matching_pattern)
def test_compress_level_error():
# Verify that passing an invalid compress argument raises an error.
wrong_compress = (-1, 10, 'wrong')
for wrong in wrong_compress:
exception_msg = 'Non valid compress level given: "{0}"'.format(wrong)
assert_raises_regex(ValueError,
exception_msg,
numpy_pickle.dump, 'dummy', 'foo', compress=wrong)
def test_hashing_pickling_error():
try:
raise Exception
except Exception:
ex_type, ex, non_picklable = sys.exc_info()
assert_raises_regex((pickle.PicklingError, TypeError),
'PicklingError while hashing',
hash, non_picklable)
def test_compress_level_error():
# Verify that passing an invalid compress argument raises an error.
wrong_compress = (-1, 10, 'wrong')
for wrong in wrong_compress:
exception_msg = 'Non valid compress level given: "{0}"'.format(wrong)
assert_raises_regex(ValueError,
exception_msg,
numpy_pickle.dump,
'dummy',
'foo',
compress=wrong)
def test_check_subprocess_call_non_zero_return_code():
code_with_non_zero_exit = '\n'.join([
'import sys', 'print("writing on stdout")',
'sys.stderr.write("writing on stderr")', 'sys.exit(123)'
])
pattern = re.compile(
'Non-zero return code: 123.+'
'Stdout:\nwriting on stdout.+'
'Stderr:\nwriting on stderr', re.DOTALL)
assert_raises_regex(ValueError, pattern, check_subprocess_call,
[sys.executable, '-c', code_with_non_zero_exit])
def test_check_subprocess_call_non_zero_return_code():
code_with_non_zero_exit = '\n'.join([
'import sys',
'print("writing on stdout")',
'sys.stderr.write("writing on stderr")',
'sys.exit(123)'])
pattern = re.compile('Non-zero return code: 123.+'
'Stdout:\nwriting on stdout.+'
'Stderr:\nwriting on stderr', re.DOTALL)
assert_raises_regex(ValueError,
pattern,
check_subprocess_call,
[sys.executable, '-c', code_with_non_zero_exit])
def test_check_subprocess_call_timeout():
code_timing_out = '\n'.join([
'import time', 'import sys', 'print("before sleep on stdout")',
'sys.stdout.flush()', 'sys.stderr.write("before sleep on stderr")',
'sys.stderr.flush()', 'time.sleep(1.1)',
'print("process should have be killed before")', 'sys.stdout.flush()'
])
pattern = re.compile(
'Non-zero return code:.+'
'Stdout:\nbefore sleep on stdout\s+'
'Stderr:\nbefore sleep on stderr', re.DOTALL)
assert_raises_regex(ValueError,
pattern,
check_subprocess_call,
[sys.executable, '-c', code_timing_out],
timeout=1)
def test_filter_args_python_3():
assert (
filter_args(func_with_kwonly_args, [], (1, 2),
{'kw1': 3, 'kw2': 4}) ==
{'a': 1, 'b': 2, 'kw1': 3, 'kw2': 4})
# filter_args doesn't care about keyword-only arguments so you
# can pass 'kw1' into *args without any problem
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
filter_args,
func_with_kwonly_args, [], (1, 2, 3), {'kw2': 2})
assert (
filter_args(func_with_kwonly_args, ['b', 'kw2'], (1, 2),
{'kw1': 3, 'kw2': 4}) ==
{'a': 1, 'kw1': 3})
assert (filter_args(func_with_signature, ['b'], (1, 2)) == {'a': 1})
def test_compress_tuple_argument():
compress_tuples = (('zlib', 3),
('gzip', 3))
# Verify the tuple is correctly taken into account.
filename = env['filename'] + str(random.randint(0, 1000))
for compress in compress_tuples:
numpy_pickle.dump("dummy", filename,
compress=compress)
# Verify the file contains the right magic number
with open(filename, 'rb') as f:
assert _detect_compressor(f) == compress[0]
# Verify setting a wrong compress tuple raises a ValueError.
assert_raises_regex(ValueError,
'Compress argument tuple should contain exactly '
'2 elements',
numpy_pickle.dump, "dummy", filename,
compress=('zlib', 3, 'extra'))
# Verify a tuple with a wrong compress method raises a ValueError.
msg = 'Non valid compression method given: "{}"'.format('wrong')
assert_raises_regex(ValueError, msg,
numpy_pickle.dump, "dummy", filename,
compress=('wrong', 3))
# Verify a tuple with a wrong compress level raises a ValueError.
msg = 'Non valid compress level given: "{}"'.format('wrong')
assert_raises_regex(ValueError, msg,
numpy_pickle.dump, "dummy", filename,
compress=('zlib', 'wrong'))
def test_check_subprocess_call_timeout():
code_timing_out = '\n'.join([
'import time',
'import sys',
'print("before sleep on stdout")',
'sys.stdout.flush()',
'sys.stderr.write("before sleep on stderr")',
'sys.stderr.flush()',
'time.sleep(1.1)',
'print("process should have be killed before")',
'sys.stdout.flush()'])
pattern = re.compile('Non-zero return code:.+'
'Stdout:\nbefore sleep on stdout\s+'
'Stderr:\nbefore sleep on stderr',
re.DOTALL)
assert_raises_regex(ValueError,
pattern,
check_subprocess_call,
[sys.executable, '-c', code_timing_out],
timeout=1)
def test_filter_args_python_3():
nose.tools.assert_equal(
filter_args(func_with_kwonly_args, [], (1, 2), {"kw1": 3, "kw2": 4}), {"a": 1, "b": 2, "kw1": 3, "kw2": 4}
)
# filter_args doesn't care about keyword-only arguments so you
# can pass 'kw1' into *args without any problem
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
filter_args,
func_with_kwonly_args,
[],
(1, 2, 3),
{"kw2": 2},
)
nose.tools.assert_equal(
filter_args(func_with_kwonly_args, ["b", "kw2"], (1, 2), {"kw1": 3, "kw2": 4}), {"a": 1, "kw1": 3}
)
nose.tools.assert_equal(filter_args(func_with_signature, ["b"], (1, 2)), {"a": 1})
def test_joblib_compression_formats():
compresslevels = (1, 3, 6)
filename = env['filename'] + str(random.randint(0, 1000))
objects = (np.ones(shape=(100, 100), dtype='f8'), range(10), {
'a': 1,
2: 'b'
}, [], (), {}, 0, 1.0)
for compress in compresslevels:
for cmethod in _COMPRESSORS:
dump_filename = filename + "." + cmethod
for obj in objects:
if not PY3_OR_LATER and cmethod in ('xz', 'lzma'):
# Lzma module only available for python >= 3.3
msg = "{0} compression is only available".format(cmethod)
assert_raises_regex(NotImplementedError,
msg,
numpy_pickle.dump,
obj,
dump_filename,
compress=(cmethod, compress))
else:
numpy_pickle.dump(obj,
dump_filename,
compress=(cmethod, compress))
# Verify the file contains the right magic number
with open(dump_filename, 'rb') as f:
nose.tools.assert_equal(_detect_compressor(f), cmethod)
# Verify the reloaded object is correct
obj_reloaded = numpy_pickle.load(dump_filename)
nose.tools.assert_true(isinstance(obj_reloaded, type(obj)))
if isinstance(obj, np.ndarray):
np.testing.assert_array_equal(obj_reloaded, obj)
else:
nose.tools.assert_equal(obj_reloaded, obj)
os.remove(dump_filename)
def test_compress_tuple_argument():
compress_tuples = (('zlib', 3), ('gzip', 3))
# Verify the tuple is correctly taken into account.
filename = env['filename'] + str(random.randint(0, 1000))
for compress in compress_tuples:
numpy_pickle.dump("dummy", filename, compress=compress)
# Verify the file contains the right magic number
with open(filename, 'rb') as f:
nose.tools.assert_equal(_detect_compressor(f), compress[0])
# Verify setting a wrong compress tuple raises a ValueError.
assert_raises_regex(ValueError,
'Compress argument tuple should contain exactly '
'2 elements',
numpy_pickle.dump,
"dummy",
filename,
compress=('zlib', 3, 'extra'))
# Verify a tuple with a wrong compress method raises a ValueError.
msg = 'Non valid compression method given: "{0}"'.format('wrong')
assert_raises_regex(ValueError,
msg,
numpy_pickle.dump,
"dummy",
filename,
compress=('wrong', 3))
# Verify a tuple with a wrong compress level raises a ValueError.
msg = 'Non valid compress level given: "{0}"'.format('wrong')
assert_raises_regex(ValueError,
msg,
numpy_pickle.dump,
"dummy",
filename,
compress=('zlib', 'wrong'))
