def test_compress_string_argument(tmpdir, compress_string): # Verify the string is correctly taken into account. filename = tmpdir.join('test.pkl').strpath numpy_pickle.dump("dummy", filename, compress=compress_string) # Verify the file contains the right magic number with open(filename, 'rb') as f: assert _detect_compressor(f) == compress_string
def test_joblib_compression_formats(tmpdir, compress, cmethod): filename = tmpdir.join('test.pkl').strpath objects = (np.ones(shape=(100, 100), dtype='f8'), range(10), { 'a': 1, 2: 'b' }, [], (), {}, 0, 1.0) if cmethod in ("lzma", "xz") and lzma is None: pytest.skip("lzma is support not available") elif cmethod == 'lz4' and with_lz4.args[0]: # Skip the test if lz4 is not installed. We here use the with_lz4 # skipif fixture whose argument is True when lz4 is not installed pytest.skip("lz4 is not installed.") dump_filename = filename + "." + cmethod for obj in objects: numpy_pickle.dump(obj, dump_filename, compress=(cmethod, compress)) # Verify the file contains the right magic number with open(dump_filename, 'rb') as f: assert _detect_compressor(f) == cmethod # Verify the reloaded object is correct obj_reloaded = numpy_pickle.load(dump_filename) assert isinstance(obj_reloaded, type(obj)) if isinstance(obj, np.ndarray): np.testing.assert_array_equal(obj_reloaded, obj) else: assert obj_reloaded == obj
def test_joblib_compression_formats(tmpdir, compress, cmethod): filename = tmpdir.join('test.pkl').strpath objects = (np.ones(shape=(100, 100), dtype='f8'), range(10), { 'a': 1, 2: 'b' }, [], (), {}, 0, 1.0) dump_filename = filename + "." + cmethod for obj in objects: if not PY3_OR_LATER and cmethod in ('lzma', 'xz', 'lz4'): # Lzma module only available for python >= 3.3 msg = "{} compression is only available".format(cmethod) error = NotImplementedError if cmethod == 'lz4': error = ValueError with raises(error) as excinfo: numpy_pickle.dump(obj, dump_filename, compress=(cmethod, compress)) excinfo.match(msg) else: numpy_pickle.dump(obj, dump_filename, compress=(cmethod, compress)) # Verify the file contains the right magic number with open(dump_filename, 'rb') as f: assert _detect_compressor(f) == cmethod # Verify the reloaded object is correct obj_reloaded = numpy_pickle.load(dump_filename) assert isinstance(obj_reloaded, type(obj)) if isinstance(obj, np.ndarray): np.testing.assert_array_equal(obj_reloaded, obj) else: assert obj_reloaded == obj
def test_compression_using_file_extension(): # test that compression method corresponds to the given filename extension. extensions_dict = { # valid compressor extentions '.z': 'zlib', '.gz': 'gzip', '.bz2': 'bz2', '.lzma': 'lzma', '.xz': 'xz', # invalid compressor extensions '.pkl': 'not-compressed', '': 'not-compressed' } filename = env['filename'] + str(random.randint(0, 1000)) obj = "object to dump" for ext, cmethod in extensions_dict.items(): dump_fname = filename + ext if not PY3_OR_LATER and cmethod in ('xz', 'lzma'): # Lzma module only available for python >= 3.3 msg = "{0} compression is only available".format(cmethod) assert_raises_regex(NotImplementedError, msg, numpy_pickle.dump, obj, dump_fname) else: numpy_pickle.dump(obj, dump_fname) # Verify the file contains the right magic number with open(dump_fname, 'rb') as f: nose.tools.assert_equal(_detect_compressor(f), cmethod) # Verify the reloaded object is correct obj_reloaded = numpy_pickle.load(dump_fname) nose.tools.assert_true(isinstance(obj_reloaded, type(obj))) nose.tools.assert_equal(obj_reloaded, obj) os.remove(dump_fname)
def test_compression_using_file_extension(): # test that compression method corresponds to the given filename extension. extensions_dict = { # valid compressor extentions '.z': 'zlib', '.gz': 'gzip', '.bz2': 'bz2', '.lzma': 'lzma', '.xz': 'xz', # invalid compressor extensions '.pkl': 'not-compressed', '': 'not-compressed' } filename = env['filename'] + str(random.randint(0, 1000)) obj = "object to dump" for ext, cmethod in extensions_dict.items(): dump_fname = filename + ext if not PY3_OR_LATER and cmethod in ('xz', 'lzma'): # Lzma module only available for python >= 3.3 msg = "{} compression is only available".format(cmethod) assert_raises_regex(NotImplementedError, msg, numpy_pickle.dump, obj, dump_fname) else: numpy_pickle.dump(obj, dump_fname) # Verify the file contains the right magic number with open(dump_fname, 'rb') as f: assert _detect_compressor(f) == cmethod # Verify the reloaded object is correct obj_reloaded = numpy_pickle.load(dump_fname) assert isinstance(obj_reloaded, type(obj)) assert obj_reloaded == obj os.remove(dump_fname)
def test_joblib_compression_formats(): compresslevels = (1, 3, 6) filename = env['filename'] + str(random.randint(0, 1000)) objects = (np.ones(shape=(100, 100), dtype='f8'), range(10), {'a': 1, 2: 'b'}, [], (), {}, 0, 1.0) for compress in compresslevels: for cmethod in _COMPRESSORS: dump_filename = filename + "." + cmethod for obj in objects: if not PY3_OR_LATER and cmethod in ('xz', 'lzma'): # Lzma module only available for python >= 3.3 msg = "{} compression is only available".format(cmethod) assert_raises_regex(NotImplementedError, msg, numpy_pickle.dump, obj, dump_filename, compress=(cmethod, compress)) else: numpy_pickle.dump(obj, dump_filename, compress=(cmethod, compress)) # Verify the file contains the right magic number with open(dump_filename, 'rb') as f: assert _detect_compressor(f) == cmethod # Verify the reloaded object is correct obj_reloaded = numpy_pickle.load(dump_filename) assert isinstance(obj_reloaded, type(obj)) if isinstance(obj, np.ndarray): np.testing.assert_array_equal(obj_reloaded, obj) else: assert obj_reloaded == obj os.remove(dump_filename)
def test_compress_tuple_argument(): compress_tuples = (('zlib', 3), ('gzip', 3)) # Verify the tuple is correctly taken into account. filename = env['filename'] + str(random.randint(0, 1000)) for compress in compress_tuples: numpy_pickle.dump("dummy", filename, compress=compress) # Verify the file contains the right magic number with open(filename, 'rb') as f: assert _detect_compressor(f) == compress[0] # Verify setting a wrong compress tuple raises a ValueError. assert_raises_regex(ValueError, 'Compress argument tuple should contain exactly ' '2 elements', numpy_pickle.dump, "dummy", filename, compress=('zlib', 3, 'extra')) # Verify a tuple with a wrong compress method raises a ValueError. msg = 'Non valid compression method given: "{}"'.format('wrong') assert_raises_regex(ValueError, msg, numpy_pickle.dump, "dummy", filename, compress=('wrong', 3)) # Verify a tuple with a wrong compress level raises a ValueError. msg = 'Non valid compress level given: "{}"'.format('wrong') assert_raises_regex(ValueError, msg, numpy_pickle.dump, "dummy", filename, compress=('zlib', 'wrong'))
def test_compression_using_file_extension(tmpdir, extension, cmethod): # test that compression method corresponds to the given filename extension. filename = tmpdir.join('test.pkl').strpath obj = "object to dump" dump_fname = filename + extension numpy_pickle.dump(obj, dump_fname) # Verify the file contains the right magic number with open(dump_fname, 'rb') as f: assert _detect_compressor(f) == cmethod # Verify the reloaded object is correct obj_reloaded = numpy_pickle.load(dump_fname) assert isinstance(obj_reloaded, type(obj)) assert obj_reloaded == obj
def test_compression_using_file_extension(tmpdir, extension, cmethod): # test that compression method corresponds to the given filename extension. filename = tmpdir.join('test.pkl').strpath obj = "object to dump" dump_fname = filename + extension if not PY3_OR_LATER and cmethod in ('xz', 'lzma'): # Lzma module only available for python >= 3.3 msg = "{} compression is only available".format(cmethod) with raises(NotImplementedError) as excinfo: numpy_pickle.dump(obj, dump_fname) excinfo.match(msg) else: numpy_pickle.dump(obj, dump_fname) # Verify the file contains the right magic number with open(dump_fname, 'rb') as f: assert _detect_compressor(f) == cmethod # Verify the reloaded object is correct obj_reloaded = numpy_pickle.load(dump_fname) assert isinstance(obj_reloaded, type(obj)) assert obj_reloaded == obj
def test_compress_tuple_argument(): compress_tuples = (('zlib', 3), ('gzip', 3)) # Verify the tuple is correctly taken into account. filename = env['filename'] + str(random.randint(0, 1000)) for compress in compress_tuples: numpy_pickle.dump("dummy", filename, compress=compress) # Verify the file contains the right magic number with open(filename, 'rb') as f: nose.tools.assert_equal(_detect_compressor(f), compress[0]) # Verify setting a wrong compress tuple raises a ValueError. assert_raises_regex(ValueError, 'Compress argument tuple should contain exactly ' '2 elements', numpy_pickle.dump, "dummy", filename, compress=('zlib', 3, 'extra')) # Verify a tuple with a wrong compress method raises a ValueError. msg = 'Non valid compression method given: "{0}"'.format('wrong') assert_raises_regex(ValueError, msg, numpy_pickle.dump, "dummy", filename, compress=('wrong', 3)) # Verify a tuple with a wrong compress level raises a ValueError. msg = 'Non valid compress level given: "{0}"'.format('wrong') assert_raises_regex(ValueError, msg, numpy_pickle.dump, "dummy", filename, compress=('zlib', 'wrong'))
def test_joblib_compression_formats(): compresslevels = (1, 3, 6) filename = env['filename'] + str(random.randint(0, 1000)) objects = (np.ones(shape=(100, 100), dtype='f8'), range(10), { 'a': 1, 2: 'b' }, [], (), {}, 0, 1.0) for compress in compresslevels: for cmethod in _COMPRESSORS: dump_filename = filename + "." + cmethod for obj in objects: if not PY3_OR_LATER and cmethod in ('xz', 'lzma'): # Lzma module only available for python >= 3.3 msg = "{0} compression is only available".format(cmethod) assert_raises_regex(NotImplementedError, msg, numpy_pickle.dump, obj, dump_filename, compress=(cmethod, compress)) else: numpy_pickle.dump(obj, dump_filename, compress=(cmethod, compress)) # Verify the file contains the right magic number with open(dump_filename, 'rb') as f: nose.tools.assert_equal(_detect_compressor(f), cmethod) # Verify the reloaded object is correct obj_reloaded = numpy_pickle.load(dump_filename) nose.tools.assert_true(isinstance(obj_reloaded, type(obj))) if isinstance(obj, np.ndarray): np.testing.assert_array_equal(obj_reloaded, obj) else: nose.tools.assert_equal(obj_reloaded, obj) os.remove(dump_filename)