def runTest(self): """testing zlib and shuffle compression filters""" uncompressed_size = os.stat(self.files[0]).st_size # check compressed data. f = Dataset(self.files[1]) size = os.stat(self.files[1]).st_size assert_almost_equal(array,f.variables['data'][:]) assert f.variables['data'].filters() == {'zlib':True,'shuffle':False,'complevel':6,'fletcher32':False} assert(size < 0.95*uncompressed_size) f.close() # check compression with shuffle f = Dataset(self.files[2]) size = os.stat(self.files[2]).st_size assert_almost_equal(array,f.variables['data'][:]) assert f.variables['data'].filters() == {'zlib':True,'shuffle':True,'complevel':6,'fletcher32':False} assert(size < 0.85*uncompressed_size) f.close() # check lossy compression without shuffle f = Dataset(self.files[3]) size = os.stat(self.files[3]).st_size checkarray = _quantize(array,lsd) assert_almost_equal(checkarray,f.variables['data'][:]) assert(size < 0.27*uncompressed_size) f.close() # check lossy compression with shuffle f = Dataset(self.files[4]) size = os.stat(self.files[4]).st_size assert_almost_equal(checkarray,f.variables['data'][:]) assert(size < 0.20*uncompressed_size) size_save = size f.close() # check lossy compression with shuffle and fletcher32 checksum. f = Dataset(self.files[5]) size = os.stat(self.files[5]).st_size assert_almost_equal(checkarray,f.variables['data'][:]) assert f.variables['data'].filters() == {'zlib':True,'shuffle':True,'complevel':6,'fletcher32':True} assert(size < 0.20*uncompressed_size) # should be slightly larger than without fletcher32 assert(size > size_save) # check chunksizes f.close() f = Dataset(self.files[6]) checkarray2 = _quantize(array2,lsd) assert_almost_equal(checkarray2,f.variables['data2'][:]) assert f.variables['data2'].filters() == {'zlib':True,'shuffle':True,'complevel':6,'fletcher32':True} assert f.variables['data2'].chunking() == [chunk1,chunk2] f.close()
# benchmark reads and writes, with and without compression. # tests all four supported file formats. from numpy.random.mtrand import uniform import netCDF4 from timeit import Timer import os, sys # create an n1dim by n2dim by n3dim random array. n1dim = 30 n2dim = 15 n3dim = 73 n4dim = 144 ntrials = 10 sys.stdout.write('reading and writing a %s by %s by %s by %s random array ..\n'%(n1dim,n2dim,n3dim,n4dim)) sys.stdout.write('(average of %s trials)\n' % ntrials) array = netCDF4._quantize(uniform(size=(n1dim,n2dim,n3dim,n4dim)),4) def write_netcdf(filename,zlib=False,shuffle=False): file = netCDF4.Dataset(filename,'w',format='NETCDF4') file.createDimension('n1', n1dim) file.createDimension('n2', n2dim) file.createDimension('n3', n3dim) file.createDimension('n4', n4dim) foo = file.createVariable('data',\ 'f8',('n1','n2','n3','n4'),zlib=zlib,shuffle=shuffle) foo[:] = array file.close() def read_netcdf(filename): file = netCDF4.Dataset(filename)
def runTest(self): """testing zlib and shuffle compression filters""" uncompressed_size = os.stat(self.files[0]).st_size # check compressed data. f = Dataset(self.files[1]) size = os.stat(self.files[1]).st_size assert_almost_equal(array, f.variables['data'][:]) assert f.variables['data'].filters() == { 'zlib': True, 'shuffle': False, 'complevel': 6, 'fletcher32': False } assert (size < 0.95 * uncompressed_size) f.close() # check compression with shuffle f = Dataset(self.files[2]) size = os.stat(self.files[2]).st_size assert_almost_equal(array, f.variables['data'][:]) assert f.variables['data'].filters() == { 'zlib': True, 'shuffle': True, 'complevel': 6, 'fletcher32': False } assert (size < 0.85 * uncompressed_size) f.close() # check lossy compression without shuffle f = Dataset(self.files[3]) size = os.stat(self.files[3]).st_size checkarray = _quantize(array, lsd) assert_almost_equal(checkarray, f.variables['data'][:]) assert (size < 0.27 * uncompressed_size) f.close() # check lossy compression with shuffle f = Dataset(self.files[4]) size = os.stat(self.files[4]).st_size assert_almost_equal(checkarray, f.variables['data'][:]) assert (size < 0.20 * uncompressed_size) size_save = size f.close() # check lossy compression with shuffle and fletcher32 checksum. f = Dataset(self.files[5]) size = os.stat(self.files[5]).st_size assert_almost_equal(checkarray, f.variables['data'][:]) assert f.variables['data'].filters() == { 'zlib': True, 'shuffle': True, 'complevel': 6, 'fletcher32': True } assert (size < 0.20 * uncompressed_size) # should be slightly larger than without fletcher32 assert (size > size_save) # check chunksizes f.close() f = Dataset(self.files[6]) checkarray2 = _quantize(array2, lsd) assert_almost_equal(checkarray2, f.variables['data2'][:]) assert f.variables['data2'].filters() == { 'zlib': True, 'shuffle': True, 'complevel': 6, 'fletcher32': True } assert f.variables['data2'].chunking() == [chunk1, chunk2] f.close()
from timeit import Timer import os, sys # create an n1dim by n2dim by n3dim random array. n1dim = 30 n2dim = 15 n3dim = 73 n4dim = 144 ntrials = 10 sys.stdout.write( 'reading and writing a %s by %s by %s by %s random array ..\n' % (n1dim, n2dim, n3dim, n4dim)) sys.stdout.write('(average of %s trials)\n' % ntrials) lsd = 4 if lsd is not None: array = netCDF4._quantize(uniform(size=(n1dim, n2dim, n3dim, n4dim)), 4) else: array = uniform(size=(n1dim, n2dim, n3dim, n4dim)) def write_netcdf(filename, complevel): file = netCDF4.Dataset(filename, 'w', format='NETCDF4') file.createDimension('n1', n1dim) file.createDimension('n2', n2dim) file.createDimension('n3', n3dim) file.createDimension('n4', n4dim) foo = file.createVariable('data',\ 'f8',('n1','n2','n3','n4'),zlib=True,shuffle=True,complevel=complevel) foo[:] = array file.close()