def test_10_posix_chunk_write(self):
self.f = Dataset('./testnc.nc','w')
self.f.test = 'Created for SemSL tests'
dim1 = self.f.createDimension('T', DIMSIZE)
dim1d = self.f.createVariable('T', 'i4', ('T',))
dim1d[:] = range(DIMSIZE)
dim2 = self.f.createDimension('Z', DIMSIZE)
dim2d = self.f.createVariable('Z', 'i4', ('Z',))
dim2d[:] = range(DIMSIZE)
dim3 = self.f.createDimension('Y', DIMSIZE)
dim3d = self.f.createVariable('Y', 'i4', ('Y',))
dim3d[:] = range(DIMSIZE)
dim4 = self.f.createDimension('X', DIMSIZE)
dim4d = self.f.createVariable('X', 'i4', ('X',))
dim4d[:] = range(DIMSIZE)
dim1d.axis = "T"
dim2d.axis = "Z"
dim3d.axis = "Y"
dim4d.axis = "X"
self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), chunksizes=(5,5,5,5))
np.random.seed(0)
self.var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE)
self.var.setncattr('units', 'test unit')
self.f.close()
# check that the subfile has the right chunking
f = Dataset('./testnc/testnc_var_[0].nc')
v = f.variables['var']
self.assertEqual([5,5,5,5],v.chunking())
f.close()
def test_disk_format(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
v = g.createVariable('var','f8')
v[:] = 1
self.assertEqual(g.disk_format, None)
f.close()
def test_set_var_chunk_cache(self):
f = Dataset('s3://test/databucket/testnc_methods.nc', 'a')
v = f.variables['var']
v.set_var_chunk_cache(size=10485760)
self.assertEqual(v.get_var_chunk_cache(), (10485760, 521, 0.75))
f.close()
def test_var_set_auto_chartostring(self):
f = Dataset('s3://test/databucket/testnc_methods.nc', 'a')
v = f.variables['var']
self.assertTrue(f.variables['var'].chartostring)
v.set_auto_chartostring(False)
self.assertFalse(f.variables['var'].chartostring)
f.close()
def test_variables(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
v = g.createVariable('var', 'f8',('T','Y','X'))
v[:] = np.zeros((20,20,20))
self.assertEqual([x for x in g.variables.keys()], ['var'])
f.close()
def test_var_set_auto_mask(self):
f = Dataset('./testnc_methods.nc', 'a')
v = f.variables['var']
self.assertTrue(f.variables['var'].mask)
v.set_auto_mask(False)
self.assertFalse(f.variables['var'].mask)
f.close()
def test_createDimension(self):
f = Dataset('./testnc_methods.nc','a')
g = f.createGroup('testgroup')
g.createDimension('extradim',10)
g.createVariable('extradim','i4',('extradim',))
v = g.createVariable('var','i4',('T','extradim',))
v[:] = np.zeros((20,10))
f.close()
f = Dataset('./testnc_methods.nc', 'r')
for i,j in zip(f.dimensions.keys(),['T','Z','Y','X']):
self.assertEqual(i,j)
g = f.groups['testgroup']
for i,j in zip(g.dimensions.keys(),['extradim']):
self.assertEqual(i,j)
f.close()
f = Dataset('./testnc_methods/testnc_methods_testgroup_var_[0].nc','r')
for i, j in zip(f.dimensions.keys(), ['T', 'Z', 'Y', 'X']):
self.assertEqual(i, j)
g = f.groups['testgroup']
for i, j in zip(g.dimensions.keys(), ['extradim']):
self.assertEqual(i, j)
f.close()
def test_ncattrs(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
g.setncattr('testattr', 'val')
g.setncattr('testattr2', 'val2')
self.assertEqual(g.ncattrs(),['testattr','testattr2'])
f.close()
def test_var_set_auto_scale(self):
f = Dataset('s3://test/databucket/testnc_methods.nc', 'a')
v = f.variables['var']
self.assertTrue(f.variables['var'].scale)
v.set_auto_scale(False)
self.assertFalse(f.variables['var'].scale)
f.close()
def test_renameDimension(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
g.createDimension('testdim',50)
self.assertEqual([x for x in g.dimensions.keys()],['testdim'])
g.renameDimension('testdim','renameddim')
self.assertEqual([x for x in g.dimensions.keys()],['renameddim'])
f.close()
def test_dimensions(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
g.createDimension('extradim', 10)
for i,j in zip(g.dimensions.keys(),['extradim']):
self.assertEqual(i,j)
f.close()
def test_renameGroup(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
gg = g.createGroup('subgroup')
self.assertEqual([x for x in g.groups.keys()],['subgroup'])
g.renameGroup('subgroup','renamedgroup')
self.assertEqual([x for x in g.groups.keys()],['renamedgroup'])
f.close()
def test_renameVariable(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
g.createVariable('var','f8')
self.assertEqual([x for x in g.variables.keys()], ['var'])
g.renameVariable('var','renamedvar')
self.assertEqual([x for x in g.variables.keys()], ['renamedvar'])
f.close()
def test_setncatts(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
v = g.createVariable('var', 'f8',('T','Y','X'))
v[:] = np.zeros((20,20,20))
g.setncatts({'testattr':'val','at2':'val2'})
self.assertEqual(g.ncattrs(), ['testattr','at2'])
f.close()
def test_isopen(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
v = g.createVariable('var', 'f8',('T','Y','X'))
v[:] = np.zeros((20,20,20))
self.assertFalse(g.isopen())
f.close()
self.assertFalse(g.isopen())
def test_renameDimension(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
v = g.createVariable('var', 'f8',('T','Y','X'))
v[:] = np.zeros((20,20,20))
g.createDimension('testdim',50)
self.assertEqual([x for x in g.dimensions.keys()],['testdim'])
g.renameDimension('testdim','renameddim')
self.assertEqual([x for x in g.dimensions.keys()],['renameddim'])
f.close()
def test_renameGroup(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
v = g.createVariable('var', 'f8',('T','Y','X'))
v[:] = np.zeros((20,20,20))
gg = g.createGroup('subgroup')
self.assertEqual([x for x in g.groups.keys()],['subgroup'])
g.renameGroup('subgroup','renamedgroup')
self.assertEqual([x for x in g.groups.keys()],['renamedgroup'])
f.close()
def test_var_delncattr(self):
f = Dataset('s3://test/databucket/testnc_methods.nc', 'a')
v = f.variables['var']
v.delncattr('units')
f.close()
f = Dataset('s3://test/databucket/testnc_methods.nc', 'a')
v = f.variables['var']
test_bool = False
try:
units = v.getncattr('units')
except AttributeError:
test_bool = True
self.assertTrue(test_bool)
f.close()
# now check a subfile...
f = Dataset(
's3://test/databucket/testnc_methods/testnc_methods_var_[0].nc',
'r')
v = f.variables['var']
test_bool = False
try:
units = v.getncattr('units')
except AttributeError:
test_bool = True
self.assertTrue(test_bool)
f.close()
def test_onegroup(self):
f = Dataset('./testnc_group.nc', 'a', format='NETCDF4')
# create group
g = f.createGroup('testgroup')
v = g.createVariable('var', 'f8', (
'T',
'Z',
'Y',
'X',
))
v[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE)
f.close()
def test_onegroup(self):
f = Dataset('./testnc_group.nc', 'a', format='NETCDF4')
# create group
g = f.createGroup('testgroup')
v = g.createVariable('var', 'f8', (
'T',
'Z',
'Y',
'X',
))
# TODO I think this throws an error because it can't find the dimensions that belong to the rootgroup
v[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE)
f.close()
def test_01_posix_write(self):
self.f = Dataset('./testnc.nc','w')
self.f.test = 'Created for SemSL tests'
dim1 = self.f.createDimension('T', DIMSIZE)
dim1d = self.f.createVariable('T', 'i4', ('T',))
dim1d[:] = range(DIMSIZE)
dim2 = self.f.createDimension('Z', DIMSIZE)
dim2d = self.f.createVariable('Z', 'i4', ('Z',))
dim2d[:] = range(DIMSIZE)
dim3 = self.f.createDimension('Y', DIMSIZE)
dim3d = self.f.createVariable('Y', 'i4', ('Y',))
dim3d[:] = range(DIMSIZE)
dim4 = self.f.createDimension('X', DIMSIZE)
dim4d = self.f.createVariable('X', 'i4', ('X',))
dim4d[:] = range(DIMSIZE)
dim1d.axis = "T"
dim2d.axis = "Z"
dim3d.axis = "Y"
dim4d.axis = "X"
self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True)
np.random.seed(0)
self.var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE)
self.var.setncattr('units', 'test unit')
self.f.close()
def setUp(self):
# Create test dataset
self.f = Dataset('s3://test/databucket/testnc_methods.nc',
'w',
format='NETCDF4')
self.f.setncattr('test', 'Created for SemSL tests')
dim1 = self.f.createDimension('T', DIMSIZE)
dim1d = self.f.createVariable('T', 'i4', ('T', ))
dim1d[:] = range(DIMSIZE)
dim2 = self.f.createDimension('Z', DIMSIZE)
dim2d = self.f.createVariable('Z', 'i4', ('Z', ))
dim2d[:] = range(DIMSIZE)
dim3 = self.f.createDimension('Y', DIMSIZE)
dim3d = self.f.createVariable('Y', 'i4', ('Y', ))
dim3d[:] = range(DIMSIZE)
dim4 = self.f.createDimension('X', DIMSIZE)
dim4d = self.f.createVariable('X', 'i4', ('X', ))
dim4d[:] = range(DIMSIZE)
dim1d.axis = "T"
dim2d.axis = "Z"
dim3d.axis = "Y"
dim4d.axis = "X"
self.var = self.f.createVariable(VARNAME,
'f8', ('T', 'Z', 'Y', 'X'),
contiguous=True)
self.var.units = 'test unit'
np.random.seed(0)
self.var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE)
self.f.close()
def test_get_variables_by_attributes(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
v = g.createVariable('testgroupvar', 'f8')
v.testattr = 'val'
v2 = g.createVariable('testgroupvar2', 'f8')
v2.testattr = 'val'
v3 = g.createVariable('tesgroupvar3', 'f8')
sg = g.createGroup('nestedgroup')
sv = sg.createVariable('nestedgroupvar', 'f8')
sv.testattr = 'val'
varlist = [x.name for x in g.get_variables_by_attributes(testattr='val')]
self.assertEqual(varlist,['testgroupvar','testgroupvar2'])
f.close()
def test_02_read_posix(self):
self.f = Dataset('./testnc.nc', 'r')
v = self.f.getVariable('var')
data = v[:]
self.assertTrue(data[0,0,0,0])
self.f.close()
def test_var_setncattr_string(self):
f = Dataset('./testnc_methods.nc', 'a')
v = f.variables['var']
v.setncattr_string('newtest', 'newtestvalue')
f.close()
f = Dataset('./testnc_methods.nc', 'r')
v = f.variables['var']
self.assertEqual(v.getncattr('newtest'), 'newtestvalue')
f.close()
# now check a subfile...
f = Dataset('./testnc_methods/testnc_methods_var_[0].nc', 'r')
v = f.variables['var']
self.assertEqual(v.getncattr('newtest'), 'newtestvalue')
f.close()
def test_var_setncatts(self):
f = Dataset('./testnc_methods.nc', 'a')
v = f.variables['var']
v.setncatts({'newtest': 'newtestvalue', 'secondnew': 'secondnewval'})
f.close()
f = Dataset('./testnc_methods.nc', 'r')
v = f.variables['var']
self.assertEqual(v.getncattr('newtest'), 'newtestvalue')
self.assertEqual(v.getncattr('secondnew'), 'secondnewval')
f.close()
# now check a subfile...
f = Dataset('./testnc_methods/testnc_methods_var_[0].nc', 'r')
v = f.variables['var']
self.assertEqual(v.getncattr('newtest'), 'newtestvalue')
self.assertEqual(v.getncattr('secondnew'), 'secondnewval')
f.close()
def test_renameAttribute(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
v = g.createVariable('var', 'f8',('T','Y','X'))
v[:] = np.zeros((20,20,20))
g.setncattr('testattr', 'val')
g.setncattr('testattr2', 'val2')
self.assertEqual(g.ncattrs(), ['testattr', 'testattr2'])
f.close()
f = Dataset('./testnc_methods.nc', 'a')
g = f.groups['testgroup']
g.renameAttribute('testattr','renamedattr')
self.assertEqual(g.ncattrs(), ['renamedattr', 'testattr2'])
f.close()
f = Dataset('./testnc_methods/testnc_methods_testgroup_var_[0].nc', 'r')
g = f.groups['testgroup']
#TODO fix
#self.assertEqual(g.ncattrs(), ['renamedattr', 'testattr2'])
f.close()
def test_getncattr(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
g.setncattr('testattr','val')
f.close()
f = Dataset('./testnc_methods.nc', 'r')
g = f.groups['testgroup']
self.assertEqual(g.getncattr('testattr'),'val')
f.close()
def test_name(self):
f = Dataset('./testnc_methods.nc', 'a')
g = f.createGroup('testgroup')
v = g.createVariable('var', 'f8',('T','Y','X'))
v[:] = np.zeros((20,20,20))
self.assertEqual(g.name, 'testgroup')
f.close()
f = Dataset('./testnc_methods/testnc_methods_testgroup_var_[0].nc', 'r')
g = f.groups['testgroup']
self.assertEqual(g.name, 'testgroup')
f.close()
def test_04_s3_write(self):
self.f = Dataset('s3://test/databucket/testnc.nc', 'w')
self.f.test = 'Created for SemSL tests'
sl_config = slConfig()
dim1 = self.f.createDimension('T', DIMSIZE)
dim1d = self.f.createVariable('T', 'i4', ('T',))
dim1d[:] = range(DIMSIZE)
dim2 = self.f.createDimension('Z', DIMSIZE)
dim2d = self.f.createVariable('Z', 'i4', ('Z',))
dim2d[:] = range(DIMSIZE)
dim3 = self.f.createDimension('Y', DIMSIZE)
dim3d = self.f.createVariable('Y', 'i4', ('Y',))
dim3d[:] = range(DIMSIZE)
dim4 = self.f.createDimension('X', DIMSIZE)
dim4d = self.f.createVariable('X', 'i4', ('X',))
dim4d[:] = range(DIMSIZE)
dim1d.axis = "T"
dim2d.axis = "Z"
dim3d.axis = "Y"
dim4d.axis = "X"
self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True)
self.var.units = 'test unit'
np.random.seed(0)
self.var[:] = np.random.rand(DIMSIZE,DIMSIZE,DIMSIZE,DIMSIZE)
self.f.close()
# Check what is in the cacheDB and cache area
slDB = slCacheDB()
allcachedfids = slDB.get_all_fids()
cachedir = sl_config['cache']['location']
cacheareafiles = glob('{}/*.nc'.format(cachedir))
cacheareasubfiles = glob('{}/testnc/*.nc'.format(cachedir))
all_cache = []
for i in cacheareasubfiles:
all_cache.append(i.split('/')[-1])
for i in cacheareafiles:
all_cache.append(i.split('/')[-1])
allcached = []
for i in allcachedfids:
allcached.append(i.split('/')[-1])
all_cache.sort()
#print(all_cache)
allcached.sort()
#print(allcached)
self.assertEqual(all_cache,allcached)
# remove from cache
sl_cache = slCacheManager()
sl_cache._clear_cache()
