def test_10_posix_chunk_write(self): self.f = Dataset('./testnc.nc','w') self.f.test = 'Created for SemSL tests' dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T',)) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z',)) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y',)) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X',)) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), chunksizes=(5,5,5,5)) np.random.seed(0) self.var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) self.var.setncattr('units', 'test unit') self.f.close() # check that the subfile has the right chunking f = Dataset('./testnc/testnc_var_[0].nc') v = f.variables['var'] self.assertEqual([5,5,5,5],v.chunking()) f.close()
def test_disk_format(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') v = g.createVariable('var','f8') v[:] = 1 self.assertEqual(g.disk_format, None) f.close()
def test_set_var_chunk_cache(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.set_var_chunk_cache(size=10485760) self.assertEqual(v.get_var_chunk_cache(), (10485760, 521, 0.75)) f.close()
def test_var_set_auto_chartostring(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] self.assertTrue(f.variables['var'].chartostring) v.set_auto_chartostring(False) self.assertFalse(f.variables['var'].chartostring) f.close()
def test_variables(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') v = g.createVariable('var', 'f8',('T','Y','X')) v[:] = np.zeros((20,20,20)) self.assertEqual([x for x in g.variables.keys()], ['var']) f.close()
def test_var_set_auto_mask(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] self.assertTrue(f.variables['var'].mask) v.set_auto_mask(False) self.assertFalse(f.variables['var'].mask) f.close()
def test_createDimension(self): f = Dataset('./testnc_methods.nc','a') g = f.createGroup('testgroup') g.createDimension('extradim',10) g.createVariable('extradim','i4',('extradim',)) v = g.createVariable('var','i4',('T','extradim',)) v[:] = np.zeros((20,10)) f.close() f = Dataset('./testnc_methods.nc', 'r') for i,j in zip(f.dimensions.keys(),['T','Z','Y','X']): self.assertEqual(i,j) g = f.groups['testgroup'] for i,j in zip(g.dimensions.keys(),['extradim']): self.assertEqual(i,j) f.close() f = Dataset('./testnc_methods/testnc_methods_testgroup_var_[0].nc','r') for i, j in zip(f.dimensions.keys(), ['T', 'Z', 'Y', 'X']): self.assertEqual(i, j) g = f.groups['testgroup'] for i, j in zip(g.dimensions.keys(), ['extradim']): self.assertEqual(i, j) f.close()
def test_ncattrs(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') g.setncattr('testattr', 'val') g.setncattr('testattr2', 'val2') self.assertEqual(g.ncattrs(),['testattr','testattr2']) f.close()
def test_var_set_auto_scale(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] self.assertTrue(f.variables['var'].scale) v.set_auto_scale(False) self.assertFalse(f.variables['var'].scale) f.close()
def test_renameDimension(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') g.createDimension('testdim',50) self.assertEqual([x for x in g.dimensions.keys()],['testdim']) g.renameDimension('testdim','renameddim') self.assertEqual([x for x in g.dimensions.keys()],['renameddim']) f.close()
def test_dimensions(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') g.createDimension('extradim', 10) for i,j in zip(g.dimensions.keys(),['extradim']): self.assertEqual(i,j) f.close()
def test_renameGroup(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') gg = g.createGroup('subgroup') self.assertEqual([x for x in g.groups.keys()],['subgroup']) g.renameGroup('subgroup','renamedgroup') self.assertEqual([x for x in g.groups.keys()],['renamedgroup']) f.close()
def test_renameVariable(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') g.createVariable('var','f8') self.assertEqual([x for x in g.variables.keys()], ['var']) g.renameVariable('var','renamedvar') self.assertEqual([x for x in g.variables.keys()], ['renamedvar']) f.close()
def test_setncatts(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') v = g.createVariable('var', 'f8',('T','Y','X')) v[:] = np.zeros((20,20,20)) g.setncatts({'testattr':'val','at2':'val2'}) self.assertEqual(g.ncattrs(), ['testattr','at2']) f.close()
def test_isopen(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') v = g.createVariable('var', 'f8',('T','Y','X')) v[:] = np.zeros((20,20,20)) self.assertFalse(g.isopen()) f.close() self.assertFalse(g.isopen())
def test_renameDimension(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') v = g.createVariable('var', 'f8',('T','Y','X')) v[:] = np.zeros((20,20,20)) g.createDimension('testdim',50) self.assertEqual([x for x in g.dimensions.keys()],['testdim']) g.renameDimension('testdim','renameddim') self.assertEqual([x for x in g.dimensions.keys()],['renameddim']) f.close()
def test_renameGroup(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') v = g.createVariable('var', 'f8',('T','Y','X')) v[:] = np.zeros((20,20,20)) gg = g.createGroup('subgroup') self.assertEqual([x for x in g.groups.keys()],['subgroup']) g.renameGroup('subgroup','renamedgroup') self.assertEqual([x for x in g.groups.keys()],['renamedgroup']) f.close()
def test_var_delncattr(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.delncattr('units') f.close() f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] test_bool = False try: units = v.getncattr('units') except AttributeError: test_bool = True self.assertTrue(test_bool) f.close() # now check a subfile... f = Dataset( 's3://test/databucket/testnc_methods/testnc_methods_var_[0].nc', 'r') v = f.variables['var'] test_bool = False try: units = v.getncattr('units') except AttributeError: test_bool = True self.assertTrue(test_bool) f.close()
def test_onegroup(self): f = Dataset('./testnc_group.nc', 'a', format='NETCDF4') # create group g = f.createGroup('testgroup') v = g.createVariable('var', 'f8', ( 'T', 'Z', 'Y', 'X', )) v[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) f.close()
def test_onegroup(self): f = Dataset('./testnc_group.nc', 'a', format='NETCDF4') # create group g = f.createGroup('testgroup') v = g.createVariable('var', 'f8', ( 'T', 'Z', 'Y', 'X', )) # TODO I think this throws an error because it can't find the dimensions that belong to the rootgroup v[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) f.close()
def test_01_posix_write(self): self.f = Dataset('./testnc.nc','w') self.f.test = 'Created for SemSL tests' dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T',)) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z',)) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y',)) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X',)) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) np.random.seed(0) self.var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) self.var.setncattr('units', 'test unit') self.f.close()
def setUp(self): # Create test dataset self.f = Dataset('s3://test/databucket/testnc_methods.nc', 'w', format='NETCDF4') self.f.setncattr('test', 'Created for SemSL tests') dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T', )) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z', )) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y', )) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X', )) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) self.var.units = 'test unit' np.random.seed(0) self.var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) self.f.close()
def test_get_variables_by_attributes(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') v = g.createVariable('testgroupvar', 'f8') v.testattr = 'val' v2 = g.createVariable('testgroupvar2', 'f8') v2.testattr = 'val' v3 = g.createVariable('tesgroupvar3', 'f8') sg = g.createGroup('nestedgroup') sv = sg.createVariable('nestedgroupvar', 'f8') sv.testattr = 'val' varlist = [x.name for x in g.get_variables_by_attributes(testattr='val')] self.assertEqual(varlist,['testgroupvar','testgroupvar2']) f.close()
def test_02_read_posix(self): self.f = Dataset('./testnc.nc', 'r') v = self.f.getVariable('var') data = v[:] self.assertTrue(data[0,0,0,0]) self.f.close()
def test_var_setncattr_string(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] v.setncattr_string('newtest', 'newtestvalue') f.close() f = Dataset('./testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') f.close() # now check a subfile... f = Dataset('./testnc_methods/testnc_methods_var_[0].nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') f.close()
def test_var_setncatts(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] v.setncatts({'newtest': 'newtestvalue', 'secondnew': 'secondnewval'}) f.close() f = Dataset('./testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') self.assertEqual(v.getncattr('secondnew'), 'secondnewval') f.close() # now check a subfile... f = Dataset('./testnc_methods/testnc_methods_var_[0].nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') self.assertEqual(v.getncattr('secondnew'), 'secondnewval') f.close()
def test_renameAttribute(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') v = g.createVariable('var', 'f8',('T','Y','X')) v[:] = np.zeros((20,20,20)) g.setncattr('testattr', 'val') g.setncattr('testattr2', 'val2') self.assertEqual(g.ncattrs(), ['testattr', 'testattr2']) f.close() f = Dataset('./testnc_methods.nc', 'a') g = f.groups['testgroup'] g.renameAttribute('testattr','renamedattr') self.assertEqual(g.ncattrs(), ['renamedattr', 'testattr2']) f.close() f = Dataset('./testnc_methods/testnc_methods_testgroup_var_[0].nc', 'r') g = f.groups['testgroup'] #TODO fix #self.assertEqual(g.ncattrs(), ['renamedattr', 'testattr2']) f.close()
def test_getncattr(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') g.setncattr('testattr','val') f.close() f = Dataset('./testnc_methods.nc', 'r') g = f.groups['testgroup'] self.assertEqual(g.getncattr('testattr'),'val') f.close()
def test_name(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') v = g.createVariable('var', 'f8',('T','Y','X')) v[:] = np.zeros((20,20,20)) self.assertEqual(g.name, 'testgroup') f.close() f = Dataset('./testnc_methods/testnc_methods_testgroup_var_[0].nc', 'r') g = f.groups['testgroup'] self.assertEqual(g.name, 'testgroup') f.close()
def test_04_s3_write(self): self.f = Dataset('s3://test/databucket/testnc.nc', 'w') self.f.test = 'Created for SemSL tests' sl_config = slConfig() dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T',)) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z',)) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y',)) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X',)) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) self.var.units = 'test unit' np.random.seed(0) self.var[:] = np.random.rand(DIMSIZE,DIMSIZE,DIMSIZE,DIMSIZE) self.f.close() # Check what is in the cacheDB and cache area slDB = slCacheDB() allcachedfids = slDB.get_all_fids() cachedir = sl_config['cache']['location'] cacheareafiles = glob('{}/*.nc'.format(cachedir)) cacheareasubfiles = glob('{}/testnc/*.nc'.format(cachedir)) all_cache = [] for i in cacheareasubfiles: all_cache.append(i.split('/')[-1]) for i in cacheareafiles: all_cache.append(i.split('/')[-1]) allcached = [] for i in allcachedfids: allcached.append(i.split('/')[-1]) all_cache.sort() #print(all_cache) allcached.sort() #print(allcached) self.assertEqual(all_cache,allcached) # remove from cache sl_cache = slCacheManager() sl_cache._clear_cache()