def test_var_setncatts(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.setncatts({'newtest': 'newtestvalue', 'secondnew': 'secondnewval'}) f.close() f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') self.assertEqual(v.getncattr('secondnew'), 'secondnewval') f.close() # now check a subfile... f = Dataset( 's3://test/databucket/testnc_methods/testnc_methods_var_[0].nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') self.assertEqual(v.getncattr('secondnew'), 'secondnewval') f.close()
def test_renameAttribute(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') v = g.createVariable('var', 'f8',('T','Y','X')) v[:] = np.zeros((20,20,20)) g.setncattr('testattr', 'val') g.setncattr('testattr2', 'val2') self.assertEqual(g.ncattrs(), ['testattr', 'testattr2']) f.close() f = Dataset('./testnc_methods.nc', 'a') g = f.groups['testgroup'] g.renameAttribute('testattr','renamedattr') self.assertEqual(g.ncattrs(), ['renamedattr', 'testattr2']) f.close() f = Dataset('./testnc_methods/testnc_methods_testgroup_var_[0].nc', 'r') g = f.groups['testgroup'] #TODO fix #self.assertEqual(g.ncattrs(), ['renamedattr', 'testattr2']) f.close()
def test_getncattr(self): f = Dataset('./testnc_methods.nc', 'a') f.setncattr('rootgroupattr', 'val') g = f.createGroup('testgroup') g.setncattr('testattr','val') v = g.createVariable('var', 'f8',('T','Y','X')) v.setncattr('varattr','val') v[:] = np.zeros((20,20,20)) f.close() f = Dataset('./testnc_methods.nc', 'r') g = f.groups['testgroup'] self.assertEqual(g.getncattr('testattr'),'val') f.close() f = Dataset('./testnc_methods/testnc_methods_testgroup_var_[0].nc', 'r') g = f.groups['testgroup'] v = g.variables['var'] #self.assertEqual(v.getncattr('varattr'),'val') self.assertEqual(g.getncattr('testattr'),'val') #self.assertEqual(f.getncattr('rootgroupattr'),'val') f.close()
def test_createGroup(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') v = g.createVariable('testgroupvar','f8',('T','Z')) #sg = g.createGroup('nestedgroup') # TODO nested groups might not get supported... #v2 = sg.createVariable('nestedgroupvar','f8',('Y','X')) v[:] = np.zeros((20,20)) #v2[:] = np.zeros((20,20)) f.close() f = Dataset('./testnc_methods.nc', 'a') g = f.groups['testgroup'] #sg = g.groups['nestedgroup'] self.assertEqual(g.path,'/testgroup') #self.assertEqual(sg.path,'/testgroup/nestedgroup') f.close() f = Dataset('./testnc_methods/testnc_methods_testgroup_testgroupvar_[0].nc', 'r') self.assertEqual(f.groups['testgroup'].path, '/testgroup') f.close()
def test_setncattr_string(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') g.setncattr('testattr', 'val') self.assertEqual(g.ncattrs(), ['testattr']) f.close()
def test_path(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') self.assertEqual(g.path, '/testgroup') f.close()
def test_isopen(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') self.assertFalse(g.isopen()) f.close() self.assertFalse(g.isopen())
def read_file(): f = Dataset('s3://test/databucket/testnc.nc', 'r') v = f.variables['var'] data = v[:] f.close()
def test_changearrayvalues(self): # create file in backend self.f = Dataset('s3://test/databucket/testnc_varchange.nc', 'w') self.f.setncattr('test', 'Created for SemSL tests') dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T', )) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z', )) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y', )) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X', )) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable('var', 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) self.var.setncattr('units', 'test unit') self.var[:] = np.zeros((DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE)) self.f.close() # remove the files in cache sl_cache = slCacheManager() sl_cache._clear_cache() # Now reopen f = Dataset('s3://test/databucket/testnc_varchange.nc', 'a') var = f.variables['var'] self.assertEqual(var[0, 0, 0, 0], 0) var[0, 0, 0, :] = np.ones((1, 1, 1, DIMSIZE)) self.assertEqual(var[0, 0, 0, 0], 1) f.close() # remove the files in cache sl_cache = slCacheManager() sl_cache._clear_cache() # now reopen again and check the change f = Dataset('s3://test/databucket/testnc_varchange.nc', 'r') var = f.variables['var'] self.assertEqual(var[0, 0, 0, 0], 1) f.close() # cleanup sl_config = slConfig() slDB = slCacheDB() conn_man = slConnectionManager(sl_config) conn = conn_man.open("s3://test") sl_cache = slCacheManager() sl_cache._clear_cache() s3 = conn.get() subfiles = s3.list_objects(Bucket='databucket', Prefix='testnc_varchange/')['Contents'] for sf in subfiles: s3.delete_object(Bucket='databucket', Key=sf['Key']) s3.delete_object(Bucket='databucket', Key='testnc_varchange.nc') s3.delete_bucket(Bucket='databucket')
def test_get_var_chunk_cache(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.get_var_chunk_cache(), (1048576, 521, 0.75)) f.close()
def test_var_getncattr(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] self.assertEqual(v.getncattr('units'), 'test unit') f.close()
def test_var_ncattrs(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] attrs = v.ncattrs() self.assertEqual(attrs, ['units']) f.close()
class test_set4_Methods_s3_noncfa(unittest.TestCase): def setUp(self): # Create test dataset self.f = Dataset('s3://test/databucket/testnc_methods.nc', 'w', format='NETCDF4') self.f.setncattr('test', 'Created for SemSL tests') dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T', )) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z', )) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y', )) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X', )) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) self.var.units = 'test unit' np.random.seed(0) self.var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) self.f.close() def tearDown(self): # remove test file sl_cache = slCacheManager() sl_config = slConfig() slDB = slCacheDB() cache_loc = sl_config['cache']['location'] conn_man = slConnectionManager(sl_config) conn = conn_man.open("s3://test") sl_cache._clear_cache() s3 = conn.get() s3.delete_object(Bucket='databucket', Key='testnc_methods.nc') s3.delete_bucket(Bucket='databucket') def test_var_ncattrs(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] attrs = v.ncattrs() self.assertEqual(attrs, ['units']) f.close() def test_var_setncattr(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.setncattr('newtest', 'newtestvalue') f.close() f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') f.close() def test_var_setncattr_string(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.setncattr_string('newtest', 'newtestvalue') f.close() f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') f.close() def test_var_setncatts(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.setncatts({'newtest': 'newtestvalue', 'secondnew': 'secondnewval'}) f.close() f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') self.assertEqual(v.getncattr('secondnew'), 'secondnewval') f.close() def test_var_getncattr(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] self.assertEqual(v.getncattr('units'), 'test unit') f.close() def test_var_delncattr(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.delncattr('units') f.close() f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] test_bool = False try: units = v.getncattr('units') except AttributeError: test_bool = True self.assertTrue(test_bool) f.close() def test_get_var_chunk_cache(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.get_var_chunk_cache(), (1048576, 521, 0.75)) f.close() def test_set_var_chunk_cache(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.set_var_chunk_cache(size=10485760) self.assertEqual(v.get_var_chunk_cache(), (10485760, 521, 0.75)) f.close() def test_var_renameAttribute(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.renameAttribute('units', 'renamedattr') f.close() f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] test_bool = False try: attr = f.getncattr('units') except AttributeError: test_bool = True self.assertTrue(test_bool) self.assertEqual(v.getncattr('renamedattr'), 'test unit') f.close() def test_var_set_auto_chartostring(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] self.assertTrue(f.variables['var'].chartostring) v.set_auto_chartostring(False) self.assertFalse(f.variables['var'].chartostring) f.close() def test_var_set_auto_mask(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] self.assertTrue(f.variables['var'].mask) v.set_auto_mask(False) self.assertFalse(f.variables['var'].mask) f.close() def test_var_set_auto_maskandscale(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] self.assertTrue(f.variables['var'].mask) self.assertTrue(f.variables['var'].scale) v.set_auto_maskandscale(False) self.assertFalse(f.variables['var'].mask) self.assertFalse(f.variables['var'].scale) f.close() def test_var_set_auto_scale(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] self.assertTrue(f.variables['var'].scale) v.set_auto_scale(False) self.assertFalse(f.variables['var'].scale) f.close()
class test_set5_Methods_s3_cfa(unittest.TestCase): def setUp(self): # Create test dataset self.f = Dataset('s3://test/databucket/testnc_methods.nc', 'w') self.f.setncattr('test', 'Created for SemSL tests') dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T', )) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z', )) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y', )) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X', )) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) self.var.setncattr('units', 'test unit') np.random.seed(0) self.var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) self.f.close() def tearDown(self): # remove test file sl_cache = slCacheManager() sl_config = slConfig() slDB = slCacheDB() cache_loc = sl_config['cache']['location'] conn_man = slConnectionManager(sl_config) conn = conn_man.open("s3://test") # construct list of subfiles subfiles = glob('{}/testnc_methods/*.nc'.format(cache_loc)) subfiles = [ 'testnc_methods/' + file.split('/')[-1] for file in subfiles ] sl_cache._clear_cache() s3 = conn.get() s3.delete_object(Bucket='databucket', Key='testnc_methods.nc') for file in subfiles: #print(file) s3.delete_object(Bucket='databucket', Key=file) s3.delete_bucket(Bucket='databucket') def test_var_setncattr(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.setncattr('newtest', 'newtestvalue') f.close() f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') f.close() # now check a subfile... f = Dataset( 's3://test/databucket/testnc_methods/testnc_methods_var_[0].nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') f.close() def test_var_ncattrs(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] attrs = v.ncattrs() self.assertEqual(attrs, ['cf_role', 'cf_dimensions', 'cfa_array', 'units']) f.close() # now check a subfile... f = Dataset( 's3://test/databucket/testnc_methods/testnc_methods_var_[0].nc', 'r') v = f.variables['var'] attrs = v.ncattrs() self.assertEqual(attrs, ['cf_role', 'cf_dimensions', 'cfa_array', 'units']) f.close() def test_var_setncattr_string(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.setncattr_string('newtest', 'newtestvalue') f.close() f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') f.close() # now check a subfile... f = Dataset( 's3://test/databucket/testnc_methods/testnc_methods_var_[0].nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') f.close() def test_var_setncatts(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.setncatts({'newtest': 'newtestvalue', 'secondnew': 'secondnewval'}) f.close() f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') self.assertEqual(v.getncattr('secondnew'), 'secondnewval') f.close() # now check a subfile... f = Dataset( 's3://test/databucket/testnc_methods/testnc_methods_var_[0].nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') self.assertEqual(v.getncattr('secondnew'), 'secondnewval') f.close() def test_var_getncattr(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] self.assertEqual(v.getncattr('units'), 'test unit') f.close() # now check a subfile... f = Dataset( 's3://test/databucket/testnc_methods/testnc_methods_var_[0].nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('units'), 'test unit') f.close() def test_var_delncattr(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.delncattr('units') f.close() f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] test_bool = False try: units = v.getncattr('units') except AttributeError: test_bool = True self.assertTrue(test_bool) f.close() # now check a subfile... f = Dataset( 's3://test/databucket/testnc_methods/testnc_methods_var_[0].nc', 'r') v = f.variables['var'] test_bool = False try: units = v.getncattr('units') except AttributeError: test_bool = True self.assertTrue(test_bool) f.close() def test_get_var_chunk_cache(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.get_var_chunk_cache(), (1048576, 521, 0.75)) f.close() def test_set_var_chunk_cache(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.set_var_chunk_cache(size=10485760) self.assertEqual(v.get_var_chunk_cache(), (10485760, 521, 0.75)) f.close() def test_var_renameAttribute(self): f = Dataset('s3://test/databucket/testnc_methods.nc', 'a') v = f.variables['var'] v.renameAttribute('units', 'renamedattr') f.close() f = Dataset('s3://test/databucket/testnc_methods.nc', 'r') v = f.variables['var'] test_bool = False try: attr = f.getncattr('units') except AttributeError: test_bool = True self.assertTrue(test_bool) self.assertEqual(v.getncattr('renamedattr'), 'test unit') f.close() # now check a subfile... f = Dataset( 's3://test/databucket/testnc_methods/testnc_methods_var_[0].nc', 'r') v = f.variables['var'] test_bool = False try: attr = f.getncattr('units') except AttributeError: test_bool = True self.assertTrue(test_bool) self.assertEqual(v.getncattr('renamedattr'), 'test unit') f.close()
class test_set2_Methods_posix_noncfa(unittest.TestCase): def setUp(self): # Create test dataset self.f = Dataset('./testnc_methods.nc', 'w', format='NETCDF4') self.f.setncattr('test', 'Created for SemSL tests') dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T', )) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z', )) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y', )) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X', )) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) self.var.units = 'test unit' np.random.seed(0) self.var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) self.f.close() def tearDown(self): # remove test file os.remove('./testnc_methods.nc') def test_var_ncattrs(self): f = Dataset('./testnc_methods.nc', 'r') v = f.variables['var'] attrs = v.ncattrs() self.assertEqual(attrs, ['units']) f.close() def test_var_setncattr(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] v.setncattr('newtest', 'newtestvalue') f.close() f = Dataset('./testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') f.close() def test_var_setncattr_string(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] v.setncattr_string('newtest', 'newtestvalue') f.close() f = Dataset('./testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') f.close() def test_var_setncatts(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] v.setncatts({'newtest': 'newtestvalue', 'secondnew': 'secondnewval'}) f.close() f = Dataset('./testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.getncattr('newtest'), 'newtestvalue') self.assertEqual(v.getncattr('secondnew'), 'secondnewval') f.close() def test_var_getncattr(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] self.assertEqual(v.getncattr('units'), 'test unit') f.close() def test_var_delncattr(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] v.delncattr('units') f.close() f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] test_bool = False try: units = v.getncattr('units') except AttributeError: test_bool = True self.assertTrue(test_bool) f.close() def test_get_var_chunk_cache(self): f = Dataset('./testnc_methods.nc', 'r') v = f.variables['var'] self.assertEqual(v.get_var_chunk_cache(), (1048576, 521, 0.75)) f.close() def test_set_var_chunk_cache(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] v.set_var_chunk_cache(size=10485760) self.assertEqual(v.get_var_chunk_cache(), (10485760, 521, 0.75)) f.close() def test_var_renameAttribute(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] v.renameAttribute('units', 'renamedattr') f.close() f = Dataset('./testnc_methods.nc', 'r') v = f.variables['var'] test_bool = False try: attr = f.getncattr('units') except AttributeError: test_bool = True self.assertTrue(test_bool) self.assertEqual(v.getncattr('renamedattr'), 'test unit') f.close() def test_var_set_auto_chartostring(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] self.assertTrue(f.variables['var'].chartostring) v.set_auto_chartostring(False) self.assertFalse(f.variables['var'].chartostring) f.close() def test_var_set_auto_mask(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] self.assertTrue(f.variables['var'].mask) v.set_auto_mask(False) self.assertFalse(f.variables['var'].mask) f.close() def test_var_set_auto_maskandscale(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] self.assertTrue(f.variables['var'].mask) self.assertTrue(f.variables['var'].scale) v.set_auto_maskandscale(False) self.assertFalse(f.variables['var'].mask) self.assertFalse(f.variables['var'].scale) f.close() def test_var_set_auto_scale(self): f = Dataset('./testnc_methods.nc', 'a') v = f.variables['var'] self.assertTrue(f.variables['var'].scale) v.set_auto_scale(False) self.assertFalse(f.variables['var'].scale) f.close()
def test_setncatts(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') g.setncatts({'testattr':'val','at2':'val2'}) self.assertEqual(g.ncattrs(), ['testattr','at2']) f.close()
def test_variables(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') g.createVariable('var', 'f8') self.assertEqual([x for x in g.variables.keys()], ['var']) f.close()
def read_file(): f = Dataset('s3://caringo/mjones07/testnc.nc', 'r') v = f.variables['var'] data = v[:] f.close()
def test_data_model(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') self.assertEqual(g.data_model,'NETCDF4') f.close()
def test_file_format(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') self.assertEqual(g.file_format,'NETCDF4') f.close()
class test_set1_variables(unittest.TestCase): def test_multicfavar(self): # Create test dataset f = Dataset('./testnc_multivar.nc', 'w') f.setncattr('test', 'Created for SemSL tests') dim1 = f.createDimension('T', DIMSIZE) dim1d = f.createVariable('T', 'i4', ('T', )) dim1d[:] = range(DIMSIZE) dim2 = f.createDimension('Z', DIMSIZE) dim2d = f.createVariable('Z', 'i4', ('Z', )) dim2d[:] = range(DIMSIZE) dim3 = f.createDimension('Y', DIMSIZE) dim3d = f.createVariable('Y', 'i4', ('Y', )) dim3d[:] = range(DIMSIZE) dim4 = f.createDimension('X', DIMSIZE) dim4d = f.createVariable('X', 'i4', ('X', )) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" np.random.seed(0) var = f.createVariable('var', 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) var.setncattr('units', 'test unit') var2 = f.createVariable('var2', 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) var2[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) var2.setncattr('units', 'test unit') f.close() # Now remove the files os.remove('./testnc_multivar.nc') subfiles = glob('./testnc_multivar/*.nc') # print('IN TEARDOWN {}'.format(subfiles)) for f in subfiles: os.remove(f) os.rmdir('./testnc_multivar') self.assertFalse(os.path.exists('./testnc_multivar/')) def test_changearrayvalues(self): # create file in backend self.f = Dataset('s3://test/databucket/testnc_varchange.nc', 'w') self.f.setncattr('test', 'Created for SemSL tests') dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T', )) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z', )) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y', )) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X', )) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable('var', 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) self.var.setncattr('units', 'test unit') self.var[:] = np.zeros((DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE)) self.f.close() # remove the files in cache sl_cache = slCacheManager() sl_cache._clear_cache() # Now reopen f = Dataset('s3://test/databucket/testnc_varchange.nc', 'a') var = f.variables['var'] self.assertEqual(var[0, 0, 0, 0], 0) var[0, 0, 0, :] = np.ones((1, 1, 1, DIMSIZE)) self.assertEqual(var[0, 0, 0, 0], 1) f.close() # remove the files in cache sl_cache = slCacheManager() sl_cache._clear_cache() # now reopen again and check the change f = Dataset('s3://test/databucket/testnc_varchange.nc', 'r') var = f.variables['var'] self.assertEqual(var[0, 0, 0, 0], 1) f.close() # cleanup sl_config = slConfig() slDB = slCacheDB() conn_man = slConnectionManager(sl_config) conn = conn_man.open("s3://test") sl_cache = slCacheManager() sl_cache._clear_cache() s3 = conn.get() subfiles = s3.list_objects(Bucket='databucket', Prefix='testnc_varchange/')['Contents'] for sf in subfiles: s3.delete_object(Bucket='databucket', Key=sf['Key']) s3.delete_object(Bucket='databucket', Key='testnc_varchange.nc') s3.delete_bucket(Bucket='databucket')
def test_groups(self): f = Dataset('./testnc_methods.nc', 'a') g = f.createGroup('testgroup') sg = g.createGroup('nestedgroup') self.assertEqual([x for x in g.groups], ['nestedgroup']) f.close()
def read_file(): f = Dataset('./testnc.nc', 'r') v = f.variables['var'] data = v[:] f.close()
class test_set0_ReadWrite(unittest.TestCase): def test_01_posix_write(self): self.f = Dataset('./testnc.nc','w') self.f.test = 'Created for SemSL tests' dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T',)) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z',)) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y',)) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X',)) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) np.random.seed(0) self.var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) self.var.setncattr('units', 'test unit') self.f.close() def test_04_s3_write(self): self.f = Dataset('s3://test/databucket/testnc.nc', 'w') self.f.test = 'Created for SemSL tests' sl_config = slConfig() dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T',)) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z',)) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y',)) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X',)) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) self.var.units = 'test unit' np.random.seed(0) self.var[:] = np.random.rand(DIMSIZE,DIMSIZE,DIMSIZE,DIMSIZE) self.f.close() # Check what is in the cacheDB and cache area slDB = slCacheDB() allcachedfids = slDB.get_all_fids() cachedir = sl_config['cache']['location'] cacheareafiles = glob('{}/*.nc'.format(cachedir)) cacheareasubfiles = glob('{}/testnc/*.nc'.format(cachedir)) all_cache = [] for i in cacheareasubfiles: all_cache.append(i.split('/')[-1]) for i in cacheareafiles: all_cache.append(i.split('/')[-1]) allcached = [] for i in allcachedfids: allcached.append(i.split('/')[-1]) all_cache.sort() #print(all_cache) allcached.sort() #print(allcached) self.assertEqual(all_cache,allcached) # remove from cache sl_cache = slCacheManager() sl_cache._clear_cache() def test_02_read_posix(self): self.f = Dataset('./testnc.nc', 'r') v = self.f.getVariable('var') data = v[:] self.assertTrue(data[0,0,0,0]) self.f.close() def test_05_read_s3(self): self.f = Dataset('s3://test/databucket/testnc.nc', 'r') v = self.f.getVariable('var') data = v[:] self.assertTrue(data[0,0,0,0]) self.f.close() def test_03_remove_posix(self): os.remove('./testnc.nc') subfiles = glob('./testnc/*.nc') for f in subfiles: os.remove(f) os.rmdir('testnc') sl_cache = slCacheManager() sl_cache._clear_cache() def test_06_remove_s3(self): sl_cache = slCacheManager() sl_config = slConfig() slDB = slCacheDB() #print(slDB.get_all_fids()) cache_loc = sl_config['cache']['location'] conn_man = slConnectionManager(sl_config) conn = conn_man.open("s3://test") sl_cache._clear_cache() s3 = conn.get() subfiles = s3.list_objects(Bucket='databucket',Prefix='testnc/')['Contents'] for sf in subfiles: s3.delete_object(Bucket='databucket',Key=sf['Key']) s3.delete_object(Bucket='databucket',Key='testnc.nc') s3.delete_bucket(Bucket='databucket') def test_07_write_none_cfa(self): self.f = Dataset('s3://test/databucket/testnc_noncfa.nc', 'w', format='NETCDF4') self.f.test = 'Created for SemSL tests' sl_config = slConfig() dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T',)) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z',)) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y',)) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X',)) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True) self.var.units = 'test unit' np.random.seed(0) self.var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) self.f.close() slDB = slCacheDB() allcachedfids = slDB.get_all_fids() cachedir = sl_config['cache']['location'] cacheareafiles = glob('{}/*.nc'.format(cachedir)) cacheareasubfiles = glob('{}/testnc_noncfa/*.nc'.format(cachedir)) all_cache = [] for i in cacheareasubfiles: all_cache.append(i.split('/')[-1]) for i in cacheareafiles: all_cache.append(i.split('/')[-1]) allcached = [] for i in allcachedfids: allcached.append(i.split('/')[-1]) all_cache.sort() # print(all_cache) allcached.sort() # print(allcached) self.assertEqual(all_cache, allcached) # remove from cache sl_cache = slCacheManager() sl_cache._clear_cache() def test_08_read_none_cfa(self): self.f = Dataset('s3://test/databucket/testnc_noncfa.nc', 'r') v = self.f.variables['var'] data = v[:] self.assertTrue(data[0,0,0,0]) self.f.close() def test_09_remove_none_cfa(self): sl_cache = slCacheManager() sl_config = slConfig() slDB = slCacheDB() #print(slDB.get_all_fids()) cache_loc = sl_config['cache']['location'] conn_man = slConnectionManager(sl_config) conn = conn_man.open("s3://test") sl_cache._clear_cache() s3 = conn.get() s3.delete_object(Bucket='databucket',Key='testnc_noncfa.nc') s3.delete_bucket(Bucket='databucket') def test_10_posix_chunk_write(self): self.f = Dataset('./testnc.nc','w') self.f.test = 'Created for SemSL tests' dim1 = self.f.createDimension('T', DIMSIZE) dim1d = self.f.createVariable('T', 'i4', ('T',)) dim1d[:] = range(DIMSIZE) dim2 = self.f.createDimension('Z', DIMSIZE) dim2d = self.f.createVariable('Z', 'i4', ('Z',)) dim2d[:] = range(DIMSIZE) dim3 = self.f.createDimension('Y', DIMSIZE) dim3d = self.f.createVariable('Y', 'i4', ('Y',)) dim3d[:] = range(DIMSIZE) dim4 = self.f.createDimension('X', DIMSIZE) dim4d = self.f.createVariable('X', 'i4', ('X',)) dim4d[:] = range(DIMSIZE) dim1d.axis = "T" dim2d.axis = "Z" dim3d.axis = "Y" dim4d.axis = "X" self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), chunksizes=(5,5,5,5)) np.random.seed(0) self.var[:] = np.random.rand(DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE) self.var.setncattr('units', 'test unit') self.f.close() # check that the subfile has the right chunking f = Dataset('./testnc/testnc_var_[0].nc') v = f.variables['var'] self.assertEqual([5,5,5,5],v.chunking()) f.close() def test_11_posix_chunk_remove(self): os.remove('./testnc.nc') subfiles = glob('./testnc/*.nc') for f in subfiles: os.remove(f) os.rmdir('testnc') sl_cache = slCacheManager() sl_cache._clear_cache()