def cleanup():
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
# print(slDB.get_all_fids())
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://caringo")
sl_cache._clear_cache()
s3 = conn.get()
s3.delete_object(Bucket='mjones07', Key='testnc.nc')
def tearDown(self):
# remove test file
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
sl_cache._clear_cache()
s3 = conn.get()
s3.delete_object(Bucket='databucket', Key='testnc_methods.nc')
s3.delete_bucket(Bucket='databucket')
def test_09_remove_none_cfa(self):
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
#print(slDB.get_all_fids())
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
sl_cache._clear_cache()
s3 = conn.get()
s3.delete_object(Bucket='databucket',Key='testnc_noncfa.nc')
s3.delete_bucket(Bucket='databucket')
def test_04_s3_write(self):
self.f = Dataset('s3://test/databucket/testnc.nc', 'w')
self.f.test = 'Created for SemSL tests'
sl_config = slConfig()
dim1 = self.f.createDimension('T', DIMSIZE)
dim1d = self.f.createVariable('T', 'i4', ('T',))
dim1d[:] = range(DIMSIZE)
dim2 = self.f.createDimension('Z', DIMSIZE)
dim2d = self.f.createVariable('Z', 'i4', ('Z',))
dim2d[:] = range(DIMSIZE)
dim3 = self.f.createDimension('Y', DIMSIZE)
dim3d = self.f.createVariable('Y', 'i4', ('Y',))
dim3d[:] = range(DIMSIZE)
dim4 = self.f.createDimension('X', DIMSIZE)
dim4d = self.f.createVariable('X', 'i4', ('X',))
dim4d[:] = range(DIMSIZE)
dim1d.axis = "T"
dim2d.axis = "Z"
dim3d.axis = "Y"
dim4d.axis = "X"
self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True)
self.var.units = 'test unit'
np.random.seed(0)
self.var[:] = np.random.rand(DIMSIZE,DIMSIZE,DIMSIZE,DIMSIZE)
self.f.close()
# Check what is in the cacheDB and cache area
slDB = slCacheDB()
allcachedfids = slDB.get_all_fids()
cachedir = sl_config['cache']['location']
cacheareafiles = glob('{}/*.nc'.format(cachedir))
cacheareasubfiles = glob('{}/testnc/*.nc'.format(cachedir))
all_cache = []
for i in cacheareasubfiles:
all_cache.append(i.split('/')[-1])
for i in cacheareafiles:
all_cache.append(i.split('/')[-1])
allcached = []
for i in allcachedfids:
allcached.append(i.split('/')[-1])
all_cache.sort()
#print(all_cache)
allcached.sort()
#print(allcached)
self.assertEqual(all_cache,allcached)
# remove from cache
sl_cache = slCacheManager()
sl_cache._clear_cache()
def cleanup():
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
# print(slDB.get_all_fids())
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://caringo")
sl_cache._clear_cache()
s3 = conn.get()
subfiles = s3.list_objects(Bucket='mjones07', Prefix='testnc/')['Contents']
for sf in subfiles:
s3.delete_object(Bucket='mjones07', Key=sf['Key'])
s3.delete_object(Bucket='mjones07', Key='testnc.nc')
def test_06_remove_s3(self):
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
#print(slDB.get_all_fids())
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
sl_cache._clear_cache()
s3 = conn.get()
subfiles = s3.list_objects(Bucket='databucket',Prefix='testnc/')['Contents']
for sf in subfiles:
s3.delete_object(Bucket='databucket',Key=sf['Key'])
s3.delete_object(Bucket='databucket',Key='testnc.nc')
s3.delete_bucket(Bucket='databucket')
def tearDown(self):
# remove test file
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
# construct list of subfiles
subfiles = glob('{}/testnc_methods/*.nc'.format(cache_loc))
subfiles = [
'testnc_methods/' + file.split('/')[-1] for file in subfiles
]
sl_cache._clear_cache()
s3 = conn.get()
s3.delete_object(Bucket='databucket', Key='testnc_methods.nc')
for file in subfiles:
#print(file)
s3.delete_object(Bucket='databucket', Key=file)
s3.delete_bucket(Bucket='databucket')
def test_changearrayvalues(self):
# create file in backend
self.f = Dataset('s3://test/databucket/testnc_varchange.nc', 'w')
self.f.setncattr('test', 'Created for SemSL tests')
dim1 = self.f.createDimension('T', DIMSIZE)
dim1d = self.f.createVariable('T', 'i4', ('T', ))
dim1d[:] = range(DIMSIZE)
dim2 = self.f.createDimension('Z', DIMSIZE)
dim2d = self.f.createVariable('Z', 'i4', ('Z', ))
dim2d[:] = range(DIMSIZE)
dim3 = self.f.createDimension('Y', DIMSIZE)
dim3d = self.f.createVariable('Y', 'i4', ('Y', ))
dim3d[:] = range(DIMSIZE)
dim4 = self.f.createDimension('X', DIMSIZE)
dim4d = self.f.createVariable('X', 'i4', ('X', ))
dim4d[:] = range(DIMSIZE)
dim1d.axis = "T"
dim2d.axis = "Z"
dim3d.axis = "Y"
dim4d.axis = "X"
self.var = self.f.createVariable('var',
'f8', ('T', 'Z', 'Y', 'X'),
contiguous=True)
self.var.setncattr('units', 'test unit')
self.var[:] = np.zeros((DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE))
self.f.close()
# remove the files in cache
sl_cache = slCacheManager()
sl_cache._clear_cache()
# Now reopen
f = Dataset('s3://test/databucket/testnc_varchange.nc', 'a')
var = f.variables['var']
self.assertEqual(var[0, 0, 0, 0], 0)
var[0, 0, 0, :] = np.ones((1, 1, 1, DIMSIZE))
self.assertEqual(var[0, 0, 0, 0], 1)
f.close()
# remove the files in cache
sl_cache = slCacheManager()
sl_cache._clear_cache()
# now reopen again and check the change
f = Dataset('s3://test/databucket/testnc_varchange.nc', 'r')
var = f.variables['var']
self.assertEqual(var[0, 0, 0, 0], 1)
f.close()
# cleanup
sl_config = slConfig()
slDB = slCacheDB()
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
sl_cache = slCacheManager()
sl_cache._clear_cache()
s3 = conn.get()
subfiles = s3.list_objects(Bucket='databucket',
Prefix='testnc_varchange/')['Contents']
for sf in subfiles:
s3.delete_object(Bucket='databucket', Key=sf['Key'])
s3.delete_object(Bucket='databucket', Key='testnc_varchange.nc')
s3.delete_bucket(Bucket='databucket')