def test_slConnectionManager():
# create config manager
sl_config = slConfig()
print("===== S3 =====")
# test S3
conn_man = slConnectionManager(sl_config)
conn1 = conn_man.open("s3://test")
conn2 = conn_man.open("s3://test")
print("Total conns: {}\nOpen conns: {}\n".format(
conn_man.total_connections("s3://test"),
conn_man.open_connections("s3://test")))
conn2.release()
print("Total conns: {}\nOpen conns: {}\n".format(
conn_man.total_connections("s3://test"),
conn_man.open_connections("s3://test")))
conn2 = conn_man.open("s3://test")
print("Total conns: {}\nOpen conns: {}\n".format(
conn_man.total_connections("s3://test"),
conn_man.open_connections("s3://test")))
conn3 = conn_man.open("s3://test")
print("Total conns: {}\nOpen conns: {}\n".format(
conn_man.total_connections("s3://test"),
conn_man.open_connections("s3://test")))
conn1.close()
conn2.close()
conn3.close()
print("Total conns: {}\nOpen conns: {}\n".format(
conn_man.total_connections("s3://test"),
conn_man.open_connections("s3://test")))
def cleanup():
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
# print(slDB.get_all_fids())
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://caringo")
sl_cache._clear_cache()
s3 = conn.get()
s3.delete_object(Bucket='mjones07', Key='testnc.nc')
def tearDown(self):
# remove test file
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
sl_cache._clear_cache()
s3 = conn.get()
s3.delete_object(Bucket='databucket', Key='testnc_methods.nc')
s3.delete_bucket(Bucket='databucket')
def test_09_remove_none_cfa(self):
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
#print(slDB.get_all_fids())
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
sl_cache._clear_cache()
s3 = conn.get()
s3.delete_object(Bucket='databucket',Key='testnc_noncfa.nc')
s3.delete_bucket(Bucket='databucket')
def cleanup():
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
# print(slDB.get_all_fids())
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://caringo")
sl_cache._clear_cache()
s3 = conn.get()
subfiles = s3.list_objects(Bucket='mjones07', Prefix='testnc/')['Contents']
for sf in subfiles:
s3.delete_object(Bucket='mjones07', Key=sf['Key'])
s3.delete_object(Bucket='mjones07', Key='testnc.nc')
def test_06_remove_s3(self):
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
#print(slDB.get_all_fids())
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
sl_cache._clear_cache()
s3 = conn.get()
subfiles = s3.list_objects(Bucket='databucket',Prefix='testnc/')['Contents']
for sf in subfiles:
s3.delete_object(Bucket='databucket',Key=sf['Key'])
s3.delete_object(Bucket='databucket',Key='testnc.nc')
s3.delete_bucket(Bucket='databucket')
def tearDown(self):
try:
self.sl_cache._clear_cache()
except:
print("Couldn't clear cache in tearDown?")
conn_man = slConnectionManager(self.sl_config)
conn = conn_man.open("s3://test")
s3 = conn.get()
s3.delete_object(Bucket='cachetest',
Key='{}'.format(self.FID_IN_CACHE.split('/')[-1]))
s3.delete_object(Bucket='cachetest',
Key='{}'.format(self.FID_NOT_IN_CACHE.split('/')[-1]))
s3.delete_object(Bucket='cachetest',
Key='{}'.format(
self.EXTRA_FID_NOT_IN_CACHE.split('/')[-1]))
s3.delete_bucket(Bucket='cachetest')
def setUp(self):
self.sl_cache = slCacheManager()
# add example file to db
self.FID_IN_CACHE = 's3://test/cachetest/{}'.format(FID_IN_CACHE)
self.FID_NOT_IN_CACHE = 's3://test/cachetest/{}'.format(
FID_NOT_IN_CACHE)
self.EXTRA_FID_NOT_IN_CACHE = 's3://test/cachetest/{}'.format(
EXTRA_FID_NOT_IN_CACHE)
self.sl_cache.DB.add_entry(self.FID_IN_CACHE, size=60 * 10**6)
self.sl_config = slConfig()
self.cache_loc = self.sl_config['cache']['location']
with open(
'{}/{}'.format(self.cache_loc,
self.FID_IN_CACHE.split('/')[-1]), 'w') as f:
f.write('testwrite')
with open(
'{}/{}'.format(self.cache_loc,
self.FID_NOT_IN_CACHE.split('/')[-1]),
'w') as f:
f.write('testwrite')
with open(
'{}/{}'.format(self.cache_loc,
self.EXTRA_FID_NOT_IN_CACHE.split('/')[-1]),
'w') as f:
f.write('testwrite')
conn_man = slConnectionManager(self.sl_config)
conn = conn_man.open("s3://test")
s3 = conn.get()
s3.create_bucket(Bucket='cachetest')
s3.upload_file(
'{}/{}'.format(self.cache_loc,
self.FID_IN_CACHE.split('/')[-1]), 'cachetest',
'{}'.format(self.FID_IN_CACHE.split('/')[-1]))
s3.upload_file(
'{}/{}'.format(self.cache_loc,
self.FID_NOT_IN_CACHE.split('/')[-1]), 'cachetest',
'{}'.format(self.FID_NOT_IN_CACHE.split('/')[-1]))
s3.upload_file(
'{}/{}'.format(self.cache_loc,
self.EXTRA_FID_NOT_IN_CACHE.split('/')[-1]),
'cachetest',
'{}'.format(self.EXTRA_FID_NOT_IN_CACHE.split('/')[-1]))
os.remove('{}/{}'.format(self.cache_loc,
self.FID_NOT_IN_CACHE.split('/')[-1]))
os.remove('{}/{}'.format(self.cache_loc,
self.EXTRA_FID_NOT_IN_CACHE.split('/')[-1]))
def tearDown(self):
# remove test file
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
# construct list of subfiles
subfiles = glob('{}/testnc_methods/*.nc'.format(cache_loc))
subfiles = [
'testnc_methods/' + file.split('/')[-1] for file in subfiles
]
sl_cache._clear_cache()
s3 = conn.get()
s3.delete_object(Bucket='databucket', Key='testnc_methods.nc')
for file in subfiles:
#print(file)
s3.delete_object(Bucket='databucket', Key=file)
s3.delete_bucket(Bucket='databucket')
def test_changearrayvalues(self):
# create file in backend
self.f = Dataset('s3://test/databucket/testnc_varchange.nc', 'w')
self.f.setncattr('test', 'Created for SemSL tests')
dim1 = self.f.createDimension('T', DIMSIZE)
dim1d = self.f.createVariable('T', 'i4', ('T', ))
dim1d[:] = range(DIMSIZE)
dim2 = self.f.createDimension('Z', DIMSIZE)
dim2d = self.f.createVariable('Z', 'i4', ('Z', ))
dim2d[:] = range(DIMSIZE)
dim3 = self.f.createDimension('Y', DIMSIZE)
dim3d = self.f.createVariable('Y', 'i4', ('Y', ))
dim3d[:] = range(DIMSIZE)
dim4 = self.f.createDimension('X', DIMSIZE)
dim4d = self.f.createVariable('X', 'i4', ('X', ))
dim4d[:] = range(DIMSIZE)
dim1d.axis = "T"
dim2d.axis = "Z"
dim3d.axis = "Y"
dim4d.axis = "X"
self.var = self.f.createVariable('var',
'f8', ('T', 'Z', 'Y', 'X'),
contiguous=True)
self.var.setncattr('units', 'test unit')
self.var[:] = np.zeros((DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE))
self.f.close()
# remove the files in cache
sl_cache = slCacheManager()
sl_cache._clear_cache()
# Now reopen
f = Dataset('s3://test/databucket/testnc_varchange.nc', 'a')
var = f.variables['var']
self.assertEqual(var[0, 0, 0, 0], 0)
var[0, 0, 0, :] = np.ones((1, 1, 1, DIMSIZE))
self.assertEqual(var[0, 0, 0, 0], 1)
f.close()
# remove the files in cache
sl_cache = slCacheManager()
sl_cache._clear_cache()
# now reopen again and check the change
f = Dataset('s3://test/databucket/testnc_varchange.nc', 'r')
var = f.variables['var']
self.assertEqual(var[0, 0, 0, 0], 1)
f.close()
# cleanup
sl_config = slConfig()
slDB = slCacheDB()
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
sl_cache = slCacheManager()
sl_cache._clear_cache()
s3 = conn.get()
subfiles = s3.list_objects(Bucket='databucket',
Prefix='testnc_varchange/')['Contents']
for sf in subfiles:
s3.delete_object(Bucket='databucket', Key=sf['Key'])
s3.delete_object(Bucket='databucket', Key='testnc_varchange.nc')
s3.delete_bucket(Bucket='databucket')
