def test_slConnectionManager():
# create config manager
sl_config = slConfig()
print("===== S3 =====")
# test S3
conn_man = slConnectionManager(sl_config)
conn1 = conn_man.open("s3://test")
conn2 = conn_man.open("s3://test")
print("Total conns: {}\nOpen conns: {}\n".format(
conn_man.total_connections("s3://test"),
conn_man.open_connections("s3://test")))
conn2.release()
print("Total conns: {}\nOpen conns: {}\n".format(
conn_man.total_connections("s3://test"),
conn_man.open_connections("s3://test")))
conn2 = conn_man.open("s3://test")
print("Total conns: {}\nOpen conns: {}\n".format(
conn_man.total_connections("s3://test"),
conn_man.open_connections("s3://test")))
conn3 = conn_man.open("s3://test")
print("Total conns: {}\nOpen conns: {}\n".format(
conn_man.total_connections("s3://test"),
conn_man.open_connections("s3://test")))
conn1.close()
conn2.close()
conn3.close()
print("Total conns: {}\nOpen conns: {}\n".format(
conn_man.total_connections("s3://test"),
conn_man.open_connections("s3://test")))
def setUp(self):
self.sl_cache = slCacheManager()
# add example file to db
self.FID_IN_CACHE = 's3://test/testbucket/{}'.format(FID_IN_CACHE)
self.FID_NOT_IN_CACHE = 's3://test/testbucket/{}'.format(
FID_NOT_IN_CACHE)
self.EXTRA_FID_NOT_IN_CACHE = 's3://test/testbucket/{}'.format(
EXTRA_FID_NOT_IN_CACHE)
self.sl_cache.DB.add_entry(self.FID_IN_CACHE, size=60 * 10**6)
self.sl_config = slConfig()
def cleanup():
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
# print(slDB.get_all_fids())
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://caringo")
sl_cache._clear_cache()
s3 = conn.get()
s3.delete_object(Bucket='mjones07', Key='testnc.nc')
def setUp(self):
self.sl_config = slConfig()
#get user home directory
user_home = os.environ["HOME"]
# add the config file name
s3_user_config_filename = user_home + "/" + ".sem-sl.json"
fp = open(s3_user_config_filename)
# deserialize from the JSON
self.json_config = json.load(fp)
def tearDown(self):
# remove test file
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
sl_cache._clear_cache()
s3 = conn.get()
s3.delete_object(Bucket='databucket', Key='testnc_methods.nc')
s3.delete_bucket(Bucket='databucket')
def test_09_remove_none_cfa(self):
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
#print(slDB.get_all_fids())
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
sl_cache._clear_cache()
s3 = conn.get()
s3.delete_object(Bucket='databucket',Key='testnc_noncfa.nc')
s3.delete_bucket(Bucket='databucket')
def test_04_s3_write(self):
self.f = Dataset('s3://test/databucket/testnc.nc', 'w')
self.f.test = 'Created for SemSL tests'
sl_config = slConfig()
dim1 = self.f.createDimension('T', DIMSIZE)
dim1d = self.f.createVariable('T', 'i4', ('T',))
dim1d[:] = range(DIMSIZE)
dim2 = self.f.createDimension('Z', DIMSIZE)
dim2d = self.f.createVariable('Z', 'i4', ('Z',))
dim2d[:] = range(DIMSIZE)
dim3 = self.f.createDimension('Y', DIMSIZE)
dim3d = self.f.createVariable('Y', 'i4', ('Y',))
dim3d[:] = range(DIMSIZE)
dim4 = self.f.createDimension('X', DIMSIZE)
dim4d = self.f.createVariable('X', 'i4', ('X',))
dim4d[:] = range(DIMSIZE)
dim1d.axis = "T"
dim2d.axis = "Z"
dim3d.axis = "Y"
dim4d.axis = "X"
self.var = self.f.createVariable(VARNAME, 'f8', ('T', 'Z', 'Y', 'X'), contiguous=True)
self.var.units = 'test unit'
np.random.seed(0)
self.var[:] = np.random.rand(DIMSIZE,DIMSIZE,DIMSIZE,DIMSIZE)
self.f.close()
# Check what is in the cacheDB and cache area
slDB = slCacheDB()
allcachedfids = slDB.get_all_fids()
cachedir = sl_config['cache']['location']
cacheareafiles = glob('{}/*.nc'.format(cachedir))
cacheareasubfiles = glob('{}/testnc/*.nc'.format(cachedir))
all_cache = []
for i in cacheareasubfiles:
all_cache.append(i.split('/')[-1])
for i in cacheareafiles:
all_cache.append(i.split('/')[-1])
allcached = []
for i in allcachedfids:
allcached.append(i.split('/')[-1])
all_cache.sort()
#print(all_cache)
allcached.sort()
#print(allcached)
self.assertEqual(all_cache,allcached)
# remove from cache
sl_cache = slCacheManager()
sl_cache._clear_cache()
def cleanup():
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
# print(slDB.get_all_fids())
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://caringo")
sl_cache._clear_cache()
s3 = conn.get()
subfiles = s3.list_objects(Bucket='mjones07', Prefix='testnc/')['Contents']
for sf in subfiles:
s3.delete_object(Bucket='mjones07', Key=sf['Key'])
s3.delete_object(Bucket='mjones07', Key='testnc.nc')
def test_06_remove_s3(self):
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
#print(slDB.get_all_fids())
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
sl_cache._clear_cache()
s3 = conn.get()
subfiles = s3.list_objects(Bucket='databucket',Prefix='testnc/')['Contents']
for sf in subfiles:
s3.delete_object(Bucket='databucket',Key=sf['Key'])
s3.delete_object(Bucket='databucket',Key='testnc.nc')
s3.delete_bucket(Bucket='databucket')
def setUp(self):
self.sl_cache = slCacheManager()
# add example file to db
self.FID_IN_CACHE = 's3://test/cachetest/{}'.format(FID_IN_CACHE)
self.FID_NOT_IN_CACHE = 's3://test/cachetest/{}'.format(
FID_NOT_IN_CACHE)
self.EXTRA_FID_NOT_IN_CACHE = 's3://test/cachetest/{}'.format(
EXTRA_FID_NOT_IN_CACHE)
self.sl_cache.DB.add_entry(self.FID_IN_CACHE, size=60 * 10**6)
self.sl_config = slConfig()
self.cache_loc = self.sl_config['cache']['location']
with open(
'{}/{}'.format(self.cache_loc,
self.FID_IN_CACHE.split('/')[-1]), 'w') as f:
f.write('testwrite')
with open(
'{}/{}'.format(self.cache_loc,
self.FID_NOT_IN_CACHE.split('/')[-1]),
'w') as f:
f.write('testwrite')
with open(
'{}/{}'.format(self.cache_loc,
self.EXTRA_FID_NOT_IN_CACHE.split('/')[-1]),
'w') as f:
f.write('testwrite')
conn_man = slConnectionManager(self.sl_config)
conn = conn_man.open("s3://test")
s3 = conn.get()
s3.create_bucket(Bucket='cachetest')
s3.upload_file(
'{}/{}'.format(self.cache_loc,
self.FID_IN_CACHE.split('/')[-1]), 'cachetest',
'{}'.format(self.FID_IN_CACHE.split('/')[-1]))
s3.upload_file(
'{}/{}'.format(self.cache_loc,
self.FID_NOT_IN_CACHE.split('/')[-1]), 'cachetest',
'{}'.format(self.FID_NOT_IN_CACHE.split('/')[-1]))
s3.upload_file(
'{}/{}'.format(self.cache_loc,
self.EXTRA_FID_NOT_IN_CACHE.split('/')[-1]),
'cachetest',
'{}'.format(self.EXTRA_FID_NOT_IN_CACHE.split('/')[-1]))
os.remove('{}/{}'.format(self.cache_loc,
self.FID_NOT_IN_CACHE.split('/')[-1]))
os.remove('{}/{}'.format(self.cache_loc,
self.EXTRA_FID_NOT_IN_CACHE.split('/')[-1]))
def tearDown(self):
# remove test file
sl_cache = slCacheManager()
sl_config = slConfig()
slDB = slCacheDB()
cache_loc = sl_config['cache']['location']
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
# construct list of subfiles
subfiles = glob('{}/testnc_methods/*.nc'.format(cache_loc))
subfiles = [
'testnc_methods/' + file.split('/')[-1] for file in subfiles
]
sl_cache._clear_cache()
s3 = conn.get()
s3.delete_object(Bucket='databucket', Key='testnc_methods.nc')
for file in subfiles:
#print(file)
s3.delete_object(Bucket='databucket', Key=file)
s3.delete_bucket(Bucket='databucket')
def test_changearrayvalues(self):
# create file in backend
self.f = Dataset('s3://test/databucket/testnc_varchange.nc', 'w')
self.f.setncattr('test', 'Created for SemSL tests')
dim1 = self.f.createDimension('T', DIMSIZE)
dim1d = self.f.createVariable('T', 'i4', ('T', ))
dim1d[:] = range(DIMSIZE)
dim2 = self.f.createDimension('Z', DIMSIZE)
dim2d = self.f.createVariable('Z', 'i4', ('Z', ))
dim2d[:] = range(DIMSIZE)
dim3 = self.f.createDimension('Y', DIMSIZE)
dim3d = self.f.createVariable('Y', 'i4', ('Y', ))
dim3d[:] = range(DIMSIZE)
dim4 = self.f.createDimension('X', DIMSIZE)
dim4d = self.f.createVariable('X', 'i4', ('X', ))
dim4d[:] = range(DIMSIZE)
dim1d.axis = "T"
dim2d.axis = "Z"
dim3d.axis = "Y"
dim4d.axis = "X"
self.var = self.f.createVariable('var',
'f8', ('T', 'Z', 'Y', 'X'),
contiguous=True)
self.var.setncattr('units', 'test unit')
self.var[:] = np.zeros((DIMSIZE, DIMSIZE, DIMSIZE, DIMSIZE))
self.f.close()
# remove the files in cache
sl_cache = slCacheManager()
sl_cache._clear_cache()
# Now reopen
f = Dataset('s3://test/databucket/testnc_varchange.nc', 'a')
var = f.variables['var']
self.assertEqual(var[0, 0, 0, 0], 0)
var[0, 0, 0, :] = np.ones((1, 1, 1, DIMSIZE))
self.assertEqual(var[0, 0, 0, 0], 1)
f.close()
# remove the files in cache
sl_cache = slCacheManager()
sl_cache._clear_cache()
# now reopen again and check the change
f = Dataset('s3://test/databucket/testnc_varchange.nc', 'r')
var = f.variables['var']
self.assertEqual(var[0, 0, 0, 0], 1)
f.close()
# cleanup
sl_config = slConfig()
slDB = slCacheDB()
conn_man = slConnectionManager(sl_config)
conn = conn_man.open("s3://test")
sl_cache = slCacheManager()
sl_cache._clear_cache()
s3 = conn.get()
subfiles = s3.list_objects(Bucket='databucket',
Prefix='testnc_varchange/')['Contents']
for sf in subfiles:
s3.delete_object(Bucket='databucket', Key=sf['Key'])
s3.delete_object(Bucket='databucket', Key='testnc_varchange.nc')
s3.delete_bucket(Bucket='databucket')
def setUp(self):
self.sl_cache = slCacheManager()
self.sl_config = slConfig()
self.cache_loc = self.sl_config['cache']['location']
with open('{}test'.format(self.cache_loc), 'w') as f:
f.write('test')
__copyright__ = "(C) 2012 Science and Technology Facilities Council"
__license__ = "BSD - see LICENSE file in top-level directory"
from SemSL._slConfigManager import slConfig
from SemSL import Backends
from SemSL._slExceptions import slConfigFileException
sl_config = slConfig()
def _get_alias(fid):
# get all host keys
hosts = sl_config['hosts']
host_keys = hosts.keys()
# Iterate through config to get all aliases
aliases = []
for host_name in iter(host_keys):
aliases.append(sl_config['hosts'][host_name]['alias'])
for alias in aliases:
if alias in fid:
return alias
# return None if alias isn't found in list
return None
def _get_hostname(fid):
alias = _get_alias(fid)