def __init__(self, fabnet_host, security_provider, parallel_put_count=3, \
parallel_get_count=3, cache_dir='/tmp', cache_size=None):
if not isinstance(security_provider, AbstractSecurityManager):
raise Exception('Invalid security provider type!')
self.__parallel_put_count = parallel_put_count
self.__parallel_get_count = parallel_get_count
self.security_provider = security_provider
self.fabnet_gateway = FabnetGateway(fabnet_host, security_provider)
prikey = self.security_provider.get_prikey()
self.metadata_key = hashlib.sha1(str(prikey)).hexdigest()
self.metadata_f_name = 'md-%s.bin' % self.metadata_key
self.db_cache = DataBlockCache(cache_dir, cache_size,
self.metadata_key)
self.journal = None
self.metadata = None
self.transactions_manager = None
self.put_manager = None
self.get_manager = None
self.delete_manager = None
DataBlock.SECURITY_MANAGER = self.security_provider
DataBlock.LOCK_MANAGER = DBLocksManager()
self.journal = Journal(self.metadata_key, \
self.db_cache.get_static_cache_path('journal.bin'), self.fabnet_gateway)
def __init__(self, fabnet_host, security_provider, parallel_put_count=3, \
parallel_get_count=3, cache_dir='/tmp', cache_size=None):
if not isinstance(security_provider, AbstractSecurityManager):
raise Exception('Invalid security provider type!')
self.__parallel_put_count = parallel_put_count
self.__parallel_get_count = parallel_get_count
self.security_provider = security_provider
self.fabnet_gateway = FabnetGateway(fabnet_host, security_provider)
prikey = self.security_provider.get_prikey()
self.metadata_key = hashlib.sha1(str(prikey)).hexdigest()
self.metadata_f_name = 'md-%s.bin'%self.metadata_key
self.db_cache = DataBlockCache(cache_dir, cache_size, self.metadata_key)
self.journal = None
self.metadata = None
self.transactions_manager = None
self.put_manager = None
self.get_manager = None
self.delete_manager = None
DataBlock.SECURITY_MANAGER = self.security_provider
DataBlock.LOCK_MANAGER = DBLocksManager()
self.journal = Journal(self.metadata_key, \
self.db_cache.get_static_cache_path('journal.bin'), self.fabnet_gateway)
class Nibbler:
def __init__(self, fabnet_host, security_provider, parallel_put_count=3, \
parallel_get_count=3, cache_dir='/tmp', cache_size=None):
if not isinstance(security_provider, AbstractSecurityManager):
raise Exception('Invalid security provider type!')
self.__parallel_put_count = parallel_put_count
self.__parallel_get_count = parallel_get_count
self.security_provider = security_provider
self.fabnet_gateway = FabnetGateway(fabnet_host, security_provider)
prikey = self.security_provider.get_prikey()
self.metadata_key = hashlib.sha1(str(prikey)).hexdigest()
self.metadata_f_name = 'md-%s.bin' % self.metadata_key
self.db_cache = DataBlockCache(cache_dir, cache_size,
self.metadata_key)
self.journal = None
self.metadata = None
self.transactions_manager = None
self.put_manager = None
self.get_manager = None
self.delete_manager = None
DataBlock.SECURITY_MANAGER = self.security_provider
DataBlock.LOCK_MANAGER = DBLocksManager()
self.journal = Journal(self.metadata_key, \
self.db_cache.get_static_cache_path('journal.bin'), self.fabnet_gateway)
def get_security_provider(self):
return self.security_provider
def start(self):
self.fabnet_gateway.init_socket_processor()
if not self.journal.foreign_exists():
raise NoJournalFoundException('No journal for key = %s' %
self.metadata_key)
self.metadata = MetadataFile(
self.db_cache.get_static_cache_path(self.metadata_f_name),
self.journal)
self.transactions_manager = TransactionsManager(
self.metadata, self.db_cache, user_id=self.metadata_key)
SmartFileObject.setup_transaction_manager(self.transactions_manager)
self.put_manager = WorkersManager(PutWorker, self.fabnet_gateway, \
self.transactions_manager, self.__parallel_put_count)
self.get_manager = WorkersManager(GetWorker, self.fabnet_gateway, \
self.transactions_manager, self.__parallel_get_count)
self.delete_manager = WorkersManager(DeleteWorker, self.fabnet_gateway, \
self.transactions_manager, 2)
self.put_manager.start()
self.get_manager.start()
self.delete_manager.start()
def is_registered(self):
self.fabnet_gateway.init_socket_processor()
return self.journal.foreign_exists()
def register_user(self):
self.fabnet_gateway.init_socket_processor()
self.journal.init()
if self.metadata:
self.metadata.close()
self.metadata = MetadataFile(
self.db_cache.get_static_cache_path(self.metadata_f_name),
self.journal)
def on_error(self, error_msg):
pass
def stop(self):
self.fabnet_gateway.force_close_all_connections()
if self.put_manager:
self.put_manager.stop()
if self.get_manager:
self.get_manager.stop()
if self.delete_manager:
self.delete_manager.stop()
if self.metadata:
self.metadata.close()
if self.journal:
self.journal.close()
if self.transactions_manager:
self.transactions_manager.close()
self.db_cache.stop()
def __make_item_fs(self, item_md):
return FSItem(item_md.name, item_md.is_dir(), item_md.create_date,
item_md.last_modify_date, item_md.size)
def find(self, path):
path = to_nimbus_path(path)
path_obj = self.transactions_manager.find_inprogress_file(path)
if not path_obj:
try:
path_obj = self.metadata.find(path)
except PathException, err:
#logger.debug('[get_resource] %s'%str(err))
return
if path_obj:
return self.__make_item_fs(path_obj)
def test00_base(self):
ks = FileBasedSecurityManager(CLIENT_KS_PATH, PASSWD)
DataBlock.SECURITY_MANAGER = ks
DataBlock.LOCK_MANAGER = DBLocksManager()
remove_dir(tmp('smart_file_test'))
os.makedirs(tmp('smart_file_test/dynamic_cache'))
os.makedirs(tmp('smart_file_test/static_cache'))
db_cache = DataBlockCache(tmp('smart_file_test'), user_id=sha1('test').hexdigest())
md = MetadataFile(db_cache.get_static_cache_path('test_md.bin'))
tr_manager = None
try:
tr_manager = TransactionsManager(md, db_cache)
p_queue = tr_manager.get_upload_queue()
g_queue = tr_manager.get_download_queue()
SmartFileObject.setup_transaction_manager(tr_manager)
e_file = SmartFileObject('/empty_file', for_write=True)
e_file.close()
e_file = SmartFileObject('/empty_file')
data = e_file.read()
self.assertEqual(data, '')
e_file.close()
test_file = SmartFileObject('/test.file', for_write=True)
test_file.write('this is test message for one data block!')
test_file.close()
put_obj = p_queue.get(False)
transaction, seek = put_obj
self.assertEqual(seek, 0)
data_block, next_seek, _ = transaction.get_data_block(seek, noclone=True)
self.assertNotEqual(data_block, None)
self.assertEqual(next_seek, None)
data_block.close()
tr_manager.update_transaction(transaction.get_id(), seek, is_failed=False, foreign_name='%040x'%123456)
self.assertEqual(transaction.get_status(), Transaction.TS_FINISHED)
db_path = '%s.%s'%(transaction.get_id(), seek)
self.assertFalse(DataBlock.is_locked(db_cache.get_cache_path(db_path)))
test_file = SmartFileObject('/test.file')
data = test_file.read(4)
self.assertEqual(data, 'this')
self.assertTrue(DataBlock.is_locked(db_cache.get_cache_path(db_path)))
data = test_file.read()
self.assertEqual(data, ' is test message for one data block!')
test_file.close()
self.assertFalse(DataBlock.is_locked(db_cache.get_cache_path(db_path)))
with self.assertRaises(Empty):
get_obj = g_queue.get(False)
mgt = MockedGetThread(g_queue)
mgt.start()
open(db_cache.get_cache_path(db_path), 'w').write('invalid data') #failed local data block
test_file = SmartFileObject('/test.file')
data = test_file.read()
self.assertEqual(data, 'this is test message for one data block!')
test_file.close()
db_cache.clear_all()
test_file = SmartFileObject('/test.file')
data = test_file.read()
self.assertEqual(data, 'this is test message for one data block!')
test_file.close()
self.assertFalse(DataBlock.is_locked(db_cache.get_cache_path('%040x'%123456)))
finally:
if tr_manager:
g_queue.put((None,None))
tr_manager.close()
md.close()
db_cache.stop()
def test01_trans_manager(self):
db_cache = DataBlockCache(tmp('smart_file_test'), user_id=sha1('test').hexdigest())
open(db_cache.get_static_cache_path('transactions-share.log'), 'w').close()
try:
db_cache.clear_all()
md = MetadataFile(db_cache.get_static_cache_path('test_md.bin'))
tr_manager = TransactionsManager(md, db_cache, 2)
transaction_id = tr_manager.start_upload_transaction('/not_cached_test.file')
tr_manager.transfer_data_block(transaction_id, 0, 3500, \
DataBlock(db_cache.get_cache_path('fake_for_delete'), force_create=True))
self.assertTrue(os.path.exists(db_cache.get_cache_path('fake_for_delete')))
fake_for_delete_name = '%s.%s'%(transaction_id, 0)
transaction_id = tr_manager.start_upload_transaction('/my_second_test.file')
tr_manager.transfer_data_block(transaction_id, 0, 1000, DataBlock(db_cache.get_cache_path('fake')))
tr_manager.transfer_data_block(transaction_id, 1000, 2000, DataBlock(db_cache.get_cache_path('fake1')))
tr_manager.update_transaction_state(transaction_id, Transaction.TS_LOCAL_SAVED)
tr_manager.transfer_data_block(transaction_id, 0, 1000, DataBlock(db_cache.get_cache_path('fake2')), '%040x'%123456)
transaction = tr_manager.start_download_transaction('/test.file')
db,_,_ = transaction.get_data_block(0)
read_block_name = '%s.%s'%(transaction.get_id(), 0)
self.assertTrue(os.path.exists(db_cache.get_cache_path(read_block_name)))
db.close()
md.close()
tr_manager.close()
self.assertFalse(DataBlock.is_locked(db_cache.get_cache_path(read_block_name)))
md = MetadataFile(db_cache.get_static_cache_path('test_md.bin'))
tr_manager = TransactionsManager(md, db_cache, 2)
up_queue = tr_manager.get_upload_queue()
self.assertEqual(up_queue.qsize(), 1)
self.assertFalse(os.path.exists(db_cache.get_cache_path(fake_for_delete_name)))
self.assertFalse(os.path.exists(db_cache.get_cache_path(read_block_name)), read_block_name)
md.close()
tr_manager.close()
open(db_cache.get_static_cache_path('transactions-share.log'), 'w').close()
md = MetadataFile(db_cache.get_static_cache_path('test_md.bin'))
tr_manager = TransactionsManager(md, db_cache, 5)
transactions = []
for i in xrange(7):
transaction_id = tr_manager.start_upload_transaction('/%s_test.file'%i)
transactions.append(transaction_id)
cnt = 0
for i, (is_up, path, stat, size, progress) in enumerate(tr_manager.iterate_transactions()):
self.assertEqual(path, '/%s_test.file'%i)
cnt += 1
self.assertEqual(cnt, 7)
for tr_id in transactions:
tr_manager.update_transaction_state(tr_id, Transaction.TS_FAILED)
cnt = 0
tr_manager.start_upload_transaction('/7_test.file')
for i, (is_up, path, stat, size, progress) in enumerate(tr_manager.iterate_transactions()):
self.assertEqual(path, '/%s_test.file'%(i+3))
cnt += 1
self.assertEqual(cnt, 5)
for i in xrange(5):
tr_manager.start_upload_transaction('/%s_2_test.file'%i)
finally:
db_cache.stop()
class Nibbler:
def __init__(self, fabnet_host, security_provider, parallel_put_count=3, \
parallel_get_count=3, cache_dir='/tmp', cache_size=None):
if not isinstance(security_provider, AbstractSecurityManager):
raise Exception('Invalid security provider type!')
self.__parallel_put_count = parallel_put_count
self.__parallel_get_count = parallel_get_count
self.security_provider = security_provider
self.fabnet_gateway = FabnetGateway(fabnet_host, security_provider)
prikey = self.security_provider.get_prikey()
self.metadata_key = hashlib.sha1(str(prikey)).hexdigest()
self.metadata_f_name = 'md-%s.bin'%self.metadata_key
self.db_cache = DataBlockCache(cache_dir, cache_size, self.metadata_key)
self.journal = None
self.metadata = None
self.transactions_manager = None
self.put_manager = None
self.get_manager = None
self.delete_manager = None
DataBlock.SECURITY_MANAGER = self.security_provider
DataBlock.LOCK_MANAGER = DBLocksManager()
self.journal = Journal(self.metadata_key, \
self.db_cache.get_static_cache_path('journal.bin'), self.fabnet_gateway)
def get_security_provider(self):
return self.security_provider
def start(self):
self.fabnet_gateway.init_socket_processor()
if not self.journal.foreign_exists():
raise NoJournalFoundException('No journal for key = %s'%self.metadata_key)
self.metadata = MetadataFile(self.db_cache.get_static_cache_path(self.metadata_f_name), self.journal)
self.transactions_manager = TransactionsManager(self.metadata, self.db_cache, user_id=self.metadata_key)
SmartFileObject.setup_transaction_manager(self.transactions_manager)
self.put_manager = WorkersManager(PutWorker, self.fabnet_gateway, \
self.transactions_manager, self.__parallel_put_count)
self.get_manager = WorkersManager(GetWorker, self.fabnet_gateway, \
self.transactions_manager, self.__parallel_get_count)
self.delete_manager = WorkersManager(DeleteWorker, self.fabnet_gateway, \
self.transactions_manager, 2)
self.put_manager.start()
self.get_manager.start()
self.delete_manager.start()
def is_registered(self):
self.fabnet_gateway.init_socket_processor()
return self.journal.foreign_exists()
def register_user(self):
self.fabnet_gateway.init_socket_processor()
self.journal.init()
if self.metadata:
self.metadata.close()
self.metadata = MetadataFile(self.db_cache.get_static_cache_path(self.metadata_f_name), self.journal)
def on_error(self, error_msg):
pass
def stop(self):
self.fabnet_gateway.force_close_all_connections()
if self.put_manager:
self.put_manager.stop()
if self.get_manager:
self.get_manager.stop()
if self.delete_manager:
self.delete_manager.stop()
if self.metadata:
self.metadata.close()
if self.journal:
self.journal.close()
if self.transactions_manager:
self.transactions_manager.close()
self.db_cache.stop()
def __make_item_fs(self, item_md):
return FSItem(item_md.name, item_md.is_dir(), item_md.create_date, item_md.last_modify_date, item_md.size)
def find(self, path):
path = to_nimbus_path(path)
path_obj = self.transactions_manager.find_inprogress_file(path)
if not path_obj:
try:
path_obj = self.metadata.find(path)
except PathException, err:
#logger.debug('[get_resource] %s'%str(err))
return
if path_obj:
return self.__make_item_fs(path_obj)