def __load_md_db(self, md_file_path):
self.db = anydbm.open(md_file_path, 'c')
self.__last_item_id = long(self.__get_db_val('last_item_id', 0))
self.__last_journal_rec_id = long(self.__get_db_val('last_journal_rec_id', 0))
if self.__journal:
j_key = self.__get_db_val('journal_key', None)
if j_key != self.__journal.get_journal_key():
logger.info('Invalid journal key in metadata database! Recreating it...')
self.db.close()
self.__remove_md_file(md_file_path)
self.db = anydbm.open(md_file_path, 'c')
self.db['journal_key'] = self.__journal.get_journal_key()
self.__last_item_id = 0
self.__last_journal_rec_id = 0
try:
self.__init_from_journal(self.__last_journal_rec_id)
except NimbusException, err:
logger.error('Metadata was not restored from journal! Details: %s'%err)
logger.info('Trying restoring full journal records...')
self.db.close()
self.__remove_md_file(md_file_path)
self.db = anydbm.open(md_file_path, 'c')
self.__init_from_journal(0)
def __load_md_db(self, md_file_path):
self.db = anydbm.open(md_file_path, 'c')
self.__last_item_id = long(self.__get_db_val('last_item_id', 0))
self.__last_journal_rec_id = long(
self.__get_db_val('last_journal_rec_id', 0))
if self.__journal:
j_key = self.__get_db_val('journal_key', None)
if j_key != self.__journal.get_journal_key():
logger.info(
'Invalid journal key in metadata database! Recreating it...'
)
self.db.close()
self.__remove_md_file(md_file_path)
self.db = anydbm.open(md_file_path, 'c')
self.db['journal_key'] = self.__journal.get_journal_key()
self.__last_item_id = 0
self.__last_journal_rec_id = 0
try:
self.__init_from_journal(self.__last_journal_rec_id)
except NimbusException, err:
logger.error(
'Metadata was not restored from journal! Details: %s' % err)
logger.info('Trying restoring full journal records...')
self.db.close()
self.__remove_md_file(md_file_path)
self.db = anydbm.open(md_file_path, 'c')
self.__init_from_journal(0)
def run(self):
logger.info("thread is started")
while not self.__stop_flag.is_set():
if self.__journal.status() in (Journal.JS_NOT_SYNC, Journal.JS_SYNC_FAILED):
try:
self.__journal._synchronize()
except Exception, err:
events_provider.error("journal", "journal synchronization is failed with message: %s" % err)
for i in xrange(JOURNAL_SYNC_CHECK_TIME):
if self.__stop_flag.is_set():
break
time.sleep(1)
class JournalSyncThread(threading.Thread):
def __init__(self, journal):
threading.Thread.__init__(self)
self.__journal = journal
self.__stop_flag = threading.Event()
self.setName('JournalSyncThread')
def stop(self):
self.__stop_flag.set()
self.join()
def run(self):
logger.info('thread is started')
while not self.__stop_flag.is_set():
if self.__journal.status() in (Journal.JS_NOT_SYNC,
Journal.JS_SYNC_FAILED):
try:
self.__journal._synchronize()
except Exception, err:
events_provider.error(
'journal',
'journal synchronization is failed with message: %s' %
err)
for i in xrange(JOURNAL_SYNC_CHECK_TIME):
if self.__stop_flag.is_set():
break
time.sleep(1)
logger.info('thread is stopped')
def run(self):
logger.info('thread is started')
while not self.__stop_flag.is_set():
if self.__journal.status() in (Journal.JS_NOT_SYNC,
Journal.JS_SYNC_FAILED):
try:
self.__journal._synchronize()
except Exception, err:
events_provider.error(
'journal',
'journal synchronization is failed with message: %s' %
err)
for i in xrange(JOURNAL_SYNC_CHECK_TIME):
if self.__stop_flag.is_set():
break
time.sleep(1)
def __init_from_journal(self, start_rec_id):
if self.__journal:
logger.info('Restoring journal from ID=%s ...'%start_rec_id)
for record_id, operation_type, item_md in self.__journal.iter(start_rec_id):
try:
if operation_type == Journal.OT_APPEND:
self.append(None, item_md)
self.__last_item_id = item_md.item_id
elif operation_type == Journal.OT_UPDATE:
self.update(item_md)
elif operation_type == Journal.OT_REMOVE:
try:
item_md = self.__get_item_md(item_md)
self.remove(item_md)
except NotFoundException, err:
logger.warning('Can not remove item with ID=%s, bcs it does not found!'%item_md)
except NotEmptyException, err:
logger.warning('Can not remove item with ID=%s, bcs it has children!'%item_md)
def __init_from_journal(self, start_rec_id):
if self.__journal:
logger.info('Restoring journal from ID=%s ...' % start_rec_id)
for record_id, operation_type, item_md in self.__journal.iter(
start_rec_id):
try:
if operation_type == Journal.OT_APPEND:
self.append(None, item_md)
self.__last_item_id = item_md.item_id
elif operation_type == Journal.OT_UPDATE:
self.update(item_md)
elif operation_type == Journal.OT_REMOVE:
try:
item_md = self.__get_item_md(item_md)
self.remove(item_md)
except NotFoundException, err:
logger.warning(
'Can not remove item with ID=%s, bcs it does not found!'
% item_md)
except NotEmptyException, err:
logger.warning(
'Can not remove item with ID=%s, bcs it has children!'
% item_md)
def info(self, event_provider, message):
logger.info(message)
self.emit(Event.ET_INFO, event_provider, message)
except NotFoundException, err:
logger.warning(
'Can not remove item with ID=%s, bcs it does not found!'
% item_md)
except NotEmptyException, err:
logger.warning(
'Can not remove item with ID=%s, bcs it has children!'
% item_md)
except AlreadyExistsException, err:
logger.warning(
'Can not append/update item %s, bcs it is already exists!'
% item_md)
self.__last_journal_rec_id = self.__journal.get_last_id()
logger.info(
'Metadata is restored from journal. Last journal record ID=%s'
% self.__last_journal_rec_id)
else:
self.append(
None, DirectoryMD(name=ROOT_NAME, item_id=0, parent_dir_id=0))
self.__valid = True
def __hash(self, str_data):
return zlib.adler32(str_data)
def __set_raw_value(self, key, value):
self.db[key.dump()] = value
def __get_raw_value(self, key, default=None):
try:
ret = self.db[key.dump()]
self.__last_item_id = item_md.item_id
elif operation_type == Journal.OT_UPDATE:
self.update(item_md)
elif operation_type == Journal.OT_REMOVE:
try:
item_md = self.__get_item_md(item_md)
self.remove(item_md)
except NotFoundException, err:
logger.warning('Can not remove item with ID=%s, bcs it does not found!'%item_md)
except NotEmptyException, err:
logger.warning('Can not remove item with ID=%s, bcs it has children!'%item_md)
except AlreadyExistsException, err:
logger.warning('Can not append/update item %s, bcs it is already exists!'%item_md)
self.__last_journal_rec_id = self.__journal.get_last_id()
logger.info('Metadata is restored from journal. Last journal record ID=%s'%self.__last_journal_rec_id)
else:
self.append(None, DirectoryMD(name=ROOT_NAME, item_id=0, parent_dir_id=0))
self.__valid = True
def __hash(self, str_data):
return zlib.adler32(str_data)
def __set_raw_value(self, key, value):
self.db[key.dump()] = value
def __get_raw_value(self, key, default=None):
try:
ret = self.db[key.dump()]
return ret
except KeyError, err: