def __load_md_db(self, md_file_path): self.db = anydbm.open(md_file_path, 'c') self.__last_item_id = long(self.__get_db_val('last_item_id', 0)) self.__last_journal_rec_id = long(self.__get_db_val('last_journal_rec_id', 0)) if self.__journal: j_key = self.__get_db_val('journal_key', None) if j_key != self.__journal.get_journal_key(): logger.info('Invalid journal key in metadata database! Recreating it...') self.db.close() self.__remove_md_file(md_file_path) self.db = anydbm.open(md_file_path, 'c') self.db['journal_key'] = self.__journal.get_journal_key() self.__last_item_id = 0 self.__last_journal_rec_id = 0 try: self.__init_from_journal(self.__last_journal_rec_id) except NimbusException, err: logger.error('Metadata was not restored from journal! Details: %s'%err) logger.info('Trying restoring full journal records...') self.db.close() self.__remove_md_file(md_file_path) self.db = anydbm.open(md_file_path, 'c') self.__init_from_journal(0)
def __load_md_db(self, md_file_path): self.db = anydbm.open(md_file_path, 'c') self.__last_item_id = long(self.__get_db_val('last_item_id', 0)) self.__last_journal_rec_id = long( self.__get_db_val('last_journal_rec_id', 0)) if self.__journal: j_key = self.__get_db_val('journal_key', None) if j_key != self.__journal.get_journal_key(): logger.info( 'Invalid journal key in metadata database! Recreating it...' ) self.db.close() self.__remove_md_file(md_file_path) self.db = anydbm.open(md_file_path, 'c') self.db['journal_key'] = self.__journal.get_journal_key() self.__last_item_id = 0 self.__last_journal_rec_id = 0 try: self.__init_from_journal(self.__last_journal_rec_id) except NimbusException, err: logger.error( 'Metadata was not restored from journal! Details: %s' % err) logger.info('Trying restoring full journal records...') self.db.close() self.__remove_md_file(md_file_path) self.db = anydbm.open(md_file_path, 'c') self.__init_from_journal(0)
def run(self): logger.info("thread is started") while not self.__stop_flag.is_set(): if self.__journal.status() in (Journal.JS_NOT_SYNC, Journal.JS_SYNC_FAILED): try: self.__journal._synchronize() except Exception, err: events_provider.error("journal", "journal synchronization is failed with message: %s" % err) for i in xrange(JOURNAL_SYNC_CHECK_TIME): if self.__stop_flag.is_set(): break time.sleep(1)
class JournalSyncThread(threading.Thread): def __init__(self, journal): threading.Thread.__init__(self) self.__journal = journal self.__stop_flag = threading.Event() self.setName('JournalSyncThread') def stop(self): self.__stop_flag.set() self.join() def run(self): logger.info('thread is started') while not self.__stop_flag.is_set(): if self.__journal.status() in (Journal.JS_NOT_SYNC, Journal.JS_SYNC_FAILED): try: self.__journal._synchronize() except Exception, err: events_provider.error( 'journal', 'journal synchronization is failed with message: %s' % err) for i in xrange(JOURNAL_SYNC_CHECK_TIME): if self.__stop_flag.is_set(): break time.sleep(1) logger.info('thread is stopped')
def run(self): logger.info('thread is started') while not self.__stop_flag.is_set(): if self.__journal.status() in (Journal.JS_NOT_SYNC, Journal.JS_SYNC_FAILED): try: self.__journal._synchronize() except Exception, err: events_provider.error( 'journal', 'journal synchronization is failed with message: %s' % err) for i in xrange(JOURNAL_SYNC_CHECK_TIME): if self.__stop_flag.is_set(): break time.sleep(1)
def __init_from_journal(self, start_rec_id): if self.__journal: logger.info('Restoring journal from ID=%s ...'%start_rec_id) for record_id, operation_type, item_md in self.__journal.iter(start_rec_id): try: if operation_type == Journal.OT_APPEND: self.append(None, item_md) self.__last_item_id = item_md.item_id elif operation_type == Journal.OT_UPDATE: self.update(item_md) elif operation_type == Journal.OT_REMOVE: try: item_md = self.__get_item_md(item_md) self.remove(item_md) except NotFoundException, err: logger.warning('Can not remove item with ID=%s, bcs it does not found!'%item_md) except NotEmptyException, err: logger.warning('Can not remove item with ID=%s, bcs it has children!'%item_md)
def __init_from_journal(self, start_rec_id): if self.__journal: logger.info('Restoring journal from ID=%s ...' % start_rec_id) for record_id, operation_type, item_md in self.__journal.iter( start_rec_id): try: if operation_type == Journal.OT_APPEND: self.append(None, item_md) self.__last_item_id = item_md.item_id elif operation_type == Journal.OT_UPDATE: self.update(item_md) elif operation_type == Journal.OT_REMOVE: try: item_md = self.__get_item_md(item_md) self.remove(item_md) except NotFoundException, err: logger.warning( 'Can not remove item with ID=%s, bcs it does not found!' % item_md) except NotEmptyException, err: logger.warning( 'Can not remove item with ID=%s, bcs it has children!' % item_md)
def info(self, event_provider, message): logger.info(message) self.emit(Event.ET_INFO, event_provider, message)
except NotFoundException, err: logger.warning( 'Can not remove item with ID=%s, bcs it does not found!' % item_md) except NotEmptyException, err: logger.warning( 'Can not remove item with ID=%s, bcs it has children!' % item_md) except AlreadyExistsException, err: logger.warning( 'Can not append/update item %s, bcs it is already exists!' % item_md) self.__last_journal_rec_id = self.__journal.get_last_id() logger.info( 'Metadata is restored from journal. Last journal record ID=%s' % self.__last_journal_rec_id) else: self.append( None, DirectoryMD(name=ROOT_NAME, item_id=0, parent_dir_id=0)) self.__valid = True def __hash(self, str_data): return zlib.adler32(str_data) def __set_raw_value(self, key, value): self.db[key.dump()] = value def __get_raw_value(self, key, default=None): try: ret = self.db[key.dump()]
self.__last_item_id = item_md.item_id elif operation_type == Journal.OT_UPDATE: self.update(item_md) elif operation_type == Journal.OT_REMOVE: try: item_md = self.__get_item_md(item_md) self.remove(item_md) except NotFoundException, err: logger.warning('Can not remove item with ID=%s, bcs it does not found!'%item_md) except NotEmptyException, err: logger.warning('Can not remove item with ID=%s, bcs it has children!'%item_md) except AlreadyExistsException, err: logger.warning('Can not append/update item %s, bcs it is already exists!'%item_md) self.__last_journal_rec_id = self.__journal.get_last_id() logger.info('Metadata is restored from journal. Last journal record ID=%s'%self.__last_journal_rec_id) else: self.append(None, DirectoryMD(name=ROOT_NAME, item_id=0, parent_dir_id=0)) self.__valid = True def __hash(self, str_data): return zlib.adler32(str_data) def __set_raw_value(self, key, value): self.db[key.dump()] = value def __get_raw_value(self, key, default=None): try: ret = self.db[key.dump()] return ret except KeyError, err: