def update_transaction_state(self, transaction_id, status):
transaction = self.__get_transaction(transaction_id)
if transaction.get_transaction_type() == Transaction.TT_UPLOAD:
if status == Transaction.TS_FINISHED:
try:
self.__save_metadata(transaction)
except Exception, err:
events_provider.critical(
"Metadata", "Can't update metadata! Details: %s" % err)
logger.traceback_info()
return self.update_transaction_state(
transaction_id, Transaction.TS_FAILED)
elif status == Transaction.TS_FAILED:
if transaction.get_status() != Transaction.TS_FAILED:
events_provider.critical("Transaction", "File %s was not uploaded to NimbusFS backend!"%\
transaction.get_file_path())
#should be removed data block from backend
replica_count = transaction.get_replica_count()
for _, _, data_block, foreign_name in transaction.iter_data_blocks(
):
self.__cancel_data_block_upload(foreign_name, data_block,
replica_count)
try:
self.__metadata.cancel_item_id_reserve(transaction_id)
except Exception, err:
logger.warning("Can't cancel item_id=%s reserve" %
transaction_id)
logger.traceback_info()
def __restore_db(self):
cdb = DataBlock(self.__path, actsize=True)
try:
while True:
data = cdb.read(BUF_LEN)
if not data:
break
self.write(data)
except Exception, err:
logger.warning('Data block %s does not restored with error: %s. Removing this data block...'%(self.get_name(), err))
self.remove()
raise err
def __restore_db(self):
cdb = DataBlock(self.__path, actsize=True)
try:
while True:
data = cdb.read(BUF_LEN)
if not data:
break
self.write(data)
except Exception, err:
logger.warning(
'Data block %s does not restored with error: %s. Removing this data block...'
% (self.get_name(), err))
self.remove()
raise err
def exec_openssl(cls, command, stdin=None, cwd=None):
'''Run openssl command. PKI_OPENSSL_BIN doesn't need to be specified'''
c = [os.environ.get('OPENSSL_EXEC', 'openssl')]
c.extend(command)
try:
proc = Subprocess(c, with_input=True)
stdout_value, stderr_value = proc.communicate(stdin)
out = stdout_value
if stderr_value:
out += '\n%s'%stderr_value
if proc.returncode != 0:
logger.warning('OpenSSL error: %s'%out)
except Exception, err:
logger.error('openssl call error! "%s" failed: %s'%(' '.join(c), err))
raise err
def __init_from_journal(self, start_rec_id):
if self.__journal:
logger.info('Restoring journal from ID=%s ...'%start_rec_id)
for record_id, operation_type, item_md in self.__journal.iter(start_rec_id):
try:
if operation_type == Journal.OT_APPEND:
self.append(None, item_md)
self.__last_item_id = item_md.item_id
elif operation_type == Journal.OT_UPDATE:
self.update(item_md)
elif operation_type == Journal.OT_REMOVE:
try:
item_md = self.__get_item_md(item_md)
self.remove(item_md)
except NotFoundException, err:
logger.warning('Can not remove item with ID=%s, bcs it does not found!'%item_md)
except NotEmptyException, err:
logger.warning('Can not remove item with ID=%s, bcs it has children!'%item_md)
def exec_openssl(cls, command, stdin=None, cwd=None):
'''Run openssl command. PKI_OPENSSL_BIN doesn't need to be specified'''
c = [os.environ.get('OPENSSL_EXEC', 'openssl')]
c.extend(command)
try:
proc = Subprocess(c, with_input=True)
stdout_value, stderr_value = proc.communicate(stdin)
out = stdout_value
if stderr_value:
out += '\n%s' % stderr_value
if proc.returncode != 0:
logger.warning('OpenSSL error: %s' % out)
except Exception, err:
logger.error('openssl call error! "%s" failed: %s' %
(' '.join(c), err))
raise err
def __init_from_journal(self, start_rec_id):
if self.__journal:
logger.info('Restoring journal from ID=%s ...' % start_rec_id)
for record_id, operation_type, item_md in self.__journal.iter(
start_rec_id):
try:
if operation_type == Journal.OT_APPEND:
self.append(None, item_md)
self.__last_item_id = item_md.item_id
elif operation_type == Journal.OT_UPDATE:
self.update(item_md)
elif operation_type == Journal.OT_REMOVE:
try:
item_md = self.__get_item_md(item_md)
self.remove(item_md)
except NotFoundException, err:
logger.warning(
'Can not remove item with ID=%s, bcs it does not found!'
% item_md)
except NotEmptyException, err:
logger.warning(
'Can not remove item with ID=%s, bcs it has children!'
% item_md)
def update_transaction_state(self, transaction_id, status):
transaction = self.__get_transaction(transaction_id)
if transaction.get_transaction_type() == Transaction.TT_UPLOAD:
if status == Transaction.TS_FINISHED:
try:
self.__save_metadata(transaction)
except Exception, err:
events_provider.critical("Metadata", "Can't update metadata! Details: %s"%err)
logger.traceback_info()
return self.update_transaction_state(transaction_id, Transaction.TS_FAILED)
elif status == Transaction.TS_FAILED:
if transaction.get_status() != Transaction.TS_FAILED:
events_provider.critical("Transaction", "File %s was not uploaded to NimbusFS backend!"%\
transaction.get_file_path())
#should be removed data block from backend
replica_count = transaction.get_replica_count()
for _,_,data_block, foreign_name in transaction.iter_data_blocks():
self.__cancel_data_block_upload(foreign_name, data_block, replica_count)
try:
self.__metadata.cancel_item_id_reserve(transaction_id)
except Exception, err:
logger.warning("Can't cancel item_id=%s reserve"%transaction_id)
logger.traceback_info()
def warning(self, event_provider, message):
logger.warning(message)
self.emit(Event.ET_WARNING, event_provider, message)
self.update(item_md)
elif operation_type == Journal.OT_REMOVE:
try:
item_md = self.__get_item_md(item_md)
self.remove(item_md)
except NotFoundException, err:
logger.warning(
'Can not remove item with ID=%s, bcs it does not found!'
% item_md)
except NotEmptyException, err:
logger.warning(
'Can not remove item with ID=%s, bcs it has children!'
% item_md)
except AlreadyExistsException, err:
logger.warning(
'Can not append/update item %s, bcs it is already exists!'
% item_md)
self.__last_journal_rec_id = self.__journal.get_last_id()
logger.info(
'Metadata is restored from journal. Last journal record ID=%s'
% self.__last_journal_rec_id)
else:
self.append(
None, DirectoryMD(name=ROOT_NAME, item_id=0, parent_dir_id=0))
self.__valid = True
def __hash(self, str_data):
return zlib.adler32(str_data)
def __set_raw_value(self, key, value):
try:
if operation_type == Journal.OT_APPEND:
self.append(None, item_md)
self.__last_item_id = item_md.item_id
elif operation_type == Journal.OT_UPDATE:
self.update(item_md)
elif operation_type == Journal.OT_REMOVE:
try:
item_md = self.__get_item_md(item_md)
self.remove(item_md)
except NotFoundException, err:
logger.warning('Can not remove item with ID=%s, bcs it does not found!'%item_md)
except NotEmptyException, err:
logger.warning('Can not remove item with ID=%s, bcs it has children!'%item_md)
except AlreadyExistsException, err:
logger.warning('Can not append/update item %s, bcs it is already exists!'%item_md)
self.__last_journal_rec_id = self.__journal.get_last_id()
logger.info('Metadata is restored from journal. Last journal record ID=%s'%self.__last_journal_rec_id)
else:
self.append(None, DirectoryMD(name=ROOT_NAME, item_id=0, parent_dir_id=0))
self.__valid = True
def __hash(self, str_data):
return zlib.adler32(str_data)
def __set_raw_value(self, key, value):
self.db[key.dump()] = value
def __get_raw_value(self, key, default=None):
try: