def read(self, read_len=None):
if self.__for_write:
raise PermissionsException('File %s is openned for write!'%self.__file_path)
if self.__closed:
raise ClosedFileException('closed file!')
self.seek(0)
try:
ret_data = ''
while True:
if not self.__cur_data_block:
if self.__seek is None:
break
self.__cur_data_block, self.__seek, _ = self.__transaction.get_data_block(self.__seek)
if not self.__cur_data_block:
break
data = self.__cur_data_block.read(read_len)
if data:
ret_data += data
if read_len and len(ret_data) >= read_len:
break
self.__cur_data_block.close()
self.__cur_data_block = None
except Exception, err:
self.__failed_transaction(err)
logger.traceback_debug()
raise err
def close(self):
if self.__closed:
return
try:
if self.__failed_flag:
return
if self.__for_write:
try:
if self.__unsync and self.__cur_data_block:
self.__send_data_block()
elif not self.__transaction_id:
self.TRANSACTIONS_MANAGER.save_empty_file(self.__file_path)
return
except Exception, err:
self.__failed_transaction(err)
logger.traceback_debug()
raise err
if self.__is_tmp_file:
status = Transaction.TS_FINISHED
else:
status = Transaction.TS_LOCAL_SAVED
if self.__transaction_id:
self.TRANSACTIONS_MANAGER.update_transaction_state(self.__transaction_id, status)
else:
def close(self):
if self.__closed:
return
try:
if self.__failed_flag:
return
if self.__for_write:
try:
if self.__unsync and self.__cur_data_block:
self.__send_data_block()
elif not self.__transaction_id:
self.TRANSACTIONS_MANAGER.save_empty_file(
self.__file_path)
return
except Exception, err:
self.__failed_transaction(err)
logger.traceback_debug()
raise err
if self.__is_tmp_file:
status = Transaction.TS_FINISHED
else:
status = Transaction.TS_LOCAL_SAVED
if self.__transaction_id:
self.TRANSACTIONS_MANAGER.update_transaction_state(
self.__transaction_id, status)
else:
def read(self, read_len=None):
if self.__for_write:
raise PermissionsException('File %s is openned for write!' %
self.__file_path)
if self.__closed:
raise ClosedFileException('closed file!')
self.seek(0)
try:
ret_data = ''
while True:
if not self.__cur_data_block:
if self.__seek is None:
break
self.__cur_data_block, self.__seek, _ = self.__transaction.get_data_block(
self.__seek)
if not self.__cur_data_block:
break
data = self.__cur_data_block.read(read_len)
if data:
ret_data += data
if read_len and len(ret_data) >= read_len:
break
self.__cur_data_block.close()
self.__cur_data_block = None
except Exception, err:
self.__failed_transaction(err)
logger.traceback_debug()
raise err
def put(self,
data_block,
key=None,
replica_count=DEFAULT_REPLICA_COUNT,
wait_writes_count=2,
allow_rewrite=True):
packet = FabnetPacketRequest(method='PutKeysInfo',
parameters={'key': key},
sync=True)
resp = self.fri_client.call_sync(self.fabnet_hostname, packet,
FRI_CLIENT_TIMEOUT)
if resp.ret_code != 0:
raise Exception('Key info error: %s' % resp.ret_message)
if not resp.ret_parameters.has_key('key_info'):
raise Exception(
'Invalid PutKeysInfo response! key_info is expected')
key_info = resp.ret_parameters['key_info']
key, node_addr = key_info
params = {'key':key, 'replica_count':replica_count, \
'wait_writes_count': wait_writes_count}
packet = FabnetPacketRequest(method='ClientPutData', parameters=params, \
binary_data=ChunkedBinaryData.prepare(data_block, FILE_ITER_BLOCK_SIZE), sync=True)
resp = self.fri_client.call_sync(node_addr, packet, FRI_CLIENT_TIMEOUT)
try:
if resp.ret_code != 0:
raise Exception('ClientPutData error: %s' % resp.ret_message)
if not resp.ret_parameters.has_key('key'):
raise Exception(
'put data block error: no data key found in response message "%s"'
% resp)
primary_key = resp.ret_parameters['key']
checksum = resp.ret_parameters['checksum']
if isinstance(data_block, DataBlock):
db_checksum = data_block.checksum()
else:
db_checksum = hashlib.sha1(data_block).hexdigest()
if checksum != db_checksum:
raise Exception('Invalid data block checksum!')
except Exception, err:
logger.error('[put] %s' % err)
logger.traceback_debug()
if not allow_rewrite:
self.remove(key, replica_count)
raise err
def run(self):
while True:
job = self.queue.get()
try:
if job == QUIT_JOB or self.stop_flag.is_set():
break
db_key, replica_count = job
self.fabnet_gateway.remove(db_key, replica_count)
except Exception, err:
logger.error('DeleteWorker error: %s'%err)
logger.traceback_debug()
finally:
def run(self):
while True:
job = self.queue.get()
try:
if job == QUIT_JOB or self.stop_flag.is_set():
break
db_key, replica_count = job
self.fabnet_gateway.remove(db_key, replica_count)
except Exception, err:
logger.error('DeleteWorker error: %s' % err)
logger.traceback_debug()
finally:
def run(self):
while True:
job = self.queue.get()
transaction = None
data_block = None
key = None
try:
if job == QUIT_JOB or self.stop_flag.is_set():
break
transaction, seek = job
data_block, _, _ = transaction.get_data_block(seek)
if not data_block.exists():
raise Exception(
'Data block %s does not found at local cache!' %
data_block.get_name())
try:
key = self.fabnet_gateway.put(data_block, replica_count=transaction.get_replica_count(), \
allow_rewrite=False)
except Exception, err:
logger.error('Put data block error: %s' % err)
logger.error('Cant put data block from file %s. Wait %s seconds and try again...'%\
(transaction.get_file_path(), FG_ERROR_TIMEOUT))
time.sleep(FG_ERROR_TIMEOUT)
data_block.reopen()
self.queue.put(job)
continue
data_block.close()
self.transactions_manager.update_transaction(
transaction.get_id(),
seek,
is_failed=False,
foreign_name=key)
except Exception, err:
events_provider.critical('PutWorker',
'%s failed: %s' % (transaction, err))
logger.traceback_debug()
try:
if transaction:
self.transactions_manager.update_transaction(transaction.get_id(), seek, \
is_failed=True)
except Exception, err:
logger.error('[PutWorker.__on_error] %s' % err)
logger.traceback_debug()
def run(self):
while True:
out_streem = data = None
job = self.queue.get()
data_block = None
transaction = None
seek = None
try:
if job == QUIT_JOB or self.stop_flag.is_set():
break
transaction, seek = job
data_block, _, foreign_name = transaction.get_data_block(
seek, noclone=False)
if not foreign_name:
raise Exception('foreign name does not found for seek=%s' %
seek)
if transaction.is_failed():
logger.debug(
'Transaction {%s} is failed! Skipping data block downloading...'
% transaction.get_id())
data_block.remove()
continue
self.fabnet_gateway.get(foreign_name,
transaction.get_replica_count(),
data_block)
data_block.close()
self.transactions_manager.update_transaction(transaction.get_id(), seek, \
is_failed=False, foreign_name=data_block.get_name())
except Exception, err:
events_provider.error('GetWorker',
'%s failed: %s' % (transaction, err))
logger.traceback_debug()
try:
if transaction and data_block:
self.transactions_manager.update_transaction(transaction.get_id(), seek, \
is_failed=True, foreign_name=data_block.get_name())
data_block.remove()
except Exception, err:
logger.error('[GetWorker.__on_error] %s' % err)
logger.traceback_debug()
def start_download_transaction(self, file_path):
file_md, item_id = self.__find_file_from_inprogress(file_path)
if (not file_md) or (not file_md.is_file()):
raise PathException('No file found at %s' % file_path)
transaction_id = item_id
transaction = Transaction(Transaction.TT_DOWNLOAD, file_path,
file_md.replica_count, transaction_id)
stored_transaction = False
try:
for chunk in file_md.chunks:
db_path = self.__db_cache.get_cache_path('%s.%s' %
(item_id, chunk.seek))
if os.path.exists(db_path):
data_block = self.new_data_block(item_id, chunk.seek,
chunk.size)
if data_block.full():
transaction.append_data_block(chunk.seek, \
chunk.size, data_block, chunk.key, no_transfer=True)
continue
else:
logger.error('Removing corrupted data block: %s' %
data_block.get_name())
data_block.remove()
if file_md.is_local:
raise NoLocalFileFound('No local chunk found for file %s (%s.%s)'%\
(file_md.name, item_id, chunk.seek))
if not stored_transaction:
self.__transactions[transaction_id] = transaction
self.__tr_log_start_transaction(transaction)
stored_transaction = True
data_block = self.new_data_block(item_id, chunk.seek,
chunk.size)
self.transfer_data_block(transaction_id, chunk.seek,
chunk.size, data_block, chunk.key)
except Exception, err:
logger.traceback_debug()
if stored_transaction:
self.update_transaction_state(transaction_id,
Transaction.TS_FAILED)
raise err
def write(self, data):
if not self.__for_write:
raise PermissionsException('File %s is openned for read!' %
self.__file_path)
if self.__closed:
raise ClosedFileException('closed file!')
if not data:
return
try:
if not self.__transaction_id:
self.__transaction_id = self.TRANSACTIONS_MANAGER.start_upload_transaction(self.__file_path, \
is_local=self.__is_tmp_file)
if self.__cur_data_block is None:
self.__cur_data_block = self.TRANSACTIONS_MANAGER.new_data_block(
self.__transaction_id, self.__seek)
data_len = len(data)
if self.__is_tmp_file:
rest = 0
else:
rest = self.__cur_db_seek + data_len - MAX_DATA_BLOCK_SIZE
if rest > 0:
rest_data = data[data_len - rest:]
data = data[:data_len - rest]
else:
rest_data = ''
self.__cur_data_block.write(data)
self.__cur_db_seek += len(data)
self.__unsync = True
if rest_data:
self.__send_data_block()
self.write(rest_data)
#logger.debug('write %s bytes to %s ...'%(len(data), self.__file_path))
except Exception, err:
self.__failed_transaction(err)
logger.traceback_debug()
raise err
def write(self, data):
if not self.__for_write:
raise PermissionsException('File %s is openned for read!'%self.__file_path)
if self.__closed:
raise ClosedFileException('closed file!')
if not data:
return
try:
if not self.__transaction_id:
self.__transaction_id = self.TRANSACTIONS_MANAGER.start_upload_transaction(self.__file_path, \
is_local=self.__is_tmp_file)
if self.__cur_data_block is None:
self.__cur_data_block = self.TRANSACTIONS_MANAGER.new_data_block(self.__transaction_id, self.__seek)
data_len = len(data)
if self.__is_tmp_file:
rest = 0
else:
rest = self.__cur_db_seek + data_len - MAX_DATA_BLOCK_SIZE
if rest > 0:
rest_data = data[data_len-rest:]
data = data[:data_len-rest]
else:
rest_data = ''
self.__cur_data_block.write(data)
self.__cur_db_seek += len(data)
self.__unsync = True
if rest_data:
self.__send_data_block()
self.write(rest_data)
#logger.debug('write %s bytes to %s ...'%(len(data), self.__file_path))
except Exception, err:
self.__failed_transaction(err)
logger.traceback_debug()
raise err
def run(self):
while True:
job = self.queue.get()
transaction = None
data_block = None
key = None
try:
if job == QUIT_JOB or self.stop_flag.is_set():
break
transaction, seek = job
data_block,_,_ = transaction.get_data_block(seek)
if not data_block.exists():
raise Exception('Data block %s does not found at local cache!'%data_block.get_name())
try:
key = self.fabnet_gateway.put(data_block, replica_count=transaction.get_replica_count(), \
allow_rewrite=False)
except Exception, err:
logger.error('Put data block error: %s'%err)
logger.error('Cant put data block from file %s. Wait %s seconds and try again...'%\
(transaction.get_file_path(), FG_ERROR_TIMEOUT))
time.sleep(FG_ERROR_TIMEOUT)
data_block.reopen()
self.queue.put(job)
continue
data_block.close()
self.transactions_manager.update_transaction(transaction.get_id(), seek, is_failed=False, foreign_name=key)
except Exception, err:
events_provider.critical('PutWorker', '%s failed: %s'%(transaction, err))
logger.traceback_debug()
try:
if transaction:
self.transactions_manager.update_transaction(transaction.get_id(), seek, \
is_failed=True)
except Exception, err:
logger.error('[PutWorker.__on_error] %s'%err)
logger.traceback_debug()
def put(self, data_block, key=None, replica_count=DEFAULT_REPLICA_COUNT, wait_writes_count=2, allow_rewrite=True):
packet = FabnetPacketRequest(method='PutKeysInfo', parameters={'key': key}, sync=True)
resp = self.fri_client.call_sync(self.fabnet_hostname, packet, FRI_CLIENT_TIMEOUT)
if resp.ret_code != 0:
raise Exception('Key info error: %s'%resp.ret_message)
if not resp.ret_parameters.has_key('key_info'):
raise Exception('Invalid PutKeysInfo response! key_info is expected')
key_info = resp.ret_parameters['key_info']
key, node_addr = key_info
params = {'key':key, 'replica_count':replica_count, \
'wait_writes_count': wait_writes_count}
packet = FabnetPacketRequest(method='ClientPutData', parameters=params, \
binary_data=ChunkedBinaryData.prepare(data_block, FILE_ITER_BLOCK_SIZE), sync=True)
resp = self.fri_client.call_sync(node_addr, packet, FRI_CLIENT_TIMEOUT)
try:
if resp.ret_code != 0:
raise Exception('ClientPutData error: %s'%resp.ret_message)
if not resp.ret_parameters.has_key('key'):
raise Exception('put data block error: no data key found in response message "%s"'%resp)
primary_key = resp.ret_parameters['key']
checksum = resp.ret_parameters['checksum']
if isinstance(data_block, DataBlock):
db_checksum = data_block.checksum()
else:
db_checksum = hashlib.sha1(data_block).hexdigest()
if checksum != db_checksum:
raise Exception('Invalid data block checksum!')
except Exception, err:
logger.error('[put] %s'%err)
logger.traceback_debug()
if not allow_rewrite:
self.remove(key, replica_count)
raise err
def run(self):
while True:
out_streem = data = None
job = self.queue.get()
data_block = None
transaction = None
seek = None
try:
if job == QUIT_JOB or self.stop_flag.is_set():
break
transaction, seek = job
data_block,_,foreign_name = transaction.get_data_block(seek, noclone=False)
if not foreign_name:
raise Exception('foreign name does not found for seek=%s'%seek)
if transaction.is_failed():
logger.debug('Transaction {%s} is failed! Skipping data block downloading...'%transaction.get_id())
data_block.remove()
continue
self.fabnet_gateway.get(foreign_name, transaction.get_replica_count(), data_block)
data_block.close()
self.transactions_manager.update_transaction(transaction.get_id(), seek, \
is_failed=False, foreign_name=data_block.get_name())
except Exception, err:
events_provider.error('GetWorker','%s failed: %s'%(transaction, err))
logger.traceback_debug()
try:
if transaction and data_block:
self.transactions_manager.update_transaction(transaction.get_id(), seek, \
is_failed=True, foreign_name=data_block.get_name())
data_block.remove()
except Exception, err:
logger.error('[GetWorker.__on_error] %s'%err)
logger.traceback_debug()
def start_download_transaction(self, file_path):
file_md, item_id = self.__find_file_from_inprogress(file_path)
if (not file_md) or (not file_md.is_file()):
raise PathException('No file found at %s'%file_path)
transaction_id = item_id
transaction = Transaction(Transaction.TT_DOWNLOAD, file_path, file_md.replica_count, transaction_id)
stored_transaction = False
try:
for chunk in file_md.chunks:
db_path = self.__db_cache.get_cache_path('%s.%s'%(item_id, chunk.seek))
if os.path.exists(db_path):
data_block = self.new_data_block(item_id, chunk.seek, chunk.size)
if data_block.full():
transaction.append_data_block(chunk.seek, \
chunk.size, data_block, chunk.key, no_transfer=True)
continue
else:
logger.error('Removing corrupted data block: %s'% data_block.get_name())
data_block.remove()
if file_md.is_local:
raise NoLocalFileFound('No local chunk found for file %s (%s.%s)'%\
(file_md.name, item_id, chunk.seek))
if not stored_transaction:
self.__transactions[transaction_id] = transaction
self.__tr_log_start_transaction(transaction)
stored_transaction = True
data_block = self.new_data_block(item_id, chunk.seek, chunk.size)
self.transfer_data_block(transaction_id, chunk.seek, chunk.size, data_block, chunk.key)
except Exception, err:
logger.traceback_debug()
if stored_transaction:
self.update_transaction_state(transaction_id, Transaction.TS_FAILED)
raise err