def __open_file(self, open_flags): self.__f_obj = open(self.__path, open_flags) if 'r+' in open_flags: if self.get_actual_size(): logger.debug('rewrite data block %s for appending...' % self.get_name()) self.__restore_db()
def _synchronize(self): try: logger.debug("synchronizing journal...") self.__journal.flush() j_data = DataBlock(self.__journal_path, actsize=True) is_send = self.__fabnet_gateway.put(j_data, key=self.__journal_key) if is_send: self.__is_sync = True self.__sync_failed = False except Exception, err: self.__sync_failed = True raise err
def _synchronize(self): try: logger.debug('synchronizing journal...') self.__journal.flush() j_data = DataBlock(self.__journal_path, actsize=True) is_send = self.__fabnet_gateway.put(j_data, key=self.__journal_key) if is_send: self.__is_sync = True self.__sync_failed = False except Exception, err: self.__sync_failed = True raise err
def transfer_data_block(self, transaction_id, seek, size, data_block, foreign_name=None): logger.debug('data block %s (seek=%s, size=%s) is ready for transfer'%(data_block.get_name(), seek, size)) transaction = self.__get_transaction(transaction_id) transaction.append_data_block(seek, size, data_block, foreign_name) if transaction.is_local(): return self.__tr_log_update(transaction_id, seek, size, data_block.get_name(), foreign_name) if transaction.is_uploading(): self.__put_queue.put((transaction, seek)) else: self.__get_queue.put((transaction, seek))
def __init__(self, file_path, for_write=False): self.__file_path = file_path self.__seek = 0 self.__cur_data_block = None self.__cur_db_seek = 0 self.__transaction_id = None #used for write() self.__transaction = None #used for read() self.__unsync = False self.__closed = False self.__for_write = for_write self.__failed_flag = False self.__is_tmp_file = self.__tmp_file() logger.debug('opening file %s for %s...'%(file_path, 'write' if for_write else 'read'))
def __parse_tr_log(self): def parse_int(val): if val in [None, 'None']: return None return int(val) def parse_str(val): if val in [None, 'None']: return None return val transactions = {} tr_log = open(self.__trlog_path, 'r') for line in tr_log.readlines(): if not line: break logger.debug('processing "%s"' % line.strip()) parts = line.split() transaction_id = int(parts[0]) r_type = parts[1] if r_type == 'ST': transaction = Transaction(int(parts[2]), base64.b64decode(parts[3]), int(parts[4]), transaction_id) transactions[transaction_id] = [transaction, {}] elif r_type == 'US': transactions[transaction_id][0].change_status(int(parts[2])) elif r_type == 'UT': seek = int(parts[2]) size = parse_int(parts[3]) local_name = parse_str(parts[4]) try: foreign_name = parse_str(parts[5]) except IndexError: foreign_name = None cur_vals = transactions[transaction_id][1].get( seek, [None, None, None]) if size: cur_vals[0] = size if local_name: cur_vals[1] = local_name if foreign_name: cur_vals[2] = foreign_name transactions[transaction_id][1][seek] = cur_vals tr_log.close() return sorted(transactions.values(), cmp=lambda x, y: cmp(x[0].get_start_datetime(), y[0]. get_start_datetime()))
def __init__(self, file_path, for_write=False): self.__file_path = file_path self.__seek = 0 self.__cur_data_block = None self.__cur_db_seek = 0 self.__transaction_id = None #used for write() self.__transaction = None #used for read() self.__unsync = False self.__closed = False self.__for_write = for_write self.__failed_flag = False self.__is_tmp_file = self.__tmp_file() logger.debug('opening file %s for %s...' % (file_path, 'write' if for_write else 'read'))
def iter(self, start_record_id=None): JLock.lock() try: j_data = DataBlock(self.__journal_path, actsize=True) buf = '' while True: if len(buf) < self.RECORD_STRUCT_SIZE: buf += j_data.read(1024) #logger.debug('J_ITER: buf=%s'%buf.encode('hex').upper()) if not buf: break #logger.debug('J_ITER: header=%s'%buf[:self.RECORD_STRUCT_SIZE].encode('hex').upper()) item_dump_len, operation_type, record_id = struct.unpack( self.RECORD_STRUCT, buf[:self.RECORD_STRUCT_SIZE]) #logger.debug('J_ITER: buf_len=%s, item_dump_len=%s, operation_type=%s, record_id=%s'%(len(buf), item_dump_len, operation_type, record_id)) if operation_type not in (self.OT_APPEND, self.OT_UPDATE, self.OT_REMOVE): #logger.debug('J_ITER: buf=%s'%buf.encode('hex').upper()) raise RuntimeError( 'Invalid journal!!! Unknown operation type: %s' % operation_type) if len(buf) < (self.RECORD_STRUCT_SIZE + item_dump_len): buf += j_data.read(1024) item_dump = buf[self. RECORD_STRUCT_SIZE:self.RECORD_STRUCT_SIZE + item_dump_len] remaining_len = BLOCK_SIZE - self.RECORD_STRUCT_SIZE - item_dump_len to_pad_len = remaining_len % BLOCK_SIZE #logger.debug('J_ITER: record=%s'%buf[:self.RECORD_STRUCT_SIZE+item_dump_len+to_pad_len].encode('hex').upper()) buf = buf[self.RECORD_STRUCT_SIZE + item_dump_len + to_pad_len:] self.__last_record_id = record_id if (start_record_id is None) or (record_id > start_record_id): if operation_type == self.OT_REMOVE: item_md = struct.unpack('<I', item_dump)[0] else: item_md = AbstractMetadataObject.load_md(item_dump) logger.debug( 'J_ITER: record_id=%s, operation_type=%s, item_md=%s' % (record_id, operation_type, item_md)) yield record_id, operation_type, item_md finally: JLock.unlock()
def run(self): while True: out_streem = data = None job = self.queue.get() data_block = None transaction = None seek = None try: if job == QUIT_JOB or self.stop_flag.is_set(): break transaction, seek = job data_block, _, foreign_name = transaction.get_data_block( seek, noclone=False) if not foreign_name: raise Exception('foreign name does not found for seek=%s' % seek) if transaction.is_failed(): logger.debug( 'Transaction {%s} is failed! Skipping data block downloading...' % transaction.get_id()) data_block.remove() continue self.fabnet_gateway.get(foreign_name, transaction.get_replica_count(), data_block) data_block.close() self.transactions_manager.update_transaction(transaction.get_id(), seek, \ is_failed=False, foreign_name=data_block.get_name()) except Exception, err: events_provider.error('GetWorker', '%s failed: %s' % (transaction, err)) logger.traceback_debug() try: if transaction and data_block: self.transactions_manager.update_transaction(transaction.get_id(), seek, \ is_failed=True, foreign_name=data_block.get_name()) data_block.remove() except Exception, err: logger.error('[GetWorker.__on_error] %s' % err) logger.traceback_debug()
def mkdir(self, path, recursive=False): path = to_nimbus_path(path) logger.debug('mkdir %s ...'%path) mdf = self.metadata if mdf.exists(path): raise AlreadyExistsException('Directory "%s" is already exists!'%path) base_path, new_dir = os.path.split(path) if not mdf.exists(base_path): if recursive: self.mkdir(base_path, recursive) else: raise PathException('Directory "%s" does not exists!'%base_path) new_dir_obj = DirectoryMD(name=new_dir) mdf.append(base_path, new_dir_obj)
def __parse_tr_log(self): def parse_int(val): if val in [None, 'None']: return None return int(val) def parse_str(val): if val in [None, 'None']: return None return val transactions = {} tr_log = open(self.__trlog_path, 'r') for line in tr_log.readlines(): if not line: break logger.debug('processing "%s"'%line.strip()) parts = line.split() transaction_id = int(parts[0]) r_type = parts[1] if r_type == 'ST': transaction = Transaction(int(parts[2]), base64.b64decode(parts[3]), int(parts[4]), transaction_id) transactions[transaction_id] = [transaction, {}] elif r_type == 'US': transactions[transaction_id][0].change_status(int(parts[2])) elif r_type == 'UT': seek = int(parts[2]) size = parse_int(parts[3]) local_name = parse_str(parts[4]) try: foreign_name = parse_str(parts[5]) except IndexError: foreign_name = None cur_vals = transactions[transaction_id][1].get(seek, [None, None, None]) if size: cur_vals[0] = size if local_name: cur_vals[1] = local_name if foreign_name: cur_vals[2] = foreign_name transactions[transaction_id][1][seek] = cur_vals tr_log.close() return sorted(transactions.values(), cmp=lambda x,y: cmp(x[0].get_start_datetime(), y[0].get_start_datetime()))
def mkdir(self, path, recursive=False): path = to_nimbus_path(path) logger.debug('mkdir %s ...' % path) mdf = self.metadata if mdf.exists(path): raise AlreadyExistsException('Directory "%s" is already exists!' % path) base_path, new_dir = os.path.split(path) if not mdf.exists(base_path): if recursive: self.mkdir(base_path, recursive) else: raise PathException('Directory "%s" does not exists!' % base_path) new_dir_obj = DirectoryMD(name=new_dir) mdf.append(base_path, new_dir_obj)
def iter(self, start_record_id=None): JLock.lock() try: j_data = DataBlock(self.__journal_path, actsize=True) buf = "" while True: if len(buf) < self.RECORD_STRUCT_SIZE: buf += j_data.read(1024) # logger.debug('J_ITER: buf=%s'%buf.encode('hex').upper()) if not buf: break # logger.debug('J_ITER: header=%s'%buf[:self.RECORD_STRUCT_SIZE].encode('hex').upper()) item_dump_len, operation_type, record_id = struct.unpack( self.RECORD_STRUCT, buf[: self.RECORD_STRUCT_SIZE] ) # logger.debug('J_ITER: buf_len=%s, item_dump_len=%s, operation_type=%s, record_id=%s'%(len(buf), item_dump_len, operation_type, record_id)) if operation_type not in (self.OT_APPEND, self.OT_UPDATE, self.OT_REMOVE): # logger.debug('J_ITER: buf=%s'%buf.encode('hex').upper()) raise RuntimeError("Invalid journal!!! Unknown operation type: %s" % operation_type) if len(buf) < (self.RECORD_STRUCT_SIZE + item_dump_len): buf += j_data.read(1024) item_dump = buf[self.RECORD_STRUCT_SIZE : self.RECORD_STRUCT_SIZE + item_dump_len] remaining_len = BLOCK_SIZE - self.RECORD_STRUCT_SIZE - item_dump_len to_pad_len = remaining_len % BLOCK_SIZE # logger.debug('J_ITER: record=%s'%buf[:self.RECORD_STRUCT_SIZE+item_dump_len+to_pad_len].encode('hex').upper()) buf = buf[self.RECORD_STRUCT_SIZE + item_dump_len + to_pad_len :] self.__last_record_id = record_id if (start_record_id is None) or (record_id > start_record_id): if operation_type == self.OT_REMOVE: item_md = struct.unpack("<I", item_dump)[0] else: item_md = AbstractMetadataObject.load_md(item_dump) logger.debug( "J_ITER: record_id=%s, operation_type=%s, item_md=%s" % (record_id, operation_type, item_md) ) yield record_id, operation_type, item_md finally: JLock.unlock()
def move(self, s_path, d_path): s_path = to_nimbus_path(s_path) d_path = to_nimbus_path(d_path) logger.debug('moving %s to %s ...'%(s_path, d_path)) mdf = self.metadata source = mdf.find(s_path) if mdf.exists(d_path): d_obj = mdf.find(d_path) if d_obj.is_file(): raise AlreadyExistsException('File %s is already exists!'%d_path) source.parent_dir_id = d_obj.item_id else: dst_path, new_name = os.path.split(d_path) source.name = new_name mdf.find(dst_path) #check existance mdf.update(source) logger.debug('%s is moved to %s!'%(s_path, d_path))
def transfer_data_block(self, transaction_id, seek, size, data_block, foreign_name=None): logger.debug('data block %s (seek=%s, size=%s) is ready for transfer' % (data_block.get_name(), seek, size)) transaction = self.__get_transaction(transaction_id) transaction.append_data_block(seek, size, data_block, foreign_name) if transaction.is_local(): return self.__tr_log_update(transaction_id, seek, size, data_block.get_name(), foreign_name) if transaction.is_uploading(): self.__put_queue.put((transaction, seek)) else: self.__get_queue.put((transaction, seek))
def move(self, s_path, d_path): s_path = to_nimbus_path(s_path) d_path = to_nimbus_path(d_path) logger.debug('moving %s to %s ...' % (s_path, d_path)) mdf = self.metadata source = mdf.find(s_path) if mdf.exists(d_path): d_obj = mdf.find(d_path) if d_obj.is_file(): raise AlreadyExistsException('File %s is already exists!' % d_path) source.parent_dir_id = d_obj.item_id else: dst_path, new_name = os.path.split(d_path) source.name = new_name mdf.find(dst_path) #check existance mdf.update(source) logger.debug('%s is moved to %s!' % (s_path, d_path))
def rmdir(self, path, recursive=False): path = to_nimbus_path(path) logger.debug('rmdir %s ...' % path) mdf = self.metadata dir_obj = mdf.find(path) if not dir_obj.is_dir(): raise NotDirectoryException('%s is a file!' % path) items = mdf.listdir(path) if items and not recursive: raise NotEmptyException('Directory "%s" is not empty!' % path) for item in items: full_path = '%s/%s' % (path, item.name) if item.is_file(): self.remove_file(full_path) else: self.rmdir(full_path, recursive) mdf.remove(dir_obj)
def rmdir(self, path, recursive=False): path = to_nimbus_path(path) logger.debug('rmdir %s ...'%path) mdf = self.metadata dir_obj = mdf.find(path) if not dir_obj.is_dir(): raise NotDirectoryException('%s is a file!'%path) items = mdf.listdir(path) if items and not recursive: raise NotEmptyException('Directory "%s" is not empty!'%path) for item in items: full_path = '%s/%s'%(path, item.name) if item.is_file(): self.remove_file(full_path) else: self.rmdir(full_path, recursive) mdf.remove(dir_obj)
def run(self): while True: out_streem = data = None job = self.queue.get() data_block = None transaction = None seek = None try: if job == QUIT_JOB or self.stop_flag.is_set(): break transaction, seek = job data_block,_,foreign_name = transaction.get_data_block(seek, noclone=False) if not foreign_name: raise Exception('foreign name does not found for seek=%s'%seek) if transaction.is_failed(): logger.debug('Transaction {%s} is failed! Skipping data block downloading...'%transaction.get_id()) data_block.remove() continue self.fabnet_gateway.get(foreign_name, transaction.get_replica_count(), data_block) data_block.close() self.transactions_manager.update_transaction(transaction.get_id(), seek, \ is_failed=False, foreign_name=data_block.get_name()) except Exception, err: events_provider.error('GetWorker','%s failed: %s'%(transaction, err)) logger.traceback_debug() try: if transaction and data_block: self.transactions_manager.update_transaction(transaction.get_id(), seek, \ is_failed=True, foreign_name=data_block.get_name()) data_block.remove() except Exception, err: logger.error('[GetWorker.__on_error] %s'%err) logger.traceback_debug()
def __open_file(self, open_flags): self.__f_obj = open(self.__path, open_flags) if 'r+' in open_flags: if self.get_actual_size(): logger.debug('rewrite data block %s for appending...'%self.get_name()) self.__restore_db()
logger.traceback_debug() raise err if self.__is_tmp_file: status = Transaction.TS_FINISHED else: status = Transaction.TS_LOCAL_SAVED if self.__transaction_id: self.TRANSACTIONS_MANAGER.update_transaction_state(self.__transaction_id, status) else: if self.__cur_data_block: self.__cur_data_block.close() finally: self.__closed = True logger.debug('file %s is closed!'%self.__file_path) def __send_data_block(self): self.__cur_data_block.finalize() if self.__cur_data_block.get_actual_size(): self.TRANSACTIONS_MANAGER.transfer_data_block(self.__transaction_id, \ self.__seek, self.__cur_db_seek, self.__cur_data_block) self.__seek += self.__cur_db_seek self.__cur_db_seek = 0 self.__cur_data_block = None self.__unsync = False def __failed_transaction(self, err): self.__failed_flag = True events_provider.critical("File", "File %s IO error: %s"%\
def remove_file(self, file_path): file_path = to_nimbus_path(file_path) logger.debug('removing file %s ...'%file_path) self.transactions_manager.remove_file(file_path) logger.debug('file %s is removed!'%file_path)
def remove_file(self, file_path): file_path = to_nimbus_path(file_path) logger.debug('removing file %s ...' % file_path) self.transactions_manager.remove_file(file_path) logger.debug('file %s is removed!' % file_path)
raise err if self.__is_tmp_file: status = Transaction.TS_FINISHED else: status = Transaction.TS_LOCAL_SAVED if self.__transaction_id: self.TRANSACTIONS_MANAGER.update_transaction_state( self.__transaction_id, status) else: if self.__cur_data_block: self.__cur_data_block.close() finally: self.__closed = True logger.debug('file %s is closed!' % self.__file_path) def __send_data_block(self): self.__cur_data_block.finalize() if self.__cur_data_block.get_actual_size(): self.TRANSACTIONS_MANAGER.transfer_data_block(self.__transaction_id, \ self.__seek, self.__cur_db_seek, self.__cur_data_block) self.__seek += self.__cur_db_seek self.__cur_db_seek = 0 self.__cur_data_block = None self.__unsync = False def __failed_transaction(self, err): self.__failed_flag = True events_provider.critical("File", "File %s IO error: %s"%\