def test_data_block(self): ks = FileBasedSecurityManager(CLIENT_KS_1024_PATH, PASSWD) DataBlock.SECURITY_MANAGER = ks DB_PATH = tmp('test_data_block.kst') DATA_LEN = 10 if os.path.exists(DB_PATH): os.remove(DB_PATH) db = DataBlock(DB_PATH, DATA_LEN, force_create=True) checksum = hashlib.sha1() for i in xrange(DATA_LEN / 10): data = ''.join( random.choice(string.letters) for i in xrange(DATA_LEN / (DATA_LEN / 10))) checksum.update(data) db.write(data) db.close() db.close() #should be OK or_checksum = checksum.hexdigest() enc_checksum = db.checksum() db = DataBlock(DB_PATH, DATA_LEN) ret_data = '' checksum = hashlib.sha1() while True: data = db.read(100) if not data: break ret_data += data checksum.update(data) self.assertEqual(or_checksum, checksum.hexdigest()) self.assertEqual(db.checksum(), enc_checksum) db = DataBlock(DB_PATH, DATA_LEN) raw = db.read_raw() self.assertEqual(db.checksum(), enc_checksum) db = DataBlock(DB_PATH, DATA_LEN) raw = db.read() self.assertEqual(ret_data, raw) app_db = DataBlock(DB_PATH) app_db.write('The end!') app_db.finalize() app_db.close() db = DataBlock(DB_PATH, actsize=True) raw = db.read() self.assertEqual(ret_data + 'The end!', raw) db.close()
def test_data_block(self): ks = FileBasedSecurityManager(CLIENT_KS_1024_PATH, PASSWD) DataBlock.SECURITY_MANAGER = ks DB_PATH = tmp('test_data_block.kst') DATA_LEN = 10 if os.path.exists(DB_PATH): os.remove(DB_PATH) db = DataBlock(DB_PATH, DATA_LEN, force_create=True) checksum = hashlib.sha1() for i in xrange(DATA_LEN/10): data = ''.join(random.choice(string.letters) for i in xrange(DATA_LEN/(DATA_LEN/10))) checksum.update(data) db.write(data) db.close() db.close() #should be OK or_checksum = checksum.hexdigest() enc_checksum = db.checksum() db = DataBlock(DB_PATH, DATA_LEN) ret_data = '' checksum = hashlib.sha1() while True: data = db.read(100) if not data: break ret_data += data checksum.update(data) self.assertEqual(or_checksum, checksum.hexdigest()) self.assertEqual(db.checksum(), enc_checksum) db = DataBlock(DB_PATH, DATA_LEN) raw = db.read_raw() self.assertEqual(db.checksum(), enc_checksum) db = DataBlock(DB_PATH, DATA_LEN) raw = db.read() self.assertEqual(ret_data, raw) app_db = DataBlock(DB_PATH) app_db.write('The end!') app_db.finalize() app_db.close() db = DataBlock(DB_PATH, actsize=True) raw = db.read() self.assertEqual(ret_data+'The end!', raw) db.close()
def iter(self, start_record_id=None): JLock.lock() try: j_data = DataBlock(self.__journal_path, actsize=True) buf = '' while True: if len(buf) < self.RECORD_STRUCT_SIZE: buf += j_data.read(1024) #logger.debug('J_ITER: buf=%s'%buf.encode('hex').upper()) if not buf: break #logger.debug('J_ITER: header=%s'%buf[:self.RECORD_STRUCT_SIZE].encode('hex').upper()) item_dump_len, operation_type, record_id = struct.unpack( self.RECORD_STRUCT, buf[:self.RECORD_STRUCT_SIZE]) #logger.debug('J_ITER: buf_len=%s, item_dump_len=%s, operation_type=%s, record_id=%s'%(len(buf), item_dump_len, operation_type, record_id)) if operation_type not in (self.OT_APPEND, self.OT_UPDATE, self.OT_REMOVE): #logger.debug('J_ITER: buf=%s'%buf.encode('hex').upper()) raise RuntimeError( 'Invalid journal!!! Unknown operation type: %s' % operation_type) if len(buf) < (self.RECORD_STRUCT_SIZE + item_dump_len): buf += j_data.read(1024) item_dump = buf[self. RECORD_STRUCT_SIZE:self.RECORD_STRUCT_SIZE + item_dump_len] remaining_len = BLOCK_SIZE - self.RECORD_STRUCT_SIZE - item_dump_len to_pad_len = remaining_len % BLOCK_SIZE #logger.debug('J_ITER: record=%s'%buf[:self.RECORD_STRUCT_SIZE+item_dump_len+to_pad_len].encode('hex').upper()) buf = buf[self.RECORD_STRUCT_SIZE + item_dump_len + to_pad_len:] self.__last_record_id = record_id if (start_record_id is None) or (record_id > start_record_id): if operation_type == self.OT_REMOVE: item_md = struct.unpack('<I', item_dump)[0] else: item_md = AbstractMetadataObject.load_md(item_dump) logger.debug( 'J_ITER: record_id=%s, operation_type=%s, item_md=%s' % (record_id, operation_type, item_md)) yield record_id, operation_type, item_md finally: JLock.unlock()
def iter(self, start_record_id=None): JLock.lock() try: j_data = DataBlock(self.__journal_path, actsize=True) buf = "" while True: if len(buf) < self.RECORD_STRUCT_SIZE: buf += j_data.read(1024) # logger.debug('J_ITER: buf=%s'%buf.encode('hex').upper()) if not buf: break # logger.debug('J_ITER: header=%s'%buf[:self.RECORD_STRUCT_SIZE].encode('hex').upper()) item_dump_len, operation_type, record_id = struct.unpack( self.RECORD_STRUCT, buf[: self.RECORD_STRUCT_SIZE] ) # logger.debug('J_ITER: buf_len=%s, item_dump_len=%s, operation_type=%s, record_id=%s'%(len(buf), item_dump_len, operation_type, record_id)) if operation_type not in (self.OT_APPEND, self.OT_UPDATE, self.OT_REMOVE): # logger.debug('J_ITER: buf=%s'%buf.encode('hex').upper()) raise RuntimeError("Invalid journal!!! Unknown operation type: %s" % operation_type) if len(buf) < (self.RECORD_STRUCT_SIZE + item_dump_len): buf += j_data.read(1024) item_dump = buf[self.RECORD_STRUCT_SIZE : self.RECORD_STRUCT_SIZE + item_dump_len] remaining_len = BLOCK_SIZE - self.RECORD_STRUCT_SIZE - item_dump_len to_pad_len = remaining_len % BLOCK_SIZE # logger.debug('J_ITER: record=%s'%buf[:self.RECORD_STRUCT_SIZE+item_dump_len+to_pad_len].encode('hex').upper()) buf = buf[self.RECORD_STRUCT_SIZE + item_dump_len + to_pad_len :] self.__last_record_id = record_id if (start_record_id is None) or (record_id > start_record_id): if operation_type == self.OT_REMOVE: item_md = struct.unpack("<I", item_dump)[0] else: item_md = AbstractMetadataObject.load_md(item_dump) logger.debug( "J_ITER: record_id=%s, operation_type=%s, item_md=%s" % (record_id, operation_type, item_md) ) yield record_id, operation_type, item_md finally: JLock.unlock()
class DBReader(threading.Thread): def __init__(self, path, f_len): threading.Thread.__init__(self) self.path = path self.f_len = f_len self.checksum = hashlib.sha1() def run(self): try: self.db = DataBlock(self.path, self.f_len) while True: data = self.db.read(1000) if not data: break self.checksum.update(data) self.db.close() os.remove(self.path) except Exception, err: print 'ERROR: %s'%err
class DBReader(threading.Thread): def __init__(self, path, f_len): threading.Thread.__init__(self) self.path = path self.f_len = f_len self.checksum = hashlib.sha1() def run(self): try: self.db = DataBlock(self.path, self.f_len) while True: data = self.db.read(1000) if not data: break self.checksum.update(data) self.db.close() os.remove(self.path) except Exception, err: print 'ERROR: %s' % err