def test_data_block(self):
ks = FileBasedSecurityManager(CLIENT_KS_1024_PATH, PASSWD)
DataBlock.SECURITY_MANAGER = ks
DB_PATH = tmp('test_data_block.kst')
DATA_LEN = 10
if os.path.exists(DB_PATH):
os.remove(DB_PATH)
db = DataBlock(DB_PATH, DATA_LEN, force_create=True)
checksum = hashlib.sha1()
for i in xrange(DATA_LEN / 10):
data = ''.join(
random.choice(string.letters)
for i in xrange(DATA_LEN / (DATA_LEN / 10)))
checksum.update(data)
db.write(data)
db.close()
db.close() #should be OK
or_checksum = checksum.hexdigest()
enc_checksum = db.checksum()
db = DataBlock(DB_PATH, DATA_LEN)
ret_data = ''
checksum = hashlib.sha1()
while True:
data = db.read(100)
if not data:
break
ret_data += data
checksum.update(data)
self.assertEqual(or_checksum, checksum.hexdigest())
self.assertEqual(db.checksum(), enc_checksum)
db = DataBlock(DB_PATH, DATA_LEN)
raw = db.read_raw()
self.assertEqual(db.checksum(), enc_checksum)
db = DataBlock(DB_PATH, DATA_LEN)
raw = db.read()
self.assertEqual(ret_data, raw)
app_db = DataBlock(DB_PATH)
app_db.write('The end!')
app_db.finalize()
app_db.close()
db = DataBlock(DB_PATH, actsize=True)
raw = db.read()
self.assertEqual(ret_data + 'The end!', raw)
db.close()
def test_data_block(self):
ks = FileBasedSecurityManager(CLIENT_KS_1024_PATH, PASSWD)
DataBlock.SECURITY_MANAGER = ks
DB_PATH = tmp('test_data_block.kst')
DATA_LEN = 10
if os.path.exists(DB_PATH):
os.remove(DB_PATH)
db = DataBlock(DB_PATH, DATA_LEN, force_create=True)
checksum = hashlib.sha1()
for i in xrange(DATA_LEN/10):
data = ''.join(random.choice(string.letters) for i in xrange(DATA_LEN/(DATA_LEN/10)))
checksum.update(data)
db.write(data)
db.close()
db.close() #should be OK
or_checksum = checksum.hexdigest()
enc_checksum = db.checksum()
db = DataBlock(DB_PATH, DATA_LEN)
ret_data = ''
checksum = hashlib.sha1()
while True:
data = db.read(100)
if not data:
break
ret_data += data
checksum.update(data)
self.assertEqual(or_checksum, checksum.hexdigest())
self.assertEqual(db.checksum(), enc_checksum)
db = DataBlock(DB_PATH, DATA_LEN)
raw = db.read_raw()
self.assertEqual(db.checksum(), enc_checksum)
db = DataBlock(DB_PATH, DATA_LEN)
raw = db.read()
self.assertEqual(ret_data, raw)
app_db = DataBlock(DB_PATH)
app_db.write('The end!')
app_db.finalize()
app_db.close()
db = DataBlock(DB_PATH, actsize=True)
raw = db.read()
self.assertEqual(ret_data+'The end!', raw)
db.close()
def iter(self, start_record_id=None):
JLock.lock()
try:
j_data = DataBlock(self.__journal_path, actsize=True)
buf = ''
while True:
if len(buf) < self.RECORD_STRUCT_SIZE:
buf += j_data.read(1024)
#logger.debug('J_ITER: buf=%s'%buf.encode('hex').upper())
if not buf:
break
#logger.debug('J_ITER: header=%s'%buf[:self.RECORD_STRUCT_SIZE].encode('hex').upper())
item_dump_len, operation_type, record_id = struct.unpack(
self.RECORD_STRUCT, buf[:self.RECORD_STRUCT_SIZE])
#logger.debug('J_ITER: buf_len=%s, item_dump_len=%s, operation_type=%s, record_id=%s'%(len(buf), item_dump_len, operation_type, record_id))
if operation_type not in (self.OT_APPEND, self.OT_UPDATE,
self.OT_REMOVE):
#logger.debug('J_ITER: buf=%s'%buf.encode('hex').upper())
raise RuntimeError(
'Invalid journal!!! Unknown operation type: %s' %
operation_type)
if len(buf) < (self.RECORD_STRUCT_SIZE + item_dump_len):
buf += j_data.read(1024)
item_dump = buf[self.
RECORD_STRUCT_SIZE:self.RECORD_STRUCT_SIZE +
item_dump_len]
remaining_len = BLOCK_SIZE - self.RECORD_STRUCT_SIZE - item_dump_len
to_pad_len = remaining_len % BLOCK_SIZE
#logger.debug('J_ITER: record=%s'%buf[:self.RECORD_STRUCT_SIZE+item_dump_len+to_pad_len].encode('hex').upper())
buf = buf[self.RECORD_STRUCT_SIZE + item_dump_len +
to_pad_len:]
self.__last_record_id = record_id
if (start_record_id is None) or (record_id > start_record_id):
if operation_type == self.OT_REMOVE:
item_md = struct.unpack('<I', item_dump)[0]
else:
item_md = AbstractMetadataObject.load_md(item_dump)
logger.debug(
'J_ITER: record_id=%s, operation_type=%s, item_md=%s' %
(record_id, operation_type, item_md))
yield record_id, operation_type, item_md
finally:
JLock.unlock()
def iter(self, start_record_id=None):
JLock.lock()
try:
j_data = DataBlock(self.__journal_path, actsize=True)
buf = ""
while True:
if len(buf) < self.RECORD_STRUCT_SIZE:
buf += j_data.read(1024)
# logger.debug('J_ITER: buf=%s'%buf.encode('hex').upper())
if not buf:
break
# logger.debug('J_ITER: header=%s'%buf[:self.RECORD_STRUCT_SIZE].encode('hex').upper())
item_dump_len, operation_type, record_id = struct.unpack(
self.RECORD_STRUCT, buf[: self.RECORD_STRUCT_SIZE]
)
# logger.debug('J_ITER: buf_len=%s, item_dump_len=%s, operation_type=%s, record_id=%s'%(len(buf), item_dump_len, operation_type, record_id))
if operation_type not in (self.OT_APPEND, self.OT_UPDATE, self.OT_REMOVE):
# logger.debug('J_ITER: buf=%s'%buf.encode('hex').upper())
raise RuntimeError("Invalid journal!!! Unknown operation type: %s" % operation_type)
if len(buf) < (self.RECORD_STRUCT_SIZE + item_dump_len):
buf += j_data.read(1024)
item_dump = buf[self.RECORD_STRUCT_SIZE : self.RECORD_STRUCT_SIZE + item_dump_len]
remaining_len = BLOCK_SIZE - self.RECORD_STRUCT_SIZE - item_dump_len
to_pad_len = remaining_len % BLOCK_SIZE
# logger.debug('J_ITER: record=%s'%buf[:self.RECORD_STRUCT_SIZE+item_dump_len+to_pad_len].encode('hex').upper())
buf = buf[self.RECORD_STRUCT_SIZE + item_dump_len + to_pad_len :]
self.__last_record_id = record_id
if (start_record_id is None) or (record_id > start_record_id):
if operation_type == self.OT_REMOVE:
item_md = struct.unpack("<I", item_dump)[0]
else:
item_md = AbstractMetadataObject.load_md(item_dump)
logger.debug(
"J_ITER: record_id=%s, operation_type=%s, item_md=%s" % (record_id, operation_type, item_md)
)
yield record_id, operation_type, item_md
finally:
JLock.unlock()
class DBReader(threading.Thread):
def __init__(self, path, f_len):
threading.Thread.__init__(self)
self.path = path
self.f_len = f_len
self.checksum = hashlib.sha1()
def run(self):
try:
self.db = DataBlock(self.path, self.f_len)
while True:
data = self.db.read(1000)
if not data:
break
self.checksum.update(data)
self.db.close()
os.remove(self.path)
except Exception, err:
print 'ERROR: %s'%err
class DBReader(threading.Thread):
def __init__(self, path, f_len):
threading.Thread.__init__(self)
self.path = path
self.f_len = f_len
self.checksum = hashlib.sha1()
def run(self):
try:
self.db = DataBlock(self.path, self.f_len)
while True:
data = self.db.read(1000)
if not data:
break
self.checksum.update(data)
self.db.close()
os.remove(self.path)
except Exception, err:
print 'ERROR: %s' % err
