def test_opens_existing_tree_without_node_and_key_sizes_given(self):
f = larch.open_forest(allow_writes=True,
key_size=self.key_size,
node_size=self.node_size,
dirname=self.tempdir)
f.commit()
f2 = larch.open_forest(dirname=self.tempdir, allow_writes=True)
self.assertEqual(f2.node_store.node_size, self.node_size)
self.assertEqual(f2.node_store.codec.key_bytes, self.key_size)
def test_opens_existing_tree_with_compatible_key_and_node_size(self):
f = larch.open_forest(key_size=self.key_size,
node_size=self.node_size,
dirname=self.tempdir,
allow_writes=True)
f.commit()
f2 = larch.open_forest(key_size=self.key_size,
node_size=self.node_size,
dirname=self.tempdir,
allow_writes=True)
self.assert_(True)
def test_opens_existing_tree_with_incompatible_node_size(self):
f = larch.open_forest(allow_writes=True,
key_size=self.key_size,
node_size=self.node_size,
dirname=self.tempdir)
f.commit()
new_size = self.node_size + 1
f2 = larch.open_forest(key_size=self.key_size,
node_size=new_size,
dirname=self.tempdir,
allow_writes=True)
self.assertEqual(int(f2.node_store.get_metadata('node_size')),
self.node_size)
def test_creates_new_forest(self):
f = larch.open_forest(key_size=self.key_size,
node_size=self.node_size,
dirname=self.tempdir,
allow_writes=True)
self.assertEqual(f.node_store.codec.key_bytes, self.key_size)
self.assertEqual(f.node_store.node_size, self.node_size)
def test_CheckIndexNodeFixReferences(self):
# Create an index node with a missing reference:
index_node_missing_reference = self.get_any_node(index_node=True)
id_index_node = index_node_missing_reference.id
id_node_to_remove = index_node_missing_reference.values()[0]
mkey = index_node_missing_reference.keys()[0]
# Let's remove the node_to_remove from the FS:
nodepath = self.forest.node_store.idpath.convert(id_node_to_remove)
os.unlink(nodepath)
del self.forest
Fforest = larch.open_forest(allow_writes=False, dirname=self.dirname)
Ffsck = larch.fsck.Fsck(Fforest,
warning=self.log_warning,
error=self.log_error,
fix=True)
index_node_missing_reference = Fforest.node_store.get_node(
id_index_node)
cin = larch.fsck.CheckIndexNode(Ffsck, index_node_missing_reference)
for work in cin.do():
work.do() # Expect dropped keys path
self.failUnless(
'index node %s: dropped key %s' %
(id_index_node, mkey.encode('hex')) in self.logged_warnings[0],
self.logged_warnings[0])
# We have a "node %d is missing" error as well:
self.failUnless(
'node %s is missing' % id_node_to_remove in self.logged_errors[0],
self.logged_errors[0])
def test_fsck(self):
Fforest = larch.open_forest(allow_writes=False,dirname=self.dirname)
Ffsck = larch.fsck.Fsck(Fforest,
warning=self.log_warning, error=self.log_error,
fix = False)
# Run fsck
ts = ttystatus.TerminalStatus()
Ffsck.run_fsck(ts=ts)
self.failIf( self.logged_warnings )
self.failIf( self.logged_errors )
def test_CheckRefcounts(self):
Fforest = larch.open_forest(allow_writes=True, dirname=self.dirname)
Ffsck = larch.fsck.Fsck(Fforest,
warning=self.log_warning,
error=self.log_error,
fix=False)
# Populate self.refcounts
Ffsck.run_fsck()
self.failIf(self.logged_warnings)
self.failIf(self.logged_errors)
leafnode = self.get_any_node()
self.failUnless(Ffsck.refcounts[leafnode.id] == 1,
Ffsck.refcounts[leafnode.id])
# Change a refcount so that it is bad:
Ffsck2 = larch.fsck.Fsck(Fforest,
warning=self.log_warning,
error=self.log_error,
fix=False)
self.failUnless(1 == Fforest.node_store.rs.get_refcount(leafnode.id))
Fforest.node_store.rs.set_refcount(leafnode.id, 2)
Fforest.node_store.rs.save_refcounts()
Fforest.commit()
# Check the refcounts
Ffsck2.run_fsck()
self.failIf(self.logged_warnings)
self.failUnless(
'node %s: refcount is %s but should be %s' % (leafnode.id, 2, 1)
in self.logged_errors[0], self.logged_errors[0])
self.failUnless(2 == Fforest.node_store.rs.get_refcount(leafnode.id))
# Fix the refcounts:
self.logged_errors = []
self.logged_warnings = []
Ffsck3 = larch.fsck.Fsck(Fforest,
warning=self.log_warning,
error=self.log_error,
fix=True)
Ffsck3.run_fsck()
self.failUnless(
'node %s: refcount is %s but should be %s' % (leafnode.id, 2, 1)
in self.logged_errors[0], self.logged_errors[0])
self.failUnless(
'node %s: refcount was set to %s' % (leafnode.id, 1)
in self.logged_warnings[0], self.logged_warnings[0])
self.failUnless(1 == Fforest.node_store.rs.get_refcount(leafnode.id))
# No more errors expected:
self.logged_errors = []
self.logged_warnings = []
Ffsck4 = larch.fsck.Fsck(Fforest,
warning=self.log_warning,
error=self.log_error,
fix=True)
Ffsck4.run_fsck()
self.failIf(self.logged_warnings)
self.failIf(self.logged_errors)
def test_fsck(self):
Fforest = larch.open_forest(allow_writes=False, dirname=self.dirname)
Ffsck = larch.fsck.Fsck(Fforest,
warning=self.log_warning,
error=self.log_error,
fix=False)
# Run fsck
ts = ttystatus.TerminalStatus()
Ffsck.run_fsck(ts=ts)
self.failIf(self.logged_warnings)
self.failIf(self.logged_errors)
def open_tree(allow_writes, dirname):
key_size = len(compute('/dev/null'))
node_size = 4096
forest = larch.open_forest(
allow_writes=allow_writes, key_size=key_size, node_size=node_size,
dirname=dirname)
if forest.trees:
tree = forest.trees[0]
else:
tree = forest.new_tree()
return forest, tree
def new_disk_forest(self):
forest = larch.open_forest(
allow_writes=True,
key_size=self.KEY_SIZE, node_size=self.NODE_SIZE,
dirname=self.dirname, node_store=larch.NodeStoreDisk)
t1=forest.new_tree()
for i in xrange(self.NB_KEYS):
value='%0*d' % (self.VALUES_SIZE, 1*i)
key ='%0*d' % (self.KEY_SIZE, 1*i)
t1.insert(key,value)
forest.commit()
return forest
def test_fail_if_existing_tree_has_incompatible_key_size(self):
f = larch.open_forest(key_size=self.key_size,
node_size=self.node_size,
dirname=self.tempdir,
allow_writes=True)
f.commit()
self.assertRaises(larch.BadKeySize,
larch.open_forest,
key_size=self.key_size + 1,
node_size=self.node_size,
dirname=self.tempdir,
allow_writes=True)
def new_disk_forest(self):
forest = larch.open_forest(allow_writes=True,
key_size=self.KEY_SIZE,
node_size=self.NODE_SIZE,
dirname=self.dirname,
node_store=larch.NodeStoreDisk)
t1 = forest.new_tree()
for i in xrange(self.NB_KEYS):
value = '%0*d' % (self.VALUES_SIZE, 1 * i)
key = '%0*d' % (self.KEY_SIZE, 1 * i)
t1.insert(key, value)
forest.commit()
return forest
def open_tree(allow_writes, dirname):
key_size = len(compute('/dev/null'))
node_size = 4096
forest = larch.open_forest(allow_writes=allow_writes,
key_size=key_size,
node_size=node_size,
dirname=dirname)
if forest.trees:
tree = forest.trees[0]
else:
tree = forest.new_tree()
return forest, tree
def test_missing_index_node(self):
# Let's delete a index node
indexnode = self.get_any_node(index_node=True)
self.failUnless(indexnode is not None)
# delete the node:
nodepath = self.forest.node_store.idpath.convert(indexnode.id)
os.unlink( nodepath )
# fsck the forest
Fforest = larch.open_forest(allow_writes=False,dirname=self.dirname)
Ffsck = larch.fsck.Fsck(Fforest,
warning=self.log_warning, error=self.log_error,
fix = False)
Ffsck.run_fsck()
# Make sure that the missing node is noticed
self.failUnless( True in [ 'node %d is missing' % indexnode.id in x for x in self.logged_errors ] )
def do(self):
if self.settings[self.skip_setting]:
return
if not self.fs.exists(self.dirname):
logging.debug('B-tree %s does not exist, skipping' % self.dirname)
return
logging.debug('Checking B-tree %s' % self.dirname)
fix = self.settings['fsck-fix']
forest = larch.open_forest(
allow_writes=fix, dirname=self.dirname, vfs=self.fs)
fsck = larch.fsck.Fsck(forest, self.warning, self.error, fix)
for work in fsck.find_work():
yield work
def do(self):
if self.settings[self.skip_setting]:
return
if not self.fs.exists(self.dirname):
logging.debug('B-tree %s does not exist, skipping', self.dirname)
return
logging.debug('Checking B-tree %s', self.dirname)
fix = self.settings['fsck-fix']
forest = larch.open_forest(
allow_writes=fix, dirname=self.dirname, vfs=self.fs)
fsck = larch.fsck.Fsck(forest, self.warning, self.error, fix)
for work in fsck.find_work():
yield work
def init_forest(self, allow_writes=False):
if self.forest is None:
tracing.trace('initializing forest dirname=%s', self.dirname)
assert self.tree is None
if not self.fs.exists(self.dirname):
tracing.trace('%s does not exist', self.dirname)
return False
self.forest = larch.open_forest(key_size=self.key_bytes,
node_size=self.node_size,
dirname=self.dirname,
upload_max=self.upload_queue_size,
lru_size=self.lru_size,
vfs=self.fs,
allow_writes=allow_writes)
self.forest_allows_writes = allow_writes
return True
def init_forest(self, allow_writes=False):
if self.forest is None:
tracing.trace('initializing forest dirname=%s', self.dirname)
assert self.tree is None
if not self.fs.exists(self.dirname):
tracing.trace('%s does not exist', self.dirname)
return False
self.forest = larch.open_forest(key_size=self.key_bytes,
node_size=self.node_size,
dirname=self.dirname,
upload_max=self.upload_queue_size,
lru_size=self.lru_size,
vfs=self.fs,
allow_writes=allow_writes)
self.forest_allows_writes = allow_writes
return True
def test_missing_leaf_node(self):
# Let's delete a leaf node
leafnode = self.get_any_node()
self.failUnless(leafnode is not None)
# delete the node:
nodepath = self.forest.node_store.idpath.convert(leafnode.id)
os.unlink(nodepath)
# fsck the forest
Fforest = larch.open_forest(allow_writes=False, dirname=self.dirname)
Ffsck = larch.fsck.Fsck(Fforest,
warning=self.log_warning,
error=self.log_error,
fix=False)
Ffsck.run_fsck()
# Make sure that the missing node is noticed
self.failUnless(True in [
'node %d is missing' % leafnode.id in x for x in self.logged_errors
])
def test_CheckRefcounts(self):
Fforest = larch.open_forest(allow_writes=True,dirname=self.dirname)
Ffsck = larch.fsck.Fsck(Fforest,
warning=self.log_warning, error=self.log_error,
fix = False)
# Populate self.refcounts
Ffsck.run_fsck()
self.failIf( self.logged_warnings )
self.failIf( self.logged_errors )
leafnode = self.get_any_node()
self.failUnless( Ffsck.refcounts[leafnode.id] == 1 , Ffsck.refcounts[leafnode.id] )
# Change a refcount so that it is bad:
Ffsck2 = larch.fsck.Fsck(Fforest,
warning=self.log_warning, error=self.log_error,
fix = False)
self.failUnless( 1 == Fforest.node_store.rs.get_refcount(leafnode.id) )
Fforest.node_store.rs.set_refcount(leafnode.id,2)
Fforest.node_store.rs.save_refcounts()
Fforest.commit()
# Check the refcounts
Ffsck2.run_fsck()
self.failIf( self.logged_warnings )
self.failUnless( 'node %s: refcount is %s but should be %s' % (leafnode.id, 2,1) in self.logged_errors[0], self.logged_errors[0] )
self.failUnless( 2 == Fforest.node_store.rs.get_refcount(leafnode.id) )
# Fix the refcounts:
self.logged_errors = []
self.logged_warnings = []
Ffsck3 = larch.fsck.Fsck(Fforest,
warning=self.log_warning, error=self.log_error,
fix = True)
Ffsck3.run_fsck()
self.failUnless( 'node %s: refcount is %s but should be %s' % (leafnode.id, 2,1) in self.logged_errors[0], self.logged_errors[0] )
self.failUnless( 'node %s: refcount was set to %s' % (leafnode.id, 1) in self.logged_warnings[0], self.logged_warnings[0] )
self.failUnless( 1 == Fforest.node_store.rs.get_refcount(leafnode.id) )
# No more errors expected:
self.logged_errors = []
self.logged_warnings = []
Ffsck4 = larch.fsck.Fsck(Fforest,
warning=self.log_warning, error=self.log_error,
fix = True)
Ffsck4.run_fsck()
self.failIf( self.logged_warnings )
self.failIf( self.logged_errors )
def test_CheckIndexNodeFixReferences(self):
# Create an index node with a missing reference:
index_node_missing_reference = self.get_any_node(index_node=True)
id_index_node = index_node_missing_reference.id
id_node_to_remove = index_node_missing_reference.values()[0]
mkey = index_node_missing_reference.keys()[0]
# Let's remove the node_to_remove from the FS:
nodepath = self.forest.node_store.idpath.convert(id_node_to_remove)
os.unlink( nodepath )
del self.forest
Fforest = larch.open_forest(allow_writes=False,dirname=self.dirname)
Ffsck = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix=True)
index_node_missing_reference = Fforest.node_store.get_node(id_index_node)
cin = larch.fsck.CheckIndexNode(Ffsck, index_node_missing_reference )
for work in cin.do():
work.do() # Expect dropped keys path
self.failUnless(
'index node %s: dropped key %s' %
(id_index_node, mkey.encode('hex')) in self.logged_warnings[0],
self.logged_warnings[0] )
# We have a "node %d is missing" error as well:
self.failUnless( 'node %s is missing' % id_node_to_remove in self.logged_errors[0], self.logged_errors[0] )
