def test_opens_existing_tree_without_node_and_key_sizes_given(self): f = larch.open_forest(allow_writes=True, key_size=self.key_size, node_size=self.node_size, dirname=self.tempdir) f.commit() f2 = larch.open_forest(dirname=self.tempdir, allow_writes=True) self.assertEqual(f2.node_store.node_size, self.node_size) self.assertEqual(f2.node_store.codec.key_bytes, self.key_size)
def test_opens_existing_tree_with_compatible_key_and_node_size(self): f = larch.open_forest(key_size=self.key_size, node_size=self.node_size, dirname=self.tempdir, allow_writes=True) f.commit() f2 = larch.open_forest(key_size=self.key_size, node_size=self.node_size, dirname=self.tempdir, allow_writes=True) self.assert_(True)
def test_opens_existing_tree_with_incompatible_node_size(self): f = larch.open_forest(allow_writes=True, key_size=self.key_size, node_size=self.node_size, dirname=self.tempdir) f.commit() new_size = self.node_size + 1 f2 = larch.open_forest(key_size=self.key_size, node_size=new_size, dirname=self.tempdir, allow_writes=True) self.assertEqual(int(f2.node_store.get_metadata('node_size')), self.node_size)
def test_creates_new_forest(self): f = larch.open_forest(key_size=self.key_size, node_size=self.node_size, dirname=self.tempdir, allow_writes=True) self.assertEqual(f.node_store.codec.key_bytes, self.key_size) self.assertEqual(f.node_store.node_size, self.node_size)
def test_CheckIndexNodeFixReferences(self): # Create an index node with a missing reference: index_node_missing_reference = self.get_any_node(index_node=True) id_index_node = index_node_missing_reference.id id_node_to_remove = index_node_missing_reference.values()[0] mkey = index_node_missing_reference.keys()[0] # Let's remove the node_to_remove from the FS: nodepath = self.forest.node_store.idpath.convert(id_node_to_remove) os.unlink(nodepath) del self.forest Fforest = larch.open_forest(allow_writes=False, dirname=self.dirname) Ffsck = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix=True) index_node_missing_reference = Fforest.node_store.get_node( id_index_node) cin = larch.fsck.CheckIndexNode(Ffsck, index_node_missing_reference) for work in cin.do(): work.do() # Expect dropped keys path self.failUnless( 'index node %s: dropped key %s' % (id_index_node, mkey.encode('hex')) in self.logged_warnings[0], self.logged_warnings[0]) # We have a "node %d is missing" error as well: self.failUnless( 'node %s is missing' % id_node_to_remove in self.logged_errors[0], self.logged_errors[0])
def test_fsck(self): Fforest = larch.open_forest(allow_writes=False,dirname=self.dirname) Ffsck = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix = False) # Run fsck ts = ttystatus.TerminalStatus() Ffsck.run_fsck(ts=ts) self.failIf( self.logged_warnings ) self.failIf( self.logged_errors )
def test_CheckRefcounts(self): Fforest = larch.open_forest(allow_writes=True, dirname=self.dirname) Ffsck = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix=False) # Populate self.refcounts Ffsck.run_fsck() self.failIf(self.logged_warnings) self.failIf(self.logged_errors) leafnode = self.get_any_node() self.failUnless(Ffsck.refcounts[leafnode.id] == 1, Ffsck.refcounts[leafnode.id]) # Change a refcount so that it is bad: Ffsck2 = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix=False) self.failUnless(1 == Fforest.node_store.rs.get_refcount(leafnode.id)) Fforest.node_store.rs.set_refcount(leafnode.id, 2) Fforest.node_store.rs.save_refcounts() Fforest.commit() # Check the refcounts Ffsck2.run_fsck() self.failIf(self.logged_warnings) self.failUnless( 'node %s: refcount is %s but should be %s' % (leafnode.id, 2, 1) in self.logged_errors[0], self.logged_errors[0]) self.failUnless(2 == Fforest.node_store.rs.get_refcount(leafnode.id)) # Fix the refcounts: self.logged_errors = [] self.logged_warnings = [] Ffsck3 = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix=True) Ffsck3.run_fsck() self.failUnless( 'node %s: refcount is %s but should be %s' % (leafnode.id, 2, 1) in self.logged_errors[0], self.logged_errors[0]) self.failUnless( 'node %s: refcount was set to %s' % (leafnode.id, 1) in self.logged_warnings[0], self.logged_warnings[0]) self.failUnless(1 == Fforest.node_store.rs.get_refcount(leafnode.id)) # No more errors expected: self.logged_errors = [] self.logged_warnings = [] Ffsck4 = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix=True) Ffsck4.run_fsck() self.failIf(self.logged_warnings) self.failIf(self.logged_errors)
def test_fsck(self): Fforest = larch.open_forest(allow_writes=False, dirname=self.dirname) Ffsck = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix=False) # Run fsck ts = ttystatus.TerminalStatus() Ffsck.run_fsck(ts=ts) self.failIf(self.logged_warnings) self.failIf(self.logged_errors)
def open_tree(allow_writes, dirname): key_size = len(compute('/dev/null')) node_size = 4096 forest = larch.open_forest( allow_writes=allow_writes, key_size=key_size, node_size=node_size, dirname=dirname) if forest.trees: tree = forest.trees[0] else: tree = forest.new_tree() return forest, tree
def new_disk_forest(self): forest = larch.open_forest( allow_writes=True, key_size=self.KEY_SIZE, node_size=self.NODE_SIZE, dirname=self.dirname, node_store=larch.NodeStoreDisk) t1=forest.new_tree() for i in xrange(self.NB_KEYS): value='%0*d' % (self.VALUES_SIZE, 1*i) key ='%0*d' % (self.KEY_SIZE, 1*i) t1.insert(key,value) forest.commit() return forest
def test_fail_if_existing_tree_has_incompatible_key_size(self): f = larch.open_forest(key_size=self.key_size, node_size=self.node_size, dirname=self.tempdir, allow_writes=True) f.commit() self.assertRaises(larch.BadKeySize, larch.open_forest, key_size=self.key_size + 1, node_size=self.node_size, dirname=self.tempdir, allow_writes=True)
def new_disk_forest(self): forest = larch.open_forest(allow_writes=True, key_size=self.KEY_SIZE, node_size=self.NODE_SIZE, dirname=self.dirname, node_store=larch.NodeStoreDisk) t1 = forest.new_tree() for i in xrange(self.NB_KEYS): value = '%0*d' % (self.VALUES_SIZE, 1 * i) key = '%0*d' % (self.KEY_SIZE, 1 * i) t1.insert(key, value) forest.commit() return forest
def open_tree(allow_writes, dirname): key_size = len(compute('/dev/null')) node_size = 4096 forest = larch.open_forest(allow_writes=allow_writes, key_size=key_size, node_size=node_size, dirname=dirname) if forest.trees: tree = forest.trees[0] else: tree = forest.new_tree() return forest, tree
def test_missing_index_node(self): # Let's delete a index node indexnode = self.get_any_node(index_node=True) self.failUnless(indexnode is not None) # delete the node: nodepath = self.forest.node_store.idpath.convert(indexnode.id) os.unlink( nodepath ) # fsck the forest Fforest = larch.open_forest(allow_writes=False,dirname=self.dirname) Ffsck = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix = False) Ffsck.run_fsck() # Make sure that the missing node is noticed self.failUnless( True in [ 'node %d is missing' % indexnode.id in x for x in self.logged_errors ] )
def do(self): if self.settings[self.skip_setting]: return if not self.fs.exists(self.dirname): logging.debug('B-tree %s does not exist, skipping' % self.dirname) return logging.debug('Checking B-tree %s' % self.dirname) fix = self.settings['fsck-fix'] forest = larch.open_forest( allow_writes=fix, dirname=self.dirname, vfs=self.fs) fsck = larch.fsck.Fsck(forest, self.warning, self.error, fix) for work in fsck.find_work(): yield work
def do(self): if self.settings[self.skip_setting]: return if not self.fs.exists(self.dirname): logging.debug('B-tree %s does not exist, skipping', self.dirname) return logging.debug('Checking B-tree %s', self.dirname) fix = self.settings['fsck-fix'] forest = larch.open_forest( allow_writes=fix, dirname=self.dirname, vfs=self.fs) fsck = larch.fsck.Fsck(forest, self.warning, self.error, fix) for work in fsck.find_work(): yield work
def init_forest(self, allow_writes=False): if self.forest is None: tracing.trace('initializing forest dirname=%s', self.dirname) assert self.tree is None if not self.fs.exists(self.dirname): tracing.trace('%s does not exist', self.dirname) return False self.forest = larch.open_forest(key_size=self.key_bytes, node_size=self.node_size, dirname=self.dirname, upload_max=self.upload_queue_size, lru_size=self.lru_size, vfs=self.fs, allow_writes=allow_writes) self.forest_allows_writes = allow_writes return True
def test_missing_leaf_node(self): # Let's delete a leaf node leafnode = self.get_any_node() self.failUnless(leafnode is not None) # delete the node: nodepath = self.forest.node_store.idpath.convert(leafnode.id) os.unlink(nodepath) # fsck the forest Fforest = larch.open_forest(allow_writes=False, dirname=self.dirname) Ffsck = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix=False) Ffsck.run_fsck() # Make sure that the missing node is noticed self.failUnless(True in [ 'node %d is missing' % leafnode.id in x for x in self.logged_errors ])
def test_CheckRefcounts(self): Fforest = larch.open_forest(allow_writes=True,dirname=self.dirname) Ffsck = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix = False) # Populate self.refcounts Ffsck.run_fsck() self.failIf( self.logged_warnings ) self.failIf( self.logged_errors ) leafnode = self.get_any_node() self.failUnless( Ffsck.refcounts[leafnode.id] == 1 , Ffsck.refcounts[leafnode.id] ) # Change a refcount so that it is bad: Ffsck2 = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix = False) self.failUnless( 1 == Fforest.node_store.rs.get_refcount(leafnode.id) ) Fforest.node_store.rs.set_refcount(leafnode.id,2) Fforest.node_store.rs.save_refcounts() Fforest.commit() # Check the refcounts Ffsck2.run_fsck() self.failIf( self.logged_warnings ) self.failUnless( 'node %s: refcount is %s but should be %s' % (leafnode.id, 2,1) in self.logged_errors[0], self.logged_errors[0] ) self.failUnless( 2 == Fforest.node_store.rs.get_refcount(leafnode.id) ) # Fix the refcounts: self.logged_errors = [] self.logged_warnings = [] Ffsck3 = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix = True) Ffsck3.run_fsck() self.failUnless( 'node %s: refcount is %s but should be %s' % (leafnode.id, 2,1) in self.logged_errors[0], self.logged_errors[0] ) self.failUnless( 'node %s: refcount was set to %s' % (leafnode.id, 1) in self.logged_warnings[0], self.logged_warnings[0] ) self.failUnless( 1 == Fforest.node_store.rs.get_refcount(leafnode.id) ) # No more errors expected: self.logged_errors = [] self.logged_warnings = [] Ffsck4 = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix = True) Ffsck4.run_fsck() self.failIf( self.logged_warnings ) self.failIf( self.logged_errors )
def test_CheckIndexNodeFixReferences(self): # Create an index node with a missing reference: index_node_missing_reference = self.get_any_node(index_node=True) id_index_node = index_node_missing_reference.id id_node_to_remove = index_node_missing_reference.values()[0] mkey = index_node_missing_reference.keys()[0] # Let's remove the node_to_remove from the FS: nodepath = self.forest.node_store.idpath.convert(id_node_to_remove) os.unlink( nodepath ) del self.forest Fforest = larch.open_forest(allow_writes=False,dirname=self.dirname) Ffsck = larch.fsck.Fsck(Fforest, warning=self.log_warning, error=self.log_error, fix=True) index_node_missing_reference = Fforest.node_store.get_node(id_index_node) cin = larch.fsck.CheckIndexNode(Ffsck, index_node_missing_reference ) for work in cin.do(): work.do() # Expect dropped keys path self.failUnless( 'index node %s: dropped key %s' % (id_index_node, mkey.encode('hex')) in self.logged_warnings[0], self.logged_warnings[0] ) # We have a "node %d is missing" error as well: self.failUnless( 'node %s is missing' % id_node_to_remove in self.logged_errors[0], self.logged_errors[0] )