def _verify_damage( self, nodes ): for node in nodes: self.assertFalse( self._content_matches( node ), 'Failed to damage AU file: %s' % node.url ) if nodes: log.info( 'Damaged the following node(s) on client %s:\n\t\t\t%s' % ( self.victim, '\n\t\t\t'.join( str( node ) for node in nodes ) ) ) else: log.info( 'No nodes damaged on client %s' % self.victim )
def _verify_damage(self, nodes): for node in nodes: self._ensure_victim_node_deleted(node) log.info( "Deleted the following node(s) on client %s:\n\t\t\t%s" % (self.victim, "\n\t\t\t".join(str(node) for node in nodes)) )
def _damage_AU( self ): nodes = self.victim.getAuNodesWithContent( self.AU ) log.info( 'Backing up cache configuration on victim cache...' ) self.victim.backupConfiguration() log.info( 'Backed up successfully' ) self.victim.daemon.stop() log.info( 'Stopped daemon running on UI port %s' % self.victim.port ) self.victim.simulateDiskFailure() log.info( 'Deleted entire contents of cache on stopped daemon' ) # Write a TitleDB entry for the simulated AU so it will be marked 'publisher down' when restored. self.framework.appendLocalConfig( { 'org.lockss.auconfig.allowEditDefaultOnlyParams': True, 'org.lockss.title.sim1.journalTitle': 'Simulated Content', 'org.lockss.title.sim1.param.1.key': 'root', 'org.lockss.title.sim1.param.1.value': 'simContent', 'org.lockss.title.sim1.param.2.key': 'depth', 'org.lockss.title.sim1.param.2.value': 0, 'org.lockss.title.sim1.param.3.key': 'branch', 'org.lockss.title.sim1.param.3.value': 0, 'org.lockss.title.sim1.param.4.key': 'numFiles', 'org.lockss.title.sim1.param.4.value': 30, 'org.lockss.title.sim1.param.pub_down.key': 'pub_down', 'org.lockss.title.sim1.param.pub_down.value': True, 'org.lockss.title.sim1.plugin': 'org.lockss.plugin.simulated.SimulatedPlugin', 'org.lockss.title.sim1.title': 'Simulated Content: simContent' }, self.victim ) time.sleep( 5 ) # Settling time self.victim.daemon.start() # Wait for the client to come up self.assert_( self.victim.waitForDaemonReady(), 'Daemon is not ready' ) log.info( 'Started daemon running on UI port %s' % self.victim.port) return nodes
def runTest( self ): HTML = self.tinyUiClient.getAdminUi().read() pattern = 'This LOCKSS box \(.*\) has not started because it is unable to load configuration data' self.assert_( re.search( pattern, HTML, re.DOTALL | re.MULTILINE ), 'No match for "%s" in\n%s' % ( pattern, HTML ) ) pattern = "Shouldn't happen" self.assertFalse( re.search( pattern, HTML, re.DOTALL | re.IGNORECASE | re.MULTILINE ), 'Unexpected match for "%s"' % pattern ) self.assert_( re.search( self.expected_pattern, HTML, re.DOTALL | re.MULTILINE ), 'No match for "%s" in\n%s' % ( self.expected_pattern, HTML ) ) log.info( 'Found "%s"' % self.expected_pattern )
def _update_configuration( self ): self.framework.appendLocalConfig( { 'org.lockss.poll.v3.toplevelPollInterval': 10 }, self.victim ) self.victim.reloadConfiguration() if self.toggle_AU_activation: log.info( 'Deactivating AU' ) self.victim.deactivateAu( self.AU ) log.info( 'Reactivating AU' ) self.victim.reactivateAu( self.AU )
def _check_v3_result( self, nodes ): log.info( 'Waiting for V3 repair...' ) self._await_repair( nodes ) self._verify_repair( nodes ) self._verify_poll_results() self._verify_voter_agreements() self._verify_voters_counts() log.info( 'AU successfully repaired' )
def setUp( self ): LockssTestCases.setUp( self ) self._start_framework() # Block return until all clients are ready to go. log.info( 'Waiting for framework to become ready' ) self.tinyUiClient = self.clients[ 0 ] time.sleep( 2 ) self.tinyUiClient.waitForCanConnectToHost( sleep = 2 )
def _create_AU_nodes(self, minimum, maximum): # Damage the AU by creating extra nodes nodes = self.victim.randomCreateRandomNodes(self.AU, minimum, maximum) log.info( "Created the following nodes on client %s:\n\t\t\t%s" % (self.victim, "\n\t\t\t".join(str(node) for node in nodes)) ) self._set_expected_agreement_from_extra(nodes) return nodes
def runTest( self ): self._setup_AU() nodes = self._damage_AU() self._verify_damage( nodes ) # enable polling self._enableVictimPoller() log.info( 'Waiting for a V3 poll to be called...' ) self.assert_( self.victim.waitForV3Poller( self.AU ), 'Timed out while waiting for V3 poll' ) log.info( 'Successfully called a V3 poll' ) self._check_v3_result( nodes )
def _verify_damage( self, nodes ): self.assertFalse( self.victim.hasAu( self.AU ), 'AU still intact' ) # Restore the backup file log.info( 'Restoring cache configuration...' ) self.victim.restoreConfiguration( self.AU ) log.info( 'Restored successfully' ) # These should be equal AU IDs, so both should return true self.assert_( self.victim.hasAu( self.AU ) ) self.assert_( self.victim.isPublisherDown( self.AU ) )
def runTest(self): self._setup_AU() # disable polling? nodes = self._damage_AU() self._verify_damage(nodes) # enable polling? log.info("Waiting for a V3 poll to be called...") self.assert_(self.victim.waitForV3Poller(self.AU), "Timed out while waiting for V3 poll") log.info("Successfully called a V3 poll") self._check_v3_result(nodes)
def _verify_repair( self, nodes ): # XXX need to identify files not count them node_count = 0 repair_count = 0 for node in nodes: node_count = node_count + 1 if self.victim.isAuNode( self.AU, node.url ): if self._content_matches( node ): repair_count = repair_count + 1 log.info( '%i nodes %i repaired' % ( node_count, repair_count ) ) # There is a small chance that the following tests will generate # a false negative, about the chance of tossing 120 heads in a row self.assert_( repair_count > 0 ) self.assert_( repair_count < node_count )
def _verify_damage( self, nodes ): V3TestCases._verify_damage( self, nodes ) log.debug( 'victim.getNoAuPeers( self.AU ): %s' % self.victim.getNoAuPeers( self.AU ) ) self.assertEqual( self.victim.getNoAuPeers( self.AU ), [ self.client_without_AU.getV3Identity() ], 'Peer without AU disappeared!' ) log.info( 'Waiting for a V3 poll to be called...' ) # Ignores first victim poll self.assert_( self.victim.waitForV3Poller( self.AU, [ self.victim_first_poll_key ] ), 'Timed out while waiting for V3 poll' ) log.info( 'Successfully called a V3 poll' ) victim_second_poll_key = self.victim.getV3PollKey( self.AU, self.victim_first_poll_key ) log.debug( "Victim's second poll key: " + victim_second_poll_key ) invitees = self.victim.getV3PollInvitedPeers( victim_second_poll_key ) log.debug( 'invitedPeers: %s' % invitees ) self.assertFalse( self.client_without_AU.getV3Identity() in invitees, 'Peer without AU invited in 2nd poll' )
def tearDown( self ): # Dump threads and look for deadlocks (independent of success) deadlockLogs = self.framework.checkForDeadlock() if deadlockLogs: log.error( 'Deadlocks detected!' ) self.fail( 'Failing due to deadlock detection. Check the following log file(s): ' + ', '.join( deadlockLogs ) ) else: log.info( 'No deadlocks detected' ) if self.delayShutdown: raw_input( '>>> Delaying shutdown. Press Enter to continue...' ) log.info( 'Stopping framework' ) self.framework.stop() self.failIf( self.framework.isRunning, 'Framework did not stop' ) unittest.TestCase.tearDown( self )
def tearDown(self): # Dump threads and look for deadlocks (independent of success) deadlockLogs = self.framework.checkForDeadlock() if deadlockLogs: log.error("Deadlocks detected!") self.fail("Failing due to deadlock detection. Check the following log file(s): " + ", ".join(deadlockLogs)) else: log.info("No deadlocks detected") if self.delayShutdown: raw_input(">>> Delaying shutdown. Press Enter to continue...") log.info("Stopping framework") self.framework.stop() self.failIf(self.framework.isRunning, "Framework did not stop") unittest.TestCase.tearDown(self)
def setUp( self ): LockssTestCases.setUp( self ) self.victim = self.clients[ 0 ] self.nonVictim = self.clients[ 1 ] for client in self.clients: extraConf = { 'org.lockss.auconfig.allowEditDefaultOnlyParams': True, 'org.lockss.id.initialV3PeerList': ';'.join( [ peer.getV3Identity() for peer in self.clients ] + self.offline_peers ), 'org.lockss.platform.v3.identity': client.getV3Identity(), 'org.lockss.poll.v3.enableV3Poller': client is self.victim, 'org.lockss.poll.v3.enableV3Voter': True } extraConf.update( self.local_configuration ) self.framework.appendLocalConfig( extraConf, client ) self._start_framework() # Block return until all clients are ready to go. log.info( 'Waiting for framework to become ready' ) self.framework.waitForFrameworkReady()
def _await_V3_poll_agreement( self ): # Expect to see a top level content poll called by all peers log.info( 'Waiting for a V3 poll by all simulated caches' ) for client in self.clients: self.assert_( client.waitForV3Poller( self.AU ), 'Never called V3 poll' ) log.info( 'Client on port %s called V3 poll...' % client.port ) # Expect each client to have won a top-level v3 poll log.info( 'Waiting for all peers to win their polls' ) for client in self.clients: self.assert_( client.waitForWonV3Poll( self.AU, self.timeout ), 'Client on port %s did not win V3 poll' % client.port ) log.info( 'Client on port %s won V3 poll...' % client.port )
def _verify_voter_agreements( self ): poll_key = self.victim.getV3PollKey( self.AU ) for client in self.clients: if client != self.victim: repairer_info = client.getAuRepairerInfo( self.AU, 'LastPercentAgreement' ) repairer_count = len( repairer_info) for ( box, agreement ) in repairer_info.iteritems(): self.assertEqual(self.expected_voter_agreement, agreement, 'Client %s wrong agreement %s with box %s' % ( client, agreement, box)) if self.symmetric == True: log.info( 'Symmetric client %s repairers OK' % client ) else: log.info( 'Asymmetric client %s repairers OK' % client ) repairer_info = self.victim.getAuRepairerInfo( self.AU, 'LastPercentAgreement' ) self.assertEqual( len( self.clients ) - 1, len( repairer_info ) ) for ( box, agreement ) in repairer_info.iteritems(): log.debug( "Client %s box %s agree %s" % ( self.victim, box, agreement ) ) self.assertEqual( agreement, self.expected_agreement, 'Voter %s had actual agreement: %s expected: %s' % ( box, agreement, self.expected_agreement ) )
def _damage_AU(self): nodes = self.victim.getAuNodesWithContent(self.AU) log.info("Backing up cache configuration on victim cache...") self.victim.backupConfiguration() log.info("Backed up successfully") self.victim.daemon.stop() log.info("Stopped daemon running on UI port %s" % self.victim.port) self.victim.simulateDiskFailure() log.info("Deleted entire contents of cache on stopped daemon") # Write a TitleDB entry for the simulated AU so it will be marked 'publisher down' when restored. self.framework.appendLocalConfig( { "org.lockss.auconfig.allowEditDefaultOnlyParams": True, "org.lockss.title.sim1.journalTitle": "Simulated Content", "org.lockss.title.sim1.param.1.key": "root", "org.lockss.title.sim1.param.1.value": "simContent", "org.lockss.title.sim1.param.2.key": "depth", "org.lockss.title.sim1.param.2.value": 0, "org.lockss.title.sim1.param.3.key": "branch", "org.lockss.title.sim1.param.3.value": 0, "org.lockss.title.sim1.param.4.key": "numFiles", "org.lockss.title.sim1.param.4.value": 30, "org.lockss.title.sim1.param.pub_down.key": "pub_down", "org.lockss.title.sim1.param.pub_down.value": True, "org.lockss.title.sim1.plugin": "org.lockss.plugin.simulated.SimulatedPlugin", "org.lockss.title.sim1.title": "Simulated Content: simContent", }, self.victim, ) time.sleep(5) # Settling time self.victim.daemon.start() # Wait for the client to come up self.assert_(self.victim.waitForDaemonReady(), "Daemon is not ready") log.info("Started daemon running on UI port %s" % self.victim.port) return nodes
def runTest( self ): self._setup_AU() # disable polling? nodes = self._damage_AU() self._verify_damage( nodes ) # enable polling? log.info( 'Waiting for a V3 poll to be called...' ) self.assert_( self.victim.waitForV3Poller( self.AU ), 'Timed out while waiting for V3 poll' ) log.info( 'Successfully called a V3 poll' ) log.info( 'Waiting for V3 repair...' ) self._await_repair( nodes ) self._verify_repair( nodes ) log.info( 'AU successfully repaired' )
def _setup_AU( self ): self.AU = lockss_daemon.Simulated_AU( **self.simulated_AU_parameters ) log.info( "Creating simulated AU's" ) for client in self.clients: client.createAu( self.AU ) for client in self.clients: client.waitAu( self.AU ) log.info( "Waiting for simulated AU's to crawl" ) for client in self.clients: self.assert_( client.waitForSuccessfulCrawl( self.AU ), "AU's did not complete initial crawl" ) log.info( "AU's completed initial crawl" )
def setUp( self ): unittest.TestCase.setUp( self ) # Log start of test log.info( len( self.__doc__ )*'=' ) log.info( self.__doc__ ) log.info( len( self.__doc__ )*'-' ) # Create a framework for the test self.framework = lockss_daemon.Framework( self.daemon_count, self.config_URLs, self.start_UI_port ) # List of daemon clients self.clients = self.framework.clientList # Enable clean-up after user interruption frameworkList.append( self.framework )
def runTest( self ): self._setup_AU() self.expected_agreement = '100.00' self.expected_voter_agreement = '100.00' self._enableVictimPoller() # Wait for the first poll to finish log.info( 'Waiting for a V3 poll to be called...' ) self.assert_( self.victim.waitForV3Poller( self.AU ), 'Timed out while waiting for first V3 poll' ) log.info( 'Successfully called first V3 poll' ) self.assert_( self.victim.waitForCompletedV3Poll( self.AU ), 'Timed out while waiting for poll to complete' ) self._verify_poll_results() self._verify_voter_agreements() self._verify_voters_counts() # Destroy the AU nodes = self._damage_AU() self._verify_damage( nodes ) log.info( 'Waiting for second V3 poll to be called...' ) self.assert_( self.victim.waitForV3Poller( self.AU ), 'Timed out while waiting for second V3 poll' ) log.info( 'Successfully called second V3 poll' ) # Check the result self._await_repair(nodes) self._check_v3_result( nodes )
def _check_v3_result( self, nodes ): log.info( 'Waiting for poll to complete...' ) self._await_complete( nodes ) self._verify_poll_results() log.info( 'AU successfully repaired' )
def _await_complete( self, nodes ): log.info( 'Waiting for V3 poll to complete...' ) self.assert_( self.victim.waitForCompletedV3Poll( self.AU ), 'Timed out while waiting for poll to complete' ) log.info( 'Poll successfully completed' )
def _damage_AU( self ): # Damage the AU by creating an extra node node = self.victim.createNode( self.AU, '000extrafile.txt' ) log.info( 'Created file %s on client %s' % ( node.url, self.victim ) ) self._set_expected_agreement_from_extra( [ node ] ) return [ node ]
def _damage_AU( self ): # Damage the AU by creating an extra node that should sort LAST in the list of CachedUrls node = self.victim.createNode( self.AU, 'zzzzzzzzzzzzz.txt' ) log.info( 'Created file %s on client %s' % ( node.url, self.victim ) ) self._set_expected_agreement_from_extra( [ node ] ) return [ node ]
def _await_repair( self, nodes ): log.info( 'Waiting for V3 poll to report no quorum...' ) self.assert_( self.victim.waitForV3NoQuorum( self.AU ), 'Timed out while waiting for no quorum' ) log.info( 'AU successfully reported No Quorum' )
def _await_repair( self, nodes ): # Expect to see the AU successfully repaired log.info( 'Waiting for successful V3 repair of entire AU' ) self.assert_( self.victim.waitForV3Repair( self.AU, timeout = self.timeout ), 'AU was not repaired by V3' )
def _check_v3_result( self, nodes ): log.info( 'Waiting for poll to complete...' ) self._await_complete( nodes ) self._verify_poll_results() log.info( 'AU repair not verified, as expected' )