def test_multiple_instances(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq self.assertEquals(self.A.instance, 1) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution]) self.A.set_proposal('reqid', 'foobar') r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar'))]) self.assertEquals(self.A.instance, 2) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution]) self.A.set_proposal('reqid', 'baz') r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'baz')), ((1, 'A'), ('reqid', 'baz')), ((1, 'A'), ('reqid', 'baz'))]) self.assertEquals(self.A.instance, 3)
def test_multiple_instances(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq self.assertEquals( self.A.instance, 1 ) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution] ) self.A.set_proposal( 'reqid', 'foobar' ) r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar'))] ) self.assertEquals( self.A.instance, 2 ) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution] ) self.A.set_proposal( 'reqid', 'baz' ) r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'baz')), ((1, 'A'), ('reqid', 'baz')), ((1, 'A'), ('reqid', 'baz'))] ) self.assertEquals( self.A.instance, 3 )
def test_multiple_instances_with_crash_recovery(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq self.assertEquals(self.A.instance, 1) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution]) self.A.set_proposal('reqid', 'foobar') r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar'))]) self.assertEquals(self.A.instance, 2) for uid in 'ABC': self.fail_and_recover(uid, False) self.A.net.link_up = True self.B.net.link_up = True self.C.net.link_up = True #------------------------------------------------ # Recovery will return to the previous instance. We'll # need to re-resolve the same result. # self.assertEquals(self.A.instance, 1) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution]) r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar'))]) #------------------------------------------------ # Back to instance 2 # self.assertEquals(self.A.instance, 2) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution]) self.A.set_proposal('reqid', 'baz') r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'baz')), ((1, 'A'), ('reqid', 'baz')), ((1, 'A'), ('reqid', 'baz'))]) self.assertEquals(self.A.instance, 3)
def test_proposal_advocate_retry_with_crash_recovery(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq self.B.net.link_up = False self.B.advocate.retry_delay = 0.01 self.B.set_proposal('reqid', 'foobar') self.B.persist() yield delay(0.03) self.assertTrue(not self.A.dresolution.called) self.fail_node('B') yield self.recover_node('B') d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution]) r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar'))])
def test_accept_nack(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution]) self.A.net.link_up = False self.A.set_proposal('reqid', 'foobar') self.assertEquals(self.A.pax.next_proposal_number, 2) self.assertTrue(self.A.pax.leader) self.A.receive_accept_nack( 'B', dict(proposal_id=self.A.pax.proposal_id, instance=1, promised_id=(2, 'B'))) self.A.receive_accept_nack( 'C', dict(proposal_id=self.A.pax.proposal_id, instance=1, promised_id=(2, 'B'))) self.assertEquals(self.A.pax.next_proposal_number, 3) self.assertTrue(not self.A.pax.leader)
def test_simple_commit(self): da = self.A.propose_result('tx1', 'commit') db = self.B.propose_result('tx1', 'commit') r = yield gatherResults([da, db]) self.assertEquals(r, [('tx1','committed'), ('tx1','committed')])
def test_accept_nack(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution] ) self.A.net.link_up = False self.A.set_proposal( 'reqid', 'foobar' ) self.assertEquals( self.A.pax.next_proposal_number, 2 ) self.assertTrue( self.A.pax.leader ) self.A.receive_accept_nack( 'B', dict(proposal_id=self.A.pax.proposal_id, instance=1, promised_id=(2, 'B'))) self.A.receive_accept_nack( 'C', dict(proposal_id=self.A.pax.proposal_id, instance=1, promised_id=(2, 'B'))) self.assertEquals( self.A.pax.next_proposal_number, 3 ) self.assertTrue( not self.A.pax.leader )
def test_simple_commit(self): da = self.A.propose_result('tx1', 'commit') db = self.B.propose_result('tx1', 'commit') r = yield gatherResults([da, db]) self.assertEquals(r, [('tx1', 'committed'), ('tx1', 'committed')])
def test_resolution_with_leadership_failure_and_isolated_node(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq d = gatherResults( [self.B.dresolution, self.C.dresolution] ) self.A.net.link_up = False self.B.net.link_up = False self.B.advocate.retry_delay = 0.01 self.B.set_proposal( 'reqid', 'foobar' ) yield delay( 0.05 ) self.assertTrue( not self.A.dresolution.called ) self.B.net.link_up = True r = yield d self.assertTrue(r in ( [((1, 'B'), ('reqid', 'foobar')), ((1, 'B'), ('reqid', 'foobar'))], [((1, 'C'), ('reqid', 'foobar')), ((1, 'C'), ('reqid', 'foobar'))]) )
def test_proposal_advocate_retry_with_crash_recovery(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq self.B.net.link_up = False self.B.advocate.retry_delay = 0.01 self.B.set_proposal( 'reqid', 'foobar' ) self.B.persist() yield delay( 0.03 ) self.assertTrue( not self.A.dresolution.called ) self.fail_node( 'B' ) yield self.recover_node( 'B' ) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution] ) r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar'))] )
def test_resolution_with_leadership_failure_and_isolated_node(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq d = gatherResults([self.B.dresolution, self.C.dresolution]) self.A.net.link_up = False self.B.net.link_up = False self.B.advocate.retry_delay = 0.01 self.B.set_proposal('reqid', 'foobar') yield delay(0.05) self.assertTrue(not self.A.dresolution.called) self.B.net.link_up = True r = yield d self.assertTrue(r in ([((1, 'B'), ('reqid', 'foobar')), ((1, 'B'), ('reqid', 'foobar'))], [((1, 'C'), ('reqid', 'foobar')), ((1, 'C'), ('reqid', 'foobar'))]))
def test_behind_in_sequence(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq self.assertEquals( self.A.instance, 1 ) self.B.net.link_up = False d = gatherResults( [self.A.dresolution, self.C.dresolution] ) self.A.set_proposal( 'reqid', 'foobar' ) r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar'))] ) self.assertEquals( self.A.instance, 2 ) self.assertEquals( self.B.instance, 1 ) self.B.net.link_up = True yield delay(0.05) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution] ) self.A.set_proposal( 'reqid', 'baz' ) r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'baz')), ((1, 'A'), ('reqid', 'baz')), ((1, 'A'), ('reqid', 'baz'))] ) self.assertEquals( self.A.instance, 3 ) self.assertEquals( self.B.instance, 3 )
def test_non_leader_resolution(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution]) self.B.set_proposal('reqid', 'foobar') r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar'))])
def test_simple_abort(self): self.A.net.link_up = False da = self.A.propose_result('tx1', 'commit') self.A.net.link_up = True self.A.heartbeat(6, True) db = self.B.propose_result('tx1', 'commit') dc = self.C.propose_result('tx1', 'commit') r = yield gatherResults([da, db, dc]) self.assertEquals(r, [('tx1','aborted'), ('tx1','aborted'), ('tx1','aborted')])
def test_behind_in_sequence(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq self.assertEquals(self.A.instance, 1) self.B.net.link_up = False d = gatherResults([self.A.dresolution, self.C.dresolution]) self.A.set_proposal('reqid', 'foobar') r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar'))]) self.assertEquals(self.A.instance, 2) self.assertEquals(self.B.instance, 1) self.B.net.link_up = True yield delay(0.05) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution]) self.A.set_proposal('reqid', 'baz') r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'baz')), ((1, 'A'), ('reqid', 'baz')), ((1, 'A'), ('reqid', 'baz'))]) self.assertEquals(self.A.instance, 3) self.assertEquals(self.B.instance, 3)
def test_simple_abort(self): self.A.net.link_up = False da = self.A.propose_result('tx1', 'commit') self.A.net.link_up = True self.A.heartbeat(6, True) db = self.B.propose_result('tx1', 'commit') dc = self.C.propose_result('tx1', 'commit') r = yield gatherResults([da, db, dc]) self.assertEquals(r, [('tx1', 'aborted'), ('tx1', 'aborted'), ('tx1', 'aborted')])
def test_leader_resolution(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution] ) self.A.set_proposal( 'reqid', 'foobar' ) r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar'))] )
def test_complex_abort(self): da = self.A.propose_result('tx1', 'commit') self.assertEquals( self.A.get_transaction('tx1').num_committed, 1 ) self.B.net.link_up = False self.C.net.link_up = False db = self.B.propose_result('tx1', 'commit') dc = self.C.propose_result('tx1', 'commit') self.B.net.link_up = True self.C.net.link_up = True self.A.heartbeat(6, True) r = yield gatherResults([da, db, dc]) self.assertEquals(r, [('tx1','aborted'), ('tx1','aborted'), ('tx1','aborted')])
def test_complex_abort(self): da = self.A.propose_result('tx1', 'commit') self.assertEquals(self.A.get_transaction('tx1').num_committed, 1) self.B.net.link_up = False self.C.net.link_up = False db = self.B.propose_result('tx1', 'commit') dc = self.C.propose_result('tx1', 'commit') self.B.net.link_up = True self.C.net.link_up = True self.A.heartbeat(6, True) r = yield gatherResults([da, db, dc]) self.assertEquals(r, [('tx1', 'aborted'), ('tx1', 'aborted'), ('tx1', 'aborted')])
def test_multiple_instances_with_crash_recovery(self): self.A.pax.acquire_leadership() yield self.A.dleader_acq self.assertEquals( self.A.instance, 1 ) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution] ) self.A.set_proposal( 'reqid', 'foobar' ) r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar'))] ) self.assertEquals( self.A.instance, 2 ) for uid in 'ABC': self.fail_and_recover(uid, False) self.A.net.link_up = True self.B.net.link_up = True self.C.net.link_up = True #------------------------------------------------ # Recovery will return to the previous instance. We'll # need to re-resolve the same result. # self.assertEquals( self.A.instance, 1 ) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution] ) r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar')), ((1, 'A'), ('reqid', 'foobar'))] ) #------------------------------------------------ # Back to instance 2 # self.assertEquals( self.A.instance, 2 ) d = gatherResults( [self.A.dresolution, self.B.dresolution, self.C.dresolution] ) self.A.set_proposal( 'reqid', 'baz' ) r = yield d self.assertEquals(r, [((1, 'A'), ('reqid', 'baz')), ((1, 'A'), ('reqid', 'baz')), ((1, 'A'), ('reqid', 'baz'))] ) self.assertEquals( self.A.instance, 3 )