def test_multiple_instances(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
self.assertEquals(self.A.instance, 1)
d = gatherResults(
[self.A.dresolution, self.B.dresolution, self.C.dresolution])
self.A.set_proposal('reqid', 'foobar')
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar'))])
self.assertEquals(self.A.instance, 2)
d = gatherResults(
[self.A.dresolution, self.B.dresolution, self.C.dresolution])
self.A.set_proposal('reqid', 'baz')
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'baz')),
((1, 'A'), ('reqid', 'baz')),
((1, 'A'), ('reqid', 'baz'))])
self.assertEquals(self.A.instance, 3)
def test_multiple_instances(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
self.assertEquals( self.A.instance, 1 )
d = gatherResults( [self.A.dresolution,
self.B.dresolution,
self.C.dresolution] )
self.A.set_proposal( 'reqid', 'foobar' )
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar'))] )
self.assertEquals( self.A.instance, 2 )
d = gatherResults( [self.A.dresolution,
self.B.dresolution,
self.C.dresolution] )
self.A.set_proposal( 'reqid', 'baz' )
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'baz')),
((1, 'A'), ('reqid', 'baz')),
((1, 'A'), ('reqid', 'baz'))] )
self.assertEquals( self.A.instance, 3 )
def test_multiple_instances_with_crash_recovery(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
self.assertEquals(self.A.instance, 1)
d = gatherResults(
[self.A.dresolution, self.B.dresolution, self.C.dresolution])
self.A.set_proposal('reqid', 'foobar')
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar'))])
self.assertEquals(self.A.instance, 2)
for uid in 'ABC':
self.fail_and_recover(uid, False)
self.A.net.link_up = True
self.B.net.link_up = True
self.C.net.link_up = True
#------------------------------------------------
# Recovery will return to the previous instance. We'll
# need to re-resolve the same result.
#
self.assertEquals(self.A.instance, 1)
d = gatherResults(
[self.A.dresolution, self.B.dresolution, self.C.dresolution])
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar'))])
#------------------------------------------------
# Back to instance 2
#
self.assertEquals(self.A.instance, 2)
d = gatherResults(
[self.A.dresolution, self.B.dresolution, self.C.dresolution])
self.A.set_proposal('reqid', 'baz')
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'baz')),
((1, 'A'), ('reqid', 'baz')),
((1, 'A'), ('reqid', 'baz'))])
self.assertEquals(self.A.instance, 3)
def test_proposal_advocate_retry_with_crash_recovery(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
self.B.net.link_up = False
self.B.advocate.retry_delay = 0.01
self.B.set_proposal('reqid', 'foobar')
self.B.persist()
yield delay(0.03)
self.assertTrue(not self.A.dresolution.called)
self.fail_node('B')
yield self.recover_node('B')
d = gatherResults(
[self.A.dresolution, self.B.dresolution, self.C.dresolution])
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar'))])
def test_accept_nack(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
d = gatherResults(
[self.A.dresolution, self.B.dresolution, self.C.dresolution])
self.A.net.link_up = False
self.A.set_proposal('reqid', 'foobar')
self.assertEquals(self.A.pax.next_proposal_number, 2)
self.assertTrue(self.A.pax.leader)
self.A.receive_accept_nack(
'B',
dict(proposal_id=self.A.pax.proposal_id,
instance=1,
promised_id=(2, 'B')))
self.A.receive_accept_nack(
'C',
dict(proposal_id=self.A.pax.proposal_id,
instance=1,
promised_id=(2, 'B')))
self.assertEquals(self.A.pax.next_proposal_number, 3)
self.assertTrue(not self.A.pax.leader)
def test_simple_commit(self):
da = self.A.propose_result('tx1', 'commit')
db = self.B.propose_result('tx1', 'commit')
r = yield gatherResults([da, db])
self.assertEquals(r, [('tx1','committed'), ('tx1','committed')])
def test_accept_nack(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
d = gatherResults( [self.A.dresolution,
self.B.dresolution,
self.C.dresolution] )
self.A.net.link_up = False
self.A.set_proposal( 'reqid', 'foobar' )
self.assertEquals( self.A.pax.next_proposal_number, 2 )
self.assertTrue( self.A.pax.leader )
self.A.receive_accept_nack( 'B', dict(proposal_id=self.A.pax.proposal_id,
instance=1,
promised_id=(2, 'B')))
self.A.receive_accept_nack( 'C', dict(proposal_id=self.A.pax.proposal_id,
instance=1,
promised_id=(2, 'B')))
self.assertEquals( self.A.pax.next_proposal_number, 3 )
self.assertTrue( not self.A.pax.leader )
def test_simple_commit(self):
da = self.A.propose_result('tx1', 'commit')
db = self.B.propose_result('tx1', 'commit')
r = yield gatherResults([da, db])
self.assertEquals(r, [('tx1', 'committed'), ('tx1', 'committed')])
def test_resolution_with_leadership_failure_and_isolated_node(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
d = gatherResults( [self.B.dresolution,
self.C.dresolution] )
self.A.net.link_up = False
self.B.net.link_up = False
self.B.advocate.retry_delay = 0.01
self.B.set_proposal( 'reqid', 'foobar' )
yield delay( 0.05 )
self.assertTrue( not self.A.dresolution.called )
self.B.net.link_up = True
r = yield d
self.assertTrue(r in ( [((1, 'B'), ('reqid', 'foobar')),
((1, 'B'), ('reqid', 'foobar'))],
[((1, 'C'), ('reqid', 'foobar')),
((1, 'C'), ('reqid', 'foobar'))]) )
def test_proposal_advocate_retry_with_crash_recovery(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
self.B.net.link_up = False
self.B.advocate.retry_delay = 0.01
self.B.set_proposal( 'reqid', 'foobar' )
self.B.persist()
yield delay( 0.03 )
self.assertTrue( not self.A.dresolution.called )
self.fail_node( 'B' )
yield self.recover_node( 'B' )
d = gatherResults( [self.A.dresolution,
self.B.dresolution,
self.C.dresolution] )
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar'))] )
def test_resolution_with_leadership_failure_and_isolated_node(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
d = gatherResults([self.B.dresolution, self.C.dresolution])
self.A.net.link_up = False
self.B.net.link_up = False
self.B.advocate.retry_delay = 0.01
self.B.set_proposal('reqid', 'foobar')
yield delay(0.05)
self.assertTrue(not self.A.dresolution.called)
self.B.net.link_up = True
r = yield d
self.assertTrue(r in ([((1, 'B'),
('reqid',
'foobar')), ((1, 'B'), ('reqid', 'foobar'))],
[((1, 'C'),
('reqid',
'foobar')), ((1, 'C'), ('reqid', 'foobar'))]))
def test_behind_in_sequence(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
self.assertEquals( self.A.instance, 1 )
self.B.net.link_up = False
d = gatherResults( [self.A.dresolution,
self.C.dresolution] )
self.A.set_proposal( 'reqid', 'foobar' )
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar'))] )
self.assertEquals( self.A.instance, 2 )
self.assertEquals( self.B.instance, 1 )
self.B.net.link_up = True
yield delay(0.05)
d = gatherResults( [self.A.dresolution,
self.B.dresolution,
self.C.dresolution] )
self.A.set_proposal( 'reqid', 'baz' )
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'baz')),
((1, 'A'), ('reqid', 'baz')),
((1, 'A'), ('reqid', 'baz'))] )
self.assertEquals( self.A.instance, 3 )
self.assertEquals( self.B.instance, 3 )
def test_non_leader_resolution(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
d = gatherResults(
[self.A.dresolution, self.B.dresolution, self.C.dresolution])
self.B.set_proposal('reqid', 'foobar')
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar'))])
def test_simple_abort(self):
self.A.net.link_up = False
da = self.A.propose_result('tx1', 'commit')
self.A.net.link_up = True
self.A.heartbeat(6, True)
db = self.B.propose_result('tx1', 'commit')
dc = self.C.propose_result('tx1', 'commit')
r = yield gatherResults([da, db, dc])
self.assertEquals(r, [('tx1','aborted'), ('tx1','aborted'), ('tx1','aborted')])
def test_behind_in_sequence(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
self.assertEquals(self.A.instance, 1)
self.B.net.link_up = False
d = gatherResults([self.A.dresolution, self.C.dresolution])
self.A.set_proposal('reqid', 'foobar')
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar'))])
self.assertEquals(self.A.instance, 2)
self.assertEquals(self.B.instance, 1)
self.B.net.link_up = True
yield delay(0.05)
d = gatherResults(
[self.A.dresolution, self.B.dresolution, self.C.dresolution])
self.A.set_proposal('reqid', 'baz')
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'baz')),
((1, 'A'), ('reqid', 'baz')),
((1, 'A'), ('reqid', 'baz'))])
self.assertEquals(self.A.instance, 3)
self.assertEquals(self.B.instance, 3)
def test_simple_abort(self):
self.A.net.link_up = False
da = self.A.propose_result('tx1', 'commit')
self.A.net.link_up = True
self.A.heartbeat(6, True)
db = self.B.propose_result('tx1', 'commit')
dc = self.C.propose_result('tx1', 'commit')
r = yield gatherResults([da, db, dc])
self.assertEquals(r, [('tx1', 'aborted'), ('tx1', 'aborted'),
('tx1', 'aborted')])
def test_leader_resolution(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
d = gatherResults( [self.A.dresolution,
self.B.dresolution,
self.C.dresolution] )
self.A.set_proposal( 'reqid', 'foobar' )
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar'))] )
def test_complex_abort(self):
da = self.A.propose_result('tx1', 'commit')
self.assertEquals( self.A.get_transaction('tx1').num_committed, 1 )
self.B.net.link_up = False
self.C.net.link_up = False
db = self.B.propose_result('tx1', 'commit')
dc = self.C.propose_result('tx1', 'commit')
self.B.net.link_up = True
self.C.net.link_up = True
self.A.heartbeat(6, True)
r = yield gatherResults([da, db, dc])
self.assertEquals(r, [('tx1','aborted'), ('tx1','aborted'), ('tx1','aborted')])
def test_complex_abort(self):
da = self.A.propose_result('tx1', 'commit')
self.assertEquals(self.A.get_transaction('tx1').num_committed, 1)
self.B.net.link_up = False
self.C.net.link_up = False
db = self.B.propose_result('tx1', 'commit')
dc = self.C.propose_result('tx1', 'commit')
self.B.net.link_up = True
self.C.net.link_up = True
self.A.heartbeat(6, True)
r = yield gatherResults([da, db, dc])
self.assertEquals(r, [('tx1', 'aborted'), ('tx1', 'aborted'),
('tx1', 'aborted')])
def test_multiple_instances_with_crash_recovery(self):
self.A.pax.acquire_leadership()
yield self.A.dleader_acq
self.assertEquals( self.A.instance, 1 )
d = gatherResults( [self.A.dresolution,
self.B.dresolution,
self.C.dresolution] )
self.A.set_proposal( 'reqid', 'foobar' )
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar'))] )
self.assertEquals( self.A.instance, 2 )
for uid in 'ABC':
self.fail_and_recover(uid, False)
self.A.net.link_up = True
self.B.net.link_up = True
self.C.net.link_up = True
#------------------------------------------------
# Recovery will return to the previous instance. We'll
# need to re-resolve the same result.
#
self.assertEquals( self.A.instance, 1 )
d = gatherResults( [self.A.dresolution,
self.B.dresolution,
self.C.dresolution] )
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar')),
((1, 'A'), ('reqid', 'foobar'))] )
#------------------------------------------------
# Back to instance 2
#
self.assertEquals( self.A.instance, 2 )
d = gatherResults( [self.A.dresolution,
self.B.dresolution,
self.C.dresolution] )
self.A.set_proposal( 'reqid', 'baz' )
r = yield d
self.assertEquals(r, [((1, 'A'), ('reqid', 'baz')),
((1, 'A'), ('reqid', 'baz')),
((1, 'A'), ('reqid', 'baz'))] )
self.assertEquals( self.A.instance, 3 )
