def test_new_peer_replace_old(self): # make sure that if all original peers crash the committee can proceed
peers = make_sawtooth_committee(7)
blockchain_size = 1
for i in range(13):
peers.append(SawtoothContainer())
peers[-1].join_sawtooth([p.ip() for p in peers])
# wait for blockchain catch up
done = False
start = time.time()
while not done:
if len(peers[-1].blocks()['data']) >= blockchain_size - 1: # catch up cant get last tx
done = True
elif time.time() - start > 30:
self.fail("New peers block catch up failed")
peers[i % 4].update_committee([p.val_key() for p in peers])
self.assertTrue(check_for_confirmation(peers, blockchain_size))
blockchain_size += 1
for _ in range(4):
del peers[0]
gc.collect() # make sure that the containers are shutdown
time.sleep(VIEW_CHANGE_WAIT_TIME_SEC + 30) # wait for view change (timeout + time to do the view change)
peers[0].submit_tx('test', '999')
blockchain_size += 1
self.assertTrue(check_for_confirmation(peers, blockchain_size, 'test'))
for p in peers:
print(p.ip())
self.assertEqual(blockchain_size, len(p.blocks()['data']))
def test_peer_leave(self):
intersections = make_peer_committees(7)
number_of_tx = 1
id_a = intersections[0].committee_id_a
id_b = intersections[0].committee_id_b
old_peer = intersections.pop()
intersections[0].update_committee(id_a, [inter.val_key(id_a) for inter in intersections])
intersections[0].update_committee(id_b, [inter.val_key(id_b) for inter in intersections])
number_of_tx += 1
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx))
self.assertTrue(check_for_confirmation([i.instance_b for i in intersections], number_of_tx))
for inter in intersections:
self.assertEqual(number_of_tx, len(inter.blocks(id_a)))
self.assertEqual(number_of_tx, len(inter.blocks(id_b)))
del old_peer
tx_a = Transaction(id_a, 1)
tx_a.key = 'test'
tx_a.value = '999'
tx_b = Transaction(id_b, 1)
tx_b.key = 'test'
tx_b.value = '888'
# check consensus still works
intersections[-1].submit(tx_a)
intersections[-1].submit(tx_b)
number_of_tx += 1
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx, 'test'))
self.assertTrue(check_for_confirmation([i.instance_b for i in intersections], number_of_tx, 'test'))
for inter in intersections:
self.assertEqual(number_of_tx, len(inter.blocks(id_a)))
self.assertEqual(number_of_tx, len(inter.blocks(id_b)))
def test_concurrent_large_committee(self):
# if this fails then the following setting may need to be set in /etc/sysctl.conf
#
# # Setup DNS threshold for arp
# net.ipv4.neigh.default.gc_thresh3 = 16384
# net.ipv4.neigh.default.gc_thresh2 = 8192
# net.ipv4.neigh.default.gc_thresh1 = 4096
# this should work if default linux settings are being used
peers1 = make_sawtooth_committee(30)
peers2 = make_sawtooth_committee(30)
number_of_tx = 1 # genesis
peers1[0].submit_tx('test', '999')
peers2[0].submit_tx('test', '999')
number_of_tx += 1
time.sleep(0.5)
peers1[1].submit_tx('test1', '888')
peers2[1].submit_tx('test1', '888')
number_of_tx += 1
time.sleep(0.5)
peers1[2].submit_tx('test2', '777')
peers2[2].submit_tx('test2', '777')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers1, number_of_tx, 'test2'))
self.assertTrue(check_for_confirmation(peers2, number_of_tx, 'test2'))
for p in peers1:
blocks = p.blocks()['data']
self.assertEqual(number_of_tx, len(blocks), p.ip())
for p in peers2:
blocks = p.blocks()['data']
self.assertEqual(number_of_tx, len(blocks), p.ip())
def test_peer_join(self):
peers = make_sawtooth_committee(7)
number_of_tx = 1 # genesis
peers[0].submit_tx('test', '999')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test'))
peers[1].submit_tx('test1', '888')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test1'))
peers.append(SawtoothContainer())
peers[-1].join_sawtooth([p.ip() for p in peers])
# wait for block catch up
done = False
start = time.time()
while not done:
if len(peers[-1].blocks()['data']) >= number_of_tx - 1: # catch up cant get last tx
done = True
elif time.time() - start > 30:
self.fail("New peers block catch up failed")
peers[0].update_committee([p.val_key() for p in peers])
number_of_tx += 1 # +1 for membership of new peer
time.sleep(VIEW_CHANGE_WAIT_TIME_SEC) # wait in case leader has to change
# submit new tx with new peer so it can finally match all the others (i.e. has full blockchain)
peers[2].submit_tx('test2', '777')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test2'))
# makes sure all peers are configured to work with each other (this is not a test of connectivity just config)
# and make sure they all have the whole blockchain
ips = [p.ip() for p in peers]
for p in peers:
peers_blocks = len(p.blocks()['data'])
self.assertEqual(number_of_tx, peers_blocks, p.ip())
peers_config = p.sawtooth_api('http://localhost:8008/peers')['data']
for ip in ips:
if ip != p.ip(): # the peer it's self is not reported in the list
self.assertIn("tcp://{}:8800".format(ip), peers_config)
# check consensus still works
peers[-1].submit_tx('test3', '666')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test3'))
for p in peers:
peers_blockchain = len(p.blocks()['data'])
self.assertEqual(number_of_tx, peers_blockchain, p.ip())
def test_committee_independent_leave(self):
intersections = make_peer_committees(7)
number_of_tx_a = 1
number_of_tx_b = 1
id_a = intersections[0].committee_id_a
id_b = intersections[0].committee_id_b
# we need to drop only one instance make sure other committee is unaffected
old_instance = intersections[-1].instance_b
committee_val_b = [inter.val_key(id_b) for inter in intersections[:-1]]
intersections[0].update_committee(id_b, committee_val_b)
self.assertTrue(check_for_confirmation([i.instance_b for i in intersections], number_of_tx_b))
number_of_tx_b += 1
del old_instance
gc.collect()
intersections[-1].instance_b = None
intersections[-1].committee_id_b = None
tx_a = Transaction(id_a, 1)
tx_a.key = 'test'
tx_a.value = '999'
tx_b = Transaction(id_b, 1)
tx_b.key = 'test'
tx_b.value = '888'
intersections[-1].submit(tx_a)
number_of_tx_a += 1
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx_a, 'test'))
intersections[0].submit(tx_b)
number_of_tx_b += 1
self.assertTrue(check_for_confirmation([i.instance_b for i in intersections[:-1]], number_of_tx_b, 'test'))
for inter in intersections:
self.assertEqual(number_of_tx_a, len(inter.blocks(id_a)))
for inter in intersections[:-1]:
self.assertEqual(number_of_tx_b, len(inter.blocks(id_b)))
self.assertEqual(None, intersections[-1].blocks(id_b))
# confirm membership
committee_b = intersections.copy()[:-1]
committee_ips_b = [inter.ip(id_b) for inter in intersections[:-1]]
for inter in committee_b:
intersections_config = inter.sawtooth_api(id_b, 'http://localhost:8008/peers')['data']
for ip in committee_ips_b:
if ip != inter.ip(id_b): # the peer it's self is not reported in the list
self.assertIn("tcp://{}:8800".format(ip), intersections_config)
def test_committee_churn(self):
self.skipTest("Complete committee churn not supported")
peers = make_sawtooth_committee(7)
blockchain_size = 1
peers[0].submit_tx('start', '1')
blockchain_size += 1
self.assertTrue(check_for_confirmation(peers, blockchain_size, 'start'))
for i in range(7):
# add new peer
peers.append(SawtoothContainer())
peers[-1].join_sawtooth([p.ip() for p in peers])
# wait for blockchain catch up
done = False
start = time.time()
while not done:
if len(peers[-1].blocks()['data']) >= blockchain_size - 1: # catch up cant get last tx
done = True
elif time.time() - start > 30:
self.fail("New peers block catch up failed")
peers[0].update_committee([p.val_key() for p in peers])
peers[0].submit_tx("update_{}".format(i), 999)
blockchain_size += 1
self.assertTrue(check_for_confirmation(peers, blockchain_size, "update_{}".format(i),
timeout=VIEW_CHANGE_WAIT_TIME_SEC))
blockchain_size += 1
for p in peers:
self.assertEqual(blockchain_size, len(p.blocks()['data']))
old_peer = peers.pop(0)
peers[0].update_committee([p.val_key() for p in peers])
self.assertTrue(check_for_confirmation(peers, blockchain_size))
blockchain_size += 1
self.assertTrue(check_for_confirmation(peers, blockchain_size, 'test_{}'.format(i)))
for p in peers:
self.assertEqual(blockchain_size, len(p.blocks()['data']))
del old_peer
gc.collect() # make sure that the containers are shutdown
peers[0].submit_tx('test', '999')
blockchain_size += 1
self.assertTrue(check_for_confirmation(peers, blockchain_size, 'test'))
for p in peers:
self.assertEqual(blockchain_size, len(p.blocks()['data']))
self.assertEqual('999', p.get_tx('test'))
def test_concurrent_committees(self):
set_a = make_sawtooth_committee(20)
set_b = make_sawtooth_committee(20)
tx_a = 'test_a'
tx_b = 'test_b'
set_a[0].submit_tx(tx_a, '999')
set_b[0].submit_tx(tx_b, '888')
self.assertTrue(check_for_confirmation(set_a, 2, tx_a))
self.assertTrue(check_for_confirmation(set_b, 2, tx_b))
for p in set_a:
self.assertEqual(2, len(p.blocks()['data']))
for p in set_b:
self.assertEqual(2, len(p.blocks()['data']))
set_b[0].submit_tx('test_b_2', '777')
self.assertTrue(check_for_confirmation(set_b, 3, tx_b))
def test_committee_shrink(self):
peers = make_sawtooth_committee(15)
blockchain_size = 1
for i in range(11):
old_peer = peers.pop()
peers[0].update_committee([p.val_key() for p in peers])
self.assertTrue(check_for_confirmation(peers, blockchain_size))
blockchain_size += 1
time.sleep(VIEW_CHANGE_WAIT_TIME_SEC) # wait for potential view change
del old_peer
gc.collect() # make sure that the containers are shutdown
peers[0].submit_tx('test', '999')
blockchain_size += 1
self.assertTrue(check_for_confirmation(peers, blockchain_size, 'test'))
for p in peers:
self.assertEqual(blockchain_size, len(p.blocks()['data']))
self.assertEqual('999', p.get_tx('test'))
def test_committee_growth(self):
peers = make_sawtooth_committee(7)
number_of_tx = 1
peers[0].submit_tx('test', '999')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test', timeout=120))
peers[1].submit_tx('test1', '888')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test1', timeout=120))
for i in range(13):
peers.append(SawtoothContainer())
peers[-1].join_sawtooth([p.ip() for p in peers])
# wait for blockchain catch up
done = False
start = time.time()
while not done:
if len(peers[-1].blocks()['data']) >= number_of_tx - 1: # catch up cant get last tx
done = True
elif time.time() - start > 30:
self.fail("New peers block catch up failed")
peers[i].update_committee([p.val_key() for p in peers])
self.assertTrue(check_for_confirmation(peers, number_of_tx, timeout=120))
number_of_tx += 1 # +1 for membership of new peer
# check consensus still works
peers[i].submit_tx('test_{}'.format(i), '777')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test_{}'.format(i), timeout=120))
# makes sure all peers are configured to work with each other (this only tests config not connectivity)
ips = [p.ip() for p in peers]
for p in peers:
peers_blocks = len(p.blocks()['data'])
self.assertGreaterEqual(number_of_tx, peers_blocks)
peers_config = p.sawtooth_api('http://localhost:8008/peers')['data']
for ip in ips:
if ip != p.ip(): # the peer it's self is not reported in the list
self.assertIn("tcp://{}:8800".format(ip), peers_config)
def test_peer_leave(self):
peers = make_sawtooth_committee(8)
blockchain_size = 1
old_peer = peers.pop()
peers[0].update_committee([p.val_key() for p in peers])
self.assertTrue(check_for_confirmation(peers, blockchain_size))
blockchain_size += 1
for p in peers:
self.assertEqual(blockchain_size, len(p.blocks()['data']))
del old_peer
gc.collect() # make sure that the containers are shutdown
# make sure consensus still works
peers[0].submit_tx('test', '999')
blockchain_size += 1
self.assertTrue(check_for_confirmation(peers, blockchain_size, 'test'))
for p in peers:
self.assertEqual('999', p.get_tx('test'))
self.assertEqual(blockchain_size, len(p.blocks()['data']))
def test_high_transaction_load(self):
peers = make_sawtooth_committee(5)
number_of_tx = 1
i = 0
for _ in range(10): # peers start to drop old blocks at 100
peers[i].submit_tx('test{}'.format(number_of_tx), '999')
i = i + 1 if i < (len(peers) - 1) else 0 # cycle peers
number_of_tx += 1
time.sleep(1.5) # prevent DOS attack counter measure
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test{}'.format(number_of_tx - 1)))
for p in peers:
self.assertEqual(number_of_tx, len(p.blocks()['data']))
def test_committee_independent_confirmation(self):
intersections = make_peer_committees(5)
number_of_tx = 1
id_a = intersections[0].committee_id_a
id_b = intersections[0].committee_id_b
number_of_tx_a = number_of_tx
number_of_tx_b = number_of_tx
tx_a = Transaction(id_a, 1)
tx_a.key = 'test2'
tx_a.value = '999'
intersections[-1].submit(tx_a)
number_of_tx_a += 1
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx, 'test2'))
for inter in intersections:
a_blocks = len(inter.blocks(id_a))
b_blocks = len(inter.blocks(id_b))
self.assertEqual(number_of_tx_a, a_blocks)
self.assertEqual(number_of_tx_b, b_blocks)
def test_fault_tolerance(self):
peers = make_sawtooth_committee(9)
number_of_tx = 1 # genesis
del peers[0]
peers[0].submit_tx('test', '999')
number_of_tx += 1
# can take some time for peers to commit (potentially 5 min for failed leader)
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test{}'.format(number_of_tx - 1),
timeout=VIEW_CHANGE_WAIT_TIME_SEC * len(peers)))
for p in peers:
self.assertEqual(number_of_tx, len(p.blocks()['data']), "Peers did not commit tx in time")
# should fail now
del peers[:5]
peers[0].submit_tx('fail', '000')
time.sleep(VIEW_CHANGE_WAIT_TIME_SEC * len(peers)) # give it plenty of time to be confirmed + 30 for buffer
for p in peers:
peers_blockchain = len(p.blocks()['data'])
self.assertEqual(number_of_tx, peers_blockchain, p.ip())
def test_large_committee(self):
peers = make_sawtooth_committee(30)
number_of_tx = 1 # genesis
peers[0].submit_tx('test', '999')
number_of_tx += 1
time.sleep(0.5)
peers[1].submit_tx('test1', '888')
number_of_tx += 1
time.sleep(0.5)
peers[2].submit_tx('test2', '777')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test2'))
for p in peers:
blocks = p.blocks()['data']
self.assertEqual(number_of_tx, len(blocks), p.ip())
def test_peer_join(self):
intersections = make_peer_committees(7)
id_a = intersections[0].committee_id_a
id_b = intersections[0].committee_id_b
intersections[0].submit(Transaction(id_a, 'update_a_1', 999))
number_of_tx = 2
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx, 'update_a_1', ))
intersections[0].submit(Transaction(id_b, 'update_b_1', 999))
self.assertTrue(check_for_confirmation([i.instance_b for i in intersections], number_of_tx, 'update_b_1', ))
intersections.append(Intersection(SawtoothContainer(), SawtoothContainer(), id_a, id_b))
committee_ips_a = [inter.instance_a.ip() for inter in intersections]
committee_ips_b = [inter.instance_b.ip() for inter in intersections]
intersections[-1].peer_join(id_a, committee_ips_a)
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx - 1)) # catch up
intersections[0].update_committee(id_a, [inter.instance_a.val_key() for inter in intersections])
number_of_tx += 1
intersections[0].submit(Transaction(id_a, 'update_a_2', 888))
number_of_tx += 1
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx, 'update_a_2'))
intersections[-1].peer_join(id_b, committee_ips_b)
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx - 1)) # catch up
intersections[0].update_committee(id_b, [inter.instance_b.val_key() for inter in intersections])
intersections[0].submit(Transaction(id_b, 'update_b_2', 888))
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx, 'update_b_2'))
# makes sure all intersections are configured to work with each other (this is not a test of connectivity
# just config) and make sure they all have the three tx
for inter in intersections:
intersections_config = inter.sawtooth_api(id_a, 'http://localhost:8008/peers')['data']
for ip in committee_ips_a:
if ip != inter.ip(id_a): # the peer it's self is not reported in the list
self.assertIn("tcp://{}:8800".format(ip), intersections_config)
intersections_config = inter.sawtooth_api(id_b, 'http://localhost:8008/peers')['data']
for ip in committee_ips_b:
if ip != inter.ip(id_b):
self.assertIn("tcp://{}:8800".format(ip), intersections_config)
def test_committee_independent_join(self):
intersections = make_peer_committees(7)
number_of_tx_a = 1
id_a = intersections[0].committee_id_a
new_peer = Intersection(SawtoothContainer(), None, id_a, None)
committee_ips_a = [inter.ip(id_a) for inter in intersections]
committee_ips_a.append(new_peer.ip(id_a))
committee_val_a = [inter.val_key(id_a) for inter in intersections]
committee_val_a.append(new_peer.val_key(id_a))
intersections[0].submit(Transaction(id_a, 'update_a_1', 999))
number_of_tx_a += 1
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx_a, 'update_a_1'))
new_peer.peer_join(id_a, committee_ips_a)
intersections[0].update_committee(id_a, committee_val_a)
number_of_tx_a += 1
intersections[0].submit(Transaction(id_a, 'update_a_2', 888))
number_of_tx_a += 1
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx_a, 'update_a_2'))
# confirm membership
committee_a = intersections.copy()
committee_a.append(new_peer)
self.assertEqual(None, new_peer.committee_id_b)
for inter in committee_a:
intersections_config = inter.sawtooth_api(id_a, 'http://localhost:8008/peers')['data']
for ip in committee_ips_a:
if ip != inter.ip(id_a): # the peer it's self is not reported in the list
self.assertIn("tcp://{}:8800".format(ip), intersections_config)
for inter in committee_a:
self.assertEqual(number_of_tx_a, len(inter.blocks(id_a)))
# now committee B
id_b = intersections[0].committee_id_b
number_of_tx_b = 1
new_peer = Intersection(None, SawtoothContainer(), None, id_b)
committee_ips_b = [inter.ip(id_b) for inter in intersections]
committee_ips_b.append(new_peer.ip(id_b))
committee_val_b = [inter.val_key(id_b) for inter in intersections]
committee_val_b.append(new_peer.val_key(id_b))
intersections[0].submit(Transaction(id_b, 'update_b_1', 999))
number_of_tx_b += 1
self.assertTrue(check_for_confirmation([i.instance_b for i in intersections], number_of_tx_b, 'update_b_1'))
new_peer.peer_join(id_b, committee_ips_b)
intersections[0].update_committee(id_b, committee_val_b)
number_of_tx_b += 1
self.assertTrue(check_for_confirmation([i.instance_b for i in intersections], number_of_tx_b))
intersections[0].submit(Transaction(id_b, 'update_b_2', 888))
number_of_tx_b += 1
self.assertTrue(check_for_confirmation([i.instance_b for i in intersections], number_of_tx_b, 'update_b_2'))
# confirm membership
committee_b = intersections.copy()
committee_b.append(new_peer)
for inter in committee_b:
intersections_config = inter.sawtooth_api(id_b, 'http://localhost:8008/peers')['data']
for ip in committee_ips_b:
if ip != inter.ip(id_b): # the peer it's self is not reported in the list
self.assertIn("tcp://{}:8800".format(ip), intersections_config)
for inter in committee_b:
self.assertEqual(number_of_tx_b, len(inter.blocks(id_b)))
