def start(quorum_id_a=None, quorum_id_b=None):
if app.config[PBFT_INSTANCES] is not None:
app.logger.warning("peer has already started, restarting")
del app.config[PBFT_INSTANCES]
instance_a = SawtoothContainer()
instance_b = SawtoothContainer()
app.config[PBFT_INSTANCES] = Intersection(instance_a, instance_b,
quorum_id_a, quorum_id_b)
app.config[QUORUMS][quorum_id_a] = []
app.config[QUORUMS][quorum_id_b] = []
return ROUTE_EXECUTED_CORRECTLY
def make_sawtooth_committee(size: int, network=DEFAULT_DOCKER_NETWORK):
if size < 4:
logging.error(
"COMMITTEE IMPOSSIBLE: can not make committees of less then 4 members, {} asked for"
.format(size))
return []
if size < 7:
logging.warning(
"COMMITTEE UNSTABLE: making committees of less then 7 members can lead to issues with adding "
"and removing. ")
peers = [SawtoothContainer(network) for _ in range(size)]
peers[0].make_genesis([p.val_key() for p in peers],
[p.user_key() for p in peers])
committee_ips = [p.ip() for p in peers]
for p in peers:
p.join_sawtooth(committee_ips)
# if the there are a lot of containers running wait longer for process to start
time.sleep(5 * size)
done = False
while not done:
done = True
for p in peers:
if len(p.blocks()['data']) < 1:
logging.info("Peer {ip} could not get genesis block\n"
" blocks:{b}".format(ip=p.ip(),
b=p.blocks()['data']))
done = False
time.sleep(0.5)
break
return peers
def test_new_peer_replace_old(self): # make sure that if all original peers crash the committee can proceed
peers = make_sawtooth_committee(7)
blockchain_size = 1
for i in range(13):
peers.append(SawtoothContainer())
peers[-1].join_sawtooth([p.ip() for p in peers])
# wait for blockchain catch up
done = False
start = time.time()
while not done:
if len(peers[-1].blocks()['data']) >= blockchain_size - 1: # catch up cant get last tx
done = True
elif time.time() - start > 30:
self.fail("New peers block catch up failed")
peers[i % 4].update_committee([p.val_key() for p in peers])
self.assertTrue(check_for_confirmation(peers, blockchain_size))
blockchain_size += 1
for _ in range(4):
del peers[0]
gc.collect() # make sure that the containers are shutdown
time.sleep(VIEW_CHANGE_WAIT_TIME_SEC + 30) # wait for view change (timeout + time to do the view change)
peers[0].submit_tx('test', '999')
blockchain_size += 1
self.assertTrue(check_for_confirmation(peers, blockchain_size, 'test'))
for p in peers:
print(p.ip())
self.assertEqual(blockchain_size, len(p.blocks()['data']))
def test_start_with_peer(self):
a = SawtoothContainer()
b = SawtoothContainer()
id_a = 'a'
id_b = 'b'
p = Intersection(a, b, id_a, id_b)
app = api.create_app(p)
app.config['TESTING'] = True
app.config['DEBUG'] = False
client = app.test_client()
docker = docker_api.from_env()
# get info on container a
container_ip = docker.containers.list()[1].exec_run(
"hostname -i").output.decode('utf-8').strip()
container_val_key = docker.containers.list()[1].exec_run("cat {val_pub}".format(val_pub=VKEY["pub"])) \
.output.decode('utf-8').strip()
container_user_key = docker.containers.list()[1].exec_run("cat {user_pub}".format(user_pub=UKEY["pub"])) \
.output.decode('utf-8').strip()
self.assertEqual(get_plain_text(client.get('/ip/a')), container_ip)
self.assertEqual(get_plain_text(client.get('/val+key/a')),
container_val_key)
self.assertEqual(get_plain_text(client.get('/user+key/a')),
container_user_key)
self.assertNotEqual(get_plain_text(client.get('/val+key/a')),
get_plain_text(client.get('/user+key/a')))
# get info on container b
container_ip = docker.containers.list()[0].exec_run(
"hostname -i").output.decode('utf-8').strip()
container_val_key = docker.containers.list()[0].exec_run("cat {val_pub}".format(val_pub=VKEY["pub"])) \
.output.decode('utf-8').strip()
container_user_key = docker.containers.list()[0].exec_run("cat {user_pub}".format(user_pub=UKEY["pub"])) \
.output.decode('utf-8').strip()
self.assertEqual(get_plain_text(client.get('/ip/b')), container_ip)
self.assertEqual(get_plain_text(client.get('/val+key/b')),
container_val_key)
self.assertEqual(get_plain_text(client.get('/user+key/b')),
container_user_key)
self.assertNotEqual(get_plain_text(client.get('/val+key/b')),
get_plain_text(client.get('/user+key/b')))
def test_peer_join(self):
intersections = make_peer_committees(7)
id_a = intersections[0].committee_id_a
id_b = intersections[0].committee_id_b
intersections[0].submit(Transaction(id_a, 'update_a_1', 999))
number_of_tx = 2
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx, 'update_a_1', ))
intersections[0].submit(Transaction(id_b, 'update_b_1', 999))
self.assertTrue(check_for_confirmation([i.instance_b for i in intersections], number_of_tx, 'update_b_1', ))
intersections.append(Intersection(SawtoothContainer(), SawtoothContainer(), id_a, id_b))
committee_ips_a = [inter.instance_a.ip() for inter in intersections]
committee_ips_b = [inter.instance_b.ip() for inter in intersections]
intersections[-1].peer_join(id_a, committee_ips_a)
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx - 1)) # catch up
intersections[0].update_committee(id_a, [inter.instance_a.val_key() for inter in intersections])
number_of_tx += 1
intersections[0].submit(Transaction(id_a, 'update_a_2', 888))
number_of_tx += 1
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx, 'update_a_2'))
intersections[-1].peer_join(id_b, committee_ips_b)
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx - 1)) # catch up
intersections[0].update_committee(id_b, [inter.instance_b.val_key() for inter in intersections])
intersections[0].submit(Transaction(id_b, 'update_b_2', 888))
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx, 'update_b_2'))
# makes sure all intersections are configured to work with each other (this is not a test of connectivity
# just config) and make sure they all have the three tx
for inter in intersections:
intersections_config = inter.sawtooth_api(id_a, 'http://localhost:8008/peers')['data']
for ip in committee_ips_a:
if ip != inter.ip(id_a): # the peer it's self is not reported in the list
self.assertIn("tcp://{}:8800".format(ip), intersections_config)
intersections_config = inter.sawtooth_api(id_b, 'http://localhost:8008/peers')['data']
for ip in committee_ips_b:
if ip != inter.ip(id_b):
self.assertIn("tcp://{}:8800".format(ip), intersections_config)
def test_setting_instances(self):
a = SawtoothContainer()
b = SawtoothContainer()
inter = Intersection(a, b, 'a', 'b')
peer = SmartShardPeer(inter)
peer.start()
client = peer.app.api.test_client()
docker = docker_api.from_env()
# get info on container a
container_ip = docker.containers.list()[1].exec_run(
"hostname -i").output.decode('utf-8').strip()
container_val_key = docker.containers.list()[1].exec_run("cat {val_pub}".format(val_pub=VKEY["pub"])) \
.output.decode('utf-8').strip()
container_user_key = docker.containers.list()[1].exec_run("cat {user_pub}".format(user_pub=UKEY["pub"])) \
.output.decode('utf-8').strip()
self.assertEqual(get_plain_text(client.get('/ip/a')), container_ip)
self.assertEqual(get_plain_text(client.get('/val+key/a')),
container_val_key)
self.assertEqual(get_plain_text(client.get('/user+key/a')),
container_user_key)
self.assertNotEqual(get_plain_text(client.get('/val+key/a')),
get_plain_text(client.get('/user+key/a')))
# get info on container b
container_ip = docker.containers.list()[0].exec_run(
"hostname -i").output.decode('utf-8').strip()
container_val_key = docker.containers.list()[0].exec_run("cat {val_pub}".format(val_pub=VKEY["pub"])) \
.output.decode('utf-8').strip()
container_user_key = docker.containers.list()[0].exec_run("cat {user_pub}".format(user_pub=UKEY["pub"])) \
.output.decode('utf-8').strip()
self.assertEqual(get_plain_text(client.get('/ip/b')), container_ip)
self.assertEqual(get_plain_text(client.get('/val+key/b')),
container_val_key)
self.assertEqual(get_plain_text(client.get('/user+key/b')),
container_user_key)
self.assertNotEqual(get_plain_text(client.get('/val+key/b')),
get_plain_text(client.get('/user+key/b')))
def test_committee_setup_single(self):
id_a = '1'
id_b = '2'
containers_a = [SawtoothContainer() for _ in range(4)]
user_keys_a = [i.user_key() for i in containers_a]
val_keys_a = [i.val_key() for i in containers_a]
committee_ips_a = [i.ip() for i in containers_a]
containers_b = [SawtoothContainer() for _ in range(4)]
user_keys_b = [i.user_key() for i in containers_b]
val_keys_b = [i.val_key() for i in containers_b]
committee_ips_b = [i.ip() for i in containers_b]
intersections = [Intersection(containers_a[i], containers_b[i], id_a, id_b) for i in range(4)]
intersections[0].make_genesis(id_a, val_keys_a, user_keys_a)
intersections[0].make_genesis(id_b, val_keys_b, user_keys_b)
for inter in intersections:
inter.start_sawtooth(committee_ips_a, committee_ips_b)
# make sure genesis is in every peer and that they can communicate with other committee members
self.assertEqual(len(intersections), 4)
for inter in intersections:
self.assertEqual(len(inter.blocks(id_a)), 1)
for inter in intersections:
self.assertEqual(len(inter.blocks(id_b)), 1)
# make sure util func works
del containers_a
del containers_b
intersections = make_peer_committees(4)
self.assertEqual(len(intersections), 4)
for inter in intersections:
self.assertEqual(len(inter.blocks(id_a)), 1)
for inter in intersections:
self.assertEqual(len(inter.blocks(id_b)), 1)
def test_peer_join(self):
peers = make_sawtooth_committee(7)
number_of_tx = 1 # genesis
peers[0].submit_tx('test', '999')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test'))
peers[1].submit_tx('test1', '888')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test1'))
peers.append(SawtoothContainer())
peers[-1].join_sawtooth([p.ip() for p in peers])
# wait for block catch up
done = False
start = time.time()
while not done:
if len(peers[-1].blocks()['data']) >= number_of_tx - 1: # catch up cant get last tx
done = True
elif time.time() - start > 30:
self.fail("New peers block catch up failed")
peers[0].update_committee([p.val_key() for p in peers])
number_of_tx += 1 # +1 for membership of new peer
time.sleep(VIEW_CHANGE_WAIT_TIME_SEC) # wait in case leader has to change
# submit new tx with new peer so it can finally match all the others (i.e. has full blockchain)
peers[2].submit_tx('test2', '777')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test2'))
# makes sure all peers are configured to work with each other (this is not a test of connectivity just config)
# and make sure they all have the whole blockchain
ips = [p.ip() for p in peers]
for p in peers:
peers_blocks = len(p.blocks()['data'])
self.assertEqual(number_of_tx, peers_blocks, p.ip())
peers_config = p.sawtooth_api('http://localhost:8008/peers')['data']
for ip in ips:
if ip != p.ip(): # the peer it's self is not reported in the list
self.assertIn("tcp://{}:8800".format(ip), peers_config)
# check consensus still works
peers[-1].submit_tx('test3', '666')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test3'))
for p in peers:
peers_blockchain = len(p.blocks()['data'])
self.assertEqual(number_of_tx, peers_blockchain, p.ip())
def test_committee_churn(self):
self.skipTest("Complete committee churn not supported")
peers = make_sawtooth_committee(7)
blockchain_size = 1
peers[0].submit_tx('start', '1')
blockchain_size += 1
self.assertTrue(check_for_confirmation(peers, blockchain_size, 'start'))
for i in range(7):
# add new peer
peers.append(SawtoothContainer())
peers[-1].join_sawtooth([p.ip() for p in peers])
# wait for blockchain catch up
done = False
start = time.time()
while not done:
if len(peers[-1].blocks()['data']) >= blockchain_size - 1: # catch up cant get last tx
done = True
elif time.time() - start > 30:
self.fail("New peers block catch up failed")
peers[0].update_committee([p.val_key() for p in peers])
peers[0].submit_tx("update_{}".format(i), 999)
blockchain_size += 1
self.assertTrue(check_for_confirmation(peers, blockchain_size, "update_{}".format(i),
timeout=VIEW_CHANGE_WAIT_TIME_SEC))
blockchain_size += 1
for p in peers:
self.assertEqual(blockchain_size, len(p.blocks()['data']))
old_peer = peers.pop(0)
peers[0].update_committee([p.val_key() for p in peers])
self.assertTrue(check_for_confirmation(peers, blockchain_size))
blockchain_size += 1
self.assertTrue(check_for_confirmation(peers, blockchain_size, 'test_{}'.format(i)))
for p in peers:
self.assertEqual(blockchain_size, len(p.blocks()['data']))
del old_peer
gc.collect() # make sure that the containers are shutdown
peers[0].submit_tx('test', '999')
blockchain_size += 1
self.assertTrue(check_for_confirmation(peers, blockchain_size, 'test'))
for p in peers:
self.assertEqual(blockchain_size, len(p.blocks()['data']))
self.assertEqual('999', p.get_tx('test'))
def test_peer_setup(self):
a = SawtoothContainer()
b = SawtoothContainer()
id_a = '1'
id_b = '2'
inter = Intersection(a, b, id_a, id_b)
self.assertEqual(inter.committee_id_a, id_a)
self.assertEqual(inter.committee_id_b, id_b)
self.assertEqual(inter.ip(id_a), a.ip())
self.assertEqual(inter.ip(id_b), b.ip())
self.assertEqual(inter.attached_network(), DEFAULT_DOCKER_NETWORK)
self.assertTrue(inter.in_committee(id_a))
self.assertTrue(inter.in_committee(id_b))
self.assertFalse(inter.in_committee('c'))
self.assertFalse(inter.in_committee(0))
del a, b, inter
a = SawtoothContainer('host')
b = SawtoothContainer('host')
id_a = '1'
id_b = '2'
inter = Intersection(a, b, id_a, id_b)
self.assertEqual(inter.attached_network(), 'host')
def test_committee_init_setup(self):
docker = docker_api.from_env()
# a committee needs a min of 4 members. Any less peers can not join or leave cleanly and they can not confirm
# new transactions
peers = [None, None, None, None]
for i in range(len(peers)):
peers[i] = SawtoothContainer()
# make sure all containers have started
self.assertEqual(4, len(docker.containers.list()))
peers[0].make_genesis([p.val_key() for p in peers], [p.user_key() for p in peers])
committee_ips = [p.ip() for p in peers]
for p in peers:
p.join_sawtooth(committee_ips)
# make sure all peers are running
process_names = []
for p in peers:
for process in p.top()['Processes']:
process_names.append(process[-1])
self.assertIn('sawtooth-validator', [i for i in process_names if 'sawtooth-validator' in i][0])
self.assertIn('/usr/bin/python3 /usr/bin/sawtooth-rest-api -v', process_names)
self.assertIn('settings-tp -v', process_names)
self.assertIn('/usr/bin/python3 /usr/bin/intkey-tp-python -v', process_names)
self.assertIn('pbft-engine -vv --connect',
[i for i in process_names if 'pbft-engine -vv --connect' in i][0])
# give the genesis block some time to get to all peers
time.sleep(3)
# makes sure genesis block is in each peer
for p in peers:
blocks = p.blocks()['data']
self.assertEqual(1, len(blocks))
# makes sure all peers are configured to work with each other (this is not a test of connectivity just config)
ips = [p.ip() for p in peers]
admin = peers[0].admin_key() # peer 0 made genesis so it has the admin key make sure all other peers get it
for p in peers:
self.assertEqual(admin, p.admin_key())
peers_config = p.sawtooth_api('http://localhost:8008/peers')['data']
for ip in ips:
if ip != p.ip(): # the peer it's self is not reported in the list
self.assertIn("tcp://{}:8800".format(ip), peers_config)
docker.close()
def test_committee_growth(self):
peers = make_sawtooth_committee(7)
number_of_tx = 1
peers[0].submit_tx('test', '999')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test', timeout=120))
peers[1].submit_tx('test1', '888')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test1', timeout=120))
for i in range(13):
peers.append(SawtoothContainer())
peers[-1].join_sawtooth([p.ip() for p in peers])
# wait for blockchain catch up
done = False
start = time.time()
while not done:
if len(peers[-1].blocks()['data']) >= number_of_tx - 1: # catch up cant get last tx
done = True
elif time.time() - start > 30:
self.fail("New peers block catch up failed")
peers[i].update_committee([p.val_key() for p in peers])
self.assertTrue(check_for_confirmation(peers, number_of_tx, timeout=120))
number_of_tx += 1 # +1 for membership of new peer
# check consensus still works
peers[i].submit_tx('test_{}'.format(i), '777')
number_of_tx += 1
self.assertTrue(check_for_confirmation(peers, number_of_tx, 'test_{}'.format(i), timeout=120))
# makes sure all peers are configured to work with each other (this only tests config not connectivity)
ips = [p.ip() for p in peers]
for p in peers:
peers_blocks = len(p.blocks()['data'])
self.assertGreaterEqual(number_of_tx, peers_blocks)
peers_config = p.sawtooth_api('http://localhost:8008/peers')['data']
for ip in ips:
if ip != p.ip(): # the peer it's self is not reported in the list
self.assertIn("tcp://{}:8800".format(ip), peers_config)
def test_kill_container(self):
docker = docker_api.from_env()
sawtooth_instance = SawtoothContainer()
self.assertEqual(1, len(docker.containers.list()))
self.assertIn(sawtooth_instance.id(), get_container_ids())
sawtooth_instance_2nd = SawtoothContainer()
self.assertEqual(2, len(docker.containers.list()))
self.assertIn(sawtooth_instance.id(), get_container_ids())
self.assertIn(sawtooth_instance_2nd.id(), get_container_ids())
# test that if one instance is stop only one instance stops
del sawtooth_instance
self.assertEqual(1, len(docker.containers.list()))
self.assertIn(sawtooth_instance_2nd.id(), get_container_ids())
del sawtooth_instance_2nd
self.assertEqual(0, len(docker.containers.list()))
# clean up
docker.close()
def test_start_container(self):
docker = docker_api.from_env()
# test that once an instance is started that it has an id, ip and key
sawtooth_instance = SawtoothContainer()
self.assertEqual(1, len(docker.containers.list()))
self.assertIsNot(sawtooth_instance.id(), None)
self.assertIsNot(sawtooth_instance.ip(), None)
self.assertIsNot(sawtooth_instance.val_key(), None)
container_ip = docker.containers.list()[0].exec_run("hostname -i").output.decode('utf-8').strip()
container_val_key = docker.containers.list()[0].exec_run("cat {val_pub}".format(val_pub=VALIDATOR_KEY["pub"])) \
.output.decode('utf-8').strip()
container_user_key = docker.containers.list()[0].exec_run("cat {user_pub}".format(user_pub=USER_KEY["pub"])) \
.output.decode('utf-8').strip()
container_network = DEFAULT_DOCKER_NETWORK
self.assertEqual(sawtooth_instance.id(), docker.containers.list()[0].id)
self.assertEqual(sawtooth_instance.ip(), container_ip)
self.assertEqual(sawtooth_instance.val_key(), container_val_key)
self.assertEqual(sawtooth_instance.user_key(), container_user_key)
self.assertNotEqual(sawtooth_instance.user_key(), sawtooth_instance.val_key())
self.assertEqual(sawtooth_instance.attached_network(), container_network)
number_of_running_processes = len(docker.containers.list()[0].top()['Processes'][0])
# should only be 2 processes bash and tail -f /dev/null
# each process has 4 columns so 2*4 = 8
self.assertEqual(8, number_of_running_processes)
# test that containers are made unique
sawtooth_instance_2nd = SawtoothContainer()
self.assertEqual(2, len(docker.containers.list()))
# tests that the two instance to not have the same IP or Key
self.assertNotEqual(sawtooth_instance.id(), sawtooth_instance_2nd.id())
self.assertNotEqual(sawtooth_instance.ip(), sawtooth_instance_2nd.ip())
self.assertNotEqual(sawtooth_instance.val_key(), sawtooth_instance_2nd.val_key())
self.assertNotEqual(sawtooth_instance.user_key(), sawtooth_instance_2nd.user_key())
# clean up
docker.close()
def test_committee_independent_join(self):
intersections = make_peer_committees(7)
number_of_tx_a = 1
id_a = intersections[0].committee_id_a
new_peer = Intersection(SawtoothContainer(), None, id_a, None)
committee_ips_a = [inter.ip(id_a) for inter in intersections]
committee_ips_a.append(new_peer.ip(id_a))
committee_val_a = [inter.val_key(id_a) for inter in intersections]
committee_val_a.append(new_peer.val_key(id_a))
intersections[0].submit(Transaction(id_a, 'update_a_1', 999))
number_of_tx_a += 1
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx_a, 'update_a_1'))
new_peer.peer_join(id_a, committee_ips_a)
intersections[0].update_committee(id_a, committee_val_a)
number_of_tx_a += 1
intersections[0].submit(Transaction(id_a, 'update_a_2', 888))
number_of_tx_a += 1
self.assertTrue(check_for_confirmation([i.instance_a for i in intersections], number_of_tx_a, 'update_a_2'))
# confirm membership
committee_a = intersections.copy()
committee_a.append(new_peer)
self.assertEqual(None, new_peer.committee_id_b)
for inter in committee_a:
intersections_config = inter.sawtooth_api(id_a, 'http://localhost:8008/peers')['data']
for ip in committee_ips_a:
if ip != inter.ip(id_a): # the peer it's self is not reported in the list
self.assertIn("tcp://{}:8800".format(ip), intersections_config)
for inter in committee_a:
self.assertEqual(number_of_tx_a, len(inter.blocks(id_a)))
# now committee B
id_b = intersections[0].committee_id_b
number_of_tx_b = 1
new_peer = Intersection(None, SawtoothContainer(), None, id_b)
committee_ips_b = [inter.ip(id_b) for inter in intersections]
committee_ips_b.append(new_peer.ip(id_b))
committee_val_b = [inter.val_key(id_b) for inter in intersections]
committee_val_b.append(new_peer.val_key(id_b))
intersections[0].submit(Transaction(id_b, 'update_b_1', 999))
number_of_tx_b += 1
self.assertTrue(check_for_confirmation([i.instance_b for i in intersections], number_of_tx_b, 'update_b_1'))
new_peer.peer_join(id_b, committee_ips_b)
intersections[0].update_committee(id_b, committee_val_b)
number_of_tx_b += 1
self.assertTrue(check_for_confirmation([i.instance_b for i in intersections], number_of_tx_b))
intersections[0].submit(Transaction(id_b, 'update_b_2', 888))
number_of_tx_b += 1
self.assertTrue(check_for_confirmation([i.instance_b for i in intersections], number_of_tx_b, 'update_b_2'))
# confirm membership
committee_b = intersections.copy()
committee_b.append(new_peer)
for inter in committee_b:
intersections_config = inter.sawtooth_api(id_b, 'http://localhost:8008/peers')['data']
for ip in committee_ips_b:
if ip != inter.ip(id_b): # the peer it's self is not reported in the list
self.assertIn("tcp://{}:8800".format(ip), intersections_config)
for inter in committee_b:
self.assertEqual(number_of_tx_b, len(inter.blocks(id_b)))