def _poll_for_convergence(self, timeout=256, tolerance=2, standard=5):
convergent = False
with Progress('awaiting convergence') as p:
to = TimeOut(timeout)
while convergent is False:
self.assertFalse(to.is_timed_out(),
'timed out awaiting convergence')
p.step()
time.sleep(4)
try:
convergent = is_convergent(self.urls, standard=standard,
tolerance=tolerance)
except MessageException:
pass
sit_rep(self.urls, verbosity=1)
return convergent
def _poll_for_convergence(self, timeout=256, tolerance=2, standard=5):
convergent = False
with Progress('awaiting convergence') as p:
to = TimeOut(timeout)
while convergent is False:
self.assertFalse(to.is_timed_out(),
'timed out awaiting convergence')
p.step()
time.sleep(4)
try:
convergent = is_convergent(self.urls, standard=standard,
tolerance=tolerance)
except MessageException:
pass
sit_rep(self.urls, verbosity=1)
return convergent
def test_validator_shutdown_restart_ext(self):
vnm = None
print
try:
vnm = get_default_vnm(5)
vnm.do_genesis()
vnm.launch()
keys = 10
rounds = 2
txn_intv = 0
print "Testing transaction load."
test = IntKeyLoadTest()
urls = vnm.urls()
self.assertEqual(5, len(urls))
test.setup(vnm.urls(), keys)
test.run(keys, rounds, txn_intv)
test.validate()
print "test validator shutdown w/ SIGINT"
vnm.deactivate_node(2, sig='SIGINT', timeout=8, force=False)
print 'check state of validators:'
sit_rep(vnm.urls(), verbosity=2)
print "sending more txns after SIGINT"
urls = vnm.urls()
self.assertEqual(4, len(urls))
test.setup(urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
print("relaunching removed_validator", 2)
vnm.activate_node(2)
print 'check state of validators:'
sit_rep(vnm.urls(), verbosity=2)
if is_convergent(vnm.urls(), tolerance=2, standard=5) is True:
print "all validators are on the same chain"
else:
print "all validators are not on the same chain"
print "sending more txns after relaunching validator 2"
urls = vnm.urls()
self.assertEqual(5, len(urls))
test.setup(urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
print "test validator shutdown w/ SIGTERM"
vnm.deactivate_node(4, sig='SIGTERM', timeout=8, force=False)
print 'check state of validators:'
sit_rep(vnm.urls(), verbosity=2)
print "sending more txns after SIGTERM"
urls = vnm.urls()
self.assertEqual(4, len(urls))
test.setup(urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
print("relaunching removed_validator", 4)
vnm.activate_node(4)
print 'check state of validators:'
sit_rep(vnm.urls(), verbosity=2)
if is_convergent(vnm.urls(), tolerance=2, standard=5) is True:
print "all validators are on the same chain"
else:
print "all validators are not on the same chain"
print "sending more txns after relaunching validator 4"
urls = vnm.urls()
self.assertEqual(5, len(urls))
test.setup(urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
finally:
vnm.shutdown(archive_name='TestValidatorShutdown')
def test_two_clique(self):
# this topology forms 2 exclusive cliques when n2 is severed
vulnerable_mat = [
[1, 1, 0, 0, 0],
[1, 1, 1, 0, 0],
[0, 1, 1, 1, 0],
[0, 0, 1, 1, 1],
[0, 0, 0, 1, 1],
]
two_clique_mat = copy.deepcopy(vulnerable_mat)
two_clique_mat[2][2] = 0
n = len(vulnerable_mat)
top = SimController(n)
print
try:
print 'phase 0: build vulnerably connected 5-net:'
from txnintegration.netconfig import NetworkConfigProvider
net_cfg = NetworkConfig(gen_dfl_cfg_poet0(), n,
provider=NetworkConfigProvider())
net_cfg.set_nodes(vulnerable_mat)
net_cfg.set_peers(vulnerable_mat)
net_cfg.set_blacklist()
vnm = ValidatorCollectionController(net_cfg)
top.initialize(net_cfg, vnm, NopEdgeController(net_cfg))
print 'phase 1: launch vulnerably connected 5-net:'
top.do_genesis(probe_seconds=0)
top.launch(probe_seconds=0)
print 'phase 2: validate state across 5-net:'
sit_rep(top.urls(), verbosity=2)
print 'phase 3: morph 5-net into two exclusive 2-net cliques:'
top.update(node_mat=two_clique_mat, probe_seconds=0, reg_seconds=0)
print 'providing time for convergence (likely partial)...'
time.sleep(32)
sit_rep(top.urls())
print 'phase 4: generate chain-ext A on clique {0, 1}:'
url = top.urls()[0]
print 'sending transactions to %s...' % (url)
ikcA = IntegerKeyClient(baseurl=url, keystring=gen_pk())
self._do_work(ikcA, 5, 2)
print 'providing time for partial convergence...'
time.sleep(8)
sit_rep(top.urls())
print 'phase 5: generate chain-ext B on clique {3, 4}, |B| = 2|A|:'
url = top.urls()[-1]
print 'sending transactions to %s...' % (url)
ikcB = IntegerKeyClient(baseurl=url, keystring=gen_pk())
self._do_work(ikcB, 1, 4)
print 'providing time for partial convergence...'
time.sleep(8)
sit_rep(top.urls())
print 'TEST 1: asserting network is forked'
self.assertEquals(False, is_convergent(top.urls(), standard=3))
print 'phase 6: reconnect 5-net:'
print 'rezzing validator-2 with InitialConnectivity = |Peers|...'
cfg = top.get_configuration(2)
cfg['InitialConnectivity'] = 2
top.set_configuration(2, cfg)
top.update(node_mat=vulnerable_mat, probe_seconds=0, reg_seconds=0)
print 'phase 7: validate state across 5-net:'
print 'providing time for global convergence...'
time.sleep(64)
sit_rep(top.urls())
print 'TEST 2: asserting network is convergent'
self.assertEquals(True, is_convergent(top.urls(), standard=4))
except Exception as e:
print 'Exception encountered: %s' % (e.message)
traceback.print_exc()
sit_rep(top.urls())
raise
finally:
top.shutdown(archive_name="TestPartitionRecoveryResults")
def test_validator_shutdown_restart_restore_ext(self):
print
try:
print "launching a validator network of 5"
vnm = get_default_vnm(5)
vnm.do_genesis()
vnm.launch()
keys = 10
rounds = 2
txn_intv = 0
timeout = 5
print "Testing transaction load."
test = IntKeyLoadTest()
urls = vnm.urls()
self.assertEqual(5, len(urls))
test.setup(vnm.urls(), keys)
test.run(keys, rounds, txn_intv)
test.validate()
to = TimeOut(timeout)
convergent = False
with Progress("Checking for validators convergence") as p:
while convergent is False or not to.is_timed_out():
time.sleep(timeout)
p.step()
convergent = is_convergent(vnm.urls(),
tolerance=2,
standard=5)
self.assertTrue(convergent, "All validators are "
"not on the same chain.")
print "all validators are on the same chain"
sit_rep(vnm.urls(), verbosity=1)
report_before_shutdown = sit_rep(vnm.urls(), verbosity=1)
validator_report = report_before_shutdown[2]
valid_dict_value = validator_report['Status']
validator_blocks_shutdown = valid_dict_value['Blocks']
print "validator_blocks", validator_blocks_shutdown
print "turn off entire validator network"
vnm.update(node_mat=numpy.zeros(shape=(5, 5)), timeout=8)
# set InitialConnectivity of individual
# node to zero before relaunching
cfg = vnm.get_configuration(2)
cfg['InitialConnectivity'] = 0
vnm.set_configuration(2, cfg)
print "relaunch validator 2"
vnm.activate_node(2)
report_after_relaunch = sit_rep(vnm.urls(), verbosity=1)
validator_report = report_after_relaunch[0]
valid_dict_value = validator_report['Status']
validator_blocks_relaunch = valid_dict_value['Blocks']
print "validator_blocks_relaunch", validator_blocks_relaunch
# the length of post-shutdown validator blocks might be bigger
# than the length of pre-shutdown validator blocks
for i in range(0, len(validator_blocks_shutdown)):
self.assertEqual(
validator_blocks_relaunch[i], validator_blocks_shutdown[i],
"mismatch in post-shutdown validator blocks. "
"Validator didn't restore fr local db")
break
print "relaunched validator restored from local database"
finally:
if vnm is not None:
# Validator network shutting down
vnm.shutdown(archive_name='TestValidatorShutdownRestore')
def test_validator_shutdown_restart_ext(self):
try:
keys = 10
rounds = 2
txn_intv = 0
print("Testing transaction load.")
test = IntKeyLoadTest()
urls = self.urls
self.assertEqual(5, len(urls))
test.setup(self.urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
print("test validator shutdown w/ SIGTERM")
node_names = self.node_controller.get_node_names()
node_names.sort()
self.node_controller.stop(node_names[4])
to = TimeOut(120)
while len(self.node_controller.get_node_names()) > 4:
if to.is_timed_out():
self.fail("Timed Out")
print('check state of validators:')
sit_rep(self.urls[:-1], verbosity=2)
print("sending more txns after SIGTERM")
urls = self.urls[:-1]
self.assertEqual(4, len(urls))
test.setup(urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
print(("relaunching removed_validator", 4))
self.node_controller.start(self.nodes[4])
to = TimeOut(120)
while len(self.node_controller.get_node_names()) < 1:
if to.is_timed_out():
self.fail("Timed Out")
report_after_relaunch = None
while report_after_relaunch is None:
try:
report_after_relaunch = \
sit_rep([self.urls[4]], verbosity=1)
except MessageException:
if to.is_timed_out():
self.fail("Timed Out")
time.sleep(4)
print('check state of validators:')
sit_rep(self.urls, verbosity=2)
if is_convergent(self.urls, tolerance=2, standard=5) is True:
print("all validators are on the same chain")
else:
print("all validators are not on the same chain")
print("sending more txns after relaunching validator 4")
urls = self.urls
self.assertEqual(5, len(urls))
test.setup(urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
finally:
print("No validators")
def test_validator_shutdown_sigkill_restart_ext(self):
print
try:
print "launching a validator network of 5"
vnm = get_default_vnm(5)
vnm.do_genesis()
vnm.launch()
keys = 10
rounds = 2
txn_intv = 0
timeout = 5
print "Testing transaction load."
test = IntKeyLoadTest()
urls = vnm.urls()
self.assertEqual(5, len(urls))
test.setup(vnm.urls(), keys)
test.run(keys, rounds, txn_intv)
test.validate()
to = TimeOut(timeout)
convergent = False
with Progress("Checking for validators convergence") as p:
while convergent is False or not to.is_timed_out():
time.sleep(timeout)
p.step()
convergent = is_convergent(vnm.urls(),
tolerance=2,
standard=5)
self.assertTrue(convergent, "All validators are "
"not on the same chain.")
print "all validators are on the same chain"
sit_rep(vnm.urls(), verbosity=1)
report_before_shutdown = sit_rep(vnm.urls(), verbosity=1)
validator_report = report_before_shutdown[4]
valid_dict_value = validator_report['Status']
validator_blocks_shutdown = valid_dict_value['Blocks']
print "validator_blocks", validator_blocks_shutdown
print "shutdown validator 4 w/ SIGKILL"
vnm.deactivate_node(4, sig='SIGKILL', timeout=8)
print 'check state of validators:'
sit_rep(vnm.urls(), verbosity=2)
print "sending more txns after SIGKILL"
urls = vnm.urls()
self.assertEqual(4, len(urls))
test.setup(urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
print "turn off entire validator network"
for i in range(0, 4):
vnm.deactivate_node(i, sig='SIGINT', timeout=8, force=False)
print 'check state of validators after graceful shutdown:'
sit_rep(vnm.urls(), verbosity=2)
# set InitialConnectivity of individual
# node to zero before relaunching
cfg = vnm.get_configuration(4)
cfg['InitialConnectivity'] = 0
vnm.set_configuration(4, cfg)
print "relaunch validator 4"
vnm.activate_node(4)
report_after_relaunch = sit_rep(vnm.urls(), verbosity=1)
validator_report = report_after_relaunch[0]
valid_dict_value = validator_report['Status']
validator_blocks_relaunch = valid_dict_value['Blocks']
print "validator_blocks_relaunch", validator_blocks_relaunch
if len(validator_blocks_relaunch) == \
len(validator_blocks_shutdown):
if validator_blocks_shutdown == validator_blocks_relaunch:
print "relaunched validator restored from local db"
else:
for i in range(0, len(validator_blocks_shutdown)):
self.assertEqual(validator_blocks_relaunch[i],
validator_blocks_shutdown[i],
"relaunched validator didn't"
" restore fr local db")
break
print "relaunched validator restored from local database"
finally:
if vnm is not None:
# shutting down validator network
vnm.shutdown(archive_name='TestValidatorShutdownRestore')
def test_two_clique(self):
# this topology forms 2 exclusive cliques when n2 is severed
vulnerable_mat = [
[1, 1, 0, 0, 0],
[1, 1, 1, 0, 0],
[0, 1, 1, 1, 0],
[0, 0, 1, 1, 1],
[0, 0, 0, 1, 1],
]
two_clique_mat = copy.deepcopy(vulnerable_mat)
two_clique_mat[2][2] = 0
n = len(vulnerable_mat)
top = SimController(n)
print
try:
print 'phase 0: build vulnerably connected 5-net:'
from txnintegration.netconfig import NetworkConfigProvider
net_cfg = NetworkConfig(gen_dfl_cfg_poet0(),
n,
provider=NetworkConfigProvider())
net_cfg.set_nodes(vulnerable_mat)
net_cfg.set_peers(vulnerable_mat)
net_cfg.set_blacklist()
vnm = ValidatorCollectionController(net_cfg)
top.initialize(net_cfg, vnm, NopEdgeController(net_cfg))
print 'phase 1: launch vulnerably connected 5-net:'
top.do_genesis(probe_seconds=0)
top.launch(probe_seconds=0)
print 'phase 2: validate state across 5-net:'
sit_rep(top.urls(), verbosity=2)
print 'phase 3: morph 5-net into two exclusive 2-net cliques:'
top.update(node_mat=two_clique_mat, probe_seconds=0, reg_seconds=0)
print 'providing time for convergence (likely partial)...'
time.sleep(32)
sit_rep(top.urls())
print 'phase 4: generate chain-ext A on clique {0, 1}:'
url = top.urls()[0]
print 'sending transactions to %s...' % (url)
ikcA = IntegerKeyClient(baseurl=url, keystring=gen_pk())
self._do_work(ikcA, 5, 2)
print 'providing time for partial convergence...'
time.sleep(8)
sit_rep(top.urls())
print 'phase 5: generate chain-ext B on clique {3, 4}, |B| = 2|A|:'
url = top.urls()[-1]
print 'sending transactions to %s...' % (url)
ikcB = IntegerKeyClient(baseurl=url, keystring=gen_pk())
self._do_work(ikcB, 1, 4)
print 'providing time for partial convergence...'
time.sleep(8)
sit_rep(top.urls())
print 'TEST 1: asserting network is forked'
self.assertEquals(False, is_convergent(top.urls(), standard=3))
print 'phase 6: reconnect 5-net:'
print 'rezzing validator-2 with InitialConnectivity = |Peers|...'
cfg = top.get_configuration(2)
cfg['InitialConnectivity'] = 2
top.set_configuration(2, cfg)
top.update(node_mat=vulnerable_mat, probe_seconds=0, reg_seconds=0)
print 'phase 7: validate state across 5-net:'
print 'providing time for global convergence...'
time.sleep(64)
sit_rep(top.urls())
print 'TEST 2: asserting network is convergent'
self.assertEquals(True, is_convergent(top.urls(), standard=4))
except Exception as e:
print 'Exception encountered: %s' % (e.message)
traceback.print_exc()
sit_rep(top.urls())
raise
finally:
top.shutdown(archive_name="TestPartitionRecoveryResults")
def test_validator_shutdown_restart_restore_ext(self):
try:
keys = 10
rounds = 2
txn_intv = 0
timeout = 20
print("Testing transaction load.")
test = IntKeyLoadTest()
urls = self.urls
self.assertEqual(5, len(urls))
test.setup(self.urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
to = TimeOut(timeout)
convergent = False
with Progress("Checking for validators convergence") as p:
while convergent is False or not to.is_timed_out():
time.sleep(5)
p.step()
convergent = is_convergent(self.urls,
tolerance=2,
standard=5)
self.assertTrue(convergent, "All validators are "
"not on the same chain.")
print("all validators are on the same chain")
sit_rep(self.urls, verbosity=1)
report_before_shutdown = sit_rep(self.urls, verbosity=1)
validator_report = report_before_shutdown[2]
valid_dict_value = validator_report['Status']
validator_blocks_shutdown = valid_dict_value['Blocks']
print("validator_blocks", validator_blocks_shutdown)
print("turn off entire validator network")
nodes_names = self.node_controller.get_node_names()
for node in nodes_names:
self.node_controller.stop(node)
to = TimeOut(120)
while len(self.node_controller.get_node_names()) > 0:
if to.is_timed_out():
self.fail("Timed Out")
print("relaunch validator 0")
self.node_controller.start(self.nodes[0])
to = TimeOut(120)
while len(self.node_controller.get_node_names()) < 1:
if to.is_timed_out():
self.fail("Timed Out")
report_after_relaunch = None
while report_after_relaunch is None:
try:
report_after_relaunch = \
sit_rep([self.urls[0]], verbosity=1)
except MessageException:
if to.is_timed_out():
self.fail("Timed Out")
time.sleep(4)
report_after_relaunch = sit_rep([self.urls[0]], verbosity=1)
validator_report = report_after_relaunch[0]
valid_dict_value = validator_report['Status']
validator_blocks_relaunch = valid_dict_value['Blocks']
print("validator_blocks_relaunch", validator_blocks_relaunch)
# the length of post-shutdown validator blocks might be bigger
# than the length of pre-shutdown validator blocks
for i in range(0, len(validator_blocks_shutdown)):
self.assertEqual(validator_blocks_relaunch[i],
validator_blocks_shutdown[i],
"mismatch in post-shutdown validator blocks. "
"Validator didn't restore fr local db")
break
print("relaunched validator restored from local database")
finally:
print("restart validators ")
for node in self.nodes:
self.node_controller.start(node)
to = TimeOut(120)
while len(self.node_controller.get_node_names()) < 5:
pass
report_after_relaunch = None
while report_after_relaunch is None:
try:
report_after_relaunch = \
sit_rep(self.urls, verbosity=1)
except MessageException:
if to.is_timed_out():
self.fail("Timed Out")
time.sleep(4)
def test_validator_shutdown_restart_restore_ext(self):
try:
keys = 10
rounds = 2
txn_intv = 0
timeout = 20
print "Testing transaction load."
test = IntKeyLoadTest()
urls = self.urls
self.assertEqual(5, len(urls))
test.setup(self.urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
to = TimeOut(timeout)
convergent = False
with Progress("Checking for validators convergence") as p:
while convergent is False or not to.is_timed_out():
time.sleep(5)
p.step()
convergent = is_convergent(self.urls,
tolerance=2,
standard=5)
self.assertTrue(convergent, "All validators are "
"not on the same chain.")
print "all validators are on the same chain"
sit_rep(self.urls, verbosity=1)
report_before_shutdown = sit_rep(self.urls, verbosity=1)
validator_report = report_before_shutdown[2]
valid_dict_value = validator_report['Status']
validator_blocks_shutdown = valid_dict_value['Blocks']
print "validator_blocks", validator_blocks_shutdown
print "turn off entire validator network"
nodes_names = self.node_controller.get_node_names()
for node in nodes_names:
self.node_controller.stop(node)
to = TimeOut(120)
while len(self.node_controller.get_node_names()) > 0:
if to.is_timed_out():
self.fail("Timed Out")
print "relaunch validator 0"
self.node_controller.start(self.nodes[0])
to = TimeOut(120)
while len(self.node_controller.get_node_names()) < 1:
if to.is_timed_out():
self.fail("Timed Out")
report_after_relaunch = None
while report_after_relaunch is None:
try:
report_after_relaunch = \
sit_rep([self.urls[0]], verbosity=1)
except MessageException:
if to.is_timed_out():
self.fail("Timed Out")
time.sleep(4)
report_after_relaunch = sit_rep([self.urls[0]], verbosity=1)
validator_report = report_after_relaunch[0]
valid_dict_value = validator_report['Status']
validator_blocks_relaunch = valid_dict_value['Blocks']
print "validator_blocks_relaunch", validator_blocks_relaunch
# the length of post-shutdown validator blocks might be bigger
# than the length of pre-shutdown validator blocks
for i in range(0, len(validator_blocks_shutdown)):
self.assertEqual(validator_blocks_relaunch[i],
validator_blocks_shutdown[i],
"mismatch in post-shutdown validator blocks. "
"Validator didn't restore fr local db")
break
print "relaunched validator restored from local database"
finally:
print "restart validators "
for node in self.nodes:
self.node_controller.start(node)
to = TimeOut(120)
while len(self.node_controller.get_node_names()) < 5:
pass
report_after_relaunch = None
while report_after_relaunch is None:
try:
report_after_relaunch = \
sit_rep(self.urls, verbosity=1)
except MessageException:
if to.is_timed_out():
self.fail("Timed Out")
time.sleep(4)
def test_validator_shutdown_restart_ext(self):
try:
keys = 10
rounds = 2
txn_intv = 0
print "Testing transaction load."
test = IntKeyLoadTest()
urls = self.urls
self.assertEqual(5, len(urls))
test.setup(self.urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
print "test validator shutdown w/ SIGTERM"
node_names = self.node_controller.get_node_names()
node_names.sort()
self.node_controller.stop(node_names[4])
to = TimeOut(120)
while len(self.node_controller.get_node_names()) > 4:
if to.is_timed_out():
self.fail("Timed Out")
print 'check state of validators:'
sit_rep(self.urls[:-1], verbosity=2)
print "sending more txns after SIGTERM"
urls = self.urls[:-1]
self.assertEqual(4, len(urls))
test.setup(urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
print ("relaunching removed_validator", 4)
self.node_controller.start(self.nodes[4])
to = TimeOut(120)
while len(self.node_controller.get_node_names()) < 1:
if to.is_timed_out():
self.fail("Timed Out")
report_after_relaunch = None
while report_after_relaunch is None:
try:
report_after_relaunch = \
sit_rep([self.urls[4]], verbosity=1)
except MessageException:
if to.is_timed_out():
self.fail("Timed Out")
time.sleep(4)
print 'check state of validators:'
sit_rep(self.urls, verbosity=2)
if is_convergent(self.urls, tolerance=2, standard=5) is True:
print "all validators are on the same chain"
else:
print "all validators are not on the same chain"
print "sending more txns after relaunching validator 4"
urls = self.urls
self.assertEqual(5, len(urls))
test.setup(urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
finally:
print "No validators"
def test_validator_shutdown_sigkill_restart_ext(self):
try:
keys = 10
rounds = 2
txn_intv = 0
timeout = 5
print "Testing transaction load."
test = IntKeyLoadTest()
urls = self.urls
self.assertEqual(5, len(urls))
test.setup(self.urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
to = TimeOut(timeout)
convergent = False
with Progress("Checking for validators convergence") as p:
while convergent is False or not to.is_timed_out():
time.sleep(timeout)
p.step()
convergent = is_convergent(self.urls,
tolerance=2,
standard=5)
self.assertTrue(convergent, "All validators are "
"not on the same chain.")
print "all validators are on the same chain"
report_before_shutdown = sit_rep(self.urls, verbosity=1)
validator_report = report_before_shutdown[4]
valid_dict_value = validator_report['Status']
validator_blocks_shutdown = valid_dict_value['Blocks']
print "validator_blocks", validator_blocks_shutdown
print "shutdown validator 4 w/ SIGKILL"
node_names = self.node_controller.get_node_names()
node_names.sort()
self.node_controller.kill(node_names[4])
to = TimeOut(120)
while len(self.node_controller.get_node_names()) > 4:
if to.is_timed_out():
self.fail("Timed Out")
print 'check state of validators:'
sit_rep(self.urls[:-1], verbosity=2)
print "sending more txns after SIGKILL"
urls = self.urls[:-1]
self.assertEqual(4, len(urls))
test.setup(urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
print "turn off entire validator network"
for node in node_names:
self.node_controller.stop(node)
to = TimeOut(120)
while len(self.node_controller.get_node_names()) > 0:
if to.is_timed_out():
self.fail("Timed Out")
print "relaunch validator 4"
self.node_controller.start(self.nodes[4])
report_after_relaunch = None
to = TimeOut(120)
while report_after_relaunch is None:
try:
report_after_relaunch = \
sit_rep([self.urls[4]], verbosity=1)
except MessageException:
if to.is_timed_out():
self.fail("Timed Out")
time.sleep(4)
validator_report = report_after_relaunch[0]
valid_dict_value = validator_report['Status']
validator_blocks_relaunch = valid_dict_value['Blocks']
print "validator_blocks_relaunch", validator_blocks_relaunch
if len(validator_blocks_relaunch) == \
len(validator_blocks_shutdown):
if validator_blocks_shutdown == validator_blocks_relaunch:
print "relaunched validator restored from local db"
else:
for i in range(0, len(validator_blocks_shutdown)):
self.assertEqual(validator_blocks_relaunch[i],
validator_blocks_shutdown[i],
"relaunched validator didn't"
" restore fr local db")
break
print "relaunched validator restored from local database"
finally:
print "restart validators "
for node in self.nodes:
self.node_controller.start(node)
to = TimeOut(120)
while len(self.node_controller.get_node_names()) < 5:
if to.is_timed_out():
self.fail("Timed Out")
report_after_relaunch = None
while report_after_relaunch is None:
try:
report_after_relaunch = \
sit_rep(self.urls, verbosity=1)
except MessageException:
if to.is_timed_out():
self.fail("Timed Out")
time.sleep(4)
print "No validators"
def test_validator_shutdown_sigkill_restart_ext(self):
try:
keys = 10
rounds = 2
txn_intv = 0
timeout = 5
print("Testing transaction load.")
test = IntKeyLoadTest()
urls = self.urls
self.assertEqual(5, len(urls))
test.setup(self.urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
to = TimeOut(timeout)
convergent = False
with Progress("Checking for validators convergence") as p:
while convergent is False or not to.is_timed_out():
time.sleep(timeout)
p.step()
convergent = is_convergent(self.urls,
tolerance=2,
standard=5)
self.assertTrue(convergent, "All validators are "
"not on the same chain.")
print("all validators are on the same chain")
report_before_shutdown = sit_rep(self.urls, verbosity=1)
validator_report = report_before_shutdown[4]
valid_dict_value = validator_report['Status']
validator_blocks_shutdown = valid_dict_value['Blocks']
print("validator_blocks", validator_blocks_shutdown)
print("shutdown validator 4 w/ SIGKILL")
node_names = self.node_controller.get_node_names()
node_names.sort()
self.node_controller.kill(node_names[4])
to = TimeOut(120)
while len(self.node_controller.get_node_names()) > 4:
if to.is_timed_out():
self.fail("Timed Out")
print('check state of validators:')
sit_rep(self.urls[:-1], verbosity=2)
print("sending more txns after SIGKILL")
urls = self.urls[:-1]
self.assertEqual(4, len(urls))
test.setup(urls, keys)
test.run(keys, rounds, txn_intv)
test.validate()
print("turn off entire validator network")
for node in node_names:
self.node_controller.stop(node)
to = TimeOut(120)
while len(self.node_controller.get_node_names()) > 0:
if to.is_timed_out():
self.fail("Timed Out")
print("relaunch validator 4")
self.node_controller.start(self.nodes[4])
report_after_relaunch = None
to = TimeOut(120)
while report_after_relaunch is None:
try:
report_after_relaunch = \
sit_rep([self.urls[4]], verbosity=1)
except MessageException:
if to.is_timed_out():
self.fail("Timed Out")
time.sleep(4)
validator_report = report_after_relaunch[0]
valid_dict_value = validator_report['Status']
validator_blocks_relaunch = valid_dict_value['Blocks']
print("validator_blocks_relaunch", validator_blocks_relaunch)
if len(validator_blocks_relaunch) == \
len(validator_blocks_shutdown):
if validator_blocks_shutdown == validator_blocks_relaunch:
print("relaunched validator restored from local db")
else:
for i in range(0, len(validator_blocks_shutdown)):
self.assertEqual(
validator_blocks_relaunch[i],
validator_blocks_shutdown[i],
"relaunched validator didn't"
" restore fr local db")
break
print("relaunched validator restored from local database")
finally:
print("restart validators ")
for node in self.nodes:
self.node_controller.start(node)
to = TimeOut(120)
while len(self.node_controller.get_node_names()) < 5:
if to.is_timed_out():
self.fail("Timed Out")
report_after_relaunch = None
while report_after_relaunch is None:
try:
report_after_relaunch = \
sit_rep(self.urls, verbosity=1)
except MessageException:
if to.is_timed_out():
self.fail("Timed Out")
time.sleep(4)
print("No validators")