def doit(seq=[]):
global nodes, all_stakes, sequence
sequence = seq
config = None
nodes = start_cluster(
2, 1, 1, config, [["epoch_length", EPOCH_LENGTH],
["block_producer_kickout_threshold", 40],
["chunk_producer_kickout_threshold", 40]], {
0: {
"view_client_throttle_period": {
"secs": 0,
"nanos": 0
},
"consensus": {
"state_sync_timeout": {
"secs": 2,
"nanos": 0
}
}
},
1: {
"view_client_throttle_period": {
"secs": 0,
"nanos": 0
},
"consensus": {
"state_sync_timeout": {
"secs": 2,
"nanos": 0
}
}
},
2: {
"tracked_shards": [0],
"view_client_throttle_period": {
"secs": 0,
"nanos": 0
},
"consensus": {
"state_sync_timeout": {
"secs": 2,
"nanos": 0
}
}
}
})
started = time.time()
last_iter = started
height, hash_ = nodes[2].get_latest_block()
for i in range(3):
nodes[i].stop_checking_store()
logger.info("Initial stakes: %s" % get_stakes())
all_stakes.append(get_stakes())
do_moar_stakes(hash_, True)
last_fake_stakes_height = FAKE_OFFSET
last_staked_height = REAL_OFFSET
while True:
if time.time() - started >= TIMEOUT:
break
assert time.time() - last_iter < TIMEOUT_PER_ITER
height, hash_ = nodes[0].get_latest_block()
send_fakes = send_reals = False
if (height + EPOCH_LENGTH - FAKE_OFFSET) // EPOCH_LENGTH > (
last_fake_stakes_height + EPOCH_LENGTH -
FAKE_OFFSET) // EPOCH_LENGTH:
last_iter = time.time()
send_fakes = True
if (height + EPOCH_LENGTH - REAL_OFFSET) // EPOCH_LENGTH > (
last_staked_height + EPOCH_LENGTH -
REAL_OFFSET) // EPOCH_LENGTH:
send_reals = True
if send_fakes or send_reals:
cur_stakes = get_stakes()
logger.info("Current stakes: %s" % cur_stakes)
if len(all_stakes) > 1:
expected_stakes = get_expected_stakes()
logger.info("Expect stakes: %s" % expected_stakes)
for (cur, expected) in zip(cur_stakes, expected_stakes):
if cur % 1000000 == 0:
assert cur == expected
else:
assert expected <= cur <= expected * 1.1
do_moar_stakes(hash_, update_expected=send_reals)
if send_fakes:
last_fake_stakes_height += EPOCH_LENGTH
elif send_reals:
last_staked_height += EPOCH_LENGTH
import base58, base64
import json
import struct
import sys
import threading
import deepdiff
sys.path.append('lib')
from cluster import start_cluster
from key import Key
from utils import load_test_contract
import transaction
nodes = start_cluster(
4, 0, 1, None,
[["epoch_length", 1000], ["block_producer_kickout_threshold", 80]], {})
def assert_changes_in_block_response(request, expected_response):
for node_index, node in enumerate(nodes):
response = node.get_changes_in_block(request)
assert 'result' in response, "the request did not succeed: %r" % response
response = response['result']
diff = deepdiff.DeepDiff(expected_response, response)
assert not diff, \
"query node #%d same changes gives different results %r (expected VS actual):\n%r\n%r" \
% (node_index, diff, expected_response, response)
def assert_changes_response(request, expected_response, **kwargs):
async def handle(self, msg, fr, to):
if msg.enum == "RoutingTableSync":
logger.info("RoutingTableSync")
if msg.enum == "RoutingTableSyncV2" and msg.RoutingTableSyncV2.enum == "Version2":
if msg.RoutingTableSyncV2.Version2.routing_state.enum == "Done":
success.value = 1
logger.info("ROUTING_STATE %s" %
msg.RoutingTableSyncV2.Version2.routing_state.enum)
logger.info("* known_edges %s" %
msg.RoutingTableSyncV2.Version2.known_edges)
logger.info("* edges %s" %
len(msg.RoutingTableSyncV2.Version2.edges))
return True
nodes = start_cluster(2, 0, 1, None, [], {}, Handler)
time.sleep(10)
signing_key, verifying_key = ed25519.create_keypair()
logger.info(str(signing_key))
def gen():
return base58.b58encode(
ed25519.create_keypair()[0].to_bytes()).decode('ascii')
def new_edge(peer_a, peer_b, nonce):
s1 = base58.b58encode(str(peer_a).zfill(32)).decode("ascii")
EPOCH_LENGTH = 100
CONFIG = utils.figure_out_sandbox_binary()
CONFIG.update({
"consensus": {
"min_block_production_delay": {
"secs": MIN_BLOCK_PROD_TIME,
"nanos": 0,
},
"max_block_production_delay": {
"secs": MAX_BLOCK_PROD_TIME,
"nanos": 0,
},
}
})
nodes = start_cluster(1, 0, 1, CONFIG, [["epoch_length", EPOCH_LENGTH]], {})
# start at block_height = 10
utils.wait_for_blocks(nodes[0], target=10)
# fast forward to about block_height=190 and then test for boundaries
nodes[0].json_rpc('sandbox_fast_forward', {"delta_height": 180}, timeout=60)
for i in range(20):
utils.wait_for_blocks(nodes[0], target=190 + i)
block_height = nodes[0].get_latest_block().height
epoch_height = nodes[0].get_validators()['result']['epoch_height']
assert epoch_height == 2 if block_height > 200 else 1
# check that we still have correct epoch heights after consecutive fast forwards:
utils.wait_for_blocks(nodes[0], target=220)
nodes[0].json_rpc('sandbox_fast_forward', {"delta_height": 70}, timeout=60)
for i in range(20):
# Unstakes for them, makes sure they stop being a validator
import sys, time, base58, random, datetime
sys.path.append('lib')
from cluster import start_cluster
from configured_logger import logger
from transaction import sign_staking_tx
TIMEOUT = 150
config = None
nodes = start_cluster(
2, 1, 1, config,
[["epoch_length", 10], ["block_producer_kickout_threshold", 40]],
{2: {
"tracked_shards": [0]
}})
started = time.time()
def get_validators():
return set([x['account_id'] for x in nodes[0].get_status()['validators']])
def get_stakes():
return [
int(nodes[2].get_account("test%s" % i)['result']['locked'])
for i in range(3)
]
from configured_logger import logger
from utils import TxContext
from transaction import sign_payment_tx
TIMEOUT = 240
nodes = start_cluster(
num_nodes=4,
num_observers=1,
num_shards=4,
config=None,
genesis_config_changes=[
["min_gas_price", 0],
["max_inflation_rate", [0, 1]],
["epoch_length", 10],
["block_producer_kickout_threshold", 70]
],
client_config_changes={
0: {"consensus": {"state_sync_timeout": {"secs": 2, "nanos": 0}}},
1: {"consensus": {"state_sync_timeout": {"secs": 2, "nanos": 0}}},
2: {"consensus": {"state_sync_timeout": {"secs": 2, "nanos": 0}}},
3: {"consensus": {"state_sync_timeout": {"secs": 2, "nanos": 0}}},
4: {"consensus": {"state_sync_timeout": {"secs": 2, "nanos": 0}},
"tracked_shards": [0, 1, 2, 3]}
}
)
started = time.time()
act_to_val = [4, 4, 4, 4, 4]
ctx = TxContext(act_to_val, nodes)
# and ensures the balances get to the expected state in a timely manner.
# Sets epoch length to 10
import sys, time, base58, random
sys.path.append('lib')
from cluster import start_cluster
from utils import TxContext
from transaction import sign_payment_tx
TIMEOUT = 240
nodes = start_cluster(4, 0, 4, {
'local': True,
'near_root': '../target/debug/'
}, [["max_inflation_rate", 0], ["epoch_length", 10],
["block_producer_kickout_threshold", 70]], {})
started = time.time()
act_to_val = [3, 2, 0, 3]
ctx = TxContext(act_to_val, nodes)
last_balances = [x for x in ctx.expected_balances]
step = 0
sent_height = -1
while True:
}
node1_config = {
"consensus": {
"header_sync_initial_timeout": {
"secs": 3,
"nanos": 0
},
"header_sync_stall_ban_timeout": {
"secs": 5,
"nanos": 0
}
},
"tracked_shards": [0]
}
nodes = start_cluster(1, 1, 1, None, [["epoch_length", EPOCH_LENGTH]], {
0: node0_config,
1: node1_config
}, Handler)
status = nodes[0].get_status()
cur_height = status['sync_info']['latest_block_height']
while cur_height <= 110:
status = nodes[0].get_status()
cur_height = status['sync_info']['latest_block_height']
time.sleep(2)
should_sync.value = True
logger.info("sync node 1")
start = time.time()
HEIGHT_GOAL = 150
TIMEOUT = HEIGHT_GOAL * 3
EPOCH_LENGTH = 20
config = None
nodes = start_cluster(
2, 2, 1, config,
[["epoch_length", EPOCH_LENGTH], ["block_producer_kickout_threshold", 40]],
{
0: {
"view_client_throttle_period": {
"secs": 0,
"nanos": 0
}
},
1: {
"view_client_throttle_period": {
"secs": 0,
"nanos": 0
}
},
2: {
"tracked_shards": [0]
}
})
started = time.time()
def get_validators():
return set([x['account_id'] for x in nodes[0].get_status()['validators']])
#!/usr/bin/env python3
"""
Spawn a cluster with four nodes. Check that no node tries to
connect to another node that is currently connected.
"""
import sys, time
import pathlib
sys.path.append(str(pathlib.Path(__file__).resolve().parents[2] / 'lib'))
import cluster
import utils
BLOCKS = 20
nodes = cluster.start_cluster(4, 0, 4, None, [], {})
trackers = [utils.LogTracker(node) for node in nodes]
utils.wait_for_blocks(nodes[0], target=BLOCKS)
assert all(not tracker.check('Dropping handshake (Active Peer).')
for tracker in trackers)
]
nightly_genesis_change = [
["minimum_validators_per_shard", 2], ["min_gas_price", 0],
["max_inflation_rate", [0, 1]], ["epoch_length", 10],
["block_producer_kickout_threshold", 60],
["chunk_producer_kickout_threshold", 60],
["validators", 0, "amount", "110000000000000000000000000000000"],
[
"records", 0, "Account", "account", "locked",
"110000000000000000000000000000000"
], ["total_supply", "4060000000000000000000000000000000"]
]
# give more stake to the bootnode so that it can produce the blocks alone
nodes = start_cluster(2, 1, 8, None,
nightly_genesis_change if nightly else genesis_change,
{})
time.sleep(3)
nodes[1].kill()
started = time.time()
act_to_val = [0, 0, 0]
ctx = utils.TxContext(act_to_val, nodes)
last_balances = [x for x in ctx.expected_balances]
sent_height = -1
caught_up_times = 0
for height, hash_ in utils.poll_blocks(nodes[0],
TIMEOUT = 300
BLOCKS = 30
# Low sync_check_period to sync from a new peer with greater height
client_config_change = {
"consensus": {
"sync_check_period": {
"secs": 0,
"nanos": 100000000
}
}
}
nodes = start_cluster(
2, 0, 2, None,
[["epoch_length", 100], ["block_producer_kickout_threshold", 80]],
{0: client_config_change})
if not doomslug:
# we expect inconsistency in store in node 0
# because we're going to turn off doomslug
# and allow applying blocks without proper validation
nodes[0].stop_checking_store()
started = time.time()
time.sleep(2)
print("Waiting for %s blocks..." % BLOCKS)
while True:
assert time.time() - started < TIMEOUT
status = nodes[0].get_status()
logging.info(f"Height: {h}")
with success.get_lock():
if h >= 10 and success.value == 0:
logging.info(
f'SUCCESS DROP={self.dropped} TOTAL={self.total}')
success.value = 1
drop = random.random() < DROP_RATIO and 'Handshake' not in msg.enum
if drop:
self.dropped += 1
self.total += 1
return not drop
start_cluster(3, 0, 1, None, [["epoch_length", 500]], {}, Handler)
started = time.time()
while True:
logging.info(f"Time: {time.time() - started:0.2}, Fin: {success.value}")
assert time.time() - started < TIMEOUT
time.sleep(1)
if success.value == 1:
break
logging.info("Success")
consensus_config = {
"consensus": {
"block_fetch_horizon": 20,
"block_header_fetch_horizon": 20
}
}
nodes = start_cluster(
4, 0, 1, None,
[["epoch_length", 10],
["validators", 0, "amount", "12500000000000000000000000000000"],
[
"records", 0, "Account", "account", "locked",
"12500000000000000000000000000000"
], ["validators", 1, "amount", "12500000000000000000000000000000"],
[
"records", 2, "Account", "account", "locked",
"12500000000000000000000000000000"
], ['total_supply', "4925000000000000000000000000000000"],
["block_producer_kickout_threshold", 40],
["chunk_producer_kickout_threshold", 40], ["num_block_producer_seats", 10],
["num_block_producer_seats_per_shard", [10]]], {1: consensus_config})
logger.info('Kill node 1')
nodes[1].kill()
node0_height, _ = utils.wait_for_blocks(nodes[0],
target=TARGET_HEIGHT1,
verbose=True)
# process transactions. Makes large-ish number of block producers per shard to minimize
# the chance of the second block producer occupying all the seats in one of the shards
import sys, time, base58, random
sys.path.append('lib')
from cluster import start_cluster
from utils import TxContext
from transaction import sign_payment_tx
TIMEOUT = 240
nodes = start_cluster(
2, 1, 8, None,
[["num_block_producer_seats", 199],
["num_block_producer_seats_per_shard", [24, 25, 25, 25, 25, 25, 25, 25]],
["min_gas_price", 0], ["max_inflation_rate", 0], ["epoch_length", 10],
["block_producer_kickout_threshold", 70]], {})
time.sleep(3)
nodes[1].kill()
started = time.time()
act_to_val = [0, 0, 0]
ctx = TxContext(act_to_val, nodes)
last_balances = [x for x in ctx.expected_balances]
max_height = 0
sent_height = -1
caught_up_times = 0
from cluster import start_cluster
BLOCK_WAIT = 40
EPOCH_LENGTH = 80
consensus_config = {
"consensus": {
"block_fetch_horizon": 10,
"block_header_fetch_horizon": 10
}
}
nodes = start_cluster(2, 0, 1, {
'local': True,
'near_root': '../target/debug/'
}, [["epoch_length", EPOCH_LENGTH], ["block_producer_kickout_threshold", 10],
["chunk_producer_kickout_threshold", 10]], {
0: consensus_config,
1: consensus_config
})
time.sleep(2)
nodes[1].kill()
cur_height = 0
print("step 1")
while cur_height < BLOCK_WAIT:
status = nodes[0].get_status()
cur_height = status['sync_info']['latest_block_height']
time.sleep(2)
nodes[1].start(nodes[1].node_key.pk, nodes[1].addr())
time.sleep(2)
sys.path.append('lib')
from cluster import start_cluster
from configured_logger import logger
from utils import TxContext
from transaction import sign_payment_tx
TIMEOUT = 180
# give more stake to the bootnode so that it can produce the blocks alone
nodes = start_cluster(
2, 1, 8, None,
[["num_block_producer_seats", 199],
["num_block_producer_seats_per_shard", [24, 25, 25, 25, 25, 25, 25, 25]],
["min_gas_price", 0], ["max_inflation_rate", [0, 1]], ["epoch_length", 10],
["block_producer_kickout_threshold", 60],
["chunk_producer_kickout_threshold", 60],
["validators", 0, "amount", "110000000000000000000000000000000"],
[
"records", 0, "Account", "account", "locked",
"110000000000000000000000000000000"
], ["total_supply", "4060000000000000000000000000000000"]], {})
time.sleep(3)
nodes[1].kill()
started = time.time()
act_to_val = [0, 0, 0]
ctx = TxContext(act_to_val, nodes)
last_balances = [x for x in ctx.expected_balances]
sys.path.append('lib')
from cluster import start_cluster
from utils import wait_for_blocks_or_timeout
TIMEOUT = 60
EPOCH_LENGTH = 20
config = None
nodes = start_cluster(
4, 1, 1, config,
[
["epoch_length", EPOCH_LENGTH], ["block_producer_kickout_threshold", 40],
['protocol_version', 35], ['max_inflation_rate', [0, 1]],
['protocol_reward_rate', [0, 1]], ['block_producer_kickout_threshold', 70],
['chunk_producer_kickout_threshold', 70]
],
{4: {
"tracked_shards": [0]
}})
started = time.time()
old_accounts = [
nodes[0].get_account("test%s" % x)['result']for x in range(4)
]
near_account_old = nodes[0].get_account("near")
wait_for_blocks_or_timeout(nodes[4], EPOCH_LENGTH * 3 + 2, TIMEOUT)
for i in range(4):
# test various ways of submitting transactions (broadcast_tx_async, broadcast_tx_sync, broadcast_tx_commit)
import sys, time, base58, base64
sys.path.append('lib')
from cluster import start_cluster
from configured_logger import logger
from utils import TxContext
from transaction import sign_payment_tx
nodes = start_cluster(
2, 1, 1,
None, [["min_gas_price", 0], ['max_inflation_rate', [0, 1]],
["epoch_length", 100], ['transaction_validity_period', 200],
["block_producer_kickout_threshold", 70]], {})
time.sleep(3)
started = time.time()
old_balances = [
int(nodes[0].get_account("test%s" % x)['result']['amount'])
for x in [0, 1]
]
logger.info(f"BALANCES BEFORE {old_balances}")
status = nodes[0].get_status()
hash1 = status['sync_info']['latest_block_hash']
for i in range(3):
tx = sign_payment_tx(nodes[0].signer_key, 'test1', 100 + i, i + 1,
node0_config = {"gc_blocks_limit": 10}
node1_config = {
"consensus": {
"block_fetch_horizon": 10,
"block_header_fetch_horizon": 10,
"state_fetch_horizon": 0
},
"tracked_shards": [0],
"gc_blocks_limit": 10,
}
nodes = start_cluster(
1, 1, 1, None,
[["epoch_length", 10], ["block_producer_kickout_threshold", 80],
["chunk_producer_kickout_threshold", 80]], {
0: node0_config,
1: node1_config
})
status1 = nodes[1].get_status()
height = status1['sync_info']['latest_block_height']
start = time.time()
while height < TARGET_HEIGHT1:
assert time.time() - start < TIME_OUT
time.sleep(1)
status1 = nodes[1].get_status()
height = status1['sync_info']['latest_block_height']
logger.info('Kill node 1')
from transaction import sign_staking_tx, sign_create_account_with_full_access_key_and_balance_tx
MAX_SYNC_WAIT = 30
EPOCH_LENGTH = 10
node1_config = {
"consensus": {
"sync_step_period": {
"secs": 0,
"nanos": 100
}
},
"tracked_shards": [0]
}
nodes = start_cluster(
1, 1, 1, None,
[["epoch_length", EPOCH_LENGTH], ["block_producer_kickout_threshold", 10],
["chunk_producer_kickout_threshold", 10]], {1: node1_config})
time.sleep(2)
nodes[1].kill()
print('node1 is killed')
status = nodes[0].get_status()
block_hash = status['sync_info']['latest_block_hash']
cur_height = status['sync_info']['latest_block_height']
num_new_accounts = 10
balance = 50000000000000000000000000000000
account_keys = []
for i in range(num_new_accounts):
signer_key = Key(f'test_account{i}', nodes[0].signer_key.pk,
nodes[0].signer_key.sk)
"nanos": 400000000
},
"max_block_wait_delay": {
"secs": 0,
"nanos": 400000000
}
}
}
nodes = start_cluster(
3, 0, 1, None,
[["epoch_length", 10], ["num_block_producer_seats", 5],
["num_block_producer_seats_per_shard", [5]],
["total_supply", "4210000000000000000000000000000000"],
["validators", 0, "amount", "260000000000000000000000000000000"],
[
"records", 0, "Account", "account", "locked",
"260000000000000000000000000000000"
]], {
0: consensus_config,
1: consensus_config,
2: consensus_config
})
logger.info('kill node1 and node2')
nodes[1].kill()
nodes[2].kill()
node0_height, _ = utils.wait_for_blocks(nodes[0], target=TARGET_HEIGHT)
logger.info('Restart node 1')
nodes[1].start(boot_node=nodes[1])
client_config_changes = {}
if not config['local']:
client_config_changes = {
"min_block_production_delay": {
"secs": 4,
},
"max_block_production_delay": {
"secs": 8,
},
"max_block_wait_delay": {
"secs": 24,
},
}
TIMEOUT = 600
nodes = start_cluster(
4, 0, 4, None,
[["epoch_length", 6], ["block_producer_kickout_threshold", 80]],
client_config_changes)
started = time.time()
hash_to_height = {}
hash_to_epoch = {}
hash_to_next_epoch = {}
height_to_hash = {}
epochs = []
block_producers_map = {}
def get_light_client_block(hash_, last_known_block):
global block_producers_map
# Make sure all nodes can still sync.
import sys, time, base58
sys.path.append('lib')
from cluster import start_cluster
from transaction import sign_staking_tx
EPOCH_LENGTH = 20
tracked_shards = {"tracked_shards": [0, 1, 2, 3]}
nodes = start_cluster(3, 1, 4, {
'local': True,
'near_root': '../target/debug/'
}, [["epoch_length", EPOCH_LENGTH], ["block_producer_kickout_threshold", 10],
["chunk_producer_kickout_threshold", 10]], {
0: tracked_shards,
1: tracked_shards
})
time.sleep(3)
status = nodes[0].get_status()
hash_ = status['sync_info']['latest_block_hash']
for i in range(4):
stake = 100000000000000000000000000 if i == 3 else 0
tx = sign_staking_tx(nodes[i].signer_key, nodes[i].validator_key, stake, 1,
base58.b58decode(hash_.encode('utf8')))
nodes[0].send_tx(tx)
print("test%s stakes %d" % (i, stake))
"nanos": 400000000
},
"max_block_wait_delay": {
"secs": 0,
"nanos": 400000000
}
}
}
nodes = start_cluster(
2, 0, 1, None,
[["epoch_length", 10], ["num_block_producer_seats", 5],
["num_block_producer_seats_per_shard", [5]],
["chunk_producer_kickout_threshold", 80],
["validators", 0, "amount", "110000000000000000000000000000000"],
[
"records", 0, "Account", "account", "AccountV1", "locked",
"110000000000000000000000000000000"
], ["total_supply", "3060000000000000000000000000000000"]], {
0: consensus_config,
1: consensus_config
})
print('Kill node 1')
nodes[1].kill()
node0_height = 0
while node0_height < TARGET_HEIGHT:
status = nodes[0].get_status()
node0_height = status['sync_info']['latest_block_height']
time.sleep(2)
expect receiving HandshakeFailure. Use that information to send valid handshake and
connect to the node.
"""
import asyncio
import socket
import sys
import time
sys.path.append('lib')
import base58
import nacl.signing
from cluster import start_cluster
from peer import ED_PREFIX, connect, create_handshake, sign_handshake
nodes = start_cluster(1, 0, 4, None, [], {})
async def main():
my_key_pair_nacl = nacl.signing.SigningKey.generate()
conn = await connect(nodes[0].addr())
handshake = create_handshake(my_key_pair_nacl, nodes[0].node_key.pk, 12345)
# First handshake attempt. Should fail with Protocol Version Mismatch
sign_handshake(my_key_pair_nacl, handshake.HandshakeV2)
await conn.send(handshake)
response = await conn.recv()
assert response.enum == 'HandshakeFailure', response.enum
}
}
}
consensus_config1 = {
"consensus": {
"sync_step_period": {
"secs": 0,
"nanos": 1000
}
},
"tracked_shards": [0]
}
nodes = start_cluster(
1, 1, 1, None,
[["epoch_length", EPOCH_LENGTH], ["block_producer_kickout_threshold", 10],
["chunk_producer_kickout_threshold", 10]], {
0: consensus_config0,
1: consensus_config1
})
time.sleep(2)
nodes[1].kill()
print("step 1")
node0_height = 0
while node0_height <= EPOCH_LENGTH + 600:
status = nodes[0].get_status()
node0_height = status['sync_info']['latest_block_height']
time.sleep(5)
nodes[1].start(nodes[1].node_key.pk, nodes[1].addr())
time.sleep(2)
# test various ways of submitting transactions (broadcast_tx_async, broadcast_tx_sync, broadcast_tx_commit)
import sys, time, base58, base64
sys.path.append('lib')
from cluster import start_cluster
from utils import TxContext
from transaction import sign_payment_tx
nodes = start_cluster(
2, 0, 1,
None, [["min_gas_price", 0], ['max_inflation_rate', [0, 1]],
["epoch_length", 10], ["block_producer_kickout_threshold", 70]], {})
time.sleep(3)
started = time.time()
old_balances = [
int(nodes[0].get_account("test%s" % x)['result']['amount'])
for x in [0, 1]
]
print("BALANCES BEFORE", old_balances)
status = nodes[0].get_status()
hash_ = status['sync_info']['latest_block_hash']
for i in range(3):
tx = sign_payment_tx(nodes[0].signer_key, 'test1', 100 + i, i + 1,
base58.b58decode(hash_.encode('utf8')))
if i == 0:
async def main():
# start a cluster with two shards
nodes = start_cluster(2, 0, 2, None, [], {})
started = time.time()
while True:
if time.time() - started > 10:
assert False, "Giving up waiting for two blocks"
status = nodes[0].get_status()
hash_ = status['sync_info']['latest_block_hash']
height = status['sync_info']['latest_block_height']
if height > 2:
block = nodes[0].get_block(hash_)
chunk_hashes = [
base58.b58decode(x['chunk_hash'])
for x in block['result']['chunks']
]
assert len(chunk_hashes) == 2
assert all([len(x) == 32 for x in chunk_hashes])
break
my_key_pair_nacl = nacl.signing.SigningKey.generate()
received_responses = [None, None]
# step = 0: before the node is killed
# step = 1: after the node is killed
for step in range(2):
conn0 = await connect(nodes[0].addr())
await run_handshake(conn0, nodes[0].node_key.pk, my_key_pair_nacl)
for shard_ord, chunk_hash in enumerate(chunk_hashes):
request = PartialEncodedChunkRequestMsg()
request.chunk_hash = chunk_hash
request.part_ords = []
request.tracking_shards = [0, 1]
routed_msg_body = RoutedMessageBody()
routed_msg_body.enum = 'PartialEncodedChunkRequest'
routed_msg_body.PartialEncodedChunkRequest = request
peer_message = create_and_sign_routed_peer_message(
routed_msg_body, nodes[0], my_key_pair_nacl)
await conn0.send(peer_message)
received_response = False
def predicate(response):
return response.enum == 'Routed' and response.Routed.body.enum == 'PartialEncodedChunkResponse'
try:
response = await asyncio.wait_for(conn0.recv(predicate), 5)
except concurrent.futures._base.TimeoutError:
response = None
if response is not None:
logger.info("Received response for shard %s" % shard_ord)
received_response = True
else:
logger.info("Didn't receive response for shard %s" % shard_ord)
if step == 0:
received_responses[shard_ord] = received_response
else:
assert received_responses[
shard_ord] == received_response, "The response doesn't match for the chunk in shard %s. Received response before node killed: %s, after: %s" % (
shard_ord, received_responses[shard_ord],
received_response)
# we expect first node to only respond to one of the chunk requests, for the shard assigned to it
assert received_responses[0] != received_responses[
1], received_responses
if step == 0:
logger.info("Killing and restarting nodes")
nodes[1].kill()
nodes[0].kill()
nodes[0].start(None, None)
time.sleep(1)
STOP_HEIGHT1 = 35
TIMEOUT = 50
config1 = {
"network": {
"ttl_account_id_router": {
"secs": 1,
"nanos": 0
},
}
}
nodes = start_cluster(
2, 0, 1, None,
[["epoch_length", EPOCH_LENGTH], ["block_producer_kickout_threshold", 30],
["chunk_producer_kickout_threshold", 30], ["num_block_producer_seats", 4],
["num_block_producer_seats_per_shard", [4]],
["validators", 0, "amount", "150000000000000000000000000000000"],
[
"records", 0, "Account", "account", "locked",
"150000000000000000000000000000000"
], ["total_supply", "3100000000000000000000000000000000"]], {1: config1})
time.sleep(2)
status1 = nodes[1].get_status()
height1 = status1['sync_info']['latest_block_height']
block = nodes[1].get_block(height1)
epoch_id = block['result']['header']['epoch_id']
start = time.time()
while True:
assert time.time() - start < TIMEOUT
time.sleep(1)
