def __init__(self):
node_config = {
'archive': True,
'tracked_shards': [0],
}
self._config = cluster.load_config()
self._near_root, self._node_dirs = cluster.init_cluster(
num_nodes=1,
num_observers=2,
num_shards=1,
config=self._config,
genesis_config_changes=[['epoch_length', EPOCH_LENGTH],
['block_producer_kickout_threshold', 80]],
client_config_changes={
0: node_config,
1: node_config,
2: node_config,
3: node_config
})
self._nodes = [None] * len(self._node_dirs)
import sys, time, base58
sys.path.append('lib')
from cluster import init_cluster, spin_up_node
from transaction import sign_staking_tx
from utils import TxContext
TIMEOUT = 300
TWENTY_FIVE = 25
config = {'local': True, 'near_root': '../target/debug/'}
near_root, node_dirs = init_cluster(
2, 1, 2, config, [["max_inflation_rate", 0], ["epoch_length", 7],
["validator_kickout_threshold", 80]],
{2: {
"tracked_shards": [0, 1]
}})
started = time.time()
boot_node = spin_up_node(config, near_root, node_dirs[0], 0, None, None)
#node1 = spin_up_node(config, near_root, node_dirs[1], 1, boot_node.node_key.pk, boot_node.addr())
observer = spin_up_node(config, near_root, node_dirs[2], 2,
boot_node.node_key.pk, boot_node.addr())
ctx = TxContext([0, 0, 0], [boot_node, None, observer])
initial_balances = ctx.get_balances()
total_supply = sum(initial_balances)
near_root, node_dirs = init_cluster(
2,
3,
2,
config,
[
["min_gas_price", 0],
["max_inflation_rate", [0, 1]],
["epoch_length", EPOCH_LENGTH],
['num_block_producer_seats', 4],
["block_producer_kickout_threshold", 20],
["chunk_producer_kickout_threshold", 20],
["validators", 0, "amount", "110000000000000000000000000000000"],
["validators", 1, "amount", "110000000000000000000000000000000"],
[
"records", 0, "Account", "account", "locked",
"110000000000000000000000000000000"
],
# each validator account is two records, thus the index of a record for the second is 2, not 1
[
"records", 2, "Account", "account", "locked",
"110000000000000000000000000000000"
],
["total_supply", "6120000000000000000000000000000000"]
],
{
4: {
"tracked_shards": [0, 1],
"archive": True
},
3: {
"archive": True,
"tracked_shards": [1],
"network": {
"ttl_account_id_router": {
"secs": 1,
"nanos": 0
}
}
},
2: {
"archive": True,
"tracked_shards": [0],
"network": {
"ttl_account_id_router": {
"secs": 1,
"nanos": 0
}
}
}
})
from cluster import init_cluster, spin_up_node, load_config
from transaction import sign_staking_tx
from utils import TxContext
TIMEOUT = 600
# the height we spin up the second node
TARGET_HEIGHT = 35
config = load_config()
# give more stake to the bootnode so that it can produce the blocks alone
near_root, node_dirs = init_cluster(
4, 1, 4, config,
[["min_gas_price", 0], ["max_inflation_rate", [0, 1]], ["epoch_length", 12],
["block_producer_kickout_threshold", 20], ["chunk_producer_kickout_threshold", 20]],
{0: {"view_client_throttle_period": {"secs": 0, "nanos": 0}, "consensus": {"state_sync_timeout": {"secs": 2, "nanos": 0}}},
1: {"view_client_throttle_period": {"secs": 0, "nanos": 0}, "consensus": {"state_sync_timeout": {"secs": 2, "nanos": 0}}},
2: {"view_client_throttle_period": {"secs": 0, "nanos": 0}, "consensus": {"state_sync_timeout": {"secs": 2, "nanos": 0}}},
3: {"view_client_throttle_period": {"secs": 0, "nanos": 0}, "consensus": {"state_sync_timeout": {"secs": 2, "nanos": 0}}}, 4: {
"tracked_shards": [0, 1, 2, 3],
"view_client_throttle_period": {"secs": 0, "nanos": 0}
}})
started = time.time()
boot_node = spin_up_node(config, near_root, node_dirs[0], 0, None, None)
boot_node.stop_checking_store()
node3 = spin_up_node(config, near_root, node_dirs[2], 2, boot_node.node_key.pk,
boot_node.addr())
node4 = spin_up_node(config, near_root, node_dirs[3], 3, boot_node.node_key.pk,
boot_node.addr())
observer = spin_up_node(config, near_root, node_dirs[4], 4,
if len(sys.argv) < 3:
print("python state_sync.py [notx, onetx, manytx] <launch_at_block>")
exit(1)
mode = sys.argv[1]
assert mode in ['notx', 'onetx', 'manytx']
from cluster import init_cluster, spin_up_node, load_config
from utils import TxContext, LogTracker
START_AT_BLOCK = int(sys.argv[2])
TIMEOUT = 150 + START_AT_BLOCK * 10
config = load_config()
near_root, node_dirs = init_cluster(2, 1, 1, config, [["min_gas_price", 0], ["max_inflation_rate", [0, 1]], ["epoch_length", 10], ["block_producer_kickout_threshold", 80]], {2: {"tracked_shards": [0]}})
started = time.time()
boot_node = spin_up_node(config, near_root, node_dirs[0], 0, None, None)
node1 = spin_up_node(config, near_root, node_dirs[1], 1, boot_node.node_key.pk, boot_node.addr())
ctx = TxContext([0, 0], [boot_node, node1])
sent_txs = False
observed_height = 0
while observed_height < START_AT_BLOCK:
assert time.time() - started < TIMEOUT
status = boot_node.get_status()
new_height = status['sync_info']['latest_block_height']
def doit(s, n, N, k, monkeys, timeout):
global block_timeout, balances_timeout, tx_tolerance
assert 2 <= n <= N
config = {'local': True, 'near_root': '../target/debug/'}
local_config_changes = {}
for i in range(N, N + k + 1):
# make all the observers track all the shards
local_config_changes[i] = {"tracked_shards": list(range(s))}
near_root, node_dirs = init_cluster(N, s, k + 1, config, [["max_inflation_rate", 0], ["epoch_length", EPOCH_LENGTH], ["validator_kickout_threshold", 75]], local_config_changes)
started = time.time()
boot_node = spin_up_node(config, near_root, node_dirs[0], 0, None, None)
boot_node.mess_with = False
nodes = [boot_node]
for i in range(1, N + k + 1):
node = spin_up_node(config, near_root, node_dirs[i], i, boot_node.node_key.pk, boot_node.addr())
nodes.append(node)
if i >= n and i < N:
node.kill()
node.mess_with = True
else:
node.mess_with = False
monkey_names = [x.__name__ for x in monkeys]
print(monkey_names)
if 'monkey_local_network' in monkey_names or 'monkey_global_network' in monkey_names:
print("There are monkeys messing up with network, initializing the infra")
init_network_pillager()
expect_network_issues()
block_timeout += 10
tx_tolerance += 0.3
if 'monkey_node_restart' in monkey_names:
expect_network_issues()
if 'monkey_node_restart' in monkey_names or 'monkey_node_set' in monkey_names:
block_timeout += 10
balances_timeout += 10
tx_tolerance += 0.4
stopped = Value('i', 0)
error = Value('i', 0)
ps = []
nonces = [(Value('i', 1), Lock()) for _ in range(N + k + 1)]
def launch_process(func):
nonlocal stopped, error, ps
p = Process(target=func, args=(stopped, error, nodes, nonces))
p.start()
ps.append((p, func.__name__))
def check_errors():
nonlocal error, ps
if error.value != 0:
for (p, _) in ps:
p.terminate()
assert False, "At least one process failed, check error messages above"
for monkey in monkeys:
launch_process(monkey)
launch_process(blocks_tracker)
started = time.time()
while time.time() - started < timeout:
check_errors()
time.sleep(1)
print("")
print("==========================================")
print("# TIMEOUT IS HIT, SHUTTING DOWN THE TEST #")
print("==========================================")
stopped.value = 1
started_shutdown = time.time()
while True:
check_errors()
still_running = [name for (p, name) in ps if p.is_alive()]
if len(still_running) == 0:
break
if time.time() - started_shutdown > TIMEOUT_SHUTDOWN:
for (p, _) in ps:
p.terminate()
assert False, "The test didn't gracefully shut down in time\nStill running: %s" % (still_running)
check_errors()
def doit(s, n, N, k, monkeys):
assert 2 <= n <= N
config = {'local': True, 'near_root': '../target/debug/'}
local_config_changes = {}
for i in range(N, N + k + 1):
# make all the observers track all the shards
local_config_changes[i] = {"tracked_shards": list(range(s))}
near_root, node_dirs = init_cluster(
N, s, k + 1, config,
[["max_inflation_rate", 0], ["epoch_length", EPOCH_LENGTH],
["validator_kickout_threshold", 75]], local_config_changes)
started = time.time()
boot_node = spin_up_node(config, near_root, node_dirs[0], 0, None, None)
nodes = [boot_node]
for i in range(1, N + k + 1):
if i < n or i >= N:
node = spin_up_node(config, near_root, node_dirs[i], i,
boot_node.node_key.pk, boot_node.addr())
nodes.append(node)
else:
nodes.append(None)
stopped = Value('i', 0)
error = Value('i', 0)
ps = []
nonces = [(Value('i', 1), Lock()) for _ in range(N + k + 1)]
def launch_process(func):
nonlocal stopped, error, ps
p = Process(target=func, args=(stopped, error, nodes, nonces))
p.start()
ps.append((p, func.__name__))
def check_errors():
nonlocal error, ps
if error.value != 0:
for (p, _) in ps:
p.terminate()
assert False, "At least one process failed, check error messages above"
for monkey in monkeys:
launch_process(globals()['monkey_%s' % monkey])
launch_process(blocks_tracker)
started = time.time()
while time.time() - started < TIMEOUT:
check_errors()
time.sleep(1)
print("")
print("==========================================")
print("# TIMEOUT IS HIT, SHUTTING DOWN THE TEST #")
print("==========================================")
stopped.value = 1
started_shutdown = time.time()
while True:
check_errors()
still_running = [name for (p, name) in ps if p.is_alive()]
if len(still_running) == 0:
break
if time.time() - started_shutdown > TIMEOUT_SHUTDOWN:
for (p, _) in ps:
p.terminate()
assert False, "The test didn't gracefully shut down in time\nStill running: %s" % (
still_running)
check_errors()
def doit(s, n, N, k, monkeys, timeout):
global block_timeout, balances_timeout, tx_tolerance, epoch_length, wait_if_restart, wipe_data, restart_sync_timeout
assert 2 <= n <= N
config = load_config()
local_config_changes = {}
monkey_names = [x.__name__ for x in monkeys]
proxy = None
logging.info(monkey_names)
for i in range(N + k + 1):
local_config_changes[i] = {
"consensus": {"block_header_fetch_horizon": BLOCK_HEADER_FETCH_HORIZON, "state_sync_timeout": {"secs": 5, "nanos": 0}},
"view_client_throttle_period": {"secs": 0, "nanos": 0}
}
for i in range(N, N + k + 1):
# make all the observers track all the shards
local_config_changes[i]["tracked_shards"] = list(range(s))
if 'monkey_wipe_data' in monkey_names:
# When data can be deleted, with the short epoch length while the node with deleted data folder is syncing,
# other nodes can run sufficiently far ahead to GC the old data. Have one archival node to address it.
# It is also needed, because the balances timeout is longer, and the txs can get GCed on the observer node
# by the time it gets to checking their status.
local_config_changes[N + k]['archive'] = True
if 'monkey_local_network' in monkey_names or 'monkey_packets_drop' in monkey_names or 'monkey_node_restart' in monkey_names:
expect_network_issues()
block_timeout += 40
if 'monkey_local_network' in monkey_names or 'monkey_packets_drop' in monkey_names:
assert config['local'], 'Network stress operations only work on local nodes'
drop_probability = 0.05 if 'monkey_packets_drop' in monkey_names else 0
reject_list = RejectListProxy.create_reject_list(1)
proxy = RejectListProxy(reject_list, drop_probability)
tx_tolerance += 0.3
if 'monkey_local_network' in monkey_names or 'monkey_packets_drop' in monkey_names:
# add 15 seconds + 10 seconds for each unique network-related monkey
balances_timeout += 15
if 'monkey_local_network' in monkey_names:
balances_timeout += 10
if 'monkey_packets_drop' in monkey_names:
wait_if_restart = True
balances_timeout += 10
if 'monkey_node_restart' in monkey_names or 'monkey_node_set' in monkey_names:
balances_timeout += 10
tx_tolerance += 0.5
if 'monkey_wipe_data' in monkey_names:
assert 'monkey_node_restart' in monkey_names or 'monkey_node_set' in monkey_names
wipe_data = True
balances_timeout += 25
# if nodes can restart, we should give them way more time to sync.
# if packets can also be dropped, each state-sync-related request or response lost adds 10 seconds
# to the sync process.
restart_sync_timeout = 45 if 'monkey_packets_drop' not in monkey_names else 90
block_timeout += (10 if 'monkey_packets_drop' not in monkey_names else 40)
# We need to make sure that the blocks that include txs are not garbage collected. From the first tx sent until
# we check balances time equal to `balances_timeout * 2` passes, and the block production is capped at 1.7/s.
# The GC keeps five epochs of blocks.
min_epoch_length = (int((balances_timeout * 2) * 1.7) + 4) // 5
epoch_length = max(epoch_length, min_epoch_length)
near_root, node_dirs = init_cluster(
N, k + 1, s, config,
[["min_gas_price", 0], ["max_inflation_rate", [0, 1]],
["epoch_length", epoch_length],
["block_producer_kickout_threshold", 10],
["chunk_producer_kickout_threshold", 10]], local_config_changes)
started = time.time()
boot_node = spin_up_node(config, near_root, node_dirs[0], 0, None, None, proxy=proxy)
boot_node.stop_checking_store()
boot_node.mess_with = False
nodes = [boot_node]
for i in range(1, N + k + 1):
node = spin_up_node(config, near_root, node_dirs[i], i,
boot_node.node_key.pk, boot_node.addr(), proxy=proxy)
node.stop_checking_store()
nodes.append(node)
if i >= n and i < N:
node.kill()
node.mess_with = True
else:
node.mess_with = False
stopped = Value('i', 0)
error = Value('i', 0)
ps = []
nonces = [(Value('i', 1), Lock()) for _ in range(N + k + 1)]
def launch_process(func):
nonlocal stopped, error, ps
p = Process(target=func, args=(stopped, error, nodes, nonces))
p.start()
ps.append((p, func.__name__))
def check_errors():
nonlocal error, ps
if error.value != 0:
for (p, _) in ps:
p.terminate()
assert False, "At least one process failed, check error messages above"
for monkey in monkeys:
launch_process(monkey)
launch_process(blocks_tracker)
started = time.time()
while time.time() - started < timeout:
check_errors()
time.sleep(1)
logging.info("")
logging.info("==========================================")
logging.info("# TIMEOUT IS HIT, SHUTTING DOWN THE TEST #")
logging.info("==========================================")
stopped.value = 1
started_shutdown = time.time()
proxies_stopped = False
while True:
check_errors()
still_running = [name for (p, name) in ps if p.is_alive()]
if len(still_running) == 0:
break
# If the test is running with proxies, `node_restart` and `node_set` can get
# stuck because the proxies now are their child processes. We can't kill the
# proxies rigth away, because that would interfere with block production, and
# might prevent other workers (e.g. block_tracker) from completing in a timely
# manner. Thus, kill the proxies some time into the shut down process.
if time.time() - started_shutdown > TIMEOUT_SHUTDOWN / 2 and not proxies_stopped:
logging.info("Shutdown is %s seconds in, shutting down proxies if any" % (TIMEOUT_SHUTDOWN / 2))
if boot_node.proxy is not None:
boot_node.proxy.global_stopped.value = 1
for p in boot_node.proxy.ps:
p.terminate()
proxies_stopped = True
if time.time() - started_shutdown > TIMEOUT_SHUTDOWN:
for (p, _) in ps:
p.terminate()
assert False, "The test didn't gracefully shut down in time\nStill running: %s" % (
still_running)
check_errors()
logging.info("Shut down complete, executing store validity checks")
for node in nodes:
node.is_check_store = True
node.check_store()
def doit(s, n, N, k, monkeys, timeout):
global block_timeout, balances_timeout, tx_tolerance
assert 2 <= n <= N
config = load_config()
local_config_changes = {}
for i in range(N, N + k + 1):
# make all the observers track all the shards
local_config_changes[i] = {"tracked_shards": list(range(s))}
near_root, node_dirs = init_cluster(
N, k + 1, s, config,
[["min_gas_price", 0], ["max_inflation_rate", [0, 1]],
["epoch_length", EPOCH_LENGTH],
["block_producer_kickout_threshold", 10],
["chunk_producer_kickout_threshold", 10]], local_config_changes)
monkey_names = [x.__name__ for x in monkeys]
proxy = None
logging.info(monkey_names)
if 'monkey_local_network' in monkey_names or 'monkey_global_network' in monkey_names:
assert config[
'local'], 'Network stress operations only work on local nodes'
reject_list = RejectListProxy.create_reject_list(1)
proxy = RejectListProxy(reject_list)
expect_network_issues()
block_timeout += 40
balances_timeout += 20
tx_tolerance += 0.3
if 'monkey_node_restart' in monkey_names:
expect_network_issues()
if 'monkey_node_restart' in monkey_names or 'monkey_node_set' in monkey_names:
block_timeout += 40
balances_timeout += 10
tx_tolerance += 0.5
started = time.time()
boot_node = spin_up_node(config,
near_root,
node_dirs[0],
0,
None,
None,
proxy=proxy)
boot_node.stop_checking_store()
boot_node.mess_with = False
nodes = [boot_node]
for i in range(1, N + k + 1):
node = spin_up_node(config,
near_root,
node_dirs[i],
i,
boot_node.node_key.pk,
boot_node.addr(),
proxy=proxy)
node.stop_checking_store()
nodes.append(node)
if i >= n and i < N:
node.kill()
node.mess_with = True
else:
node.mess_with = False
stopped = Value('i', 0)
error = Value('i', 0)
ps = []
nonces = [(Value('i', 1), Lock()) for _ in range(N + k + 1)]
def launch_process(func):
nonlocal stopped, error, ps
p = Process(target=func, args=(stopped, error, nodes, nonces))
p.start()
ps.append((p, func.__name__))
def check_errors():
nonlocal error, ps
if error.value != 0:
for (p, _) in ps:
p.terminate()
assert False, "At least one process failed, check error messages above"
for monkey in monkeys:
launch_process(monkey)
launch_process(blocks_tracker)
started = time.time()
while time.time() - started < timeout:
check_errors()
time.sleep(1)
logging.info("")
logging.info("==========================================")
logging.info("# TIMEOUT IS HIT, SHUTTING DOWN THE TEST #")
logging.info("==========================================")
stopped.value = 1
started_shutdown = time.time()
while True:
check_errors()
still_running = [name for (p, name) in ps if p.is_alive()]
if len(still_running) == 0:
break
if time.time() - started_shutdown > TIMEOUT_SHUTDOWN:
for (p, _) in ps:
p.terminate()
assert False, "The test didn't gracefully shut down in time\nStill running: %s" % (
still_running)
check_errors()
logging.info("Shut down complete, executing store validity checks")
for node in nodes:
node.is_check_store = True
node.check_store()
sys.path.append('lib')
from cluster import init_cluster, spin_up_node, load_config
from transaction import sign_staking_tx
from utils import TxContext
TIMEOUT = 600
# the height we spin up the second node
TARGET_HEIGHT = 35
config = load_config()
# give more stake to the bootnode so that it can produce the blocks alone
near_root, node_dirs = init_cluster(
4, 1, 4,
config, [["min_gas_price", 0], ["max_inflation_rate", [0, 1]],
["epoch_length", 12], ["block_producer_kickout_threshold", 20],
["chunk_producer_kickout_threshold", 20]],
{4: {
"tracked_shards": [0, 1, 2, 3]
}})
started = time.time()
boot_node = spin_up_node(config, near_root, node_dirs[0], 0, None, None)
node3 = spin_up_node(config, near_root, node_dirs[2], 2, boot_node.node_key.pk,
boot_node.addr())
node4 = spin_up_node(config, near_root, node_dirs[3], 3, boot_node.node_key.pk,
boot_node.addr())
observer = spin_up_node(config, near_root, node_dirs[4], 4,
boot_node.node_key.pk, boot_node.addr())
ctx = TxContext([0, 0, 0, 0, 0], [boot_node, None, node3, node4, observer])
config = load_config()
near_root, node_dirs = init_cluster(
2, 1, 1, config,
[["min_gas_price", 0], ["max_inflation_rate", [0, 1]], ["epoch_length", 10],
["protocol_version", 47],
[
"simple_nightshade_shard_layout", {
"V1": {
"fixed_shards": [],
"boundary_accounts":
["aurora", "aurora-0", "kkuuue2akv_1630967379.near"],
"shards_split_map": [[0, 1, 2, 3]],
"to_parent_shard_map": [0, 0, 0, 0],
"version": 1
}
}
], ["block_producer_kickout_threshold", 80]], {
0: {
"tracked_shards": [0]
},
1: {
"tracked_shards": [0]
},
2: {
"tracked_shards": [0],
"consensus": {
"block_fetch_horizon": 20,
},
}
})
from cluster import init_cluster, spin_up_node, load_config
from transaction import sign_staking_tx
from utils import TxContext
TIMEOUT = 600
TWENTY_FIVE = 25
config = load_config()
# give more stake to the bootnode so that it can produce the blocks alone
near_root, node_dirs = init_cluster(
2, 1, 2, config,
[["min_gas_price", 0], ["max_inflation_rate", 0], ["epoch_length", 7],
["block_producer_kickout_threshold", 80],
["validators", 0, "amount", "60000000000000000000000000000000"],
[
"records", 0, "Account", "account", "locked",
"60000000000000000000000000000000"
]], {2: {
"tracked_shards": [0, 1]
}})
started = time.time()
boot_node = spin_up_node(config, near_root, node_dirs[0], 0, None, None)
#node1 = spin_up_node(config, near_root, node_dirs[1], 1, boot_node.node_key.pk, boot_node.addr())
observer = spin_up_node(config, near_root, node_dirs[2], 2,
boot_node.node_key.pk, boot_node.addr())
# It takes a while for test2 account to appear
ctx = TxContext([0, 0, 0], [boot_node, None, observer])
import os
sys.path.append('lib')
from cluster import init_cluster, start_cluster, GCloudNode
import retry
args = (3, 1, 1, {
'local': False,
'near_root': '../target/debug/',
'remote': {
'instance_name': 'near-pytest',
}
}, [], [])
init_cluster(*args)
subprocess.run([os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../scripts/create_instance_pool.sh"),
"near-pytest",
"us-west2-a us-west2-b us-west2-c us-west2-a"])
node_dirs = subprocess.check_output("find ~/.near/* -maxdepth 0", shell=True).decode('utf-8').strip().split('\n')
g = GCloudNode('near-pytest-0', node_dirs[0])
assert g.machine_status() == 'RUNNING'
g.is_ready()
print(g.addr())
start_cluster(*args)
g.change_version('staging')
retry.retry(lambda: g.is_ready(), 1200)
