def monkey_staking(stopped, error, nodes, nonces):
while stopped.value == 0:
validator_ids = get_validator_ids(nodes)
whom = random.randint(0, len(nonces) - 2)
status = nodes[-1].get_status()
hash_, _ = get_recent_hash(nodes[-1])
who_can_unstake = get_the_guy_to_mess_up_with(nodes)
nonce_val, nonce_lock = nonces[whom]
with nonce_lock:
stake = random.randint(0.7 * MAX_STAKE // 1000000, MAX_STAKE // 1000000) * 1000000
if whom == who_can_unstake:
stake = 0
tx = sign_staking_tx(nodes[whom].signer_key, nodes[whom].validator_key, stake, nonce_val.value, base58.b58decode(hash_.encode('utf8')))
for validator_id in validator_ids:
try:
nodes[validator_id].send_tx(tx)
except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError):
if not network_issues_expected and not nodes[validator_id].mess_with:
raise
nonce_val.value = nonce_val.value + 1
time.sleep(1)
def do_moar_stakes(last_block_hash, update_expected):
global next_nonce, all_stakes, fake_stakes, sequence
if len(sequence) == 0:
stakes = [0, 0, 0]
# have 1-2 validators with stake, and the remaining without
# make numbers dibisable by 1M so that we can easily distinguish a situation when the current locked amt has some reward added to it (not divisable by 1M) vs not (divisable by 1M)
stakes[random.randint(0, 2)] = random.randint(
70000000000000000000000000, 100000000000000000000000000) * 1000000
stakes[random.randint(0, 2)] = random.randint(
70000000000000000000000000, 100000000000000000000000000) * 1000000
else:
stakes = sequence[0]
sequence = sequence[1:]
vals = get_validators()
val_id = int(list(vals)[0][4:])
for i in range(3):
tx = sign_staking_tx(nodes[i].signer_key, nodes[i].validator_key,
stakes[i], next_nonce,
base58.b58decode(last_block_hash.encode('utf8')))
nodes[val_id].send_tx(tx)
next_nonce += 1
if update_expected:
fake_stakes = [0, 0, 0]
all_stakes.append(stakes)
print("")
else:
fake_stakes = stakes
print("Sent %s staking txs: %s" %
("REAL" if update_expected else "fake", stakes))
def monkey_staking(stopped, error, nodes, nonces):
while stopped.value == 0:
validator_ids = get_validator_ids(nodes)
whom = random.randint(0, len(nonces) - 1)
status = nodes[-1].get_status()
hash_, _ = get_recent_hash(nodes[-1])
who_can_unstake = get_the_guy_to_mess_up_with(nodes)
nonce_val, nonce_lock = nonces[whom]
with nonce_lock:
stake = random.randint(0.7 * MAX_STAKE // 1000000,
MAX_STAKE // 1000000) * 1000000
if whom == who_can_unstake:
stake = 0
tx = sign_staking_tx(nodes[whom].signer_key,
nodes[whom].validator_key, stake,
nonce_val.value,
base58.b58decode(hash_.encode('utf8')))
for validator_id in validator_ids:
nodes[validator_id].send_tx(tx)
nonce_val.value = nonce_val.value + 1
time.sleep(1)
def send_staking_tx(self, stake):
status = self.get_status()
hash_ = status['sync_info']['latest_block_hash']
if self.account_key_nonce is None:
self.account_key_nonce = self.get_nonce_for_pk(self.signer_key.account_id, self.signer_key.pk)
self.account_key_nonce += 1
tx = sign_staking_tx(nodes[index].signer_key, nodes[index].validator_key, stake, self.account_key_nonce, base58.b58decode(hash_.encode('utf8')))
print(f'{self.signer_key.account_id} stakes {stake}')
res = self.send_tx_and_wait(tx, timeout=15)
if 'error' in res or 'Failure' in res['result']['status']:
print(res)
def send_staking_tx(self, stake):
hash_ = self.get_latest_block().hash_bytes
if self.account_key_nonce is None:
self.account_key_nonce = self.get_nonce_for_pk(
self.signer_key.account_id, self.signer_key.pk)
self.account_key_nonce += 1
tx = sign_staking_tx(nodes[index].signer_key,
nodes[index].validator_key, stake,
self.account_key_nonce, hash_)
logger.info(f'{self.signer_key.account_id} stakes {stake}')
res = self.send_tx_and_wait(tx, timeout=15)
if 'error' in res or 'Failure' in res['result']['status']:
logger.info(res)
# we expect no skipped heights
height_to_num_approvals[height] = len(
block['result']['header']['approvals'])
if height > largest_height:
logger.info("... %s" % height)
logger.info(block['result']['header']['approvals'])
largest_height = height
if height > HEIGHT_GOAL:
break
if height > epoch_switch_height + 2:
for val_ord in next_vals:
tx = sign_staking_tx(nodes[val_ord].signer_key,
nodes[val_ord].validator_key, 0, next_nonce,
base58.b58decode(prev_hash.encode('utf8')))
for target in range(0, 4):
nodes[target].send_tx(tx)
next_nonce += 1
for val_ord in cur_vals:
tx = sign_staking_tx(nodes[val_ord].signer_key,
nodes[val_ord].validator_key,
50000000000000000000000000000000, next_nonce,
base58.b58decode(prev_hash.encode('utf8')))
for target in range(0, 4):
nodes[target].send_tx(tx)
next_nonce += 1
if epoch_id not in seen_epochs:
["chunk_producer_kickout_threshold", 10]], {1: client_config, 2: client_config})
time.sleep(2)
nodes[2].kill()
validator_key = Key(nodes[1].validator_key.account_id, nodes[2].signer_key.pk, nodes[2].signer_key.sk)
nodes[2].reset_validator_key(validator_key)
nodes[2].reset_data()
nodes[2].start(nodes[0].node_key.pk, nodes[0].addr())
time.sleep(3)
status = nodes[0].get_status()
block_hash = status['sync_info']['latest_block_hash']
block_height = status['sync_info']['latest_block_height']
tx = sign_staking_tx(nodes[1].signer_key, validator_key, 50000000000000000000000000000000, 1,
base58.b58decode(block_hash.encode('utf8')))
res = nodes[0].send_tx_and_wait(tx, timeout=15)
assert 'error' not in res
start_time = time.time()
while True:
if time.time() - start_time > TIMEOUT:
assert False, "Validators get stuck"
status1 = nodes[1].get_status()
node1_height = status1['sync_info']['latest_block_height']
status2 = nodes[2].get_status()
node2_height = status2['sync_info']['latest_block_height']
if node1_height > block_height + 4 * EPOCH_LENGTH and node2_height > block_height + 4 * EPOCH_LENGTH:
break
time.sleep(2)
def send_stake_tx(self, stake_amount, base_block_hash=None):
self.prep_tx()
tx = sign_staking_tx(self.key, self.key, stake_amount, self.nonce,
base_block_hash or self.base_block_hash)
return self.send_tx(tx)
logging.info("Waiting for the new nodes to sync")
while True:
if (not node2.get_status()['sync_info']['syncing']
and not node3.get_status()['sync_info']['syncing']):
break
time.sleep(1)
for stake, nodes, expected_vals in [
(100000000000000000000000000000000, [node2, node3],
["test0", "test1", "test2", "test3"]),
(0, [boot_node, node1], ["test2", "test3"]),
]:
logging.info("Rotating validators")
for ord_, node in enumerate(reversed(nodes)):
tx = sign_staking_tx(node.signer_key, node.validator_key, stake,
10, hash_)
boot_node.send_tx(tx)
logging.info("Waiting for rotation to occur")
while True:
assert timeout.check(), get_validators(boot_node)
if set(get_validators(boot_node)) == set(expected_vals):
break
else:
time.sleep(1)
start_height = boot_node.get_latest_block().height
logging.info("Killing old nodes")
boot_node.kill()
node1.kill()
nodes = start_cluster(
3, 1, 4, None,
[["epoch_length", EPOCH_LENGTH], ["block_producer_kickout_threshold", 10],
["chunk_producer_kickout_threshold", 10]], {
0: tracked_shards,
1: tracked_shards
})
time.sleep(3)
hash_ = nodes[0].get_latest_block().hash_bytes
for i in range(4):
stake = 50000000000000000000000000000000 if i == 3 else 0
tx = sign_staking_tx(nodes[i].signer_key, nodes[i].validator_key, stake, 1,
hash_)
nodes[0].send_tx(tx)
logger.info("test%s stakes %d" % (i, stake))
for cur_height, _ in utils.poll_blocks(nodes[0], poll_interval=1):
if cur_height >= EPOCH_LENGTH * 2:
break
if cur_height > EPOCH_LENGTH + 1:
info = nodes[0].json_rpc('validators', 'latest')
count = len(info['result']['next_validators'])
assert count == 1, 'Number of validators do not match'
validator = info['result']['next_validators'][0]['account_id']
assert validator == 'test3'
while cur_height <= EPOCH_LENGTH * 3:
statuses = sorted((enumerate(node.get_latest_block() for node in nodes)),
nodes = start_cluster(
3, 1, 4, None,
[["epoch_length", EPOCH_LENGTH], ["block_producer_kickout_threshold", 10],
["chunk_producer_kickout_threshold", 10]], {
0: tracked_shards,
1: tracked_shards
})
time.sleep(3)
status = nodes[0].get_status()
hash_ = status['sync_info']['latest_block_hash']
for i in range(4):
stake = 50000000000000000000000000000000 if i == 3 else 0
tx = sign_staking_tx(nodes[i].signer_key, nodes[i].validator_key, stake, 1,
base58.b58decode(hash_.encode('utf8')))
nodes[0].send_tx(tx)
logger.info("test%s stakes %d" % (i, stake))
cur_height = 0
while cur_height < EPOCH_LENGTH * 2:
status = nodes[0].get_status()
cur_height = status['sync_info']['latest_block_height']
if cur_height > EPOCH_LENGTH + 1:
validator_info = nodes[0].json_rpc('validators', 'latest')
assert len(
validator_info['result']
['next_validators']) == 1, "Number of validators do not match"
assert validator_info['result']['next_validators'][0][
'account_id'] == "test3"
time.sleep(1)
def get_validators():
return set([x['account_id'] for x in nodes[0].get_status()['validators']])
def get_stakes():
return [
int(nodes[2].get_account("test%s" % i)['result']['locked'])
for i in range(3)
]
status = nodes[2].get_status()
hash_ = status['sync_info']['latest_block_hash']
tx = sign_staking_tx(nodes[2].signer_key, nodes[2].validator_key,
100000000000000000000000000000000, 2,
base58.b58decode(hash_.encode('utf8')))
nodes[0].send_tx(tx)
max_height = 0
print("Initial stakes: %s" % get_stakes())
while True:
assert time.time() - started < TIMEOUT
status = nodes[0].get_status()
height = status['sync_info']['latest_block_height']
if 'test2' in get_validators():
print("Normalin, normalin")
def stake(source_account, base_block_hash):
nonce = get_nonce_for_pk(source_account.account_id, source_account.pk)
tx = sign_staking_tx(source_account, source_account, 1, nonce + 1,
base_block_hash)
send_tx(tx)
account_keys.append(signer_key)
res = nodes[0].send_tx_and_wait(create_account_tx, timeout=15)
assert 'error' not in res, res
target_height = 50
while cur_height < target_height:
status = nodes[0].get_status()
cur_height = status['sync_info']['latest_block_height']
time.sleep(1)
status = nodes[0].get_status()
block_hash = status['sync_info']['latest_block_hash']
for signer_key in account_keys:
staking_tx = sign_staking_tx(signer_key, nodes[0].validator_key,
balance // (num_new_accounts * 2), 1,
base58.b58decode(block_hash.encode('utf8')))
res = nodes[0].send_tx_and_wait(staking_tx, timeout=15)
assert 'error' not in res
target_height = 80
while cur_height < target_height:
status = nodes[0].get_status()
cur_height = status['sync_info']['latest_block_height']
time.sleep(1)
print('restart node1')
nodes[1].start(nodes[1].node_key.pk, nodes[1].addr())
print('node1 restarted')
time.sleep(3)
def get_validators():
return set([x['account_id'] for x in nodes[0].get_status()['validators']])
def get_stakes():
return [
int(nodes[2].get_account("test%s" % i)['result']['locked'])
for i in range(3)
]
hash_ = nodes[2].get_latest_block().hash_bytes
tx = sign_staking_tx(nodes[2].signer_key, nodes[2].validator_key,
100000000000000000000000000000000, 2, hash_)
nodes[0].send_tx(tx)
logger.info("Initial stakes: %s" % get_stakes())
for height, _ in utils.poll_blocks(nodes[0], timeout=TIMEOUT):
if 'test2' in get_validators():
logger.info("Normalin, normalin")
assert 20 <= height <= 25, height
break
tx = sign_staking_tx(nodes[2].signer_key, nodes[2].validator_key, 0, 3, hash_)
nodes[2].send_tx(tx)
for height, _ in utils.poll_blocks(nodes[0], timeout=TIMEOUT):
if 'test2' not in get_validators():
logger.info("DONE")
assert (sum(balances) == total_supply)
initial_balances = balances
# 4. Stake for the second node to bring it back up as a validator and wait until it actually
# becomes one
def get_validators():
return set([x['account_id'] for x in boot_node.get_status()['validators']])
print(get_validators())
tx = sign_staking_tx(node2.signer_key, node2.validator_key,
50000000000000000000000000, 20,
base58.b58decode(hash_.encode('utf8')))
boot_node.send_tx(tx)
assert (get_validators() == set(["test0"]))
while True:
if time.time() - started > TIMEOUT:
print(get_validators())
assert False
if get_validators() == set(["test0", "test1"]):
break
time.sleep(1)
})
time.sleep(2)
nodes[2].kill()
validator_key = Key(nodes[1].validator_key.account_id, nodes[2].signer_key.pk,
nodes[2].signer_key.sk)
nodes[2].reset_validator_key(validator_key)
nodes[2].reset_data()
nodes[2].start(boot_node=nodes[0])
time.sleep(3)
block = nodes[0].get_latest_block()
block_height = block.height
block_hash = block.hash_bytes
tx = sign_staking_tx(nodes[1].signer_key, validator_key,
50000000000000000000000000000000, 1, block_hash)
res = nodes[0].send_tx_and_wait(tx, timeout=15)
assert 'error' not in res
start_time = time.time()
while True:
assert time.time() - start_time < TIMEOUT, 'Validators got stuck'
node1_height = nodes[1].get_latest_block().height
node2_height = nodes[2].get_latest_block().height
if (node1_height > block_height + 4 * EPOCH_LENGTH
and node2_height > block_height + 4 * EPOCH_LENGTH):
break
time.sleep(2)
def stake(source_account):
last_block_hash = get_latest_block_hash()
nonce = get_nonce_for_pk(source_account.account_id, source_account.pk)
tx = sign_staking_tx(source_account, source_account, 1, nonce + 1,
last_block_hash)
send_tx(tx)