def create_default_state(env, diff_calc=None):
r_state = RootState(env=env, diff_calc=diff_calc)
s_state_list = dict()
for full_shard_id in env.quark_chain_config.get_full_shard_ids():
shard_state = ShardState(env=env,
full_shard_id=full_shard_id,
db=quarkchain.db.InMemoryDb())
mblock, coinbase_amount_map = shard_state.init_genesis_state(
r_state.get_tip_block())
block_hash = mblock.header.get_hash()
r_state.add_validated_minor_block_hash(block_hash,
coinbase_amount_map.balance_map)
s_state_list[full_shard_id] = shard_state
# add a root block so that later minor blocks will be broadcasted to neighbor shards
minor_header_list = []
for state in s_state_list.values():
minor_header_list.append(state.header_tip)
root_block = r_state.create_block_to_mine(minor_header_list)
assert r_state.add_block(root_block)
for state in s_state_list.values():
assert state.add_root_block(root_block)
return r_state, s_state_list
def handle_root_print_by_height(env, args):
root_state = RootState(env)
header = root_state.db.get_root_block_header_by_height(args.height)
if header is None:
print("header not found")
return 1
pprint(header.to_dict())
print("hash: {}".format(header.get_hash().hex()))
return 0
def handle_root_print_block(env, args):
root_state = RootState(env)
block = root_state.db.get_root_block_by_hash(bytes.fromhex(args.hash))
if block is None:
print("header not found")
return 1
pprint(block.to_dict())
print("hash: {}".format(block.header.get_hash().hex()))
return 0
def main():
os.chdir(
os.path.join(os.path.dirname(os.path.abspath(__file__)), "../cluster"))
env, args = parse_args()
rs = RootState(env)
rb = rs.get_tip_block()
print("Root block height: %d" % rb.header.height)
if args.all_shards:
print_all_shard_balances(env, rb, args)
elif args.recipient:
print_recipient_balance(env, rb, args)
elif args.full_shard_id:
print_shard_balance(env, rb, args.full_shard_id)
elif args.minor_block_height:
print_balance_by_block_height(env, rb, args.minor_block_height)
def create_default_state(env, diff_calc=None):
r_state = RootState(env=env, diff_calc=diff_calc)
s_state_list = dict()
for full_shard_id in env.quark_chain_config.get_full_shard_ids():
shard_state = ShardState(env=env,
full_shard_id=full_shard_id,
db=quarkchain.db.InMemoryDb())
shard_state.init_genesis_state(r_state.get_tip_block())
s_state_list[full_shard_id] = shard_state
for state in s_state_list.values():
block_hash = state.header_tip.get_hash()
for dst_state in s_state_list.values():
if state == dst_state:
continue
dst_state.add_cross_shard_tx_list_by_minor_block_hash(
block_hash, CrossShardTransactionList(tx_list=[]))
r_state.add_validated_minor_block_hash(block_hash)
return (r_state, s_state_list)
def create_default_state(env, diff_calc=None):
r_state = RootState(env=env, diff_calc=diff_calc)
s_state_list = []
for shard_id in range(env.quark_chain_config.SHARD_SIZE):
shard_state = ShardState(env=env,
shard_id=shard_id,
db=quarkchain.db.InMemoryDb())
shard_state.init_genesis_state(r_state.get_tip_block())
s_state_list.append(shard_state)
for state in s_state_list:
block_hash = state.header_tip.get_hash()
for dst_state in s_state_list:
if state == dst_state:
continue
dst_state.add_cross_shard_tx_list_by_minor_block_hash(
block_hash, CrossShardTransactionList(tx_list=[]))
r_state.add_validated_minor_block_hash(block_hash)
return (r_state, s_state_list)
def create_default_state(env, diff_calc=None):
r_state = RootState(env=env, diff_calc=diff_calc)
s_state_list = dict()
for full_shard_id in env.quark_chain_config.get_full_shard_ids():
shard_state = ShardState(env=env,
full_shard_id=full_shard_id,
db=quarkchain.db.InMemoryDb())
shard_state.init_genesis_state(r_state.get_tip_block())
s_state_list[full_shard_id] = shard_state
# add a root block so that later minor blocks will be broadcasted to neighbor shards
root_block = r_state.tip.create_block_to_append()
for state in s_state_list.values():
root_block.add_minor_block_header(state.header_tip)
block_hash = state.header_tip.get_hash()
r_state.add_validated_minor_block_hash(block_hash)
root_block.finalize()
assert r_state.add_block(root_block)
for state in s_state_list.values():
assert state.add_root_block(root_block)
return (r_state, s_state_list)
def create_test_clusters(
num_cluster,
genesis_account,
chain_size,
shard_size,
num_slaves,
genesis_root_heights,
genesis_minor_quarkash,
remote_mining=False,
small_coinbase=False,
loadtest_accounts=None,
connect=True, # connect the bootstrap node by default
should_set_gas_price_limit=False,
mblock_coinbase_amount=None,
):
# so we can have lower minimum diff
easy_diff_calc = EthDifficultyCalculator(
cutoff=45, diff_factor=2048, minimum_diff=10
)
bootstrap_port = get_next_port() # first cluster will listen on this port
cluster_list = []
loop = asyncio.get_event_loop()
for i in range(num_cluster):
env = get_test_env(
genesis_account,
genesis_minor_quarkash=genesis_minor_quarkash,
chain_size=chain_size,
shard_size=shard_size,
genesis_root_heights=genesis_root_heights,
remote_mining=remote_mining,
)
env.cluster_config.P2P_PORT = bootstrap_port if i == 0 else get_next_port()
env.cluster_config.JSON_RPC_PORT = get_next_port()
env.cluster_config.PRIVATE_JSON_RPC_PORT = get_next_port()
env.cluster_config.SIMPLE_NETWORK = SimpleNetworkConfig()
env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT = bootstrap_port
env.quark_chain_config.loadtest_accounts = loadtest_accounts or []
if should_set_gas_price_limit:
env.quark_chain_config.MIN_TX_POOL_GAS_PRICE = 10
env.quark_chain_config.MIN_MINING_GAS_PRICE = 10
if small_coinbase:
# prevent breaking previous tests after tweaking default rewards
env.quark_chain_config.ROOT.COINBASE_AMOUNT = 5
for c in env.quark_chain_config.shards.values():
c.COINBASE_AMOUNT = 5
if mblock_coinbase_amount is not None:
for c in env.quark_chain_config.shards.values():
c.COINBASE_AMOUNT = mblock_coinbase_amount
env.cluster_config.SLAVE_LIST = []
check(is_p2(num_slaves))
for j in range(num_slaves):
slave_config = SlaveConfig()
slave_config.ID = "S{}".format(j)
slave_config.PORT = get_next_port()
slave_config.FULL_SHARD_ID_LIST = []
env.cluster_config.SLAVE_LIST.append(slave_config)
full_shard_ids = [
(i << 16) + shard_size + j
for i in range(chain_size)
for j in range(shard_size)
]
for i, full_shard_id in enumerate(full_shard_ids):
slave = env.cluster_config.SLAVE_LIST[i % num_slaves]
slave.FULL_SHARD_ID_LIST.append(full_shard_id)
slave_server_list = []
for j in range(num_slaves):
slave_env = env.copy()
slave_env.db = InMemoryDb()
slave_env.slave_config = env.cluster_config.get_slave_config(
"S{}".format(j)
)
slave_server = SlaveServer(slave_env, name="cluster{}_slave{}".format(i, j))
slave_server.start()
slave_server_list.append(slave_server)
root_state = RootState(env, diff_calc=easy_diff_calc)
master_server = MasterServer(env, root_state, name="cluster{}_master".format(i))
master_server.start()
# Wait until the cluster is ready
loop.run_until_complete(master_server.cluster_active_future)
# Substitute diff calculate with an easier one
for slave in slave_server_list:
for shard in slave.shards.values():
shard.state.diff_calc = easy_diff_calc
# Start simple network and connect to seed host
network = SimpleNetwork(env, master_server, loop)
network.start_server()
if connect and i != 0:
peer = call_async(network.connect("127.0.0.1", bootstrap_port))
else:
peer = None
cluster_list.append(Cluster(master_server, slave_server_list, network, peer))
return cluster_list
def test_root_state_simple(self):
env = get_test_env()
state = RootState(env=env)
self.assertEqual(state.tip.height, 0)
def test_root_state_recovery(self):
env = get_test_env()
r_state, s_states = create_default_state(env)
s_state0 = s_states[2 | 0]
s_state1 = s_states[2 | 1]
b0 = s_state0.create_block_to_mine()
add_minor_block_to_cluster(s_states, b0)
b1 = s_state1.create_block_to_mine()
add_minor_block_to_cluster(s_states, b1)
r_state.add_validated_minor_block_hash(
b0.header.get_hash(), b0.header.coinbase_amount_map.balance_map)
r_state.add_validated_minor_block_hash(
b1.header.get_hash(), b1.header.coinbase_amount_map.balance_map)
root_block0 = r_state.create_block_to_mine([b0.header, b1.header])
root_block00 = r_state.create_block_to_mine([b0.header, b1.header])
self.assertTrue(r_state.add_block(root_block0))
# create a fork
root_block00.header.create_time += 1
root_block00.finalize(
coinbase_tokens=r_state._calculate_root_block_coinbase(
[
header.get_hash()
for header in root_block00.minor_block_header_list
],
root_block00.header.height,
))
self.assertNotEqual(root_block0.header.get_hash(),
root_block00.header.get_hash())
self.assertFalse(r_state.add_block(root_block00))
self.assertEqual(
r_state.db.get_root_block_by_hash(root_block00.header.get_hash()),
root_block00,
)
b2 = s_state0.create_block_to_mine()
add_minor_block_to_cluster(s_states, b2)
b3 = s_state1.create_block_to_mine()
add_minor_block_to_cluster(s_states, b3)
r_state.add_validated_minor_block_hash(
b2.header.get_hash(), b2.header.coinbase_amount_map.balance_map)
r_state.add_validated_minor_block_hash(
b3.header.get_hash(), b3.header.coinbase_amount_map.balance_map)
root_block1 = r_state.create_block_to_mine([b2.header, b3.header])
self.assertTrue(r_state.add_block(root_block1))
# now the longest chain is root_block0 <-- root_block1
# but root_block0 will become the new tip after recovery
recovered_state = RootState(env=env)
self.assertEqual(recovered_state.tip, root_block0.header)
self.assertEqual(recovered_state.db.get_root_block_by_height(2),
root_block0)
tip_height = recovered_state.tip.height
self.assertEqual(
recovered_state.db.get_root_block_by_height(tip_height),
root_block0)
# fork is pruned from recovered state
self.assertIsNone(
recovered_state.db.get_root_block_by_hash(
root_block00.header.get_hash()))
self.assertEqual(
recovered_state.db.get_root_block_by_hash(
root_block00.header.get_hash(), consistency_check=False),
root_block00,
)
def create_test_clusters(
num_cluster,
genesis_account,
shard_size,
num_slaves,
genesis_root_heights,
remote_mining=False,
):
# so we can have lower minimum diff
easy_diff_calc = EthDifficultyCalculator(cutoff=45,
diff_factor=2048,
minimum_diff=10)
bootstrap_port = get_next_port() # first cluster will listen on this port
cluster_list = []
loop = asyncio.get_event_loop()
for i in range(num_cluster):
env = get_test_env(
genesis_account,
genesis_minor_quarkash=1000000,
shard_size=shard_size,
genesis_root_heights=genesis_root_heights,
remote_mining=remote_mining,
)
env.cluster_config.P2P_PORT = bootstrap_port if i == 0 else get_next_port(
)
env.cluster_config.JSON_RPC_PORT = get_next_port()
env.cluster_config.PRIVATE_JSON_RPC_PORT = get_next_port()
env.cluster_config.SIMPLE_NETWORK = SimpleNetworkConfig()
env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT = bootstrap_port
env.cluster_config.SLAVE_LIST = []
for j in range(num_slaves):
slave_config = SlaveConfig()
slave_config.ID = "S{}".format(j)
slave_config.PORT = get_next_port()
slave_config.SHARD_MASK_LIST = [ShardMask(num_slaves | j)]
slave_config.DB_PATH_ROOT = None # TODO: fix the db in config
env.cluster_config.SLAVE_LIST.append(slave_config)
slave_server_list = []
for j in range(num_slaves):
slave_env = env.copy()
slave_env.db = InMemoryDb()
slave_env.slave_config = env.cluster_config.get_slave_config(
"S{}".format(j))
slave_server = SlaveServer(slave_env,
name="cluster{}_slave{}".format(i, j))
slave_server.start()
slave_server_list.append(slave_server)
root_state = RootState(env, diff_calc=easy_diff_calc)
master_server = MasterServer(env,
root_state,
name="cluster{}_master".format(i))
master_server.start()
# Wait until the cluster is ready
loop.run_until_complete(master_server.cluster_active_future)
# Substitute diff calculate with an easier one
for slave in slave_server_list:
for shard in slave.shards.values():
shard.state.diff_calc = easy_diff_calc
# Start simple network and connect to seed host
network = SimpleNetwork(env, master_server)
network.start_server()
if i != 0:
peer = call_async(network.connect("127.0.0.1", bootstrap_port))
else:
peer = None
cluster_list.append(
Cluster(master_server, slave_server_list, network, peer))
return cluster_list
def test_root_state_recovery(self):
env = get_test_env()
r_state, s_states = create_default_state(env)
s_state0 = s_states[2 | 0]
s_state1 = s_states[2 | 1]
b0 = s_state0.get_tip().create_block_to_append()
add_minor_block_to_cluster(s_states, b0)
b1 = s_state1.get_tip().create_block_to_append()
add_minor_block_to_cluster(s_states, b1)
r_state.add_validated_minor_block_hash(b0.header.get_hash())
r_state.add_validated_minor_block_hash(b1.header.get_hash())
root_block0 = (
r_state.tip.create_block_to_append().add_minor_block_header(
s_state0.db.get_minor_block_by_height(0).header
).add_minor_block_header(b0.header).add_minor_block_header(
s_state1.db.get_minor_block_by_height(
0).header).add_minor_block_header(b1.header).finalize())
root_block00 = (
r_state.tip.create_block_to_append().add_minor_block_header(
s_state0.db.get_minor_block_by_height(0).header
).add_minor_block_header(b0.header).add_minor_block_header(
s_state1.db.get_minor_block_by_height(
0).header).add_minor_block_header(b1.header).finalize())
self.assertTrue(r_state.add_block(root_block0))
# create a fork
root_block00.header.create_time += 1
root_block00.finalize()
self.assertNotEqual(root_block0.header.get_hash(),
root_block00.header.get_hash())
self.assertFalse(r_state.add_block(root_block00))
self.assertEqual(
r_state.db.get_root_block_by_hash(root_block00.header.get_hash()),
root_block00,
)
b2 = s_state0.get_tip().create_block_to_append()
add_minor_block_to_cluster(s_states, b2)
b3 = s_state1.get_tip().create_block_to_append()
add_minor_block_to_cluster(s_states, b3)
r_state.add_validated_minor_block_hash(b2.header.get_hash())
r_state.add_validated_minor_block_hash(b3.header.get_hash())
root_block1 = (
r_state.tip.create_block_to_append().add_minor_block_header(
b2.header).add_minor_block_header(b3.header).finalize())
self.assertTrue(r_state.add_block(root_block1))
# now the longest chain is root_block0 <-- root_block1
# but root_block0 will become the new tip after recovery
recovered_state = RootState(env=env)
self.assertEqual(recovered_state.tip, root_block0.header)
self.assertEqual(recovered_state.db.get_root_block_by_height(1),
root_block0)
self.assertEqual(recovered_state.get_root_block_by_height(None),
root_block0)
# fork is pruned from recovered state
self.assertIsNone(
recovered_state.db.get_root_block_by_hash(
root_block00.header.get_hash()))
self.assertEqual(
recovered_state.db.get_root_block_by_hash(
root_block00.header.get_hash(), consistency_check=False),
root_block00,
)
def handle_root_print_tip(env, args):
root_state = RootState(env)
pprint(root_state.tip.to_dict())
print("hash: {}".format(root_state.tip.get_hash().hex()))
return 0
def get_root_state():
env, args = parse_args()
return RootState(env)
def create_test_clusters(num_cluster, genesis_account, shard_size, num_slaves,
genesis_root_heights):
bootstrap_port = get_next_port() # first cluster will listen on this port
cluster_list = []
loop = asyncio.get_event_loop()
for i in range(num_cluster):
env = get_test_env(
genesis_account,
genesis_minor_quarkash=1000000,
shard_size=shard_size,
genesis_root_heights=genesis_root_heights,
)
env.cluster_config.P2P_PORT = bootstrap_port if i == 0 else get_next_port(
)
env.cluster_config.JSON_RPC_PORT = get_next_port()
env.cluster_config.PRIVATE_JSON_RPC_PORT = get_next_port()
env.cluster_config.SIMPLE_NETWORK = SimpleNetworkConfig()
env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT = bootstrap_port
env.cluster_config.SLAVE_LIST = []
for j in range(num_slaves):
slave_config = SlaveConfig()
slave_config.ID = "S{}".format(j)
slave_config.PORT = get_next_port()
slave_config.SHARD_MASK_LIST = [ShardMask(num_slaves | j)]
slave_config.DB_PATH_ROOT = None # TODO: fix the db in config
env.cluster_config.SLAVE_LIST.append(slave_config)
slave_server_list = []
for j in range(num_slaves):
slave_env = env.copy()
slave_env.db = InMemoryDb()
slave_env.slave_config = env.cluster_config.get_slave_config(
"S{}".format(j))
slave_server = SlaveServer(slave_env,
name="cluster{}_slave{}".format(i, j))
slave_server.start()
slave_server_list.append(slave_server)
root_state = RootState(env)
master_server = MasterServer(env,
root_state,
name="cluster{}_master".format(i))
master_server.start()
# Wait until the cluster is ready
loop.run_until_complete(master_server.cluster_active_future)
# Start simple network and connect to seed host
network = SimpleNetwork(env, master_server)
network.start_server()
if i != 0:
peer = call_async(network.connect("127.0.0.1", bootstrap_port))
else:
peer = None
cluster_list.append(
Cluster(master_server, slave_server_list, network, peer))
return cluster_list