def __create_from_args_internal():
check(
is_p2(args.num_shards_per_chain),
"--num_shards_per_chain must be power of 2",
)
check(is_p2(args.num_slaves), "--num_slaves must be power of 2")
config = ClusterConfig()
config.LOG_LEVEL = args.log_level
config.DB_PATH_ROOT = args.db_path_root
config.P2P_PORT = args.p2p_port
config.JSON_RPC_PORT = args.json_rpc_port
config.PRIVATE_JSON_RPC_PORT = args.json_rpc_private_port
config.CLEAN = args.clean
config.START_SIMULATED_MINING = args.start_simulated_mining
config.ENABLE_TRANSACTION_HISTORY = args.enable_transaction_history
config.QUARKCHAIN.update(
args.num_chains,
args.num_shards_per_chain,
args.root_block_interval_sec,
args.minor_block_interval_sec,
)
config.QUARKCHAIN.NETWORK_ID = args.network_id
config.GENESIS_DIR = args.genesis_dir
config.MONITORING.KAFKA_REST_ADDRESS = args.monitoring_kafka_rest_address
if args.p2p:
config.SIMPLE_NETWORK = None
config.P2P = P2PConfig()
# p2p module
config.P2P.BOOT_NODES = args.bootnodes
config.P2P.PRIV_KEY = args.privkey
config.P2P.MAX_PEERS = args.max_peers
config.P2P.UPNP = args.upnp
else:
config.P2P = None
config.SIMPLE_NETWORK = SimpleNetworkConfig()
config.SIMPLE_NETWORK.BOOTSTRAP_HOST = (
args.simple_network_bootstrap_host)
config.SIMPLE_NETWORK.BOOTSTRAP_PORT = (
args.simple_network_bootstrap_port)
config.SLAVE_LIST = []
for i in range(args.num_slaves):
slave_config = SlaveConfig()
slave_config.PORT = args.port_start + i
slave_config.ID = "S{}".format(i)
slave_config.SHARD_MASK_LIST = [ShardMask(i | args.num_slaves)]
config.SLAVE_LIST.append(slave_config)
fd, config.json_filepath = tempfile.mkstemp()
with os.fdopen(fd, "w") as tmp:
tmp.write(config.to_json())
return config
def __init__(
self,
num_cluster,
genesis_account=Address.create_empty_account(),
chain_size=2,
shard_size=2,
num_slaves=None,
genesis_root_heights=None,
remote_mining=False,
small_coinbase=False,
loadtest_accounts=None,
connect=True,
should_set_gas_price_limit=False,
mblock_coinbase_amount=None,
genesis_minor_quarkash=1000000,
):
self.num_cluster = num_cluster
self.genesis_account = genesis_account
self.chain_size = chain_size
self.shard_size = shard_size
self.num_slaves = num_slaves if num_slaves else chain_size
self.genesis_root_heights = genesis_root_heights
self.remote_mining = remote_mining
self.small_coinbase = small_coinbase
self.loadtest_accounts = loadtest_accounts
self.connect = connect
self.should_set_gas_price_limit = should_set_gas_price_limit
self.mblock_coinbase_amount = mblock_coinbase_amount
self.genesis_minor_quarkash = genesis_minor_quarkash
check(is_p2(self.num_slaves))
check(is_p2(self.shard_size))
def is_neighbor(b1: Branch, b2: Branch):
"""A naive algorithm to decide neighbor relationship
TODO: a better algorithm, because the current one ensures 32 neighbors ONLY when there are 2^32 shards
"""
if b1.get_chain_id() == b2.get_chain_id():
return is_p2(abs(b1.get_shard_id() - b2.get_shard_id()))
if b1.get_shard_id() == b2.get_shard_id():
return is_p2(abs(b1.get_chain_id() - b2.get_chain_id()))
return False
def init_and_validate(self):
self._chain_id_to_shard_size = dict()
chain_id_to_shard_ids = dict()
for full_shard_id, shard_config in self.shards.items():
chain_id = shard_config.CHAIN_ID
shard_size = shard_config.SHARD_SIZE
shard_id = shard_config.SHARD_ID
check(full_shard_id == (chain_id << 16 | shard_size | shard_id))
check(is_p2(shard_size))
check(shard_config.ETH_CHAIN_ID == self.BASE_ETH_CHAIN_ID +
chain_id + 1)
if chain_id in self._chain_id_to_shard_size:
check(shard_size == self._chain_id_to_shard_size[chain_id])
else:
self._chain_id_to_shard_size[chain_id] = shard_size
chain_id_to_shard_ids.setdefault(chain_id, set()).add(shard_id)
# check the number of ShardConfigs matches SHARD_SIZE for each chain
# and the SHARD_ID starts from 0 to (SHARD_SIZE - 1)
for chain_id, shard_ids in chain_id_to_shard_ids.items():
shard_size = self.get_shard_size_by_chain_id(chain_id)
check(shard_ids == set(range(shard_size)))
# check the chain id starts from 0 to (CHAIN_SIZE - 1)
check(set(chain_id_to_shard_ids.keys()) == set(range(self.CHAIN_SIZE)))
def get_test_env(
genesis_account=Address.create_empty_account(),
genesis_minor_quarkash=0,
chain_size=2,
shard_size=2,
genesis_root_heights=None, # dict(full_shard_id, genesis_root_height)
remote_mining=False,
genesis_minor_token_balances={},
):
check(is_p2(shard_size))
env = DEFAULT_ENV.copy()
env.db = InMemoryDb()
env.set_network_id(1234567890)
env.cluster_config = ClusterConfig()
env.quark_chain_config.update(chain_size, shard_size, 10, 1,
env.quark_chain_config.GENESIS_TOKEN)
if remote_mining:
env.quark_chain_config.ROOT.CONSENSUS_CONFIG.REMOTE_MINE = True
env.quark_chain_config.ROOT.CONSENSUS_TYPE = ConsensusType.POW_DOUBLESHA256
env.quark_chain_config.ROOT.GENESIS.DIFFICULTY = 10
env.quark_chain_config.ROOT.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME = 40
env.quark_chain_config.ROOT.DIFFICULTY_ADJUSTMENT_FACTOR = 1024
if genesis_root_heights:
check(len(genesis_root_heights) == shard_size * chain_size)
for chain_id in range(chain_size):
for shard_id in range(shard_size):
full_shard_id = chain_id << 16 | shard_size | shard_id
shard = env.quark_chain_config.shards[full_shard_id]
shard.GENESIS.ROOT_HEIGHT = genesis_root_heights[full_shard_id]
# fund genesis account in all shards
for full_shard_id, shard in env.quark_chain_config.shards.items():
addr = genesis_account.address_in_shard(
full_shard_id).serialize().hex()
if len(genesis_minor_token_balances) != 0:
shard.GENESIS.ALLOC[addr] = genesis_minor_token_balances
else:
shard.GENESIS.ALLOC[addr] = genesis_minor_quarkash
shard.CONSENSUS_CONFIG.REMOTE_MINE = remote_mining
shard.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME = 7
shard.DIFFICULTY_ADJUSTMENT_FACTOR = 512
if remote_mining:
shard.CONSENSUS_TYPE = ConsensusType.POW_DOUBLESHA256
shard.GENESIS.DIFFICULTY = 10
shard.POSW_CONFIG.WINDOW_SIZE = 2
env.quark_chain_config.SKIP_MINOR_DIFFICULTY_CHECK = True
env.quark_chain_config.SKIP_ROOT_DIFFICULTY_CHECK = True
env.cluster_config.ENABLE_TRANSACTION_HISTORY = True
env.cluster_config.DB_PATH_ROOT = ""
check(env.cluster_config.use_mem_db())
return env
def __init__(
self,
num_cluster,
genesis_account=Address.create_empty_account(),
chain_size=2,
shard_size=2,
num_slaves=None,
genesis_root_heights=None,
remote_mining=False,
small_coinbase=False,
):
self.num_cluster = num_cluster
self.genesis_account = genesis_account
self.chain_size = chain_size
self.shard_size = shard_size
self.num_slaves = num_slaves if num_slaves else chain_size
self.genesis_root_heights = genesis_root_heights
self.remote_mining = remote_mining
self.small_coinbase = small_coinbase
check(is_p2(self.num_slaves))
check(is_p2(self.shard_size))
def is_neighbor(b1: Branch, b2: Branch):
"""A naive algorithm to decide neighbor relationship
Two shards are neighbor iff there is only 1 bit difference in their shard ids.
This only applies if there are more than 32 shards in the network.
Otherwise all shards are neighbor to each other.
TODO: a better algorithm
"""
check(b1.get_shard_size() == b2.get_shard_size())
check(b1.get_shard_id() != b2.get_shard_id())
if b1.get_shard_size() <= 32:
return True
return is_p2(abs(b1.get_shard_id() - b2.get_shard_id()))
def create(shard_size: int, shard_id: int):
assert is_p2(shard_size)
return Branch(shard_size | shard_id)
def create_test_clusters(
num_cluster,
genesis_account,
chain_size,
shard_size,
num_slaves,
genesis_root_heights,
genesis_minor_quarkash,
remote_mining=False,
small_coinbase=False,
loadtest_accounts=None,
connect=True, # connect the bootstrap node by default
should_set_gas_price_limit=False,
mblock_coinbase_amount=None,
):
# so we can have lower minimum diff
easy_diff_calc = EthDifficultyCalculator(
cutoff=45, diff_factor=2048, minimum_diff=10
)
bootstrap_port = get_next_port() # first cluster will listen on this port
cluster_list = []
loop = asyncio.get_event_loop()
for i in range(num_cluster):
env = get_test_env(
genesis_account,
genesis_minor_quarkash=genesis_minor_quarkash,
chain_size=chain_size,
shard_size=shard_size,
genesis_root_heights=genesis_root_heights,
remote_mining=remote_mining,
)
env.cluster_config.P2P_PORT = bootstrap_port if i == 0 else get_next_port()
env.cluster_config.JSON_RPC_PORT = get_next_port()
env.cluster_config.PRIVATE_JSON_RPC_PORT = get_next_port()
env.cluster_config.SIMPLE_NETWORK = SimpleNetworkConfig()
env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT = bootstrap_port
env.quark_chain_config.loadtest_accounts = loadtest_accounts or []
if should_set_gas_price_limit:
env.quark_chain_config.MIN_TX_POOL_GAS_PRICE = 10
env.quark_chain_config.MIN_MINING_GAS_PRICE = 10
if small_coinbase:
# prevent breaking previous tests after tweaking default rewards
env.quark_chain_config.ROOT.COINBASE_AMOUNT = 5
for c in env.quark_chain_config.shards.values():
c.COINBASE_AMOUNT = 5
if mblock_coinbase_amount is not None:
for c in env.quark_chain_config.shards.values():
c.COINBASE_AMOUNT = mblock_coinbase_amount
env.cluster_config.SLAVE_LIST = []
check(is_p2(num_slaves))
for j in range(num_slaves):
slave_config = SlaveConfig()
slave_config.ID = "S{}".format(j)
slave_config.PORT = get_next_port()
slave_config.FULL_SHARD_ID_LIST = []
env.cluster_config.SLAVE_LIST.append(slave_config)
full_shard_ids = [
(i << 16) + shard_size + j
for i in range(chain_size)
for j in range(shard_size)
]
for i, full_shard_id in enumerate(full_shard_ids):
slave = env.cluster_config.SLAVE_LIST[i % num_slaves]
slave.FULL_SHARD_ID_LIST.append(full_shard_id)
slave_server_list = []
for j in range(num_slaves):
slave_env = env.copy()
slave_env.db = InMemoryDb()
slave_env.slave_config = env.cluster_config.get_slave_config(
"S{}".format(j)
)
slave_server = SlaveServer(slave_env, name="cluster{}_slave{}".format(i, j))
slave_server.start()
slave_server_list.append(slave_server)
root_state = RootState(env, diff_calc=easy_diff_calc)
master_server = MasterServer(env, root_state, name="cluster{}_master".format(i))
master_server.start()
# Wait until the cluster is ready
loop.run_until_complete(master_server.cluster_active_future)
# Substitute diff calculate with an easier one
for slave in slave_server_list:
for shard in slave.shards.values():
shard.state.diff_calc = easy_diff_calc
# Start simple network and connect to seed host
network = SimpleNetwork(env, master_server, loop)
network.start_server()
if connect and i != 0:
peer = call_async(network.connect("127.0.0.1", bootstrap_port))
else:
peer = None
cluster_list.append(Cluster(master_server, slave_server_list, network, peer))
return cluster_list
def __create_from_args_internal():
check(
is_p2(args.num_shards_per_chain),
"--num_shards_per_chain must be power of 2",
)
check(is_p2(args.num_slaves), "--num_slaves must be power of 2")
config = ClusterConfig()
config.LOG_LEVEL = args.log_level
config.DB_PATH_ROOT = args.db_path_root
config.P2P_PORT = args.p2p_port
config.JSON_RPC_PORT = args.json_rpc_port
config.PRIVATE_JSON_RPC_PORT = args.json_rpc_private_port
config.JSON_RPC_HOST = args.json_rpc_host
config.PRIVATE_JSON_RPC_HOST = args.json_rpc_private_host
config.CLEAN = args.clean
config.START_SIMULATED_MINING = args.start_simulated_mining
config.ENABLE_TRANSACTION_HISTORY = args.enable_transaction_history
config.QUARKCHAIN.update(
args.num_chains,
args.num_shards_per_chain,
args.root_block_interval_sec,
args.minor_block_interval_sec,
args.default_token,
)
config.QUARKCHAIN.NETWORK_ID = args.network_id
config.GENESIS_DIR = args.genesis_dir
config.MONITORING.KAFKA_REST_ADDRESS = args.monitoring_kafka_rest_address
if args.p2p:
config.SIMPLE_NETWORK = None
config.P2P = P2PConfig()
# p2p module
config.P2P.BOOT_NODES = args.bootnodes
config.P2P.PRIV_KEY = args.privkey
config.P2P.MAX_PEERS = args.max_peers
config.P2P.UPNP = args.upnp
else:
config.P2P = None
config.SIMPLE_NETWORK = SimpleNetworkConfig()
config.SIMPLE_NETWORK.BOOTSTRAP_HOST = (
args.simple_network_bootstrap_host)
config.SIMPLE_NETWORK.BOOTSTRAP_PORT = (
args.simple_network_bootstrap_port)
if args.prom:
config.PROMETHEUS = PrometheusConfig()
config.PROMETHEUS.INTERVAL = args.prom_interval
config.PROMETHEUS.TOKENS = args.prom_tokens
config.PROMETHEUS.PORT = args.prom_port
config.SLAVE_LIST = []
for i in range(args.num_slaves):
slave_config = SlaveConfig()
slave_config.PORT = args.port_start + i
slave_config.ID = "S{}".format(i)
slave_config.FULL_SHARD_ID_LIST = []
config.SLAVE_LIST.append(slave_config)
# assign full shard IDs to each slave, using hex strings to write into JSON
full_shard_ids = [(i << 16) + args.num_shards_per_chain + j
for i in range(args.num_chains)
for j in range(args.num_shards_per_chain)]
for i, full_shard_id in enumerate(full_shard_ids):
slave = config.SLAVE_LIST[i % args.num_slaves]
slave.FULL_SHARD_ID_LIST.append(full_shard_id)
fd, config.json_filepath = tempfile.mkstemp()
with os.fdopen(fd, "w") as tmp:
tmp.write(config.to_json())
return config
def get_full_shard_id(self, shard_size: int):
if not is_p2(shard_size):
raise RuntimeError("Invalid shard size {}".format(shard_size))
chain_id = self.full_shard_key >> 16
shard_id = self.full_shard_key & (shard_size - 1)
return (chain_id << 16) | shard_size | shard_id
def set_shard_size(self, shard_size):
check(is_p2(shard_size))
self.shard_size = shard_size
def create(shard_size, reshard_vote=False):
assert is_p2(shard_size)
reshard_vote = 1 if reshard_vote else 0
return ShardInfo(
int_left_most_bit(shard_size) - 1 + (reshard_vote << 31))
def create_test_clusters(
num_cluster,
genesis_account,
chain_size,
shard_size,
num_slaves,
genesis_root_heights,
remote_mining=False,
small_coinbase=False,
):
# so we can have lower minimum diff
easy_diff_calc = EthDifficultyCalculator(cutoff=45,
diff_factor=2048,
minimum_diff=10)
bootstrap_port = get_next_port() # first cluster will listen on this port
cluster_list = []
loop = asyncio.get_event_loop()
for i in range(num_cluster):
env = get_test_env(
genesis_account,
genesis_minor_quarkash=1000000,
chain_size=chain_size,
shard_size=shard_size,
genesis_root_heights=genesis_root_heights,
remote_mining=remote_mining,
)
env.cluster_config.P2P_PORT = bootstrap_port if i == 0 else get_next_port(
)
env.cluster_config.JSON_RPC_PORT = get_next_port()
env.cluster_config.PRIVATE_JSON_RPC_PORT = get_next_port()
env.cluster_config.SIMPLE_NETWORK = SimpleNetworkConfig()
env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT = bootstrap_port
if small_coinbase:
# prevent breaking previous tests after tweaking default rewards
env.quark_chain_config.ROOT.COINBASE_AMOUNT = 5
for c in env.quark_chain_config.shards.values():
c.COINBASE_AMOUNT = 5
env.cluster_config.SLAVE_LIST = []
check(is_p2(num_slaves))
for j in range(num_slaves):
slave_config = SlaveConfig()
slave_config.ID = "S{}".format(j)
slave_config.PORT = get_next_port()
slave_config.CHAIN_MASK_LIST = [ChainMask(num_slaves | j)]
env.cluster_config.SLAVE_LIST.append(slave_config)
slave_server_list = []
for j in range(num_slaves):
slave_env = env.copy()
slave_env.db = InMemoryDb()
slave_env.slave_config = env.cluster_config.get_slave_config(
"S{}".format(j))
slave_server = SlaveServer(slave_env,
name="cluster{}_slave{}".format(i, j))
slave_server.start()
slave_server_list.append(slave_server)
root_state = RootState(env, diff_calc=easy_diff_calc)
master_server = MasterServer(env,
root_state,
name="cluster{}_master".format(i))
master_server.start()
# Wait until the cluster is ready
loop.run_until_complete(master_server.cluster_active_future)
# Substitute diff calculate with an easier one
for slave in slave_server_list:
for shard in slave.shards.values():
shard.state.diff_calc = easy_diff_calc
# Start simple network and connect to seed host
network = SimpleNetwork(env, master_server, loop)
network.start_server()
if i != 0:
peer = call_async(network.connect("127.0.0.1", bootstrap_port))
else:
peer = None
cluster_list.append(
Cluster(master_server, slave_server_list, network, peer))
return cluster_list
def get_shard_id(self, shard_size):
if not is_p2(shard_size):
raise RuntimeError("Invalid shard size {}".format(shard_size))
return self.full_shard_id & (shard_size - 1)
def __create_from_args_internal():
check(is_p2(args.num_shards), "--num_shards must be power of 2")
check(is_p2(args.num_slaves), "--num_slaves must be power of 2")
config = ClusterConfig()
config.LOG_LEVEL = args.log_level
config.DB_PATH_ROOT = args.db_path_root
config.P2P_PORT = args.p2p_port
config.JSON_RPC_PORT = args.json_rpc_port
config.PRIVATE_JSON_RPC_PORT = args.json_rpc_private_port
config.CLEAN = args.clean
config.MINE = args.mine
config.SLAVE_IDS = args.slave_ids
config.IS_MASTER = args.is_master
config.ENABLE_TRANSACTION_HISTORY = args.enable_transaction_history
config.QUARKCHAIN.update(
args.num_shards,
args.root_block_interval_sec,
args.minor_block_interval_sec,
)
config.QUARKCHAIN.NETWORK_ID = args.network_id
config.GENESIS_DIR = args.genesis_dir
config.MONITORING.KAFKA_REST_ADDRESS = args.monitoring_kafka_rest_address
if args.devp2p_enable:
config.SIMPLE_NETWORK = None
config.P2P = P2PConfig()
config.P2P.IP = args.devp2p_ip
config.P2P.DISCOVERY_PORT = args.devp2p_port
config.P2P.BOOTSTRAP_HOST = args.devp2p_bootstrap_host
config.P2P.BOOTSTRAP_PORT = args.devp2p_bootstrap_port
config.P2P.MIN_PEERS = args.devp2p_min_peers
config.P2P.MAX_PEERS = args.devp2p_max_peers
config.P2P.ADDITIONAL_BOOTSTRAPS = args.devp2p_additional_bootstraps
else:
config.P2P = None
config.SIMPLE_NETWORK = SimpleNetworkConfig()
config.SIMPLE_NETWORK.BOOTSTRAP_HOST = (
args.simple_network_bootstrap_host)
config.SIMPLE_NETWORK.BOOTSTRAP_PORT = (
args.simple_network_bootstrap_port)
config.SLAVE_LIST = []
slave_ip_list = args.slave_ips.split(",")
slave_ip_len = len(slave_ip_list)
# if len(slave_ip_list) > 1:
# args.num_slaves = len(slave_ip_list)
for i in range(args.num_slaves):
slave_config = SlaveConfig()
slave_config.IP = slave_ip_list[(i % slave_ip_len)]
slave_config.PORT = args.port_start + i
slave_config.ID = "S{}".format(i)
slave_config.SHARD_MASK_LIST = [ShardMask(i | args.num_slaves)]
config.SLAVE_LIST.append(slave_config)
fd, config.json_filepath = tempfile.mkstemp()
with os.fdopen(fd, "w") as tmp:
tmp.write(config.to_json())
return config
def create(chain_id: int, shard_size: int, shard_id: int):
assert is_p2(shard_size)
return Branch((chain_id << 16) + shard_size | shard_id)
