def __create_from_args_internal():
check(
is_p2(args.num_shards_per_chain),
"--num_shards_per_chain must be power of 2",
)
check(is_p2(args.num_slaves), "--num_slaves must be power of 2")
config = ClusterConfig()
config.LOG_LEVEL = args.log_level
config.DB_PATH_ROOT = args.db_path_root
config.P2P_PORT = args.p2p_port
config.JSON_RPC_PORT = args.json_rpc_port
config.PRIVATE_JSON_RPC_PORT = args.json_rpc_private_port
config.CLEAN = args.clean
config.START_SIMULATED_MINING = args.start_simulated_mining
config.ENABLE_TRANSACTION_HISTORY = args.enable_transaction_history
config.QUARKCHAIN.update(
args.num_chains,
args.num_shards_per_chain,
args.root_block_interval_sec,
args.minor_block_interval_sec,
)
config.QUARKCHAIN.NETWORK_ID = args.network_id
config.GENESIS_DIR = args.genesis_dir
config.MONITORING.KAFKA_REST_ADDRESS = args.monitoring_kafka_rest_address
if args.p2p:
config.SIMPLE_NETWORK = None
config.P2P = P2PConfig()
# p2p module
config.P2P.BOOT_NODES = args.bootnodes
config.P2P.PRIV_KEY = args.privkey
config.P2P.MAX_PEERS = args.max_peers
config.P2P.UPNP = args.upnp
else:
config.P2P = None
config.SIMPLE_NETWORK = SimpleNetworkConfig()
config.SIMPLE_NETWORK.BOOTSTRAP_HOST = (
args.simple_network_bootstrap_host)
config.SIMPLE_NETWORK.BOOTSTRAP_PORT = (
args.simple_network_bootstrap_port)
config.SLAVE_LIST = []
for i in range(args.num_slaves):
slave_config = SlaveConfig()
slave_config.PORT = args.port_start + i
slave_config.ID = "S{}".format(i)
slave_config.SHARD_MASK_LIST = [ShardMask(i | args.num_slaves)]
config.SLAVE_LIST.append(slave_config)
fd, config.json_filepath = tempfile.mkstemp()
with os.fdopen(fd, "w") as tmp:
tmp.write(config.to_json())
return config
def test_shard_mask_iterate(self):
sm0 = ShardMask(0b11)
self.assertEqual(sorted(l for l in sm0.iterate(4)), [1, 0b11])
self.assertEqual(sorted(l for l in sm0.iterate(8)), [1, 0b11, 0b101, 0b111])
sm1 = ShardMask(0b101)
self.assertEqual(sorted(l for l in sm1.iterate(8)), [1, 0b101])
self.assertEqual(sorted(l for l in sm1.iterate(16)), [1, 0b101, 0b1001, 0b1101])
def __init__(self):
self.QUARKCHAIN = QuarkChainConfig()
self.MASTER = MasterConfig()
self.SLAVE_LIST = [] # type: List[SlaveConfig]
self.SIMPLE_NETWORK = SimpleNetworkConfig()
self._json_filepath = None
self.MONITORING = MonitoringConfig()
self.kafka_logger = KafkaSampleLogger(self)
slave_config = SlaveConfig()
slave_config.PORT = 38000
slave_config.ID = "S0"
slave_config.SHARD_MASK_LIST = [ShardMask(1)]
self.SLAVE_LIST.append(slave_config)
fd, self.json_filepath = tempfile.mkstemp()
with os.fdopen(fd, "w") as tmp:
tmp.write(self.to_json())
def test_shard_mask(self):
sm0 = ShardMask(0b1)
self.assertTrue(sm0.contain_shard_id(0))
self.assertTrue(sm0.contain_shard_id(1))
self.assertTrue(sm0.contain_shard_id(0b1111111))
sm1 = ShardMask(0b101)
self.assertFalse(sm1.contain_shard_id(0))
self.assertTrue(sm1.contain_shard_id(0b1))
self.assertFalse(sm1.contain_shard_id(0b10))
self.assertFalse(sm1.contain_shard_id(0b11))
self.assertFalse(sm1.contain_shard_id(0b100))
self.assertTrue(sm1.contain_shard_id(0b101))
self.assertFalse(sm1.contain_shard_id(0b110))
self.assertFalse(sm1.contain_shard_id(0b111))
self.assertFalse(sm1.contain_shard_id(0b1000))
self.assertTrue(sm1.contain_shard_id(0b1001))
def create_test_clusters(
num_cluster,
genesis_account,
shard_size,
num_slaves,
genesis_root_heights,
remote_mining=False,
):
# so we can have lower minimum diff
easy_diff_calc = EthDifficultyCalculator(cutoff=45,
diff_factor=2048,
minimum_diff=10)
bootstrap_port = get_next_port() # first cluster will listen on this port
cluster_list = []
loop = asyncio.get_event_loop()
for i in range(num_cluster):
env = get_test_env(
genesis_account,
genesis_minor_quarkash=1000000,
shard_size=shard_size,
genesis_root_heights=genesis_root_heights,
remote_mining=remote_mining,
)
env.cluster_config.P2P_PORT = bootstrap_port if i == 0 else get_next_port(
)
env.cluster_config.JSON_RPC_PORT = get_next_port()
env.cluster_config.PRIVATE_JSON_RPC_PORT = get_next_port()
env.cluster_config.SIMPLE_NETWORK = SimpleNetworkConfig()
env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT = bootstrap_port
env.cluster_config.SLAVE_LIST = []
for j in range(num_slaves):
slave_config = SlaveConfig()
slave_config.ID = "S{}".format(j)
slave_config.PORT = get_next_port()
slave_config.SHARD_MASK_LIST = [ShardMask(num_slaves | j)]
slave_config.DB_PATH_ROOT = None # TODO: fix the db in config
env.cluster_config.SLAVE_LIST.append(slave_config)
slave_server_list = []
for j in range(num_slaves):
slave_env = env.copy()
slave_env.db = InMemoryDb()
slave_env.slave_config = env.cluster_config.get_slave_config(
"S{}".format(j))
slave_server = SlaveServer(slave_env,
name="cluster{}_slave{}".format(i, j))
slave_server.start()
slave_server_list.append(slave_server)
root_state = RootState(env, diff_calc=easy_diff_calc)
master_server = MasterServer(env,
root_state,
name="cluster{}_master".format(i))
master_server.start()
# Wait until the cluster is ready
loop.run_until_complete(master_server.cluster_active_future)
# Substitute diff calculate with an easier one
for slave in slave_server_list:
for shard in slave.shards.values():
shard.state.diff_calc = easy_diff_calc
# Start simple network and connect to seed host
network = SimpleNetwork(env, master_server)
network.start_server()
if i != 0:
peer = call_async(network.connect("127.0.0.1", bootstrap_port))
else:
peer = None
cluster_list.append(
Cluster(master_server, slave_server_list, network, peer))
return cluster_list
def from_dict(cls, d):
config = super().from_dict(d)
config.SHARD_MASK_LIST = [ShardMask(v) for v in config.SHARD_MASK_LIST]
return config
def __create_from_args_internal():
check(is_p2(args.num_shards), "--num_shards must be power of 2")
check(is_p2(args.num_slaves), "--num_slaves must be power of 2")
config = ClusterConfig()
config.LOG_LEVEL = args.log_level
config.DB_PATH_ROOT = args.db_path_root
config.P2P_PORT = args.p2p_port
config.JSON_RPC_PORT = args.json_rpc_port
config.PRIVATE_JSON_RPC_PORT = args.json_rpc_private_port
config.CLEAN = args.clean
config.MINE = args.mine
config.SLAVE_IDS = args.slave_ids
config.IS_MASTER = args.is_master
config.ENABLE_TRANSACTION_HISTORY = args.enable_transaction_history
config.QUARKCHAIN.update(
args.num_shards,
args.root_block_interval_sec,
args.minor_block_interval_sec,
)
config.QUARKCHAIN.NETWORK_ID = args.network_id
config.GENESIS_DIR = args.genesis_dir
config.MONITORING.KAFKA_REST_ADDRESS = args.monitoring_kafka_rest_address
if args.devp2p_enable:
config.SIMPLE_NETWORK = None
config.P2P = P2PConfig()
config.P2P.IP = args.devp2p_ip
config.P2P.DISCOVERY_PORT = args.devp2p_port
config.P2P.BOOTSTRAP_HOST = args.devp2p_bootstrap_host
config.P2P.BOOTSTRAP_PORT = args.devp2p_bootstrap_port
config.P2P.MIN_PEERS = args.devp2p_min_peers
config.P2P.MAX_PEERS = args.devp2p_max_peers
config.P2P.ADDITIONAL_BOOTSTRAPS = args.devp2p_additional_bootstraps
else:
config.P2P = None
config.SIMPLE_NETWORK = SimpleNetworkConfig()
config.SIMPLE_NETWORK.BOOTSTRAP_HOST = (
args.simple_network_bootstrap_host)
config.SIMPLE_NETWORK.BOOTSTRAP_PORT = (
args.simple_network_bootstrap_port)
config.SLAVE_LIST = []
slave_ip_list = args.slave_ips.split(",")
slave_ip_len = len(slave_ip_list)
# if len(slave_ip_list) > 1:
# args.num_slaves = len(slave_ip_list)
for i in range(args.num_slaves):
slave_config = SlaveConfig()
slave_config.IP = slave_ip_list[(i % slave_ip_len)]
slave_config.PORT = args.port_start + i
slave_config.ID = "S{}".format(i)
slave_config.SHARD_MASK_LIST = [ShardMask(i | args.num_slaves)]
config.SLAVE_LIST.append(slave_config)
fd, config.json_filepath = tempfile.mkstemp()
with os.fdopen(fd, "w") as tmp:
tmp.write(config.to_json())
return config
def create_test_clusters(num_cluster, genesis_account, shard_size, num_slaves,
genesis_root_heights):
bootstrap_port = get_next_port() # first cluster will listen on this port
cluster_list = []
loop = asyncio.get_event_loop()
for i in range(num_cluster):
env = get_test_env(
genesis_account,
genesis_minor_quarkash=1000000,
shard_size=shard_size,
genesis_root_heights=genesis_root_heights,
)
env.cluster_config.P2P_PORT = bootstrap_port if i == 0 else get_next_port(
)
env.cluster_config.JSON_RPC_PORT = get_next_port()
env.cluster_config.PRIVATE_JSON_RPC_PORT = get_next_port()
env.cluster_config.SIMPLE_NETWORK = SimpleNetworkConfig()
env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT = bootstrap_port
env.cluster_config.SLAVE_LIST = []
for j in range(num_slaves):
slave_config = SlaveConfig()
slave_config.ID = "S{}".format(j)
slave_config.PORT = get_next_port()
slave_config.SHARD_MASK_LIST = [ShardMask(num_slaves | j)]
slave_config.DB_PATH_ROOT = None # TODO: fix the db in config
env.cluster_config.SLAVE_LIST.append(slave_config)
slave_server_list = []
for j in range(num_slaves):
slave_env = env.copy()
slave_env.db = InMemoryDb()
slave_env.slave_config = env.cluster_config.get_slave_config(
"S{}".format(j))
slave_server = SlaveServer(slave_env,
name="cluster{}_slave{}".format(i, j))
slave_server.start()
slave_server_list.append(slave_server)
root_state = RootState(env)
master_server = MasterServer(env,
root_state,
name="cluster{}_master".format(i))
master_server.start()
# Wait until the cluster is ready
loop.run_until_complete(master_server.cluster_active_future)
# Start simple network and connect to seed host
network = SimpleNetwork(env, master_server)
network.start_server()
if i != 0:
peer = call_async(network.connect("127.0.0.1", bootstrap_port))
else:
peer = None
cluster_list.append(
Cluster(master_server, slave_server_list, network, peer))
return cluster_list
