def __create_from_args_internal(): check( is_p2(args.num_shards_per_chain), "--num_shards_per_chain must be power of 2", ) check(is_p2(args.num_slaves), "--num_slaves must be power of 2") config = ClusterConfig() config.LOG_LEVEL = args.log_level config.DB_PATH_ROOT = args.db_path_root config.P2P_PORT = args.p2p_port config.JSON_RPC_PORT = args.json_rpc_port config.PRIVATE_JSON_RPC_PORT = args.json_rpc_private_port config.CLEAN = args.clean config.START_SIMULATED_MINING = args.start_simulated_mining config.ENABLE_TRANSACTION_HISTORY = args.enable_transaction_history config.QUARKCHAIN.update( args.num_chains, args.num_shards_per_chain, args.root_block_interval_sec, args.minor_block_interval_sec, ) config.QUARKCHAIN.NETWORK_ID = args.network_id config.GENESIS_DIR = args.genesis_dir config.MONITORING.KAFKA_REST_ADDRESS = args.monitoring_kafka_rest_address if args.p2p: config.SIMPLE_NETWORK = None config.P2P = P2PConfig() # p2p module config.P2P.BOOT_NODES = args.bootnodes config.P2P.PRIV_KEY = args.privkey config.P2P.MAX_PEERS = args.max_peers config.P2P.UPNP = args.upnp else: config.P2P = None config.SIMPLE_NETWORK = SimpleNetworkConfig() config.SIMPLE_NETWORK.BOOTSTRAP_HOST = ( args.simple_network_bootstrap_host) config.SIMPLE_NETWORK.BOOTSTRAP_PORT = ( args.simple_network_bootstrap_port) config.SLAVE_LIST = [] for i in range(args.num_slaves): slave_config = SlaveConfig() slave_config.PORT = args.port_start + i slave_config.ID = "S{}".format(i) slave_config.SHARD_MASK_LIST = [ShardMask(i | args.num_slaves)] config.SLAVE_LIST.append(slave_config) fd, config.json_filepath = tempfile.mkstemp() with os.fdopen(fd, "w") as tmp: tmp.write(config.to_json()) return config
def __init__( self, num_cluster, genesis_account=Address.create_empty_account(), chain_size=2, shard_size=2, num_slaves=None, genesis_root_heights=None, remote_mining=False, small_coinbase=False, loadtest_accounts=None, connect=True, should_set_gas_price_limit=False, mblock_coinbase_amount=None, genesis_minor_quarkash=1000000, ): self.num_cluster = num_cluster self.genesis_account = genesis_account self.chain_size = chain_size self.shard_size = shard_size self.num_slaves = num_slaves if num_slaves else chain_size self.genesis_root_heights = genesis_root_heights self.remote_mining = remote_mining self.small_coinbase = small_coinbase self.loadtest_accounts = loadtest_accounts self.connect = connect self.should_set_gas_price_limit = should_set_gas_price_limit self.mblock_coinbase_amount = mblock_coinbase_amount self.genesis_minor_quarkash = genesis_minor_quarkash check(is_p2(self.num_slaves)) check(is_p2(self.shard_size))
def is_neighbor(b1: Branch, b2: Branch): """A naive algorithm to decide neighbor relationship TODO: a better algorithm, because the current one ensures 32 neighbors ONLY when there are 2^32 shards """ if b1.get_chain_id() == b2.get_chain_id(): return is_p2(abs(b1.get_shard_id() - b2.get_shard_id())) if b1.get_shard_id() == b2.get_shard_id(): return is_p2(abs(b1.get_chain_id() - b2.get_chain_id())) return False
def init_and_validate(self): self._chain_id_to_shard_size = dict() chain_id_to_shard_ids = dict() for full_shard_id, shard_config in self.shards.items(): chain_id = shard_config.CHAIN_ID shard_size = shard_config.SHARD_SIZE shard_id = shard_config.SHARD_ID check(full_shard_id == (chain_id << 16 | shard_size | shard_id)) check(is_p2(shard_size)) check(shard_config.ETH_CHAIN_ID == self.BASE_ETH_CHAIN_ID + chain_id + 1) if chain_id in self._chain_id_to_shard_size: check(shard_size == self._chain_id_to_shard_size[chain_id]) else: self._chain_id_to_shard_size[chain_id] = shard_size chain_id_to_shard_ids.setdefault(chain_id, set()).add(shard_id) # check the number of ShardConfigs matches SHARD_SIZE for each chain # and the SHARD_ID starts from 0 to (SHARD_SIZE - 1) for chain_id, shard_ids in chain_id_to_shard_ids.items(): shard_size = self.get_shard_size_by_chain_id(chain_id) check(shard_ids == set(range(shard_size))) # check the chain id starts from 0 to (CHAIN_SIZE - 1) check(set(chain_id_to_shard_ids.keys()) == set(range(self.CHAIN_SIZE)))
def get_test_env( genesis_account=Address.create_empty_account(), genesis_minor_quarkash=0, chain_size=2, shard_size=2, genesis_root_heights=None, # dict(full_shard_id, genesis_root_height) remote_mining=False, genesis_minor_token_balances={}, ): check(is_p2(shard_size)) env = DEFAULT_ENV.copy() env.db = InMemoryDb() env.set_network_id(1234567890) env.cluster_config = ClusterConfig() env.quark_chain_config.update(chain_size, shard_size, 10, 1, env.quark_chain_config.GENESIS_TOKEN) if remote_mining: env.quark_chain_config.ROOT.CONSENSUS_CONFIG.REMOTE_MINE = True env.quark_chain_config.ROOT.CONSENSUS_TYPE = ConsensusType.POW_DOUBLESHA256 env.quark_chain_config.ROOT.GENESIS.DIFFICULTY = 10 env.quark_chain_config.ROOT.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME = 40 env.quark_chain_config.ROOT.DIFFICULTY_ADJUSTMENT_FACTOR = 1024 if genesis_root_heights: check(len(genesis_root_heights) == shard_size * chain_size) for chain_id in range(chain_size): for shard_id in range(shard_size): full_shard_id = chain_id << 16 | shard_size | shard_id shard = env.quark_chain_config.shards[full_shard_id] shard.GENESIS.ROOT_HEIGHT = genesis_root_heights[full_shard_id] # fund genesis account in all shards for full_shard_id, shard in env.quark_chain_config.shards.items(): addr = genesis_account.address_in_shard( full_shard_id).serialize().hex() if len(genesis_minor_token_balances) != 0: shard.GENESIS.ALLOC[addr] = genesis_minor_token_balances else: shard.GENESIS.ALLOC[addr] = genesis_minor_quarkash shard.CONSENSUS_CONFIG.REMOTE_MINE = remote_mining shard.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME = 7 shard.DIFFICULTY_ADJUSTMENT_FACTOR = 512 if remote_mining: shard.CONSENSUS_TYPE = ConsensusType.POW_DOUBLESHA256 shard.GENESIS.DIFFICULTY = 10 shard.POSW_CONFIG.WINDOW_SIZE = 2 env.quark_chain_config.SKIP_MINOR_DIFFICULTY_CHECK = True env.quark_chain_config.SKIP_ROOT_DIFFICULTY_CHECK = True env.cluster_config.ENABLE_TRANSACTION_HISTORY = True env.cluster_config.DB_PATH_ROOT = "" check(env.cluster_config.use_mem_db()) return env
def __init__( self, num_cluster, genesis_account=Address.create_empty_account(), chain_size=2, shard_size=2, num_slaves=None, genesis_root_heights=None, remote_mining=False, small_coinbase=False, ): self.num_cluster = num_cluster self.genesis_account = genesis_account self.chain_size = chain_size self.shard_size = shard_size self.num_slaves = num_slaves if num_slaves else chain_size self.genesis_root_heights = genesis_root_heights self.remote_mining = remote_mining self.small_coinbase = small_coinbase check(is_p2(self.num_slaves)) check(is_p2(self.shard_size))
def is_neighbor(b1: Branch, b2: Branch): """A naive algorithm to decide neighbor relationship Two shards are neighbor iff there is only 1 bit difference in their shard ids. This only applies if there are more than 32 shards in the network. Otherwise all shards are neighbor to each other. TODO: a better algorithm """ check(b1.get_shard_size() == b2.get_shard_size()) check(b1.get_shard_id() != b2.get_shard_id()) if b1.get_shard_size() <= 32: return True return is_p2(abs(b1.get_shard_id() - b2.get_shard_id()))
def create(shard_size: int, shard_id: int): assert is_p2(shard_size) return Branch(shard_size | shard_id)
def create_test_clusters( num_cluster, genesis_account, chain_size, shard_size, num_slaves, genesis_root_heights, genesis_minor_quarkash, remote_mining=False, small_coinbase=False, loadtest_accounts=None, connect=True, # connect the bootstrap node by default should_set_gas_price_limit=False, mblock_coinbase_amount=None, ): # so we can have lower minimum diff easy_diff_calc = EthDifficultyCalculator( cutoff=45, diff_factor=2048, minimum_diff=10 ) bootstrap_port = get_next_port() # first cluster will listen on this port cluster_list = [] loop = asyncio.get_event_loop() for i in range(num_cluster): env = get_test_env( genesis_account, genesis_minor_quarkash=genesis_minor_quarkash, chain_size=chain_size, shard_size=shard_size, genesis_root_heights=genesis_root_heights, remote_mining=remote_mining, ) env.cluster_config.P2P_PORT = bootstrap_port if i == 0 else get_next_port() env.cluster_config.JSON_RPC_PORT = get_next_port() env.cluster_config.PRIVATE_JSON_RPC_PORT = get_next_port() env.cluster_config.SIMPLE_NETWORK = SimpleNetworkConfig() env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT = bootstrap_port env.quark_chain_config.loadtest_accounts = loadtest_accounts or [] if should_set_gas_price_limit: env.quark_chain_config.MIN_TX_POOL_GAS_PRICE = 10 env.quark_chain_config.MIN_MINING_GAS_PRICE = 10 if small_coinbase: # prevent breaking previous tests after tweaking default rewards env.quark_chain_config.ROOT.COINBASE_AMOUNT = 5 for c in env.quark_chain_config.shards.values(): c.COINBASE_AMOUNT = 5 if mblock_coinbase_amount is not None: for c in env.quark_chain_config.shards.values(): c.COINBASE_AMOUNT = mblock_coinbase_amount env.cluster_config.SLAVE_LIST = [] check(is_p2(num_slaves)) for j in range(num_slaves): slave_config = SlaveConfig() slave_config.ID = "S{}".format(j) slave_config.PORT = get_next_port() slave_config.FULL_SHARD_ID_LIST = [] env.cluster_config.SLAVE_LIST.append(slave_config) full_shard_ids = [ (i << 16) + shard_size + j for i in range(chain_size) for j in range(shard_size) ] for i, full_shard_id in enumerate(full_shard_ids): slave = env.cluster_config.SLAVE_LIST[i % num_slaves] slave.FULL_SHARD_ID_LIST.append(full_shard_id) slave_server_list = [] for j in range(num_slaves): slave_env = env.copy() slave_env.db = InMemoryDb() slave_env.slave_config = env.cluster_config.get_slave_config( "S{}".format(j) ) slave_server = SlaveServer(slave_env, name="cluster{}_slave{}".format(i, j)) slave_server.start() slave_server_list.append(slave_server) root_state = RootState(env, diff_calc=easy_diff_calc) master_server = MasterServer(env, root_state, name="cluster{}_master".format(i)) master_server.start() # Wait until the cluster is ready loop.run_until_complete(master_server.cluster_active_future) # Substitute diff calculate with an easier one for slave in slave_server_list: for shard in slave.shards.values(): shard.state.diff_calc = easy_diff_calc # Start simple network and connect to seed host network = SimpleNetwork(env, master_server, loop) network.start_server() if connect and i != 0: peer = call_async(network.connect("127.0.0.1", bootstrap_port)) else: peer = None cluster_list.append(Cluster(master_server, slave_server_list, network, peer)) return cluster_list
def __create_from_args_internal(): check( is_p2(args.num_shards_per_chain), "--num_shards_per_chain must be power of 2", ) check(is_p2(args.num_slaves), "--num_slaves must be power of 2") config = ClusterConfig() config.LOG_LEVEL = args.log_level config.DB_PATH_ROOT = args.db_path_root config.P2P_PORT = args.p2p_port config.JSON_RPC_PORT = args.json_rpc_port config.PRIVATE_JSON_RPC_PORT = args.json_rpc_private_port config.JSON_RPC_HOST = args.json_rpc_host config.PRIVATE_JSON_RPC_HOST = args.json_rpc_private_host config.CLEAN = args.clean config.START_SIMULATED_MINING = args.start_simulated_mining config.ENABLE_TRANSACTION_HISTORY = args.enable_transaction_history config.QUARKCHAIN.update( args.num_chains, args.num_shards_per_chain, args.root_block_interval_sec, args.minor_block_interval_sec, args.default_token, ) config.QUARKCHAIN.NETWORK_ID = args.network_id config.GENESIS_DIR = args.genesis_dir config.MONITORING.KAFKA_REST_ADDRESS = args.monitoring_kafka_rest_address if args.p2p: config.SIMPLE_NETWORK = None config.P2P = P2PConfig() # p2p module config.P2P.BOOT_NODES = args.bootnodes config.P2P.PRIV_KEY = args.privkey config.P2P.MAX_PEERS = args.max_peers config.P2P.UPNP = args.upnp else: config.P2P = None config.SIMPLE_NETWORK = SimpleNetworkConfig() config.SIMPLE_NETWORK.BOOTSTRAP_HOST = ( args.simple_network_bootstrap_host) config.SIMPLE_NETWORK.BOOTSTRAP_PORT = ( args.simple_network_bootstrap_port) if args.prom: config.PROMETHEUS = PrometheusConfig() config.PROMETHEUS.INTERVAL = args.prom_interval config.PROMETHEUS.TOKENS = args.prom_tokens config.PROMETHEUS.PORT = args.prom_port config.SLAVE_LIST = [] for i in range(args.num_slaves): slave_config = SlaveConfig() slave_config.PORT = args.port_start + i slave_config.ID = "S{}".format(i) slave_config.FULL_SHARD_ID_LIST = [] config.SLAVE_LIST.append(slave_config) # assign full shard IDs to each slave, using hex strings to write into JSON full_shard_ids = [(i << 16) + args.num_shards_per_chain + j for i in range(args.num_chains) for j in range(args.num_shards_per_chain)] for i, full_shard_id in enumerate(full_shard_ids): slave = config.SLAVE_LIST[i % args.num_slaves] slave.FULL_SHARD_ID_LIST.append(full_shard_id) fd, config.json_filepath = tempfile.mkstemp() with os.fdopen(fd, "w") as tmp: tmp.write(config.to_json()) return config
def get_full_shard_id(self, shard_size: int): if not is_p2(shard_size): raise RuntimeError("Invalid shard size {}".format(shard_size)) chain_id = self.full_shard_key >> 16 shard_id = self.full_shard_key & (shard_size - 1) return (chain_id << 16) | shard_size | shard_id
def set_shard_size(self, shard_size): check(is_p2(shard_size)) self.shard_size = shard_size
def create(shard_size, reshard_vote=False): assert is_p2(shard_size) reshard_vote = 1 if reshard_vote else 0 return ShardInfo( int_left_most_bit(shard_size) - 1 + (reshard_vote << 31))
def create_test_clusters( num_cluster, genesis_account, chain_size, shard_size, num_slaves, genesis_root_heights, remote_mining=False, small_coinbase=False, ): # so we can have lower minimum diff easy_diff_calc = EthDifficultyCalculator(cutoff=45, diff_factor=2048, minimum_diff=10) bootstrap_port = get_next_port() # first cluster will listen on this port cluster_list = [] loop = asyncio.get_event_loop() for i in range(num_cluster): env = get_test_env( genesis_account, genesis_minor_quarkash=1000000, chain_size=chain_size, shard_size=shard_size, genesis_root_heights=genesis_root_heights, remote_mining=remote_mining, ) env.cluster_config.P2P_PORT = bootstrap_port if i == 0 else get_next_port( ) env.cluster_config.JSON_RPC_PORT = get_next_port() env.cluster_config.PRIVATE_JSON_RPC_PORT = get_next_port() env.cluster_config.SIMPLE_NETWORK = SimpleNetworkConfig() env.cluster_config.SIMPLE_NETWORK.BOOTSTRAP_PORT = bootstrap_port if small_coinbase: # prevent breaking previous tests after tweaking default rewards env.quark_chain_config.ROOT.COINBASE_AMOUNT = 5 for c in env.quark_chain_config.shards.values(): c.COINBASE_AMOUNT = 5 env.cluster_config.SLAVE_LIST = [] check(is_p2(num_slaves)) for j in range(num_slaves): slave_config = SlaveConfig() slave_config.ID = "S{}".format(j) slave_config.PORT = get_next_port() slave_config.CHAIN_MASK_LIST = [ChainMask(num_slaves | j)] env.cluster_config.SLAVE_LIST.append(slave_config) slave_server_list = [] for j in range(num_slaves): slave_env = env.copy() slave_env.db = InMemoryDb() slave_env.slave_config = env.cluster_config.get_slave_config( "S{}".format(j)) slave_server = SlaveServer(slave_env, name="cluster{}_slave{}".format(i, j)) slave_server.start() slave_server_list.append(slave_server) root_state = RootState(env, diff_calc=easy_diff_calc) master_server = MasterServer(env, root_state, name="cluster{}_master".format(i)) master_server.start() # Wait until the cluster is ready loop.run_until_complete(master_server.cluster_active_future) # Substitute diff calculate with an easier one for slave in slave_server_list: for shard in slave.shards.values(): shard.state.diff_calc = easy_diff_calc # Start simple network and connect to seed host network = SimpleNetwork(env, master_server, loop) network.start_server() if i != 0: peer = call_async(network.connect("127.0.0.1", bootstrap_port)) else: peer = None cluster_list.append( Cluster(master_server, slave_server_list, network, peer)) return cluster_list
def get_shard_id(self, shard_size): if not is_p2(shard_size): raise RuntimeError("Invalid shard size {}".format(shard_size)) return self.full_shard_id & (shard_size - 1)
def __create_from_args_internal(): check(is_p2(args.num_shards), "--num_shards must be power of 2") check(is_p2(args.num_slaves), "--num_slaves must be power of 2") config = ClusterConfig() config.LOG_LEVEL = args.log_level config.DB_PATH_ROOT = args.db_path_root config.P2P_PORT = args.p2p_port config.JSON_RPC_PORT = args.json_rpc_port config.PRIVATE_JSON_RPC_PORT = args.json_rpc_private_port config.CLEAN = args.clean config.MINE = args.mine config.SLAVE_IDS = args.slave_ids config.IS_MASTER = args.is_master config.ENABLE_TRANSACTION_HISTORY = args.enable_transaction_history config.QUARKCHAIN.update( args.num_shards, args.root_block_interval_sec, args.minor_block_interval_sec, ) config.QUARKCHAIN.NETWORK_ID = args.network_id config.GENESIS_DIR = args.genesis_dir config.MONITORING.KAFKA_REST_ADDRESS = args.monitoring_kafka_rest_address if args.devp2p_enable: config.SIMPLE_NETWORK = None config.P2P = P2PConfig() config.P2P.IP = args.devp2p_ip config.P2P.DISCOVERY_PORT = args.devp2p_port config.P2P.BOOTSTRAP_HOST = args.devp2p_bootstrap_host config.P2P.BOOTSTRAP_PORT = args.devp2p_bootstrap_port config.P2P.MIN_PEERS = args.devp2p_min_peers config.P2P.MAX_PEERS = args.devp2p_max_peers config.P2P.ADDITIONAL_BOOTSTRAPS = args.devp2p_additional_bootstraps else: config.P2P = None config.SIMPLE_NETWORK = SimpleNetworkConfig() config.SIMPLE_NETWORK.BOOTSTRAP_HOST = ( args.simple_network_bootstrap_host) config.SIMPLE_NETWORK.BOOTSTRAP_PORT = ( args.simple_network_bootstrap_port) config.SLAVE_LIST = [] slave_ip_list = args.slave_ips.split(",") slave_ip_len = len(slave_ip_list) # if len(slave_ip_list) > 1: # args.num_slaves = len(slave_ip_list) for i in range(args.num_slaves): slave_config = SlaveConfig() slave_config.IP = slave_ip_list[(i % slave_ip_len)] slave_config.PORT = args.port_start + i slave_config.ID = "S{}".format(i) slave_config.SHARD_MASK_LIST = [ShardMask(i | args.num_slaves)] config.SLAVE_LIST.append(slave_config) fd, config.json_filepath = tempfile.mkstemp() with os.fdopen(fd, "w") as tmp: tmp.write(config.to_json()) return config
def create(chain_id: int, shard_size: int, shard_id: int): assert is_p2(shard_size) return Branch((chain_id << 16) + shard_size | shard_id)