def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--full_shard_id", type=int, help="full shard id to operate") parser.add_argument( "--all_shards", action="store_true", default=False, help="query balances in all shards", ) parser.add_argument("--recipient", default=None, type=str, help="query a specific recipient") parser.add_argument( "--minor_block_height", default=None, type=int, help="query balance at specific minor block height", ) ClusterConfig.attach_arguments(parser) args = parser.parse_args() env = DEFAULT_ENV.copy() env.cluster_config = ClusterConfig.create_from_args(args) # initialize database if not env.cluster_config.use_mem_db(): env.db = PersistentDb( "{path}/master.db".format(path=env.cluster_config.DB_PATH_ROOT), clean=env.cluster_config.CLEAN, ) return env, args
def test_unknown_structure(self): parser = argparse.ArgumentParser() ClusterConfig.attach_arguments(parser) args = parser.parse_args(["--monitoring_kafka_rest_address=x"]) cluster_config = ClusterConfig.create_from_args(args) sample = dict(a=1, b=2, c={"x", "y"}) cluster_config.kafka_logger.log_kafka_sample( "topic", sample) # should trigger warning log
def test_kafka_log(self): parser = argparse.ArgumentParser() ClusterConfig.attach_arguments(parser) args = parser.parse_args( []) # set --kafka_rest_address correctly to see real actions cluster_config = ClusterConfig.create_from_args(args) sample = dict(a=1, b=2, c=["x", "y"]) cluster_config.kafka_logger.log_kafka_sample("dlltest", sample)
async def main(): parser = argparse.ArgumentParser() ClusterConfig.attach_arguments(parser) parser.add_argument("--num_clusters", default=2, type=int) args = parser.parse_args() clusters = [] mine_i = random.randint(0, args.num_clusters - 1) mine = args.start_simulated_mining if mine: print("cluster {} will be mining".format(mine_i)) else: print("No one will be mining") db_path_root = args.db_path_root p2p_port = args.p2p_port for i in range(args.num_clusters): args.start_simulated_mining = mine and i == mine_i args.db_path_root = "{}_C{}".format(db_path_root, i) # set up p2p bootstrapping, with fixed bootstrap key for now if args.p2p: if i == 0: args.privkey = ( "31552f186bf90908ce386fb547dd0410bf443309125cc43fd0ffd642959bf6d9" ) else: args.privkey = "" args.bootnodes = "enode://c571e0db93d17cc405cb57640826b70588a6a28785f38b21be471c609ca12fcb06cb306ac44872908f5bed99046031a5af82072d484e3ef9029560c1707193a0@127.0.0.1:{}".format( p2p_port) config = ClusterConfig.create_from_args(args) print("Cluster {} config file: {}".format(i, config.json_filepath)) print(config.to_json()) clusters.append( cl.Cluster( config, "{}C{}{}_".format(colors[i % len(colors)], i, COLOR_END))) args.p2p_port += 1 args.port_start += 100 args.json_rpc_port += 1 args.json_rpc_private_port += 1 tasks = list() tasks.append(asyncio.ensure_future(clusters[0].run())) await asyncio.sleep(3) for cluster in clusters[1:]: tasks.append(asyncio.ensure_future(cluster.run())) try: await asyncio.gather(*tasks) except KeyboardInterrupt: try: for cluster in clusters: asyncio.get_event_loop().run_until_complete(cluster.shutdown()) except Exception: pass
def test_cluster_dict(self): parser = argparse.ArgumentParser() ClusterConfig.attach_arguments(parser) args = parser.parse_args(["--num_shards=4", "--genesis_dir="]) cluster_config = ClusterConfig.create_from_args(args) args = parser.parse_args(["--cluster_config=" + cluster_config.json_filepath]) deserialized = ClusterConfig.create_from_args(args) self.assertTrue(cluster_config == deserialized)
def parse_args(): parser = argparse.ArgumentParser() ClusterConfig.attach_arguments(parser) # Unique Id identifying the node in the cluster parser.add_argument("--node_id", default="", type=str) args = parser.parse_args() env = DEFAULT_ENV.copy() env.cluster_config = ClusterConfig.create_from_args(args) env.slave_config = env.cluster_config.get_slave_config(args.node_id) return env
def main(): os.chdir(os.path.dirname(os.path.abspath(__file__))) parser = argparse.ArgumentParser() ClusterConfig.attach_arguments(parser) args = parser.parse_args() config = ClusterConfig.create_from_args(args) print("Cluster config file: {}".format(config.json_filepath)) print(config.to_json()) cluster = Cluster(config) cluster.start_and_loop()
def parse_args(): parser = argparse.ArgumentParser() ClusterConfig.attach_arguments(parser) parser.add_argument("--profile", default="", type=str) parser.add_argument("--check_db_rblock_from", default=-1, type=int) parser.add_argument("--check_db_rblock_to", default=0, type=int) parser.add_argument("--check_db_rblock_batch", default=1, type=int) args = parser.parse_args() env = DEFAULT_ENV.copy() env.cluster_config = ClusterConfig.create_from_args(args) env.arguments = args return env
def test_cluster_slave_config_legacy(self): parser = argparse.ArgumentParser() ClusterConfig.attach_arguments(parser) args = parser.parse_args( ["--num_chains=8", "--num_shards_per_chain=1", "--num_slaves=4"]) cluster_config = ClusterConfig.create_from_args(args) # remove full shard list but use chain mask d = cluster_config.to_dict() for i, s in enumerate(d["SLAVE_LIST"]): s["CHAIN_MASK_LIST"] = [i + 4] del (s["FULL_SHARD_ID_LIST"]) deserialized = ClusterConfig.from_dict(d) # chain mask translated config should equal previous full shard ID config self.assertTrue(cluster_config == deserialized)
def __init__(self, db=None, evm_config=None): self.db = db or InMemoryDb() self.__cluster_config = ClusterConfig() self.evm_config = evm_config or get_default_evm_config() self.evm_config["NETWORK_ID"] = self.quark_chain_config.NETWORK_ID self.evm_env = EvmEnv(db=self.db, config=self.evm_config)
def get_test_env( genesis_account=Address.create_empty_account(), genesis_quarkash=0, genesis_minor_quarkash=0, shard_size=2, genesis_root_heights=None, ): env = DEFAULT_ENV.copy() env.db = InMemoryDb() env.set_network_id(1234567890) env.cluster_config = ClusterConfig() env.quark_chain_config.update(shard_size, 1, 1) env.quark_chain_config.TESTNET_MASTER_ADDRESS = genesis_account.serialize( ).hex() if genesis_root_heights: check(len(genesis_root_heights) == shard_size) for shard_id in range(shard_size): env.quark_chain_config.SHARD_LIST[ shard_id].GENESIS.ROOT_HEIGHT = genesis_root_heights[shard_id] # fund genesis account in all shards for i, shard in enumerate(env.quark_chain_config.SHARD_LIST): shard.GENESIS.ALLOC[genesis_account.address_in_shard( i).serialize().hex()] = genesis_minor_quarkash env.quark_chain_config.SKIP_MINOR_DIFFICULTY_CHECK = True env.quark_chain_config.SKIP_ROOT_DIFFICULTY_CHECK = True env.cluster_config.ENABLE_TRANSACTION_HISTORY = True env.cluster_config.DB_PATH_ROOT = "" check(env.cluster_config.use_mem_db()) return env
def jrpc_server_context(master): env = DEFAULT_ENV.copy() env.cluster_config = ClusterConfig() env.cluster_config.JSON_RPC_PORT = 38391 server = JSONRPCServer.start_test_server(env, master) yield server server.shutdown()
def get_test_env( genesis_account=Address.create_empty_account(), genesis_minor_quarkash=0, chain_size=2, shard_size=2, genesis_root_heights=None, # dict(full_shard_id, genesis_root_height) remote_mining=False, genesis_minor_token_balances={}, ): check(is_p2(shard_size)) env = DEFAULT_ENV.copy() env.db = InMemoryDb() env.set_network_id(1234567890) env.cluster_config = ClusterConfig() env.quark_chain_config.update(chain_size, shard_size, 10, 1, env.quark_chain_config.GENESIS_TOKEN) if remote_mining: env.quark_chain_config.ROOT.CONSENSUS_CONFIG.REMOTE_MINE = True env.quark_chain_config.ROOT.CONSENSUS_TYPE = ConsensusType.POW_DOUBLESHA256 env.quark_chain_config.ROOT.GENESIS.DIFFICULTY = 10 env.quark_chain_config.ROOT.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME = 40 env.quark_chain_config.ROOT.DIFFICULTY_ADJUSTMENT_FACTOR = 1024 if genesis_root_heights: check(len(genesis_root_heights) == shard_size * chain_size) for chain_id in range(chain_size): for shard_id in range(shard_size): full_shard_id = chain_id << 16 | shard_size | shard_id shard = env.quark_chain_config.shards[full_shard_id] shard.GENESIS.ROOT_HEIGHT = genesis_root_heights[full_shard_id] # fund genesis account in all shards for full_shard_id, shard in env.quark_chain_config.shards.items(): addr = genesis_account.address_in_shard( full_shard_id).serialize().hex() if len(genesis_minor_token_balances) != 0: shard.GENESIS.ALLOC[addr] = genesis_minor_token_balances else: shard.GENESIS.ALLOC[addr] = genesis_minor_quarkash shard.CONSENSUS_CONFIG.REMOTE_MINE = remote_mining shard.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME = 7 shard.DIFFICULTY_ADJUSTMENT_FACTOR = 512 if remote_mining: shard.CONSENSUS_TYPE = ConsensusType.POW_DOUBLESHA256 shard.GENESIS.DIFFICULTY = 10 shard.POSW_CONFIG.WINDOW_SIZE = 2 env.quark_chain_config.SKIP_MINOR_DIFFICULTY_CHECK = True env.quark_chain_config.SKIP_ROOT_DIFFICULTY_CHECK = True env.cluster_config.ENABLE_TRANSACTION_HISTORY = True env.cluster_config.DB_PATH_ROOT = "" check(env.cluster_config.use_mem_db()) return env
async def main(): parser = argparse.ArgumentParser() ClusterConfig.attach_arguments(parser) parser.add_argument("--num_clusters", default=2, type=int) args = parser.parse_args() clusters = [] mine_i = random.randint(0, args.num_clusters - 1) if args.mine: print("cluster {} will be mining".format(mine_i)) else: print("No one will be mining") mine = args.mine db_path_root = args.db_path_root for i in range(args.num_clusters): args.mine = mine and i == mine_i args.db_path_root = "{}_C{}".format(db_path_root, i) config = ClusterConfig.create_from_args(args) print("Cluster {} config file: {}".format(i, config.json_filepath)) print(config.to_json()) clusters.append( cl.Cluster( config, "{}C{}{}_".format(colors[i % len(colors)], i, COLOR_END))) args.p2p_port += 1 args.port_start += 100 args.json_rpc_port += 1 args.json_rpc_private_port += 1 args.devp2p_port += 1 tasks = list() tasks.append(asyncio.ensure_future(clusters[0].run())) await asyncio.sleep(3) for cluster in clusters[1:]: tasks.append(asyncio.ensure_future(cluster.run())) try: await asyncio.gather(*tasks) except KeyboardInterrupt: try: for cluster in clusters: asyncio.get_event_loop().run_until_complete(cluster.shutdown()) except Exception: pass
def test_special_contract_enable_ts(self): env = Env() for addr in PRECOMPILED_CONTRACTS_AFTER_EVM_ENABLED: self.assertEqual(specials[addr][1], 0) cluster_config = ClusterConfig() cluster_config.QUARKCHAIN.ENABLE_EVM_TIMESTAMP = 123 env.cluster_config = cluster_config for addr in PRECOMPILED_CONTRACTS_AFTER_EVM_ENABLED: self.assertEqual(specials[addr][1], 123)
def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("action", type=str, help="action to take") parser.add_argument("--height", type=int, help="block height to operate") parser.add_argument("--hash", type=str, help="block hash to operate") ClusterConfig.attach_arguments(parser) args = parser.parse_args() env = DEFAULT_ENV.copy() env.cluster_config = ClusterConfig.create_from_args(args) # initialize database if not env.cluster_config.use_mem_db(): env.db = PersistentDb( "{path}/master.db".format(path=env.cluster_config.DB_PATH_ROOT), clean=env.cluster_config.CLEAN, ) return env, args
def jrpc_server_context(master): env = DEFAULT_ENV.copy() env.cluster_config = ClusterConfig() env.cluster_config.JSON_RPC_PORT = 38391 # to pass the circleCi env.cluster_config.JSON_RPC_HOST = "127.0.0.1" server = JSONRPCServer.start_test_server(env, master) try: yield server finally: server.shutdown()
def test_cluster_dict_wloadtest(self): """convert to dict and back to check if the content changed, requires `__eq__` removing --loadtest will make the test faster passing more num_shards will increase runtime linearly """ parser = argparse.ArgumentParser() ClusterConfig.attach_arguments(parser) pwd = os.path.dirname(os.path.abspath(__file__)) default_genesis_dir = os.path.join(pwd, "../../genesis_data") args = parser.parse_args( ["--num_shards=4", "--genesis_dir=" + default_genesis_dir] ) cluster_config = ClusterConfig.create_from_args(args) args = parser.parse_args(["--cluster_config=" + cluster_config.json_filepath]) deserialized = ClusterConfig.create_from_args(args) self.assertTrue(cluster_config == deserialized) self.assertTrue( len(cluster_config.QUARKCHAIN.SHARD_LIST[0].GENESIS.ALLOC) > 12000 )
def main(): logging.getLogger("asyncio").setLevel(logging.ERROR) os.chdir(os.path.dirname(os.path.abspath(__file__))) parser = argparse.ArgumentParser() ClusterConfig.attach_arguments(parser) parser.add_argument("--profile", default="", type=str) parser.add_argument("--check_db_rblock_from", default=-1, type=int) parser.add_argument("--check_db_rblock_to", default=0, type=int) parser.add_argument("--check_db_rblock_batch", default=1, type=int) args = parser.parse_args() config = ClusterConfig.create_from_args(args) print("Cluster config file: {}".format(config.json_filepath)) print(config.to_json()) cluster = Cluster(config, args=args) if args.check_db: cluster.check_db() else: cluster.start_and_loop()
def main(): if "EMAIL_FROM_ADDRESS" not in os.environ: raise ValueError( "EMAIL_FROM_ADDRESS not found in environment variables") if "PASSWORD" not in os.environ: raise ValueError("PASSWORD not found in environment variables") if "EMAIL_TO_ADDRESS" not in os.environ: raise ValueError("EMAIL_TO_ADDRESS not found in environment variables") os.chdir(os.path.dirname("../../quarkchain/cluster/")) parser = argparse.ArgumentParser() ClusterConfig.attach_arguments(parser) args = parser.parse_args() config = ClusterConfig.create_from_args(args) # creat a config tempfile for health check, which is a copy of the original config file tempfile.tempdir = "../../mainnet/singularity/" fd, config.json_filepath = tempfile.mkstemp() with os.fdopen(fd, "w") as tmp: tmp.write(config.to_json()) print("Cluster config file: {}".format(config.json_filepath)) print(config.to_json()) cluster = HealthCheckCluster(config) bootstrap_nodes = config.P2P.BOOT_NODES.split(",") count = 0 while True: bash_command_revised_config = ( "QKC_CONFIG=" + config.json_filepath + " python3 ../../quarkchain/tools/config_p2p.py --bootnodes " + bootstrap_nodes[count]) os.system(bash_command_revised_config) print("Start Bootstrap With " + bootstrap_nodes[count]) cluster.start_and_loop() time.sleep(100) count = (count + 1) % len(bootstrap_nodes)
def main(): args = parse_args() config = ClusterConfig.create_from_args(args) db = RootDb(PersistentDb(args.db), config.QUARKCHAIN, 0) header = db.get_tip_header() if not header: raise RuntimeError("Not a valid RootDb") from_height = header.height if args.root_height <= 0 else args.root_height tip_header = None block = db.get_root_block_by_hash(header.get_hash(), False) shard_to_address_count = dict() # shard -> (recipient -> count) while block.header.height > 0: if block.header.height > from_height: block = db.get_root_block_by_hash(block.header.hash_prev_block, False) continue if block.header.height == from_height: tip_header = block.header for minor_header in block.minor_block_header_list: shard = minor_header.branch.get_full_shard_id() address_hex = minor_header.coinbase_address.recipient.hex() address_to_count = shard_to_address_count.setdefault(shard, dict()) current = address_to_count.setdefault(address_hex, 0) address_to_count[address_hex] = current + 1 block = db.get_root_block_by_hash(block.header.hash_prev_block, False) algo_to_address_count = dict() # algorithm -> (recipient -> count) for shard_id, address_to_count in shard_to_address_count.items(): algo = shard_id_to_algorithm(shard_id) addr_to_count = algo_to_address_count.setdefault(algo, dict()) for address, count in address_to_count.items(): current = addr_to_count.setdefault(address, 0) addr_to_count[address] = current + count print( "Counting shard blocks from root block {} {}".format( tip_header.height, tip_header.get_hash().hex() ) ) for algo, address_count in algo_to_address_count.items(): total = sum(address_count.values()) print() print("{} has {} blocks".format(algo, total)) sorted_by_count = sorted( address_count.items(), key=operator.itemgetter(1), reverse=True ) for address, count in sorted_by_count: print("{} {} {:.2f}%".format(address, count, count / total * 100))
def get_test_env( genesis_account=Address.create_empty_account(), genesis_minor_quarkash=0, shard_size=2, genesis_root_heights=None, remote_mining=False, ): env = DEFAULT_ENV.copy() env.db = InMemoryDb() env.set_network_id(1234567890) env.cluster_config = ClusterConfig() env.quark_chain_config.update(shard_size, 10, 1) if remote_mining: env.quark_chain_config.ROOT.CONSENSUS_CONFIG.REMOTE_MINE = True env.quark_chain_config.ROOT.CONSENSUS_TYPE = ConsensusType.POW_SHA3SHA3 env.quark_chain_config.ROOT.GENESIS.DIFFICULTY = 10 env.quark_chain_config.ROOT.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME = 40 env.quark_chain_config.ROOT.DIFFICULTY_ADJUSTMENT_FACTOR = 1024 if genesis_root_heights: check(len(genesis_root_heights) == shard_size) for shard_id in range(shard_size): shard = env.quark_chain_config.SHARD_LIST[shard_id] shard.GENESIS.ROOT_HEIGHT = genesis_root_heights[shard_id] # fund genesis account in all shards for i, shard in enumerate(env.quark_chain_config.SHARD_LIST): addr = genesis_account.address_in_shard(i).serialize().hex() shard.GENESIS.ALLOC[addr] = genesis_minor_quarkash shard.CONSENSUS_CONFIG.REMOTE_MINE = remote_mining shard.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME = 7 shard.DIFFICULTY_ADJUSTMENT_FACTOR = 512 if remote_mining: shard.CONSENSUS_TYPE = ConsensusType.POW_SHA3SHA3 shard.GENESIS.DIFFICULTY = 10 env.quark_chain_config.SKIP_MINOR_DIFFICULTY_CHECK = True env.quark_chain_config.SKIP_ROOT_DIFFICULTY_CHECK = True env.cluster_config.ENABLE_TRANSACTION_HISTORY = True env.cluster_config.DB_PATH_ROOT = "" check(env.cluster_config.use_mem_db()) return env
def main(): parser = argparse.ArgumentParser() parser.add_argument( "--config", required=True, type=str, help= "<Required> path to config json file, same as the config running cluster", ) parser.add_argument( "-s", "--shards", required=True, nargs="+", help= '<Required> specify shards (identified by full_shard_key) to mine, use "R" to indicate root chain', ) parser.add_argument("--worker", type=int, help="number of worker threads", default=1) parser.add_argument("--host", type=str, help="host address of the cluster", default="localhost") args = parser.parse_args() with open(args.config) as f: cluster_config = ClusterConfig.from_json(f.read()) qkc_config = cluster_config.QUARKCHAIN global cluster_host if args.host: cluster_host = args.host # 1 worker config <-> 1 mining thread <-> 1 or more shards worker_configs = [[] for _ in range(min(args.worker, len(args.shards))) ] # type: List[List[Dict]] for worker_i, shard_str in zip(cycle(range(args.worker)), args.shards): if shard_str.isnumeric(): full_shard_key = int(shard_str) full_shard_id = qkc_config.get_full_shard_id_by_full_shard_key( full_shard_key) c = qkc_config.shards[full_shard_id] else: full_shard_id = None c = qkc_config.ROOT worker_configs[worker_i].append({ "full_shard_id": full_shard_id, "consensus_type": c.CONSENSUS_TYPE, "target_block_time": c.CONSENSUS_CONFIG.TARGET_BLOCK_TIME, }) miners = [] stopper = threading.Event() for config_list in worker_configs: ext_miner = ExternalMiner(config_list, stopper) ext_miner.start() miners.append(ext_miner) sig_handler = SigHandler(stopper, miners) signal.signal(signal.SIGINT, sig_handler)