def test_cluster_dict(self):
parser = argparse.ArgumentParser()
ClusterConfig.attach_arguments(parser)
args = parser.parse_args(["--num_shards=4", "--genesis_dir="])
cluster_config = ClusterConfig.create_from_args(args)
args = parser.parse_args(["--cluster_config=" + cluster_config.json_filepath])
deserialized = ClusterConfig.create_from_args(args)
self.assertTrue(cluster_config == deserialized)
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--full_shard_id",
type=int,
help="full shard id to operate")
parser.add_argument(
"--all_shards",
action="store_true",
default=False,
help="query balances in all shards",
)
parser.add_argument("--recipient",
default=None,
type=str,
help="query a specific recipient")
parser.add_argument(
"--minor_block_height",
default=None,
type=int,
help="query balance at specific minor block height",
)
ClusterConfig.attach_arguments(parser)
args = parser.parse_args()
env = DEFAULT_ENV.copy()
env.cluster_config = ClusterConfig.create_from_args(args)
# initialize database
if not env.cluster_config.use_mem_db():
env.db = PersistentDb(
"{path}/master.db".format(path=env.cluster_config.DB_PATH_ROOT),
clean=env.cluster_config.CLEAN,
)
return env, args
def test_kafka_log(self):
parser = argparse.ArgumentParser()
ClusterConfig.attach_arguments(parser)
args = parser.parse_args(
[]) # set --kafka_rest_address correctly to see real actions
cluster_config = ClusterConfig.create_from_args(args)
sample = dict(a=1, b=2, c=["x", "y"])
cluster_config.kafka_logger.log_kafka_sample("dlltest", sample)
def test_unknown_structure(self):
parser = argparse.ArgumentParser()
ClusterConfig.attach_arguments(parser)
args = parser.parse_args(["--monitoring_kafka_rest_address=x"])
cluster_config = ClusterConfig.create_from_args(args)
sample = dict(a=1, b=2, c={"x", "y"})
cluster_config.kafka_logger.log_kafka_sample(
"topic", sample) # should trigger warning log
async def main():
parser = argparse.ArgumentParser()
ClusterConfig.attach_arguments(parser)
parser.add_argument("--num_clusters", default=2, type=int)
args = parser.parse_args()
clusters = []
mine_i = random.randint(0, args.num_clusters - 1)
mine = args.start_simulated_mining
if mine:
print("cluster {} will be mining".format(mine_i))
else:
print("No one will be mining")
db_path_root = args.db_path_root
p2p_port = args.p2p_port
for i in range(args.num_clusters):
args.start_simulated_mining = mine and i == mine_i
args.db_path_root = "{}_C{}".format(db_path_root, i)
# set up p2p bootstrapping, with fixed bootstrap key for now
if args.p2p:
if i == 0:
args.privkey = (
"31552f186bf90908ce386fb547dd0410bf443309125cc43fd0ffd642959bf6d9"
)
else:
args.privkey = ""
args.bootnodes = "enode://c571e0db93d17cc405cb57640826b70588a6a28785f38b21be471c609ca12fcb06cb306ac44872908f5bed99046031a5af82072d484e3ef9029560c1707193a0@127.0.0.1:{}".format(
p2p_port)
config = ClusterConfig.create_from_args(args)
print("Cluster {} config file: {}".format(i, config.json_filepath))
print(config.to_json())
clusters.append(
cl.Cluster(
config, "{}C{}{}_".format(colors[i % len(colors)], i,
COLOR_END)))
args.p2p_port += 1
args.port_start += 100
args.json_rpc_port += 1
args.json_rpc_private_port += 1
tasks = list()
tasks.append(asyncio.ensure_future(clusters[0].run()))
await asyncio.sleep(3)
for cluster in clusters[1:]:
tasks.append(asyncio.ensure_future(cluster.run()))
try:
await asyncio.gather(*tasks)
except KeyboardInterrupt:
try:
for cluster in clusters:
asyncio.get_event_loop().run_until_complete(cluster.shutdown())
except Exception:
pass
def parse_args():
parser = argparse.ArgumentParser()
ClusterConfig.attach_arguments(parser)
# Unique Id identifying the node in the cluster
parser.add_argument("--node_id", default="", type=str)
args = parser.parse_args()
env = DEFAULT_ENV.copy()
env.cluster_config = ClusterConfig.create_from_args(args)
env.slave_config = env.cluster_config.get_slave_config(args.node_id)
return env
def test_cluster_dict_wloadtest(self):
"""convert to dict and back to check if the content changed, requires `__eq__`
removing --loadtest will make the test faster
passing more num_shards will increase runtime linearly
"""
parser = argparse.ArgumentParser()
ClusterConfig.attach_arguments(parser)
pwd = os.path.dirname(os.path.abspath(__file__))
default_genesis_dir = os.path.join(pwd, "../../genesis_data")
args = parser.parse_args(
["--num_shards=4", "--genesis_dir=" + default_genesis_dir]
)
cluster_config = ClusterConfig.create_from_args(args)
args = parser.parse_args(["--cluster_config=" + cluster_config.json_filepath])
deserialized = ClusterConfig.create_from_args(args)
self.assertTrue(cluster_config == deserialized)
self.assertTrue(
len(cluster_config.QUARKCHAIN.SHARD_LIST[0].GENESIS.ALLOC) > 12000
)
def main():
os.chdir(os.path.dirname(os.path.abspath(__file__)))
parser = argparse.ArgumentParser()
ClusterConfig.attach_arguments(parser)
args = parser.parse_args()
config = ClusterConfig.create_from_args(args)
print("Cluster config file: {}".format(config.json_filepath))
print(config.to_json())
cluster = Cluster(config)
cluster.start_and_loop()
def parse_args():
parser = argparse.ArgumentParser()
ClusterConfig.attach_arguments(parser)
parser.add_argument("--profile", default="", type=str)
parser.add_argument("--check_db_rblock_from", default=-1, type=int)
parser.add_argument("--check_db_rblock_to", default=0, type=int)
parser.add_argument("--check_db_rblock_batch", default=1, type=int)
args = parser.parse_args()
env = DEFAULT_ENV.copy()
env.cluster_config = ClusterConfig.create_from_args(args)
env.arguments = args
return env
def main():
args = parse_args()
config = ClusterConfig.create_from_args(args)
db = RootDb(PersistentDb(args.db), config.QUARKCHAIN, 0)
header = db.get_tip_header()
if not header:
raise RuntimeError("Not a valid RootDb")
from_height = header.height if args.root_height <= 0 else args.root_height
tip_header = None
block = db.get_root_block_by_hash(header.get_hash(), False)
shard_to_address_count = dict() # shard -> (recipient -> count)
while block.header.height > 0:
if block.header.height > from_height:
block = db.get_root_block_by_hash(block.header.hash_prev_block, False)
continue
if block.header.height == from_height:
tip_header = block.header
for minor_header in block.minor_block_header_list:
shard = minor_header.branch.get_full_shard_id()
address_hex = minor_header.coinbase_address.recipient.hex()
address_to_count = shard_to_address_count.setdefault(shard, dict())
current = address_to_count.setdefault(address_hex, 0)
address_to_count[address_hex] = current + 1
block = db.get_root_block_by_hash(block.header.hash_prev_block, False)
algo_to_address_count = dict() # algorithm -> (recipient -> count)
for shard_id, address_to_count in shard_to_address_count.items():
algo = shard_id_to_algorithm(shard_id)
addr_to_count = algo_to_address_count.setdefault(algo, dict())
for address, count in address_to_count.items():
current = addr_to_count.setdefault(address, 0)
addr_to_count[address] = current + count
print(
"Counting shard blocks from root block {} {}".format(
tip_header.height, tip_header.get_hash().hex()
)
)
for algo, address_count in algo_to_address_count.items():
total = sum(address_count.values())
print()
print("{} has {} blocks".format(algo, total))
sorted_by_count = sorted(
address_count.items(), key=operator.itemgetter(1), reverse=True
)
for address, count in sorted_by_count:
print("{} {} {:.2f}%".format(address, count, count / total * 100))
def test_cluster_slave_config_legacy(self):
parser = argparse.ArgumentParser()
ClusterConfig.attach_arguments(parser)
args = parser.parse_args(
["--num_chains=8", "--num_shards_per_chain=1", "--num_slaves=4"])
cluster_config = ClusterConfig.create_from_args(args)
# remove full shard list but use chain mask
d = cluster_config.to_dict()
for i, s in enumerate(d["SLAVE_LIST"]):
s["CHAIN_MASK_LIST"] = [i + 4]
del (s["FULL_SHARD_ID_LIST"])
deserialized = ClusterConfig.from_dict(d)
# chain mask translated config should equal previous full shard ID config
self.assertTrue(cluster_config == deserialized)
async def main():
parser = argparse.ArgumentParser()
ClusterConfig.attach_arguments(parser)
parser.add_argument("--num_clusters", default=2, type=int)
args = parser.parse_args()
clusters = []
mine_i = random.randint(0, args.num_clusters - 1)
if args.mine:
print("cluster {} will be mining".format(mine_i))
else:
print("No one will be mining")
mine = args.mine
db_path_root = args.db_path_root
for i in range(args.num_clusters):
args.mine = mine and i == mine_i
args.db_path_root = "{}_C{}".format(db_path_root, i)
config = ClusterConfig.create_from_args(args)
print("Cluster {} config file: {}".format(i, config.json_filepath))
print(config.to_json())
clusters.append(
cl.Cluster(
config, "{}C{}{}_".format(colors[i % len(colors)], i,
COLOR_END)))
args.p2p_port += 1
args.port_start += 100
args.json_rpc_port += 1
args.json_rpc_private_port += 1
args.devp2p_port += 1
tasks = list()
tasks.append(asyncio.ensure_future(clusters[0].run()))
await asyncio.sleep(3)
for cluster in clusters[1:]:
tasks.append(asyncio.ensure_future(cluster.run()))
try:
await asyncio.gather(*tasks)
except KeyboardInterrupt:
try:
for cluster in clusters:
asyncio.get_event_loop().run_until_complete(cluster.shutdown())
except Exception:
pass
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("action", type=str, help="action to take")
parser.add_argument("--height", type=int, help="block height to operate")
parser.add_argument("--hash", type=str, help="block hash to operate")
ClusterConfig.attach_arguments(parser)
args = parser.parse_args()
env = DEFAULT_ENV.copy()
env.cluster_config = ClusterConfig.create_from_args(args)
# initialize database
if not env.cluster_config.use_mem_db():
env.db = PersistentDb(
"{path}/master.db".format(path=env.cluster_config.DB_PATH_ROOT),
clean=env.cluster_config.CLEAN,
)
return env, args
def main():
logging.getLogger("asyncio").setLevel(logging.ERROR)
os.chdir(os.path.dirname(os.path.abspath(__file__)))
parser = argparse.ArgumentParser()
ClusterConfig.attach_arguments(parser)
parser.add_argument("--profile", default="", type=str)
parser.add_argument("--check_db_rblock_from", default=-1, type=int)
parser.add_argument("--check_db_rblock_to", default=0, type=int)
parser.add_argument("--check_db_rblock_batch", default=1, type=int)
args = parser.parse_args()
config = ClusterConfig.create_from_args(args)
print("Cluster config file: {}".format(config.json_filepath))
print(config.to_json())
cluster = Cluster(config, args=args)
if args.check_db:
cluster.check_db()
else:
cluster.start_and_loop()
def main():
if "EMAIL_FROM_ADDRESS" not in os.environ:
raise ValueError(
"EMAIL_FROM_ADDRESS not found in environment variables")
if "PASSWORD" not in os.environ:
raise ValueError("PASSWORD not found in environment variables")
if "EMAIL_TO_ADDRESS" not in os.environ:
raise ValueError("EMAIL_TO_ADDRESS not found in environment variables")
os.chdir(os.path.dirname("../../quarkchain/cluster/"))
parser = argparse.ArgumentParser()
ClusterConfig.attach_arguments(parser)
args = parser.parse_args()
config = ClusterConfig.create_from_args(args)
# creat a config tempfile for health check, which is a copy of the original config file
tempfile.tempdir = "../../mainnet/singularity/"
fd, config.json_filepath = tempfile.mkstemp()
with os.fdopen(fd, "w") as tmp:
tmp.write(config.to_json())
print("Cluster config file: {}".format(config.json_filepath))
print(config.to_json())
cluster = HealthCheckCluster(config)
bootstrap_nodes = config.P2P.BOOT_NODES.split(",")
count = 0
while True:
bash_command_revised_config = (
"QKC_CONFIG=" + config.json_filepath +
" python3 ../../quarkchain/tools/config_p2p.py --bootnodes " +
bootstrap_nodes[count])
os.system(bash_command_revised_config)
print("Start Bootstrap With " + bootstrap_nodes[count])
cluster.start_and_loop()
time.sleep(100)
count = (count + 1) % len(bootstrap_nodes)
