def from_dict(cls, d):
config = super().from_dict(d)
config.ROOT = RootConfig.from_dict(config.ROOT)
chains = []
shards = dict()
for s in config.CHAINS:
chain_config = ChainConfig.from_dict(s)
chains.append(chain_config)
for shard_id in range(chain_config.SHARD_SIZE):
shard_config = ShardConfig(chain_config)
shard_config.root_config = config.ROOT
shard_config.SHARD_ID = shard_id
shard_config.COINBASE_ADDRESS = (Address.create_from(
shard_config.COINBASE_ADDRESS).address_in_shard(
shard_config.get_full_shard_id()).serialize().hex())
# filter alloc addresses based on shard id
alloc = dict()
for address_hex, balances in shard_config.GENESIS.ALLOC.items(
):
address = Address.create_from(bytes.fromhex(address_hex))
address_shard_id = address.full_shard_key & (
chain_config.SHARD_SIZE - 1)
if shard_id == address_shard_id:
alloc[address_hex] = balances
shard_config.GENESIS.ALLOC = alloc
shards[shard_config.get_full_shard_id()] = shard_config
config.CHAINS = chains
config.shards = shards
config.init_and_validate()
return config
def update_genesis_alloc(cluser_config):
""" Update ShardConfig.GENESIS.ALLOC """
ALLOC_FILE_TEMPLATE = "alloc/{}.json"
LOADTEST_FILE = "loadtest.json"
if not cluser_config.GENESIS_DIR:
return
alloc_file_template = os.path.join(cluser_config.GENESIS_DIR,
ALLOC_FILE_TEMPLATE)
loadtest_file = os.path.join(cluser_config.GENESIS_DIR, LOADTEST_FILE)
qkc_config = cluser_config.QUARKCHAIN
allocation = {
qkc_config.GENESIS_TOKEN: 1000000 * (10**18),
"QETC": 2 * (10**8) * (10**18),
"QFB": 3 * (10**8) * (10**18),
"QAAPL": 4 * (10**8) * (10**18),
"QTSLA": 5 * (10**8) * (10**18),
}
old_shards = copy.deepcopy(qkc_config.shards)
try:
for chain_id in range(qkc_config.CHAIN_SIZE):
alloc_file = alloc_file_template.format(chain_id)
with open(alloc_file, "r") as f:
items = json.load(f)
for item in items:
address = Address.create_from(item["address"])
full_shard_id = qkc_config.get_full_shard_id_by_full_shard_key(
address.full_shard_key)
qkc_config.shards[full_shard_id].GENESIS.ALLOC[
item["address"]] = allocation
Logger.info(
"[{}] Imported {} genesis accounts into config from {}".format(
chain_id, len(items), alloc_file))
except Exception as e:
Logger.warning("Error importing genesis accounts from {}: {}".format(
alloc_file, e))
qkc_config.shards = old_shards
Logger.warning("Cleared all partially imported genesis accounts!")
# each account in loadtest file is funded on all the shards
try:
with open(loadtest_file, "r") as f:
items = json.load(f)
qkc_config.loadtest_accounts = items
for item in items:
address = Address.create_from(item["address"])
for full_shard_id, shard_config in qkc_config.shards.items():
shard_config.GENESIS.ALLOC[address.address_in_shard(
full_shard_id).serialize().hex()] = allocation
Logger.info("Imported {} loadtest accounts from {}".format(
len(items), loadtest_file))
except Exception:
Logger.info("No loadtest accounts imported into genesis alloc")
def test_getWork_and_submitWork(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_key=0)
with ClusterContext(
1, acc1, remote_mining=True, shard_size=1,
small_coinbase=True) as clusters, jrpc_server_context(
clusters[0].master):
master = clusters[0].master
slaves = clusters[0].slave_list
tx = create_transfer_transaction(
shard_state=clusters[0].get_shard_state(1 | 0),
key=id1.get_key(),
from_address=acc1,
to_address=acc1,
value=0,
gas_price=12,
)
self.assertTrue(slaves[0].add_tx(tx))
for shard_id in ["0x0", None]: # shard, then root
resp = send_request("getWork", [shard_id])
self.assertEqual(resp[1:], ["0x1", "0xa"]) # height and diff
header_hash_hex = resp[0]
if shard_id is not None: # shard 0
miner_address = Address.create_from(
master.env.quark_chain_config.shards[1].
COINBASE_ADDRESS)
else: # root
miner_address = Address.create_from(
master.env.quark_chain_config.ROOT.COINBASE_ADDRESS)
block = call_async(
master.get_next_block_to_mine(address=miner_address,
branch_value=shard_id
and 0b01))
# solve it and submit
work = MiningWork(bytes.fromhex(header_hash_hex[2:]), 1, 10)
solver = DoubleSHA256(work)
nonce = solver.mine(0, 10000).nonce
mixhash = "0x" + sha3_256(b"").hex()
resp = send_request(
"submitWork",
[
shard_id,
header_hash_hex,
hex(nonce),
mixhash,
"0x" + bytes(65).hex(),
],
)
self.assertTrue(resp)
# show progress on shard 0
self.assertEqual(
clusters[0].get_shard_state(1 | 0).get_tip().header.height, 1)
def update_genesis_alloc(cluser_config):
""" Update ShardConfig.GENESIS.ALLOC """
ALLOC_FILE = "alloc.json"
LOADTEST_FILE = "loadtest.json"
if not cluser_config.GENESIS_DIR:
return
alloc_file = os.path.join(cluser_config.GENESIS_DIR, ALLOC_FILE)
loadtest_file = os.path.join(cluser_config.GENESIS_DIR, LOADTEST_FILE)
qkc_config = cluser_config.QUARKCHAIN
# each account in alloc_file is only funded on the shard it belongs to
try:
with open(alloc_file, "r") as f:
items = json.load(f)
# address_hex -> key_hex for jsonrpc faucet drip
qkc_config.alloc_accounts = dict()
for item in items:
qkc_config.alloc_accounts[item["address"]] = item["key"]
address = Address.create_from(item["address"])
shard = address.get_shard_id(qkc_config.SHARD_SIZE)
qkc_config.SHARD_LIST[shard].GENESIS.ALLOC[item["address"]] = 1000000 * (
10 ** 18
)
Logger.info(
"Imported {} accounts from genesis alloc at {}".format(
len(items), alloc_file
)
)
except Exception as e:
Logger.warning("Unable to load genesis alloc from {}: {}".format(alloc_file, e))
# each account in loadtest file is funded on all the shards
try:
with open(loadtest_file, "r") as f:
items = json.load(f)
qkc_config.loadtest_accounts = items
for item in items:
address = Address.create_from(item["address"])
for i, shard in enumerate(qkc_config.SHARD_LIST):
shard.GENESIS.ALLOC[
address.address_in_shard(i).serialize().hex()
] = 1000 * (10 ** 18)
Logger.info(
"Imported {} loadtest accounts from {}".format(len(items), loadtest_file)
)
except Exception:
Logger.info("No loadtest accounts imported into genesis alloc")
def _convert_eth_call_data(data, shard):
to_address = Address.create_from(
eth_address_to_quarkchain_address_decoder(data["to"]))
if shard:
to_address = Address(to_address.recipient, shard)
data["to"] = "0x" + to_address.serialize().hex()
if "from" in data:
from_address = Address.create_from(
eth_address_to_quarkchain_address_decoder(data["from"]))
if shard:
from_address = Address(from_address.recipient, shard)
data["from"] = "0x" + from_address.serialize().hex()
return data
def update(self, chain_size, shard_size_per_chain, root_block_time,
minor_block_time):
self.CHAIN_SIZE = chain_size
self.ROOT = RootConfig()
self.ROOT.CONSENSUS_TYPE = ConsensusType.POW_SIMULATE
self.ROOT.CONSENSUS_CONFIG = POWConfig()
self.ROOT.CONSENSUS_CONFIG.TARGET_BLOCK_TIME = root_block_time
self.CHAINS = []
self.shards = dict()
for chain_id in range(chain_size):
chain_config = ChainConfig()
chain_config.CHAIN_ID = chain_id
chain_config.SHARD_SIZE = shard_size_per_chain
chain_config.CONSENSUS_TYPE = ConsensusType.POW_SIMULATE
chain_config.CONSENSUS_CONFIG = POWConfig()
chain_config.CONSENSUS_CONFIG.TARGET_BLOCK_TIME = minor_block_time
self.CHAINS.append(chain_config)
for shard_id in range(shard_size_per_chain):
shard_config = ShardConfig(chain_config)
shard_config.root_config = self.ROOT
shard_config.SHARD_ID = shard_id
shard_config.COINBASE_ADDRESS = (Address.create_from(
shard_config.COINBASE_ADDRESS).address_in_shard(
shard_config.get_full_shard_id()).serialize().hex())
self.shards[shard_config.get_full_shard_id()] = shard_config
self.init_and_validate()
async def getTransactionsByAddress(self, address, start="0x", limit="0xa"):
""" "start" should be the "next" in the response for fetching next page.
"start" can also be "0x" to fetch from the beginning (i.e., latest).
"start" can be "0x00" to fetch the pending outgoing transactions.
"""
address = Address.create_from(address)
if limit > 20:
limit = 20
result = await self.master.get_transactions_by_address(
address, start, limit)
if not result:
return None
tx_list, next = result
txs = []
for tx in tx_list:
txs.append({
"txId":
id_encoder(tx.tx_hash, tx.from_address.full_shard_id),
"fromAddress":
address_encoder(tx.from_address.serialize()),
"toAddress":
address_encoder(tx.to_address.serialize())
if tx.to_address else "0x",
"value":
quantity_encoder(tx.value),
"blockHeight":
quantity_encoder(tx.block_height),
"timestamp":
quantity_encoder(tx.timestamp),
"success":
tx.success,
})
return {"txList": txs, "next": data_encoder(next)}
def create_minor_block(self, root_block: RootBlock, full_shard_id: int,
evm_state: EvmState) -> MinorBlock:
""" Create genesis block for shard.
Genesis block's hash_prev_root_block is set to the genesis root block.
Genesis state will be committed to the given evm_state.
Based on ALLOC, genesis_token will be added to initial accounts.
"""
branch = Branch(full_shard_id)
shard_config = self._qkc_config.shards[full_shard_id]
genesis = shard_config.GENESIS
for address_hex, alloc_amount in genesis.ALLOC.items():
address = Address.create_from(bytes.fromhex(address_hex))
check(
self._qkc_config.get_full_shard_id_by_full_shard_key(
address.full_shard_key) == full_shard_id)
evm_state.full_shard_key = address.full_shard_key
if isinstance(alloc_amount, dict):
for k, v in alloc_amount.items():
evm_state.delta_token_balance(address.recipient,
token_id_encode(k), v)
else:
evm_state.delta_token_balance(address.recipient,
self._qkc_config.genesis_token,
alloc_amount)
evm_state.commit()
meta = MinorBlockMeta(
hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT),
hash_evm_state_root=evm_state.trie.root_hash,
)
local_fee_rate = 1 - self._qkc_config.reward_tax_rate # type: Fraction
coinbase_tokens = {
self._qkc_config.genesis_token:
shard_config.COINBASE_AMOUNT * local_fee_rate.numerator //
local_fee_rate.denominator
}
coinbase_address = Address.create_empty_account(full_shard_id)
header = MinorBlockHeader(
version=genesis.VERSION,
height=genesis.HEIGHT,
branch=branch,
hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK),
hash_prev_root_block=root_block.header.get_hash(),
evm_gas_limit=genesis.GAS_LIMIT,
hash_meta=sha3_256(meta.serialize()),
coinbase_amount_map=TokenBalanceMap(coinbase_tokens),
coinbase_address=coinbase_address,
create_time=genesis.TIMESTAMP,
difficulty=genesis.DIFFICULTY,
extra_data=bytes.fromhex(genesis.EXTRA_DATA),
)
return (
MinorBlock(header=header, meta=meta, tx_list=[]),
TokenBalanceMap(coinbase_tokens),
)
async def _call_or_estimate_gas(self, is_call: bool, **data):
""" Returns the result of the transaction application without putting in block chain """
if not isinstance(data, dict):
raise InvalidParams("Transaction must be an object")
def get_data_default(key, decoder, default=None):
if key in data:
return decoder(data[key])
return default
to = get_data_default("to", address_decoder, None)
if to is None:
raise InvalidParams("Missing to")
to_full_shard_key = int.from_bytes(to[20:], "big")
gas = get_data_default("gas", quantity_decoder, 0)
gas_price = get_data_default("gasPrice", quantity_decoder, 0)
value = get_data_default("value", quantity_decoder, 0)
data_ = get_data_default("data", data_decoder, b"")
sender = get_data_default("from", address_decoder,
b"\x00" * 20 + to[20:])
sender_address = Address.create_from(sender)
gas_token_id = get_data_default(
"gas_token_id", quantity_decoder,
self.env.quark_chain_config.genesis_token)
transfer_token_id = get_data_default(
"transfer_token_id",
quantity_decoder,
self.env.quark_chain_config.genesis_token,
)
network_id = self.master.env.quark_chain_config.NETWORK_ID
nonce = 0 # slave will fill in the real nonce
evm_tx = EvmTransaction(
nonce,
gas_price,
gas,
to[:20],
value,
data_,
from_full_shard_key=sender_address.full_shard_key,
to_full_shard_key=to_full_shard_key,
network_id=network_id,
gas_token_id=gas_token_id,
transfer_token_id=transfer_token_id,
)
tx = TypedTransaction(SerializedEvmTransaction.from_evm_tx(evm_tx))
if is_call:
res = await self.master.execute_transaction(
tx, sender_address, data["block_height"])
return data_encoder(res) if res is not None else None
else: # estimate gas
res = await self.master.estimate_gas(tx, sender_address)
return quantity_encoder(res) if res is not None else None
def create_minor_block(
self, root_block: RootBlock, shard_id: int, evm_state: EvmState
) -> MinorBlock:
""" Create genesis block for shard.
Genesis block's hash_prev_root_block is set to the genesis root block.
Genesis state will be committed to the given evm_state.
"""
branch = Branch.create(self._qkc_config.SHARD_SIZE, shard_id)
shard_config = self._qkc_config.SHARD_LIST[shard_id]
genesis = shard_config.GENESIS
coinbase_address = Address.create_from(
bytes.fromhex(shard_config.COINBASE_ADDRESS)
)
check(coinbase_address.get_shard_id(self._qkc_config.SHARD_SIZE) == shard_id)
for address_hex, amount_in_wei in genesis.ALLOC.items():
address = Address.create_from(bytes.fromhex(address_hex))
check(address.get_shard_id(self._qkc_config.SHARD_SIZE) == shard_id)
evm_state.full_shard_id = address.full_shard_id
evm_state.delta_balance(address.recipient, amount_in_wei)
evm_state.commit()
meta = MinorBlockMeta(
hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT),
hash_evm_state_root=evm_state.trie.root_hash,
)
header = MinorBlockHeader(
version=genesis.VERSION,
height=genesis.HEIGHT,
branch=branch,
hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK),
hash_prev_root_block=root_block.header.get_hash(),
evm_gas_limit=genesis.GAS_LIMIT,
hash_meta=sha3_256(meta.serialize()),
coinbase_address=coinbase_address,
coinbase_amount=shard_config.COINBASE_AMOUNT,
create_time=genesis.TIMESTAMP,
difficulty=genesis.DIFFICULTY,
extra_data=bytes.fromhex(genesis.EXTRA_DATA),
)
return MinorBlock(header=header, meta=meta, tx_list=[])
def __init__(self, qkc_config: QuarkChainConfig, shard):
self.qkc_config = qkc_config
self.full_shard_id = shard.full_shard_id
self.shard = shard
self.running = False
self.accounts = []
for item in qkc_config.loadtest_accounts:
account = Account(Address.create_from(item["address"]),
bytes.fromhex(item["key"]))
self.accounts.append(account)
async def fund(endpoint, genesisId, data):
network_id = await endpoint.get_network_id()
shard_size = await endpoint.get_shard_size()
futures = []
for e in GAME_ADDRESSES:
shard = e[0]
to = Address.create_from(e[1][2:])
futures.append(
fund_shard(endpoint, genesisId, to, data, network_id, shard))
results = await asyncio.gather(*futures)
print("\n\n")
for shard, result in enumerate(results):
tx_id, height = result
print('[{}, "{}"], // {}'.format(shard, height, tx_id))
def from_dict(cls, d):
config = super().from_dict(d)
config.ROOT = RootConfig.from_dict(config.ROOT)
chains = []
shards = dict()
for s in config.CHAINS:
chain_config = ChainConfig.from_dict(s)
chains.append(chain_config)
for shard_id in range(chain_config.SHARD_SIZE):
shard_config = ShardConfig(chain_config)
shard_config.root_config = config.ROOT
shard_config.SHARD_ID = shard_id
shard_config.COINBASE_ADDRESS = (Address.create_from(
shard_config.COINBASE_ADDRESS).address_in_shard(
shard_config.get_full_shard_id()).serialize().hex())
shards[shard_config.get_full_shard_id()] = shard_config
config.CHAINS = chains
config.shards = shards
config.init_and_validate()
return config
async def getTransactionsByAddress(
self, address, start="0x", limit="0xa", transfer_token_id=None
):
""" "start" should be the "next" in the response for fetching next page.
"start" can also be "0x" to fetch from the beginning (i.e., latest).
"start" can be "0x00" to fetch the pending outgoing transactions.
"""
address = Address.create_from(address)
if limit > 20:
limit = 20
result = await self.master.get_transactions_by_address(
address, transfer_token_id, start, limit
)
if not result:
return None
tx_list, next = result
return {
"txList": [tx_detail_encoder(tx) for tx in tx_list],
"next": data_encoder(next),
}
async def fund(endpoint, genesisId, addrByAmount):
network_id = await endpoint.get_network_id()
shard_size = await endpoint.get_shard_size()
for amount in addrByAmount:
addrs = addrByAmount.get(amount, [])
print("======\nstart for amount {} with {} address\n======".format(
amount, len(addrs)))
# shard -> [addr]
byShard = defaultdict(list)
for addr in addrs:
shard = int(addr[-8:], 16) & (shard_size - 1)
byShard[shard].append(addr)
while True:
toFund = []
for addrs in byShard.values():
if addrs:
toFund.append(addrs.pop())
if not toFund:
break
futures = []
for addr in toFund:
shard = int(addr[-8:], 16) & (shard_size - 1)
try:
# sorry but this is user input
to = Address.create_from(addr[2:])
except:
print("addr format invalid {}".format(addr))
continue
await asyncio.sleep(0.1) # slight delay for each call
futures.append(
fund_shard(endpoint, genesisId, to, network_id, shard,
amount))
results = await asyncio.gather(*futures)
print("\n\n")
for idx, result in enumerate(results):
tx_id, height = result
print('[{}, "{}"], // {}'.format(idx, height, tx_id))
def __init_miner(self):
miner_address = Address.create_from(
self.env.quark_chain_config.SHARD_LIST[
self.shard_id].COINBASE_ADDRESS)
async def __create_block(retry=True):
# hold off mining if the shard is syncing
while self.synchronizer.running or not self.state.initialized:
if not retry:
break
await asyncio.sleep(0.1)
return self.state.create_block_to_mine(address=miner_address)
async def __add_block(block):
# Do not add block if there is a sync in progress
if self.synchronizer.running:
return
# Do not add stale block
if self.state.header_tip.height >= block.header.height:
return
await self.handle_new_block(block)
def __get_mining_param():
return {
"target_block_time":
self.slave.artificial_tx_config.target_minor_block_time
}
shard_config = self.env.quark_chain_config.SHARD_LIST[
self.shard_id] # type: ShardConfig
self.miner = Miner(
shard_config.CONSENSUS_TYPE,
__create_block,
__add_block,
__get_mining_param,
remote=shard_config.CONSENSUS_CONFIG.REMOTE_MINE,
)
def testnet_master_address(self):
return Address.create_from(self.TESTNET_MASTER_ADDRESS)
def create_minor_block(
self, root_block: RootBlock, full_shard_id: int,
evm_state: EvmState) -> Tuple[MinorBlock, TokenBalanceMap]:
""" Create genesis block for shard.
Genesis block's hash_prev_root_block is set to the genesis root block.
Genesis state will be committed to the given evm_state.
Based on ALLOC, genesis_token will be added to initial accounts.
"""
branch = Branch(full_shard_id)
shard_config = self._qkc_config.shards[full_shard_id]
genesis = shard_config.GENESIS
for address_hex, alloc_data in genesis.ALLOC.items():
address = Address.create_from(bytes.fromhex(address_hex))
check(
self._qkc_config.get_full_shard_id_by_full_shard_key(
address.full_shard_key) == full_shard_id)
evm_state.full_shard_key = address.full_shard_key
recipient = address.recipient
if "code" in alloc_data:
code = decode_hex(remove_0x_head(alloc_data["code"]))
evm_state.set_code(recipient, code)
evm_state.set_nonce(recipient, 1)
if "storage" in alloc_data:
for k, v in alloc_data["storage"].items():
evm_state.set_storage_data(
recipient,
big_endian_to_int(decode_hex(k[2:])),
big_endian_to_int(decode_hex(v[2:])),
)
# backward compatible:
# v1: {addr: {QKC: 1234}}
# v2: {addr: {balances: {QKC: 1234}, code: 0x, storage: {0x12: 0x34}}}
balances = alloc_data
if "balances" in alloc_data:
balances = alloc_data["balances"]
for k, v in balances.items():
if k in ("code", "storage"):
continue
evm_state.delta_token_balance(recipient, token_id_encode(k), v)
evm_state.commit()
meta = MinorBlockMeta(
hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT),
hash_evm_state_root=evm_state.trie.root_hash,
xshard_tx_cursor_info=XshardTxCursorInfo(root_block.header.height,
0, 0),
)
local_fee_rate = 1 - self._qkc_config.reward_tax_rate # type: Fraction
coinbase_tokens = {
self._qkc_config.genesis_token:
shard_config.COINBASE_AMOUNT * local_fee_rate.numerator //
local_fee_rate.denominator
}
coinbase_address = Address.create_empty_account(full_shard_id)
header = MinorBlockHeader(
version=genesis.VERSION,
height=genesis.HEIGHT,
branch=branch,
hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK),
hash_prev_root_block=root_block.header.get_hash(),
evm_gas_limit=genesis.GAS_LIMIT,
hash_meta=sha3_256(meta.serialize()),
coinbase_amount_map=TokenBalanceMap(coinbase_tokens),
coinbase_address=coinbase_address,
create_time=genesis.TIMESTAMP,
difficulty=genesis.DIFFICULTY,
extra_data=bytes.fromhex(genesis.EXTRA_DATA),
)
return (
MinorBlock(header=header, meta=meta, tx_list=[]),
TokenBalanceMap(coinbase_tokens),
)
def miner_address(self):
return Address.create_from(self.MINER_ADDRESS)
