def __init_miner(self):
miner_address = self.env.quark_chain_config.testnet_master_address.address_in_branch(
Branch.create(self.__get_shard_size(), self.shard_id))
async def __create_block():
# hold off mining if the shard is syncing
while self.synchronizer.running or not self.state.initialized:
await asyncio.sleep(0.1)
return self.state.create_block_to_mine(address=miner_address)
async def __add_block(block):
# Do not add block if there is a sync in progress
if self.synchronizer.running:
return
# Do not add stale block
if self.state.header_tip.height >= block.header.height:
return
await self.handle_new_block(block)
def __get_target_block_time():
return self.slave.artificial_tx_config.target_minor_block_time
self.miner = Miner(
self.env.quark_chain_config.SHARD_LIST[
self.shard_id].CONSENSUS_TYPE,
__create_block,
__add_block,
__get_target_block_time,
self.env,
)
def miner_gen(consensus,
create_func,
add_func,
tip_func=dummy_tip,
**kwargs):
m = Miner(consensus, create_func, add_func, self.get_mining_params,
tip_func, **kwargs)
m.enabled = True
return m
def setUp(self):
super().setUp()
TestMiner.added_blocks = []
self.miner = Miner(
ConsensusType.POW_SIMULATE,
self.dummy_create_block_async,
self.dummy_add_block_async,
self.get_target_block_time,
None,
)
self.miner.enable()
TestMiner.miner = self.miner
def __init_miner(self):
miner_address = Address.create_from(
self.env.quark_chain_config.SHARD_LIST[
self.shard_id].COINBASE_ADDRESS)
async def __create_block(retry=True):
# hold off mining if the shard is syncing
while self.synchronizer.running or not self.state.initialized:
if not retry:
break
await asyncio.sleep(0.1)
return self.state.create_block_to_mine(address=miner_address)
async def __add_block(block):
# Do not add block if there is a sync in progress
if self.synchronizer.running:
return
# Do not add stale block
if self.state.header_tip.height >= block.header.height:
return
await self.handle_new_block(block)
def __get_mining_param():
return {
"target_block_time":
self.slave.artificial_tx_config.target_minor_block_time
}
shard_config = self.env.quark_chain_config.SHARD_LIST[
self.shard_id] # type: ShardConfig
self.miner = Miner(
shard_config.CONSENSUS_TYPE,
__create_block,
__add_block,
__get_mining_param,
remote=shard_config.CONSENSUS_CONFIG.REMOTE_MINE,
)
def __init_miner(self):
async def __create_block(coinbase_addr: Address, retry=True):
# hold off mining if the shard is syncing
while self.synchronizer.running or not self.state.initialized:
if not retry:
break
await asyncio.sleep(0.1)
if coinbase_addr.is_empty(): # devnet or wrong config
coinbase_addr.full_shard_key = self.full_shard_id
return self.state.create_block_to_mine(address=coinbase_addr)
async def __add_block(block):
# do not add block if there is a sync in progress
if self.synchronizer.running:
return
# do not add stale block
if self.state.header_tip.height >= block.header.height:
return
await self.handle_new_block(block)
def __get_mining_param():
return {
"target_block_time":
self.slave.artificial_tx_config.target_minor_block_time
}
shard_config = self.env.quark_chain_config.shards[
self.full_shard_id] # type: ShardConfig
self.miner = Miner(
shard_config.CONSENSUS_TYPE,
__create_block,
__add_block,
__get_mining_param,
lambda: self.state.header_tip,
remote=shard_config.CONSENSUS_CONFIG.REMOTE_MINE,
)
class Shard:
def __init__(self, env, shard_id, slave):
self.env = env
self.shard_id = shard_id
self.slave = slave
self.state = ShardState(env, shard_id, self.__init_shard_db())
self.loop = asyncio.get_event_loop()
self.synchronizer = Synchronizer()
self.peers = dict() # cluster_peer_id -> PeerShardConnection
# block hash -> future (that will return when the block is fully propagated in the cluster)
# the block that has been added locally but not have been fully propagated will have an entry here
self.add_block_futures = dict()
self.tx_generator = TransactionGenerator(self.env.quark_chain_config,
self)
self.__init_miner()
def __init_shard_db(self):
"""
Create a PersistentDB or use the env.db if DB_PATH_ROOT is not specified in the ClusterConfig.
"""
if self.env.cluster_config.use_mem_db():
return InMemoryDb()
db_path = "{path}/shard-{shard_id}.db".format(
path=self.env.cluster_config.DB_PATH_ROOT, shard_id=self.shard_id)
return PersistentDb(db_path, clean=self.env.cluster_config.CLEAN)
def __init_miner(self):
miner_address = self.env.quark_chain_config.testnet_master_address.address_in_branch(
Branch.create(self.__get_shard_size(), self.shard_id))
async def __create_block():
# hold off mining if the shard is syncing
while self.synchronizer.running or not self.state.initialized:
await asyncio.sleep(0.1)
return self.state.create_block_to_mine(address=miner_address)
async def __add_block(block):
# Do not add block if there is a sync in progress
if self.synchronizer.running:
return
# Do not add stale block
if self.state.header_tip.height >= block.header.height:
return
await self.handle_new_block(block)
def __get_target_block_time():
return self.slave.artificial_tx_config.target_minor_block_time
self.miner = Miner(
self.env.quark_chain_config.SHARD_LIST[
self.shard_id].CONSENSUS_TYPE,
__create_block,
__add_block,
__get_target_block_time,
self.env,
)
def __get_shard_size(self):
return self.env.quark_chain_config.SHARD_SIZE
@property
def genesis_root_height(self):
return self.env.quark_chain_config.get_genesis_root_height(
self.shard_id)
def add_peer(self, peer: PeerShardConnection):
self.peers[peer.cluster_peer_id] = peer
async def __init_genesis_state(self, root_block: RootBlock):
block = self.state.init_genesis_state(root_block)
xshard_list = []
await self.slave.broadcast_xshard_tx_list(block, xshard_list,
root_block.header.height)
await self.slave.send_minor_block_header_to_master(
block.header,
len(block.tx_list),
len(xshard_list),
self.state.get_shard_stats(),
)
async def init_from_root_block(self, root_block: RootBlock):
""" Either recover state from local db or create genesis state based on config"""
if root_block.header.height > self.genesis_root_height:
return self.state.init_from_root_block(root_block)
if root_block.header.height == self.genesis_root_height:
await self.__init_genesis_state(root_block)
async def add_root_block(self, root_block: RootBlock):
check(root_block.header.height >= self.genesis_root_height)
if root_block.header.height > self.genesis_root_height:
return self.state.add_root_block(root_block)
# this happens when there is a root chain fork
if root_block.header.height == self.genesis_root_height:
await self.__init_genesis_state(root_block)
def broadcast_new_block(self, block):
for cluster_peer_id, peer in self.peers.items():
peer.send_new_block(block)
def broadcast_new_tip(self):
for cluster_peer_id, peer in self.peers.items():
peer.broadcast_new_tip()
def broadcast_tx_list(self, tx_list, source_peer=None):
for cluster_peer_id, peer in self.peers.items():
if source_peer == peer:
continue
peer.broadcast_tx_list(tx_list)
async def handle_new_block(self, block):
"""
0. if local shard is syncing, doesn't make sense to add, skip
1. if block parent is not in local state/new block pool, discard
2. if already in cache or in local state/new block pool, pass
3. validate: check time, difficulty, POW
4. add it to new minor block broadcast cache
5. broadcast to all peers (minus peer that sent it, optional)
6. add_block() to local state (then remove from cache)
also, broadcast tip if tip is updated (so that peers can sync if they missed blocks, or are new)
"""
if self.synchronizer.running:
# TODO optinal: queue the block if it came from broadcast to so that once sync is over, catch up immediately
return
if block.header.get_hash() in self.state.new_block_pool:
return
if self.state.db.contain_minor_block_by_hash(block.header.get_hash()):
return
if not self.state.db.contain_minor_block_by_hash(
block.header.hash_prev_minor_block):
if block.header.hash_prev_minor_block not in self.state.new_block_pool:
return
# TODO check difficulty and POW here
# one option is to use __validate_block but we may not need the full check
if block.header.create_time > time_ms() // 1000 + 30:
return
self.state.new_block_pool[block.header.get_hash()] = block
self.broadcast_new_block(block)
await self.add_block(block)
async def add_block(self, block):
""" Returns true if block is successfully added. False on any error.
called by 1. local miner (will not run if syncing) 2. SyncTask
"""
old_tip = self.state.header_tip
try:
xshard_list = self.state.add_block(block)
except Exception as e:
Logger.error_exception()
return False
# only remove from pool if the block successfully added to state,
# this may cache failed blocks but prevents them being broadcasted more than needed
# TODO add ttl to blocks in new_block_pool
self.state.new_block_pool.pop(block.header.get_hash(), None)
# block has been added to local state, broadcast tip so that peers can sync if needed
try:
if old_tip != self.state.header_tip:
self.broadcast_new_tip()
except Exception:
Logger.warning_every_sec("broadcast tip failure", 1)
# block already existed in local shard state
# but might not have been propagated to other shards and master
# let's make sure all the shards and master got it before return
if xshard_list is None:
future = self.add_block_futures.get(block.header.get_hash(), None)
if future:
Logger.info(
"[{}] {} is being added ... waiting for it to finish".
format(block.header.branch.get_shard_id(),
block.header.height))
await future
return True
self.add_block_futures[
block.header.get_hash()] = self.loop.create_future()
# Start mining new one before propagating inside cluster
# The propagation should be done by the time the new block is mined
self.miner.mine_new_block_async()
prev_root_height = self.state.db.get_root_block_by_hash(
block.header.hash_prev_root_block).header.height
await self.slave.broadcast_xshard_tx_list(block, xshard_list,
prev_root_height)
await self.slave.send_minor_block_header_to_master(
block.header,
len(block.tx_list),
len(xshard_list),
self.state.get_shard_stats(),
)
self.add_block_futures[block.header.get_hash()].set_result(None)
del self.add_block_futures[block.header.get_hash()]
return True
async def add_block_list_for_sync(self, block_list):
""" Add blocks in batch to reduce RPCs. Will NOT broadcast to peers.
Returns true if blocks are successfully added. False on any error.
This function only adds blocks to local and propagate xshard list to other shards.
It does NOT notify master because the master should already have the minor header list,
and will add them once this function returns successfully.
"""
if not block_list:
return True
existing_add_block_futures = []
block_hash_to_x_shard_list = dict()
for block in block_list:
check(block.header.branch.get_shard_id() == self.shard_id)
block_hash = block.header.get_hash()
try:
xshard_list = self.state.add_block(block)
except Exception as e:
Logger.error_exception()
return False
# block already existed in local shard state
# but might not have been propagated to other shards and master
# let's make sure all the shards and master got it before return
if xshard_list is None:
future = self.add_block_futures.get(block_hash, None)
if future:
existing_add_block_futures.append(future)
else:
prev_root_height = self.state.db.get_root_block_by_hash(
block.header.hash_prev_root_block).header.height
block_hash_to_x_shard_list[block_hash] = (xshard_list,
prev_root_height)
self.add_block_futures[block_hash] = self.loop.create_future()
await self.slave.batch_broadcast_xshard_tx_list(
block_hash_to_x_shard_list, block_list[0].header.branch)
for block_hash in block_hash_to_x_shard_list.keys():
self.add_block_futures[block_hash].set_result(None)
del self.add_block_futures[block_hash]
await asyncio.gather(*existing_add_block_futures)
return True
def add_tx_list(self, tx_list, source_peer=None):
if not tx_list:
return
valid_tx_list = []
for tx in tx_list:
if self.add_tx(tx):
valid_tx_list.append(tx)
if not valid_tx_list:
return
self.broadcast_tx_list(valid_tx_list, source_peer)
def add_tx(self, tx: Transaction):
return self.state.add_tx(tx)
class TestMiner(unittest.TestCase):
# used for stubbing `add_block_async_func`
added_blocks = []
miner = None # type: Miner
def setUp(self):
super().setUp()
TestMiner.added_blocks = []
self.miner = Miner(
ConsensusType.POW_SIMULATE,
self.dummy_create_block_async,
self.dummy_add_block_async,
self.get_target_block_time,
None,
)
self.miner.enable()
TestMiner.miner = self.miner
@staticmethod
async def dummy_add_block_async(block) -> None:
TestMiner.added_blocks.append(block)
# keep calling mining
TestMiner.miner.mine_new_block_async()
@staticmethod
async def dummy_create_block_async() -> Optional[RootBlock]:
if len(TestMiner.added_blocks) >= 5:
return None # stop the game
return RootBlock(RootBlockHeader(create_time=int(time.time()), extra_data="{}".encode("utf-8")))
@staticmethod
def get_target_block_time() -> float:
# guarantee target time is hit
return 0.0
def test_simulate_mine(self):
# should generate 5 blocks and then end
loop = asyncio.get_event_loop()
loop.run_until_complete(self.miner.mine_new_block_async())
self.assertEqual(len(TestMiner.added_blocks), 5)
def test_simulate_mine_handle_block_exception(self):
i = 0
async def add(block):
nonlocal i
try:
if i % 2 == 0:
raise Exception("( う-´)づ︻╦̵̵̿╤── \(˚☐˚”)/")
else:
await TestMiner.dummy_add_block_async(block)
finally:
i += 1
async def create():
nonlocal i
if i >= 5:
return None
return RootBlock(RootBlockHeader(create_time=int(time.time()), extra_data="{}".encode("utf-8")))
self.miner.add_block_async_func = add
self.miner.create_block_async_func = create
loop = asyncio.get_event_loop()
loop.run_until_complete(self.miner.mine_new_block_async())
# only 2 blocks can be added
self.assertEqual(len(TestMiner.added_blocks), 2)
def miner_gen(consensus, create_func, add_func):
m = Miner(consensus, create_func, add_func, self.get_mining_params,
None)
m.enable()
return m