async def handle_new_transaction_list(self, op, cmd, rpc_id):
if len(cmd.transaction_list) > NEW_TRANSACTION_LIST_LIMIT:
self.close_with_error("Too many transactions in one command")
for tx in cmd.transaction_list:
Logger.debug("Received tx {} from peer {}".format(
tx.get_hash().hex(), self.id.hex()))
await self.master_server.add_transaction(tx, self)
def run_test(name, pairs):
Logger.debug('testing %s' % name)
def _dec(x):
if isinstance(x, str) and x.startswith('0x'):
return bytes.fromhex(str(x[2:]))
if isinstance(x, str):
return bytes(x, "ascii")
return x
pairs['in'] = [(_dec(k), _dec(v)) for k, v in pairs['in']]
deletes = [(k, v) for k, v in pairs['in'] if v is None]
N_PERMUTATIONS = 100
for i, permut in enumerate(itertools.permutations(pairs['in'])):
if i > N_PERMUTATIONS:
break
if pairs.get('nopermute', None) is not None and pairs['nopermute']:
permut = pairs['in']
N_PERMUTATIONS = 1
t = trie.Trie(InMemoryDb())
for k, v in permut:
# logger.debug('updating with (%s, %s)' %(k, v))
if v is not None:
t.update(k, v)
else:
t.delete(k)
# make sure we have deletes at the end
for k, v in deletes:
t.delete(k)
if pairs['root'] != '0x' + t.root_hash.hex():
raise Exception("Mismatch: %r %r %r %r" %
(name, pairs['root'], '0x' + t.root_hash.hex(),
(i, list(permut) + deletes)))
def _get_block_time(block: Block, target_block_time) -> float:
if isinstance(block, MinorBlock):
# Adjust the target block time to compensate computation time
gas_used_ratio = block.meta.evm_gas_used / block.header.evm_gas_limit
target_block_time = target_block_time * (1 - gas_used_ratio * 0.4)
Logger.debug("[{}] target block time {:.2f}".format(
block.header.branch.get_shard_id(), target_block_time))
return numpy.random.exponential(target_block_time)
async def handle_new_transaction_list(self, op, cmd, rpc_id):
for tx in cmd.transaction_list:
Logger.debug("Received tx {} from peer {}".format(
tx.get_hash().hex(), self.id.hex()))
await self.master_server.add_transaction(tx, self)
async def add_block_list_for_sync(self, block_list):
""" Add blocks in batch to reduce RPCs. Will NOT broadcast to peers.
Returns true if blocks are successfully added. False on any error.
Additionally, returns list of coinbase_amount_map for each block
This function only adds blocks to local and propagate xshard list to other shards.
It does NOT notify master because the master should already have the minor header list,
and will add them once this function returns successfully.
"""
coinbase_amount_list = []
if not block_list:
return True, coinbase_amount_list
existing_add_block_futures = []
block_hash_to_x_shard_list = dict()
uncommitted_block_header_list = []
uncommitted_coinbase_amount_map_list = []
for block in block_list:
check(
block.header.branch.get_full_shard_id() == self.full_shard_id)
block_hash = block.header.get_hash()
# adding the block header one assuming the block will be validated.
coinbase_amount_list.append(block.header.coinbase_amount_map)
commit_status, future = self.__get_block_commit_status_by_hash(
block_hash)
if commit_status == BLOCK_COMMITTED:
# Skip processing the block if it is already committed
Logger.warning(
"minor block to sync {} is already committed".format(
block_hash.hex()))
continue
elif commit_status == BLOCK_COMMITTING:
# Check if the block is being propagating to other slaves and the master
# Let's make sure all the shards and master got it before committing it
Logger.info(
"[{}] {} is being added ... waiting for it to finish".
format(block.header.branch.to_str(), block.header.height))
existing_add_block_futures.append(future)
continue
check(commit_status == BLOCK_UNCOMMITTED)
# Validate and add the block
try:
xshard_list, coinbase_amount_map = self.state.add_block(
block, skip_if_too_old=False, force=True)
except Exception as e:
Logger.error_exception()
return False, None
prev_root_height = self.state.db.get_root_block_header_by_hash(
block.header.hash_prev_root_block).height
block_hash_to_x_shard_list[block_hash] = (xshard_list,
prev_root_height)
self.add_block_futures[block_hash] = self.loop.create_future()
uncommitted_block_header_list.append(block.header)
uncommitted_coinbase_amount_map_list.append(
block.header.coinbase_amount_map)
await self.slave.batch_broadcast_xshard_tx_list(
block_hash_to_x_shard_list, block_list[0].header.branch)
check(
len(uncommitted_coinbase_amount_map_list) == len(
uncommitted_block_header_list))
await self.slave.send_minor_block_header_list_to_master(
uncommitted_block_header_list,
uncommitted_coinbase_amount_map_list)
# Commit all blocks and notify all rest add block operations
for block_header in uncommitted_block_header_list:
block_hash = block_header.get_hash()
self.state.commit_by_hash(block_hash)
Logger.debug("committed mblock {}".format(block_hash.hex()))
self.add_block_futures[block_hash].set_result(None)
del self.add_block_futures[block_hash]
# Wait for the other add block operations
await asyncio.gather(*existing_add_block_futures)
return True, coinbase_amount_list
async def add_block(self, block):
""" Returns true if block is successfully added. False on any error.
called by 1. local miner (will not run if syncing) 2. SyncTask
"""
block_hash = block.header.get_hash()
commit_status, future = self.__get_block_commit_status_by_hash(
block_hash)
if commit_status == BLOCK_COMMITTED:
return True
elif commit_status == BLOCK_COMMITTING:
Logger.info(
"[{}] {} is being added ... waiting for it to finish".format(
block.header.branch.to_str(), block.header.height))
await future
return True
check(commit_status == BLOCK_UNCOMMITTED)
# Validate and add the block
old_tip = self.state.header_tip
try:
xshard_list, coinbase_amount_map = self.state.add_block(block,
force=True)
except Exception as e:
Logger.error_exception()
return False
# only remove from pool if the block successfully added to state,
# this may cache failed blocks but prevents them being broadcasted more than needed
# TODO add ttl to blocks in new_block_header_pool
self.state.new_block_header_pool.pop(block_hash, None)
# block has been added to local state, broadcast tip so that peers can sync if needed
try:
if old_tip != self.state.header_tip:
self.broadcast_new_tip()
except Exception:
Logger.warning_every_sec("broadcast tip failure", 1)
# Add the block in future and wait
self.add_block_futures[block_hash] = self.loop.create_future()
prev_root_height = self.state.db.get_root_block_header_by_hash(
block.header.hash_prev_root_block).height
await self.slave.broadcast_xshard_tx_list(block, xshard_list,
prev_root_height)
await self.slave.send_minor_block_header_to_master(
block.header,
len(block.tx_list),
len(xshard_list),
coinbase_amount_map,
self.state.get_shard_stats(),
)
# Commit the block
self.state.commit_by_hash(block_hash)
Logger.debug("committed mblock {}".format(block_hash.hex()))
# Notify the rest
self.add_block_futures[block_hash].set_result(None)
del self.add_block_futures[block_hash]
return True
