def _get_block_candidates(self) -> List[MinorBlock]:
"""Use given criteria to generate potential blocks matching the bloom."""
def should_skip(block: MinorBlock) -> bool:
should_skip_block = False
# same byte order as in bloom.py
header_bloom = block.header.bloom
for bit_list in self.bloom_bits:
if not any((header_bloom & i) == i for i in bit_list):
should_skip_block = True
break
return should_skip_block
ret = []
if self.candidate_blocks is not None:
ret = [b for b in self.candidate_blocks if not should_skip(b)]
else:
end_block_hash = self.end_block_header.get_hash()
for i in range(self.size):
block = self.db.get_minor_block_by_hash(end_block_hash)
if not block:
Logger.error(
"No block found for height {} at shard {}".format(
i, self.db.branch.get_full_shard_id()))
continue
if not should_skip(block):
ret.append(block)
end_block_hash = block.header.hash_prev_block
ret = list(reversed(ret))
return ret
def _get_block_candidates(self) -> List[MinorBlock]:
"""Use given criteria to generate potential blocks matching the bloom."""
ret = []
for i in range(self.start_block, self.end_block + 1):
block = self.db.get_minor_block_by_height(i)
if not block:
Logger.error(
"No block found for height {} at shard {}".format(
i, self.db.branch.get_shard_id()
)
)
continue
should_skip_block = False
# same byte order as in bloom.py
header_bloom = block.header.bloom
for bit_list in self.bloom_bits:
if not any((header_bloom & i) == i for i in bit_list):
should_skip_block = True
break
if not should_skip_block:
ret.append(block)
if (1 + i) % 100 == 0 and time.time() - self.start_ts > Filter.TIMEOUT:
raise Exception("Filter timeout")
return ret
async def handle_create_cluster_peer_connection_request(self, req):
shard_to_conn = dict()
active_futures = []
for shard in self.shards.values():
if req.cluster_peer_id in shard.peers:
Logger.error(
"duplicated create cluster peer connection {}".format(
req.cluster_peer_id))
continue
peer_shard_conn = PeerShardConnection(
master_conn=self,
cluster_peer_id=req.cluster_peer_id,
shard=shard,
name="{}_vconn_{}".format(self.name, req.cluster_peer_id),
)
asyncio.ensure_future(peer_shard_conn.active_and_loop_forever())
active_futures.append(peer_shard_conn.active_future)
shard_to_conn[shard] = peer_shard_conn
# wait for all the connections to become active before return
await asyncio.gather(*active_futures)
# Make peer connection available to shard once they are active
for shard, peer_shard_conn in shard_to_conn.items():
shard.add_peer(peer_shard_conn)
return CreateClusterPeerConnectionResponse(error_code=0)
async def submit_work(self, header_hash: bytes, nonce: int, mixhash: bytes) -> bool:
if not self.remote:
raise ValueError("Should only be used for remote miner")
if header_hash not in self.work_map:
return False
block = self.work_map[header_hash]
header = copy.copy(block.header)
header.nonce, header.mixhash = nonce, mixhash
# lower the difficulty for root block signed by guardian
if self.guardian_private_key and isinstance(block, RootBlock):
diff = Guardian.adjust_difficulty(header.difficulty, header.height)
try:
validate_seal(header, self.consensus_type, adjusted_diff=diff)
except ValueError:
return False
# sign as a guardian
header.sign_with_private_key(self.guardian_private_key)
else: # minor block, or doesn't have guardian private key
try:
validate_seal(header, self.consensus_type)
except ValueError:
return False
block.header = header # actual update
try:
await self.add_block_async_func(block)
del self.work_map[header_hash]
self.current_work = None
return True
except Exception as ex:
Logger.error(ex)
return False
async def handle_add_xshard_tx_list_request(self, req):
if req.branch not in self.shards:
Logger.error("cannot find shard id {} locally".format(
req.branch.get_full_shard_id()))
return AddXshardTxListResponse(error_code=errno.ENOENT)
self.shards[
req.branch].state.add_cross_shard_tx_list_by_minor_block_hash(
req.minor_block_hash, req.tx_list)
return AddXshardTxListResponse(error_code=0)
async def handle_mined_block(instance: Miner):
while True:
block = await instance.output_q.coro_get()
if not block:
return
# start mining before processing and propagating mined block
instance._mine_new_block_async()
try:
await instance.add_block_async_func(block)
except Exception as ex:
Logger.error(ex)
async def handle_add_xshard_tx_list_request(self, req):
if req.branch.get_shard_size() != self.__get_shard_size():
Logger.error("add xshard tx list request shard size mismatch! "
"Expect: {}, actual: {}".format(
self.__get_shard_size(),
req.branch.get_shard_size()))
return AddXshardTxListResponse(error_code=errno.ESRCH)
if req.branch not in self.shards:
Logger.error("cannot find shard id {} locally".format(
req.branch.get_shard_id()))
return AddXshardTxListResponse(error_code=errno.ENOENT)
self.shards[
req.branch].state.add_cross_shard_tx_list_by_minor_block_hash(
req.minor_block_hash, req.tx_list)
return AddXshardTxListResponse(error_code=0)
async def handle_mined_block(instance: Miner):
while True:
block = await instance.output_q.coro_get()
if not block:
return
# start mining before processing and propagating mined block
instance._mine_new_block_async()
try:
# Root block should include latest minor block headers while it's being mined
# This is a hack to get the latest minor block included since testnet does not check difficulty
if instance.consensus_type == ConsensusType.POW_SIMULATE:
block = await self.create_block_async_func()
Simulate(block, target_block_time=0).post_process_mined_block(
block
)
await instance.add_block_async_func(block)
except Exception as ex:
Logger.error(ex)
def log_kafka_sample(self, topic: str, sample: dict):
"""This is for testing/debugging only, use async version for production"""
if self.cluster_config.MONITORING.KAFKA_REST_ADDRESS == "":
return
url = "http://{}/topics/{}".format(
self.cluster_config.MONITORING.KAFKA_REST_ADDRESS, topic)
try:
record_data = json.dumps({"records": [{"value": sample}]})
headers = {
"Content-Type": "application/vnd.kafka.json.v2+json",
"Accept": "application/vnd.kafka.v2+json",
}
response = requests.post(url, data=record_data, headers=headers)
if response.status_code != 200:
raise Exception("non-OK response status code: {}".format(
response.status_code))
except Exception as ex:
Logger.error("Failed to log sample to Kafka: {}".format(ex))
async def start_peer(self, peer: BasePeer) -> None:
self.run_child_service(peer)
await self.wait(peer.events.started.wait(), timeout=1)
try:
with peer.collect_sub_proto_messages() as buffer:
await self.wait(
peer.boot_manager.events.finished.wait(),
timeout=self._peer_boot_timeout,
)
except TimeoutError as err:
self.logger.debug("Timout waiting for peer to boot: %s", err)
await peer.disconnect(DisconnectReason.timeout)
return
else:
if peer.remote.id % CLUSTER_PEER_ID_LEN in self.cluster_peer_map:
Logger.error("{} already in cluster peer id".format(
peer.remote.id))
await peer.disconnect(DisconnectReason.already_connected)
return
self._add_peer(peer, buffer.get_messages())
async def __handle(self, request):
request = await request.text()
Logger.info(request)
d = dict()
try:
d = json.loads(request)
except Exception:
pass
method = d.get("method", "null")
if method in self.counters:
self.counters[method] += 1
else:
self.counters[method] = 1
# Use armor to prevent the handler from being cancelled when
# aiohttp server loses connection to client
response = await armor(self.handlers.dispatch(request))
if "error" in response:
Logger.error(response)
if response.is_notification:
return web.Response()
return web.json_response(response, status=response.http_status)
async def handle_mined_block():
while True:
res = await self.output_q.coro_get() # type: MiningResult
if not res:
return # empty result means ending
# start mining before processing and propagating mined block
self._mine_new_block_async()
block = self.work_map[res.header_hash]
block.header.nonce = res.nonce
block.header.mixhash = res.mixhash
del self.work_map[res.header_hash]
self._track(block)
try:
# FIXME: Root block should include latest minor block headers while it's being mined
# This is a hack to get the latest minor block included since testnet does not check difficulty
if self.consensus_type == ConsensusType.POW_SIMULATE:
block = await self.create_block_async_func()
block.header.nonce = random.randint(0, 2**32 - 1)
self._track(block)
self._log_status(block)
await self.add_block_async_func(block)
except Exception as ex:
Logger.error(ex)
async def submit_work(self, header_hash: bytes, nonce: int, mixhash: bytes) -> bool:
if not self.remote:
raise ValueError("Should only be used for remote miner")
if header_hash not in self.work_map:
return False
block = self.work_map[header_hash]
header = copy.copy(block.header)
header.nonce, header.mixhash = nonce, mixhash
try:
validate_seal(header, self.consensus_type)
except ValueError:
return False
block.header = header # actual update
try:
await self.add_block_async_func(block)
del self.work_map[header_hash]
self.current_work = None
return True
except Exception as ex:
Logger.error(ex)
return False
async def __gen(self, num_tx, x_shard_percent,
sample_tx: TypedTransaction):
Logger.info(
"[{}] start generating {} transactions with {}% cross-shard".
format(self.full_shard_id, num_tx, x_shard_percent))
try:
if num_tx <= 0:
return
start_time = time.time()
tx_list = []
total = 0
sample_evm_tx = sample_tx.tx.to_evm_tx()
for account in self.accounts:
nonce = self.shard.state.get_transaction_count(
account.address.recipient)
tx = self.create_transaction(account, nonce, x_shard_percent,
sample_evm_tx)
if not tx:
continue
tx_list.append(tx)
total += 1
if len(tx_list) >= 600 or total >= num_tx:
self.shard.add_tx_list(tx_list)
tx_list = []
await asyncio.sleep(
random.uniform(8, 12)
) # yield CPU so that other stuff won't be held for too long
if total >= num_tx:
break
end_time = time.time()
except Exception as e:
Logger.error(e)
Logger.info("[{}] generated {} transactions in {:.2f} seconds".format(
self.full_shard_id, total, end_time - start_time))
self.running = False
def close_with_error(self, error):
Logger.error("Closing shard connection with error {}".format(error))
return super().close_with_error(error)
