def _get_block_candidates(self) -> List[MinorBlock]: """Use given criteria to generate potential blocks matching the bloom.""" def should_skip(block: MinorBlock) -> bool: should_skip_block = False # same byte order as in bloom.py header_bloom = block.header.bloom for bit_list in self.bloom_bits: if not any((header_bloom & i) == i for i in bit_list): should_skip_block = True break return should_skip_block ret = [] if self.candidate_blocks is not None: ret = [b for b in self.candidate_blocks if not should_skip(b)] else: end_block_hash = self.end_block_header.get_hash() for i in range(self.size): block = self.db.get_minor_block_by_hash(end_block_hash) if not block: Logger.error( "No block found for height {} at shard {}".format( i, self.db.branch.get_full_shard_id())) continue if not should_skip(block): ret.append(block) end_block_hash = block.header.hash_prev_block ret = list(reversed(ret)) return ret
def _get_block_candidates(self) -> List[MinorBlock]: """Use given criteria to generate potential blocks matching the bloom.""" ret = [] for i in range(self.start_block, self.end_block + 1): block = self.db.get_minor_block_by_height(i) if not block: Logger.error( "No block found for height {} at shard {}".format( i, self.db.branch.get_shard_id() ) ) continue should_skip_block = False # same byte order as in bloom.py header_bloom = block.header.bloom for bit_list in self.bloom_bits: if not any((header_bloom & i) == i for i in bit_list): should_skip_block = True break if not should_skip_block: ret.append(block) if (1 + i) % 100 == 0 and time.time() - self.start_ts > Filter.TIMEOUT: raise Exception("Filter timeout") return ret
async def handle_create_cluster_peer_connection_request(self, req): shard_to_conn = dict() active_futures = [] for shard in self.shards.values(): if req.cluster_peer_id in shard.peers: Logger.error( "duplicated create cluster peer connection {}".format( req.cluster_peer_id)) continue peer_shard_conn = PeerShardConnection( master_conn=self, cluster_peer_id=req.cluster_peer_id, shard=shard, name="{}_vconn_{}".format(self.name, req.cluster_peer_id), ) asyncio.ensure_future(peer_shard_conn.active_and_loop_forever()) active_futures.append(peer_shard_conn.active_future) shard_to_conn[shard] = peer_shard_conn # wait for all the connections to become active before return await asyncio.gather(*active_futures) # Make peer connection available to shard once they are active for shard, peer_shard_conn in shard_to_conn.items(): shard.add_peer(peer_shard_conn) return CreateClusterPeerConnectionResponse(error_code=0)
async def submit_work(self, header_hash: bytes, nonce: int, mixhash: bytes) -> bool: if not self.remote: raise ValueError("Should only be used for remote miner") if header_hash not in self.work_map: return False block = self.work_map[header_hash] header = copy.copy(block.header) header.nonce, header.mixhash = nonce, mixhash # lower the difficulty for root block signed by guardian if self.guardian_private_key and isinstance(block, RootBlock): diff = Guardian.adjust_difficulty(header.difficulty, header.height) try: validate_seal(header, self.consensus_type, adjusted_diff=diff) except ValueError: return False # sign as a guardian header.sign_with_private_key(self.guardian_private_key) else: # minor block, or doesn't have guardian private key try: validate_seal(header, self.consensus_type) except ValueError: return False block.header = header # actual update try: await self.add_block_async_func(block) del self.work_map[header_hash] self.current_work = None return True except Exception as ex: Logger.error(ex) return False
async def handle_add_xshard_tx_list_request(self, req): if req.branch not in self.shards: Logger.error("cannot find shard id {} locally".format( req.branch.get_full_shard_id())) return AddXshardTxListResponse(error_code=errno.ENOENT) self.shards[ req.branch].state.add_cross_shard_tx_list_by_minor_block_hash( req.minor_block_hash, req.tx_list) return AddXshardTxListResponse(error_code=0)
async def handle_mined_block(instance: Miner): while True: block = await instance.output_q.coro_get() if not block: return # start mining before processing and propagating mined block instance._mine_new_block_async() try: await instance.add_block_async_func(block) except Exception as ex: Logger.error(ex)
async def handle_add_xshard_tx_list_request(self, req): if req.branch.get_shard_size() != self.__get_shard_size(): Logger.error("add xshard tx list request shard size mismatch! " "Expect: {}, actual: {}".format( self.__get_shard_size(), req.branch.get_shard_size())) return AddXshardTxListResponse(error_code=errno.ESRCH) if req.branch not in self.shards: Logger.error("cannot find shard id {} locally".format( req.branch.get_shard_id())) return AddXshardTxListResponse(error_code=errno.ENOENT) self.shards[ req.branch].state.add_cross_shard_tx_list_by_minor_block_hash( req.minor_block_hash, req.tx_list) return AddXshardTxListResponse(error_code=0)
async def handle_mined_block(instance: Miner): while True: block = await instance.output_q.coro_get() if not block: return # start mining before processing and propagating mined block instance._mine_new_block_async() try: # Root block should include latest minor block headers while it's being mined # This is a hack to get the latest minor block included since testnet does not check difficulty if instance.consensus_type == ConsensusType.POW_SIMULATE: block = await self.create_block_async_func() Simulate(block, target_block_time=0).post_process_mined_block( block ) await instance.add_block_async_func(block) except Exception as ex: Logger.error(ex)
def log_kafka_sample(self, topic: str, sample: dict): """This is for testing/debugging only, use async version for production""" if self.cluster_config.MONITORING.KAFKA_REST_ADDRESS == "": return url = "http://{}/topics/{}".format( self.cluster_config.MONITORING.KAFKA_REST_ADDRESS, topic) try: record_data = json.dumps({"records": [{"value": sample}]}) headers = { "Content-Type": "application/vnd.kafka.json.v2+json", "Accept": "application/vnd.kafka.v2+json", } response = requests.post(url, data=record_data, headers=headers) if response.status_code != 200: raise Exception("non-OK response status code: {}".format( response.status_code)) except Exception as ex: Logger.error("Failed to log sample to Kafka: {}".format(ex))
async def start_peer(self, peer: BasePeer) -> None: self.run_child_service(peer) await self.wait(peer.events.started.wait(), timeout=1) try: with peer.collect_sub_proto_messages() as buffer: await self.wait( peer.boot_manager.events.finished.wait(), timeout=self._peer_boot_timeout, ) except TimeoutError as err: self.logger.debug("Timout waiting for peer to boot: %s", err) await peer.disconnect(DisconnectReason.timeout) return else: if peer.remote.id % CLUSTER_PEER_ID_LEN in self.cluster_peer_map: Logger.error("{} already in cluster peer id".format( peer.remote.id)) await peer.disconnect(DisconnectReason.already_connected) return self._add_peer(peer, buffer.get_messages())
async def __handle(self, request): request = await request.text() Logger.info(request) d = dict() try: d = json.loads(request) except Exception: pass method = d.get("method", "null") if method in self.counters: self.counters[method] += 1 else: self.counters[method] = 1 # Use armor to prevent the handler from being cancelled when # aiohttp server loses connection to client response = await armor(self.handlers.dispatch(request)) if "error" in response: Logger.error(response) if response.is_notification: return web.Response() return web.json_response(response, status=response.http_status)
async def handle_mined_block(): while True: res = await self.output_q.coro_get() # type: MiningResult if not res: return # empty result means ending # start mining before processing and propagating mined block self._mine_new_block_async() block = self.work_map[res.header_hash] block.header.nonce = res.nonce block.header.mixhash = res.mixhash del self.work_map[res.header_hash] self._track(block) try: # FIXME: Root block should include latest minor block headers while it's being mined # This is a hack to get the latest minor block included since testnet does not check difficulty if self.consensus_type == ConsensusType.POW_SIMULATE: block = await self.create_block_async_func() block.header.nonce = random.randint(0, 2**32 - 1) self._track(block) self._log_status(block) await self.add_block_async_func(block) except Exception as ex: Logger.error(ex)
async def submit_work(self, header_hash: bytes, nonce: int, mixhash: bytes) -> bool: if not self.remote: raise ValueError("Should only be used for remote miner") if header_hash not in self.work_map: return False block = self.work_map[header_hash] header = copy.copy(block.header) header.nonce, header.mixhash = nonce, mixhash try: validate_seal(header, self.consensus_type) except ValueError: return False block.header = header # actual update try: await self.add_block_async_func(block) del self.work_map[header_hash] self.current_work = None return True except Exception as ex: Logger.error(ex) return False
async def __gen(self, num_tx, x_shard_percent, sample_tx: TypedTransaction): Logger.info( "[{}] start generating {} transactions with {}% cross-shard". format(self.full_shard_id, num_tx, x_shard_percent)) try: if num_tx <= 0: return start_time = time.time() tx_list = [] total = 0 sample_evm_tx = sample_tx.tx.to_evm_tx() for account in self.accounts: nonce = self.shard.state.get_transaction_count( account.address.recipient) tx = self.create_transaction(account, nonce, x_shard_percent, sample_evm_tx) if not tx: continue tx_list.append(tx) total += 1 if len(tx_list) >= 600 or total >= num_tx: self.shard.add_tx_list(tx_list) tx_list = [] await asyncio.sleep( random.uniform(8, 12) ) # yield CPU so that other stuff won't be held for too long if total >= num_tx: break end_time = time.time() except Exception as e: Logger.error(e) Logger.info("[{}] generated {} transactions in {:.2f} seconds".format( self.full_shard_id, total, end_time - start_time)) self.running = False
def close_with_error(self, error): Logger.error("Closing shard connection with error {}".format(error)) return super().close_with_error(error)