async def fetch_block_transactions(self, block): q = time.time() missed = set() tx_count = len(block["tx"]) for h in block["tx"]: try: self.tx_cache[h] except: missed.add(h) if self.utxo_data: if self.db_type == "postgresql": async with self.db_pool.acquire() as conn: rows = await conn.fetch( "SELECT distinct tx_id FROM connector_unconfirmed_stxo " "WHERE tx_id = ANY($1);", set(s2rh(t) for t in missed)) for row in rows: missed.remove(rh2s(row["tx_id"])) if missed: coinbase = await conn.fetchval( "SELECT out_tx_id FROM connector_unconfirmed_utxo " "WHERE out_tx_id = $1 LIMIT 1;", s2rh(block["tx"][0])) if coinbase: if block["tx"][0] in missed: missed.remove(block["tx"][0]) self.log.debug("Block missed transactions %s from %s" % (len(missed), tx_count)) if missed: self.missed_tx = set(missed) self.await_tx = set(missed) self.await_tx_future = {s2rh(i): asyncio.Future() for i in missed} self.block_txs_request = asyncio.Future() self.block_timestamp = block["time"] self.loop.create_task(self._get_missed()) try: await asyncio.wait_for(self.block_txs_request, timeout=self.block_timeout) except asyncio.CancelledError: # refresh rpc connection session try: await self.rpc.close() self.rpc = aiojsonrpc.rpc(self.rpc_url, self.loop, timeout=self.rpc_timeout) except: pass raise RuntimeError("block transaction request timeout") self.total_received_tx += tx_count self.total_received_tx_last += tx_count self.total_received_tx_time += time.time() - q rate = round(self.total_received_tx / self.total_received_tx_time) self.log.debug( "Transactions received: %s [%s] received tx rate tx/s ->> %s <<" % (tx_count, time.time() - q, rate))
async def get_next_block(self): if not self.active: return if not self.get_next_block_mutex.done(): await self.get_next_block_mutex try: self.get_next_block_mutex = asyncio.Future() if self.node_last_block <= self.last_block_height + self.backlog: d = await self.rpc.getblockcount() if d == self.node_last_block: self.log.info("blockchain is synchronized backlog %s" % self.backlog) return else: self.node_last_block = d d = self.node_last_block - self.last_block_height if d > self.deep_sync_limit: if not self.deep_synchronization: self.log.warning("Deep synchronization mode") self.deep_synchronization = True else: if self.deep_synchronization: self.log.warning("Normal synchronization mode") self.deep_synchronization = False h = await self.rpc.getblockhash(self.last_block_height + 1) await self._get_block_by_hash(h) except Exception as err: self.log.error("get next block failed %s" % str(err)) finally: self.get_next_block_mutex.set_result(True)
async def _send_and_wait_origin(self, operate, _tuple, timeout): # operate 枚举 message_id = await self._send_to_linda_server(operate, _tuple) ''' return future timeout futurn 被消息循环队列释放 timeout https://docs.python.org/3/library/asyncio-task.html#asyncio.wait_for ''' f = asyncio.Future() # todo asyncio future self.linda_wait_futures.append((message_id, f)) # todo 加入到队列里: (message_id, f) f.set_result(tuple) try: return await asyncio.wait_for(f, timeout=timeout) # result() 非阻塞 查询状态 except asyncio.TimeoutError: # print('timeout!') raise asyncio.TimeoutError( f'timeout: {timeout}; message_id: {message_id}')
async def _new_block(self, block): if not self.active or not self.active_block.done() or self.last_block_height >= block["height"]: return self.active_block = asyncio.Future() self.block_dependency_tx = 0 bin_block_hash = bytes_from_hex(block["hash"]) bin_prev_block_hash = block["previousblockhash"] if "previousblockhash" in block else None block_height = int(block["height"]) self.log.info("New block %s %s" % (block_height, block["hash"])) bt = tm() bpt = 0 try: # blockchain position check if self.block_cache.get(bin_block_hash) is not None: self.log.debug("duplicated block %s" % block["hash"]) return if self.block_cache.get(bin_prev_block_hash) is None and self.last_block_height: self.log.critical("Connector panic! Node out of sync no parent block in chain tail %s" % bin_prev_block_hash) return if self.last_block_height + 1 != block_height: if self.orphan_handler: tq = tm() await self.orphan_handler(self.last_block_height) self.log.info("orphan handler %s [%s]" % (self.last_block_height, tm(tq))) self.block_cache.pop_last() self.last_block_height -= 1 return # add all block transactions missed = set() for h in block["tx"]: if self.tx_cache.get(h) is None: missed.add(h) if self.before_block_handler: q = time.time() await self.before_block_handler(block) bpt = time.time() - q self.blocks_processing_time += bpt self.log.info("Transactions missed %s" % len(missed)) cq = tm() if missed: self.log.debug("Request missed transactions") self.missed_tx_list = set(missed) self.await_tx_list = missed self.await_tx_future = dict() for i in missed: self.await_tx_future[i] = asyncio.Future() self.block_txs_request = asyncio.Future() if self.deep_synchronization: self.loop.create_task(self._get_missed(block["hash"], block["time"], block["height"])) else: self.loop.create_task(self._get_missed(False, block["time"], block["height"])) try: await asyncio.wait_for(self.block_txs_request, timeout=self.block_timeout) except asyncio.CancelledError: # refresh rpc connection session await self.rpc.close() self.rpc = aiojsonrpc.rpc(self.rpc_url, self.loop, timeout=self.rpc_timeout) raise RuntimeError("block transaction request timeout") tx_count = len(block["tx"]) self.total_received_tx += tx_count self.total_received_tx_time += tm(cq) rate = round(self.total_received_tx/self.total_received_tx_time) self.log.info("Transactions received: %s [%s] rate tx/s ->> %s <<" % (tx_count, tm(cq), rate)) if self.block_handler: q = time.time() await self.block_handler(block) self.blocks_processing_time += time.time() - q bpt += time.time() - q # insert new block self.block_cache.set(block["hash"], block["height"]) self.last_block_height = block["height"] # after block added handler if self.after_block_handler: q = time.time() try: await self.after_block_handler(block) except: pass self.blocks_processing_time += time.time() - q bpt += time.time() - q self.blocks_processed_count += 1 [self.tx_cache.pop(h) for h in block["tx"]] except Exception as err: if self.await_tx_list: self.await_tx_list = set() self.log.error(str(traceback.format_exc())) self.log.error("new block error %s" % str(err)) finally: self.active_block.set_result(True) self.log.info("> %s < block [%s tx/ %s size] (dp %s) processing time %s cache [%s/%s]" % (block["height"], len(block["tx"]), block["size"] / 1000000, self.block_dependency_tx, tm(bt), len(self.block_hashes_preload._store), len(self.block_preload._store))) if self.node_last_block > self.last_block_height: self.loop.create_task(self.get_next_block())
def __init__(self, node_rpc_url, node_zerromq_url, logger, last_block_height=0, chain_tail=None, mempool_tx_list=None, tx_handler=None, orphan_handler=None, before_block_handler=None, block_handler=None, after_block_handler=None, block_timeout=30, deep_sync_limit=20, backlog=0, mempool_tx=True, rpc_batch_limit=20, rpc_threads_limit=100, rpc_timeout=100, preload=False): self.loop = asyncio.get_event_loop() # settings self.log = logger self.rpc_url = node_rpc_url self.zmq_url = node_zerromq_url self.orphan_handler = orphan_handler self.block_timeout = block_timeout self.tx_handler = tx_handler self.before_block_handler = before_block_handler self.block_handler = block_handler self.after_block_handler = after_block_handler self.deep_sync_limit = deep_sync_limit self.backlog = backlog self.mempool_tx = mempool_tx self.chain_tail = list(chain_tail) if chain_tail else [] self.mempool_tx_list = list(mempool_tx_list) if mempool_tx_list else [] self.rpc_timeout = rpc_timeout self.batch_limit = rpc_batch_limit # state and stats self.node_last_block = None self.last_block_height = int(last_block_height) if int(last_block_height) else 0 self.deep_synchronization = False self.block_dependency_tx = 0 # counter of tx that have dependencies in block self.active = True self.get_next_block_mutex = asyncio.Future() self.get_next_block_mutex.set_result(True) self.active_block = asyncio.Future() self.active_block.set_result(True) self.last_zmq_msg = int(time.time()) self.total_received_tx = 0 self.blocks_processed_count = 0 self.blocks_decode_time = 0 self.blocks_download_time = 0 self.blocks_processing_time = 0 self.total_received_tx_time = 0 # cache and system self.preload = preload self.block_preload = Cache(max_size=50000) self.block_hashes_preload = Cache(max_size=50000) self.tx_cache = Cache(max_size=50000) self.block_cache = Cache(max_size=10000) self.block_txs_request = None self.connected = asyncio.Future() self.await_tx_list = list() self.missed_tx_list = list() self.await_tx_future = dict() self.add_tx_future = dict() self.get_missed_tx_threads = 0 self.get_missed_tx_threads_limit = rpc_threads_limit self.tx_in_process = set() self.zmqContext = None self.tasks = list() self.log.info("Node connector started") asyncio.ensure_future(self.start(), loop=self.loop)
async def _new_block(self, block): if not self.active: return tq = time.time() if self.block_headers_cache.get(block["hash"]) is not None: return if self.deep_synchronization: block["height"] = self.last_block_height + 1 if self.last_block_height >= block["height"]: return if not self.active_block.done(): return try: self.active_block = asyncio.Future() if self.last_block_height < self.last_block_utxo_cached_height: if not self.cache_loading: self.log.info("Bootstrap UTXO cache ...") self.cache_loading = True else: if self.cache_loading: self.log.info("UTXO Cache bootstrap completed") self.cache_loading = False await self.verify_block_position(block) if self.deep_synchronization: await self._block_as_transactions_batch(block) if not self.cache_loading or block[ "height"] > self.app_block_height_on_start: if self.block_batch_handler: t = time.time() await self.block_batch_handler(block) self.batch_handler += time.time() - t if self.total_received_tx - self.total_received_tx_stat > 100000: self.report_sync_process(block["height"]) if self.utxo_data: if self.sync_utxo.len() > self.sync_utxo.size_limit: if not self.sync_utxo.save_process: if self.sync_utxo.checkpoints and not self.cache_loading: if self.sync_utxo.checkpoints[0] < block[ "height"]: self.sync_utxo.create_checkpoint( block["height"], self.app_last_block) if self.sync_utxo.save_process: self.loop.create_task( self.sync_utxo.commit()) else: # call before block handler if self.before_block_handler: await self.before_block_handler(block) await self.fetch_block_transactions(block) if self.utxo_data: if self.db_type == "postgresql": async with self.db_pool.acquire() as conn: async with conn.transaction(): data = await self.uutxo.apply_block_changes( [s2rh(h) for h in block["tx"]], block["height"], conn) block["mempoolInvalid"] = { "tx": data["invalid_txs"], "inputs": data["dbs_stxo"], "outputs": data["dbs_uutxo"] } if self.block_handler: await self.block_handler(block, conn) await conn.execute( "UPDATE connector_utxo_state SET value = $1 " "WHERE name = 'last_block';", block["height"]) await conn.execute( "UPDATE connector_utxo_state SET value = $1 " "WHERE name = 'last_cached_block';", block["height"]) elif self.block_handler: await self.block_handler(block, None) self.block_headers_cache.set(block["hash"], block["height"]) self.last_block_height = block["height"] self.app_last_block = block["height"] self.blocks_processed_count += 1 # after block added handler if self.after_block_handler: if not self.cache_loading or block[ "height"] > self.app_block_height_on_start: try: await self.after_block_handler(block) except: pass if not self.deep_synchronization: self.log.info( "Block %s -> %s; tx count %s;" % (block["height"], block["hash"], len(block["tx"]))) except Exception as err: if self.await_tx: self.await_tx = set() for i in self.await_tx_future: if not self.await_tx_future[i].done(): self.await_tx_future[i].cancel() self.await_tx_future = dict() self.log.error("block %s error %s" % (block["height"], str(err))) import traceback print(traceback.format_exc()) finally: if self.node_last_block > self.last_block_height: self.get_next_block_mutex = True self.loop.create_task(self.get_next_block()) self.blocks_processing_time += time.time() - tq self.active_block.set_result(True)
def __init__(self, node_rpc_url, node_zerromq_url, logger, last_block_height=0, chain_tail=None, tx_handler=None, orphan_handler=None, before_block_handler=None, block_handler=None, after_block_handler=None, block_batch_handler=None, flush_app_caches_handler=None, synchronization_completed_handler=None, block_timeout=30, deep_sync_limit=20, backlog=0, mempool_tx=True, rpc_batch_limit=50, rpc_threads_limit=100, rpc_timeout=100, utxo_data=False, utxo_cache_size=1000000, skip_opreturn=True, block_cache_workers=4, block_preload_cache_limit=1000 * 1000000, block_preload_batch_size_limit=200000000, block_hashes_cache_limit=200 * 1000000, db_type=None, db=None, app_proc_title="Connector"): self.loop = asyncio.get_event_loop() # settings self.log = logger self.rpc_url = node_rpc_url self.app_proc_title = app_proc_title self.rpc_timeout = rpc_timeout self.rpc_batch_limit = rpc_batch_limit self.zmq_url = node_zerromq_url self.orphan_handler = orphan_handler self.block_timeout = block_timeout self.tx_handler = tx_handler self.skip_opreturn = skip_opreturn self.before_block_handler = before_block_handler self.block_handler = block_handler self.after_block_handler = after_block_handler self.block_batch_handler = block_batch_handler self.flush_app_caches_handler = flush_app_caches_handler self.synchronization_completed_handler = synchronization_completed_handler self.block_preload_batch_size_limit = block_preload_batch_size_limit self.deep_sync_limit = deep_sync_limit self.backlog = backlog self.mempool_tx = mempool_tx self.db_type = db_type self.db = db self.utxo_cache_size = utxo_cache_size self.block_cache_workers = block_cache_workers self.utxo_data = utxo_data self.chain_tail = list(chain_tail) if chain_tail else [] # state and stats self.node_last_block = None self.sync_utxo = None self.uutxo = None self.cache_loading = False self.app_block_height_on_start = int(last_block_height) if int( last_block_height) else 0 self.last_block_height = 0 self.last_block_utxo_cached_height = 0 self.deep_synchronization = False self.block_dependency_tx = 0 # counter of tx that have dependencies in block self.active = True self.get_next_block_mutex = False self.active_block = asyncio.Future() self.active_block.set_result(True) self.last_zmq_msg = int(time.time()) self.total_received_tx = 0 self.total_received_tx_stat = 0 self.blocks_processed_count = 0 self.blocks_decode_time = 0 self.blocks_download_time = 0 self.blocks_processing_time = 0 self.tx_processing_time = 0 self.non_cached_blocks = 0 self.total_received_tx_time = 0 self.coins = 0 self.op_return = 0 self.destroyed_coins = 0 self.preload_cached_total = 0 self.preload_cached = 0 self.preload_cached_annihilated = 0 self.start_time = time.time() self.total_received_tx_last = 0 self.start_time_last = time.time() self.batch_time = time.time() self.batch_load_utxo = 0 self.batch_parsing = 0 self.batch_handler = 0 self.app_last_block = None # cache and system self.block_preload_cache_limit = block_preload_cache_limit self.block_hashes_cache_limit = block_hashes_cache_limit self.tx_cache_limit = 144 * 5000 self.block_headers_cache_limit = 100 * 100000 self.block_preload = Cache(max_size=self.block_preload_cache_limit, clear_tail=False) self.block_hashes = Cache(max_size=self.block_hashes_cache_limit) self.block_hashes_preload_mutex = False self.tx_cache = MRU(self.tx_cache_limit) self.block_headers_cache = Cache( max_size=self.block_headers_cache_limit) self.block_txs_request = None self.connected = asyncio.Future() self.await_tx = list() self.missed_tx = list() self.await_tx_future = dict() self.add_tx_future = dict() self.get_missed_tx_threads = 0 self.get_missed_tx_threads_limit = rpc_threads_limit self.tx_in_process = set() self.zmqContext = None self.tasks = list() self.log.info("Node connector started") asyncio.ensure_future(self.start(), loop=self.loop)