async def fetch_block_transactions(self, block):
q = time.time()
missed = set()
tx_count = len(block["tx"])
for h in block["tx"]:
try:
self.tx_cache[h]
except:
missed.add(h)
if self.utxo_data:
if self.db_type == "postgresql":
async with self.db_pool.acquire() as conn:
rows = await conn.fetch(
"SELECT distinct tx_id FROM connector_unconfirmed_stxo "
"WHERE tx_id = ANY($1);", set(s2rh(t) for t in missed))
for row in rows:
missed.remove(rh2s(row["tx_id"]))
if missed:
coinbase = await conn.fetchval(
"SELECT out_tx_id FROM connector_unconfirmed_utxo "
"WHERE out_tx_id = $1 LIMIT 1;",
s2rh(block["tx"][0]))
if coinbase:
if block["tx"][0] in missed:
missed.remove(block["tx"][0])
self.log.debug("Block missed transactions %s from %s" %
(len(missed), tx_count))
if missed:
self.missed_tx = set(missed)
self.await_tx = set(missed)
self.await_tx_future = {s2rh(i): asyncio.Future() for i in missed}
self.block_txs_request = asyncio.Future()
self.block_timestamp = block["time"]
self.loop.create_task(self._get_missed())
try:
await asyncio.wait_for(self.block_txs_request,
timeout=self.block_timeout)
except asyncio.CancelledError:
# refresh rpc connection session
try:
await self.rpc.close()
self.rpc = aiojsonrpc.rpc(self.rpc_url,
self.loop,
timeout=self.rpc_timeout)
except:
pass
raise RuntimeError("block transaction request timeout")
self.total_received_tx += tx_count
self.total_received_tx_last += tx_count
self.total_received_tx_time += time.time() - q
rate = round(self.total_received_tx / self.total_received_tx_time)
self.log.debug(
"Transactions received: %s [%s] received tx rate tx/s ->> %s <<" %
(tx_count, time.time() - q, rate))
async def get_next_block(self):
if not self.active:
return
if not self.get_next_block_mutex.done():
await self.get_next_block_mutex
try:
self.get_next_block_mutex = asyncio.Future()
if self.node_last_block <= self.last_block_height + self.backlog:
d = await self.rpc.getblockcount()
if d == self.node_last_block:
self.log.info("blockchain is synchronized backlog %s" % self.backlog)
return
else:
self.node_last_block = d
d = self.node_last_block - self.last_block_height
if d > self.deep_sync_limit:
if not self.deep_synchronization:
self.log.warning("Deep synchronization mode")
self.deep_synchronization = True
else:
if self.deep_synchronization:
self.log.warning("Normal synchronization mode")
self.deep_synchronization = False
h = await self.rpc.getblockhash(self.last_block_height + 1)
await self._get_block_by_hash(h)
except Exception as err:
self.log.error("get next block failed %s" % str(err))
finally:
self.get_next_block_mutex.set_result(True)
async def _send_and_wait_origin(self, operate, _tuple, timeout):
# operate 枚举
message_id = await self._send_to_linda_server(operate, _tuple)
'''
return future timeout
futurn 被消息循环队列释放
timeout
https://docs.python.org/3/library/asyncio-task.html#asyncio.wait_for
'''
f = asyncio.Future() # todo asyncio future
self.linda_wait_futures.append((message_id, f))
# todo 加入到队列里: (message_id, f) f.set_result(tuple)
try:
return await asyncio.wait_for(f,
timeout=timeout) # result() 非阻塞 查询状态
except asyncio.TimeoutError:
# print('timeout!')
raise asyncio.TimeoutError(
f'timeout: {timeout}; message_id: {message_id}')
async def _new_block(self, block):
if not self.active or not self.active_block.done() or self.last_block_height >= block["height"]:
return
self.active_block = asyncio.Future()
self.block_dependency_tx = 0
bin_block_hash = bytes_from_hex(block["hash"])
bin_prev_block_hash = block["previousblockhash"] if "previousblockhash" in block else None
block_height = int(block["height"])
self.log.info("New block %s %s" % (block_height, block["hash"]))
bt = tm()
bpt = 0
try:
# blockchain position check
if self.block_cache.get(bin_block_hash) is not None:
self.log.debug("duplicated block %s" % block["hash"])
return
if self.block_cache.get(bin_prev_block_hash) is None and self.last_block_height:
self.log.critical("Connector panic! Node out of sync no parent block in chain tail %s" % bin_prev_block_hash)
return
if self.last_block_height + 1 != block_height:
if self.orphan_handler:
tq = tm()
await self.orphan_handler(self.last_block_height)
self.log.info("orphan handler %s [%s]" % (self.last_block_height, tm(tq)))
self.block_cache.pop_last()
self.last_block_height -= 1
return
# add all block transactions
missed = set()
for h in block["tx"]:
if self.tx_cache.get(h) is None:
missed.add(h)
if self.before_block_handler:
q = time.time()
await self.before_block_handler(block)
bpt = time.time() - q
self.blocks_processing_time += bpt
self.log.info("Transactions missed %s" % len(missed))
cq = tm()
if missed:
self.log.debug("Request missed transactions")
self.missed_tx_list = set(missed)
self.await_tx_list = missed
self.await_tx_future = dict()
for i in missed:
self.await_tx_future[i] = asyncio.Future()
self.block_txs_request = asyncio.Future()
if self.deep_synchronization:
self.loop.create_task(self._get_missed(block["hash"], block["time"], block["height"]))
else:
self.loop.create_task(self._get_missed(False, block["time"], block["height"]))
try:
await asyncio.wait_for(self.block_txs_request, timeout=self.block_timeout)
except asyncio.CancelledError:
# refresh rpc connection session
await self.rpc.close()
self.rpc = aiojsonrpc.rpc(self.rpc_url, self.loop, timeout=self.rpc_timeout)
raise RuntimeError("block transaction request timeout")
tx_count = len(block["tx"])
self.total_received_tx += tx_count
self.total_received_tx_time += tm(cq)
rate = round(self.total_received_tx/self.total_received_tx_time)
self.log.info("Transactions received: %s [%s] rate tx/s ->> %s <<" % (tx_count, tm(cq), rate))
if self.block_handler:
q = time.time()
await self.block_handler(block)
self.blocks_processing_time += time.time() - q
bpt += time.time() - q
# insert new block
self.block_cache.set(block["hash"], block["height"])
self.last_block_height = block["height"]
# after block added handler
if self.after_block_handler:
q = time.time()
try:
await self.after_block_handler(block)
except:
pass
self.blocks_processing_time += time.time() - q
bpt += time.time() - q
self.blocks_processed_count += 1
[self.tx_cache.pop(h) for h in block["tx"]]
except Exception as err:
if self.await_tx_list:
self.await_tx_list = set()
self.log.error(str(traceback.format_exc()))
self.log.error("new block error %s" % str(err))
finally:
self.active_block.set_result(True)
self.log.info("> %s < block [%s tx/ %s size] (dp %s) processing time %s cache [%s/%s]" %
(block["height"],
len(block["tx"]),
block["size"] / 1000000,
self.block_dependency_tx,
tm(bt),
len(self.block_hashes_preload._store),
len(self.block_preload._store)))
if self.node_last_block > self.last_block_height:
self.loop.create_task(self.get_next_block())
def __init__(self, node_rpc_url, node_zerromq_url, logger,
last_block_height=0, chain_tail=None,
mempool_tx_list=None,
tx_handler=None, orphan_handler=None,
before_block_handler=None, block_handler=None, after_block_handler=None,
block_timeout=30,
deep_sync_limit=20, backlog=0, mempool_tx=True,
rpc_batch_limit=20, rpc_threads_limit=100, rpc_timeout=100,
preload=False):
self.loop = asyncio.get_event_loop()
# settings
self.log = logger
self.rpc_url = node_rpc_url
self.zmq_url = node_zerromq_url
self.orphan_handler = orphan_handler
self.block_timeout = block_timeout
self.tx_handler = tx_handler
self.before_block_handler = before_block_handler
self.block_handler = block_handler
self.after_block_handler = after_block_handler
self.deep_sync_limit = deep_sync_limit
self.backlog = backlog
self.mempool_tx = mempool_tx
self.chain_tail = list(chain_tail) if chain_tail else []
self.mempool_tx_list = list(mempool_tx_list) if mempool_tx_list else []
self.rpc_timeout = rpc_timeout
self.batch_limit = rpc_batch_limit
# state and stats
self.node_last_block = None
self.last_block_height = int(last_block_height) if int(last_block_height) else 0
self.deep_synchronization = False
self.block_dependency_tx = 0 # counter of tx that have dependencies in block
self.active = True
self.get_next_block_mutex = asyncio.Future()
self.get_next_block_mutex.set_result(True)
self.active_block = asyncio.Future()
self.active_block.set_result(True)
self.last_zmq_msg = int(time.time())
self.total_received_tx = 0
self.blocks_processed_count = 0
self.blocks_decode_time = 0
self.blocks_download_time = 0
self.blocks_processing_time = 0
self.total_received_tx_time = 0
# cache and system
self.preload = preload
self.block_preload = Cache(max_size=50000)
self.block_hashes_preload = Cache(max_size=50000)
self.tx_cache = Cache(max_size=50000)
self.block_cache = Cache(max_size=10000)
self.block_txs_request = None
self.connected = asyncio.Future()
self.await_tx_list = list()
self.missed_tx_list = list()
self.await_tx_future = dict()
self.add_tx_future = dict()
self.get_missed_tx_threads = 0
self.get_missed_tx_threads_limit = rpc_threads_limit
self.tx_in_process = set()
self.zmqContext = None
self.tasks = list()
self.log.info("Node connector started")
asyncio.ensure_future(self.start(), loop=self.loop)
async def _new_block(self, block):
if not self.active: return
tq = time.time()
if self.block_headers_cache.get(block["hash"]) is not None: return
if self.deep_synchronization:
block["height"] = self.last_block_height + 1
if self.last_block_height >= block["height"]: return
if not self.active_block.done(): return
try:
self.active_block = asyncio.Future()
if self.last_block_height < self.last_block_utxo_cached_height:
if not self.cache_loading:
self.log.info("Bootstrap UTXO cache ...")
self.cache_loading = True
else:
if self.cache_loading:
self.log.info("UTXO Cache bootstrap completed")
self.cache_loading = False
await self.verify_block_position(block)
if self.deep_synchronization:
await self._block_as_transactions_batch(block)
if not self.cache_loading or block[
"height"] > self.app_block_height_on_start:
if self.block_batch_handler:
t = time.time()
await self.block_batch_handler(block)
self.batch_handler += time.time() - t
if self.total_received_tx - self.total_received_tx_stat > 100000:
self.report_sync_process(block["height"])
if self.utxo_data:
if self.sync_utxo.len() > self.sync_utxo.size_limit:
if not self.sync_utxo.save_process:
if self.sync_utxo.checkpoints and not self.cache_loading:
if self.sync_utxo.checkpoints[0] < block[
"height"]:
self.sync_utxo.create_checkpoint(
block["height"],
self.app_last_block)
if self.sync_utxo.save_process:
self.loop.create_task(
self.sync_utxo.commit())
else:
# call before block handler
if self.before_block_handler:
await self.before_block_handler(block)
await self.fetch_block_transactions(block)
if self.utxo_data:
if self.db_type == "postgresql":
async with self.db_pool.acquire() as conn:
async with conn.transaction():
data = await self.uutxo.apply_block_changes(
[s2rh(h) for h in block["tx"]],
block["height"], conn)
block["mempoolInvalid"] = {
"tx": data["invalid_txs"],
"inputs": data["dbs_stxo"],
"outputs": data["dbs_uutxo"]
}
if self.block_handler:
await self.block_handler(block, conn)
await conn.execute(
"UPDATE connector_utxo_state SET value = $1 "
"WHERE name = 'last_block';",
block["height"])
await conn.execute(
"UPDATE connector_utxo_state SET value = $1 "
"WHERE name = 'last_cached_block';",
block["height"])
elif self.block_handler:
await self.block_handler(block, None)
self.block_headers_cache.set(block["hash"], block["height"])
self.last_block_height = block["height"]
self.app_last_block = block["height"]
self.blocks_processed_count += 1
# after block added handler
if self.after_block_handler:
if not self.cache_loading or block[
"height"] > self.app_block_height_on_start:
try:
await self.after_block_handler(block)
except:
pass
if not self.deep_synchronization:
self.log.info(
"Block %s -> %s; tx count %s;" %
(block["height"], block["hash"], len(block["tx"])))
except Exception as err:
if self.await_tx:
self.await_tx = set()
for i in self.await_tx_future:
if not self.await_tx_future[i].done():
self.await_tx_future[i].cancel()
self.await_tx_future = dict()
self.log.error("block %s error %s" % (block["height"], str(err)))
import traceback
print(traceback.format_exc())
finally:
if self.node_last_block > self.last_block_height:
self.get_next_block_mutex = True
self.loop.create_task(self.get_next_block())
self.blocks_processing_time += time.time() - tq
self.active_block.set_result(True)
def __init__(self,
node_rpc_url,
node_zerromq_url,
logger,
last_block_height=0,
chain_tail=None,
tx_handler=None,
orphan_handler=None,
before_block_handler=None,
block_handler=None,
after_block_handler=None,
block_batch_handler=None,
flush_app_caches_handler=None,
synchronization_completed_handler=None,
block_timeout=30,
deep_sync_limit=20,
backlog=0,
mempool_tx=True,
rpc_batch_limit=50,
rpc_threads_limit=100,
rpc_timeout=100,
utxo_data=False,
utxo_cache_size=1000000,
skip_opreturn=True,
block_cache_workers=4,
block_preload_cache_limit=1000 * 1000000,
block_preload_batch_size_limit=200000000,
block_hashes_cache_limit=200 * 1000000,
db_type=None,
db=None,
app_proc_title="Connector"):
self.loop = asyncio.get_event_loop()
# settings
self.log = logger
self.rpc_url = node_rpc_url
self.app_proc_title = app_proc_title
self.rpc_timeout = rpc_timeout
self.rpc_batch_limit = rpc_batch_limit
self.zmq_url = node_zerromq_url
self.orphan_handler = orphan_handler
self.block_timeout = block_timeout
self.tx_handler = tx_handler
self.skip_opreturn = skip_opreturn
self.before_block_handler = before_block_handler
self.block_handler = block_handler
self.after_block_handler = after_block_handler
self.block_batch_handler = block_batch_handler
self.flush_app_caches_handler = flush_app_caches_handler
self.synchronization_completed_handler = synchronization_completed_handler
self.block_preload_batch_size_limit = block_preload_batch_size_limit
self.deep_sync_limit = deep_sync_limit
self.backlog = backlog
self.mempool_tx = mempool_tx
self.db_type = db_type
self.db = db
self.utxo_cache_size = utxo_cache_size
self.block_cache_workers = block_cache_workers
self.utxo_data = utxo_data
self.chain_tail = list(chain_tail) if chain_tail else []
# state and stats
self.node_last_block = None
self.sync_utxo = None
self.uutxo = None
self.cache_loading = False
self.app_block_height_on_start = int(last_block_height) if int(
last_block_height) else 0
self.last_block_height = 0
self.last_block_utxo_cached_height = 0
self.deep_synchronization = False
self.block_dependency_tx = 0 # counter of tx that have dependencies in block
self.active = True
self.get_next_block_mutex = False
self.active_block = asyncio.Future()
self.active_block.set_result(True)
self.last_zmq_msg = int(time.time())
self.total_received_tx = 0
self.total_received_tx_stat = 0
self.blocks_processed_count = 0
self.blocks_decode_time = 0
self.blocks_download_time = 0
self.blocks_processing_time = 0
self.tx_processing_time = 0
self.non_cached_blocks = 0
self.total_received_tx_time = 0
self.coins = 0
self.op_return = 0
self.destroyed_coins = 0
self.preload_cached_total = 0
self.preload_cached = 0
self.preload_cached_annihilated = 0
self.start_time = time.time()
self.total_received_tx_last = 0
self.start_time_last = time.time()
self.batch_time = time.time()
self.batch_load_utxo = 0
self.batch_parsing = 0
self.batch_handler = 0
self.app_last_block = None
# cache and system
self.block_preload_cache_limit = block_preload_cache_limit
self.block_hashes_cache_limit = block_hashes_cache_limit
self.tx_cache_limit = 144 * 5000
self.block_headers_cache_limit = 100 * 100000
self.block_preload = Cache(max_size=self.block_preload_cache_limit,
clear_tail=False)
self.block_hashes = Cache(max_size=self.block_hashes_cache_limit)
self.block_hashes_preload_mutex = False
self.tx_cache = MRU(self.tx_cache_limit)
self.block_headers_cache = Cache(
max_size=self.block_headers_cache_limit)
self.block_txs_request = None
self.connected = asyncio.Future()
self.await_tx = list()
self.missed_tx = list()
self.await_tx_future = dict()
self.add_tx_future = dict()
self.get_missed_tx_threads = 0
self.get_missed_tx_threads_limit = rpc_threads_limit
self.tx_in_process = set()
self.zmqContext = None
self.tasks = list()
self.log.info("Node connector started")
asyncio.ensure_future(self.start(), loop=self.loop)