async def start(self):
try:
conn = await asyncpg.connect(dsn=self.postgresql_dsn)
await init_db(conn)
self.last_tx_id = await get_last_tx_id(conn)
self.last_block_id = await get_last_block_id(conn)
self.last_block_height = await get_last_block_height(conn)
await load_tx_cache(self, conn)
self.log.info('tx cache len %s' % (self.tx_cache.len()))
await load_pending_cache(self, conn)
self.log.info('pending cache len %s' % (self.pending_cache.len()))
await load_block_cache(self, conn)
await conn.close()
self._db_pool = await \
asyncpg.create_pool(dsn=self.postgresql_dsn, loop=self.loop, min_size=1, max_size=self.postgresql_pool_max_size)
except Exception as err:
self.log.error("Start failed")
self.log.error(str(traceback.format_exc()))
await self.stop()
return
self.rpc = aiojsonrpc.rpc(self.rpc_url,
self.loop,
timeout=self.rpc_timeout)
if self.trace_rpc_url:
self.trace_rpc = aiojsonrpc.rpc(self.trace_rpc_url,
self.loop,
timeout=self.rpc_timeout)
else:
self.trace_rpc = None
self.websocket = self.loop.create_task(self.websocket_client())
self._watchdog = self.loop.create_task(self.watchdog())
if self.preload:
self.loop.create_task(self.preload_block())
self.loop.create_task(self.get_last_block())
async def start(self):
try:
conn = await asyncpg.connect(dsn=self.postgresql_dsn)
await init_db(conn)
self.last_tx_id = await get_last_tx_id(conn)
self.last_block_id = await get_last_block_id(conn)
self.last_block_height = await get_last_block_height(conn)
if not self.external_dublicate_filter:
await load_tx_cache(self, conn)
elif self.external_cache_loader:
await self.external_cache_loader(self.tx_cache, conn)
await load_block_cache(self, conn)
await conn.close()
self._db_pool = await asyncpg.create_pool(dsn=self.postgresql_dsn,
min_size=1,
max_size=self.postgresql_pool_max_size)
except Exception as err:
self.log.error("Bitcoind connector start failed: %s", err)
self.log.debug(str(traceback.format_exc()))
await self.stop()
return
self.rpc = aiojsonrpc.rpc(self.rpc_url, self.loop, timeout=self.rpc_timeout)
self.zmqContext = zmq.asyncio.Context()
self.zmqSubSocket = self.zmqContext.socket(zmq.SUB)
self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashblock")
self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "rawtx")
self.zmqSubSocket.connect(self.zmq_url)
self.tasks.append(self.loop.create_task(self.zeromq_handler()))
self.tasks.append(self.loop.create_task(self.watchdog()))
self.connected.set_result(True)
if self.preload:
self.loop.create_task(self.preload_block())
self.loop.create_task(self.preload_block_hashes())
self.loop.create_task(self.get_last_block())
async def message_loop(self):
try:
self.rpc = aiojsonrpc.rpc(self.rpc_url,
self.loop,
timeout=self.rpc_timeout)
if self.dsn:
self.db = await asyncpg.create_pool(dsn=self.dsn,
min_size=1,
max_size=1)
self.reader = await self.get_pipe_reader(self.in_reader)
self.writer = await self.get_pipe_writer(self.out_writer)
while True:
msg_type, msg = await self.pipe_get_msg(self.reader)
if msg_type == b'pipe_read_error':
return
if msg_type == b'get':
self.loop.create_task(
self.load_blocks(bytes_to_int(msg),
self.rpc_batch_limit))
continue
if msg_type == b'rpc_batch_limit':
self.rpc_batch_limit = bytes_to_int(msg)
continue
except:
pass
async def fetch_block_transactions(self, block):
q = time.time()
missed = set()
tx_count = len(block["tx"])
for h in block["tx"]:
try:
self.tx_cache[h]
except:
missed.add(h)
if self.utxo_data:
if self.db_type == "postgresql":
async with self.db_pool.acquire() as conn:
rows = await conn.fetch(
"SELECT distinct tx_id FROM connector_unconfirmed_stxo "
"WHERE tx_id = ANY($1);", set(s2rh(t) for t in missed))
for row in rows:
missed.remove(rh2s(row["tx_id"]))
if missed:
coinbase = await conn.fetchval(
"SELECT out_tx_id FROM connector_unconfirmed_utxo "
"WHERE out_tx_id = $1 LIMIT 1;",
s2rh(block["tx"][0]))
if coinbase:
if block["tx"][0] in missed:
missed.remove(block["tx"][0])
self.log.debug("Block missed transactions %s from %s" %
(len(missed), tx_count))
if missed:
self.missed_tx = set(missed)
self.await_tx = set(missed)
self.await_tx_future = {s2rh(i): asyncio.Future() for i in missed}
self.block_txs_request = asyncio.Future()
self.block_timestamp = block["time"]
self.loop.create_task(self._get_missed())
try:
await asyncio.wait_for(self.block_txs_request,
timeout=self.block_timeout)
except asyncio.CancelledError:
# refresh rpc connection session
try:
await self.rpc.close()
self.rpc = aiojsonrpc.rpc(self.rpc_url,
self.loop,
timeout=self.rpc_timeout)
except:
pass
raise RuntimeError("block transaction request timeout")
self.total_received_tx += tx_count
self.total_received_tx_last += tx_count
self.total_received_tx_time += time.time() - q
rate = round(self.total_received_tx / self.total_received_tx_time)
self.log.debug(
"Transactions received: %s [%s] received tx rate tx/s ->> %s <<" %
(tx_count, time.time() - q, rate))
async def init_db_pool(app):
# init db pool
app["loop"] = asyncio.get_event_loop()
app["log"].info("Init db pool ... ")
app["rpc"] = aiojsonrpc.rpc(app["node_rpc_url"], app["loop"])
app["db_pool"] = await \
asyncpg.create_pool(dsn=app["dsn"],
loop=app["loop"],
min_size=10, max_size=app["pool_threads"])
app["service_db_pool"] = await \
asyncpg.create_pool(dsn=app["dsn"],
loop=app["loop"],
min_size=1, max_size=1)
async def start(self):
while True:
self.log.info("Connector initialization")
try:
self.rpc = aiojsonrpc.rpc(self.rpc_url, self.loop, timeout=self.rpc_timeout)
self.node_last_block = await self.rpc.getblockcount()
except Exception as err:
self.log.error("Get node best block error:" + str(err))
if not isinstance(self.node_last_block, int):
self.log.error("Get node best block height failed")
self.log.error("Node rpc url: "+self.rpc_url)
await asyncio.sleep(10)
continue
self.log.info("Node best block height %s" %self.node_last_block)
self.log.info("Connector last block height %s" % self.last_block_height)
if self.node_last_block < self.last_block_height:
self.log.error("Node is behind application blockchain state!")
await asyncio.sleep(10)
continue
elif self.node_last_block == self.last_block_height:
self.log.warning("Blockchain is synchronized")
else:
d = self.node_last_block - self.last_block_height
self.log.warning("%s blocks before synchronization synchronized")
if d > self.deep_sync_limit:
self.log.warning("Deep synchronization mode")
self.deep_synchronization = True
break
[self.tx_cache.set(row, True) for row in self.mempool_tx_list]
h = self.last_block_height
if h < len(self.chain_tail):
raise Exception("Chain tail len not match last block height")
for row in reversed(self.chain_tail):
self.block_cache.set(row, h)
h -= 1
self.tasks.append(self.loop.create_task(self.zeromq_handler()))
self.tasks.append(self.loop.create_task(self.watchdog()))
self.connected.set_result(True)
if self.preload:
self.loop.create_task(self.preload_block())
self.loop.create_task(self.preload_block_hashes())
self.loop.create_task(self.get_next_block())
async def start(self):
try:
conn = await asyncpg.connect(dsn=self.postgresql_dsn)
await init_db(conn)
self.last_tx_id = await get_last_tx_id(conn)
self.last_block_id = await get_last_block_id(conn)
self.last_block_height = await get_last_block_height(conn)
if not self.external_dublicate_filter:
await load_tx_cache(self, conn)
elif self.external_cache_loader:
await self.external_cache_loader(self.tx_cache, conn)
await load_block_cache(self, conn)
await conn.close()
self._db_pool = await asyncpg.create_pool(
dsn=self.postgresql_dsn,
min_size=1,
max_size=self.postgresql_pool_max_size)
except Exception as err:
self.log.error("Bitcoind connector start failed: %s", err)
self.log.debug(str(traceback.format_exc()))
await self.stop()
return
self.rpc = aiojsonrpc.rpc(self.rpc_url,
self.loop,
timeout=self.rpc_timeout)
self.zmqContext = zmq.asyncio.Context()
self.zmqSubSocket = self.zmqContext.socket(zmq.SUB)
self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashblock")
self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "rawtx")
self.zmqSubSocket.connect(self.zmq_url)
self.tasks.append(self.loop.create_task(self.zeromq_handler()))
self.tasks.append(self.loop.create_task(self.watchdog()))
self.connected.set_result(True)
if self.preload:
self.loop.create_task(self.preload_block())
self.loop.create_task(self.preload_block_hashes())
self.loop.create_task(self.get_last_block())
async def _new_block(self, block):
self.block_dependency_tx = 0
"""
0 Check if block already exist in db
1 Check parent block in db:
If no parent
get last block height from db
if last block height >= recent block height
this is orphan ignore it
else:
remove top block from db and ask block with
hrecent block height -1
return
2 add all transactions from block to db
ask full block from node
parse txs and add to db in case not exist
3 call before add block handler^ if this handler rise
exception block adding filed
4 add block to db and commit
5 after block add handelr
6 ask next block
"""
if not self.active or not self.active_block.done():
return
if block is None:
self.sync = False
self.log.debug('Block synchronization completed')
return
self.active_block = asyncio.Future()
binary_block_hash = unhexlify(block["hash"])
binary_previousblock_hash = \
unhexlify(block["previousblockhash"]) \
if "previousblockhash" in block else None
block_height = int(block["height"])
next_block_height = block_height + 1
self.log.info("New block %s %s" % (block_height, block["hash"]))
bt = q = tm()
try:
async with self._db_pool.acquire() as con:
# blockchain position check
block_exist = self.block_cache.get(binary_block_hash)
if block_exist is not None:
self.log.info("block already exist in db %s" % block["hash"])
return
# Get parent from db
if binary_previousblock_hash is not None:
parent_height = self.block_cache.get(binary_previousblock_hash)
else:
parent_height = None
# self.log.warning("parent height %s" % parent_height)
if parent_height is None:
# have no mount point in local chain
# self.log.warning("last local height %s" % self.last_block_height)
if self.last_block_height is not None:
if self.last_block_height >= block_height:
self.log.critical("bitcoin node out of sync block %s" % block["hash"])
return
if self.last_block_height+1 == block_height:
if self.orphan_handler:
tq = tm()
await self.orphan_handler(self.last_block_height, con)
self.log.info("orphan handler %s [%s]" % (self.last_block_height, tm(tq)))
tq = tm()
await remove_orphan(self, con)
self.log.info("remove orphan %s [%s]" % (self.last_block_height + 1, tm(tq)))
next_block_height -= 2
if next_block_height > self.last_block_height:
next_block_height = self.last_block_height + 1
if self.sync and next_block_height >= self.sync:
if self.sync_requested:
next_block_height = self.last_block_height + 1
else:
self.sync = next_block_height
return
else:
if self.start_block is not None and block["height"] != self.start_block:
self.log.info("Start from block %s" % self.start_block)
next_block_height = self.start_block
return
else:
if self.last_block_height + 1 != block_height:
if self.orphan_handler:
tq = tm()
await self.orphan_handler(self.last_block_height, con)
self.log.info("orphan handler %s [%s]" % (self.last_block_height, tm(tq)))
await remove_orphan(self, con)
next_block_height -= 1
self.log.debug("requested %s" % next_block_height)
return
self.log.debug("blockchain position check [%s]" % tm(q))
# add all block transactions
q = tm()
binary_tx_hash_list = [unhexlify(t)[::-1] for t in block["tx"]]
if block["height"] in (91842, 91880):
# BIP30 Fix
self.tx_cache.pop(s2rh("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599"))
self.tx_cache.pop(s2rh("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468"))
tx_id_list, missed = await get_tx_id_list(self, binary_tx_hash_list, con)
if len(tx_id_list)+len(missed) != len(block["tx"]):
raise Exception("tx count mismatch")
self.await_tx_id_list = tx_id_list
if self.before_block_handler:
sn = await self.before_block_handler(block,
missed,
self.sync_tx_lock,
self.node_last_block,
con)
if sn and missed:
self.await_tx_id_list = self.await_tx_id_list + [0 for i in range(len(missed))]
missed = []
cq = tm()
missed = [rh2s(t) for t in missed]
self.log.info("Transactions already exist: %s missed %s [%s]" % (len(tx_id_list), len(missed), tm(q)))
if missed:
self.log.debug("Request missed transactions")
self.missed_tx_list = list(missed)
self.await_tx_list = missed
self.await_tx_future = dict()
for i in missed:
self.await_tx_future[unhexlify(i)[::-1]] = asyncio.Future()
self.block_txs_request = asyncio.Future()
if len(missed) == len(block["tx"]):
self.loop.create_task(self._get_missed(block["hash"],
block["time"],
block["height"]
))
else:
self.loop.create_task(self._get_missed(False,
block["time"],
block["height"]
))
try:
await asyncio.wait_for(self.block_txs_request, timeout=self.block_timeout)
except asyncio.CancelledError:
# refresh rpc connection session
await self.rpc.close()
self.rpc = aiojsonrpc.rpc(self.rpc_url, self.loop, timeout=self.rpc_timeout)
raise RuntimeError("block transaction request timeout")
if len(block["tx"]) != len(self.await_tx_id_list):
self.log.error("get block transactions failed")
self.log.error(str(self.await_tx_id_list))
raise Exception("get block transactions failed")
tx_count = len(self.await_tx_id_list)
self.total_received_tx += tx_count
self.total_received_tx_time += tm(q)
rate = round(self.total_received_tx/self.total_received_tx_time)
self.log.info("Transactions received: %s [%s] rate tx/s ->> %s <<" % (tx_count, tm(cq), rate))
async with con.transaction():
if self.block_received_handler:
await self.block_received_handler(block, con)
# insert new block
await insert_new_block(self, binary_block_hash,
block["height"],
binary_previousblock_hash,
block["time"], con)
if not self.external_dublicate_filter:
self.loop.create_task(update_block_height(self, block["height"],
list(self.await_tx_id_list)))
if self.sync == block["height"]:
self.sync += 1
next_block_height = self.sync
# after block added handler
if self.block_handler:
await self.block_handler(block, con)
self.last_block_height = block["height"]
self.block_cache.set(binary_block_hash, block["height"])
except Exception as err:
if self.await_tx_list:
self.await_tx_list = []
self.log.error(str(traceback.format_exc()))
self.log.error("new block error %s" % str(err))
next_block_height = None
finally:
self.active_block.set_result(True)
self.log.debug("block processing completed")
if next_block_height is not None:
self.sync_requested = True
self.loop.create_task(self.get_block_by_height(next_block_height))
self.log.info("%s block [%s tx/ %s size] (dp %s) processing time %s cache [%s/%s]" %
(block["height"],
len(block["tx"]),
block["size"] / 1000000,
self.block_dependency_tx,
tm(bt),
len(self.block_hashes_preload._store),
len(self.block_preload._store)))
async def load_blocks(self, height, limit):
start_height = height
start_limit = limit
self.destroyed_coins = MRU()
self.coins = MRU()
try:
self.rpc = aiojsonrpc.rpc(self.rpc_url, self.loop, timeout=self.rpc_timeout)
blocks, missed = dict(), deque()
v, t, limit = height + limit, 0, 30
while height < v and height <= self.target_height:
batch, h_list = list(), list()
while len(batch) < limit and height < v and height <= self.target_height:
batch.append(["getblockhash", height])
h_list.append(height)
height += 1
result = await self.rpc.batch(batch)
h, batch = list(), list()
for lh, r in zip(h_list, result):
if r["result"] is not None:
batch.append(["getblock", r["result"], 0])
h.append(lh)
result = await self.rpc.batch(batch)
for x, y in zip(h, result):
if y["result"] is not None:
block = decode_block_tx(y["result"])
block["p2pkMapHash"] = []
if self.option_tx_map:
block["txMap"], block["stxo"] = set(), deque()
if self.option_block_filters:
block["filter"] = set()
if self.option_analytica:
block["stat"] = {"inputs": {"count": 0,
"amount": {"max": {"value": None, "txId": None},
"min": {"value": None, "txId": None},
"total": 0},
"typeMap": {}},
"outputs": {"count": 0,
"amount": {"max": {"value": None,
"txId": None},
"min": {"value": None,
"txId": None},
"total": 0},
"typeMap": {}},
"transactions": {"count": 0,
"amount": {"max": {"value": None, "txId": None},
"min": {"value": None, "txId": None},
"total": 0},
"size": {"max": {"value": None, "txId": None},
"min": {"value": None, "txId": None},
"total": 0},
"vSize": {"max": {"value": None, "txId": None},
"min": {"value": None, "txId": None},
"total": 0},
"fee": {"max": {"value": None, "txId": None},
"min": {"value": None, "txId": None},
"total": 0},
"feeRate": {"max": {"value": None, "txId": None},
"min": {"value": None, "txId": None}},
"amountMap": {},
"feeRateMap": {},
"typeMap": {"segwit": {"count": 0,
"amount": 0,
"size": 0},
"rbf": {"count": 0,
"amount": 0,
"size": 0}}}
}
if self.option_merkle_proof:
mt = merkle_tree(block["rawTx"][i]["txId"] for i in block["rawTx"])
coinbase = block["rawTx"][0]["vIn"][0]["scriptSig"]
block["miner"] = None
for tag in MINER_COINBASE_TAG:
if coinbase.find(tag) != -1:
block["miner"] = json.dumps(MINER_COINBASE_TAG[tag])
break
else:
try:
address_hash = block["rawTx"][0]["vOut"][0]["addressHash"]
script_hash = False if block["rawTx"][0]["vOut"][0]["nType"] == 1 else True
a = hash_to_address(address_hash, script_hash=script_hash)
if a in MINER_PAYOUT_TAG:
block["miner"] = json.dumps(MINER_PAYOUT_TAG[a])
except:
pass
if self.utxo_data:
# handle outputs
for z in block["rawTx"]:
if self.option_merkle_proof:
block["rawTx"][z]["merkleProof"] = b''.join(merkle_proof(mt, z, return_hex=False))
tx_pointer = (x << 39)+(z << 20)
for i in block["rawTx"][z]["vOut"]:
out= block["rawTx"][z]["vOut"][i]
o = b"".join((block["rawTx"][z]["txId"], int_to_bytes(i)))
pointer = (x << 39)+(z << 20)+(1 << 19) + i
out_type = out["nType"]
try:
if out_type == 2:
block["p2pkMapHash"].append((out["addressHash"], out["scriptPubKey"]))
raise Exception("P2PK")
address = b"".join((bytes([out_type]), out["addressHash"]))
except:
address = b"".join((bytes([out_type]), out["scriptPubKey"]))
if out_type in (0, 1, 2, 5, 6):
if self.option_block_filters:
e = b"".join((bytes([out_type]),
z.to_bytes(4, byteorder="little"),
out["addressHash"]))
block["filter"].add(e)
if self.option_tx_map:
block["txMap"].add((address, tx_pointer))
out["_address"] = address
self.coins[o] = (pointer, out["value"], address)
if self.option_analytica:
tx = block["rawTx"][z]
out_stat = block["stat"]["outputs"]
out_stat["count"] += 1
out_stat["amount"]["total"] += out["value"]
if out_stat["amount"]["min"]["value"] is None or \
out_stat["amount"]["min"]["value"] > out["value"]:
if out["value"] > 0:
out_stat["amount"]["min"]["value"] = out["value"]
out_stat["amount"]["min"]["txId"] = rh2s(tx["txId"])
out_stat["amount"]["max"]["vOut"] = i
if out_stat["amount"]["max"]["value"] is None or \
out_stat["amount"]["max"]["value"] < out["value"]:
out_stat["amount"]["max"]["value"] = out["value"]
out_stat["amount"]["max"]["txId"] = rh2s(tx["txId"])
out_stat["amount"]["max"]["vOut"] = i
key = None if out["value"] == 0 else str(math.floor(math.log10(out["value"])))
out_type = SCRIPT_N_TYPES[out_type]
a = out["value"]
try:
out_stat["typeMap"][out_type]["count"] += 1
out_stat["typeMap"][out_type]["amount"] += a
except:
out_stat["typeMap"][out_type] = {"count": 1, "amount": a, "amountMap": {}}
try:
out_stat["typeMap"][out_type]["amountMap"][key]["count"] += 1
out_stat["typeMap"][out_type]["amountMap"][key]["amount"] += a
except:
out_stat["typeMap"][out_type]["amountMap"][key] = {"count": 1, "amount": a}
if self.option_analytica:
tx = block["rawTx"][z]
tx["inputsAmount"] = 0
tx_stat = block["stat"]["transactions"]
tx_stat["count"] += 1
for k in ("amount", "size", "vSize"):
tx_stat[k]["total"] += tx[k]
if tx_stat[k]["min"]["value"] is None or tx_stat[k]["min"]["value"] > tx[k]:
tx_stat[k]["min"]["value"] = tx[k]
tx_stat[k]["min"]["txId"] = rh2s(tx["txId"])
if tx_stat[k]["max"]["value"] is None or tx_stat[k]["max"]["value"] < tx[k]:
tx_stat[k]["max"]["value"] = tx[k]
tx_stat[k]["max"]["txId"] = rh2s(tx["txId"])
key = None if tx["amount"] == 0 else str(math.floor(math.log10(tx["amount"])))
try:
tx_stat["amountMap"][key]["count"] += 1
tx_stat["amountMap"][key]["amount"] += tx["amount"]
tx_stat["amountMap"][key]["size"] += tx["amount"]
except:
tx_stat["amountMap"][key] = {"count": 1,
"amount": tx["amount"],
"size": tx["amount"]}
if tx["segwit"]:
tx_stat["typeMap"]["segwit"]["count"] += 1
tx_stat["typeMap"]["segwit"]["amount"] += tx["amount"]
tx_stat["typeMap"]["segwit"]["size"] += tx["size"]
if tx["rbf"]:
tx_stat["typeMap"]["rbf"]["count"] += 1
tx_stat["typeMap"]["rbf"]["amount"] += tx["amount"]
tx_stat["typeMap"]["rbf"]["size"] += tx["size"]
# handle inputs
for z in block["rawTx"]:
if not block["rawTx"][z]["coinbase"]:
for i in block["rawTx"][z]["vIn"]:
inp = block["rawTx"][z]["vIn"][i]
outpoint = b"".join((inp["txId"], int_to_bytes(inp["vOut"])))
block["rawTx"][z]["vIn"][i]["_outpoint"] = outpoint
tx_pointer = (x<<39)+(z<<20)
try:
r = self.coins.delete(outpoint)
block["rawTx"][z]["vIn"][i]["_a_"] = r
self.destroyed_coins[r[0]] = True
out_type = r[2][0]
if self.option_block_filters:
if out_type in (0, 1, 5, 6):
e = b"".join((bytes([out_type]),
z.to_bytes(4, byteorder="little"),
r[2][1:]))
block["filter"].add(e)
elif out_type == 2:
a = parse_script(r[2][1:])["addressHash"]
e = b"".join((bytes([out_type]),
z.to_bytes(4, byteorder="little"),
a[:20]))
block["filter"].add(e)
if self.option_tx_map:
block["txMap"].add((r[2], tx_pointer))
block["stxo"].append((r[0], (x<<39)+(z<<20)+i, r[2], r[1]))
t += 1
if self.option_analytica:
a = r[1]
in_type = SCRIPT_N_TYPES[r[2][0]]
try:
tx = block["rawTx"][z]
input_stat = block["stat"]["inputs"]
input_stat["count"] += 1
tx["inputsAmount"] += a
input_stat["amount"]["total"] += a
if input_stat["amount"]["min"]["value"] is None or \
input_stat["amount"]["min"]["value"] > a:
input_stat["amount"]["min"]["value"] = a
input_stat["amount"]["min"]["txId"] = rh2s(tx["txId"])
input_stat["amount"]["min"]["vIn"] = i
if input_stat["amount"]["max"]["value"] is None or \
input_stat["amount"]["max"]["value"] < a:
input_stat["amount"]["max"]["value"] = a
input_stat["amount"]["max"]["txId"] = rh2s(tx["txId"])
input_stat["amount"]["max"]["vIn"] = i
key = None if a == 0 else str(math.floor(math.log10(a)))
try:
input_stat["typeMap"][in_type]["count"] += 1
input_stat["typeMap"][in_type]["amount"] += a
except:
input_stat["typeMap"][in_type] = {"count": 1, "amount": a,
"amountMap": {}}
try:
input_stat["typeMap"][in_type]["amountMap"][key]["count"] += 1
input_stat["typeMap"][in_type]["amountMap"][key]["amount"] += a
except:
input_stat["typeMap"][in_type]["amountMap"][key] = {"count": 1,
"amount": a}
except:
print(traceback.format_exc())
except:
if self.dsn:
missed.append(outpoint)
blocks[x] = block
m, n = 0, 0
if self.utxo_data and missed and self.dsn:
async with self.db.acquire() as conn:
rows = await conn.fetch("SELECT outpoint, "
" pointer,"
" address,"
" amount "
"FROM connector_utxo "
"WHERE outpoint = ANY($1);", missed)
m += len(rows)
p = dict()
for row in rows:
p[row["outpoint"]] = (row["pointer"], row["amount"], row["address"])
for h in blocks:
for z in blocks[h]["rawTx"]:
tx_pointer = (h<<39)+(z<<20)
if not blocks[h]["rawTx"][z]["coinbase"]:
for i in blocks[h]["rawTx"][z]["vIn"]:
outpoint = blocks[h]["rawTx"][z]["vIn"][i]["_outpoint"]
try:
blocks[h]["rawTx"][z]["vIn"][i]["_l_"] = p[outpoint]
try:
out_type = p[outpoint][2][0]
if self.option_block_filters:
if out_type in (0, 1, 5, 6):
e = b"".join((bytes([out_type]),
z.to_bytes(4, byteorder="little"),
p[outpoint][2][1:]))
blocks[h]["filter"].add(e)
elif out_type == 2:
a = parse_script(p[outpoint][2][1:])["addressHash"]
e = b"".join((bytes([out_type]),
z.to_bytes(4, byteorder="little"),
a[:20]))
blocks[h]["filter"].add(e)
if self.option_tx_map:
blocks[h]["txMap"].add((p[outpoint][2], tx_pointer))
blocks[h]["stxo"].append((p[outpoint][0],
(h<<39)+(z<<20)+i,
p[outpoint][2],
p[outpoint][1]))
if self.option_analytica:
a = p[outpoint][1]
in_type = SCRIPT_N_TYPES[p[outpoint][2][0]]
tx = blocks[h]["rawTx"][z]
input_stat = blocks[h]["stat"]["inputs"]
input_stat["count"] += 1
tx["inputsAmount"] += a
input_stat["amount"]["total"] += a
if input_stat["amount"]["min"]["value"] is None or \
input_stat["amount"]["min"]["value"] > a:
input_stat["amount"]["min"]["value"] = a
input_stat["amount"]["min"]["txId"] = rh2s(tx["txId"])
input_stat["amount"]["min"]["vIn"] = i
if input_stat["amount"]["max"]["value"] is None or \
input_stat["amount"]["max"]["value"] < a:
input_stat["amount"]["max"]["value"] = a
input_stat["amount"]["max"]["txId"] = rh2s(tx["txId"])
input_stat["amount"]["max"]["vIn"] = i
key = None if a == 0 else str(math.floor(math.log10(a)))
try:
input_stat["typeMap"][in_type]["count"] += 1
input_stat["typeMap"][in_type]["amount"] += a
except:
input_stat["typeMap"][in_type] = {"count": 1, "amount": a,
"amountMap": {}}
try:
input_stat["typeMap"][in_type]["amountMap"][key]["count"] += 1
input_stat["typeMap"][in_type]["amountMap"][key]["amount"] += a
except:
input_stat["typeMap"][in_type]["amountMap"][key] = {"count": 1,
"amount": a}
t += 1
n += 1
except:
print(traceback.format_exc())
except:
pass
if self.utxo_data and blocks:
blocks[x]["checkpoint"] = x
for x in blocks:
if self.utxo_data:
for y in blocks[x]["rawTx"]:
for i in blocks[x]["rawTx"][y]["vOut"]:
try:
r = self.destroyed_coins.delete((x<<39)+(y<<20)+(1<<19)+i)
blocks[x]["rawTx"][y]["vOut"][i]["_s_"] = r
except:
pass
if self.option_block_filters:
blocks[x]["filter"] = bytearray(b"".join(blocks[x]["filter"]))
blocks[x] = pickle.dumps(blocks[x])
await self.pipe_sent_msg(b'result', pickle.dumps(blocks))
except concurrent.futures.CancelledError:
pass
except Exception as err:
self.log.error("block loader restarting: %s" % err)
print(traceback.format_exc())
await asyncio.sleep(1)
self.loop.create_task(self.load_blocks(start_height, start_limit))
finally:
try: await self.rpc.close()
except: pass
async def _new_block(self, block):
self.block_dependency_tx = 0
"""
0 Check if block already exist in db
1 Check parent block in db:
If no parent
get last block height from db
if last block height >= recent block height
this is orphan ignore it
else:
remove top block from db and ask block with
hrecent block height -1
return
2 add all transactions from block to db
ask full block from node
parse txs and add to db in case not exist
3 call before add block handler^ if this handler rise
exception block adding filed
4 add block to db and commit
5 after block add handelr
6 ask next block
"""
if not self.active or not self.active_block.done():
return
if block is None:
self.sync = False
self.log.debug('Block synchronization completed')
return
self.active_block = asyncio.Future()
binary_block_hash = unhexlify(block["hash"])
binary_previousblock_hash = \
unhexlify(block["previousblockhash"]) \
if "previousblockhash" in block else None
block_height = int(block["height"])
next_block_height = block_height + 1
self.log.info("New block %s %s" % (block_height, block["hash"]))
bt = q = tm()
try:
async with self._db_pool.acquire() as con:
# blockchain position check
block_exist = self.block_cache.get(binary_block_hash)
if block_exist is not None:
self.log.info("block already exist in db %s" %
block["hash"])
return
# Get parent from db
if binary_previousblock_hash is not None:
parent_height = self.block_cache.get(
binary_previousblock_hash)
else:
parent_height = None
# self.log.warning("parent height %s" % parent_height)
if parent_height is None:
# have no mount point in local chain
# self.log.warning("last local height %s" % self.last_block_height)
if self.last_block_height is not None:
if self.last_block_height >= block_height:
self.log.critical(
"bitcoin node out of sync block %s" %
block["hash"])
return
if self.last_block_height + 1 == block_height:
if self.orphan_handler:
tq = tm()
await self.orphan_handler(
self.last_block_height, con)
self.log.info("orphan handler %s [%s]" %
(self.last_block_height, tm(tq)))
tq = tm()
await remove_orphan(self, con)
self.log.info("remove orphan %s [%s]" %
(self.last_block_height + 1, tm(tq)))
next_block_height -= 2
if next_block_height > self.last_block_height:
next_block_height = self.last_block_height + 1
if self.sync and next_block_height >= self.sync:
if self.sync_requested:
next_block_height = self.last_block_height + 1
else:
self.sync = next_block_height
return
else:
if self.start_block is not None and block[
"height"] != self.start_block:
self.log.info("Start from block %s" %
self.start_block)
next_block_height = self.start_block
return
else:
if self.last_block_height + 1 != block_height:
if self.orphan_handler:
tq = tm()
await self.orphan_handler(self.last_block_height,
con)
self.log.info("orphan handler %s [%s]" %
(self.last_block_height, tm(tq)))
await remove_orphan(self, con)
next_block_height -= 1
self.log.debug("requested %s" % next_block_height)
return
self.log.debug("blockchain position check [%s]" % tm(q))
# add all block transactions
q = tm()
binary_tx_hash_list = [unhexlify(t)[::-1] for t in block["tx"]]
if block["height"] in (91842, 91880):
# BIP30 Fix
self.tx_cache.pop(
s2rh(
"d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599"
))
self.tx_cache.pop(
s2rh(
"e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468"
))
tx_id_list, missed = await get_tx_id_list(
self, binary_tx_hash_list, con)
if len(tx_id_list) + len(missed) != len(block["tx"]):
raise Exception("tx count mismatch")
self.await_tx_id_list = tx_id_list
if self.before_block_handler:
sn = await self.before_block_handler(
block, missed, self.sync_tx_lock, self.node_last_block,
con)
if sn and missed:
self.await_tx_id_list = self.await_tx_id_list + [
0 for i in range(len(missed))
]
missed = []
cq = tm()
missed = [rh2s(t) for t in missed]
self.log.info("Transactions already exist: %s missed %s [%s]" %
(len(tx_id_list), len(missed), tm(q)))
if missed:
self.log.debug("Request missed transactions")
self.missed_tx_list = list(missed)
self.await_tx_list = missed
self.await_tx_future = dict()
for i in missed:
self.await_tx_future[unhexlify(i)
[::-1]] = asyncio.Future()
self.block_txs_request = asyncio.Future()
if len(missed) == len(block["tx"]):
self.loop.create_task(
self._get_missed(block["hash"], block["time"],
block["height"]))
else:
self.loop.create_task(
self._get_missed(False, block["time"],
block["height"]))
try:
await asyncio.wait_for(self.block_txs_request,
timeout=self.block_timeout)
except asyncio.CancelledError:
# refresh rpc connection session
await self.rpc.close()
self.rpc = aiojsonrpc.rpc(self.rpc_url,
self.loop,
timeout=self.rpc_timeout)
raise RuntimeError("block transaction request timeout")
if len(block["tx"]) != len(self.await_tx_id_list):
self.log.error("get block transactions failed")
self.log.error(str(self.await_tx_id_list))
raise Exception("get block transactions failed")
tx_count = len(self.await_tx_id_list)
self.total_received_tx += tx_count
self.total_received_tx_time += tm(q)
rate = round(self.total_received_tx /
self.total_received_tx_time)
self.log.info(
"Transactions received: %s [%s] rate tx/s ->> %s <<" %
(tx_count, tm(cq), rate))
async with con.transaction():
if self.block_received_handler:
await self.block_received_handler(block, con)
# insert new block
await insert_new_block(self, binary_block_hash,
block["height"],
binary_previousblock_hash,
block["time"], con)
if not self.external_dublicate_filter:
self.loop.create_task(
update_block_height(self, block["height"],
list(self.await_tx_id_list)))
if self.sync == block["height"]:
self.sync += 1
next_block_height = self.sync
# after block added handler
if self.block_handler:
await self.block_handler(block, con)
self.last_block_height = block["height"]
self.block_cache.set(binary_block_hash, block["height"])
except Exception as err:
if self.await_tx_list:
self.await_tx_list = []
self.log.error(str(traceback.format_exc()))
self.log.error("new block error %s" % str(err))
next_block_height = None
finally:
self.active_block.set_result(True)
self.log.debug("block processing completed")
if next_block_height is not None:
self.sync_requested = True
self.loop.create_task(
self.get_block_by_height(next_block_height))
self.log.info(
"%s block [%s tx/ %s size] (dp %s) processing time %s cache [%s/%s]"
% (block["height"], len(block["tx"]), block["size"] / 1000000,
self.block_dependency_tx, tm(bt),
len(self.block_hashes_preload._store),
len(self.block_preload._store)))
async def _new_block(self, block):
if not self.active or not self.active_block.done() or self.last_block_height >= block["height"]:
return
self.active_block = asyncio.Future()
self.block_dependency_tx = 0
bin_block_hash = bytes_from_hex(block["hash"])
bin_prev_block_hash = block["previousblockhash"] if "previousblockhash" in block else None
block_height = int(block["height"])
self.log.info("New block %s %s" % (block_height, block["hash"]))
bt = tm()
bpt = 0
try:
# blockchain position check
if self.block_cache.get(bin_block_hash) is not None:
self.log.debug("duplicated block %s" % block["hash"])
return
if self.block_cache.get(bin_prev_block_hash) is None and self.last_block_height:
self.log.critical("Connector panic! Node out of sync no parent block in chain tail %s" % bin_prev_block_hash)
return
if self.last_block_height + 1 != block_height:
if self.orphan_handler:
tq = tm()
await self.orphan_handler(self.last_block_height)
self.log.info("orphan handler %s [%s]" % (self.last_block_height, tm(tq)))
self.block_cache.pop_last()
self.last_block_height -= 1
return
# add all block transactions
missed = set()
for h in block["tx"]:
if self.tx_cache.get(h) is None:
missed.add(h)
if self.before_block_handler:
q = time.time()
await self.before_block_handler(block)
bpt = time.time() - q
self.blocks_processing_time += bpt
self.log.info("Transactions missed %s" % len(missed))
cq = tm()
if missed:
self.log.debug("Request missed transactions")
self.missed_tx_list = set(missed)
self.await_tx_list = missed
self.await_tx_future = dict()
for i in missed:
self.await_tx_future[i] = asyncio.Future()
self.block_txs_request = asyncio.Future()
if self.deep_synchronization:
self.loop.create_task(self._get_missed(block["hash"], block["time"], block["height"]))
else:
self.loop.create_task(self._get_missed(False, block["time"], block["height"]))
try:
await asyncio.wait_for(self.block_txs_request, timeout=self.block_timeout)
except asyncio.CancelledError:
# refresh rpc connection session
await self.rpc.close()
self.rpc = aiojsonrpc.rpc(self.rpc_url, self.loop, timeout=self.rpc_timeout)
raise RuntimeError("block transaction request timeout")
tx_count = len(block["tx"])
self.total_received_tx += tx_count
self.total_received_tx_time += tm(cq)
rate = round(self.total_received_tx/self.total_received_tx_time)
self.log.info("Transactions received: %s [%s] rate tx/s ->> %s <<" % (tx_count, tm(cq), rate))
if self.block_handler:
q = time.time()
await self.block_handler(block)
self.blocks_processing_time += time.time() - q
bpt += time.time() - q
# insert new block
self.block_cache.set(block["hash"], block["height"])
self.last_block_height = block["height"]
# after block added handler
if self.after_block_handler:
q = time.time()
try:
await self.after_block_handler(block)
except:
pass
self.blocks_processing_time += time.time() - q
bpt += time.time() - q
self.blocks_processed_count += 1
[self.tx_cache.pop(h) for h in block["tx"]]
except Exception as err:
if self.await_tx_list:
self.await_tx_list = set()
self.log.error(str(traceback.format_exc()))
self.log.error("new block error %s" % str(err))
finally:
self.active_block.set_result(True)
self.log.info("> %s < block [%s tx/ %s size] (dp %s) processing time %s cache [%s/%s]" %
(block["height"],
len(block["tx"]),
block["size"] / 1000000,
self.block_dependency_tx,
tm(bt),
len(self.block_hashes_preload._store),
len(self.block_preload._store)))
if self.node_last_block > self.last_block_height:
self.loop.create_task(self.get_next_block())
async def rpctest(self):
self.rpc = aiojsonrpc.rpc(self.config["BITCOIND"]["rpc"], loop)
async def start(self):
if self.utxo_data:
await self.utxo_init()
while True:
self.log.info("Connector initialization")
try:
self.rpc = aiojsonrpc.rpc(self.rpc_url,
self.loop,
timeout=self.rpc_timeout)
self.node_last_block = await self.rpc.getblockcount()
except Exception as err:
self.log.error("Get node best block error:" + str(err))
if not isinstance(self.node_last_block, int):
self.log.error("Get node best block height failed")
self.log.error("Node rpc url: " + self.rpc_url)
await asyncio.sleep(10)
continue
self.log.info("Node best block height %s" % self.node_last_block)
self.log.info(
"Connector last block height %s [%s]" %
(self.last_block_height, self.last_block_utxo_cached_height))
self.log.info("Application last block height %s" %
self.app_block_height_on_start)
if self.node_last_block < self.last_block_height:
self.log.error("Node is behind application blockchain state!")
await asyncio.sleep(10)
continue
elif self.node_last_block == self.last_block_height:
self.log.info("Blockchain is synchronized")
else:
d = self.node_last_block - self.last_block_height
self.log.info("%s blocks before synchronization" % d)
if d > self.deep_sync_limit:
self.log.info("Deep synchronization mode")
self.deep_synchronization = True
break
if self.utxo_data:
if self.db_type == "postgresql":
db = self.db_pool
else:
db = self.db
self.sync_utxo = UTXO(self.db_type, db, self.rpc, self.loop,
self.log, self.utxo_cache_size)
self.uutxo = UUTXO(self.db_type, db, self.log)
h = self.last_block_height
if h < len(self.chain_tail):
raise Exception("Chain tail len not match last block height")
for row in reversed(self.chain_tail):
self.block_headers_cache.set(row, h)
h -= 1
if self.utxo_data and self.db_type == "postgresql":
self.block_loader = BlockLoader(self,
workers=self.block_cache_workers,
dsn=self.db)
else:
self.block_loader = BlockLoader(self,
workers=self.block_cache_workers)
self.tasks.append(self.loop.create_task(self.zeromq_handler()))
self.tasks.append(self.loop.create_task(self.watchdog()))
self.connected.set_result(True)
self.get_next_block_mutex = True
self.loop.create_task(self.get_next_block())
async def rpctest(self):
self.rpc = aiojsonrpc.rpc(self.config["BITCOIND"]["rpc"], loop)
while 1:
p = await self.rpc.getinfo()
print(p)
await asyncio.sleep(10)