async def loading(self):
self.rpc_batch_limit = 30
self.worker_tasks = [
self.loop.create_task(self.start_worker(i))
for i in range(self.worker_limit)
]
target_height = self.parent.node_last_block - self.parent.deep_sync_limit
self.height = self.parent.last_block_height + 1
while self.height < target_height:
target_height = self.parent.node_last_block - self.parent.deep_sync_limit
new_requests = 0
if self.parent.block_preload._store_size < self.parent.block_preload_cache_limit:
try:
for i in self.worker_busy:
if not self.worker_busy[i]:
self.worker_busy[i] = True
if self.height <= self.parent.last_block_height:
self.height = self.parent.last_block_height + 1
await self.pipe_sent_msg(
self.worker[i].writer, b'rpc_batch_limit',
int_to_bytes(self.rpc_batch_limit))
await self.pipe_sent_msg(self.worker[i].writer,
b'get',
int_to_bytes(self.height))
self.height += self.rpc_batch_limit
new_requests += 1
if not new_requests:
await asyncio.sleep(1)
continue
if self.last_batch_size < self.parent.block_preload_batch_size_limit:
self.rpc_batch_limit += 40
elif self.last_batch_size > self.parent.block_preload_batch_size_limit and self.rpc_batch_limit > 60:
self.rpc_batch_limit -= 40
except asyncio.CancelledError:
self.log.info("Loading task terminated")
[self.worker[p].terminate() for p in self.worker]
for p in self.worker_busy:
self.worker_busy[p] = False
return
except Exception as err:
self.log.error("Loading task error %s " % err)
else:
await asyncio.sleep(1)
self.watchdog_task.cancel()
if self.parent.block_preload._store:
while next(reversed(
self.parent.block_preload._store)) < target_height:
await asyncio.sleep(1)
self.log.info("block loader reached target block %s" %
target_height)
self.log.debug(" Cache first block %s; "
"cache last block %s;" %
(next(iter(self.parent.block_preload._store)),
next(reversed(self.parent.block_preload._store))))
[self.worker[p].terminate() for p in self.worker]
for p in self.worker_busy:
self.worker_busy[p] = False
def op_push_data(data):
if len(data) <= 0x4b:
return b''.join([bytes([len(data)]), data])
elif len(data) <= 0xff:
return b''.join([OP_PUSHDATA1, bytes([len(data)]), data])
elif len(data) <= 0xffff:
return b''.join([OP_PUSHDATA2, int_to_bytes(len(data), byteorder="little"), data])
else:
return b''.join([OP_PUSHDATA4, int_to_bytes(len(data), byteorder="little"), data])
def leveldb_atomic_batch(self):
with self.db.write_batch() as batch:
[batch.delete(k) for k in self.pending_deleted]
[batch.put(k[0], k[1]) for k in self.utxo_records]
batch.put(b"last_block", int_to_bytes(self.checkpoint))
def rocksdb_atomic_batch(self):
batch = rocksdb.WriteBatch()
[batch.delete(k) for k in self.pending_deleted]
[batch.put(k[0], k[1]) for k in self.utxo_records]
batch.put(b"last_block", int_to_bytes(self.checkpoint))
self.db.write(batch)
async def load_blocks(self, height, limit):
start_height = height
start_limit = limit
self.destroyed_coins = MRU()
self.coins = MRU()
try:
self.rpc = aiojsonrpc.rpc(self.rpc_url, self.loop, timeout=self.rpc_timeout)
blocks, missed = dict(), deque()
v, t, limit = height + limit, 0, 40
while height < v and height <= self.target_height:
batch, h_list = list(), list()
while len(batch) < limit and height < v and height <= self.target_height:
batch.append(["getblockhash", height])
h_list.append(height)
height += 1
result = await self.rpc.batch(batch)
h, batch = list(), list()
for lh, r in zip(h_list, result):
if r["result"] is not None:
batch.append(["getblock", r["result"], 0])
h.append(lh)
result = await self.rpc.batch(batch)
for x, y in zip(h, result):
if y["result"] is not None:
block = decode_block_tx(y["result"])
block["p2pkMapHash"] = []
if self.option_tx_map: block["txMap"], block["stxo"] = deque(), deque()
if self.option_block_filters:
block["filter"] = set()
if self.option_merkle_proof:
mt = merkle_tree(block["rawTx"][i]["txId"] for i in block["rawTx"])
coinbase = block["rawTx"][0]["vIn"][0]["scriptSig"]
block["miner"] = None
for tag in MINER_COINBASE_TAG:
if coinbase.find(tag) != -1:
block["miner"] = json.dumps(MINER_COINBASE_TAG[tag])
break
else:
try:
address_hash = block["rawTx"][0]["vOut"][0]["addressHash"]
script_hash = False if block["rawTx"][0]["vOut"][0]["nType"] == 1 else True
a = hash_to_address(address_hash, script_hash=script_hash)
if a in MINER_PAYOUT_TAG:
block["miner"] = json.dumps(MINER_PAYOUT_TAG[a])
except:
pass
if self.utxo_data:
# handle outputs
for z in block["rawTx"]:
if self.option_merkle_proof:
block["rawTx"][z]["merkleProof"] = b''.join(merkle_proof(mt, z, return_hex=False))
for i in block["rawTx"][z]["vOut"]:
out= block["rawTx"][z]["vOut"][i]
o = b"".join((block["rawTx"][z]["txId"], int_to_bytes(i)))
pointer = (x << 39)+(z << 20)+(1 << 19) + i
out_type = out["nType"]
try:
if out_type == 2:
block["p2pkMapHash"].append((out["addressHash"], out["scriptPubKey"]))
raise Exception("P2PK")
address = b"".join((bytes([out_type]), out["addressHash"]))
except:
address = b"".join((bytes([out_type]), out["scriptPubKey"]))
if out_type in (0, 1, 2, 5, 6):
if self.option_block_filters:
e = b"".join((bytes([out_type]),
z.to_bytes(4, byteorder="little"),
out["addressHash"][:20]))
block["filter"].add(e)
if self.option_tx_map:
block["txMap"].append((pointer, address, out["value"]))
out["_address"] = address
self.coins[o] = (pointer, out["value"], address)
# handle inputs
for z in block["rawTx"]:
if not block["rawTx"][z]["coinbase"]:
for i in block["rawTx"][z]["vIn"]:
inp = block["rawTx"][z]["vIn"][i]
outpoint = b"".join((inp["txId"], int_to_bytes(inp["vOut"])))
block["rawTx"][z]["vIn"][i]["_outpoint"] = outpoint
try:
r = self.coins.delete(outpoint)
try:
block["rawTx"][z]["vIn"][i]["_a_"] = r
self.destroyed_coins[r[0]] = True
out_type = r[2][0]
if self.option_block_filters:
if out_type in (0, 1, 5, 6):
e = b"".join((bytes([out_type]),
z.to_bytes(4, byteorder="little"),
r[2][1:21]))
block["filter"].add(e)
elif out_type == 2:
a = parse_script(r[2][1:])["addressHash"]
e = b"".join((bytes([out_type]),
z.to_bytes(4, byteorder="little"),
a[:20]))
block["filter"].add(e)
if self.option_tx_map:
block["txMap"].append(((x<<39)+(z<<20)+i, r[2], r[1]))
block["stxo"].append((r[0], (x<<39)+(z<<20)+i))
t += 1
except:
print(traceback.format_exc())
except:
if self.dsn: missed.append(outpoint)
blocks[x] = block
m, n = 0, 0
if self.utxo_data and missed and self.dsn:
if self.dsn:
async with self.db.acquire() as conn:
rows = await conn.fetch("SELECT outpoint, "
" pointer,"
" address,"
" amount "
"FROM connector_utxo "
"WHERE outpoint = ANY($1);", missed)
m += len(rows)
p = dict()
for row in rows:
p[row["outpoint"]] = (row["pointer"], row["amount"], row["address"])
for h in blocks:
for z in blocks[h]["rawTx"]:
if not blocks[h]["rawTx"][z]["coinbase"]:
for i in blocks[h]["rawTx"][z]["vIn"]:
outpoint = blocks[h]["rawTx"][z]["vIn"][i]["_outpoint"]
try:
blocks[h]["rawTx"][z]["vIn"][i]["_l_"] = p[outpoint]
try:
out_type = p[outpoint][2][0]
if self.option_block_filters:
if out_type in (0, 1, 5, 6):
e = b"".join((bytes([out_type]),
z.to_bytes(4, byteorder="little"),
p[outpoint][2][1:21]))
blocks[h]["filter"].add(e)
elif out_type == 2:
a = parse_script(p[outpoint][2][1:])["addressHash"]
e = b"".join((bytes([out_type]),
z.to_bytes(4, byteorder="little"),
a[:20]))
blocks[h]["filter"].add(e)
if self.option_tx_map:
blocks[h]["txMap"].append(((h<<39)+(z<<20)+i,
p[outpoint][2], p[outpoint][1]))
blocks[h]["stxo"].append((p[outpoint][0], (h<<39)+(z<<20)+i))
t += 1
n += 1
except:
print(traceback.format_exc())
except:
pass
if self.utxo_data and blocks:
blocks[x]["checkpoint"] = x
for x in blocks:
if self.utxo_data:
for y in blocks[x]["rawTx"]:
for i in blocks[x]["rawTx"][y]["vOut"]:
try:
r = self.destroyed_coins.delete((x<<39)+(y<<20)+(1<<19)+i)
blocks[x]["rawTx"][y]["vOut"][i]["_s_"] = r
except: pass
if self.option_block_filters:
blocks[x]["filter"] = bytearray(b"".join(blocks[x]["filter"]))
blocks[x] = pickle.dumps(blocks[x])
await self.pipe_sent_msg(b'result', pickle.dumps(blocks))
except concurrent.futures.CancelledError:
pass
except Exception as err:
self.log.error("block loader restarting: %s" % err)
print(traceback.format_exc())
await asyncio.sleep(1)
self.loop.create_task(self.load_blocks(start_height, start_limit))
finally:
try: await self.rpc.close()
except: pass
async def load_blocks(self, height, limit):
start_height = height
try:
x = None
blocks = dict()
missed = deque()
t = 0
e = height + limit
limit = 40
while height < e:
batch, h_list = list(), list()
while len(batch) < limit and height < e:
batch.append(["getblockhash", height])
h_list.append(height)
height += 1
result = await self.rpc.batch(batch)
h, batch = list(), list()
for lh, r in zip(h_list, result):
if r["result"] is not None:
batch.append(["getblock", r["result"], 0])
h.append(lh)
result = await self.rpc.batch(batch)
for x, y in zip(h, result):
if y["result"] is not None:
block = decode_block_tx(y["result"])
for z in block["rawTx"]:
for i in block["rawTx"][z]["vOut"]:
o = b"".join((block["rawTx"][z]["txId"],
int_to_bytes(i)))
pointer = (x << 39) + (z << 20) + (1 << 19) + i
try:
address = b"".join((bytes([
block["rawTx"][z]["vOut"][i]["nType"]
]), block["rawTx"][z]["vOut"][i]
["addressHash"]))
except:
address = b"".join((bytes([
block["rawTx"][z]["vOut"][i]["nType"]
]), block["rawTx"][z]["vOut"][i]
["scriptPubKey"]))
self.coins[o] = (
pointer,
block["rawTx"][z]["vOut"][i]["value"],
address)
if not block["rawTx"][z]["coinbase"]:
for i in block["rawTx"][z]["vIn"]:
inp = block["rawTx"][z]["vIn"][i]
outpoint = b"".join(
(inp["txId"],
int_to_bytes(inp["vOut"])))
try:
r = self.coins.delete(outpoint)
if r[0] >> 39 >= start_height and r[
0] >> 39 < height:
block["rawTx"][z]["vIn"][i][
"_a_"] = r
else:
block["rawTx"][z]["vIn"][i][
"_c_"] = r
t += 1
self.destroyed_coins[r[0]] = True
assert r is not None
except:
if self.dsn:
missed.append(outpoint)
blocks[x] = block
m = 0
n = 0
if missed and self.dsn:
if self.dsn:
async with self.db.acquire() as conn:
rows = await conn.fetch(
"SELECT outpoint, "
" pointer,"
" address,"
" amount "
"FROM connector_utxo "
"WHERE outpoint = ANY($1);", missed)
m += len(rows)
p = dict()
for row in rows:
p[row["outpoint"]] = (row["pointer"], row["amount"],
row["address"])
for block in blocks:
for z in blocks[block]["rawTx"]:
if not blocks[block]["rawTx"][z]["coinbase"]:
for i in blocks[block]["rawTx"][z]["vIn"]:
inp = blocks[block]["rawTx"][z]["vIn"][i]
outpoint = b"".join(
(inp["txId"],
int_to_bytes(inp["vOut"])))
try:
blocks[block]["rawTx"][z]["vIn"][i][
"_l_"] = p[outpoint]
assert p[outpoint] is not None
t += 1
n += 1
except:
pass
if blocks:
blocks[x]["checkpoint"] = x
for x in blocks:
for y in blocks[x]["rawTx"]:
for i in blocks[x]["rawTx"][y]["vOut"]:
try:
r = self.destroyed_coins.delete((x << 39) +
(y << 20) +
(1 << 19) + i)
blocks[x]["rawTx"][y]["vOut"][i]["_s_"] = r
assert r is not None
except:
pass
blocks[x] = pickle.dumps(blocks[x])
await self.pipe_sent_msg(b'result', pickle.dumps(blocks))
except Exception as err:
self.log.debug("load blocks error: %s" % str(err))
await self.pipe_sent_msg(b'result', pickle.dumps([]))
await self.pipe_sent_msg(b'failed', pickle.dumps(start_height))