async def crawl(self):
self.start = await self.get_history_state()
self.start += 1
while True:
current_height = await self.get_block_count()
time_a = CT.now()
if self.start < current_height:
stop = self.start + self.max_tasks
if stop >= current_height:
stop = current_height
self.processing.extend([i for i in range(self.start, stop)])
max_height = max(self.processing)
min_height = self.processing[0]
await asyncio.wait(
[self.cache_block(h) for h in self.processing])
if self.processing != sorted(self.cache.keys()):
msg = 'cache != processing'
logger.error(msg)
sys.exit(1)
txids = []
for block in self.cache.values():
for tx in block['tx']:
for vin in tx['vin']:
txids.append(vin['txid'])
txids = list(set(txids))
if txids:
await asyncio.wait(
[self.cache_utxo_vouts(txid) for txid in txids])
if sorted(txids) != sorted(self.cache_utxo.keys()):
msg = 'cache utxo error'
logger.error(msg)
sys.exit(1)
vins = []
vouts = []
for block in self.cache.values():
block_time = block['time']
for tx in block['tx']:
utxo_dict = {}
for vin in tx['vin']:
utxo = self.cache_utxo[vin['txid']][vin['vout']]
key = utxo['asset'] + '_' + utxo['address']
if key in utxo_dict.keys():
utxo_dict[key]['value'] = CT.sci_to_str(
str(
D(utxo_dict[key]['value']) +
D(utxo['value'])))
else:
utxo_dict[key] = utxo
vout_dict = {}
for vout in tx['vout']:
key = vout['asset'] + '_' + vout['address']
if key in vout_dict.keys():
vout_dict[key]['value'] = CT.sci_to_str(
str(
D(vout_dict[key]['value']) +
D(vout['value'])))
else:
vout_dict[key] = vout
if 1 == len(utxo_dict) == len(
vout_dict) and utxo_dict.keys(
) == vout_dict.keys():
key = list(utxo_dict.keys())[0]
if utxo_dict[key]['value'] == vout_dict[key][
'value']:
continue
utxos = list(utxo_dict.values())
for i in range(len(utxos)):
utxo = utxos[i]
key = utxo['asset'] + '_' + utxo['address']
if key in vout_dict.keys():
if D(utxo['value']) > D(
vout_dict[key]['value']):
utxo['value'] = CT.sci_to_str(
str(
D(utxo['value']) -
D(vout_dict[key]['value'])))
del vout_dict[key]
vins.append([utxo, tx['txid'], i, block_time])
voutx = list(vout_dict.values())
for k in range(len(voutx)):
vout = voutx[k]
vouts.append([vout, tx['txid'], k, block_time])
if vins:
await asyncio.wait(
[self.update_a_vin(*vin) for vin in vins])
if vouts:
await asyncio.wait(
[self.update_a_vout(*vout) for vout in vouts])
time_b = CT.now()
logger.info(
'reached %s ,cost %.6fs to sync %s blocks ,total cost: %.6fs'
% (max_height, time_b - time_a, stop - self.start,
time_b - START_TIME))
await self.update_history_state(max_height)
self.start = max_height + 1
del self.processing
del self.cache
del self.cache_utxo
self.processing = []
self.cache = {}
self.cache_utxo = {}
else:
await asyncio.sleep(0.5)
def hex_to_num_str(cls, hs):
bs = unhexlify(hs)
return CT.sci_to_str(str(D(cls.bytes_to_num(bs)) / 100000000))
