async def crawl(self):
self.start = await self.get_state()
self.start += 1
while True:
current_height = await self.get_block_count()
time_a = CT.now()
if self.start < current_height:
stop = self.start + self.max_tasks
if stop >= current_height:
stop = current_height
self.processing.extend([i for i in range(self.start, stop)])
max_height = max(self.processing)
min_height = self.processing[0]
await asyncio.wait(
[self.cache_block(h) for h in self.processing])
if self.processing != sorted(self.cache.keys()):
msg = 'cache != processing'
logger.error(msg)
raise Exception(msg)
sys.exit(1)
await self.update_sys_fee(min_height)
vins = []
vouts = []
claims = []
for block in self.cache.values():
for tx in block['tx']:
txid = tx['txid']
height = block['index']
for vin in tx['vin']:
vins.append([vin, txid, height])
for vout in tx['vout']:
vouts.append([vout, txid, height])
if 'claims' in tx.keys():
for claim in tx['claims']:
claims.append([claim, txid, height])
if vins:
await asyncio.wait(
[self.update_a_vin(*vin) for vin in vins])
if vouts:
await asyncio.wait(
[self.update_a_vout(*vout) for vout in vouts])
if claims:
await asyncio.wait(
[self.update_a_claim(*claim) for claim in claims])
#cache update addresses
if stop == current_height and 1 == len(self.processing):
uas = []
vinas = await asyncio.gather(
*[self.get_address_from_vin(vin[0]) for vin in vins])
voutas = [vout[0]['address'] for vout in vouts]
uas = list(set(vinas + voutas))
await self.update_addresses(max_height, uas)
time_b = CT.now()
logger.info(
'reached %s ,cost %.6fs to sync %s blocks ,total cost: %.6fs'
% (max_height, time_b - time_a, stop - self.start,
time_b - START_TIME))
await asyncio.wait([
self.update_block(block) for block in self.cache.values()
])
await self.update_state(max_height)
self.start = max_height + 1
del self.processing
del self.cache
self.processing = []
self.cache = {}
else:
await asyncio.sleep(0.5)
async def crawl(self):
self.start = await self.get_history_state()
self.start += 1
while True:
current_height = await self.get_block_count()
time_a = CT.now()
if self.start < current_height:
stop = self.start + self.max_tasks
if stop >= current_height:
stop = current_height
self.processing.extend([i for i in range(self.start, stop)])
max_height = max(self.processing)
min_height = self.processing[0]
await asyncio.wait(
[self.cache_block(h) for h in self.processing])
if self.processing != sorted(self.cache.keys()):
msg = 'cache != processing'
logger.error(msg)
sys.exit(1)
txids = []
for block in self.cache.values():
for tx in block['tx']:
for vin in tx['vin']:
txids.append(vin['txid'])
txids = list(set(txids))
if txids:
await asyncio.wait(
[self.cache_utxo_vouts(txid) for txid in txids])
if sorted(txids) != sorted(self.cache_utxo.keys()):
msg = 'cache utxo error'
logger.error(msg)
sys.exit(1)
vins = []
vouts = []
for block in self.cache.values():
block_time = block['time']
for tx in block['tx']:
utxo_dict = {}
for vin in tx['vin']:
utxo = self.cache_utxo[vin['txid']][vin['vout']]
key = utxo['asset'] + '_' + utxo['address']
if key in utxo_dict.keys():
utxo_dict[key]['value'] = CT.sci_to_str(
str(
D(utxo_dict[key]['value']) +
D(utxo['value'])))
else:
utxo_dict[key] = utxo
vout_dict = {}
for vout in tx['vout']:
key = vout['asset'] + '_' + vout['address']
if key in vout_dict.keys():
vout_dict[key]['value'] = CT.sci_to_str(
str(
D(vout_dict[key]['value']) +
D(vout['value'])))
else:
vout_dict[key] = vout
if 1 == len(utxo_dict) == len(
vout_dict) and utxo_dict.keys(
) == vout_dict.keys():
key = list(utxo_dict.keys())[0]
if utxo_dict[key]['value'] == vout_dict[key][
'value']:
continue
utxos = list(utxo_dict.values())
for i in range(len(utxos)):
utxo = utxos[i]
key = utxo['asset'] + '_' + utxo['address']
if key in vout_dict.keys():
if D(utxo['value']) > D(
vout_dict[key]['value']):
utxo['value'] = CT.sci_to_str(
str(
D(utxo['value']) -
D(vout_dict[key]['value'])))
del vout_dict[key]
vins.append([utxo, tx['txid'], i, block_time])
voutx = list(vout_dict.values())
for k in range(len(voutx)):
vout = voutx[k]
vouts.append([vout, tx['txid'], k, block_time])
if vins:
await asyncio.wait(
[self.update_a_vin(*vin) for vin in vins])
if vouts:
await asyncio.wait(
[self.update_a_vout(*vout) for vout in vouts])
time_b = CT.now()
logger.info(
'reached %s ,cost %.6fs to sync %s blocks ,total cost: %.6fs'
% (max_height, time_b - time_a, stop - self.start,
time_b - START_TIME))
await self.update_history_state(max_height)
self.start = max_height + 1
del self.processing
del self.cache
del self.cache_utxo
self.processing = []
self.cache = {}
self.cache_utxo = {}
else:
await asyncio.sleep(0.5)
% (max_height, time_b - time_a, stop - self.start,
time_b - START_TIME))
await asyncio.wait([
self.update_block(block) for block in self.cache.values()
])
await self.update_state(max_height)
self.start = max_height + 1
del self.processing
del self.cache
self.processing = []
self.cache = {}
else:
await asyncio.sleep(0.5)
if __name__ == "__main__":
START_TIME = CT.now()
logger.info('STARTING...')
mongo_uri = C.get_mongo_uri()
neo_uri = C.get_neo_uri()
mongo_db = C.get_mongo_db()
tasks = C.get_tasks()
loop = asyncio.get_event_loop()
crawler = Crawler(mongo_uri, mongo_db, neo_uri, loop, tasks)
try:
loop.run_until_complete(crawler.crawl())
except Exception as e:
logger.error('LOOP EXCEPTION: %s' % e)
finally:
loop.close()
async def crawl(self):
self.start = await self.get_asset_state()
self.start += 1
while True:
current_height = await self.get_block_count()
time_a = CT.now()
if self.start < current_height:
stop = self.start + self.max_tasks
if stop >= current_height:
stop = current_height
self.processing.extend([i for i in range(self.start, stop)])
max_height = max(self.processing)
min_height = self.processing[0]
await asyncio.wait(
[self.cache_block(h) for h in self.processing])
if self.processing != sorted(self.cache.keys()):
msg = 'cache != processing'
logger.error(msg)
sys.exit(1)
global_assets = {}
nep5_assets = {}
for block in self.cache.values():
block_time = block['time']
for tx in block['tx']:
if 'RegisterTransaction' == tx['type']:
global_assets[tx['txid']] = tx['asset']
global_assets[tx['txid']]['time'] = block_time
if 'InvocationTransaction' == tx[
'type'] and 490 <= int(float(tx['sys_fee'])):
if tx['script'].endswith(
'68134e656f2e436f6e74726163742e437265617465'
):
try:
asset = self.parse_script(tx['script'])
except Exception as e:
print('parse error:', e)
continue
asset['time'] = block_time
nep5_assets[asset['contract']] = asset
if global_assets:
await asyncio.wait([
self.update_a_global_asset(*i)
for i in global_assets.items()
])
if nep5_assets:
await asyncio.wait([
self.update_a_nep5_asset(*i)
for i in nep5_assets.items()
])
time_b = CT.now()
logger.info(
'reached %s ,cost %.6fs to sync %s blocks ,total cost: %.6fs'
% (max_height, time_b - time_a, stop - self.start,
time_b - START_TIME))
await self.update_asset_state(max_height)
self.start = max_height + 1
del self.processing
del self.cache
self.processing = []
self.cache = {}
else:
await asyncio.sleep(0.5)
