def main(): m = MongoDB() tasks = [] all_pairs = m.get_all_pairs() for address in all_pairs[:100]: pair = Web3.toChecksumAddress(address) task = log_loop(pair, 60) tasks.append(task) print('{} Starting...'.format(len(tasks))) loop = asyncio.get_event_loop() try: loop.run_until_complete(asyncio.gather(*tasks)) finally: loop.close()
def scrapReserves(pair_address): pair = Web3.toChecksumAddress(pair_address) contract = DeFiContract(pair, 'Pair') r0, r1, _ = contract.getReserves() print('{} {} {}'.format(pair, r0, r1)) doc = { 'address': pair, 'r0': r0, 'r1': r1, 't': datetime.utcnow().timestamp(), } return doc while True: all_pairs = m.get_all_pairs() now = datetime.now().timestamp() with concurrent.futures.ThreadPoolExecutor() as executor: futures = [] for pair in all_pairs: futures.append(executor.submit(scrapReserves, pair)) for future in concurrent.futures.as_completed(futures): doc = future.result() # k.save(doc) elapsed = datetime.now().timestamp() - now print('Sleeping after {}s...'.format(elapsed)) time.sleep(60)