async def listen_coin_minute_task(loop): err_data_List = [] has_data = 0 session = None await main(loop) while True: redis_obj = Redis.getInstance() pre_item = redis_obj.customer("coin_all_minute:queue") if pre_item: if not session: session = aiohttp.ClientSession(conn_timeout=1800) has_data = 1 pre_item = json.loads(pre_item.decode().replace("'", '"')) collect_info("listen_coin_minute_task pre_item is %s" % str(pre_item)) err_data_List = await run_paraser_item(pre_item, -1, err_data_List, session) else: if has_data: num = redis_obj.getListLen("coin_all_minute:queue") if num: continue has_data = 0 await print_error_item(-1, err_data_List) if session: await session.close() session = None err_data_List = [] time.sleep(5)
async def main(loop, day_num): await orm.create_plug_pool(loop=loop, **configs.db) session = aiohttp.ClientSession(conn_timeout=1800) data_lists = [] for num in range(10): result_data = await fetch_async(ticker_url % (num * 100 + 1), session, toJson=True) if result_data: data_list = result_data["data"].values() data_lists.extend(data_list) bit_names = {str(val["symbol"]): val for val in data_lists} all_value = await bs4_paraser(bit_names) collect_info("init_request_down main all_value is %s" % str(all_value)) redis_obj = Redis.getInstance() if day_num == 0: folios_lists = await get_folio() for folio_val in folios_lists: redis_obj.producer( "folio_24_hour:queue", json.dumps({"id": folio_val.id}, ensure_ascii=False, default=__default)) if day_num == 1: for val in all_value: redis_obj.producer("coin_all_day:queue", json.dumps(val)) folios_lists = await get_folio() for folio_val in folios_lists: redis_obj.producer( "folio_all_items:queue", json.dumps({"id": folio_val.id}, ensure_ascii=False, default=__default)) if day_num == -1: for val in all_value: redis_obj.producer("coin_all_minute:queue", json.dumps(val)) await session.close()