async def listen_coin_minute_task(loop): err_data_List = [] has_data = 0 session = None await main(loop) while True: redis_obj = Redis.getInstance() pre_item = redis_obj.customer("coin_all_minute:queue") if pre_item: if not session: session = aiohttp.ClientSession(conn_timeout=1800) has_data = 1 pre_item = json.loads(pre_item.decode().replace("'", '"')) collect_info("listen_coin_minute_task pre_item is %s" % str(pre_item)) err_data_List = await run_paraser_item(pre_item, -1, err_data_List, session) else: if has_data: num = redis_obj.getListLen("coin_all_minute:queue") if num: continue has_data = 0 await print_error_item(-1, err_data_List) if session: await session.close() session = None err_data_List = [] time.sleep(5)
async def getReportUpdateTime(key="update_report", item=""): redis_obj = Redis.getInstance() update_report = redis_obj.get(key) if update_report is None: return "" update_report = json.loads(update_report.decode()) if not isinstance( update_report, dict) else update_report res = update_report.get(item, {}).get("update_at", "") return res
async def init_db(loop): await orm.create_pool(loop=loop, **configs.db) redis_obj = Redis.getInstance() data_path = os.path.join(root_path, "data/dbData/data.json") with open(data_path, "r") as load_f: load_data = json.load(load_f) hotCoins = load_data.get("hotCoins") update_report = load_data.get("update_report") redis_obj.set("hotCoins", json.dumps(hotCoins)) redis_obj.set("update_report", json.dumps(update_report))
async def get_coin_name_list(request): result = {"error": 0, "data": "", "message": ""} redis_obj = Redis.getInstance() coin_name_list = redis_obj.get("coin_name_list") if not coin_name_list: dataList = await dao.findHotCoinList(all=1) redis_obj.set("coin_name_list", json.dumps(dataList)) else: dataList = json.loads(coin_name_list.decode()) result["data"] = dataList return result
async def update_folio(): while True: redis_obj = Redis.getInstance() pre_item = redis_obj.customer("folio_all_items:queue") if pre_item: pre_item = json.loads(pre_item.decode()) await calculate_portfolio(pre_item["id"], folio=None, re_cal=True, is_ratio=True) else: break time.sleep(0.1)
async def token2user(token): if not token: return None try: redis_obj = Redis.getInstance() user = redis_obj.get(token) if user is None: return None user = json.loads( user.decode()) if not isinstance(user, dict) else user redis_obj.set_expire(token, 3600 * 24) return user except Exception as e: error("token2user exception is: %s" % str(e)) return None
async def listen_folio_24_hour_task(loop): has_data = 1 await main(loop) while True: redis_obj = Redis.getInstance() pre_item = redis_obj.customer("folio_24_hour:queue") if pre_item: has_data = 1 pre_item = json.loads(pre_item.decode()) await updateHour24History(pre_item["id"], folio=None) else: if has_data: num = redis_obj.getListLen("folio_24_hour:queue") if num: continue has_data = 0 time.sleep(5)
async def main(loop, day_num): await orm.create_plug_pool(loop=loop, **configs.db) session = aiohttp.ClientSession(conn_timeout=1800) data_lists = [] for num in range(10): result_data = await fetch_async(ticker_url % (num * 100 + 1), session, toJson=True) if result_data: data_list = result_data["data"].values() data_lists.extend(data_list) bit_names = {str(val["symbol"]): val for val in data_lists} all_value = await bs4_paraser(bit_names) collect_info("init_request_down main all_value is %s" % str(all_value)) redis_obj = Redis.getInstance() if day_num == 0: folios_lists = await get_folio() for folio_val in folios_lists: redis_obj.producer( "folio_24_hour:queue", json.dumps({"id": folio_val.id}, ensure_ascii=False, default=__default)) if day_num == 1: for val in all_value: redis_obj.producer("coin_all_day:queue", json.dumps(val)) folios_lists = await get_folio() for folio_val in folios_lists: redis_obj.producer( "folio_all_items:queue", json.dumps({"id": folio_val.id}, ensure_ascii=False, default=__default)) if day_num == -1: for val in all_value: redis_obj.producer("coin_all_minute:queue", json.dumps(val)) await session.close()
async def update_redis(): date_time = str(int(time.time())) update_report = { "portfolio_ratios": { "update_at": date_time }, "portfolio_optimization": { "update_at": date_time }, "portfolio_basic": { "update_at": date_time }, "portfolio_ai": { "update_at": date_time }, "portfolio_style": { "update_at": date_time }, "portfolio_benchmark": { "update_at": date_time }, "portfolio_risk": { "update_at": date_time }, "portfolio_versus": { "update_at": date_time }, "coins_ai": { "update_at": date_time }, "coins_basic": { "update_at": date_time }, "coins_ratios": { "update_at": date_time }, "coins_versus": { "update_at": date_time } } redis_obj = Redis.getInstance() redis_obj.set("update_report", json.dumps(update_report))
async def frequency(remote_ip, path_url): try: fast_rep = 0 redis_obj = Redis.getInstance() data_val = redis_obj.getHash(remote_ip, path_url) try: data_val = json.loads(data_val.decode()) val = data_val.get("now_time", "") if isinstance( data_val, dict) else data_val num = data_val.get("num", "") if isinstance(data_val, dict) else 1 now_time = int(time.time() * 1000) pre_time = data_val.get("pre_time", "") if isinstance( data_val, dict) else now_time except: now_time = int(time.time() * 1000) pre_time = now_time val = "" num = 1 request_info("frequency val:%s" % str(val)) if not val: data = {"now_time": now_time, "num": 1, "pre_time": now_time} redis_obj.setHash(remote_ip, path_url, json.dumps(data)) redis_obj.setHashExpire(remote_ip, 3600) else: if now_time - int(val) < 200: fast_rep = 1 if num >= 60 and now_time - pre_time <= 60000: fast_rep = 1 num += 1 elif now_time - pre_time > 60000: pre_time = now_time num = 1 else: num += 1 data = {"now_time": now_time, "num": num, "pre_time": pre_time} redis_obj.setHash(remote_ip, path_url, json.dumps(data)) redis_obj.setHashExpire(remote_ip, 3600) return fast_rep except Exception as e: error("frequency exception is: %s" % str(e)) return None