async def listen_coin_minute_task(loop):
err_data_List = []
has_data = 0
session = None
await main(loop)
while True:
redis_obj = Redis.getInstance()
pre_item = redis_obj.customer("coin_all_minute:queue")
if pre_item:
if not session:
session = aiohttp.ClientSession(conn_timeout=1800)
has_data = 1
pre_item = json.loads(pre_item.decode().replace("'", '"'))
collect_info("listen_coin_minute_task pre_item is %s" %
str(pre_item))
err_data_List = await run_paraser_item(pre_item, -1, err_data_List,
session)
else:
if has_data:
num = redis_obj.getListLen("coin_all_minute:queue")
if num:
continue
has_data = 0
await print_error_item(-1, err_data_List)
if session:
await session.close()
session = None
err_data_List = []
time.sleep(5)
async def getReportUpdateTime(key="update_report", item=""):
redis_obj = Redis.getInstance()
update_report = redis_obj.get(key)
if update_report is None:
return ""
update_report = json.loads(update_report.decode()) if not isinstance(
update_report, dict) else update_report
res = update_report.get(item, {}).get("update_at", "")
return res
async def init_db(loop):
await orm.create_pool(loop=loop, **configs.db)
redis_obj = Redis.getInstance()
data_path = os.path.join(root_path, "data/dbData/data.json")
with open(data_path, "r") as load_f:
load_data = json.load(load_f)
hotCoins = load_data.get("hotCoins")
update_report = load_data.get("update_report")
redis_obj.set("hotCoins", json.dumps(hotCoins))
redis_obj.set("update_report", json.dumps(update_report))
async def get_coin_name_list(request):
result = {"error": 0, "data": "", "message": ""}
redis_obj = Redis.getInstance()
coin_name_list = redis_obj.get("coin_name_list")
if not coin_name_list:
dataList = await dao.findHotCoinList(all=1)
redis_obj.set("coin_name_list", json.dumps(dataList))
else:
dataList = json.loads(coin_name_list.decode())
result["data"] = dataList
return result
async def update_folio():
while True:
redis_obj = Redis.getInstance()
pre_item = redis_obj.customer("folio_all_items:queue")
if pre_item:
pre_item = json.loads(pre_item.decode())
await calculate_portfolio(pre_item["id"],
folio=None,
re_cal=True,
is_ratio=True)
else:
break
time.sleep(0.1)
async def token2user(token):
if not token:
return None
try:
redis_obj = Redis.getInstance()
user = redis_obj.get(token)
if user is None:
return None
user = json.loads(
user.decode()) if not isinstance(user, dict) else user
redis_obj.set_expire(token, 3600 * 24)
return user
except Exception as e:
error("token2user exception is: %s" % str(e))
return None
async def listen_folio_24_hour_task(loop):
has_data = 1
await main(loop)
while True:
redis_obj = Redis.getInstance()
pre_item = redis_obj.customer("folio_24_hour:queue")
if pre_item:
has_data = 1
pre_item = json.loads(pre_item.decode())
await updateHour24History(pre_item["id"], folio=None)
else:
if has_data:
num = redis_obj.getListLen("folio_24_hour:queue")
if num:
continue
has_data = 0
time.sleep(5)
async def main(loop, day_num):
await orm.create_plug_pool(loop=loop, **configs.db)
session = aiohttp.ClientSession(conn_timeout=1800)
data_lists = []
for num in range(10):
result_data = await fetch_async(ticker_url % (num * 100 + 1),
session,
toJson=True)
if result_data:
data_list = result_data["data"].values()
data_lists.extend(data_list)
bit_names = {str(val["symbol"]): val for val in data_lists}
all_value = await bs4_paraser(bit_names)
collect_info("init_request_down main all_value is %s" % str(all_value))
redis_obj = Redis.getInstance()
if day_num == 0:
folios_lists = await get_folio()
for folio_val in folios_lists:
redis_obj.producer(
"folio_24_hour:queue",
json.dumps({"id": folio_val.id},
ensure_ascii=False,
default=__default))
if day_num == 1:
for val in all_value:
redis_obj.producer("coin_all_day:queue", json.dumps(val))
folios_lists = await get_folio()
for folio_val in folios_lists:
redis_obj.producer(
"folio_all_items:queue",
json.dumps({"id": folio_val.id},
ensure_ascii=False,
default=__default))
if day_num == -1:
for val in all_value:
redis_obj.producer("coin_all_minute:queue", json.dumps(val))
await session.close()
async def update_redis():
date_time = str(int(time.time()))
update_report = {
"portfolio_ratios": {
"update_at": date_time
},
"portfolio_optimization": {
"update_at": date_time
},
"portfolio_basic": {
"update_at": date_time
},
"portfolio_ai": {
"update_at": date_time
},
"portfolio_style": {
"update_at": date_time
},
"portfolio_benchmark": {
"update_at": date_time
},
"portfolio_risk": {
"update_at": date_time
},
"portfolio_versus": {
"update_at": date_time
},
"coins_ai": {
"update_at": date_time
},
"coins_basic": {
"update_at": date_time
},
"coins_ratios": {
"update_at": date_time
},
"coins_versus": {
"update_at": date_time
}
}
redis_obj = Redis.getInstance()
redis_obj.set("update_report", json.dumps(update_report))
async def frequency(remote_ip, path_url):
try:
fast_rep = 0
redis_obj = Redis.getInstance()
data_val = redis_obj.getHash(remote_ip, path_url)
try:
data_val = json.loads(data_val.decode())
val = data_val.get("now_time", "") if isinstance(
data_val, dict) else data_val
num = data_val.get("num", "") if isinstance(data_val, dict) else 1
now_time = int(time.time() * 1000)
pre_time = data_val.get("pre_time", "") if isinstance(
data_val, dict) else now_time
except:
now_time = int(time.time() * 1000)
pre_time = now_time
val = ""
num = 1
request_info("frequency val:%s" % str(val))
if not val:
data = {"now_time": now_time, "num": 1, "pre_time": now_time}
redis_obj.setHash(remote_ip, path_url, json.dumps(data))
redis_obj.setHashExpire(remote_ip, 3600)
else:
if now_time - int(val) < 200:
fast_rep = 1
if num >= 60 and now_time - pre_time <= 60000:
fast_rep = 1
num += 1
elif now_time - pre_time > 60000:
pre_time = now_time
num = 1
else:
num += 1
data = {"now_time": now_time, "num": num, "pre_time": pre_time}
redis_obj.setHash(remote_ip, path_url, json.dumps(data))
redis_obj.setHashExpire(remote_ip, 3600)
return fast_rep
except Exception as e:
error("frequency exception is: %s" % str(e))
return None
