def get_reads():
"""
获取所有的read记录
:return:
"""
page_num = int(request.args.get('page', 1))
page_size = int(request.args.get('size', 20))
dbms = request.args.get('dbms')
try:
check_alias(db_alias=dbms)
except DbmsAliasError:
return Result.gen_failed('404', 'dbms error')
uid = request.args.get('uid')
aid = request.args.get('aid')
cons = {'uid': uid, 'aid': aid}
kwargs = {}
for key, value in cons.items():
if value is not None and value != '':
kwargs[key] = value
res = ReadService().get_reads(page_num=page_num,
page_size=page_size,
db_alias=dbms,
**kwargs)
_reads = list(read.to_dict() for read in res)
total = ReadService().count(db_alias=dbms, **kwargs)
data = {'total': total, 'list': _reads}
return Result.gen_success(data)
def get_articles():
# 获取请求参数
page_num = int(request.args.get('page', 1)) # 页码
page_size = int(request.args.get('size', 20)) # 每页数量
dbms = request.args.get('dbms') # 请求数据库
sort_by = request.args.get('sort_by')
if sort_by is not None and sort_by not in ArticleService.field_names:
if sort_by[1:] not in ArticleService.field_names:
sort_by = None
try: # 检查数据库地址正确性
check_alias(db_alias=dbms)
except DbmsAliasError:
return Result.gen_failed('404', 'dbms error')
# 获取查询参数
title = request.args.get('title')
authors = request.args.get('authors')
category = request.args.get('category')
articleTags = request.args.get('articleTags')
language = request.args.get('language')
detail = request.args.get('detail', '1')
# print(request.data)
# print(exclude)
cons = {
'title__contains': title,
'authors': authors,
'category': category,
'articleTags': articleTags,
'language': language
}
# 去除为空的查询参数
kwargs = {}
for key, value in cons.items():
if value is not None and value != '':
kwargs[key] = value
if detail != '1':
kwargs['exclude'] = ['text', 'image', 'video']
# 尝试从dbms对应的Redis中获取该数据
_REDIS_KEY_ = f"{ARTICLES_LIST}:{dbms}:{page_num}:{page_size}:{sort_by}:{kwargs}"
data = RedisService().get_dict(dbms, _REDIS_KEY_)
# 判断数据是否存在
if data is None or data == {}:
# 不存在
arts = ArticleService().get_articles(page_num=page_num, page_size=page_size, db_alias=dbms, sort_by=sort_by,
**kwargs)
arts = list(art.to_dict() for art in arts)
total = ArticleService().count(db_alias=dbms, **kwargs)
data = {
'total': total,
'list': arts
}
# 将该数据存入Redis中
RedisService().set_dict(dbms, _REDIS_KEY_, data)
return Result.gen_success(data)
pass
def get_users_list():
page_num = int(request.args.get('page', 1))
page_size = int(request.args.get('size', 20))
dbms = request.args.get('dbms')
try:
check_alias(db_alias=dbms)
except DbmsAliasError:
return Result.gen_failed('404', 'dbms error')
name = request.args.get('name')
gender = request.args.get('gender')
cons = {'name': name, 'gender': gender}
kwargs = {}
for key, value in cons.items():
if value is not None and value != '':
kwargs[key] = value
_REDIS_KEY_ = f"{USERS_LIST}:{page_num}:{page_size}:{kwargs}"
data = RedisService().get_dict(dbms, _REDIS_KEY_)
if data is None or data == {}:
res = UserService().get_users(page_num=page_num,
page_size=page_size,
db_alias=dbms,
**kwargs)
_users = list()
total = UserService().count(db_alias=dbms, **kwargs)
for user in res:
_users.append(user.to_dict())
data = {'total': total, 'list': list(_users)}
RedisService().set_dict(dbms, _REDIS_KEY_, data)
return Result.gen_success(data)
def delete_read(rid):
if rid is None:
return Result.gen_failed('404', 'uid not found')
_REDIS_KEY_ = f"{READS_ITEM}:{rid}"
RedisService().delete_key_to_all(f"{READS_ITEM}:{rid}")
ReadService().del_read_by_rid(rid)
return Result.gen_success(msg='删除成功')
def get_user_read_history(uid):
"""
获取指定用户的所有read记录
:return:
"""
region = request.args.get('region')
if region not in DBMS().region['values']:
return Result.gen_failed('404', 'region error')
else:
dbms = DBMS().get_best_dbms_by_region(region)
kwargs = {'uid': uid}
# kwargs = {}
data = None
# _REDIS_KEY_ = f"READ_LIST:{dbms}:{page_num}:{page_size}:{kwargs}"
# data = RedisService().get_redis(dbms).get_dict(_REDIS_KEY_)
if data is None or data == {}:
res = ReadService().get_reads(sort_by='-timestamp',
db_alias=dbms,
**kwargs)
records = {}
for read in res:
if read.aid not in records.keys():
article = ArticleService().get_one_by_aid(aid=read.aid,
only=['title'])
if article is not None:
# record = read.to_dict()
read.title = article.title
records[read.aid] = read
else:
records[read.aid].readOrNot = records[
read.aid].readOrNot or read.readOrNot
records[read.aid].agreeOrNot = records[
read.aid].agreeOrNot or read.agreeOrNot
records[read.aid].shareOrNot = records[
read.aid].shareOrNot or read.shareOrNot
records[read.aid].commentOrNot = records[
read.aid].commentOrNot or read.commentOrNot
records[read.aid].readTimeLength += read.readTimeLength
records[read.aid].readSequence += read.readSequence
records[read.aid].timestamp = max(records[read.aid].timestamp,
read.timestamp)
records = list(
read.to_dict(other=['title']) for (key, read) in records.items())
total = ReadService().count(db_alias=dbms, **kwargs)
data = {'total': total, 'list': records}
return Result.gen_success(data)
def delete_user_info(uid):
_REDIS_KEY_ = f"{USERS_ITEM}:{uid}"
for dbms in DBMS().get_all_dbms_by_region():
RedisService().delete_key(dbms, _REDIS_KEY_)
RedisService().delete_key_by_pattern(dbms, pattern=f'{USERS_LIST}:*')
if uid is None:
return Result.gen_failed('404', 'uid not found')
UserService().del_user_by_uid(uid=uid)
return Result.gen_success('删除成功')
def get_today_populars():
level = request.args.get('level', 'daily')
timestamp = request.args.get("t")
dbms = request.args.get('dbms')
try:
check_alias(db_alias=dbms)
except DbmsAliasError:
return Result.gen_failed('404', 'dbms error')
articles = PopularService().get_articles(timestamp, level, db_alias=dbms)
articles = list(article.to_dict(only=['aid', 'title', 'count', 'category']) for article in articles)
return Result.gen_success(data=articles)
def update_read_record():
"""
对用户的阅读行为添加记录
:return:
"""
# TODO uid 和region 通过登录的用户进行获取
# print(request.json)
uid = int(request.json.get('uid'))
aid = int(request.json.get('aid'))
region = request.json.get('region')
category = request.json.get('category')
readOrNot = int(request.json.get('readOrNot', 0))
readTimeLength = int(request.json.get('readTimeLength', 0))
readSequence = int(request.json.get('readSequence', 0))
agreeOrNot = int(request.json.get('agreeOrNot', 0))
commentOrNot = int(request.json.get('commentOrNot', 0))
shareOrNot = int(request.json.get('shareOrNot', 0))
commentDetail = request.json.get('commentDetail', '')
if region not in DBMS().region['values'] or category not in DBMS(
).category['values']:
return Result.gen_failed(500, msg='region or category error')
if uid is None or aid is None:
return Result.gen_failed(500, "uid or aid 不能为空")
rid = -343
for dbms in get_dbms_by_region(region):
rid = ReadService().new_record(aid=aid,
uid=uid,
readOrNot=readOrNot,
readTimeLength=readTimeLength,
readSequence=readSequence,
agreeOrNot=agreeOrNot,
commentOrNot=commentOrNot,
shareOrNot=shareOrNot,
commentDetail=commentDetail,
db_alias=dbms).rid
for dbms in get_dbms_by_category(category):
BeReadService().new_record(aid=aid,
uid=uid,
readOrNot=readOrNot,
commentOrNot=commentOrNot,
agreeOrNot=agreeOrNot,
shareOrNot=shareOrNot,
db_alias=dbms)
return Result.gen_success(msg='success', data={'rid': rid})
pass
def delete_popular(pid):
if pid is None:
return Result.gen_failed('404', 'pid not found')
dbms = request.args.get('dbms')
if dbms is None:
return Result.gen_failed('5000', '请带上dbms参数')
# 删除缓存
RedisService().get_redis(dbms).delete(f"POPULAR:{pid}")
RedisService().get_redis(dbms).delete_by_pattern(f"POPULAR_LIST:{dbms}:*")
PopularService().del_by_id(pid)
return Result.gen_success('删除成功')
def login():
if not request.is_json:
return Result.gen_failed(400, msg='Not JSON Format')
username = request.json.get('username')
password = request.json.get('password')
# print(username, password)
user = None
if username and password:
user = UserService().login(username, password)
if user:
access_token = create_access_token(
identity=user.to_dict(only=['uid', 'name', 'region']),
expires_delta=False)
return Result.gen_success({"token": access_token})
else:
return Result.gen_failed(404, "用户名或密码错误")
def delete_article(aid):
if aid is None:
return Result.gen_failed('404', 'aid not found')
_REDIS_KEY_ = f"{ARTICLES_ITEM}:{aid}"
category = request.args.get('category')
if category is None:
return Result.gen_failed('5000', '请带上category参数')
# 先从缓存中删除该键值以及所有list对应的键值
for dbms in DBMS().get_dbms_by_category(category):
RedisService().delete_key(dbms, _REDIS_KEY_)
RedisService().delete_key_by_pattern(dbms, f"{ARTICLES_LIST}:*")
ArticleService().del_by_aid(aid)
return Result.gen_success('删除成功')
def get_article(aid):
if aid is None:
return Result.gen_failed('404', 'aid not found')
# 获取查询参数
category = request.args.get('category', None)
# article = {}
# 尝试从Redis中获取该数据
_REDIS_KEY_ = f"{ARTICLES_ITEM}:{aid}"
if category is not None:
if category not in DBMS().category['values']:
return Result.gen_failed('404', '类别不存在')
# 从category对应的dbms的redis中取数据
dbms = get_best_dbms_by_category(category)
article = RedisService().get_dict(dbms, _REDIS_KEY_)
if article == {}:
# 缓存中无数据，则去数据库中查询
article = ArticleService().get_one_by_aid(aid=aid,
db_alias=get_best_dbms_by_category(category))
if article is not None:
# article存在，存入redis中
article = article.to_dict()
RedisService().set_dict(dbms, _REDIS_KEY_, article)
else:
# 尝试从所有redis中取该数据
article = RedisService().get_dict_from_all(_REDIS_KEY_)
if article == {}:
# 缓存中没有，去数据库中查询
article = ArticleService().get_one_by_aid(aid=aid)
if article is not None:
article = article.to_dict()
try:
for dbms in DBMS().get_dbms_by_category(article['category']):
RedisService().set_dict(dbms, _REDIS_KEY_, article)
except Exception:
# 避免出现类别不存在的情况
logger.info(f"aid: {aid} 存入缓存失败， data: {article}")
if article is None or article == {}:
return Result.gen_failed(404, '文章不存在')
return Result.gen_success(article)
def get_read(rid):
read = {}
_REDIS_KEY_ = f"{READS_ITEM}:{rid}"
for dbms in DBMS().get_all_dbms_by_region():
read = RedisService().get_dict(dbms, _REDIS_KEY_)
if read == {}:
read = ReadService().get_by_rid(rid=rid)
if read is None:
return Result.gen_failed(404, 'user not found')
read = read.to_dict()
for dbms in DBMS().get_all_dbms_by_region():
RedisService().set_dict(dbms, _REDIS_KEY_, read)
return Result.gen_success(read)
pass
def user_info():
user = current_user
info = user.to_dict(other=['avatar'])
if info['name'] == 'admin':
info['roles'] = ['admin']
else:
info['roles'] = ['user']
# info['avatar'] = "https://wpimg.wallstcn.com/f778738c-e4f8-4870-b634-56703b4acafe.gif"
return Result.gen_success(info)
pass
def dashboard():
from config import DBMS
from service.user_service import UserService
from service.article_service import ArticleService
from service.read_service import ReadService
from service.popular_service import PopularService
dbms = request.args.get('dbms')
nums = {
'users':
UserService().count(db_alias=dbms),
'articles':
ArticleService().count(db_alias=dbms),
'reads':
ReadService().count(db_alias=dbms),
'populars':
PopularService().count(temporalGranularity='daily', db_alias=dbms)
}
charts = {'users': [], 'articles': [], 'reads': []}
for dbms in DBMS().get_all_dbms_by_region():
charts['users'].append({
'name': dbms,
'value': UserService().count(db_alias=dbms)
})
charts['articles'].append({
'name':
dbms,
'value':
ArticleService().count(db_alias=dbms)
})
charts['reads'].append({
'name': dbms,
'value': ReadService().count(db_alias=dbms)
})
data = {'nums': nums, 'charts': charts}
# if dbms == 'Beijing':
# data = {
# 'users': 12354,
# 'articles': 533366,
# 'reads': 23424,
# 'populars': 90372
# }
# else:
# data = {
# 'users': 63234,
# 'articles': 1284,
# 'reads': 724933,
# 'populars': 8422
# }
return Result.gen_success(data=data)
pass
def get_user(uid):
user = {}
_REDIS_KEY_ = f"{USERS_ITEM}:{uid}"
for dbms in DBMS().get_all_dbms_by_region():
user = RedisService().get_dict(dbms, _REDIS_KEY_)
if user is not None and user != {}:
break
if user is None or user == {}:
logger.info("get from mongodb")
user = UserService().get_user_by_uid(uid=uid)
if user is None:
return Result.gen_failed(404, 'user not found')
user = user.to_dict()
for dbms in DBMS().get_dbms_by_region(user['region']):
RedisService().set_dict(dbms, _REDIS_KEY_, user)
return Result.gen_success(user)
def register_new_user():
"""
用户注册
:return:
"""
data = request.json
data.pop('uid')
region = data.get('region')
for dbms in DBMS().get_dbms_by_region(region):
RedisService().delete_key_by_pattern(dbms, f"{USERS_LIST}:*")
success, msg = UserService().register(**data)
if not success:
return Result.gen_failed(code=505, msg=msg)
return Result.gen_success(msg='success')
def update_populars():
from utils.func import timestamp_to_datetime
timestamp = request.json.get('timestamp')
if timestamp is not None:
_date = timestamp_to_datetime(timestamp).date()
print("更新热门文章： 日期 -> {}".format(_date))
PopularService().update_popular(_date=_date)
else:
PopularService().update_popular()
return Result.gen_success()
# return Result.gen_failed(234, 'shibai')
pass
def update_article(aid):
data = request.json
category = data.pop('category', None)
if category is None:
return Result.gen_failed(code=5000, msg='缺少category字段')
for key in list(data.keys())[::-1]:
if key not in ArticleService.field_names or key in ArticleService.update_forbid:
data.pop(key)
data['update_time'] = get_timestamp()
# print(data)
_REDIS_KEY_ = f"{ARTICLES_ITEM}:{aid}"
for dbms in DBMS().get_dbms_by_category(category):
RedisService().delete_key(dbms, _REDIS_KEY_)
RedisService().delete_key_by_pattern(dbms, f"{ARTICLES_LIST}:*")
ArticleService().update_by_aid(aid=aid, db_alias=dbms, **data)
return Result.gen_success(data={'aid': aid, 'category': category})
def upload():
print(request.files['picFile'].filename)
file = request.files['picFile']
if file:
file_ext = get_file_ext(file.filename)
file_category = set_type(file_ext)
hdfs_path = datetime.date.today().strftime("/%Y/%m/%d")
hdfs_filename = get_hdfs_filename()
user_id = current_user.id
img_path = f"{hdfs_path}/{file_category}/{user_id}_{hdfs_filename}.{file_ext}"
try:
HDFSService().upload(path=img_path, data=file)
except Exception as err:
logger.info(f'error: {err}')
return Result.gen_failed(5000, msg='上传出现错误')
# print(data)
print(file)
return Result.gen_success(data={'img_path': img_path})
def update_user_info(uid):
"""用户信息更新"""
user = current_user
# print(user)
is_admin = (user.name == 'admin')
if is_admin:
user = UserService().get_user_by_uid(uid)
forbid = ['uid', 'pwd', 'name', 'timestamp']
if (not is_admin) or user.uid != uid:
Result.gen_failed(code=50001, msg='无权限进行此操作')
if is_admin == 'admin':
forbid = ['uid', 'pwd', 'timestamp']
data: dict = request.json
for key in list(data.keys())[::-1]:
if key not in UserService.field_names or key in forbid:
data.pop(key)
else:
# print(f'data: {key}: {data.get(key)}, origin: {getattr(user, key)}')
if data.get(key) == getattr(user, key):
data.pop(key)
if data == {}:
return Result.gen_success(msg='无更新信息')
_REDIS_KEY_ = f"{USERS_ITEM}:{uid}"
for dbms in DBMS().get_dbms_by_region(user.region):
RedisService().delete_key(dbms, _REDIS_KEY_)
RedisService().delete_key_by_pattern(dbms, pattern=f'{USERS_LIST}:*')
UserService().update_by_uid_with_dbms(uid=uid, db_alias=dbms, **data)
return Result.gen_success(msg='success')
pass
def get_populars():
page_num = int(request.args.get('page', 1))
page_size = int(request.args.get('size', 20))
dbms = request.args.get('dbms')
temporalGranularity = request.args.get('level')
sort_by = request.args.get('sort_by') or 'timestamp'
try:
check_alias(db_alias=dbms)
except DbmsAliasError:
return Result.gen_failed('404', 'dbms error')
timestamp = request.args.get('timestamp')
cons = {
'timestamp': timestamp
}
kwargs = {}
for key, value in cons.items():
if value is not None and value != '':
kwargs[key] = value
_REDIS_KEY_ = f"POPULAR_LIST:{dbms}:{page_num}:{page_size}:{temporalGranularity}:{sort_by}:{kwargs}"
data = RedisService().get_redis(dbms).get_dict(_REDIS_KEY_)
if data is None or data == {}:
ranks = PopularService().get_ranks(temporalGranularity=temporalGranularity, db_alias=dbms,
page_num=page_num,
page_size=page_size, sort_by=sort_by, **kwargs)
ranks = list(rank.to_dict() for rank in ranks)
total = PopularService().count(temporalGranularity=temporalGranularity, db_alias=dbms, **kwargs)
data = {
'total': total,
'list': ranks
}
RedisService().get_redis(dbms).set_dict(_REDIS_KEY_, data)
return Result.gen_success(data)
pass
def new_articles():
data = request.json
for key in list(data.keys())[::-1]:
if key not in ArticleService.field_names:
data.pop(key)
logger.info(f'添加文章： {data}')
# 先删除对应的缓存，然后在添加文章
for dbms in DBMS().get_dbms_by_category(data['category']):
RedisService().delete_key_by_pattern(dbms, f'{ARTICLES_LIST}:*')
aid = ArticleService().add_an_article(**data)
return Result.gen_success(data={'aid': aid})
pass
def custom_user_loader_error(identity):
ret = {
"msg": "User {} not found".format(identity)
}
return Result.gen_failed(404, msg=ret)
def logout():
# user = get_jwt_user()
return Result.gen_success("")
pass
def get_dashboard_info():
from config import DBMS
from service.user_service import UserService
from service.article_service import ArticleService
from service.read_service import ReadService
from service.popular_service import PopularService
from service.redis_service import RedisService
dbms = request.args.get('dbms')
print('get info start time: {}'.format(datetime.datetime.now()))
if dbms not in DBMS().all:
return Result.gen_failed(code=500, msg='dbms error')
_KEY_ = "DASHBOARD"
data = RedisService().get_dict(dbms, _KEY_)
if data is not None and data != {}:
return Result.gen_success(data=data)
nums = {
'users':
UserService().count(db_alias=dbms),
'articles':
ArticleService().count(db_alias=dbms),
'reads':
ReadService().count(db_alias=dbms),
'populars':
PopularService().count(temporalGranularity='daily', db_alias=dbms)
}
nodes = get_nodes(dbms)
charts = {'users': [], 'articles': [], 'reads': []}
for dbms1 in DBMS().get_all_dbms_by_region():
if dbms1 == dbms:
charts['users'].append({'name': dbms1, 'value': nums['users']})
charts['articles'].append({
'name': dbms1,
'value': nums['articles']
})
charts['reads'].append({'name': dbms1, 'value': nums['reads']})
else:
charts['users'].append({
'name': dbms1,
'value': UserService().count(db_alias=dbms1)
})
charts['articles'].append({
'name':
dbms1,
'value':
ArticleService().count(db_alias=dbms1)
})
charts['reads'].append({
'name': dbms1,
'value': ReadService().count(db_alias=dbms1)
})
data = {'nums': nums, 'charts': charts, 'nodes': nodes}
print('end info start time: {}'.format(datetime.datetime.now()))
RedisService().set_dict(dbms, _KEY_, data)
return Result.gen_success(data=data)
pass
