async def post(self):
r_dict = {'code': 0, 'line': None}
category = self.get_argument('category')
if not category:
return r_dict
try:
category = int(category)
condition = self.get_argument('condition_value')
x_axis = self.get_argument('xAxis', '')
stats = None # 统计结果
if category == CATEGORY_MEMBER_LEARN_SUBJECT_WRONG:
stats = await SubjectWrongViewedStatistics.aggregate(
stage_list=[
MatchStage(parse_condition(condition)),
GroupStage('count', sum={'$sum': 1}),
SortStage([('_id', ASC)]),
LimitStage(6)
]).to_list(None)
if category == CATEGORY_MEMBER_LEARN_SUBJECT_RESOLVING_VIEWED:
stats = await SubjectResolvingViewedStatistics.aggregate(
stage_list=[
MatchStage(parse_condition(condition)),
GroupStage('member_cid', sum={'$sum': 1}),
GroupStage('sum', sum={'$sum': 1}),
SortStage([('_id', ASC)]),
LimitStage(6)
]).to_list(None)
if category == CATEGORY_MEMBER_LEARN_SUBJECT_RESOLVING_TREND:
stats = await SubjectResolvingViewedStatistics.aggregate(
stage_list=[
MatchStage(parse_condition(condition)),
MatchStage({'wrong_count': {
'$gt': 0
}}),
GroupStage('wrong_count', sum={'$sum': 1}),
SortStage([('_id', ASC)]),
LimitStage(6)
]).to_list(None)
if category == CATEGORY_MEMBER_LEARN_SUBJECT_PERSONAL_CENTER:
stats = await PersonalCenterViewedStatistics.aggregate(
stage_list=[
MatchStage(parse_condition(condition)),
GroupStage('count', sum={'$sum': 1}),
SortStage([('_id', ASC)]),
LimitStage(6)
]).to_list(None)
member_count = sum([s.sum for s in stats])
if not x_axis:
x_axis_data = [s.id for s in stats]
if member_count == 0:
series_data = [0] * len(x_axis_data)
else:
series_data = [s.sum / member_count * 100 for s in stats]
else:
x_axis_data = json.loads(x_axis)
stat_id_map = {s.id: s for s in stats}
if member_count == 0:
series_data = [0] * len(x_axis_data)
else:
series_data = [
stat_id_map[data.get('value')].sum / member_count *
100 if data.get('value') in stat_id_map else 0
for data in x_axis_data
]
r_dict = {
'code': 1,
'line': {
'xAxisData': x_axis_data,
'seriesData': series_data
}
}
except Exception:
logger.error(traceback.format_exc())
return r_dict
def do_statistics_member_active(cache_key, m_city_code_list,
province_code_list, city_code_list,
gender_list, age_group_list, education_list):
"""
:param cache_key:
:param m_city_code_list:
:param province_code_list:
:param city_code_list:
:param gender_list:
:param age_group_list:
:param education_list:
:return:
"""
RedisCache.set(cache_key, KEY_CACHE_REPORT_DOING_NOW, 5 * 60)
# 统计数据
stage_list = []
if m_city_code_list:
stage_list.append(MatchStage({'city_code': {'$in': m_city_code_list}}))
# 取前一天凌晨12点之前的数据
time_match = get_yesterday()
stage_list.append(MatchStage({'updated_dt': {'$lt': time_match}}))
query_dict = {}
if province_code_list:
query_dict['province_code'] = {'$in': province_code_list}
if city_code_list:
query_dict['city_code'] = {'$in': city_code_list}
if gender_list:
query_dict['gender'] = {
'$in': [int(s_gender) for s_gender in gender_list]
}
if age_group_list:
query_dict['age_group'] = {
'$in': [int(s_age_group) for s_age_group in age_group_list]
}
if education_list:
query_dict['education'] = {
'$in': [int(s_education) for s_education in education_list]
}
if query_dict:
stage_list.append(MatchStage(query_dict))
stage_list.extend([
GroupStage('learning_code', quantity={'$sum': 1}),
SortStage([('_id', ASC)]),
LimitStage(8)
])
mld_stat_cursor = MemberLearningDayStatistics.sync_aggregate(stage_list)
data = []
while True:
try:
mld_stat = mld_stat_cursor.next()
if mld_stat:
data.append({
'days': mld_stat.id,
'quantity': mld_stat.quantity
})
except StopIteration:
break
all_members = sum([d.get('quantity') for d in data])
data_dict = {'data_list': data, 'all_members': all_members}
if not data_dict:
early_warning_empty("start_statistics_member_active", cache_key,
locals(), '答题活跃度统计数据为空，请检查！')
RedisCache.set(cache_key, msgpack.packb(data_dict))
def do_statistics_member_top_n(cache_key, m_city_code_list, stat_type, top_n,
time_range):
"""
:param cache_key:
:param m_city_code_list:
:param stat_type:
:param top_n:
:param time_range:
:return:
"""
RedisCache.set(cache_key, KEY_CACHE_REPORT_DOING_NOW, 5 * 60)
stage_list = []
# 取前一天凌晨12点之前的数据
time_match = get_yesterday()
stage_list.append(MatchStage({'updated_dt': {'$lt': time_match}}))
if m_city_code_list:
stage_list.append(MatchStage({'city_code': {'$in': m_city_code_list}}))
s_code = ''
e_code = ''
if time_range:
suffix = time_range[-1:]
range_num = int(time_range.replace(suffix, ''))
delta = None
if suffix.upper() == 'D':
delta = datetime.timedelta(days=range_num)
elif suffix.upper() == 'M':
delta = datetime.timedelta(days=range_num * 30)
elif suffix.upper() == 'Y':
delta = datetime.timedelta(days=range_num * 365)
if delta:
s_code = datetime2str(datetime.datetime.now() - delta,
date_format='%Y%m%d000000')
e_code = datetime2str(datetime.datetime.now(),
date_format='%Y%m%d000000')
if s_code and e_code:
stage_list.append(
MatchStage({'daily_code': {
'$gte': s_code,
'$lte': e_code
}}))
stage_list.extend([
GroupStage({
'daily_code': '$daily_code',
'member_cid': '$member_cid',
'province_code': '$province_code'
} if stat_type == 1 else {
'daily_code': '$daily_code',
'member_cid': '$member_cid',
'province_code': '$province_code',
'city_code': '$city_code'
},
learn_times={'$sum': '$learn_times'}),
GroupStage({
'daily_code': '$_id.daily_code',
'province_code': '$_id.province_code'
} if stat_type == 1 else {
'daily_code': '$_id.daily_code',
'province_code': '$_id.province_code',
'city_code': '$_id.city_code'
},
count={'$sum': 1},
times={'$sum': '$learn_times'}),
LookupStage(AdministrativeDivision, '_id.province_code', 'post_code',
'province_list'),
LookupStage(AdministrativeDivision, '_id.city_code', 'post_code',
'city_list'),
ProjectStage(
**{
'_id':
False,
'daily_code':
'$_id.daily_code',
'count':
'$count',
'times':
'$times',
'province_code':
'$_id.province_code',
'province_title':
'$province_list.title',
'ad_code':
'$_id.province_code' if stat_type == 1 else '$_id.city_code',
'ad_title':
'$province_list.title' if stat_type ==
1 else '$city_list.title'
}),
SortStage([('daily_code', ASC), ('count', DESC)])
])
# 检索数据
data = []
stat_cursor = MemberDailyStatistics.sync_aggregate(stage_list)
t_code, t_list = None, None
top_n_found = False
while True:
try:
daily_stat = stat_cursor.next()
if daily_stat:
daily_code = daily_stat.daily_code
if not daily_code == t_code:
t_code = daily_code
t_list = []
top_n_found = False
print(t_code)
if len(t_list) < top_n:
t_list.append({
'date':
daily_code[:8],
'ad_code':
daily_stat.ad_code,
'province_title':
daily_stat.province_title[0] if
daily_stat.province_title and stat_type == 2 else '',
'title':
daily_stat.ad_title[0]
if daily_stat.ad_title else 'undefined',
'quantity':
daily_stat.count,
'times':
daily_stat.times
})
elif not top_n_found:
if t_code is not None:
data.append(t_list)
top_n_found = True
except StopIteration:
break
if not data:
early_warning_empty("start_statistics_member_top_n", cache_key,
locals(), '每日参与TOP5统计数据为空，请检查！')
RedisCache.set(cache_key, msgpack.packb(data))
def do_stat_in_history(history_model, time_match, basic_stages, stat_type):
"""
:param history_model:
:param time_match
:param basic_stages:
:param stat_type:
:return:
"""
stage_list = [
time_match,
ProjectStage(
**{
'daily_code': {
'$dateToString': {
'format': "%Y%m%d",
"date": "$updated_dt"
}
},
'member_cid': "$member_cid",
'updated_dt': '$updated_dt'
}),
GroupStage({
'member_cid': '$member_cid',
'daily_code': '$daily_code'
},
quantity={"$sum": 1},
updated_dt={'$first': "$updated_dt"}),
LookupStage(Member, '_id.member_cid', 'cid', 'member_list'),
MatchStage({'member_list': {
'$ne': []
}}),
ProjectStage(
**{
'daily_code': '$_id.daily_code',
'member_cid': '$_id.member_cid',
'updated_dt': '$updated_dt',
'quantity': '$quantity',
'city_code': {
'$arrayElemAt': ['$member_list.city_code', 0]
},
'province_code': {
'$arrayElemAt': ['$member_list.province_code', 0]
},
'sex': {
'$arrayElemAt': ['$member_list.sex', 0]
},
'age_group': {
'$arrayElemAt': ['$member_list.age_group', 0]
},
'education': {
'$arrayElemAt': ['$member_list.education', 0]
},
}),
]
if basic_stages:
stage_list.extend(basic_stages)
if stat_type == 'time':
quantity = {'$sum': "$quantity"}
else:
quantity = {'$sum': 1}
stage_list.extend([
GroupStage('daily_code', quantity=quantity),
SortStage([('_id', ASC)])
])
cursor = history_model.sync_aggregate(stage_list, allowDiskUse=True)
data = {}
while True:
try:
his = cursor.next()
data[his.id] = his.quantity
except StopIteration:
break
except Exception as e:
logger.error(str(e))
continue
return data
def do_statistics_learning_situation(cache_key,
chart_type=None,
m_city_code_list=None,
gender_list=None,
province_code_list=None,
city_code_list=None,
age_group_list=None,
education_list=None,
dimension=None,
time_range=None,
dimension_code=None):
"""
:param cache_key:
:param chart_type:
:param m_city_code_list:
:param gender_list:
:param province_code_list:
:param city_code_list:
:param age_group_list:
:param education_list:
:param dimension:
:param time_range:
:param dimension_code:
:return:
"""
RedisCache.set(cache_key, KEY_CACHE_REPORT_DOING_NOW, 5 * 60)
stage_list = []
# 取前一天凌晨12点之前的数据
time_match = get_yesterday()
stage_list.append(MatchStage({'created_dt': {'$lt': time_match}}))
s_code, e_code = '', ''
if chart_type == 1:
s_code, e_code = get_daily_code_range(time_range)
if city_code_list:
stage_list.append(MatchStage({'city_code': {'$in': m_city_code_list}}))
parent_dimension_cid, dimension_cid = get_dimension(dimension_code)
if parent_dimension_cid and dimension_cid:
stage_list.append(
MatchStage({'dimension.%s' % parent_dimension_cid: dimension_cid}))
query_dict = {}
if s_code and e_code:
query_dict['daily_code'] = {'$gte': s_code, '$lte': e_code}
if province_code_list:
query_dict['province_code'] = {'$in': province_code_list}
if city_code_list:
query_dict['city_code'] = {'$in': city_code_list}
if gender_list:
query_dict['gender'] = {
'$in': [int(s_gender) for s_gender in gender_list]
}
if age_group_list:
query_dict['age_group'] = {
'$in': [int(s_age_group) for s_age_group in age_group_list]
}
if education_list:
query_dict['education'] = {
'$in': [int(s_education) for s_education in education_list]
}
if dimension:
try:
s_dimension = json.loads(dimension)
for k, v in s_dimension.items():
query_dict['dimension.%s' % k] = {'$in': v}
except Exception:
pass
if query_dict:
stage_list.append(MatchStage(query_dict))
stage_list.append(
GroupStage('daily_code' if chart_type == 1 else 'learning_code',
total={'$sum': '$subject_total_quantity'},
correct={'$sum': '$subject_correct_quantity'}))
stage_list.append(SortStage([('_id', ASC)]))
if chart_type == 2:
stage_list.append(MatchStage({'_id': {'$lte': 20}}))
data = []
# 检索数据
if chart_type == 1:
stat_cursor = MemberDailyDimensionStatistics.sync_aggregate(stage_list)
else:
stat_cursor = MemberLearningDayDimensionStatistics.sync_aggregate(
stage_list)
while True:
try:
md_stat = stat_cursor.next()
if md_stat:
data.append({
md_stat.id[:8] if chart_type == 1 else md_stat.id: {
'total': md_stat.total,
'correct': md_stat.correct
}
})
except StopIteration:
break
if not data:
early_warning_empty("start_statistics_learning_situation", cache_key,
locals(), '学习效果中数据为空，请检查！')
RedisCache.set(cache_key, msgpack.packb(data))
async def post(self):
r_dict = {'code': 0}
open_id = self.get_i_argument('open_id')
if not open_id:
r_dict['code'] = 1001
return r_dict
try:
member = await find_member_by_open_id(open_id)
if not member:
r_dict['code'] = 1002
return r_dict
wrong_subjects = await MemberWrongSubject.aggregate([
MatchStage({'member_cid': member.cid, 'status': STATUS_WRONG_SUBJECT_ACTIVE}),
SortStage([('wrong_dt', DESC)]),
LimitStage(10),
LookupStage(Subject, 'subject_cid', 'cid', 'subject_list'),
ProjectStage(**{
'dan_grade': {'$arrayElemAt': ['$member_info.dan_grade', 0]},
'cid': '$subject_cid',
'title': {'$arrayElemAt': ['$subject_list.title', 0]},
'dimension_dict': {'$arrayElemAt': ['$subject_list.dimension_dict', 0]},
'knowledge_first': {'$arrayElemAt': ['$subject_list.knowledge_first', 0]},
'knowledge_second': {'$arrayElemAt': ['$subject_list.knowledge_second', 0]},
'resolving': {'$arrayElemAt': ['$subject_list.resolving', 0]},
'option_cid': '$option_cid',
'image_cid': {'$arrayElemAt': ['$subject_list.image_cid', 0]}
}),
LookupStage(SubjectOption, let={'cid': '$cid'},
pipeline=[{'$match': {'$expr': {'$and': [{'$eq': ['$subject_cid', '$$cid']}]}}},
SortStage([('code', ASC)])], as_list_name='option_list'),
LookupStage(UploadFiles, 'image_cid', 'cid', 'image_list')
]).to_list(10)
question_list = []
for sbj in wrong_subjects:
picture_url = ''
if sbj.image_list:
picture_url = '%s://%s%s%s%s' % (
SERVER_PROTOCOL, SERVER_HOST, STATIC_URL_PREFIX, 'files/', sbj.image_list[0].title)
correct, error = get_cached_subject_accuracy(sbj.cid)
all_amount = correct + error
difficult = await SubjectDimension.get_by_cid(
getattr(sbj.dimension_dict, "8187881ABEAD2CF4A4F24C0AA02B2C2D"))
category = await SubjectDimension.get_by_cid(
getattr(sbj.dimension_dict, "18114DC7C31B17AE35841CAE833161A2"))
question = {'id': sbj.cid,
'title': sbj.title,
'picture_url': picture_url,
'category': getattr(category, 'title', ''),
'difficulty_degree': "%s/5" % getattr(difficult, 'ordered', 1), # ordered: 难度
'knowledge_first': KNOWLEDGE_FIRST_LEVEL_DICT.get(sbj.knowledge_first, ''),
'knowledge_second': KNOWLEDGE_SECOND_LEVEL_DICT.get(sbj.knowledge_second, ''),
'resolving': sbj.resolving,
'correct_percent': format(float(correct) / float(all_amount),
'.2%') if all_amount else '0%',
'option_id': sbj.option_cid,
'option_list': [{
'id': opt.cid, 'title': opt.title, 'true_answer': opt.correct
} for opt in sbj.option_list]}
question_list.append(question)
wrong_stat = await SubjectWrongViewedStatistics.find_one({'member_cid': member.cid})
if not wrong_stat:
wrong_stat = SubjectWrongViewedStatistics(member_cid=member.cid, count=0)
wrong_stat.sex = member.sex
wrong_stat.age_group = member.age_group
wrong_stat.education = member.education
wrong_stat.province_code = member.province_code
wrong_stat.city_code = member.city_code
wrong_stat.count += 1
await wrong_stat.save()
r_dict = {'code': 1000, 'question_list': question_list}
except Exception:
logger.error(traceback.format_exc())
return r_dict
async def post(self):
r_dict = {'code': 0}
try:
pageNum = int(self.get_i_argument('pageNum', 1))
size = int(self.get_i_argument('size', 10))
search_type = self.get_i_argument('search_type', 'all')
skip = (pageNum - 1) * size if (pageNum - 1) * size > 0 else 0
filter_dict = {'db_mark': {'$ne': ''}, 'release_time': {'$ne': ''}}
dif_time = 0
if search_type == 'week':
dif_time = 7
elif search_type == 'month':
dif_time = 30
elif search_type == 'year':
dif_time = 365
if dif_time:
show_time = datetime.datetime.now().replace(
hour=0, minute=0, second=0,
microsecond=0) - datetime.timedelta(days=dif_time)
filter_dict = {
'db_mark': {
'$ne': ''
},
'release_time': {
'$ne': '',
'$gte': show_time
}
}
match = MatchStage(filter_dict)
skip = SkipStage(skip)
sort = SortStage([('db_mark', DESC)])
limit = LimitStage(int(size))
count = await Tvs.count({
'db_mark': {
'$ne': ''
},
'release_time': {
'$ne': ''
}
})
tvs = await Tvs.aggregate([match, sort, skip, limit]).to_list(None)
new_tvs = []
for tv in tvs:
new_tvs.append({
'id':
str(tv.id),
'name':
tv.name,
'pic_url':
tv.pic_url,
'db_mark':
tv.db_mark,
'actor':
tv.actor,
'source_nums':
len(tv.download),
'release_time':
tv.release_time.strftime('%Y-%m-%d'),
'label':
api_utils.get_show_source_label(tv),
'recommend_info':
'这部神剧值得一看。',
'set_num':
tv.set_num if tv.set_num else '',
's_type':
'tv'
})
r_dict['tvs'] = new_tvs
r_dict['count'] = count
r_dict['code'] = 1000
except Exception:
logger.error(traceback.format_exc())
print('score')
print(r_dict)
return r_dict
async def post(self):
r_dict = {'code': 0, 'line': None}
category = self.get_argument('category')
condition = self.get_argument('condition_value')
x_axis = self.get_argument('xAxis', '')
if not category:
member_list = await Member.aggregate(
stage_list=[GroupStage('source', sum={'$sum': 1})]
).to_list(None)
member_source = {m.id: m.sum for m in member_list}
# fix: 特殊处理，将旧版线上数据与扫码数据合并
# 详见： http://code.wenjuan.com/WolvesAU/CRSPN/issues/10
# 最新定制：调查扫码(线上扫码-6、线下扫码-1)
member_source['126'] = member_source.get(
'1', 0) + member_source.get('6', 0)
# 直接扫码
member_source['direct_scan'] = member_source.get('2', 0) + \
member_source.get('CA755167DEA9AA89650D11C10FAA5413', 0) + \
member_source.get('160631F26D00F7A2DC56DAE2A0C4AF12', 0) + \
member_source.get('F742E0C7CA5F7E175844478D74484C29', 0)
source_list = await GameMemberSource.find({
'status':
STATUS_MEMBER_SOURCE_ACTIVE
}).to_list(None)
source_dict = {source.code: source.title for source in source_list}
source_dict = adjust_source(source_dict)
r_dict = {
'code': 1,
'pie': {
'legendData': [v for _, v in source_dict.items()],
'seriesData': [{
'name': name,
'value': member_source.get(code)
} for code, name in source_dict.items()]
}
}
return r_dict
try:
# fix: 特殊处理，将旧版线上数据与扫码数据合并
# 详见： http://code.wenjuan.com/WolvesAU/CRSPN/issues/10
source_match = {'source': category}
if category == '126':
source_match = {'source': {'$in': list('16')}}
elif category == '3':
source_match = {
'source': {
'$in': [
'2', '3', 'CA755167DEA9AA89650D11C10FAA5413',
'160631F26D00F7A2DC56DAE2A0C4AF12',
'F742E0C7CA5F7E175844478D74484C29'
]
}
}
member_list = await Member.aggregate([
ProjectStage(
**{
'_id': '$id',
'created_dt': {
"$dateToString": {
"format": "%Y-%m-%d",
"date": "$created_dt"
}
},
'source': '$source',
'province_code': '$province_code',
'city_code': '$city_code',
'sex': '$sex',
'age': '$age_group',
'education': '$education'
}),
MatchStage(parse_condition(condition)),
MatchStage(source_match),
GroupStage('created_dt', sum={'$sum': 1}),
SortStage([('_id', ASC)]),
]).to_list(None)
if not x_axis:
x_axis_data = [m.id for m in member_list]
series_data = [m.sum for m in member_list]
else:
x_axis_data = json.loads(x_axis)
member_id_map = {m.id: m for m in member_list}
series_data = [
member_id_map[data.get('value')].sum
if data.get('value') in member_id_map else 0
for data in x_axis_data
]
r_dict = {
'code': 1,
'line': {
'xAxisData': x_axis_data,
'seriesData': series_data
}
}
except Exception:
logger.error(traceback.format_exc())
return r_dict
