async def post(self): r_dict = {'code': 0, 'line': None} category = self.get_argument('category') if not category: return r_dict try: category = int(category) condition = self.get_argument('condition_value') x_axis = self.get_argument('xAxis', '') stats = None # 统计结果 if category == CATEGORY_MEMBER_LEARN_SUBJECT_WRONG: stats = await SubjectWrongViewedStatistics.aggregate( stage_list=[ MatchStage(parse_condition(condition)), GroupStage('count', sum={'$sum': 1}), SortStage([('_id', ASC)]), LimitStage(6) ]).to_list(None) if category == CATEGORY_MEMBER_LEARN_SUBJECT_RESOLVING_VIEWED: stats = await SubjectResolvingViewedStatistics.aggregate( stage_list=[ MatchStage(parse_condition(condition)), GroupStage('member_cid', sum={'$sum': 1}), GroupStage('sum', sum={'$sum': 1}), SortStage([('_id', ASC)]), LimitStage(6) ]).to_list(None) if category == CATEGORY_MEMBER_LEARN_SUBJECT_RESOLVING_TREND: stats = await SubjectResolvingViewedStatistics.aggregate( stage_list=[ MatchStage(parse_condition(condition)), MatchStage({'wrong_count': { '$gt': 0 }}), GroupStage('wrong_count', sum={'$sum': 1}), SortStage([('_id', ASC)]), LimitStage(6) ]).to_list(None) if category == CATEGORY_MEMBER_LEARN_SUBJECT_PERSONAL_CENTER: stats = await PersonalCenterViewedStatistics.aggregate( stage_list=[ MatchStage(parse_condition(condition)), GroupStage('count', sum={'$sum': 1}), SortStage([('_id', ASC)]), LimitStage(6) ]).to_list(None) member_count = sum([s.sum for s in stats]) if not x_axis: x_axis_data = [s.id for s in stats] if member_count == 0: series_data = [0] * len(x_axis_data) else: series_data = [s.sum / member_count * 100 for s in stats] else: x_axis_data = json.loads(x_axis) stat_id_map = {s.id: s for s in stats} if member_count == 0: series_data = [0] * len(x_axis_data) else: series_data = [ stat_id_map[data.get('value')].sum / member_count * 100 if data.get('value') in stat_id_map else 0 for data in x_axis_data ] r_dict = { 'code': 1, 'line': { 'xAxisData': x_axis_data, 'seriesData': series_data } } except Exception: logger.error(traceback.format_exc()) return r_dict
def do_statistics_member_active(cache_key, m_city_code_list, province_code_list, city_code_list, gender_list, age_group_list, education_list): """ :param cache_key: :param m_city_code_list: :param province_code_list: :param city_code_list: :param gender_list: :param age_group_list: :param education_list: :return: """ RedisCache.set(cache_key, KEY_CACHE_REPORT_DOING_NOW, 5 * 60) # 统计数据 stage_list = [] if m_city_code_list: stage_list.append(MatchStage({'city_code': {'$in': m_city_code_list}})) # 取前一天凌晨12点之前的数据 time_match = get_yesterday() stage_list.append(MatchStage({'updated_dt': {'$lt': time_match}})) query_dict = {} if province_code_list: query_dict['province_code'] = {'$in': province_code_list} if city_code_list: query_dict['city_code'] = {'$in': city_code_list} if gender_list: query_dict['gender'] = { '$in': [int(s_gender) for s_gender in gender_list] } if age_group_list: query_dict['age_group'] = { '$in': [int(s_age_group) for s_age_group in age_group_list] } if education_list: query_dict['education'] = { '$in': [int(s_education) for s_education in education_list] } if query_dict: stage_list.append(MatchStage(query_dict)) stage_list.extend([ GroupStage('learning_code', quantity={'$sum': 1}), SortStage([('_id', ASC)]), LimitStage(8) ]) mld_stat_cursor = MemberLearningDayStatistics.sync_aggregate(stage_list) data = [] while True: try: mld_stat = mld_stat_cursor.next() if mld_stat: data.append({ 'days': mld_stat.id, 'quantity': mld_stat.quantity }) except StopIteration: break all_members = sum([d.get('quantity') for d in data]) data_dict = {'data_list': data, 'all_members': all_members} if not data_dict: early_warning_empty("start_statistics_member_active", cache_key, locals(), '答题活跃度统计数据为空,请检查!') RedisCache.set(cache_key, msgpack.packb(data_dict))
def do_statistics_member_top_n(cache_key, m_city_code_list, stat_type, top_n, time_range): """ :param cache_key: :param m_city_code_list: :param stat_type: :param top_n: :param time_range: :return: """ RedisCache.set(cache_key, KEY_CACHE_REPORT_DOING_NOW, 5 * 60) stage_list = [] # 取前一天凌晨12点之前的数据 time_match = get_yesterday() stage_list.append(MatchStage({'updated_dt': {'$lt': time_match}})) if m_city_code_list: stage_list.append(MatchStage({'city_code': {'$in': m_city_code_list}})) s_code = '' e_code = '' if time_range: suffix = time_range[-1:] range_num = int(time_range.replace(suffix, '')) delta = None if suffix.upper() == 'D': delta = datetime.timedelta(days=range_num) elif suffix.upper() == 'M': delta = datetime.timedelta(days=range_num * 30) elif suffix.upper() == 'Y': delta = datetime.timedelta(days=range_num * 365) if delta: s_code = datetime2str(datetime.datetime.now() - delta, date_format='%Y%m%d000000') e_code = datetime2str(datetime.datetime.now(), date_format='%Y%m%d000000') if s_code and e_code: stage_list.append( MatchStage({'daily_code': { '$gte': s_code, '$lte': e_code }})) stage_list.extend([ GroupStage({ 'daily_code': '$daily_code', 'member_cid': '$member_cid', 'province_code': '$province_code' } if stat_type == 1 else { 'daily_code': '$daily_code', 'member_cid': '$member_cid', 'province_code': '$province_code', 'city_code': '$city_code' }, learn_times={'$sum': '$learn_times'}), GroupStage({ 'daily_code': '$_id.daily_code', 'province_code': '$_id.province_code' } if stat_type == 1 else { 'daily_code': '$_id.daily_code', 'province_code': '$_id.province_code', 'city_code': '$_id.city_code' }, count={'$sum': 1}, times={'$sum': '$learn_times'}), LookupStage(AdministrativeDivision, '_id.province_code', 'post_code', 'province_list'), LookupStage(AdministrativeDivision, '_id.city_code', 'post_code', 'city_list'), ProjectStage( **{ '_id': False, 'daily_code': '$_id.daily_code', 'count': '$count', 'times': '$times', 'province_code': '$_id.province_code', 'province_title': '$province_list.title', 'ad_code': '$_id.province_code' if stat_type == 1 else '$_id.city_code', 'ad_title': '$province_list.title' if stat_type == 1 else '$city_list.title' }), SortStage([('daily_code', ASC), ('count', DESC)]) ]) # 检索数据 data = [] stat_cursor = MemberDailyStatistics.sync_aggregate(stage_list) t_code, t_list = None, None top_n_found = False while True: try: daily_stat = stat_cursor.next() if daily_stat: daily_code = daily_stat.daily_code if not daily_code == t_code: t_code = daily_code t_list = [] top_n_found = False print(t_code) if len(t_list) < top_n: t_list.append({ 'date': daily_code[:8], 'ad_code': daily_stat.ad_code, 'province_title': daily_stat.province_title[0] if daily_stat.province_title and stat_type == 2 else '', 'title': daily_stat.ad_title[0] if daily_stat.ad_title else 'undefined', 'quantity': daily_stat.count, 'times': daily_stat.times }) elif not top_n_found: if t_code is not None: data.append(t_list) top_n_found = True except StopIteration: break if not data: early_warning_empty("start_statistics_member_top_n", cache_key, locals(), '每日参与TOP5统计数据为空,请检查!') RedisCache.set(cache_key, msgpack.packb(data))
def do_stat_in_history(history_model, time_match, basic_stages, stat_type): """ :param history_model: :param time_match :param basic_stages: :param stat_type: :return: """ stage_list = [ time_match, ProjectStage( **{ 'daily_code': { '$dateToString': { 'format': "%Y%m%d", "date": "$updated_dt" } }, 'member_cid': "$member_cid", 'updated_dt': '$updated_dt' }), GroupStage({ 'member_cid': '$member_cid', 'daily_code': '$daily_code' }, quantity={"$sum": 1}, updated_dt={'$first': "$updated_dt"}), LookupStage(Member, '_id.member_cid', 'cid', 'member_list'), MatchStage({'member_list': { '$ne': [] }}), ProjectStage( **{ 'daily_code': '$_id.daily_code', 'member_cid': '$_id.member_cid', 'updated_dt': '$updated_dt', 'quantity': '$quantity', 'city_code': { '$arrayElemAt': ['$member_list.city_code', 0] }, 'province_code': { '$arrayElemAt': ['$member_list.province_code', 0] }, 'sex': { '$arrayElemAt': ['$member_list.sex', 0] }, 'age_group': { '$arrayElemAt': ['$member_list.age_group', 0] }, 'education': { '$arrayElemAt': ['$member_list.education', 0] }, }), ] if basic_stages: stage_list.extend(basic_stages) if stat_type == 'time': quantity = {'$sum': "$quantity"} else: quantity = {'$sum': 1} stage_list.extend([ GroupStage('daily_code', quantity=quantity), SortStage([('_id', ASC)]) ]) cursor = history_model.sync_aggregate(stage_list, allowDiskUse=True) data = {} while True: try: his = cursor.next() data[his.id] = his.quantity except StopIteration: break except Exception as e: logger.error(str(e)) continue return data
def do_statistics_learning_situation(cache_key, chart_type=None, m_city_code_list=None, gender_list=None, province_code_list=None, city_code_list=None, age_group_list=None, education_list=None, dimension=None, time_range=None, dimension_code=None): """ :param cache_key: :param chart_type: :param m_city_code_list: :param gender_list: :param province_code_list: :param city_code_list: :param age_group_list: :param education_list: :param dimension: :param time_range: :param dimension_code: :return: """ RedisCache.set(cache_key, KEY_CACHE_REPORT_DOING_NOW, 5 * 60) stage_list = [] # 取前一天凌晨12点之前的数据 time_match = get_yesterday() stage_list.append(MatchStage({'created_dt': {'$lt': time_match}})) s_code, e_code = '', '' if chart_type == 1: s_code, e_code = get_daily_code_range(time_range) if city_code_list: stage_list.append(MatchStage({'city_code': {'$in': m_city_code_list}})) parent_dimension_cid, dimension_cid = get_dimension(dimension_code) if parent_dimension_cid and dimension_cid: stage_list.append( MatchStage({'dimension.%s' % parent_dimension_cid: dimension_cid})) query_dict = {} if s_code and e_code: query_dict['daily_code'] = {'$gte': s_code, '$lte': e_code} if province_code_list: query_dict['province_code'] = {'$in': province_code_list} if city_code_list: query_dict['city_code'] = {'$in': city_code_list} if gender_list: query_dict['gender'] = { '$in': [int(s_gender) for s_gender in gender_list] } if age_group_list: query_dict['age_group'] = { '$in': [int(s_age_group) for s_age_group in age_group_list] } if education_list: query_dict['education'] = { '$in': [int(s_education) for s_education in education_list] } if dimension: try: s_dimension = json.loads(dimension) for k, v in s_dimension.items(): query_dict['dimension.%s' % k] = {'$in': v} except Exception: pass if query_dict: stage_list.append(MatchStage(query_dict)) stage_list.append( GroupStage('daily_code' if chart_type == 1 else 'learning_code', total={'$sum': '$subject_total_quantity'}, correct={'$sum': '$subject_correct_quantity'})) stage_list.append(SortStage([('_id', ASC)])) if chart_type == 2: stage_list.append(MatchStage({'_id': {'$lte': 20}})) data = [] # 检索数据 if chart_type == 1: stat_cursor = MemberDailyDimensionStatistics.sync_aggregate(stage_list) else: stat_cursor = MemberLearningDayDimensionStatistics.sync_aggregate( stage_list) while True: try: md_stat = stat_cursor.next() if md_stat: data.append({ md_stat.id[:8] if chart_type == 1 else md_stat.id: { 'total': md_stat.total, 'correct': md_stat.correct } }) except StopIteration: break if not data: early_warning_empty("start_statistics_learning_situation", cache_key, locals(), '学习效果中数据为空,请检查!') RedisCache.set(cache_key, msgpack.packb(data))
async def post(self): r_dict = {'code': 0} open_id = self.get_i_argument('open_id') if not open_id: r_dict['code'] = 1001 return r_dict try: member = await find_member_by_open_id(open_id) if not member: r_dict['code'] = 1002 return r_dict wrong_subjects = await MemberWrongSubject.aggregate([ MatchStage({'member_cid': member.cid, 'status': STATUS_WRONG_SUBJECT_ACTIVE}), SortStage([('wrong_dt', DESC)]), LimitStage(10), LookupStage(Subject, 'subject_cid', 'cid', 'subject_list'), ProjectStage(**{ 'dan_grade': {'$arrayElemAt': ['$member_info.dan_grade', 0]}, 'cid': '$subject_cid', 'title': {'$arrayElemAt': ['$subject_list.title', 0]}, 'dimension_dict': {'$arrayElemAt': ['$subject_list.dimension_dict', 0]}, 'knowledge_first': {'$arrayElemAt': ['$subject_list.knowledge_first', 0]}, 'knowledge_second': {'$arrayElemAt': ['$subject_list.knowledge_second', 0]}, 'resolving': {'$arrayElemAt': ['$subject_list.resolving', 0]}, 'option_cid': '$option_cid', 'image_cid': {'$arrayElemAt': ['$subject_list.image_cid', 0]} }), LookupStage(SubjectOption, let={'cid': '$cid'}, pipeline=[{'$match': {'$expr': {'$and': [{'$eq': ['$subject_cid', '$$cid']}]}}}, SortStage([('code', ASC)])], as_list_name='option_list'), LookupStage(UploadFiles, 'image_cid', 'cid', 'image_list') ]).to_list(10) question_list = [] for sbj in wrong_subjects: picture_url = '' if sbj.image_list: picture_url = '%s://%s%s%s%s' % ( SERVER_PROTOCOL, SERVER_HOST, STATIC_URL_PREFIX, 'files/', sbj.image_list[0].title) correct, error = get_cached_subject_accuracy(sbj.cid) all_amount = correct + error difficult = await SubjectDimension.get_by_cid( getattr(sbj.dimension_dict, "8187881ABEAD2CF4A4F24C0AA02B2C2D")) category = await SubjectDimension.get_by_cid( getattr(sbj.dimension_dict, "18114DC7C31B17AE35841CAE833161A2")) question = {'id': sbj.cid, 'title': sbj.title, 'picture_url': picture_url, 'category': getattr(category, 'title', ''), 'difficulty_degree': "%s/5" % getattr(difficult, 'ordered', 1), # ordered: 难度 'knowledge_first': KNOWLEDGE_FIRST_LEVEL_DICT.get(sbj.knowledge_first, ''), 'knowledge_second': KNOWLEDGE_SECOND_LEVEL_DICT.get(sbj.knowledge_second, ''), 'resolving': sbj.resolving, 'correct_percent': format(float(correct) / float(all_amount), '.2%') if all_amount else '0%', 'option_id': sbj.option_cid, 'option_list': [{ 'id': opt.cid, 'title': opt.title, 'true_answer': opt.correct } for opt in sbj.option_list]} question_list.append(question) wrong_stat = await SubjectWrongViewedStatistics.find_one({'member_cid': member.cid}) if not wrong_stat: wrong_stat = SubjectWrongViewedStatistics(member_cid=member.cid, count=0) wrong_stat.sex = member.sex wrong_stat.age_group = member.age_group wrong_stat.education = member.education wrong_stat.province_code = member.province_code wrong_stat.city_code = member.city_code wrong_stat.count += 1 await wrong_stat.save() r_dict = {'code': 1000, 'question_list': question_list} except Exception: logger.error(traceback.format_exc()) return r_dict
async def post(self): r_dict = {'code': 0} try: pageNum = int(self.get_i_argument('pageNum', 1)) size = int(self.get_i_argument('size', 10)) search_type = self.get_i_argument('search_type', 'all') skip = (pageNum - 1) * size if (pageNum - 1) * size > 0 else 0 filter_dict = {'db_mark': {'$ne': ''}, 'release_time': {'$ne': ''}} dif_time = 0 if search_type == 'week': dif_time = 7 elif search_type == 'month': dif_time = 30 elif search_type == 'year': dif_time = 365 if dif_time: show_time = datetime.datetime.now().replace( hour=0, minute=0, second=0, microsecond=0) - datetime.timedelta(days=dif_time) filter_dict = { 'db_mark': { '$ne': '' }, 'release_time': { '$ne': '', '$gte': show_time } } match = MatchStage(filter_dict) skip = SkipStage(skip) sort = SortStage([('db_mark', DESC)]) limit = LimitStage(int(size)) count = await Tvs.count({ 'db_mark': { '$ne': '' }, 'release_time': { '$ne': '' } }) tvs = await Tvs.aggregate([match, sort, skip, limit]).to_list(None) new_tvs = [] for tv in tvs: new_tvs.append({ 'id': str(tv.id), 'name': tv.name, 'pic_url': tv.pic_url, 'db_mark': tv.db_mark, 'actor': tv.actor, 'source_nums': len(tv.download), 'release_time': tv.release_time.strftime('%Y-%m-%d'), 'label': api_utils.get_show_source_label(tv), 'recommend_info': '这部神剧值得一看。', 'set_num': tv.set_num if tv.set_num else '', 's_type': 'tv' }) r_dict['tvs'] = new_tvs r_dict['count'] = count r_dict['code'] = 1000 except Exception: logger.error(traceback.format_exc()) print('score') print(r_dict) return r_dict
async def post(self): r_dict = {'code': 0, 'line': None} category = self.get_argument('category') condition = self.get_argument('condition_value') x_axis = self.get_argument('xAxis', '') if not category: member_list = await Member.aggregate( stage_list=[GroupStage('source', sum={'$sum': 1})] ).to_list(None) member_source = {m.id: m.sum for m in member_list} # fix: 特殊处理,将旧版线上数据与扫码数据合并 # 详见: http://code.wenjuan.com/WolvesAU/CRSPN/issues/10 # 最新定制:调查扫码(线上扫码-6、线下扫码-1) member_source['126'] = member_source.get( '1', 0) + member_source.get('6', 0) # 直接扫码 member_source['direct_scan'] = member_source.get('2', 0) + \ member_source.get('CA755167DEA9AA89650D11C10FAA5413', 0) + \ member_source.get('160631F26D00F7A2DC56DAE2A0C4AF12', 0) + \ member_source.get('F742E0C7CA5F7E175844478D74484C29', 0) source_list = await GameMemberSource.find({ 'status': STATUS_MEMBER_SOURCE_ACTIVE }).to_list(None) source_dict = {source.code: source.title for source in source_list} source_dict = adjust_source(source_dict) r_dict = { 'code': 1, 'pie': { 'legendData': [v for _, v in source_dict.items()], 'seriesData': [{ 'name': name, 'value': member_source.get(code) } for code, name in source_dict.items()] } } return r_dict try: # fix: 特殊处理,将旧版线上数据与扫码数据合并 # 详见: http://code.wenjuan.com/WolvesAU/CRSPN/issues/10 source_match = {'source': category} if category == '126': source_match = {'source': {'$in': list('16')}} elif category == '3': source_match = { 'source': { '$in': [ '2', '3', 'CA755167DEA9AA89650D11C10FAA5413', '160631F26D00F7A2DC56DAE2A0C4AF12', 'F742E0C7CA5F7E175844478D74484C29' ] } } member_list = await Member.aggregate([ ProjectStage( **{ '_id': '$id', 'created_dt': { "$dateToString": { "format": "%Y-%m-%d", "date": "$created_dt" } }, 'source': '$source', 'province_code': '$province_code', 'city_code': '$city_code', 'sex': '$sex', 'age': '$age_group', 'education': '$education' }), MatchStage(parse_condition(condition)), MatchStage(source_match), GroupStage('created_dt', sum={'$sum': 1}), SortStage([('_id', ASC)]), ]).to_list(None) if not x_axis: x_axis_data = [m.id for m in member_list] series_data = [m.sum for m in member_list] else: x_axis_data = json.loads(x_axis) member_id_map = {m.id: m for m in member_list} series_data = [ member_id_map[data.get('value')].sum if data.get('value') in member_id_map else 0 for data in x_axis_data ] r_dict = { 'code': 1, 'line': { 'xAxisData': x_axis_data, 'seriesData': series_data } } except Exception: logger.error(traceback.format_exc()) return r_dict