def change_duplicate_race_mapping(race_cid: str): print("race_cid:%s" % race_cid) match_stage = MatchStage({'race_cid': race_cid, 'record_flag': 1}) project_stage = ProjectStage(**{ "race_cid": 1, "member_cid": 1, "race_check_point_cid": 1 }) group_stage = GroupStage({'_id': '$member_cid'}, count={'$sum': 1}, duplicate_list={'$push': '$$ROOT'}) match_stage_count = MatchStage({'count': {'$gt': 1}}) project_stage_1 = ProjectStage(**{'duplicate_list': 1}) duplicate_race_mappings = RaceMapping.sync_aggregate([ match_stage, project_stage, group_stage, match_stage_count, project_stage_1 ]).to_list(None) count = 1 if len(duplicate_race_mappings) > 0: for duplicate_race_mapping in duplicate_race_mappings: print('第%d个:' % count) print(duplicate_race_mapping.duplicate_list) duplicate_record_ids = [ x._id for x in duplicate_race_mapping.duplicate_list ] not_need_index = 0 # 确定record_flag为1的元素 for index, value in enumerate( duplicate_race_mapping.duplicate_list): if value.race_check_point_cid: not_need_index = index duplicate_record_ids.pop(not_need_index) print("record_flag需置为0的记录Id:") print(duplicate_record_ids) update_requests = [] for object_id in duplicate_record_ids: update_requests.append( UpdateOne({'_id': object_id}, {'$set': { 'record_flag': 0 }})) RaceMapping.sync_update_many(update_requests) print("-------END:record_flag已置为0---------------") count += 1 else: print("-------未找到member_cid重复的记录-------") print("-------结束处理活动-------")
def get_member_info(thread_num, race_member_list, checkPointCidList, lastCheckPoint, checkPointMap): exportedMemberList = [] for race_member in race_member_list: # raceMapping = race_member.race_list[0] raceMapping = race_member race_cid = raceMapping.race_cid member = race_member.member_list[0] memberCid = member.cid red_match = MatchStage( {'race_cid': race_cid, 'member_cid': memberCid, 'draw_status': 0, 'draw_dt': {'$ne': None}, 'award_cid': {'$ne': None}, 'record_flag': 1}) red_project = ProjectStage(**{"member_cid": 1, "award_amount": 1}) red_group = GroupStage('member_cid', count={'$sum': 1}, amount={'$sum': '$award_amount'}) # redPacketsOfMember = RedPacketBox.sync_find( # {'race_cid': race_cid, 'member_cid': memberCid, 'award_cid': {'$ne': None}, 'record_flag': 1}) # eachAmountOfRedPacket = [redPacket.award_amount for redPacket in redPacketsOfMember] redPacketsOfMemberCursor = RedPacketBox.sync_aggregate([red_match, red_project, red_group]).batch_size(50) exportedMember = MemberInfoExportedModel() exportedMember.open_id = member.open_id exportedMember.member_cid = memberCid exportedMember.nick = member.nick_name exportedMember.firstTimeOfEnroll = member.created_dt city = raceMapping.auth_address.get('city', '') exportedMember.city = city if not city is None else '' district = raceMapping.auth_address.get('district', '') exportedMember.district = district if not district is None else '' mobile = getattr(raceMapping, 'mobile', '') if mobile is None: exportedMember.mobile = member.mobile else: exportedMember.mobile = mobile check_point_cid = getattr(raceMapping, 'race_check_point_cid', None) if check_point_cid is None: exportedMember.currentCheckPoint = "1" elif check_point_cid == lastCheckPoint: exportedMember.currentCheckPoint = "已通关" else: exportedMember.currentCheckPoint = str(checkPointMap[check_point_cid]) answerTimes = MemberCheckPointHistory.sync_find( {'member_cid': memberCid, 'check_point_cid': {'$in': checkPointCidList}, 'record_flag': 1}).to_list(None) exportedMember.answerTimes = len(answerTimes) try: redPacketsOfMember = redPacketsOfMemberCursor.next() if redPacketsOfMember: exportedMember.totalNumOfRedPacket = redPacketsOfMember.count exportedMember.totalAmountOfRedPacket = round(redPacketsOfMember.amount, 2) except StopIteration: pass exportedMemberList.append(exportedMember) print(thread_num, member.cid, exportedMember.nick, exportedMember.city, exportedMember.district, exportedMember.mobile, exportedMember.currentCheckPoint, exportedMember.answerTimes, exportedMember.totalNumOfRedPacket, exportedMember.totalAmountOfRedPacket) return exportedMemberList
def do_stat_in_history(history_model, city_code_list, choice_time, ad_map={}): """ :param history_model: :param city_code_list: :param ad_map: :param choice_time :return: """ # 取前一天凌晨12点之前的数据 time_match = get_yesterday() if not choice_time: match_stage = MatchStage({'updated_dt': {'$lt': time_match}}) else: # 当天下一天凌晨的时候 max_choice_time = choice_time.replace(hour=23, minute=59, second=59, microsecond=999) match_stage = MatchStage({'updated_dt': {'$gte': choice_time, '$lt': max_choice_time}}) cursor = history_model.sync_aggregate([ match_stage, GroupStage('member_cid', quantity={"$sum": 1}), LookupStage(Member, '_id', 'cid', 'member_list'), ProjectStage(**{ 'province_code': {'$arrayElemAt': ['$member_list.province_code', 0]}, 'city_code': {'$arrayElemAt': ['$member_list.city_code', 0]}, 'quantity': '$quantity' }), MatchStage({'city_code': {'$in': city_code_list}}), GroupStage('city_code', quantity={'$sum': "$quantity"}, province_code={'$first': '$province_code'}), SortStage([('quantity', DESC)]) ]) data = {} while True: try: his = cursor.next() city_data = data.get(his.province_code, {}) city = ad_map.get(his.id) if not city: city = AdministrativeDivision.sync_find_one({'code': his.id, 'parent_code': {'$ne': None}}) ad_map[city.code] = city city_data[city.title] = his.quantity data[his.province_code] = city_data except StopIteration: break except Exception as e: logger.error(str(e)) continue return data, ad_map
async def post(self): res_dict = {'code': 0} try: open_id = self.get_i_argument('open_id', '') if not open_id: res_dict['code'] = 1001 return res_dict member = await find_member_by_open_id(open_id) if not member: res_dict['code'] = 1002 return res_dict race_cid = self.get_i_argument('race_cid', '') if not race_cid: res_dict['code'] = 1003 return res_dict stat_list = await RaceCheckPointStatistics.aggregate(stage_list=[ MatchStage({'race_cid': race_cid}), SortStage([('pass_checkpoint_num', DESC), ('correct_num', DESC)]), LimitStage(30), LookupStage(Member, 'member_cid', 'cid', 'member_list'), ProjectStage( **{ 'nick_name': { '$arrayElemAt': ['$member_list.nick_name', 0] }, 'avatar': { '$arrayElemAt': ['$member_list.avatar', 0] }, 'correct_num': '$correct_num', 'pass_checkpoint_num': '$pass_checkpoint_num' }) ]).to_list(30) res_dict = {'code': 1000, 'rankings': stat_list} except Exception: logger.error(traceback.format_exc()) return res_dict
def do_init(model, skip_num, limit_num): """ :param model: :param skip_num: :param limit_num: :return: """ stage_list = [ MatchStage({ 'created_dt': { "$gte": datetime.now().replace(day=14, hour=18, minute=15, second=00), "$lte": datetime.now().replace(day=17, hour=9, minute=22, second=00) } }), ProjectStage( **{ 'daily_code': { "$dateToString": { 'format': '%Y%m%d000000', 'date': "$created_dt" } }, 'member_cid': 1, 'correct': { '$size': { '$filter': { 'input': '$result.true_answer', 'as': 'item', 'cond': { '$and': [{ '$eq': ['$$item', True] }] } } } }, 'total': { '$size': '$result' }, 'result': 1, 'created_dt': 1 }), GroupStage({ 'daily_code': '$daily_code', 'member_cid': '$member_cid' }, correct_list={'$push': "$correct"}, result_list={"$push": "$result"}, created_dt={'$first': "$created_dt"}, correct={'$sum': '$correct'}, total={'$sum': '$total'}, learn_times={'$sum': 1}), LookupStage(Member, '_id.member_cid', 'cid', 'member_list'), MatchStage({'member_list': { '$ne': [] }}), ProjectStage( **{ 'daily_code': "$_id.daily_code", 'member_cid': '$_id.member_cid', 'correct': '$correct', 'total': "$total", 'correct_list': "$correct_list", 'learn_times': '$learn_times', 'province_code': { "$arrayElemAt": ['$member_list.province_code', 0] }, 'city_code': { "$arrayElemAt": ['$member_list.city_code', 0] }, 'district_code': { "$arrayElemAt": ['$member_list.district_code', 0] }, 'gender': { "$arrayElemAt": ['$member_list.sex', 0] }, 'age_group': { "$arrayElemAt": ['$member_list.age_group', 0] }, 'education': { "$arrayElemAt": ['$member_list.education', 0] }, 'result_list': 1, 'created_dt': 1 }), SortStage([('daily_code', ASC), ('member_cid', ASC)]), ] if skip_num: stage_list.append(SkipStage(skip_num)) if limit_num: stage_list.append(LimitStage(limit_num)) cursor = MemberGameHistory.sync_aggregate( stage_list, allowDiskUse=True, read_preference=ReadPreference.PRIMARY) index = 0 while True: try: index += 1 print('has exec %s.' % index) data = cursor.next() param = { 'daily_code': data.daily_code, 'member_cid': data.member_cid, 'province_code': data.province_code, 'city_code': data.city_code, 'district_code': data.district_code, 'gender': data.gender, 'age_group': data.age_group, 'education': data.education, 'learn_times': data.learn_times, 'subject_total_quantity': data.total, 'subject_correct_quantity': data.correct, 'quantity_detail': dict(Counter(map(lambda x: str(x), data.correct_list))), 'dimension_detail': get_dimension_detail(data.result_list), 'created_dt': data.created_dt, 'updated_dt': data.created_dt } model(**param).sync_save() except StopIteration: break except AttributeError as e: print(e) continue
def daily_member_statistic(race_cid, daily_code): """ 统计每日活动下的会员 :param race_cid: :param daily_code: :return: """ start_date = str2datetime(daily_code, '%Y%m%d').replace(hour=0, minute=0, second=0, microsecond=0) end_date = start_date + datetime.timedelta(days=1) check_point_cids, _checkpoint_map = get_checkpoint_cid_list(race_cid) # 当日答题记录 history_match = MatchStage({ 'created_dt': { '$gte': start_date, '$lt': end_date }, 'check_point_cid': { '$in': check_point_cids } }) history_group = GroupStage({'member_cid': '$member_cid'}, enter_times={'$sum': 1}, results_list={'$push': '$result'}) history_project = ProjectStage( **{ 'member_cid': '$_id.member_cid', 'results_list': '$results_list', 'enter_times': '$enter_times' }) # 红包信息 member_amount_map = get_red_packet_info(race_cid, start_date, end_date) # member_cid: amount check_point_history_cursor = MemberCheckPointHistory.sync_aggregate( [history_match, history_group, history_project], allowDiskUse=True).batch_size(4) member_statistic_list = [] count = 0 while True: try: cursor = check_point_history_cursor.next() count += 1 # 根据member_cid查对应的member信息 temp_member = Member.sync_get_by_cid(cursor.member_cid) if not temp_member: print("normal未找到对应member_cid:%s" % cursor.member_cid) continue info = MemberStatisticInfo() info.daily_code = daily_code info.race_cid = race_cid # 会员信息 info.member_cid = cursor.member_cid info.nick_name = temp_member.nick_name info.open_id = temp_member.open_id info.mobile = temp_member.mobile info.first_time_login = temp_member.created_dt info.enter_times = cursor.enter_times # 答题数量、正确数 for results in cursor.results_list: info.answer_times += len(results) info.true_answer_times += len([ result for result in results if result.get('true_answer') ]) # race_mapping相关信息,地理位置 has_race_mapping = get_race_mapping_info(info) if not has_race_mapping: print("normal未找到对应race_mapping,race_cid:%s,member_cid:%s" % (info.race_cid, info.member_cid)) continue # 是否为当日新用户 is_new_user(info) # 最后一关 is_final_pass(info) # 红包信息 try: value = member_amount_map.pop(info.member_cid) except KeyError: value = None if not value: value = { 'grant_count': 0, 'grant_amount': 0, 'draw_count': 0, 'draw_amount': 0 } info.grant_red_packet_amount = value.get('grant_amount') info.grant_red_packet_count = value.get('grant_count') info.draw_red_packet_amount = value.get('draw_amount') info.draw_red_packet_count = value.get('draw_count') # 保存记录 member_statistic_list.append(info) # logger.info("Success: member_cid:%s is_final_Pass:%s is_new:%s" % ( # info.member_cid, info.is_final_passed, info.is_new_user)) print("Success: member_cid:%s is_final_Pass:%s is_new:%s" % (info.member_cid, info.is_final_passed, info.is_new_user)) if len(member_statistic_list) % 500 == 0: MemberStatisticInfo.sync_insert_many(member_statistic_list) member_statistic_list = [] except StopIteration: break except CursorNotFound: check_point_history_cursor = MemberCheckPointHistory.sync_aggregate( [history_match, history_group, history_project]).skip(count).batch_size(4) except Exception as e: logger.error(e) logger.info("Fail: daily_code:%s,race_cid: %s" % (daily_code, race_cid)) member_statistic_list = [] member_statistic_list += insert_from_member_amount_map( member_amount_map, daily_code, race_cid) if len(member_statistic_list) > 0: MemberStatisticInfo.sync_insert_many(member_statistic_list)
def deal_member_without_history(race_cid, daily_code): """ 处理报名活动但未答题的会员 :param race_cid: :param daily_code: :return: """ city_name_list, district_name_list = get_address(race_cid) checkpoint_cid, _checkpoint_map = get_checkpoint_cid_list(race_cid) start_date = str2datetime(daily_code, '%Y%m%d').replace(hour=0, minute=0, second=0, microsecond=0) end_date = start_date + datetime.timedelta(days=1) member_cid_with_history = MemberCheckPointHistory.sync_distinct( "member_cid", { 'check_point_cid': { '$in': checkpoint_cid }, 'created_dt': { '$gte': start_date, '$lt': end_date } }) race_member_match = MatchStage({ "race_cid": race_cid, 'member_cid': { '$nin': member_cid_with_history }, 'auth_address.city': { "$in": city_name_list }, 'auth_address.district': { "$in": district_name_list }, 'created_dt': { '$gte': start_date, '$lt': end_date } }) member_group = GroupStage({'member_cid': '$member_cid'}, auth_address={'$first': '$auth_address'}, company_cid={'$first': '$company_cid'}, mobile={'$first': '$mobile'}, created_dt={'$first': '$created_dt'}) member_project = ProjectStage( **{ 'cid': '$cid', 'member_cid': '$_id.member_cid', 'auth_address': '$auth_address', 'mobile': '$mobile', 'created_dt': '$created_dt', 'company_cid': '$company_cid' }) member_without_history = RaceMapping.sync_aggregate( [race_member_match, member_group, member_project]).batch_size(4) member_amount_map = get_red_packet_info(race_cid, start_date, end_date) red_member_cid_list = member_amount_map.keys() member_no_history_list = [] count = 0 while True: try: stat = member_without_history.next() count += 1 if stat.member_cid in red_member_cid_list: continue # 根据member_cid查对应的member信息 temp_member = Member.sync_get_by_cid(stat.member_cid) if not temp_member: print("no history未找到对应member_cid:%s" % stat.member_cid) continue info_special = MemberStatisticInfo() info_special.is_special_user = 1 info_special.race_cid = race_cid info_special.member_cid = stat.member_cid info_special.daily_code = format(stat.created_dt, '%Y%m%d') info_special.nick_name = temp_member.nick_name info_special.open_id = temp_member.open_id if stat.mobile: info_special.mobile = stat.mobile else: info_special.mobile = temp_member.mobile info_special.first_time_login = temp_member.created_dt info_special.enter_times = 1 info_special.answer_times = 0 info_special.true_answer_times = 0 info_special.is_final_passed = 0 info_special.is_new_user = 1 info_special.grant_red_packet_amount = 0.0 info_special.grant_red_packet_count = 0 info_special.draw_red_packet_count = 0 info_special.draw_red_packet_amount = 0.0 info_special.province = stat.auth_address.get('province') info_special.city = stat.auth_address.get('city') info_special.district = stat.auth_address.get('district') info_special.town = stat.auth_address.get('town') info_special.check_point_cid = stat.race_check_point_cid if stat.race_check_point_cid: info_special.check_point_index = _checkpoint_map[ stat.race_check_point_cid] else: info_special.check_point_index = 1 if stat.company_cid: company = Company.sync_get_by_cid(stat.company_cid) info_special.company_cid = company.cid info_special.company_name = company.title member_no_history_list.append(info_special) # logger.info("Success without history: member_cid:%s is_final_Pass:%s" % (info_special.member_cid,info_special.is_final_passed)) if len(member_no_history_list) % 500 == 0: MemberStatisticInfo.sync_insert_many(member_no_history_list) member_no_history_list = [] except StopIteration: break except CursorNotFound: member_without_history = RaceMapping.sync_aggregate( [race_member_match, member_group, member_project]).skip(count).batch_size(4) except Exception as e: logger.info( "Fail: without history daily_code:%s,member_cid:%s,race_cid: %s" % (info_special.daily_code, info_special.member_cid, race_cid)) if len(member_no_history_list) > 0: MemberStatisticInfo.sync_insert_many(member_no_history_list)
def get_stages(group_dict=None, skip_num=None): """ :param group_dict: :param skip_num: :return: """ if not group_dict or skip_num is None: logger.error('there is not group_dict(%s) or skip_num(%s)' % (group_dict, skip_num)) raise ValueError() inactive_subject_cids = Subject.sync_distinct( 'cid', { '$or': [{ 'status': STATUS_SUBJECT_INACTIVE }, { 'category_use': { '$in': [CATEGORY_SUBJECT_BENCHMARK, CATEGORY_SUBJECT_GRADUATION] } }] }) inactive_sbj = MatchStage({'subject_cid': {'$nin': inactive_subject_cids}}) group_stage = GroupStage(group_dict, t_total={'$sum': '$total'}, t_correct={'$sum': '$correct'}, created_dt={'$max': '$created_dt'}) sort_stage = SortStage([('t_total', DESC), ('t_correct', DESC), ('created_dt', ASC)]) project_stage = ProjectStage(total='$t_total', correct='$t_correct', percent={ '$cond': { 'if': { '$eq': ['$t_total', 0] }, 'then': 0, 'else': { '$divide': ['$t_correct', '$t_total'] } } }) s_lookup_stage = LookupStage(Subject, as_list_name='subject_list', let={'subject_id': "$_id.subject_cid"}, pipeline=[{ '$match': { '$expr': { '$and': [{ '$eq': ['$cid', '$$subject_id'] }] } } }]) so_lookup_stage = LookupStage( SubjectOption, as_list_name='subject_option_list', let={'subject_id': "$_id.subject_cid"}, pipeline=[{ '$match': { '$expr': { '$and': [{ '$eq': ['$subject_cid', '$$subject_id'] }] } } }, { '$sort': { 'code': ASC } }]) match_stage = MatchStage({ 'subject_list': { '$ne': [] }, 'subject_option_list': { '$ne': [] } }) project_stage2 = ProjectStage( **{ 'custom_code': { '$arrayElemAt': ['$subject_list.custom_code', 0] }, 'code': { '$arrayElemAt': ['$subject_list.code', 0] }, 'title': { '$arrayElemAt': ['$subject_list.title', 0] }, 'option_list': '$subject_option_list', 'dimension': { '$arrayElemAt': ['$subject_list.dimension_dict', 0] }, 'total': '$total', 'correct': '$correct' }) skip_stage = SkipStage(skip_num) limit_stage = LimitStage(10000) return [ inactive_sbj, group_stage, sort_stage, skip_stage, limit_stage, project_stage, s_lookup_stage, so_lookup_stage, match_stage, project_stage2 ]
def do_init_member_stat(pre_match: dict = {}, skip_num=0, limit_num=0, just_return_count=False): """ :param pre_match: :param skip_num: :param limit_num: :param just_return_count: :return: """ if not isinstance(just_return_count, bool): raise Exception('check params(just_return_count)') stage_list = [] if pre_match: stage_list.append(MatchStage(pre_match)) stage_list.extend([ UnwindStage("result"), LookupStage(Member, 'member_cid', 'cid', 'member_list'), MatchStage({'member_list': { '$ne': list() }}), ProjectStage( **{ 'subject_cid': '$result.subject_cid', 'province_code': { '$arrayElemAt': ['$member_list.province_code', 0] }, 'city_code': { '$arrayElemAt': ['$member_list.city_code', 0] }, 'district_code': { '$arrayElemAt': ['$member_list.district_code', 0] }, 'sex': { '$arrayElemAt': ['$member_list.sex', 0] }, 'age_group': { '$arrayElemAt': ['$member_list.age_group', 0] }, 'category': { '$arrayElemAt': ['$member_list.category', 0] }, 'education': { '$arrayElemAt': ['$member_list.education', 0] }, 'true_answer': { '$cond': { 'if': { '$eq': ['$result.true_answer', True] }, 'then': True, 'else': False } }, 'created_dt': '$created_dt' }), GroupStage( { 'subject_cid': '$subject_cid', 'province_code': '$province_code', 'city_code': '$city_code', 'district_code': '$district_code', 'sex': '$sex', 'age_group': '$age_group', 'category': '$category', 'education': '$education', }, answers_list={'$push': '$true_answer'}, created_dt={'$first': '$created_dt'}), ]) if just_return_count: stage_list.append(CountStage()) data = MemberGameHistory.sync_aggregate( stage_list, read_preference=ReadPreference.PRIMARY, allowDiskUse=True).to_list(1) return data[0].count stage_list.append(SortStage([('created_dt', ASC)])) if skip_num: stage_list.append(SkipStage(skip_num)) if limit_num: stage_list.append(LimitStage(limit_num)) cursor = MemberGameHistory.sync_aggregate( stage_list, read_preference=ReadPreference.PRIMARY, allowDiskUse=True).batch_size(256) index = 0 _subject_map = {} while True: try: data = cursor.next() subject_cid = data.id.get('subject_cid') subject = _subject_map.get(subject_cid) if not subject: subject = Subject.sync_get_by_cid(subject_cid) _subject_map[subject_cid] = subject write_member_subject_stat(data, subject) index += 1 print('has exec %s' % index) except StopIteration: break except Exception: print(traceback.format_exc()) continue
def do_stat(model): stage_list = [ UnwindStage('result'), ProjectStage( **{ 'daily_code': { '$dateToString': { 'format': '%Y%m%d', 'date': '$created_dt' } }, 'member_cid': '$member_cid', 'subject_cid': '$result.subject_cid', 'true_answer': '$result.true_answer' }), LookupStage(Subject, 'subject_cid', 'cid', 'subject_list'), ProjectStage( **{ 'daily_code': 1, 'member_cid': 1, 'true_answer': 1, 'dimension': { '$arrayElemAt': ['$subject_list.dimension_dict', 0] } }), GroupStage( { 'daily_code': '$daily_code', 'member_cid': '$member_cid', 'dimension': '$dimension' }, answer_list={'$push': '$true_answer'}), ProjectStage( **{ 'daily_code': '$_id.daily_code', 'member_cid': '$_id.member_cid', 'dimension': '$_id.dimension', 'correct': { '$size': { '$filter': { 'input': '$answer_list', 'as': 'item', 'cond': { '$and': [{ '$eq': ['$$item', True] }] } } } }, 'total': { "$size": "$answer_list" } }), GroupStage({ 'member_cid': '$member_cid', 'dimension': '$dimension' }, daily_list={'$push': '$daily_code'}, correct_list={'$push': '$correct'}, total_list={'$push': '$total'}), ] cursor = MemberGameHistory.sync_aggregate( stage_list, allowDiskUse=True, read_preference=ReadPreference.PRIMARY) t1 = time.time() index = 0 insert_list = [] while True: try: data = cursor.next() member = Member.sync_get_by_cid(data.id.get('member_cid')) if not member: continue dimension = data.id.get('dimension') if not dimension: continue for i, day in enumerate(data.daily_list): param = { 'learning_code': get_learning_code(day, data.daily_list), 'member_cid': member.cid, 'dimension': dimension, 'province_code': member.province_code, 'city_code': member.city_code, 'district_code': member.district_code, 'gender': member.sex, 'age_group': member.age_group, 'education': member.education, 'subject_total_quantity': data.total_list[i], 'subject_correct_quantity': data.correct_list[i] } insert_list.append(model(**param)) except StopIteration: break if len(insert_list) > 5000: model.sync_insert_many(insert_list) insert_list = [] index += 1 print('has exec', index) t2 = time.time() print('cost:', t2 - t1) model.sync_insert_many(insert_list) t3 = time.time() print('insert', t3 - t2)
async def get(self): race_cid = self.get_argument('race_cid', '') menu_list = await get_menu(self, 'config', race_cid) put_out_form = self.get_argument('put_out_form', '') red_packet_item = self.get_argument('red_packet_item', '') if race_cid: # 抽奖总览 race = await Race.find_one({'cid': race_cid, 'record_flag': 1}) # 找到该活动下面的所有rule_cid # 已经发放的红包个数 already_put_red_packet_amount_list = await RedPacketBox.aggregate([ MatchStage({ 'race_cid': race_cid, 'draw_status': STATUS_REDPACKET_AWARDED, 'member_cid': { '$ne': None }, 'award_cid': { '$ne': None }, 'record_flag': 1 }), GroupStage(None, sum={'$sum': '$award_amount'}, quantity={'$sum': 1}) ]).to_list(None) # 抽奖详情 kw_word = self.get_argument('kw_word', '') # 奖项标题 item_title = self.get_argument('item_title', '') stage_list = [ LookupStage(Member, 'member_cid', 'cid', 'member_list'), LookupStage(RaceGameCheckPoint, 'checkpoint_cid', 'cid', 'checkpoint_list'), LookupStage(RedPacketItemSetting, 'award_cid', 'cid', 'setting_list'), LookupStage(RedPacketConf, 'award_cid', 'cid', 'conf_list'), ProjectStage( **{ 'member_cid': '$member_cid', 'nick_name': { '$arrayElemAt': ['$member_list.nick_name', 0] }, 'checkpoint': { '$arrayElemAt': ['$checkpoint_list.alias', 0] }, 'category': { '$cond': { 'if': { '$ne': ['$setting_list', list()] }, 'then': '抽奖形式', 'else': '直接发放' } }, 'detail': { '$cond': { 'if': { '$ne': ['$setting_list', list()] }, 'then': { '$arrayElemAt': ['$setting_list.title', 0] }, 'else': { '$arrayElemAt': ['$conf_list.category', 0] } } }, 'award_amount': '$award_amount', 'draw_dt': '$draw_dt', 'award_cid': '$award_cid' }), ] query_dict = {} if kw_word: query_dict['$or'] = [ { 'nick_name': { '$regex': kw_word, '$options': 'i' } }, { 'member_cid': { '$regex': kw_word, '$options': 'i' } }, { 'checkpoint': { '$regex': kw_word, '$options': 'i' } }, ] if put_out_form: query_dict['category'] = put_out_form if red_packet_item and put_out_form != CATEGORY_REDPACKET_RULE_DICT.get( CATEGORY_REDPACKET_RULE_DIRECT): query_dict['detail'] = red_packet_item query = MatchStage(query_dict) stage_list.append(query) query_match_dict = { "race_cid": race_cid, 'draw_status': STATUS_REDPACKET_AWARDED, 'member_cid': { '$ne': None }, 'award_cid': { '$ne': None }, 'record_flag': 1 } per_page_quantity = int(self.get_argument('per_page_quantity', 10)) to_page_num = int(self.get_argument('page', 1)) page_url = '%s?page=$page&per_page_quantity=%s&race_cid=%s&kw_name=%s&put_out_form=%s' % ( self.reverse_url("backoffice_race_redpkt_rule_see_result"), per_page_quantity, race_cid, kw_word, put_out_form) paging = Paging(page_url, RedPacketBox, current_page=to_page_num, pipeline_stages=stage_list, sort=['award_amount'], items_per_page=per_page_quantity, **query_match_dict) await paging.pager() # 抽奖形式的奖项列表 lottery_item_list = await RedPacketItemSetting.distinct( 'title', { 'race_cid': race_cid, 'record_flag': 1 }) return locals()
def export_race_data(workbook, race_cid: str, sheet_name): """ 导出每日参与人数,每日新增人数,每日新增人次 :param race_cid: :return: """ # yesterday_time = get_yesterday() # time_match = MatchStage({'updated_dt': {'$lt': yesterday_time}}) race = Race.sync_get_by_cid(race_cid) city_list = AdministrativeDivision.sync_distinct( 'code', {'parent_code': race.province_code}) city_name_list = AdministrativeDivision.sync_distinct( 'title', {'parent_code': race.province_code}) dist_list = [] for city in city_list: dist_list += AdministrativeDivision.sync_distinct( 'title', {'parent_code': city}) match = { 'race_cid': race_cid, 'auth_address.province': { '$ne': None }, 'auth_address.city': { "$in": city_name_list }, 'auth_address.district': { "$in": dist_list } } sheet = workbook.add_worksheet(sheet_name) cursor = RaceMapping.sync_aggregate([ MatchStage(match), ProjectStage( **{ 'daily_code': { '$dateToString': { 'format': "%Y%m%d", 'date': "$created_dt" } }, 'auth_address': "$auth_address", 'member_cid': "$member_cid" }), GroupStage( { 'daily_code': '$daily_code', 'district': '$auth_address.district' }, sum={'$sum': 1}, auth_address={'$first': '$auth_address'}), SortStage([('_id.daily_code', ASC)]) ]) sheet.write_string(0, 0, '城市') sheet.write_string(0, 1, '区县') daily_list = [] prize_list = [] county_map = {} v_list = list() _max_row = 0 while True: try: data = cursor.next() _current_row = None title = data.id.get('district') if title is None: title = '未知' if title not in prize_list: prize_list.append(title) _current_row = len(prize_list) city = data.auth_address.get('city') if not city: continue sheet.write_string(_current_row, 0, city) ad_city = AdministrativeDivision.sync_find_one({'title': city}) _county = AdministrativeDivision.sync_distinct( 'title', {'parent_code': ad_city.code}) for _c in _county: county_map[_c] = city sheet.write_string(_current_row, 1, title) else: _current_row = prize_list.index(title) + 1 daily_code = data.id.get('daily_code') if not daily_code: daily_code = '未知' if daily_code not in daily_list: daily_list.append(daily_code) _current_col = len(daily_list) + 1 sheet.write_string(0, _current_col, daily_code) else: _current_col = daily_list.index(daily_code) + 2 sheet.write_number(_current_row, _current_col, data.sum) v_list.append(data.sum) if _current_row >= _max_row: _max_row = _current_row except StopIteration: break except Exception as e: raise e for k, v in county_map.items(): if k not in prize_list: _max_row += 1 sheet.write_string(_max_row, 0, v) sheet.write_string(_max_row, 1, k) if _max_row: sheet.write_string(_max_row + 1, 0, '总和') sheet.write_number(_max_row + 1, 1, sum(v_list))
def do_stat_in_history(history_model, time_match, basic_stages, stat_type): """ :param history_model: :param time_match :param basic_stages: :param stat_type: :return: """ stage_list = [ time_match, ProjectStage( **{ 'daily_code': { '$dateToString': { 'format': "%Y%m%d", "date": "$updated_dt" } }, 'member_cid': "$member_cid", 'updated_dt': '$updated_dt' }), GroupStage({ 'member_cid': '$member_cid', 'daily_code': '$daily_code' }, quantity={"$sum": 1}, updated_dt={'$first': "$updated_dt"}), LookupStage(Member, '_id.member_cid', 'cid', 'member_list'), MatchStage({'member_list': { '$ne': [] }}), ProjectStage( **{ 'daily_code': '$_id.daily_code', 'member_cid': '$_id.member_cid', 'updated_dt': '$updated_dt', 'quantity': '$quantity', 'city_code': { '$arrayElemAt': ['$member_list.city_code', 0] }, 'province_code': { '$arrayElemAt': ['$member_list.province_code', 0] }, 'sex': { '$arrayElemAt': ['$member_list.sex', 0] }, 'age_group': { '$arrayElemAt': ['$member_list.age_group', 0] }, 'education': { '$arrayElemAt': ['$member_list.education', 0] }, }), ] if basic_stages: stage_list.extend(basic_stages) if stat_type == 'time': quantity = {'$sum': "$quantity"} else: quantity = {'$sum': 1} stage_list.extend([ GroupStage('daily_code', quantity=quantity), SortStage([('_id', ASC)]) ]) cursor = history_model.sync_aggregate(stage_list, allowDiskUse=True) data = {} while True: try: his = cursor.next() data[his.id] = his.quantity except StopIteration: break except Exception as e: logger.error(str(e)) continue return data
def export_race_enter_position(race_cid: str, title, n): """ 导出活动下面参与情况 :return: """ if not isinstance(race_cid, str): raise ValueError("race_cid is not str") current = datetime.datetime.now() export_time_list = [ (current + datetime.timedelta(days=-n)).replace(hour=0, minute=0, second=0) for n in (range(1, n)) ] export_time_list.sort() export_time = [ transform_time_format(export_dt) for export_dt in export_time_list ] export_time.sort() workbook = xlsxwriter.Workbook("{title}{today}.xlsx".format(title=title, today=today)) excel_name = "{title}{today}.xlsx".format(title=title, today=today) head_list = [ "新增人数", "参与人数", "参与次数", "通关人数", "红包发放数", "红包领取数量", "红包发放金额", "红包领取金额" ] sheet = workbook.add_worksheet("{title}一周数据".format(title=title)) data_center_format = workbook.add_format({ 'valign': 'vcenter', 'align': 'center', 'font_name': 'Microsoft YaHei' }) sheet.merge_range(0, 0, 1, 0, "城市", data_center_format) sheet.merge_range(0, 1, 1, 1, "区县", data_center_format) sheet.write_string(1, 2, "累计人数") sheet.write_string(1, 3, "累计参与次数") sheet.write_string(1, 4, "累计红包发放总数") # excel填充一周日期 for i in range(1, n): sheet.merge_range(0, 7 * (i - 1) + 5 + i - 1, 0, 7 * (i - 1) + 5 + i - 1 + 7, export_time[i - 1], data_center_format) for head in range(7 * (i - 1) + 5 + i - 1, 7 * (i - 1) + 5 + i - 1 + 7 + 1): index = head - 8 * (i - 1) - 5 sheet.write_string(1, head, head_list[index]) prov_match = MatchStage({'province': {'$ne': None}}) race = Race.sync_get_by_cid(race_cid) # 市级活动,如六安市,扬州市 if race.city_code: city_code_list = AdministrativeDivision.sync_distinct( 'code', {'code': race.city_code}) # 该活动的所属城市范围 city_name_list = AdministrativeDivision.sync_distinct( 'title', {'code': race.city_code}) else: prov = AdministrativeDivision.sync_find_one( {'code': race.province_code}) city_code_list = AdministrativeDivision.sync_distinct( 'code', {'parent_code': race.province_code}) city_name_list = AdministrativeDivision.sync_distinct( 'title', {'parent_code': race.province_code}) prov_match = MatchStage({'province': prov.title}) dist_list = [] for city_code in city_code_list: # 该活动区县的范围 dist_list += AdministrativeDivision.sync_distinct( 'title', {'parent_code': city_code}) address_sort = SortStage([('_id.city', ASC), ('_id.district', ASC)]) match = { "race_cid": race_cid, 'record_flag': 1, 'city': { '$in': city_name_list }, 'district': { '$in': dist_list } } address_list = RaceMemberEnterInfoStatistic.sync_aggregate([ MatchStage(match), prov_match, ProjectStage(**{ 'province': "$province", "city": "$city", "district": "$district" }), GroupStage({ 'district': '$district', 'city': '$city' }, sum={'$sum': 1}), address_sort ]) # 拿到该活动下的所有区县和城市对应字典表 dis_map = get_race_district_mapping(sheet, address_list) # 时间与地区人数的字典 {'20190702': {'六安市-叶集区': 20}, '20190703': {'六安市-舒城县': 30}} quantity_time_dict = {} # 每日参与人数一周的数据 时间与地区人数的字典 {'20190702': {'六安市-叶集区': 20}, '20190703': {'六安市-舒城县': 30}} enter_quantity_time_dict = {} # 时间与地区次数的字典 {'20190702': {'六安市-叶集区': 20}, '20190703': {'六安市-舒城县': 30}} count_time_dict = {} # 一周通关的人数 {'20190702': {'六安市-叶集区': 20, "六安市-舒城县": 15}, '20190703': {'六安市-舒城县': 30}} pass_quantity_time_dict = {} # 一周红包发放数量和金额的字典 red_give_dict = {} red_give_amount_dict = {} # 一周红包领取数量和金额的字典 red_receive_dict = {} red_receive_amount_dict = {} base_match = { 'race_cid': race_cid, 'city': { '$in': city_name_list }, 'district': { '$in': dist_list }, 'daily_code': { '$gte': export_time[0], '$lte': transform_time_format( (export_time_list[-1] + datetime.timedelta(days=1))) } } total_match = copy.deepcopy(base_match) del total_match['daily_code'] total_district_count_dict = {} total_district_times_dict = {} total_send_red_packet_dict = {} race_total_info_cursor = RaceMemberEnterInfoStatistic.sync_aggregate([ MatchStage(total_match), GroupStage( { 'city': '$city', 'district': '$district' }, count_sum={'$sum': '$increase_enter_count'}, enter_times_sum={'$sum': '$enter_times'}, grant_red_packet_count={'$sum': "$grant_red_packet_count"}, ) ]) while True: try: data = race_total_info_cursor.next() district = data.id.get('district') if district not in total_district_count_dict: total_district_count_dict[district] = data.count_sum if district not in total_district_times_dict: total_district_times_dict[district] = data.enter_times_sum if district not in total_send_red_packet_dict: total_send_red_packet_dict[ district] = data.grant_red_packet_count except StopIteration: break except Exception as e: raise e # 每日参与人数,总计 for district, data in total_district_count_dict.items(): _col = dis_map.get(district) sheet.write_number(_col, 2, data) # 每日参与次数,总计 for district, data in total_district_times_dict.items(): _col = dis_map.get(district) sheet.write_number(_col, 3, data) # 每日累计发放红包数量 for district, data in total_send_red_packet_dict.items(): _col = dis_map.get(district) sheet.write_number(_col, 4, data) race_member_info_list = RaceMemberEnterInfoStatistic.sync_aggregate([ MatchStage(base_match), GroupStage( { "daily_code": "$daily_code", 'district': '$district', 'city': '$city' }, increase_quantity_sum={'$sum': '$increase_enter_count'}, enter_count_sum={'$sum': '$enter_count'}, pass_count_sum={'$sum': '$pass_count'}, enter_times_sum={'$sum': '$enter_times'}, grant_red_packet_count_sum={'$sum': '$grant_red_packet_count'}, grant_red_packet_amount_sum={'$sum': '$grant_red_packet_amount'}, draw_red_packet_count_sum={'$sum': '$draw_red_packet_count'}, draw_red_packet_amount_sum={'$sum': '$draw_red_packet_amount'}, ), address_sort ]) for race_member_info in race_member_info_list: daily = race_member_info.id.get('daily_code') city = race_member_info.id.get('city') district = race_member_info.id.get('district') if city and district: if daily not in quantity_time_dict: # 每日新增参与人数 quantity_time_dict[daily] = { "{city}-{district}".format(city=city, district=district): race_member_info.increase_quantity_sum } else: quantity_time_dict[daily].update({ "{city}-{district}".format(city=city, district=district): race_member_info.increase_quantity_sum }) if daily not in enter_quantity_time_dict: # 每日参与人数 enter_quantity_time_dict[daily] = { "{city}-{district}".format(city=city, district=district): race_member_info.enter_count_sum } else: enter_quantity_time_dict[daily].update({ "{city}-{district}".format(city=city, district=district): race_member_info.enter_count_sum }) if daily not in count_time_dict: # 每日参与次数 count_time_dict[daily] = { "{city}-{district}".format(city=city, district=district): race_member_info.enter_times_sum } else: count_time_dict[daily].update({ "{city}-{district}".format(city=city, district=district): race_member_info.enter_times_sum }) if daily not in pass_quantity_time_dict: # 每日通关人数 pass_quantity_time_dict[daily] = { "{city}-{district}".format(city=city, district=district): race_member_info.pass_count_sum } else: pass_quantity_time_dict[daily].update({ "{city}-{district}".format(city=city, district=district): race_member_info.pass_count_sum }) if daily not in red_give_dict: # 每日红包发放数量 red_give_dict[daily] = { "{city}-{district}".format(city=city, district=district): race_member_info.grant_red_packet_count_sum } else: red_give_dict[daily].update({ "{city}-{district}".format(city=city, district=district): race_member_info.grant_red_packet_count_sum }) if daily not in red_give_amount_dict: # 每日红包发放金额 red_give_amount_dict[daily] = { "{city}-{district}".format(city=city, district=district): race_member_info.grant_red_packet_amount_sum } else: red_give_amount_dict[daily].update({ "{city}-{district}".format(city=city, district=district): race_member_info.grant_red_packet_amount_sum }) if daily not in red_receive_dict: # 每日红包领取数量 red_receive_dict[daily] = { "{city}-{district}".format(city=city, district=district): race_member_info.draw_red_packet_count_sum } else: red_receive_dict[daily].update({ "{city}-{district}".format(city=city, district=district): race_member_info.draw_red_packet_count_sum }) if daily not in red_receive_amount_dict: # 每日红包领取金额 red_receive_amount_dict[daily] = { "{city}-{district}".format(city=city, district=district): race_member_info.draw_red_packet_amount_sum } else: red_receive_amount_dict[daily].update({ "{city}-{district}".format(city=city, district=district): race_member_info.draw_red_packet_amount_sum }) else: continue # 每日新增人数在excel的位置 quantity_increase_position_list = [4 + 8 * (i - 1) for i in range(1, n)] # 每日参与人数在excel的位置 quantity_enter_position_list = [5 + 8 * (i - 1) for i in range(1, n)] # 每日参与次数在excel的位置 count_enter_position_list = [6 + 8 * (i - 1) for i in range(1, n)] # 每日通关次数在excel的位置 pass_enter_position_list = [7 + 8 * (i - 1) for i in range(1, n)] # 红包发放数量在excel的位置 red_give_position_list = [8 + 8 * (i - 1) for i in range(1, n)] # 红包发放金额在excel的位置 red_give_amount_list = [10 + 8 * (i - 1) for i in range(1, n)] # 红包领取数量在excel的位置 red_receive_position_list = [9 + 8 * (i - 1) for i in range(1, n)] # 红包领取金额在excel的位置 red_receive_amount_list = [11 + 8 * (i - 1) for i in range(1, n)] # 填充每日新增人数 write_excel_data(sheet, quantity_time_dict, dis_map, export_time, quantity_increase_position_list, save=None) # excel 填充每日参与人数 write_excel_data(sheet, enter_quantity_time_dict, dis_map, export_time, quantity_enter_position_list, save=None) # excel 填充每日参与次数 write_excel_data(sheet, count_time_dict, dis_map, export_time, count_enter_position_list, save=None) # excel 填充每日通关人数 write_excel_data(sheet, pass_quantity_time_dict, dis_map, export_time, pass_enter_position_list, save=None) # excel 填充每日发放个数 write_excel_data(sheet, red_give_dict, dis_map, export_time, red_give_position_list, save=None) # excel 填充每日领取金额 write_excel_data(sheet, red_give_amount_dict, dis_map, export_time, red_give_amount_list, save=2) # excel 填充每日领取个数 write_excel_data(sheet, red_receive_dict, dis_map, export_time, red_receive_position_list, save=None) # excel 填充每日领取金额 write_excel_data(sheet, red_receive_amount_dict, dis_map, export_time, red_receive_amount_list, save=2) workbook.close() return excel_name
def do_init(model): """ :return: """ b_dt = datetime.now().replace(month=5, day=18, hour=0, minute=0, second=0) e_dt = datetime.now().replace(month=6, day=13, hour=0, minute=0, second=0) cursor = MemberGameHistory.sync_aggregate( [ MatchStage({'created_dt': { '$gte': b_dt, '$lte': e_dt }}), UnwindStage("result"), ProjectStage( **{ 'daily_code': { '$dateToString': { 'format': '%Y%m%d000000', 'date': '$created_dt' } }, 'member_cid': 1, 'subject_cid': '$result.subject_cid', 'true_answer': "$result.true_answer", 'created_dt': 1 }), GroupStage( { 'daily_code': '$daily_code', 'member_cid': '$member_cid', 'subject_cid': '$subject_cid' }, answer_list={'$push': '$true_answer'}, created_dt={'$first': '$created_dt'}), LookupStage(Member, '_id.member_cid', 'cid', 'member_list'), LookupStage(Subject, '_id.subject_cid', 'cid', 'subject_list'), MatchStage({ 'member_list': { '$ne': [] }, 'subject_list': { '$ne': [] } }), ProjectStage( **{ 'member_cid': '$_id.member_cid', 'dimension': { "$arrayElemAt": ['$subject_list.dimension_dict', 0] }, 'daily_code': '$_id.daily_code', 'province_code': { "$arrayElemAt": ['$member_list.province_code', 0] }, 'city_code': { "$arrayElemAt": ['$member_list.city_code', 0] }, 'district_code': { "$arrayElemAt": ['$member_list.district_code', 0] }, 'gender': { "$arrayElemAt": ['$member_list.sex', 0] }, 'age_group': { "$arrayElemAt": ['$member_list.age_group', 0] }, 'education': { "$arrayElemAt": ['$member_list.education', 0] }, 'correct': { '$size': { '$filter': { 'input': '$answer_list', 'as': 'item', 'cond': { '$and': [{ '$eq': ['$$item', True] }] } } } }, 'total': { "$size": "$answer_list" }, 'created_dt': 1 }), GroupStage( { 'daily_code': '$daily_code', 'member_cid': '$member_cid', 'dimension': '$dimension' }, province_code={'$first': "$province_code"}, city_code={'$first': "$city_code"}, district_code={'$first': "$district_code"}, gender={'$first': "$gender"}, age_group={'$first': "$age_group"}, education={'$first': "$education"}, correct={'$sum': "$correct"}, total={'$sum': '$total'}) ], allowDiskUse=True, read_preference=ReadPreference.PRIMARY) index = 0 while True: try: data = cursor.next() param = { 'daily_code': data.id.get('daily_code'), 'member_cid': data.id.get('member_cid'), 'dimension': data.id.get('dimension'), 'province_code': data.province_code, 'city_code': data.city_code, 'district_code': data.district_code, 'gender': data.gender, 'age_group': data.age_group, 'education': data.education, 'subject_total_quantity': data.total, 'subject_correct_quantity': data.correct, 'created_dt': data.created_dt } model(**param).sync_save() index += 1 print('has exec', index) except StopIteration: break except AttributeError as e: print(e) continue print('done.')
async def get(self): race_cid = self.get_argument('race_cid', '') if race_cid: # 抽奖总览 match_stage = MatchStage({'race_cid': race_cid, 'record_flag': 1}) old_match_stage = MatchStage({ 'race_cid': race_cid, 'record_flag': 0 }) a_sort = SortStage([('amount', DESC)]) # 关闭活动之前设置好的奖项配置 old_red_item_list = await RedPacketItemSetting.aggregate( stage_list=[old_match_stage, a_sort]).to_list(None) red_packet_item_list = await RedPacketItemSetting.aggregate( stage_list=[match_stage, a_sort]).to_list(None) # 中奖各个奖项的个数的列表 old_win_prize_quantity_list = [] win_prize_quantity_list = [] # 剩下的奖项的个数 remain_prize_quantity_list = [] # 已经中奖的各个奖项的数量 if red_packet_item_list: win_prize_quantity_list, remain_prize_quantity_list = await get_red_status_count( race_cid, red_packet_item_list) if old_red_item_list: old_win_prize_quantity_list, _ = await get_red_status_count( race_cid, old_red_item_list) else: red_packet_item_list = [] # 抽奖详情 kw_word = self.get_argument('kw_word', '') # 奖项标题 item_title = self.get_argument('item_title', '') memebr_lookup = LookupStage(Member, 'open_id', 'open_id', 'member_list') attr_project = ProjectStage( **{ 'member_cid': { '$arrayElemAt': ['$member_list.cid', 0] }, 'nick_name': { '$arrayElemAt': ['$member_list.nick_name', 0] }, 'province_code': { '$arrayElemAt': ['$member_list.province_code', 0] }, 'city_code': { '$arrayElemAt': ['$member_list.city_code', 0] }, 'lottery_dt': '$lottery_dt', 'award_cid': '$award_cid' }) p_lookup = LookupStage(AdministrativeDivision, 'province_code', 'code', 'p_list') c_lookup = LookupStage(AdministrativeDivision, 'city_code', 'code', 'c_list') award_lookup = LookupStage(RedPacketEntry, 'award_cid', 'cid', 'entry_list') info_project = ProjectStage( **{ 'member_cid': '$member_cid', 'nick_name': '$nick_name', 'p_list': '$p_list', 'c_list': '$c_list', 'province': { '$arrayElemAt': ['$p_list.title', 0] }, 'city': { '$arrayElemAt': ['$c_list.title', 0] }, 'lottery_dt': '$lottery_dt', 'award_cid': { '$arrayElemAt': ['$entry_list.award_cid', 0] } }) item_lookup = LookupStage(RedPacketItemSetting, 'award_cid', 'cid', 'item_list') final_project = ProjectStage( **{ 'member_cid': '$member_cid', 'nick_name': '$nick_name', 'province': '$province', 'city': '$city', 'p_list': '$p_list', 'c_list': '$c_list', 'lottery_dt': '$lottery_dt', 'award_title': { '$arrayElemAt': ['$item_list.title', 0] }, 'item_list': '$item_list' }) query_dict = {} if kw_word: query_dict['$or'] = [ { 'province': { '$regex': kw_word, '$options': 'i' } }, { 'city': { '$regex': kw_word, '$options': 'i' } }, { 'nick_name': { '$regex': kw_word, '$options': 'i' } }, { 'member_cid': { '$regex': kw_word, '$options': 'i' } }, ] if item_title: query_dict['award_title'] = { '$regex': item_title, '$options': 'i' } query = MatchStage(query_dict) query_match_dict = {'race_cid': race_cid} per_page_quantity = int(self.get_argument('per_page_quantity', 10)) to_page_num = int(self.get_argument('page', 1)) page_url = '%s?page=$page&per_page_quantity=%s&race_cid=%s&kw_name=%s&item_title=%s' % ( self.reverse_url("backoffice_race_lottery_result_list"), per_page_quantity, race_cid, kw_word, item_title) paging = Paging(page_url, RedPacketLotteryHistory, current_page=to_page_num, pipeline_stages=[ memebr_lookup, attr_project, p_lookup, c_lookup, award_lookup, info_project, item_lookup, final_project, query ], items_per_page=per_page_quantity, **query_match_dict) await paging.pager() return locals()
async def do_query_fight_subjects_by_rule_oid(race_cid: str, rule_cid: str): """ 依据抽题规则oid查询题目 :param race_cid: :param rule_cid: :return: """ if rule_cid and rule_cid: count = await get_subject_bank_quantity(race_cid, rule_cid) if count > 0: # 获取题目 race_subject_bank_list = await RaceSubjectBanks.find( dict(rule_cid=rule_cid, race_cid=race_cid, record_flag=1) ).skip(random.randint(1, count - 1)).limit(1).to_list(1) if race_subject_bank_list: race_subject_bank = await RaceSubjectBanks.find_one( {'cid': race_subject_bank_list[0].cid}) # subject_id_list = [ObjectId(subject_id) if isinstance(subject_id, str) else subject_id # for subject_id in subject_bank.subject_id_list] race_subject_cid = [ subject_cid for subject_cid in race_subject_bank.refer_subject_cid_list ] race_subject_list = await RaceSubjectRefer.aggregate( stage_list=[ MatchStage({ 'cid': { '$in': race_subject_cid }, 'race_cid': race_cid }), LookupStage( SubjectOption, let={'sub_cid': '$subject_cid'}, pipeline=[{ '$match': { '$expr': { '$and': [{ '$eq': ['$subject_cid', '$$sub_cid'] }] } } }, SortStage([('code', ASC)])], as_list_name='option_list'), LookupStage(Subject, 'subject_cid', 'cid', 'subject_list'), ProjectStage( **{ 'race_cid': '$race_cid', 'subject_cid': '$subject_cid', 'title': '$title', 'status': '$status', 'choice': '$choice', 'dimension_dict': '$dimension_dict', 'option_list': '$option_list', 'image_cid': { '$arrayElemAt': ['$subject_list.image_cid', 0] } }), LookupStage(UploadFiles, 'image_cid', 'cid', 'image_list') ]).to_list(None) return race_subject_list return None
async def do_stat_member_accuracy(race_cid: str, time_match: MatchStage, group_id, district_title="", time_num=""): """ 统计参赛正确率 :param race_cid: :param time_match: :param group_id: :param district_title :return: """ if not race_cid: return cache_key = get_cache_key( race_cid, 'member_accuracy_{district}_{time_num}_{district_title}'.format( district=group_id, time_num=time_num, district_title=district_title)) member_accuracy_data = RedisCache.get(cache_key) if member_accuracy_data: data_cache = msgpack.unpackb(member_accuracy_data, raw=False) if not member_accuracy_data or not data_cache: district_match = MatchStage({}) if district_title: district_match = MatchStage({'district': district_title}) # 安徽特殊处理,需整合六安数据(弃用) # race = await Race.get_by_cid(race_cid) all_match = {'race_cid': race_cid} # if race.province_code == '340000': # all_match = {'race_cid': {'$in': [race_cid, CITY_RACE_CID]}} cursor = RaceMemberEnterInfoStatistic.aggregate([ MatchStage(all_match), time_match, district_match, GroupStage(group_id, total_correct={'$sum': "$true_answer_times"}, total_count={'$sum': '$answer_times'}), ProjectStage( **{ 'city': "$city", 'sum': { '$cond': { 'if': { '$eq': ['$total_count', 0] }, 'then': 0, 'else': { '$divide': ['$total_correct', '$total_count'] } } } }) ]) accuracy_data = await stat_data(cursor) logger_cache.info('cache_key: %s' % cache_key) RedisCache.set(cache_key, msgpack.packb(accuracy_data), 23 * 60 * 60) return accuracy_data return msgpack.unpackb(member_accuracy_data, raw=False)
def export_race_enter_position(race_cid: str, title): """ 导出活动下面参与情况 :return: """ if not isinstance(race_cid, str): raise ValueError("race_cid is not str") now = datetime.datetime.now() export_time_list = [ now + datetime.timedelta(days=-n) for n in (range(1, 8)) ] export_time_list.sort() export_time = [ transform_time_format(export_dt) for export_dt in export_time_list ] export_time.sort() workbook = xlsxwriter.Workbook(title + today + ".xlsx") head_list = [ "新增人数", "参与人数", "参与次数", "通关人数", "红包发放数", "红包领取数量", "红包发放金额", "红包领取金额" ] sheet = workbook.add_worksheet(title + "一周数据") data_center_format = workbook.add_format({ 'valign': 'vcenter', 'align': 'center', 'font_name': 'Microsoft YaHei' }) sheet.merge_range(0, 0, 1, 0, "城市", data_center_format) sheet.merge_range(0, 1, 1, 1, "区县", data_center_format) sheet.write_string(1, 2, "累计人数") sheet.write_string(1, 3, "累计参与次数") for i in range(1, 8): sheet.merge_range(0, 7 * (i - 1) + 4 + i - 1, 0, 7 * (i - 1) + 4 + i - 1 + 7, export_time[i - 1], data_center_format) for head in range(7 * (i - 1) + 4 + i - 1, 7 * (i - 1) + 4 + i - 1 + 7 + 1): index = head - 8 * (i - 1) - 4 sheet.write_string(1, head, head_list[index]) midnight = (datetime.datetime.now()).replace(hour=0, minute=0, second=0, microsecond=0) race = Race.sync_get_by_cid(race_cid) # 市级活动,如六安市,扬州市 if race.city_code: city_code_list = AdministrativeDivision.sync_distinct( 'code', {'code': race.city_code}) # 该活动的所属城市范围 city_name_list = AdministrativeDivision.sync_distinct( 'title', {'code': race.city_code}) else: city_code_list = AdministrativeDivision.sync_distinct( 'code', {'parent_code': race.province_code}) city_name_list = AdministrativeDivision.sync_distinct( 'title', {'parent_code': race.province_code}) dist_list = [] for city_code in city_code_list: # 该活动区县的范围 dist_list += AdministrativeDivision.sync_distinct( 'title', {'parent_code': city_code}) # 最基本的人数match base_quantity_match = { 'race_cid': race_cid, 'auth_address.province': { '$ne': None }, "record_flag": 1, 'auth_address.city': { "$in": city_name_list }, 'auth_address.district': { "$in": dist_list } } # 最基本的次数match base_count_match = { 'race_cid': race_cid, 'province': { '$ne': None }, "record_$pushflag": 1, 'city': { "$in": city_name_list }, 'district': { "$in": dist_list } } # 最基本的参与人数match base_enter_quantity_match = { 'race_cid': race_cid, 'auth_address.province': { '$ne': None }, "record_flag": 1, 'auth_address.city': { "$in": city_name_list }, 'auth_address.district': { "$in": dist_list } } # 累计人数 quantity_match_stage = MatchStage({ 'race_cid': race_cid, 'auth_address.province': { '$ne': None }, "record_flag": 1, 'auth_address.city': { "$in": city_name_list }, 'auth_address.district': { "$in": dist_list }, 'created_dt': { '$lte': midnight } }) # 累计次数 daily_code = datetime2str(datetime.datetime.now(), date_format='%Y%m%d') count_match = { 'race_cid': race_cid, 'province': { '$ne': None }, 'city': { '$in': city_name_list }, 'district': { '$in': dist_list }, 'daily_code': { '$lt': daily_code } } lookup_stage = LookupStage(Member, 'member_cid', 'cid', 'member_list') address_sort = SortStage([('_id.city', ASC), ('_id.district', ASC)]) quantity_list = RaceMapping.sync_aggregate([ quantity_match_stage, lookup_stage, ProjectStage(**{ 'auth_address': "$auth_address", "member_list": "$member_list" }), MatchStage({'member_list': { '$ne': [] }}), GroupStage( { 'district': '$auth_address.district', 'city': '$auth_address.city' }, sum={'$sum': 1}), address_sort ]) count_list = ReportRacePeopleStatistics.sync_aggregate([ MatchStage(count_match), GroupStage({ 'city': '$city', 'district': '$district' }, sum={'$sum': "$total_num"}) ]) dis_map = {} quantity = 0 for index, address in enumerate(quantity_list): quantity += address.sum sheet.write_string(index + 2, 0, address.id.get('city')) sheet.write_string(index + 2, 1, address.id.get('district')) sheet.write_number(index + 2, 2, address.sum) if address.id.get('district') not in dis_map: dis_map[address.id.get('district')] = index + 2 else: dis_map[address.id.get('district')] += index + 2 count_map = {} for count in count_list: district = count.id.get('district') if district not in count_map: count_map[district] = count.sum print(count_map, 'count') print(dis_map, 'dis_map') for k, v in count_map.items(): position = dis_map.get(k) if position: sheet.write_number(position, 3, v) # 有答题次数,没有人数的情况,跳过 else: continue # 时间与地区人数的字典 {'20190702': {'六安市-叶集区': 20}, '20190703': {'六安市-舒城县': 30}} quantity_time_dict = {} # 一个星期的人数的数据 base_quantity_match['created_dt'] = { '$gte': export_time_list[0].replace(hour=0, minute=0, second=0), '$lte': (export_time_list[-1] + datetime.timedelta(days=1)).replace(hour=0, minute=0, second=0) } one_week_quantity_list = RaceMapping.sync_aggregate([ MatchStage(base_quantity_match), lookup_stage, ProjectStage( **{ 'daily_code': { '$dateToString': { 'format': "%Y%m%d", 'date': "$created_dt" } }, 'auth_address': "$auth_address", "member_list": "$member_list" }), MatchStage({'member_list': { '$ne': [] }}), GroupStage( { "daily_code": "$daily_code", 'district': '$auth_address.district', 'city': '$auth_address.city' }, sum={'$sum': 1}), address_sort ]) for quantity_data in one_week_quantity_list: daily = quantity_data.id.get('daily_code') city = quantity_data.id.get('city') district = quantity_data.id.get('district') if city and district: if daily not in quantity_time_dict: temp_dict = {} temp_dict["{city}-{district}".format( city=city, district=district)] = quantity_data.sum quantity_time_dict[daily] = temp_dict else: quantity_time_dict[daily].update({ "{city}-{district}".format(city=city, district=district): quantity_data.sum }) else: continue # 每日参与人数一周的数据 # 时间与地区人数的字典 {'20190702': {'六安市-叶集区': 20}, '20190703': {'六安市-舒城县': 30}} base_enter_quantity_match['updated_dt'] = { '$gte': export_time_list[0].replace(hour=0, minute=0, second=0), '$lte': (export_time_list[-1] + datetime.timedelta(days=1)).replace(hour=0, minute=0, second=0) } enter_quantity_time_dict = {} one_week_enter_quantity_list = RaceMapping.sync_aggregate([ MatchStage(base_enter_quantity_match), lookup_stage, ProjectStage( **{ 'daily_code': { '$dateToString': { 'format': "%Y%m%d", 'date': "$updated_dt" } }, 'auth_address': "$auth_address", "member_list": "$member_list" }), MatchStage({'member_list': { '$ne': [] }}), GroupStage( { "daily_code": "$daily_code", 'district': '$auth_address.district', 'city': '$auth_address.city' }, sum={'$sum': 1}), address_sort ]) for quantity_data in one_week_enter_quantity_list: daily = quantity_data.id.get('daily_code') city = quantity_data.id.get('city') district = quantity_data.id.get('district') if city and district: if daily not in enter_quantity_time_dict: temp_dict = {} temp_dict["{city}-{district}".format( city=city, district=district)] = quantity_data.sum enter_quantity_time_dict[daily] = temp_dict else: enter_quantity_time_dict[daily].update({ "{city}-{district}".format(city=city, district=district): quantity_data.sum }) else: continue # print(enter_quantity_time_dict, 'enter_quantity') # 每日新增参与次数一周的数据 base_count_match['created_dt'] = { '$gte': export_time_list[0].replace(hour=0, minute=0, second=0), '$lte': (export_time_list[-1] + datetime.timedelta(days=1)).replace(hour=0, minute=0, second=0) } one_week_count_list = ReportRacePeopleStatistics.sync_aggregate([ MatchStage(base_count_match), GroupStage( { 'city': '$city', 'district': '$district', 'daily_code': '$daily_code' }, sum={'$sum': "$total_num"}) ]) # 时间与地区次数的字典 {'20190702': {'六安市-叶集区': 20}, '20190703': {'六安市-舒城县': 30}} count_time_dict = {} for quantity_data in one_week_count_list: daily = quantity_data.id.get('daily_code') city = quantity_data.id.get('city') district = quantity_data.id.get('district') if city and district: if daily not in count_time_dict: temp_dict = {} temp_dict["{city}-{district}".format( city=city, district=district)] = quantity_data.sum count_time_dict[daily] = temp_dict else: count_time_dict[daily].update({ "{city}-{district}".format(city=city, district=district): quantity_data.sum }) else: continue # 一周通关的人数 # {'20190702': {'六安市-叶集区': 20, "六安市-舒城县": 15}, '20190703': {'六安市-舒城县': 30}} pass_quantity_time_dict = {} last_checkpoint_cid, _, _, _ = get_export_param(race_cid) # 拿到最后一关的cid pass_match_stage = MatchStage({ 'check_point_cid': last_checkpoint_cid, 'status': 1, 'record_flag': 1, 'created_dt': { '$gte': export_time_list[0].replace(hour=0, minute=0, second=0), '$lte': (export_time_list[-1] + datetime.timedelta(days=1)).replace(hour=0, minute=0, second=0) } }) check_point_cursor = MemberCheckPointHistory.sync_aggregate( [pass_match_stage, lookup_stage]) member_cid_list = [] while True: try: check_point = check_point_cursor.next() if check_point.member_list: member = check_point.member_list[0] race_mapping = RaceMapping.sync_find_one({ 'race_cid': race_cid, 'member_cid': member.cid }) if race_mapping: if race_mapping.auth_address and race_mapping.auth_address.get( 'province'): if check_point.member_cid not in member_cid_list: district = race_mapping.auth_address.get( 'district') city = race_mapping.auth_address.get('city') if district and city: member_cid_list.append(check_point.member_cid) created_time = format(check_point.created_dt, "%Y%m%d") if created_time not in pass_quantity_time_dict: pass_quantity_time_dict[created_time] = { "{city}-{district}".format(city=city, district=district): 1 } else: city_district = "{city}-{district}".format( city=city, district=district) v_dict = pass_quantity_time_dict.get( created_time) if city_district in v_dict: v_dict[city_district] += 1 else: v_dict[city_district] = 1 else: continue except StopIteration: break except Exception as e: raise e # 每日新增人数在excel的位置 quantity_increase_position_list = [4 + 8 * (i - 1) for i in range(1, 8)] # 每日参与人数在excel的位置 quantity_enter_position_list = [5 + 8 * (i - 1) for i in range(1, 8)] # 每日参与次数在excel的位置 count_enter_position_list = [6 + 8 * (i - 1) for i in range(1, 8)] # 每日通关次数在excel的位置 pass_enter_position_list = [7 + 8 * (i - 1) for i in range(1, 8)] # 红包发放数量在excel的位置 red_give_position_list = [8 + 8 * (i - 1) for i in range(1, 8)] # 红包发放金额在excel的位置 red_give_amount_list = [10 + 8 * (i - 1) for i in range(1, 8)] # 红包领取数量在excel的位置 red_receive_position_list = [9 + 8 * (i - 1) for i in range(1, 8)] # 红包领取金额在excel的位置 red_receive_amount_list = [11 + 8 * (i - 1) for i in range(1, 8)] print(quantity_increase_position_list) print(dis_map, 'dis_map6') print(quantity, 'quan') print(quantity_time_dict, 'quantity_time') # 填充每日新增人数 write_excel_data(sheet, quantity_time_dict, dis_map, export_time, quantity_increase_position_list, save=None) # excel 填充每日参与人数 write_excel_data(sheet, enter_quantity_time_dict, dis_map, export_time, quantity_enter_position_list, save=None) # excel 填充每日参与次数 write_excel_data(sheet, count_time_dict, dis_map, export_time, count_enter_position_list, save=None) # excel 填充每日通关人数 write_excel_data(sheet, pass_quantity_time_dict, dis_map, export_time, pass_enter_position_list, save=None) red_give_dict = {} red_give_amount_dict = {} # 红包发放个数 red_give_out_match = { "race_cid": race_cid, 'record_flag': 1, "award_cid": { '$ne': None }, 'member_cid': { '$ne': None }, 'draw_dt': { '$gte': export_time_list[0].replace(hour=0, minute=0, second=0), '$lte': (export_time_list[-1] + datetime.timedelta(days=1)).replace(hour=0, minute=0, second=0) } } red_give_out_cursor = RedPacketBox.sync_aggregate([ MatchStage(red_give_out_match), lookup_stage, ProjectStage( **{ 'daily_code': { '$dateToString': { 'format': "%Y%m%d", 'date': "$draw_dt" } }, "member_cid": '$member_cid', "member_list": "$member_list", "award_amount": '$award_amount', }), MatchStage({'member_list': { '$ne': [] }}), GroupStage({"daily_code": "$daily_code"}, amount_list={'$push': '$award_amount'}, cid_list={'$push': '$member_cid'}, sum={'$sum': 1}) ]) while True: try: red_packet = red_give_out_cursor.next() if red_packet and red_packet.cid_list: cid_list = red_packet.cid_list amount_list = red_packet.amount_list print(len(amount_list), 'len_m') print(len(cid_list), 'len') daily = red_packet.id.get('daily_code') for cid, amount in zip(cid_list, amount_list): race_mapping = RaceMapping.sync_find_one({ 'race_cid': race_cid, 'member_cid': cid, 'auth_address.city': { '$in': city_name_list }, 'auth_address.district': { '$in': dist_list } }) if race_mapping: city = race_mapping.auth_address.get('city') district = race_mapping.auth_address.get('district') city_district = "{city}-{district}".format( city=city, district=district) if daily not in red_give_dict: red_give_dict[daily] = {city_district: 1} else: v_dict = red_give_dict.get(daily) if city_district not in v_dict: v_dict[city_district] = 1 else: v_dict[city_district] += 1 if daily not in red_give_amount_dict: red_give_amount_dict[daily] = { city_district: amount } else: v_dict = red_give_amount_dict.get(daily) if city_district not in v_dict: v_dict[city_district] = amount else: v_dict[city_district] += amount except StopIteration: break except Exception as e: raise e print(red_give_dict, 'dict2') print(red_give_amount_dict, 'amount-dict2') # excel 填充每日发放个数 write_excel_data(sheet, red_give_dict, dis_map, export_time, red_give_position_list, save=None) # excel 填充每日领取金额 write_excel_data(sheet, red_give_amount_dict, dis_map, export_time, red_give_amount_list, save=2) # 红包领取个数 red_receive_dict = {} red_receive_amount_dict = {} red_receive_match = { "race_cid": race_cid, 'record_flag': 1, "award_cid": { '$ne': None }, 'draw_status': 0, 'member_cid': { '$ne': None }, 'draw_dt': { '$gte': export_time_list[0].replace(hour=0, minute=0, second=0), '$lte': (export_time_list[-1] + datetime.timedelta(days=1)).replace(hour=0, minute=0, second=0) } } red_receive_cursor = RedPacketBox.sync_aggregate([ MatchStage(red_receive_match), lookup_stage, ProjectStage( **{ 'daily_code': { '$dateToString': { 'format': "%Y%m%d", 'date': "$draw_dt" } }, "member_cid": '$member_cid', "member_list": "$member_list", "award_amount": '$award_amount', }), MatchStage({'member_list': { '$ne': [] }}), GroupStage({"daily_code": "$daily_code"}, receive_amount_list={'$push': '$award_amount'}, cid_list={'$push': '$member_cid'}, sum={'$sum': 1}) ]) while True: try: red_packet = red_receive_cursor.next() if red_packet and red_packet.cid_list: amount_list = red_packet.receive_amount_list cid_list = list(set(red_packet.cid_list)) print(len(cid_list), 'len') daily = red_packet.id.get('daily_code') for cid, amount in zip(cid_list, amount_list): race_mapping = RaceMapping.sync_find_one({ 'race_cid': race_cid, 'member_cid': cid, 'auth_address.city': { '$in': city_name_list }, 'auth_address.district': { '$in': dist_list } }) if race_mapping: city = race_mapping.auth_address.get('city') district = race_mapping.auth_address.get('district') city_district = "{city}-{district}".format( city=city, district=district) if daily not in red_receive_dict: red_receive_dict[daily] = {city_district: 1} else: v_dict = red_receive_dict.get(daily) if city_district not in v_dict: v_dict[city_district] = 1 else: v_dict[city_district] += 1 if daily not in red_receive_amount_dict: red_receive_amount_dict[daily] = { city_district: amount } else: v_dict = red_receive_amount_dict.get(daily) if city_district not in v_dict: v_dict[city_district] = amount else: v_dict[city_district] += amount except StopIteration: break except Exception as e: raise e print(red_receive_dict, 'receive_cursor') print(red_receive_amount_dict, 'rece_amount') # excel 填充每日领取个数 write_excel_data(sheet, red_receive_dict, dis_map, export_time, red_receive_position_list, save=None) # excel 填充每日领取金额 write_excel_data(sheet, red_receive_amount_dict, dis_map, export_time, red_receive_amount_list, save=2) workbook.close()
def do_statistics_member_top_n(cache_key, m_city_code_list, stat_type, top_n, time_range): """ :param cache_key: :param m_city_code_list: :param stat_type: :param top_n: :param time_range: :return: """ RedisCache.set(cache_key, KEY_CACHE_REPORT_DOING_NOW, 5 * 60) stage_list = [] # 取前一天凌晨12点之前的数据 time_match = get_yesterday() stage_list.append(MatchStage({'updated_dt': {'$lt': time_match}})) if m_city_code_list: stage_list.append(MatchStage({'city_code': {'$in': m_city_code_list}})) s_code = '' e_code = '' if time_range: suffix = time_range[-1:] range_num = int(time_range.replace(suffix, '')) delta = None if suffix.upper() == 'D': delta = datetime.timedelta(days=range_num) elif suffix.upper() == 'M': delta = datetime.timedelta(days=range_num * 30) elif suffix.upper() == 'Y': delta = datetime.timedelta(days=range_num * 365) if delta: s_code = datetime2str(datetime.datetime.now() - delta, date_format='%Y%m%d000000') e_code = datetime2str(datetime.datetime.now(), date_format='%Y%m%d000000') if s_code and e_code: stage_list.append( MatchStage({'daily_code': { '$gte': s_code, '$lte': e_code }})) stage_list.extend([ GroupStage({ 'daily_code': '$daily_code', 'member_cid': '$member_cid', 'province_code': '$province_code' } if stat_type == 1 else { 'daily_code': '$daily_code', 'member_cid': '$member_cid', 'province_code': '$province_code', 'city_code': '$city_code' }, learn_times={'$sum': '$learn_times'}), GroupStage({ 'daily_code': '$_id.daily_code', 'province_code': '$_id.province_code' } if stat_type == 1 else { 'daily_code': '$_id.daily_code', 'province_code': '$_id.province_code', 'city_code': '$_id.city_code' }, count={'$sum': 1}, times={'$sum': '$learn_times'}), LookupStage(AdministrativeDivision, '_id.province_code', 'post_code', 'province_list'), LookupStage(AdministrativeDivision, '_id.city_code', 'post_code', 'city_list'), ProjectStage( **{ '_id': False, 'daily_code': '$_id.daily_code', 'count': '$count', 'times': '$times', 'province_code': '$_id.province_code', 'province_title': '$province_list.title', 'ad_code': '$_id.province_code' if stat_type == 1 else '$_id.city_code', 'ad_title': '$province_list.title' if stat_type == 1 else '$city_list.title' }), SortStage([('daily_code', ASC), ('count', DESC)]) ]) # 检索数据 data = [] stat_cursor = MemberDailyStatistics.sync_aggregate(stage_list) t_code, t_list = None, None top_n_found = False while True: try: daily_stat = stat_cursor.next() if daily_stat: daily_code = daily_stat.daily_code if not daily_code == t_code: t_code = daily_code t_list = [] top_n_found = False print(t_code) if len(t_list) < top_n: t_list.append({ 'date': daily_code[:8], 'ad_code': daily_stat.ad_code, 'province_title': daily_stat.province_title[0] if daily_stat.province_title and stat_type == 2 else '', 'title': daily_stat.ad_title[0] if daily_stat.ad_title else 'undefined', 'quantity': daily_stat.count, 'times': daily_stat.times }) elif not top_n_found: if t_code is not None: data.append(t_list) top_n_found = True except StopIteration: break if not data: early_warning_empty("start_statistics_member_top_n", cache_key, locals(), '每日参与TOP5统计数据为空,请检查!') RedisCache.set(cache_key, msgpack.packb(data))