def statistic_daily_award_of_check_point(check_point_cid: str, daily_code): """ 统计每一关每天参与人数、抽奖人数、中奖人数 :param check_point_cid: 关卡cid :return: """ # print("正在统计关卡:", check_point.alias, check_point.cid) match = {'check_point_cid': check_point_cid, 'record_flag': 1} red_match = {'checkpoint_cid': check_point_cid} s_date = str2datetime(daily_code, date_format='%Y%m%d').replace(hour=0, minute=0, second=0) e_date = s_date + datetime.timedelta(days=1) match['fight_datetime'] = {'$gte': s_date, '$lt': e_date} red_match['draw_dt'] = {'$gte': s_date, '$lt': e_date} # 参与人数 attend_member = MemberCheckPointHistory.sync_distinct("member_cid", match) attend_count = len(attend_member) # 当前关卡的红包 current_checkpoint_reds = RedPacketBox.sync_find( red_match, read_preference=ReadPreference.PRIMARY).batch_size(2000) lottery_count = 0 win_count = 0 for current_checkpoint_red in current_checkpoint_reds: lottery_count += 1 if current_checkpoint_red.award_cid: win_count += 1 daily_award = DailyAward(daily_code, attend_count, lottery_count, win_count) # print(check_point_cid, daily_award) return daily_award
def is_new_user(info: MemberStatisticInfo): """ 判断是否为当日新用户 :param info: :return: """ race_cid = info.race_cid member_cid = info.member_cid current_date = str2datetime(info.daily_code, '%Y%m%d').replace(hour=0, minute=0, second=0, microsecond=0) checkpoint_cid_list, _checkpoint_map = get_checkpoint_cid_list(race_cid) has_history = MemberCheckPointHistory.sync_find_one( {'member_cid': member_cid, 'check_point_cid': {'$in': checkpoint_cid_list}, 'created_dt': {'$lt': current_date}}) if has_history: race_mapping = RaceMapping.sync_find_one({'race_cid': race_cid, 'member_cid': member_cid}) if format(race_mapping.created_dt, "%Y%m%d") == info.daily_code: info.is_new_user = 1 else: info.is_new_user = 0 else: # 之前报名活动,但未答题的情况 race_mapping = RaceMapping.sync_find_one({'race_cid': race_cid, 'member_cid': member_cid}) if format(race_mapping.created_dt, "%Y%m%d") == info.daily_code: info.is_new_user = 1 else: info.is_new_user = 0
def transform_date(time_str): """ 将时间字符串转换成日期 20190627转换成Date(20190627) :param time_str: :return: """ date = time_str[:4] + '/' + time_str[4:6] + '/' + time_str[6:] date = date.replace("/0", "/") return str2datetime(date, "%Y/%m/%d")
def test(race_cid): member_cid_list = MemberStatisticInfo.sync_distinct( "member_cid", { 'race_cid': race_cid, 'is_new_user': 1 }) list2 = MemberStatisticInfo.sync_distinct('member_cid', {'race_cid': race_cid}) cursor2 = MemberStatisticInfo.sync_find({'race_cid': race_cid}) print(len(member_cid_list)) print(len(list2)) count = 0 count1 = 0 dailiy = set() for member in cursor2: count += 1 if member.member_cid not in member_cid_list: count1 += 1 print('-----------------------') dailiy.add(member.daily_code) print(member.daily_code, member.member_cid, member.is_new_user, member.draw_red_packet_amount) race_mapping = RaceMapping.sync_find_one({ 'race_cid': race_cid, 'member_cid': member.member_cid }) print(11, race_mapping.cid, format(race_mapping.created_dt, "%Y%m%d"), race_mapping.updated_dt) current_date = str2datetime(member.daily_code, '%Y%m%d').replace(hour=0, minute=0, second=0, microsecond=0) has_history = MemberCheckPointHistory.sync_find_one({ 'member_cid': member.member_cid, 'created_dt': { '$lt': current_date } }) if has_history: print(222, has_history) else: print(None) print('total', count) print('no', count1) temp = list(dailiy) temp.sort() print(temp) return temp
def deal_member_without_history(race_cid, daily_code): """ 处理报名活动但未答题的会员 :param race_cid: :param daily_code: :return: """ city_name_list, district_name_list = get_address(race_cid) checkpoint_cid, _checkpoint_map = get_checkpoint_cid_list(race_cid) start_date = str2datetime(daily_code, '%Y%m%d').replace(hour=0, minute=0, second=0, microsecond=0) end_date = start_date + datetime.timedelta(days=1) member_cid_with_history = MemberCheckPointHistory.sync_distinct( "member_cid", { 'check_point_cid': { '$in': checkpoint_cid }, 'created_dt': { '$gte': start_date, '$lt': end_date } }) race_member_match = MatchStage({ "race_cid": race_cid, 'member_cid': { '$nin': member_cid_with_history }, 'auth_address.city': { "$in": city_name_list }, 'auth_address.district': { "$in": district_name_list }, 'created_dt': { '$gte': start_date, '$lt': end_date } }) member_group = GroupStage({'member_cid': '$member_cid'}, auth_address={'$first': '$auth_address'}, company_cid={'$first': '$company_cid'}, mobile={'$first': '$mobile'}, created_dt={'$first': '$created_dt'}) member_project = ProjectStage( **{ 'cid': '$cid', 'member_cid': '$_id.member_cid', 'auth_address': '$auth_address', 'mobile': '$mobile', 'created_dt': '$created_dt', 'company_cid': '$company_cid' }) member_without_history = RaceMapping.sync_aggregate( [race_member_match, member_group, member_project]).batch_size(4) member_amount_map = get_red_packet_info(race_cid, start_date, end_date) red_member_cid_list = member_amount_map.keys() member_no_history_list = [] count = 0 while True: try: stat = member_without_history.next() count += 1 if stat.member_cid in red_member_cid_list: continue # 根据member_cid查对应的member信息 temp_member = Member.sync_get_by_cid(stat.member_cid) if not temp_member: print("no history未找到对应member_cid:%s" % stat.member_cid) continue info_special = MemberStatisticInfo() info_special.is_special_user = 1 info_special.race_cid = race_cid info_special.member_cid = stat.member_cid info_special.daily_code = format(stat.created_dt, '%Y%m%d') info_special.nick_name = temp_member.nick_name info_special.open_id = temp_member.open_id if stat.mobile: info_special.mobile = stat.mobile else: info_special.mobile = temp_member.mobile info_special.first_time_login = temp_member.created_dt info_special.enter_times = 1 info_special.answer_times = 0 info_special.true_answer_times = 0 info_special.is_final_passed = 0 info_special.is_new_user = 1 info_special.grant_red_packet_amount = 0.0 info_special.grant_red_packet_count = 0 info_special.draw_red_packet_count = 0 info_special.draw_red_packet_amount = 0.0 info_special.province = stat.auth_address.get('province') info_special.city = stat.auth_address.get('city') info_special.district = stat.auth_address.get('district') info_special.town = stat.auth_address.get('town') info_special.check_point_cid = stat.race_check_point_cid if stat.race_check_point_cid: info_special.check_point_index = _checkpoint_map[ stat.race_check_point_cid] else: info_special.check_point_index = 1 if stat.company_cid: company = Company.sync_get_by_cid(stat.company_cid) info_special.company_cid = company.cid info_special.company_name = company.title member_no_history_list.append(info_special) # logger.info("Success without history: member_cid:%s is_final_Pass:%s" % (info_special.member_cid,info_special.is_final_passed)) if len(member_no_history_list) % 500 == 0: MemberStatisticInfo.sync_insert_many(member_no_history_list) member_no_history_list = [] except StopIteration: break except CursorNotFound: member_without_history = RaceMapping.sync_aggregate( [race_member_match, member_group, member_project]).skip(count).batch_size(4) except Exception as e: logger.info( "Fail: without history daily_code:%s,member_cid:%s,race_cid: %s" % (info_special.daily_code, info_special.member_cid, race_cid)) if len(member_no_history_list) > 0: MemberStatisticInfo.sync_insert_many(member_no_history_list)
def daily_member_statistic(race_cid, daily_code): """ 统计每日活动下的会员 :param race_cid: :param daily_code: :return: """ start_date = str2datetime(daily_code, '%Y%m%d').replace(hour=0, minute=0, second=0, microsecond=0) end_date = start_date + datetime.timedelta(days=1) check_point_cids, _checkpoint_map = get_checkpoint_cid_list(race_cid) # 当日答题记录 history_match = MatchStage({ 'created_dt': { '$gte': start_date, '$lt': end_date }, 'check_point_cid': { '$in': check_point_cids } }) history_group = GroupStage({'member_cid': '$member_cid'}, enter_times={'$sum': 1}, results_list={'$push': '$result'}) history_project = ProjectStage( **{ 'member_cid': '$_id.member_cid', 'results_list': '$results_list', 'enter_times': '$enter_times' }) # 红包信息 member_amount_map = get_red_packet_info(race_cid, start_date, end_date) # member_cid: amount check_point_history_cursor = MemberCheckPointHistory.sync_aggregate( [history_match, history_group, history_project], allowDiskUse=True).batch_size(4) member_statistic_list = [] count = 0 while True: try: cursor = check_point_history_cursor.next() count += 1 # 根据member_cid查对应的member信息 temp_member = Member.sync_get_by_cid(cursor.member_cid) if not temp_member: print("normal未找到对应member_cid:%s" % cursor.member_cid) continue info = MemberStatisticInfo() info.daily_code = daily_code info.race_cid = race_cid # 会员信息 info.member_cid = cursor.member_cid info.nick_name = temp_member.nick_name info.open_id = temp_member.open_id info.mobile = temp_member.mobile info.first_time_login = temp_member.created_dt info.enter_times = cursor.enter_times # 答题数量、正确数 for results in cursor.results_list: info.answer_times += len(results) info.true_answer_times += len([ result for result in results if result.get('true_answer') ]) # race_mapping相关信息,地理位置 has_race_mapping = get_race_mapping_info(info) if not has_race_mapping: print("normal未找到对应race_mapping,race_cid:%s,member_cid:%s" % (info.race_cid, info.member_cid)) continue # 是否为当日新用户 is_new_user(info) # 最后一关 is_final_pass(info) # 红包信息 try: value = member_amount_map.pop(info.member_cid) except KeyError: value = None if not value: value = { 'grant_count': 0, 'grant_amount': 0, 'draw_count': 0, 'draw_amount': 0 } info.grant_red_packet_amount = value.get('grant_amount') info.grant_red_packet_count = value.get('grant_count') info.draw_red_packet_amount = value.get('draw_amount') info.draw_red_packet_count = value.get('draw_count') # 保存记录 member_statistic_list.append(info) # logger.info("Success: member_cid:%s is_final_Pass:%s is_new:%s" % ( # info.member_cid, info.is_final_passed, info.is_new_user)) print("Success: member_cid:%s is_final_Pass:%s is_new:%s" % (info.member_cid, info.is_final_passed, info.is_new_user)) if len(member_statistic_list) % 500 == 0: MemberStatisticInfo.sync_insert_many(member_statistic_list) member_statistic_list = [] except StopIteration: break except CursorNotFound: check_point_history_cursor = MemberCheckPointHistory.sync_aggregate( [history_match, history_group, history_project]).skip(count).batch_size(4) except Exception as e: logger.error(e) logger.info("Fail: daily_code:%s,race_cid: %s" % (daily_code, race_cid)) member_statistic_list = [] member_statistic_list += insert_from_member_amount_map( member_amount_map, daily_code, race_cid) if len(member_statistic_list) > 0: MemberStatisticInfo.sync_insert_many(member_statistic_list)
async def post(self): res = dict(code=0) name = self.get_argument('name') sex = int(self.get_argument('sex', 1)) email = self.get_argument('email') mobile = self.get_argument('mobile') birthday = self.get_argument('birthday') province_code = self.get_argument('province_code') # 省份编码 city_code = self.get_argument('city_code') # 城市编码 vehicle_code = self.get_argument('vehicle_code') # 车型编号 purchase_datetime = self.get_argument('purchase_datetime') # 车辆购买时间 content = self.get_argument('content') # 备注 status = int(self.get_argument('status', STATUS_USER_ACTIVE)) if name and mobile and email: # 校验用户名 if mobile: exist_count = await Member.count(dict(mobile=mobile)) if exist_count: res['code'] = -2 return res if email: email_valid = re.match( r"^\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$", email) exist_count = await Member.count(dict(email=email)) if exist_count: res['code'] = -3 return res if not email_valid: res['code'] = -4 return res if mobile: member = Member() member.mobile = mobile member.code = get_increase_code(KEY_INCREASE_MEMBER_CODE) if email: member.email = email else: member = Member() member.email = email member.code = get_increase_code(KEY_INCREASE_MEMBER_CODE) if name: member.name = name member.sex = sex if birthday: member.birthday = str2datetime(birthday, '%Y-%m-%d') if vehicle_code: vehicle = await Vehicle.find_one( dict(code=vehicle_code, record_flag=1)) if vehicle: member.needless['vehicle_title'] = vehicle.title member.needless['vehicle_brand'] = vehicle.needless.get( 'brand') member.needless['vehicle_category'] = vehicle.needless.get( 'category') member.needless['vehicle_series'] = vehicle.needless.get( 'series') member.needless['vehicle_config'] = vehicle.needless.get( 'config') member.needless['vehicle_colour'] = vehicle.needless.get( 'colour') member.needless['vehicle_displace'] = vehicle.needless.get( 'displace') else: for attr in [ 'vehicle_brand', 'vehicle_category', 'vehicle_series', 'vehicle_config', 'vehicle_colour', 'vehicle_displace', 'vehicle_title' ]: try: member.needless.pop(attr) except KeyError: pass member.content = content member.status = status # 冗余信息 if province_code: province = await AdministrativeDivision.find_one( dict(parent_code=None, code=province_code)) if province: member.needless['province'] = province.title city = await AdministrativeDivision.find_one( dict(parent_code=province_code, code=city_code)) if city: member.needless['city'] = city.title member.province_code = province_code member.city_code = city_code member.source = SOURCE_TYPE_MEMBER_SYSTEM if vehicle_code: if vehicle_code: vehicle = await Vehicle.find_one( dict(code=vehicle_code, record_flag=1)) if vehicle: member.needless['vehicle_title'] = vehicle.title member.needless[ 'vehicle_brand'] = vehicle.needless.get('brand') member.needless[ 'vehicle_category'] = vehicle.needless.get( 'category') member.needless[ 'vehicle_series'] = vehicle.needless.get('series') member.needless[ 'vehicle_config'] = vehicle.needless.get('config') member.needless[ 'vehicle_colour'] = vehicle.needless.get('colour') member.needless[ 'vehicle_displace'] = vehicle.needless.get( 'displace') member.vehicle_code = vehicle_code if purchase_datetime: member.purchase_datetime = str2datetime( purchase_datetime, '%Y-%m') member.status = status member_id = await member.save() res['code'] = 1 res['member_id'] = member_id else: res['code'] = -1 return res
async def post(self): r_dict = {'code': 0} race_cid = self.get_argument('race_cid') race = await Race.get_by_cid(race_cid) if not race: r_dict['code'] = -1 return r_dict code = self.get_argument('code') title = self.get_argument('title') start_dt = self.get_argument('start_dt') end_dt = self.get_argument('end_dt') if not code or not title or not start_dt or not end_dt: r_dict['code'] = -2 return r_dict if await Race.count(dict(code=code)) > 0: r_dict['code'] = -5 return r_dict if await Race.count(dict(title=title)) > 0: r_dict['code'] = -6 return r_dict new_cid = get_new_cid() try: new_race = copy.deepcopy(race) new_race.cid = new_cid new_race.code = code new_race.title = title new_race.start_datetime = str2datetime(start_dt) new_race.end_datetime = str2datetime(end_dt) new_race.status = STATUS_RACE_INACTIVE new_race.owner_list = [self.current_user.cid] await Race.insert_many([new_race]) # 开始复制 await do_copy_subject_refer(race_cid, new_race.cid, self.current_user.cid) await do_copy_red_packet_rule(race_cid, new_race.cid, self.current_user.cid) await do_copy_choice_rule(race_cid, new_race.cid, self.current_user.cid) await do_copy_checkpoint(race_cid, new_race.cid, self.current_user.cid) await do_copy_subject_bank(race_cid, new_race.cid, self.current_user.cid) await do_copy_company(race_cid, new_race.cid, self.current_user.cid) await do_copy_red_packet_box(race_cid, new_race.cid, self.current_user.cid) r_dict['code'] = 1 except Exception: logger.error(traceback.format_exc()) await Race.delete_many({'cid': new_cid}) await RaceSubjectRefer.delete_many({'race_cid': new_cid}) await RaceSubjectChoiceRules.delete_many({'race_cid': new_cid}) await RaceSubjectBanks.delete_many({'race_cid': new_cid}) await RedPacketItemSetting.delete_many({'race_cid': new_cid}) await Company.delete_many({'race_cid': new_cid}) await RedPacketConf.delete_many({'race_cid': new_cid}) await RedPacketRule.delete_many({'race_cid': new_cid}) await RaceGameCheckPoint.delete_many({'race_cid': new_cid}) finally: pass return r_dict
async def get(self): pipelines = list() # 展示当前用户可看的竞赛活动 user = self.current_user # 隐藏了筛选条件,编辑,状态变更等按钮,不是超级管理员的情况下 button_hide = False # 不是超管的情况 if not user.superuser: if PERMISSION_TYPE_MINIAPP_RACE_ADD_OR_EDIT_MANAGEMENT not in user.permission_code_list: button_hide = True city_title = user.city # 省级账号可以看到该省和该省下面所有的市的活动 province_title = user.province try: if province_title and not city_title: province = await AdministrativeDivision.find_one({ 'title': province_title, 'record_flag': 1 }) if province: # 该省下面的所有市 city_list = await AdministrativeDivision.find({ 'parent_code': province.code }).to_list(None) city_code_list = [ city.code for city in city_list if city_list ] pipelines.append( MatchStage({ "$or": [{ "city_code": { '$in': city_code_list } }, { 'province_code': province.code }] })) # 市级账号只能看到该市的活动 elif city_title: city = await AdministrativeDivision.find_one({ 'title': { '$regex': city_title }, 'record_flag': 1 }) if city: pipelines.append(MatchStage({"city_code": city.code})) except Exception as e: logger.error(traceback.format_exc()) search_keywords = self.get_argument('search_keywords', '') search_date = self.get_argument('search_date', '') sort_type = int(self.get_argument('sort_type', 1)) search_province = self.get_argument('search_province', '') search_city = self.get_argument('search_city', '') page = int(self.get_argument('page', 1)) per_page_quantity = int(self.get_argument('per_page_quantity', 10)) if search_keywords: pipelines.append( MatchStage({ '$or': [{ 'code': { '$regex': search_keywords } }, { 'title': { '$regex': search_keywords } }] })) if search_date: try: start_date_str, end_date_str = search_date.split(' - ') start_datetime = str2datetime(start_date_str) end_datetime = str2datetime(end_date_str) pipelines.append( MatchStage({ 'start_datetime': { '$gte': start_datetime }, 'end_datetime': { '$lte': end_datetime } })) except ValueError or TypeError: logger.error(traceback.format_exc()) search_date = '' sort_list = [] if sort_type: if int(sort_type) == 1: sort_list.append('-start_datetime') if int(sort_type) == 2: sort_list.append('start_datetime') left_pipes = [ LookupStage(AdministrativeDivision, 'province_code', 'code', 'province_list'), LookupStage(AdministrativeDivision, 'city_code', 'code', 'city_list'), ] try: query_param = {'record_flag': 1} if search_province: query_param['province_code'] = search_province if search_city: query_param['city_code'] = search_city page_url = '%s?page=$page&per_page_quantity=%s&search_keywords=%s&search_date=%s&sort_type=%s&' \ 'search_province=%s&search_city=%s' % ( self.reverse_url("backoffice_race_list"), per_page_quantity, search_keywords, search_date, sort_type, search_province, search_city) paging = Paging(page_url, Race, current_page=page, items_per_page=per_page_quantity, pipeline_stages=pipelines, left_pipeline_stages=left_pipes, sort=sort_list, **query_param) await paging.pager() city_list = None if search_province: city_list = await AdministrativeDivision.find( dict(parent_code=search_province)).to_list(None) province_list = await AdministrativeDivision.find( dict(parent_code=None)).to_list(None) now = datetime.now() except Exception: logger.error(traceback.format_exc()) return locals()
def do_init(): """ 初始该活动数据 :return: """ cache_key = 'race_report_script' RedisCache.delete(cache_key) cp_map = get_all_race_checkpoint_map() last_map = get_all_last_checkpoint() with open('./chekpt_history.csv', encoding='utf-8') as f: csv_reader = csv.reader(f) for index, line in enumerate(csv_reader): try: if index == 0: continue # member_cid,check_point_cid,status,created_dt his = MemberCheckPointHistory() his.member_cid = line[0] his.check_point_cid = line[1] his.status = line[2] line3 = line[3].replace('T', ' ').split('.')[0] c_dt = str2datetime(line3) his.created_dt = c_dt mapping = RaceMapping.sync_find_one({ 'member_cid': his.member_cid, 'race_cid': cp_map.get(his.check_point_cid) }) member = Member.sync_get_by_cid(his.member_cid) auth_address = mapping.auth_address if mapping else None if not auth_address: continue race_cid = cp_map[his.check_point_cid] daily_code = __get_daily_code(his.created_dt) param = { 'race_cid': race_cid, 'province': auth_address.get('province'), 'city': auth_address.get('city'), 'district': auth_address.get('district'), 'town': auth_address.get('town'), 'sex': member.sex, 'education': member.education, 'category': member.category, 'daily_code': daily_code, 'company_cid': mapping.company_cid } stat = ReportRacePeopleStatisticsTemp.sync_find_one( param, read_preference=ReadPreference.PRIMARY) if not stat: stat = ReportRacePeopleStatisticsTemp(**param) stat.created_dt = his.created_dt stat.total_num += 1 # 初次通关 if his.check_point_cid == last_map[ race_cid] and his.status == STATUS_RESULT_CHECK_POINT_WIN and RedisCache.hget( cache_key, member.cid) is None: stat.pass_num += 1 RedisCache.hset(cache_key, member.cid, 1) # 当日人数 day_member_string = md5(daily_code + member.cid) if RedisCache.hget(cache_key, day_member_string) is None: RedisCache.hset(cache_key, day_member_string, 1) stat.people_num += 1 # # 当日新增人数 # old_his = MemberCheckPointHistory.sync_find_one({'member_cid': member.cid, 'created_dt': { # '$lt': his.updated_dt.replace(hour=0, minute=0, second=0, microsecond=0)}}) # if not old_his: # stat.incre_people += 1 stat.updated_dt = his.created_dt stat.sync_save() print('has exec %s' % index) except Exception: print(traceback.format_exc())