def statistic_daily_award_of_check_point(check_point_cid: str, daily_code):
"""
统计每一关每天参与人数、抽奖人数、中奖人数
:param check_point_cid: 关卡cid
:return:
"""
# print("正在统计关卡:", check_point.alias, check_point.cid)
match = {'check_point_cid': check_point_cid, 'record_flag': 1}
red_match = {'checkpoint_cid': check_point_cid}
s_date = str2datetime(daily_code, date_format='%Y%m%d').replace(hour=0,
minute=0,
second=0)
e_date = s_date + datetime.timedelta(days=1)
match['fight_datetime'] = {'$gte': s_date, '$lt': e_date}
red_match['draw_dt'] = {'$gte': s_date, '$lt': e_date}
# 参与人数
attend_member = MemberCheckPointHistory.sync_distinct("member_cid", match)
attend_count = len(attend_member)
# 当前关卡的红包
current_checkpoint_reds = RedPacketBox.sync_find(
red_match, read_preference=ReadPreference.PRIMARY).batch_size(2000)
lottery_count = 0
win_count = 0
for current_checkpoint_red in current_checkpoint_reds:
lottery_count += 1
if current_checkpoint_red.award_cid:
win_count += 1
daily_award = DailyAward(daily_code, attend_count, lottery_count,
win_count)
# print(check_point_cid, daily_award)
return daily_award
def is_new_user(info: MemberStatisticInfo):
"""
判断是否为当日新用户
:param info:
:return:
"""
race_cid = info.race_cid
member_cid = info.member_cid
current_date = str2datetime(info.daily_code, '%Y%m%d').replace(hour=0, minute=0, second=0, microsecond=0)
checkpoint_cid_list, _checkpoint_map = get_checkpoint_cid_list(race_cid)
has_history = MemberCheckPointHistory.sync_find_one(
{'member_cid': member_cid, 'check_point_cid': {'$in': checkpoint_cid_list},
'created_dt': {'$lt': current_date}})
if has_history:
race_mapping = RaceMapping.sync_find_one({'race_cid': race_cid, 'member_cid': member_cid})
if format(race_mapping.created_dt, "%Y%m%d") == info.daily_code:
info.is_new_user = 1
else:
info.is_new_user = 0
else:
# 之前报名活动，但未答题的情况
race_mapping = RaceMapping.sync_find_one({'race_cid': race_cid, 'member_cid': member_cid})
if format(race_mapping.created_dt, "%Y%m%d") == info.daily_code:
info.is_new_user = 1
else:
info.is_new_user = 0
def transform_date(time_str):
"""
将时间字符串转换成日期 20190627转换成Date(20190627)
:param time_str:
:return:
"""
date = time_str[:4] + '/' + time_str[4:6] + '/' + time_str[6:]
date = date.replace("/0", "/")
return str2datetime(date, "%Y/%m/%d")
def test(race_cid):
member_cid_list = MemberStatisticInfo.sync_distinct(
"member_cid", {
'race_cid': race_cid,
'is_new_user': 1
})
list2 = MemberStatisticInfo.sync_distinct('member_cid',
{'race_cid': race_cid})
cursor2 = MemberStatisticInfo.sync_find({'race_cid': race_cid})
print(len(member_cid_list))
print(len(list2))
count = 0
count1 = 0
dailiy = set()
for member in cursor2:
count += 1
if member.member_cid not in member_cid_list:
count1 += 1
print('-----------------------')
dailiy.add(member.daily_code)
print(member.daily_code, member.member_cid, member.is_new_user,
member.draw_red_packet_amount)
race_mapping = RaceMapping.sync_find_one({
'race_cid':
race_cid,
'member_cid':
member.member_cid
})
print(11, race_mapping.cid,
format(race_mapping.created_dt, "%Y%m%d"),
race_mapping.updated_dt)
current_date = str2datetime(member.daily_code,
'%Y%m%d').replace(hour=0,
minute=0,
second=0,
microsecond=0)
has_history = MemberCheckPointHistory.sync_find_one({
'member_cid':
member.member_cid,
'created_dt': {
'$lt': current_date
}
})
if has_history:
print(222, has_history)
else:
print(None)
print('total', count)
print('no', count1)
temp = list(dailiy)
temp.sort()
print(temp)
return temp
def deal_member_without_history(race_cid, daily_code):
"""
处理报名活动但未答题的会员
:param race_cid:
:param daily_code:
:return:
"""
city_name_list, district_name_list = get_address(race_cid)
checkpoint_cid, _checkpoint_map = get_checkpoint_cid_list(race_cid)
start_date = str2datetime(daily_code, '%Y%m%d').replace(hour=0,
minute=0,
second=0,
microsecond=0)
end_date = start_date + datetime.timedelta(days=1)
member_cid_with_history = MemberCheckPointHistory.sync_distinct(
"member_cid", {
'check_point_cid': {
'$in': checkpoint_cid
},
'created_dt': {
'$gte': start_date,
'$lt': end_date
}
})
race_member_match = MatchStage({
"race_cid": race_cid,
'member_cid': {
'$nin': member_cid_with_history
},
'auth_address.city': {
"$in": city_name_list
},
'auth_address.district': {
"$in": district_name_list
},
'created_dt': {
'$gte': start_date,
'$lt': end_date
}
})
member_group = GroupStage({'member_cid': '$member_cid'},
auth_address={'$first': '$auth_address'},
company_cid={'$first': '$company_cid'},
mobile={'$first': '$mobile'},
created_dt={'$first': '$created_dt'})
member_project = ProjectStage(
**{
'cid': '$cid',
'member_cid': '$_id.member_cid',
'auth_address': '$auth_address',
'mobile': '$mobile',
'created_dt': '$created_dt',
'company_cid': '$company_cid'
})
member_without_history = RaceMapping.sync_aggregate(
[race_member_match, member_group, member_project]).batch_size(4)
member_amount_map = get_red_packet_info(race_cid, start_date, end_date)
red_member_cid_list = member_amount_map.keys()
member_no_history_list = []
count = 0
while True:
try:
stat = member_without_history.next()
count += 1
if stat.member_cid in red_member_cid_list:
continue
# 根据member_cid查对应的member信息
temp_member = Member.sync_get_by_cid(stat.member_cid)
if not temp_member:
print("no history未找到对应member_cid:%s" % stat.member_cid)
continue
info_special = MemberStatisticInfo()
info_special.is_special_user = 1
info_special.race_cid = race_cid
info_special.member_cid = stat.member_cid
info_special.daily_code = format(stat.created_dt, '%Y%m%d')
info_special.nick_name = temp_member.nick_name
info_special.open_id = temp_member.open_id
if stat.mobile:
info_special.mobile = stat.mobile
else:
info_special.mobile = temp_member.mobile
info_special.first_time_login = temp_member.created_dt
info_special.enter_times = 1
info_special.answer_times = 0
info_special.true_answer_times = 0
info_special.is_final_passed = 0
info_special.is_new_user = 1
info_special.grant_red_packet_amount = 0.0
info_special.grant_red_packet_count = 0
info_special.draw_red_packet_count = 0
info_special.draw_red_packet_amount = 0.0
info_special.province = stat.auth_address.get('province')
info_special.city = stat.auth_address.get('city')
info_special.district = stat.auth_address.get('district')
info_special.town = stat.auth_address.get('town')
info_special.check_point_cid = stat.race_check_point_cid
if stat.race_check_point_cid:
info_special.check_point_index = _checkpoint_map[
stat.race_check_point_cid]
else:
info_special.check_point_index = 1
if stat.company_cid:
company = Company.sync_get_by_cid(stat.company_cid)
info_special.company_cid = company.cid
info_special.company_name = company.title
member_no_history_list.append(info_special)
# logger.info("Success without history: member_cid:%s is_final_Pass:%s" % (info_special.member_cid,info_special.is_final_passed))
if len(member_no_history_list) % 500 == 0:
MemberStatisticInfo.sync_insert_many(member_no_history_list)
member_no_history_list = []
except StopIteration:
break
except CursorNotFound:
member_without_history = RaceMapping.sync_aggregate(
[race_member_match, member_group,
member_project]).skip(count).batch_size(4)
except Exception as e:
logger.info(
"Fail: without history daily_code:%s,member_cid:%s,race_cid: %s"
% (info_special.daily_code, info_special.member_cid, race_cid))
if len(member_no_history_list) > 0:
MemberStatisticInfo.sync_insert_many(member_no_history_list)
def daily_member_statistic(race_cid, daily_code):
"""
统计每日活动下的会员
:param race_cid:
:param daily_code:
:return:
"""
start_date = str2datetime(daily_code, '%Y%m%d').replace(hour=0,
minute=0,
second=0,
microsecond=0)
end_date = start_date + datetime.timedelta(days=1)
check_point_cids, _checkpoint_map = get_checkpoint_cid_list(race_cid)
# 当日答题记录
history_match = MatchStage({
'created_dt': {
'$gte': start_date,
'$lt': end_date
},
'check_point_cid': {
'$in': check_point_cids
}
})
history_group = GroupStage({'member_cid': '$member_cid'},
enter_times={'$sum': 1},
results_list={'$push': '$result'})
history_project = ProjectStage(
**{
'member_cid': '$_id.member_cid',
'results_list': '$results_list',
'enter_times': '$enter_times'
})
# 红包信息
member_amount_map = get_red_packet_info(race_cid, start_date, end_date)
# member_cid: amount
check_point_history_cursor = MemberCheckPointHistory.sync_aggregate(
[history_match, history_group, history_project],
allowDiskUse=True).batch_size(4)
member_statistic_list = []
count = 0
while True:
try:
cursor = check_point_history_cursor.next()
count += 1
# 根据member_cid查对应的member信息
temp_member = Member.sync_get_by_cid(cursor.member_cid)
if not temp_member:
print("normal未找到对应member_cid:%s" % cursor.member_cid)
continue
info = MemberStatisticInfo()
info.daily_code = daily_code
info.race_cid = race_cid
# 会员信息
info.member_cid = cursor.member_cid
info.nick_name = temp_member.nick_name
info.open_id = temp_member.open_id
info.mobile = temp_member.mobile
info.first_time_login = temp_member.created_dt
info.enter_times = cursor.enter_times
# 答题数量、正确数
for results in cursor.results_list:
info.answer_times += len(results)
info.true_answer_times += len([
result for result in results if result.get('true_answer')
])
# race_mapping相关信息,地理位置
has_race_mapping = get_race_mapping_info(info)
if not has_race_mapping:
print("normal未找到对应race_mapping,race_cid:%s,member_cid:%s" %
(info.race_cid, info.member_cid))
continue
# 是否为当日新用户
is_new_user(info)
# 最后一关
is_final_pass(info)
# 红包信息
try:
value = member_amount_map.pop(info.member_cid)
except KeyError:
value = None
if not value:
value = {
'grant_count': 0,
'grant_amount': 0,
'draw_count': 0,
'draw_amount': 0
}
info.grant_red_packet_amount = value.get('grant_amount')
info.grant_red_packet_count = value.get('grant_count')
info.draw_red_packet_amount = value.get('draw_amount')
info.draw_red_packet_count = value.get('draw_count')
# 保存记录
member_statistic_list.append(info)
# logger.info("Success: member_cid:%s is_final_Pass:%s is_new:%s" % (
# info.member_cid, info.is_final_passed, info.is_new_user))
print("Success: member_cid:%s is_final_Pass:%s is_new:%s" %
(info.member_cid, info.is_final_passed, info.is_new_user))
if len(member_statistic_list) % 500 == 0:
MemberStatisticInfo.sync_insert_many(member_statistic_list)
member_statistic_list = []
except StopIteration:
break
except CursorNotFound:
check_point_history_cursor = MemberCheckPointHistory.sync_aggregate(
[history_match, history_group,
history_project]).skip(count).batch_size(4)
except Exception as e:
logger.error(e)
logger.info("Fail: daily_code:%s,race_cid: %s" %
(daily_code, race_cid))
member_statistic_list = []
member_statistic_list += insert_from_member_amount_map(
member_amount_map, daily_code, race_cid)
if len(member_statistic_list) > 0:
MemberStatisticInfo.sync_insert_many(member_statistic_list)
async def post(self):
res = dict(code=0)
name = self.get_argument('name')
sex = int(self.get_argument('sex', 1))
email = self.get_argument('email')
mobile = self.get_argument('mobile')
birthday = self.get_argument('birthday')
province_code = self.get_argument('province_code') # 省份编码
city_code = self.get_argument('city_code') # 城市编码
vehicle_code = self.get_argument('vehicle_code') # 车型编号
purchase_datetime = self.get_argument('purchase_datetime') # 车辆购买时间
content = self.get_argument('content') # 备注
status = int(self.get_argument('status', STATUS_USER_ACTIVE))
if name and mobile and email:
# 校验用户名
if mobile:
exist_count = await Member.count(dict(mobile=mobile))
if exist_count:
res['code'] = -2
return res
if email:
email_valid = re.match(
r"^\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$", email)
exist_count = await Member.count(dict(email=email))
if exist_count:
res['code'] = -3
return res
if not email_valid:
res['code'] = -4
return res
if mobile:
member = Member()
member.mobile = mobile
member.code = get_increase_code(KEY_INCREASE_MEMBER_CODE)
if email:
member.email = email
else:
member = Member()
member.email = email
member.code = get_increase_code(KEY_INCREASE_MEMBER_CODE)
if name:
member.name = name
member.sex = sex
if birthday:
member.birthday = str2datetime(birthday, '%Y-%m-%d')
if vehicle_code:
vehicle = await Vehicle.find_one(
dict(code=vehicle_code, record_flag=1))
if vehicle:
member.needless['vehicle_title'] = vehicle.title
member.needless['vehicle_brand'] = vehicle.needless.get(
'brand')
member.needless['vehicle_category'] = vehicle.needless.get(
'category')
member.needless['vehicle_series'] = vehicle.needless.get(
'series')
member.needless['vehicle_config'] = vehicle.needless.get(
'config')
member.needless['vehicle_colour'] = vehicle.needless.get(
'colour')
member.needless['vehicle_displace'] = vehicle.needless.get(
'displace')
else:
for attr in [
'vehicle_brand', 'vehicle_category', 'vehicle_series',
'vehicle_config', 'vehicle_colour', 'vehicle_displace',
'vehicle_title'
]:
try:
member.needless.pop(attr)
except KeyError:
pass
member.content = content
member.status = status
# 冗余信息
if province_code:
province = await AdministrativeDivision.find_one(
dict(parent_code=None, code=province_code))
if province:
member.needless['province'] = province.title
city = await AdministrativeDivision.find_one(
dict(parent_code=province_code, code=city_code))
if city:
member.needless['city'] = city.title
member.province_code = province_code
member.city_code = city_code
member.source = SOURCE_TYPE_MEMBER_SYSTEM
if vehicle_code:
if vehicle_code:
vehicle = await Vehicle.find_one(
dict(code=vehicle_code, record_flag=1))
if vehicle:
member.needless['vehicle_title'] = vehicle.title
member.needless[
'vehicle_brand'] = vehicle.needless.get('brand')
member.needless[
'vehicle_category'] = vehicle.needless.get(
'category')
member.needless[
'vehicle_series'] = vehicle.needless.get('series')
member.needless[
'vehicle_config'] = vehicle.needless.get('config')
member.needless[
'vehicle_colour'] = vehicle.needless.get('colour')
member.needless[
'vehicle_displace'] = vehicle.needless.get(
'displace')
member.vehicle_code = vehicle_code
if purchase_datetime:
member.purchase_datetime = str2datetime(
purchase_datetime, '%Y-%m')
member.status = status
member_id = await member.save()
res['code'] = 1
res['member_id'] = member_id
else:
res['code'] = -1
return res
async def post(self):
r_dict = {'code': 0}
race_cid = self.get_argument('race_cid')
race = await Race.get_by_cid(race_cid)
if not race:
r_dict['code'] = -1
return r_dict
code = self.get_argument('code')
title = self.get_argument('title')
start_dt = self.get_argument('start_dt')
end_dt = self.get_argument('end_dt')
if not code or not title or not start_dt or not end_dt:
r_dict['code'] = -2
return r_dict
if await Race.count(dict(code=code)) > 0:
r_dict['code'] = -5
return r_dict
if await Race.count(dict(title=title)) > 0:
r_dict['code'] = -6
return r_dict
new_cid = get_new_cid()
try:
new_race = copy.deepcopy(race)
new_race.cid = new_cid
new_race.code = code
new_race.title = title
new_race.start_datetime = str2datetime(start_dt)
new_race.end_datetime = str2datetime(end_dt)
new_race.status = STATUS_RACE_INACTIVE
new_race.owner_list = [self.current_user.cid]
await Race.insert_many([new_race])
# 开始复制
await do_copy_subject_refer(race_cid, new_race.cid,
self.current_user.cid)
await do_copy_red_packet_rule(race_cid, new_race.cid,
self.current_user.cid)
await do_copy_choice_rule(race_cid, new_race.cid,
self.current_user.cid)
await do_copy_checkpoint(race_cid, new_race.cid,
self.current_user.cid)
await do_copy_subject_bank(race_cid, new_race.cid,
self.current_user.cid)
await do_copy_company(race_cid, new_race.cid,
self.current_user.cid)
await do_copy_red_packet_box(race_cid, new_race.cid,
self.current_user.cid)
r_dict['code'] = 1
except Exception:
logger.error(traceback.format_exc())
await Race.delete_many({'cid': new_cid})
await RaceSubjectRefer.delete_many({'race_cid': new_cid})
await RaceSubjectChoiceRules.delete_many({'race_cid': new_cid})
await RaceSubjectBanks.delete_many({'race_cid': new_cid})
await RedPacketItemSetting.delete_many({'race_cid': new_cid})
await Company.delete_many({'race_cid': new_cid})
await RedPacketConf.delete_many({'race_cid': new_cid})
await RedPacketRule.delete_many({'race_cid': new_cid})
await RaceGameCheckPoint.delete_many({'race_cid': new_cid})
finally:
pass
return r_dict
async def get(self):
pipelines = list()
# 展示当前用户可看的竞赛活动
user = self.current_user
# 隐藏了筛选条件，编辑，状态变更等按钮，不是超级管理员的情况下
button_hide = False
# 不是超管的情况
if not user.superuser:
if PERMISSION_TYPE_MINIAPP_RACE_ADD_OR_EDIT_MANAGEMENT not in user.permission_code_list:
button_hide = True
city_title = user.city
# 省级账号可以看到该省和该省下面所有的市的活动
province_title = user.province
try:
if province_title and not city_title:
province = await AdministrativeDivision.find_one({
'title':
province_title,
'record_flag':
1
})
if province:
# 该省下面的所有市
city_list = await AdministrativeDivision.find({
'parent_code':
province.code
}).to_list(None)
city_code_list = [
city.code for city in city_list if city_list
]
pipelines.append(
MatchStage({
"$or": [{
"city_code": {
'$in': city_code_list
}
}, {
'province_code': province.code
}]
}))
# 市级账号只能看到该市的活动
elif city_title:
city = await AdministrativeDivision.find_one({
'title': {
'$regex': city_title
},
'record_flag':
1
})
if city:
pipelines.append(MatchStage({"city_code": city.code}))
except Exception as e:
logger.error(traceback.format_exc())
search_keywords = self.get_argument('search_keywords', '')
search_date = self.get_argument('search_date', '')
sort_type = int(self.get_argument('sort_type', 1))
search_province = self.get_argument('search_province', '')
search_city = self.get_argument('search_city', '')
page = int(self.get_argument('page', 1))
per_page_quantity = int(self.get_argument('per_page_quantity', 10))
if search_keywords:
pipelines.append(
MatchStage({
'$or': [{
'code': {
'$regex': search_keywords
}
}, {
'title': {
'$regex': search_keywords
}
}]
}))
if search_date:
try:
start_date_str, end_date_str = search_date.split(' - ')
start_datetime = str2datetime(start_date_str)
end_datetime = str2datetime(end_date_str)
pipelines.append(
MatchStage({
'start_datetime': {
'$gte': start_datetime
},
'end_datetime': {
'$lte': end_datetime
}
}))
except ValueError or TypeError:
logger.error(traceback.format_exc())
search_date = ''
sort_list = []
if sort_type:
if int(sort_type) == 1:
sort_list.append('-start_datetime')
if int(sort_type) == 2:
sort_list.append('start_datetime')
left_pipes = [
LookupStage(AdministrativeDivision, 'province_code', 'code',
'province_list'),
LookupStage(AdministrativeDivision, 'city_code', 'code',
'city_list'),
]
try:
query_param = {'record_flag': 1}
if search_province:
query_param['province_code'] = search_province
if search_city:
query_param['city_code'] = search_city
page_url = '%s?page=$page&per_page_quantity=%s&search_keywords=%s&search_date=%s&sort_type=%s&' \
'search_province=%s&search_city=%s' % (
self.reverse_url("backoffice_race_list"), per_page_quantity, search_keywords, search_date,
sort_type, search_province, search_city)
paging = Paging(page_url,
Race,
current_page=page,
items_per_page=per_page_quantity,
pipeline_stages=pipelines,
left_pipeline_stages=left_pipes,
sort=sort_list,
**query_param)
await paging.pager()
city_list = None
if search_province:
city_list = await AdministrativeDivision.find(
dict(parent_code=search_province)).to_list(None)
province_list = await AdministrativeDivision.find(
dict(parent_code=None)).to_list(None)
now = datetime.now()
except Exception:
logger.error(traceback.format_exc())
return locals()
def do_init():
"""
初始该活动数据
:return:
"""
cache_key = 'race_report_script'
RedisCache.delete(cache_key)
cp_map = get_all_race_checkpoint_map()
last_map = get_all_last_checkpoint()
with open('./chekpt_history.csv', encoding='utf-8') as f:
csv_reader = csv.reader(f)
for index, line in enumerate(csv_reader):
try:
if index == 0:
continue
# member_cid,check_point_cid,status,created_dt
his = MemberCheckPointHistory()
his.member_cid = line[0]
his.check_point_cid = line[1]
his.status = line[2]
line3 = line[3].replace('T', ' ').split('.')[0]
c_dt = str2datetime(line3)
his.created_dt = c_dt
mapping = RaceMapping.sync_find_one({
'member_cid':
his.member_cid,
'race_cid':
cp_map.get(his.check_point_cid)
})
member = Member.sync_get_by_cid(his.member_cid)
auth_address = mapping.auth_address if mapping else None
if not auth_address:
continue
race_cid = cp_map[his.check_point_cid]
daily_code = __get_daily_code(his.created_dt)
param = {
'race_cid': race_cid,
'province': auth_address.get('province'),
'city': auth_address.get('city'),
'district': auth_address.get('district'),
'town': auth_address.get('town'),
'sex': member.sex,
'education': member.education,
'category': member.category,
'daily_code': daily_code,
'company_cid': mapping.company_cid
}
stat = ReportRacePeopleStatisticsTemp.sync_find_one(
param, read_preference=ReadPreference.PRIMARY)
if not stat:
stat = ReportRacePeopleStatisticsTemp(**param)
stat.created_dt = his.created_dt
stat.total_num += 1
# 初次通关
if his.check_point_cid == last_map[
race_cid] and his.status == STATUS_RESULT_CHECK_POINT_WIN and RedisCache.hget(
cache_key, member.cid) is None:
stat.pass_num += 1
RedisCache.hset(cache_key, member.cid, 1)
# 当日人数
day_member_string = md5(daily_code + member.cid)
if RedisCache.hget(cache_key, day_member_string) is None:
RedisCache.hset(cache_key, day_member_string, 1)
stat.people_num += 1
# # 当日新增人数
# old_his = MemberCheckPointHistory.sync_find_one({'member_cid': member.cid, 'created_dt': {
# '$lt': his.updated_dt.replace(hour=0, minute=0, second=0, microsecond=0)}})
# if not old_his:
# stat.incre_people += 1
stat.updated_dt = his.created_dt
stat.sync_save()
print('has exec %s' % index)
except Exception:
print(traceback.format_exc())
