def query():
amount = 0
miss_members = set()
with open('./rdp.txt', 'r') as f:
for line in f.readlines():
rid = line.replace('\n', '')
box = RedPacketBox.sync_find_one({'redpkt_rid': rid})
if not box:
try:
race_cid, member_cid, redpacket_cid = line.split('-')
box = RedPacketBox.sync_get_by_cid(redpacket_cid)
if not box:
print('miss box', rid)
member = Member.sync_get_by_cid(member_cid)
if member:
miss_members.add(member.nick_name)
else:
print('miss member', member_cid)
continue
except ValueError:
print('miss box', rid)
continue
if box.draw_status == STATUS_REDPACKET_NOT_AWARDED:
box.draw_status = 0
box.sync_save()
print('not draw', rid)
else:
print('success ', rid)
amount += box.award_amount
print(amount)
print('-----')
print(len(miss_members), miss_members)
def do_resend():
new_rule_cid = 'CC5C14D38A130733D94DACF109D337C0'
has_money_box = RedPacketBox.sync_find_one({'rule_cid': new_rule_cid})
print(has_money_box)
cursor = RedPacketBox.sync_find({
'rule_cid': "1F22A64622D5DC2A5D8660C809550F38",
'award_cid': None
})
index = 0
while True:
try:
box: RedPacketBox = cursor.next()
box.award_cid = has_money_box.award_cid
box.award_amount = has_money_box.award_amount
box.award_msg = has_money_box.award_msg
box.draw_dt = None
box.draw_status = None
box.rule_cid = new_rule_cid
box.needless['msg'] = 'resend'
box.sync_save()
index += 1
print('has resend', index)
add_notice(box.member_cid,
box.race_cid,
box.checkpoint_cid,
msg_type=TYPE_MSG_DRAW,
redpkt_box=box)
except StopIteration:
break
def statistic_daily_award_of_check_point(check_point_cid: str, daily_code):
"""
统计每一关每天参与人数、抽奖人数、中奖人数
:param check_point_cid: 关卡cid
:return:
"""
# print("正在统计关卡:", check_point.alias, check_point.cid)
match = {'check_point_cid': check_point_cid, 'record_flag': 1}
red_match = {'checkpoint_cid': check_point_cid}
s_date = str2datetime(daily_code, date_format='%Y%m%d').replace(hour=0,
minute=0,
second=0)
e_date = s_date + datetime.timedelta(days=1)
match['fight_datetime'] = {'$gte': s_date, '$lt': e_date}
red_match['draw_dt'] = {'$gte': s_date, '$lt': e_date}
# 参与人数
attend_member = MemberCheckPointHistory.sync_distinct("member_cid", match)
attend_count = len(attend_member)
# 当前关卡的红包
current_checkpoint_reds = RedPacketBox.sync_find(
red_match, read_preference=ReadPreference.PRIMARY).batch_size(2000)
lottery_count = 0
win_count = 0
for current_checkpoint_red in current_checkpoint_reds:
lottery_count += 1
if current_checkpoint_red.award_cid:
win_count += 1
daily_award = DailyAward(daily_code, attend_count, lottery_count,
win_count)
# print(check_point_cid, daily_award)
return daily_award
def export_redpacket(race_cid):
"""
:param race_cid:
:return:
"""
box_list = set()
cursor = RedPacketBox.sync_find(
{
'race_cid': race_cid,
'award_cid': {
'$ne': None
},
'member_cid': {
"$ne": None
}
},
read_preference=ReadPreference.PRIMARY).batch_size(2048)
index = 0
while True:
try:
data: RedPacketBox = cursor.next()
box_list.add('%s-%s' % (data.checkpoint_cid, data.member_cid))
index += 1
print('has exec', index)
except StopIteration:
break
with open('./box-%s.json' % race_cid, 'w', encoding='utf-8') as f:
dist_list = json.dumps(list(box_list), ensure_ascii=False)
f.write(dist_list)
def get_member_info(thread_num, race_member_list, checkPointCidList, lastCheckPoint, checkPointMap):
exportedMemberList = []
for race_member in race_member_list:
# raceMapping = race_member.race_list[0]
raceMapping = race_member
race_cid = raceMapping.race_cid
member = race_member.member_list[0]
memberCid = member.cid
red_match = MatchStage(
{'race_cid': race_cid, 'member_cid': memberCid, 'draw_status': 0, 'draw_dt': {'$ne': None},
'award_cid': {'$ne': None}, 'record_flag': 1})
red_project = ProjectStage(**{"member_cid": 1, "award_amount": 1})
red_group = GroupStage('member_cid', count={'$sum': 1}, amount={'$sum': '$award_amount'})
# redPacketsOfMember = RedPacketBox.sync_find(
# {'race_cid': race_cid, 'member_cid': memberCid, 'award_cid': {'$ne': None}, 'record_flag': 1})
# eachAmountOfRedPacket = [redPacket.award_amount for redPacket in redPacketsOfMember]
redPacketsOfMemberCursor = RedPacketBox.sync_aggregate([red_match, red_project, red_group]).batch_size(50)
exportedMember = MemberInfoExportedModel()
exportedMember.open_id = member.open_id
exportedMember.member_cid = memberCid
exportedMember.nick = member.nick_name
exportedMember.firstTimeOfEnroll = member.created_dt
city = raceMapping.auth_address.get('city', '')
exportedMember.city = city if not city is None else ''
district = raceMapping.auth_address.get('district', '')
exportedMember.district = district if not district is None else ''
mobile = getattr(raceMapping, 'mobile', '')
if mobile is None:
exportedMember.mobile = member.mobile
else:
exportedMember.mobile = mobile
check_point_cid = getattr(raceMapping, 'race_check_point_cid', None)
if check_point_cid is None:
exportedMember.currentCheckPoint = "1"
elif check_point_cid == lastCheckPoint:
exportedMember.currentCheckPoint = "已通关"
else:
exportedMember.currentCheckPoint = str(checkPointMap[check_point_cid])
answerTimes = MemberCheckPointHistory.sync_find(
{'member_cid': memberCid, 'check_point_cid': {'$in': checkPointCidList}, 'record_flag': 1}).to_list(None)
exportedMember.answerTimes = len(answerTimes)
try:
redPacketsOfMember = redPacketsOfMemberCursor.next()
if redPacketsOfMember:
exportedMember.totalNumOfRedPacket = redPacketsOfMember.count
exportedMember.totalAmountOfRedPacket = round(redPacketsOfMember.amount, 2)
except StopIteration:
pass
exportedMemberList.append(exportedMember)
print(thread_num, member.cid, exportedMember.nick, exportedMember.city, exportedMember.district,
exportedMember.mobile,
exportedMember.currentCheckPoint, exportedMember.answerTimes, exportedMember.totalNumOfRedPacket,
exportedMember.totalAmountOfRedPacket)
return exportedMemberList
def get_red_packet_info(race_cid, start_date, end_date):
"""
统计当日该会员参加活动时，红包领取数量、领取金额、发放数量
:param race_cid:
:param start_date:
:param end_date:
:return:
"""
cursor = RedPacketBox.sync_aggregate([
MatchStage({
'draw_dt': {
"$gte": start_date,
"$lt": end_date
},
'race_cid': race_cid,
'award_cid': {
'$ne': None
}
}),
GroupStage({
'member_cid': '$member_cid',
'draw_status': '$draw_status'
},
amount={'$sum': '$award_amount'},
sum={"$sum": 1})
])
ret = {}
while True:
try:
data = cursor.next()
_member = data.id.get('member_cid')
_status = data.id.get('draw_status')
_value = ret.get(_member)
if not _value:
_value = {
'grant_count': 0,
'grant_amount': 0,
'draw_count': 0,
'draw_amount': 0
}
_value['grant_count'] += data.sum
_value['grant_amount'] += data.amount
if _status == 0:
_value['draw_count'] += data.sum
_value['draw_amount'] += data.amount
ret[_member] = _value
except StopIteration:
break
return ret
def get_all_miss_history_box(race_cid):
"""
获取没有历史记录的红包游标
:param race_cid
:return:
"""
match_stage = MatchStage({'draw_status': 0})
if race_cid:
match_stage = MatchStage({'draw_status': 0, 'race_cid': race_cid})
cursor = RedPacketBox.sync_aggregate(stage_list=[
match_stage,
LookupStage(MemberCheckPointHistory,
as_list_name='history_list',
let={
'member_cid': '$member_cid',
'checkpoint': '$check_point_cid'
},
pipeline=[{
'$match': {
'$expr': {
'$and': [
{
'$eq': ['$member_cid', '$$member_cid']
},
{
'$eq':
['$checkpoint_cid', '$$checkpoint']
},
]
}
}
}]),
MatchStage({'history_list': []}),
])
index = 0
repair_list = []
while True:
try:
box = cursor.next()
repair_list.append('%s-%s-%s' %
(box.cid, box.member_cid, box.checkpoint_cid))
index += 1
except StopIteration:
break
with open('./repair-%s.json' % race_cid, 'w', encoding='utf-8') as f:
dist_list = json.dumps(list(repair_list), ensure_ascii=False)
f.write(dist_list)
return index
async def generate_awards_by_config(config: RedPacketConf):
"""
:param config:
:return:
"""
if not config or config.quantity == 0:
raise Exception('parameter error')
await RedPacketBox.delete_many({'rule_cid': config.rule_cid})
award_list = list()
random_list = []
if config.category == CATEGORY_REDPACKET_RANDOM:
random_list = generate_random_list(config.quantity,
config.total_amount)
index = 0
while index < config.quantity:
index += 1
box = RedPacketBox()
box.race_cid = config.race_cid
box.rule_cid = config.rule_cid
box.award_cid = config.cid
box.award_msg = config.msg
if config.category == CATEGORY_REDPACKET_CONST:
box.award_amount = config.total_amount / config.quantity
if config.category == CATEGORY_REDPACKET_RANDOM:
current_sum_amount = sum(
[award.award_amount for award in award_list])
if current_sum_amount < config.total_amount:
box.award_amount = random_list[len(award_list)]
award_list.append(box)
if len(award_list) > 2000:
await RedPacketBox.insert_many(award_list)
award_list = []
if award_list:
await RedPacketBox.insert_many(award_list)
def do_repair():
cursor = RedPacketBox.sync_find(
{
'needless.repair_flag': 1,
'draw_status': 0
},
read_preference=ReadPreference.PRIMARY).batch_size(128)
for index, data in enumerate(cursor):
history = MemberCheckPointHistory.sync_find_one({
'check_point_cid':
data.checkpoint_cid,
'member_cid':
data.member_cid,
'status':
STATUS_RESULT_CHECK_POINT_WIN
})
if not history:
print('-- miss --', index, data.cid)
continue
data.draw_dt = history.created_dt
data.sync_save()
print('-- success --', index)
def start_lottery_queuing(self, member_cid, race_cid, rule: RedPacketRule, checkpoint_cid):
"""
:param self:
:param member_cid:
:param race_cid:
:param rule:
:param checkpoint_cid:
:return:
"""
logger.info('START(%s): lottery queuing, race_cid=%s, checkpoint_cid=%s, member_id=%s ' % (
self.request.id, race_cid, checkpoint_cid, member_cid))
try:
if not (member_cid and race_cid and rule):
raise Exception('There is not member_cid or race_cid or rule')
top_limit, fail_msg = None, None
if rule.category == CATEGORY_REDPACKET_RULE_DIRECT:
conf = RedPacketConf.sync_find_one({'rule_cid': rule.cid})
top_limit = conf.top_limit
fail_msg = conf.over_msg
if rule.category == CATEGORY_REDPACKET_RULE_LOTTERY:
conf = RedPacketBasicSetting.sync_find_one({'rule_cid': rule.cid})
top_limit = conf.top_limit
fail_msg = conf.fail_msg
has_sent_count = RedPacketBox.sync_count({'rule_cid': rule.cid, 'draw_dt': {
'$gte': datetime.now().replace(hour=0, minute=0, second=0),
'$lte': datetime.now().replace(hour=23, minute=59, second=59)}, 'record_flag': 1},
read_preference=ReadPreference.PRIMARY)
msg = None
if has_sent_count >= top_limit:
msg = conf.over_msg
redpkt_box = RedPacketBox()
result = RESULT_RACE_LOTTERY_LOSE_LATE
else:
redpkt_box = RedPacketBox.sync_find_one({'rule_cid': rule.cid, 'draw_dt': None, 'record_flag': 1})
if not redpkt_box:
msg = fail_msg
redpkt_box = RedPacketBox()
result = RESULT_RACE_LOTTERY_LOSE
else:
# 拿到了红包
if not redpkt_box.award_msg:
item_setting = RedPacketItemSetting.sync_find_one({'rule_cid': rule.cid})
msg = item_setting.message
logger.error('item_msg: %s' % msg)
if not redpkt_box.award_cid:
result = RESULT_RACE_LOTTERY_LOSE
redpkt_box.race_cid = race_cid
redpkt_box.rule_cid = rule.cid
redpkt_box.checkpoint_cid = checkpoint_cid
redpkt_box.member_cid = member_cid
redpkt_box.draw_dt = datetime.now()
if msg:
redpkt_box.award_msg = msg
redpkt_box.sync_save()
win = bool(redpkt_box.award_cid)
if win:
result = RESULT_RACE_LOTTERY_WIN
add_notice(member_cid, race_cid, checkpoint_cid, msg_type=TYPE_MSG_DRAW, redpkt_box=redpkt_box)
RedisCache.hset(KEY_RACE_LOTTERY_RESULT.format(checkpoint_cid), member_cid, result)
logger.info(' END (%s): lottery end , checkpoint_cid=%s, member_id=%s, result=%s ' % (
self.request.id, checkpoint_cid, member_cid, result))
except Exception:
logger.error(traceback.format_exc())
logger.error(
'ERROR(%s): lottery error, checkpoint_cid=%s, member_id=%s, rule_cid=%s' % (
self.request.id, checkpoint_cid, member_cid, rule.cid))
def filter_data(race_cid):
"""
:param race_cid:
:return:
"""
with open('./box-%s.json' % race_cid, 'r') as f:
box = set(json.load(f))
with open('./history-%s.json' % race_cid, 'r') as f:
history = set(json.load(f))
with open('./repair-%s.json' % race_cid, 'r') as f:
repair_list = json.load(f)
box = box.difference(history)
history = history.difference(box)
box_dict = {}
for b in box:
_c, _m = b.split('-')
m_list = box_dict.get(_c, [])
m_list.append(_m)
box_dict[_c] = m_list
history_dict = {}
for b in history:
_c, _m = b.split('-')
m_list = history_dict.get(_c, [])
m_list.append(_m)
history_dict[_c] = m_list
print('box key:', len(box_dict))
print('his key:', len(history_dict))
for k, v in box_dict.items():
print(k, len(v))
print('-----------------------------')
for k, v in history_dict.items():
print(k, len(v))
print('----------init finish--------')
print('--------- repair start ------')
fail_box = {}
index = 0
change_list = []
for repair in repair_list:
_box_cid, _member_cid, _checkpoint_cid = repair.split('-')
_box = RedPacketBox.sync_find_one({'cid': _box_cid})
try:
_cid = history_dict[_checkpoint_cid].pop()
while True:
if RedPacketBox.sync_count(
{
'race_cid': race_cid,
'member_cid': _cid
},
read_preference=ReadPreference.PRIMARY) < NUM:
break
else:
_cid = history_dict[_checkpoint_cid].pop()
print(' --- skip ---')
change_msg = '%s-%s-->%s-%s' % (
_box.checkpoint_cid, _box.member_cid, _checkpoint_cid, _cid)
logger.info(change_msg)
_box.member_cid = _cid
_box.needless['repair_flag'] = 1
# _box.draw_dt =
_box.sync_save()
change_list.append(change_msg)
index += 1
print('has deal', index)
except IndexError:
_mm = fail_box.get(_checkpoint_cid, [])
_mm.append(repair)
fail_box[_checkpoint_cid] = _mm
fail_box2 = {}
new_his_map = []
for c, m_list in history_dict.items():
for m in m_list:
new_his_map.append([m, c])
for k, repair_list in fail_box.items():
for repair in repair_list:
pdb.set_trace()
_box_cid, _member_cid, _checkpoint_cid = repair.split('-')
_box = RedPacketBox.sync_find_one({'cid': _box_cid})
try:
member_cid, checkpoint_cid = new_his_map.pop()
while True:
if RedPacketBox.sync_count(
{
'race_cid': race_cid,
'member_cid': member_cid
},
read_preference=ReadPreference.PRIMARY) < NUM:
break
else:
member_cid, checkpoint_cid = new_his_map.pop()
print(' --- skip ---')
change_msg = '%s-%s-->%s-%s' % (_box.checkpoint_cid,
_box.member_cid,
_checkpoint_cid, member_cid)
logger.info(change_msg)
_box.checkpoint_cid = checkpoint_cid
_box.member_cid = member_cid
_box.needless['repair_flag'] = 1
# _box.draw_dt =
_box.sync_save()
change_list.append(change_msg)
index += 1
print('has deal', index)
except IndexError:
_mm = fail_box2.get(_checkpoint_cid, [])
_mm.append(repair)
fail_box2[_checkpoint_cid] = _mm
assert fail_box2 == {}, 'fail box not empty'
def export_race_enter_position(race_cid: str, title):
"""
导出活动下面参与情况
:return:
"""
if not isinstance(race_cid, str):
raise ValueError("race_cid is not str")
now = datetime.datetime.now()
export_time_list = [
now + datetime.timedelta(days=-n) for n in (range(1, 8))
]
export_time_list.sort()
export_time = [
transform_time_format(export_dt) for export_dt in export_time_list
]
export_time.sort()
workbook = xlsxwriter.Workbook(title + today + ".xlsx")
head_list = [
"新增人数", "参与人数", "参与次数", "通关人数", "红包发放数", "红包领取数量", "红包发放金额", "红包领取金额"
]
sheet = workbook.add_worksheet(title + "一周数据")
data_center_format = workbook.add_format({
'valign': 'vcenter',
'align': 'center',
'font_name': 'Microsoft YaHei'
})
sheet.merge_range(0, 0, 1, 0, "城市", data_center_format)
sheet.merge_range(0, 1, 1, 1, "区县", data_center_format)
sheet.write_string(1, 2, "累计人数")
sheet.write_string(1, 3, "累计参与次数")
for i in range(1, 8):
sheet.merge_range(0, 7 * (i - 1) + 4 + i - 1, 0,
7 * (i - 1) + 4 + i - 1 + 7, export_time[i - 1],
data_center_format)
for head in range(7 * (i - 1) + 4 + i - 1,
7 * (i - 1) + 4 + i - 1 + 7 + 1):
index = head - 8 * (i - 1) - 4
sheet.write_string(1, head, head_list[index])
midnight = (datetime.datetime.now()).replace(hour=0,
minute=0,
second=0,
microsecond=0)
race = Race.sync_get_by_cid(race_cid)
# 市级活动，如六安市，扬州市
if race.city_code:
city_code_list = AdministrativeDivision.sync_distinct(
'code', {'code': race.city_code})
# 该活动的所属城市范围
city_name_list = AdministrativeDivision.sync_distinct(
'title', {'code': race.city_code})
else:
city_code_list = AdministrativeDivision.sync_distinct(
'code', {'parent_code': race.province_code})
city_name_list = AdministrativeDivision.sync_distinct(
'title', {'parent_code': race.province_code})
dist_list = []
for city_code in city_code_list:
# 该活动区县的范围
dist_list += AdministrativeDivision.sync_distinct(
'title', {'parent_code': city_code})
# 最基本的人数match
base_quantity_match = {
'race_cid': race_cid,
'auth_address.province': {
'$ne': None
},
"record_flag": 1,
'auth_address.city': {
"$in": city_name_list
},
'auth_address.district': {
"$in": dist_list
}
}
# 最基本的次数match
base_count_match = {
'race_cid': race_cid,
'province': {
'$ne': None
},
"record_$pushflag": 1,
'city': {
"$in": city_name_list
},
'district': {
"$in": dist_list
}
}
# 最基本的参与人数match
base_enter_quantity_match = {
'race_cid': race_cid,
'auth_address.province': {
'$ne': None
},
"record_flag": 1,
'auth_address.city': {
"$in": city_name_list
},
'auth_address.district': {
"$in": dist_list
}
}
# 累计人数
quantity_match_stage = MatchStage({
'race_cid': race_cid,
'auth_address.province': {
'$ne': None
},
"record_flag": 1,
'auth_address.city': {
"$in": city_name_list
},
'auth_address.district': {
"$in": dist_list
},
'created_dt': {
'$lte': midnight
}
})
# 累计次数
daily_code = datetime2str(datetime.datetime.now(), date_format='%Y%m%d')
count_match = {
'race_cid': race_cid,
'province': {
'$ne': None
},
'city': {
'$in': city_name_list
},
'district': {
'$in': dist_list
},
'daily_code': {
'$lt': daily_code
}
}
lookup_stage = LookupStage(Member, 'member_cid', 'cid', 'member_list')
address_sort = SortStage([('_id.city', ASC), ('_id.district', ASC)])
quantity_list = RaceMapping.sync_aggregate([
quantity_match_stage, lookup_stage,
ProjectStage(**{
'auth_address': "$auth_address",
"member_list": "$member_list"
}),
MatchStage({'member_list': {
'$ne': []
}}),
GroupStage(
{
'district': '$auth_address.district',
'city': '$auth_address.city'
},
sum={'$sum': 1}), address_sort
])
count_list = ReportRacePeopleStatistics.sync_aggregate([
MatchStage(count_match),
GroupStage({
'city': '$city',
'district': '$district'
},
sum={'$sum': "$total_num"})
])
dis_map = {}
quantity = 0
for index, address in enumerate(quantity_list):
quantity += address.sum
sheet.write_string(index + 2, 0, address.id.get('city'))
sheet.write_string(index + 2, 1, address.id.get('district'))
sheet.write_number(index + 2, 2, address.sum)
if address.id.get('district') not in dis_map:
dis_map[address.id.get('district')] = index + 2
else:
dis_map[address.id.get('district')] += index + 2
count_map = {}
for count in count_list:
district = count.id.get('district')
if district not in count_map:
count_map[district] = count.sum
print(count_map, 'count')
print(dis_map, 'dis_map')
for k, v in count_map.items():
position = dis_map.get(k)
if position:
sheet.write_number(position, 3, v)
# 有答题次数，没有人数的情况，跳过
else:
continue
# 时间与地区人数的字典 {'20190702': {'六安市-叶集区': 20}, '20190703': {'六安市-舒城县': 30}}
quantity_time_dict = {}
# 一个星期的人数的数据
base_quantity_match['created_dt'] = {
'$gte':
export_time_list[0].replace(hour=0, minute=0, second=0),
'$lte':
(export_time_list[-1] + datetime.timedelta(days=1)).replace(hour=0,
minute=0,
second=0)
}
one_week_quantity_list = RaceMapping.sync_aggregate([
MatchStage(base_quantity_match), lookup_stage,
ProjectStage(
**{
'daily_code': {
'$dateToString': {
'format': "%Y%m%d",
'date': "$created_dt"
}
},
'auth_address': "$auth_address",
"member_list": "$member_list"
}),
MatchStage({'member_list': {
'$ne': []
}}),
GroupStage(
{
"daily_code": "$daily_code",
'district': '$auth_address.district',
'city': '$auth_address.city'
},
sum={'$sum': 1}), address_sort
])
for quantity_data in one_week_quantity_list:
daily = quantity_data.id.get('daily_code')
city = quantity_data.id.get('city')
district = quantity_data.id.get('district')
if city and district:
if daily not in quantity_time_dict:
temp_dict = {}
temp_dict["{city}-{district}".format(
city=city, district=district)] = quantity_data.sum
quantity_time_dict[daily] = temp_dict
else:
quantity_time_dict[daily].update({
"{city}-{district}".format(city=city, district=district):
quantity_data.sum
})
else:
continue
# 每日参与人数一周的数据
# 时间与地区人数的字典 {'20190702': {'六安市-叶集区': 20}, '20190703': {'六安市-舒城县': 30}}
base_enter_quantity_match['updated_dt'] = {
'$gte':
export_time_list[0].replace(hour=0, minute=0, second=0),
'$lte':
(export_time_list[-1] + datetime.timedelta(days=1)).replace(hour=0,
minute=0,
second=0)
}
enter_quantity_time_dict = {}
one_week_enter_quantity_list = RaceMapping.sync_aggregate([
MatchStage(base_enter_quantity_match), lookup_stage,
ProjectStage(
**{
'daily_code': {
'$dateToString': {
'format': "%Y%m%d",
'date': "$updated_dt"
}
},
'auth_address': "$auth_address",
"member_list": "$member_list"
}),
MatchStage({'member_list': {
'$ne': []
}}),
GroupStage(
{
"daily_code": "$daily_code",
'district': '$auth_address.district',
'city': '$auth_address.city'
},
sum={'$sum': 1}), address_sort
])
for quantity_data in one_week_enter_quantity_list:
daily = quantity_data.id.get('daily_code')
city = quantity_data.id.get('city')
district = quantity_data.id.get('district')
if city and district:
if daily not in enter_quantity_time_dict:
temp_dict = {}
temp_dict["{city}-{district}".format(
city=city, district=district)] = quantity_data.sum
enter_quantity_time_dict[daily] = temp_dict
else:
enter_quantity_time_dict[daily].update({
"{city}-{district}".format(city=city, district=district):
quantity_data.sum
})
else:
continue
# print(enter_quantity_time_dict, 'enter_quantity')
# 每日新增参与次数一周的数据
base_count_match['created_dt'] = {
'$gte':
export_time_list[0].replace(hour=0, minute=0, second=0),
'$lte':
(export_time_list[-1] + datetime.timedelta(days=1)).replace(hour=0,
minute=0,
second=0)
}
one_week_count_list = ReportRacePeopleStatistics.sync_aggregate([
MatchStage(base_count_match),
GroupStage(
{
'city': '$city',
'district': '$district',
'daily_code': '$daily_code'
},
sum={'$sum': "$total_num"})
])
# 时间与地区次数的字典 {'20190702': {'六安市-叶集区': 20}, '20190703': {'六安市-舒城县': 30}}
count_time_dict = {}
for quantity_data in one_week_count_list:
daily = quantity_data.id.get('daily_code')
city = quantity_data.id.get('city')
district = quantity_data.id.get('district')
if city and district:
if daily not in count_time_dict:
temp_dict = {}
temp_dict["{city}-{district}".format(
city=city, district=district)] = quantity_data.sum
count_time_dict[daily] = temp_dict
else:
count_time_dict[daily].update({
"{city}-{district}".format(city=city, district=district):
quantity_data.sum
})
else:
continue
# 一周通关的人数
# {'20190702': {'六安市-叶集区': 20, "六安市-舒城县": 15}, '20190703': {'六安市-舒城县': 30}}
pass_quantity_time_dict = {}
last_checkpoint_cid, _, _, _ = get_export_param(race_cid) # 拿到最后一关的cid
pass_match_stage = MatchStage({
'check_point_cid': last_checkpoint_cid,
'status': 1,
'record_flag': 1,
'created_dt': {
'$gte':
export_time_list[0].replace(hour=0, minute=0, second=0),
'$lte': (export_time_list[-1] +
datetime.timedelta(days=1)).replace(hour=0,
minute=0,
second=0)
}
})
check_point_cursor = MemberCheckPointHistory.sync_aggregate(
[pass_match_stage, lookup_stage])
member_cid_list = []
while True:
try:
check_point = check_point_cursor.next()
if check_point.member_list:
member = check_point.member_list[0]
race_mapping = RaceMapping.sync_find_one({
'race_cid':
race_cid,
'member_cid':
member.cid
})
if race_mapping:
if race_mapping.auth_address and race_mapping.auth_address.get(
'province'):
if check_point.member_cid not in member_cid_list:
district = race_mapping.auth_address.get(
'district')
city = race_mapping.auth_address.get('city')
if district and city:
member_cid_list.append(check_point.member_cid)
created_time = format(check_point.created_dt,
"%Y%m%d")
if created_time not in pass_quantity_time_dict:
pass_quantity_time_dict[created_time] = {
"{city}-{district}".format(city=city,
district=district):
1
}
else:
city_district = "{city}-{district}".format(
city=city, district=district)
v_dict = pass_quantity_time_dict.get(
created_time)
if city_district in v_dict:
v_dict[city_district] += 1
else:
v_dict[city_district] = 1
else:
continue
except StopIteration:
break
except Exception as e:
raise e
# 每日新增人数在excel的位置
quantity_increase_position_list = [4 + 8 * (i - 1) for i in range(1, 8)]
# 每日参与人数在excel的位置
quantity_enter_position_list = [5 + 8 * (i - 1) for i in range(1, 8)]
# 每日参与次数在excel的位置
count_enter_position_list = [6 + 8 * (i - 1) for i in range(1, 8)]
# 每日通关次数在excel的位置
pass_enter_position_list = [7 + 8 * (i - 1) for i in range(1, 8)]
# 红包发放数量在excel的位置
red_give_position_list = [8 + 8 * (i - 1) for i in range(1, 8)]
# 红包发放金额在excel的位置
red_give_amount_list = [10 + 8 * (i - 1) for i in range(1, 8)]
# 红包领取数量在excel的位置
red_receive_position_list = [9 + 8 * (i - 1) for i in range(1, 8)]
# 红包领取金额在excel的位置
red_receive_amount_list = [11 + 8 * (i - 1) for i in range(1, 8)]
print(quantity_increase_position_list)
print(dis_map, 'dis_map6')
print(quantity, 'quan')
print(quantity_time_dict, 'quantity_time')
# 填充每日新增人数
write_excel_data(sheet,
quantity_time_dict,
dis_map,
export_time,
quantity_increase_position_list,
save=None)
# excel 填充每日参与人数
write_excel_data(sheet,
enter_quantity_time_dict,
dis_map,
export_time,
quantity_enter_position_list,
save=None)
# excel 填充每日参与次数
write_excel_data(sheet,
count_time_dict,
dis_map,
export_time,
count_enter_position_list,
save=None)
# excel 填充每日通关人数
write_excel_data(sheet,
pass_quantity_time_dict,
dis_map,
export_time,
pass_enter_position_list,
save=None)
red_give_dict = {}
red_give_amount_dict = {}
# 红包发放个数
red_give_out_match = {
"race_cid": race_cid,
'record_flag': 1,
"award_cid": {
'$ne': None
},
'member_cid': {
'$ne': None
},
'draw_dt': {
'$gte':
export_time_list[0].replace(hour=0, minute=0, second=0),
'$lte': (export_time_list[-1] +
datetime.timedelta(days=1)).replace(hour=0,
minute=0,
second=0)
}
}
red_give_out_cursor = RedPacketBox.sync_aggregate([
MatchStage(red_give_out_match), lookup_stage,
ProjectStage(
**{
'daily_code': {
'$dateToString': {
'format': "%Y%m%d",
'date': "$draw_dt"
}
},
"member_cid": '$member_cid',
"member_list": "$member_list",
"award_amount": '$award_amount',
}),
MatchStage({'member_list': {
'$ne': []
}}),
GroupStage({"daily_code": "$daily_code"},
amount_list={'$push': '$award_amount'},
cid_list={'$push': '$member_cid'},
sum={'$sum': 1})
])
while True:
try:
red_packet = red_give_out_cursor.next()
if red_packet and red_packet.cid_list:
cid_list = red_packet.cid_list
amount_list = red_packet.amount_list
print(len(amount_list), 'len_m')
print(len(cid_list), 'len')
daily = red_packet.id.get('daily_code')
for cid, amount in zip(cid_list, amount_list):
race_mapping = RaceMapping.sync_find_one({
'race_cid': race_cid,
'member_cid': cid,
'auth_address.city': {
'$in': city_name_list
},
'auth_address.district': {
'$in': dist_list
}
})
if race_mapping:
city = race_mapping.auth_address.get('city')
district = race_mapping.auth_address.get('district')
city_district = "{city}-{district}".format(
city=city, district=district)
if daily not in red_give_dict:
red_give_dict[daily] = {city_district: 1}
else:
v_dict = red_give_dict.get(daily)
if city_district not in v_dict:
v_dict[city_district] = 1
else:
v_dict[city_district] += 1
if daily not in red_give_amount_dict:
red_give_amount_dict[daily] = {
city_district: amount
}
else:
v_dict = red_give_amount_dict.get(daily)
if city_district not in v_dict:
v_dict[city_district] = amount
else:
v_dict[city_district] += amount
except StopIteration:
break
except Exception as e:
raise e
print(red_give_dict, 'dict2')
print(red_give_amount_dict, 'amount-dict2')
# excel 填充每日发放个数
write_excel_data(sheet,
red_give_dict,
dis_map,
export_time,
red_give_position_list,
save=None)
# excel 填充每日领取金额
write_excel_data(sheet,
red_give_amount_dict,
dis_map,
export_time,
red_give_amount_list,
save=2)
# 红包领取个数
red_receive_dict = {}
red_receive_amount_dict = {}
red_receive_match = {
"race_cid": race_cid,
'record_flag': 1,
"award_cid": {
'$ne': None
},
'draw_status': 0,
'member_cid': {
'$ne': None
},
'draw_dt': {
'$gte':
export_time_list[0].replace(hour=0, minute=0, second=0),
'$lte': (export_time_list[-1] +
datetime.timedelta(days=1)).replace(hour=0,
minute=0,
second=0)
}
}
red_receive_cursor = RedPacketBox.sync_aggregate([
MatchStage(red_receive_match), lookup_stage,
ProjectStage(
**{
'daily_code': {
'$dateToString': {
'format': "%Y%m%d",
'date': "$draw_dt"
}
},
"member_cid": '$member_cid',
"member_list": "$member_list",
"award_amount": '$award_amount',
}),
MatchStage({'member_list': {
'$ne': []
}}),
GroupStage({"daily_code": "$daily_code"},
receive_amount_list={'$push': '$award_amount'},
cid_list={'$push': '$member_cid'},
sum={'$sum': 1})
])
while True:
try:
red_packet = red_receive_cursor.next()
if red_packet and red_packet.cid_list:
amount_list = red_packet.receive_amount_list
cid_list = list(set(red_packet.cid_list))
print(len(cid_list), 'len')
daily = red_packet.id.get('daily_code')
for cid, amount in zip(cid_list, amount_list):
race_mapping = RaceMapping.sync_find_one({
'race_cid': race_cid,
'member_cid': cid,
'auth_address.city': {
'$in': city_name_list
},
'auth_address.district': {
'$in': dist_list
}
})
if race_mapping:
city = race_mapping.auth_address.get('city')
district = race_mapping.auth_address.get('district')
city_district = "{city}-{district}".format(
city=city, district=district)
if daily not in red_receive_dict:
red_receive_dict[daily] = {city_district: 1}
else:
v_dict = red_receive_dict.get(daily)
if city_district not in v_dict:
v_dict[city_district] = 1
else:
v_dict[city_district] += 1
if daily not in red_receive_amount_dict:
red_receive_amount_dict[daily] = {
city_district: amount
}
else:
v_dict = red_receive_amount_dict.get(daily)
if city_district not in v_dict:
v_dict[city_district] = amount
else:
v_dict[city_district] += amount
except StopIteration:
break
except Exception as e:
raise e
print(red_receive_dict, 'receive_cursor')
print(red_receive_amount_dict, 'rece_amount')
# excel 填充每日领取个数
write_excel_data(sheet,
red_receive_dict,
dis_map,
export_time,
red_receive_position_list,
save=None)
# excel 填充每日领取金额
write_excel_data(sheet,
red_receive_amount_dict,
dis_map,
export_time,
red_receive_amount_list,
save=2)
workbook.close()
def do_query(race_cid):
cursor = RedPacketBox.sync_find({
'member_cid': {
'$ne': None
},
'award_cid': {
'$ne': None
},
'draw_status': 1,
'race_cid': race_cid
}).batch_size(128)
index = 0
while True:
try:
box: RedPacketBox = cursor.next()
race = Race.sync_get_by_cid(box.race_cid)
rid = generate_rid(box.race_cid, box.member_cid, box.cid)
json_data = json.dumps(
deal_param(REDPACKET_PLATFORM_QUERY_RESULT_URL,
redpkt_account=race.redpkt_account,
rid=rid))
res = requests.get(REDPACKET_PLATFORM_HOST +
REDPACKET_PLATFORM_QUERY_RESULT_URL,
data=json_data)
res = res.json()
if res.get('data'):
status = res.get('data')[0].get('status')
else:
print('member has not click pick_url')
continue
if status == 2:
print('[RedPacket Drawing] the redpacket info: '
'member_cid: %s, checkpoint_cid: %s.' %
(box.member_cid, box.checkpoint_cid))
if status == 3:
box.draw_status = STATUS_REDPACKET_AWARDED
box.issue_status = STATUS_READPACKET_ISSUE_SUCCESS
box.sync_save()
notice_list = MemberNotice.sync_find({
'member_cid': box.member_cid,
'category': CATEGORY_NOTICE_AWARD,
'status': STATUS_NOTICE_UNREAD,
'checkpoint_cid': box.checkpoint_cid,
'record_flag': 1
}).to_list(None)
for notice in notice_list:
notice.status = STATUS_NOTICE_READ
notice.sync_save()
print(
'[RedPacket Drew] the redpacket info: member_cid: %s, checkpoint_cid: %s.'
% (box.member_cid, box.checkpoint_cid))
break
if status == 5:
box.status = STATUS_REDPACKET_AWARD_FAILED
box.issue_status = STATUS_READPACKET_ISSUE_FAIL
box.sync_save()
break
if status == 6:
logger.warning(
'[RedPacket Waiting] the redpacket info: member_cid: %s, checkpoint_cid: %s.'
% (box.member_cid, box.checkpoint_cid))
if status == 8:
pass
index += 1
print('has query', index)
except StopIteration:
break
(box.cid, box.member_cid, box.checkpoint_cid))
index += 1
except StopIteration:
break
with open('./repair-%s.json' % race_cid, 'w', encoding='utf-8') as f:
dist_list = json.dumps(list(repair_list), ensure_ascii=False)
f.write(dist_list)
return index
if __name__ == '__main__':
print('------ begin ------')
# _cid = "3040737C97F7C7669B04BC39A660065D" # 安徽
_cid = "F742E0C7CA5F7E175844478D74484C29"
print(
'发放红包总数:',
RedPacketBox.sync_count({
'race_cid': _cid,
'award_cid': {
'$ne': None
},
'member_cid': {
'$ne': None
}
}))
print('缺少历史纪录的红包数量：', get_all_miss_history_box(_cid))
print('------ end ------')
def repair_records(race_cid):
"""
单个关卡中存在多个抽奖的记录
:param race_cid:
:return:
"""
cursor = RedPacketBox.sync_aggregate([
MatchStage({'race_cid': race_cid, 'member_cid': {'$ne': None}}),
GroupStage({'member_cid': '$member_cid', 'checkpoint_cid': '$checkpoint_cid'}, count={'$sum': 1},
box_list={'$push': '$cid'}),
MatchStage({'count': {'$gte': 2}})
], read_preference=ReadPreference.PRIMARY, allowDiskUse=True).batch_size(256)
with open('./no-history-members.json', 'r') as f:
member_list = json.load(f)
index = count = 0
ck_member_map = {}
fail_list = []
while True:
try:
data = cursor.next()
box_list = RedPacketBox.sync_find({'cid': {'$in': data.box_list}}).to_list(None)
yes_list = list(filter(lambda x: x.award_cid is not None, box_list)) # draw_status --> award_cid
noo_list = list(filter(lambda x: x.award_cid is None, box_list))
# 对于多余的记录，直接改给没有历史记录的游客
if len(yes_list) == 0:
noo_list.pop()
elif len(yes_list) == 1:
pass
elif len(yes_list) >= 2:
noo_list += list(filter(lambda x: x.draw_status == 1, yes_list))
for noo_box in noo_list:
try:
_m_list = ck_member_map.get(noo_box.checkpoint_cid)
if _m_list == 'empty':
fail_list.append(noo_box.cid)
continue
if not _m_list:
_m_list = copy.deepcopy(member_list)
count += 1
while True:
try:
fit_cid = _m_list.pop()
if RedPacketBox.sync_count(
{'checkpoint_cid': noo_box.checkpoint_cid, 'member_cid': fit_cid},
read_preference=ReadPreference.PRIMARY) == 0:
break
except IndexError:
_m_list = 'empty'
print('empty: %s' % noo_box.checkpoint_cid)
break
ck_member_map[noo_box.checkpoint_cid] = _m_list
if _m_list == 'empty':
continue
if not fit_cid:
fail_list.append(noo_box.cid)
continue
noo_box.member_cid = fit_cid
noo_box.award_cid = None
noo_box.award_amount = 0
noo_box.award_msg = None
noo_box.needless['repair_flag'] = 1
noo_box.sync_save()
index += 1
print('has done', index)
except IndexError:
fail_list.append(noo_box.cid)
except StopIteration:
break
print(index, count)
assert fail_list == [], 'fail list not empty, length: %s' % len(fail_list)
def export_lottery_detail(race_cid: str):
"""
:param race_cid:
:return:
"""
race = Race.sync_find_one({'cid': race_cid})
workbook = xlsxwriter.Workbook('%s中奖人员名单.xlsx' % race.title)
worksheet = workbook.add_worksheet()
_b_dt = datetime.now().replace(2019, 5, 20, 0, 0, 0, 0)
_e_dt = datetime.now()
cursor = RedPacketBox.sync_aggregate(
[
MatchStage({
'draw_dt': {
'$gt': _b_dt,
'$lt': _e_dt
},
'draw_status': 0,
'race_cid': race_cid
}),
LookupStage(Member, 'member_cid', 'cid', 'member_list'),
# LookupStage(RaceMapping, 'member_cid', 'member_cid', 'map_list'),
# LookupStage(RedPacketItemSetting, 'award_cid', 'cid', 'item_list')
],
read_preference=ReadPreference.PRIMARY).batch_size(128)
cols = [
'序号', '时间', '用户OPENID', '微信昵称', '奖品', '奖励金额', '领取状态', '省', '市', '区',
'领取时间', '关卡序号', '获奖时间', '答题时间', '红包id'
]
for col_index, col_name in enumerate(cols):
worksheet.write(0, col_index, col_name)
_row = 1
while True:
try:
box = cursor.next()
worksheet.write_number(_row, 0, _row)
worksheet.write_string(_row, 1, datetime2str(box.draw_dt))
worksheet.write_string(
_row, 2,
box.member_list[0].open_id if box.member_list else '未知')
worksheet.write_string(
_row, 3,
box.member_list[0].nick_name if box.member_list else '未知')
# worksheet.write_string(_row, 4, box.item_list[0].title if box.item_list else '未知')
worksheet.write_number(_row, 5, box.award_amount)
worksheet.write_string(
_row, 6, STATUS_REDPACKET_AWARD_DICT.get(box.draw_status))
# if box.map_list:
# worksheet.write_string(_row, 7, box.map_list[0].auth_address.get('province', ''))
# worksheet.write_string(_row, 8, box.map_list[0].auth_address.get('city', ''))
# worksheet.write_string(_row, 9, box.map_list[0].auth_address.get('district', ''))
worksheet.write_string(_row, 10, datetime2str(box.request_dt))
checkpt = RaceGameCheckPoint.sync_get_by_cid(box.checkpoint_cid)
if checkpt:
worksheet.write_number(_row, 11, checkpt.index)
worksheet.write_string(_row, 12, datetime2str(box.draw_dt))
# his = MemberCheckPointHistory.sync_find_one(
# {'member_cid': box.member_cid, 'check_point_cid': box.checkpoint_cid})
# if his:
# worksheet.write_string(_row, 13, datetime2str(his.created_dt))
worksheet.write_string(_row, 14, box.cid)
print('has exec', _row)
_row += 1
except StopIteration:
break
workbook.close()
def get_lottery_result(self, member: Member, reward_cid, rid):
"""
获取到小程序抽奖结果
:param self:
:param member:
:param reward_cid:
:param rid:
:return:
"""
logger.info(
'START(%s): lottery result query, member_cid=%s, reward_cid=%s, rid=%s '
% (self.request.id, member.cid, reward_cid, rid))
try:
redpacket = RedPacketBox.sync_find_one(
{
'member_cid': member.cid,
'cid': reward_cid,
'record_flag': 1
},
read_preference=ReadPreference.PRIMARY)
race = Race.sync_get_by_cid(redpacket.race_cid)
time_start = time.time()
while time.time() - time_start < 30:
time.sleep(1)
json_data = json.dumps(
deal_param(REDPACKET_PLATFORM_QUERY_RESULT_URL,
redpkt_account=race.redpkt_account,
rid=rid))
res = requests.get(REDPACKET_PLATFORM_HOST +
REDPACKET_PLATFORM_QUERY_RESULT_URL,
data=json_data)
res = res.json()
status = res.get('data')[0].get('status')
if status == 2:
logger.warning('[RedPacket Drawing] the redpacket info: '
'member_cid: %s, checkpoint_cid: %s.' %
(member.cid, redpacket.checkpoint_cid))
if status == 3:
redpacket.draw_status = STATUS_REDPACKET_AWARDED
redpacket.issue_status = STATUS_READPACKET_ISSUE_SUCCESS
redpacket.sync_save()
notice_list = MemberNotice.sync_find({
'member_cid':
member.cid,
'category':
CATEGORY_NOTICE_AWARD,
'status':
STATUS_NOTICE_UNREAD,
'checkpoint_cid':
redpacket.checkpoint_cid,
'record_flag':
1
}).to_list(None)
for notice in notice_list:
notice.status = STATUS_NOTICE_READ
notice.sync_save()
logger.info(
'[RedPacket Drew] the redpacket info: member_cid: %s, checkpoint_cid: %s.'
% (member.cid, redpacket.checkpoint_cid))
break
if status == 5:
redpacket.status = STATUS_REDPACKET_AWARD_FAILED
redpacket.issue_status = STATUS_READPACKET_ISSUE_FAIL
redpacket.sync_save()
break
if status == 6:
logger.warning(
'[RedPacket Waiting] the redpacket info: member_cid: %s, checkpoint_cid: %s.'
% (member.cid, redpacket.checkpoint_cid))
if status == 8:
pass
except Exception:
logger.error(str(res.content))
logger.error(traceback.format_exc())
logger.info(
' END (%s): lottery result query, member_cid=%s, reward_cid=%s, rid=%s '
% (self.request.id, member.cid, reward_cid, rid))
basic = RedPacketBasicSetting.sync_find_one({'race_cid': race.cid})
basic.rule_cid = rule.cid
basic.sync_save()
# db.TBL_RED_PACKET_BASIC_SETTING.update({}, {$rename : {"expect_people_num" : "expect_num", "top_limit_each_day": "top_limit"}}, false, true)
settings = RedPacketItemSetting.sync_find({'race_cid': race.cid})
setting_map = {}
for s in settings:
s.rule_cid = rule.cid
setting_map[s.cid] = s
s.sync_save()
# db.TBL_RED_PACKET_ITEM_SETTING.update({}, {$rename : {"count" : "quantity"}}, false, true)
# 奖池
for entry in RedPacketEntry.sync_find({'race_cid': race.cid}):
item = setting_map[entry.award_cid]
box = RedPacketBox()
box.race_cid = race.cid
box.rule_cid = rule.cid
box.award_cid = entry.award_cid
box.award_msg = item.message
box.award_amount = item.amount
if entry.open_id:
member = Member.sync_find_one({'open_id': entry.open_id})
box.member_cid = member.cid
hist = RedPacketAwardHistory.sync_find_one({'race_cid': race.cid, 'award_cid': entry.cid, 'issue_status': {'$ne': 1}})
if hist:
box.draw_status = STATUS_REDPACKET_AWARDED
box.draw_dt = hist.request_dt
box.error_msg = hist.error_msg
async def generate_awards_by_item_settings(basic_setting, settings):
"""
:param basic_setting:
:param settings:
:return:
"""
if not (basic_setting and settings):
raise Exception('no basic_setting or settings')
# 删除还未发出的红包
await RedPacketBox.delete_many({
'rule_cid': basic_setting.rule_cid,
'member_cid': None
})
box_list = await RedPacketBox.aggregate(
[
MatchStage({
'rule_cid': basic_setting.rule_cid,
'award_cid': {
'$ne': None
}
}),
GroupStage('award_cid', sum={'$sum': 1})
],
read_preference=ReadPreference.PRIMARY).to_list(None)
# 红包的发放情况
get_situ = {box.id: box.sum for box in box_list}
award_list = list()
for config in settings:
for _ in range(config.quantity - get_situ.get(config.cid, 0)):
box = RedPacketBox()
box.race_cid = basic_setting.race_cid
box.rule_cid = basic_setting.rule_cid
box.award_cid = config.cid
box.award_msg = config.message
box.award_amount = config.amount
award_list.append(box)
has_sent_count = sum(get_situ.values())
while len(award_list) < basic_setting.expect_num - has_sent_count:
box = RedPacketBox()
box.race_cid = basic_setting.race_cid
box.rule_cid = basic_setting.rule_cid
box.award_msg = basic_setting.fail_msg
award_list.append(box)
shuffle(award_list)
if award_list:
for award in award_list:
await award.save()