def _sync_get_learning_code(history):
"""
获取学习日编码
:param member_cid: 会员CID
:return:
"""
if history:
l_code = RedisCache.get('LEARNING_STATISTICS_CODE_%s' % history.cid)
if not l_code:
prev_datetime = copy.deepcopy(history.fight_datetime).replace(
hour=23, minute=59, second=59,
microsecond=999999) - datetime.timedelta(days=1)
match_stage = MatchStage({
'member_cid': history.member_cid,
'fight_datetime': {
'$lte': prev_datetime
}
})
project_stage = ProjectStage(date={
'$dateToString': {
'format': '%Y%m%d',
'date': '$fight_datetime'
}
})
group_stage = GroupStage('date')
mgh_cursor = MemberGameHistory.sync_aggregate(
[match_stage, project_stage, group_stage],
read_preference=ReadPreference.PRIMARY)
mch_cursor = MemberCheckPointHistory.sync_aggregate(
[match_stage, project_stage, group_stage],
read_preference=ReadPreference.PRIMARY)
tmp_dict = {}
for mgh in mgh_cursor:
if mgh:
tmp_dict[mgh.id] = int(mgh.id)
for mch in mch_cursor:
if mch:
tmp_dict[mch.id] = int(mch.id)
l_code = 1
if tmp_dict:
l_code = len(tmp_dict.keys()) + 1
remain_seconds = get_day_remain_seconds()
if remain_seconds:
RedisCache.set(
'LEARNING_STATISTICS_CODE_%s' % history.member_cid, l_code,
remain_seconds)
else:
l_code = int(l_code)
return l_code
return None
def do_init(model, skip_num, limit_num):
"""
:param model:
:param skip_num:
:param limit_num:
:return:
"""
stage_list = [
MatchStage({
'created_dt': {
"$gte":
datetime.now().replace(day=14, hour=18, minute=15, second=00),
"$lte":
datetime.now().replace(day=17, hour=9, minute=22, second=00)
}
}),
ProjectStage(
**{
'daily_code': {
"$dateToString": {
'format': '%Y%m%d000000',
'date': "$created_dt"
}
},
'member_cid': 1,
'correct': {
'$size': {
'$filter': {
'input': '$result.true_answer',
'as': 'item',
'cond': {
'$and': [{
'$eq': ['$$item', True]
}]
}
}
}
},
'total': {
'$size': '$result'
},
'result': 1,
'created_dt': 1
}),
GroupStage({
'daily_code': '$daily_code',
'member_cid': '$member_cid'
},
correct_list={'$push': "$correct"},
result_list={"$push": "$result"},
created_dt={'$first': "$created_dt"},
correct={'$sum': '$correct'},
total={'$sum': '$total'},
learn_times={'$sum': 1}),
LookupStage(Member, '_id.member_cid', 'cid', 'member_list'),
MatchStage({'member_list': {
'$ne': []
}}),
ProjectStage(
**{
'daily_code': "$_id.daily_code",
'member_cid': '$_id.member_cid',
'correct': '$correct',
'total': "$total",
'correct_list': "$correct_list",
'learn_times': '$learn_times',
'province_code': {
"$arrayElemAt": ['$member_list.province_code', 0]
},
'city_code': {
"$arrayElemAt": ['$member_list.city_code', 0]
},
'district_code': {
"$arrayElemAt": ['$member_list.district_code', 0]
},
'gender': {
"$arrayElemAt": ['$member_list.sex', 0]
},
'age_group': {
"$arrayElemAt": ['$member_list.age_group', 0]
},
'education': {
"$arrayElemAt": ['$member_list.education', 0]
},
'result_list': 1,
'created_dt': 1
}),
SortStage([('daily_code', ASC), ('member_cid', ASC)]),
]
if skip_num:
stage_list.append(SkipStage(skip_num))
if limit_num:
stage_list.append(LimitStage(limit_num))
cursor = MemberGameHistory.sync_aggregate(
stage_list, allowDiskUse=True, read_preference=ReadPreference.PRIMARY)
index = 0
while True:
try:
index += 1
print('has exec %s.' % index)
data = cursor.next()
param = {
'daily_code':
data.daily_code,
'member_cid':
data.member_cid,
'province_code':
data.province_code,
'city_code':
data.city_code,
'district_code':
data.district_code,
'gender':
data.gender,
'age_group':
data.age_group,
'education':
data.education,
'learn_times':
data.learn_times,
'subject_total_quantity':
data.total,
'subject_correct_quantity':
data.correct,
'quantity_detail':
dict(Counter(map(lambda x: str(x), data.correct_list))),
'dimension_detail':
get_dimension_detail(data.result_list),
'created_dt':
data.created_dt,
'updated_dt':
data.created_dt
}
model(**param).sync_save()
except StopIteration:
break
except AttributeError as e:
print(e)
continue
def do_init_member_stat(pre_match: dict = {},
skip_num=0,
limit_num=0,
just_return_count=False):
"""
:param pre_match:
:param skip_num:
:param limit_num:
:param just_return_count:
:return:
"""
if not isinstance(just_return_count, bool):
raise Exception('check params(just_return_count)')
stage_list = []
if pre_match:
stage_list.append(MatchStage(pre_match))
stage_list.extend([
UnwindStage("result"),
LookupStage(Member, 'member_cid', 'cid', 'member_list'),
MatchStage({'member_list': {
'$ne': list()
}}),
ProjectStage(
**{
'subject_cid': '$result.subject_cid',
'province_code': {
'$arrayElemAt': ['$member_list.province_code', 0]
},
'city_code': {
'$arrayElemAt': ['$member_list.city_code', 0]
},
'district_code': {
'$arrayElemAt': ['$member_list.district_code', 0]
},
'sex': {
'$arrayElemAt': ['$member_list.sex', 0]
},
'age_group': {
'$arrayElemAt': ['$member_list.age_group', 0]
},
'category': {
'$arrayElemAt': ['$member_list.category', 0]
},
'education': {
'$arrayElemAt': ['$member_list.education', 0]
},
'true_answer': {
'$cond': {
'if': {
'$eq': ['$result.true_answer', True]
},
'then': True,
'else': False
}
},
'created_dt': '$created_dt'
}),
GroupStage(
{
'subject_cid': '$subject_cid',
'province_code': '$province_code',
'city_code': '$city_code',
'district_code': '$district_code',
'sex': '$sex',
'age_group': '$age_group',
'category': '$category',
'education': '$education',
},
answers_list={'$push': '$true_answer'},
created_dt={'$first': '$created_dt'}),
])
if just_return_count:
stage_list.append(CountStage())
data = MemberGameHistory.sync_aggregate(
stage_list,
read_preference=ReadPreference.PRIMARY,
allowDiskUse=True).to_list(1)
return data[0].count
stage_list.append(SortStage([('created_dt', ASC)]))
if skip_num:
stage_list.append(SkipStage(skip_num))
if limit_num:
stage_list.append(LimitStage(limit_num))
cursor = MemberGameHistory.sync_aggregate(
stage_list, read_preference=ReadPreference.PRIMARY,
allowDiskUse=True).batch_size(256)
index = 0
_subject_map = {}
while True:
try:
data = cursor.next()
subject_cid = data.id.get('subject_cid')
subject = _subject_map.get(subject_cid)
if not subject:
subject = Subject.sync_get_by_cid(subject_cid)
_subject_map[subject_cid] = subject
write_member_subject_stat(data, subject)
index += 1
print('has exec %s' % index)
except StopIteration:
break
except Exception:
print(traceback.format_exc())
continue
def do_stat(model):
stage_list = [
UnwindStage('result'),
ProjectStage(
**{
'daily_code': {
'$dateToString': {
'format': '%Y%m%d',
'date': '$created_dt'
}
},
'member_cid': '$member_cid',
'subject_cid': '$result.subject_cid',
'true_answer': '$result.true_answer'
}),
LookupStage(Subject, 'subject_cid', 'cid', 'subject_list'),
ProjectStage(
**{
'daily_code': 1,
'member_cid': 1,
'true_answer': 1,
'dimension': {
'$arrayElemAt': ['$subject_list.dimension_dict', 0]
}
}),
GroupStage(
{
'daily_code': '$daily_code',
'member_cid': '$member_cid',
'dimension': '$dimension'
},
answer_list={'$push': '$true_answer'}),
ProjectStage(
**{
'daily_code': '$_id.daily_code',
'member_cid': '$_id.member_cid',
'dimension': '$_id.dimension',
'correct': {
'$size': {
'$filter': {
'input': '$answer_list',
'as': 'item',
'cond': {
'$and': [{
'$eq': ['$$item', True]
}]
}
}
}
},
'total': {
"$size": "$answer_list"
}
}),
GroupStage({
'member_cid': '$member_cid',
'dimension': '$dimension'
},
daily_list={'$push': '$daily_code'},
correct_list={'$push': '$correct'},
total_list={'$push': '$total'}),
]
cursor = MemberGameHistory.sync_aggregate(
stage_list, allowDiskUse=True, read_preference=ReadPreference.PRIMARY)
t1 = time.time()
index = 0
insert_list = []
while True:
try:
data = cursor.next()
member = Member.sync_get_by_cid(data.id.get('member_cid'))
if not member:
continue
dimension = data.id.get('dimension')
if not dimension:
continue
for i, day in enumerate(data.daily_list):
param = {
'learning_code': get_learning_code(day, data.daily_list),
'member_cid': member.cid,
'dimension': dimension,
'province_code': member.province_code,
'city_code': member.city_code,
'district_code': member.district_code,
'gender': member.sex,
'age_group': member.age_group,
'education': member.education,
'subject_total_quantity': data.total_list[i],
'subject_correct_quantity': data.correct_list[i]
}
insert_list.append(model(**param))
except StopIteration:
break
if len(insert_list) > 5000:
model.sync_insert_many(insert_list)
insert_list = []
index += 1
print('has exec', index)
t2 = time.time()
print('cost:', t2 - t1)
model.sync_insert_many(insert_list)
t3 = time.time()
print('insert', t3 - t2)
def do_init(model):
"""
:return:
"""
b_dt = datetime.now().replace(month=5, day=18, hour=0, minute=0, second=0)
e_dt = datetime.now().replace(month=6, day=13, hour=0, minute=0, second=0)
cursor = MemberGameHistory.sync_aggregate(
[
MatchStage({'created_dt': {
'$gte': b_dt,
'$lte': e_dt
}}),
UnwindStage("result"),
ProjectStage(
**{
'daily_code': {
'$dateToString': {
'format': '%Y%m%d000000',
'date': '$created_dt'
}
},
'member_cid': 1,
'subject_cid': '$result.subject_cid',
'true_answer': "$result.true_answer",
'created_dt': 1
}),
GroupStage(
{
'daily_code': '$daily_code',
'member_cid': '$member_cid',
'subject_cid': '$subject_cid'
},
answer_list={'$push': '$true_answer'},
created_dt={'$first': '$created_dt'}),
LookupStage(Member, '_id.member_cid', 'cid', 'member_list'),
LookupStage(Subject, '_id.subject_cid', 'cid', 'subject_list'),
MatchStage({
'member_list': {
'$ne': []
},
'subject_list': {
'$ne': []
}
}),
ProjectStage(
**{
'member_cid': '$_id.member_cid',
'dimension': {
"$arrayElemAt": ['$subject_list.dimension_dict', 0]
},
'daily_code': '$_id.daily_code',
'province_code': {
"$arrayElemAt": ['$member_list.province_code', 0]
},
'city_code': {
"$arrayElemAt": ['$member_list.city_code', 0]
},
'district_code': {
"$arrayElemAt": ['$member_list.district_code', 0]
},
'gender': {
"$arrayElemAt": ['$member_list.sex', 0]
},
'age_group': {
"$arrayElemAt": ['$member_list.age_group', 0]
},
'education': {
"$arrayElemAt": ['$member_list.education', 0]
},
'correct': {
'$size': {
'$filter': {
'input': '$answer_list',
'as': 'item',
'cond': {
'$and': [{
'$eq': ['$$item', True]
}]
}
}
}
},
'total': {
"$size": "$answer_list"
},
'created_dt': 1
}),
GroupStage(
{
'daily_code': '$daily_code',
'member_cid': '$member_cid',
'dimension': '$dimension'
},
province_code={'$first': "$province_code"},
city_code={'$first': "$city_code"},
district_code={'$first': "$district_code"},
gender={'$first': "$gender"},
age_group={'$first': "$age_group"},
education={'$first': "$education"},
correct={'$sum': "$correct"},
total={'$sum': '$total'})
],
allowDiskUse=True,
read_preference=ReadPreference.PRIMARY)
index = 0
while True:
try:
data = cursor.next()
param = {
'daily_code': data.id.get('daily_code'),
'member_cid': data.id.get('member_cid'),
'dimension': data.id.get('dimension'),
'province_code': data.province_code,
'city_code': data.city_code,
'district_code': data.district_code,
'gender': data.gender,
'age_group': data.age_group,
'education': data.education,
'subject_total_quantity': data.total,
'subject_correct_quantity': data.correct,
'created_dt': data.created_dt
}
model(**param).sync_save()
index += 1
print('has exec', index)
except StopIteration:
break
except AttributeError as e:
print(e)
continue
print('done.')