def save_reviews(self, reviews):
for review in reviews:
try:
review.update({
'id': self._ids.next_id("reviews"),
'user_id': int(review.get('user_id', 0)),
'user_name': review.get('user_name', ''),
'app_id': int(review.get('app_id', 0)),
'rate': float(review.get('rate', 0)),
'comment': review.get('comment', ''),
'created_time': timestamp_utc_now()
})
self._db.reviews.save(review)
except Exception, e:
print e
continue
if review['app_id'] == 0:
continue
app = self._db.apps.find_one({'id': review['app_id']}, fields=APP_FIELDS_RATING)
if app is None:
continue
self._db.reviews.save(review)
app_rate_count = app['rate_count'] if 'rate_count' in app else 0
app_rate = app['rate'] if 'rate' in app else 0
app_rate = (app_rate + review['rate']) / (app_rate_count + 1.0)
self._db.apps.update({'_id': app['_id']}, {'$set': {'rate': app_rate, 'rate_count': app_rate_count}})
def _get_valid_messages(self):
now = timestamp_utc_now()
return cached_query('push_valid_nmessages',
lambda sender : [message for message in sender._db.messages.find({ 'not_valid_before': {'$lte': now}, 'not_valid_after': {'$gt': now} }, fields = message_fields_detail)],
query_func_state = self,
timeout = 120,
return_cache_hit_status = True)
def query_login_pictures(self, start_index=0, count=20, order=None):
if order is None:
order = [('start_date', self.ORDER_DESC)]
now = timestamp_utc_now()
cond = {'start_date': {'$lte': now}, 'end_date': {'$gt': now}}
return self._db.login_pictures.find(cond,
skip=start_index,
limit=count,
sort=order)
def can_query_update(self, client_id):
now = timestamp_utc_now()
limit = self._db.limits.find_and_modify({'_id': client_id}, {
'_id': client_id,
'last_query': now
}, True)
if not limit or (now - limit['last_query'] >
self.UPDATE_QUERY_LIMIT_TIME):
return True
return False
def _get_valid_messages(self):
now = timestamp_utc_now()
return cached_query('push_valid_nmessages',
lambda sender: [
message
for message in sender._db.messages.find(
{
'not_valid_before': {
'$lte': now
},
'not_valid_after': {
'$gt': now
}
},
fields=message_fields_detail)
],
query_func_state=self,
timeout=120,
return_cache_hit_status=True)
def query_login_pictures(self, start_index=0, count=20, order=None):
if order is None:
order = [('start_date', self.ORDER_DESC)]
now = timestamp_utc_now()
cond = {'start_date': {'$lte': now}, 'end_date': {'$gt': now}}
return self._db.login_pictures.find(cond, skip = start_index, limit = count, sort = order)
def client_statistics(request):
_ERR_CODE_ILL_PARAM = "Illegal Parameter"
_ERR_CODE_PARAM_ERR = "Get Parameter Error"
_ERR_CODE_SERV_ERR = "Server Error"
_ERR_CODE = {
_ERR_CODE_ILL_PARAM: 'IllegalParameters',
_ERR_CODE_PARAM_ERR: 'GetParameterError',
_ERR_CODE_SERV_ERR: 'ServerError',
}
key_rename_map = {
'chn': 'chn',
'clientid': 'clientid',
'appid': 'appid',
'appname': 'appname',
'md_id': 'id',
'md_type': 'type',
'md_num': 'num',
'md_appid': 'appid',
'md_appname': 'appname',
'md_categid': 'categid',
'md_categname': 'categname',
'sd_subjectid': 'subjectid',
'sd_subjectname': 'subjectname',
'sd_csubjectid': 'csubjectid',
'sd_csubjectname': 'csubjectname',
'log_date': 'log_date',
}
db_value_map = {
'_md_type': 'md_type',
'_md_appid': 'md_appid',
'_md_appname': 'md_appname',
'_md_num': 'md_num',
'_md_categid': 'md_categid',
'_md_categname': 'md_categname',
'_sd_subjectid': 'sd_subjectid',
'_sd_subjectname': 'sd_subjectname',
'_sd_csubjectid': 'sd_csubjectid',
'_sd_csubjectname': 'sd_csubjectname',
'_log_date': 'log_date',
'_client_id': 'client_id',
'_chn': 'chn',
}
query_dict = {}
statist_db = StatistcMongoDB(settings.MONGODB_CONF)
log_date = timestamp_utc_now()
try:
client_id = get_parameter_GET(
request,
key_rename_map['clientid'])
chn = get_parameter_GET(
request,
key_rename_map['chn'])
md_type = get_parameter_GET(
request,
key_rename_map['md_type'])
md_appid = get_parameter_GET(
request,
key_rename_map['md_appid'])
md_appname = get_parameter_GET(
request,
key_rename_map['md_appname'])
md_num = get_parameter_GET(
request,
key_rename_map['md_num'])
md_categid = get_parameter_GET(
request,
key_rename_map['md_categid'])
md_categname = get_parameter_GET(
request,
key_rename_map['md_categname'])
except Exception:
return json_response_error(-1, _ERR_CODE[_ERR_CODE_PARAM_ERR])
else:
if md_type not in ('1', '2', '3', '4', '5', '6', '6', '7', '8', '9', '10', '11', '12'):
return json_response_error(-1, _ERR_CODE[_ERR_CODE_ILL_PARAM])
if md_type == '8' or md_type == '9':
try:
sd_subjectid = get_parameter_GET(
request,
key_rename_map['sd_subjectid'])
sd_subjectname = get_parameter_GET(
request,
key_rename_map['sd_subjectname'])
sd_csubjectid = get_parameter_GET(
request,
key_rename_map['sd_csubjectid'])
sd_csubjectname = get_parameter_GET(
request,
key_rename_map['sd_csubjectname'])
except Exception:
return json_response_error(-1, _ERR_CODE[_ERR_CODE_PARAM_ERR])
else:
query_dict = {
db_value_map['_md_type']: md_type,
db_value_map['_md_appid']: md_appid,
db_value_map['_md_appname']: md_appname,
db_value_map['_md_num']: md_num,
db_value_map['_md_categid']: md_categid,
db_value_map['_md_categname']: md_categname,
db_value_map['_sd_subjectid']: sd_subjectid,
db_value_map['_sd_subjectname']: sd_subjectname,
db_value_map['_sd_csubjectid']: sd_csubjectid,
db_value_map['_sd_csubjectname']: sd_csubjectname,
db_value_map['_log_date']: log_date,
db_value_map['_client_id']: client_id,
db_value_map['_chn']: chn,
}
else:
query_dict = {
db_value_map['_md_type']: md_type,
db_value_map['_md_appid']: md_appid,
db_value_map['_md_appname']: md_appname,
db_value_map['_md_num']: md_num,
db_value_map['_md_categid']: md_categid,
db_value_map['_md_categname']: md_categname,
db_value_map['_log_date']: log_date,
db_value_map['_client_id']: client_id,
db_value_map['_chn']: chn,
}
try:
if not statist_db.insert_to_StatisticDB(query_dict):
return json_response_error(-1, _ERR_CODE[_ERR_CODE_SERV_ERR])
except Exception:
return json_response_error(-1, _ERR_CODE[_ERR_CODE_SERV_ERR])
else:
return json_response_ok()
finally:
query_dict.clear()
def execute(self, request):
chn = get_parameter_POST_or_GET(
request,
'chn',
required=False,
str_max_length=255)
client_id = get_parameter_POST_or_GET(request, 'clientid')
source = get_parameter_POST_or_GET(
request,
'source',
str_max_length=255)
app_id = get_parameter_POST_or_GET(
request,
'app_id',
convert_func=int)
app_version_code = get_parameter_POST_or_GET(
request,
'app_version_code',
convert_func=int)
app_version_name = get_parameter_POST_or_GET(
request,
'app_version_name')
user_name = get_parameter_POST_or_GET(request, 'user_name')
review_rate = get_parameter_POST_or_GET(
request,
'rate',
convert_func=int)
review_content = get_parameter_POST_or_GET(
request,
'content',
str_max_length=1000)
ip = get_parameter_META(
request,
'REMOTE_ADDR',
required=False,
default=u'')
for q in (client_id, source, app_id, app_version_code, app_version_name, user_name, review_rate, review_content):
if isinstance(q, HttpResponse):
return q
error_code = PostReviewAction.ERROR_CODE_OK
status = PostReviewAction.REVIEW_STATUS_OK
if self.contains_block_words(app_version_name + user_name + review_content):
status = PostReviewAction.REVIEW_STATUS_AWAITING_MODERATION
error_code = PostReviewAction.ERROR_CODE_AWAITING_MODERATION
review = {
'app_id': app_id,
'app_version_code': app_version_code,
'app_version_name': app_version_name,
'user_name': user_name,
'rate': review_rate,
'content': review_content,
'source': source,
'client_id': client_id,
'ip': ip,
'created_time': timestamp_utc_now(),
'status': status,
}
self.app_db.save_reviews(review)
del review['app_id']
del review['app_version_code']
del review['source']
del review['client_id']
del review['ip']
del review['status']
del review['_id']
return {'error_code': error_code, 'reviews': [review]}
'start_index',
convert_func=int)
count = get_parameter_GET(request, 'count', convert_func=int)
ip = get_parameter_META(
request,
'REMOTE_ADDR',
required=False,
default=u'')
for q in (client_id, source, app_id, app_version_code):
if isinstance(q, HttpResponse):
return q
return (
{'reviews': self.query_reviews(
app_id,
start_index=start_index,
count=count)}
)
if __name__ == '__main__':
action = PostReviewAction(None, None)
print action.contains_block_words('test')
print action.contains_block_words(u'东西acdsf东西南北论坛sdfsdf')
print action.contains_block_words(u'a[av]asfasdf')
print timestamp_utc_now()
for i in xrange(0, 500):
action.contains_block_words(
u'东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a东西a')
print timestamp_utc_now()
def can_query_update(self, client_id):
now = timestamp_utc_now()
limit = self._db.limits.find_and_modify({'_id':client_id}, {'_id':client_id, 'last_query':now }, True)
if not limit or (now - limit['last_query'] > self.UPDATE_QUERY_LIMIT_TIME):
return True
return False
def select_messages(
messages, client_id, source, chn, is_test_device, count, device_model='', os_v=0, screensize='', sim_operator=0,
user_phone='', client_version=0, ip=None, debug=0, debug_user_tags=None, perf=None, push_db=None, user_db=None, location_db=None):
if not messages:
if debug:
messages.append({'debug_message': 'no message found'})
return messages
time_now = timestamp_utc_now()
final_messages = messages
if perf:
perf.set('pre')
# push time check
user_pushed_messages = push_db.get_pushed_messages(client_id)
if user_pushed_messages:
print 'pushed'
final_messages = [message for message in final_messages if str(
message['message_id']) not in user_pushed_messages]
last_message_push_time = max(message[1]
for message in user_pushed_messages.iteritems())
# no two messages within 12 hours
if time_now - last_message_push_time < MESSAGE_PUSH_INTERVAL:
# push only if there's direct message
if all(_get_message_group_name(message) != '' for message in final_messages):
final_messages = []
if debug:
final_messages.append(
{'debug_message': 'no direct message',
'last_message_push_time': last_message_push_time,
'now': time_now,
'messages': messages,
'user_pushed_messages': user_pushed_messages})
return final_messages
if perf:
perf.set('ptc')
print 'end .....'
# filter messages which run out of quota
pushed_counts = push_db.get_message_pushed_counts(final_messages)
pushed_counts = dict([(item['_id'], item)
for item in pushed_counts]) if pushed_counts else {}
date_now = _date_key_from_time(time_now)
final_messages_new = []
for message in final_messages:
push_total = _get_message_target_value(
message,
'push_total',
defauld_value=0)
push_daily = _get_message_target_value(
message,
'push_daily',
defauld_value=0)
pushed_total = 0
pushed_today = 0
pushed_count = pushed_counts.get(message['message_id'], None)
if pushed_count:
pushed_total = pushed_count.get('total', 0)
pushed_today = pushed_count.get(date_now, 0)
if push_total != 0 and push_total <= pushed_total:
continue
if push_daily != 0 and push_daily <= pushed_today:
continue
# 1000000, this will be used for message sampling
message['push_today'] = (
push_daily - pushed_today) if push_daily != 0 else 1000000
final_messages_new.append(message)
final_messages = final_messages_new
if perf:
perf.set('qt')
if not final_messages:
if debug:
final_messages.append(
{'debug_message': 'all messages run out of quota',
'messages': messages,
'pushed_counts': pushed_counts})
return final_messages
# step 0: get user data
user_tags = user_db.query_user_tags(client_id)
if perf:
perf.set('qut')
user_apps = user_db.query_apps_installed(client_id)
if perf:
perf.set('qua')
user_tags = _update_user_tags(
user_tags,
user_apps,
chn,
device_model,
os_v,
screensize,
sim_operator,
user_phone,
client_version,
ip,
user_pushed_messages,
debug_user_tags,
location_db,
perf)
if perf:
perf.set('uut')
# step 1: filter message
print '11111'
print final_messages
message_matcher = MessageMatcher(user_tags, time_now, debug=debug)
final_messages = message_matcher.do_match(final_messages)
# no message filter for test device:
if is_test_device:
final_messages = messages
if perf:
perf.set('fltr')
# step 2: rank and select message
message_ranker = MessageRanker(user_tags, debug=debug)
final_messages = message_ranker.do_rank(final_messages, count)
if perf:
perf.set('rnk')
if client_id == 'apitest':
# client: apitest is a very special test id, it could always get all
# valid messages
final_messages = messages
else:
for message in final_messages:
if 'targets' in message:
del message['targets']
if 'push_today' in message:
del message['push_today']
if not debug:
_update_pushed_messages(
client_id,
time_now,
user_pushed_messages,
final_messages,
push_db)
if perf:
perf.set('ums')
if debug:
debug_data = {'debug': debug,
'user_tags': user_tags,
'message_matcher': message_matcher.debug_logs,
'message_ranker': message_ranker.debug_logs,
'user_pushed_messages': user_pushed_messages,
'pushed_counts': pushed_counts}
final_messages.append(debug_data)
return final_messages