def push_load(self, phone_type, udid, last_id, limit):
if is_blank(udid):
return [], False
# 取出分组(ios/android)推送、个性化推送记录
group_id = NOTIFICATION_GROUP.get(phone_type, 1)
query = Q(group_id=group_id, target_id__ne='')
if not is_blank(last_id):
query = query & Q(pk__lt=last_id)
items = list(
NotificationInbox.objects(query).order_by('-id').limit(limit + 1))
final_text_items = []
if not is_blank(udid):
# TODO embeded document optimize
profile_item = NotificationInbox.objects(
udid=udid, action_type=ACTION_PUSH_PROFILE).first()
if profile_item:
items.insert(0, profile_item)
udid_text_item = NotificationInbox.objects(
udid=udid, action_type=ACTION_PUSH_SUBGROUP).first()
if udid_text_item:
udid_text_nids = {
x.target_id: x.notification_id
for x in udid_text_item.target_items
}
query = Q(group_id=GROUP_MATCH,
action_type=ACTION_PUSH_SUBGROUP)
if last_id:
query = query & Q(pk__lt=last_id)
text_items = {
x.notification_id: x
for x in list(
NotificationInbox.objects(query).order_by('-id').limit(
limit + 1))
}
for tid, nid in udid_text_nids.items():
if nid in text_items:
text_items[nid].target_id = tid
final_text_items.append(text_items[nid])
items = self.filter_group_for_profile(udid, items)
items.extend(final_text_items)
items = sorted(items,
key=functools.cmp_to_key(lambda x, y: int(
(y.ctime - x.ctime).total_seconds())))
has_more = False
if len(items) > limit:
has_more = True
items = items[:limit]
return items, has_more
def is_android_outcast(unique_device_id):
# 空的设备udid不进审
if is_blank(unique_device_id):
return False
# 不合格的udid进审
if len(unique_device_id) < 23:
return True
ts = int(unique_device_id[:17])
# 2018-07-23 22:00:00
if ts > 15323544000000000:
return True
return False
def message_load(self, pk, last_id, limit):
if is_blank(pk):
return [], False
kwargs = {
'to_uid': pk,
'status': OLD_MESSAGE_NORMAL,
}
if not is_blank(last_id):
kwargs['pk__lt'] = last_id
rs = MessageOldStage.objects(**kwargs).order_by('-id').limit(limit + 1)
comments = [item for item in rs]
has_more = False
if len(comments) > limit:
has_more = True
comments = comments[:limit]
return comments, has_more
def normalize(self, query_str):
if self.is_active == EnumActive.DISABLE:
return None
if self.language != EnumLanguageType.JA:
self.title = ''
if not isinstance(self.published_at, int):
self.published_at = try_parse_datetime_ts(self.published_at)
# 视频是youtube视频时需要处理的数据
if self.source_type == EnumVideoSourceType.YOUTUBE:
self.original_site_url = 'https://www.youtube.com/watch?v={}'.format(
self.y_video_id)
self.source_name = 'Youtube' if is_blank(
self.channel_name) else self.channel_name
self.source_pic = DEFAULT_YOUTUBE_ICON if is_blank(
self.avatar_url) else self.avatar_url
# 如果source_type是URL的, 需要从media中抽取相应的数据
if self.source_type == EnumVideoSourceType.URL and self.medias:
medias = extract_medias(self.medias, logger=exception_logger)
if medias:
v = medias[0]
if not isinstance(v, MediaVideo):
raise ValueError(
'video[mp4] has invalid format.{}'.format(v))
self.__dict__.update(v.get_urls())
self.duration_interval = v.duration
if self.cover_image_urls:
self.cover_image = self.cover_image_urls[0]
self.style = EnumListItemStyle.VIDEO_PLAYABLE
self.source_pic = self.avatar_url
if hasattr(query_str, 'channel_type'):
if query_str.channel_type == EnumChannelType.ARTICLE:
self.style = EnumListItemStyle.VIDEO_LARGE_COVER
# 客户端需要type始终为1, 数据端可能有很多类型都是video
self.type = EnumItemType.VIDEO
self.share_url = SHARE_URL.format(self.type, self.aid)
return self
def queue_jump(self, y_video_id):
"""
由于更新环是rpush-blpop的顺序, 所以使用lpush插队,可以让y_video_id能够进到队列头部, 提前盗链
:param y_video_id:
:return:
"""
try:
if is_blank(y_video_id):
return
self.cli.lpush(VIDEO_HOTLINK_QUEUE_KEY, '*_{}'.format(y_video_id))
except:
exception_logger.exception(
'video_hotlink_queue_jump_err: {}'.format(y_video_id))
def get_pk(request):
"""
获取用户主键, 如果有 uid, 使用 uid; 如果没有, 使用 unique_device_id
:type request: django.http.HttpRequest
:rtype: (str, pyutil.api.api_util.QueryStr)
"""
# return request.GET.get('unique_device_id'), None
# FIXME
from pyutil.api.api_util import parse_query_str
from pyutil.text.conv import is_blank
uid = request.session.get('uid', '')
query_str = parse_query_str(request)
if is_blank(uid):
uid = query_str.unique_device_id
return uid, query_str
def rule_is_pure(query_str):
"""
受苹果商店推荐影响的用户
"""
return False
if query_str.unique_device_id in PURE_TEST_UDIDS:
return True
udid = query_str.unique_device_id
if query_str.phone_type == QueryStr.PHONE_TYPE_ANDROID or is_blank(
udid) or udid.startswith('mg'):
return False
if len(udid) < 23:
return False
try:
ts = int(udid[:17])
if ts > PURE_MORE_START_TS:
return True
finally:
return False
def eval(e_flag, pk):
if is_blank(e_flag):
return ABEval()
experiments = e_flag.split(EXP_SEP)
r = ABEval()
r.pk = pk
for exp in experiments:
try:
layer_id, exp_name = exp.split(LAYER_SEP)
if layer_id in r.experiments:
r.experiments[layer_id].add(exp_name)
else:
r.experiments[layer_id] = {
exp_name,
}
except:
logging.getLogger('exc').exception(
'ABEval.eval()_error: e_flag={},pk={}'.format(e_flag, pk))
return r
def ab_test_first(e_flag, layer_id, exp_name):
"""
一次性检查逻辑, 在一个请求只有一个实验的场景下使用
:param e_flag:
:param layer_id:
:param exp_name:
:return:
"""
if is_blank(e_flag):
return False
experiments = e_flag.split(EXP_SEP)
for exp in experiments:
try:
layer_id, exp_name = exp.split(LAYER_SEP)
if layer_id == layer_id and exp_name == exp_name:
return True
except:
logging.getLogger('exc').exception(
'ab_test_first()_error: e_flag={}'.format(e_flag))
return False
def verify_request_encryption(request, redis_pool):
"""
校验写接口,防止重复提交. True 为通过验证, False 为没有通过.
:type request: django.http.HttpRequest
:type redis_pool: redis.ConnectionPool
:rtype: bool
"""
encrypted_data = request.META.get(ENCRYPTION_KEY, '')
# 如果加密数据为空, 则返回无效
if is_blank(encrypted_data):
return False, 'Empty data'
# 解出加密数据,如果解数据出错,则返回无效
try:
data = base64.b64decode(encrypted_data)
data = rncryptor.decrypt(data, ENCRYPTION_PKEY)
timestamp, udid = data.split()
timestamp = int(timestamp)
except Exception as e:
return False, 'Error data, {}'.format(e)
# 验证时间, 如果当前时间比
now_ts = int(time.time() * 1000)
if now_ts > timestamp + ENCRYPTION_TIMEDELTA or now_ts < timestamp - ENCRYPTION_TIMEDELTA:
return False, 'Invalid time, now={}, args={}, {}, date={}'.format(
now_ts, timestamp, udid, encrypted_data)
try:
redis_db = redis.StrictRedis(connection_pool=redis_pool)
key = '{}_{}'.format(udid, timestamp)
if redis_db.exists(key):
return False, 'Repeat request, now={}, args={}, {}, date={}'.format(
now_ts, timestamp, udid, encrypted_data)
redis_db.set(key, '1', px=ENCRYPTION_TIMEDELTA)
return True, 'Success'
except Exception as e:
# 降级逻辑, REDIS 挂掉, 默认不进行校验.
return True, 'Redis error, {}'.format(e)
def set_device_push_group_count(self, group_id, udid, val):
if is_blank(udid):
return
self.write_redis.set(get_device_push_group_count(group_id, udid), val)
def clear_user_message_count(self, uid):
if is_blank(uid):
return
self.write_redis.set(get_user_message_count(uid), '0')