def get_item_list_by_user(user_info, resource_type, count):
""" get item list by user from offline recommendation
@return RecommendItemContainer
"""
logger.debug("begin get_item_list_by_user")
uid = user_info.uid
result = RecommendItemContainer()
errCode, kv_dict = model.get_offline_user_recommend_item_list(uid, [resource_type])
if not is_valid_data(errCode, kv_dict, "offline_user_recommend_item_list", {"uid":uid, "resource_type":resource_type}):
return result
if not kv_dict:
return result
try:
assert(resource_type in kv_dict)
reason2itemidlist = FeatureNameItemIdList()
reason2itemidlist.ParseFromString(kv_dict[resource_type])
for reason2itemid in reason2itemidlist.feature_name_item_id_list:
if result.size() >= count:
logger.debug("break for enough result")
break
reason = reason2itemid.feature_name
item_id_list = ItemIdList()
for i in reason2itemid.item_id:
item_id_list.item_id.append(i)
filter_and_compose_recommend_item_list(user_info, reason, item_id_list, result)
except:
logger.error("bad ItemIdList from get_offline_user_recommend_item_list:%s" % traceback.format_exc())
return result
def new_user_recommend(user_info, count):
""" request recommend for new user which has no favor action(点击、收藏等)
@return (True|False, RecommendItemContainer)
"""
logger.debug("begin new_user_recommend")
uid = user_info.uid
need_global_features = False
result = RecommendItemContainer()
if not user_info.user_feature_info:
for resource_type in ResourceType.RESOURCE_TYPE_LIST:
(errCode, user_feature_list) = get_user_feature_list(uid, resource_type)
if errCode:
if len(user_feature_list.feature):
user_info.user_feature_info[resource_type] = user_feature_list
else:
need_global_features = True
else:
logger.error("fail to get user info,uid:%s" % uid)
return (False, result)
#TODO:may not need to get hot features again
#TODO:may join with old_user recommend
logger.debug("need_global_features:%s" % need_global_features)
if need_global_features:
logger.debug("need global features for new user recommend,uid:%s" % uid)
resouce_feature_name_list = get_resource_hot_feature_list(user_info, count)
if not resouce_feature_name_list:
return (False, result)
logger.debug("resouce_feature_name_list:%s" % resouce_feature_name_list)
#update user feature feature
for resource_type, feature_name_list_proto in resouce_feature_name_list.iteritems():
if resource_type in user_info.user_feature_info and len(user_info.user_feature_info[resource_type].feature):
continue
user_feature_list = UserFeatureList()
for item in feature_name_list_proto.feature:
cur = user_feature_list.feature.add()
cur.feature_name = item.feature_name
cur.visit_info.pv_count = 0
cur.visit_info.click_count = 0
cur.visit_info.weight = item.weight
user_info.user_feature_info[resource_type] = user_feature_list
resouce_feature_name_list = user_info.user_feature_info
for resource_type, feature_name_list_proto in resouce_feature_name_list.iteritems():
#@note:简化逻辑，直接取平均,不考虑余数不足的情况
resource_count = (count+len(resouce_feature_name_list)-1)/len(resouce_feature_name_list)
reason_item_list = get_item_list_by_feature_list(user_info,
resource_type, feature_name_list_proto, resource_count)
logger.info("current size after hot feature list,uid:%s,resource_type:%s,count:%s" % (uid,
resource_type, reason_item_list.size()))
logger.debug("get new_user_recommend, resource_type:%s, result:%s" % (resource_type, reason_item_list))
if not reason_item_list:
continue
add_and_dedup_recommend_item(user_info, reason_item_list, result, count)
if result.size() >= count:
break
return (True, result)
def get_item_list_by_item(user_info, resource_type, count):
""" get item_list by item recommendation
@return RecommendItemContainer
"""
logger.debug("begin get_item_list_by_item")
uid = user_info.uid
result = RecommendItemContainer()
#get favor item_id列表
errCode, kv_dict = model.get_online_user_favor_item_list(uid, [resource_type])
if not is_valid_data(errCode, kv_dict, "online_user_favor_item_list", {"uid":uid, "resource_type":resource_type}):
return result
if not kv_dict:
return result
assert(resource_type in kv_dict)
try:
item_id_list_proto = ItemIdList()
item_id_list_proto.ParseFromString(kv_dict[resource_type])
reason_list = [resource_type+ITEM_RECOMMEND_FEATURE_TYPE+v for v in item_id_list_proto.item_id]
item_id_list = []
for v in item_id_list_proto.item_id:
if not check_reason(user_info, resource_type+ITEM_RECOMMEND_FEATURE_TYPE+v):
continue
item_id_list.append(v)
if not item_id_list:
logger.info("all item id list have no item2item result:%s" % item_id_list)
return result
#@note: use latest item first
item_id_list.reverse()
#recommend by item id
errCode, kv_dict = model.get_offline_item_recommend_item_list(item_id_list)
if not is_valid_data(errCode, kv_dict, "offline_item_recommend_item_list", {"item_id_list":"..."}):
return result
if not kv_dict:
for item_id in item_id_list:
reason = resource_type + ITEM_RECOMMEND_FEATURE_TYPE + item_id
ignore_reason(user_info, reason)
return result
for item_id in item_id_list:
if result.size() >= count:
break
if item_id not in kv_dict:
logger.info("no item2itemid list for id:%s" % item_id)
ignore_reason(user_info, reason)
continue
k = item_id
v = kv_dict[k]
reason = resource_type + ITEM_RECOMMEND_FEATURE_TYPE + k
item_id_list_proto = ItemIdList()
item_id_list_proto.ParseFromString(v)
filter_and_compose_recommend_item_list(user_info, reason, item_id_list_proto, result)
except:
logger.error("bad get_item_list_by_item:%s" % traceback.format_exc())
return result
def get_item_list_by_feature_list(user_info, resource_type, feature_name_list_proto, count):
""" get item list by check with all feature list
@return RecommendItemContainer
"""
logger.debug("begin get_item_list_by_feature_list")
feature_name_list = []
result = RecommendItemContainer()
filtered_feature_name_list_proto = UserFeatureList()
for v in feature_name_list_proto.feature:
feature_name = v.feature_name
if not check_reason(user_info, feature_name):
continue
feature_name_list.append(feature_name)
cur = filtered_feature_name_list_proto.feature.add()
cur.feature_name = feature_name
if not feature_name_list:
logger.info("no new reason from feature list,current feature_name size:%s" % len(feature_name_list_proto.feature))
logger.debug("current feature_name_list_proto:%s" % feature_name_list_proto)
return result
#news like
if ResourceType.RESOURCE_TYPE_NEWS == resource_type:
return get_news_online_item_by_feature_list(user_info, resource_type, filtered_feature_name_list_proto, count)
#items like
errCode, kv_dict = model.get_offline_feature_hot_item_list(feature_name_list)
if not is_valid_data(errCode, kv_dict, "offline_feature_hot_item_list", {"feature_name_list":"..."}):
return result
if not kv_dict:
for k in feature_name_list:
ignore_reason(user_info, k)
return result
try:
#logger.debug("%s" % feature_name_list)
for k in feature_name_list: #@note:ordered
logger.debug("check result with feature_name:%s" % k)
if result.size() >= count:
logger.debug("fetch enough recommend result, count:%s" % count)
break
if k not in kv_dict:
logger.info("no result for feature_name:%s" % k)
ignore_reason(user_info, k)
continue
v = kv_dict[k]
item_id_list = ItemIdList()
item_id_list.ParseFromString(v)
filter_and_compose_recommend_item_list(user_info, k, item_id_list, result)
logger.debug("get_item_list_by_feature_list,result_size:%s, uid:%s, resource_type:%s" % (result.size(), user_info.uid, resource_type))
except:
logger.error("bad ItemIdList for get_item_list_by_feature_list:%s" % traceback.format_exc())
return result
def old_user_recommend(user_info, resource_distribution_info, count):
""" recommend for old user
@return (True|False, RecommendItemContainer)
"""
logger.debug("begin old_user_recommend")
result = RecommendItemContainer()
for resource_type, cur_count in resource_distribution_info.iteritems():
reason_item_list = old_user_recommend_for_resource_type(user_info, resource_type, cur_count)
logger.debug("get old_user_recommend, resource_type:%s, result:%s" % (resource_type, reason_item_list))
if not reason_item_list:
continue
add_and_dedup_recommend_item(user_info, reason_item_list, result, count)
if result.size() >= count:
break
return (True, result)
def get_item_list_by_user_feature(user_info, resource_type, count):
""" get item list by: first get user feature list, then get item list by feature list
@return RecommendItemContainer
"""
logger.debug("begin get_item_list_by_user_feature")
uid = user_info.uid
(errCode, user_feature_list) = get_user_feature_list(uid, resource_type)
if errCode:
user_info.user_feature_info[resource_type] = user_feature_list
result = RecommendItemContainer()
if errCode and len(user_feature_list.feature):
result = get_item_list_by_feature_list(user_info, resource_type,
user_feature_list, count)
if result.size() >= count:
return result
return result
def old_user_recommend_for_news_like(user_info, resource_type, count):
""" get news recommend for old user
@return RecommendItemContainer
"""
logger.debug("begin old_user_recommend_for_news_like")
uid = user_info.uid
assert(count > 0)
(errCode, user_feature_list) = get_user_feature_list(uid, resource_type)
if errCode:
user_info.user_feature_info[resource_type] = user_feature_list
result = RecommendItemContainer()
if errCode and len(user_feature_list.feature):
result = get_news_online_item_by_feature_list(user_info, resource_type, user_feature_list, count)
logger.info("current size after news::features,uid:%s,resource_type:%s,count:%s" % (uid,
resource_type, result.size()))
logger.debug("get old_user_recommend_news::features, resource_type:%s, result:%s" % (resource_type, result))
if result.size() >= count:
logger.info("finish old_user_recommend_for_news_like::features, uid:%s, count:%s" % (uid, result.size()))
return result
logger.info("finish old_user_recommend_for_news_like::recent, uid:%s, count:%s" % (uid, result.size()))
return result
def get_request_recommend(uid, count):
""" interface for request recommend,return ordered reason to RecommedItem list
@return (True|False, [(reason,[RecommendItem])])
"""
try:
logger.info("begin request recommend, uid:%s, count:%s" % (uid,count))
result = RecommendItemContainer()
logger.debug("begin get_request_recommend")
user_info = UserInfo(uid, logger)
#@note:new_user判断依据为是否有过点击/收藏等明确的意图行为;可以有过推送行为
new_user = True
for k,v in user_info.final_resource_visit_info.iteritems():
if v.click_count != 0:
new_user = False
break
if new_user:
errCode, result = new_user_recommend(user_info, count)
logger.info("new_user_recommend:uid:%s,status:%s,result count:%s" % (uid,errCode, result.size()))
else:
resource_distribution_info = get_resource_distribution_info(user_info.final_resource_visit_info, count)
logger.info("resource_distribution_info, uid:%s, resource_distribution_info:%s" % (uid, resource_distribution_info))
if not resource_distribution_info:
logger.error("get resource_count error for user:%s" % uid)
errCode, result = new_user_recommend(user_info, count)
logger.info("new_user_recommend:uid:%s,status:%s,result count:%s" % (uid,errCode, result.size()))
else:
errCode, result = old_user_recommend(user_info, resource_distribution_info, count)
logger.info("old_user_recommend:uid:%s,status:%s,result count:%s" % (uid,errCode, result.size()))
if result.size() < count:
logger.debug("old_user_recommend result:%s" % result)
errCode, new_user_result = new_user_recommend(user_info, count - result.size())
logger.info("patch new_user_recommend:uid:%s,status:%s,result count:%s" % (uid,errCode, new_user_result.size()))
result.update(new_user_result)
update_user_info(user_info, result)
final_result = result.to_list(user_info)
logger.info("request recommend result, uid:%s, expect_count:%s, real_count:%s, result:%s" % (uid,
count, result.size(), result.get_summary_str(final_result)))
return (True, final_result)
except:
logger.error("bad exception in get_request_recommend, exception:%s" % traceback.format_exc())
return (False, {})
def old_user_recommend_for_item(user_info, resource_type, count):
""" get item like recommend for old user
@return RecommendItemContainer
"""
result = RecommendItemContainer()
uid = user_info.uid
logger.debug("begin old_user_recommend_for_item")
#policy *:user item list
result = old_user_recommend_for_item_user2itemlist(user_info, resource_type, result, count)
logger.info("current size after user2itemlist,uid:%s,resource_type:%s,count:%s" % (uid,
resource_type, result.size()))
if result.size() >= count:
return result
#policy *:user favor item list => item list
result = old_user_recommend_for_item_item2itemlist(user_info, resource_type, result, count)
logger.info("current size after item2itemlist,uid:%s,resource_type:%s,count:%s" % (uid,
resource_type, result.size()))
if result.size() >= count:
return result
#policy *:user feature list => item list
result = old_user_recommend_for_item_userfeaturelist2itemlist(user_info, resource_type, result, count)
logger.info("current size after userfeaturelist2itemlist,uid:%s,resource_type:%s,count:%s" % (uid,
resource_type, result.size()))
return result
def raw_get_news_online_item_by_feature_list(user_info, resource_type, feature_name_list_proto,
count, max_recommend_item_count_per_reason, fetch_more_times, backup_feature_count):
""" talk with news online item interface by feature list, @return RecommendItemContainer
@note: should filter visited feature in user_info for user_feature_list, and update user_info
"""
result = RecommendItemContainer()
input = FeatureNameLimitList()
logger.debug("feature list proto:%s" % feature_name_list_proto)
for i in feature_name_list_proto.feature:
if not check_reason(user_info, i.feature_name):
continue
#@note: temp logic to ignore some feature type
continue_out = False
for prefix in NEWS_IGNORE_FEATURE_NAME_PREFIX:
if i.feature_name.startswith(prefix):
logger.info("ignore feature:%s" % i.feature_name)
continue_out = True
break
if continue_out:
continue
cur = input.feature_name_limit.add()
cur.feature_name = i.feature_name
cur.limit = int(count * max_recommend_item_count_per_reason * fetch_more_times)
if len(input.feature_name_limit) >= count + backup_feature_count:
logger.info("break for enough feature count:%s,feature list:%s" %
(len(input.feature_name_limit), input))
break
#logger.debug("feature_name:%s,limit:%s" % (cur.feature_name, cur.limit))
if not input.feature_name_limit:
logger.info("no new feature to get for news:%s, feature_list size:%s" % (resource_type, len(feature_name_list_proto.feature)))
logger.debug("current feature_list :%s" % (feature_name_list_proto))
return result
try:
#socket_out = urllib2.urlopen(NEWS_ONLINE_URL_SEARCH_BY_QUERY, urllib.urlencode(input.SerializeToString()))
socket_out = urllib2.urlopen(NEWS_ONLINE_URL_SEARCH_BY_QUERY, input.SerializeToString(),
timeout=online_clicklog_feedback.NEWS_TIMEOUT_SECONDS)
return_data = socket_out.read()
except:
logger.error("fail to talk with online news server,featurelists:%s,exceptions:%s" %
(input, traceback.format_exc()))
return result
try:
output = NewsFeatureItemList()
output.ParseFromString(return_data)
logger.debug("get news online item list result,user_info:%s,result:%s" % (user_info, output))
except:
logger.error("bad parse news return value:%s" % traceback.format_exc())
return result
for reason_item_id_list in output.feature_item:
if not len(reason_item_id_list.item_feature_info):
continue
reason = reason_item_id_list.feature_name
resource_type = ResourceType.get_resource_type(reason)
item_list = []
for i in reason_item_id_list.item_feature_info:
if i.item_id in user_info.recent_push_item_id_list:
logger.debug("ignore existing item id:%s" % i.item_id)
continue
item = RecommendItem()
item.item_id = i.item_id
item.item_info_json = i.item_info
item.feature_name_list = i.item_feature_list
item_list.append(item)
user_info.recent_push_item_id_list.add(i.item_id)
logger.info("item_list:%s" %item_list)
if len(item_list) >= max_recommend_item_count_per_reason:
logger.debug("real_list size:%s, select part:%s" % (len(reason_item_id_list.item_feature_info),
max_recommend_item_count_per_reason))
break
if not item_list or len(item_list) < max_recommend_item_count_per_reason:
logger.info("not enough data, discard:%s" % (item_list))
ignore_reason(user_info, reason)
#fetch not enough data, discard this reason
continue
result.add(reason, item_list)
add_reason(user_info, reason)
if result.size() >= count:
break
if not result:
logger.info("get empty get_news_online_item_by_feature_list")
return result