def fetch(self, logger):
"""update user info, return None"""
uid = self.uid
#offline_resource_visit_info
try:
errCode, kv_dict = model.get_offline_user_resource_visitinfo(uid, ResourceType.RESOURCE_TYPE_LIST)
if not is_valid_data(errCode, kv_dict, "offline_user_resource_visitinfo", {"uid":uid, "resource_type_list":"..."}):
pass
for k, v in kv_dict.iteritems():
self.offline_resource_visit_info[k] = VisitInfo()
self.offline_resource_visit_info[k].ParseFromString(v)
except:
logger.error("bad offline_resource_visit_info:%s" % traceback.format_exc())
#recent_resource_visit_info
try:
errCode, kv_dict = model.get_online_user_resource_visitinfo(uid, ResourceType.RESOURCE_TYPE_LIST)
if not is_valid_data(errCode, kv_dict, "online_user_resource_visitinfo", {"uid":uid, "resource_type_list":"..."}):
pass
for k, v in kv_dict.iteritems():
self.recent_resource_visit_info[k] = VisitInfo()
self.recent_resource_visit_info[k].ParseFromString(v)
#final_resource_visit_info
self.final_resource_visit_info = UserInfo.merge_offline_recent_resource_visit_info(self.offline_resource_visit_info,
self.recent_resource_visit_info)
except:
logger.error("bad recent_resource_visit_info:%s" % traceback.format_exc())
self.final_resource_visit_info = self.offline_resource_visit_info
#pushed_item_id_list
try:
errCode, v = model.get_online_user_push_item_list(uid)
if not is_valid_data(errCode, v, "online_user_push_item_list", {"uid":uid}):
pass
if v:
item_id_list_proto = ItemIdList()
item_id_list_proto.ParseFromString(v)
self.recent_push_item_id_list = set()
self.recent_push_item_id_list.update(item_id_list_proto.item_id)
except:
logger.error("bad pushed_item_id_list:%s" % traceback.format_exc())
#pushed_feature_list
try:
errCode, v = model.get_online_user_feature_reason(uid)
if not is_valid_data(errCode, v, "online_user_feature_reason", {"uid":uid}):
pass
if v:
feature_list_proto = ItemFeatureList()
feature_list_proto.ParseFromString(v)
for u in feature_list_proto.feature:
self.recent_push_reason_info[u.feature_name] = u.weight
except:
logger.error("bad pushed_feature_list:%s" % traceback.format_exc())
logger.debug("user_info:%s" % self)
return None
def get_item_list_by_item(user_info, resource_type, count):
""" get item_list by item recommendation
@return RecommendItemContainer
"""
logger.debug("begin get_item_list_by_item")
uid = user_info.uid
result = RecommendItemContainer()
#get favor item_id列表
errCode, kv_dict = model.get_online_user_favor_item_list(uid, [resource_type])
if not is_valid_data(errCode, kv_dict, "online_user_favor_item_list", {"uid":uid, "resource_type":resource_type}):
return result
if not kv_dict:
return result
assert(resource_type in kv_dict)
try:
item_id_list_proto = ItemIdList()
item_id_list_proto.ParseFromString(kv_dict[resource_type])
reason_list = [resource_type+ITEM_RECOMMEND_FEATURE_TYPE+v for v in item_id_list_proto.item_id]
item_id_list = []
for v in item_id_list_proto.item_id:
if not check_reason(user_info, resource_type+ITEM_RECOMMEND_FEATURE_TYPE+v):
continue
item_id_list.append(v)
if not item_id_list:
logger.info("all item id list have no item2item result:%s" % item_id_list)
return result
#@note: use latest item first
item_id_list.reverse()
#recommend by item id
errCode, kv_dict = model.get_offline_item_recommend_item_list(item_id_list)
if not is_valid_data(errCode, kv_dict, "offline_item_recommend_item_list", {"item_id_list":"..."}):
return result
if not kv_dict:
for item_id in item_id_list:
reason = resource_type + ITEM_RECOMMEND_FEATURE_TYPE + item_id
ignore_reason(user_info, reason)
return result
for item_id in item_id_list:
if result.size() >= count:
break
if item_id not in kv_dict:
logger.info("no item2itemid list for id:%s" % item_id)
ignore_reason(user_info, reason)
continue
k = item_id
v = kv_dict[k]
reason = resource_type + ITEM_RECOMMEND_FEATURE_TYPE + k
item_id_list_proto = ItemIdList()
item_id_list_proto.ParseFromString(v)
filter_and_compose_recommend_item_list(user_info, reason, item_id_list_proto, result)
except:
logger.error("bad get_item_list_by_item:%s" % traceback.format_exc())
return result
def get_item_list_by_user(user_info, resource_type, count):
""" get item list by user from offline recommendation
@return RecommendItemContainer
"""
logger.debug("begin get_item_list_by_user")
uid = user_info.uid
result = RecommendItemContainer()
errCode, kv_dict = model.get_offline_user_recommend_item_list(uid, [resource_type])
if not is_valid_data(errCode, kv_dict, "offline_user_recommend_item_list", {"uid":uid, "resource_type":resource_type}):
return result
if not kv_dict:
return result
try:
assert(resource_type in kv_dict)
reason2itemidlist = FeatureNameItemIdList()
reason2itemidlist.ParseFromString(kv_dict[resource_type])
for reason2itemid in reason2itemidlist.feature_name_item_id_list:
if result.size() >= count:
logger.debug("break for enough result")
break
reason = reason2itemid.feature_name
item_id_list = ItemIdList()
for i in reason2itemid.item_id:
item_id_list.item_id.append(i)
filter_and_compose_recommend_item_list(user_info, reason, item_id_list, result)
except:
logger.error("bad ItemIdList from get_offline_user_recommend_item_list:%s" % traceback.format_exc())
return result
def update_user_recent_resource_visit_info(uid, resource_type, favor_type):
""" update user recent resource visit info, @return True|False
"""
errCode, kv_dict = model.get_online_user_resource_visitinfo(uid, [resource_type])
if not is_valid_data(errCode, kv_dict, "online_user_resource_visitinfo", {"uid":uid, "resource_type":resource_type}):
return False
logger.debug("resource_type:%s, kv_dict:%s" % (resource_type, kv_dict))
visit_info_proto = VisitInfo()
try:
if kv_dict:
assert(resource_type in kv_dict)
visit_info_proto.ParseFromString(kv_dict[resource_type])
logger.debug("resource_visit info,resource_type:%s,visit_info:%s" % (resource_type, visit_info_proto))
except:
logger.error("bad ItemIdList from get_online_user_favor_item_list:uid:%s, resource_type:%s, exception:%s" % (uid, resource_type, traceback.format_exc()))
return False
#TODO:define constants outside
if favor_type in ["click", "like"]:
logger.info("update uid:%s, click_count:%s" % (uid, visit_info_proto.click_count))
visit_info_proto.click_count += 1
elif favor_type == "show":
visit_info_proto.pv_count += 1
else:
logger.warn("ignore unknown type:%s" % favor_type)
return False
errCode = model.set_online_user_resource_visitinfo(uid, resource_type, visit_info_proto.SerializeToString())
if not errCode:
logger.warn("fail to update_user_recent_resource_visit_info, uid:%s, resource_type:%s" % (uid, resource_type))
else:
logger.info("successfully to update_user_recent_resource_visit_info, uid:%s, resource_type:%s" % (uid, resource_type))
return errCode
def update_user_favor_item_id_list(uid, resource_type, item_id, favor_type):
""" update user favor item id list, @return True|False
"""
resource_type = ResourceType.get_resource_type(item_id)
assert(resource_type)
errCode, kv_dict = model.get_online_user_favor_item_list(uid, [resource_type])
if not is_valid_data(errCode, kv_dict, "online_user_favor_item_list", {"uid":uid, "resource_type":resource_type}):
return False
logger.debug("resource_type:%s, kv_dict:%s" % (resource_type, kv_dict))
item_id_list_proto = ItemIdList()
try:
if kv_dict:
assert(resource_type in kv_dict)
item_id_list_proto.ParseFromString(kv_dict[resource_type])
logger.debug("favor item_id_list:%s" % item_id_list_proto)
except:
logger.error("bad ItemIdList from get_online_user_favor_item_list:uid:%s, resource_type:%s, exception:%s" % (uid, resource_type, traceback.format_exc()))
return False
item_id_list_proto.item_id.append(item_id)
limit_protobuf_repeated_scala_field_size(item_id_list_proto, "item_id", MAX_FAVOR_ITEM_LIST_COUNT_PER_PERSON)
logger.debug("new item_id_list:%s" % item_id_list_proto)
errCode = model.set_online_user_favor_item_list(uid, resource_type, item_id_list_proto.SerializeToString())
if not errCode:
logger.warn("fail to set_online_user_favor_item_list, uid:%s, resource_type:%s" % (uid, resource_type))
else:
logger.info("successfully to set_online_user_favor_item_list, uid:%s, resource_type:%s" % (uid, resource_type))
return errCode
def get_item_list_by_feature_list(user_info, resource_type, feature_name_list_proto, count):
""" get item list by check with all feature list
@return RecommendItemContainer
"""
logger.debug("begin get_item_list_by_feature_list")
feature_name_list = []
result = RecommendItemContainer()
filtered_feature_name_list_proto = UserFeatureList()
for v in feature_name_list_proto.feature:
feature_name = v.feature_name
if not check_reason(user_info, feature_name):
continue
feature_name_list.append(feature_name)
cur = filtered_feature_name_list_proto.feature.add()
cur.feature_name = feature_name
if not feature_name_list:
logger.info("no new reason from feature list,current feature_name size:%s" % len(feature_name_list_proto.feature))
logger.debug("current feature_name_list_proto:%s" % feature_name_list_proto)
return result
#news like
if ResourceType.RESOURCE_TYPE_NEWS == resource_type:
return get_news_online_item_by_feature_list(user_info, resource_type, filtered_feature_name_list_proto, count)
#items like
errCode, kv_dict = model.get_offline_feature_hot_item_list(feature_name_list)
if not is_valid_data(errCode, kv_dict, "offline_feature_hot_item_list", {"feature_name_list":"..."}):
return result
if not kv_dict:
for k in feature_name_list:
ignore_reason(user_info, k)
return result
try:
#logger.debug("%s" % feature_name_list)
for k in feature_name_list: #@note:ordered
logger.debug("check result with feature_name:%s" % k)
if result.size() >= count:
logger.debug("fetch enough recommend result, count:%s" % count)
break
if k not in kv_dict:
logger.info("no result for feature_name:%s" % k)
ignore_reason(user_info, k)
continue
v = kv_dict[k]
item_id_list = ItemIdList()
item_id_list.ParseFromString(v)
filter_and_compose_recommend_item_list(user_info, k, item_id_list, result)
logger.debug("get_item_list_by_feature_list,result_size:%s, uid:%s, resource_type:%s" % (result.size(), user_info.uid, resource_type))
except:
logger.error("bad ItemIdList for get_item_list_by_feature_list:%s" % traceback.format_exc())
return result
def get_item_features_for_item(resource_type, item_id):
""" get item features list proto for item, fail if item_id not exist or other failure
@return errCode, item_feature_list_proto
"""
errCode, data = model.get_offline_item_features([item_id])
if not is_valid_data(errCode, data, "offline_item_features", {"item_id":item_id}):
return False, None
try:
assert(item_id in data)
result = ItemFeatureList()
result.ParseFromString(data[item_id])
return True, result
except:
logger.error("bad get_offline_item_features:%s" % traceback.format_exc())
return False, None
def convert_user_feature_list(errCode, kv_dict, uid, resource_type, desc):
""" convert user feature list kv_dict to user_feature_list_proto
@return (errCode, user_feature_list_proto)
"""
if not is_valid_data(errCode, kv_dict, desc, {"uid":uid, "resource_type":resource_type}):
return (False, None)
if not kv_dict:
return (True, UserFeatureList())
result = UserFeatureList()
assert(resource_type in kv_dict)
try:
result.ParseFromString(kv_dict[resource_type])
except:
logger.error("bad convert_user_feature_list:%s" % traceback.format_exc())
return False, result
return True, result
def update_user_recent_feature_list(uid, resource_type, item_id, card_feature, favor_type):
""" update user feature list, @return True|False
"""
errCode, kv_dict = model.get_online_user_feature_list(uid, [resource_type])
if not is_valid_data(errCode, kv_dict, "online_user_feature_list", {"uid":uid, "resource_type":resource_type}):
return False
user_feature_list = UserFeatureList()
try:
if kv_dict:
assert(resource_type in kv_dict)
user_feature_list.ParseFromString(kv_dict[resource_type])
logger.debug("user_feature_list:%s" % user_feature_list)
except:
logger.error("bad get_online_user_feature_list:%s" % traceback.format_exc())
return False
errCode, item_feature_list_proto = get_item_features(resource_type, item_id)
logger.debug("resource_type:%s, item_id:%s, user_feature_list:%s, item_feature_list:%s, card_feature:%s, favor_type:%s" % (
resource_type, item_id, user_feature_list, item_feature_list_proto, card_feature, favor_type))
if not errCode:
logger.info("fail to get item feature list for click feedback:%s" % item_id)
#still continue process for update by card_feature
return raw_update_user_action(uid, resource_type, user_feature_list, item_feature_list_proto, card_feature, favor_type)
def get_resource_hot_feature_list(user_info, count):
""" get resource hot feature list, maybe top level category for per resource type, filter existing feature_name list pushed to user
@return {resource_type:feature_name_list}
"""
#@note:下期推荐策略：从最重要的resource开始.这里均匀选取
#1.确定各个resource_type的数量比例:均匀选取
#2.取每个resource的feature list按照重要性从高到低排列
result = {}
errCode, kv_dict = model.get_offline_global_hot_feature_list(ResourceType.RESOURCE_TYPE_LIST)
if not is_valid_data(errCode, kv_dict, "global_hot_feature_list", {"resource_type_list":"..."}):
return {}
if not kv_dict:
return {}
try:
for k,v in kv_dict.iteritems():
item_feature_list = ItemFeatureList()
item_feature_list.ParseFromString(v)
result[k] = item_feature_list
logger.debug("resource_type:%s, hot_feature_list:%s" % (k, result[k]))
except:
logger.error("bad get_offline_global_hot_feature_list:%s" % traceback.format_exc())
logger.info("input resource_type_size:%s, result resource_type_size:%s" % (len(ResourceType.RESOURCE_TYPE_LIST), len(kv_dict)))
logger.debug("get_resource_hot_feature_list, user_info:%s, count:%s, result:%s" % (user_info, count, result))
return result
