def Recommend_list(uid, num, end=None, pid=None, lookback=5 * 61.0):
# return : [{'id':111,'type':'topic','title':'xxxx'},{'id':222,'type':'news','title':'yyy'}...]
# ******************************************
# if not IS_ONLINE_WEB_SERVER:
# return TEST_RETURN_Recommend_list
# ******************************************
bad_return = []
log_mark = "recommend_topn"
info_logger.info(
"%s===============start=========uid=%s==============pid=%s===============",
log_mark, uid, str(pid))
# assert uid
try:
uid = int(uid)
except:
info_logger.info("%s=====failed in recommend==bad uid=%s=========",
log_mark, uid)
return bad_return
if uid == -1:
info_logger.info("%s=====failed in recommend==bad uid=%s=========",
log_mark, uid)
return bad_return
# time window
if not end:
end = time.time()
else:
end = ensure_second_timestamp(end)
begin = end - lookback
end += 5.0 # 结束点顺延5s，防止hbase表里还没有实时数据
if pid: # qa触发由传入的problem_id查询信息
user_info0 = one_user_last_qa_info(pid)
else:
user_info0 = cy_time_event_one_user_kernel(uid, begin, end)
res_dict = Recommend_by_user_info(user_info0,
uid,
log_mark=log_mark,
num=num)
res = res_dict['res']
status = res_dict['status']
if not res:
info_logger.info("%s==failed in recommend==%s===uid=%s===========",
log_mark, status, uid)
return bad_return
for item in res:
best_id, title, mtype = item
info_logger.info(
"%s==succeed in recommend===id=%s==title=%s====type=%s===uid=%s===========",
log_mark, best_id, title, mtype, uid)
return [{'id': item[0], 'title': item[1], 'type': item[2]} for item in res]
def Recommend_news(uid, num, solr_first=True, redis_second=True):
bad_return = BAD_TEST_RETURN_Recommend_plan
if not IS_ONLINE_WEB_SERVER:
shuffle(TEST_RETURN_Recommend_news)
return TEST_RETURN_Recommend_news[:num]
log_mark = "recommend_news"
info_logger.info("%s===uid=%s====start====num=%s=======", log_mark, uid,
num)
if uid == -1:
return bad_return
try:
uid = str(uid)
except:
info_logger.info("%s===uid=%s============uid unvalid=======", log_mark,
uid)
return bad_return
# 首先尝试去solr取结果
if solr_first:
solr_res = get_ramdom_topn_news_from_solr(uid, num)
if solr_res:
log_string = '|'.join([str(x) for x in solr_res])
info_logger.info("%s===uid=%s=====solr_newsids=%s============",
log_mark, uid, log_string)
return solr_res[:num]
# 再尝试去redis里取结果
if redis_second:
redis_ids = get_random_topn_news_from_redis(uid, num)
if redis_ids:
log_string = '|'.join([str(x) for x in redis_ids])
info_logger.info("%s===uid=%s=====redis_newsids=%s============",
log_mark, uid, log_string)
return redis_ids[:num]
res = recommend_news_kernel(uid=uid, test=False, num=num + 10)
ids = res['ids']
# 如果使用redis，则将计算结果存入redis，以便下次刷新时使用
if redis_second:
write_user_all_news_to_redis(uid, ids)
# 随机选取num个文章
ids = select_newsid(ids, num)
if redis_second:
write_user_showed_news_to_redis(uid, ids)
log_string = '|'.join([str(x) for x in ids])
info_logger.info("%s===newsids=%s=======uid=%s=============", log_mark,
log_string, uid)
return ids[:num]
def do_one(self, sql, commit=False):
cur = self.conn.cursor()
# print "sql"
# print sql
try:
cur.execute(sql)
return cur.fetchall()
except Exception, e:
print "Exception:", e
info_logger.info("sql exception %s", e)
return None
def Recommend_tags(uid):
# ******************************************
# if uid == -1:
# return BAD_TEST_RETURN_Recommend_tags
# return TEST_RETURN_Recommend_tags
# ******************************************
# 取用户最后一个query
# 返回 最相似疾病词*2，最相似症状词*1，最相似药品词*2（按顺序给出词），和 解决方案
# 别忘了打log
log_mark = "recommend_tag_tag_tag"
info_logger.info("%s===uid=%s====start==========", log_mark, uid)
bad_return = BAD_TEST_RETURN_Recommend_tags
# return TEST_RETURN_Recommend_tags
if uid == -1:
return bad_return
last_query = get_user_last_query(uid)
info_logger.info("%s===uid=%s====last_query=%s=======", log_mark, uid,
last_query)
tags = get_similar_entities(query=last_query)
plans = get_relation_plan(query=last_query)
info_logger.info("%s===tags=%s=======uid=%s=============", log_mark,
'|||'.join(tags), uid)
plans_string = '|||'.join([item['name'] for item in plans])
info_logger.info("%s===plans=%s=======uid=%s=============", log_mark,
plans_string, uid)
return {'words': tags, 'plan': plans}
def Recommend(uid, lookback, end=None, pid=None):
# if not IS_ONLINE_WEB_SERVER:
# return choice(TEST_RETURN)
# recommed top 1
bad_return = [-1, "", "nothing"] # material_id, title, material_type
log_mark = "recommend_one"
info_logger.info(
"%s===============start=========uid=%s==============pid=%s===============",
log_mark, uid, str(pid))
try:
uid = int(uid)
except:
info_logger.info("%s=====failed in recommend==bad uid=%s=========",
log_mark, uid)
return bad_return
if uid == -1:
info_logger.info("%s=====failed in recommend==bad uid=%s=========",
log_mark, uid)
return bad_return
if not end:
end = time.time()
else:
end = ensure_second_timestamp(end)
begin = end - lookback
end += 5.0 # 结束点顺延5s，防止hbase表里还没有实时数据
if pid: # qa触发由传入的problem_id查询信息
user_info0 = one_user_last_qa_info(pid)
else:
user_info0 = cy_time_event_one_user_kernel(uid, begin, end)
res_dict = Recommend_by_user_info(user_info0, uid, log_mark=log_mark)
res = res_dict['res']
status = res_dict['status']
if not res:
info_logger.info("%s==failed in recommend==%s===uid=%s===========",
log_mark, status, uid)
return bad_return
best_id, title, mtype = res[0]
info_logger.info(
"%s==succeed in recommend===id=%s==title=%s====type=%s===uid=%s===========",
log_mark, best_id, title, mtype, uid)
return [int(best_id), title, mtype]
def Recommend_topics(uid, num, timestamp=None, test=False):
# if uid == -1:
# return BAD_TEST_RETURN_Recomend_topics
# return TEST_RETURN_Recomend_topics
log_mark = "recommend_topic_topic"
info_logger.info("%s===uid=%s====start====num=%s=======", log_mark, uid,
num)
topic_ids = Recommend_topics_kernel(uid=uid,
num=num,
timestamp=timestamp,
test=test)
info_logger.info("%s===uid=%s====return====topic_ids=%s=======", log_mark,
uid, topic_ids)
return topic_ids
def cy_time_event_one_user_viewnews(uid, begin, end):
connection = happybase.Connection('hbase_server',
compat='0.90',
port=19090,
timeout=SMALL_TIMEOUT)
table = connection.table("cy_real_time_event")
res = {}
for key, data in table.scan(row_prefix=str(uid) + '|'):
uid, timestamp, event_type = key.split('|')
if event_type not in ["view_news"]:
continue
timestamp = ensure_second_timestamp(timestamp)
info_logger.info("real timestamp=%s", timestamp)
if timestamp >= begin and timestamp <= end:
news_id = int(data["info:news_id"])
res[news_id] = timestamp
return res
def Recommend_plan(uid, num):
bad_return = BAD_TEST_RETURN_Recommend_plan
# if not IS_ONLINE_WEB_SERVER:
# return TEST_RETURN_Recommend_plan
log_mark = "recommend_plan"
info_logger.info("%s===uid=%s====start====num=%s=======", log_mark, uid,
num)
if uid == -1:
return bad_return
try:
uid = str(uid)
except:
info_logger.info("%s===uid=%s============uid unvalid=======", log_mark,
uid)
return bad_return
res = get_relation_plan3(uid, num)
systagids = res.get('ids', [])
status = res['status']
plan_ids = systagid_2_planid(systagids)
plans_string = '|||'.join([str(item) for item in plan_ids])
info_logger.info("%s===plans=%s=======uid=%s======status=%s=======",
log_mark, plans_string, uid, status)
return plan_ids
def is_valid_user_info(user_info):
if len(user_info['tags']) == 0 or len(user_info['vecs']) == 0:
info_logger.info("=failed no tags=======")
return False
return True
def parse_user_info(user_info):
last_event = user_info['last_event']
info_logger.info("last_event %s", str(last_event))
if last_event is None:
return None
timestamp = user_info['last_event_time']
# output = {} # trigger,timestamp,tags,vecs,weights,center,texts,sex,age,special_population
if last_event[0] == 'big_search':
texts = [item[0] for item in user_info['big_search']]
text = ' '.join(texts).strip()
# tags, weights, cates, _ = get_medical_entities_info([text], [1.0], 1.0)
# 改为不仅仅使用医学实体词
tags, weights, cates, counts, entities, entity_counts = get_medical_entities_info2_cyseg(
text_list=[text],
weight_list=[1.0],
weights_is_dict=False # weights以list形式返回
)
vecs, keep_indices = get_vecs_weighted3(tags)
weights = [weights[i] for i in keep_indices]
tags = [tags[i] for i in keep_indices] # 去掉没有向量的tag
entities = [x for x in entities if x in tags] # 去掉没有向量的实体词
center = get_center(vecs)
special_population, sex = special_population_big(text)
age = None
output = {
"trigger": 'big_search',
"timestamp": timestamp, # 这个没有用
"tags": tags,
"vecs": vecs,
"entities": entities,
"weights": weights,
"cates": cates,
"center": center,
"texts": texts,
"sex": sex,
"age": age,
"special_population": special_population,
}
elif last_event[0] == 'free_problem_create':
text, sex, age = last_event[1]
if sex:
s_text = text + ' （%s，%s）' % (sex, age)
else:
s_text = text
text, sex, age = qa_ask_info(text)
special_population, sex = special_population_big(s_text)
print special_population
tags, weights, cates, counts, entities, entity_counts = get_medical_entities_info2_cyseg(
text_list=[text],
weight_list=[1.0],
weights_is_dict=False # weights以list形式返回
)
vecs, keep_indices = get_vecs_weighted3(tags)
weights = [weights[i] for i in keep_indices]
# cates is a dict
tags = [tags[i] for i in keep_indices] # 去掉没有向量的tag
entities = [x for x in entities if x in tags] # 去掉没有向量的实体词
info_logger.info("len vecs %s", len(vecs))
center = get_center(vecs)
output = {
"trigger": 'free_problem_create',
"timestamp": timestamp,
"tags": tags,
"entities": entities,
"vecs": vecs,
"weights": weights,
"cates": cates,
"center": center,
"texts": [text],
"sex": sex,
"age": age,
"special_population": special_population,
}
else:
print "output is None"
output = None
return output
def Recommend_topics_kernel(uid, num, timestamp=None, test=False):
if not test:
bad_return = []
else:
bad_return = [], None, None
log_mark = "recommend_topic_topic"
# step 1 先从solr里找，看有没有被离线计算过，8ms左右
if not test:
solr_res = get_caled_user_topn_topics_yesterday0(uid)
if solr_res is not None:
info_logger.info(
"%s===uid=%s====return==offline_data==topic_ids=%s=======",
log_mark, uid, solr_res)
print 'hehehehe', uid, solr_res
return solr_res
user_info0 = get_yesterday_user_info(uid, timestamp)
if user_info0['last_event'] is None:
info_logger.info("%s==failed in no last event=uid=%s========",
log_mark, uid)
return bad_return
# print json.dumps(user_info0)
t1 = time.time()
user_info = parse_user_info2(user_info0)
t2 = time.time()
print "Recommend_topics_kernel parse_user_info2 time", t2 - t1
texts = user_info['texts']
tags = user_info['tags']
u_weights = user_info['weights']
u_cates = user_info['cates']
special_population = user_info['special_population']
center = None
trigger = user_info['trigger']
u_vecs = user_info['vecs']
if len(tags) == 0:
info_logger.info("%s==failed in no tags=uid=%s========", log_mark, uid)
return bad_return
t3 = time.time()
recall_ids, title_dict, score_dict = recall(
text=' '.join(texts),
tags=tags,
weights=u_weights,
cates=u_cates,
special_population=special_population,
center=center,
trigger_type=trigger,
only_topic=True,
yxjt=True)
t4 = time.time()
print "Recommend_topics_kernel recall time", t4 - t3
info_logger.info((log_mark + "===recall_ids", recall_ids))
t5 = time.time()
topn_ids_scores, title_dict, v_score_dict = rank(
uid=uid,
recall_ids=recall_ids,
title_dict=title_dict,
solr_score_dict=score_dict,
u_vecs=u_vecs,
u_weights=u_weights,
u_tags=tags,
u_cates=u_cates,
keep=num)
t6 = time.time()
print "Recommend_topics_kernel parse_user_info2 rank", t6 - t5
threshhold = trigger_threshhold(trigger)
if test:
return [
int(item[0].split('_')[-1]) for item in topn_ids_scores
if v_score_dict[item[0]] >= threshhold
], user_info, v_score_dict
return [
int(item[0].split('_')[-1]) for item in topn_ids_scores
if v_score_dict[item[0]] >= threshhold
]
def Recommend_by_user_info(user_info0, uid, log_mark, num=1, test=False):
if test:
bad_return = {
"user_info": None,
"res": None,
"topn_ids_scores": None,
"only_topic": None,
"v_score_dict": None
}
else:
bad_return = {"res": None}
t1 = time.time()
user_info = parse_user_info(user_info0)
t2 = time.time()
print "Recommend_by_user_info parse_user_info time", t2 - t1
if user_info is None:
bad_return["status"] = "user info is None"
return bad_return
if not is_valid_user_info(user_info):
bad_return["status"] = "not valid user info"
return bad_return
only_topic = False
if not filter_user_info(user_info):
only_topic = True
texts = user_info["texts"]
tags = user_info["tags"]
special_population = user_info["special_population"]
center = user_info["center"]
u_vecs = user_info["vecs"]
u_weights = user_info["weights"]
u_cates = user_info["cates"]
trigger = user_info["trigger"]
info_logger.info("%s=texts=%s======uid=%s===============", log_mark,
'|||'.join(texts), uid)
info_logger.info("%s=tags=%s======uid=%s===============", log_mark,
'|||'.join(tags), uid)
info_logger.info("%s=special_population=%s======uid=%s===============",
log_mark, special_population, uid)
info_logger.info("%s=trigger=%s======uid=%s===============", log_mark,
trigger, uid)
info_logger.info("%s=only_topic=%s======uid=%s===============", log_mark,
only_topic, uid)
###############
if trigger not in ("big_search", "free_problem_create"):
bad_return["status"] = "trigger_type not bs or qa"
return bad_return
################
# 召回
t3 = time.time()
recall_ids, title_dict, score_dict = recall(
text=' '.join(texts),
tags=tags,
weights=u_weights,
cates=u_cates,
special_population=special_population,
center=center,
trigger_type=trigger,
only_topic=only_topic)
t4 = time.time()
print "Recommend_by_user_info recall time", t4 - t3
info_logger.info("%s=recall_ids=%s===============uid=%s========", log_mark,
'-'.join(recall_ids), uid)
# 排序
t5 = time.time()
topn_ids_scores, title_dict, v_score_dict = rank(
uid=uid,
recall_ids=recall_ids,
title_dict=title_dict,
solr_score_dict=score_dict,
u_vecs=u_vecs,
u_weights=u_weights,
u_tags=tags,
u_cates=u_cates,
keep=num + 9)
t6 = time.time()
print "Recommend_by_user_info rank time", t6 - t5
info_logger.info("%s=topn_ids_scores_len=%s================uid=%s========",
log_mark, len(topn_ids_scores), uid)
if len(topn_ids_scores) == 0:
bad_return["status"] = "topn_ids_scores empty" # 往往因为物料不足或者召回量过低导致
return bad_return
t7 = time.time()
best = aftertreatment(topn_ids_scores, {}, special_population, num)
t8 = time.time()
print "Recommend_by_user_info aftertreatment time", t8 - t7
if not best:
bad_return["status"] = "no best"
return bad_return
# get threshhold
if not test:
threshhold = trigger_threshhold(trigger)
else:
threshhold = -9999999.9
best_id, _ = best[0]
best_score = v_score_dict[best_id]
if best_score < threshhold: # 最高分低于阈值
bad_return["status"] = "best_score too low"
return bad_return
res = [[
int(item[0].split('_')[1]),
title_dict.get(item[0], u''), item[0].split('_')[0]
] for item in best] # [[111,title1,"news"],[222,title2],...]
if not test:
return {"res": res, "status": "succeed"}
else:
return {
"res": res,
"user_info": user_info,
"topn_ids_scores": topn_ids_scores,
"only_topic": only_topic,
"status": "succeed",
"v_score_dict": v_score_dict
}