def update_topic(uid_list=[]):
if not uid_list:
uid_list = load_uid_list()
fb_flow_text_index_list = get_facebook_flow_text_index_list(load_timestamp())
user_topic_data = get_filter_keywords(fb_flow_text_index_list, uid_list)
user_topic_dict, user_topic_list = topic_classfiy(uid_list, user_topic_data)
user_topic_string = {}
for uid, topic_list in user_topic_list.items():
li = []
for t in topic_list:
li.append(zh_data[name_list.index(t)].decode('utf8'))
user_topic_string[uid] = '&'.join(li)
user_topic = {}
for uid in uid_list:
if uid in user_topic_dict:
user_topic[uid] = {
'filter_keywords': json.dumps(user_topic_data[uid]),
'topic': json.dumps(user_topic_dict[uid]),
'topic_string': user_topic_string[uid]
}
else:
user_topic[uid] = {
'filter_keywords': json.dumps({}),
'topic': json.dumps({}),
'topic_string': ''
}
return save_data2es(user_topic)
def update_keywords(uid_list=[]):
if not uid_list:
uid_list = load_uid_list()
fb_flow_text_index_list = get_facebook_flow_text_index_list(load_timestamp())
keywords_query_body = {
'query':{
"filtered":{
"filter": {
"bool": {
"must": [
{"terms": {"uid": uid_list}},
]
}
}
}
},
'size': MAX_SEARCH_SIZE,
"sort": {"timestamp": {"order": "desc"}},
"fields": ["keywords_dict", "uid"]
}
user_keywords = {}
for index_name in fb_flow_text_index_list:
try:
search_results = es.search(index=index_name, doc_type=facebook_flow_text_index_type, body=keywords_query_body)['hits']['hits']
for item in search_results:
content = item['fields']
uid = content['uid'][0]
if not uid in user_keywords:
user_keywords[uid] = {
'keywords': {}
}
if content.has_key('keywords_dict'):
user_keywords[uid]['keywords'] = merge_dict(user_keywords[uid]['keywords'], json.loads(content['keywords_dict'][0]))
except Exception,e:
print e
def update_domain(uid_list=[]):
if not uid_list:
uid_list = load_uid_list()
fb_flow_text_index_list = get_facebook_flow_text_index_list(
load_timestamp(), TEST_MAX_FLOW_TEXT_DAYS)
user_domain_data = {}
#load num of text
count_result = count_text_num(uid_list, fb_flow_text_index_list)
#load baseinfo
fb_user_query_body = {
'post_filter': {
'exists': {
'field': 'bio_str'
}
},
'query': {
"filtered": {
"filter": {
"bool": {
"must": [
{
"terms": {
"uid": uid_list
}
},
]
}
}
}
},
'size': MAX_SEARCH_SIZE,
"fields": ["bio_str", "category", "uid"]
}
try:
search_results = es.search(index=facebook_user_index_name,
doc_type=facebook_user_index_type,
body=fb_user_query_body)['hits']['hits']
for item in search_results:
content = item['fields']
uid = content['uid'][0]
if not uid in user_domain_data:
text_num = count_result[uid]
user_domain_data[uid] = {
'bio_str': '',
'category': '',
'number_of_text': text_num
}
#对于长文本，Goslate 会在标点换行等分隔处把文本分拆为若干接近 2000 字节的子文本，再一一查询，最后将翻译结果拼接后返回用户。通过这种方式，Goslate 突破了文本长度的限制。
if content.has_key('category'):
category = content.get('category')[0]
else:
category = ''
if content.has_key('bio_str'):
bio_str = content.get('bio_str')[0]
else:
bio_str = ''
user_domain_data[uid]['bio_str'] = bio_str
user_domain_data[uid]['category'] = category
except Exception, e:
print e
def update_sensitive(uid_list=[]):
if not uid_list:
uid_list = load_uid_list()
fb_flow_text_index_list = get_facebook_flow_text_index_list(
load_timestamp(), TEST_MAX_FLOW_TEXT_DAYS)
sensitive_query_body = {
'query': {
"filtered": {
"filter": {
"bool": {
"must": [
{
"terms": {
"uid": uid_list
}
},
]
}
}
}
},
'size': MAX_SEARCH_SIZE,
"sort": {
"timestamp": {
"order": "desc"
}
},
"fields": ["sensitive_words_dict", "sensitive", "uid"]
}
user_sensitive = {}
for index_name in fb_flow_text_index_list:
try:
search_results = es.search(
index=index_name,
doc_type=facebook_flow_text_index_type,
body=sensitive_query_body)['hits']['hits']
for item in search_results:
content = item['fields']
uid = content['uid'][0]
if not uid in user_sensitive:
user_sensitive[uid] = {
'sensitive_dict': {},
'sensitive_list': []
}
if content.has_key('sensitive_words_dict'):
user_sensitive[uid]['sensitive_dict'] = merge_dict(
user_sensitive[uid]['sensitive_dict'],
json.loads(content['sensitive_words_dict'][0]))
if content.has_key('sensitive'):
user_sensitive[uid]['sensitive_list'].append(
float(content.get('sensitive')[0]))
except Exception, e:
print e
def update_sentiment(uid_list=[]):
'''
SENTIMENT_DICT_NEW = {'0':u'中性', '1':u'积极', '2':u'生气', '3':'焦虑', \
'4':u'悲伤', '5':u'厌恶', '6':u'消极其他', '7':u'消极'}
'''
if not uid_list:
uid_list = load_uid_list()
fb_flow_text_index_list = get_facebook_flow_text_index_list(
load_timestamp())
sentiment_query_body = {
'query': {
"filtered": {
"filter": {
"bool": {
"must": [
{
"terms": {
"uid": uid_list
}
},
]
}
}
}
},
'size': MAX_SEARCH_SIZE,
"sort": {
"timestamp": {
"order": "desc"
}
},
"fields": ["sentiment", "uid"]
}
user_sentiment = {}
for index_name in fb_flow_text_index_list:
try:
search_results = es.search(
index=index_name,
doc_type=facebook_flow_text_index_type,
body=sentiment_query_body)['hits']['hits']
for item in search_results:
content = item['fields']
uid = content['uid'][0]
if not uid in user_sentiment:
user_sentiment[uid] = {'sentiment_list': []}
if content.has_key('sentiment'):
user_sentiment[uid]['sentiment_list'].append(
int(content.get('sentiment')[0]))
except Exception, e:
print e
def update_hashtag(uid_list=[]):
if not uid_list:
uid_list = load_uid_list()
fb_flow_text_index_list = get_facebook_flow_text_index_list(
load_timestamp())
keywords_query_body = {
'query': {
"filtered": {
"filter": {
"bool": {
"must": [
{
"terms": {
"uid": uid_list
}
},
]
}
}
}
},
'size': MAX_SEARCH_SIZE,
"sort": {
"timestamp": {
"order": "desc"
}
},
"fields": ["hashtag", "uid"]
}
user_hashtag = {}
for index_name in fb_flow_text_index_list:
try:
search_results = es.search(
index=index_name,
doc_type=facebook_flow_text_index_type,
body=keywords_query_body)['hits']['hits']
for item in search_results:
content = item['fields']
uid = content['uid'][0]
if not uid in user_hashtag:
user_hashtag[uid] = {'hashtag_list': []}
if content.has_key('hashtag'):
hashtag = content['hashtag'][0]
if hashtag:
hashtag_list = hashtag.split('&')
user_hashtag[uid]['hashtag_list'].extend(hashtag_list)
except Exception, e:
print e
def update_domain(uid_list=[]):
if not uid_list:
uid_list = load_uid_list()
fb_flow_text_index_list = get_facebook_flow_text_index_list(load_timestamp())
user_domain_data = {}
#load num of text
count_result = count_text_num(uid_list, fb_flow_text_index_list)
#load baseinfo
fb_user_query_body = {
'query':{
"filtered":{
"filter": {
"bool": {
"must": [
{"terms": {"uid": uid_list}},
]
}
}
}
},
'size': MAX_SEARCH_SIZE,
"fields": ["bio", "about", "description", "quotes", "category", "uid"]
}
try:
search_results = es.search(index=facebook_user_index_name, doc_type=facebook_user_index_type, body=fb_user_query_body)['hits']['hits']
for item in search_results:
content = item['fields']
uid = content['uid'][0]
if not uid in user_domain_data:
text_num = count_result[uid]
user_domain_data[uid] = {
'bio_str': '',
'bio_list': [],
'category': '',
'number_of_text': text_num
}
#对于长文本，Goslate 会在标点换行等分隔处把文本分拆为若干接近 2000 字节的子文本，再一一查询，最后将翻译结果拼接后返回用户。通过这种方式，Goslate 突破了文本长度的限制。
if content.has_key('category'):
category = content.get('category')[0]
else:
category = ''
if content.has_key('description'):
description = content.get('description')[0][:1000] #有的用户描述信息之类的太长了……3000+，没有卵用，而且翻译起来会出现一些问题，截取一部分就行了
else:
description = ''
if content.has_key('quotes'):
quotes = content.get('quotes')[0][:1000]
else:
quotes = ''
if content.has_key('bio'):
bio = content.get('bio')[0][:1000]
else:
bio = ''
if content.has_key('about'):
about = content.get('about')[0][:1000]
else:
about = ''
user_domain_data[uid]['bio_list'] = [quotes, bio, about, description]
user_domain_data[uid]['category'] = category
except Exception,e:
print e
