def get_save_step_two(task_detail):
#update
user_no_max = get_tw_xnr_no()
user_no_current = user_no_max + 1
r.set(tw_xnr_max_no, user_no_current)
task_detail['user_no'] = user_no_current
task_id = user_no2tw_id(user_no_current) #五位数 TXNR0001
item_exist = dict()
item_exist['submitter'] = task_detail['submitter']
item_exist['user_no'] = task_detail['user_no']
item_exist['domain_name'] = task_detail['domain_name']
item_exist['role_name'] = task_detail['role_name']
item_exist['psy_feature'] = '&'.join(task_detail['psy_feature'].encode('utf-8').split('，'))
item_exist['political_side'] = task_detail['political_side']
item_exist['business_goal'] = '&'.join(task_detail['business_goal'].encode('utf-8').split('，'))
# item_exist['daily_interests'] = '&'.join(task_detail['daily_interests'].encode('utf-8').split('，'))
item_exist['monitor_keywords'] = ','.join(task_detail['monitor_keywords'].encode('utf-8').split('，'))
item_exist['active_time'] = '&'.join(task_detail['active_time'].split('-'))
item_exist['day_post_average'] = json.dumps(task_detail['day_post_average'].split('-'))
item_exist['create_status'] = 1 # 第二步完成
item_exist['xnr_user_no'] = task_id # 虚拟人编号
item_exist['create_time'] = int(time.time())
print es.index(index=tw_xnr_index_name,doc_type=tw_xnr_index_type,id=task_id,body=item_exist)
mark = True
return mark,task_id
def get_add_opinion_corpus(task_detail):
mark = False
corpus_name = task_detail['corpus_name']
corpus_pinyin = pinyin.get(corpus_name, format='strip', delimiter='_')
item_dict = {}
item_dict['corpus_name'] = corpus_name
item_dict['corpus_pinyin'] = corpus_pinyin
item_dict['submitter'] = task_detail['submitter']
item_dict['status'] = 0
try:
es.get(index=opinion_corpus_index_name,
doc_type=opinion_corpus_index_type,
id=corpus_pinyin)
return 'exists'
except:
try:
es.index(index=opinion_corpus_index_name,
doc_type=opinion_corpus_index_type,
body=item_dict,
id=corpus_pinyin)
mark = True
r_r.lpush(opinion_expand_task_queue_name, json.dumps(item_dict))
except:
pass
return mark
def domain_create_task(domain_name,
create_type,
create_time,
submitter,
description,
remark,
compute_status=0):
task_id = pinyin.get(domain_name, format='strip', delimiter='_')
try:
es.get(index=weibo_domain_index_name,
doc_type=weibo_domain_index_type,
id=task_id)['_source']
return 'domain name exists!'
except:
try:
domain_task_dict = dict()
#domain_task_dict['xnr_user_no'] = xnr_user_no
domain_task_dict['domain_pinyin'] = pinyin.get(domain_name,
format='strip',
delimiter='_')
domain_task_dict['domain_name'] = domain_name
domain_task_dict['create_type'] = json.dumps(create_type)
domain_task_dict['create_time'] = create_time
domain_task_dict['submitter'] = submitter
domain_task_dict['description'] = description
domain_task_dict['remark'] = remark
domain_task_dict['compute_status'] = compute_status
r.lpush(weibo_target_domain_detect_queue_name,
json.dumps(domain_task_dict))
item_exist = dict()
#item_exist['xnr_user_no'] = domain_task_dict['xnr_user_no']
item_exist['domain_pinyin'] = domain_task_dict['domain_pinyin']
item_exist['domain_name'] = domain_task_dict['domain_name']
item_exist['create_type'] = domain_task_dict['create_type']
item_exist['create_time'] = domain_task_dict['create_time']
item_exist['submitter'] = domain_task_dict['submitter']
item_exist['description'] = domain_task_dict['description']
item_exist['remark'] = domain_task_dict['remark']
item_exist['group_size'] = ''
item_exist['compute_status'] = 0 # 存入创建信息
es.index(index=weibo_domain_index_name,
doc_type=weibo_domain_index_type,
id=item_exist['domain_pinyin'],
body=item_exist)
mark = True
except:
mark = False
return mark
def get_save_step_three_2(task_detail):
task_id = task_detail['task_id']
# nick_name = task_detail['nick_name']
try:
item_fans_followers = dict()
followers_uids = list(set(task_detail['followers_uids'].split('，')))
item_fans_followers['followers_list'] = followers_uids
item_fans_followers['xnr_user_no'] = task_id
print es.index(index=fb_xnr_fans_followers_index_name,
doc_type=fb_xnr_fans_followers_index_type,
id=task_id,
body=item_fans_followers)
#把关注任务加到redis队列中
for followers_uid in followers_uids:
queue_dict = {
'channel': 'facebook',
'operate_type': 'add',
'content': {
'xnr_user_no': task_id,
'uid': followers_uid
}
}
if not add_operate2redis(queue_dict):
mark = False
return mark
mark = True
except:
mark = False
return mark
def addto_warning_corpus(task_detail):
flow_text_index_name = flow_text_index_name_pre + ts2datetime(task_detail['timestamp'])
try:
corpus_result = es_flow_text.get(index=flow_text_index_name,doc_type=flow_text_index_type,id=task_detail['mid'])['_source']
corpus_result['xnr_user_no'] = task_detail['xnr_user_no']
corpus_result['warning_source'] = task_detail['warning_source']
corpus_result['create_time'] = task_detail['create_time']
corpus_result['validity'] = 1
corpus_result['nick_name'] = get_user_nickname(task_detail['uid'])
#查询关注列表
lookup_type = 'followers_list'
followers_list = lookup_xnr_fans_followers(task_detail['xnr_user_no'],lookup_type)
set_mark = set_intersection(task_detail['uid'],followers_list)
if set_mark > 0:
corpus_result['content_type']='follow'
else:
corpus_result['content_type']='unfollow'
es_xnr.index(index=weibo_warning_corpus_index_name,doc_type=weibo_warning_corpus_index_type,id=task_detail['mid'],body=corpus_result)
mark=True
except:
mark=False
return mark
def save_sent_msg(self, m, to_puid, to_name):
data = {
'msg_type': m.type,
'text': m.text,
'timestamp': int(time.mktime(m.create_time.timetuple())),
'xnr_id': self.self.puid,
'xnr_name': self.self.name,
'group_id': to_puid,
'group_name': to_name,
'speaker_id': self.self.puid,
'speaker_name': self.self.name
}
nowDate = datetime.datetime.now().strftime('%Y-%m-%d')
#虚拟人发送的消息在wx_sent_group_message中存一份
sent_index_name = wx_sent_group_message_index_name_pre + str(nowDate)
if not es_xnr.indices.exists(index=sent_index_name):
print 'get mapping'
print wx_group_message_mappings(sent_index_name)
print es_xnr.index(index=sent_index_name,
doc_type=wx_group_message_index_type,
body=data)
#虚拟人发送的消息在wx_group_message中再存一份
index_name = wx_group_message_index_name_pre + str(nowDate)
if not es_xnr.indices.exists(index=index_name):
print 'get mapping'
print wx_group_message_mappings(index_name)
print es_xnr.index(index=index_name,
doc_type=wx_group_message_index_type,
body=data)
def get_save_step_three_2(task_detail):
task_id = task_detail['task_id']
nick_name = task_detail['nick_name']
#query_body = {'query':{'term':{'nick_name':nick_name}},'sort':{'user_no':{'order':'desc'}}}
#es_result = es.search(index=weibo_xnr_index_name,doc_type=weibo_xnr_index_type,body=query_body)['hits']['hits']
#task_id = es_result[0]['_source']['xnr_user_no']
#插入 weibo_xnr_fans_followers表
try:
item_fans_followers = dict()
#followers_nickname_list = task_detail['followers_nickname'].encode('utf-8').split('，')
#print 'followers_nickname_list::',followers_nickname_list
#followers_list = nickname2uid(followers_nickname_list)
#print 'followers_list::',followers_list
followers_uids = list(set(task_detail['followers_uids'].split('，')))
# print 'followers_uids::',followers_uids
item_fans_followers['followers_list'] = followers_uids
item_fans_followers['xnr_user_no'] = task_id
# print 'item_fans_followers::',item_fans_followers
es.index(index=weibo_xnr_fans_followers_index_name,
doc_type=weibo_xnr_fans_followers_index_type,
id=task_id,
body=item_fans_followers)
mark = True
except:
mark = False
return mark
def addto_weibo_corpus(task_detail):
corpus_type = task_detail[0]
theme_daily_name = task_detail[1]
text = task_detail[2]
uid = task_detail[3]
mid = task_detail[4]
timestamp = task_detail[5]
retweeted = task_detail[6]
comment = task_detail[7]
like = task_detail[8]
create_type = task_detail[9]
xnr_user_no = task_detail[10]
create_time = int(time.time())
corpus_id = task_detail[4] #mid
try:
es_xnr.index(index=weibo_xnr_corpus_index_name,doc_type=weibo_xnr_corpus_index_type,id=corpus_id,\
body={'corpus_type':corpus_type,'theme_daily_name':theme_daily_name,'text':text,'uid':uid,'mid':mid,\
'timestamp':timestamp,'retweeted':retweeted,'comment':comment,'like':like,'create_type':create_type,\
'create_time':create_time,'xnr_user_no':xnr_user_no})
result = True
except:
result = False
return result
def get_save_step_one(task_detail):
es_results = es.search(index=weibo_xnr_index_name,doc_type=weibo_xnr_index_type,body={'query':{'match_all':{}},\
'sort':{'user_no':{'order':'desc'}}})['hits']['hits']
if es_results:
user_no_max = es_results[0]['_source']['user_no']
user_no_current = user_no_max + 1
else:
user_no_current = 1
task_detail['user_no'] = user_no_current
task_id = user_no2_id(user_no_current) #五位数 WXNR0001
try:
item_exist = dict()
item_exist['user_no'] = task_detail['user_no']
item_exist['domain_name'] = task_detail['domain_name']
item_exist['role_name'] = task_detail['role_name']
item_exist['psy_feature'] = '&'.join(task_detail['psy_feature'].encode('utf-8').split('，'))
item_exist['political_side'] = task_detail['political_side']
item_exist['business_goal'] = '&'.join(task_detail['business_goal'].encode('utf-8').split('，'))
item_exist['daily_interests'] = '&'.join(task_detail['daily_interests'].encode('utf-8').split('，'))
item_exist['monitor_keywords'] = '&'.join(task_detail['monitor_keywords'].encode('utf-8').split('，'))
item_exist['create_status'] = 0 # 第一步完成
es.index(index=weibo_xnr_index_name,doc_type=weibo_xnr_index_type,id=task_id,body=item_exist)
mark = True
except:
mark = False
return mark
def report_warming_content(report_info, user_info, weibo_info):
report_dict = dict()
report_dict['report_type'] = report_info[0]
report_dict['report_time'] = int(report_info[1])
report_dict['xnr_user_no'] = report_info[2]
report_dict['event_name'] = report_info[3]
report_dict['uid'] = report_info[4]
report_id = report_info[2] + '_' + str(report_info[1])
#对用户信息进行
user_list = []
if user_info:
#print 'aaaaaa'
user_info_item = user_info.encode('utf-8').split('*')
for user_item in user_info_item:
user_detail = user_item.split(',')
user_dict = dict()
user_dict['uid'] = user_detail[0]
user_dict['nick_name'] = user_detail[1]
user_dict['fansnum'] = user_detail[2]
user_dict['friendsnum'] = user_detail[3]
user_list.append(user_dict)
#对微博信息进行处理
weibo_list = []
if weibo_info:
#print 'bbbbbb'
#print 'weibo_info:::',weibo_info
weibo_info_item = weibo_info.split('*')
#print weibo_info_item
for weibo_item in weibo_info_item:
#print 'weibo_item：：：',weibo_item
weibo_detail = weibo_item.split(',')
weibo_dict = dict()
weibo_dict['mid'] = weibo_detail[0]
weibo_dict['text'] = weibo_detail[1]
weibo_dict['timestamp'] = weibo_detail[2]
weibo_dict['retweeted'] = weibo_detail[3]
weibo_dict['like'] = weibo_detail[4]
weibo_dict['comment'] = weibo_detail[5]
weibo_dict['sensitive'] = weibo_detail[6]
weibo_dict['sensitive_words_string'] = weibo_detail[7]
weibo_list.append(weibo_dict)
report_content = dict()
report_content['user_list'] = user_list
report_content['weibo_list'] = weibo_list
report_dict['report_content'] = json.dumps(report_content)
try:
es_xnr.index(index=weibo_report_management_index_name,
doc_type=weibo_report_management_index_type,
id=report_id,
body=report_dict)
mark = True
except:
mark = False
return mark
def add_xnr_map_relationship(xnr_map_detail):
xnr_map_id = xnr_map_detail['main_user'] + '_' + str(
xnr_map_detail['timestamp'])
try:
es.index(index=xnr_map_index_name,
doc_type=xnr_map_index_type,
body=xnr_map_detail,
id=xnr_map_id)
result = True
except:
result = False
return result
def domain_create_task(domain_name,
create_type,
create_time,
submitter,
description,
remark,
compute_status=0):
task_id = pinyin.get(domain_name, format='strip', delimiter='_')
try:
es.get(index=fb_domain_index_name,
doc_type=fb_domain_index_type,
id=task_id)['_source']
return 'domain name exists!'
except:
try:
domain_task_dict = dict()
domain_task_dict['domain_pinyin'] = pinyin.get(domain_name,
format='strip',
delimiter='_')
domain_task_dict['domain_name'] = domain_name
domain_task_dict['create_type'] = json.dumps(create_type)
domain_task_dict['create_time'] = create_time
domain_task_dict['submitter'] = submitter
domain_task_dict['description'] = description
domain_task_dict['remark'] = remark
domain_task_dict['compute_status'] = compute_status
# print 'domain_task_dict'
# print domain_task_dict
# print 'before: r.lrange'
# print r.lrange(fb_target_domain_detect_queue_name,0,100)
r.lpush(fb_target_domain_detect_queue_name,
json.dumps(domain_task_dict))
# print 'after: r.lrange'
# print r.lrange(fb_target_domain_detect_queue_name,0,100)
item_exist = dict()
item_exist['domain_pinyin'] = domain_task_dict['domain_pinyin']
item_exist['domain_name'] = domain_task_dict['domain_name']
item_exist['create_type'] = domain_task_dict['create_type']
item_exist['create_time'] = domain_task_dict['create_time']
item_exist['submitter'] = domain_task_dict['submitter']
item_exist['description'] = domain_task_dict['description']
item_exist['remark'] = domain_task_dict['remark']
item_exist['group_size'] = ''
item_exist['compute_status'] = 0 # 存入创建信息
print es.index(index=fb_domain_index_name,
doc_type=fb_domain_index_type,
id=item_exist['domain_pinyin'],
body=item_exist)
mark = True
except Exception, e:
print e
mark = False
return mark
def homepage():
ip = request.remote_addr
timestamp = int(time.time())
user_name = ''
_id = current_user.get_id()
cx = sqlite3.connect("/home/ubuntu8/yuanhuiru/xnr/xnr1/xnr/flask-admin.db")
#cx = sqlite3.connect("sqlite:///flask-admin.db")
cu=cx.cursor()
users = cu.execute("select id,email from user")
for row in users:
if row[0] == int(_id):
user_name = row[1]
break
cx.close()
current_date = ts2datetime(timestamp)
current_time_new = datetime2ts(current_date)
log_id = user_name +'_'+ current_date
exist_item = es_xnr.exists(index=weibo_log_management_index_name,doc_type=weibo_log_management_index_type,\
id=log_id)
if exist_item:
get_result = es_xnr.get(index=weibo_log_management_index_name,doc_type=weibo_log_management_index_type,\
id=log_id)['_source']
login_ip_list = get_result['login_ip']
login_time_list = get_result['login_time']
login_ip_list.append(ip)
login_time_list.append(timestamp)
es_xnr.update(index=weibo_log_management_index_name,doc_type=weibo_log_management_index_type,\
id=log_id,body={'doc':{'login_ip':login_ip_list,'login_time':login_time_list}})
else:
item_dict = {}
item_dict['user_name'] = user_name
item_dict['login_ip'] = [ip]
item_dict['login_time'] = [timestamp]
item_dict['operate_date'] = current_date
item_dict['operate_time'] = current_time_new
item_dict['user_id'] = ''
#item_dict['user_name'] = ''
item_dict['operate_content'] = ''
es_xnr.index(index=weibo_log_management_index_name,doc_type=weibo_log_management_index_type,\
id=log_id,body=item_dict)
return render_template('index/navigationMain.html')
def get_save_step_three_2(task_detail):
task_id = task_detail['task_id']
nick_name = task_detail['nick_name']
try:
item_fans_followers = dict()
followers_uids = list(set(task_detail['followers_uids'].split('，')))
item_fans_followers['followers_list'] = followers_uids
item_fans_followers['xnr_user_no'] = task_id
print es.index(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,id=task_id,body=item_fans_followers)
mark = True
except:
mark = False
return mark
def get_save_step_two(task_detail):
task_id = task_detail['task_id']
# es_results = es.search(index=weibo_xnr_index_name,doc_type=weibo_xnr_index_type,body={'query':{'match_all':{}},\
# 'sort':{'user_no':{'order':'desc'}}})['hits']['hits']
# if es_results:
# user_no_max = es_results[0]['_source']['user_no']
# user_no_current = user_no_max + 1
# else:
# user_no_current = 1
# 如果是首次创建时，点到了第二步中，下一步，则 task_id 尚未分配，则首先分配 task_id
# 如果是从未完成虚拟人中，点到了第二步中，下一步，则task_id已分配，则沿用原来的task_id
if not task_id:
user_no_current = get_wb_xnr_no()
task_detail['user_no'] = user_no_current
task_id = user_no2_id(user_no_current) #五位数 WXNR0001
item_exist = dict()
item_exist['submitter'] = task_detail['submitter']
item_exist['user_no'] = task_detail['user_no']
item_exist['domain_name'] = task_detail['domain_name']
item_exist['role_name'] = task_detail['role_name']
item_exist['psy_feature'] = '&'.join(
task_detail['psy_feature'].encode('utf-8').split('，'))
item_exist['political_side'] = task_detail['political_side']
item_exist['business_goal'] = '&'.join(
task_detail['business_goal'].encode('utf-8').split('，'))
item_exist['daily_interests'] = '&'.join(
task_detail['daily_interests'].encode('utf-8').split('，'))
item_exist['monitor_keywords'] = ','.join(
task_detail['monitor_keywords'].encode('utf-8').split('，'))
item_exist['active_time'] = '&'.join(task_detail['active_time'].split('-'))
item_exist['day_post_average'] = json.dumps(
task_detail['day_post_average'].split('-'))
item_exist['create_status'] = 1 # 第二步完成
item_exist['xnr_user_no'] = task_id # 虚拟人编号
item_exist['create_time'] = int(time.time())
es.index(index=weibo_xnr_index_name,
doc_type=weibo_xnr_index_type,
id=task_id,
body=item_exist)
mark = True
#except:
# mark = False
return mark, task_id
def update_xnr_map_relationship(xnr_map_detail, xnr_map_id):
try:
es.delete(index=xnr_map_index_name,
doc_type=xnr_map_index_type,
id=xnr_map_id)
es.index(index=xnr_map_index_name,
doc_type=xnr_map_index_type,
body=xnr_map_detail,
id=xnr_map_id)
# es.update(index=xnr_map_index_name,doc_type=xnr_map_index_type,body=xnr_map_detail,id=xnr_map_id)
result = True
except:
result = False
return result
def create_role_authority(role_name, description):
authority_detail = dict()
authority_detail['role_name'] = role_name
authority_detail['description'] = description
authority_id = role_name
try:
es.index(index=weibo_authority_management_index_name,
doc_type=weibo_authority_management_index_type,
id=authority_id,
body=authority_detail)
result = True
except:
result = False
return result
def get_create_hidden_expression(origin_word,evolution_words_string,create_type,create_time,submitter):
task_detail = dict()
task_detail['origin_word'] = origin_word
task_detail['evolution_words_string'] = evolution_words_string
task_detail['create_type'] = create_type
task_detail['create_time'] = create_time
task_detail['submitter']=submitter
task_id = origin_word
try:
es.index(index=weibo_hidden_expression_index_name,doc_type=weibo_hidden_expression_index_type,id=task_id,body=task_detail)
mark = True
except:
mark = False
return mark
def get_create_sensitive_words(rank,sensitive_words,create_type,create_time,submitter):
task_detail = dict()
task_detail['rank'] = rank
task_detail['sensitive_words'] = sensitive_words
task_detail['create_type'] = create_type
task_detail['create_time'] = create_time
task_detail['submitter'] = submitter
task_id = sensitive_words
try:
es.index(index=weibo_sensitive_words_index_name,doc_type=weibo_sensitive_words_index_type,id=task_id,body=task_detail)
mark = True
except:
mark = False
return mark
def create_user_account(user_account_info):
account_detail = dict()
account_detail['user_id'] = user_account_info[0]
account_detail['user_name'] = user_account_info[1]
account_detail['my_xnrs'] = user_account_info[2]
account_id = user_account_info[0]
try:
es.index(index=weibo_account_management_index_name,
doc_type=weibo_account_management_index_type,
id=account_id,
body=account_detail)
result = True
except:
result = False
return result
def new_addto_weibo_corpus(task_detail):
flow_text_index_name = flow_text_index_name_pre + ts2datetime(task_detail['timestamp'])
try:
corpus_result = es_flow_text.get(index=flow_text_index_name,doc_type=flow_text_index_type,id=task_detail['mid'])['_source']
task_detail['text']=corpus_result['text']
task_detail['retweeted']=corpus_result['retweeted']
task_detail['comment']=corpus_result['comment']
task_detail['like']=corpus_result['like']
except:
mark=False
try:
es_xnr.index(index=weibo_xnr_corpus_index_name,doc_type=weibo_xnr_corpus_index_type,id=task_detail['mid'],body=task_detail)
mark=True
except:
mark=False
return mark
def get_save_step_two(task_detail):
item_exist = dict()
item_exist['submitter'] = task_detail['submitter']
item_exist['domain_name'] = task_detail['domain_name']
item_exist['role_name'] = task_detail['role_name']
item_exist['psy_feature'] = '&'.join(
task_detail['psy_feature'].encode('utf-8').split('，'))
item_exist['political_side'] = task_detail['political_side']
item_exist['business_goal'] = '&'.join(
task_detail['business_goal'].encode('utf-8').split('，'))
item_exist['daily_interests'] = '&'.join(
task_detail['daily_interests'].encode('utf-8').split('，'))
item_exist['monitor_keywords'] = ','.join(
task_detail['monitor_keywords'].encode('utf-8').split('，'))
#item_exist['sex'] = task_detail['sex']
# item_exist['nick_name'] = task_detail['nick_name']
# item_exist['age'] = task_detail['age']
# item_exist['location'] = task_detail['location']
# item_exist['career'] = task_detail['career']
# item_exist['description'] = task_detail['description']
item_exist['active_time'] = '&'.join(task_detail['active_time'].split('-'))
item_exist['day_post_average'] = json.dumps(
task_detail['day_post_average'].split('-'))
item_exist['create_status'] = 1 # 第二步完成
item_exist['create_time'] = int(time.time())
task_source = task_detail['task_source']
task_id = ''
if task_source == 'new':
user_no_current = get_wb_xnr_no()
item_exist['user_no'] = user_no_current #task_detail['user_no']
task_id = user_no2_id(user_no_current) #五位数 WXNR0001
item_exist['xnr_user_no'] = task_id # 虚拟人编号
es.index(index=weibo_xnr_index_name,
doc_type=weibo_xnr_index_type,
id=task_id,
body=item_exist)
mark = True
#except:
# mark = False
return mark, task_id
def utils_report_warning_content(wxbot_id, report_type, report_time, speaker_id, wx_content_info_str):
xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid']
report_dict = {
'report_type': report_type,
'report_time': report_time,
'xnr_user_no': wxbot_id,
'xnr_puid': xnr_puid,
'speaker_id': speaker_id,
'report_content': wx_content_info_str
}
report_id = wxbot_id + '_' + str(report_time)
mark = 0
try:
wx_report_management_mappings()
es_xnr.index(index=wx_report_management_index_name, doc_type=wx_report_management_index_type, id=report_id,body=report_dict)
mark = 1
except Exception,e:
print e
def report_warming_content(report_type, report_time, xnr_user_no,\
qq_number, qq_content_info):
report_dict = dict()
report_dict['report_type'] = report_type
report_dict['report_time'] = int(report_time)
report_dict['xnr_user_no'] = xnr_user_no
report_dict['qq_number'] = qq_number
report_dict['qq_nick'] = ''
report_dict['report_content'] = json.dumps(qq_content_info)
report_id = xnr_user_no + '_' + str(report_time)
try:
es_xnr.index(index=qq_report_management_index_name, \
doc_type=qq_report_management_index_type, id=report_id,\
body=report_dict)
mark = True
except:
mark = False
return mark
def get_create_date_remind(date_name,timestamp,keywords,create_type,create_time,content_recommend,submitter):
task_detail = dict()
#task_detail['date_time'] = ts2datetime(int(timestamp))[5:10]
task_detail['date_name']=date_name
task_detail['date_time']=timestamp[5:10]
task_detail['keywords'] = keywords
task_detail['create_type'] = create_type
task_detail['create_time'] = create_time
task_detail['content_recommend']=content_recommend
task_detail['submitter']=submitter
task_id = create_time
try:
es.index(index=weibo_date_remind_index_name,doc_type=weibo_date_remind_index_type,id=task_id,body=task_detail)
mark = True
except:
mark = False
return mark
def report_warming_content_new(task_detail):
report_dict = dict()
report_dict['report_type'] = task_detail['report_type']
report_dict['report_time'] = task_detail['report_time']
report_dict['xnr_user_no'] = task_detail['xnr_user_no']
report_dict['qq_number'] = task_detail['qq_number']
report_dict['qq_nickname'] = task_detail['qq_nickname']
if task_detail['report_type'] == '人物':
user_info = []
for item in task_detail['user_info']:
qq_groups = item['qq_groups'].values()
item['text'] = get_user_text(item['qq_nick'], qq_groups,
item['last_speak_ts'])
user_info.append(item)
report_id = task_detail['xnr_user_no'] + '_' + task_detail[
'qq_nickname']
report_dict['report_content'] = json.dumps(user_info)
elif task_detail['report_type'] == '言论':
report_id = task_detail['report_id']
report_content = []
for item in task_detail['content_info']:
index_name = group_message_index_name_pre + ts2datetime(
int(item['timestamp']))
try:
content_result = es_xnr.get(index=index_name,
doc_type=group_message_index_type,
id=item['_id'])['_source']
#print 'content_result:',content_result,type(content_result)
report_content.append(content_result)
except:
print 'content error!'
report_dict['report_content'] = json.dumps(report_content)
try:
es_xnr.index(index=qq_report_management_index_name, \
doc_type=qq_report_management_index_type, id=report_id,\
body=report_dict)
mark = True
except:
mark = False
return mark
def create_corpus(corpus_info):
corpus_detail=dict()
corpus_detail['corpus_type']=corpus_info[0]
corpus_detail['theme_daily_name']=corpus_info[1]
corpus_detail['text']=corpus_info[2]
corpus_detail['uid']=corpus_info[3]
corpus_detail['mid']=corpus_info[4]
corpus_detail['timestamp']=corpus_info[5]
corpus_detail['retweeted']=corpus_info[6]
corpus_detail['comment']=corpus_info[7]
corpus_detail['like']=corpus_info[8]
corpus_detail['create_type']=corpus_info[9]
corpus_id=corpus_info[4] #mid
#print corpus_info
try:
es.index(index=weibo_xnr_corpus_index_name,doc_type=weibo_xnr_corpus_index_type,id=corpus_id,body=corpus_detail)
mark=True
except:
mark=False
return mark
def cron_compute_mark_wx():
current_time = int(time.time() - DAY)
current_date = ts2datetime(current_time)
current_time_new = datetime2ts(current_date)
#加载数据库中虚拟人信息
xnr_query_body = {'query': {'match_all': {}}, 'size': MAX_VALUE}
xnr_results = es_xnr.search(index=wx_xnr_index_name,
doc_type=wx_xnr_index_type,
body=xnr_query_body)['hits']['hits']
flag = False
for result in xnr_results:
xnr_user_no = result['_source']['xnr_user_no']
puid = result['_source']['puid']
#计算
influence_dict = get_influence_at_num(puid)
penetration_dict = get_penetration_num(xnr_user_no)
safe_dict = wx_history_count(xnr_user_no, puid, current_time_new)
#整理
_id = xnr_user_no + '_' + current_date
xnr_user_detail = {
'influence': influence_dict['mark'],
'penetration': penetration_dict['mark'],
'safe': safe_dict['mark'],
'daily_be_at_num': influence_dict['daily_be_at_num'],
'total_be_at_num': influence_dict['total_be_at_num'],
'daily_sensitive_num': penetration_dict['sensitive_info'],
'total_post_num': safe_dict['total_post_num'],
'daily_post_num': safe_dict['daily_post_num'],
'date_time': current_date,
'timestamp': current_time_new,
'xnr_user_no': xnr_user_no,
'puid': puid
}
#并存储
wx_xnr_history_count_mappings() #先确保数据库存在
try:
if es_xnr.exists(index=wx_xnr_history_count_index_name,
doc_type=wx_xnr_history_count_index_type,
id=_id):
msg = es_xnr.update(index=wx_xnr_history_count_index_name,
doc_type=wx_xnr_history_count_index_type,
body={'doc': xnr_user_detail},
id=_id)
else:
msg = es_xnr.index(index=wx_xnr_history_count_index_name,
doc_type=wx_xnr_history_count_index_type,
id=_id,
body=xnr_user_detail)
flag = True
print msg
except Exception, e:
print 'cron_compute_mark_wx Exception: ', str(e)
return False
def addto_facebook_corpus(task_detail):
flow_text_index_name = facebook_flow_text_index_name_pre + ts2datetime(
task_detail['timestamp'])
try:
corpus_result = es_xnr.get(index=flow_text_index_name,
doc_type=facebook_flow_text_index_type,
id=task_detail['fid'])['_source']
task_detail['text'] = corpus_result['text']
#查询三个指标字段
fid_result = lookup_fid_attend_index(task_detail['fid'],
task_detail['timestamp'],
task_detail['timestamp'])
if fid_result:
task_detail['comment'] = fid_result['comment']
task_detail['share'] = fid_result['share']
task_detail['favorite'] = fid_result['favorite']
else:
task_detail['comment'] = 0
task_detail['share'] = 0
task_detail['favorite'] = 0
#查询用户昵称
task_detail['nick_name'] = get_user_nickname(corpus_result['uid'])
# task_detail['retweeted']=corpus_result['retweeted']
# task_detail['comment']=corpus_result['comment']
# task_detail['like']=corpus_result['like']
except:
mark = False
try:
es_xnr.index(index=facebook_xnr_corpus_index_name,
doc_type=facebook_xnr_corpus_index_type,
id=task_detail['fid'],
body=task_detail)
mark = True
except:
mark = False
return mark
def addto_twitter_corpus(task_detail):
flow_text_index_name = twitter_flow_text_index_name_pre + ts2datetime(
task_detail['timestamp'])
try:
corpus_result = es_xnr.get(index=flow_text_index_name,
doc_type=twitter_flow_text_index_type,
id=task_detail['tid'])['_source']
task_detail['text'] = corpus_result['text']
#查询三个指标字段
tid_result = lookup_tid_attend_index(task_detail['tid'],
task_detail['timestamp'],
task_detail['timestamp'])
if tid_result:
task_detail['comment'] = tid_result['comment']
task_detail['share'] = tid_result['share']
task_detail['favorite'] = tid_result['favorite']
else:
task_detail['comment'] = 0
task_detail['share'] = 0
task_detail['favorite'] = 0
#查询用户昵称
task_detail['nick_name'] = get_user_nickname(item['_source']['uid'])
except:
mark = False
try:
es_xnr.index(index=twitter_xnr_corpus_index_name,
doc_type=twitter_xnr_corpus_index_type,
id=task_detail['tid'],
body=task_detail)
mark = True
except:
mark = False
return mark