def get_theme_flow(theme_name, submit_user):
topic_id = p.get_pinyin(theme_name)
topic_id = topic_id.lower()
eid_string = es_event.get(index=special_event_name, doc_type=special_event_type, id=topic_id, fields=['event'])
event_list = eid_string['fields']['event'][0].split('&')
query_body = {
'query':{
'terms':{'en_name':event_list}
},
"sort": [{'start_ts':'asc'}]
}
name_list = es_event.search(index=event_analysis_name, doc_type=event_text_type, \
body=query_body, fields=['name', 'en_name'])['hits']['hits']
query_body2 = {
'query':{"match_all":{}},
"sort": [{'retweeted':'desc'}],
'size':1
}
event_name_list = []
for i in name_list:
event_name_list.append(i['fields']['en_name'][0])
print event_name_list
result_list = []
for i in event_name_list:
max_retweet = es_event.search(index=i, doc_type='text', body=query_body2, \
fields=['text', 'timestamp'])['hits']['hits']
print max_retweet,'00000000000'
text = max_retweet[0]['fields']['text'][0]
t_datetime = ts2date(max_retweet[0]['fields']['timestamp'][0])
result_list.append([i, text, t_datetime])
return result_list
def event_river_search(eid_list):
query_body = {
'query': {
'terms': {
'en_name': eid_list
}
}
# "sort": [{sort_flag:'desc'}]
}
fields_list = ['time_type_weibo', 'name']
print eid_list
event_detail = es_event.search(index=event_analysis_name, doc_type=event_type, \
body=query_body, _source=False, fields=fields_list)['hits']['hits']
detail_result = {}
for i in event_detail:
name_i = i['fields']['name'][0]
single_river = []
river = json.loads(i['fields']['time_type_weibo'][0])
print name_i.encode('utf-8')
for r in river:
time_r = []
river_value = 0
for ki, vi in r[1].iteritems():
river_value += vi
time_r.append(ts2date(r[0]))
time_r.append(river_value)
single_river.append(time_r)
detail_result[name_i] = single_river
return detail_result
def ajax_show_event_task():
results = es_event.search(index=event_task_name,
doc_type=event_task_type,
body={"query": {
"match_all": {}
}})['hits']['hits']
result_list = []
for i in results:
result_list.append(i['_source'])
return json.dumps(result_list) #True submit_num
def search_event_time_limit(item, field, start_ts, end_ts, submit_user):
query_body = {
"query": {
"bool": {
"must": [{
"range": {
"submit_ts": {
"gte": start_ts,
"lte": end_ts
}
}
}],
'should': [{
"wildcard": {
'keywords': '*' + str(item.encode('utf-8')) + '*'
}
}, {
"wildcard": {
'en_name': '*' + str(item.encode('utf-8')) + '*'
}
}, {
"wildcard": {
'name': '*' + str(item.encode('utf-8')) + '*'
}
}]
}
}
}
e_nodes_list = {}
# event_relation =[]
result = []
try:
event_result = es_event.search(index=event_analysis_name, doc_type=event_text_type, \
body=query_body, fields=field)['hits']['hits'] #fields=['name','en_name']
except:
return 'does not exist'
for i in event_result:
event = []
i_fields = i['fields']
for j in field:
if not i_fields.has_key(j):
event.append('')
continue
if j == 'keywords':
keywords = i_fields[j][0].split('&')
keywords = keywords[:5]
event.append(keywords)
elif j == 'work_tag':
tag = deal_editor_tag(i_fields[j][0], submit_user)[0]
event.append(tag)
else:
event.append(i_fields[j][0])
result.append(event)
return result
def search_event(item, field, submit_user):
query_body = {
"query": {
'bool': {
'should': [
{
"wildcard": {
'keywords': '*' + str(item.encode('utf-8')) + '*'
}
},
{
"wildcard": {
'en_name': '*' + str(item.encode('utf-8')) + '*'
}
},
# {"wildcard":{'name':'*'+str(item.encode('utf-8'))+'*'}}
]
}
},
'size': 10
}
result = []
try:
event_result = es_event.search(index=event_analysis_name, doc_type=event_text_type, \
body=query_body, fields=field)['hits']['hits'] #fields=['name','en_name']
except:
return 'does not exist'
for i in event_result:
event = []
i_fields = i['fields']
for j in field:
if not i_fields.has_key(j):
event.append('')
continue
if j == 'keywords':
keywords = i_fields[j][0].split('&')
keywords = keywords[:5]
event.append(keywords)
elif j == 'work_tag':
tag = deal_editor_tag(i_fields[j][0], submit_user)[0]
event.append(tag)
else:
event.append(i_fields[j][0])
result.append(event)
return result
def search_event(item, field):
query_body = {
"query": {
'bool': {
'should': [
{
"wildcard": {
'keywords': '*' + str(item.encode('utf-8')) + '*'
}
},
{
"wildcard": {
'en_name': '*' + str(item.encode('utf-8')) + '*'
}
},
# {"wildcard":{'name':'*'+str(item.encode('utf-8'))+'*'}}
]
}
},
'size': 10
}
only_eid = []
event_id_list = []
u_nodes_list = {}
e_nodes_list = {}
event_relation = []
try:
name_results = es_event.search(index=event_analysis_name, doc_type=event_text_type, \
body=query_body, fields=field)['hits']['hits'] #fields=['name','en_name']
except:
return 'does not exist'
for i in name_results:
field_list = []
for key in field:
try:
key1 = i['fields'][key][0]
except:
key1 = ''
field_list.append(key1)
event_id_list.append(field_list)
return event_id_list
def get_es_status(index_name):
query_search = {
"query": {
"bool": {
"must": [{
"match_all": {}
}],
"must_not": [],
"should": []
}
},
"from": 0,
"size": 3,
"sort": [],
"aggs": {}
}
print "ss"
result = es.search(index=index_name, doc_type="text",
body=query_search)['hits']['hits']
return result
def get_es_node():
query_search = {
"query": {
"bool": {
"must": [{
"match_all": {}
}],
"must_not": [],
"should": []
}
},
"from": 0,
"size": 100,
"sort": [],
"facets": {}
}
print "ss"
result = es.search(index="user_portrait",
doc_type="user",
body=query_search)['hits']['hits']
return result
def search_event(item):
query_body = {
"query": {
'bool': {
'should': [{
"wildcard": {
'keywords': '*' + str(item.encode('utf-8')) + '*'
}
}, {
"wildcard": {
'en_name': '*' + str(item.encode('utf-8')) + '*'
}
}, {
"wildcard": {
'name': '*' + str(item.encode('utf-8')) + '*'
}
}]
}
},
'size': 10
}
only_eid = []
event_id_list = []
u_nodes_list = {}
e_nodes_list = {}
event_relation = []
try:
name_results = es_event.search(index=event_name, doc_type=event_type, \
body=query_body, fields=['name','en_name'])['hits']['hits']
except:
return 'does not exist'
for i in name_results:
print i
name = i['fields']['name'][0]
en_name = i['fields']['en_name'][0]
only_eid.append(en_name)
e_nodes_list[en_name] = name
event_id_list.append([en_name, name])
return event_id_list
def search_event_time_limit(item, field, start_ts, end_ts):
query_body = {
"query": {
"bool": {
"must": [{
"range": {
"submit_ts": {
"gte": start_ts,
"lte": end_ts
}
}
}]
}
}
}
only_eid = []
event_id_list = []
u_nodes_list = {}
e_nodes_list = {}
event_relation = []
try:
name_results = es_event.search(index=event_analysis_name, doc_type=event_text_type, \
body=query_body, fields=field)['hits']['hits'] #fields=['name','en_name']
except:
return 'does not exist'
for i in name_results:
field_list = []
for key in field:
try:
key1 = i['fields'][key][0]
except:
key1 = ''
field_list.append(key1)
event_id_list.append(field_list)
return event_id_list
def search_related_e_card(item, submit_user, theme_name):
if theme_name:
theme_name = theme_name + '_' + submit_user
theme_name_pinyin = p.get_pinyin(theme_name)
event_list_string = es_event.get(index=special_event_name, doc_type=special_event_type, id=theme_name_pinyin,\
fields=['event'])
eid_list = []
eid_list = event_list_string['fields']['event'][0].split('&')
else:
eid_list = []
query_body = {
"query": {
'bool': {
'should': [{
"wildcard": {
'keywords': '*' + str(item.encode('utf-8')) + '*'
}
}, {
"wildcard": {
'en_name': '*' + str(item.encode('utf-8')) + '*'
}
}, {
"wildcard": {
'name': '*' + str(item.encode('utf-8')) + '*'
}
}]
}
},
'size': 1000
}
fields_list = [
'en_name', 'name', 'event_type', 'real_time', 'real_geo', 'uid_counts',
'weibo_counts', 'keywords', 'work_tag'
]
only_eid = []
event_id_list = []
u_nodes_list = {}
e_nodes_list = {}
event_relation = []
try:
event_result = es_event.search(index=event_analysis_name, doc_type=event_text_type, \
body=query_body, fields=['en_name'])['hits']['hits']
except:
return 'node does not exist'
# print event_result
search_eid = []
result = []
for i in event_result:
i_fields = i['fields']
search_eid.append(i_fields['en_name'][0])
show_id_set = set(search_eid) - set(eid_list)
show_id = [i for i in show_id_set]
if not show_id:
return []
event_result = es_event.mget(index=event_analysis_name, doc_type=event_text_type, \
body={'ids':show_id}, fields=fields_list)['docs']
for i in event_result:
event = []
i_fields = i['fields']
for j in fields_list:
if not i_fields.has_key(j):
event.append('')
continue
if j == 'keywords':
keywords = i_fields[j][0].split('&')
keywords = keywords[:5]
event.append(keywords)
elif j == 'work_tag':
tag = deal_event_tag(i_fields[j][0], submit_user)[0]
event.append(tag)
else:
event.append(i_fields[j][0])
result.append(event)
return result
def search_related_e_card(item, layer):
query_body = {
"query": {
'bool': {
'should': [{
"wildcard": {
'keywords': '*' + str(item.encode('utf-8')) + '*'
}
}, {
"wildcard": {
'en_name': '*' + str(item.encode('utf-8')) + '*'
}
}, {
"wildcard": {
'name': '*' + str(item.encode('utf-8')) + '*'
}
}]
}
},
'size': 10
}
only_eid = []
event_id_list = []
u_nodes_list = {}
e_nodes_list = {}
event_relation = []
try:
name_results = es_event.search(index=event_name, doc_type=event_type, \
body=query_body, fields=['name','en_name'])['hits']['hits']
print name_results, '@@@@@@@@@@@@@@@@@'
except:
return 'node does not exist'
print name_results, '@@@@@@@@@@@@@@@@@------------#######################'
if len(name_results) == 0:
return 'node does not exist'
for i in name_results:
print i
name = i['fields']['name'][0]
en_name = i['fields']['en_name'][0]
only_eid.append(en_name)
e_nodes_list[en_name] = name
event_id_list.append([en_name, name])
print len(event_id_list), '========='
if layer == '1':
for eid_value in event_id_list:
c_string = 'START s0 = node:event_index(event="' + str(
eid_value[0]) + '") '
c_string += 'MATCH (s0)-[r1]-(s1:Event) return s0,r1,s1 LIMIT 100'
result = graph.run(c_string)
for i in list(result):
m_id = dict(i['s1'])['event_id']
if m_id not in only_eid:
only_eid.append(m_id)
result_card = event_detail_search(only_eid, 'start_ts')
if layer == '2':
for eid_value in event_id_list:
c_string = 'START s0 = node:event_index(event="' + str(
eid_value[0]) + '") '
c_string += 'MATCH (s0)-[r1]-(ss)-[r]-(s1:Event) return ss, s1 LIMIT 100'
result = graph.run(c_string)
for i in list(result):
ss_id = dict(i['ss'])
# print ss_id,'???????????/'
if ss_id.has_key('uid') or ss_id.has_key('event_id'):
m_id = dict(i['s1'])['event_id']
if m_id not in only_eid:
only_eid.append(m_id)
else:
print '00000'
print only_eid, '000000000000'
result_card = event_detail_search(only_eid, 'start_ts')
if layer == 'all':
eid_list_all = []
result = search_related_event_f(item)
if result == 'node does not exist':
return 'node does not exist'
eid_dict = result['event_nodes']
for k, v in eid_dict.iteritems():
eid_list_all.append(k)
result_card = event_detail_search(eid_list_all, 'start_ts')
return result_card
def search_related_event_f(item):
query_body = {
"query": {
'bool': {
'should': [{
"wildcard": {
'keywords': '*' + str(item.encode('utf-8')) + '*'
}
}, {
"wildcard": {
'en_name': '*' + str(item.encode('utf-8')) + '*'
}
}, {
"wildcard": {
'name': '*' + str(item.encode('utf-8')) + '*'
}
}]
}
},
'size': 10
}
only_eid = []
event_id_list = []
u_nodes_list = {}
e_nodes_list = {}
event_relation = []
try:
name_results = es_event.search(index=event_name, doc_type=event_type, \
body=query_body, fields=['name','en_name'])['hits']['hits']
except:
return 'node does not exist'
if len(name_results) == 0:
return 'node does not exist'
print name_results, '*********************'
for i in name_results:
name = i['fields']['name'][0]
en_name = i['fields']['en_name'][0]
only_eid.append(en_name)
e_nodes_list[en_name] = name
event_id_list.append([en_name, name])
for event_value in event_id_list:
c_string = 'START s0 = node:event_index(event="' + str(
event_value[0]) + '") '
c_string += 'MATCH (s0)-[r1]-(s1) return s0,r1,s1 LIMIT 10'
# print c_string,'==========='
mid_eid_list = [] #存放第一层的数据，再以这些为起始点，扩展第二层
mid_uid_list = []
result = graph.run(c_string)
# print list(result),'-----------------'
for i in list(result):
print i
start_id = i['s0']['event_id']
# start_id = s0['event']
relation1 = i['r1'].type()
m_id = dict(i['s1'])
if m_id.has_key('uid'):
middle_id = m_id['uid']
mid_uid_list.append(middle_id)
user_name = user_name_search(middle_id)
u_nodes_list[middle_id] = user_name
event_relation.append([start_id, relation1, middle_id])
if m_id.has_key('envent_id'):
middle_id = m_id['envent_id']
mid_eid_list.append(middle_id)
event_name2 = event_name_search(middle_id)
e_nodes_list[middle_id] = event_name2
event_relation.append([start_id, relation1, middle_id])
# print mid_uid_list
# print mid_eid_list,'++++++++++++++++'
for mid_uid in mid_uid_list:
c_string = 'START s1 = node:node_index(uid="' + str(mid_uid) + '") '
c_string += 'MATCH (s1)-[r2]->(s2:Event) return s1,r2,s2 LIMIT 5'
uid_result = graph.run(c_string)
for i in uid_result:
relation2 = i['r2'].type()
end_id = dict(i['s2'])
if end_id.has_key('uid'):
user_name = user_name_search(end_id['uid'])
u_nodes_list[end_id['uid']] = user_name
event_relation.append([mid_uid, relation2, end_id['uid']])
if end_id.has_key('envent_id'):
event_name2 = event_name_search(end_id['envent_id'])
e_nodes_list[end_id['envent_id']] = event_name2
event_relation.append(
[mid_uid, relation2, end_id['envent_id']])
for mid_eid in mid_eid_list:
c_string = 'START s1 = node:event_index(event="' + str(mid_eid) + '") '
c_string += 'MATCH (s1)-[r2]->(s2:Event) return s1,r2,s2 LIMIT 5'
eid_result = graph.run(c_string)
for i in eid_result:
relation2 = i['r2'].type()
end_id = dict(i['s2'])
if end_id.has_key('uid'):
user_name = user_name_search(end_id['uid'])
u_nodes_list[end_id['uid']] = user_name
event_relation.append([mid_eid, relation2, end_id['uid']])
if end_id.has_key('envent_id'):
event_name2 = event_name_search(end_id['envent_id'])
e_nodes_list[end_id['envent_id']] = event_name2
event_relation.append(
[mid_eid, relation2, end_id['envent_id']])
return {'total_event':len(event_id_list),'user_nodes':u_nodes_list,'event_nodes':e_nodes_list,\
'relation':event_relation}
