def get_theme_flow(theme_name, submit_user): topic_id = p.get_pinyin(theme_name) topic_id = topic_id.lower() eid_string = es_event.get(index=special_event_name, doc_type=special_event_type, id=topic_id, fields=['event']) event_list = eid_string['fields']['event'][0].split('&') query_body = { 'query':{ 'terms':{'en_name':event_list} }, "sort": [{'start_ts':'asc'}] } name_list = es_event.search(index=event_analysis_name, doc_type=event_text_type, \ body=query_body, fields=['name', 'en_name'])['hits']['hits'] query_body2 = { 'query':{"match_all":{}}, "sort": [{'retweeted':'desc'}], 'size':1 } event_name_list = [] for i in name_list: event_name_list.append(i['fields']['en_name'][0]) print event_name_list result_list = [] for i in event_name_list: max_retweet = es_event.search(index=i, doc_type='text', body=query_body2, \ fields=['text', 'timestamp'])['hits']['hits'] print max_retweet,'00000000000' text = max_retweet[0]['fields']['text'][0] t_datetime = ts2date(max_retweet[0]['fields']['timestamp'][0]) result_list.append([i, text, t_datetime]) return result_list
def event_river_search(eid_list): query_body = { 'query': { 'terms': { 'en_name': eid_list } } # "sort": [{sort_flag:'desc'}] } fields_list = ['time_type_weibo', 'name'] print eid_list event_detail = es_event.search(index=event_analysis_name, doc_type=event_type, \ body=query_body, _source=False, fields=fields_list)['hits']['hits'] detail_result = {} for i in event_detail: name_i = i['fields']['name'][0] single_river = [] river = json.loads(i['fields']['time_type_weibo'][0]) print name_i.encode('utf-8') for r in river: time_r = [] river_value = 0 for ki, vi in r[1].iteritems(): river_value += vi time_r.append(ts2date(r[0])) time_r.append(river_value) single_river.append(time_r) detail_result[name_i] = single_river return detail_result
def ajax_show_event_task(): results = es_event.search(index=event_task_name, doc_type=event_task_type, body={"query": { "match_all": {} }})['hits']['hits'] result_list = [] for i in results: result_list.append(i['_source']) return json.dumps(result_list) #True submit_num
def search_event_time_limit(item, field, start_ts, end_ts, submit_user): query_body = { "query": { "bool": { "must": [{ "range": { "submit_ts": { "gte": start_ts, "lte": end_ts } } }], 'should': [{ "wildcard": { 'keywords': '*' + str(item.encode('utf-8')) + '*' } }, { "wildcard": { 'en_name': '*' + str(item.encode('utf-8')) + '*' } }, { "wildcard": { 'name': '*' + str(item.encode('utf-8')) + '*' } }] } } } e_nodes_list = {} # event_relation =[] result = [] try: event_result = es_event.search(index=event_analysis_name, doc_type=event_text_type, \ body=query_body, fields=field)['hits']['hits'] #fields=['name','en_name'] except: return 'does not exist' for i in event_result: event = [] i_fields = i['fields'] for j in field: if not i_fields.has_key(j): event.append('') continue if j == 'keywords': keywords = i_fields[j][0].split('&') keywords = keywords[:5] event.append(keywords) elif j == 'work_tag': tag = deal_editor_tag(i_fields[j][0], submit_user)[0] event.append(tag) else: event.append(i_fields[j][0]) result.append(event) return result
def search_event(item, field, submit_user): query_body = { "query": { 'bool': { 'should': [ { "wildcard": { 'keywords': '*' + str(item.encode('utf-8')) + '*' } }, { "wildcard": { 'en_name': '*' + str(item.encode('utf-8')) + '*' } }, # {"wildcard":{'name':'*'+str(item.encode('utf-8'))+'*'}} ] } }, 'size': 10 } result = [] try: event_result = es_event.search(index=event_analysis_name, doc_type=event_text_type, \ body=query_body, fields=field)['hits']['hits'] #fields=['name','en_name'] except: return 'does not exist' for i in event_result: event = [] i_fields = i['fields'] for j in field: if not i_fields.has_key(j): event.append('') continue if j == 'keywords': keywords = i_fields[j][0].split('&') keywords = keywords[:5] event.append(keywords) elif j == 'work_tag': tag = deal_editor_tag(i_fields[j][0], submit_user)[0] event.append(tag) else: event.append(i_fields[j][0]) result.append(event) return result
def search_event(item, field): query_body = { "query": { 'bool': { 'should': [ { "wildcard": { 'keywords': '*' + str(item.encode('utf-8')) + '*' } }, { "wildcard": { 'en_name': '*' + str(item.encode('utf-8')) + '*' } }, # {"wildcard":{'name':'*'+str(item.encode('utf-8'))+'*'}} ] } }, 'size': 10 } only_eid = [] event_id_list = [] u_nodes_list = {} e_nodes_list = {} event_relation = [] try: name_results = es_event.search(index=event_analysis_name, doc_type=event_text_type, \ body=query_body, fields=field)['hits']['hits'] #fields=['name','en_name'] except: return 'does not exist' for i in name_results: field_list = [] for key in field: try: key1 = i['fields'][key][0] except: key1 = '' field_list.append(key1) event_id_list.append(field_list) return event_id_list
def get_es_status(index_name): query_search = { "query": { "bool": { "must": [{ "match_all": {} }], "must_not": [], "should": [] } }, "from": 0, "size": 3, "sort": [], "aggs": {} } print "ss" result = es.search(index=index_name, doc_type="text", body=query_search)['hits']['hits'] return result
def get_es_node(): query_search = { "query": { "bool": { "must": [{ "match_all": {} }], "must_not": [], "should": [] } }, "from": 0, "size": 100, "sort": [], "facets": {} } print "ss" result = es.search(index="user_portrait", doc_type="user", body=query_search)['hits']['hits'] return result
def search_event(item): query_body = { "query": { 'bool': { 'should': [{ "wildcard": { 'keywords': '*' + str(item.encode('utf-8')) + '*' } }, { "wildcard": { 'en_name': '*' + str(item.encode('utf-8')) + '*' } }, { "wildcard": { 'name': '*' + str(item.encode('utf-8')) + '*' } }] } }, 'size': 10 } only_eid = [] event_id_list = [] u_nodes_list = {} e_nodes_list = {} event_relation = [] try: name_results = es_event.search(index=event_name, doc_type=event_type, \ body=query_body, fields=['name','en_name'])['hits']['hits'] except: return 'does not exist' for i in name_results: print i name = i['fields']['name'][0] en_name = i['fields']['en_name'][0] only_eid.append(en_name) e_nodes_list[en_name] = name event_id_list.append([en_name, name]) return event_id_list
def search_event_time_limit(item, field, start_ts, end_ts): query_body = { "query": { "bool": { "must": [{ "range": { "submit_ts": { "gte": start_ts, "lte": end_ts } } }] } } } only_eid = [] event_id_list = [] u_nodes_list = {} e_nodes_list = {} event_relation = [] try: name_results = es_event.search(index=event_analysis_name, doc_type=event_text_type, \ body=query_body, fields=field)['hits']['hits'] #fields=['name','en_name'] except: return 'does not exist' for i in name_results: field_list = [] for key in field: try: key1 = i['fields'][key][0] except: key1 = '' field_list.append(key1) event_id_list.append(field_list) return event_id_list
def search_related_e_card(item, submit_user, theme_name): if theme_name: theme_name = theme_name + '_' + submit_user theme_name_pinyin = p.get_pinyin(theme_name) event_list_string = es_event.get(index=special_event_name, doc_type=special_event_type, id=theme_name_pinyin,\ fields=['event']) eid_list = [] eid_list = event_list_string['fields']['event'][0].split('&') else: eid_list = [] query_body = { "query": { 'bool': { 'should': [{ "wildcard": { 'keywords': '*' + str(item.encode('utf-8')) + '*' } }, { "wildcard": { 'en_name': '*' + str(item.encode('utf-8')) + '*' } }, { "wildcard": { 'name': '*' + str(item.encode('utf-8')) + '*' } }] } }, 'size': 1000 } fields_list = [ 'en_name', 'name', 'event_type', 'real_time', 'real_geo', 'uid_counts', 'weibo_counts', 'keywords', 'work_tag' ] only_eid = [] event_id_list = [] u_nodes_list = {} e_nodes_list = {} event_relation = [] try: event_result = es_event.search(index=event_analysis_name, doc_type=event_text_type, \ body=query_body, fields=['en_name'])['hits']['hits'] except: return 'node does not exist' # print event_result search_eid = [] result = [] for i in event_result: i_fields = i['fields'] search_eid.append(i_fields['en_name'][0]) show_id_set = set(search_eid) - set(eid_list) show_id = [i for i in show_id_set] if not show_id: return [] event_result = es_event.mget(index=event_analysis_name, doc_type=event_text_type, \ body={'ids':show_id}, fields=fields_list)['docs'] for i in event_result: event = [] i_fields = i['fields'] for j in fields_list: if not i_fields.has_key(j): event.append('') continue if j == 'keywords': keywords = i_fields[j][0].split('&') keywords = keywords[:5] event.append(keywords) elif j == 'work_tag': tag = deal_event_tag(i_fields[j][0], submit_user)[0] event.append(tag) else: event.append(i_fields[j][0]) result.append(event) return result
def search_related_e_card(item, layer): query_body = { "query": { 'bool': { 'should': [{ "wildcard": { 'keywords': '*' + str(item.encode('utf-8')) + '*' } }, { "wildcard": { 'en_name': '*' + str(item.encode('utf-8')) + '*' } }, { "wildcard": { 'name': '*' + str(item.encode('utf-8')) + '*' } }] } }, 'size': 10 } only_eid = [] event_id_list = [] u_nodes_list = {} e_nodes_list = {} event_relation = [] try: name_results = es_event.search(index=event_name, doc_type=event_type, \ body=query_body, fields=['name','en_name'])['hits']['hits'] print name_results, '@@@@@@@@@@@@@@@@@' except: return 'node does not exist' print name_results, '@@@@@@@@@@@@@@@@@------------#######################' if len(name_results) == 0: return 'node does not exist' for i in name_results: print i name = i['fields']['name'][0] en_name = i['fields']['en_name'][0] only_eid.append(en_name) e_nodes_list[en_name] = name event_id_list.append([en_name, name]) print len(event_id_list), '=========' if layer == '1': for eid_value in event_id_list: c_string = 'START s0 = node:event_index(event="' + str( eid_value[0]) + '") ' c_string += 'MATCH (s0)-[r1]-(s1:Event) return s0,r1,s1 LIMIT 100' result = graph.run(c_string) for i in list(result): m_id = dict(i['s1'])['event_id'] if m_id not in only_eid: only_eid.append(m_id) result_card = event_detail_search(only_eid, 'start_ts') if layer == '2': for eid_value in event_id_list: c_string = 'START s0 = node:event_index(event="' + str( eid_value[0]) + '") ' c_string += 'MATCH (s0)-[r1]-(ss)-[r]-(s1:Event) return ss, s1 LIMIT 100' result = graph.run(c_string) for i in list(result): ss_id = dict(i['ss']) # print ss_id,'???????????/' if ss_id.has_key('uid') or ss_id.has_key('event_id'): m_id = dict(i['s1'])['event_id'] if m_id not in only_eid: only_eid.append(m_id) else: print '00000' print only_eid, '000000000000' result_card = event_detail_search(only_eid, 'start_ts') if layer == 'all': eid_list_all = [] result = search_related_event_f(item) if result == 'node does not exist': return 'node does not exist' eid_dict = result['event_nodes'] for k, v in eid_dict.iteritems(): eid_list_all.append(k) result_card = event_detail_search(eid_list_all, 'start_ts') return result_card
def search_related_event_f(item): query_body = { "query": { 'bool': { 'should': [{ "wildcard": { 'keywords': '*' + str(item.encode('utf-8')) + '*' } }, { "wildcard": { 'en_name': '*' + str(item.encode('utf-8')) + '*' } }, { "wildcard": { 'name': '*' + str(item.encode('utf-8')) + '*' } }] } }, 'size': 10 } only_eid = [] event_id_list = [] u_nodes_list = {} e_nodes_list = {} event_relation = [] try: name_results = es_event.search(index=event_name, doc_type=event_type, \ body=query_body, fields=['name','en_name'])['hits']['hits'] except: return 'node does not exist' if len(name_results) == 0: return 'node does not exist' print name_results, '*********************' for i in name_results: name = i['fields']['name'][0] en_name = i['fields']['en_name'][0] only_eid.append(en_name) e_nodes_list[en_name] = name event_id_list.append([en_name, name]) for event_value in event_id_list: c_string = 'START s0 = node:event_index(event="' + str( event_value[0]) + '") ' c_string += 'MATCH (s0)-[r1]-(s1) return s0,r1,s1 LIMIT 10' # print c_string,'===========' mid_eid_list = [] #存放第一层的数据,再以这些为起始点,扩展第二层 mid_uid_list = [] result = graph.run(c_string) # print list(result),'-----------------' for i in list(result): print i start_id = i['s0']['event_id'] # start_id = s0['event'] relation1 = i['r1'].type() m_id = dict(i['s1']) if m_id.has_key('uid'): middle_id = m_id['uid'] mid_uid_list.append(middle_id) user_name = user_name_search(middle_id) u_nodes_list[middle_id] = user_name event_relation.append([start_id, relation1, middle_id]) if m_id.has_key('envent_id'): middle_id = m_id['envent_id'] mid_eid_list.append(middle_id) event_name2 = event_name_search(middle_id) e_nodes_list[middle_id] = event_name2 event_relation.append([start_id, relation1, middle_id]) # print mid_uid_list # print mid_eid_list,'++++++++++++++++' for mid_uid in mid_uid_list: c_string = 'START s1 = node:node_index(uid="' + str(mid_uid) + '") ' c_string += 'MATCH (s1)-[r2]->(s2:Event) return s1,r2,s2 LIMIT 5' uid_result = graph.run(c_string) for i in uid_result: relation2 = i['r2'].type() end_id = dict(i['s2']) if end_id.has_key('uid'): user_name = user_name_search(end_id['uid']) u_nodes_list[end_id['uid']] = user_name event_relation.append([mid_uid, relation2, end_id['uid']]) if end_id.has_key('envent_id'): event_name2 = event_name_search(end_id['envent_id']) e_nodes_list[end_id['envent_id']] = event_name2 event_relation.append( [mid_uid, relation2, end_id['envent_id']]) for mid_eid in mid_eid_list: c_string = 'START s1 = node:event_index(event="' + str(mid_eid) + '") ' c_string += 'MATCH (s1)-[r2]->(s2:Event) return s1,r2,s2 LIMIT 5' eid_result = graph.run(c_string) for i in eid_result: relation2 = i['r2'].type() end_id = dict(i['s2']) if end_id.has_key('uid'): user_name = user_name_search(end_id['uid']) u_nodes_list[end_id['uid']] = user_name event_relation.append([mid_eid, relation2, end_id['uid']]) if end_id.has_key('envent_id'): event_name2 = event_name_search(end_id['envent_id']) e_nodes_list[end_id['envent_id']] = event_name2 event_relation.append( [mid_eid, relation2, end_id['envent_id']]) return {'total_event':len(event_id_list),'user_nodes':u_nodes_list,'event_nodes':e_nodes_list,\ 'relation':event_relation}