def get_all_forum_pages(dict_name, start_date, end_date):
start_timestamps, end_timestamps = get_time(start_date, end_date)
if start_timestamps == 0 or end_timestamps == 0:
return None
list = []
for item in dict_name.keys():
dict = {}
dict["term"] = {"keywords": dict_name[item]}
list.append(dict)
query = {
"query": {
"bool": {
"must": [{
"range": {
"timestamps": {
"gte": start_timestamps,
"lte": end_timestamps
}
}
}, {
"bool": {
"should": list
}
}]
}
},
"from": 0,
"size": 10000,
"sort": [],
"aggs": {}
}
result = es.search(index=es_forum_index,
doc_type=es_forum_type,
body=query)['hits']['hits']
return (len(result))
def get_forum_pages(name, start_date, end_date):
start_timestamps, end_timestamps = get_time(start_date, end_date)
if start_timestamps == 0 or end_timestamps == 0:
return None
query = {
"query": {
"bool": {
"must": [{
"range": {
"timestamps": {
"gte": start_timestamps,
"lte": end_timestamps
}
}
}, {
"term": {
"keywords": name
}
}],
"must_not": [],
"should": []
}
},
"from": 0,
"size": 10000,
"sort": [],
"aggs": {}
}
result = es.search(index=es_forum_index,
doc_type=es_forum_type,
body=query)['hits']['hits']
return len(result)
def get_all_forum_data(dict_name, page_number, page_size, start_date,
end_date):
start_from = (page_number - 1) * page_size
start_timestamps, end_timestamps = get_time(start_date, end_date)
if start_timestamps == 0 or end_timestamps == 0:
return None
list = []
for item in dict_name.keys():
dict = {}
dict["term"] = {"keywords": dict_name[item]}
list.append(dict)
query = {
"query": {
"bool": {
"must": [{
"range": {
"timestamps": {
"gte": start_timestamps,
"lte": end_timestamps
}
}
}, {
"bool": {
"should": list
}
}]
}
},
"from": start_from,
"size": page_size,
"sort": [],
"aggs": {}
}
result = es.search(index=es_forum_index,
doc_type=es_forum_type,
body=query)['hits']['hits']
list = []
for item in result:
dict = {}
dict["context"] = item["_source"]["context"]
dict["title"] = item["_source"]["title"]
dict["name"] = item["_source"]["keywords"]
dict["author"] = item["_source"]["author"]
dict["address"] = item["_source"]["address"]
dict["comment"] = item["_source"]["comments"]
dict["reviews"] = item["_source"]["reviews"]
dict["source"] = item["_source"]["source"]
dict["likes"] = item["_source"]["likes"]
dict["buddha_operation"] = item["_source"]["buddha_operation"]
dict["tread"] = item["_source"]["tread"]
dict["time"] = item["_source"]["time"].split(" ")[0]
list.append(dict)
return list
def get_popularity(start_time, end_time, dict_name):
try:
start_time, end_time = get_time(start_time, end_time)
fb = get_fb_aver_link(dict_name, start_time, end_time)
tw = get_tw_count(dict_name, start_time, end_time)
news = get_news_count(dict_name, start_time, end_time)
ptt = get_ptt_popularity(dict_name, start_time, end_time)
if fb == 0:
fb = {}
for key, value in dict_name.items():
fb[value] = 0
if tw == 0:
tw = {}
for key, value in dict_name.items():
tw[value] = 0
if news == 0:
news = {}
for key, value in dict_name.items():
news[value] = 0
if ptt == 0:
ptt = {}
for key, value in dict_name.items():
ptt[value] = 0
count = 1
dict = {}
flat = 1
sum = 0
result_dict = {}
for id in dict_name.keys():
sorce = round((fb[dict_name[id]] * fb_weight +
ptt[dict_name[id]] * ptt_weight +
tw[dict_name[id]] * tw_weight +
news[dict_name[id]] * news_weight), 3)
dict[dict_name[id]] = sorce
sum += sorce
for id in dict_name.keys():
map_dict = {}
map_dict["partisan"] = get_partisan(id)
if count == len(dict_name):
map_dict["sorce"] = round(flat, 3)
result_dict[dict_name[id]] = map_dict
else:
sorce = dict[dict_name[id]] / sum
map_dict["sorce"] = sorce
result_dict[dict_name[id]] = map_dict
flat -= sorce
count += 1
return result_dict
except:
return 0
def get_information_news_statistics_count(start_time, end_time, dict_name):
start_timestamps, end_timestamps = get_time(start_time, end_time)
data_list = get_date(start_time, end_time)
if start_timestamps == 0 or end_timestamps == 0 or dict_name == 0:
return None
statistics = {}
dict = {}
for item in dict_name.keys():
count_list = []
data_dict = {}
name = dict_name[item]
query = {
"query": {
"bool": {
"must": [{
"range": {
"timestamps": {
"gte": start_timestamps,
"lte": end_timestamps
}
}
}, {
"term": {
"keywords": name
}
}],
"must_not": [],
"should": []
}
},
"from": 0,
"size": 10000,
"sort": [],
"aggs": {}
}
result = es.search(index=es_news_index,
doc_type=es_news_type,
body=query)['hits']['hits']
for item in data_list:
data_dict[item] = 0
for item in result:
item_data = item["_source"]["time"].split(" ")[0]
data_dict[item_data] = data_dict[item_data] + 1
for key in data_dict.keys():
count_list.append(data_dict[key])
dict[name] = count_list
statistics["person"] = dict
statistics["date"] = data_list
return statistics
def get_news_data(name, page_number, page_size, start_date, end_date):
start_from = (page_number - 1) * page_size
start_timestamps, end_timestamps = get_time(start_date, end_date)
if start_timestamps == 0 or end_timestamps == 0:
return None
query = {
"query": {
"bool": {
"must": [{
"range": {
"timestamps": {
"gte": start_timestamps,
"lte": end_timestamps
}
}
}, {
"term": {
"keywords": name
}
}],
"must_not": [],
"should": []
}
},
"from": start_from,
"size": page_size,
"sort": [],
"aggs": {}
}
result = es.search(index=es_news_index, doc_type=es_news_type,
body=query)['hits']['hits']
list = []
for item in result:
dict = {}
dict["context"] = item["_source"]["context"]
dict["source"] = item["_source"]["news_source"]
dict["keywords"] = item["_source"]["keywords"]
dict["title"] = item["_source"]["title"]
dict["summary"] = item["_source"]["summary"]
dict["url"] = item["_source"]["url"]
dict["images"] = item["_source"]["images"]
dict["time"] = item["_source"]["time"].split(" ")[0]
list.append(dict)
return list
def get_facebook_trend(start_time, end_time, dict_name):
list_data = get_date(start_time, end_time)
start_time, end_time = get_time(start_time, end_time)
series_dict = {}
table_list = []
upshot = {}
for id, name in dict_name.items():
link_dict = {}
post_dict = {}
for item in list_data:
link_dict[item] = 0
post_dict[item] = 0
query = {"query": {"bool": {"must": [{"term": {"facebook_name": name}},
{"range": {"timestamps": {"gte": start_time, "lte": end_time}}}]}},
"from": 0,
"size": 9999}
result = es.search(index=es_facebook_index, doc_type=es_facebook_type, body=query)['hits']['hits']
for item in result:
date = get_timestamps_to_date(item["_source"]["timestamps"])
post_dict[date] = post_dict[date] + 1
link_dict[date] = link_dict[date] + int(item["_source"]["likes"])
aver_list = []
sum_post = 0
sum_link = 0
for item in list_data:
post = int(post_dict[item])
link = int(link_dict[item])
if post == 0 or link == 0:
aver_list.append(0)
else:
aver_list.append(int(link / post))
sum_link += link
sum_post += post
series_dict[name] = aver_list
dict = {}
dict["name"] = name
dict["total_likes"] = sum_link
dict["total_post"] = sum_post
dict["aver_likes"] = int(sum_link / sum_post)
table_list.append(dict)
upshot["echart"] = {"xAxis": list_data, "series": series_dict}
upshot["table"] = table_list
return upshot
def get_twitter_data(name, page_number, page_size, start_date, end_date):
start_from = (page_number - 1) * page_size
start_timestamps, end_timestamps = get_time(start_date, end_date)
query = {
"query": {
"bool": {
"must": [{
"range": {
"timestamps": {
"gte": start_timestamps,
"lte": end_timestamps
}
}
}, {
"term": {
"twitter_search": name
}
}],
"must_not": [],
"should": []
}
},
"from": start_from,
"size": page_size,
"sort": [],
"aggs": {}
}
result = es.search(index=es_twitter_index,
doc_type=es_twitter_type,
body=query)['hits']['hits']
list = []
for item in result:
dict = {}
dict["context"] = item["_source"]["context"]
dict["likes"] = item["_source"]["likes"]
dict["name"] = item["_source"]["twitter_search"]
dict["twitter_name"] = item["_source"]["twitter_name"]
dict["comment"] = item["_source"]["comment"]
dict["time"] = item["_source"]["time"].split(" ")[0]
list.append(dict)
return list
def get_candidates(dict_name):
try:
date = time.strftime("%Y-%m-%d")
start_time, end_time = get_time(date, date)
sum = 0
dict = {}
face_dict = {}
face_sum = 0
for item in dict_name.keys():
name = dict_name[item]
query = {
"query": {
"bool": {
"must": [{
"term": {
"keywords": name
}
}, {
"range": {
"timestamps": {
"gte": start_time,
"lte": end_time
}
}
}]
}
},
"from": 0,
"size": 9999
}
result = es.search(index=es_news_index,
doc_type=es_news_type,
body=query)['hits']['hits']
new_count = len(result)
query = {
"query": {
"bool": {
"must": [{
"term": {
"twitter_search": name
}
}, {
"range": {
"timestamps": {
"gte": start_time,
"lte": end_time
}
}
}]
}
},
"from": 0,
"size": 9999
}
result = es.search(index=es_twitter_index,
doc_type=es_twitter_type,
body=query)['hits']['hits']
twitter_count = len(result)
name_count = twitter_count + new_count
sum += name_count
dict[name] = name_count
query = {
"query": {
"bool": {
"must": [{
"term": {
"facebook_name": name
}
}, {
"range": {
"timestamps": {
"gte": start_time,
"lte": end_time
}
}
}]
}
},
"from": 0,
"size": 9999
}
result = es.search(index=es_facebook_index,
doc_type=es_facebook_type,
body=query)['hits']['hits']
facebook_count = 0
for item in result:
facebook_count += int(item["_source"]["likes"])
face_sum += facebook_count
face_dict[name] = facebook_count
if sum == 0:
pass
else:
flag = len(dict)
duty = 1.0
i = 1
for item in dict.keys():
if i != flag:
score = round(float(dict[item]) / sum, 3)
duty -= score
dict[item] = score
else:
dict[item] = duty
if face_sum == 0:
pass
else:
flag = len(face_sum)
duty = 1
i = 1
for item in face_dict.keys():
if i != flag:
score = round(float(face_dict[item]) / face_sum, 3)
duty -= score
face_dict[item] = score
else:
face_dict[item] = duty
result_dict = {}
result_dict["heat"] = dict
result_dict["activity"] = face_dict
return result_dict
except Exception as e:
return 0