def get_google_trend(start_time, end_time, dict_name):
conn = getconn()
cur = conn.cursor()
list_data = get_date(start_time, end_time)
start_time = get_date_item(start_time)
end_time = get_date_item(end_time)
series_dict = {}
try:
dict = {}
for id in dict_name.keys():
series = []
sql = "select trend,date_time from google_trend where DATE(date_time) >= %s and DATE(date_time)<= %s and candidate_id=%s"
count = cur.execute(sql, (start_time, end_time, id))
result = cur.fetchmany(count)
dict_item = {}
# Can be optimized
# ---------------------------------------------------------------------------------------
for item in result:
dict_item[item[1].strftime("%Y-%m-%d")] = item[0]
for item in list_data:
if item not in dict_item:
dict_item[item] = 0
dict[dict_name[id]] = dict_item
for item in list_data:
series.append(dict[dict_name[id]][item])
series_dict[dict_name[id]] = series
result = {}
result["xAxis"] = list_data
result["series"] = series_dict
# -----------------------------------------------------------------------------------------
return result
except:
return 0
finally:
closeAll(conn, cur)
def get_information_all_statistics_count(start_time, end_time, dict_name):
try:
data_list = get_date(start_time, end_time)
dict = {}
result = {}
forum_result = get_information_forum_statistics_count(
start_time, end_time, dict_name)["person"]
facebook_result = get_information_facebook_statistics_count(
start_time, end_time, dict_name)["person"]
twitter_result = get_information_twitter_statistics_count(
start_time, end_time, dict_name)["person"]
news_result = get_information_news_statistics_count(
start_time, end_time, dict_name)["person"]
for item in dict_name.keys():
list = []
for i in range(0, len(data_list)):
list.append(
int(forum_result[dict_name[item]][i]) +
int(facebook_result[dict_name[item]][i]) +
int(twitter_result[dict_name[item]][i]) +
int(news_result[dict_name[item]][i]))
dict[dict_name[item]] = list
result["xAxis"] = dict
result["series"] = data_list
return result
except:
return 0
def get_information_news_statistics_count(start_time, end_time, dict_name):
start_timestamps, end_timestamps = get_time(start_time, end_time)
data_list = get_date(start_time, end_time)
if start_timestamps == 0 or end_timestamps == 0 or dict_name == 0:
return None
statistics = {}
dict = {}
for item in dict_name.keys():
count_list = []
data_dict = {}
name = dict_name[item]
query = {
"query": {
"bool": {
"must": [{
"range": {
"timestamps": {
"gte": start_timestamps,
"lte": end_timestamps
}
}
}, {
"term": {
"keywords": name
}
}],
"must_not": [],
"should": []
}
},
"from": 0,
"size": 10000,
"sort": [],
"aggs": {}
}
result = es.search(index=es_news_index,
doc_type=es_news_type,
body=query)['hits']['hits']
for item in data_list:
data_dict[item] = 0
for item in result:
item_data = item["_source"]["time"].split(" ")[0]
data_dict[item_data] = data_dict[item_data] + 1
for key in data_dict.keys():
count_list.append(data_dict[key])
dict[name] = count_list
statistics["person"] = dict
statistics["date"] = data_list
return statistics
def get_facebook_trend(start_time, end_time, dict_name):
list_data = get_date(start_time, end_time)
start_time, end_time = get_time(start_time, end_time)
series_dict = {}
table_list = []
upshot = {}
for id, name in dict_name.items():
link_dict = {}
post_dict = {}
for item in list_data:
link_dict[item] = 0
post_dict[item] = 0
query = {"query": {"bool": {"must": [{"term": {"facebook_name": name}},
{"range": {"timestamps": {"gte": start_time, "lte": end_time}}}]}},
"from": 0,
"size": 9999}
result = es.search(index=es_facebook_index, doc_type=es_facebook_type, body=query)['hits']['hits']
for item in result:
date = get_timestamps_to_date(item["_source"]["timestamps"])
post_dict[date] = post_dict[date] + 1
link_dict[date] = link_dict[date] + int(item["_source"]["likes"])
aver_list = []
sum_post = 0
sum_link = 0
for item in list_data:
post = int(post_dict[item])
link = int(link_dict[item])
if post == 0 or link == 0:
aver_list.append(0)
else:
aver_list.append(int(link / post))
sum_link += link
sum_post += post
series_dict[name] = aver_list
dict = {}
dict["name"] = name
dict["total_likes"] = sum_link
dict["total_post"] = sum_post
dict["aver_likes"] = int(sum_link / sum_post)
table_list.append(dict)
upshot["echart"] = {"xAxis": list_data, "series": series_dict}
upshot["table"] = table_list
return upshot
def get_popular_info(start_time, end_time, message):
conn = getconn()
cur = conn.cursor()
id_list = message.keys()
list_data = get_date(start_time, end_time)
try:
hxr_dic = {}
series_dict = {}
for name_id in id_list:
sql = """SELECT CAST(`create_data` AS CHAR),`popularity_score`,`candidate_id` FROM popularity WHERE %s <= `create_data` AND `create_data` <= %s AND `candidate_id` = %s ORDER BY `create_data` ASC"""
re = cur.execute(sql, (start_time, end_time, name_id))
if re < 1:
return 0
else:
result = cur.fetchall()
every_dic = {}
for re in result:
every_dic[re[0]] = re[1]
# hxr_dic[message.get(name_id)] = every_dic
for date in list_data:
if date not in every_dic:
every_dic[date] = 0
hxr_dic[message.get(name_id)] = every_dic
series = []
for item in list_data:
series.append(hxr_dic[message.get(name_id)][item])
series_dict[message.get(name_id)] = series
result = {}
result["xAxis"] = list_data
result["series"] = series_dict
return result
except Exception as erro:
app.logger.error(erro)
return 0
finally:
closeAll(conn, cur)
else:
result_dict[item] = round(1 / len(dict), 3)
flat -= round(1 / len(dict), 3)
count += 1
else:
for item in dict.keys():
if count == len(dict):
result_dict[item] = flat
else:
result_dict[item] = round(float(dict[item]) / sum, 3)
flat -= round(float(dict[item]) / sum, 3)
count += 1
return result_dict
except:
return 0
if __name__ == '__main__':
# end_time = datetime.datetime.now().strftime("%Y-%m-%d")
# list = get_date("2018-2-15",end_time)
# for item in list:
# start_time = get_before_time(item,45)
# insert_popularity_demo(start_time,item)
end_time = datetime.datetime.now().strftime("%Y-%m-%d")
list = get_date("2018-2-15", end_time)
for item in list:
start_time = get_before_time(item, 45)
# end_time = datetime.datetime.now().strftime("%Y-%m-%d")
start_time = get_before_time(item, 45)
print(update_popularity_demo(start_time, item))