def view_tweets_about_keyword(keyword):
period, start, end, cache_time = get_period(request, "week")
params = {
"start": start.strftime(time_format),
"end": end.strftime(time_format)
}
keyword_data = cache(process_details,
keyword,
params,
cache_time=cache_time,
path=get_req_path(request))
if isinstance(keyword_data, Response):
return keyword_data
num_tweets = keyword_data["num_tweets"]
tweets = keyword_data["tweets"]
retweets = keyword_data["retweets"]
template_data = {
"keyword": keyword,
"num_tweets": num_tweets,
"num_unique_tweets": len(tweets),
"tweets": tweets,
"retweets": retweets,
"period": period,
"start": display_datetime(start),
"end": display_datetime(end)
}
return render_template("tweets.html",
title=make_title(keyword),
**template_data)
def view_token_co_occurrences(keyword):
period, start, end, cache_time = get_period(request, "week")
params = {
"start": start.strftime(time_format),
"end": end.strftime(time_format)
}
keyword_data = cache(process_tokens,
keyword,
params,
cache_time=cache_time,
path=get_req_path(request))
if isinstance(keyword_data, Response):
return keyword_data
occurrences = []
for k in keyword_data["occurrences"]:
if k["text"] != keyword:
k["pos"] = parse_pos(k["pos"].split("(")[0])
occurrences.append(k)
nums = range(1, len(occurrences) + 1)
template_data = {
"keyword": keyword,
"period": period,
"start": display_datetime(start),
"end": display_datetime(end),
"occurrences": zip(nums, occurrences)
}
return render_template("occurrences.html",
title=make_title(keyword),
**template_data)
def view_news(keyword):
period, start, end, cache_time = get_period(request, "week")
news_data = cache(process_news,
keyword,
start,
end,
cache_time=cache_time,
path=get_req_path(request))
if isinstance(news_data, Response):
return news_data
period_name = {
"day": "dag",
"week": "week",
"month": "maand"
}.get(period, "dag")
news = []
for item in news_data:
item["pubdate"] = display_datetime(item["pubdate"])
del item["nid"]
news.append(item)
template_data = {
"keyword": keyword,
"start": display_datetime(start),
"end": display_datetime(end),
"period": period,
"period_name": period_name,
"news": news
}
return render_template("news.html",
title=make_title(keyword),
**template_data)
def view_anomalies(group):
keywords = cache(tweety.get_group,
group,
cache_time=60 * 60,
path=get_req_path(request))
if isinstance(keywords, Response):
return keywords
keywords = [k["lemma"] for k in keywords]
anomalies = json.loads(redis.get("anomalies"))
anomalies = [a for a in anomalies if a[0] in keywords]
start = datetime.strptime(json.loads(redis.get("anomalies_start")),
time_format)
end = datetime.strptime(json.loads(redis.get("anomalies_end")),
time_format)
template_data = {
"peaks": anomalies,
"start": display_datetime(start),
"end": display_datetime(end),
"num_peaks": len(anomalies)
}
return render_template("anomaly.html",
title=make_title("Piekdetectie"),
**template_data)
def top(group):
max_amount = request.args.get("k", 10, type=int)
period, start, end, cache_time = get_period(request, "day")
params = {
"start": start.strftime(time_format),
"end": end.strftime(time_format),
"group": group
}
data = cache(process_top,
group,
max_amount,
params,
cache_time=cache_time,
path=get_req_path(request))
if isinstance(data, Response):
return data
if len(data) < max_amount:
max_amount = len(data)
template_data = {
"data": data,
"group": group,
"disp_group": display_group(group),
"max_amount": str(max_amount),
"period": period,
"start": display_datetime(start),
"end": display_datetime(end),
"title": make_title("Top {} {}".format(max_amount,
display_group(group)))
}
return render_template("top.html", **template_data)
def view_group(group):
period, start, end, cache_time = get_period(request, "day")
params = {
"start": start.strftime(time_format),
"end": end.strftime(time_format),
"group": group
}
keywords = cache(tweety.get_keywords,
cache_time=cache_time,
path=get_req_path(request),
**params)
if isinstance(keywords, Response):
return keywords
total = sum([entry["count"] for entry in keywords])
for keyword in keywords:
keyword["percentage"] = "{:.2f}".format(keyword["count"] / total * 100)
keyword["count"] = display_number(keyword["count"])
nums = range(1, len(keywords) + 1)
template_data = {
"nums_keywords": zip(nums, keywords),
"group": group,
"disp_group": display_group(group),
"nums": nums,
"total": display_number(total),
"period": period,
"start": display_datetime(start),
"end": display_datetime(end)
}
return render_template("group.html",
title=make_title(template_data["disp_group"]),
**template_data)
def home():
sync_time = redis.get("sync_time")
if sync_time:
sync_time = sync_time.decode("utf-8")
max_amount = request.args.get("k", 10, type=int)
period, start, end, cache_time = get_period(request, "day")
params = {
"start": start.strftime(time_format),
"end": end.strftime(time_format),
"group": "bloemen"
}
bloemen = cache(process_top,
"bloemen",
max_amount,
params,
cache_time=cache_time,
path=get_req_path(request))
params["group"] = "groente_en_fruit"
groente_en_fruit = cache(process_top,
"groente_en_fruit",
max_amount,
params,
cache_time=cache_time,
path=get_req_path(request))
if isinstance(bloemen, Response):
return bloemen
if isinstance(groente_en_fruit, Response):
return groente_en_fruit
template_data = {
"bloemen": bloemen,
"groente_en_fruit": groente_en_fruit,
"sync_time": sync_time,
"start": display_datetime(start),
"end": display_datetime(end),
"period": period
}
return render_template("home.html",
title=make_title("BigTU research project"),
**template_data)
def view_keywords_in_group(group):
"""Show a list of all the keywords in the group."""
keywords = cache(tweety.get_group,
group,
cache_time=60 * 60,
path=get_req_path(request))
if isinstance(keywords, Response):
return keywords
if keywords:
keywords.sort(key=lambda x: x["lemma"])
for k in keywords:
k["pos"] = display_pos(k["pos"])
template_data = {
"disp_group": display_group(group),
"title": make_title("Trefwoorden in {}".format(display_group(group))),
"keywords": keywords,
"total": len(keywords)
}
return render_template("group_keywords.html", **template_data)
def view_stories(group):
period, start, end, cache_time = get_period(request, "week")
params = {
"start": start.strftime(time_format),
"end": end.strftime(time_format)
}
story_data = cache(process_stories,
group,
params,
cache_time=cache_time,
path=get_req_path(request))
if isinstance(story_data, Response):
return story_data
active_stories, closed_stories = story_data
storify_data = []
timeline_data = []
timeline_start = timegm(start.timetuple()) * 1000
timeline_end = timegm(end.timetuple()) * 1000
display_tweets = 11
display_active_stories = 10
display_closed_stories = 5
for story in active_stories:
if not (len(storify_data) < display_active_stories):
break
story = filter_story(story, display_tweets)
timeline_info = {
"label": len(storify_data),
"times": story["cluster_details"]
}
del story["cluster_details"]
storify_data.append(story)
timeline_data.append(timeline_info)
for story in closed_stories:
if not (len(storify_data) <
display_active_stories + display_closed_stories):
break
story = filter_story(story, display_tweets)
timeline_info = {
"label": len(storify_data),
"times": story["cluster_details"]
}
del story["cluster_details"]
storify_data.append(story)
timeline_data.append(timeline_info)
template_data = {
"group":
display_group(group),
"storify_data":
json.dumps(storify_data),
"timeline_data":
json.dumps(timeline_data),
"timeline_start_ts":
timeline_start,
"timeline_end_ts":
timeline_end,
"display_tweets":
display_tweets,
"num_stories":
min(display_active_stories + display_closed_stories,
len(storify_data)),
"start":
display_datetime(start),
"end":
display_datetime(end),
"period":
period
}
return render_template("storify.html",
title=make_title(group),
**template_data)
def view_keyword(keyword):
deluxe = is_deluxe(
current_user
) # users in the "deluxe" group can specify their own time period
period, start, end, cache_time = get_period(request, "week")
if period == "custom":
if not deluxe:
flash(
"Deze functionaliteit is alleen beschikbaar voor goedgekeurde gebruikers.",
"error")
return redirect(url_for("horti.home"))
if (end - start).days > 31:
flash("Periode langer dan een maand is niet toegestaan", "error")
return redirect(url_for("horti.home"))
if start > end:
flash("De einddatum moet na de begindatum zijn.", "error")
return redirect(url_for("horti.home"))
params = {
"start": start.strftime(time_format),
"end": end.strftime(time_format)
}
keyword_data = cache(process_details,
keyword,
params,
cache_time=cache_time,
path=get_req_path(request))
if isinstance(keyword_data, Response):
return keyword_data
urls = keyword_data["URLs"][:16]
for url in urls:
url["display_url"] = shorten(url["link"], 80)
del keyword_data["URLs"]
keyword_data["tagCloud"] = keyword_data["tagCloud"][:200]
photos = enumerate(keyword_data["photos"]
) # number of photo's is limited in processing.py
del keyword_data["photos"]
display_tweets = 11
max_tweets = 200
keyword_data["tweets"] = keyword_data["tweets"][:max_tweets]
keyword_data["retweets"] = keyword_data["retweets"][:display_tweets]
keyword_data["interaction_tweets"] = keyword_data[
"interaction_tweets"][:max_tweets]
num_tweets = keyword_data["num_tweets"]
del keyword_data["num_tweets"]
graph = keyword_data["graph"]
del keyword_data["graph"]
polarity = keyword_data["polarity"]
del keyword_data["polarity"]
polarity_face = display_polarity(polarity)
gtrends_period = {
"day": "now 1-d",
"week": "now 7-d",
"month": "today 1-m"
}.get(period, "now 1-d")
period_name = {
"day": "dag",
"week": "week",
"month": "maand"
}.get(period, "dag")
news = []
for item in keyword_data["news"]:
item["pubdate"] = display_datetime(item["pubdate"])
del item["nid"]
news.append(item)
del keyword_data["news"]
template_data = {
"keyword": keyword,
"keyword_data": json.dumps(keyword_data),
"deluxe": deluxe,
"num_tweets": display_number(num_tweets),
"urls": urls,
"graph": json.dumps(graph),
"photos": photos,
"display_tweets": display_tweets,
"start": display_datetime(start),
"end": display_datetime(end),
"period": period,
"period_name": period_name,
"polarity": polarity,
"polarity_face": polarity_face,
"gtrends_period": gtrends_period,
"news": news
}
return render_template("keyword.html",
title=make_title(keyword),
**template_data)
def edit_group(group):
roles = get_roles(current_user)
if ("g:" + group) not in roles and "admin" not in roles:
flash(
"U heeft geen rechten om de groep \"{}\" aan te passen.".format(
display_group(group)), "error")
return redirect(url_for("horti.home"))
if request.method == "GET":
keywords = cache(tweety.get_group,
group,
cache_time=60 * 60,
path=get_req_path(request))
if isinstance(keywords, Response):
return keywords
if keywords:
keywords.sort(key=lambda x: x["lemma"])
for k in keywords:
k["pos"] = display_pos(k["pos"])
template_data = {
"keywords": keywords,
"group": group,
"disp_group": display_group(group),
"title": make_title("{} aanpassen".format(group))
}
return render_template("edit_group.html", **template_data)
elif request.method == "POST":
data = json.loads(request.data)
keywords = data["keywords"]
if data["action"] == "delete":
for k in keywords:
k["pos"] = parse_pos(k["pos"])
keywords = [(k["lemma"], k["pos"]) for k in keywords]
current_keywords = json.loads(tweety.get_group(group))
current_keywords = [(k["lemma"], k["pos"])
for k in current_keywords]
new_keywords = set(current_keywords) - set(keywords)
new_keywords = [{"lemma": k[0], "pos": k[1]} for k in new_keywords]
tweety.put_group(group, data=json.dumps(new_keywords))
cache(tweety.get_group,
group,
cache_time=60 * 60,
force_refresh=True)
return jsonify({"status": "ok"})
elif data["action"] == "add":
keywords = [(k["lemma"], k["pos"]) for k in keywords]
user_lemmas = [k[0] for k in keywords]
key = "".join([choice(ascii_letters) for _ in range(11)])
lemmatize.apply_async((key, user_lemmas), queue="workers")
current_keywords = json.loads(tweety.get_group(group))
current_keywords = [(k["lemma"], k["pos"])
for k in current_keywords]
processed_lemmas = None
while processed_lemmas is None:
processed_lemmas = redis.get(key)
sleep(0.1)
processed_lemmas = json.loads(processed_lemmas)
diff = {}
for (p, u) in zip(processed_lemmas, user_lemmas):
if u != p:
diff[u] = p
if diff:
return jsonify({"status": "diff", "diff": diff})
new_keywords = set(current_keywords) | set(keywords)
new_keywords = [{"lemma": k[0], "pos": k[1]} for k in new_keywords]
tweety.put_group(group, data=json.dumps(new_keywords))
cache(tweety.get_group,
group,
cache_time=60 * 60,
force_refresh=True)
return jsonify({"status": "ok"})