def article_id():
"""
returns the article at that URL or creates an article and returns it
takes url as URL argument
NOTE: the url should be encoded with quote_plus (Pyton) and encodeURIComponent(Javascript)
:return: article id
"""
url = request.args.get("url", "")
if not url:
flask.abort(400)
try:
article = Article.find_or_create(db_session, url)
return json_result(dict(article_id=article.id))
except Exception as e:
from sentry_sdk import capture_exception
capture_exception(e)
zeeguu.core.log(e)
flask.abort(500)
def get_cohorts_for_article(article_id):
"""
Gets all the cohorts for this article
"""
article = Article.find_by_id(article_id)
return json.dumps(CohortArticleMap.get_cohorts_for_article(article))
def __init__(self, real=False):
super().__init__()
if real:
self.article = Article.find_or_create(ArticleRule.db.session,
url_diesel_fahrverbote)
else:
self.article = self._create_model_object()
self.save(self.article)
def teacher_texts():
"""
Gets all the articles of this teacher
"""
articles = Article.own_texts_for_user(flask.g.user)
article_info_dicts = [article.article_info_for_teacher() for article in articles]
return json.dumps(article_info_dicts)
def upload_own_text():
db_session.rollback()
language = Language.find_or_create(request.form.get("language", ""))
content = request.form.get("content", "")
title = request.form.get("title", "")
new_article_id = Article.create_from_upload(db_session, title, content,
flask.g.user, language)
return str(new_article_id)
def _create_model_object(self):
title = " ".join(self.faker.text().split()[:4])
authors = self.faker.name()
content = self.faker.text()
summary = self.faker.text()
published = datetime.now() - timedelta(minutes=randint(0, 7200))
rss_feed = RSSFeedRule().feed
language = LanguageRule().random
url = UrlRule().url
article = Article(url, title, authors, content, summary, published,
rss_feed, language)
if self._exists_in_db(article):
return self._create_model_object()
return article
def add_article_to_cohort():
"""
Gets all the articles of this teacher
"""
cohort = Cohort.find(request.form.get("cohort_id"))
check_permission_for_cohort(cohort.id)
article = Article.find_by_id(request.form.get("article_id"))
if not CohortArticleMap.find(cohort.id, article.id):
now = datetime.now()
new_mapping = CohortArticleMap(cohort, article, now)
db.session.add(new_mapping)
db.session.commit()
return "OK"
def delete_article_from_cohort():
"""
Gets all the articles of this teacher
"""
cohort = Cohort.find(request.form.get("cohort_id"))
check_permission_for_cohort(cohort.id)
article = Article.find_by_id(request.form.get("article_id"))
mapping = CohortArticleMap.find(cohort.id, article.id)
if mapping:
db.session.delete(mapping)
db.session.commit()
return "OK"
else:
return make_error(401, "That article does not belong to the cohort!")
def upload_articles(cohort_id):
"""
uploads articles for a cohort with input from a POST request
"""
check_permission_for_cohort(cohort_id)
try:
for article_data in json.loads(request.data):
url = Url("userarticle/{}".format(uuid.uuid4().hex))
title = article_data["title"]
authors = article_data["authors"]
content = article_data["content"]
summary = article_data["summary"]
published_time = datetime.now()
language_code = article_data["language_code"]
language = Language.find(language_code)
new_article = Article(
url,
title,
authors,
content,
summary,
published_time,
None, # rss feed
language,
)
db.session.add(new_article)
db.session.flush()
db.session.refresh(new_article)
cohort = Cohort.find(cohort_id)
now = datetime.now()
new_cohort_article_map = CohortArticleMap(cohort, new_article, now)
db.session.add(new_cohort_article_map)
db.session.commit()
return "OK"
except ValueError:
flask.abort(400)
return "ValueError"
def more_like_this_article(user, count, article_id):
"""
Given a article ID find more articles like that one via Elasticsearchs "more_like_this" method
"""
article = Article.find_by_id(article_id)
query_body = build_more_like_this_query(count, article.content,
article.language)
es = Elasticsearch(ES_CONN_STRING)
res = es.search(index=ES_ZINDEX, body=query_body) # execute search
hit_list = res["hits"].get("hits")
# TODO need to make sure either that the searched on article is always a part of the list \
# or that it is never there.
# it could be used to show on website; you searched on X, here is what we found related to X
final_article_mix = _to_articles_from_ES_hits(hit_list)
return [
UserArticle.user_article_info(user, article)
for article in final_article_mix
]
def user_article_info(cls,
user: User,
article: Article,
with_content=False,
with_translations=True):
from zeeguu.core.model import Bookmark
# Initialize returned info with the default article info
returned_info = article.article_info(with_content=with_content)
user_article_info = UserArticle.find(user, article)
if not user_article_info:
returned_info["starred"] = False
returned_info["opened"] = False
returned_info["liked"] = None
returned_info["translations"] = []
return returned_info
returned_info["starred"] = user_article_info.starred is not None
returned_info["opened"] = user_article_info.opened is not None
returned_info["liked"] = user_article_info.liked
if user_article_info.starred:
returned_info["starred_time"] = datetime_to_json(
user_article_info.starred)
if with_translations:
translations = Bookmark.find_all_for_user_and_article(
user, article)
returned_info["translations"] = [
each.serializable_dictionary() for each in translations
]
return returned_info
def _find_article_in_value_or_extra_data(self, db_session):
"""
Finds or creates an article_id
return: articleID or NONE
NOTE: When the article cannot be downloaded anymore,
either because the article is no longer available or the newspaper.parser() fails
"""
if self.event in ALL_ARTICLE_INTERACTION_ACTIONS:
if self.value.startswith("http"):
url = self.value
else:
url = self.find_url_in_extra_data()
if url:
return Article.find_or_create(db_session,
url,
sleep_a_bit=True).id
return None
def _to_articles_from_ES_hits(hits):
articles = []
for hit in hits:
articles.append(Article.find_by_id(hit.get("_id")))
return articles
def test_find_or_create(self):
self.new_art = Article.find_or_create(session,
url_formation_professionnelle)
assert (self.new_art.fk_difficulty)
def test_load_article_without_language_information(self):
art = Article.find_or_create(session, url_plane_crashes)
assert (art)
found = 1
for text in texts:
# print(text.article_id)
if not text.article:
article_id = None
if 'articleID' in text.url.as_canonical_string():
article_id = text.url.as_canonical_string().split("articleID=")[-1]
# print(f'extracted id: {article_id}')
if article_id:
article = Article.query.filter_by(id=article_id).one()
else:
article = Article.find(text.url.as_canonical_string())
# print(text.url.as_canonical_string())
if not article:
not_found += 1
print(f'not found: {not_found}')
else:
found += 1
text.article = article
zeeguu.core.db.session.add(text)
# print(text)
# print(article)
# print(text.url.as_string())
# print(text.article.url.as_string())
print(f'found: {found}')
def _exists_in_db(obj):
return Article.exists(obj)
def own_texts():
r = [e.article_info() for e in Article.own_texts_for_user(flask.g.user)]
return json_result(r)
def articles_correlations():
articles_df = pd.DataFrame(columns=[
"id", "lang", "difficulty", "word_count", "title_length", "opened",
"translated", "spoken", "liked", "closed"
])
all_users = User.find_all()
print(len(all_users))
for reading_language in languages_to_analyze:
print("\nLANGUAGE:", reading_language)
language_id = Language.find(reading_language).id
for user in tqdm(all_users):
if user.learned_language_id == language_id:
events = UserActivityData.find(user)
for event in events:
article_id = event.article_id
if article_id:
article_data = Article.find_by_id(article_id)
if article_data.language_id == language_id:
if not (articles_df['id'] == article_id).any():
title_len = len(article_data.title.split())
df = {
"id": article_id,
"lang": article_data.language_id,
"difficulty": article_data.fk_difficulty,
"word_count": article_data.word_count,
"title_length": title_len,
"opened": 0,
"translated": 0,
"spoken": 0,
"liked": 0,
"closed": 0
}
articles_df = articles_df.append(
df, ignore_index=True)
if event.event == "UMR - OPEN ARTICLE":
articles_df.loc[articles_df.id == article_id,
'opened'] += 1
if event.event == "UMR - TRANSLATE TEXT":
articles_df.loc[articles_df.id == article_id,
'translated'] += 1
if event.event == "UMR - SPEAK TEXT":
articles_df.loc[articles_df.id == article_id,
'spoken'] += 1
if event.event == "UMR - LIKE ARTICLE":
articles_df.loc[articles_df.id == article_id,
'liked'] += 1
if event.event == "UMR - ARTICLE CLOSED":
articles_df.loc[articles_df.id == article_id,
'closed'] += 1
print("Articles:", len(articles_df))
correlation_variables = [
"word_count", "difficulty", "liked", "translated", "spoken",
"opened", "closed", "title_length"
]
# word count & fk_difficulty
spearman_corr = stats.spearmanr(articles_df[correlation_variables[0]],
articles_df[correlation_variables[1]])
print(correlation_variables[0], correlation_variables[1],
spearman_corr[0], spearman_corr[1])
# liked & fk_difficulty
spearman_corr = stats.spearmanr(articles_df[correlation_variables[2]],
articles_df[correlation_variables[1]])
print(correlation_variables[2], correlation_variables[1],
spearman_corr[0], spearman_corr[1])
# number of translations & difficulty
spearman_corr = stats.spearmanr(articles_df[correlation_variables[3]],
articles_df[correlation_variables[1]])
print(correlation_variables[3], correlation_variables[1],
spearman_corr[0], spearman_corr[1])
# number of spoken words & difficulty
spearman_corr = stats.spearmanr(articles_df[correlation_variables[4]],
articles_df[correlation_variables[1]])
print(correlation_variables[4], correlation_variables[1],
spearman_corr[0], spearman_corr[1])
# number of times article is opened & difficulty
spearman_corr = stats.spearmanr(articles_df[correlation_variables[5]],
articles_df[correlation_variables[1]])
print(correlation_variables[5], correlation_variables[1],
spearman_corr[0], spearman_corr[1])
# number of times article is closed & difficulty
spearman_corr = stats.spearmanr(articles_df[correlation_variables[6]],
articles_df[correlation_variables[1]])
print(correlation_variables[6], correlation_variables[1],
spearman_corr[0], spearman_corr[1])
# title length & fk_difficulty
spearman_corr = stats.spearmanr(articles_df[correlation_variables[7]],
articles_df[correlation_variables[1]])
print(correlation_variables[7], correlation_variables[1],
spearman_corr[0], spearman_corr[1])
# title length & number of times article is opened
spearman_corr = stats.spearmanr(articles_df[correlation_variables[5]],
articles_df[correlation_variables[7]])
print(correlation_variables[5], correlation_variables[7],
spearman_corr[0], spearman_corr[1])
def get_one_translation(from_lang_code, to_lang_code):
"""
To think about:
- it would also make sense to separate translation from
logging; or at least, allow for situations where a translation
is not associated with an url... or?
- jul 2021 - Bjorn would like to have the possibility of getting
a translation without an article; can be done; allow for the
articleID to be empty; what would be the downside of that?
- hmm. maybe he can simply work with get_multiple_translations
:return: json array with translations
"""
word_str = request.form["word"].strip(punctuation_extended)
url = request.form.get("url")
title_str = request.form.get("title", "")
context = request.form.get("context", "")
article_id = request.form.get("articleID", None)
if not article_id:
# the url comes from elsewhere not from the reader, so we find or create the article
article = Article.find_or_create(db_session, url)
article_id = article.id
minimal_context, query = minimize_context(context, from_lang_code,
word_str)
# if we have an own / teacher translation that is our first "best guess"
# ML: TODO: word translated in the same text / articleID / url should still be considered
# as an own translation; currently only if the "context" is the same counts;
# which means that translating afslore in a previous paragraph does not count
best_guess = get_own_past_translation(flask.g.user, word_str,
from_lang_code, to_lang_code,
context)
if best_guess:
likelihood = 1
source = "Own past translation"
else:
translations = get_next_results(
{
"from_lang_code": from_lang_code,
"to_lang_code": to_lang_code,
"url": url,
"word": word_str,
"title": title_str,
"query": query,
"context": minimal_context,
},
number_of_results=1,
).translations
best_guess = translations[0]["translation"]
likelihood = translations[0].pop("quality")
source = translations[0].pop("service_name")
bookmark = Bookmark.find_or_create(
db_session,
flask.g.user,
word_str,
from_lang_code,
best_guess,
to_lang_code,
minimal_context,
url,
title_str,
article_id,
)
print(bookmark)
return json_result({
"translation": best_guess,
"bookmark_id": bookmark.id,
"source": source,
"likelihood": likelihood,
})
import zeeguu.core
from zeeguu.core.model import Article, UserArticle
from zeeguu.core.model.starred_article import StarredArticle
session = zeeguu.core.db.session
for sa in StarredArticle.query.all():
try:
article = Article.find_or_create(session, sa.url.as_string())
ua = UserArticle.find_or_create(session,
sa.user,
article,
starred=sa.starred_date)
session.add(ua)
session.commit()
print(f'{sa.starred_date} x {ua.user.name} x {ua.article.title}')
except Exception as ex:
print(f'could not import {sa.url.as_string()}')
print(ex)
def contribute_translation(from_lang_code, to_lang_code):
"""
User contributes a translation they think is appropriate for
a given :param word in :param from_lang_code in a given :param context
The :param translation is in :param to_lang_code
Together with the two words and the textual context, you must submit
also the :param url, :param title of the page where the original
word and context occurred.
:return: in case of success, the bookmark_id and main translation
"""
# All these POST params are mandatory
word_str = unquote_plus(request.form["word"])
translation_str = request.form["translation"]
url = request.form.get("url", "")
context_str = request.form.get("context", "")
title_str = request.form.get("title", "")
# when a translation is added by hand, the servicename_translation is None
# thus we set it to MANUAL
service_name = request.form.get("servicename_translation", "MANUAL")
article_id = None
if "articleID" in url:
article_id = url.split("articleID=")[-1]
url = Article.query.filter_by(
id=article_id).one().url.as_canonical_string()
elif "articleURL" in url:
url = url.split("articleURL=")[-1]
elif "article?id=" in url:
article_id = url.split("article?id=")[-1]
url = Article.query.filter_by(
id=article_id).one().url.as_canonical_string()
else:
# the url comes from elsewhere not from the reader, so we find or create the article
article = Article.find_or_create(db_session, url)
article_id = article.id
# Optional POST param
selected_from_predefined_choices = request.form.get(
"selected_from_predefined_choices", "")
minimal_context, _ = minimize_context(context_str, from_lang_code,
word_str)
bookmark = Bookmark.find_or_create(
db_session,
flask.g.user,
word_str,
from_lang_code,
translation_str,
to_lang_code,
minimal_context,
url,
title_str,
article_id,
)
# Inform apimux about translation selection
data = {
"word_str": word_str,
"translation_str": translation_str,
"url": url,
"context_size": len(context_str),
"service_name": service_name,
}
contribute_trans(data)
return json_result(dict(bookmark_id=bookmark.id))