def test_add_new_word_to_DB(self):
word = "baum"
rank = UserWord.find_rank(word, self.de)
new_word = UserWord(word, self.de, rank)
db.session.add(new_word)
self.mir.star(new_word)
db.session.commit()
def _create_model_object(self, user, force_quality=True, **kwargs):
"""
Creates a Bookmark object with random data.
Behind the random words, a random number is added since the Faker library does not have too many RANDOM words
and random words get repeated whenever many random bookmarks are created. To overcome the problem of bookmarks
with duplicate words in the database, a random number is added.
Also, if force_quality is set (default) and the bookmark is not .quality_bookmark() the process is
reiterated till this is true. This simplifies some of the tests
:param user: User Object, to which the bookmark is assigned.
:param kwargs: Holds any of the 'props' as key if a field should not be random
:return:
"""
bookmark = None
while not bookmark:
from tests_zeeguu.rules.text_rule import TextRule
random_text = TextRule().text
random_origin_word = self.faker.word() + str(random.random())
random_origin_language = LanguageRule().random
random_translation_word = self.faker.word() + str(random.random())
random_translation_language = LanguageRule().random
if UserWord.exists(random_origin_word, random_origin_language) \
or UserWord.exists(random_translation_word, random_translation_language):
return self._create_model_object(user)
random_origin = UserWordRule(random_origin_word,
random_origin_language).user_word
random_translation = UserWordRule(
random_translation_word, random_translation_language).user_word
random_date = self.faker.date_time_this_month()
from tests_zeeguu.rules.article_rule import ArticleRule
random_article = ArticleRule(real=True).article
bookmark = Bookmark(random_origin, random_translation, user,
random_text, random_date)
if force_quality and not bookmark.quality_bookmark():
# print ("random bookmark was of low quality. retrying...")
bookmark = False
for k in kwargs:
if k in self.props:
setattr(bookmark, k, kwargs.get(k))
if self._exists_in_db(bookmark):
return self._create_model_object(user)
return bookmark
def find_or_create(cls, session, user, _origin: str, _origin_lang: str,
_translation: str, _translation_lang: str,
_context: str, _url: str, _url_title: str,
article_id: int):
"""
if the bookmark does not exist, it creates it and returns it
if it exists, it ** updates the translation** and returns the bookmark object
:param _origin:
:param _context:
:param _url:
:return:
"""
origin_lang = Language.find_or_create(_origin_lang)
translation_lang = Language.find_or_create(_translation_lang)
origin = UserWord.find_or_create(session, _origin, origin_lang)
article = Article.query.filter_by(id=article_id).one()
url = Url.find_or_create(session, article.url.as_string(), _url_title)
context = Text.find_or_create(session, _context, origin_lang, url,
article)
translation = UserWord.find_or_create(session, _translation,
translation_lang)
now = datetime.now()
try:
# try to find this bookmark
bookmark = Bookmark.find_by_user_word_and_text(
user, origin, context)
# update the translation
bookmark.translation = translation
except sqlalchemy.orm.exc.NoResultFound as e:
bookmark = cls(origin, translation, user, context, now)
except Exception as e:
raise e
session.add(bookmark)
session.commit()
return bookmark
def __getitem__(self, args):
word = self.cache.get(args, None)
if word is None:
word = UserWord(*args)
zeeguu.db.session.add(word)
self.cache[args] = word
return word
def get_possible_translations(from_lang_code, to_lang_code):
"""
Returns a list of possible translations for this
:param word: word to be translated
:param from_lang_code:
:param to_lang_code:
:return: json array with dictionaries. each of the dictionaries contains at least
one 'translation' and one 'translation_id' key.
In the future we envision that the dict will contain
other types of information, such as relative frequency,
"""
translations_json = []
context = request.form.get('context', '')
url = request.form.get('url', '')
word = request.form['word']
main_translation, alternatives = zeeguu.api.translation_service.translate_from_to(
word, from_lang_code, to_lang_code)
lan = Language.find(from_lang_code)
likelihood = 1.0
for translation in alternatives:
wor = UserWord.find(translation, lan)
zeeguu.db.session.add(wor)
zeeguu.db.session.commit()
t_dict = dict(translation_id=wor.id,
translation=translation,
likelihood=likelihood)
translations_json.append(t_dict)
likelihood -= 0.01
return json_result(dict(translations=translations_json))
def get_possible_translations(from_lang_code, to_lang_code):
"""
Returns a list of possible translations for this
:param word: word to be translated
:param from_lang_code:
:param to_lang_code:
:return: json array with dictionaries. each of the dictionaries contains at least
one 'translation' and one 'translation_id' key.
In the future we envision that the dict will contain
other types of information, such as relative frequency,
"""
translations_json = []
context = request.form.get('context', '')
url = request.form.get('url', '')
word = request.form['word']
main_translation, alternatives = zeeguu.api.translation_service.translate_from_to(word, from_lang_code, to_lang_code)
lan = Language.find(from_lang_code)
likelihood = 1.0
for translation in alternatives:
wor = UserWord.find(translation, lan)
zeeguu.db.session.add(wor)
zeeguu.db.session.commit()
t_dict = dict(translation_id= wor.id,
translation=translation,
likelihood=likelihood)
translations_json.append(t_dict)
likelihood -= 0.01
return json_result(dict(translations=translations_json))
def add_bookmark(user, original_language, original_word, translation_language, translation_word, date, the_context, the_url, the_url_title):
url = Url.find (the_url)
text = Text.find_or_create(the_context, translation_language, url)
origin = UserWord.find(original_word.lower(), original_language)
translation = UserWord.find(translation_word.lower(), translation_language)
zeeguu.db.session.add(url)
zeeguu.db.session.add(text)
zeeguu.db.session.add(origin)
zeeguu.db.session.add(translation)
t1= Bookmark(origin, translation, user, text, date)
zeeguu.db.session.add(t1)
zeeguu.db.session.commit()
add_probability_to_existing_words_of_user(user,t1,original_language)
def test_add_new_word_to_DB(self):
word = "baum"
rank = UserWord.find_rank(word, self.de)
new_word = UserWord(word, self.de, rank)
db.session.add(new_word)
self.mir.star(new_word)
db.session.commit()
def bookmark_with_context(from_lang_code, to_lang_code, word_str, url_str,
title_str, context_str, translation_str):
"""
This function will lookup a given word-text pair, and if found, it will return
that bookmark rather than a new one
:param from_lang_code:
:param to_lang_code:
:param word_str:
:param url_str:
:param title_str:
:param context_str:
:param translation_str:
:return:
"""
from_lang = Language.find(from_lang_code)
to_lang = Language.find(to_lang_code)
user_word = UserWord.find(word_str, from_lang)
url = Url.find(url_str, title_str)
zeeguu.db.session.add(url)
zeeguu.db.session.commit()
context = Text.find_or_create(context_str, from_lang, url)
zeeguu.db.session.add(context)
zeeguu.db.session.commit()
translation = UserWord.find(translation_str, to_lang)
try:
bookmark = Bookmark.find_all_by_user_word_and_text(
flask.g.user, user_word, context)[0]
# TODO: Think about updating the date of this bookmark, or maybe creating a duplicate
# otherwise, in the history this translation will not be visible!
except Exception:
bookmark = Bookmark(user_word, translation, flask.g.user, context,
datetime.now())
zeeguu.db.session.add(bookmark)
bookmark.calculate_probabilities_after_adding_a_bookmark(
flask.g.user, bookmark.origin.language)
zeeguu.db.session.commit()
return str(bookmark.id)
def add_bookmark(db, user, original_language, original_word,
translation_language, translation_word, date, the_context,
the_url, the_url_title):
session = db.session
url = Url.find_or_create(session, the_url, the_url_title)
text = Text.find_or_create(session, the_context, translation_language, url)
origin = UserWord.find_or_create(session, original_word, original_language)
translation = UserWord.find_or_create(session, translation_word,
translation_language)
b1 = Bookmark(origin, translation, user, text, date)
db.session.add(b1)
db.session.commit()
return b1
def _create_model_object(self):
random_word = self.faker.word()
random_language = LanguageRule().random
user_word = UserWord(random_word, random_language)
if self._exists_in_db(user_word):
return self._create_model_object()
return user_word
def __init__(self, word=None, language=None):
super().__init__()
if word is None or language is None:
self.user_word = self._create_model_object()
else:
self.user_word = UserWord(word, language)
if not self._exists_in_db(self.user_word):
self.save(self.user_word)
def bookmark_with_context(from_lang_code, to_lang_code, word_str, url_str, title_str, context_str, translation_str):
"""
This function will lookup a given word-text pair, and if found, it will return
that bookmark rather than a new one
:param from_lang_code:
:param to_lang_code:
:param word_str:
:param url_str:
:param title_str:
:param context_str:
:param translation_str:
:return:
"""
from_lang = Language.find(from_lang_code)
to_lang = Language.find(to_lang_code)
user_word = UserWord.find(word_str, from_lang)
url = Url.find(url_str, title_str)
zeeguu.db.session.add(url)
zeeguu.db.session.commit()
context = Text.find_or_create(context_str, from_lang, url)
zeeguu.db.session.add(context)
zeeguu.db.session.commit()
translation = UserWord.find(translation_str, to_lang)
try:
bookmark = Bookmark.find_all_by_user_word_and_text(flask.g.user, user_word, context)[0]
# TODO: Think about updating the date of this bookmark, or maybe creating a duplicate
# otherwise, in the history this translation will not be visible!
except Exception:
bookmark = Bookmark(user_word, translation, flask.g.user, context, datetime.now())
zeeguu.db.session.add(bookmark)
bookmark.calculate_probabilities_after_adding_a_bookmark(flask.g.user, bookmark.origin.language)
zeeguu.db.session.commit()
return str(bookmark.id)
def _create_model_object(self, user, **kwargs):
bookmark = None
while not bookmark:
random_url = UrlRule().url
random_text = TextRule().text
random_origin_word = self.faker.word()
random_origin_language = LanguageRule().random
random_translation_word = self.faker.word()
random_translation_language = LanguageRule().random
if UserWord.exists(random_origin_word, random_origin_language) \
or UserWord.exists(random_translation_word, random_translation_language):
return self._create_model_object(user)
random_origin = UserWordRule(random_origin_word,
random_origin_language).user_word
random_translation = UserWordRule(
random_translation_word, random_translation_language).user_word
random_date = self.faker.date_time_this_month()
bookmark = Bookmark(random_origin, random_translation, user,
random_text, random_date)
if not bookmark.quality_bookmark():
bookmark = False
# print ("random bookmark was of low quality. retrying...")
for k in kwargs:
if k in self.props:
setattr(bookmark, k, kwargs.get(k))
if self._exists_in_db(bookmark):
return self._create_model_object(user)
return bookmark
def add_new_translation_to_bookmark(word_translation, bookmark_id):
bookmark = Bookmark.query.filter_by(id=bookmark_id).first()
translations_of_bookmark = bookmark.translations_list
for transl in translations_of_bookmark:
if transl.word == word_translation:
return 'FAIL'
translation_user_word = UserWord.find(word_translation,
translations_of_bookmark[0].language)
bookmark.add_new_translation(translation_user_word)
zeeguu.db.session.add(translation_user_word)
zeeguu.db.session.commit()
return "OK"
def set_know_word_prob():
zeeguu.app.test_request_context().push()
zeeguu.db.session.commit()
enc_probs = EncounterBasedProbability.find_all()
ex_probs = ExerciseBasedProbability.find_all()
for prob in enc_probs:
user = prob.user
word = prob.ranked_word.word
language = prob.ranked_word.language
user_word = None
if UserWord.exists(word, language):
user_word = UserWord.find(word, language)
if ExerciseBasedProbability.exists(user, user_word):
ex_prob = ExerciseBasedProbability.find(user, user_word)
known_word_prob = KnownWordProbability.calculateKnownWordProb(
ex_prob.probability, prob.probability)
known_word_probability_obj = KnownWordProbability.find(
user, user_word, prob.ranked_word, known_word_prob)
else:
known_word_probability_obj = KnownWordProbability.find(
user, None, prob.ranked_word, prob.probability)
zeeguu.db.session.add(known_word_probability_obj)
zeeguu.db.session.commit()
for prob in ex_probs:
user = prob.user
language = prob.user_word.language
word = prob.user_word.word
ranked_word = None
if RankedWord.exists(word, language):
ranked_word = RankedWord.find(word, language)
if not EncounterBasedProbability.exists(user, ranked_word):
if UserWord.exists(word, language):
user_word = UserWord.find(word, language)
known_word_probability_obj = KnownWordProbability(
user, user_word, ranked_word, prob.probability)
zeeguu.db.session.add(known_word_probability_obj)
zeeguu.db.session.commit()
print('job3')
def add_new_translation_to_bookmark(word_translation, bookmark_id):
bookmark = Bookmark.query.filter_by(
id=bookmark_id
).first()
translations_of_bookmark = bookmark.translations_list
for transl in translations_of_bookmark:
if transl.word == word_translation:
return 'FAIL'
translation_user_word = UserWord.find(word_translation, translations_of_bookmark[0].language)
bookmark.add_new_translation(translation_user_word)
zeeguu.db.session.add(translation_user_word)
zeeguu.db.session.commit()
return "OK"
def test_importance_level(self):
word = "beschloss"
if zeeguu.model.ranked_word.RankedWord.exists(word.lower(), self.de):
rank = model.UserWord.find_rank(word.lower(), self.de)
new_word = model.UserWord.find(word, self.de)
else:
new_word = model.UserWord.find(word, self.de)
db.session.add(new_word)
db.session.commit()
word = "unexistingword"
beschloss = UserWord.find(word, self.de)
assert beschloss
assert beschloss.importance_level() == 0
def test_importance_level(self):
word = "beschloss"
if zeeguu.model.ranked_word.RankedWord.exists(word.lower(), self.de):
rank = model.UserWord.find_rank(word.lower(), self.de)
new_word = model.UserWord.find(word,self.de)
else:
new_word = model.UserWord.find(word,self.de)
db.session.add(new_word)
db.session.commit()
word = "unexistingword"
beschloss = UserWord.find(word, self.de)
assert beschloss
assert beschloss.importance_level() == 0
def _create_model_object(self, word=None, language=None):
tmp_word = word
tmp_language = language
if tmp_word is None:
tmp_word = self.faker.word()
if tmp_language is None:
tmp_language = LanguageRule().random
user_word = UserWord(tmp_word, tmp_language)
if self._exists_in_db(user_word):
return self._create_model_object(word, language)
return user_word
def set_default_exercise_based_prob():
zeeguu.app.test_request_context().push()
zeeguu.db.session.commit()
users = User.find_all()
languages = Language.all()
for user in users:
for language in languages:
user_words_by_language = UserWord.find_by_language(language)
for word in user_words_by_language:
if ExerciseBasedProbability.exists(user, word):
prob = ExerciseBasedProbability.find(user, word)
bookmarks_by_user_and_word = Bookmark.find_all_by_user_and_word(
user, word)
total_prob = 0
for bookmark in bookmarks_by_user_and_word:
prob.calculate_known_bookmark_probability(bookmark)
total_prob += float(prob.probability)
if bookmarks_by_user_and_word:
prob.probability = total_prob / len(
bookmarks_by_user_and_word)
zeeguu.db.session.commit()
print('job1')
def test_find_word(self):
assert UserWord.find(self.random_origin_word,
self.random_origin_language)
def words(self):
for word in re.split(re.compile("[^\\w]+", re.U), self.content):
yield UserWord.find(word, self.language)
def update_existing_word_ranks(language):
for user_word in UserWord.find_by_language(language):
new_rank = RankedWord.find(user_word.word, language)
user_word.set_rank(new_rank)
zeeguu.db.session.add(user_word)
zeeguu.db.session.commit()
def test_search_1(self):
word = UserWord.find("hauen345", self.de)
s = model.Search(self.mir, word, self.de)
db.session.add(s)
db.session.commit()
def test_find_word(self):
word = "baum"
assert UserWord.find(word, self.de)
def test_find_word(self):
word = "baum"
assert UserWord.find(word, self.de)
def test_search_1(self):
word = UserWord.find("hauen345", self.de)
s = model.Search(self.mir, word, self.de)
db.session.add(s)
db.session.commit()
def _exists_in_db(obj):
return UserWord.exists(obj.word, obj.language)
def calculate_probabilities_after_adding_a_bookmark(self, user,language):
"""
ML: This has to be refactored.
It's a mess.
The idea is: you've just added a bookmark.
There are two things to do:
1. update the probabilities of the context words (they have been
encountered, and not translated)
2. update the probabilities of the word itself
-
:param user:
:param language:
:return:
"""
# 1. computations for adding encounter based probability for the context words
for word in self.context_words_with_rank():
enc_prob = EncounterBasedProbability.find_or_create(word, user, language)
zeeguu.db.session.add(enc_prob)
zeeguu.db.session.commit()
user_word = None
ranked_word = enc_prob.ranked_word
if UserWord.exists(word,language):
user_word = UserWord.find(word,language)
if ExerciseBasedProbability.exists(user,user_word): #checks if exercise based probability exists for words in context
ex_prob = ExerciseBasedProbability.find(user,user_word)
known_word_prob = KnownWordProbability.find(user,user_word,ranked_word)
known_word_prob.probability = known_word_prob.calculateKnownWordProb(ex_prob.probability, enc_prob.probability) #updates known word probability as exercise based probability already existed.
else:
if KnownWordProbability.exists(user, user_word,ranked_word):
known_word_prob = KnownWordProbability.find(user,user_word,ranked_word)
known_word_prob.probability = enc_prob.probability # updates known word probability as encounter based probability already existed
else:
known_word_prob = KnownWordProbability.find(user,user_word,ranked_word, enc_prob.probability) # new known word probability created as it did not exist
zeeguu.db.session.add(known_word_prob)
# 2. Update the probabilities of the word itself
# 2.a) exercise based prob
# ML: Should this thing change?
# The ex based probability should probably not change after I add a bookmark
# Commenting out the following lines: s
# ex_prob = ExerciseBasedProbability.find(user, self.origin)
# if ex_prob:
# ex_prob.update_probability_after_adding_bookmark_with_same_word(self,user)
# zeeguu.db.session.add(ex_prob)
# 2.b) encounter based prob
ranked_word = RankedWord.find(self.origin.word, language)
if ranked_word: #checks if ranked_word exists for that looked up word
if EncounterBasedProbability.exists(user, ranked_word): # checks if encounter based probability exists for that looked up word
enc_prob = EncounterBasedProbability.find(user, ranked_word)
enc_prob.word_has_just_beek_bookmarked()
db.session.add(enc_prob)
db.session.commit()
# 2.c) update known word probability if it exists
if KnownWordProbability.exists(user, self.origin,ranked_word):
known_word_prob = KnownWordProbability.find(user,self.origin,ranked_word)
known_word_prob.word_has_just_beek_bookmarked()
db.session.add(known_word_prob)
db.session.commit()
def calculate_probabilities_after_adding_a_bookmark(self, user, language):
"""
ML: This has to be refactored.
It's a mess.
The idea is: you've just added a bookmark.
There are two things to do:
1. update the probabilities of the context words (they have been
encountered, and not translated)
2. update the probabilities of the word itself
-
:param user:
:param language:
:return:
"""
# 1. computations for adding encounter based probability for the context words
for word in self.context_words_with_rank():
enc_prob = EncounterBasedProbability.find_or_create(
word, user, language)
zeeguu.db.session.add(enc_prob)
zeeguu.db.session.commit()
user_word = None
ranked_word = enc_prob.ranked_word
if UserWord.exists(word, language):
user_word = UserWord.find(word, language)
if ExerciseBasedProbability.exists(
user, user_word
): #checks if exercise based probability exists for words in context
ex_prob = ExerciseBasedProbability.find(user, user_word)
known_word_prob = KnownWordProbability.find(
user, user_word, ranked_word)
known_word_prob.probability = known_word_prob.calculateKnownWordProb(
ex_prob.probability, enc_prob.probability
) #updates known word probability as exercise based probability already existed.
else:
if KnownWordProbability.exists(user, user_word, ranked_word):
known_word_prob = KnownWordProbability.find(
user, user_word, ranked_word)
known_word_prob.probability = enc_prob.probability # updates known word probability as encounter based probability already existed
else:
known_word_prob = KnownWordProbability.find(
user, user_word, ranked_word, enc_prob.probability
) # new known word probability created as it did not exist
zeeguu.db.session.add(known_word_prob)
# 2. Update the probabilities of the word itself
# 2.a) exercise based prob
# ML: Should this thing change?
# The ex based probability should probably not change after I add a bookmark
# Commenting out the following lines: s
# ex_prob = ExerciseBasedProbability.find(user, self.origin)
# if ex_prob:
# ex_prob.update_probability_after_adding_bookmark_with_same_word(self,user)
# zeeguu.db.session.add(ex_prob)
# 2.b) encounter based prob
ranked_word = RankedWord.find(self.origin.word, language)
if ranked_word: #checks if ranked_word exists for that looked up word
if EncounterBasedProbability.exists(
user, ranked_word
): # checks if encounter based probability exists for that looked up word
enc_prob = EncounterBasedProbability.find(user, ranked_word)
enc_prob.word_has_just_beek_bookmarked()
db.session.add(enc_prob)
db.session.commit()
# 2.c) update known word probability if it exists
if KnownWordProbability.exists(user, self.origin, ranked_word):
known_word_prob = KnownWordProbability.find(
user, self.origin, ranked_word)
known_word_prob.word_has_just_beek_bookmarked()
db.session.add(known_word_prob)
db.session.commit()