def _create_model_object(self, user, force_quality=True, **kwargs):
"""
Creates a Bookmark object with random data.
Behind the random words, a random number is added since the Faker library does not have too many RANDOM words
and random words get repeated whenever many random bookmarks are created. To overcome the problem of bookmarks
with duplicate words in the database, a random number is added.
Also, if force_quality is set (default) and the bookmark is not .quality_bookmark() the process is
reiterated till this is true. This simplifies some of the tests
:param user: User Object, to which the bookmark is assigned.
:param kwargs: Holds any of the 'props' as key if a field should not be random
:return:
"""
bookmark = None
while not bookmark:
from tests_zeeguu.rules.text_rule import TextRule
random_text = TextRule().text
random_origin_word = self.faker.word() + str(random.random())
random_origin_language = LanguageRule().random
random_translation_word = self.faker.word() + str(random.random())
random_translation_language = LanguageRule().random
if UserWord.exists(random_origin_word, random_origin_language) \
or UserWord.exists(random_translation_word, random_translation_language):
return self._create_model_object(user)
random_origin = UserWordRule(random_origin_word,
random_origin_language).user_word
random_translation = UserWordRule(
random_translation_word, random_translation_language).user_word
random_date = self.faker.date_time_this_month()
from tests_zeeguu.rules.article_rule import ArticleRule
random_article = ArticleRule(real=True).article
bookmark = Bookmark(random_origin, random_translation, user,
random_text, random_date)
if force_quality and not bookmark.quality_bookmark():
# print ("random bookmark was of low quality. retrying...")
bookmark = False
for k in kwargs:
if k in self.props:
setattr(bookmark, k, kwargs.get(k))
if self._exists_in_db(bookmark):
return self._create_model_object(user)
return bookmark
def create_new_exercise(exercise_outcome, exercise_source,
exercise_solving_speed, bookmark_id):
"""
OBSOLETE!
Use the /report_exercise_outcome/... API endpoint
In the model parlance, an exercise is an entry in a table that
logs the performance of an exercise. Every such performance, has a source, and an outcome.
:param exercise_outcome:
:param exercise_source:
:param exercise_solving_speed:
:param bookmark_id:
:return:
"""
try:
bookmark = Bookmark.find(bookmark_id)
new_source = ExerciseSource.find_by_source(exercise_source)
new_outcome = ExerciseOutcome.find(exercise_outcome)
if not new_source or not new_outcome:
return "FAIL"
exercise = Exercise(new_outcome, new_source, exercise_solving_speed,
datetime.datetime.now())
bookmark.add_new_exercise(exercise)
zeeguu.db.session.add(exercise)
zeeguu.db.session.commit()
update_probabilities_for_word(bookmark.origin)
return "OK"
except:
return "FAIL"
def set_default_encounter_based_prob():
zeeguu.app.test_request_context().push()
zeeguu.db.session.commit()
default_probability = 0.5
languages = Language.all()
users = User.find_all()
for user in users:
for lang in languages:
marked_words_of_user_in_text = []
words_of_all_bookmarks_content = []
for bookmark in Bookmark.find_by_specific_user(user):
if bookmark.origin.language == lang:
# bookmark_content_words = re.sub("[^\w]", " ", bookmark.text.content).split()
bookmark_content_words = re.findall(r'(?u)\w+', bookmark.text.content)
words_of_all_bookmarks_content.extend(bookmark_content_words)
marked_words_of_user_in_text.append(bookmark.origin.word)
words_known_from_user= [word for word in words_of_all_bookmarks_content if word not in marked_words_of_user_in_text]
for word_known in words_known_from_user:
if RankedWord.exists(word_known, lang):
rank = RankedWord.find(word_known, lang)
if EncounterBasedProbability.exists(user, rank):
prob = EncounterBasedProbability.find(user,rank, default_probability)
prob.not_looked_up_counter +=1
else:
prob = EncounterBasedProbability.find(user,rank,default_probability)
zeeguu.db.session.add(prob)
zeeguu.db.session.commit()
print 'job2'
def report_exercise_outcome(exercise_outcome,exercise_source,exercise_solving_speed,bookmark_id):
"""
In the model parlance, an exercise is an entry in a table that
logs the performance of an exercise. Every such performance, has a source, and an outcome.
:param exercise_outcome: One of: Correct, Retry, Wrong, Typo, Too easy
:param exercise_source: has been assigned to your app by zeeguu
:param exercise_solving_speed: in milliseconds
:param bookmark_id: the bookmark for which the data is reported
:return:
"""
try:
bookmark = Bookmark.find(bookmark_id)
new_source = ExerciseSource.find_by_source(exercise_source)
new_outcome = ExerciseOutcome.find(exercise_outcome)
if not new_source:
return "could not find source"
if not new_outcome:
return "could not find outcome"
exercise = Exercise(new_outcome,new_source,exercise_solving_speed, datetime.now())
bookmark.add_new_exercise(exercise)
zeeguu.db.session.add(exercise)
zeeguu.db.session.commit()
from zeeguu.language.knowledge_estimator import update_probabilities_for_word
update_probabilities_for_word(bookmark.origin)
return "OK"
except :
traceback.print_exc()
return "FAIL"
def set_default_encounter_based_prob():
zeeguu.app.test_request_context().push()
zeeguu.db.session.commit()
default_probability = 0.5
languages = Language.all()
users = User.find_all()
for user in users:
for lang in languages:
marked_words_of_user_in_text = []
words_of_all_bookmarks_content = []
for bookmark in Bookmark.find_by_specific_user(user):
if bookmark.origin.language == lang:
# bookmark_content_words = re.sub("[^\w]", " ", bookmark.text.content).split()
bookmark_content_words = re.findall(
r'(?u)\w+', bookmark.text.content)
words_of_all_bookmarks_content.extend(
bookmark_content_words)
marked_words_of_user_in_text.append(bookmark.origin.word)
words_known_from_user = [
word for word in words_of_all_bookmarks_content
if word not in marked_words_of_user_in_text
]
for word_known in words_known_from_user:
if RankedWord.exists(word_known, lang):
rank = RankedWord.find(word_known, lang)
if EncounterBasedProbability.exists(user, rank):
prob = EncounterBasedProbability.find(
user, rank, default_probability)
prob.not_looked_up_counter += 1
else:
prob = EncounterBasedProbability.find(
user, rank, default_probability)
zeeguu.db.session.add(prob)
zeeguu.db.session.commit()
print 'job2'
def bookmark_with_context(from_lang_code, to_lang_code, word_str, url_str,
title_str, context_str, translation_str):
"""
This function will lookup a given word-text pair, and if found, it will return
that bookmark rather than a new one
:param from_lang_code:
:param to_lang_code:
:param word_str:
:param url_str:
:param title_str:
:param context_str:
:param translation_str:
:return:
"""
from_lang = Language.find(from_lang_code)
to_lang = Language.find(to_lang_code)
user_word = UserWord.find(word_str, from_lang)
url = Url.find(url_str, title_str)
zeeguu.db.session.add(url)
zeeguu.db.session.commit()
context = Text.find_or_create(context_str, from_lang, url)
zeeguu.db.session.add(context)
zeeguu.db.session.commit()
translation = UserWord.find(translation_str, to_lang)
try:
bookmark = Bookmark.find_all_by_user_word_and_text(
flask.g.user, user_word, context)[0]
# TODO: Think about updating the date of this bookmark, or maybe creating a duplicate
# otherwise, in the history this translation will not be visible!
except Exception:
bookmark = Bookmark(user_word, translation, flask.g.user, context,
datetime.now())
zeeguu.db.session.add(bookmark)
bookmark.calculate_probabilities_after_adding_a_bookmark(
flask.g.user, bookmark.origin.language)
zeeguu.db.session.commit()
return str(bookmark.id)
def test_watch_event(self):
glance = self.test_watch_event_type()
a_bookmark = Bookmark.find(1)
new_glance = WatchInteractionEvent(glance, 1, datetime.now())
db.session.add(new_glance)
db.session.commit()
assert len(WatchInteractionEvent.events_for_bookmark(a_bookmark)) == 1
def set_default_exercise_based_prob():
zeeguu.app.test_request_context().push()
zeeguu.db.session.commit()
bookmarks = Bookmark.find_all()
for bookmark in bookmarks:
prob = ExerciseBasedProbability.find(bookmark.user, bookmark.origin)
zeeguu.db.session.add(prob)
zeeguu.db.session.commit()
print 'job1'
def test_watch_event(self):
glance = self.test_watch_event_type()
a_bookmark = Bookmark.find(1)
new_glance = WatchInteractionEvent(glance, 1, datetime.now())
db.session.add(new_glance)
db.session.commit()
assert len(WatchInteractionEvent.events_for_bookmark(a_bookmark)) == 1
def set_default_exercise_based_prob():
zeeguu.app.test_request_context().push()
zeeguu.db.session.commit()
bookmarks = Bookmark.find_all()
for bookmark in bookmarks:
prob = ExerciseBasedProbability.find(bookmark.user, bookmark.origin)
zeeguu.db.session.add(prob)
zeeguu.db.session.commit()
print 'job1'
def update_probability_after_adding_bookmark_with_same_word(
self, bookmark, user):
from zeeguu.model.bookmark import Bookmark
count_bookmarks_with_same_word = len(
Bookmark.find_all_by_user_and_word(user, bookmark.origin))
self.probability = (
float(self.probability * count_bookmarks_with_same_word) + 0.1) / (
count_bookmarks_with_same_word + 1
) # compute avg probability of all bookmarks with same word
def words_being_learned(self, language):
# Get the words the user is currently learning
words_learning = {}
bookmarks = Bookmark.find_by_specific_user(self.user)
for bookmark in bookmarks:
learning = not bookmark.check_is_latest_outcome_too_easy()
user_word = bookmark.origin
if learning and user_word.language == language:
words_learning[user_word.word] = user_word.word
return words_learning
def test_delete_bookmark3(self):
with zeeguu.app.app_context():
form_data = dict(url='http://mir.lu',
context=u'Die kleine Jägermeister',
word="Die")
bookmark1 = self.json_from_api_post(
'/translate_and_bookmark/de/en', form_data)
Bookmark.find(bookmark1["bookmark_id"])
form_data["word"] = "kleine"
bookmark2 = self.json_from_api_post(
'/translate_and_bookmark/de/en', form_data)
b2 = Bookmark.find(bookmark2["bookmark_id"])
assert len(b2.text.all_bookmarks()) == 2
self.api_post("delete_bookmark/" + str(b2.id))
assert len(b2.text.all_bookmarks()) == 1
def bookmark_with_context(from_lang_code, to_lang_code, word_str, url_str, title_str, context_str, translation_str):
"""
This function will lookup a given word-text pair, and if found, it will return
that bookmark rather than a new one
:param from_lang_code:
:param to_lang_code:
:param word_str:
:param url_str:
:param title_str:
:param context_str:
:param translation_str:
:return:
"""
from_lang = Language.find(from_lang_code)
to_lang = Language.find(to_lang_code)
user_word = UserWord.find(word_str, from_lang)
url = Url.find(url_str, title_str)
zeeguu.db.session.add(url)
zeeguu.db.session.commit()
context = Text.find_or_create(context_str, from_lang, url)
zeeguu.db.session.add(context)
zeeguu.db.session.commit()
translation = UserWord.find(translation_str, to_lang)
try:
bookmark = Bookmark.find_all_by_user_word_and_text(flask.g.user, user_word, context)[0]
# TODO: Think about updating the date of this bookmark, or maybe creating a duplicate
# otherwise, in the history this translation will not be visible!
except Exception:
bookmark = Bookmark(user_word, translation, flask.g.user, context, datetime.now())
zeeguu.db.session.add(bookmark)
bookmark.calculate_probabilities_after_adding_a_bookmark(flask.g.user, bookmark.origin.language)
zeeguu.db.session.commit()
return str(bookmark.id)
def _create_model_object(self, user, **kwargs):
bookmark = None
while not bookmark:
random_url = UrlRule().url
random_text = TextRule().text
random_origin_word = self.faker.word()
random_origin_language = LanguageRule().random
random_translation_word = self.faker.word()
random_translation_language = LanguageRule().random
if UserWord.exists(random_origin_word, random_origin_language) \
or UserWord.exists(random_translation_word, random_translation_language):
return self._create_model_object(user)
random_origin = UserWordRule(random_origin_word,
random_origin_language).user_word
random_translation = UserWordRule(
random_translation_word, random_translation_language).user_word
random_date = self.faker.date_time_this_month()
bookmark = Bookmark(random_origin, random_translation, user,
random_text, random_date)
if not bookmark.quality_bookmark():
bookmark = False
# print ("random bookmark was of low quality. retrying...")
for k in kwargs:
if k in self.props:
setattr(bookmark, k, kwargs.get(k))
if self._exists_in_db(bookmark):
return self._create_model_object(user)
return bookmark
def words_being_learned(self):
"""
The words the user is currently learning
:return:
"""
words_learning = []
bookmarks = Bookmark.find_by_specific_user(self.user)
for bookmark in bookmarks:
learning = False
if bookmark.latest_exercise_outcome():
learning = not bookmark.latest_exercise_outcome().too_easy()
user_word = bookmark.origin
if learning and user_word.language == self.language:
words_learning.append(user_word.word)
return words_learning
def add_bookmark(user, original_language, original_word, translation_language, translation_word, date, the_context, the_url, the_url_title):
url = Url.find (the_url)
text = Text.find_or_create(the_context, translation_language, url)
origin = UserWord.find(original_word.lower(), original_language)
translation = UserWord.find(translation_word.lower(), translation_language)
zeeguu.db.session.add(url)
zeeguu.db.session.add(text)
zeeguu.db.session.add(origin)
zeeguu.db.session.add(translation)
t1= Bookmark(origin, translation, user, text, date)
zeeguu.db.session.add(t1)
zeeguu.db.session.commit()
add_probability_to_existing_words_of_user(user,t1,original_language)
def add_bookmark(db, user, original_language, original_word,
translation_language, translation_word, date, the_context,
the_url, the_url_title):
session = db.session
url = Url.find_or_create(session, the_url, the_url_title)
text = Text.find_or_create(session, the_context, translation_language, url)
origin = UserWord.find_or_create(session, original_word, original_language)
translation = UserWord.find_or_create(session, translation_word,
translation_language)
b1 = Bookmark(origin, translation, user, text, date)
db.session.add(b1)
db.session.commit()
return b1
def test_bookmark_has_been_learned(self):
to_study = self.json_from_api_get("bookmarks_to_study/50")
to_study_count_before = len(to_study)
# Create an learnedIt event
learned_bookmark_id = to_study[0]["id"]
events = [
dict(bookmark_id=to_study[0]["id"],
time="2016-05-05T10:10:10",
event="learnedIt"),
dict(bookmark_id=to_study[1]["id"],
time="2016-05-05T10:11:10",
event="wrongTranslation")
]
result = self.api_post('/upload_smartwatch_events',
dict(events=json.dumps(events)))
assert (result.data == "OK")
with zeeguu.app.app_context():
b = Bookmark.find(learned_bookmark_id)
print b.has_been_learned()
def update_probabilities_for_word(word):
try:
bookmarks_for_this_word = Bookmark.find_all_by_user_and_word(
flask.g.user, word)
ex_prob = ExerciseBasedProbability.find(flask.g.user, word)
total_prob = 0
for b in bookmarks_for_this_word:
ex_prob.calculate_known_bookmark_probability(b)
total_prob += float(ex_prob.probability)
ex_prob.probability = total_prob / len(bookmarks_for_this_word)
if RankedWord.exists(word.word, word.language):
ranked_word = RankedWord.find(word.word, word.language)
if EncounterBasedProbability.exists(flask.g.user, ranked_word):
enc_prob = EncounterBasedProbability.find(
flask.g.user, ranked_word)
known_word_prob = KnownWordProbability.find(
flask.g.user, word, ranked_word)
print "!known word prob before: " + str(
known_word_prob.probability)
print "!ex_prob: " + str(ex_prob.probability)
print "!enc_prob: " + str(enc_prob.probability)
known_word_prob.probability = KnownWordProbability.calculateKnownWordProb(
ex_prob.probability, enc_prob.probability)
print "!known word prob after: " + str(
known_word_prob.probability)
else:
known_word_prob = KnownWordProbability.find(
flask.g.user, word, ranked_word)
known_word_prob.probability = ex_prob.probability
db.session.commit()
except:
print "failed to update probabilities for word with id: " + str(
word.id)
print "!successfully updated probabilities for word with id: " + str(
word.id)
def create_default_bookmarks(session, user, language_code):
bookmarks = []
try:
print("trying to load default bookmarks for " + str(user.name))
for data_point in bookmark_data[language_code]:
bookmark = Bookmark.find_or_create(session, user, data_point[0],
language_code, data_point[1],
"en", data_point[2],
data_point[3],
"Zeeguu Exercises")
bookmarks.append(bookmark)
except Exception as e:
zeeguu.log(
"could not load default bookmarks for {0}".format(language_code))
#raise e
return []
return bookmarks
def set_default_exercise_based_prob():
zeeguu.app.test_request_context().push()
zeeguu.db.session.commit()
users = User.find_all()
languages = Language.all()
for user in users:
for language in languages:
user_words_by_language = UserWord.find_by_language(language)
for word in user_words_by_language:
if ExerciseBasedProbability.exists(user, word):
prob = ExerciseBasedProbability.find(user, word)
bookmarks_by_user_and_word = Bookmark.find_all_by_user_and_word(
user, word)
total_prob = 0
for bookmark in bookmarks_by_user_and_word:
prob.calculate_known_bookmark_probability(bookmark)
total_prob += float(prob.probability)
if bookmarks_by_user_and_word:
prob.probability = total_prob / len(
bookmarks_by_user_and_word)
zeeguu.db.session.commit()
print('job1')
def _update_bookmark_probability(cls, db, user, word):
try:
bookmarks_for_this_word = Bookmark.find_all_by_user_and_word(
user, word)
ex_prob = ExerciseBasedProbability.find_or_create(user, word)
total_prob = 0
for each_bookmark in bookmarks_for_this_word:
ex_prob.calculate_known_bookmark_probability(each_bookmark)
total_prob += float(ex_prob.probability)
ex_prob.probability = total_prob / len(bookmarks_for_this_word)
# TODO: experiment also with max instead of the average here: that would be
# TODO: consider the best known bookmark to represent the knowledge of the user
db.session.add(ex_prob)
db.session.commit()
return ex_prob
print(
"!exercise based probability for word with id {1}: {0}".format(
ex_prob.probability, word.id))
except Exception as e:
print("failed to update probabilities for word with id: " +
str(word.id))
print((e.message))
def report_exercise_outcome(exercise_outcome, exercise_source,
exercise_solving_speed, bookmark_id):
"""
In the model parlance, an exercise is an entry in a table that
logs the performance of an exercise. Every such performance, has a source, and an outcome.
:param exercise_outcome: One of: Correct, Retry, Wrong, Typo, Too easy
:param exercise_source: has been assigned to your app by zeeguu
:param exercise_solving_speed: in milliseconds
:param bookmark_id: the bookmark for which the data is reported
:return:
"""
try:
bookmark = Bookmark.find(bookmark_id)
new_source = ExerciseSource.find_by_source(exercise_source)
new_outcome = ExerciseOutcome.find(exercise_outcome)
if not new_source:
return "could not find source"
if not new_outcome:
return "could not find outcome"
exercise = Exercise(new_outcome, new_source, exercise_solving_speed,
datetime.now())
bookmark.add_new_exercise(exercise)
zeeguu.db.session.add(exercise)
zeeguu.db.session.commit()
from zeeguu.language.knowledge_estimator import update_probabilities_for_word
update_probabilities_for_word(bookmark.origin)
return "OK"
except:
traceback.print_exc()
return "FAIL"
def all_bookmarks(self):
return Bookmark.find_all_for_text(self)
def _exists_in_db(obj):
return Bookmark.exists(obj)
def update_probability_after_adding_bookmark_with_same_word(self, bookmark, user):
from zeeguu.model.bookmark import Bookmark
count_bookmarks_with_same_word = len(Bookmark.find_all_by_user_and_word(user, bookmark.origin))
self.probability = (float(self.probability * count_bookmarks_with_same_word) + 0.1)/(count_bookmarks_with_same_word+1)# compute avg probability of all bookmarks with same word