def test_add_new_word_to_DB(self): word = "baum" rank = UserWord.find_rank(word, self.de) new_word = UserWord(word, self.de, rank) db.session.add(new_word) self.mir.star(new_word) db.session.commit()
def _create_model_object(self, user, force_quality=True, **kwargs): """ Creates a Bookmark object with random data. Behind the random words, a random number is added since the Faker library does not have too many RANDOM words and random words get repeated whenever many random bookmarks are created. To overcome the problem of bookmarks with duplicate words in the database, a random number is added. Also, if force_quality is set (default) and the bookmark is not .quality_bookmark() the process is reiterated till this is true. This simplifies some of the tests :param user: User Object, to which the bookmark is assigned. :param kwargs: Holds any of the 'props' as key if a field should not be random :return: """ bookmark = None while not bookmark: from tests_zeeguu.rules.text_rule import TextRule random_text = TextRule().text random_origin_word = self.faker.word() + str(random.random()) random_origin_language = LanguageRule().random random_translation_word = self.faker.word() + str(random.random()) random_translation_language = LanguageRule().random if UserWord.exists(random_origin_word, random_origin_language) \ or UserWord.exists(random_translation_word, random_translation_language): return self._create_model_object(user) random_origin = UserWordRule(random_origin_word, random_origin_language).user_word random_translation = UserWordRule( random_translation_word, random_translation_language).user_word random_date = self.faker.date_time_this_month() from tests_zeeguu.rules.article_rule import ArticleRule random_article = ArticleRule(real=True).article bookmark = Bookmark(random_origin, random_translation, user, random_text, random_date) if force_quality and not bookmark.quality_bookmark(): # print ("random bookmark was of low quality. retrying...") bookmark = False for k in kwargs: if k in self.props: setattr(bookmark, k, kwargs.get(k)) if self._exists_in_db(bookmark): return self._create_model_object(user) return bookmark
def find_or_create(cls, session, user, _origin: str, _origin_lang: str, _translation: str, _translation_lang: str, _context: str, _url: str, _url_title: str, article_id: int): """ if the bookmark does not exist, it creates it and returns it if it exists, it ** updates the translation** and returns the bookmark object :param _origin: :param _context: :param _url: :return: """ origin_lang = Language.find_or_create(_origin_lang) translation_lang = Language.find_or_create(_translation_lang) origin = UserWord.find_or_create(session, _origin, origin_lang) article = Article.query.filter_by(id=article_id).one() url = Url.find_or_create(session, article.url.as_string(), _url_title) context = Text.find_or_create(session, _context, origin_lang, url, article) translation = UserWord.find_or_create(session, _translation, translation_lang) now = datetime.now() try: # try to find this bookmark bookmark = Bookmark.find_by_user_word_and_text( user, origin, context) # update the translation bookmark.translation = translation except sqlalchemy.orm.exc.NoResultFound as e: bookmark = cls(origin, translation, user, context, now) except Exception as e: raise e session.add(bookmark) session.commit() return bookmark
def __getitem__(self, args): word = self.cache.get(args, None) if word is None: word = UserWord(*args) zeeguu.db.session.add(word) self.cache[args] = word return word
def get_possible_translations(from_lang_code, to_lang_code): """ Returns a list of possible translations for this :param word: word to be translated :param from_lang_code: :param to_lang_code: :return: json array with dictionaries. each of the dictionaries contains at least one 'translation' and one 'translation_id' key. In the future we envision that the dict will contain other types of information, such as relative frequency, """ translations_json = [] context = request.form.get('context', '') url = request.form.get('url', '') word = request.form['word'] main_translation, alternatives = zeeguu.api.translation_service.translate_from_to( word, from_lang_code, to_lang_code) lan = Language.find(from_lang_code) likelihood = 1.0 for translation in alternatives: wor = UserWord.find(translation, lan) zeeguu.db.session.add(wor) zeeguu.db.session.commit() t_dict = dict(translation_id=wor.id, translation=translation, likelihood=likelihood) translations_json.append(t_dict) likelihood -= 0.01 return json_result(dict(translations=translations_json))
def get_possible_translations(from_lang_code, to_lang_code): """ Returns a list of possible translations for this :param word: word to be translated :param from_lang_code: :param to_lang_code: :return: json array with dictionaries. each of the dictionaries contains at least one 'translation' and one 'translation_id' key. In the future we envision that the dict will contain other types of information, such as relative frequency, """ translations_json = [] context = request.form.get('context', '') url = request.form.get('url', '') word = request.form['word'] main_translation, alternatives = zeeguu.api.translation_service.translate_from_to(word, from_lang_code, to_lang_code) lan = Language.find(from_lang_code) likelihood = 1.0 for translation in alternatives: wor = UserWord.find(translation, lan) zeeguu.db.session.add(wor) zeeguu.db.session.commit() t_dict = dict(translation_id= wor.id, translation=translation, likelihood=likelihood) translations_json.append(t_dict) likelihood -= 0.01 return json_result(dict(translations=translations_json))
def add_bookmark(user, original_language, original_word, translation_language, translation_word, date, the_context, the_url, the_url_title): url = Url.find (the_url) text = Text.find_or_create(the_context, translation_language, url) origin = UserWord.find(original_word.lower(), original_language) translation = UserWord.find(translation_word.lower(), translation_language) zeeguu.db.session.add(url) zeeguu.db.session.add(text) zeeguu.db.session.add(origin) zeeguu.db.session.add(translation) t1= Bookmark(origin, translation, user, text, date) zeeguu.db.session.add(t1) zeeguu.db.session.commit() add_probability_to_existing_words_of_user(user,t1,original_language)
def bookmark_with_context(from_lang_code, to_lang_code, word_str, url_str, title_str, context_str, translation_str): """ This function will lookup a given word-text pair, and if found, it will return that bookmark rather than a new one :param from_lang_code: :param to_lang_code: :param word_str: :param url_str: :param title_str: :param context_str: :param translation_str: :return: """ from_lang = Language.find(from_lang_code) to_lang = Language.find(to_lang_code) user_word = UserWord.find(word_str, from_lang) url = Url.find(url_str, title_str) zeeguu.db.session.add(url) zeeguu.db.session.commit() context = Text.find_or_create(context_str, from_lang, url) zeeguu.db.session.add(context) zeeguu.db.session.commit() translation = UserWord.find(translation_str, to_lang) try: bookmark = Bookmark.find_all_by_user_word_and_text( flask.g.user, user_word, context)[0] # TODO: Think about updating the date of this bookmark, or maybe creating a duplicate # otherwise, in the history this translation will not be visible! except Exception: bookmark = Bookmark(user_word, translation, flask.g.user, context, datetime.now()) zeeguu.db.session.add(bookmark) bookmark.calculate_probabilities_after_adding_a_bookmark( flask.g.user, bookmark.origin.language) zeeguu.db.session.commit() return str(bookmark.id)
def add_bookmark(db, user, original_language, original_word, translation_language, translation_word, date, the_context, the_url, the_url_title): session = db.session url = Url.find_or_create(session, the_url, the_url_title) text = Text.find_or_create(session, the_context, translation_language, url) origin = UserWord.find_or_create(session, original_word, original_language) translation = UserWord.find_or_create(session, translation_word, translation_language) b1 = Bookmark(origin, translation, user, text, date) db.session.add(b1) db.session.commit() return b1
def _create_model_object(self): random_word = self.faker.word() random_language = LanguageRule().random user_word = UserWord(random_word, random_language) if self._exists_in_db(user_word): return self._create_model_object() return user_word
def __init__(self, word=None, language=None): super().__init__() if word is None or language is None: self.user_word = self._create_model_object() else: self.user_word = UserWord(word, language) if not self._exists_in_db(self.user_word): self.save(self.user_word)
def bookmark_with_context(from_lang_code, to_lang_code, word_str, url_str, title_str, context_str, translation_str): """ This function will lookup a given word-text pair, and if found, it will return that bookmark rather than a new one :param from_lang_code: :param to_lang_code: :param word_str: :param url_str: :param title_str: :param context_str: :param translation_str: :return: """ from_lang = Language.find(from_lang_code) to_lang = Language.find(to_lang_code) user_word = UserWord.find(word_str, from_lang) url = Url.find(url_str, title_str) zeeguu.db.session.add(url) zeeguu.db.session.commit() context = Text.find_or_create(context_str, from_lang, url) zeeguu.db.session.add(context) zeeguu.db.session.commit() translation = UserWord.find(translation_str, to_lang) try: bookmark = Bookmark.find_all_by_user_word_and_text(flask.g.user, user_word, context)[0] # TODO: Think about updating the date of this bookmark, or maybe creating a duplicate # otherwise, in the history this translation will not be visible! except Exception: bookmark = Bookmark(user_word, translation, flask.g.user, context, datetime.now()) zeeguu.db.session.add(bookmark) bookmark.calculate_probabilities_after_adding_a_bookmark(flask.g.user, bookmark.origin.language) zeeguu.db.session.commit() return str(bookmark.id)
def _create_model_object(self, user, **kwargs): bookmark = None while not bookmark: random_url = UrlRule().url random_text = TextRule().text random_origin_word = self.faker.word() random_origin_language = LanguageRule().random random_translation_word = self.faker.word() random_translation_language = LanguageRule().random if UserWord.exists(random_origin_word, random_origin_language) \ or UserWord.exists(random_translation_word, random_translation_language): return self._create_model_object(user) random_origin = UserWordRule(random_origin_word, random_origin_language).user_word random_translation = UserWordRule( random_translation_word, random_translation_language).user_word random_date = self.faker.date_time_this_month() bookmark = Bookmark(random_origin, random_translation, user, random_text, random_date) if not bookmark.quality_bookmark(): bookmark = False # print ("random bookmark was of low quality. retrying...") for k in kwargs: if k in self.props: setattr(bookmark, k, kwargs.get(k)) if self._exists_in_db(bookmark): return self._create_model_object(user) return bookmark
def add_new_translation_to_bookmark(word_translation, bookmark_id): bookmark = Bookmark.query.filter_by(id=bookmark_id).first() translations_of_bookmark = bookmark.translations_list for transl in translations_of_bookmark: if transl.word == word_translation: return 'FAIL' translation_user_word = UserWord.find(word_translation, translations_of_bookmark[0].language) bookmark.add_new_translation(translation_user_word) zeeguu.db.session.add(translation_user_word) zeeguu.db.session.commit() return "OK"
def set_know_word_prob(): zeeguu.app.test_request_context().push() zeeguu.db.session.commit() enc_probs = EncounterBasedProbability.find_all() ex_probs = ExerciseBasedProbability.find_all() for prob in enc_probs: user = prob.user word = prob.ranked_word.word language = prob.ranked_word.language user_word = None if UserWord.exists(word, language): user_word = UserWord.find(word, language) if ExerciseBasedProbability.exists(user, user_word): ex_prob = ExerciseBasedProbability.find(user, user_word) known_word_prob = KnownWordProbability.calculateKnownWordProb( ex_prob.probability, prob.probability) known_word_probability_obj = KnownWordProbability.find( user, user_word, prob.ranked_word, known_word_prob) else: known_word_probability_obj = KnownWordProbability.find( user, None, prob.ranked_word, prob.probability) zeeguu.db.session.add(known_word_probability_obj) zeeguu.db.session.commit() for prob in ex_probs: user = prob.user language = prob.user_word.language word = prob.user_word.word ranked_word = None if RankedWord.exists(word, language): ranked_word = RankedWord.find(word, language) if not EncounterBasedProbability.exists(user, ranked_word): if UserWord.exists(word, language): user_word = UserWord.find(word, language) known_word_probability_obj = KnownWordProbability( user, user_word, ranked_word, prob.probability) zeeguu.db.session.add(known_word_probability_obj) zeeguu.db.session.commit() print('job3')
def add_new_translation_to_bookmark(word_translation, bookmark_id): bookmark = Bookmark.query.filter_by( id=bookmark_id ).first() translations_of_bookmark = bookmark.translations_list for transl in translations_of_bookmark: if transl.word == word_translation: return 'FAIL' translation_user_word = UserWord.find(word_translation, translations_of_bookmark[0].language) bookmark.add_new_translation(translation_user_word) zeeguu.db.session.add(translation_user_word) zeeguu.db.session.commit() return "OK"
def test_importance_level(self): word = "beschloss" if zeeguu.model.ranked_word.RankedWord.exists(word.lower(), self.de): rank = model.UserWord.find_rank(word.lower(), self.de) new_word = model.UserWord.find(word, self.de) else: new_word = model.UserWord.find(word, self.de) db.session.add(new_word) db.session.commit() word = "unexistingword" beschloss = UserWord.find(word, self.de) assert beschloss assert beschloss.importance_level() == 0
def test_importance_level(self): word = "beschloss" if zeeguu.model.ranked_word.RankedWord.exists(word.lower(), self.de): rank = model.UserWord.find_rank(word.lower(), self.de) new_word = model.UserWord.find(word,self.de) else: new_word = model.UserWord.find(word,self.de) db.session.add(new_word) db.session.commit() word = "unexistingword" beschloss = UserWord.find(word, self.de) assert beschloss assert beschloss.importance_level() == 0
def _create_model_object(self, word=None, language=None): tmp_word = word tmp_language = language if tmp_word is None: tmp_word = self.faker.word() if tmp_language is None: tmp_language = LanguageRule().random user_word = UserWord(tmp_word, tmp_language) if self._exists_in_db(user_word): return self._create_model_object(word, language) return user_word
def set_default_exercise_based_prob(): zeeguu.app.test_request_context().push() zeeguu.db.session.commit() users = User.find_all() languages = Language.all() for user in users: for language in languages: user_words_by_language = UserWord.find_by_language(language) for word in user_words_by_language: if ExerciseBasedProbability.exists(user, word): prob = ExerciseBasedProbability.find(user, word) bookmarks_by_user_and_word = Bookmark.find_all_by_user_and_word( user, word) total_prob = 0 for bookmark in bookmarks_by_user_and_word: prob.calculate_known_bookmark_probability(bookmark) total_prob += float(prob.probability) if bookmarks_by_user_and_word: prob.probability = total_prob / len( bookmarks_by_user_and_word) zeeguu.db.session.commit() print('job1')
def test_find_word(self): assert UserWord.find(self.random_origin_word, self.random_origin_language)
def words(self): for word in re.split(re.compile("[^\\w]+", re.U), self.content): yield UserWord.find(word, self.language)
def update_existing_word_ranks(language): for user_word in UserWord.find_by_language(language): new_rank = RankedWord.find(user_word.word, language) user_word.set_rank(new_rank) zeeguu.db.session.add(user_word) zeeguu.db.session.commit()
def test_search_1(self): word = UserWord.find("hauen345", self.de) s = model.Search(self.mir, word, self.de) db.session.add(s) db.session.commit()
def test_find_word(self): word = "baum" assert UserWord.find(word, self.de)
def _exists_in_db(obj): return UserWord.exists(obj.word, obj.language)
def calculate_probabilities_after_adding_a_bookmark(self, user,language): """ ML: This has to be refactored. It's a mess. The idea is: you've just added a bookmark. There are two things to do: 1. update the probabilities of the context words (they have been encountered, and not translated) 2. update the probabilities of the word itself - :param user: :param language: :return: """ # 1. computations for adding encounter based probability for the context words for word in self.context_words_with_rank(): enc_prob = EncounterBasedProbability.find_or_create(word, user, language) zeeguu.db.session.add(enc_prob) zeeguu.db.session.commit() user_word = None ranked_word = enc_prob.ranked_word if UserWord.exists(word,language): user_word = UserWord.find(word,language) if ExerciseBasedProbability.exists(user,user_word): #checks if exercise based probability exists for words in context ex_prob = ExerciseBasedProbability.find(user,user_word) known_word_prob = KnownWordProbability.find(user,user_word,ranked_word) known_word_prob.probability = known_word_prob.calculateKnownWordProb(ex_prob.probability, enc_prob.probability) #updates known word probability as exercise based probability already existed. else: if KnownWordProbability.exists(user, user_word,ranked_word): known_word_prob = KnownWordProbability.find(user,user_word,ranked_word) known_word_prob.probability = enc_prob.probability # updates known word probability as encounter based probability already existed else: known_word_prob = KnownWordProbability.find(user,user_word,ranked_word, enc_prob.probability) # new known word probability created as it did not exist zeeguu.db.session.add(known_word_prob) # 2. Update the probabilities of the word itself # 2.a) exercise based prob # ML: Should this thing change? # The ex based probability should probably not change after I add a bookmark # Commenting out the following lines: s # ex_prob = ExerciseBasedProbability.find(user, self.origin) # if ex_prob: # ex_prob.update_probability_after_adding_bookmark_with_same_word(self,user) # zeeguu.db.session.add(ex_prob) # 2.b) encounter based prob ranked_word = RankedWord.find(self.origin.word, language) if ranked_word: #checks if ranked_word exists for that looked up word if EncounterBasedProbability.exists(user, ranked_word): # checks if encounter based probability exists for that looked up word enc_prob = EncounterBasedProbability.find(user, ranked_word) enc_prob.word_has_just_beek_bookmarked() db.session.add(enc_prob) db.session.commit() # 2.c) update known word probability if it exists if KnownWordProbability.exists(user, self.origin,ranked_word): known_word_prob = KnownWordProbability.find(user,self.origin,ranked_word) known_word_prob.word_has_just_beek_bookmarked() db.session.add(known_word_prob) db.session.commit()
def calculate_probabilities_after_adding_a_bookmark(self, user, language): """ ML: This has to be refactored. It's a mess. The idea is: you've just added a bookmark. There are two things to do: 1. update the probabilities of the context words (they have been encountered, and not translated) 2. update the probabilities of the word itself - :param user: :param language: :return: """ # 1. computations for adding encounter based probability for the context words for word in self.context_words_with_rank(): enc_prob = EncounterBasedProbability.find_or_create( word, user, language) zeeguu.db.session.add(enc_prob) zeeguu.db.session.commit() user_word = None ranked_word = enc_prob.ranked_word if UserWord.exists(word, language): user_word = UserWord.find(word, language) if ExerciseBasedProbability.exists( user, user_word ): #checks if exercise based probability exists for words in context ex_prob = ExerciseBasedProbability.find(user, user_word) known_word_prob = KnownWordProbability.find( user, user_word, ranked_word) known_word_prob.probability = known_word_prob.calculateKnownWordProb( ex_prob.probability, enc_prob.probability ) #updates known word probability as exercise based probability already existed. else: if KnownWordProbability.exists(user, user_word, ranked_word): known_word_prob = KnownWordProbability.find( user, user_word, ranked_word) known_word_prob.probability = enc_prob.probability # updates known word probability as encounter based probability already existed else: known_word_prob = KnownWordProbability.find( user, user_word, ranked_word, enc_prob.probability ) # new known word probability created as it did not exist zeeguu.db.session.add(known_word_prob) # 2. Update the probabilities of the word itself # 2.a) exercise based prob # ML: Should this thing change? # The ex based probability should probably not change after I add a bookmark # Commenting out the following lines: s # ex_prob = ExerciseBasedProbability.find(user, self.origin) # if ex_prob: # ex_prob.update_probability_after_adding_bookmark_with_same_word(self,user) # zeeguu.db.session.add(ex_prob) # 2.b) encounter based prob ranked_word = RankedWord.find(self.origin.word, language) if ranked_word: #checks if ranked_word exists for that looked up word if EncounterBasedProbability.exists( user, ranked_word ): # checks if encounter based probability exists for that looked up word enc_prob = EncounterBasedProbability.find(user, ranked_word) enc_prob.word_has_just_beek_bookmarked() db.session.add(enc_prob) db.session.commit() # 2.c) update known word probability if it exists if KnownWordProbability.exists(user, self.origin, ranked_word): known_word_prob = KnownWordProbability.find( user, self.origin, ranked_word) known_word_prob.word_has_just_beek_bookmarked() db.session.add(known_word_prob) db.session.commit()