コード例 #1
0
ファイル: populate.py プロジェクト: LarsHoldijk/Zeeguu-Core
def create_minimal_test_db(db):
    drop_current_tables(db)

    # Some common test fixtures
    de = Language("de", "German")
    en = Language("en", "English")
    nl = Language("nl", "Dutch")
    es = Language("es", "Spanish")
    fr = Language("fr", "French")

    db.session.add_all([en, de, nl, es, fr])

    mir = User(TEST_EMAIL, "Mircea", TEST_PASS, de, en)

    db.session.add(mir)

    show_solution = ExerciseOutcome("Show solution")
    retry = ExerciseOutcome("Retry")
    correct = ExerciseOutcome("Correct")
    wrong = ExerciseOutcome("Wrong")
    typo = ExerciseOutcome("Typo")
    too_easy = ExerciseOutcome("Too easy")

    outcomes = [show_solution, retry, correct, wrong, typo, too_easy]

    db.session.add_all(outcomes)

    recognize = ExerciseSource("Recognize")
    translate = ExerciseSource("Translate")

    sources = [recognize, translate]

    db.session.add_all(sources)

    b1 = add_bookmark(db, mir, de, "Schaf", en, "sheep",
                      datetime.datetime(2011, 1, 1, 1, 1,
                                        1), "Bitte... zeichne mir ein Schaf!",
                      "http://www.derkleineprinz-online.de/text/2-kapitel/",
                      "Der Kleine Prinz - Kapitel 2")

    b2 = add_bookmark(db, mir, de, "sprang", en, "jumped",
                      datetime.datetime(2011, 1, 1, 1, 1,
                                        1), "Ich sprang auf die Fusse.",
                      "http://www.derkleineprinz-online.de/text/2-kapitel/",
                      "Der Kleine Prinz - Kapitel 2")

    bookmarks = [b1, b2]

    for i in range(0, 5):
        random_source = sources[random.randint(0, len(sources) - 1)]
        random_outcome = outcomes[random.randint(0, len(outcomes) - 1)]
        random_solving_speed = random.randint(500, 5000)
        exercise = Exercise(random_outcome, random_source,
                            random_solving_speed, datetime.datetime.now())
        random_bookmark = bookmarks[random.randint(0, len(bookmarks) - 1)]
        random_bookmark.add_new_exercise(exercise)

    global TEST_BOOKMARKS_COUNT
    TEST_BOOKMARKS_COUNT = 2
    db.session.commit()
コード例 #2
0
 def __init__(self,
              email,
              name,
              password,
              learned_language=None,
              native_language=None,
              invitation_code=None):
     self.email = email
     self.name = name
     self.update_password(password)
     self.learned_language = learned_language or Language.default_learned()
     self.native_language = native_language or Language.default_native_language(
     )
     self.invitation_code = invitation_code
コード例 #3
0
ファイル: bookmark.py プロジェクト: alinbalutoiu/Zeeguu-Core
    def find_or_create(cls, session, user, _origin: str, _origin_lang: str,
                       _translation: str, _translation_lang: str,
                       _context: str, _url: str, _url_title: str,
                       article_id: int):
        """
            if the bookmark does not exist, it creates it and returns it
            if it exists, it ** updates the translation** and returns the bookmark object

        :param _origin:
        :param _context:
        :param _url:
        :return:
        """

        origin_lang = Language.find_or_create(_origin_lang)
        translation_lang = Language.find_or_create(_translation_lang)

        origin = UserWord.find_or_create(session, _origin, origin_lang)

        article = Article.query.filter_by(id=article_id).one()

        url = Url.find_or_create(session, article.url.as_string(), _url_title)

        context = Text.find_or_create(session, _context, origin_lang, url,
                                      article)

        translation = UserWord.find_or_create(session, _translation,
                                              translation_lang)

        now = datetime.now()

        try:
            # try to find this bookmark
            bookmark = Bookmark.find_by_user_word_and_text(
                user, origin, context)

            # update the translation
            bookmark.translation = translation

        except sqlalchemy.orm.exc.NoResultFound as e:
            bookmark = cls(origin, translation, user, context, now)
        except Exception as e:
            raise e

        session.add(bookmark)
        session.commit()

        return bookmark
コード例 #4
0
def get_possible_translations(from_lang_code, to_lang_code):
    """
    Returns a list of possible translations for this
    :param word: word to be translated
    :param from_lang_code:
    :param to_lang_code:
    :return: json array with dictionaries. each of the dictionaries contains at least
        one 'translation' and one 'translation_id' key.

        In the future we envision that the dict will contain
        other types of information, such as relative frequency,
    """

    translations_json = []
    context = request.form.get('context', '')
    url = request.form.get('url', '')
    word = request.form['word']

    main_translation, alternatives = zeeguu.api.translation_service.translate_from_to(word, from_lang_code, to_lang_code)

    lan = Language.find(from_lang_code)
    likelihood = 1.0
    for translation in alternatives:
        wor = UserWord.find(translation, lan)
        zeeguu.db.session.add(wor)
        zeeguu.db.session.commit()
        t_dict = dict(translation_id= wor.id,
                 translation=translation,
                 likelihood=likelihood)
        translations_json.append(t_dict)
        likelihood -= 0.01

    return json_result(dict(translations=translations_json))
コード例 #5
0
 def __init__(self, user, lang_code=None):
     self.user = user
     if lang_code:
         self.lang_code = lang_code
     else:
         self.lang_code = self.user.learned_language_id
     self.language = Language.find(self.lang_code)
コード例 #6
0
def start_following_feed():
    """
    Start following a feed for which the client provides all the
    metadata. This is useful for the cases where badly formed
    feeds can't be parsed by feedparser.

    :return:
    """

    feed_info = json.loads(request.form.get('feed_info', ''), "utf-8")

    image_url = feed_info["image"]
    language = Language.find(feed_info["language"])
    url_string = feed_info["url"]
    title = feed_info["title"]
    description = feed_info["description"]

    url = Url.find(url_string)
    zeeguu.db.session.add(url)
    # Important to commit this url first; otherwise we end up creating
    # two domains with the same name for both the urls...
    zeeguu.db.session.commit()

    feed_image_url = Url.find(image_url)

    feed_object = RSSFeed.find_or_create(url, title, description,
                                         feed_image_url, language)
    feed_registration = RSSFeedRegistration.find_or_create(
        flask.g.user, feed_object)

    zeeguu.db.session.add_all([feed_image_url, feed_object, feed_registration])
    zeeguu.db.session.commit()

    return "OK"
コード例 #7
0
def get_possible_translations(from_lang_code, to_lang_code):
    """
    Returns a list of possible translations for this
    :param word: word to be translated
    :param from_lang_code:
    :param to_lang_code:
    :return: json array with dictionaries. each of the dictionaries contains at least
        one 'translation' and one 'translation_id' key.

        In the future we envision that the dict will contain
        other types of information, such as relative frequency,
    """

    translations_json = []
    context = request.form.get('context', '')
    url = request.form.get('url', '')
    word = request.form['word']

    main_translation, alternatives = zeeguu.api.translation_service.translate_from_to(
        word, from_lang_code, to_lang_code)

    lan = Language.find(from_lang_code)
    likelihood = 1.0
    for translation in alternatives:
        wor = UserWord.find(translation, lan)
        zeeguu.db.session.add(wor)
        zeeguu.db.session.commit()
        t_dict = dict(translation_id=wor.id,
                      translation=translation,
                      likelihood=likelihood)
        translations_json.append(t_dict)
        likelihood -= 0.01

    return json_result(dict(translations=translations_json))
コード例 #8
0
    def find_or_create(cls, session, user: User, _url, _title: str, _language):
        """

            create a new object and add it to the db if it's not already there
            otherwise retrieve the existing object and update

            in case of creation, the created object is incomplete

\        """

        language = Language.find(_language)
        url = Url.find_or_create(session, _url, _title)

        try:
            return cls.query.filter_by(user=user, url=url).one()
        except NoResultFound:
            try:
                new = cls(user, url, _title, language)
                session.add(new)
                session.commit()
                return new
            except Exception as e:
                print("seems we avoided a race condition")
                session.rollback()
                return cls.query.filter_by(user=user, url=url).one()
コード例 #9
0
def set_default_encounter_based_prob():
    zeeguu.app.test_request_context().push()
    zeeguu.db.session.commit()
    default_probability = 0.5
    languages = Language.all()
    users = User.find_all()
    for user in users:
        for lang in languages:
            marked_words_of_user_in_text = []
            words_of_all_bookmarks_content = []
            for bookmark in Bookmark.find_by_specific_user(user):
                if bookmark.origin.language == lang:
                    # bookmark_content_words = re.sub("[^\w]", " ",  bookmark.text.content).split()
                    bookmark_content_words = re.findall(
                        r'(?u)\w+', bookmark.text.content)
                    words_of_all_bookmarks_content.extend(
                        bookmark_content_words)
                    marked_words_of_user_in_text.append(bookmark.origin.word)
            words_known_from_user = [
                word for word in words_of_all_bookmarks_content
                if word not in marked_words_of_user_in_text
            ]
            for word_known in words_known_from_user:
                if RankedWord.exists(word_known, lang):
                    rank = RankedWord.find(word_known, lang)
                    if EncounterBasedProbability.exists(user, rank):
                        prob = EncounterBasedProbability.find(
                            user, rank, default_probability)
                        prob.not_looked_up_counter += 1
                    else:
                        prob = EncounterBasedProbability.find(
                            user, rank, default_probability)
                        zeeguu.db.session.add(prob)
                        zeeguu.db.session.commit()
    print 'job2'
コード例 #10
0
 def cache_ranked_words(cls):
     cls.ranked_words_cache = {}
     for language in Language.all():
         ranked_words = cls.find_all(language)
         for ranked_word in ranked_words:
             ranked_word_key = language.id + '_' + ranked_word.word
             cls.ranked_words_cache[ranked_word_key] = ranked_word
コード例 #11
0
def set_default_encounter_based_prob():
    zeeguu.app.test_request_context().push()
    zeeguu.db.session.commit()
    default_probability = 0.5
    languages = Language.all()
    users = User.find_all()
    for user in users:
        for lang in languages:
            marked_words_of_user_in_text = []
            words_of_all_bookmarks_content = []
            for bookmark in Bookmark.find_by_specific_user(user):
                if bookmark.origin.language == lang:
                    # bookmark_content_words = re.sub("[^\w]", " ",  bookmark.text.content).split()
                    bookmark_content_words = re.findall(r'(?u)\w+', bookmark.text.content)
                    words_of_all_bookmarks_content.extend(bookmark_content_words)
                    marked_words_of_user_in_text.append(bookmark.origin.word)
            words_known_from_user= [word for word in words_of_all_bookmarks_content if word not in marked_words_of_user_in_text]
            for word_known in words_known_from_user:
                if RankedWord.exists(word_known, lang):
                   rank = RankedWord.find(word_known, lang)
                   if EncounterBasedProbability.exists(user, rank):
                       prob = EncounterBasedProbability.find(user,rank, default_probability)
                       prob.not_looked_up_counter +=1
                   else:
                       prob = EncounterBasedProbability.find(user,rank,default_probability)
                       zeeguu.db.session.add(prob)
     		       zeeguu.db.session.commit()
    print 'job2'
コード例 #12
0
def create_account():

    # A cool way of passing the arguments to the flask template
    template_arguments = dict (
         languages= Language.all(),
         native_languages = Language.native_languages(),
         default_learned= Language.default_learned()
    )

    # GET
    if flask.request.method == "GET":
        return flask.render_template("create_account.html", **template_arguments)

    # POST
    form = flask.request.form
    password = form.get("password", None)
    email = form.get("email", None)
    name = form.get("name", None)
    code = form.get("code", None)
    language = Language.find(form.get("language", None))
    native_language = Language.find(form.get("native_language", None))

    if not (code == "Kairo" or code == "unibe" or code == "rug" or code =="42"):
        flash("Invitation code is not recognized. Please contact us.")

    elif password is None or email is None or name is None:
        flash("Please enter your name, email address, and password")

    else:
        try:

            zeeguu.db.session.add(User(email, name, password, language, native_language))
            zeeguu.db.session.commit()
            user = User.authorize(email, password)
            flask.session["user"] = user.id
            return flask.redirect(flask.url_for("account.my_account"))

        except ValueError:
            flash("Username could not be created. Please contact us.")
        except sqlalchemy.exc.IntegrityError:
            flash(email + " is already in use. Please select a different email.")
        except:
            flash("Something went wrong. Please contact us.")
        finally:
            zeeguu.db.session.rollback()

    return flask.render_template("create_account.html", **template_arguments)
コード例 #13
0
def available_languages():
    """
    :return: jason with language codes for the
    supported languages.
    e.g. ["en", "fr", "de", "it", "no", "ro"]
    """
    available_language_codes = map((lambda x: x.id), Language.available_languages())
    return json.dumps(available_language_codes)
コード例 #14
0
def available_native_languages():
    """
    :return: jason with language codes for the
    supported native languages. curently only english...
    e.g. ["en", "fr", "de", "it", "no", "ro"]unquote_plus(flask.r
    """
    available_language_codes = map((lambda x: x.id), Language.native_languages())
    return json.dumps(available_language_codes)
コード例 #15
0
    def setUp(self):
        # Superclass does prepare the DB before each of the tests
        super(Dbtest, self).setUp()

        # Some common test fixtures
        self.mir = User.find("*****@*****.**")
        assert self.mir
        self.de = Language.find("de")
コード例 #16
0
ファイル: db_tests.py プロジェクト: MrAlexDeluxe/Zeeguu-API
    def setUp(self):
        # Superclass does prepare the DB before each of the tests
        super(Dbtest, self).setUp()

        # Some common test fixtures
        self.mir = User.find("*****@*****.**")
        assert self.mir
        self.de = Language.find("de")
コード例 #17
0
def bookmark_with_context(from_lang_code, to_lang_code, word_str, url_str,
                          title_str, context_str, translation_str):
    """
        This function will lookup a given word-text pair, and if found, it will return
     that bookmark rather than a new one

    :param from_lang_code:
    :param to_lang_code:
    :param word_str:
    :param url_str:
    :param title_str:
    :param context_str:
    :param translation_str:
    :return:
    """
    from_lang = Language.find(from_lang_code)
    to_lang = Language.find(to_lang_code)

    user_word = UserWord.find(word_str, from_lang)

    url = Url.find(url_str, title_str)
    zeeguu.db.session.add(url)
    zeeguu.db.session.commit()

    context = Text.find_or_create(context_str, from_lang, url)
    zeeguu.db.session.add(context)
    zeeguu.db.session.commit()

    translation = UserWord.find(translation_str, to_lang)

    try:
        bookmark = Bookmark.find_all_by_user_word_and_text(
            flask.g.user, user_word, context)[0]
    #     TODO: Think about updating the date of this bookmark, or maybe creating a duplicate
    #       otherwise, in the history this translation will not be visible!

    except Exception:
        bookmark = Bookmark(user_word, translation, flask.g.user, context,
                            datetime.now())
        zeeguu.db.session.add(bookmark)
        bookmark.calculate_probabilities_after_adding_a_bookmark(
            flask.g.user, bookmark.origin.language)
        zeeguu.db.session.commit()

    return str(bookmark.id)
コード例 #18
0
ファイル: user.py プロジェクト: alinbalutoiu/Zeeguu-Core
    def create_anonymous(cls,
                         uuid,
                         password,
                         learned_language_code=None,
                         native_language_code=None):
        """

        :param uuid:
        :param password:
        :param learned_language_code:
        :param native_language_code:
        :return:
        """

        # since the DB must have an email we generate a fake one
        fake_email = uuid + cls.ANONYMOUS_EMAIL_DOMAIN

        if learned_language_code is not None:
            try:
                learned_language = Language.find_or_create(
                    learned_language_code)
            except NoResultFound as e:
                learned_language = None
        else:
            learned_language = None

        if native_language_code is not None:
            try:
                native_language = Language.find_or_create(native_language_code)
            except NoResultFound as e:
                native_language = None
        else:
            native_language = None

        new_user = cls(fake_email,
                       uuid,
                       password,
                       learned_language=learned_language,
                       native_language=native_language)

        # # Until we find_or_create a better way of adding exercises for anonymous and new users... we simply
        # from zeeguu.temporary.default_words import default_bookmarks
        # default_bookmarks(new_user, learned_language_code)

        return new_user
コード例 #19
0
def available_languages():
    """
    :return: jason with language codes for the
    supported languages.
    e.g. ["en", "fr", "de", "it", "no", "ro"]
    """
    available_language_codes = map((lambda x: x.id),
                                   Language.available_languages())
    return json.dumps(available_language_codes)
コード例 #20
0
def available_native_languages():
    """
    :return: jason with language codes for the
    supported native languages. curently only english...
    e.g. ["en", "fr", "de", "it", "no", "ro"]unquote_plus(flask.r
    """
    available_language_codes = map((lambda x: x.id),
                                   Language.native_languages())
    return json.dumps(available_language_codes)
コード例 #21
0
    def test_languages_exists(self):
        language_should_be = LanguageRule().random

        try:
            language_to_check = Language.find(language_should_be.code)
        except NoResultFound:
            assert False, "No Language found in database"

        assert language_should_be.code == language_to_check.code \
               and language_should_be.name == language_to_check.name
コード例 #22
0
def add_ranked_word_to_db(lang_code):
    zeeguu.app.test_request_context().push()
    zeeguu.db.session.commit()
    from_lang = Language.find(lang_code)
    initial_line_number = 1

    for word in filter_word_list(test_word_list(lang_code)):
        r = RankedWord(word.lower(), from_lang,initial_line_number)
        zeeguu.db.session.add(r)
        initial_line_number+=1
    zeeguu.db.session.commit()
コード例 #23
0
    def __create_new_language(cls, language_id):
        language_name = cls.languages.get(language_id)

        if language_name is None:
            raise KeyError

        new_language = Language(language_id, language_name)

        cls.save(new_language)

        return new_language
コード例 #24
0
def bookmark_with_context(from_lang_code, to_lang_code, word_str, url_str, title_str, context_str, translation_str):
    """
        This function will lookup a given word-text pair, and if found, it will return
     that bookmark rather than a new one

    :param from_lang_code:
    :param to_lang_code:
    :param word_str:
    :param url_str:
    :param title_str:
    :param context_str:
    :param translation_str:
    :return:
    """
    from_lang = Language.find(from_lang_code)
    to_lang = Language.find(to_lang_code)

    user_word = UserWord.find(word_str, from_lang)

    url = Url.find(url_str, title_str)
    zeeguu.db.session.add(url)
    zeeguu.db.session.commit()

    context = Text.find_or_create(context_str, from_lang, url)
    zeeguu.db.session.add(context)
    zeeguu.db.session.commit()

    translation = UserWord.find(translation_str, to_lang)

    try:
        bookmark = Bookmark.find_all_by_user_word_and_text(flask.g.user, user_word, context)[0]
    #     TODO: Think about updating the date of this bookmark, or maybe creating a duplicate
    #       otherwise, in the history this translation will not be visible!

    except Exception:
        bookmark = Bookmark(user_word, translation, flask.g.user, context, datetime.now())
        zeeguu.db.session.add(bookmark)
        bookmark.calculate_probabilities_after_adding_a_bookmark(flask.g.user, bookmark.origin.language)
        zeeguu.db.session.commit()

    return str(bookmark.id)
コード例 #25
0
def get_learned_bookmarks(lang):
    lang = Language.find(lang)

    estimator = SethiKnowledgeEstimator(flask.g.user, lang.id)
    bk_list = [
        dict(
            id=bookmark.id,
            origin=bookmark.origin.word,
            text=bookmark.text.content)
        for bookmark in estimator.learned_bookmarks()]

    return json_result(bk_list)
コード例 #26
0
def get_difficulty_for_text(lang_code):
    """
    URL parameters:
    :param lang_code: the language of the text

    Json data:
    :param texts: json array that contains the texts to calculate the difficulty for. Each text consists of an array
        with the text itself as 'content' and an additional 'id' which gets roundtripped unchanged
    :param difficulty_computer (optional): calculate difficulty score using a specific algorithm
    :param rank_boundary (deprecated): upper boundary for word frequency rank (between 1 and 10'000)
    :param personalized (deprecated): by default we always compute the personalized difficulty

    For an example of how the Json data looks like, see
        ../tests/api_tests.py#test_txt_difficulty(self):

    :return difficulties: json array, which contains for each text:
      * estimated_difficulty - one of three: "EASY", "MEDIUM", "HARD"
      * id - identifies the text
      * [deprecated] score_average - average difficulty of the words in the text
      * [deprecated] score_median - median difficulty of the words in the text
    """
    language = Language.find(lang_code)
    if not language:
        return 'FAIL'

    data = request.get_json()

    if not 'texts' in data:
        return 'FAIL'

    texts = []
    for text in data['texts']:
        texts.append(text)

    difficulty_computer = 'default'
    if 'difficulty_computer' in data:
        difficulty_computer = data['difficulty_computer'].lower()

    user = flask.g.user
    known_probabilities = KnownWordProbability.find_all_by_user_cached(user)

    difficulties = []
    for text in texts:
        difficulty = text_difficulty(
                text["content"],
                language,
                known_probabilities,
                difficulty_computer
                )
        difficulty["id"] = text["id"]
        difficulties.append(difficulty)

    return json_result(dict(difficulties=difficulties))
コード例 #27
0
def get_learned_bookmarks(lang):
    lang = Language.find(lang)

    estimator = SethiKnowledgeEstimator(flask.g.user, lang.id)
    bk_list = [
        dict(id=bookmark.id,
             origin=bookmark.origin.word,
             text=bookmark.text.content)
        for bookmark in estimator.learned_bookmarks()
    ]

    return json_result(bk_list)
コード例 #28
0
ファイル: user.py プロジェクト: alinbalutoiu/Zeeguu-Core
 def __init__(self,
              email,
              name,
              password,
              learned_language=None,
              native_language=None,
              invitation_code=None,
              cohort=None):
     self.email = email
     self.name = name
     self.update_password(password)
     self.learned_language = learned_language or Language.default_learned()
     self.native_language = native_language or Language.default_native_language(
     )
     self.invitation_code = invitation_code
     self.cohort = cohort
     # Add the learned language to user languages and set reading_news to True
     # so that the user has articles in the reader when opening it for the first time.
     from zeeguu.model import UserLanguage
     UserLanguage(self,
                  learned_language or Language.default_learned(),
                  reading_news=True)
コード例 #29
0
def get_difficulty_for_text(lang_code):
    """
    URL parameters:
    :param lang_code: the language of the text

    Json data:
    :param texts: json array that contains the texts to calculate the difficulty for. Each text consists of an array
        with the text itself as 'content' and an additional 'id' which gets roundtripped unchanged
    :param difficulty_computer (optional): calculate difficulty score using a specific algorithm
    :param rank_boundary (deprecated): upper boundary for word frequency rank (between 1 and 10'000)
    :param personalized (deprecated): by default we always compute the personalized difficulty

    For an example of how the Json data looks like, see
        ../tests/api_tests.py#test_txt_difficulty(self):

    :return difficulties: json array, which contains for each text:
      * estimated_difficulty - one of three: "EASY", "MEDIUM", "HARD"
      * id - identifies the text
      * [deprecated] score_average - average difficulty of the words in the text
      * [deprecated] score_median - median difficulty of the words in the text
    """
    language = Language.find(lang_code)
    if not language:
        return 'FAIL'

    data = request.get_json()

    if not 'texts' in data:
        return 'FAIL'

    texts = []
    for text in data['texts']:
        texts.append(text)

    difficulty_computer = 'default'
    if 'difficulty_computer' in data:
        difficulty_computer = data['difficulty_computer'].lower()

    user = flask.g.user
    known_probabilities = KnownWordProbability.find_all_by_user_cached(user)

    difficulties = []
    for text in texts:
        difficulty = text_difficulty(text["content"], language,
                                     known_probabilities, difficulty_computer)
        difficulty["id"] = text["id"]
        difficulties.append(difficulty)

    return json_result(dict(difficulties=difficulties))
コード例 #30
0
    def test_same_text_does_not_get_created_multiple_Times(self):

        context = u'Die kleine Jägermeister'
        with zeeguu.app.app_context():
            url = Url.find('http://mir.lu/stories/german/jagermeister',
                           "Die Kleine Jagermeister (Mircea's Stories)")
            source_language = Language.find('de')

            form_data = dict(url=url.as_string(), context=context, word="Die")

            self.api_post('/translate_and_bookmark/de/en', form_data)
            text1 = Text.find_or_create(context, source_language, url)
            self.api_post('/translate_and_bookmark/de/en', form_data)
            text2 = Text.find_or_create(context, source_language, url)
            assert (text1 == text2)
コード例 #31
0
def start_following_feeds():
    """
    A user can start following multiple feeds at once.

    The feeds are passed as the post parameter :feeds:
     which contains a json list with URLs for the feeds to be followed.

    :return:
    """

    json_array_with_feeds = json.loads(request.form.get('feeds', ''))

    for urlString in json_array_with_feeds:
        feed = feedparser.parse(urlString).feed

        feed_image_url_string = ""
        if "image" in feed:
            feed_image_url_string = feed.image["href"]

        lan = None
        if "language" in feed:
            lan = Language.find(two_letter_language_code(feed))

        url = Url.find(urlString)
        zeeguu.db.session.add(url)
        # Important to commit this url first; otherwise we end up creating
        # two domains with the same name for both the urls...
        zeeguu.db.session.commit()

        feed_object = RSSFeed.find_by_url(url)
        if not feed_object:
            feed_image_url = Url.find(feed_image_url_string)
            title = url
            if "title" in feed:
                title = feed.title
            feed_object = RSSFeed.find_or_create(url, title, feed.description,
                                                 feed_image_url, lan)
            zeeguu.db.session.add_all([feed_image_url, feed_object])
            zeeguu.db.session.commit()

        feed_registration = RSSFeedRegistration.find_or_create(
            flask.g.user, feed_object)

        zeeguu.db.session.add(feed_registration)
        zeeguu.db.session.commit()

    return "OK"
コード例 #32
0
 def known_words_list(self):
     lang_id = Language.find(self.lang_code)
     bookmarks = self.user.all_bookmarks()
     known_words = []
     filtered_known_words_from_user = []
     filtered_known_words_dict_list = []
     for bookmark in bookmarks:
         if bookmark.check_is_latest_outcome_too_easy():
             known_words.append(bookmark.origin.word)
     for word_known in known_words:
         if RankedWord.exists(word_known, lang_id):
             filtered_known_words_from_user.append(word_known)
             # zeeguu.db.session.commit()
     filtered_known_words_from_user = list(
         set(filtered_known_words_from_user))
     for word in filtered_known_words_from_user:
         filtered_known_words_dict_list.append({'word': word})
     return filtered_known_words_dict_list
コード例 #33
0
def get_interesting_reading_languages():
    """
    'Interesting languages' are defined as languages the user
    isn't subscribed to already and thus might subscribe to.

    :return: a json list with languages the user isn't reading yet.
    every language in this list is a dictionary with the following info:
                id = unique id of the language;
                language = <unicode string>
    """

    all_languages = Language.available_languages()
    learned_languages = UserLanguage.all_reading_for_user(flask.g.user)

    interesting_languages = []

    for lan in all_languages:
        if lan not in learned_languages:
            interesting_languages.append(lan.as_dictionary())

    return json_result(interesting_languages)
コード例 #34
0
def get_learnability_for_text(lang_code):
    """
    URL parameters:
    :param lang_code: the language of the text

    Json data:
    :param texts: json array that contains the texts to calculate the learnability for. Each text consists of an array
        with the text itself as 'content' and an additional 'id' which gets roundtripped unchanged
        For an example of how the Json data looks like, see
            ../tests/api_tests.py#test_text_learnability(self):


    :return learnabilities: json array, contains the learnabilities as arrays with the key 'score' for the learnability
        value (percentage of words from the text that the user is currently learning), the 'count' of the learned
        words in the text and the 'id' parameter to identify the corresponding text
    """
    user = flask.g.user

    language = Language.find(lang_code)
    if language is None:
        return 'FAIL'

    data = request.get_json()

    texts = []
    if 'texts' in data:
        for text in data['texts']:
            texts.append(text)
    else:
        return 'FAIL'

    learnabilities = []
    for text in texts:
        e = SethiKnowledgeEstimator(user)
        count, learnability = text_learnability(
            text, e.words_being_learned(language))
        learnabilities.append(
            dict(score=learnability, count=count, id=text['id']))

    return json_result(dict(learnabilities=learnabilities))
コード例 #35
0
def add_ranked_word_to_db(lang_code, word_list_file, number_of_words):
    """
      Adds the ranks of the words to the DB.


    :param lang_code:
    :param word_list_file:
    :return:
    """
    zeeguu.app.test_request_context().push()
    zeeguu.db.session.commit()

    print ("Looking for language ..." + lang_code)
    language = Language.find(lang_code)

    print "-> Starting to import words in the DB"
    current_line_number = 1
    word_list = remove_duplicates_based_on_case(read_words_from_file(word_list_file))
    word_list = word_list[0:number_of_words]

    for word in word_list:

        if RankedWord.exists(word, language):
            ranked_word = RankedWord.find(word, language)
            ranked_word.rank = current_line_number
        else:
            ranked_word = RankedWord(word, language, current_line_number)
        zeeguu.db.session.add(ranked_word)

        current_line_number += 1
        print_progress_stats(current_line_number, word)

    # Commit everything at once - twice as fast as committing  after every word
    zeeguu.db.session.commit()

    print ('-> Done importing the ranked words in the DB')

    print ('-> Updating word ranks for words already in the DB...')
    update_existing_word_ranks(language)
コード例 #36
0
def get_learnability_for_text(lang_code):
    """
    URL parameters:
    :param lang_code: the language of the text

    Json data:
    :param texts: json array that contains the texts to calculate the learnability for. Each text consists of an array
        with the text itself as 'content' and an additional 'id' which gets roundtripped unchanged
        For an example of how the Json data looks like, see
            ../tests/api_tests.py#test_text_learnability(self):


    :return learnabilities: json array, contains the learnabilities as arrays with the key 'score' for the learnability
        value (percentage of words from the text that the user is currently learning), the 'count' of the learned
        words in the text and the 'id' parameter to identify the corresponding text
    """
    user = flask.g.user

    language = Language.find(lang_code)
    if language is None:
        return 'FAIL'

    data = request.get_json()

    texts = []
    if 'texts' in data:
        for text in data['texts']:
            texts.append(text)
    else:
        return 'FAIL'

    learnabilities = []
    for text in texts:
        e = SethiKnowledgeEstimator(user)
        count, learnability = text_learnability(text, e.words_being_learned(language))
        learnabilities.append(dict(score=learnability, count=count, id=text['id']))

    return json_result(dict(learnabilities=learnabilities))
コード例 #37
0
def set_default_exercise_based_prob():
    zeeguu.app.test_request_context().push()
    zeeguu.db.session.commit()
    users = User.find_all()
    languages = Language.all()

    for user in users:
        for language in languages:
            user_words_by_language = UserWord.find_by_language(language)
            for word in user_words_by_language:
                if ExerciseBasedProbability.exists(user, word):
                    prob = ExerciseBasedProbability.find(user, word)
                    bookmarks_by_user_and_word = Bookmark.find_all_by_user_and_word(
                        user, word)
                    total_prob = 0
                    for bookmark in bookmarks_by_user_and_word:
                        prob.calculate_known_bookmark_probability(bookmark)
                        total_prob += float(prob.probability)
                    if bookmarks_by_user_and_word:
                        prob.probability = total_prob / len(
                            bookmarks_by_user_and_word)
                    zeeguu.db.session.commit()
    print('job1')
コード例 #38
0
def modify_user_language():
    """
    This endpoint is for modifying a user language.
    It tries to find the user_language, and otherwise create it.
    It then sets all the parameters given.

    :return: "OK" in case of success
    """
    language_id = int(request.form.get('language_id', ''))
    try:
        language_reading = int(request.form.get('language_reading', ''))
    except:
        language_reading = None
    try:
        language_exercises = int(request.form.get('language_exercises', ''))
    except:
        language_exercises = None
    try:
        language_level = int(request.form.get('language_level', ''))
    except:
        language_level = None

    language_object = Language.find_by_id(language_id)
    user_language = UserLanguage.find_or_create(session, flask.g.user,
                                                language_object)
    if language_reading is not None:
        user_language.reading_news = language_reading
        recompute_recommender_cache_if_needed(flask.g.user, session)
    if language_exercises is not None:
        user_language.doing_exercises = language_exercises
    if language_level is not None:
        user_language.declared_level = language_level
    session.add(user_language)
    session.commit()

    return "OK"
コード例 #39
0
    def find_or_create(cls, session, _url:str, language=None, sleep_a_bit=False):
        """

            If not found, download and extract all
            the required info for this article.

        :param url:
        :return:
        """
        from zeeguu.model import Url, Article, Language
        import newspaper

        url = Url.extract_canonical_url(_url)

        try:
            found = cls.find(url)
            if found:
                return found

            art = newspaper.Article(url=url)
            art.download()
            art.parse()

            if art.text == '':
                raise Exception("Newspaper got empty article from: " + url)

            if sleep_a_bit:
                import time
                from random import randint
                print("GOT: " + url)
                sleep_time = randint(3, 33)
                print(f"sleeping for {sleep_time}s... so we don't annoy our friendly servers")
                time.sleep(sleep_time)

            if not language:
                if art.meta_lang == '':
                    art.meta_lang = detect(art.text)
                    zeeguu.log(f"langdetect: {art.meta_lang} for {url}")
                language = Language.find_or_create(art.meta_lang)

            # Create new article and save it to DB
            url_object = Url.find_or_create(session, url)

            new_article = Article(
                url_object,
                art.title,
                ', '.join(art.authors),
                art.text[0:32000],  # any article longer than this will be truncated...
                art.summary,
                None,
                None,
                language
            )
            session.add(new_article)

            session.commit()

            return new_article
        except sqlalchemy.exc.IntegrityError or sqlalchemy.exc.DatabaseError:
            for i in range(10):
                try:
                    session.rollback()
                    u = cls.find(url)
                    print("Found article by url after recovering from race")
                    return u
                except:
                    print("Exception of second degree in article..." + str(i))
                    time.sleep(0.3)
                    continue
                break
コード例 #40
0
def get_content_from_url():
    """
    Json data:
    :param urls: json array that contains the urls to get the article content for. Each url consists of an array
        with the url itself as 'url' and an additional 'id' which gets roundtripped unchanged.
        For an example of how the Json data looks like, see
            ../tests/api_tests.py#test_content_from_url(self):

    :param timeout (optional): maximal time in seconds to wait for the results

    :param lang_code (optional): If the user sends along the language, then we compute the difficulty of the texts

    :return contents: json array, contains the contents of the urls that responded within the timeout as arrays
        with the key 'content' for the article content, the url of the main image as 'image' and the 'id' parameter
        to identify the corresponding url

    """
    data = request.get_json()
    queue = Queue.Queue()

    urls = []
    if 'urls' in data:
        for url in data['urls']:
            urls.append(url)
    else:
        return 'FAIL'

    if 'timeout' in data:
        timeout = int(data['timeout'])
    else:
        timeout = 10

    # Start worker threads to get url contents
    threads = []
    for url in urls:
        thread = threading.Thread(target=PageExtractor.worker, args=(url['url'], url['id'], queue))
        thread.daemon = True
        threads.append(thread)
        thread.start()

    # Wait for workers to finish until timeout
    stop = time.time() + timeout
    while any(t.isAlive() for t in threads) and time.time() < stop:
        time.sleep(0.1)

    contents = []
    for i in xrange(len(urls)):
        try:
            contents.append(queue.get_nowait())
        except Queue.Empty:
            pass

    # If the user sends along the language, then we can compute the difficulty
    if 'lang_code' in data:
        lang_code = data['lang_code']
        language = Language.find(lang_code)
        if language is not None:
            print "got language"
            user = flask.g.user
            known_probabilities = KnownWordProbability.find_all_by_user_cached(user)
            for each_content_dict in contents:
                    difficulty = text_difficulty(
                            each_content_dict["content"],
                            language,
                            known_probabilities
                            )
                    each_content_dict["difficulty"] = difficulty

    return json_result(dict(contents=contents))
コード例 #41
0
ファイル: user.py プロジェクト: MrAlexDeluxe/Zeeguu-API
 def __init__(self, email, username, password, learned_language=None, native_language = None):
     self.email = email
     self.name = username
     self.update_password(password)
     self.learned_language = learned_language or Language.default_learned()
     self.native_language = native_language or Language.default_native_language()
コード例 #42
0
ファイル: user.py プロジェクト: MrAlexDeluxe/Zeeguu-API
 def set_learned_language(self, code):
     self.learned_language = Language.find(code)
コード例 #43
0
ファイル: user.py プロジェクト: MrAlexDeluxe/Zeeguu-API
 def set_native_language(self, code):
     self.native_language = Language.find(code)
コード例 #44
0
def get_not_encountered_words(lang_code):
    return json_result(flask.g.user.get_not_encountered_words(Language.find(lang_code)))