Exemplo n.º 1
0
def show_post(slug):
    main_post = mongo_col.find_one({"slug": slug})
    main_post = {
        "url": main_post["canonical_url"],
        "title": main_post["title"],
        "slug": main_post["slug"],
        "content": main_post["contents"]
    }

    # preprocessing
    content = markdown_to_text(main_post["content"])
    text_corpus = make_texts_corpus([content])
    bow = id2word.doc2bow(next(text_corpus))
    doc_distribution = np.array(
        [doc_top[1] for doc_top in lda_model.get_document_topics(bow=bow)])

    # recommender posts
    most_sim_ids = list(
        get_most_similar_documents(doc_distribution, doc_topic_dist))[1:]

    most_sim_ids = [int(id_) for id_ in most_sim_ids]
    posts = mongo_col.find({"idrs": {"$in": most_sim_ids}})
    related_posts = [{
        "url": post["canonical_url"],
        "title": post["title"],
        "slug": post["slug"]
    } for post in posts][1:]

    return render_template('index.html',
                           main_post=main_post,
                           posts=related_posts)
Exemplo n.º 2
0
def show_post(slug):
    main_post = mongo_col.find_one({"slug": slug})
    md = get_content_of_post(slug)
    main_post = {
        "url": main_post["url"],
        "title": main_post["title"],
        "slug": main_post["slug"],
        "content": md2html(md)
    }

    # preprocessing
    content = markdown_to_text(main_post["content"])
    text_corpus = make_texts_corpus([content])
    bow = id2word.doc2bow(next(text_corpus))
    # sử dụng dictionary và LDA model đã train và lưu lại để thu được vector document_dist, ứng với phân bố các topic của document đó
    doc_distribution = np.array(
        [doc_top[1] for doc_top in lda_model.get_document_topics(bow=bow)])

    # recommender posts
    most_sim_ids = list(
        get_most_similar_documents(doc_distribution, doc_topic_dist))[1:]

    most_sim_ids = [int(id_) for id_ in most_sim_ids]
    posts = mongo_col.find({"idrs": {"$in": most_sim_ids}})
    related_posts = [{
        "url": post["url"],
        "title": post["title"],
        "slug": post["slug"]
    } for post in posts][1:]

    return render_template('index.html',
                           main_post=main_post,
                           posts=related_posts)
Exemplo n.º 3
0
def search():
    if request.method == 'POST':
        search_text = request.form['search']
    else:
        search_text = request.args.get('search')

    if request.args.get('is_content') is None:
        posts = mongo_col.find({
            "title": {
                "$regex": search_text
            }
        }).sort("title").limit(30)
        random_posts = [{
            "idrs": post["idrs"],
            "url": post["url"],
            "title": post["title"],
            "slug": post["slug"],
            "id": post["_id"],
            "is_active": post["is_active"],
            "views_count": post["views_count"]
        } for post in posts]
    else:
        content = markdown_to_text(search_text)
        text_corpus = make_texts_corpus([content])
        bow = id2word.doc2bow(next(text_corpus))
        doc_distribution = np.array(
            [doc_top[1] for doc_top in lda_model.get_document_topics(bow=bow)])
        # recommender posts
        most_sim_ids = list(
            get_most_similar_documents(doc_distribution, doc_topic_dist))[1:]

        most_sim_ids = [int(id_) for id_ in most_sim_ids]
        posts = mongo_col.find({"idrs": {"$in": most_sim_ids}})
        random_posts = [{
            "idrs": post["idrs"],
            "url": post["url"],
            "title": post["title"],
            "slug": post["slug"],
            "id": post["_id"],
            "is_active": post["is_active"],
            "views_count": post["views_count"]
        } for post in posts][1:]
    return render_template('list-post.html', random_posts=random_posts)
Exemplo n.º 4
0
def search():
    search_text = request.form['search']
    print('search text : ', search_text)
    content = markdown_to_text(search_text)
    text_corpus = make_texts_corpus([content])
    bow = id2word.doc2bow(next(text_corpus))
    doc_distribution = np.array(
        [doc_top[1] for doc_top in lda_model.get_document_topics(bow=bow)])
    # recommender posts
    most_sim_ids = list(
        get_most_similar_documents(doc_distribution, doc_topic_dist))[1:]

    most_sim_ids = [int(id_) for id_ in most_sim_ids]
    posts = mongo_col.find({"idrs": {"$in": most_sim_ids}})
    related_posts = [{
        "url": post["url"],
        "title": post["title"],
        "slug": post["slug"]
    } for post in posts][1:]
    return render_template('search.html', search=related_posts)
 def predict(self, document_dist):
     doc_topic_dist = self.documents_topic_distribution()
     return get_most_similar_documents(document_dist, doc_topic_dist)
Exemplo n.º 6
0
def show_post(slug):
    main_post = mongo_col.find_one({"slug": slug})
    mongo_col.update_one(
        {"_id": ObjectId(main_post["_id"])},
        {"$set": {
            "views_count": main_post["views_count"] + 1
        }})

    data = pd.read_csv("../src/preprocessing/major_word.csv")
    features = data.values
    tags = []
    dictionary = []
    for dic in features:
        if dic[1] >= 2000:
            dictionary.append([dic[0], dic[1]])
    dictionary = np.array(dictionary)

    text = main_post["content"]
    content_list = text.split()
    for content in list(set(content_list)):
        if content in dictionary[:, 0]:
            text = [{
                "text": word[0],
                "total": int(word[1])
            } for word in dictionary if word[0] == content]
            tags.append(text[0])

    print(tags)
    main_post = {
        "url": main_post["url"],
        "title": main_post["title"],
        "slug": main_post["slug"],
        # "content": md2html(text),
        "content": md2html(main_post["content"])
    }

    # # md = get_content_of_post(slug)
    # main_post = {
    #     "url": main_post["url"],
    #     "title": main_post["title"],
    #     "slug": main_post["slug"],
    #     "content": md2html(main_post["content"])
    # }

    # preprocessing
    content = markdown_to_text(main_post["content"])
    text_corpus = make_texts_corpus([content])
    bow = id2word.doc2bow(next(text_corpus))
    # sử dụng dictionary và LDA model đã train và lưu lại để thu được vector document_dist, ứng với phân bố các topic của document đó
    doc_distribution = np.array(
        [doc_top[1] for doc_top in lda_model.get_document_topics(bow=bow)])

    # recommender posts
    most_sim_ids = list(
        get_most_similar_documents(doc_distribution, doc_topic_dist))[1:]

    most_sim_ids = [int(id_) for id_ in most_sim_ids]
    posts = mongo_col.find({"idrs": {"$in": most_sim_ids}})
    related_posts = [{
        "url": post["url"],
        "title": post["title"],
        "slug": post["slug"],
    } for post in posts][1:]

    return render_template('index.html',
                           main_post=main_post,
                           posts=related_posts,
                           tags=tags)