def get_interesting_topics(): """ Get a list of interesting topics for the given language. Interesting topics are for now defined as: - The topic is not followed yet - The topic is not in the filters list - There are articles with that topic in the language :return: """ topic_data = [] already_filtered = [each.topic for each in TopicFilter.all_for_user(flask.g.user)] already_subscribed = [ each.topic for each in TopicSubscription.all_for_user(flask.g.user) ] reading_languages = Language.all_reading_for_user(flask.g.user) loc_topics = [] for each in reading_languages: loc_topics.extend(LocalizedTopic.all_for_language(each)) topics = [each.topic for each in loc_topics] for topic in topics: if (topic not in already_filtered) and (topic not in already_subscribed): topic_data.append(topic.as_dictionary()) return json_result(topic_data)
def _reading_preferences_hash(user): """ Method to retrieve the hash, as this is done several times. :param user: :return: articles_hash: ArticlesHash """ user_filter_subscriptions = TopicFilter.all_for_user(user) filters = [topic_id.topic for topic_id in user_filter_subscriptions] user_topic_subscriptions = TopicSubscription.all_for_user(user) topics = [topic_id.topic for topic_id in user_topic_subscriptions] user_languages = Language.all_reading_for_user(user) user_search_filters = SearchFilter.all_for_user(user) search_filters = [search_id.search for search_id in user_search_filters] user_searches = SearchSubscription.all_for_user(user) searches = [search_id.search for search_id in user_searches] articles_hash = ArticlesCache.calculate_hash( user, topics, filters, searches, search_filters, user_languages ) return articles_hash
def filter_subscribed_articles(subscribed_articles, user_languages, user): """ :param subscribed_articles: :param user_filters: :param user_languages: :param user_search_filters: :return: a generator which retrieves articles as needed """ def _article_matches_user_topic_filters(article, filters): return not set(article.topics).isdisjoint([each.topic for each in filters]) user_search_filters = SearchFilter.all_for_user(user) user_filters = TopicFilter.all_for_user(user) keywords_to_avoid = [] for user_search_filter in user_search_filters: keywords_to_avoid.append(user_search_filter.search.keywords) subscribed_articles = (art for art in subscribed_articles if (art.language in user_languages) and not art.broken and (UserLanguage.appropriate_level(art, user)) and not _article_matches_user_topic_filters(art, user_filters) and not (art.contains_any_of(keywords_to_avoid))) return subscribed_articles
def get_interesting_topics(): """ Get a list of interesting topics for the given language. Interesting topics are for now defined as: - The topic is not followed yet - The topic is not in the filters list :return: """ topic_data = [] already_filtered = [each.topic for each in TopicFilter.all_for_user(flask.g.user)] already_subscribed = [each.topic for each in TopicSubscription.all_for_user(flask.g.user)] for topic in Topic.get_all_topics(): if (topic not in already_filtered) and (topic not in already_subscribed): topic_data.append(topic.as_dictionary()) return json_result(topic_data)
def get_subscribed_filters(): """ A user might be subscribed to multiple filters at once. This endpoint returns them as a list. :return: a json list with filters for which the user is registered; every filter in this list is a dictionary with the following info: id = unique id of the topic; title = <unicode string> """ filters = TopicFilter.all_for_user(flask.g.user) filter_list = [] for fil in filters: try: filter_list.append(fil.topic.as_dictionary()) except Exception as e: zeeguu_core.log(str(e)) return json_result(filter_list)
def get_subscribed_filters(): """ A user might be subscribed to multiple filters at once. This endpoint returns them as a list. :return: a json list with filters for which the user is registered; every filter in this list is a dictionary with the following info: id = unique id of the topic; title = <unicode string> """ filters = TopicFilter.all_for_user(flask.g.user) filter_list = [] for fil in filters: try: filter_list.append(fil.topic.as_dictionary()) except Exception as e: zeeguu_core.log(str(e)) return json_result(filter_list)
def get_interesting_topics(): """ Get a list of interesting topics for the given language. Interesting topics are for now defined as: - The topic is not followed yet - The topic is not in the filters list :return: """ topic_data = [] already_filtered = [ each.topic for each in TopicFilter.all_for_user(flask.g.user) ] already_subscribed = [ each.topic for each in TopicSubscription.all_for_user(flask.g.user) ] for topic in Topic.get_all_topics(): if (topic not in already_filtered) and (topic not in already_subscribed): topic_data.append(topic.as_dictionary()) return json_result(topic_data)
def _filter_subscribed_articles( search_subscriptions, topic_subscriptions, user_languages, user ): """ :param subscribed_articles: :param user_filters: :param user_languages: :param user_search_filters: :return: a generator which retrieves articles as needed """ from zeeguu_core.model import Topic user_search_filters = SearchFilter.all_for_user(user) # TODO: shouldn't this be passed down from upstream? total_article_count = 30 per_language_article_count = total_article_count / len(user_languages) final_article_mix = set() for language in user_languages: print(f"language: {language}") query = Article.query query = query.order_by(Article.id.desc()) query = query.filter(Article.language == language) query = query.filter(Article.broken == False) # speed up a bit the stuff # query = query.filter(Article.id > 500000) # 0. Ensure appropriate difficulty declared_level_min, declared_level_max = user.levels_for(language) lower_bounds = declared_level_min * 10 upper_bounds = declared_level_max * 10 query = query.filter(lower_bounds < Article.fk_difficulty) query = query.filter(Article.fk_difficulty < upper_bounds) # 1. Keywords to exclude # ============================== keywords_to_avoid = [] for user_search_filter in user_search_filters: keywords_to_avoid.append(user_search_filter.search.keywords) print(f"keywords to exclude: {keywords_to_avoid}") for keyword_to_avoid in keywords_to_avoid: query = query.filter( not_( or_( Article.title.contains(keyword_to_avoid), Article.content.contains(keyword_to_avoid), ) ) ) # title does not contain keywords # 2. Topics to exclude / filter out # ================================= user_filters = TopicFilter.all_for_user(user) to_exclude_topic_ids = [each.topic.id for each in user_filters] print(f"to exlcude topic ids: {to_exclude_topic_ids}") print(f"topics to exclude: {user_filters}") query = query.filter( not_(Article.topics.any(Topic.id.in_(to_exclude_topic_ids))) ) # 3. Topics subscribed, and thus to include # ========================================= ids_of_topics_to_include = [ subscription.topic.id for subscription in topic_subscriptions ] # print(f"topics to include: {topic_subscriptions}") print(f"topics ids to include: {ids_of_topics_to_include}") # we comment out this line, because we want to do an or_between it and the # one corresponding to searches later below! # query = query.filter(Article.topics.any(Topic.id.in_(topic_ids))) # 4. Searches to include # ====================== print(f"Search subscriptions: {search_subscriptions}") ids_for_articles_containing_search_terms = set() for user_search in search_subscriptions: search_string = user_search.search.keywords.lower() articles_for_word = ArticleWord.get_articles_for_word(search_string) ids_for_articles_containing_search_terms.update( [article.id for article in articles_for_word] ) # commenting out this line, in favor of it being part of a merge later # query = query.filter(Article.id.in_(article_ids)) if ids_of_topics_to_include or ids_for_articles_containing_search_terms: query = query.filter( or_( Article.topics.any(Topic.id.in_(ids_of_topics_to_include)), Article.id.in_(ids_for_articles_containing_search_terms), ) ) query = query.limit(per_language_article_count) final_article_mix.update(query.all()) return final_article_mix
def article_search_for_user(user, count, search_terms): """ Handles searching. Find the relational values from the database and use them to search in elasticsearch for relative articles. :param user: :param count: max amount of articles to return :param search_terms: the inputed search string by the user :return: articles """ user_languages = Language.all_reading_for_user(user) per_language_article_count = count / len(user_languages) final_article_mix = [] for language in user_languages: print(f"language: {language}") # 0. Ensure appropriate difficulty declared_level_min, declared_level_max = user.levels_for(language) lower_bounds = declared_level_min * 10 upper_bounds = declared_level_max * 10 # 1. Unwanted user topics # ============================== user_search_filters = SearchFilter.all_for_user(user) unwanted_user_topics = [] for user_search_filter in user_search_filters: unwanted_user_topics.append(user_search_filter.search.keywords) print(f"keywords to exclude: {unwanted_user_topics}") # 2. Topics to exclude / filter out # ================================= excluded_topics = TopicFilter.all_for_user(user) topics_to_exclude = [each.topic.title for each in excluded_topics] print(f"topics to exclude: {topics_to_exclude}") # 3. Topics subscribed, and thus to include # ========================================= topic_subscriptions = TopicSubscription.all_for_user(user) topics_to_include = [ subscription.topic.title for subscription in TopicSubscription.all_for_user(user) ] print(f"topics to include: {topic_subscriptions}") # 4. Wanted user topics # ========================================= user_subscriptions = SearchSubscription.all_for_user(user) wanted_user_topics = [] for sub in user_subscriptions: wanted_user_topics.append(sub.search.keywords) print(f"keywords to include: {wanted_user_topics}") # build the query using elastic_query_builder query_body = build_elastic_query( per_language_article_count, search_terms, _list_to_string(topics_to_include), _list_to_string(topics_to_exclude), _list_to_string(wanted_user_topics), _list_to_string(unwanted_user_topics), language, upper_bounds, lower_bounds, ) es = Elasticsearch(ES_CONN_STRING) res = es.search(index=ES_ZINDEX, body=query_body) hit_list = res["hits"].get("hits") final_article_mix.extend(_to_articles_from_ES_hits(hit_list)) # convert to article_info and return return [ UserArticle.user_article_info(user, article) for article in final_article_mix if article is not None ]