def fetch_packages(self): package_ids = None key = 'popular_packages.' + str(self.user.id) if self.use_cache: try: package_ids = cache.get(key) except Exception as e: app.logger.exception(e) app.logger.info("An exception catched, disable cache") self.use_cache = False if package_ids is None: packages = db.session.query(Subscribe.feed_id, db.func.count(Subscribe.id).label('total')) \ .filter(Subscribe.type == Subscribe.TYPE_FEED) \ .filter(Subscribe.feed_id.notin_(db.session.query(Subscribe.feed_id).filter( (Subscribe.user_id == self.user.id) & (Subscribe.feed_id.isnot(None)) ).group_by(Subscribe.feed_id))) \ .group_by(Subscribe.feed_id) \ .order_by(db.desc('total')) \ .limit(500) package_ids = [] for (feed_id, _count) in packages: package_ids.append(feed_id) if self.random_order: import random random.shuffle(package_ids) package_ids = package_ids[:self.packages_count] if self.use_cache: cache.set(key, package_ids, timeout=self.timeout) feeds = Feed.query.filter(Feed.id.in_(package_ids)) return feeds
def recalculate_categories(): from rdr.modules.subscribes.models import Subscribe from rdr.modules.feeds.models import Feed from rdr.tasks.schedule.feeds import categories from rdr.application.database import db from math import ceil feed_ids = db.session.query(Subscribe.feed_id, db.func.count(Subscribe.id).label('total')) \ .filter(Subscribe.type == Subscribe.TYPE_FEED) \ .group_by(Subscribe.feed_id) \ .order_by(db.desc('total')) feed_ids = [x[0] for x in feed_ids] count = len(feed_ids) offset = 0 results = [] for num in xrange(0, len(categories)): category = categories[num][0] percent = categories[num][1] if offset < count: if num != len(categories) - 1: percent_count = int(ceil(float(percent) / 100 * count)) results.append((category, feed_ids[offset:offset+percent_count])) offset += percent_count else: results.append((category, feed_ids[offset:])) for res in results: if res[1]: q = db.update(Feed).values({'category': res[0]}) \ .where(Feed.id.in_(res[1])) db.session.execute(q) db.session.commit()
def generate(self): date_from = self._get_date_from() if date_from is None: return [] stats = db.session.query(ArticleStatus.article_id, db.func.count(ArticleStatus.id).label('total')) \ .join(ArticleStatus.article) \ .join(Subscribe, ((Subscribe.type == Subscribe.TYPE_FEED) & (Subscribe.feed_id == Article.feed_id))) \ .group_by(ArticleStatus.article_id).filter( (ArticleStatus.status == ArticleStatus.STATUS_READ) & (Subscribe.user_id == self.user.id) & (Article.fetched >= date_from) ).order_by(db.desc('total')).limit(self.articles_count) article_ids = [] for (article_id, _count) in stats: article_ids.append(article_id) article_ids = list(unique(article_ids)) return article_ids
def generate_stats(self): feed_ids = self._get_feed_ids() from rdr.modules.feeds.models import Article articles = Article.query.filter((Article.feed_id.in_(feed_ids))) if self.dates == StatsGenerator.DATES_LAST_WEEK: days_count = 7 elif self.dates == StatsGenerator.DATES_LAST_MONTH: days_count = 30 else: raise Exception('Unknown date format') end_date = datetime.now() start_date = end_date - timedelta(days=days_count) articles = articles.filter((Article.published >= start_date) & (Article.published <= end_date)) stats = {} dates = OrderedDict() days_of_week = self._get_days_of_week() for day in days_of_week: dates[day] = { 'count': 0, 'read': 0, 'articles': [] } articles = articles.all() for article in articles: if article.published: date = article.published.strftime('%a') if date in dates: dates[date]['count'] += 1 dates[date]['articles'].append(article.id) from rdr.modules.feeds.models import ArticleStatus graph = [] for (day, dat) in dates.items(): articles_for_date = dat['articles'] del dat['articles'] if dat['count'] > 0: count = ArticleStatus.query.filter((ArticleStatus.user_id == self.user.id) & (ArticleStatus.article_id.in_(articles_for_date)) & (ArticleStatus.status == ArticleStatus.STATUS_READ)).count() dat['read'] = int(count) day = gettext(day) graph.append([day, dat['count'], dat['read']]) stats['graph'] = graph # fetch read table date from rdr.modules.feeds.models import Feed reading_stats = db.session.query(Feed, db.func.count(ArticleStatus.id).label('total')) \ .join(ArticleStatus.article) \ .join(Article.feed) \ .group_by(Feed.id) \ .filter((ArticleStatus.status == ArticleStatus.STATUS_READ) & (ArticleStatus.user_id == self.user.id) & (ArticleStatus.read_date >= start_date) & (ArticleStatus.read_date <= end_date)) \ .order_by(db.desc('total')).limit(self.tables_limit) reading_table = [] for (feed, count) in reading_stats: overall_count = Article.query.filter((Article.feed_id == feed.id) & (Article.fetched >= start_date) & (Article.fetched <= end_date)).count() if overall_count > 0: read_percent = (float(count) * 100) / overall_count if read_percent > 100: read_percent = 100 read_percent = ('%.2f' % read_percent).rstrip('0').rstrip('.') else: read_percent = '-' reading_table.append({ 'id': feed.id, 'title': feed.title, 'icon': feed.small_icon_src, 'read_count': int(count), 'all_count': int(overall_count), 'read_percent': read_percent }) stats['readingTable'] = reading_table # fetch subscriptions table data overall_stats = db.session.query(Feed, db.func.count(Article.id).label('total')) \ .join(Article.feed) \ .group_by(Feed.id) \ .filter((Article.fetched >= start_date) & (Article.fetched <= end_date) & (Article.feed_id.in_(feed_ids))) \ .order_by(db.desc('total')).limit(self.tables_limit) overall_table = [] for (feed, count) in overall_stats: read_count = ArticleStatus.query.join(ArticleStatus.article) \ .filter((ArticleStatus.user_id == self.user.id) & (ArticleStatus.status == ArticleStatus.STATUS_READ) & (Article.feed_id == feed.id) & (ArticleStatus.read_date >= start_date) & (ArticleStatus.read_date <= end_date)).count() read_percent = (float(read_count) * 100) / count if read_percent > 100: read_percent = 100 read_percent = ('%.2f' % read_percent).rstrip('0').rstrip('.') if read_percent == '0': read_percent = '-' overall_table.append({ 'id': feed.id, 'title': feed.title, 'icon': feed.small_icon_src, 'published_count': int(count), 'items_per_day': ('%.2f' % (float(count) / days_count)).rstrip('0').rstrip('.'), 'read_count': int(read_count), 'read_percent': read_percent }) stats['subscriptionsTable'] = overall_table read_count = ArticleStatus.query.filter((ArticleStatus.user_id == self.user.id) & (ArticleStatus.status == ArticleStatus.STATUS_READ)) \ .join(ArticleStatus.article) \ .filter((Article.feed_id.in_(feed_ids) & (ArticleStatus.read_date >= start_date) & (ArticleStatus.read_date <= end_date))).count() from rdr.modules.feeds.models import ArticleFavorite fav_count = ArticleFavorite.query.join(ArticleFavorite.article) \ .filter((ArticleFavorite.user_id == self.user.id) & (Article.feed_id.in_(feed_ids)) & (ArticleFavorite.fav_date >= start_date) & (ArticleFavorite.fav_date <= end_date)).count() stats['subscriptionsCount'] = len(feed_ids) stats['publishedCount'] = len(articles) stats['readCount'] = read_count stats['favCount'] = fav_count return stats