def fetch_packages(self):
package_ids = None
key = 'popular_packages.' + str(self.user.id)
if self.use_cache:
try:
package_ids = cache.get(key)
except Exception as e:
app.logger.exception(e)
app.logger.info("An exception catched, disable cache")
self.use_cache = False
if package_ids is None:
packages = db.session.query(Subscribe.feed_id, db.func.count(Subscribe.id).label('total')) \
.filter(Subscribe.type == Subscribe.TYPE_FEED) \
.filter(Subscribe.feed_id.notin_(db.session.query(Subscribe.feed_id).filter(
(Subscribe.user_id == self.user.id) & (Subscribe.feed_id.isnot(None))
).group_by(Subscribe.feed_id))) \
.group_by(Subscribe.feed_id) \
.order_by(db.desc('total')) \
.limit(500)
package_ids = []
for (feed_id, _count) in packages:
package_ids.append(feed_id)
if self.random_order:
import random
random.shuffle(package_ids)
package_ids = package_ids[:self.packages_count]
if self.use_cache:
cache.set(key, package_ids, timeout=self.timeout)
feeds = Feed.query.filter(Feed.id.in_(package_ids))
return feeds
def recalculate_categories():
from rdr.modules.subscribes.models import Subscribe
from rdr.modules.feeds.models import Feed
from rdr.tasks.schedule.feeds import categories
from rdr.application.database import db
from math import ceil
feed_ids = db.session.query(Subscribe.feed_id, db.func.count(Subscribe.id).label('total')) \
.filter(Subscribe.type == Subscribe.TYPE_FEED) \
.group_by(Subscribe.feed_id) \
.order_by(db.desc('total'))
feed_ids = [x[0] for x in feed_ids]
count = len(feed_ids)
offset = 0
results = []
for num in xrange(0, len(categories)):
category = categories[num][0]
percent = categories[num][1]
if offset < count:
if num != len(categories) - 1:
percent_count = int(ceil(float(percent) / 100 * count))
results.append((category, feed_ids[offset:offset+percent_count]))
offset += percent_count
else:
results.append((category, feed_ids[offset:]))
for res in results:
if res[1]:
q = db.update(Feed).values({'category': res[0]}) \
.where(Feed.id.in_(res[1]))
db.session.execute(q)
db.session.commit()
def generate(self):
date_from = self._get_date_from()
if date_from is None:
return []
stats = db.session.query(ArticleStatus.article_id, db.func.count(ArticleStatus.id).label('total')) \
.join(ArticleStatus.article) \
.join(Subscribe, ((Subscribe.type == Subscribe.TYPE_FEED) & (Subscribe.feed_id == Article.feed_id))) \
.group_by(ArticleStatus.article_id).filter(
(ArticleStatus.status == ArticleStatus.STATUS_READ) &
(Subscribe.user_id == self.user.id) &
(Article.fetched >= date_from)
).order_by(db.desc('total')).limit(self.articles_count)
article_ids = []
for (article_id, _count) in stats:
article_ids.append(article_id)
article_ids = list(unique(article_ids))
return article_ids
def generate_stats(self):
feed_ids = self._get_feed_ids()
from rdr.modules.feeds.models import Article
articles = Article.query.filter((Article.feed_id.in_(feed_ids)))
if self.dates == StatsGenerator.DATES_LAST_WEEK:
days_count = 7
elif self.dates == StatsGenerator.DATES_LAST_MONTH:
days_count = 30
else:
raise Exception('Unknown date format')
end_date = datetime.now()
start_date = end_date - timedelta(days=days_count)
articles = articles.filter((Article.published >= start_date) & (Article.published <= end_date))
stats = {}
dates = OrderedDict()
days_of_week = self._get_days_of_week()
for day in days_of_week:
dates[day] = {
'count': 0,
'read': 0,
'articles': []
}
articles = articles.all()
for article in articles:
if article.published:
date = article.published.strftime('%a')
if date in dates:
dates[date]['count'] += 1
dates[date]['articles'].append(article.id)
from rdr.modules.feeds.models import ArticleStatus
graph = []
for (day, dat) in dates.items():
articles_for_date = dat['articles']
del dat['articles']
if dat['count'] > 0:
count = ArticleStatus.query.filter((ArticleStatus.user_id == self.user.id) &
(ArticleStatus.article_id.in_(articles_for_date)) &
(ArticleStatus.status == ArticleStatus.STATUS_READ)).count()
dat['read'] = int(count)
day = gettext(day)
graph.append([day, dat['count'], dat['read']])
stats['graph'] = graph
# fetch read table date
from rdr.modules.feeds.models import Feed
reading_stats = db.session.query(Feed, db.func.count(ArticleStatus.id).label('total')) \
.join(ArticleStatus.article) \
.join(Article.feed) \
.group_by(Feed.id) \
.filter((ArticleStatus.status == ArticleStatus.STATUS_READ) &
(ArticleStatus.user_id == self.user.id) &
(ArticleStatus.read_date >= start_date) &
(ArticleStatus.read_date <= end_date)) \
.order_by(db.desc('total')).limit(self.tables_limit)
reading_table = []
for (feed, count) in reading_stats:
overall_count = Article.query.filter((Article.feed_id == feed.id) &
(Article.fetched >= start_date) &
(Article.fetched <= end_date)).count()
if overall_count > 0:
read_percent = (float(count) * 100) / overall_count
if read_percent > 100:
read_percent = 100
read_percent = ('%.2f' % read_percent).rstrip('0').rstrip('.')
else:
read_percent = '-'
reading_table.append({
'id': feed.id,
'title': feed.title,
'icon': feed.small_icon_src,
'read_count': int(count),
'all_count': int(overall_count),
'read_percent': read_percent
})
stats['readingTable'] = reading_table
# fetch subscriptions table data
overall_stats = db.session.query(Feed, db.func.count(Article.id).label('total')) \
.join(Article.feed) \
.group_by(Feed.id) \
.filter((Article.fetched >= start_date) &
(Article.fetched <= end_date) &
(Article.feed_id.in_(feed_ids))) \
.order_by(db.desc('total')).limit(self.tables_limit)
overall_table = []
for (feed, count) in overall_stats:
read_count = ArticleStatus.query.join(ArticleStatus.article) \
.filter((ArticleStatus.user_id == self.user.id) &
(ArticleStatus.status == ArticleStatus.STATUS_READ) &
(Article.feed_id == feed.id) &
(ArticleStatus.read_date >= start_date) &
(ArticleStatus.read_date <= end_date)).count()
read_percent = (float(read_count) * 100) / count
if read_percent > 100:
read_percent = 100
read_percent = ('%.2f' % read_percent).rstrip('0').rstrip('.')
if read_percent == '0':
read_percent = '-'
overall_table.append({
'id': feed.id,
'title': feed.title,
'icon': feed.small_icon_src,
'published_count': int(count),
'items_per_day': ('%.2f' % (float(count) / days_count)).rstrip('0').rstrip('.'),
'read_count': int(read_count),
'read_percent': read_percent
})
stats['subscriptionsTable'] = overall_table
read_count = ArticleStatus.query.filter((ArticleStatus.user_id == self.user.id) &
(ArticleStatus.status == ArticleStatus.STATUS_READ)) \
.join(ArticleStatus.article) \
.filter((Article.feed_id.in_(feed_ids) &
(ArticleStatus.read_date >= start_date) &
(ArticleStatus.read_date <= end_date))).count()
from rdr.modules.feeds.models import ArticleFavorite
fav_count = ArticleFavorite.query.join(ArticleFavorite.article) \
.filter((ArticleFavorite.user_id == self.user.id) &
(Article.feed_id.in_(feed_ids)) &
(ArticleFavorite.fav_date >= start_date) &
(ArticleFavorite.fav_date <= end_date)).count()
stats['subscriptionsCount'] = len(feed_ids)
stats['publishedCount'] = len(articles)
stats['readCount'] = read_count
stats['favCount'] = fav_count
return stats
