def bootstrap_feedicons():
print "Mongo DB feed_icons: %s" % MFeedIcon.objects().count()
db.feed_icons.drop()
print "Dropped! Mongo DB feed_icons: %s" % MFeedIcon.objects().count()
print "FeedIcons: %s" % FeedIcon.objects.count()
pprint(db.feed_icons.index_information())
feeds = Feed.objects.all().order_by('-average_stories_per_month')
feed_count = feeds.count()
i = 0
for feed in feeds:
i += 1
print "%s/%s: %s" % (
i,
feed_count,
feed,
)
sys.stdout.flush()
if not MFeedIcon.objects(feed_id=feed.pk):
feed_icon = FeedIcon.objects.filter(feed=feed).values()
if feed_icon:
try:
MFeedIcon(**feed_icon[0]).save()
except:
print '\n\n!\n\n'
continue
print "\nMongo DB feed_icons: %s" % MFeedIcon.objects().count()
def bootstrap_feedicons():
print "Mongo DB feed_icons: %s" % MFeedIcon.objects().count()
db.feed_icons.drop()
print "Dropped! Mongo DB feed_icons: %s" % MFeedIcon.objects().count()
print "FeedIcons: %s" % FeedIcon.objects.count()
pprint(db.feed_icons.index_information())
feeds = Feed.objects.all().order_by('-average_stories_per_month')
feed_count = feeds.count()
i = 0
for feed in feeds:
i += 1
print "%s/%s: %s" % (i, feed_count, feed,)
sys.stdout.flush()
if not MFeedIcon.objects(feed_id=feed.pk):
feed_icon = FeedIcon.objects.filter(feed=feed).values()
if feed_icon:
try:
MFeedIcon(**feed_icon[0]).save()
except:
print '\n\n!\n\n'
continue
print "\nMongo DB feed_icons: %s" % MFeedIcon.objects().count()
def load_recommended_feed(request):
user = get_user(request)
page = int(request.REQUEST.get('page', 0))
usersub = None
refresh = request.REQUEST.get('refresh')
now = datetime.datetime.now
unmoderated = request.REQUEST.get('unmoderated', False) == 'true'
if unmoderated:
recommended_feeds = RecommendedFeed.objects.filter(is_public=False, declined_date__isnull=True)[page:page+2]
else:
recommended_feeds = RecommendedFeed.objects.filter(is_public=True, approved_date__lte=now)[page:page+2]
if recommended_feeds and request.user.is_authenticated():
usersub = UserSubscription.objects.filter(user=user, feed=recommended_feeds[0].feed)
if refresh != 'true' and page > 0:
logging.user(request, "~FBBrowse recommended feed: ~SBPage #%s" % (page+1))
recommended_feed = recommended_feeds and recommended_feeds[0]
feed_icon = MFeedIcon.objects(feed_id=recommended_feed.feed.pk)
if recommended_feed:
return render_to_response('recommendations/render_recommended_feed.xhtml', {
'recommended_feed' : recommended_feed,
'description' : recommended_feed.description or recommended_feed.feed.data.feed_tagline,
'usersub' : usersub,
'feed_icon' : feed_icon and feed_icon[0],
'has_next_page' : len(recommended_feeds) > 1,
'has_previous_page' : page != 0,
'unmoderated' : unmoderated,
'today' : datetime.datetime.now(),
}, context_instance=RequestContext(request))
else:
return HttpResponse("")
def render_recommended_feed(context, recommended_feeds, unmoderated=False):
user = get_user(context['user'])
usersub = None
if context['user'].is_authenticated:
usersub = UserSubscription.objects.filter(
user=user, feed=recommended_feeds[0].feed)
recommended_feed = recommended_feeds and recommended_feeds[0]
feed_icon = MFeedIcon.objects(feed_id=recommended_feed.feed_id)
if recommended_feed:
return {
'recommended_feed':
recommended_feed,
'description':
recommended_feed.description
or recommended_feed.feed.data.feed_tagline,
'usersub':
usersub,
'feed_icon':
feed_icon and feed_icon[0],
'user':
context['user'],
'has_next_page':
len(recommended_feeds) > 1,
'unmoderated':
unmoderated,
'today':
datetime.datetime.now(),
}
def refresh_feeds(request):
start = datetime.datetime.utcnow()
user = get_user(request)
feed_ids = request.REQUEST.getlist('feed_id')
feeds = {}
user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True)
feed_ids = [f for f in feed_ids if f and not f.startswith('river')]
if feed_ids:
user_subs = user_subs.filter(feed__in=feed_ids)
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
favicons_fetching = [int(f) for f in request.REQUEST.getlist('favicons_fetching') if f]
feed_icons = dict([(i.feed_id, i) for i in MFeedIcon.objects(feed_id__in=favicons_fetching)])
for i, sub in enumerate(user_subs):
pk = sub.feed.pk
if (sub.needs_unread_recalc or
sub.unread_count_updated < UNREAD_CUTOFF or
sub.oldest_unread_story_date < UNREAD_CUTOFF):
sub = sub.calculate_feed_scores(silent=True)
if not sub: continue # TODO: Figure out the correct sub and give it a new feed_id
feeds[pk] = {
'ps': sub.unread_count_positive,
'nt': sub.unread_count_neutral,
'ng': sub.unread_count_negative,
}
if sub.feed.has_feed_exception or sub.feed.has_page_exception:
feeds[pk]['has_exception'] = True
feeds[pk]['exception_type'] = 'feed' if sub.feed.has_feed_exception else 'page'
feeds[pk]['feed_address'] = sub.feed.feed_address
feeds[pk]['exception_code'] = sub.feed.exception_code
if request.REQUEST.get('check_fetch_status', False):
feeds[pk]['not_yet_fetched'] = not sub.feed.fetched_once
if sub.feed.pk in favicons_fetching and sub.feed.pk in feed_icons:
feeds[pk]['favicon'] = feed_icons[sub.feed.pk].data
feeds[pk]['favicon_color'] = feed_icons[sub.feed.pk].color
feeds[pk]['favicon_fetching'] = sub.feed.favicon_fetching
user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True)
if favicons_fetching:
sub_feed_ids = [s.feed.pk for s in user_subs]
moved_feed_ids = [f for f in favicons_fetching if f not in sub_feed_ids]
for moved_feed_id in moved_feed_ids:
duplicate_feeds = DuplicateFeed.objects.filter(duplicate_feed_id=moved_feed_id)
if duplicate_feeds and duplicate_feeds[0].feed.pk in feeds:
feeds[moved_feed_id] = feeds[duplicate_feeds[0].feed.pk]
feeds[moved_feed_id]['dupe_feed_id'] = duplicate_feeds[0].feed.pk
if settings.DEBUG or request.REQUEST.get('check_fetch_status'):
diff = datetime.datetime.utcnow()-start
timediff = float("%s.%.2s" % (diff.seconds, (diff.microseconds / 1000)))
logging.user(request, "~FBRefreshing %s feeds (%s seconds) (%s/%s)" % (user_subs.count(), timediff, request.REQUEST.get('check_fetch_status', False), len(favicons_fetching)))
return {'feeds': feeds}
def refresh_feeds(request):
start = datetime.datetime.utcnow()
user = get_user(request)
feed_ids = request.REQUEST.getlist('feed_id')
feeds = {}
user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True)
feed_ids = [f for f in feed_ids if f and not f.startswith('river')]
if feed_ids:
user_subs = user_subs.filter(feed__in=feed_ids)
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
favicons_fetching = [int(f) for f in request.REQUEST.getlist('favicons_fetching') if f]
feed_icons = dict([(i.feed_id, i) for i in MFeedIcon.objects(feed_id__in=favicons_fetching)])
for i, sub in enumerate(user_subs):
pk = sub.feed.pk
if (sub.needs_unread_recalc or
sub.unread_count_updated < UNREAD_CUTOFF or
sub.oldest_unread_story_date < UNREAD_CUTOFF):
sub = sub.calculate_feed_scores(silent=True)
if not sub: continue # TODO: Figure out the correct sub and give it a new feed_id
feeds[pk] = {
'ps': sub.unread_count_positive,
'nt': sub.unread_count_neutral,
'ng': sub.unread_count_negative,
}
if sub.feed.has_feed_exception or sub.feed.has_page_exception:
feeds[pk]['has_exception'] = True
feeds[pk]['exception_type'] = 'feed' if sub.feed.has_feed_exception else 'page'
feeds[pk]['feed_address'] = sub.feed.feed_address
feeds[pk]['exception_code'] = sub.feed.exception_code
if request.REQUEST.get('check_fetch_status', False):
feeds[pk]['not_yet_fetched'] = not sub.feed.fetched_once
if sub.feed.pk in favicons_fetching and sub.feed.pk in feed_icons:
feeds[pk]['favicon'] = feed_icons[sub.feed.pk].data
feeds[pk]['favicon_color'] = feed_icons[sub.feed.pk].color
feeds[pk]['favicon_fetching'] = sub.feed.favicon_fetching
user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True)
if favicons_fetching:
sub_feed_ids = [s.feed.pk for s in user_subs]
moved_feed_ids = [f for f in favicons_fetching if f not in sub_feed_ids]
for moved_feed_id in moved_feed_ids:
duplicate_feeds = DuplicateFeed.objects.filter(duplicate_feed_id=moved_feed_id)
if duplicate_feeds and duplicate_feeds[0].feed.pk in feeds:
feeds[moved_feed_id] = feeds[duplicate_feeds[0].feed.pk]
feeds[moved_feed_id]['dupe_feed_id'] = duplicate_feeds[0].feed.pk
if settings.DEBUG or request.REQUEST.get('check_fetch_status'):
diff = datetime.datetime.utcnow()-start
timediff = float("%s.%.2s" % (diff.seconds, (diff.microseconds / 1000)))
logging.user(request, "~FBRefreshing %s feeds (%s seconds) (%s/%s)" % (user_subs.count(), timediff, request.REQUEST.get('check_fetch_status', False), len(favicons_fetching)))
return {'feeds': feeds}
def load_feed_favicons(request):
user = get_user(request)
feed_ids = request.REQUEST.getlist('feed_ids')
user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True)
if feed_ids and len(feed_ids) > 0:
user_subs = user_subs.filter(feed__in=feed_ids)
feed_ids = [sub['feed__pk'] for sub in user_subs.values('feed__pk')]
feed_icons = dict([(i.feed_id, i.data) for i in MFeedIcon.objects(feed_id__in=feed_ids)])
return feed_icons
def load_feed_favicons(request):
user = get_user(request)
feed_ids = request.REQUEST.getlist('feed_ids')
user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True)
if feed_ids and len(feed_ids) > 0:
user_subs = user_subs.filter(feed__in=feed_ids)
feed_ids = [sub['feed__pk'] for sub in user_subs.values('feed__pk')]
feed_icons = dict([(i.feed_id, i.data) for i in MFeedIcon.objects(feed_id__in=feed_ids)])
return feed_icons
def load_recommended_feed(request):
user = get_user(request)
page = max(int(request.REQUEST.get('page', 0)), 0)
usersub = None
refresh = request.REQUEST.get('refresh')
now = datetime.datetime.now
unmoderated = request.REQUEST.get('unmoderated', False) == 'true'
if unmoderated:
recommended_feeds = RecommendedFeed.objects.filter(
is_public=False, declined_date__isnull=True)[page:page + 2]
else:
recommended_feeds = RecommendedFeed.objects.filter(
is_public=True, approved_date__lte=now)[page:page + 2]
if recommended_feeds and request.user.is_authenticated():
usersub = UserSubscription.objects.filter(
user=user, feed=recommended_feeds[0].feed)
if refresh != 'true' and page > 0:
logging.user(request,
"~FBBrowse recommended feed: ~SBPage #%s" % (page + 1))
recommended_feed = recommended_feeds and recommended_feeds[0]
if not recommended_feeds:
return HttpResponse("")
feed_icon = MFeedIcon.objects(feed_id=recommended_feed.feed_id)
if recommended_feed:
return render_to_response(
'recommendations/render_recommended_feed.xhtml', {
'recommended_feed':
recommended_feed,
'description':
recommended_feed.description
or recommended_feed.feed.data.feed_tagline,
'usersub':
usersub,
'feed_icon':
feed_icon and feed_icon[0],
'has_next_page':
len(recommended_feeds) > 1,
'has_previous_page':
page != 0,
'unmoderated':
unmoderated,
'today':
datetime.datetime.now(),
'page':
page,
},
context_instance=RequestContext(request))
else:
return HttpResponse("")
def render_recommended_feed(context, recommended_feeds):
user = get_user(context['user'])
usersub = None
if context['user'].is_authenticated():
usersub = UserSubscription.objects.filter(user=user, feed=recommended_feeds[0].feed)
recommended_feed = recommended_feeds and recommended_feeds[0]
feed_icon = MFeedIcon.objects(feed_id=recommended_feed.feed.pk)
if recommended_feed:
return {
'recommended_feed': recommended_feed,
'description': recommended_feed.description or recommended_feed.feed.data.feed_tagline,
'usersub': usersub,
'feed_icon': feed_icon and feed_icon[0],
'user': context['user'],
'has_next_page': len(recommended_feeds) > 1
}
def refresh_feeds(request):
user = get_user(request)
feed_ids = request.REQUEST.getlist('feed_id')
check_fetch_status = request.REQUEST.get('check_fetch_status')
favicons_fetching = request.REQUEST.getlist('favicons_fetching')
start = datetime.datetime.utcnow()
feeds = UserSubscription.feeds_with_updated_counts(user, feed_ids=feed_ids,
check_fetch_status=check_fetch_status)
favicons_fetching = [int(f) for f in favicons_fetching if f]
feed_icons = dict([(i.feed_id, i) for i in MFeedIcon.objects(feed_id__in=favicons_fetching)])
for feed_id, feed in feeds.items():
if feed_id in favicons_fetching and feed_id in feed_icons:
feeds[feed_id]['favicon'] = feed_icons[feed_id].data
feeds[feed_id]['favicon_color'] = feed_icons[feed_id].color
feeds[feed_id]['favicon_fetching'] = feed.get('favicon_fetching')
user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True)
sub_feed_ids = [s.feed_id for s in user_subs]
if favicons_fetching:
moved_feed_ids = [f for f in favicons_fetching if f not in sub_feed_ids]
for moved_feed_id in moved_feed_ids:
duplicate_feeds = DuplicateFeed.objects.filter(duplicate_feed_id=moved_feed_id)
if duplicate_feeds and duplicate_feeds[0].feed.pk in feeds:
feeds[moved_feed_id] = feeds[duplicate_feeds[0].feed.pk]
feeds[moved_feed_id]['dupe_feed_id'] = duplicate_feeds[0].feed.pk
if check_fetch_status:
missing_feed_ids = list(set(feed_ids) - set(sub_feed_ids))
if missing_feed_ids:
duplicate_feeds = DuplicateFeed.objects.filter(duplicate_feed_id__in=missing_feed_ids)
for duplicate_feed in duplicate_feeds:
feeds[duplicate_feed.duplicate_feed_id] = {'id': duplicate_feed.feed.pk}
if settings.DEBUG or check_fetch_status:
diff = datetime.datetime.utcnow()-start
timediff = float("%s.%.2s" % (diff.seconds, (diff.microseconds / 1000)))
logging.user(request, "~FBRefreshing %s feeds (%s seconds) (%s/%s)" % (
len(feeds.keys()), timediff, check_fetch_status, len(favicons_fetching)))
return {'feeds': feeds}
def refresh_feeds(request):
start = datetime.datetime.utcnow()
user = get_user(request)
feed_ids = request.REQUEST.getlist('feed_id')
feeds = {}
user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True)
if feed_ids:
user_subs = user_subs.filter(feed__in=feed_ids)
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
favicons_fetching = [int(f) for f in request.REQUEST.getlist('favicons_fetching') if f]
feed_icons = dict([(i.feed_id, i) for i in MFeedIcon.objects(feed_id__in=favicons_fetching)])
for sub in user_subs:
if (sub.needs_unread_recalc or
sub.unread_count_updated < UNREAD_CUTOFF or
sub.oldest_unread_story_date < UNREAD_CUTOFF):
sub.calculate_feed_scores(silent=True)
feeds[sub.feed.pk] = {
'ps': sub.unread_count_positive,
'nt': sub.unread_count_neutral,
'ng': sub.unread_count_negative,
}
if sub.feed.has_feed_exception or sub.feed.has_page_exception:
feeds[sub.feed.pk]['has_exception'] = True
feeds[sub.feed.pk]['exception_type'] = 'feed' if sub.feed.has_feed_exception else 'page'
feeds[sub.feed.pk]['feed_address'] = sub.feed.feed_address
feeds[sub.feed.pk]['exception_code'] = sub.feed.exception_code
if request.REQUEST.get('check_fetch_status', False):
feeds[sub.feed.pk]['not_yet_fetched'] = not sub.feed.fetched_once
if sub.feed.pk in favicons_fetching and sub.feed.pk in feed_icons:
feeds[sub.feed.pk]['favicon'] = feed_icons[sub.feed.pk].data
feeds[sub.feed.pk]['favicon_color'] = feed_icons[sub.feed.pk].color
feeds[sub.feed.pk]['favicon_fetching'] = bool(not (feed_icons[sub.feed.pk].not_found or
feed_icons[sub.feed.pk].data))
if settings.DEBUG:
diff = datetime.datetime.utcnow()-start
timediff = float("%s.%.2s" % (diff.seconds, (diff.microseconds / 1000)))
logging.user(request.user, "~FBRefreshing %s feeds (%s seconds)" % (user_subs.count(), timediff))
return {'feeds': feeds}
def load_recommended_feed(request):
user = get_user(request)
page = max(int(request.REQUEST.get("page", 0)), 0)
usersub = None
refresh = request.REQUEST.get("refresh")
now = datetime.datetime.now
unmoderated = request.REQUEST.get("unmoderated", False) == "true"
if unmoderated:
recommended_feeds = RecommendedFeed.objects.filter(is_public=False, declined_date__isnull=True)[page : page + 2]
else:
recommended_feeds = RecommendedFeed.objects.filter(is_public=True, approved_date__lte=now)[page : page + 2]
if recommended_feeds and request.user.is_authenticated():
usersub = UserSubscription.objects.filter(user=user, feed=recommended_feeds[0].feed)
if refresh != "true" and page > 0:
logging.user(request, "~FBBrowse recommended feed: ~SBPage #%s" % (page + 1))
recommended_feed = recommended_feeds and recommended_feeds[0]
if not recommended_feeds:
return HttpResponse("")
feed_icon = MFeedIcon.objects(feed_id=recommended_feed.feed_id)
if recommended_feed:
return render_to_response(
"recommendations/render_recommended_feed.xhtml",
{
"recommended_feed": recommended_feed,
"description": recommended_feed.description or recommended_feed.feed.data.feed_tagline,
"usersub": usersub,
"feed_icon": feed_icon and feed_icon[0],
"has_next_page": len(recommended_feeds) > 1,
"has_previous_page": page != 0,
"unmoderated": unmoderated,
"today": datetime.datetime.now(),
"page": page,
},
context_instance=RequestContext(request),
)
else:
return HttpResponse("")
def __init__(self, feed, page_data=None, force=False):
self.feed = feed
self.force = force
self.page_data = page_data
self.feed_icon = MFeedIcon.get_feed(feed_id=self.feed.pk)
def __init__(self, feed, page_data=None, force=False):
self.feed = feed
self.force = force
self.page_data = page_data
self.feed_icon = MFeedIcon.get_feed(feed_id=self.feed.pk)
