def bootstrap_feedicons(): print "Mongo DB feed_icons: %s" % MFeedIcon.objects().count() db.feed_icons.drop() print "Dropped! Mongo DB feed_icons: %s" % MFeedIcon.objects().count() print "FeedIcons: %s" % FeedIcon.objects.count() pprint(db.feed_icons.index_information()) feeds = Feed.objects.all().order_by('-average_stories_per_month') feed_count = feeds.count() i = 0 for feed in feeds: i += 1 print "%s/%s: %s" % ( i, feed_count, feed, ) sys.stdout.flush() if not MFeedIcon.objects(feed_id=feed.pk): feed_icon = FeedIcon.objects.filter(feed=feed).values() if feed_icon: try: MFeedIcon(**feed_icon[0]).save() except: print '\n\n!\n\n' continue print "\nMongo DB feed_icons: %s" % MFeedIcon.objects().count()
def bootstrap_feedicons(): print "Mongo DB feed_icons: %s" % MFeedIcon.objects().count() db.feed_icons.drop() print "Dropped! Mongo DB feed_icons: %s" % MFeedIcon.objects().count() print "FeedIcons: %s" % FeedIcon.objects.count() pprint(db.feed_icons.index_information()) feeds = Feed.objects.all().order_by('-average_stories_per_month') feed_count = feeds.count() i = 0 for feed in feeds: i += 1 print "%s/%s: %s" % (i, feed_count, feed,) sys.stdout.flush() if not MFeedIcon.objects(feed_id=feed.pk): feed_icon = FeedIcon.objects.filter(feed=feed).values() if feed_icon: try: MFeedIcon(**feed_icon[0]).save() except: print '\n\n!\n\n' continue print "\nMongo DB feed_icons: %s" % MFeedIcon.objects().count()
def load_recommended_feed(request): user = get_user(request) page = int(request.REQUEST.get('page', 0)) usersub = None refresh = request.REQUEST.get('refresh') now = datetime.datetime.now unmoderated = request.REQUEST.get('unmoderated', False) == 'true' if unmoderated: recommended_feeds = RecommendedFeed.objects.filter(is_public=False, declined_date__isnull=True)[page:page+2] else: recommended_feeds = RecommendedFeed.objects.filter(is_public=True, approved_date__lte=now)[page:page+2] if recommended_feeds and request.user.is_authenticated(): usersub = UserSubscription.objects.filter(user=user, feed=recommended_feeds[0].feed) if refresh != 'true' and page > 0: logging.user(request, "~FBBrowse recommended feed: ~SBPage #%s" % (page+1)) recommended_feed = recommended_feeds and recommended_feeds[0] feed_icon = MFeedIcon.objects(feed_id=recommended_feed.feed.pk) if recommended_feed: return render_to_response('recommendations/render_recommended_feed.xhtml', { 'recommended_feed' : recommended_feed, 'description' : recommended_feed.description or recommended_feed.feed.data.feed_tagline, 'usersub' : usersub, 'feed_icon' : feed_icon and feed_icon[0], 'has_next_page' : len(recommended_feeds) > 1, 'has_previous_page' : page != 0, 'unmoderated' : unmoderated, 'today' : datetime.datetime.now(), }, context_instance=RequestContext(request)) else: return HttpResponse("")
def render_recommended_feed(context, recommended_feeds, unmoderated=False): user = get_user(context['user']) usersub = None if context['user'].is_authenticated: usersub = UserSubscription.objects.filter( user=user, feed=recommended_feeds[0].feed) recommended_feed = recommended_feeds and recommended_feeds[0] feed_icon = MFeedIcon.objects(feed_id=recommended_feed.feed_id) if recommended_feed: return { 'recommended_feed': recommended_feed, 'description': recommended_feed.description or recommended_feed.feed.data.feed_tagline, 'usersub': usersub, 'feed_icon': feed_icon and feed_icon[0], 'user': context['user'], 'has_next_page': len(recommended_feeds) > 1, 'unmoderated': unmoderated, 'today': datetime.datetime.now(), }
def refresh_feeds(request): start = datetime.datetime.utcnow() user = get_user(request) feed_ids = request.REQUEST.getlist('feed_id') feeds = {} user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True) feed_ids = [f for f in feed_ids if f and not f.startswith('river')] if feed_ids: user_subs = user_subs.filter(feed__in=feed_ids) UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD) favicons_fetching = [int(f) for f in request.REQUEST.getlist('favicons_fetching') if f] feed_icons = dict([(i.feed_id, i) for i in MFeedIcon.objects(feed_id__in=favicons_fetching)]) for i, sub in enumerate(user_subs): pk = sub.feed.pk if (sub.needs_unread_recalc or sub.unread_count_updated < UNREAD_CUTOFF or sub.oldest_unread_story_date < UNREAD_CUTOFF): sub = sub.calculate_feed_scores(silent=True) if not sub: continue # TODO: Figure out the correct sub and give it a new feed_id feeds[pk] = { 'ps': sub.unread_count_positive, 'nt': sub.unread_count_neutral, 'ng': sub.unread_count_negative, } if sub.feed.has_feed_exception or sub.feed.has_page_exception: feeds[pk]['has_exception'] = True feeds[pk]['exception_type'] = 'feed' if sub.feed.has_feed_exception else 'page' feeds[pk]['feed_address'] = sub.feed.feed_address feeds[pk]['exception_code'] = sub.feed.exception_code if request.REQUEST.get('check_fetch_status', False): feeds[pk]['not_yet_fetched'] = not sub.feed.fetched_once if sub.feed.pk in favicons_fetching and sub.feed.pk in feed_icons: feeds[pk]['favicon'] = feed_icons[sub.feed.pk].data feeds[pk]['favicon_color'] = feed_icons[sub.feed.pk].color feeds[pk]['favicon_fetching'] = sub.feed.favicon_fetching user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True) if favicons_fetching: sub_feed_ids = [s.feed.pk for s in user_subs] moved_feed_ids = [f for f in favicons_fetching if f not in sub_feed_ids] for moved_feed_id in moved_feed_ids: duplicate_feeds = DuplicateFeed.objects.filter(duplicate_feed_id=moved_feed_id) if duplicate_feeds and duplicate_feeds[0].feed.pk in feeds: feeds[moved_feed_id] = feeds[duplicate_feeds[0].feed.pk] feeds[moved_feed_id]['dupe_feed_id'] = duplicate_feeds[0].feed.pk if settings.DEBUG or request.REQUEST.get('check_fetch_status'): diff = datetime.datetime.utcnow()-start timediff = float("%s.%.2s" % (diff.seconds, (diff.microseconds / 1000))) logging.user(request, "~FBRefreshing %s feeds (%s seconds) (%s/%s)" % (user_subs.count(), timediff, request.REQUEST.get('check_fetch_status', False), len(favicons_fetching))) return {'feeds': feeds}
def load_feed_favicons(request): user = get_user(request) feed_ids = request.REQUEST.getlist('feed_ids') user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True) if feed_ids and len(feed_ids) > 0: user_subs = user_subs.filter(feed__in=feed_ids) feed_ids = [sub['feed__pk'] for sub in user_subs.values('feed__pk')] feed_icons = dict([(i.feed_id, i.data) for i in MFeedIcon.objects(feed_id__in=feed_ids)]) return feed_icons
def load_recommended_feed(request): user = get_user(request) page = max(int(request.REQUEST.get('page', 0)), 0) usersub = None refresh = request.REQUEST.get('refresh') now = datetime.datetime.now unmoderated = request.REQUEST.get('unmoderated', False) == 'true' if unmoderated: recommended_feeds = RecommendedFeed.objects.filter( is_public=False, declined_date__isnull=True)[page:page + 2] else: recommended_feeds = RecommendedFeed.objects.filter( is_public=True, approved_date__lte=now)[page:page + 2] if recommended_feeds and request.user.is_authenticated(): usersub = UserSubscription.objects.filter( user=user, feed=recommended_feeds[0].feed) if refresh != 'true' and page > 0: logging.user(request, "~FBBrowse recommended feed: ~SBPage #%s" % (page + 1)) recommended_feed = recommended_feeds and recommended_feeds[0] if not recommended_feeds: return HttpResponse("") feed_icon = MFeedIcon.objects(feed_id=recommended_feed.feed_id) if recommended_feed: return render_to_response( 'recommendations/render_recommended_feed.xhtml', { 'recommended_feed': recommended_feed, 'description': recommended_feed.description or recommended_feed.feed.data.feed_tagline, 'usersub': usersub, 'feed_icon': feed_icon and feed_icon[0], 'has_next_page': len(recommended_feeds) > 1, 'has_previous_page': page != 0, 'unmoderated': unmoderated, 'today': datetime.datetime.now(), 'page': page, }, context_instance=RequestContext(request)) else: return HttpResponse("")
def render_recommended_feed(context, recommended_feeds): user = get_user(context['user']) usersub = None if context['user'].is_authenticated(): usersub = UserSubscription.objects.filter(user=user, feed=recommended_feeds[0].feed) recommended_feed = recommended_feeds and recommended_feeds[0] feed_icon = MFeedIcon.objects(feed_id=recommended_feed.feed.pk) if recommended_feed: return { 'recommended_feed': recommended_feed, 'description': recommended_feed.description or recommended_feed.feed.data.feed_tagline, 'usersub': usersub, 'feed_icon': feed_icon and feed_icon[0], 'user': context['user'], 'has_next_page': len(recommended_feeds) > 1 }
def refresh_feeds(request): user = get_user(request) feed_ids = request.REQUEST.getlist('feed_id') check_fetch_status = request.REQUEST.get('check_fetch_status') favicons_fetching = request.REQUEST.getlist('favicons_fetching') start = datetime.datetime.utcnow() feeds = UserSubscription.feeds_with_updated_counts(user, feed_ids=feed_ids, check_fetch_status=check_fetch_status) favicons_fetching = [int(f) for f in favicons_fetching if f] feed_icons = dict([(i.feed_id, i) for i in MFeedIcon.objects(feed_id__in=favicons_fetching)]) for feed_id, feed in feeds.items(): if feed_id in favicons_fetching and feed_id in feed_icons: feeds[feed_id]['favicon'] = feed_icons[feed_id].data feeds[feed_id]['favicon_color'] = feed_icons[feed_id].color feeds[feed_id]['favicon_fetching'] = feed.get('favicon_fetching') user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True) sub_feed_ids = [s.feed_id for s in user_subs] if favicons_fetching: moved_feed_ids = [f for f in favicons_fetching if f not in sub_feed_ids] for moved_feed_id in moved_feed_ids: duplicate_feeds = DuplicateFeed.objects.filter(duplicate_feed_id=moved_feed_id) if duplicate_feeds and duplicate_feeds[0].feed.pk in feeds: feeds[moved_feed_id] = feeds[duplicate_feeds[0].feed.pk] feeds[moved_feed_id]['dupe_feed_id'] = duplicate_feeds[0].feed.pk if check_fetch_status: missing_feed_ids = list(set(feed_ids) - set(sub_feed_ids)) if missing_feed_ids: duplicate_feeds = DuplicateFeed.objects.filter(duplicate_feed_id__in=missing_feed_ids) for duplicate_feed in duplicate_feeds: feeds[duplicate_feed.duplicate_feed_id] = {'id': duplicate_feed.feed.pk} if settings.DEBUG or check_fetch_status: diff = datetime.datetime.utcnow()-start timediff = float("%s.%.2s" % (diff.seconds, (diff.microseconds / 1000))) logging.user(request, "~FBRefreshing %s feeds (%s seconds) (%s/%s)" % ( len(feeds.keys()), timediff, check_fetch_status, len(favicons_fetching))) return {'feeds': feeds}
def refresh_feeds(request): start = datetime.datetime.utcnow() user = get_user(request) feed_ids = request.REQUEST.getlist('feed_id') feeds = {} user_subs = UserSubscription.objects.select_related('feed').filter(user=user, active=True) if feed_ids: user_subs = user_subs.filter(feed__in=feed_ids) UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD) favicons_fetching = [int(f) for f in request.REQUEST.getlist('favicons_fetching') if f] feed_icons = dict([(i.feed_id, i) for i in MFeedIcon.objects(feed_id__in=favicons_fetching)]) for sub in user_subs: if (sub.needs_unread_recalc or sub.unread_count_updated < UNREAD_CUTOFF or sub.oldest_unread_story_date < UNREAD_CUTOFF): sub.calculate_feed_scores(silent=True) feeds[sub.feed.pk] = { 'ps': sub.unread_count_positive, 'nt': sub.unread_count_neutral, 'ng': sub.unread_count_negative, } if sub.feed.has_feed_exception or sub.feed.has_page_exception: feeds[sub.feed.pk]['has_exception'] = True feeds[sub.feed.pk]['exception_type'] = 'feed' if sub.feed.has_feed_exception else 'page' feeds[sub.feed.pk]['feed_address'] = sub.feed.feed_address feeds[sub.feed.pk]['exception_code'] = sub.feed.exception_code if request.REQUEST.get('check_fetch_status', False): feeds[sub.feed.pk]['not_yet_fetched'] = not sub.feed.fetched_once if sub.feed.pk in favicons_fetching and sub.feed.pk in feed_icons: feeds[sub.feed.pk]['favicon'] = feed_icons[sub.feed.pk].data feeds[sub.feed.pk]['favicon_color'] = feed_icons[sub.feed.pk].color feeds[sub.feed.pk]['favicon_fetching'] = bool(not (feed_icons[sub.feed.pk].not_found or feed_icons[sub.feed.pk].data)) if settings.DEBUG: diff = datetime.datetime.utcnow()-start timediff = float("%s.%.2s" % (diff.seconds, (diff.microseconds / 1000))) logging.user(request.user, "~FBRefreshing %s feeds (%s seconds)" % (user_subs.count(), timediff)) return {'feeds': feeds}
def load_recommended_feed(request): user = get_user(request) page = max(int(request.REQUEST.get("page", 0)), 0) usersub = None refresh = request.REQUEST.get("refresh") now = datetime.datetime.now unmoderated = request.REQUEST.get("unmoderated", False) == "true" if unmoderated: recommended_feeds = RecommendedFeed.objects.filter(is_public=False, declined_date__isnull=True)[page : page + 2] else: recommended_feeds = RecommendedFeed.objects.filter(is_public=True, approved_date__lte=now)[page : page + 2] if recommended_feeds and request.user.is_authenticated(): usersub = UserSubscription.objects.filter(user=user, feed=recommended_feeds[0].feed) if refresh != "true" and page > 0: logging.user(request, "~FBBrowse recommended feed: ~SBPage #%s" % (page + 1)) recommended_feed = recommended_feeds and recommended_feeds[0] if not recommended_feeds: return HttpResponse("") feed_icon = MFeedIcon.objects(feed_id=recommended_feed.feed_id) if recommended_feed: return render_to_response( "recommendations/render_recommended_feed.xhtml", { "recommended_feed": recommended_feed, "description": recommended_feed.description or recommended_feed.feed.data.feed_tagline, "usersub": usersub, "feed_icon": feed_icon and feed_icon[0], "has_next_page": len(recommended_feeds) > 1, "has_previous_page": page != 0, "unmoderated": unmoderated, "today": datetime.datetime.now(), "page": page, }, context_instance=RequestContext(request), ) else: return HttpResponse("")
def __init__(self, feed, page_data=None, force=False): self.feed = feed self.force = force self.page_data = page_data self.feed_icon = MFeedIcon.get_feed(feed_id=self.feed.pk)