def load_feed_statistics(request, feed_id): user = get_user(request) stats = dict() feed = get_object_or_404(Feed, pk=feed_id) feed.update_all_statistics() feed.set_next_scheduled_update(verbose=True, skip_scheduling=True) feed.save_feed_story_history_statistics() feed.save_classifier_counts() # Dates of last and next update stats['active'] = feed.active stats['last_update'] = relative_timesince(feed.last_update) stats['next_update'] = relative_timeuntil(feed.next_scheduled_update) stats['push'] = feed.is_push # Minutes between updates update_interval_minutes = feed.get_next_scheduled_update(force=True, verbose=False) stats['update_interval_minutes'] = update_interval_minutes original_active_premium_subscribers = feed.active_premium_subscribers original_premium_subscribers = feed.premium_subscribers feed.active_premium_subscribers = max(feed.active_premium_subscribers + 1, 1) feed.premium_subscribers += 1 premium_update_interval_minutes = feed.get_next_scheduled_update( force=True, verbose=False) feed.active_premium_subscribers = original_active_premium_subscribers feed.premium_subscribers = original_premium_subscribers stats['premium_update_interval_minutes'] = premium_update_interval_minutes stats['errors_since_good'] = feed.errors_since_good # Stories per month - average and month-by-month breakout average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history stats['average_stories_per_month'] = average_stories_per_month stats['story_count_history'] = story_count_history and json.decode( story_count_history) # Subscribers stats['subscriber_count'] = feed.num_subscribers stats['stories_last_month'] = feed.stories_last_month stats['last_load_time'] = feed.last_load_time stats['premium_subscribers'] = feed.premium_subscribers stats['active_subscribers'] = feed.active_subscribers stats['active_premium_subscribers'] = feed.active_premium_subscribers # Classifier counts stats['classifier_counts'] = json.decode(feed.data.feed_classifier_counts) # Fetch histories timezone = user.profile.timezone fetch_history = MFetchHistory.feed(feed_id, timezone=timezone) stats['feed_fetch_history'] = fetch_history['feed_fetch_history'] stats['page_fetch_history'] = fetch_history['page_fetch_history'] stats['feed_push_history'] = fetch_history['push_history'] logging.user(request, "~FBStatistics: ~SB%s" % (feed)) return stats
def load_feed_statistics(request, feed_id): user = get_user(request) stats = dict() feed = get_object_or_404(Feed, pk=feed_id) feed.update_all_statistics() feed.set_next_scheduled_update(verbose=True, skip_scheduling=True) feed.save_feed_story_history_statistics() feed.save_classifier_counts() # Dates of last and next update stats['active'] = feed.active stats['last_update'] = relative_timesince(feed.last_update) stats['next_update'] = relative_timeuntil(feed.next_scheduled_update) stats['push'] = feed.is_push # Minutes between updates update_interval_minutes = feed.get_next_scheduled_update(force=True, verbose=False) stats['update_interval_minutes'] = update_interval_minutes original_active_premium_subscribers = feed.active_premium_subscribers original_premium_subscribers = feed.premium_subscribers feed.active_premium_subscribers = max(feed.active_premium_subscribers+1, 1) feed.premium_subscribers += 1 premium_update_interval_minutes = feed.get_next_scheduled_update(force=True, verbose=False) feed.active_premium_subscribers = original_active_premium_subscribers feed.premium_subscribers = original_premium_subscribers stats['premium_update_interval_minutes'] = premium_update_interval_minutes stats['errors_since_good'] = feed.errors_since_good # Stories per month - average and month-by-month breakout average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history stats['average_stories_per_month'] = average_stories_per_month stats['story_count_history'] = story_count_history and json.decode(story_count_history) # Subscribers stats['subscriber_count'] = feed.num_subscribers stats['num_subscribers'] = feed.num_subscribers stats['stories_last_month'] = feed.stories_last_month stats['last_load_time'] = feed.last_load_time stats['premium_subscribers'] = feed.premium_subscribers stats['active_subscribers'] = feed.active_subscribers stats['active_premium_subscribers'] = feed.active_premium_subscribers # Classifier counts stats['classifier_counts'] = json.decode(feed.data.feed_classifier_counts) # Fetch histories timezone = user.profile.timezone fetch_history = MFetchHistory.feed(feed_id, timezone=timezone) stats['feed_fetch_history'] = fetch_history['feed_fetch_history'] stats['page_fetch_history'] = fetch_history['page_fetch_history'] stats['feed_push_history'] = fetch_history['push_history'] logging.user(request, "~FBStatistics: ~SB%s" % (feed)) return stats
def push_callback(request, push_id): if request.method == 'GET': mode = request.GET['hub.mode'] topic = request.GET['hub.topic'] challenge = request.GET['hub.challenge'] lease_seconds = request.GET.get('hub.lease_seconds') verify_token = request.GET.get('hub.verify_token', '') if mode == 'subscribe': if not verify_token.startswith('subscribe'): raise Http404 subscription = get_object_or_404(PushSubscription, pk=push_id, topic=topic, verify_token=verify_token) subscription.verified = True subscription.set_expiration(int(lease_seconds)) subscription.save() subscription.feed.setup_push() logging.debug(' ---> [%-30s] [%s] ~BBVerified PuSH' % (unicode(subscription.feed)[:30], subscription.feed_id)) verified.send(sender=subscription) return HttpResponse(challenge, content_type='text/plain') elif request.method == 'POST': subscription = get_object_or_404(PushSubscription, pk=push_id) fetch_history = MFetchHistory.feed(subscription.feed_id) latest_push_date_delta = None if fetch_history and fetch_history.get('push_history'): latest_push = fetch_history['push_history'][0]['push_date'] latest_push_date = datetime.datetime.strptime(latest_push, '%Y-%m-%d %H:%M:%S') latest_push_date_delta = datetime.datetime.now() - latest_push_date if latest_push_date > datetime.datetime.now() - datetime.timedelta(minutes=1): logging.debug(' ---> [%-30s] ~SN~FBSkipping feed fetch, pushed %s seconds ago' % (unicode(subscription.feed)[:30], latest_push_date_delta.seconds)) return HttpResponse('Slow down, you just pushed %s seconds ago...' % latest_push_date_delta.seconds, status=429) # XXX TODO: Optimize this by removing feedparser. It just needs to find out # the hub_url or topic has changed. ElementTree could do it. if random.random() < 0.1: parsed = feedparser.parse(request.raw_post_data) subscription.check_urls_against_pushed_data(parsed) # Don't give fat ping, just fetch. # subscription.feed.queue_pushed_feed_xml(request.raw_post_data) if subscription.feed.active_premium_subscribers >= 1: subscription.feed.queue_pushed_feed_xml("Fetch me", latest_push_date_delta=latest_push_date_delta) MFetchHistory.add(feed_id=subscription.feed_id, fetch_type='push') else: logging.debug(' ---> [%-30s] ~FBSkipping feed fetch, no actives: %s' % (unicode(subscription.feed)[:30], subscription.feed)) return HttpResponse('OK') return Http404
def load_feed_settings(request, feed_id): stats = dict() feed = get_object_or_404(Feed, pk=feed_id) user = get_user(request) timezone = user.profile.timezone fetch_history = MFetchHistory.feed(feed_id, timezone=timezone) stats['feed_fetch_history'] = fetch_history['feed_fetch_history'] stats['page_fetch_history'] = fetch_history['page_fetch_history'] stats['feed_push_history'] = fetch_history['push_history'] stats['duplicate_addresses'] = feed.duplicate_addresses.all() return stats
def exception_change_feed_link(request): feed_id = request.POST['feed_id'] feed = get_object_or_404(Feed, pk=feed_id) original_feed = feed feed_link = request.POST['feed_link'] timezone = request.user.profile.timezone code = -1 if feed.has_page_exception or feed.has_feed_exception: # Fix broken feed logging.user(request, "~FRFixing feed exception by link: ~SB%s~SN to ~SB%s" % (feed.feed_link, feed_link)) feed_address = feedfinder.feed(feed_link) if feed_address: code = 1 feed.has_page_exception = False feed.active = True feed.fetched_once = False feed.feed_link = feed_link feed.feed_address = feed_address duplicate_feed = feed.schedule_feed_fetch_immediately() if duplicate_feed: new_feed = Feed.objects.get(pk=duplicate_feed.pk) feed = new_feed new_feed.schedule_feed_fetch_immediately() new_feed.has_page_exception = False new_feed.active = True new_feed.save() else: # Branch good feed logging.user(request, "~FRBranching feed by link: ~SB%s~SN to ~SB%s" % (feed.feed_link, feed_link)) feed, _ = Feed.objects.get_or_create(feed_address=feed.feed_address, feed_link=feed_link) code = 1 if feed.pk != original_feed.pk: try: feed.branch_from_feed = original_feed.branch_from_feed or original_feed except Feed.DoesNotExist: feed.branch_from_feed = original_feed feed.feed_link_locked = True feed.save() feed = feed.update() feed = Feed.get_by_id(feed.pk) try: usersub = UserSubscription.objects.get(user=request.user, feed=feed) except UserSubscription.DoesNotExist: usersubs = UserSubscription.objects.filter(user=request.user, feed=original_feed) if usersubs: usersub = usersubs[0] usersub.switch_feed(feed, original_feed) else: fetch_history = MFetchHistory.feed(feed_id, timezone=timezone) return { 'code': -1, 'feed_fetch_history': fetch_history['feed_fetch_history'], 'page_fetch_history': fetch_history['page_fetch_history'], 'push_history': fetch_history['push_history'], } usersub.calculate_feed_scores(silent=False) feed.update_all_statistics() classifiers = get_classifiers_for_user(usersub.user, feed_id=usersub.feed_id) if feed and feed.has_feed_exception: code = -1 feeds = { original_feed.pk: usersub.canonical(full=True, classifiers=classifiers), } fetch_history = MFetchHistory.feed(feed_id, timezone=timezone) return { 'code': code, 'feeds': feeds, 'new_feed_id': usersub.feed_id, 'feed_fetch_history': fetch_history['feed_fetch_history'], 'page_fetch_history': fetch_history['page_fetch_history'], 'push_history': fetch_history['push_history'], }
def load_feed_statistics(request, feed_id): user = get_user(request) timezone = user.profile.timezone stats = dict() feed = get_object_or_404(Feed, pk=feed_id) feed.update_all_statistics() feed.set_next_scheduled_update(verbose=True, skip_scheduling=True) feed.save_feed_story_history_statistics() feed.save_classifier_counts() # Dates of last and next update stats['active'] = feed.active stats['last_update'] = relative_timesince(feed.last_update) stats['next_update'] = relative_timeuntil(feed.next_scheduled_update) stats['push'] = feed.is_push if feed.is_push: try: stats['push_expires'] = localtime_for_timezone(feed.push.lease_expires, timezone).strftime("%Y-%m-%d %H:%M:%S") except PushSubscription.DoesNotExist: stats['push_expires'] = 'Missing push' feed.is_push = False feed.save() # Minutes between updates update_interval_minutes = feed.get_next_scheduled_update(force=True, verbose=False) stats['update_interval_minutes'] = update_interval_minutes original_active_premium_subscribers = feed.active_premium_subscribers original_premium_subscribers = feed.premium_subscribers feed.active_premium_subscribers = max(feed.active_premium_subscribers+1, 1) feed.premium_subscribers += 1 premium_update_interval_minutes = feed.get_next_scheduled_update(force=True, verbose=False, premium_speed=True) feed.active_premium_subscribers = original_active_premium_subscribers feed.premium_subscribers = original_premium_subscribers stats['premium_update_interval_minutes'] = premium_update_interval_minutes stats['errors_since_good'] = feed.errors_since_good # Stories per month - average and month-by-month breakout average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history stats['average_stories_per_month'] = average_stories_per_month story_count_history = story_count_history and json.decode(story_count_history) if story_count_history and isinstance(story_count_history, dict): stats['story_count_history'] = story_count_history['months'] stats['story_days_history'] = story_count_history['days'] stats['story_hours_history'] = story_count_history['hours'] else: stats['story_count_history'] = story_count_history # Rotate hours to match user's timezone offset localoffset = timezone.utcoffset(datetime.datetime.utcnow()) hours_offset = int(localoffset.total_seconds() / 3600) rotated_hours = {} for hour, value in stats['story_hours_history'].items(): rotated_hours[str(int(hour)+hours_offset)] = value stats['story_hours_history'] = rotated_hours # Subscribers stats['subscriber_count'] = feed.num_subscribers stats['num_subscribers'] = feed.num_subscribers stats['stories_last_month'] = feed.stories_last_month stats['last_load_time'] = feed.last_load_time stats['premium_subscribers'] = feed.premium_subscribers stats['active_subscribers'] = feed.active_subscribers stats['active_premium_subscribers'] = feed.active_premium_subscribers # Classifier counts stats['classifier_counts'] = json.decode(feed.data.feed_classifier_counts) # Fetch histories fetch_history = MFetchHistory.feed(feed_id, timezone=timezone) stats['feed_fetch_history'] = fetch_history['feed_fetch_history'] stats['page_fetch_history'] = fetch_history['page_fetch_history'] stats['feed_push_history'] = fetch_history['push_history'] logging.user(request, "~FBStatistics: ~SB%s" % (feed)) return stats
def exception_change_feed_address(request): feed_id = request.POST["feed_id"] feed = get_object_or_404(Feed, pk=feed_id) original_feed = feed feed_address = request.POST["feed_address"] timezone = request.user.profile.timezone code = -1 if not feed.known_good and (feed.has_page_exception or feed.has_feed_exception): # Fix broken feed logging.user( request, "~FRFixing feed exception by address: ~SB%s~SN to ~SB%s" % (feed.feed_address, feed_address) ) feed.has_feed_exception = False feed.active = True feed.fetched_once = False feed.feed_address = feed_address duplicate_feed = feed.schedule_feed_fetch_immediately() code = 1 if duplicate_feed: new_feed = Feed.objects.get(pk=duplicate_feed.pk) feed = new_feed new_feed.schedule_feed_fetch_immediately() new_feed.has_feed_exception = False new_feed.active = True new_feed = new_feed.save() if new_feed.pk != feed.pk: merge_feeds(new_feed.pk, feed.pk) else: # Branch good feed logging.user(request, "~FRBranching feed by address: ~SB%s~SN to ~SB%s" % (feed.feed_address, feed_address)) try: feed = Feed.objects.get( hash_address_and_link=Feed.generate_hash_address_and_link(feed_address, feed.feed_link) ) except Feed.DoesNotExist: feed = Feed.objects.create(feed_address=feed_address, feed_link=feed.feed_link) code = 1 if feed.pk != original_feed.pk: try: feed.branch_from_feed = original_feed.branch_from_feed or original_feed except Feed.DoesNotExist: feed.branch_from_feed = original_feed feed.feed_address_locked = True feed = feed.save() feed = feed.update() feed = Feed.get_by_id(feed.pk) try: usersub = UserSubscription.objects.get(user=request.user, feed=feed) except UserSubscription.DoesNotExist: usersubs = UserSubscription.objects.filter(user=request.user, feed=original_feed) if usersubs: usersub = usersubs[0] usersub.switch_feed(feed, original_feed) else: fetch_history = MFetchHistory.feed(feed_id, timezone=timezone) return { "code": -1, "feed_fetch_history": fetch_history["feed_fetch_history"], "page_fetch_history": fetch_history["page_fetch_history"], "push_history": fetch_history["push_history"], } usersub.calculate_feed_scores(silent=False) feed.update_all_statistics() classifiers = get_classifiers_for_user(usersub.user, feed_id=usersub.feed_id) feeds = {original_feed.pk: usersub and usersub.canonical(full=True, classifiers=classifiers)} if feed and feed.has_feed_exception: code = -1 fetch_history = MFetchHistory.feed(feed_id, timezone=timezone) return { "code": code, "feeds": feeds, "new_feed_id": usersub.feed_id, "feed_fetch_history": fetch_history["feed_fetch_history"], "page_fetch_history": fetch_history["page_fetch_history"], "push_history": fetch_history["push_history"], }
def load_feed_statistics(request, feed_id): user = get_user(request) timezone = user.profile.timezone stats = dict() feed = get_object_or_404(Feed, pk=feed_id) feed.update_all_statistics() feed.set_next_scheduled_update(verbose=True, skip_scheduling=True) feed.save_feed_story_history_statistics() feed.save_classifier_counts() # Dates of last and next update stats["active"] = feed.active stats["last_update"] = relative_timesince(feed.last_update) stats["next_update"] = relative_timeuntil(feed.next_scheduled_update) stats["push"] = feed.is_push if feed.is_push: try: stats["push_expires"] = localtime_for_timezone(feed.push.lease_expires, timezone).strftime( "%Y-%m-%d %H:%M:%S" ) except PushSubscription.DoesNotExist: stats["push_expires"] = "Missing push" feed.is_push = False feed.save() # Minutes between updates update_interval_minutes = feed.get_next_scheduled_update(force=True, verbose=False) stats["update_interval_minutes"] = update_interval_minutes original_active_premium_subscribers = feed.active_premium_subscribers original_premium_subscribers = feed.premium_subscribers feed.active_premium_subscribers = max(feed.active_premium_subscribers + 1, 1) feed.premium_subscribers += 1 premium_update_interval_minutes = feed.get_next_scheduled_update(force=True, verbose=False, premium_speed=True) feed.active_premium_subscribers = original_active_premium_subscribers feed.premium_subscribers = original_premium_subscribers stats["premium_update_interval_minutes"] = premium_update_interval_minutes stats["errors_since_good"] = feed.errors_since_good # Stories per month - average and month-by-month breakout average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history stats["average_stories_per_month"] = average_stories_per_month stats["story_count_history"] = story_count_history and json.decode(story_count_history) # Subscribers stats["subscriber_count"] = feed.num_subscribers stats["num_subscribers"] = feed.num_subscribers stats["stories_last_month"] = feed.stories_last_month stats["last_load_time"] = feed.last_load_time stats["premium_subscribers"] = feed.premium_subscribers stats["active_subscribers"] = feed.active_subscribers stats["active_premium_subscribers"] = feed.active_premium_subscribers # Classifier counts stats["classifier_counts"] = json.decode(feed.data.feed_classifier_counts) # Fetch histories fetch_history = MFetchHistory.feed(feed_id, timezone=timezone) stats["feed_fetch_history"] = fetch_history["feed_fetch_history"] stats["page_fetch_history"] = fetch_history["page_fetch_history"] stats["feed_push_history"] = fetch_history["push_history"] logging.user(request, "~FBStatistics: ~SB%s" % (feed)) return stats
def load_feed_statistics(request, feed_id): user = get_user(request) timezone = user.profile.timezone stats = dict() feed = get_object_or_404(Feed, pk=feed_id) feed.update_all_statistics() feed.set_next_scheduled_update(verbose=True, skip_scheduling=True) feed.save_feed_story_history_statistics() feed.save_classifier_counts() # Dates of last and next update stats['active'] = feed.active stats['last_update'] = relative_timesince(feed.last_update) stats['next_update'] = relative_timeuntil(feed.next_scheduled_update) stats['push'] = feed.is_push if feed.is_push: try: stats['push_expires'] = localtime_for_timezone( feed.push.lease_expires, timezone).strftime("%Y-%m-%d %H:%M:%S") except PushSubscription.DoesNotExist: stats['push_expires'] = 'Missing push' feed.is_push = False feed.save() # Minutes between updates update_interval_minutes = feed.get_next_scheduled_update(force=True, verbose=False) stats['update_interval_minutes'] = update_interval_minutes original_active_premium_subscribers = feed.active_premium_subscribers original_premium_subscribers = feed.premium_subscribers feed.active_premium_subscribers = max(feed.active_premium_subscribers + 1, 1) feed.premium_subscribers += 1 premium_update_interval_minutes = feed.get_next_scheduled_update( force=True, verbose=False, premium_speed=True) feed.active_premium_subscribers = original_active_premium_subscribers feed.premium_subscribers = original_premium_subscribers stats['premium_update_interval_minutes'] = premium_update_interval_minutes stats['errors_since_good'] = feed.errors_since_good # Stories per month - average and month-by-month breakout average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history stats['average_stories_per_month'] = average_stories_per_month story_count_history = story_count_history and json.decode( story_count_history) if story_count_history and isinstance(story_count_history, dict): stats['story_count_history'] = story_count_history['months'] stats['story_days_history'] = story_count_history['days'] stats['story_hours_history'] = story_count_history['hours'] else: stats['story_count_history'] = story_count_history # Rotate hours to match user's timezone offset localoffset = timezone.utcoffset(datetime.datetime.utcnow()) hours_offset = int(localoffset.total_seconds() / 3600) rotated_hours = {} for hour, value in stats['story_hours_history'].items(): rotated_hours[str(int(hour) + hours_offset)] = value stats['story_hours_history'] = rotated_hours # Subscribers stats['subscriber_count'] = feed.num_subscribers stats['num_subscribers'] = feed.num_subscribers stats['stories_last_month'] = feed.stories_last_month stats['last_load_time'] = feed.last_load_time stats['premium_subscribers'] = feed.premium_subscribers stats['active_subscribers'] = feed.active_subscribers stats['active_premium_subscribers'] = feed.active_premium_subscribers # Classifier counts stats['classifier_counts'] = json.decode(feed.data.feed_classifier_counts) # Fetch histories fetch_history = MFetchHistory.feed(feed_id, timezone=timezone) stats['feed_fetch_history'] = fetch_history['feed_fetch_history'] stats['page_fetch_history'] = fetch_history['page_fetch_history'] stats['feed_push_history'] = fetch_history['push_history'] logging.user(request, "~FBStatistics: ~SB%s" % (feed)) return stats