def delete_old_history():
MFeedFetchHistory.objects(fetch_date__lt=last_day,
status_code__in=[200, 304]).delete()
MPageFetchHistory.objects(fetch_date__lt=last_day,
status_code__in=[200, 304]).delete()
MFeedFetchHistory.objects(fetch_date__lt=last_month).delete()
MPageFetchHistory.objects(fetch_date__lt=last_month).delete()
MFeedPushHistory.objects(push_date__lt=last_month).delete()
def calculate_metrics():
import datetime
from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory
last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1)
return {
'feed_errors': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(),
'feed_success': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(),
'page_errors': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(),
'page_success': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(),
}
def load_feed_statistics(request, feed_id):
user = get_user(request)
stats = dict()
feed = get_object_or_404(Feed, pk=feed_id)
feed.count_subscribers()
feed.set_next_scheduled_update(verbose=True, skip_scheduling=True)
feed.save_feed_story_history_statistics()
feed.save_classifier_counts()
# Dates of last and next update
stats['active'] = feed.active
stats['last_update'] = relative_timesince(feed.last_update)
if feed.is_push:
stats['next_update'] = "real-time..."
else:
stats['next_update'] = relative_timeuntil(feed.next_scheduled_update)
# Minutes between updates
update_interval_minutes, _ = feed.get_next_scheduled_update(force=True, verbose=False)
if feed.is_push:
stats['update_interval_minutes'] = 0
else:
stats['update_interval_minutes'] = update_interval_minutes
original_active_premium_subscribers = feed.active_premium_subscribers
original_premium_subscribers = feed.premium_subscribers
feed.active_premium_subscribers = max(feed.active_premium_subscribers+1, 1)
feed.premium_subscribers += 1
premium_update_interval_minutes, _ = feed.get_next_scheduled_update(force=True, verbose=False)
feed.active_premium_subscribers = original_active_premium_subscribers
feed.premium_subscribers = original_premium_subscribers
if feed.is_push:
stats['premium_update_interval_minutes'] = 0
else:
stats['premium_update_interval_minutes'] = premium_update_interval_minutes
# Stories per month - average and month-by-month breakout
average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history
stats['average_stories_per_month'] = average_stories_per_month
stats['story_count_history'] = story_count_history and json.decode(story_count_history)
# Subscribers
stats['subscriber_count'] = feed.num_subscribers
stats['stories_last_month'] = feed.stories_last_month
stats['last_load_time'] = feed.last_load_time
stats['premium_subscribers'] = feed.premium_subscribers
stats['active_subscribers'] = feed.active_subscribers
stats['active_premium_subscribers'] = feed.active_premium_subscribers
# Classifier counts
stats['classifier_counts'] = json.decode(feed.data.feed_classifier_counts)
# Fetch histories
timezone = user.profile.timezone
stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id, timezone=timezone)
stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id, timezone=timezone)
stats['feed_push_history'] = MFeedPushHistory.feed_history(feed_id, timezone=timezone)
logging.user(request, "~FBStatistics: ~SB%s ~FG(%s/%s/%s subs)" % (feed, feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers,))
return stats
def load_feed_statistics(request):
stats = dict()
feed_id = request.GET['feed_id']
feed = get_object_or_404(Feed, pk=feed_id)
feed.save_feed_story_history_statistics()
# Dates of last and next update
stats['last_update'] = relative_timesince(feed.last_update)
stats['next_update'] = relative_timeuntil(feed.next_scheduled_update)
# Minutes between updates
update_interval_minutes, random_factor = feed.get_next_scheduled_update()
stats['update_interval_minutes'] = update_interval_minutes
# Stories per month - average and month-by-month breakout
average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history
stats['average_stories_per_month'] = average_stories_per_month
stats['story_count_history'] = story_count_history and json.decode(story_count_history)
# Subscribers
stats['subscriber_count'] = feed.num_subscribers
stats['stories_last_month'] = feed.stories_last_month
stats['last_load_time'] = feed.last_load_time
stats['premium_subscribers'] = feed.premium_subscribers
stats['active_subscribers'] = feed.active_subscribers
# Fetch histories
stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id)
stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id)
logging.user(request.user, "~FBStatistics: ~SB%s ~FG(%s/%s/%s subs)" % (feed, feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers,))
return stats
def load_feed_settings(request, feed_id):
stats = dict()
feed = get_object_or_404(Feed, pk=feed_id)
stats['duplicate_addresses'] = feed.duplicate_addresses.all()
stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id)
stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id)
return stats
def load_feed_settings(request, feed_id):
stats = dict()
feed = get_object_or_404(Feed, pk=feed_id)
stats['duplicate_addresses'] = feed.duplicate_addresses.all()
stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id)
stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id)
return stats
def delete_old_history():
print "Deleting old history."
feed_fetch_last_day = MFeedFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304])
page_fetch_last_day = MPageFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304])
feed_fetch_last_month = MFeedFetchHistory.objects(fetch_date__lt=last_month)
page_fetch_last_month = MPageFetchHistory.objects(fetch_date__lt=last_month)
push_last_month = MFeedPushHistory.objects(push_date__lt=last_month)
print "Found %s/%s/%s/%s/%s (feed/page day, feed/page month, push month)" % (
feed_fetch_last_day.count(),
page_fetch_last_day.count(),
feed_fetch_last_month.count(),
page_fetch_last_month.count(),
push_last_month.count(),
)
feed_fetch_last_day.delete()
page_fetch_last_day.delete()
feed_fetch_last_month.delete()
page_fetch_last_month.delete()
push_last_month.delete()
def load_feed_settings(request, feed_id):
stats = dict()
feed = get_object_or_404(Feed, pk=feed_id)
user = get_user(request)
timezone = user.profile.timezone
stats['duplicate_addresses'] = feed.duplicate_addresses.all()
stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id, timezone=timezone)
stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id, timezone=timezone)
return stats
def calculate_metrics():
import datetime
from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory
last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1)
return {
'feed_errors':
MFeedFetchHistory.objects(fetch_date__gte=last_day,
status_code__nin=[200, 304]).count(),
'feed_success':
MFeedFetchHistory.objects(fetch_date__gte=last_day,
status_code__in=[200, 304]).count(),
'page_errors':
MPageFetchHistory.objects(fetch_date__gte=last_day,
status_code__nin=[200, 304]).count(),
'page_success':
MPageFetchHistory.objects(fetch_date__gte=last_day,
status_code__in=[200, 304]).count(),
}
def collect_statistics_feeds_fetched(cls, last_day=None):
if not last_day:
last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
feeds_fetched = MFeedFetchHistory.objects(fetch_date__gte=last_day).count()
cls.objects(key='feeds_fetched').update_one(upsert=True, key='feeds_fetched', value=feeds_fetched)
old_fetch_histories = MFeedFetchHistory.objects(fetch_date__lte=last_day)
for history in old_fetch_histories:
history.delete()
old_page_histories = MPageFetchHistory.objects(fetch_date__lte=last_day)
for history in old_page_histories:
history.delete()
return feeds_fetched
def load_feed_statistics(request, feed_id):
stats = dict()
feed = get_object_or_404(Feed, pk=feed_id)
feed.save_feed_story_history_statistics()
feed.save_classifier_counts()
# Dates of last and next update
stats['last_update'] = relative_timesince(feed.last_update)
stats['next_update'] = relative_timeuntil(feed.next_scheduled_update)
# Minutes between updates
update_interval_minutes, random_factor = feed.get_next_scheduled_update(
force=True)
stats['update_interval_minutes'] = update_interval_minutes
# Stories per month - average and month-by-month breakout
average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history
stats['average_stories_per_month'] = average_stories_per_month
stats['story_count_history'] = story_count_history and json.decode(
story_count_history)
# Subscribers
stats['subscriber_count'] = feed.num_subscribers
stats['stories_last_month'] = feed.stories_last_month
stats['last_load_time'] = feed.last_load_time
stats['premium_subscribers'] = feed.premium_subscribers
stats['active_subscribers'] = feed.active_subscribers
stats['active_premium_subscribers'] = feed.active_premium_subscribers
# Classifier counts
stats['classifier_counts'] = json.decode(feed.data.feed_classifier_counts)
# Fetch histories
stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id)
stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id)
logging.user(
request, "~FBStatistics: ~SB%s ~FG(%s/%s/%s subs)" % (
feed,
feed.num_subscribers,
feed.active_subscribers,
feed.premium_subscribers,
))
return stats
def collect_statistics_feeds_fetched(cls, last_day=None):
if not last_day:
last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
feeds_fetched = MFeedFetchHistory.objects(
fetch_date__gte=last_day).count()
cls.objects(key='feeds_fetched').update_one(upsert=True,
key='feeds_fetched',
value=feeds_fetched)
old_fetch_histories = MFeedFetchHistory.objects(
fetch_date__lte=last_day)
for history in old_fetch_histories:
history.delete()
old_page_histories = MPageFetchHistory.objects(
fetch_date__lte=last_day)
for history in old_page_histories:
history.delete()
return feeds_fetched
graph_config = {
'graph_category': 'NewsBlur',
'graph_title': 'NewsBlur Fetching History',
'graph_vlabel': 'errors',
'feed_errors.label': 'Feed Errors',
'feed_success.label': 'Feed Success',
'page_errors.label': 'Page Errors',
'page_success.label': 'Page Success',
}
last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1)
metrics = {
'feed_errors':
MFeedFetchHistory.objects(fetch_date__gte=last_day,
status_code__nin=[200, 304]).count(),
'feed_success':
MFeedFetchHistory.objects(fetch_date__gte=last_day,
status_code__in=[200, 304]).count(),
'page_errors':
MPageFetchHistory.objects(fetch_date__gte=last_day,
status_code__nin=[200, 304]).count(),
'page_success':
MPageFetchHistory.objects(fetch_date__gte=last_day,
status_code__in=[200, 304]).count(),
}
if __name__ == '__main__':
MuninGraph(graph_config, metrics).run()
#!/usr/bin/env python
from utils.munin.base import MuninGraph
from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory
import datetime
graph_config = {
'graph_category' : 'NewsBlur',
'graph_title' : 'NewsBlur Fetching History',
'graph_vlabel' : 'errors',
'feed_errors.label': 'Feed Errors',
'feed_success.label': 'Feed Success',
'page_errors.label': 'Page Errors',
'page_success.label': 'Page Success',
}
last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1)
metrics = {
'feed_errors': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(),
'feed_success': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(),
'page_errors': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(),
'page_success': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(),
}
if __name__ == '__main__':
MuninGraph(graph_config, metrics).run()
def exception_change_feed_link(request):
feed_id = request.POST['feed_id']
feed = get_object_or_404(Feed, pk=feed_id)
original_feed = feed
feed_link = request.POST['feed_link']
timezone = request.user.profile.timezone
code = -1
if feed.has_page_exception or feed.has_feed_exception:
# Fix broken feed
logging.user(request, "~FRFixing feed exception by link: ~SB%s~SN to ~SB%s" % (feed.feed_link, feed_link))
feed_address = feedfinder.feed(feed_link)
if feed_address:
code = 1
feed.has_page_exception = False
feed.active = True
feed.fetched_once = False
feed.feed_link = feed_link
feed.feed_address = feed_address
duplicate_feed = feed.schedule_feed_fetch_immediately()
if duplicate_feed:
new_feed = Feed.objects.get(pk=duplicate_feed.pk)
feed = new_feed
new_feed.schedule_feed_fetch_immediately()
new_feed.has_page_exception = False
new_feed.active = True
new_feed.save()
else:
# Branch good feed
logging.user(request, "~FRBranching feed by link: ~SB%s~SN to ~SB%s" % (feed.feed_link, feed_link))
feed, _ = Feed.objects.get_or_create(feed_address=feed.feed_address, feed_link=feed_link)
code = 1
if feed.pk != original_feed.pk:
try:
feed.branch_from_feed = original_feed.branch_from_feed or original_feed
except Feed.DoesNotExist:
feed.branch_from_feed = original_feed
feed.feed_link_locked = True
feed.save()
feed = feed.update()
feed = Feed.get_by_id(feed.pk)
try:
usersub = UserSubscription.objects.get(user=request.user, feed=feed)
except UserSubscription.DoesNotExist:
usersubs = UserSubscription.objects.filter(user=request.user, feed=original_feed)
if usersubs:
usersub = usersubs[0]
usersub.switch_feed(feed, original_feed)
else:
return {
'code': -1,
'feed_fetch_history': MFeedFetchHistory.feed_history(feed_id, timezone=timezone),
'page_fetch_history': MPageFetchHistory.feed_history(feed_id, timezone=timezone),
}
usersub.calculate_feed_scores(silent=False)
feed.update_all_statistics()
classifiers = get_classifiers_for_user(usersub.user, feed_id=usersub.feed_id)
if feed and feed.has_feed_exception:
code = -1
feeds = {
original_feed.pk: usersub.canonical(full=True, classifiers=classifiers),
}
return {
'code': code,
'feeds': feeds,
'new_feed_id': usersub.feed_id,
'feed_fetch_history': MFeedFetchHistory.feed_history(feed_id, timezone=timezone),
'page_fetch_history': MPageFetchHistory.feed_history(feed_id, timezone=timezone),
}
def delete_old_history():
MFeedFetchHistory.objects(fetch_date__lt=last_day).delete()
MPageFetchHistory.objects(fetch_date__lt=last_day).delete()
def delete_old_history():
MFeedFetchHistory.objects(fetch_date__lt=last_day).delete()
MPageFetchHistory.objects(fetch_date__lt=last_day).delete()
def delete_old_history():
MFeedFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete()
MPageFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete()
MFeedFetchHistory.objects(fetch_date__lt=last_month).delete()
MPageFetchHistory.objects(fetch_date__lt=last_month).delete()
MFeedPushHistory.objects(push_date__lt=last_month).delete()