def delete_old_history():
MFeedFetchHistory.objects(fetch_date__lt=last_day,
status_code__in=[200, 304]).delete()
MPageFetchHistory.objects(fetch_date__lt=last_day,
status_code__in=[200, 304]).delete()
MFeedFetchHistory.objects(fetch_date__lt=last_month).delete()
MPageFetchHistory.objects(fetch_date__lt=last_month).delete()
MFeedPushHistory.objects(push_date__lt=last_month).delete()
def calculate_metrics():
import datetime
from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory
last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1)
return {
'feed_errors': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(),
'feed_success': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(),
'page_errors': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(),
'page_success': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(),
}
def delete_old_history():
print "Deleting old history."
feed_fetch_last_day = MFeedFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304])
page_fetch_last_day = MPageFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304])
feed_fetch_last_month = MFeedFetchHistory.objects(fetch_date__lt=last_month)
page_fetch_last_month = MPageFetchHistory.objects(fetch_date__lt=last_month)
push_last_month = MFeedPushHistory.objects(push_date__lt=last_month)
print "Found %s/%s/%s/%s/%s (feed/page day, feed/page month, push month)" % (
feed_fetch_last_day.count(),
page_fetch_last_day.count(),
feed_fetch_last_month.count(),
page_fetch_last_month.count(),
push_last_month.count(),
)
feed_fetch_last_day.delete()
page_fetch_last_day.delete()
feed_fetch_last_month.delete()
page_fetch_last_month.delete()
push_last_month.delete()
def calculate_metrics():
import datetime
from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory
last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1)
return {
'feed_errors':
MFeedFetchHistory.objects(fetch_date__gte=last_day,
status_code__nin=[200, 304]).count(),
'feed_success':
MFeedFetchHistory.objects(fetch_date__gte=last_day,
status_code__in=[200, 304]).count(),
'page_errors':
MPageFetchHistory.objects(fetch_date__gte=last_day,
status_code__nin=[200, 304]).count(),
'page_success':
MPageFetchHistory.objects(fetch_date__gte=last_day,
status_code__in=[200, 304]).count(),
}
def collect_statistics_feeds_fetched(cls, last_day=None):
if not last_day:
last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
feeds_fetched = MFeedFetchHistory.objects(fetch_date__gte=last_day).count()
cls.objects(key='feeds_fetched').update_one(upsert=True, key='feeds_fetched', value=feeds_fetched)
old_fetch_histories = MFeedFetchHistory.objects(fetch_date__lte=last_day)
for history in old_fetch_histories:
history.delete()
old_page_histories = MPageFetchHistory.objects(fetch_date__lte=last_day)
for history in old_page_histories:
history.delete()
return feeds_fetched
def collect_statistics_feeds_fetched(cls, last_day=None):
if not last_day:
last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
feeds_fetched = MFeedFetchHistory.objects(
fetch_date__gte=last_day).count()
cls.objects(key='feeds_fetched').update_one(upsert=True,
key='feeds_fetched',
value=feeds_fetched)
old_fetch_histories = MFeedFetchHistory.objects(
fetch_date__lte=last_day)
for history in old_fetch_histories:
history.delete()
old_page_histories = MPageFetchHistory.objects(
fetch_date__lte=last_day)
for history in old_page_histories:
history.delete()
return feeds_fetched
#!/usr/bin/env python
from utils.munin.base import MuninGraph
from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory
import datetime
graph_config = {
'graph_category' : 'NewsBlur',
'graph_title' : 'NewsBlur Fetching History',
'graph_vlabel' : 'errors',
'feed_errors.label': 'Feed Errors',
'feed_success.label': 'Feed Success',
'page_errors.label': 'Page Errors',
'page_success.label': 'Page Success',
}
last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1)
metrics = {
'feed_errors': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(),
'feed_success': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(),
'page_errors': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(),
'page_success': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(),
}
if __name__ == '__main__':
MuninGraph(graph_config, metrics).run()
def delete_old_history():
MFeedFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete()
MPageFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete()
MFeedFetchHistory.objects(fetch_date__lt=last_month).delete()
MPageFetchHistory.objects(fetch_date__lt=last_month).delete()
MFeedPushHistory.objects(push_date__lt=last_month).delete()
def delete_old_history():
MFeedFetchHistory.objects(fetch_date__lt=last_day).delete()
MPageFetchHistory.objects(fetch_date__lt=last_day).delete()
graph_config = {
'graph_category': 'NewsBlur',
'graph_title': 'NewsBlur Fetching History',
'graph_vlabel': 'errors',
'feed_errors.label': 'Feed Errors',
'feed_success.label': 'Feed Success',
'page_errors.label': 'Page Errors',
'page_success.label': 'Page Success',
}
last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1)
metrics = {
'feed_errors':
MFeedFetchHistory.objects(fetch_date__gte=last_day,
status_code__nin=[200, 304]).count(),
'feed_success':
MFeedFetchHistory.objects(fetch_date__gte=last_day,
status_code__in=[200, 304]).count(),
'page_errors':
MPageFetchHistory.objects(fetch_date__gte=last_day,
status_code__nin=[200, 304]).count(),
'page_success':
MPageFetchHistory.objects(fetch_date__gte=last_day,
status_code__in=[200, 304]).count(),
}
if __name__ == '__main__':
MuninGraph(graph_config, metrics).run()
def delete_old_history():
MFeedFetchHistory.objects(fetch_date__lt=last_day).delete()
MPageFetchHistory.objects(fetch_date__lt=last_day).delete()
