def delete_old_history(): MFeedFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete() MPageFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete() MFeedFetchHistory.objects(fetch_date__lt=last_month).delete() MPageFetchHistory.objects(fetch_date__lt=last_month).delete() MFeedPushHistory.objects(push_date__lt=last_month).delete()
def calculate_metrics(): import datetime from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1) return { 'feed_errors': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(), 'feed_success': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(), 'page_errors': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(), 'page_success': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(), }
def delete_old_history(): print "Deleting old history." feed_fetch_last_day = MFeedFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]) page_fetch_last_day = MPageFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]) feed_fetch_last_month = MFeedFetchHistory.objects(fetch_date__lt=last_month) page_fetch_last_month = MPageFetchHistory.objects(fetch_date__lt=last_month) push_last_month = MFeedPushHistory.objects(push_date__lt=last_month) print "Found %s/%s/%s/%s/%s (feed/page day, feed/page month, push month)" % ( feed_fetch_last_day.count(), page_fetch_last_day.count(), feed_fetch_last_month.count(), page_fetch_last_month.count(), push_last_month.count(), ) feed_fetch_last_day.delete() page_fetch_last_day.delete() feed_fetch_last_month.delete() page_fetch_last_month.delete() push_last_month.delete()
def collect_statistics_feeds_fetched(cls, last_day=None): if not last_day: last_day = datetime.datetime.now() - datetime.timedelta(hours=24) feeds_fetched = MFeedFetchHistory.objects(fetch_date__gte=last_day).count() cls.objects(key='feeds_fetched').update_one(upsert=True, key='feeds_fetched', value=feeds_fetched) old_fetch_histories = MFeedFetchHistory.objects(fetch_date__lte=last_day) for history in old_fetch_histories: history.delete() old_page_histories = MPageFetchHistory.objects(fetch_date__lte=last_day) for history in old_page_histories: history.delete() return feeds_fetched
def collect_statistics_feeds_fetched(cls, last_day=None): if not last_day: last_day = datetime.datetime.now() - datetime.timedelta(hours=24) feeds_fetched = MFeedFetchHistory.objects( fetch_date__gte=last_day).count() cls.objects(key='feeds_fetched').update_one(upsert=True, key='feeds_fetched', value=feeds_fetched) old_fetch_histories = MFeedFetchHistory.objects( fetch_date__lte=last_day) for history in old_fetch_histories: history.delete() old_page_histories = MPageFetchHistory.objects( fetch_date__lte=last_day) for history in old_page_histories: history.delete() return feeds_fetched
#!/usr/bin/env python from utils.munin.base import MuninGraph from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory import datetime graph_config = { 'graph_category' : 'NewsBlur', 'graph_title' : 'NewsBlur Fetching History', 'graph_vlabel' : 'errors', 'feed_errors.label': 'Feed Errors', 'feed_success.label': 'Feed Success', 'page_errors.label': 'Page Errors', 'page_success.label': 'Page Success', } last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1) metrics = { 'feed_errors': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(), 'feed_success': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(), 'page_errors': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(), 'page_success': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(), } if __name__ == '__main__': MuninGraph(graph_config, metrics).run()
def delete_old_history(): MFeedFetchHistory.objects(fetch_date__lt=last_day).delete() MPageFetchHistory.objects(fetch_date__lt=last_day).delete()
graph_config = { 'graph_category': 'NewsBlur', 'graph_title': 'NewsBlur Fetching History', 'graph_vlabel': 'errors', 'feed_errors.label': 'Feed Errors', 'feed_success.label': 'Feed Success', 'page_errors.label': 'Page Errors', 'page_success.label': 'Page Success', } last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1) metrics = { 'feed_errors': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(), 'feed_success': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(), 'page_errors': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(), 'page_success': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(), } if __name__ == '__main__': MuninGraph(graph_config, metrics).run()