コード例 #1
0
ファイル: stats.py プロジェクト: 1flow/1flow
def synchronize_mongodb_statsd_articles_gauges(full=False):
    """ synchronize all articles-related gauges on our statsd server. """

    with benchmark('synchronize statsd gauges for Article.*'):

        empty               = Article.objects(content_type=0).no_cache()
        # empty_pending       = empty.filter(content_error='', url_error='')
        # empty_content_error = empty.filter(content_error__ne='')
        # empty_url_error     = empty.filter(url_error__ne='')

        parsed             = Article.objects(
            content_type__ne=CONTENT_TYPES.NONE)
        html               = parsed.filter(content_type=CONTENT_TYPES.HTML)
        markdown           = parsed.filter(content_type=CONTENT_TYPES.MARKDOWN)

        absolutes          = Article.objects(url_absolute=True).no_cache()
        duplicates         = Article.objects(duplicate_of__ne=None).no_cache()
        orphaned           = Article.objects(orphaned=True).no_cache()
        content_errors     = Article.objects(content_error__ne='').no_cache()
        url_errors         = Article.objects(url_error__ne='').no_cache()

        statsd.gauge('mongo.articles.counts.total',
                     Article._get_collection().count())
        statsd.gauge('mongo.articles.counts.markdown', markdown.count())
        statsd.gauge('mongo.articles.counts.html', html.count())
        statsd.gauge('mongo.articles.counts.empty', empty.count())
        statsd.gauge('mongo.articles.counts.content_errors',
                     content_errors.count())
        statsd.gauge('mongo.articles.counts.url_errors', url_errors.count())

        if full:
            statsd.gauge('mongo.articles.counts.orphaned', orphaned.count())
            statsd.gauge('mongo.articles.counts.absolutes', absolutes.count())
            statsd.gauge('mongo.articles.counts.duplicates', duplicates.count())
コード例 #2
0
ファイル: stats.py プロジェクト: dolanor-galaxy/1flow
def synchronize_mongodb_statsd_articles_gauges(full=False):
    """ synchronize all articles-related gauges on our statsd server. """

    with benchmark('synchronize statsd gauges for Article.*'):

        empty = Article.objects(content_type=0).no_cache()
        # empty_pending       = empty.filter(content_error='', url_error='')
        # empty_content_error = empty.filter(content_error__ne='')
        # empty_url_error     = empty.filter(url_error__ne='')

        parsed = Article.objects(content_type__ne=CONTENT_TYPES.NONE)
        html = parsed.filter(content_type=CONTENT_TYPES.HTML)
        markdown = parsed.filter(content_type=CONTENT_TYPES.MARKDOWN)

        absolutes = Article.objects(url_absolute=True).no_cache()
        duplicates = Article.objects(duplicate_of__ne=None).no_cache()
        orphaned = Article.objects(orphaned=True).no_cache()
        content_errors = Article.objects(content_error__ne='').no_cache()
        url_errors = Article.objects(url_error__ne='').no_cache()

        statsd.gauge('mongo.articles.counts.total',
                     Article._get_collection().count())
        statsd.gauge('mongo.articles.counts.markdown', markdown.count())
        statsd.gauge('mongo.articles.counts.html', html.count())
        statsd.gauge('mongo.articles.counts.empty', empty.count())
        statsd.gauge('mongo.articles.counts.content_errors',
                     content_errors.count())
        statsd.gauge('mongo.articles.counts.url_errors', url_errors.count())

        if full:
            statsd.gauge('mongo.articles.counts.orphaned', orphaned.count())
            statsd.gauge('mongo.articles.counts.absolutes', absolutes.count())
            statsd.gauge('mongo.articles.counts.duplicates',
                         duplicates.count())
コード例 #3
0
ファイル: stats.py プロジェクト: 1flow/1flow
def article_content_error_types():
    """ Return an error classifier on the ``content_error`` attribute. """

    return ContentErrorClassifier(
        Article.objects(content_error__ne='').no_cache(),
        'content_error'
    ).classify()
コード例 #4
0
ファイル: stats.py プロジェクト: dolanor-galaxy/1flow
def article_url_error_types():
    """ Return an error classifier on the ``url_error`` attribute. """

    # Next to investigate:
    #    list index out of range: 758
    #    'NoneType' object has no attribute 'findAll': 137

    return UrlErrorClassifier(
        Article.objects(url_error__ne='').no_cache(), 'url_error').classify()
コード例 #5
0
ファイル: stats.py プロジェクト: 1flow/1flow
def article_url_error_types():
    """ Return an error classifier on the ``url_error`` attribute. """

    # Next to investigate:
    #    list index out of range: 758
    #    'NoneType' object has no attribute 'findAll': 137

    return UrlErrorClassifier(
        Article.objects(url_error__ne='').no_cache(),
        'url_error'
    ).classify()
コード例 #6
0
 def tearDownClass(cls):
     Article.drop_collection()
     Feed.drop_collection()
コード例 #7
0
from oneflow.base.tests import (connect_mongodb_testsuite, TEST_REDIS)

#from unittest import skip

DjangoUser = get_user_model()
LOGGER     = logging.getLogger(__file__)

# Use the test database not to pollute the production/development one.
RedisStatsCounter.REDIS = TEST_REDIS

TEST_REDIS.flushdb()

connect_mongodb_testsuite()

# Empty the database before starting in case an old test failed to tearDown().
Article.drop_collection()
Read.drop_collection()
User.drop_collection()
Feed.drop_collection()
Tag.drop_collection()
WebSite.drop_collection()
Author.drop_collection()


@override_settings(STATICFILES_STORAGE=
                   'pipeline.storage.NonPackagingPipelineStorage',
                   CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
                   CELERY_ALWAYS_EAGER=True,
                   BROKER_BACKEND='memory',)
class HomeAndPreferencesViewTest(TestCase):
コード例 #8
0
ファイル: stats.py プロジェクト: dolanor-galaxy/1flow
def article_content_error_types():
    """ Return an error classifier on the ``content_error`` attribute. """

    return ContentErrorClassifier(
        Article.objects(content_error__ne='').no_cache(),
        'content_error').classify()