def synchronize_mongodb_statsd_articles_gauges(full=False):
""" synchronize all articles-related gauges on our statsd server. """
with benchmark('synchronize statsd gauges for Article.*'):
empty = Article.objects(content_type=0).no_cache()
# empty_pending = empty.filter(content_error='', url_error='')
# empty_content_error = empty.filter(content_error__ne='')
# empty_url_error = empty.filter(url_error__ne='')
parsed = Article.objects(
content_type__ne=CONTENT_TYPES.NONE)
html = parsed.filter(content_type=CONTENT_TYPES.HTML)
markdown = parsed.filter(content_type=CONTENT_TYPES.MARKDOWN)
absolutes = Article.objects(url_absolute=True).no_cache()
duplicates = Article.objects(duplicate_of__ne=None).no_cache()
orphaned = Article.objects(orphaned=True).no_cache()
content_errors = Article.objects(content_error__ne='').no_cache()
url_errors = Article.objects(url_error__ne='').no_cache()
statsd.gauge('mongo.articles.counts.total',
Article._get_collection().count())
statsd.gauge('mongo.articles.counts.markdown', markdown.count())
statsd.gauge('mongo.articles.counts.html', html.count())
statsd.gauge('mongo.articles.counts.empty', empty.count())
statsd.gauge('mongo.articles.counts.content_errors',
content_errors.count())
statsd.gauge('mongo.articles.counts.url_errors', url_errors.count())
if full:
statsd.gauge('mongo.articles.counts.orphaned', orphaned.count())
statsd.gauge('mongo.articles.counts.absolutes', absolutes.count())
statsd.gauge('mongo.articles.counts.duplicates', duplicates.count())
def synchronize_mongodb_statsd_articles_gauges(full=False):
""" synchronize all articles-related gauges on our statsd server. """
with benchmark('synchronize statsd gauges for Article.*'):
empty = Article.objects(content_type=0).no_cache()
# empty_pending = empty.filter(content_error='', url_error='')
# empty_content_error = empty.filter(content_error__ne='')
# empty_url_error = empty.filter(url_error__ne='')
parsed = Article.objects(content_type__ne=CONTENT_TYPES.NONE)
html = parsed.filter(content_type=CONTENT_TYPES.HTML)
markdown = parsed.filter(content_type=CONTENT_TYPES.MARKDOWN)
absolutes = Article.objects(url_absolute=True).no_cache()
duplicates = Article.objects(duplicate_of__ne=None).no_cache()
orphaned = Article.objects(orphaned=True).no_cache()
content_errors = Article.objects(content_error__ne='').no_cache()
url_errors = Article.objects(url_error__ne='').no_cache()
statsd.gauge('mongo.articles.counts.total',
Article._get_collection().count())
statsd.gauge('mongo.articles.counts.markdown', markdown.count())
statsd.gauge('mongo.articles.counts.html', html.count())
statsd.gauge('mongo.articles.counts.empty', empty.count())
statsd.gauge('mongo.articles.counts.content_errors',
content_errors.count())
statsd.gauge('mongo.articles.counts.url_errors', url_errors.count())
if full:
statsd.gauge('mongo.articles.counts.orphaned', orphaned.count())
statsd.gauge('mongo.articles.counts.absolutes', absolutes.count())
statsd.gauge('mongo.articles.counts.duplicates',
duplicates.count())
def article_content_error_types():
""" Return an error classifier on the ``content_error`` attribute. """
return ContentErrorClassifier(
Article.objects(content_error__ne='').no_cache(),
'content_error'
).classify()
def article_url_error_types():
""" Return an error classifier on the ``url_error`` attribute. """
# Next to investigate:
# list index out of range: 758
# 'NoneType' object has no attribute 'findAll': 137
return UrlErrorClassifier(
Article.objects(url_error__ne='').no_cache(), 'url_error').classify()
def article_url_error_types():
""" Return an error classifier on the ``url_error`` attribute. """
# Next to investigate:
# list index out of range: 758
# 'NoneType' object has no attribute 'findAll': 137
return UrlErrorClassifier(
Article.objects(url_error__ne='').no_cache(),
'url_error'
).classify()
def tearDownClass(cls):
Article.drop_collection()
Feed.drop_collection()
from oneflow.base.tests import (connect_mongodb_testsuite, TEST_REDIS)
#from unittest import skip
DjangoUser = get_user_model()
LOGGER = logging.getLogger(__file__)
# Use the test database not to pollute the production/development one.
RedisStatsCounter.REDIS = TEST_REDIS
TEST_REDIS.flushdb()
connect_mongodb_testsuite()
# Empty the database before starting in case an old test failed to tearDown().
Article.drop_collection()
Read.drop_collection()
User.drop_collection()
Feed.drop_collection()
Tag.drop_collection()
WebSite.drop_collection()
Author.drop_collection()
@override_settings(STATICFILES_STORAGE=
'pipeline.storage.NonPackagingPipelineStorage',
CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
CELERY_ALWAYS_EAGER=True,
BROKER_BACKEND='memory',)
class HomeAndPreferencesViewTest(TestCase):
def article_content_error_types():
""" Return an error classifier on the ``content_error`` attribute. """
return ContentErrorClassifier(
Article.objects(content_error__ne='').no_cache(),
'content_error').classify()