def reload_wiki_traffic_stats():
if settings.STAGE:
print('Skipped reload_wiki_traffic_stats(). '
'Set settings.STAGE to False to run it for real.')
return
for period, _ in PERIODS:
WikiDocumentVisits.reload_period_from_analytics(period)
def reload_wiki_traffic_stats():
transaction.enter_transaction_management()
transaction.managed(True)
for period, _ in PERIODS:
try:
WikiDocumentVisits.reload_period_from_analytics(period)
except:
transaction.rollback()
raise
else:
transaction.commit()
# Nice but not necessary when the process is about to exit:
transaction.leave_transaction_management()
def reload_wiki_traffic_stats():
transaction.enter_transaction_management()
transaction.managed(True)
for period, _ in PERIODS:
try:
WikiDocumentVisits.reload_period_from_analytics(period)
except:
transaction.rollback()
raise
else:
transaction.commit()
# Nice but not necessary when the process is about to exit:
transaction.leave_transaction_management()
def test_unknown_view(self):
"""Skip URLs that don't resolve."""
eq_({},
WikiDocumentVisits._visit_counts(
'{"data": {"12/01/2010-12/07/'
'2010": {"SubRows":{"http://support.mozilla.com/%s/unknown/":8}}}}'
% settings.LANGUAGE_CODE))
def test_good_visit_count(self):
"""Extract visit counts from good data.
It has some nasty non-ASCII chars in it.
"""
d = revision(document=document(slug='hellỗ', save=True),
is_approved=True,
save=True).document
d2 = revision(document=document(slug='there', save=True),
is_approved=True,
save=True).document
# We get a str, not a unicode obj, out of the urllib call.
eq_({
d.pk: 1037639,
d2.pk: 213817
},
WikiDocumentVisits._visit_counts(
'{"data": {"12/01/2010-12/07/2010": {"SubRows":{'
'"http://support.mozilla.com/%s/kb/hellỗ":{"Attributes":{"Title":'
'"Firefox Support Home Page | Firefox Support","UrlLink":'
'"http://support.mozilla.com/en-US/home/"},"measures":'
'{"Visits":1037639.0,"Views":3357731.0,"Average Time Viewed":23.0'
'},"SubRows":null},"http://support.mozilla.com/%s/kb/there":'
'{"Attributes":{"Title":"Startseite der Firefox-Hilfe | Firefox'
'Support","UrlLink":"http://support.mozilla.com/de/home/"},'
'"measures":{"Visits":213817.0,"Views":595329.0,"Average Time '
'Viewed":25.0},"SubRows":null}}}}}' %
((settings.LANGUAGE_CODE, ) * 2)))
def test_no_pages(self):
"""Don't pave over current data if WebTrends returns well-formatted
data structure with no interesting data in it."""
# Get some JSON that contains no interesting data.
no_pages = '{"data": {"12/01/2010-12/07/2010": {"SubRows": {}}}}'
counts = WikiDocumentVisits._visit_counts(no_pages)
eq_({}, counts) # Make sure nothing interesting is there.
# Try to reload visits table from the uninteresting data:
d = document()
d.save()
v = WikiDocumentVisits.objects.create(document=d, visits=12, period=THIS_WEEK)
WikiDocumentVisits.reload_period_from_json(THIS_WEEK, no_pages)
# Visits table should remain unchanged:
eq_(1, WikiDocumentVisits.objects.filter(pk=v.pk).count())
def test_good_visit_count(self):
"""Extract visit counts from good data.
It has some nasty non-ASCII chars in it.
"""
d = document(slug="hellỗ")
d.save()
d2 = document(slug="there")
d2.save()
# We get a str, not a unicode obj, out of the urllib call.
eq_(
{d.pk: 1037639, d2.pk: 213817},
WikiDocumentVisits._visit_counts(
'{"data": {"12/01/2010-12/07/2010": {"SubRows":{'
'"http://support.mozilla.com/%s/kb/hellỗ":{"Attributes":{"Title":'
'"Firefox Support Home Page | Firefox Support","UrlLink":'
'"http://support.mozilla.com/en-US/home/"},"measures":'
'{"Visits":1037639.0,"Views":3357731.0,"Average Time Viewed":23.0'
'},"SubRows":null},"http://support.mozilla.com/%s/kb/there":'
'{"Attributes":{"Title":"Startseite der Firefox-Hilfe | Firefox'
'Support","UrlLink":"http://support.mozilla.com/de/home/"},'
'"measures":{"Visits":213817.0,"Views":595329.0,"Average Time '
'Viewed":25.0},"SubRows":null}}}}}' % ((settings.LANGUAGE_CODE,) * 2)
),
)
def test_non_document_view(self):
"""Skip URLs that don't resolve to the wiki document view."""
eq_({},
WikiDocumentVisits._visit_counts(
'{"data": {"12/01/2010-12/07/'
'2010": {"SubRows":{"http://support.mozilla.com/%s/contributors":8'
'}}}}' % settings.LANGUAGE_CODE))
def test_bad_visit_count(self):
"""Skip URLs whose visit counts aren't ints."""
d = revision(is_approved=True, save=True).document
eq_({}, WikiDocumentVisits._visit_counts('{"data": {"12/01/2010-12/07/'
'2010": {"SubRows":{"http://support.mozilla.com/%s/kb/%s":{'
'"measures":{"Visits":"non-integer"}}}}}}'
% (settings.LANGUAGE_CODE, d.slug)))
def test_foreign_locale(self):
"""Skip URLs with non-English locale."""
eq_({},
WikiDocumentVisits._visit_counts(
'{"data": {"12/01/2010-12/07/'
'2010": {"SubRows":{"http://support.mozilla.com/zh/home/":8}}}}'
))
def test_bad_visit_count(self):
"""Skip URLs whose visit counts aren't ints."""
d = revision(is_approved=True, save=True).document
eq_({}, WikiDocumentVisits._visit_counts('{"data": {"12/01/2010-12/07/'
'2010": {"SubRows":{"http://support.mozilla.com/%s/kb/%s":{'
'"measures":{"Visits":"non-integer"}}}}}}'
% (settings.LANGUAGE_CODE, d.slug)))
def test_foreign_locale(self):
"""Skip URLs with non-English locale."""
eq_(
{},
WikiDocumentVisits._visit_counts(
'{"data": {"12/01/2010-12/07/' '2010": {"SubRows":{"http://support.mozilla.com/zh/home/":8}}}}'
),
)
def test_no_pages(self):
"""Don't pave over current data if WebTrends returns well-formatted
data structure with no interesting data in it."""
# Get some JSON that contains no interesting data.
no_pages = '{"data": {"12/01/2010-12/07/2010": {"SubRows": {}}}}'
counts = WikiDocumentVisits._visit_counts(no_pages)
eq_({}, counts) # Make sure nothing interesting is there.
# Try to reload visits table from the uninteresting data:
d = document()
d.save()
v = WikiDocumentVisits.objects.create(document=d, visits=12,
period=THIS_WEEK)
WikiDocumentVisits.reload_period_from_json(THIS_WEEK, no_pages)
# Visits table should remain unchanged:
eq_(1, WikiDocumentVisits.objects.filter(pk=v.pk).count())
def test_bad_page_info(self):
"""Skip URLs whose page info is unsubscriptable."""
d = revision(is_approved=True, save=True).document
eq_({},
WikiDocumentVisits._visit_counts(
'{"data": {"12/01/2010-12/07/'
'2010": {"SubRows":{"http://support.mozilla.com/%s/kb/%s":8}}}}'
% (settings.LANGUAGE_CODE, d.slug)))
def test_unknown_view(self):
"""Skip URLs that don't resolve."""
eq_(
{},
WikiDocumentVisits._visit_counts(
'{"data": {"12/01/2010-12/07/'
'2010": {"SubRows":{"http://support.mozilla.com/%s/unknown/":8}}}}' % settings.LANGUAGE_CODE
),
)
def test_non_document_view(self):
"""Skip URLs that don't resolve to the wiki document view."""
eq_(
{},
WikiDocumentVisits._visit_counts(
'{"data": {"12/01/2010-12/07/'
'2010": {"SubRows":{"http://support.mozilla.com/%s/contributors":8'
"}}}}" % settings.LANGUAGE_CODE
),
)
def test_bad_page_info(self):
"""Skip URLs whose page info is unsubscriptable."""
d = document()
d.save()
eq_(
{},
WikiDocumentVisits._visit_counts(
'{"data": {"12/01/2010-12/07/'
'2010": {"SubRows":{"http://support.mozilla.com/%s/kb/%s":8}}}}' % (settings.LANGUAGE_CODE, d.slug)
),
)
def test_visit_count_from_analytics(self, _build_request):
"""Verify stored visit counts from mocked analytics data.
It has some nasty non-ASCII chars in it.
"""
execute = _build_request.return_value.get.return_value.execute
execute.return_value = PAGEVIEWS_BY_DOCUMENT_RESPONSE
d1 = revision(document=document(slug='hellỗ', save=True),
is_approved=True, save=True).document
d2 = revision(document=document(slug='there', save=True),
is_approved=True, save=True).document
WikiDocumentVisits.reload_period_from_analytics(LAST_7_DAYS)
eq_(2, WikiDocumentVisits.objects.count())
wdv1 = WikiDocumentVisits.objects.get(document=d1)
eq_(27, wdv1.visits)
eq_(LAST_7_DAYS, wdv1.period)
wdv2 = WikiDocumentVisits.objects.get(document=d2)
eq_(LAST_7_DAYS, wdv2.period)
def test_visit_count_from_analytics(self, _build_request):
"""Verify stored visit counts from mocked analytics data.
It has some nasty non-ASCII chars in it.
"""
execute = _build_request.return_value.get.return_value.execute
execute.return_value = PAGEVIEWS_BY_DOCUMENT_RESPONSE
d1 = revision(document=document(slug='hellỗ', save=True),
is_approved=True,
save=True).document
d2 = revision(document=document(slug='there', save=True),
is_approved=True,
save=True).document
WikiDocumentVisits.reload_period_from_analytics(LAST_7_DAYS)
eq_(2, WikiDocumentVisits.objects.count())
wdv1 = WikiDocumentVisits.objects.get(document=d1)
eq_(27, wdv1.visits)
eq_(LAST_7_DAYS, wdv1.period)
wdv2 = WikiDocumentVisits.objects.get(document=d2)
eq_(LAST_7_DAYS, wdv2.period)
def test_no_locale(self):
"""Skip URLs with no locale."""
eq_({}, WikiDocumentVisits._visit_counts('{"data": {"12/01/2010-12/07/'
'2010": {"SubRows":{"http://support.mozilla.com/home":8}}}}'))
def test_not_subscriptable(self):
"""Raise a nice err if WebTrends returns an unsubscriptable obj."""
WikiDocumentVisits._visit_counts('8')
def test_no_data_attr(self):
"""Raise a nice err if WebTrends returns an obj with no 'data' attr."""
WikiDocumentVisits._visit_counts('{}')
def test_bad_json(self):
"""Raise a nice error if WebTrends hands us bad JSON."""
WikiDocumentVisits._visit_counts('{')
def reload_wiki_traffic_stats():
for period, _ in PERIODS:
WikiDocumentVisits.reload_period_from_analytics(period)
def test_bad_json(self):
"""Raise a nice error if WebTrends hands us bad JSON."""
WikiDocumentVisits._visit_counts('{')
def test_no_data_attr(self):
"""Raise a nice err if WebTrends returns an obj with no 'data' attr."""
WikiDocumentVisits._visit_counts('{}')
def test_not_subscriptable(self):
"""Raise a nice err if WebTrends returns an unsubscriptable obj."""
WikiDocumentVisits._visit_counts('8')