def get_mailman_user(self):
# Only cache the user_id, not the whole user instance, because
# mailmanclient is not pickle-safe
cache_key = "User:%s:mailman_user_id" % self.id
user_id = cache.get(cache_key)
try:
mm_client = get_mailman_client()
if user_id is None:
try:
mm_user = mm_client.get_user(self.user.email)
except HTTPError as e:
if e.code != 404:
raise # will be caught down there
mm_user = mm_client.create_user(self.user.email, self.user.get_full_name())
# XXX The email is not set as verified, because we don't
# know if the registration that was used verified it.
logger.info("Created Mailman user for %s (%s)", self.user.username, self.user.email)
cache.set(cache_key, mm_user.user_id, None)
return mm_user
else:
return mm_client.get_user(user_id)
except (HTTPError, MailmanConnectionError) as e:
logger.warning(
"Error getting or creating the Mailman user of %s (%s): %s", self.user.username, self.user.email, e
)
return None
def recent_threads_count(self):
begin_date, end_date = self.get_recent_dates()
cache_key = "MailingList:%s:recent_threads_count" % self.name
result = cache.get(cache_key)
if result is None:
result = self.get_threads_between(begin_date, end_date).count()
cache.set(cache_key, result, 3600 * 12) # 12 hours
return result
def test_clear_recent_threads_cache(self):
# The recent threads cache must be cleared when a new message arrives
mlist = MailingList.objects.create(name="example-list")
cache.set("MailingList:example-list:recent_threads", "test-value")
cache.set("MailingList:example-list:recent_threads_count", "test-value")
msg = Message()
msg["From"] = "*****@*****.**"
msg["Subject"] = "Fake Subject"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Fake Message")
m_hash = add_to_list("example-list", msg)
thread = Thread.objects.get(thread_id=m_hash)
self.assertEqual(
cache.get("MailingList:example-list:recent_threads"), [thread.id])
self.assertEqual(
[t.thread_id for t in mlist.recent_threads], [m_hash])
self.assertEqual(
cache.get("MailingList:example-list:recent_threads_count"), 1)
def on_thread_added(self, thread):
cache_key = "MailingList:%s:recent_threads" % self.name
recent_thread_ids = cache.get(cache_key)
if recent_thread_ids is not None and len(recent_thread_ids) >= 1000:
# It's a high-volume list, just append to the cache
recent_thread_ids.append(thread.id)
cache.set(cache_key, recent_thread_ids, 3600 * 12) # 12 hours
cache.set("%s_count" % cache_key, len(recent_thread_ids), 3600 * 12) # 12 hours
else:
# Low-volume list, rebuild the cache
self._recent_threads_cache_rebuild()
def recent_threads(self):
begin_date, end_date = self.get_recent_dates()
# Only cache the list of thread ids, or it may go over memcached's size
# limit (1MB)
cache_key = "MailingList:%s:recent_threads" % self.name
thread_ids = cache.get(cache_key)
if thread_ids is None:
threads = self.get_threads_between(begin_date, end_date)
cache.set(cache_key, [t.id for t in threads], 3600 * 6) # 6 hours
else:
threads = Thread.objects.filter(id__in=thread_ids)
return threads
def test_recent_threads_cache_high_volume(self):
# On high volume lists, the recent threads cache is just appended to
# instead of rebuilt (it will be rebuilt by a cron job daily).
# High volume lists are those with more than 1000 recent threads.
MailingList.objects.create(name="example-list")
existing = list(range(1000))
cache.set("MailingList:example-list:recent_threads", existing)
cache.set("MailingList:example-list:recent_threads_count",
len(existing))
msg = Message()
msg["From"] = "*****@*****.**"
msg["Subject"] = "Fake Subject"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Fake Message")
m_hash = add_to_list("example-list", msg)
thread = Thread.objects.get(thread_id=m_hash)
self.assertEqual(
cache.get("MailingList:example-list:recent_threads"),
existing + [thread.id])
self.assertEqual(
cache.get("MailingList:example-list:recent_threads_count"),
len(existing) + 1)
def get_mailman_user(self):
# Only cache the user_id, not the whole user instance, because
# mailmanclient is not pickle-safe
cache_key = "User:%s:mailman_user_id" % self.id
user_id = cache.get(cache_key)
try:
mm_client = get_mailman_client()
if user_id is None:
mm_user = mm_client.get_user(self.user.email)
cache.set(cache_key, mm_user.user_id, None)
return mm_user
else:
return mm_client.get_user(user_id)
except (HTTPError, MailmanConnectionError):
return None
def test_subscribe_not_subscribed(self):
self.ml.settings["subscription_policy"] = "open"
self.ml.get_member.side_effect = ValueError
cache.set("User:%s:subscriptions" % self.user.id, "test-value")
class Prefs(dict):
save = Mock()
member = Mock()
member.preferences = Prefs()
self.ml.subscribe.side_effect = lambda *a: member
mailman.subscribe("*****@*****.**", self.user)
self.assertTrue(self.ml.get_member.called)
self.assertTrue(self.ml.subscribe.called)
self.assertEqual(member.preferences["delivery_status"], "by_user")
self.assertTrue(member.preferences.save.called)
self.assertEqual(cache.get("User:%s:subscriptions" % self.user.id),
None)
def test_subscribe_not_subscribed(self):
self.ml.settings["subscription_policy"] = "open"
self.ml.get_member.side_effect = ValueError
cache.set("User:%s:subscriptions" % self.user.id, "test-value")
class Prefs(dict):
save = Mock()
member = Mock()
member.preferences = Prefs()
self.ml.subscribe.side_effect = lambda *a, **kw: member
mailman.subscribe("*****@*****.**", self.user)
self.assertTrue(self.ml.get_member.called)
self.ml.subscribe.assert_called_with('*****@*****.**',
' ',
pre_verified=True,
pre_confirmed=True)
self.assertEqual(member.preferences["delivery_status"], "by_user")
self.assertTrue(member.preferences.save.called)
self.assertEqual(cache.get("User:%s:subscriptions" % self.user.id),
None)
def test_subscribe_moderate_undetected(self):
# The list requires moderation but we failed to detect it in the
# possible subscription policies. If the subscription requires a
# confirmation, Mailman will reply with a 202 code, and mailman.client
# will return the response content (a dict) instead of a Member
# instance. Make sure we can handle that.
cache.set("User:%s:subscriptions" % self.user.id,
"test-value", version=2)
self.ml.settings["subscription_policy"] = "open"
self.ml.get_member.side_effect = ValueError
response_dict = {'token_owner': 'subscriber', 'http_etag': '"deadbeef"',
'token': 'deadbeefdeadbeef'}
self.ml.subscribe.side_effect = lambda *a, **kw: response_dict
try:
mailman.subscribe("*****@*****.**", self.user)
except AttributeError:
self.fail("This use case was not properly handled")
self.assertTrue(self.ml.get_member.called)
# There must be no exception even if the response is not a Member.
# Cache was not cleared because the subscription was not done
self.assertEqual(
cache.get("User:%s:subscriptions" % self.user.id, version=2),
"test-value")
