def get_mailman_user(self): # Only cache the user_id, not the whole user instance, because # mailmanclient is not pickle-safe cache_key = "User:%s:mailman_user_id" % self.id user_id = cache.get(cache_key) try: mm_client = get_mailman_client() if user_id is None: try: mm_user = mm_client.get_user(self.user.email) except HTTPError as e: if e.code != 404: raise # will be caught down there mm_user = mm_client.create_user(self.user.email, self.user.get_full_name()) # XXX The email is not set as verified, because we don't # know if the registration that was used verified it. logger.info("Created Mailman user for %s (%s)", self.user.username, self.user.email) cache.set(cache_key, mm_user.user_id, None) return mm_user else: return mm_client.get_user(user_id) except (HTTPError, MailmanConnectionError) as e: logger.warning( "Error getting or creating the Mailman user of %s (%s): %s", self.user.username, self.user.email, e ) return None
def recent_threads_count(self): begin_date, end_date = self.get_recent_dates() cache_key = "MailingList:%s:recent_threads_count" % self.name result = cache.get(cache_key) if result is None: result = self.get_threads_between(begin_date, end_date).count() cache.set(cache_key, result, 3600 * 12) # 12 hours return result
def test_clear_recent_threads_cache(self): # The recent threads cache must be cleared when a new message arrives mlist = MailingList.objects.create(name="example-list") cache.set("MailingList:example-list:recent_threads", "test-value") cache.set("MailingList:example-list:recent_threads_count", "test-value") msg = Message() msg["From"] = "*****@*****.**" msg["Subject"] = "Fake Subject" msg["Message-ID"] = "<dummy>" msg.set_payload("Fake Message") m_hash = add_to_list("example-list", msg) thread = Thread.objects.get(thread_id=m_hash) self.assertEqual( cache.get("MailingList:example-list:recent_threads"), [thread.id]) self.assertEqual( [t.thread_id for t in mlist.recent_threads], [m_hash]) self.assertEqual( cache.get("MailingList:example-list:recent_threads_count"), 1)
def on_thread_added(self, thread): cache_key = "MailingList:%s:recent_threads" % self.name recent_thread_ids = cache.get(cache_key) if recent_thread_ids is not None and len(recent_thread_ids) >= 1000: # It's a high-volume list, just append to the cache recent_thread_ids.append(thread.id) cache.set(cache_key, recent_thread_ids, 3600 * 12) # 12 hours cache.set("%s_count" % cache_key, len(recent_thread_ids), 3600 * 12) # 12 hours else: # Low-volume list, rebuild the cache self._recent_threads_cache_rebuild()
def recent_threads(self): begin_date, end_date = self.get_recent_dates() # Only cache the list of thread ids, or it may go over memcached's size # limit (1MB) cache_key = "MailingList:%s:recent_threads" % self.name thread_ids = cache.get(cache_key) if thread_ids is None: threads = self.get_threads_between(begin_date, end_date) cache.set(cache_key, [t.id for t in threads], 3600 * 6) # 6 hours else: threads = Thread.objects.filter(id__in=thread_ids) return threads
def test_recent_threads_cache_high_volume(self): # On high volume lists, the recent threads cache is just appended to # instead of rebuilt (it will be rebuilt by a cron job daily). # High volume lists are those with more than 1000 recent threads. MailingList.objects.create(name="example-list") existing = list(range(1000)) cache.set("MailingList:example-list:recent_threads", existing) cache.set("MailingList:example-list:recent_threads_count", len(existing)) msg = Message() msg["From"] = "*****@*****.**" msg["Subject"] = "Fake Subject" msg["Message-ID"] = "<dummy>" msg.set_payload("Fake Message") m_hash = add_to_list("example-list", msg) thread = Thread.objects.get(thread_id=m_hash) self.assertEqual( cache.get("MailingList:example-list:recent_threads"), existing + [thread.id]) self.assertEqual( cache.get("MailingList:example-list:recent_threads_count"), len(existing) + 1)
def get_mailman_user(self): # Only cache the user_id, not the whole user instance, because # mailmanclient is not pickle-safe cache_key = "User:%s:mailman_user_id" % self.id user_id = cache.get(cache_key) try: mm_client = get_mailman_client() if user_id is None: mm_user = mm_client.get_user(self.user.email) cache.set(cache_key, mm_user.user_id, None) return mm_user else: return mm_client.get_user(user_id) except (HTTPError, MailmanConnectionError): return None
def test_subscribe_not_subscribed(self): self.ml.settings["subscription_policy"] = "open" self.ml.get_member.side_effect = ValueError cache.set("User:%s:subscriptions" % self.user.id, "test-value") class Prefs(dict): save = Mock() member = Mock() member.preferences = Prefs() self.ml.subscribe.side_effect = lambda *a: member mailman.subscribe("*****@*****.**", self.user) self.assertTrue(self.ml.get_member.called) self.assertTrue(self.ml.subscribe.called) self.assertEqual(member.preferences["delivery_status"], "by_user") self.assertTrue(member.preferences.save.called) self.assertEqual(cache.get("User:%s:subscriptions" % self.user.id), None)
def test_subscribe_not_subscribed(self): self.ml.settings["subscription_policy"] = "open" self.ml.get_member.side_effect = ValueError cache.set("User:%s:subscriptions" % self.user.id, "test-value") class Prefs(dict): save = Mock() member = Mock() member.preferences = Prefs() self.ml.subscribe.side_effect = lambda *a, **kw: member mailman.subscribe("*****@*****.**", self.user) self.assertTrue(self.ml.get_member.called) self.ml.subscribe.assert_called_with('*****@*****.**', ' ', pre_verified=True, pre_confirmed=True) self.assertEqual(member.preferences["delivery_status"], "by_user") self.assertTrue(member.preferences.save.called) self.assertEqual(cache.get("User:%s:subscriptions" % self.user.id), None)
def test_subscribe_moderate_undetected(self): # The list requires moderation but we failed to detect it in the # possible subscription policies. If the subscription requires a # confirmation, Mailman will reply with a 202 code, and mailman.client # will return the response content (a dict) instead of a Member # instance. Make sure we can handle that. cache.set("User:%s:subscriptions" % self.user.id, "test-value", version=2) self.ml.settings["subscription_policy"] = "open" self.ml.get_member.side_effect = ValueError response_dict = {'token_owner': 'subscriber', 'http_etag': '"deadbeef"', 'token': 'deadbeefdeadbeef'} self.ml.subscribe.side_effect = lambda *a, **kw: response_dict try: mailman.subscribe("*****@*****.**", self.user) except AttributeError: self.fail("This use case was not properly handled") self.assertTrue(self.ml.get_member.called) # There must be no exception even if the response is not a Member. # Cache was not cleared because the subscription was not done self.assertEqual( cache.get("User:%s:subscriptions" % self.user.id, version=2), "test-value")