def _get_msg(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["Message-ID-Hash"] = "QKODQBCADMDSP5YPOPKECXQWEQAMXZL3"
msg.set_payload("Dummy message")
return msg
def test_add_in_classical_thread(self):
# msg1
# |-msg2
# | `-msg4
# `-msg3
ml = FakeList("example-list")
msgs = []
for num in range(1, 5):
msg = Message()
msg["From"] = "*****@*****.**" % num
msg["Message-ID"] = "<msg%d>" % num
msg.set_payload("message %d" % num)
msgs.append(msg)
msgs[1]["In-Reply-To"] = "<msg1>"
msgs[2]["In-Reply-To"] = "<msg1>"
msgs[3]["In-Reply-To"] = "<msg2>"
for msg in msgs:
self.store.add_to_list(ml, msg)
msgs = []
for num in range(1, 5):
msg = self.store.get_message_by_id_from_list(
"example-list", "msg%d" % num)
msgs.append(msg)
msg1, msg2, msg3, msg4 = msgs
self.assertEqual(msg1.thread_order, 0)
self.assertEqual(msg1.thread_depth, 0)
self.assertEqual(msg2.thread_order, 1)
self.assertEqual(msg2.thread_depth, 1)
self.assertEqual(msg3.thread_order, 3)
self.assertEqual(msg3.thread_depth, 1)
self.assertEqual(msg4.thread_order, 2)
self.assertEqual(msg4.thread_depth, 2)
def test_no_from(self):
msg = Message()
msg.set_payload("Dummy message")
try:
name, email = kittystore.utils.parseaddr(msg["From"])
except AttributeError, e:
self.fail(e)
def test_on_new_message_invalidate(self):
# Check that the cache is invalidated on new message
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
today = datetime.datetime.utcnow().date() # don't use datetime.date.today(), we need UTC
self.store.add_to_list("example-list", msg)
# calls to cache.delete() -- invalidation
delete_args = [ call[0][0] for call in
self.store.db.cache.delete.call_args_list ]
#from pprint import pprint; pprint(delete_args)
self.assertEqual(set(delete_args), set([
u'list:example-list:recent_participants_count',
u'list:example-list:recent_threads_count',
u'list:example-list:participants_count:%d:%d' % (today.year, today.month),
u'list:example-list:thread:QKODQBCADMDSP5YPOPKECXQWEQAMXZL3:emails_count',
u'list:example-list:thread:QKODQBCADMDSP5YPOPKECXQWEQAMXZL3:participants_count'
]))
# calls to cache.get_or_create() -- repopulation
goc_args = [ call[0][0] for call in
self.store.db.cache.get_or_create.call_args_list ]
#from pprint import pprint; pprint(goc_args)
self.assertEqual(set(goc_args), set([
u'list:example-list:recent_participants_count',
u'list:example-list:recent_threads_count',
u'list:example-list:participants_count:%d:%d' % (today.year, today.month),
u'list:example-list:threads_count:%d:%d' % (today.year, today.month),
u'list:example-list:thread:QKODQBCADMDSP5YPOPKECXQWEQAMXZL3:emails_count',
u'list:example-list:thread:QKODQBCADMDSP5YPOPKECXQWEQAMXZL3:participants_count',
u'list:example-list:thread:QKODQBCADMDSP5YPOPKECXQWEQAMXZL3:starting_email_id',
]))
def test_sync_mailman_user(self):
# Check that the user_id is set when sync_mailman_user is run
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
self.store.add_to_list("example-list", msg)
dbmsg = self.store.get_message_by_id_from_list(
"example-list", "dummy")
self.assertEqual(dbmsg.sender.user_id, None)
# setup Mailman's reply
uid = uuid.uuid1()
new_user_id = FakeMMUser()
new_user_id.user_id = uid.int
self.mm_client.get_user.side_effect = lambda addr: new_user_id
# do the test and check
mailman_user.sync_mailman_user(self.store)
#dbmsg = self.store.get_message_by_id_from_list(
# "example-list", "dummy")
self.assertEqual(dbmsg.sender.user_id, uid)
self.assertTrue(dbmsg.sender.user is not None,
"A 'User' instance was not created")
self.assertEqual(dbmsg.sender.user.id, uid)
self.assertEqual(1,
self.store.get_message_count_by_user_id(uid))
def test_on_new_message_userid(self):
# Check that the user_id is set on a new message
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
# setup Mailman's reply
new_user_id = FakeMMUser()
uid = uuid.uuid1()
new_user_id.user_id = uid.int
self.mm_client.get_user.side_effect = lambda addr: new_user_id
# check the User does not exist yet
self.assertEqual(0,
self.store.get_message_count_by_user_id(uid))
# do the test and check
self.store.add_to_list("example-list", msg)
dbmsg = self.store.get_message_by_id_from_list(
"example-list", "dummy")
self.assertEqual(dbmsg.sender.user_id, uid)
self.assertTrue(dbmsg.sender.user is not None,
"A 'User' instance was not created")
self.assertEqual(dbmsg.sender.user.id, uid)
self.assertEqual(1,
self.store.get_message_count_by_user_id(uid))
self.assertEqual(self.store.get_users_count(), 1)
def setUp(self):
self.user = User.objects.create_user('testuser', '*****@*****.**', 'testPass')
self.client.login(username='******', password='******')
store = kittystore.get_store(SettingsModule(), debug=False, auto_create=True)
self.client.defaults = {"kittystore.store": store,
"HTTP_USER_AGENT": "testbot",
}
# Create test data
ml = FakeList("*****@*****.**")
ml.subject_prefix = u"[example] "
# Create 3 threads
messages = []
for msgnum in range(3):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<id%d>" % (msgnum+1)
msg["Subject"] = "Dummy message"
msg.set_payload("Dummy message")
store.add_to_list(ml, msg)
messages.append(msg)
# 1st is unread, 2nd is read, 3rd is updated
LastView.objects.create(list_address="*****@*****.**", user=self.user,
threadid=get_message_id_hash("<id2>"))
LastView.objects.create(list_address="*****@*****.**", user=self.user,
threadid=get_message_id_hash("<id3>"))
msg4 = Message()
msg4["From"] = "*****@*****.**"
msg4["Message-ID"] = "<id4>"
msg4["Subject"] = "Dummy message"
msg4["In-Reply-To"] = "<id3>"
msg4.set_payload("Dummy message")
store.add_to_list(ml, msg4)
def test_vote_cancel(self):
ml = FakeList("*****@*****.**")
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg1>"
msg.set_payload("Dummy message")
self.store.add_to_list(ml, msg)
msg.replace_header("Message-ID", "<msg2>")
self.store.add_to_list(ml, msg)
msg1 = self.store.get_message_by_id_from_list("*****@*****.**", "msg1")
msg1.vote(1, u"testuser")
msg2 = self.store.get_message_by_id_from_list("*****@*****.**", "msg2")
msg2.vote(-1, u"testuser")
self.assertEqual(msg1.likes, 1)
self.assertEqual(msg2.dislikes, 1)
for msg in (msg1, msg2):
url = reverse('message_vote', args=("*****@*****.**",
msg.message_id_hash))
resp = self.client.post(url, {"vote": "0"})
self.assertEqual(resp.status_code, 200)
self.assertEqual(msg.likes, 0)
self.assertEqual(msg.dislikes, 0)
result = json.loads(resp.content)
self.assertEqual(result["like"], 0)
self.assertEqual(result["dislike"], 0)
def setUp(self):
self.user = User.objects.create_user('testuser', '*****@*****.**',
'testPass')
self.user.is_staff = True
self.user.save()
self.client.login(username='******', password='******')
self.store = kittystore.get_store(SettingsModule(),
debug=False,
auto_create=True)
self.client.defaults = {
"kittystore.store": self.store,
"HTTP_USER_AGENT": "testbot",
}
ml = FakeList("*****@*****.**")
ml.subject_prefix = u"[example] "
# Create 2 threads
self.messages = []
for msgnum in range(2):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<id%d>" % (msgnum + 1)
msg["Subject"] = "Dummy message"
msg.set_payload("Dummy message")
msg["Message-ID-Hash"] = self.store.add_to_list(ml, msg)
self.messages.append(msg)
def setUp(self):
self.user = User.objects.create_user('testuser', '*****@*****.**', 'testPass')
self.client.login(username='******', password='******')
# Create test data
ml = FakeList("*****@*****.**")
ml.subject_prefix = u"[example] "
# Create 3 threads
messages = []
for msgnum in range(3):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<id%d>" % (msgnum+1)
msg["Subject"] = "Dummy message"
msg.set_payload("Dummy message")
self.store.add_to_list(ml, msg)
messages.append(msg)
# 1st is unread, 2nd is read, 3rd is updated
LastView.objects.create(list_address="*****@*****.**", user=self.user,
threadid=get_message_id_hash("<id2>"))
LastView.objects.create(list_address="*****@*****.**", user=self.user,
threadid=get_message_id_hash("<id3>"))
msg4 = Message()
msg4["From"] = "*****@*****.**"
msg4["Message-ID"] = "<id4>"
msg4["Subject"] = "Dummy message"
msg4["In-Reply-To"] = "<id3>"
msg4.set_payload("Dummy message")
self.store.add_to_list(ml, msg4)
def test_vote_cancel(self):
ml = FakeList("*****@*****.**")
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg1>"
msg.set_payload("Dummy message")
self.store.add_to_list(ml, msg)
msg.replace_header("Message-ID", "<msg2>")
self.store.add_to_list(ml, msg)
msg1 = self.store.get_message_by_id_from_list("*****@*****.**",
"msg1")
msg1.vote(1, u"testuser")
msg2 = self.store.get_message_by_id_from_list("*****@*****.**",
"msg2")
msg2.vote(-1, u"testuser")
self.assertEqual(msg1.likes, 1)
self.assertEqual(msg2.dislikes, 1)
for msg in (msg1, msg2):
url = reverse('message_vote',
args=("*****@*****.**", msg.message_id_hash))
resp = self.client.post(url, {"vote": "0"})
self.assertEqual(resp.status_code, 200)
self.assertEqual(msg.likes, 0)
self.assertEqual(msg.dislikes, 0)
result = json.loads(resp.content)
self.assertEqual(result["like"], 0)
self.assertEqual(result["dislike"], 0)
def test_on_new_message_invalidate(self):
# Check that the cache is invalidated on new message
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
today = datetime.datetime.utcnow().date() # don't use datetime.date.today(), we need UTC
self.store.add_to_list("example-list", msg)
# calls to cache.delete() -- invalidation
delete_args = [ call[0][0] for call in
self.store.db.cache.delete.call_args_list ]
#from pprint import pprint; pprint(delete_args)
self.assertEqual(set(delete_args), set([
u'list:example-list:recent_participants_count',
u'list:example-list:recent_threads_count',
u'list:example-list:participants_count:%d:%d' % (today.year, today.month),
u'list:example-list:thread:QKODQBCADMDSP5YPOPKECXQWEQAMXZL3:emails_count',
u'list:example-list:thread:QKODQBCADMDSP5YPOPKECXQWEQAMXZL3:participants_count'
]))
# calls to cache.get_or_create() -- repopulation
goc_args = [ call[0][0] for call in
self.store.db.cache.get_or_create.call_args_list ]
#from pprint import pprint; pprint(goc_args)
self.assertEqual(set(goc_args), set([
u'list:example-list:recent_participants_count',
u'list:example-list:recent_threads_count',
u'list:example-list:participants_count:%d:%d' % (today.year, today.month),
u'list:example-list:threads_count:%d:%d' % (today.year, today.month),
u'list:example-list:thread:QKODQBCADMDSP5YPOPKECXQWEQAMXZL3:emails_count',
u'list:example-list:thread:QKODQBCADMDSP5YPOPKECXQWEQAMXZL3:participants_count',
u'list:example-list:thread:QKODQBCADMDSP5YPOPKECXQWEQAMXZL3:starting_email_id',
]))
def test_on_new_message_userid(self):
# Check that the user_id is set on a new message
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
# setup Mailman's reply
new_user_id = FakeMMUser()
uid = uuid.uuid1()
new_user_id.user_id = uid.int
self.mm_client.get_user.side_effect = lambda addr: new_user_id
# check the User does not exist yet
self.assertEqual(0,
self.store.get_message_count_by_user_id(uid))
# do the test and check
self.store.add_to_list("example-list", msg)
dbmsg = self.store.get_message_by_id_from_list(
"example-list", "dummy")
self.assertEqual(dbmsg.sender.user_id, uid)
self.assertTrue(dbmsg.sender.user is not None,
"A 'User' instance was not created")
self.assertEqual(dbmsg.sender.user.id, uid)
self.assertEqual(1,
self.store.get_message_count_by_user_id(uid))
self.assertEqual(self.store.get_users_count(), 1)
def test_properties_on_new_message(self):
ml = FakeList("example-list")
ml.display_name = u"name 1"
ml.subject_prefix = u"[prefix 1]"
ml.description = u"desc 1"
kittystore.utils.MM_CLIENT.get_list.side_effect = lambda n: ml
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
self.store.add_to_list("example-list", msg)
ml_db = self.store.get_lists()[0]
self.assertEqual(ml_db.display_name, "name 1")
self.assertEqual(ml_db.subject_prefix, "[prefix 1]")
ml.display_name = u"name 2"
ml.subject_prefix = u"[prefix 2]"
ml.description = u"desc 2"
ml.archive_policy = "private"
msg.replace_header("Message-ID", "<dummy2>")
self.store.add_to_list("example-list", msg)
ml_db = self.store.get_lists()[0]
#ml_db = self.store.db.find(List).one()
self.assertEqual(ml_db.display_name, "name 2")
self.assertEqual(ml_db.subject_prefix, "[prefix 2]")
self.assertEqual(ml_db.description, "desc 2")
self.assertEqual(ml_db.archive_policy, ArchivePolicy.private)
def test_sync_mailman_user(self):
# Check that the user_id is set when sync_mailman_user is run
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
self.store.add_to_list("example-list", msg)
dbmsg = self.store.get_message_by_id_from_list(
"example-list", "dummy")
self.assertEqual(dbmsg.sender.user_id, None)
# setup Mailman's reply
uid = uuid.uuid1()
new_user_id = FakeMMUser()
new_user_id.user_id = uid.int
self.mm_client.get_user.side_effect = lambda addr: new_user_id
# do the test and check
mailman_user.sync_mailman_user(self.store)
#dbmsg = self.store.get_message_by_id_from_list(
# "example-list", "dummy")
self.assertEqual(dbmsg.sender.user_id, uid)
self.assertTrue(dbmsg.sender.user is not None,
"A 'User' instance was not created")
self.assertEqual(dbmsg.sender.user.id, uid)
self.assertEqual(1,
self.store.get_message_count_by_user_id(uid))
def _get_msg(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["Message-ID-Hash"] = "QKODQBCADMDSP5YPOPKECXQWEQAMXZL3"
msg.set_payload("Dummy message")
return msg
def test_properties_on_new_message(self):
ml = FakeList("example-list")
ml.display_name = "name 1"
ml.subject_prefix = "[prefix 1]"
ml.description = "desc 1"
kittystore.utils.MM_CLIENT.get_list.side_effect = lambda n: ml
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
self.store.add_to_list("example-list", msg)
ml_db = self.store.get_lists()[0]
self.assertEqual(ml_db.display_name, "name 1")
self.assertEqual(ml_db.subject_prefix, "[prefix 1]")
ml.display_name = "name 2"
ml.subject_prefix = "[prefix 2]"
ml.description = "desc 2"
ml.archive_policy = "private"
msg.replace_header("Message-ID", "<dummy2>")
self.store.add_to_list("example-list", msg)
ml_db = self.store.get_lists()[0]
#ml_db = self.store.db.find(List).one()
self.assertEqual(ml_db.display_name, "name 2")
self.assertEqual(ml_db.subject_prefix, "[prefix 2]")
self.assertEqual(ml_db.description, "desc 2")
self.assertEqual(ml_db.archive_policy, ArchivePolicy.private)
def setUp(self):
self.user = User.objects.create_user('testuser', '*****@*****.**',
'testPass')
self.client.login(username='******', password='******')
store = kittystore.get_store(SettingsModule(), debug=False)
ml = FakeList("*****@*****.**")
ml.subject_prefix = u"[example] "
# Create 3 threads
messages = []
for msgnum in range(3):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<id%d>" % (msgnum + 1)
msg["Subject"] = "Dummy message"
msg.set_payload("Dummy message")
store.add_to_list(ml, msg)
messages.append(msg)
# 1st is unread, 2nd is read, 3rd is updated
LastView.objects.create(list_address="*****@*****.**",
user=self.user,
threadid=get_message_id_hash("<id2>"))
LastView.objects.create(list_address="*****@*****.**",
user=self.user,
threadid=get_message_id_hash("<id3>"))
msg4 = Message()
msg4["From"] = "*****@*****.**"
msg4["Message-ID"] = "<id4>"
msg4["Subject"] = "Dummy message"
msg4["In-Reply-To"] = "<id3>"
msg4.set_payload("Dummy message")
store.add_to_list(ml, msg4)
# Factory
defaults = {"kittystore.store": store, "HTTP_USER_AGENT": "testbot"}
self.factory = RequestFactory(**defaults)
def test_add_in_classical_thread(self):
# msg1
# |-msg2
# | `-msg4
# `-msg3
ml = FakeList("example-list")
msgs = []
for num in range(1, 5):
msg = Message()
msg["From"] = "*****@*****.**" % num
msg["Message-ID"] = "<msg%d>" % num
msg.set_payload("message %d" % num)
msgs.append(msg)
msgs[1]["In-Reply-To"] = "<msg1>"
msgs[2]["In-Reply-To"] = "<msg1>"
msgs[3]["In-Reply-To"] = "<msg2>"
for msg in msgs:
self.store.add_to_list(ml, msg)
msgs = []
for num in range(1, 5):
msg = self.store.get_message_by_id_from_list(
"example-list", "msg%d" % num)
msgs.append(msg)
msg1, msg2, msg3, msg4 = msgs
self.assertEqual(msg1.thread_order, 0)
self.assertEqual(msg1.thread_depth, 0)
self.assertEqual(msg2.thread_order, 1)
self.assertEqual(msg2.thread_depth, 1)
self.assertEqual(msg3.thread_order, 3)
self.assertEqual(msg3.thread_depth, 1)
self.assertEqual(msg4.thread_order, 2)
self.assertEqual(msg4.thread_depth, 2)
def test_multiple_reference(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["References"] = " <ref-1> <ref-2> "
msg.set_payload("Dummy message")
ref_id = kittystore.utils.get_ref(msg)
self.assertEqual(ref_id, "ref-2")
def test_in_reply_to(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["In-Reply-To"] = " <ref-1> "
msg.set_payload("Dummy message")
ref_id = kittystore.utils.get_ref(msg)
self.assertEqual(ref_id, "ref-1")
def setUp(self):
# Create the list by adding a dummy message
ml = FakeList("*****@*****.**")
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg>"
msg.set_payload("Dummy message")
self.store.add_to_list(ml, msg)
def test_multiple_reference(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["References"] = " <ref-1> <ref-2> "
msg.set_payload("Dummy message")
ref_id = kittystore.utils.get_ref(msg)
self.assertEqual(ref_id, "ref-2")
def test_in_reply_to(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["In-Reply-To"] = " <ref-1> "
msg.set_payload("Dummy message")
ref_id = kittystore.utils.get_ref(msg)
self.assertEqual(ref_id, "ref-1")
def test_get_sender_name(self):
msg = Message()
msg["From"] = "Sender Name <*****@*****.**>"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
self.store.add_to_list(FakeList("example-list"), msg)
stored_msg = self.store.db.query(Email).one()
user_id = stored_msg.sender.user_id
self.assertEqual(self.store.get_sender_name(user_id), "Sender Name")
def test_on_new_message_no_reply_from_mailman(self):
# Check that the user_id is set on a new message
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
self.store.add_to_list("example-list", msg)
dbmsg = self.store.get_message_by_id_from_list("example-list", "dummy")
self.assertEqual(dbmsg.sender.user_id, None)
def _create_email(self, num, reply_to=None):
ml = FakeList("example-list")
msg = Message()
msg["From"] = "*****@*****.**" % num
msg["Message-ID"] = "<msg%d>" % num
msg.set_payload("message %d" % num)
if reply_to is not None:
msg["In-Reply-To"] = "<msg%d>" % reply_to
self.store.add_to_list(ml, msg)
def _make_msg(self, msgid, reply_to=None):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<%s>" % msgid
msg["Subject"] = "Dummy message"
msg.set_payload("Dummy message")
if reply_to is not None:
msg["In-Reply-To"] = "<%s>" % reply_to
msg["Message-ID-Hash"] = self.store.add_to_list(self.ml, msg)
return msg
def test_empty_reference(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["References"] = " "
msg.set_payload("Dummy message")
try:
kittystore.utils.get_ref(msg)
except IndexError:
self.fail("Empty 'References' tag should be handled")
def test_in_reply_to_and_reference(self):
"""The In-Reply-To header should win over References"""
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["In-Reply-To"] = " <ref-1> "
msg["References"] = " <ref-2> "
msg.set_payload("Dummy message")
ref_id = kittystore.utils.get_ref(msg)
self.assertEqual(ref_id, "ref-1")
def test_no_date(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
now = datetime.datetime.utcnow()
try:
self.store.add_to_list(FakeList("example-list"), msg)
except IntegrityError, e:
self.fail(e)
def test_empty_reference(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["References"] = " "
msg.set_payload("Dummy message")
try:
ref_id = kittystore.utils.get_ref(msg)
except IndexError, e:
self.fail("Empty 'References' tag should be handled")
def test_in_reply_to_and_reference(self):
"""The In-Reply-To header should win over References"""
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["In-Reply-To"] = " <ref-1> "
msg["References"] = " <ref-2> "
msg.set_payload("Dummy message")
ref_id = kittystore.utils.get_ref(msg)
self.assertEqual(ref_id, "ref-1")
def test_date_aware(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["Date"] = "Fri, 02 Nov 2012 16:07:54 +0100"
msg.set_payload("Dummy message")
try:
self.store.add_to_list(FakeList("example-list"), msg)
except IntegrityError, e:
self.fail(e)
def test_date_aware(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["Date"] = "Fri, 02 Nov 2012 16:07:54 +0100"
msg.set_payload("Dummy message")
try:
self.store.add_to_list(FakeList("example-list"), msg)
except IntegrityError, e:
self.fail(e)
def test_duplicate(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
self.store.add_to_list(FakeList("example-list"), msg)
self.assertEqual(self.store.get_list_size("example-list"), 1)
self.assertTrue(self.store.is_message_in_list("example-list", "dummy"))
self.store.add_to_list(FakeList("example-list"), msg)
self.assertEqual(self.store.get_list_size("example-list"), 1)
def test_non_ascii_email_address(self):
"""Non-ascii email addresses should raise a ValueError exception"""
msg = Message()
msg["From"] = b"dummy-non-ascii-\xc3\[email protected]"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
try:
self.store.add_to_list(FakeList("example-list"), msg)
except ValueError, e:
self.assertEqual(e.__class__.__name__, "ValueError")
def test_no_date(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
now = datetime.datetime.utcnow()
try:
self.store.add_to_list(FakeList("example-list"), msg)
except IntegrityError, e:
self.fail(e)
def test_on_new_message_no_reply_from_mailman(self):
# Check that the user_id is set on a new message
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
self.store.add_to_list("example-list", msg)
dbmsg = self.store.get_message_by_id_from_list(
"example-list", "dummy")
self.assertEqual(dbmsg.sender.user_id, None)
def test_on_new_message_bad_reply_from_mailman(self):
# Check that errors from mailmanclient are handled gracefully
self.mm_client.get_user.side_effect = ValueError
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
try:
self.store.add_to_list("example-list", msg)
except ValueError, e:
self.fail("Errors from mailmanclient should be handled gracefully")
def test_on_new_message_bad_reply_from_mailman(self):
# Check that errors from mailmanclient are handled gracefully
self.mm_client.get_user.side_effect = ValueError
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
try:
self.store.add_to_list("example-list", msg)
except ValueError, e:
self.fail("Errors from mailmanclient should be handled gracefully")
def test_private_list(self):
# emails on private lists must not be found by a search on all lists
ml = FakeList("example-list")
ml.archive_policy = ArchivePolicy.private
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
self.store.add_to_list(ml, msg)
result = self.store.search("dummy")
self.assertEqual(result["total"], 0)
def test_subject(self):
ml = FakeList("example-list")
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummymsg>"
msg["Date"] = "Fri, 02 Nov 2012 16:07:54 +0000"
msg["Subject"] = "Dummy subject"
msg.set_payload("Dummy message")
self.store.add_to_list(ml, msg)
thread = self.store.db.find(Thread).one()
self.assertEqual(thread.subject, "Dummy subject")
def test_subject(self):
ml = FakeList("example-list")
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummymsg>"
msg["Date"] = "Fri, 02 Nov 2012 16:07:54 +0000"
msg["Subject"] = "Dummy subject"
msg.set_payload("Dummy message")
self.store.add_to_list(ml, msg)
thread = self.store.db.query(Thread).one()
self.assertEqual(thread.subject, "Dummy subject")
def test_long_message_id(self):
# Some message-ids are more than 255 chars long
# Check with assert here because SQLite will not enforce the limit
# (http://www.sqlite.org/faq.html#q9)
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "X" * 260
msg.set_payload("Dummy message")
try:
self.store.add_to_list(FakeList("example-list"), msg)
except IntegrityError, e:
self.fail(e)
def test_on_old_message(self):
kittystore.utils.MM_CLIENT = None
olddate = datetime.datetime.utcnow() - datetime.timedelta(days=40)
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["Date"] = olddate.isoformat()
msg.set_payload("Dummy message")
self.store.add_to_list("example-list", msg)
ml_db = self.store.get_lists()[0]
self.assertEqual(ml_db.recent_participants_count, 0)
self.assertEqual(ml_db.recent_threads_count, 0)
def test_on_old_message(self):
kittystore.utils.MM_CLIENT = None
olddate = datetime.datetime.utcnow() - datetime.timedelta(days=40)
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["Date"] = olddate.isoformat()
msg.set_payload("Dummy message")
self.store.add_to_list("example-list", msg)
ml_db = self.store.get_lists()[0]
self.assertEqual(ml_db.recent_participants_count, 0)
self.assertEqual(ml_db.recent_threads_count, 0)
def add_fetch_data(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Subject"] = "Fake Subject"
msg["Message-ID"] = "<dummy>"
msg["Date"] = "Fri, 02 Nov 2012 16:07:54"
msg.set_payload("Fake Message")
ml = FakeList("example-list")
ml.display_name = u"name 1"
ml.subject_prefix = u"[prefix 1]"
return ml.fqdn_listname, self.store.add_to_list(ml, msg)
def setUp(self):
store = kittystore.get_store(SettingsModule(),
debug=False, auto_create=True)
self.client.defaults = {"kittystore.store": store,
"HTTP_USER_AGENT": "testbot",
}
# Create the list by adding a dummy message
ml = FakeList("*****@*****.**")
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg>"
msg.set_payload("Dummy message")
store.add_to_list(ml, msg)
def test_duplicate_nonascii(self):
msg = Message()
msg["From"] = b"*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
self.store.add_to_list(FakeList("example-list"), msg)
self.assertEqual(self.store.get_list_size("example-list"), 1)
self.assertTrue(self.store.is_message_in_list("example-list", "dummy"))
msg.replace_header("From", b"dummy-non-ascii\xc3\[email protected]")
try:
self.store.add_to_list(FakeList("example-list"), msg)
except UnicodeDecodeError, e:
self.fail("Died on a non-ascii header message: %s" % unicode(e))
def setUp(self):
store = kittystore.get_store(SettingsModule(),
debug=False,
auto_create=True)
self.client.defaults = {
"kittystore.store": store,
"HTTP_USER_AGENT": "testbot",
}
# Create the list by adding a dummy message
ml = FakeList("*****@*****.**")
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg>"
msg.set_payload("Dummy message")
store.add_to_list(ml, msg)
def test_starting_message_1(self):
# A basic thread: msg2 replies to msg1
ml = FakeList("example-list")
msg1 = Message()
msg1["From"] = "*****@*****.**"
msg1["Message-ID"] = "<msg1>"
msg1.set_payload("message 1")
self.store.add_to_list(ml, msg1)
msg2 = Message()
msg2["From"] = "*****@*****.**"
msg2["Message-ID"] = "<msg2>"
msg2.set_payload("message 2")
msg2["In-Reply-To"] = msg1["Message-ID"]
self.store.add_to_list(ml, msg2)
thread = self.store.db.find(Thread).one()
self.assertEqual(thread.starting_email.message_id, "msg1")
def test_starting_message_2(self):
# A partially-imported thread: msg1 replies to something we don't have
ml = FakeList("example-list")
msg1 = Message()
msg1["From"] = "*****@*****.**"
msg1["Message-ID"] = "<msg1>"
msg1["In-Reply-To"] = "<msg0>"
msg1.set_payload("message 1")
self.store.add_to_list(ml, msg1)
msg2 = Message()
msg2["From"] = "*****@*****.**"
msg2["Message-ID"] = "<msg2>"
msg2["In-Reply-To"] = msg1["Message-ID"]
msg2.set_payload("message 2")
self.store.add_to_list(ml, msg2)
thread = self.store.db.find(Thread).one()
self.assertEqual(thread.starting_email.message_id, "msg1")
def test_on_new_thread_invalidate(self):
# Check that the cache is invalidated on new message
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
self.store.add_to_list("example-list", msg)
msg.replace_header("Message-ID", "<dummy2>")
msg["In-Reply-To"] = "<dummy>"
self.store.add_to_list("example-list", msg)
call_args = [ call[0][0] for call in self.store.db.cache.set.call_args_list ]
# we have duplicates because both the Storm and the SQLAlchemy model
# subscribe to the event, so we must deduplicate
call_args = set(call_args)
#from pprint import pprint; pprint(call_args)
#print(repr(call_args))
self.assertEqual(call_args, set([
u'list:example-list:thread:QKODQBCADMDSP5YPOPKECXQWEQAMXZL3:subject'
]))
def test_update_list(self):
"""List records must be updated when changed in Mailman"""
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg.set_payload("Dummy message")
ml = FakeList("example-list")
ml.display_name = u"name 1"
ml.subject_prefix = u"[prefix 1]"
self.store.add_to_list(ml, msg)
ml_db = self.store.db.find(List).one()
self.assertEqual(ml_db.display_name, "name 1")
self.assertEqual(ml_db.subject_prefix, "[prefix 1]")
ml.display_name = u"name 2"
ml.subject_prefix = u"[prefix 2]"
self.store.add_to_list(ml, msg)
ml_db = self.store.db.find(List).one()
self.assertEqual(ml_db.display_name, "name 2")
self.assertEqual(ml_db.subject_prefix, "[prefix 2]")
def test_starting_message_3(self):
# A thread where the reply has an anterior date to the first email
# (the In-Reply-To header must win over the date sort)
ml = FakeList("example-list")
msg1 = Message()
msg1["From"] = "*****@*****.**"
msg1["Message-ID"] = "<msg1>"
msg1["Date"] = "Fri, 02 Nov 2012 16:07:54 +0000"
msg1.set_payload("message 1")
self.store.add_to_list(ml, msg1)
msg2 = Message()
msg2["From"] = "*****@*****.**"
msg2["Message-ID"] = "<msg2>"
msg2["Date"] = "Fri, 01 Nov 2012 16:07:54 +0000"
msg2.set_payload("message 2")
msg2["In-Reply-To"] = msg1["Message-ID"]
self.store.add_to_list(ml, msg2)
thread = self.store.db.find(Thread).one()
self.assertEqual(thread.starting_email.message_id, "msg1")
def test_thread_neighbors(self):
ml = FakeList("example-list")
# Create 3 threads
msg_t1_1 = Message()
msg_t1_1["From"] = "*****@*****.**"
msg_t1_1["Message-ID"] = "<id1_1>"
msg_t1_1.set_payload("Dummy message")
self.store.add_to_list(ml, msg_t1_1)
msg_t2_1 = Message()
msg_t2_1["From"] = "*****@*****.**"
msg_t2_1["Message-ID"] = "<id2_1>"
msg_t2_1.set_payload("Dummy message")
self.store.add_to_list(ml, msg_t2_1)
msg_t3_1 = Message()
msg_t3_1["From"] = "*****@*****.**"
msg_t3_1["Message-ID"] = "<id3_1>"
msg_t3_1.set_payload("Dummy message")
self.store.add_to_list(ml, msg_t3_1)
# Check the neighbors
def check_neighbors(thread, expected_prev, expected_next):
thread_id = get_message_id_hash("<id%s_1>" % thread)
prev_th, next_th = self.store.get_thread_neighbors(
"example-list", thread_id)
# convert to something I can compare
prev_th = prev_th and prev_th.thread_id
expected_prev = expected_prev and \
get_message_id_hash("<id%s_1>" % expected_prev)
next_th = next_th and next_th.thread_id
expected_next = expected_next and \
get_message_id_hash("<id%s_1>" % expected_next)
# compare
self.assertEqual(prev_th, expected_prev)
self.assertEqual(next_th, expected_next)
# Order should be: 1, 2, 3
check_neighbors(1, None, 2)
check_neighbors(2, 1, 3)
check_neighbors(3, 2, None)
# now add a new message in thread 1, which becomes the most recently
# active
msg_t1_2 = Message()
msg_t1_2["From"] = "*****@*****.**"
msg_t1_2["Message-ID"] = "<id1_2>"
msg_t1_2["In-Reply-To"] = "<id1_1>"
msg_t1_2.set_payload("Dummy message")
self.store.add_to_list(ml, msg_t1_2)
# Order should be: 2, 3, 1
check_neighbors(2, None, 3)
check_neighbors(3, 2, 1)
check_neighbors(1, 3, None)
def setUp(self):
self.tmpdir = mkdtemp(prefix="hyperkitty-testing-")
self.user = User.objects.create_user('testuser', '*****@*****.**',
'testPass')
settings = SettingsModule()
settings.KITTYSTORE_SEARCH_INDEX = self.tmpdir
self.store = kittystore.get_store(settings,
debug=False,
auto_create=True)
self.client.defaults = {
"kittystore.store": self.store,
"HTTP_USER_AGENT": "testbot",
}
ml = FakeList("*****@*****.**")
ml.subject_prefix = u"[example] "
ml.archive_policy = ArchivePolicy.private
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msgid>"
msg["Subject"] = "Dummy message"
msg.set_payload("Dummy message")
msg["Message-ID-Hash"] = self.msgid = self.store.add_to_list(ml, msg)
def setUp(self):
self.user = User.objects.create_user('testuser', '*****@*****.**',
'testPass')
self.client.login(username='******', password='******')
# use a temp variable below because self.client.session is actually a
# property which returns a new instance en each call :-/
session = self.client.session
session["user_id"] = u"testuser"
session.save()
self.store = kittystore.get_store(SettingsModule(),
debug=False,
auto_create=True)
self.client.defaults = {
"kittystore.store": self.store,
"HTTP_USER_AGENT": "testbot",
}
# Create a dummy message to test on
ml = FakeList("*****@*****.**")
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg>"
msg.set_payload("Dummy message")
self.store.add_to_list(ml, msg)
def process(mlist, msg, msgdata):
"""Decorate the message with headers and footers."""
# Digests and Mailman-craft messages should not get additional headers.
if msgdata.get('isdigest') or msgdata.get('nodecorate'):
return
# Kludge to not decorate mail for Mail-Archive.com.
if ('recipients' in msgdata and len(msgdata['recipients']) == 1
and list(msgdata['recipients'])[0] == MailArchive().recipient):
return
d = {}
member = msgdata.get('member')
if member is not None:
# Calculate the extra personalization dictionary.
# member.subscriber can be a User instance or an Address instance, and
# member.address can be None and so can member._user.preferred_address.
if member._address is not None:
_address = member._address
else:
_address = (member._user.preferred_address
or list(member._user.addresses)[0])
recipient = msgdata.get('recipient', _address.original_email)
d['member'] = formataddr((_address.display_name, _address.email))
d['user_email'] = recipient
d['user_delivered_to'] = _address.original_email
d['user_language'] = member.preferred_language.description
d['user_name'] = member.display_name
d['user_name_or_address'] = member.display_name or recipient
# For backward compatibility.
d['user_address'] = recipient
# Calculate the archiver permalink substitution variables. This provides
# the $<archive-name>_url placeholder for every enabled archiver.
for archiver in IListArchiverSet(mlist).archivers:
if archiver.is_enabled:
# Get the permalink of the message from the archiver. Watch out
# for exceptions in the archiver plugin.
try:
archive_url = archiver.system_archiver.permalink(mlist, msg)
except Exception:
alog.exception('Exception in "{}" archiver'.format(
archiver.system_archiver.name))
archive_url = None
if archive_url is not None:
placeholder = '{}_url'.format(archiver.system_archiver.name)
d[placeholder] = archive_url
# These strings are descriptive for the log file and shouldn't be i18n'd
d.update(msgdata.get('decoration-data', {}))
header = decorate('list:member:regular:header', mlist, d)
footer = decorate('list:member:regular:footer', mlist, d)
# Escape hatch if both the footer and header are empty or None.
if len(header) == 0 and len(footer) == 0:
return
# Be MIME smart here. We only attach the header and footer by
# concatenation when the message is a non-multipart of type text/plain.
# Otherwise, if it is not a multipart, we make it a multipart, and then we
# add the header and footer as text/plain parts.
#
# BJG: In addition, only add the footer if the message's character set
# matches the charset of the list's preferred language. This is a
# suboptimal solution, and should be solved by allowing a list to have
# multiple headers/footers, for each language the list supports.
#
# Also, if the list's preferred charset is us-ascii, we can always
# safely add the header/footer to a plain text message since all
# charsets Mailman supports are strict supersets of us-ascii --
# no, UTF-16 emails are not supported yet.
#
# TK: Message with 'charset=' cause trouble. So, instead of
# mgs.get_content_charset('us-ascii') ...
mcset = msg.get_content_charset() or 'us-ascii'
lcset = mlist.preferred_language.charset
msgtype = msg.get_content_type()
# BAW: If the charsets don't match, should we add the header and footer by
# MIME multipart chroming the message?
wrap = True
if not msg.is_multipart() and msgtype == 'text/plain':
# Save the RFC-3676 format parameters.
format_param = msg.get_param('format')
delsp = msg.get_param('delsp')
# Save 'Content-Transfer-Encoding' header in case decoration fails.
cte = msg.get('content-transfer-encoding')
# header/footer is now in unicode.
try:
oldpayload = msg.get_payload(decode=True).decode(mcset)
del msg['content-transfer-encoding']
frontsep = endsep = ''
if len(header) > 0 and not header.endswith('\n'):
frontsep = '\n'
if len(footer) > 0 and not oldpayload.endswith('\n'):
endsep = '\n'
payload = header + frontsep + oldpayload + endsep + footer
# When setting the payload for the message, try various charset
# encodings until one does not produce a UnicodeError. We'll try
# charsets in this order: the list's charset, the message's
# charset, then utf-8. It's okay if some of these are duplicates.
for cset in (lcset, mcset, 'utf-8'):
try:
msg.set_payload(payload.encode(cset), cset)
except UnicodeError:
pass
else:
if format_param:
msg.set_param('format', format_param)
if delsp:
msg.set_param('delsp', delsp)
wrap = False
break
except (LookupError, UnicodeError):
if cte:
# Restore the original c-t-e.
del msg['content-transfer-encoding']
msg['Content-Transfer-Encoding'] = cte
elif msg.get_content_type() == 'multipart/mixed':
# The next easiest thing to do is just prepend the header and append
# the footer as additional subparts
payload = msg.get_payload()
if not isinstance(payload, list):
payload = [payload]
if len(footer) > 0:
mimeftr = MIMEText(footer.encode(lcset, errors='replace'), 'plain',
lcset)
mimeftr['Content-Disposition'] = 'inline'
payload.append(mimeftr)
if len(header) > 0:
mimehdr = MIMEText(header.encode(lcset, errors='replace'), 'plain',
lcset)
mimehdr['Content-Disposition'] = 'inline'
payload.insert(0, mimehdr)
msg.set_payload(payload)
wrap = False
# If we couldn't add the header or footer in a less intrusive way, we can
# at least do it by MIME encapsulation. We want to keep as much of the
# outer chrome as possible.
if not wrap:
return
# Because of the way Message objects are passed around to process(), we
# need to play tricks with the outer message -- i.e. the outer one must
# remain the same instance. So we're going to create a clone of the outer
# message, with all the header chrome intact, then copy the payload to it.
# This will give us a clone of the original message, and it will form the
# basis of the interior, wrapped Message.
inner = Message()
# Which headers to copy? Let's just do the Content-* headers
for h, v in msg.items():
if h.lower().startswith('content-'):
inner[h] = v
inner.set_payload(msg.get_payload())
# For completeness
inner.set_unixfrom(msg.get_unixfrom())
inner.preamble = msg.preamble
inner.epilogue = msg.epilogue
# Don't copy get_charset, as this might be None, even if
# get_content_charset isn't. However, do make sure there is a default
# content-type, even if the original message was not MIME.
inner.set_default_type(msg.get_default_type())
# BAW: HACK ALERT.
if hasattr(msg, '__version__'):
inner.__version__ = msg.__version__
# Now, play games with the outer message to make it contain three
# subparts: the header (if any), the wrapped message, and the footer (if
# any).
payload = [inner]
if len(header) > 0:
mimehdr = MIMEText(header.encode(lcset, errors='replace'), 'plain',
lcset)
mimehdr['Content-Disposition'] = 'inline'
payload.insert(0, mimehdr)
if len(footer) > 0:
mimeftr = MIMEText(footer.encode(lcset, errors='replace'), 'plain',
lcset)
mimeftr['Content-Disposition'] = 'inline'
payload.append(mimeftr)
msg.set_payload(payload)
del msg['content-type']
del msg['content-transfer-encoding']
del msg['content-disposition']
msg['Content-Type'] = 'multipart/mixed'
