def test_parse_multipart(self) -> None:
header = b'subject: multipart test\n' \
b'content-type: multipart/mixed;\n boundary="testbound"\n\n'
part1 = b'\n' \
b'part one!\n\n' \
b'lorem ipsum etc.\n'
part2 = b'content-type: text/html\n' \
b'\n' \
b'<html><body><h1>part two</h1></body></html>\n'
body = b'preamble\n' \
b'--testbound\n' + part1 + \
b'--testbound\n' + part2 + \
b'--testbound--\n' \
b'epilogue\n'
raw = header + body
msg = MessageContent.parse(raw)
self.assertEqual(raw, bytes(msg))
self.assertEqual(header, bytes(msg.header))
self.assertEqual({b'subject': ['multipart test'],
b'content-type': ['multipart/mixed; '
'boundary="testbound"']},
msg.header.parsed)
self.assertEqual(body, bytes(msg.body))
self.assertTrue(msg.body.has_nested)
self.assertEqual(2, len(msg.body.nested))
self.assertEqual(part1, bytes(msg.body.nested[0]))
self.assertEqual({}, msg.body.nested[0].header.parsed)
self.assertEqual(part2, bytes(msg.body.nested[1]))
self.assertEqual({b'content-type': ['text/html']},
msg.body.nested[1].header.parsed)
async def _load_full(self, redis: Redis, ct_keys: ContentKeys) \
-> MessageContent:
pipe = unwatch_pipe(redis)
pipe.hmget(ct_keys.data, b'full', b'full-json')
_, (literal, full_json) = await pipe.execute()
if literal is None or full_json is None:
raise ValueError(f'Missing message content: {self.email_id}')
return MessageContent.from_json(literal, json.loads(full_json))
def get_actions(self, sender: str, recipient: str,
append_msg: AppendMessage) -> Sequence[Command]:
actions: List[Command] = []
try:
content = MessageContent.parse(append_msg.literal)
self._get_actions(actions, sender, recipient, append_msg, content)
except StopRunning:
pass
return actions
def get_actions(self, sender: str, recipient: str,
append_msg: AppendMessage) -> Sequence[Command]:
actions: List[Command] = []
try:
content = MessageContent.parse(append_msg.message)
self._get_actions(actions, sender, recipient, append_msg, content)
except StopRunning:
pass
return actions
async def _load_header(self, redis: Redis, ct_keys: ContentKeys) \
-> MessageContent:
pipe = unwatch_pipe(redis)
pipe.hmget(ct_keys.data, b'header', b'header-json')
_, (literal, header_json) = await pipe.execute()
if literal is None or header_json is None:
raise ValueError(f'Missing message header: {self.email_id}')
header = MessageHeader.from_json(literal, json.loads(header_json))
body = MessageBody.empty()
return MessageContent(literal, header, body)
async def load_content(self, requirement: FetchRequirement) \
-> LoadedMessage:
if self._key is None or self._maildir is None \
or requirement.has_none(FetchRequirement.CONTENT):
return LoadedMessage(self, requirement, None)
try:
maildir_msg = self._maildir.get_message(self._key)
except (KeyError, FileNotFoundError):
return LoadedMessage(self, requirement, None)
else:
content = MessageContent.parse(bytes(maildir_msg))
return LoadedMessage(self, requirement, content)
async def add(self, append_msg: AppendMessage, recent: bool = False) \
-> Message:
redis = self._redis
prefix = self._prefix
is_deleted = Deleted in append_msg.flag_set
is_unseen = Seen not in append_msg.flag_set
msg_content = MessageContent.parse(append_msg.message)
msg_flags = [flag.value for flag in append_msg.flag_set]
msg_time = append_msg.when.isoformat().encode('ascii')
while True:
await redis.watch(prefix + b':max-mod', self._abort_key)
max_uid, max_mod, abort = await redis.mget(
prefix + b':max-uid', prefix + b':max-mod', self._abort_key)
MailboxAbort.assertFalse(abort)
new_uid = int(max_uid or 0) + 1
new_mod = int(max_mod or 0) + 1
msg_prefix = prefix + b':msg:%d' % new_uid
multi = redis.multi_exec()
multi.set(prefix + b':max-uid', new_uid)
multi.set(prefix + b':max-mod', new_mod)
multi.sadd(prefix + b':uids', new_uid)
multi.zadd(prefix + b':mod-sequence', new_mod, new_uid)
multi.zadd(prefix + b':sequence', new_uid, new_uid)
if recent:
multi.sadd(prefix + b':recent', new_uid)
if is_deleted:
multi.sadd(prefix + b':deleted', new_uid)
if is_unseen:
multi.zadd(prefix + b':unseen', new_uid, new_uid)
if msg_flags:
multi.sadd(msg_prefix + b':flags', *msg_flags)
multi.set(msg_prefix + b':time', msg_time)
multi.set(msg_prefix + b':header', bytes(msg_content.header))
multi.set(msg_prefix + b':body', bytes(msg_content.body))
try:
await multi.execute()
except MultiExecError:
if await _check_errors(multi):
raise
else:
break
return Message(new_uid, append_msg.flag_set, append_msg.when,
recent=recent, content=msg_content)
def test_parse(self) -> None:
header = b'from: [email protected] \n' \
b'to: [email protected],\n [email protected]\n' \
b'to: [email protected]\r\n' \
b'subject: hello world \xff\r\n' \
b'test:\n more stuff\n\n'
body = b'abc\n'
raw = header + body
msg = MessageContent.parse(raw)
self.assertEqual(raw, bytes(msg))
self.assertEqual(header, bytes(msg.header))
self.assertEqual({b'from': ['*****@*****.**'],
b'to': ['[email protected], [email protected]',
'*****@*****.**'],
b'subject': ['hello world \ufffd'],
b'test': [' more stuff']}, msg.header.parsed)
self.assertEqual('hello world \ufffd', msg.header.parsed.subject)
self.assertEqual(body, bytes(msg.body))
self.assertFalse(msg.body.has_nested)
def test_parse_rfc822(self) -> None:
header = b'subject: rfc822 test\n' \
b'content-type: message/rfc822\n\n'
sub_header = b'content-type: text/html\n\n'
sub_body = b'<html><body><h1>part two</h1></body></html>'
body = sub_header + sub_body
raw = header + body
msg = MessageContent.parse(raw)
self.assertEqual(raw, bytes(msg))
self.assertEqual(header, bytes(msg.header))
self.assertEqual({b'subject': ['rfc822 test'],
b'content-type': ['message/rfc822']},
msg.header.parsed)
self.assertEqual(body, bytes(msg.body))
self.assertTrue(msg.body.has_nested)
self.assertEqual(1, len(msg.body.nested))
self.assertEqual(body, bytes(msg.body.nested[0]))
self.assertEqual(sub_header, bytes(msg.body.nested[0].header))
self.assertEqual(sub_body, bytes(msg.body.nested[0].body))
self.assertEqual({b'content-type': ['text/html']},
msg.body.nested[0].header.parsed)
async def get(self, uid: int, cached_msg: CachedMessage = None,
requirement: FetchRequirement = FetchRequirement.METADATA) \
-> Optional[Message]:
redis = self._redis
prefix = self._prefix
msg_prefix = prefix + b':msg:%d' % uid
multi = redis.multi_exec()
multi.sismember(prefix + b':uids', uid)
multi.smembers(msg_prefix + b':flags')
multi.get(msg_prefix + b':time')
multi.sismember(prefix + b':recent', uid)
if requirement & FetchRequirement.BODY:
multi.get(msg_prefix + b':header')
multi.get(msg_prefix + b':body')
elif requirement & FetchRequirement.HEADERS:
multi.get(msg_prefix + b':header')
multi.echo(b'')
else:
multi.echo(b'')
multi.echo(b'')
multi.get(self._abort_key)
exists, flags, time, recent, header, body, abort = \
await multi.execute()
MailboxAbort.assertFalse(abort)
if not exists:
if cached_msg is None:
return None
else:
return Message(cached_msg.uid, cached_msg.permanent_flags,
cached_msg.internal_date, expunged=True)
msg_flags = {Flag(flag) for flag in flags}
msg_time = datetime.fromisoformat(time.decode('ascii'))
msg_recent = bool(recent)
if header:
msg_content = MessageContent.parse_split(header, body)
return Message(uid, msg_flags, msg_time, recent=msg_recent,
content=msg_content)
else:
return Message(uid, msg_flags, msg_time, recent=msg_recent)
async def save(self, message: bytes) -> SavedMessage:
redis = self._redis
ns_keys = self._ns_keys
content = MessageContent.parse(message)
new_email_id = ObjectId.random_email_id()
msg_hash = HashStream(hashlib.sha1()).digest(content)
thread_keys = ThreadKey.get_all(content.header)
thread_key_keys = [
b'\0'.join(thread_key) for thread_key in thread_keys
]
await redis.unwatch()
multi = redis.multi_exec()
multi.hsetnx(ns_keys.email_ids, msg_hash, new_email_id.value)
multi.hget(ns_keys.email_ids, msg_hash)
if thread_key_keys:
multi.hmget(ns_keys.thread_ids, *thread_key_keys)
else:
multi.hmget(ns_keys.thread_ids, b'')
_, email_id, thread_ids = await multi.execute()
thread_id_b = next(
(thread_id for thread_id in thread_ids if thread_id is not None),
None)
if thread_id_b is None:
thread_id = ObjectId.random_thread_id()
else:
thread_id = ObjectId(thread_id_b)
ct_keys = ContentKeys(ns_keys, email_id)
multi = redis.multi_exec()
multi.hset(ct_keys.data, b'full', message)
multi.hset(ct_keys.data, b'full-json', json.dumps(content.json))
multi.hset(ct_keys.data, b'header', bytes(content.header))
multi.hset(ct_keys.data, b'header-json',
json.dumps(content.header.json))
multi.expire(ct_keys.data, self._cleanup.content_expire)
for thread_key_key in thread_key_keys:
multi.hsetnx(ns_keys.thread_ids, thread_key_key, thread_id.value)
await multi.execute()
return SavedMessage(ObjectId(email_id), thread_id, None)
async def save(self, message: bytes) -> SavedMessage:
content = MessageContent.parse(message)
email_id = self._content_cache.add(content)
thread_id = self._thread_cache.add(content)
return SavedMessage(email_id, thread_id, content)
def test_base64_cte(self) -> None:
data = b'Content-Transfer-Encoding: base64\n\n' + _b64_body
msg = MessageContent.parse(data)
decoded = MessageDecoder.of(msg.header).decode(msg.body)
self.assertEqual(b'Testing\x01\x00\nBase 64 \n', bytes(decoded))
def test_quopri_cte(self) -> None:
data = b'Content-Transfer-Encoding: quoted-printable\n\n' + _qp_body
msg = MessageContent.parse(data)
decoded = MessageDecoder.of(msg.header).decode(msg.body)
self.assertEqual(b'Testing\x01Quoted=Printable\n', bytes(decoded))
def test_7bit_cte(self) -> None:
data = b'\n' + _7bit_body
msg = MessageContent.parse(data)
decoded = MessageDecoder.of(msg.header).decode(msg.body)
self.assertEqual(b'Testing 7bit\n', bytes(decoded))