def get_parts(response):
"""Extract parts from headers.
Params:
response: a request object
Returns:
an array of content-ids
"""
head_lines = ""
for k, v in response.raw.getheaders().items():
head_lines += str(k) + ":" + str(v) + "\n"
full = head_lines.encode() + response.content
parser = email.parser.BytesParser()
decoded_reply = parser.parsebytes(full)
parts = {}
start = decoded_reply.get_param("start").lstrip("<").rstrip(">")
i = 0
for part in decoded_reply.get_payload():
cid = part.get("content-Id", "").lstrip("<").rstrip(">")
if (not start or start == cid) and "start" not in parts:
parts["start"] = part.get_payload(decode=True)
else:
parts[cid or "Attachment%d" % i] = part.get_payload(decode=True)
i += 1
return parts
def load_from_checkpoint(self, checkpoint_data):
parser = email.parser.BytesParser()
message = parser.parsebytes(checkpoint_data)
version = int(message['Version'])
if version not in self.SUPPORTED_VERSIONS:
raise storage.UnsupportedFileVersionError()
if message.get_content_type() != 'application/json':
raise storage.CorruptedProjectError(
"Unexpected content type %s" % message.get_content_type())
serialized_checkpoint = message.get_payload()
checkpoint = json.loads(serialized_checkpoint, cls=JSONDecoder)
self.deserialize(checkpoint)
self.init_references()
def validate_node(root, parent, node):
assert node.parent is parent, (node.parent, parent)
assert node.root is root, (node.root, root)
for c in node.list_children():
validate_node(root, node, c)
validate_node(self, None, self)
def read(self):
data, entry_type = super().read()
headers_length = data.index(b'\n\n') + 2
parser = email.parser.BytesParser()
message = parser.parsebytes(data[:headers_length])
content = data[headers_length:]
if 'Checksum' in message:
should_checksum = message['Checksum'].split(';')[0]
checksum_type = message.get_param('type', None, 'Checksum')
if checksum_type is None:
raise BadFileFormatError("Checksum type not specified")
if checksum_type == 'md5':
have_checksum = hashlib.md5(content).hexdigest()
else:
raise BadFileFormatError(
"Unsupported checksum type '%s'" % checksum_type)
if have_checksum != should_checksum:
logger.error("%r", content)
raise CorruptedFileError(
"Checksum mismatch (%s != %s)"
% (have_checksum, should_checksum))
encoding = message.get_param('charset', 'ascii')
return content.decode(encoding), message, entry_type
def read(self) -> Tuple[str, email.message.Message, bytes]:
data, entry_type = self.__file.read()
headers_length = data.index(b'\n\n') + 2
parser = email.parser.BytesParser()
message = parser.parsebytes(data[:headers_length])
content = data[headers_length:]
if 'Checksum' in message:
should_checksum = cast(str, message['Checksum']).split(';')[0]
checksum_type = cast(str,
message.get_param('type', None, 'Checksum'))
if checksum_type is None:
raise BadFileFormatError("Checksum type not specified")
if checksum_type == 'md5':
have_checksum = hashlib.md5(content).hexdigest()
else:
raise BadFileFormatError("Unsupported checksum type '%s'" %
checksum_type)
if have_checksum != should_checksum:
logger.error("%r", content)
raise CorruptedFileError("Checksum mismatch (%s != %s)" %
(have_checksum, should_checksum))
encoding = cast(str, message.get_param('charset', 'ascii'))
return content.decode(encoding), message, entry_type
def main():
# get user login
user_email = get_login()
if not user_email:
exit_with_error("Unauthorized")
# read stdin and parse to message object
input_bytes = sys.stdin.buffer.read()
parser = email.parser.BytesParser()
try:
message = parser.parsebytes(input_bytes)
except MessageError as e:
exit_with_error("MessageError: %s" % str(e))
# add a header with the user's email
message[USER_TRACKING_HEADER] = user_email
# add a header with the server hostname
# scripts.mit.edu runs multiple machines and load-balances between them
message[SCRIPTS_HOSTNAME_HEADER] = platform.node()
# add date if missing
if "Date" not in message:
message["Date"] = email.utils.formatdate()
if "--debug" in sys.argv:
# print the email without sending
print(message.as_string(maxheaderlen=78))
else:
# send the email
send_email(message)
def headers(self):
if self._content and len(self._content):
if not self._headers:
parser = email.parser.BytesParser(policy=email.policy.default)
msg = parser.parsebytes(self._content, headersonly=True)
self._headers = msg
return self._headers
else:
return None
def deserialize_command(self, cmd_data):
parser = email.parser.BytesParser()
message = parser.parsebytes(cmd_data)
target_id = message['Target']
cmd_state = json.loads(message.get_payload(), cls=JSONDecoder)
cmd = commands.Command.create_from_state(cmd_state)
return cmd, target_id
def test_mbox_with_8bit_tags(self):
self.cli_login()
self.cli_import("0028-tags-need-8bit-encoding.mbox.gz")
self.cli_logout()
mbox = self.client.get('/QEMU/[email protected]/mbox')
parser = email.parser.BytesParser(policy=email.policy.SMTP)
msg = parser.parsebytes(mbox.content)
payload = decode_payload(msg)
self.assertIn('SynICState *synic = get_synic(cs);', payload)
self.assertIn('Reviewed-by: Philippe Mathieu-Daudé <*****@*****.**>', payload)
def _body(self):
if not hasattr(self, "_raw_body"):
status, data = self._client.uid("FETCH", str(self.uid),
"(BODY.PEEK[])")
if status != "OK":
raise RuntimeError()
parser = email.parser.BytesParser(policy=email.policy.default)
self._raw_body = parser.parsebytes(data[0][1])
return self._raw_body
def test_mbox_with_8bit_tags(self):
self.cli_login()
self.cli_import("0028-tags-need-8bit-encoding.mbox.gz")
self.cli_logout()
mbox = self.client.get(
"/QEMU/[email protected]/mbox")
parser = email.parser.BytesParser(policy=email.policy.SMTP)
msg = parser.parsebytes(mbox.content)
payload = decode_payload(msg)
self.assertIn("SynICState *synic = get_synic(cs);", payload)
self.assertIn(
"Reviewed-by: Philippe Mathieu-Daudé <*****@*****.**>", payload)
def read(self) -> Tuple[FileInfo, bytes]:
if not os.path.exists(self.path):
raise NotFoundError()
with open(self.path, 'rb') as fp:
magic = fp.read(len(self.MAGIC))
if magic != self.MAGIC:
raise BadFileFormatError("Not an noisicaä file")
# email.parser's headersonly attribute doesn't seem to work the
# way I would expect it.
headers = b''
while headers[-2:] != b'\n\n':
b = fp.read(1)
if not b:
break
headers += b
parser = email.parser.BytesParser()
message = parser.parsebytes(headers)
content = fp.read()
if 'Checksum' in message:
should_checksum = cast(str, message['Checksum']).split(';')[0]
checksum_type = cast(
str, message.get_param('type', None, 'Checksum'))
if checksum_type is None:
raise BadFileFormatError("Checksum type not specified")
if checksum_type == 'md5':
have_checksum = hashlib.md5(content).hexdigest()
else:
raise BadFileFormatError("Unsupported checksum type '%s'" %
checksum_type)
if have_checksum != should_checksum:
raise CorruptedFileError("Checksum mismatch (%s != %s)" %
(have_checksum, should_checksum))
file_info = FileInfo()
file_info.content_type = message.get_content_type()
file_info.encoding = cast(str,
message.get_param('charset', 'ascii'))
if 'Version' in message:
file_info.version = int(cast(str, message['Version']))
if 'File-Type' in message:
file_info.filetype = cast(str, message['File-Type'])
return file_info, content
def from_bytes(cls, data, timestamp=None, flags=None, custom_flags=None):
if timestamp is None:
timestamp = time.time()
if flags is None:
flags = set()
else:
assert isinstance(flags, set)
if custom_flags is None:
custom_flags = set()
else:
assert isinstance(flags, set)
parser = cls.msg_parser()
msg = parser.parsebytes(data)
return cls(msg, timestamp, flags, custom_flags)
def read(self):
if not os.path.exists(self.path):
raise NotFoundError()
with open(self.path, 'rb') as fp:
magic = fp.read(len(self.MAGIC))
if magic != self.MAGIC:
raise BadFileFormatError("Not an noisicaä file")
# email.parser's headersonly attribute doesn't seem to work the
# way I would expect it.
headers = b''
while headers[-2:] != b'\n\n':
b = fp.read(1)
if not b:
break
headers += b
parser = email.parser.BytesParser()
message = parser.parsebytes(headers)
content = fp.read()
if 'Checksum' in message:
should_checksum = message['Checksum'].split(';')[0]
checksum_type = message.get_param('type', None, 'Checksum')
if checksum_type is None:
raise BadFileFormatError("Checksum type not specified")
if checksum_type == 'md5':
have_checksum = hashlib.md5(content).hexdigest()
else:
raise BadFileFormatError(
"Unsupported checksum type '%s'" % checksum_type)
if have_checksum != should_checksum:
raise CorruptedFileError(
"Checksum mismatch (%s != %s)"
% (have_checksum, should_checksum))
file_info = FileInfo()
file_info.content_type = message.get_content_type()
file_info.encoding = message.get_param('charset', 'ascii')
if 'Version' in message:
file_info.version = int(message['Version'])
if 'File-Type' in message:
file_info.filetype = message['File-Type']
return file_info, content
def git_am_patch_split(f, encoding=None):
"""Parse a git-am-style patch and split it up into bits.
:param f: File-like object to parse
:param encoding: Encoding to use when creating Git objects
:return: Tuple with commit object, diff contents and git version
"""
encoding = encoding or getattr(f, "encoding", "ascii")
contents = f.read()
if isinstance(contents, bytes) and getattr(email.parser, "BytesParser", None):
parser = email.parser.BytesParser()
msg = parser.parsebytes(contents)
else:
parser = email.parser.Parser()
msg = parser.parsestr(contents)
return parse_patch_message(msg, encoding)
def git_am_patch_split(f, encoding=None):
"""Parse a git-am-style patch and split it up into bits.
:param f: File-like object to parse
:param encoding: Encoding to use when creating Git objects
:return: Tuple with commit object, diff contents and git version
"""
encoding = encoding or getattr(f, "encoding", "ascii")
contents = f.read()
if type(contents) is bytes and getattr(email.parser, "BytesParser", None):
parser = email.parser.BytesParser()
msg = parser.parsebytes(contents)
else:
parser = email.parser.Parser()
msg = parser.parsestr(contents)
return parse_patch_message(msg, encoding)
def parse_message(data):
# Parse the email to its constituent parts
# https://bugs.python.org/issue25545 says
# > A unicode string has no RFC defintion as an email, so things do not work right...
# > You do have to conditionalize your 2/3 code to use the bytes parser and generator if you are dealing with 8-bit
# > messages. There's just no way around that.
if six.PY2:
parser = email.feedparser.FeedParser()
parser.feed(data)
msg = parser.close()
else:
# works the same as BytesFeedParser, and better than non-"Bytes" parsers for some messages
parser = email.parser.BytesParser()
msg = parser.parsebytes(data.encode('utf-8'))
# Extract relevant data
result = {}
result['multipart'] = multipart = msg.is_multipart()
result['headers'] = dict(msg)
result['message_id'] = _parse_message_id(msg.get('Message-ID'))
result['in_reply_to'] = _parse_message_id(msg.get('In-Reply-To'))
result['references'] = _parse_message_id(msg.get('References'))
if result['message_id'] == []:
result['message_id'] = h.gen_message_id()
else:
result['message_id'] = result['message_id'][0]
if multipart:
result['parts'] = []
for part in msg.walk():
dpart = dict(headers=dict(part),
message_id=result['message_id'],
in_reply_to=result['in_reply_to'],
references=result['references'],
content_type=part.get_content_type(),
filename=part.get_filename(None),
payload=part.get_payload(decode=True))
# payload is sometimes already unicode (due to being saved in mongo?)
if part.get_content_maintype() == 'text':
dpart['payload'] = six.ensure_text(dpart['payload'])
result['parts'].append(dpart)
else:
result['payload'] = msg.get_payload(decode=True)
# payload is sometimes already unicode (due to being saved in mongo?)
if msg.get_content_maintype() == 'text':
result['payload'] = six.ensure_text(result['payload'])
return result
def _on_email(num, data):
parser = email.parser.BytesParser()
headers_b = data[0][1]
headers = parser.parsebytes(headers_b, headersonly=True)
if on_email is not None:
result = on_email(num, data, headers)
if result:
return
xfr = headers.get('X-Failed-Recipients')
if xfr is not None:
for xfr_email in filter(None, map(
lambda s: s.strip(), xfr.split(','))):
if on_fail_email is not None:
on_fail_email(xfr_email)
def _on_email(num, data):
parser = email.parser.BytesParser()
headers_b = data[0][1]
headers = parser.parsebytes(headers_b, headersonly=True)
if on_email is not None:
result = on_email(num, data, headers)
if result:
return
xfr = headers.get('X-Failed-Recipients')
if xfr is not None:
for xfr_email in filter(None,
map(lambda s: s.strip(), xfr.split(','))):
if on_fail_email is not None:
on_fail_email(xfr_email)
async def handle_DATA(self, server, session, envelope):
logger = logging.getLogger('orouboros.localhandler')
logger.debug(f'Session auth? {session.is_authenticated}')
logger.debug(f'Message is from {envelope.mail_from!r}')
logger.debug(f'Message is for {envelope.rcpt_tos!r}')
valid_recipients = [
recipient for recipient in envelope.rcpt_tos
if recipient.rpartition('@')[2] in self.ok_domains
]
# TODO: remove False and -W. Werner, 2018-03-01
if False and not valid_recipients:
logger.error(f'No valid recipients in {envelope.rcpt_tos}')
return '554 Transaction failed'
else:
logger.info(f'Sending mail to {valid_recipients}')
parser = email.parser.BytesParser()
msg = parser.parsebytes(envelope.original_content)
self.deliver(msg, valid_recipients)
return '250 Message accepted for delivery, eh?'
def fuzz(parser):
with open(sys.argv[1], "rb") as fp:
message = parser.parse(fp)
message.as_bytes(policy=email.policy.default)
message.is_multipart()
message.get_unixfrom()
keys = message.keys()
for key in keys:
message.get(key)
message.get_all(key)
message.values()
message.get_content_type()
message.get_content_maintype()
message.get_content_subtype()
message.get_default_type()
message.get_filename()
message.get_boundary()
message.get_content_charset()
message.is_attachment()
message.get_content_disposition()
for part in message.walk():
pass
parser = email.parser.BytesParser(policy=email.policy.default)
parser.parsebytes(b"")
afl.init()
fuzz(parser)
os._exit(0)
def run_query(mailbox, query, cmd, a_box, boxlst, mail, print_mutex, mqueue, name):
dqueue = queue.Queue()
mqueue.put((name.update, 'Running query.'))
search_query = '('
for key in query.keys():
if query[key]:
if (len(search_query) > 1):
search_query = search_query+' '
search_query = search_query+'%s \"%s\"' % (key, query[key])
search_query = search_query+')'
if (mailbox != 'ALL'):
t = email_search(mail, search_query, mailbox, dqueue, print_mutex)
t.start()
while(True):
time.sleep(.5)
try:
(result, lst) = dqueue.get(block=False)
except queue.Empty:
pass
else:
mqueue.put((name.update, 'Query complete.'))
break
else:
i = 0
box = a_box
for mailbox in (boxlst):
if(box == mailbox or mailbox == 'ALL'): continue
t = email_search(mail, search_query, mailbox, dqueue, print_mutex)
t.start()
while(True):
time.sleep(.5)
try:
(result, tmp) = dqueue.get(block=False)
except queue.Empty:
pass
else:
break
i += len(tmp)
if(len(tmp) > 0):
#mail.copy(a, box)
t = email_copy(mail, tmp, box, dqueue, print_mutex)
t.start()
while(True):
try:
result = dqueue.get(block=False)
except queue.Empty:
pass
else:
break
mqueue.put((name.update, ('%i Messages found.' % i)))
return
#If messages were found ask for what to do, else inform user that not messages matchedthe query
if(len(lst)==0):
mqueue.put((name.update, ('No messages that matched query.')))
return
else:
ans = cmd
#Delete
if ans == 'delete':
mqueue.put((confirm, ((mail.get_name(), '%i Messages found in %s, \ndo you really want to delete these messages?' % (len(lst), mailbox), \
delete, mail, mailbox, lst, print_mutex, mqueue, name))))
return
#Move - actually copy and delete
elif ans == 'move':
mqueue.put((confirm, ((mail.get_name(), '%i Messages found in %s, \ndo you really want to move these messages?' % (len(lst), mailbox), \
move, mail, mailbox, a_box, lst, print_mutex, dqueue, mqueue, name))))
return
#Copy
elif ans == 'copy':
mqueue.put((name.update, ('%i Messages found. Copied to %s' % (len(lst), a_box))))
t = email_copy(mail, lst, a_box, dqueue, print_mutex)
t.start()
while(True):
time.sleep(.5)
try:
resukt = dqueue.get(block=False)
except queue.Empty:
pass
else:
break
return len(lst)
#Fetch message
elif ans == 'fetch':
#pdb.set_trace()
mqueue.put((name.update, ('%i Messages found.' % len(lst))))
log(mail.get_name(), '%i Messages found.' % len(lst))
for a in lst:
try:
#typ, data = mail.fetch(a, '(BODY.PEEK[TEXT])')
typ, data = mail.fetch(a, '(RFC822)')
#msg = email.message_from_string(data[0][1])
except Exception as e:
log (e)
#log(mail.get_name(), data)
sdata = str(data[0][1])
#sdata = sdata[2:]
#sdata = sdata[:-1]
#msg = email.message_from_string(str(data[0][1]))
parser = email.parser.BytesParser()
#sdata= data[0][1].decode('utf-8')
msg = parser.parsebytes(data[0][1])
maintype = msg.get_content_maintype()
if (maintype == 'multipart'):
with print_mutex: print('Multipart!!!')
for part in msg.walk():
if part.get_content_type() == 'text/plain':
with print_mutex: print(part.get_payload())
elif (maintype == 'text'):
with print_mutex: print('Text!!!')
with print_mutex: print(msg.get_payload())
return len(lst)
def parse_email(email_string):
return parser.parsebytes(email_string)
async def handle_DATA(self, server, session, envelope):
parser = email.parser.BytesParser(policy=email.policy.SMTP)
message = parser.parsebytes(envelope.content)
try:
parsed_message = ParsedMailMessage.from_email_object(message, envelope.mail_from)
except Exception:
err_msg = ''.join(traceback.format_exception(*sys.exc_info()))
self._error_writer('Failed to parse Message:\n{}'.format(err_msg))
django.core.mail.send_mail(
'LMTP Handling Error', err_msg, from_email=POSTMASTER_ADDRESS,
recipient_list=[POSTMASTER_ADDRESS], fail_silently=True)
return '500 Failed to parse incoming message'
success_count = 0
error_messages = []
internal_problems = []
for recipient in envelope.rcpt_tos:
try:
self._contribution_processor.process_message_for_recipient(parsed_message,
recipient)
success_count += 1
self._success_writer('Processed recipient {:d}/{:d} ({}) successfully'
.format(success_count, len(envelope.rcpt_tos), recipient))
except MailProcessingFailure as exc:
error_messages.append((recipient, str(exc)))
self._error_writer('Failed to process recipient <{}>: {}'.format(recipient, exc))
except Exception as exc:
# unknown internal problems end up here
internal_problems.append((recipient,
''.join(traceback.format_exception(*sys.exc_info()))))
self._error_writer('Internal error while processing recipient <{}>: {}\n{}'
.format(recipient, exc, internal_problems[-1][1]))
if error_messages:
# The mail was not acceptable for us (e.g. due to permissions). We send a human
# readable separate response and tell the mail server, that the mail was handled.
if len(error_messages) > 1:
# format multiple errors into one string
error_response_text = '\n\n'.join('= {} =\n\n{}'.format(recipient, error_message)
for recipient, error_message in error_messages)
else:
error_response_text = error_messages[0][1]
else:
error_response_text = None
if internal_problems:
# At least one recipient failed mysteriously.
problems_text = '\n\n'.join('= {} =\n{}\n'.format(recipient, exc)
for recipient, exc in internal_problems)
# inform developers
self._contribution_processor.send_error_mail_response(
parsed_message, problems_text, recipient=POSTMASTER_ADDRESS, fail_silently=True)
if success_count > 0:
# Never reject a message that was at least partly processed. Otherwise a
# contribution could be published multiple times - which would be quite annoying
# for the subscribers.
return '250 Message partially accepted for delivery (ignoring internal errors)'
else:
# The mail server should ask us again later. Hopefully the problem is fixed until
# then. An error message mail is not yet returned to the sender.
return '451 Requested action aborted: error in processing'
elif error_response_text is not None:
# Try to send an helpful error message mail back to sender.
# Only return a bounce, if we failed to submit this error message mail.
try:
self._contribution_processor.send_error_mail_response(parsed_message,
error_response_text,
recipient=envelope.mail_from)
return ('250 The incoming message was at least partly not accepted for '
'publication. Instead a separate error mails was returned.')
except smtplib.SMTPException:
# emergency bounce in case of mail delivery problems
return '550 Rejected incoming mail due to errors'
elif success_count > 0:
return '250 Message accepted for delivery'
else:
self._error_writer('Unexpectedly encountered an empty list of recipients.')
def parse_message_from_bytes(raw_bytes):
parser = email.parser.BytesParser(policy=email.policy.SMTP)
return parser.parsebytes(raw_bytes)
