def write_wheelfile(self,
wheelfile_base,
generator='bdist_wheel (' + wheel_version + ')'):
from email.message import Message
# Workaround for Python 2.7 for when "generator" is unicode
if sys.version_info < (3, ) and not isinstance(generator, str):
generator = generator.encode('utf-8')
msg = Message()
msg['Wheel-Version'] = '1.0' # of the spec
msg['Generator'] = generator
msg['Root-Is-Purelib'] = str(self.root_is_pure).lower()
if self.build_number is not None:
msg['Build'] = self.build_number
# Doesn't work for bdist_wininst
impl_tag, abi_tag, plat_tag = self.get_tag()
for impl in impl_tag.split('.'):
for abi in abi_tag.split('.'):
for plat in plat_tag.split('.'):
msg['Tag'] = '-'.join((impl, abi, plat))
wheelfile_path = os.path.join(wheelfile_base, 'WHEEL')
logger.info('creating %s', wheelfile_path)
buffer = BytesIO()
BytesGenerator(buffer, maxheaderlen=0).flatten(msg)
with open(wheelfile_path, 'wb') as f:
f.write(buffer.getvalue().replace(b'\r\n', b'\r'))
def message2email(msg: Message) -> EmailMessage:
"""
Convert an instance of the old `Message` class (or one of its subclasses,
like a `mailbox` message class) to an instance of the new `EmailMessage`
class with the ``default`` policy. If ``msg`` is already an
`EmailMessage`, it is returned unchanged.
"""
if isinstance(msg, EmailMessage):
return msg
# Message.as_bytes() refolds long header lines (which can result in changes
# in whitespace after reparsing) and doesn't give a way to change this, so
# we need to use a BytesGenerator manually.
fp = BytesIO()
# TODO: Instead of maxheaderlen, use a policy with refold_source=None?
g = BytesGenerator(fp, mangle_from_=False, maxheaderlen=0)
g.flatten(msg, unixfrom=msg.get_unixfrom() is not None)
fp.seek(0)
emsg = email.message_from_binary_file(fp, policy=policy.default)
assert isinstance(emsg, EmailMessage)
# MMDFMessage and mboxMessage make their "From " lines available though a
# different method than normal Messages, so we have to copy it over
# manually.
if isinstance(msg, (MMDFMessage, mboxMessage)):
emsg.set_unixfrom("From " + msg.get_from())
return emsg
def test_cte_type_7bit_transforms_8bit_cte(self):
source = textwrap.dedent("""\
From: [email protected]
To: Dinsdale
Subject: Nudge nudge, wink, wink
Mime-Version: 1.0
Content-Type: text/plain; charset="latin-1"
Content-Transfer-Encoding: 8bit
oh là là, know what I mean, know what I mean?
""").encode('latin1')
msg = message_from_bytes(source)
expected = textwrap.dedent("""\
From: [email protected]
To: Dinsdale
Subject: Nudge nudge, wink, wink
Mime-Version: 1.0
Content-Type: text/plain; charset="iso-8859-1"
Content-Transfer-Encoding: quoted-printable
oh l=E0 l=E0, know what I mean, know what I mean?
""").encode('ascii')
s = io.BytesIO()
g = BytesGenerator(s,
policy=self.policy.clone(cte_type='7bit',
linesep='\n'))
g.flatten(msg)
self.assertEqual(s.getvalue(), expected)
def dump_message(file, headers, props=None, content=None):
msg = Message()
for (name, value) in headers:
msg[name] = value
payload = BytesIO()
if props is not None:
start = payload.tell()
for (key, value) in props.items():
payload.write("K {}\n".format(len(key)).encode("ascii"))
payload.writelines((key.encode("ascii"), b"\n"))
payload.write("V {}\n".format(len(value)).encode("ascii"))
payload.writelines((value.encode("ascii"), b"\n"))
payload.write(b"PROPS-END\n")
msg["Prop-content-length"] = format(payload.tell() - start)
if content is not None:
msg["Text-content-length"] = format(len(content))
payload.write(content)
if props is not None or content is not None:
payload = payload.getvalue()
msg["Content-length"] = format(len(payload))
# Workaround for Python issue 18324, "set_payload does not handle
# binary payloads correctly", http://bugs.python.org/issue18324
msg.set_payload(payload.decode("ascii", "surrogateescape"))
BytesGenerator(file, mangle_from_=False).flatten(msg)
def get_payload(self):
"""
decode and return part payload. if I{type} is 'text/*' and I{charset}
not C{None}, be careful to take care of the text encoding. Use
something like C{part.get_payload().decode(part.charset)}
"""
payload=None
if self.type.startswith('message/'):
# I don't use msg.as_string() because I want to use mangle_from_=False
if sys.version_info<(3, 0):
# python 2.x
from email.generator import Generator
fp = StringIO.StringIO()
g = Generator(fp, mangle_from_=False)
g.flatten(self.part, unixfrom=False)
payload=fp.getvalue()
else:
# support only for python >= 3.2
from email.generator import BytesGenerator
import io
fp = io.BytesIO()
g = BytesGenerator(fp, mangle_from_=False)
g.flatten(self.part, unixfrom=False)
payload=fp.getvalue()
else:
payload=self.part.get_payload(decode=True)
return payload
def write_wheelfile(self,
wheelfile_base,
generator="bdist_wheel (" + wheel_version + ")"):
from email.message import Message
msg = Message()
msg["Wheel-Version"] = "1.0" # of the spec
msg["Generator"] = generator
msg["Root-Is-Purelib"] = str(self.root_is_pure).lower()
if self.build_number is not None:
msg["Build"] = self.build_number
# Doesn't work for bdist_wininst
impl_tag, abi_tag, plat_tag = self.get_tag()
for impl in impl_tag.split("."):
for abi in abi_tag.split("."):
for plat in plat_tag.split("."):
msg["Tag"] = "-".join((impl, abi, plat))
wheelfile_path = os.path.join(wheelfile_base, "WHEEL")
log.info(f"creating {wheelfile_path}")
buffer = BytesIO()
BytesGenerator(buffer, maxheaderlen=0).flatten(msg)
with open(wheelfile_path, "wb") as f:
f.write(buffer.getvalue().replace(b"\r\n", b"\r"))
def do_GET(self):
"""
Echo a request without a body.
"""
message = self.get_message()
self.send_head()
BytesGenerator(self.wfile).flatten(message, unixfrom=False)
def _flatten_message(self, message, flattening_policy):
bytes_io = BytesIO()
generator = BytesGenerator(bytes_io,
mangle_from_=False,
policy=flattening_policy)
generator.flatten(message, unixfrom=False, linesep='\r\n')
return bytes_io.getvalue()
def _flatten_message(
self, message,
_): #3: remove whole method definition (keeping the above one)
bytes_io = BytesIO()
generator = BytesGenerator(bytes_io, mangle_from_=False)
generator.flatten(message, unixfrom=False)
return bytes_io.getvalue()
def _msg_generator(self, msg):
outfp = BytesIO()
if pycompat.PY3:
BytesGenerator(outfp, policy=SMTP).flatten(msg, False)
return outfp.getvalue()
else:
Generator(outfp).flatten(msg, False)
return re.sub(_LINE_BREAK, b'\r\n', outfp.getvalue())
def test_cte_type_7bit_handles_unknown_8bit(self):
source = ("Subject: Maintenant je vous présente mon "
"collègue\n\n").encode('utf-8')
expected = ('Subject: Maintenant je vous =?unknown-8bit?q?'
'pr=C3=A9sente_mon_coll=C3=A8gue?=\n\n').encode('ascii')
msg = message_from_bytes(source)
s = io.BytesIO()
g = BytesGenerator(s, policy=self.policy.clone(cte_type='7bit'))
g.flatten(msg)
self.assertEqual(s.getvalue(), expected)
def mail_to_bytes(mail):
"""Get bytes based on a mail.
Based on email.Message.as_bytes, but we reimplement it here because python
3.3 lacks it.
"""
fp = BytesIO()
g = BytesGenerator(fp, mangle_from_=False, policy=mail.policy)
g.flatten(mail, unixfrom=False)
return fp.getvalue()
def main(do_it=False):
with open('results.pkl', 'rb') as f:
results = pickle.load(f)
if not do_it:
print('This only send data if you give it --ok')
print('Would send the following mails:')
print('')
n = len(results)
i = 0
for result in results:
gifter = result[0]
gifted = result[1]
#mail_body = mail_body_fmt.format(gifter=result[0], gifted=result[1])
#mail_from = f'{sender_email}'
#mail_to = f'{gifter.email}'
mail_from = f'{sender_name} <{sender_email}>'
mail_to = f'{gifter.name} <{gifter.email}>'
mail_subject = mail_subject_fmt.format()
mail_body = mail_body_fmt.format(gifter=gifter, gifted=gifted)
msg = EmailMessage()
msg['From'] = mail_from
msg['To'] = mail_to
msg['Subject'] = mail_subject
msg.set_payload(mail_body, charset='utf-8')
if do_it:
#p = Popen(["/usr/sbin/sendmail", "-t", "-oi"], stdin=PIPE)
#p.communicate(msg.as_bytes())
p = Popen(['sendmail', '-F', '-oi', gifter.email], stdin=PIPE)
g = BytesGenerator(p.stdin,
policy=msg.policy.clone(linesep='\r\n'))
g.flatten(msg)
p.stdin.close()
rc = p.wait()
i += 1
print(f'{i}/{n} mails sent.')
else:
# because utf8 is 8b and transport is 7b, mail is normally encoded in base64
# this is just a way to add the un-encoded text to the message before printing
# trying to send it will give errors
msg.set_payload(mail_body)
print('-' * 78)
print(msg)
print('-' * 78)
if do_it and i != n:
print('There were some errors sending the mails')
os.remove('results.pkl')
def do_POST(self):
""" Echo a request with a body. """
message = self.get_message()
try:
length = int(self.headers["Content-Length"])
except (TypeError, ValueError) as exc:
message.set_payload("Invalid Content-Length: {exc}".format(exc))
else:
message.set_payload(self.rfile.read(length))
finally:
self.send_head()
BytesGenerator(self.wfile).flatten(message, unixfrom=False)
def as_bytes(self, unixfrom=False, policy=None):
"""Return the entire formatted message as a bytes object.
Optional 'unixfrom', when true, means include the Unix From_ envelope
header. 'policy' is passed to the BytesGenerator instance used to
serialize the message; if not specified the policy associated with
the message instance is used.
"""
from email.generator import BytesGenerator
policy = self.policy if policy is None else policy
fp = BytesIO()
g = BytesGenerator(fp, mangle_from_=False, policy=policy)
g.flatten(self, unixfrom=unixfrom)
return fp.getvalue()
def as_bytes(self):
"""Return the entire formatted message as a bytes object."""
# Instead of using self.message.as_bytes() directly,
# we copy and edit the implementation of email.Message.as_bytes
# since it does not accept maxheaderlen, which we wish to set to 0
# for transparency.
policy = self.message.policy
fp = BytesIO()
g = BytesGenerator(fp,
mangle_from_=False,
maxheaderlen=0,
policy=policy)
g.flatten(self.message, unixfrom=None)
return fp.getvalue()
def parse_msg(self, msg):
"""Parses the given :class:`~email.message.Message` to
populate the :attr:`headers` and :attr:`message` attributes.
:param data: The complete message, headers and message body.
:type data: :class:`~email.message.Message`
"""
# Can't use non-six BytesIO here cause python2 BytesGenerator will fail
# to decode headers
outfp = six.BytesIO()
BytesGenerator(outfp).flatten(msg, False)
data = outfp.getvalue()
if six.PY2:
data = data.encode()
self.parse(data)
def test_smtputf8_policy(self):
msg = EmailMessage()
msg['From'] = 'Páolo <fő[email protected]>'
msg['To'] = 'Dinsdale'
msg['Subject'] = 'Nudge nudge, wink, wink ὠ9'
msg.set_content('oh là là, know what I mean, know what I mean?')
expected = textwrap.dedent(""" From: Páolo <fő[email protected]>
To: Dinsdale
Subject: Nudge nudge, wink, wink ὠ9
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0
oh là là, know what I mean, know what I mean?
""").encode('utf-8').replace(b'\n', b'\r\n')
s = io.BytesIO()
g = BytesGenerator(s, policy=policy.SMTPUTF8)
g.flatten(msg)
self.assertEqual(s.getvalue(), expected)
def test_smtp_policy(self):
msg = EmailMessage()
msg["From"] = Address(addr_spec="*****@*****.**", display_name="Páolo")
msg["To"] = Address(addr_spec="*****@*****.**", display_name="Dinsdale")
msg["Subject"] = "Nudge nudge, wink, wink"
msg.set_content("oh boy, know what I mean, know what I mean?")
expected = textwrap.dedent("""\
From: =?utf-8?q?P=C3=A1olo?= <*****@*****.**>
To: Dinsdale <*****@*****.**>
Subject: Nudge nudge, wink, wink
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: 7bit
MIME-Version: 1.0
oh boy, know what I mean, know what I mean?
""").encode().replace(b"\n", b"\r\n")
s = io.BytesIO()
g = BytesGenerator(s, policy=policy.SMTP)
g.flatten(msg)
self.assertEqual(s.getvalue(), expected)
def ingest(self, file_path, entity):
mbox = mailbox.mbox(file_path)
entity.schema = model.get('Package')
entity.add('mimeType', self.DEFAULT_MIME)
for i, msg in enumerate(mbox.itervalues(), 1):
# Is there a risk of https://bugs.python.org/issue27321 ?
try:
msg_path = self.make_work_file('%s.eml' % i)
with open(msg_path, 'wb') as fh:
gen = BytesGenerator(fh, policy=default)
gen.flatten(msg)
checksum = self.manager.store(msg_path, mime_type=RFC822)
msg_path.unlink()
child = self.manager.make_entity('Email', parent=entity)
child.make_id(checksum)
child.add('contentHash', checksum)
child.add('mimeType', RFC822)
self.manager.queue_entity(child)
except Exception:
log.exception("[%r] Cannot extract message %s", entity, i)
def encode_multipart_message(message):
# The message must be multipart.
assert message.is_multipart()
# The body length cannot yet be known.
assert "Content-Length" not in message
# So line-endings can be fixed-up later on, component payloads must have
# no Content-Length and their Content-Transfer-Encoding must be base64
# (and not quoted-printable, which Django doesn't appear to understand).
for part in message.get_payload():
assert "Content-Length" not in part
assert part["Content-Transfer-Encoding"] == "base64"
# Flatten the message without headers.
buf = BytesIO()
generator = BytesGenerator(buf, False) # Don't mangle "^From".
generator._write_headers = lambda self: None # Ignore.
generator.flatten(message)
# Ensure the body has CRLF-delimited lines. See
# http://bugs.python.org/issue1349106.
body = b"\r\n".join(buf.getvalue().splitlines())
# Only now is it safe to set the content length.
message.add_header("Content-Length", "%d" % len(body))
return message.items(), body
def msg_as_input(self, msg):
m = message_from_bytes(msg, policy=policy.SMTP)
b = io.BytesIO()
g = BytesGenerator(b)
g.flatten(m)
self.assertEqual(b.getvalue(), msg)
def write_pkg_info(path, message):
with open(path, "wb") as out:
BytesGenerator(out, maxheaderlen=0).flatten(message)
def msg2bytes(msg):
f = MyBytesIO()
BytesGenerator(f).flatten(msg)
return f.getvalue()
text = lsb_release + '\n' + '\n'.join(env)
attachment = MIMEText(text, _charset='UTF-8')
attachment.add_header('Content-Disposition', 'inline')
message = MIMEMultipart()
message.add_header('Tags', 'snap')
message.attach(attachment)
blob = message.as_string().encode('UTF-8')
url = 'https://{}/+storeblob'.format(LAUNCHPAD)
data = MIMEMultipart()
submit = MIMEText('1')
submit.add_header('Content-Disposition', 'form-data; name="FORM_SUBMIT"')
data.attach(submit)
form_blob = MIMEBase('application', 'octet-stream')
form_blob.add_header('Content-Disposition',
'form-data; name="field.blob"; filename="x"')
form_blob.set_payload(blob.decode('ascii'))
data.attach(form_blob)
data_flat = BytesIO()
gen = BytesGenerator(data_flat, mangle_from_=False)
gen.flatten(data)
request = Request(url, data_flat.getvalue())
request.add_header('Content-Type',
'multipart/form-data; boundary=' + data.get_boundary())
opener = build_opener(HTTPSHandler)
result = opener.open(request)
handle = result.info().get('X-Launchpad-Blob-Token')
summary = '[snap] SUMMARY HERE'.encode('UTF-8')
params = urlencode({'field.title': summary})
filebug_url = 'https://bugs.{}/ubuntu/+source/libreoffice/+filebug/{}?{}'
subprocess.run(["xdg-open", filebug_url.format(LAUNCHPAD, handle, params)])
