async def send_message(self,
message,
sender=None,
recipients=None,
mail_options=None,
rcpt_options=None):
'''
Converts message to a bytestring and passes it to sendmail.
The arguments are as for sendmail, except that messsage is an
email.message.Message object. If sender is None or recipients is
None, these arguments are taken from the headers of the Message as
described in RFC 2822 (a ValueError is raised if there is more than
one set of 'Resent-' headers). Regardless of the values of sender and
recipients, any Bcc field (or Resent-Bcc field, when the Message is a
resent) of the Message object won't be transmitted. The Message
object is then serialized using email.generator.BytesGenerator and
sendmail is called to transmit the message.
'Resent-Date' is a mandatory field if the Message is resent (RFC 2822
Section 3.6.6). In such a case, we use the 'Resent-*' fields.
However, if there is more than one 'Resent-' block there's no way to
unambiguously determine which one is the most recent in all cases,
so rather than guess we raise a ValueError in that case.
'''
resent_dates = message.get_all('Resent-Date')
if resent_dates and len(resent_dates) > 1:
raise ValueError(
"Message has more than one 'Resent-' header block")
if sender is None:
sender = extract_sender(message, resent_dates=resent_dates)
if recipients is None:
recipients = extract_recipients(message, resent_dates=resent_dates)
# Make a local copy so we can delete the bcc headers.
message_copy = copy.copy(message)
del message_copy['Bcc']
del message_copy['Resent-Bcc']
messageio = io.StringIO()
generator = email.generator.Generator(messageio)
generator.flatten(message_copy, linesep='\r\n')
flat_message = messageio.getvalue()
result = await self.sendmail(sender,
recipients,
flat_message,
mail_options=mail_options,
rcpt_options=rcpt_options)
return result
def IncomingMail(self, message):
print "Incoming message - saving to the sent file"
pathSent = os.path.join(self.amconfig, "mboxout")
try:
fp = open(pathSent, "ab")
generator = email.generator.Generator(fp)
generator.flatten(message)
fp.close()
print "Successfully written mail message to send stack"
except Exception, e:
print "Failed to update send stack (%s)" % e
raise
async def send_message(self, message, sender=None, recipients=None,
mail_options=None, rcpt_options=None):
'''
Converts message to a bytestring and passes it to sendmail.
The arguments are as for sendmail, except that messsage is an
email.message.Message object. If sender is None or recipients is
None, these arguments are taken from the headers of the Message as
described in RFC 2822 (a ValueError is raised if there is more than
one set of 'Resent-' headers). Regardless of the values of sender and
recipients, any Bcc field (or Resent-Bcc field, when the Message is a
resent) of the Message object won't be transmitted. The Message
object is then serialized using email.generator.BytesGenerator and
sendmail is called to transmit the message.
'Resent-Date' is a mandatory field if the Message is resent (RFC 2822
Section 3.6.6). In such a case, we use the 'Resent-*' fields.
However, if there is more than one 'Resent-' block there's no way to
unambiguously determine which one is the most recent in all cases,
so rather than guess we raise a ValueError in that case.
'''
resent_dates = message.get_all('Resent-Date')
if resent_dates and len(resent_dates) > 1:
raise ValueError(
"Message has more than one 'Resent-' header block")
if sender is None:
sender = extract_sender(message, resent_dates=resent_dates)
if recipients is None:
recipients = extract_recipients(message, resent_dates=resent_dates)
# Make a local copy so we can delete the bcc headers.
message_copy = copy.copy(message)
del message_copy['Bcc']
del message_copy['Resent-Bcc']
messageio = io.StringIO()
generator = email.generator.Generator(messageio)
generator.flatten(message_copy, linesep='\r\n')
flat_message = messageio.getvalue()
result = await self.sendmail(
sender, recipients, flat_message, mail_options=mail_options,
rcpt_options=rcpt_options)
return result
def processMail(self, mail):
"""
Processes a specific mail by saving it.
Parameters
----------
mail : email
The mail to process.
"""
filename = '%s-%s.eml' % (self.getDate(mail), mail.get('Subject'))
filename = self.validateFilename(filename)
filename = os.path.join(self.basepath, filename)
with open(filename, 'w') as f:
generator = email.generator.Generator(f)
generator.flatten(mail)
if mail.is_multipart(): self.processAttachments(mail)
def flatten_message(message: Message) -> Tuple[str, List[str], str]:
resent_dates = message.get_all('Resent-Date')
if resent_dates is not None and len(resent_dates) > 1:
raise ValueError("Message has more than one 'Resent-' header block")
sender = _extract_sender(message, resent_dates=resent_dates)
recipients = _extract_recipients(message, resent_dates=resent_dates)
# Make a local copy so we can delete the bcc headers.
message_copy = copy.copy(message)
del message_copy['Bcc']
del message_copy['Resent-Bcc']
messageio = io.StringIO()
generator = email.generator.Generator(messageio)
generator.flatten(message_copy, linesep='\r\n')
flat = messageio.getvalue()
return str(sender), [str(recipient) for recipient in recipients], flat
def flatten_message(
message: Union[email.message.EmailMessage, email.message.Message],
utf8: bool = False,
cte_type: str = "8bit",
) -> bytes:
# Make a local copy so we can delete the bcc headers.
message_copy = copy.copy(message)
del message_copy["Bcc"]
del message_copy["Resent-Bcc"]
if isinstance(message.policy, email.policy.Compat32): # type: ignore
# Compat32 cannot use UTF8
policy = message.policy.clone( # type: ignore
linesep=LINE_SEP, cte_type=cte_type)
else:
policy = message.policy.clone( # type: ignore
linesep=LINE_SEP, utf8=utf8, cte_type=cte_type)
with io.BytesIO() as messageio:
generator = email.generator.BytesGenerator(messageio, policy=policy)
generator.flatten(message_copy)
flat_message = messageio.getvalue()
return flat_message
def flatten_message(
message: Message, utf8: bool = False, cte_type: str = "8bit"
) -> bytes:
# Make a local copy so we can delete the bcc headers.
message_copy = copy.copy(message)
del message_copy["Bcc"]
del message_copy["Resent-Bcc"]
if utf8:
policy = email.policy.SMTPUTF8 # type: email.policy.Policy
else:
policy = email.policy.SMTP
if policy.cte_type != cte_type:
policy = policy.clone(cte_type=cte_type)
with io.BytesIO() as messageio:
generator = email.generator.BytesGenerator( # type: ignore
messageio, policy=policy
)
generator.flatten(message_copy)
flat_message = messageio.getvalue()
return flat_message
#!/usr/bin/env python
import email
import email.generator
import io
import sys
raw_message = sys.stdin.read()
message = email.message_from_string(raw_message)
date = message['Date']
if date:
date = email.utils.parsedate_to_datetime(date)
local_date = date.astimezone()
message.add_header('X-Local-Date', email.utils.format_datetime(local_date))
stream = io.StringIO()
generator = email.generator.Generator(stream, mangle_from_=False)
generator.flatten(message)
raw_message = stream.getvalue()
print(raw_message, end='', flush=True)
def send_message(self, message, sender=None, recipients=None,
mail_options=[], rcpt_options=[]):
"""Converts message to a bytestring and passes it to sendmail.
The arguments are as for sendmail, except that messsage is an
email.message.Message object. If sender is None or recipients is
None, these arguments are taken from the headers of the Message as
described in RFC 2822 (a ValueError is raised if there is more than
one set of 'Resent-' headers). Regardless of the values of sender and
recipients, any Bcc field (or Resent-Bcc field, when the Message is a
resent) of the Message object won't be transmitted. The Message
object is then serialized using email.generator.BytesGenerator and
sendmail is called to transmit the message.
"""
# 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822
# Section 3.6.6). In such a case, we use the 'Resent-*' fields.
# However, if there is more than one 'Resent-' block there's no way to
# unambiguously determine which one is the most recent in all cases,
# so rather than guess we raise a ValueError in that case.
#
# TODO implement heuristics to guess the correct Resent-* block with an
# option allowing the user to enable the heuristics. (It should be
# possible to guess correctly almost all of the time.)
resent = message.get_all('Resent-Date')
if resent is None:
header_prefix = lambda s: s
elif len(resent) == 1:
header_prefix = lambda s: "{}{}".format('Resent-', s)
else:
raise ValueError(
"Message has more than one 'Resent-' header block")
if not sender:
# Prefer the sender field per RFC 2822:3.6.2.
if header_prefix('Sender') in message:
sender = message[header_prefix('Sender')]
else:
sender = message[header_prefix('From')]
if not recipients:
recipients = []
address_fields = []
for field in ('To', 'Cc', 'Bcc'):
address_fields.extend(
message.get_all(header_prefix(field), []))
for address in email.utils.getaddresses(address_fields):
recipients.append(address)
# Make a local copy so we can delete the bcc headers.
message_copy = copy.copy(message)
del message_copy['Bcc']
del message_copy['Resent-Bcc']
# Generate into string
with io.BytesIO() as messageio:
generator = email.generator.BytesGenerator(messageio)
generator.flatten(message_copy, linesep='\r\n')
flat_message = messageio.getvalue()
# finally, send the message
result = yield from self.sendmail(sender, recipients, flat_message,
mail_options, rcpt_options)
return result
def send_message(self,
message,
sender=None,
recipients=None,
mail_options=[],
rcpt_options=[]):
"""Converts message to a bytestring and passes it to sendmail.
The arguments are as for sendmail, except that messsage is an
email.message.Message object. If sender is None or recipients is
None, these arguments are taken from the headers of the Message as
described in RFC 2822 (a ValueError is raised if there is more than
one set of 'Resent-' headers). Regardless of the values of sender and
recipients, any Bcc field (or Resent-Bcc field, when the Message is a
resent) of the Message object won't be transmitted. The Message
object is then serialized using email.generator.BytesGenerator and
sendmail is called to transmit the message.
"""
# 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822
# Section 3.6.6). In such a case, we use the 'Resent-*' fields.
# However, if there is more than one 'Resent-' block there's no way to
# unambiguously determine which one is the most recent in all cases,
# so rather than guess we raise a ValueError in that case.
#
# TODO implement heuristics to guess the correct Resent-* block with an
# option allowing the user to enable the heuristics. (It should be
# possible to guess correctly almost all of the time.)
resent = message.get_all('Resent-Date')
if resent is None:
header_prefix = lambda s: s
elif len(resent) == 1:
header_prefix = lambda s: "{}{}".format('Resent-', s)
else:
raise ValueError(
"Message has more than one 'Resent-' header block")
if not sender:
# Prefer the sender field per RFC 2822:3.6.2.
if header_prefix('Sender') in message:
sender = message[header_prefix('Sender')]
else:
sender = message[header_prefix('From')]
if not recipients:
recipients = []
address_fields = []
for field in ('To', 'Cc', 'Bcc'):
address_fields.extend(message.get_all(header_prefix(field),
[]))
for address in email.utils.getaddresses(address_fields):
recipients.append(address)
# Make a local copy so we can delete the bcc headers.
message_copy = copy.copy(message)
del message_copy['Bcc']
del message_copy['Resent-Bcc']
# Generate into string
with io.BytesIO() as messageio:
generator = email.generator.BytesGenerator(messageio)
generator.flatten(message_copy, linesep='\r\n')
flat_message = messageio.getvalue()
# finally, send the message
result = yield from self.sendmail(sender, recipients, flat_message,
mail_options, rcpt_options)
return result
def _run_customizer(self):
"""Executes the entire process of customize a mailing for a recipient
and returns its full path.
This may take some time and shouldn't be run from the reactor thread.
"""
try:
fullpath = os.path.join(
self.temp_path,
MailCustomizer.make_file_name(self.recipient.mailing.id,
self.recipient.id))
if os.path.exists(fullpath):
self.log.debug("Customized email found here: %s", fullpath)
parser = email.parser.Parser()
with file(fullpath, 'rt') as fd:
header = parser.parse(fd, headersonly=True)
return header['Message-ID'], fullpath
contact_data = self.make_contact_data_dict(self.recipient)
message = self._parse_message()
assert (isinstance(contact_data, dict))
assert (isinstance(message, Message))
#email.iterators._structure(message)
mixed_attachments = []
related_attachments = []
for attachment in contact_data.get('attachments', []):
if 'content-id' in attachment:
related_attachments.append(attachment)
else:
mixed_attachments.append(attachment)
#bodies = MailingBody.objects.filter(relay = self.recipient.mailing_queue).order_by('header_pos')
def convert_to_mixed(part, mixed_attachments, subtype):
import email.mime.multipart
part2 = email.mime.multipart.MIMEMultipart(_subtype=subtype)
part2.set_payload(part.get_payload())
del part['Content-Type']
part['Content-Type'] = 'multipart/mixed'
part.set_payload(None)
part.attach(part2)
for attachment in mixed_attachments:
part.attach(self._make_mime_part(attachment))
def personalise_bodies(part,
mixed_attachments=[],
related_attachments=[]):
import email.message
assert (isinstance(part, email.message.Message))
if part.is_multipart():
subtype = part.get_content_subtype()
if subtype == 'mixed':
personalise_bodies(
part.get_payload(0),
related_attachments=related_attachments)
for attachment in mixed_attachments:
part.attach(self._make_mime_part(attachment))
elif subtype == 'alternative':
for p in part.get_payload():
personalise_bodies(
p, related_attachments=related_attachments)
if mixed_attachments:
convert_to_mixed(part,
mixed_attachments,
subtype="alternative")
elif subtype == 'digest':
raise email.errors.MessageParseError, "multipart/digest not supported"
elif subtype == 'parallel':
raise email.errors.MessageParseError, "multipart/parallel not supported"
elif subtype == 'related':
personalise_bodies(part.get_payload(0))
for attachment in related_attachments:
part.attach(self._make_mime_part(attachment))
if mixed_attachments:
convert_to_mixed(part,
mixed_attachments,
subtype="related")
else:
self.log.warn("Unknown multipart subtype '%s'" %
subtype)
else:
maintype = part.get_content_maintype()
if maintype == 'text':
self._customize_message(part, contact_data)
if mixed_attachments:
import email.mime.text
part2 = email.mime.text.MIMEText(
part.get_payload(decode=True))
del part['Content-Type']
part['Content-Type'] = 'multipart/mixed'
part.set_payload(None)
part.attach(part2)
for attachment in mixed_attachments:
part.attach(self._make_mime_part(attachment))
else:
self.log.warn(
"personalise_bodies(): can't handle '%s' parts" %
part.get_content_type())
personalise_bodies(message, mixed_attachments, related_attachments)
# Customize the subject
subject = self._do_customization(
header_to_unicode(message.get("Subject", "")), contact_data)
# Remove some headers
for header in ('Subject', 'Received', 'To', 'From', 'User-Agent',
'Date', 'Message-ID', 'List-Unsubscribe',
'DKIM-Signature', 'Authentication-Results',
'Received-SPF', 'Received-SPF', 'X-Received',
'Delivered-To', 'Feedback-ID', 'Precedence',
'Return-Path'):
if header in message:
del message[header]
message['Subject'] = Header(subject)
# Adding missing headers
# message['Precedence'] = "bulk"
h = Header(self.recipient.sender_name or '')
h.append("<%s>" % self.recipient.mail_from)
message['From'] = h
h = Header()
h.append(contact_data.get('firstname') or '')
h.append(contact_data.get('lastname') or '')
h.append("<%s>" % contact_data['email'])
message['To'] = h
message['Date'] = email.utils.formatdate()
# message['Message-ID'] = email.utils.make_msgid() # very very slow on certain circumstance
message['Message-ID'] = "<%s.%d@cm.%s>" % (
self.recipient.id, self.recipient.mailing.id,
self.recipient.domain_name)
if self.unsubscribe_url:
message['List-Unsubscribe'] = self.unsubscribe_url
fp = cStringIO.StringIO()
generator = email.generator.Generator(fp, mangle_from_=False)
generator.flatten(message)
flattened_message = fp.getvalue()
flattened_message = self.add_dkim_signature(flattened_message)
flattened_message = self.add_fbl(flattened_message)
with open(fullpath + '.tmp', 'wt') as fp:
fp.write(flattened_message)
fp.close()
if os.path.exists(fullpath):
os.remove(fullpath)
os.rename(fullpath + '.tmp', fullpath)
return message['Message-ID'], fullpath
except Exception:
self.log.exception(
"Failed to customize mailing '%s' for recipient '%s'" %
(self.recipient.mail_from, self.recipient.email))
raise
def main(
*inputs:
dict(metavar="input", help="input dump streams (default: stdin)"),
log: dict(short="-l", help="input log stream") = None,
):
"""Merge dumps of different parts of a Subversion repository
Multiple dump streams may be concatenated into a single input stream if
they have contiguous revision numbers.
Separate dump streams can also be given of separate paths in the
repository. In this case a log input is also required to determine
whether there are any copies between the paths to restore.
"""
if log:
with open(log, "rb") as log:
copies = dict()
for [rev, rev_copies] in svnlog.iter_svn_copies(log):
if rev_copies is None:
continue
copies[rev] = rev_copies
else:
copies = None
with ExitStack() as cleanup:
dumps = list()
for input in inputs:
stream = cleanup.enter_context(open(input, "rb"))
dumps.append({"stream": stream})
if not dumps:
dumps = ({"stream": stdin.buffer},)
out_version = None
for dump in dumps:
[record, content] = read_record(dump["stream"])
[version] = record.get_all("SVN-fs-dump-format-version", ())
dump["version"] = int(version)
if out_version is None:
out_version = dump["version"]
else:
out_version = max(out_version, dump["version"])
version = ("SVN-fs-dump-format-version", format(out_version))
write_message_fields(stdout.buffer, (version,))
out_uuid = None
out_record = None
end = False
for dump in dumps:
try:
[record, content] = read_record(dump["stream"])
uuid = record.get_all("UUID", ())
if uuid:
if dump["version"] < 2:
warn(f"{dump['stream'].name}: UUID record only "
"expected in version >= 2")
[uuid] = uuid
if out_uuid is None:
out_uuid = uuid
elif out_uuid != uuid:
warn(f"{dump['stream'].name}: Conflicting UUID {uuid}; "
f"expected {out_uuid}")
[record, content] = read_record(dump["stream"])
assert content is not None
if out_record is None:
out_record = record
out_content = content
elif (record.items() != out_record.items() or
content != out_content):
new_record = email.message.Message()
for field in ("Revision-number", "Prop-content-length",
"Content-length"):
values = out_record.get_all(field, ())
if (record.get_all(field, ()) != values):
warn(f"{dump['stream'].name}: Conflicting "
f"{field} field")
for value in values:
new_record[field] = value
out_record = new_record
if content != out_content:
warn(f"{dump['stream'].name}: Conflicting content")
except EOFError:
end = True
if out_uuid is not None and out_version >= 2:
write_message_fields(stdout.buffer, (("UUID", out_uuid),))
if not end:
generator = email.generator.BytesGenerator(stdout.buffer,
mangle_from_=False)
generator.flatten(out_record)
stdout.buffer.write(out_content)
def main(
*inputs: dict(metavar="input", help="input dump streams (default: stdin)"),
log: dict(short="-l", help="input log stream") = None,
):
"""Merge dumps of different parts of a Subversion repository
Multiple dump streams may be concatenated into a single input stream if
they have contiguous revision numbers.
Separate dump streams can also be given of separate paths in the
repository. In this case a log input is also required to determine
whether there are any copies between the paths to restore.
"""
if log:
with open(log, "rb") as log:
copies = dict()
for [rev, rev_copies] in svnlog.iter_svn_copies(log):
if rev_copies is None:
continue
copies[rev] = rev_copies
else:
copies = None
with ExitStack() as cleanup:
dumps = list()
for input in inputs:
stream = cleanup.enter_context(open(input, "rb"))
dumps.append({"stream": stream})
if not dumps:
dumps = ({"stream": stdin.buffer}, )
out_version = None
for dump in dumps:
[record, content] = read_record(dump["stream"])
[version] = record.get_all("SVN-fs-dump-format-version", ())
dump["version"] = int(version)
if out_version is None:
out_version = dump["version"]
else:
out_version = max(out_version, dump["version"])
version = ("SVN-fs-dump-format-version", format(out_version))
write_message_fields(stdout.buffer, (version, ))
out_uuid = None
out_record = None
end = False
for dump in dumps:
try:
[record, content] = read_record(dump["stream"])
uuid = record.get_all("UUID", ())
if uuid:
if dump["version"] < 2:
warn(f"{dump['stream'].name}: UUID record only "
"expected in version >= 2")
[uuid] = uuid
if out_uuid is None:
out_uuid = uuid
elif out_uuid != uuid:
warn(
f"{dump['stream'].name}: Conflicting UUID {uuid}; "
f"expected {out_uuid}")
[record, content] = read_record(dump["stream"])
assert content is not None
if out_record is None:
out_record = record
out_content = content
elif (record.items() != out_record.items()
or content != out_content):
new_record = email.message.Message()
for field in ("Revision-number", "Prop-content-length",
"Content-length"):
values = out_record.get_all(field, ())
if (record.get_all(field, ()) != values):
warn(f"{dump['stream'].name}: Conflicting "
f"{field} field")
for value in values:
new_record[field] = value
out_record = new_record
if content != out_content:
warn(f"{dump['stream'].name}: Conflicting content")
except EOFError:
end = True
if out_uuid is not None and out_version >= 2:
write_message_fields(stdout.buffer, (("UUID", out_uuid), ))
if not end:
generator = email.generator.BytesGenerator(stdout.buffer,
mangle_from_=False)
generator.flatten(out_record)
stdout.buffer.write(out_content)
def main(argv):
nome = ''
try:
opts, args = getopt.getopt(argv, "hi:o:", ["ifile=", "ofile="])
except getopt.GetoptError:
print 'convert.py -i <filename>.imm'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'convert.py -i <inputfile>.imm'
sys.exit()
conn = sqlite3.connect('Containers.db')
c = conn.cursor()
ca = conn.cursor()
Consulta = conn
cons = Consulta.execute(
"Select ContainerID from Containers where FileName ='" + nome +
"'").fetchall()
if len(cons) == 0:
print "File or table not found " + nome
exit()
else:
for caa in cons:
cid = caa[0]
dirsaida = './recovered/' + nome
try:
os.stat(dirsaida)
except:
os.mkdir(dirsaida)
consulta = "Select HeaderID, FromSender,ToSender,MsgPos,MsgSize,ReceivedDate,Subject from Headers where ContainerID like'" + cid + "' and Deleted = 0 ;"
nom = nome + '.imm'
f = open(nom)
cnt = 0
for s in c.execute(consulta):
header = s[0] #HeaderID
fromsender = s[1] #FromSender
tosender = 'Delivered-To:' + s[2] #ToSender
msgpos = s[3] #MsgPos
msgsize = s[4] #MsgSize
rec = 'Received: by 10.0.0.1 with SMTP id 123123aa'
v = datetime.datetime.fromtimestamp(s[5]) #ReceivedDate ja convertido
con = "select Path from Attachments where HeaderID = '" + header + "';"
f.seek(msgpos)
data = f.read(msgsize)
filename = nome + '-part-%03d%s' % (cnt, '.eml')
fp = open(os.path.join('./tmp', filename), 'w')
fp.write(tosender.encode('utf8'))
fp.write('\n')
fp.write(data)
fp.close()
d = open(os.path.join('./tmp', filename))
cnt += 1
msg = email.message_from_file(d)
for anexos in ca.execute(con):
an = anexos[0].replace("\\", "/")
if os.path.isfile(os.path.join('./Attachments',
an.encode('utf8'))):
path = os.path.join('./Attachments', an.encode('utf8'))
ctype, encoding = mimetypes.guess_type(path)
if ctype is None or encoding is not None:
ctype = 'application/octet-stream'
maintype, subtype = ctype.split('/', 1)
fp = open(path, 'rb')
print ctype
if subtype == 'text':
part = MIMEText(fp.read(), _subtype=subtype)
elif subtype == 'image':
part = MIMEImage(fp.read(), _subtype=subtype)
elif subtype == 'audio':
part = MIMEAudio(fp.read(), _subtype=subtype)
else:
part = MIMEBase(maintype, subtype)
part.set_payload(fp.read())
Encoders.encode_base64(part)
fp.close()
nan1 = "recuperado-" + an.encode('utf8')
part.add_header('Content-Disposition',
'attachment;filename=\"' + nan1 + '\"')
try:
msg.attach(part)
print subtype
except Exception, e:
#print part
print header + " " + nan1.decode('utf8')
print maintype + ' ' + ctype
print e
continue
d.close()
dirsaida = './recovered/' + nome
saida = open(os.path.join(dirsaida, filename), 'w')
generator = email.generator.Generator(saida)
generator.flatten(msg)
