def parsestr(self, text, headersonly=None):
soup = BeautifulSoup(text)
body = self._text_for_selector(soup, '#body')
msg = MIMEText(body, 'plain', 'utf-8')
from_text = self._parse_dfn_header(
self._text_for_selector(soup, '#from'))
from_name = from_text.split('<')[0].strip()
from_address = self._text_for_selector(soup, '#from a')
from_addr = email.utils.formataddr((from_name, from_address))
msg['From'] = from_addr
subject = self._text_for_selector(soup, 'h1')
msg['Subject'] = subject
message_id = self._parse_dfn_header(
self._text_for_selector(soup, '#message-id'))
msg['Message-ID'] = message_id.strip()
message_date = self._parse_dfn_header(
self._text_for_selector(soup, '#date'))
msg['Date'] = message_date.strip()
mbox_message = mailbox.mboxMessage(msg)
mbox_message.set_from(from_address,
email.utils.parsedate(message_date))
return mbox_message
def create(self):
'''
Loop through source emails and construct a fake email
'''
try:
self.src_mbox.lock()
self.dest_mbox.lock()
for msg in self.src_mbox:
fmsg = mailbox.mboxMessage()
fmsg.set_from('MAILER-DAEMON',
time_=self.extract_time(msg.get_from()))
for k, v in msg.items():
if k in ('To', 'From', 'Cc'):
fmsg[k] = self.anonomize(v)
elif k == 'Subject':
fmsg[k] = self.faker.company()
elif k == 'Date':
fmsg[k] = v
fmsg.set_payload(generate_paragraph()[-1])
self.dest_mbox.add(fmsg)
self.dest_mbox.flush()
finally:
self.src_mbox.unlock()
self.src_mbox.close()
self.dest_mbox.unlock()
self.dest_mbox.close()
def parsestr(self, text, headersonly=None):
soup = BeautifulSoup(text)
body = self._text_for_selector(soup, '#body')
msg = MIMEText(body,'plain','utf-8')
from_text = self._parse_dfn_header(self._text_for_selector(soup, '#from'))
from_name = from_text.split('<')[0].strip()
from_address = self._text_for_selector(soup,'#from a')
from_addr = email.utils.formataddr((from_name, from_address))
msg['From'] = from_addr
subject = self._text_for_selector(soup,'h1')
msg['Subject'] = subject
message_id = self._parse_dfn_header(self._text_for_selector(soup,'#message-id'))
msg['Message-ID'] = message_id.strip()
message_date = self._parse_dfn_header(self._text_for_selector(soup,'#date'))
msg['Date'] = message_date.strip()
mbox_message = mailbox.mboxMessage(msg)
mbox_message.set_from(from_address, email.utils.parsedate(message_date))
return mbox_message
def build_item(
username,
email_address,
subject,
category,
priority,
revision,
content,
attachments,
):
msg = email.MIMEText.MIMEText(content)
user_string = email.utils.formataddr((username, email_address))
msg[HEAD_FROM] = user_string
msg[HEAD_SUBJECT] = subject
msg[HEAD_DATE] = email.utils.formatdate()
msg[HEAD_MSG_ID] = email.utils.make_msgid(
transform_subject_to_filename(subject))
msg[HEAD_PRIORITY] = make_priority_string(priority)
if revision:
msg[HEAD_REVISION] = revision
msg.preamble = content
msg = mailbox.mboxMessage(msg)
msg.set_from(user_string)
msg.set_unixfrom(user_string)
return msg
def add(self, msg, folder, flags):
mmsg = mailbox.mboxMessage(msg)
if GMVaultExporter.GM_SEEN in flags:
mmsg.add_flag('R')
if GMVaultExporter.GM_FLAGGED in flags:
mmsg.add_flag('F')
self.subdir(folder).add(mmsg)
def add(self, msg, folder, flags):
mmsg = mailbox.mboxMessage(msg)
if GMVaultExporter.GM_SEEN in flags:
mmsg.add_flag('R')
if GMVaultExporter.GM_FLAGGED in flags:
mmsg.add_flag('F')
self.subdir(folder).add(mmsg)
def parsestr(self, text, headersonly=None):
soup = BeautifulSoup(text)
body = self._text_for_selector(soup, "#body")
msg = MIMEText(body, "plain", "utf-8")
from_text = str(
self._parse_dfn_header(self._text_for_selector(soup, "#from")))
from_name = from_text.split("<")[0].strip()
from_address = str(self._text_for_selector(soup, "#from a"))
from_addr = email.utils.formataddr((from_name, from_address))
msg["From"] = from_addr
subject = self._text_for_selector(soup, "h1")
msg["Subject"] = str(subject)
message_id = self._parse_dfn_header(
self._text_for_selector(soup, "#message-id"))
msg["Message-ID"] = str(message_id.strip())
message_date = self._parse_dfn_header(
self._text_for_selector(soup, "#date"))
msg["Date"] = str(message_date.strip())
mbox_message = mailbox.mboxMessage(msg)
mbox_message.set_from(from_address,
email.utils.parsedate(message_date))
return mbox_message
def custom_reader(data_stream):
data = data_stream.read()
try:
content = data.decode("ascii")
except (UnicodeDecodeError, UnicodeEncodeError) as e:
content = data.decode("cp1252", errors="replace")
return mailbox.mboxMessage(content)
def create (self):
'''
Loop through source emails and construct a fake email
'''
try:
self.src_mbox.lock()
self.dest_mbox.lock()
for msg in self.src_mbox:
fmsg = mailbox.mboxMessage()
fmsg.set_from('MAILER-DAEMON',time_=self.extract_time(msg.get_from()))
for k,v in msg.items():
if k in ('To', 'From', 'Cc'):
fmsg[k] = self.anonomize(v)
elif k =='Subject':
fmsg[k] = self.faker.company()
elif k == 'Date':
fmsg[k] = v
fmsg.set_payload(generate_paragraph()[-1])
self.dest_mbox.add(fmsg)
self.dest_mbox.flush()
finally:
self.src_mbox.unlock()
self.src_mbox.close()
self.dest_mbox.unlock()
self.dest_mbox.close()
def get_mbox_content(fn, options):
total, decode_err, parse_err, print_err = 0, 0, 0, 0
for bin_msg in iterate_mbox(fn):
id_ = get_from(bin_msg)
total += 1
try:
str_msg = bin_msg.decode('utf-8')
except Exception as e:
logging.error(f'failed to decode message {id_}: {e}')
decode_err += 1
continue
try:
msg = mboxMessage(str_msg)
except Exception as e:
parse_err += 1
logging.error(f'failed to parse message {id_}: {e}')
continue
if msg.is_multipart():
text = ''.join([p.as_string() for p in msg.get_payload()])
else:
text = msg.get_payload()
if not options.no_fix:
text = fix_text(text)
try:
print(text)
except Exception as e:
print_err += 1
logging.error(f'failed to print message {id_}: {e}')
continue
print(
f'{total} total, {decode_err} decode errors, '
f'{parse_err} parse errors, {print_err} write errors',
file=sys.stderr)
def mbox_reader(stream):
"""Read a non-ascii message from mailbox.
Based on https://stackoverflow.com/questions/37890123/how-to-trap-an-exception-that-occurs-in-code-underlying-python-for-loop
"""
data = stream.read()
text = data.decode(encoding="utf-8")
return mailbox.mboxMessage(text)
def create_mail(args: configargparse.Namespace) -> mailbox.mboxMessage:
msg = mailbox.mboxMessage()
msg['Date'] = email.utils.formatdate()
msg['Subject'] = args.subject
msg['From'] = args.sender
msg['To'] = args.recipient
msg.set_payload(sys.stdin.read())
return msg
def add(self, raw):
mail = mailbox.mboxMessage(raw)
self.lock()
try:
mailbox.Maildir.add(self, mail)
self.flush()
finally:
self.unlock()
def add(self, raw):
mail = mailbox.mboxMessage(raw)
self.lock()
try:
super(Mails, self).add(mail)
self.flush()
finally:
self.unlock()
def _decode_msg(msg: IO[Any]) -> mailbox.mboxMessage:
"""
Custom decode function
by default this uses 'ascii' which can cause fatal errors
on UnicodeDecodeErrors
"""
msg_str = try_decode_buf(msg.read())
return mailbox.mboxMessage(mailbox.Message(msg_str))
def create_mailbox_message(e_msg):
m = mailbox.mboxMessage(e_msg.mime_content)
if e_msg.is_read:
print("A")
m.set_flags('S')
return m
def save_message(path, email):
mbox = mailbox.mbox(path, create=True)
mbox.lock()
try:
msg = mailbox.mboxMessage(email)
mbox.add(msg)
mbox.flush()
finally:
mbox.unlock()
return
def save_message(path, email):
mbox = mailbox.mbox(path, create=True)
mbox.lock()
try:
msg = mailbox.mboxMessage(email)
mbox.add(msg)
mbox.flush()
finally:
mbox.unlock()
return
def create_message_from_string(self, data, filename=None):
if data.startswith('From '):
msg = mailbox.mboxMessage(data)
from_addr = re.sub(r"^From ", "", msg.get_unixfrom())
msg.from_addr = from_addr
msg.set_from(from_addr)
else:
msg = mailbox.Message(data)
msg.from_addr = msg.get('From')
return msg
def deliver_mbox(self,suspect):
mbox_msg=mailbox.mboxMessage(suspect.get_message_rep())
mbox_path=apply_template(self.config.get(self.section,'path'), suspect)
mbox=mailbox.mbox( mbox_path)
try:
mbox.lock()
mbox.add(mbox_msg)
mbox.flush()
except Exception,e:
self.logger.error("Could not store message %s to %s: %s"%(suspect.id,mbox_path,str(e)))
def deliver_mbox(self, suspect):
mbox_msg = mailbox.mboxMessage(suspect.get_message_rep())
mbox_path = apply_template(self.config.get(self.section, "path"), suspect)
mbox = mailbox.mbox(mbox_path)
try:
mbox.lock()
mbox.add(mbox_msg)
mbox.flush()
except Exception, e:
self.logger.error("Could not store message %s to %s: %s" % (suspect.id, mbox_path, str(e)))
def mbox_reader(stream):
"""
This func is an hack to fix encoding problem
More info
https://stackoverflow.com/questions/37890123/how-to-trap-an-exception-that-occurs-in-code-underlying-python-for-loop
"""
data = stream.read()
encoding = get_encoding(data)
text = data.decode(encoding=encoding)
return mailbox.mboxMessage(text)
def create_new_mail(self):
msg = mailbox.mboxMessage()
address = email.utils.formataddr(
('InetSim Relayer', '*****@*****.**'))
# See https://pymotw.com/3/mailbox/
#msg.set_unixfrom("InetSimRelayer " + datetime.now().strftime("%a %b %d %H:%M:%S %Y"))
msg["Date"] = email.utils.formatdate()
msg["Message-ID"] = email.utils.make_msgid("logs-" + self.todayDate)
msg["From"] = address
msg["To"] = address
msg["Subject"] = "Logs from " + self.todayDate
return msg
def process_message(self, peer, mailfrom, rcpttos, data):
headers = email.parser.Parser().parsestr(data)
values = {
'peer': ':'.join(map(str, peer)),
'mailfrom': mailfrom,
'rcpttos': ', '.join(rcpttos),
'subject': headers.get('subject'),
}
log.info('%(peer)s: %(mailfrom)s -> %(rcpttos)s [%(subject)s]', values)
if self.config.directory:
with create_maildir(self.config.directory, create=False) as mbox:
mbox.add(mailbox.mboxMessage(data))
def _add_message(self, mbox, msg_id, flags):
m = mailbox.mboxMessage()
m.set_payload('Hello world!', 'ascii')
m.add_header('from', '*****@*****.**')
m.add_header('to', '*****@*****.**')
m.add_header('subject', 'Hi!')
m.add_header('message-id', msg_id)
m.set_flags(flags)
mbox.lock()
try:
mbox.add(m)
finally:
mbox.unlock()
def _add_message(self, mbox, msg_id, flags):
m = mailbox.mboxMessage()
m.set_payload('Hello world!', 'ascii')
m.add_header('from', '*****@*****.**')
m.add_header('to', '*****@*****.**')
m.add_header('subject', 'Hi!')
m.add_header('message-id', msg_id)
m.set_flags(flags)
mbox.lock()
try:
mbox.add(m)
finally:
mbox.unlock()
def process_message(self, peer, mailfrom, rcpttos, data):
headers = email.parser.Parser().parsestr(data)
values = {
'peer': ':'.join(map(str, peer)),
'mailfrom': mailfrom,
'rcpttos': ', '.join(rcpttos),
'subject': headers.get('subject'),
}
log.info('%(peer)s: %(mailfrom)s -> %(rcpttos)s [%(subject)s]', values)
if self.config.directory:
with create_maildir(self.config.directory) as mbox:
mbox.add(mailbox.mboxMessage(data))
def send_error_mail(TO, FROM, MAILBOX, Text):
import mailbox
import email.utils
from_addr = email.utils.formataddr('offlineimap', FROM)
to_addr = email.utils.formataddr('OI Admin', TO)
msg = mailbox.mboxMessage()
msg['From'] = from_addr
msg['To'] = to_addr
msg['Date'] = email.utils.formatdate(localtime=True)
msg['Subject'] = 'Problems with offlineimap'
msg.set_payload(Text + "\n")
mbox = mailbox.Maildir(MAILBOX)
mbox.add(msg)
logging.info('Wrote mail to maildir')
def addmsg(self, em):
"""Convert an Xmpdf email to mboxMessage format and add to MBOX."""
self.mbox.lock()
try:
msg = mailbox.mboxMessage()
msg.set_unixfrom(
self._encode('Author' + self._encode(em.header.date)))
msg['Date'] = em.header.date
msg['From'] = self._encode(em.header.from_email)
msg['To'] = self._encode(em.header.to)
msg['Subject'] = self._encode(em.header.subject)
msg.set_payload(self._encode(em.body))
self.mbox.add(msg)
self.mbox.flush()
finally:
self.mbox.unlock()
def save_message(m, event):
num = event['Sender']
from_addr = email.utils.formataddr(('', num))
mbox = mailbox.mbox('events.mbox')
mbox.lock()
try:
msg = mailbox.mboxMessage()
msg['From'] = from_addr
msg['Subject'] = event['message']
msg.set_payload(event['message'])
mbox.add(msg)
mbox.flush()
finally:
mbox.unlock()
def make_phish(self):
has_sender = None
random_msg = None
random_from = None
while not has_sender:
random_msg = self.inbox[randint(0, len(self.inbox) - 1)]
random_from = self.inbox[randint(0, len(self.inbox) - 1)]
has_sender = self.extract_from(random_from)
phish = mailbox.mboxMessage()
phish['From'] = random_from['From']
phish['To'] = random_msg['To']
phish['Subject'] = random_msg['Subject']
phish = self.modify_phish(phish, random_msg)
phish.set_payload("This is the body for a generated phishing email.\n")
return phish
def make_phish(self):
has_sender = None
random_msg = None
random_from = None
while not has_sender:
random_msg = self.inbox[randint(0, len(self.inbox)-1)]
random_from = self.inbox[randint(0, len(self.inbox)-1)]
has_sender = self.extract_from(random_from)
phish = mailbox.mboxMessage()
phish['From'] = random_from['From']
phish['To'] = random_msg['To']
phish['Subject'] = random_msg['Subject']
phish = self.modify_phish(phish, random_msg)
phish.set_payload("This is the body for a generated phishing email.\n")
return phish
def create_email_message(
self,
archived_at: str,
body: str,
**header,
) -> Message:
"""
Parameters
----------
archived_at : URL to the Email message.
body : String that contains the body of the message.
header : Dictionary that contains all available header fields of the
message.
"""
msg = email.message.EmailMessage()
if body is not None:
try:
msg.set_content(body) # don't use charset="utf-16"
except Exception:
# UnicodeEncodeError: 'utf-16' codec can't encode character
# '\ud83d' in position 8638: surrogates not allowed
pass
for key, value in header.items():
if "content-type" == key:
msg.set_param("Content-Type", value)
elif "mime-version" == key:
msg.set_param("MIME-Version", value)
elif "content-transfer-encoding" == key:
msg.set_param("Content-Transfer-Encoding", value)
else:
try:
# TODO: find out why it sometimes raises
# email/_header_value_parser.py
# IndexError: list index out of range.
# Also look into UTF-8 encoding.
msg[key] = value
except Exception:
pass
if ((msg["Message-ID"] is None) and (msg["Date"] is not None)
and (msg["From"] is not None)):
msg["Message-ID"] = archived_at.split("/")[-1]
# convert to `EmailMessage` to `mboxMessage`
mbox_msg = mboxMessage(msg)
mbox_msg.add_header("Archived-At", "<" + archived_at + ">")
return mbox_msg
def load_mailset_from_csv():
csv_data = request.form.keys()[0]
csv_string = StringIO.StringIO(csv_data)
csv_reader = csv.reader(utf_8_encoder(csv_string))
headers = csv_reader.next()
messages = []
for row in csv_reader:
mail = ""
row[3] = ', '.join(filter(len, row[3].split(' ')))
for header, value in zip(headers, row):
if header == 'Body':
mail += "\n"
else:
mail += header + ": "
mail += value + "\n"
messages.append(mailbox.mboxMessage(mail))
mail_service.index_messages(messages)
return respond_json(None)
def entry2msg(e, title, encoding):
body = '<head><meta http-equiv="Content-Type" content="text/html; charset=' + encoding + '"></head>\n'
body += e.get('id', '') + '<br>\n'
body += e.get('summary', '') + '<br>\n'
for i in e.get('content', []):
body += i.value + '<br>\n'
m = MIMEText(body.encode(encoding, 'replace'), 'html', encoding)
m['From'] = Header(title, encoding)
m['Subject'] = Header(e.get('title', 'No title'), encoding)
m['Date'] = strftime('%a, %d %b %Y %H:%M:%S +0000',
e.get('published_parsed', gmtime()))
return mailbox.mboxMessage(m)
def create_mailbox(self, email: EmailModel,
app_date: TreatmentModel) -> None:
sentdate = app_date.treatmentDate - timedelta(days=20)
mbox = mailbox.mbox("./data/mail/mailbox.mbox")
mbox.lock()
try:
msg = mailbox.mboxMessage()
msg.set_unixfrom('author Sat Feb 7 01:05:34 2009')
msg.set_from(
"MAILER DAEMON",
time.strptime(str(sentdate) + " 12:00", "%Y-%m-%d %H:%M"))
msg['From'] = email.email_from
msg['To'] = email.email_to
msg['Subject'] = email.subject
msg.set_payload(email.content)
mbox.add(msg)
mbox.flush()
finally:
mbox.unlock()
def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
if isinstance(data, bytes):
headers = email.parser.BytesHeaderParser(
policy=policy.compat32).parsebytes(data)
else:
# in python 3.5 instance(data, str) is True -> we use the old code
headers = email.parser.Parser().parsestr(data)
values = {
'peer': ':'.join(map(str, peer)),
'mailfrom': mailfrom,
'rcpttos': ', '.join(rcpttos),
'subject': headers.get('subject'),
}
log.info('%(peer)s: %(mailfrom)s -> %(rcpttos)s [%(subject)s]', values)
if self.config.directory:
with create_maildir(self.config.directory, create=False) as mbox:
mbox.add(mailbox.mboxMessage(data))
def make_phish(self):
has_sender = None
random_msg = None
random_from = None
validEmail = None
while not has_sender or not validEmail:
random_msg = self.inbox[randint(0, len(self.inbox) - 1)]
random_from = self.inbox[randint(0, len(self.inbox) - 1)]
has_sender = random_from["From"]
validEmail = not ("List-Unsubscribe" in random_from.keys()
or "*****@*****.**"
in random_from["From"])
phish = mailbox.mboxMessage()
phish['From'] = random_from['From']
phish['To'] = random_msg['To']
phish['Subject'] = random_msg['Subject']
phish = self.modify_phish(phish, random_msg)
phish.set_payload("This is the body for a generated phishing email.\n")
return phish
def run(self):
while not self.kill_received:
try:
msg = self.errorQueue.get_nowait()
reason = msg[0]
email = msg[1]
if email[2] == '':
mboxFile = luser+"-"+email[1]+"@"+email[0]+"-"+ str(reason) +".mbox"
else:
mboxFile = luser+"-"+email[1]+"@"+email[0]+"-"+email[2]+"-"+ str(reason) +".mbox"
dest = mailbox.mbox(mboxes+mboxFile)
dest.lock()
dest.add(mailbox.mboxMessage(email[5]))
dest.flush()
dest.unlock()
self.logQueue.put("wrote to " + mboxFile + " " + email[3], True, 10)
except Queue.Empty:
time.sleep(1)
else:
exit()
def make_phish(self):
has_sender = None
random_msg = None
random_from = None
validEmail = None
while not has_sender or not validEmail:
random_msg = self.inbox[randint(0, len(self.inbox) - 1)]
random_from = self.inbox[randint(0, len(self.inbox) - 1)]
has_sender = random_from["From"]
validEmail = not (
"List-Unsubscribe" in random_from.keys() or "*****@*****.**" in random_from["From"]
)
phish = mailbox.mboxMessage()
phish["From"] = random_from["From"]
phish["To"] = random_msg["To"]
phish["Subject"] = random_msg["Subject"]
phish = self.modify_phish(phish, random_msg)
phish.set_payload("This is the body for a generated phishing email.\n")
return phish
def parse_bounces(filename, mintime=None, maxtime=None):
mb = mbox(filename)
errors = set()
count = 0
for key, msg in mb.iteritems():
message = mboxMessage(msg)
msgtime = parsedate(message['Date'])
count += 1
print >> sys.stderr, '\r', "%6d" % count, \
strftime("%Y/%m/%d %H:%M:%S", msgtime),
sys.stdout.flush()
t = timegm(msgtime)
if mintime and t < mintime:
continue
if maxtime and t > maxtime:
continue
temporary, permanent = all_failures(msg)
errors.update([p for p in permanent if '@' in p])
print >> sys.stderr, ""
print >> sys.stderr, "%s issues" % len(errors)
return errors
def parse_bounces(filename, mintime=None, maxtime=None):
mb = mbox(filename)
errors = set()
count = 0
for key, msg in mb.iteritems():
message = mboxMessage(msg)
msgtime = parsedate(message['Date'])
count += 1
print >> sys.stderr, '\r', "%6d" % count, \
strftime("%Y/%m/%d %H:%M:%S", msgtime),
sys.stdout.flush()
t = timegm(msgtime)
if mintime and t < mintime:
continue
if maxtime and t > maxtime:
continue
temporary, permanent = all_failures(msg)
errors.update([p for p in permanent if '@' in p])
print >> sys.stderr, ""
print >> sys.stderr, "%s issues" % len(errors)
return errors
def process_message(self, peer, mailfrom, rcpttos, data):
if self.mailboxFile is not None:
#print "-----------"+peer
inheaders = 1
lines = data.split('\n')
print '---------- MESSAGE FOLLOWS ----------'
for line in lines:
# headers first
if inheaders and not line:
print 'X-Peer:', peer[0]
inheaders = 0
print line
print '------------ END MESSAGE ------------'
mbox = self.mailboxFile
parser = email.Parser.Parser()
#parse data and complete fields
msg = parser.parsestr(data)
id = msg.get('Message-ID', '')
date = msg.get('Date', '')
subject = msg.get('Subject', '')
payload = msg.get_payload(decode=1)
charset = msg.get_charset()
content_type = msg.get_content_type()
print payload
mbox.lock()
try:
msg = mailbox.mboxMessage()
msg['From'] = mailfrom
msg['To'] = rcpttos
msg['Subject'] = subject
msg['Message-ID'] = id
msg['Date'] = date
msg.set_charset(charset)
msg.set_payload(payload)
msg.set_type(content_type)
mbox.add(msg)
mbox.flush()
finally:
mbox.unlock()
def test_unixfrom(self):
# Make sure the UNIX From line is forwarded to the incoming function.
msg = mailbox.mboxMessage()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg1>"
msg["Date"] = "2008-01-01 12:00:00"
msg.set_payload("msg1")
msg.set_from("[email protected] Mon Jul 21 11:44:51 2008")
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
# Message must have been accepted
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
email = Email.objects.first()
self.assertEqual(email.archived_date,
datetime(2008, 7, 21, 11, 44, 51, tzinfo=utc))
def demangle(correct_list, orig_mbx, out_mbx):
cnf_p = ConfigParser(dict_type=OrderedDict)
cnf_p.read(correct_list)
# pairs = dict(cnf_p.items(ADDR_SEC_LABEL))
pairs = dict(
(k, {"repl": v, "RE": re.compile(r"\b%s\b" % k, re.IGNORECASE)})
for (k, v) in cnf_p.items(ADDR_SEC_LABEL)
if v is not None
)
counter = 0
if os.path.exists(out_mbx):
shutil.move(out_mbx, "{0}.bak".format(out_mbx))
in_mbx = mailbox.mbox(orig_mbx)
out_mbx = mailbox.mbox(out_mbx)
out_mbx.lock()
for msg in in_mbx.itervalues():
msg_str = str(msg)
matches = MANGLED_ADDR_RE.search(msg_str)
if matches is not None:
u_from = msg.get_from()
for orig, fixed in pairs.items():
# if (orig is None) or (fixed is None):
# continue
# msg_str = msg_str.replace(orig, fixed)
msg_str = fixed["RE"].sub(fixed["repl"], msg_str)
counter += 1 # This is wrong
out_msg = mailbox.mboxMessage(msg_str)
out_msg.set_from(u_from)
out_mbx.add(out_msg)
else:
out_mbx.add(msg)
out_mbx.close()
in_mbx.close()
logging.info("Change counter = {0}".format(counter))
def test_unixfrom(self):
# Make sure the UNIX From line is forwarded to the incoming function.
msg = mailbox.mboxMessage()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg1>"
msg["Date"] = "2008-01-01 12:00:00"
msg.set_payload("msg1")
msg.set_from("[email protected] Mon Jul 21 11:44:51 2008")
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
self.command.execute(os.path.join(self.tmpdir, "test.mbox"), **kw)
#print(output.getvalue())
# Message must have been accepted
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
email = Email.objects.first()
self.assertEqual(
email.archived_date,
datetime(2008, 7, 21, 11, 44, 51, tzinfo=utc))
def demangle(correct_list, orig_mbx, out_mbx):
cnf_p = ConfigParser(dict_type=OrderedDict)
cnf_p.read(correct_list)
# pairs = dict(cnf_p.items(ADDR_SEC_LABEL))
pairs = dict((k, {
'repl': v,
'RE': re.compile(r'\b%s\b' % k, re.IGNORECASE)
}) for (k, v) in cnf_p.items(ADDR_SEC_LABEL) if v is not None)
counter = 0
if os.path.exists(out_mbx):
shutil.move(out_mbx, '{0}.bak'.format(out_mbx))
in_mbx = mailbox.mbox(orig_mbx)
out_mbx = mailbox.mbox(out_mbx)
out_mbx.lock()
for msg in in_mbx.itervalues():
msg_str = str(msg)
matches = MANGLED_ADDR_RE.search(msg_str)
if matches is not None:
u_from = msg.get_from()
for orig, fixed in pairs.items():
# if (orig is None) or (fixed is None):
# continue
# msg_str = msg_str.replace(orig, fixed)
msg_str = fixed['RE'].sub(fixed['repl'], msg_str)
counter += 1 # This is wrong
out_msg = mailbox.mboxMessage(msg_str)
out_msg.set_from(u_from)
out_mbx.add(out_msg)
else:
out_mbx.add(msg)
out_mbx.close()
in_mbx.close()
logging.info('Change counter = {0}'.format(counter))
def main(mboxfile, threadsfile=None, labelsfile=None, username=None,
password=None, verbose=False, label=None, delete=False,
msg_delay=0, thread_delay=0, skip_thread_delay = 0,
nodownload=False, query = None):
""" Archive Emails from Gmail to an mbox """
if username is None:
username = raw_input("Gmail account name: ")
if password is None:
from getpass import getpass
password = getpass("Password: "******"\n"): username = username[:-1]
if password.endswith("\n"): password = password[:-1]
ga = libgmail.GmailAccount(username, password)
if verbose: print "\nPlease wait, logging in..."
try:
ga.login()
except libgmail.GmailLoginFailure:
print "\nLogin failed. (Wrong username/password?)"
else:
if verbose: print "Log in successful.\n"
searches = libgmail.STANDARD_FOLDERS + ga.getLabelNames()
while label is None and query is None:
print "Select folder or label to archive: (Ctrl-C to exit)"
for optionId, optionName in enumerate(searches):
print " %d. %s" % (optionId, optionName)
print " %d. %s" % (len(searches), "QUERY")
try:
label = searches[int(raw_input("Choice: "))]
except ValueError:
print "Please select a folder or label by typing in the " \
"number in front of it."
raw_input()
label = None
except IndexError:
query = raw_input("Query: ")
except (KeyboardInterrupt, EOFError):
print ""
return 1
print
if verbose:
if label is not None:
print "Selected folder: %s" % label
else:
print "Selected query: %s" % query
if query is None:
if label in libgmail.STANDARD_FOLDERS:
result = ga.getMessagesByFolder(label, True)
else:
result = ga.getMessagesByLabel(label, True)
else:
result = ga.getMessagesByQuery(query, True)
if len(result):
archive_mbox = mbox(mboxfile)
archive_mbox.lock()
gmail_ids_in_mbox = {}
# get dict of gmail_ids already in mbox
for i, mbox_msg in enumerate(archive_mbox):
gmail_ids_in_mbox[mbox_msg['X-GmailID']] = i
labels_fh = StringIO()
if labelsfile is not None:
labels_fh = open(labelsfile, "w")
threads_fh = StringIO()
if threadsfile is not None:
threads_fh = open(threadsfile, "w")
gmail_ids = []
try:
for i, thread in enumerate(result):
if verbose:
print "\nThread ID: ", thread.id, " LEN ", \
len(thread)
labels_fh.write("%s: %s\n"
% (thread.id, thread.getLabels()))
gmail_ids_in_thread = []
local_thread_delay = thread_delay
for gmail_msg in thread:
if verbose:
print " ", gmail_msg.id, gmail_msg.number
gmail_ids_in_thread.append(str(gmail_msg.id))
gmail_ids.append(str(gmail_msg.id))
if nodownload:
if verbose: print " skipped (no download)"
local_thread_delay = skip_thread_delay
continue
if gmail_ids_in_mbox.has_key(str(gmail_msg.id)):
if verbose: print " skipped"
local_thread_delay = skip_thread_delay
continue # skip messages already in mbox
mbox_msg = mboxMessage(gmail_msg.source)
mbox_msg.add_header("X-GmailID",
gmail_msg.id.encode('ascii'))
archive_mbox.add(mbox_msg)
sleep(msg_delay)
threads_fh.write("%s\n" % gmail_ids_in_thread)
sleep(local_thread_delay)
if delete:
for gmail_id in gmail_ids_in_mbox.keys():
if gmail_id not in gmail_ids:
mbox_id = gmail_ids_in_mbox[gmail_id]
if verbose:
print "Delete id %s from mbox" % mbox_id
archive_mbox.remove(mbox_id)
except KeyboardInterrupt:
print "Keyboard Interrrupt"
finally:
if verbose: print "Flushing and closing mbox file"
threads_fh.close()
labels_fh.close()
archive_mbox.close()
else:
if label is not None:
print "No threads found in `%s`." % label
else:
print "No threads found in query `%s`." % query
if verbose: print "\n\nDone."
def mbox_reader(stream):
data = stream.read()
text = data.decode(encoding="utf-8")
return mailbox.mboxMessage(text)
# beginning of a line in the message body, they are escaped by prefixing the line with >.
import mailbox
import email.utils
from_addr = email.utils.formataddr(('Author', '*****@*****.**'))
to_addr = email.utils.formataddr(('Recipient', '*****@*****.**'))
payload = '''This is the body.
From (will not be escaped).
There are 3 lines.
'''
mbox = mailbox.mbox('example.mbox')
mbox.lock()
try:
msg = mailbox.mboxMessage()
msg.set_unixfrom('author Sat Feb 7 01:05:34 2009')
msg['From'] = from_addr
msg['To'] = to_addr
msg['Subject'] = 'Sample message 1'
msg.set_payload(payload)
mbox.add(msg)
mbox.flush()
msg = mailbox.mboxMessage()
msg.set_unixfrom('author')
msg['From'] = from_addr
msg['To'] = to_addr
msg['Subject'] = 'Sample message 2'
msg.set_payload('This is the second body.\n')
mbox.add(msg)
"""
"""
__version__ = "$Id: mailbox_mbox_create.py 1882 2009-01-04 15:38:33Z dhellmann $"
#end_pymotw_header
import mailbox
import email.utils
from_addr = email.utils.formataddr(('Author', '*****@*****.**'))
to_addr = email.utils.formataddr(('Recipient', '*****@*****.**'))
mbox = mailbox.mbox('example.mbox')
mbox.lock()
try:
msg = mailbox.mboxMessage()
msg.set_unixfrom('author')
msg['From'] = from_addr
msg['To'] = to_addr
msg['Subject'] = 'Sample message 1'
msg.set_payload('This is the body.\nFrom (should be escaped).\nThere are 3 lines.\n')
mbox.add(msg)
mbox.flush()
msg = mailbox.mboxMessage()
msg.set_unixfrom('author')
msg['From'] = from_addr
msg['To'] = to_addr
msg['Subject'] = 'Sample message 2'
msg.set_payload('This is the second body.\n')
mbox.add(msg)
#!/usr/bin/env python3
import sys
from glob import glob
from mailbox import mboxMessage
from os.path import join, splitext
path = '.'
if len(sys.argv) > 1:
path = sys.argv[1]
for path in glob(join(path, '*.mbox')):
with open(path, 'r') as mail_file:
message = mboxMessage(mail_file.read())
content = message.get_payload()
if isinstance(content, str):
html = message.get_payload(decode=True).decode('utf-8')
else:
html = content[0].get_payload(decode=True).decode('utf-8')
html = html.replace('charset=iso-8859-1', 'charset=utf-8')
file_name, _ = splitext(path)
with open(f'{file_name}.html', 'w') as html_file:
html_file.write(html)
def deliver(self, message):
self.box.lock()
self.box.add(mailbox.mboxMessage(bytes(message)))
self.box.unlock()
def archivebox(mbox, user):
disposition = 0
(flags, delimiter, mbox_name) = mbox
# Don't archive boxes in the archive
if mbox_name.startswith("Archive") or \
mbox_name.startswith("_ARCHIVE"):
return disposition
# If a box can't be selected, it can't be archived
if b'\NoSelect' in flags:
return disposition
tdir = archivemymail.config.target_dir
if tdir.startswith('~/'):
tdir = tdir.replace('~', '/home/' +
(user.split('@', 1))[0], 1)
if '%u' in tdir:
tdir = tdir.replace('%u', user)
logging.info("")
logging.info("====== %s ======", mbox_name)
archivemymail.server.select_folder(mbox_name)
msg_list = archivemymail.server.search(['OLD', 'BEFORE', '{:%d-%b-%Y}'.format(archivemymail.archivedate)])
archivemymail.mboxman = archivemymail.mboxman(user,
archivemymail.config.target_dir,
archivemymail.statsman,
archivemymail.config.dry_run,
archivemymail.config.compression)
progress = archivemymail.progress.Progress(len(msg_list))
logging.debug("%d messages to archive", len(msg_list))
for msg_num in msg_list:
disposition &= archivemymail.HAVE_ARCHIVED
imapmessage = archivemymail.server.fetch(msg_num, ['FLAGS', 'RFC822', 'INTERNALDATE'])[msg_num]
mboxflags = ''
if imapclient.RECENT in imapmessage['FLAGS']:
mboxflags += 'R'
if imapclient.SEEN in imapmessage['FLAGS']:
mboxflags += 'O'
if imapclient.DELETED in imapmessage['FLAGS']:
mboxflags += 'D'
if imapclient.FLAGGED in imapmessage['FLAGS']:
mboxflags += 'F'
if imapclient.ANSWERED in imapmessage['FLAGS']:
mboxflags += 'A'
logging.debug(' + Flags: %s', mboxflags)
# Get the body of the message
fp = email.parser.BytesFeedParser()
fp.feed(imapmessage['RFC822'])
message = mailbox.mboxMessage(fp.close())
message.set_flags(mboxflags)
# Use the internal date to determine which file to archive to
boxname = os.path.join(tdir,
"{boxpath}/{year}/{month:02}".format(boxpath=mbox_name,
year=imapmessage['INTERNALDATE'].year,
month=imapmessage['INTERNALDATE'].month))
logging.debug(" + File is %s.%s", boxname, archivemymail.config.compression)
archivemymail.mboxman.set_box(boxname, mbox_name.lower().find('spam') == -1)
# Log Progress
progress.log(message, boxname, boxname.lower().find('spam') == 1)
# Pass the message through SpamAssassin
if archivemymail.config.do_learning and not archivemymail.config.dry_run:
subprocess.run(['sa-learn', '--spam', '--no-sync',
'--dbpath', archivemymail.config.bayes_dir],
input=message.as_string()).check_returncode()
elif not archivemymail.config.dry_run:
subprocess.run(['sa-learn', '--ham', '--no-sync',
'--dbpath', archivemymail.config.bayes_dir],
input=message.as_string()).check_returncode()
# Add the message to the mbox
archivemymail.mboxman.add(message)
# And remove it from the mailbox
archivemymail.server.delete_messages(msg_num)
# Archived all the necessary messages
# Delete the folder, too, if there's nothing left
if mbox_name.lower() not in ('inbox', # Don't delete the special folders
'drafts',
'sent',
'spam',
'trash',
'learnspam',
'queue'):
msg_list = archivemymail.server.search(['NOT', 'DELETED'])
if len(msg_list) <= 0:
disposition &= archivemymail.MBOX_DELETED
if archivemymail.config.dry_run:
logging.info("Would delete now-empty folder %s", mbox_name)
else:
logging.debug("Deleting now-empty folder %s")
archivemymail.server.delete_folder(mbox_name)
archivemymail.server.close_folder()
return disposition
'./Mail/friend1/in/': ['*****@*****.**'],
'./Mail/friend2/in/': ['*****@*****.**'],
}
if not mail.get('To', None):
del mail["To"]
for p in folder_to_map:
if filepath.startswith(p):
for m in folder_to_map[p]:
mail.add_header('To', m)
break
try:
assert mail.get('From', None) is not None
assert mail.get('To', None)
assert mail.get('Subject', None) is not None
assert mail.get('Date', None) is not None
except:
import pdb; pdb.set_trace()
mbox.add(mailbox.mboxMessage(mail))
# The mail was successfuly migrated, remove its source
os.remove(filepath)
# Remove empty folders
for root, dirs, files in os.walk(ARCHIVE_FOLDER, topdown=False):
for name in dirs:
fname = os.path.join(root, name)
if os.path.exists(fname) and not os.listdir(fname):
print 'Remove %s ...' % fname
os.removedirs(fname)