def scan(self, filters=None):
filters = filters or []
for root, dirs, files in os.walk(self.path):
for filename in files:
if not filename.lower().endswith('.json'):
continue
with open(os.path.join(root, filename), 'rb') as fd:
data = json.load(fd)
email_uuid = data.pop('email_uuid',
data.pop('uuid', None))
data['filename'] = filename
ims = data['body'].encode('utf-8')
data['uuid'] = email_uuid
data.update(EmailMessage((email_uuid, mailbox.Message(ims))).to_dict())
to = data.get('to', None)
if not to:
continue
if isinstance(to, basestring) and not any([domain in to for domain in self.capture_subdomains]):
continue
if isinstance(to, (tuple, list)) and not any([domain in x for x in to for domain in self.capture_subdomains]):
continue
for validate in filter(callable, filters):
if not validate(data):
continue
self.box.append(data)
yield data
def filler_message( mid ):
m = mailbox.Message()
m['message-id'] = mid
m['subject'] = '[Not in archive]'
m['date'] = '[Not in archive]'
m['from'] = ''
return m
def deliver(self, user, domain, message, from_):
self.logger.info("Delivering message to: {}[@{}]".format(user, domain))
message = mailbox.Message(message)
success = True
path = self.mailbox_path.format(user=user, domain=domain)
user_mailbox = self.mailbox_format(path, create=True)
self.logger.debug("Mailboxpath for {}[@{}] is: {}".format(
user, domain, path))
try:
user_mailbox.lock()
user_mailbox.add(message)
except mailbox.ExternalClashError:
success = False
self.logger.warning("Mailbox not available of user {user}: "
"ExternalClash".format(user=user))
except Exception as e:
success = False
self.logger.error(
("Mailbox not available of user {user}: "
"" + str(e) + " " + str(e.args)).format(user=user))
else:
self.logger.info("Delivery to {}[@{}] successful".format(
user, domain))
finally:
user_mailbox.unlock()
user_mailbox.close()
return success
def get_calendar_items(username,
password,
host,
port=143,
mark_read=True,
criteria=['UNSEEN']):
imap = imaplib.IMAP4(host, port)
imap.login(username, password)
imap.select('Inbox', True)
unseen_nums = imap.search(None, 'UNSEEN')[1][0].split()
typ, data = imap.search(None, *criteria)
items = []
for num in data[0].split():
typ, message_data = imap.fetch(num, '(RFC822)')
# make the message unread again if it was before, if the 'mark_read' flag is not set
if not mark_read and num in unseen_nums:
imap.store(num, '-FLAGS', '\\Seen')
msg = mailbox.Message(message_data[0][1])
for part in msg.walk():
try:
if part.get_content_type() == "text/calendar":
calendar = part.get_payload(decode=1)
item = vobject.readOne(calendar).vevent
# ignore event acceptance etc. todo: update attendance list on reciept of one of these...
if item.summary.value.split(":")[0] not in [
'Accepted', 'Declined', 'Tentative', 'Canceled'
]:
items.append(item)
except Exception:
pass
imap.close()
imap.logout()
return items
def _decode_msg(msg: IO[Any]) -> mailbox.mboxMessage:
"""
Custom decode function
by default this uses 'ascii' which can cause fatal errors
on UnicodeDecodeErrors
"""
msg_str = try_decode_buf(msg.read())
return mailbox.mboxMessage(mailbox.Message(msg_str))
def create_message(frm, to, content, headers = None):
if not headers:
headers = {}
m = mailbox.Message()
m['from'] = frm
m['to'] = to
for h,v in headers.iteritems():
m[h] = v
m.set_payload(content)
return m
def unmangle_email(email_text, split_pattern):
# strip out the occasional headers
stripped = re.sub(
r"\n?cimsreport_open internet [A-Z0-9a-z_-]+\.txt\[[A-Z0-9\s:/]+\]\n",
"", email_text)
# getting rid of the initial integer and dividing line
stripped = re.split(r'^\d+\n={87}', stripped)[-1]
# next, split into individual email messages
split = re.split(split_pattern, stripped)
messages = []
utc = dateutil.tz.tzutc()
# then go through them pair-wise
for _number, message in chunkwise(itertools.islice(split, 1, None)):
number = _number.replace(',', '')
try:
parsed = mailbox.Message(message.encode('utf8'))
messages.append({
'id':
number,
'applicant':
parsed['From'].decode('utf8'),
'email_to':
parsed['To'].decode('utf8'),
'email_subject':
parsed['Subject'].decode('utf8'),
'dateRcpt':
dateutil.parser.parse(
parsed['Date']).replace(tzinfo=dateutil.tz.tzutc()).
isoformat(), # are they really UTC? Who knows?
'text':
parsed.get_payload().decode('utf8'),
'preprocessed':
True
})
except:
messages.append({
'id': number,
'text': message,
'preprocessed': True
})
return messages
def store_mail(mbx, mail):
"""
stores given mail in mailbox. If mailbox is maildir, set the S-flag and
return path to newly added mail. Oherwise this will return `None`.
:param mbx: mailbox to use
:type mbx: :class:`mailbox.Mailbox`
:param mail: the mail to store
:type mail: :class:`email.message.Message` or str
:returns: absolute path of mail-file for Maildir or None if mail was
successfully stored
:rtype: str or None
:raises: StoreMailError
"""
if not isinstance(mbx, mailbox.Mailbox):
logging.debug('Not a mailbox')
return False
mbx.lock()
if isinstance(mbx, mailbox.Maildir):
logging.debug('Maildir')
msg = mailbox.MaildirMessage(mail)
msg.set_flags('S')
else:
logging.debug('no Maildir')
msg = mailbox.Message(mail)
try:
message_id = mbx.add(msg)
mbx.flush()
mbx.unlock()
logging.debug('got mailbox msg id : %s', message_id)
except Exception as e:
raise StoreMailError(e)
path = None
# add new Maildir message to index and add tags
if isinstance(mbx, mailbox.Maildir):
# this is a dirty hack to get the path to the newly added file
# I wish the mailbox module were more helpful...
plist = glob.glob1(os.path.join(mbx._path, 'new'),
message_id + '*')
if plist:
path = os.path.join(mbx._path, 'new', plist[0])
logging.debug('path of saved msg: %s', path)
return path
def filename_from_content_disposition(response: httpx.Response) -> str:
"""
Extract and validate filename from a Content-Disposition header.
Eg...
Content-Disposition: attachment; filename=example.tar.gz
"""
content_disposition = response.headers.get("Content-Disposition")
if content_disposition:
msg = mailbox.Message("Content-Disposition: %s" % content_disposition)
filename = msg.get_filename()
if filename:
# Basic sanitation.
filename = os.path.basename(filename).lstrip(".").strip()
if filename:
return filename
return ""
def unmangle_email_two(email_text, split_pattern):
def _parse_special(t):
p_from = re.compile(r'\nFrom:([^\n]*)', re.MULTILINE)
p_date = re.compile(r'\nDate:([^\n]*)', re.MULTILINE)
p_subj = re.compile(r'\nSubject:([^\n]*)', re.MULTILINE)
def _remove_name(s, name):
try:
signature = re.search(re.compile(re.escape(name)), s)
return s[:signature.start()]
except:
return s
def _parse_fields(s):
name = ''
date = ''
subj = ''
name_match = re.findall(p_from, s)
date_match = re.findall(p_date, s)
subj_match = re.findall(p_subj, s)
ct = s[:]
for p in [p_from, p_date, p_subj]:
ct = p.sub('', ct)
if name_match:
name = name_match[0].strip()
ct = _remove_name(ct, name)
if date_match:
try:
date = dateutil.parser.parse(
date_match[0].strip()).replace(
tzinfo=dateutil.tz.tzutc()).isoformat()
except:
date = None
if subj_match:
subj = subj_match[0].strip()
return {'name': name, 'date': date, 'subj': subj, 'text': ct}
return _parse_fields(t)
# strip out the occasional headers
stripped = re.sub(
r"\n?cimsreport_open internet [A-Z0-9a-z_-]+\.txt\[[A-Z0-9\s:/]+\]\n",
"", email_text)
# getting rid of the initial integer and dividing line
stripped = re.split(r'^\d+\n={87}', stripped)[-1]
# next, split into individual email messages
split = re.split(split_pattern, stripped)
messages = []
utc = dateutil.tz.tzutc()
# then go through them pair-wise
for _number, message in zip(range(len(split)), split):
number = unicode(_number)
try:
parsed = mailbox.Message(message.encode('utf8'))
messages.append({
'id':
number,
'applicant':
parsed['From'].decode('utf8'),
'email_to':
parsed['To'].decode('utf8'),
'email_subject':
parsed['Subject'].decode('utf8'),
'dateRcpt':
dateutil.parser.parse(
parsed['Date']).replace(tzinfo=dateutil.tz.tzutc()).
isoformat(), # are they really UTC? Who knows?
'text':
parsed.get_payload().decode('utf8'),
'preprocessed':
True
})
except:
try:
parsed = _parse_special(message)
messages.append({
'id': number,
'text': parsed['text'],
'applicant': parsed.get('name', ''),
'dateRcpt': parsed.get('date', ''),
'email_subject': parsed.get('subj', ''),
'preprocessed': True
})
except:
messages.append({
'id': number,
'text': message,
'preprocessed': True
})
return messages
def decodeHeader(headerMsg):
L = header.decode_header(headerMsg)
s = ''
for s1, chset in L:
if (type(s1) == bytes):
s += s1.decode(chset) if chset else s1.decode()
else:
s += s1
return s
host = '' # 적당한 값을 써 넣을 것
userid = '' # 적당한 값을 써 넣을 것
passwd = '' # 적당한 값을 써 넣을 것
mbox = poplib.POP3(host)
mbox.user(userid)
mbox.pass_(passwd)
noMsg, tsize = mbox.stat()
for k in range(1, noMsg + 1):
res = mbox.top(k, 0)[1]
headerMsg = '\n'.join([b.decode('utf-8') for b in res])
msg = mailbox.Message(headerMsg)
print(
k, '보낸이: %s, 받는이: %s, 제목: %s, 날짜: %s' %
(decodeHeader(msg['from']), decodeHeader(
msg['to']), decodeHeader(msg['subject']), msg['date']))
mbox.quit()
"B.FinEco.Aswath-Damodaran",
"http://www.schneier.com/blog/index.rdf": "B.M.Bruce-Schenier",
"http://ftalphaville.ft.com/blog/?feed=rss2": "B.NP.FT_Alphaville",
}
# behave like sendmail and read message from stdin
msg = sys.stdin.read()
# pick up the feed from the X-RSS-Feed header set by rss2email
feed = Parser().parsestr(msg)['X-RSS-Feed']
# some websites now automatically redirect http:// requests to https://
# so we check for http:// even if X-RSS-Feed says https://
feed2 = feed.replace('https://', 'http://')
if feed in labels.keys():
folder = labels[feed]
elif feed2 in labels.keys():
folder = labels[feed2]
else:
# we have encountered a feed that we cannot handle because it is not
# there in labels so we write an error log and return a failure code
# to rss2email
f.write(
time.strftime("%d %b %y") + '\n' + 'Not found:' + feed +
'\n AND :' + feed2 + '\n')
sys.exit(1)
# add the message to the correct maildir subfolder
myfolder = rootfolder + folder
md = mailbox.Maildir(myfolder)
md.add(mailbox.Message(msg))
sys.exit(0)
def from_internet_message_string(cls, string):
message = mailbox.Message(string)
return cls((message['Message-ID'], message))
def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
msg = mailbox.Message(data)
mbox = mailbox.mbox('/var/mail/user')
mbox.lock()
mbox.add(msg)
mbox.close()
