def fetch_by_uid(self, uid):
message, data = self.connection.uid('fetch', uid, '(BODY.PEEK[])')
raw_email = data[0][1]
email_object = parse_email(raw_email)
return email_object
def fetch_by_uid(self, uid):
message, data = self.connection.uid('fetch', uid, '(BODY.PEEK[])')
raw_email = data[0][1]
email_object = parse_email(raw_email)
return email_object
def fetch_by_uid(self, uid):
message, data = self.connection.uid('fetch', uid, '(BODY.PEEK[])') # Don't mark the messages as read, save bandwidth with PEEK
raw_email = data[0][1]
email_object = parse_email(raw_email)
return email_object
async def get_list(request, username, tag):
global clients
client, timestamp = clients.get(username, [None, None])
if client and time.time() - timestamp > 60 * 60: # over an hour
del clients[username]
client = None
if not client:
return json(None, status=401)
messages = []
if client.selected is not tag:
for item in client.connection.list()[1]:
real_tag = shlex.split(item.decode())[-1]
if real_tag.lower().count(tag):
break
client.connection.select(quote_folder_name(real_tag))
client.selected = tag
uids = client.connection.uid('search', None, '(ALL)')[1][0].split()
if not uids:
return json([])
response = client.connection.uid(
"fetch", b",".join(uids[-100:]),
"(BODY.PEEK[HEADER.FIELDS (FROM TO DATE SUBJECT)] FLAGS)")
for labels, message in response[1][::2]:
message = parser.parse_email(message)
message.uid = labels.split()[2]
message.flags = imaplib.ParseFlags(labels)
del message.raw_email
messages.append(message)
return json(reversed(messages))
def fetch_by_uid(self, uid, **kwargs):
folder = kwargs.get('folder', None)
if folder:
self.select_folder(folder)
try:
if self.gmail:
message, data = self.connection.uid('fetch', uid, '(X-GM-MSGID X-GM-THRID UID FLAGS BODY.PEEK[])') # Don't mark the messages as read, save bandwidth with PEEK
else:
message, data = self.connection.uid('fetch', uid, '(UID FLAGS BODY.PEEK[])')
except imaplib.IMAP4.abort:
self.select_folder(folder)
if self.gmail:
message, data = self.connection.uid('fetch', uid, '(X-GM-MSGID X-GM-THRID UID FLAGS BODY.PEEK[])')
else:
message, data = self.connection.uid('fetch', uid, '(UID FLAGS BODY.PEEK[])')
raw_email = data[0][1]
self.maildata = {}
groups = None
if self.gmail and data is not None:
pattern = re.compile("^(\d+) \(X-GM-THRID (?P<gthrid>\d+) X-GM-MSGID (?P<gmsgid>\d+) UID (?P<uid>\d+) FLAGS \((?P<flags>[^\)]*)\)")
headers = data[0][0]
groups = pattern.match(headers).groups()
self.maildata['GTHRID'] = groups[1]
self.maildata['GMSGID'] = groups[2]
self.maildata['UID'] = groups[3]
self.maildata['FLAGS'] = groups[4]
if not self.gmail and data is not None:
# Try this pattern first
pattern = re.compile("^(\d+) \(UID (?P<uid>\d+) FLAGS \((?P<flags>[^\)]*)\)")
match = pattern.search(data[0][0])
if match:
groups = pattern.search(data[0][0]).groupdict()
self.maildata['FLAGS'] = groups.get('flags', None)
self.maildata['UID'] = groups.get('uid', None)
# If no match, try this pattern (its usually yahoo f*****g things up)
else:
pattern = re.compile("^(\d+) \(FLAGS \((?P<flags>[^\)]*)\) UID (?P<uid>\d+)")
match = pattern.search(data[0][0])
if match:
groups = pattern.search(data[0][0]).groupdict()
self.maildata['FLAGS'] = groups.get('flags', None)
self.maildata['UID'] = groups.get('uid', None)
# Last resort
else:
pattern = re.compile("^(\d+) \(UID (?P<uid>\d+)")
groups = pattern.search(data[0][0]).groupdict()
self.maildata['UID'] = groups.get('uid', None)
self.maildata['FLAGS'] = groups.get('flags', None)
self.maildata['data'] = raw_email
email_object = parse_email(self.maildata)
return email_object
def fetch_by_uid(self, uid):
message, data = self.connection.uid('fetch', uid, '(BODY.PEEK[])')
logger.debug("Fetched message for UID {}".format(int(uid)))
raw_email = data[0][1]
email_object = parse_email(raw_email, policy=self.parser_policy)
return email_object
def fetch_by_uid(self, uid):
message, data = self.connection.uid('fetch', uid, '(BODY.PEEK[])')
logger.debug("Fetched message for UID {}".format(int(uid)))
raw_email = data[0][1]
email_object = parse_email(raw_email)
return email_object
def fetch_by_uid(self, uid):
message, data = self.connection.uid(
'fetch', uid, '(BODY.PEEK[])'
) # Don't mark the messages as read, save bandwidth with PEEK
raw_email = data[0][1]
email_object = parse_email(raw_email)
return email_object
def fetch_by_uid(self, uid):
message, data = self.connection.uid('fetch', uid, '(BODY.PEEK[])')
raw_email = data[0][1]
if not isinstance(raw_email, unicode):
raw_email = raw_email.decode('utf8')
email_object = parse_email(raw_email)
return email_object
async def get_message_text(request, username, tag, uid):
global text_cache
response = text_cache.get(uid)
if not response:
client, _ = clients.get(username)
response = client.connection.uid("fetch", uid, "(BODY.PEEK[])")
labels, message = response[1][0]
message = parser.parse_email(message)
response = message.body["plain"][0] if message.body[
"plain"] else "No text"
text_cache[uid] = decode_bytes(response)
return text(response)
async def get_message_html(request, username, tag, uid):
global html_cache
response = html_cache.get(uid)
if not response:
client, _ = clients.get(username)
response = client.connection.uid("fetch", uid, "(BODY.PEEK[])")
labels, message = response[1][0]
message = parser.parse_email(message)
if not message.body["html"] and message.body["plain"]:
message.body["html"] = message.body["plain"]
response = iframe_script + decode_bytes(
message.body["html"][0]) if message.body["html"] else "No html"
return html(response)
def main(args):
try:
passwd = args.pop(0)
except IndexError:
passwd = getpass.getpass()
imaplib.Debug = 4
imap = IMAP4(IMAP_HOST, IMAP_PORT)
imap.starttls()
imap.login(USER, passwd)
imap.select(FOLDER)
type_, data = imap.search(None, b'ALL')
try:
for msgnum in data[0].split():
type_, data = imap.fetch(msgnum,
b'(BODY[HEADER.FIELDS (SUBJECT FROM)])')
msg = parse_email(data[0][1])
print('[msg #{}]\nFrom: {}\nSubject: {}\n'.format(
int(msgnum), msg.from_[0]['email'], msg.subject))
finally:
imap.close()
imap.logout()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 10 17:10:54 2019
@author: sebastian
"""
import os
if os.getenv('INTELMQ_TEST_EXOTIC'):
from imbox.parser import parse_email
with open(os.path.join(os.path.dirname(__file__),
'foobarzip.eml')) as handle:
EMAIL_ZIP_FOOBAR = parse_email(handle.read())
with open(os.path.join(os.path.dirname(__file__),
'foobartxt.eml')) as handle:
EMAIL_TXT_FOOBAR = parse_email(handle.read())
class MockedImbox():
def __init__(self,
hostname,
username=None,
password=None,
ssl=True,
port=None,
ssl_context=None,
policy=None,
starttls=False):
pass
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 10 17:10:54 2019
@author: sebastian
"""
import os
from copy import deepcopy
if os.getenv('INTELMQ_TEST_EXOTIC'):
from imbox.parser import parse_email
with open(os.path.join(os.path.dirname(__file__),
'foobarzip.eml')) as handle:
EMAIL_ZIP_FOOBAR = parse_email(handle.read())
with open(os.path.join(os.path.dirname(__file__),
'foobartxt.eml')) as handle:
EMAIL_TXT_FOOBAR = parse_email(handle.read())
with open(os.path.join(os.path.dirname(__file__),
'fake_attachment.eml')) as handle:
EMAIL_FAKE_ATTACHMENT = parse_email(handle.read())
class MockedImbox():
def __init__(self,
hostname,
username=None,
password=None,
ssl=True,
port=None,
ssl_context=None,
policy=None,
def run_against_stdin(self):
print('Running crawler on stdin')
message = parse_email(sys.stdin.read())
self.process_message(message)
print('Done')
