def flags(self) -> (str, ):
"""
Message flags
*This attribute will not be changed after actions: flag, seen
"""
result = []
if type(self._raw_flag_data) is bytes:
result = imaplib.ParseFlags(self._raw_flag_data)
else:
for raw_flag_item in self._raw_flag_data:
result.extend(imaplib.ParseFlags(raw_flag_item))
return tuple(i.decode().strip().replace('\\', '').upper()
for i in result)
def flags(self):
'Get flags'
r, data = self.server.uid('fetch', self.uid, '(FLAGS)')
if r != 'OK':
raise IMAPError(self.format_error('Could not get flags', data))
string = data[0]
return imaplib.ParseFlags(string) if string else ()
def moveMsg(self,boxname):
# Move messages from defined boxname
selectedbox = self.From.select(boxname)
print("Fetching emails from %s"%boxname)
# Clear up the flag emails (incase it has stalled and you've started again)
self.expunge()
typ, data = self.From.search(None, 'ALL')
print(">>>>" + typ)
print("Found %d Messages"%len(data[0].split()))
regex = r"Message-ID:[\s|\S]?<(.*?)>\s"
for muid in data[0].split():
# start messageUID process
status, msg = self.From.fetch(muid,"(FLAGS INTERNALDATE BODY.PEEK[])") #Get email
Email = msg[0][1]
flags = imaplib.ParseFlags(msg[0][0]) # get Flags
if "\\Recent" in flags:
flag_str = "\\Seen"
else:
flag_str = " ".join(flags)
print(">>>>>" + flag_str)
date = imaplib.Time2Internaldate(imaplib.Internaldate2tuple(msg[0][0])) #get date
copy_result = self.To.append(boxname, flag_str, date, Email) # Copy to Archive
if copy_result[0] == 'OK':
del_msg = self.From.store(muid, '+FLAGS', '\\Deleted') # Mark for deletion (this is deleted via expunge)
self.expunge()
def list_last_emails(self, nb=20):
"""List last messages"""
res, data = self.connection.search(None, 'ALL')
if res != "OK":
logger.error("Unable to fetch mails")
return []
lst = []
for mail_id in data[0].split()[-nb:]:
res, flags_data = self.connection.fetch(mail_id, '(FLAGS)')
if res != "OK":
logger.error(
"Unable to fetch flags for mail {0}".format(mail_id))
continue
flags = imaplib.ParseFlags(flags_data[0])
mail = self.get_mail(mail_id)
if not mail:
continue
parser = Parser(mail)
parsed_orig_mail = MailParser(mail)
if isinstance(mail_id, bytes):
mail_id = mail_id.decode()
flags = [fl.decode() for fl in flags]
lst.append(u"{}: '{}', '{}', {}, ''{}".format(
mail_id, parsed_orig_mail.subject, parser.headers['Subject'],
parsed_orig_mail.headers['From'], flags))
logger.info(lst[-1])
return lst
def parse_mail(self):
"""
Retrieves a message's header from the IMAP server and parses its
headers in the email header data structure used by the user's filters.
This method is invoked by the parent class' constructor.
"""
# Mail has already been parsed in this case
if not (self._headers is None and self.message_flags is None):
return True
self._processor.log("")
self._processor.log("New mail detected with UID {0}:".format(self.uid))
try:
ret, data = self._processor.imap.uid('fetch', self.uid,
"(FLAGS BODY.PEEK[HEADER])")
except self._processor.imap.error as e:
# Anything imaplib raises an exception for is fatal here.
self._processor.fatal_error("Error retrieving message "
"with UID %s: %s" % (self.uid, e))
if ret != 'OK':
self._processor.log_error(
"Error: Could not retrieve message {0}: {1}".format(
self.uid, ret))
return False
flags = imaplib.ParseFlags(data[0][0])
if self.message_flags is None:
self.message_flags = []
for flag in flags:
self.message_flags.append(flag.decode('ascii'))
headers = email_parser.Parser().parsestr(data[0][1].decode('ascii'),
headersonly=True)
if self._headers is None:
self._headers = {}
for name in headers.keys():
value_parts = []
for header in headers.get_all(name, []):
try:
for (s, c) in email_header.decode_header(header):
# email.header.decode_header in Python 3.x may
# return either [(str, None)] or [(bytes,
# None), ..., (bytes, encoding)]. We must
# compensate for this.
if not isinstance(s, str):
s = s.decode(c if c else "ascii")
value_parts.append(s)
except (email_errors.HeaderParseError, LookupError,
ValueError):
self._processor.log_error(
"Error: Could not decode header {0} in message "
"UID {1}".format(ascii(header), self.uid))
value_parts.append(header)
self._headers[name.lower()] = " ".join(value_parts)
return True
def refresh_flags(self):
"""
Refreshes message flags for all messages in header cache.
"""
self.log_info("==> Updating message flags...")
for folder in self.header_cache:
try:
ret, data = self.get_flags(folder)
except self.imap.abort:
self.log_error("IMAP connection aborted, reconnecting.")
# Reconnect if the connection has timed out due to the header
# cache update taking too long (may happen on mailboxes with
# lots of messages).
self.reconnect()
ret, data = self.get_flags(folder)
for msg in data:
flags = []
flags_raw = imaplib.ParseFlags(msg)
for flag in flags_raw:
flags.append(flag.decode('ascii'))
uid = re.search('UID (\d+)',
msg.decode('ascii')).group().split(" ")[1]
self.log_debug("UID: %s" % uid)
self.log_debug(" server flags: %s" % flags)
self.log_debug(" cache flags: %s" %
self.header_cache[folder]['uids'][uid]['flags'])
self.header_cache[folder]['uids'][uid]['flags'] = flags
self.header_cache[folder]['uids'][uid]['obj'].message_flags = flags
self.log_info("==> Message flags updated.")
def refresh_flags(self):
"""
Refreshes message flags for all messages in header cache.
"""
self.log_info("==> Updating message flags...")
for folder in self.header_cache:
self.select(folder)
uids = ",".join(self.header_cache[folder]['uids'])
try:
ret, data = self.imap.uid('fetch', uids, "FLAGS")
except self.imap.error as e:
self.fatal_error(
"Error forwarding: Could not retrieve message flags for folder {0}: {1}"
"{1}".format(folder, e))
for msg in data:
flags = []
flags_raw = imaplib.ParseFlags(msg)
for flag in flags_raw:
flags.append(flag.decode('ascii'))
uid = m = re.search('UID (\d+)',
msg.decode('ascii')).group().split(" ")[1]
self.log_debug ("UID: %s" % uid)
self.log_debug(" server flags: %s" % flags)
self.log_debug(" cache flags: %s" %
self.header_cache[folder]['uids'][uid]['flags'])
self.header_cache[folder]['uids'][uid]['flags'] = flags
self.header_cache[folder]['uids'][uid]['obj'].message_flags = flags
self.log_info("==> Message flags updated.")
def parse_imap_flags(flagstr: List[bytes]) -> List[EmailFlag]:
parsed_flag_strings = [f.decode() for f in imaplib.ParseFlags(flagstr[0])]
flags = []
for flag in EmailFlag:
if MAP_FLAG_TO_IMAP[flag] in parsed_flag_strings:
flags.append(flag)
return flags
def parse_flags(headers):
"""Copied from https://github.com/girishramnani/gmail/blob/master/gmail/message.py"""
if len(headers) == 0:
return []
if sys.version_info[0] == 3:
headers = bytes(headers, "ascii")
return list(imaplib.ParseFlags(headers))
async def get_list(request, username, tag):
global clients
client, timestamp = clients.get(username, [None, None])
if client and time.time() - timestamp > 60 * 60: # over an hour
del clients[username]
client = None
if not client:
return json(None, status=401)
messages = []
if client.selected is not tag:
for item in client.connection.list()[1]:
real_tag = shlex.split(item.decode())[-1]
if real_tag.lower().count(tag):
break
client.connection.select(quote_folder_name(real_tag))
client.selected = tag
uids = client.connection.uid('search', None, '(ALL)')[1][0].split()
if not uids:
return json([])
response = client.connection.uid(
"fetch", b",".join(uids[-100:]),
"(BODY.PEEK[HEADER.FIELDS (FROM TO DATE SUBJECT)] FLAGS)")
for labels, message in response[1][::2]:
message = parser.parse_email(message)
message.uid = labels.split()[2]
message.flags = imaplib.ParseFlags(labels)
del message.raw_email
messages.append(message)
return json(reversed(messages))
def __iter__(self):
from kryptomime.mail import protect_mail
filter = "(OR UNSEEN FLAGGED)" if self.filter else 'ALL'
status, response = self.imap.uid('search', None, filter)
if not status == 'OK': return
email_ids = response[0].split()
for e_id in email_ids:
status, response = self.imap.uid('fetch', e_id, '(FLAGS)')
flags = ''
for flag in imaplib.ParseFlags(response[0]):
if 'Flagged' in flag: flags += 'F'
elif 'Seen' in flag: flags += 'S'
elif 'Deleted' in flag: flags += 'D'
if 'D' in flags: continue
# mark for processing
status, response = self.imap.uid('store', e_id, '+FLAGS',
r'(\Flagged)')
status, response = self.imap.uid('fetch', e_id, '(RFC822)')
raw = response[0][1]
yield protect_mail(raw), flags
if self.keep:
status, response = self.imap.uid('store', e_id, '-FLAGS',
r'(\Flagged)')
else:
status, response = self.imap.uid('store', e_id, '+FLAGS',
r'(\Deleted)')
async def add_flag(request, username, tag, uid, flag):
client, _ = clients.get(username)
flag = FLAGS[flag]
flags = client.connection.uid("store", uid, "+FLAGS", flag)[1][0]
if flags is None:
flags = ""
flags = imaplib.ParseFlags(flags)
return json(flags)
def get_flags_from_struct(data):
"""Get flags to copy old flags to new message."""
flags = imaplib.ParseFlags(data[1])
flags = b" ".join(flags) if flags != () else b""
flags = flags.decode("utf-8")
flags = flags.replace("\\Recent", "") # read-only attribute
return flags.strip()
def _clean_message_data(fetch_data):
"""
:param fetch_data: Message object model
:returns [message_data: bytes, uid_data: bytes, flag_data: list]
*Elements may contain byte strings in any order, like: b'4517 (FLAGS (\\Recent NonJunk))'
"""
message_data = b''
uid_data = b''
flag_data = []
for fetch_item in fetch_data:
# flags
if type(fetch_item) is bytes and imaplib.ParseFlags(fetch_item):
flag_data.extend(imaplib.ParseFlags(fetch_item))
# data, uid
if type(fetch_item) is tuple:
uid_data = fetch_item[0]
message_data = fetch_item[1]
return message_data, uid_data, flag_data
def flags(self) -> [str]:
"""
Message flags
*This attribute will not be changed after actions: flag, seen
"""
result = []
for raw_flag_item in self._raw_flag_data:
result.extend(imaplib.ParseFlags(raw_flag_item))
return [i.decode().strip().replace('\\', '').upper() for i in result]
def should_handle(self, mail_id):
res, flags_data = self.connection.fetch(mail_id, '(FLAGS)')
if res != "OK":
logger.error("Unable to fetch flags for mail {0}".format(mail_id))
return False
flags = imaplib.ParseFlags(flags_data[0])
if b"imported" in flags or b"error" in flags or b"unsupported" in flags:
return False
return True
def flags(self) -> Tuple[str, ...]:
"""
Message flags
*This attribute will not be changed after "flag" actions
"""
result = []
for raw_flag_item in self._raw_flag_data:
result.extend(imaplib.ParseFlags(raw_flag_item))
return tuple(i.decode().strip() for i in result) # noqa
def __init__(self, email, uid, folders, msg, flags=None):
self.email = email
self.uid = uid
self.folders = folders
self.msg = email_lib.message_from_bytes(msg[1])
try:
self.flags = imaplib.ParseFlags(flags)
except:
self.flags = None
def flag_from_string(self, data_set):
result = []
for raw_flag_item in data_set:
if isinstance(raw_flag_item, string_types):
raw_flag_item = raw_flag_item.encode("utf-8")
result.extend(imaplib.ParseFlags(raw_flag_item))
flags = tuple(item.decode().strip().replace('\\', '').upper()
for item in result)
self.flags = flags
return flags
def getMsgHdr(self, num):
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
num = str(num)
typ, data = self.mb.fetch(num, '(FLAGS RFC822.HEADER)')
hdr = data[0][1]
imh = email.parser.Parser(IMAPMsgHdr).parsestr(hdr, True)
imh.msgid = num
imh.flags = imaplib.ParseFlags(data[0][0])
imh.server = self
return imh
def fetch_header(self, mid):
"ヘッダ情報を取得する"
if not self.imap: raise IOError(self.not_connected)
parts = '(UID FLAGS INTERNALDATE RFC822.HEADER)'
rc, res = self.imap.fetch(mid, parts)
if not rc == 'OK':
raise IOError('unkown fetch response(%s)' % res)
msgs = []
rlen = len(res)
#print mid, 'rlen:', rlen
ct = 0
ect = 0
mid = 0
while ct + 2 <= rlen:
try:
#print res[ct:ct+2]
((res1, rawhead), ep) = res[ct:ct + 2]
emsg = EMessage()
mid = res1.split()[0]
emsg.id = int(mid)
emsg.received = imaplib.Internaldate2tuple(res1)
flags = imaplib.ParseFlags(res1)
emsg.flags = flag_text(flags)
mr = UID_PATTERN.search(res1)
if not mr: raise IOError('no uid response(%s)' % res1)
emsg.uid = int(mr.group(1))
mr = SIZE_PATTERN.search(res1)
emsg.size = int(mr.group(1)) if mr else None
#print res1, ep
#print 'uid', uid, 'received', received
emsg.parse(rawhead, headersonly=True)
msgs.append(emsg)
except Exception as e:
ect += 1
if log:
if ect == 1:
log.exception('parse: %s: %s', mid, e)
else:
log.warn('parse: %s: %s', mid, e)
else:
log.debug('WARN: parse: %s: %s', mid, e)
finally:
ct += 2
return msgs
def createMail(imap, mailbox, new_message, flags):
now = time.time()
if not new_message.has_key("date"):
new_message["date"] = imaplib.Time2Internaldate(now)
response = imap.append(mailbox, imaplib.ParseFlags(flags), now, new_message.as_string(True))
if (response[0] != "OK"):
return -1
else:
uid = response[1][0].split("]")[0].split(" ")
return int(uid[len(uid)-1])
def _get_message_data_parts(fetch_data) -> (bytes, bytes, [bytes]):
raw_message_data = b''
raw_uid_data = b''
raw_flag_data = []
for fetch_item in fetch_data:
# flags
if type(fetch_item) is bytes and imaplib.ParseFlags(fetch_item):
raw_flag_data.append(fetch_item)
# data, uid
if type(fetch_item) is tuple:
raw_uid_data = fetch_item[0]
raw_message_data = fetch_item[1]
return raw_message_data, raw_uid_data, raw_flag_data
def __flag(self, msgset, cmd, flags, with_silent=True):
ft = type(flags)
if ft == list or ft == tuple or ft == set:
flags = ' '.join(flags)
rc, res = self.imap.store(msgset, cmd, flags)
if not rc == 'OK' or with_silent: return None
rc = []
for rt in res:
mid = int(rt[0:rt.find(' ')])
flags = imaplib.ParseFlags(rt)
rc.append((mid, flags))
def get_imapflags(self, uid):
""" Return a list of imap flags for the message with UID
Raise exception if there if there is no message with that UID.
"""
try:
(code, data) = self._server.uid('fetch', uid, '(FLAGS)')
flagresult = data[0]
if code != 'OK':
raise ImapNotOkError("%s in get_imapflags(%s)" % (code, uid))
return list(imaplib.ParseFlags(flagresult))
except (TypeError, ValueError):
raise ValueError("Unexpected results while fetching flags " \
+ "from server for message %s; response was (%s, %s)" \
% (uid, code, data))
def get_email_seen_status(self, uid, flag_string):
""" parse the email FLAGS response """
if not flag_string:
return None
flags = []
for flag in imaplib.ParseFlags(flag_string) or []:
pattern = re.compile("\w+")
match = re.search(pattern, frappe.as_unicode(flag))
flags.append(match.group(0))
if "Seen" in flags:
self.seen_status.update({uid: "SEEN"})
else:
self.seen_status.update({uid: "UNSEEN"})
def _get_message_data_parts(fetch_data) -> (bytes, bytes, [bytes]):
"""
:param fetch_data: Message object model
:returns (raw_message_data: bytes, raw_uid_data: bytes, raw_flag_data: [bytes])
*Elements may contain byte strings in any order, like: b'4517 (FLAGS (\\Recent NonJunk))'
"""
raw_message_data = b''
raw_uid_data = b''
raw_flag_data = []
for fetch_item in fetch_data:
# flags
if type(fetch_item) is bytes and imaplib.ParseFlags(fetch_item):
raw_flag_data.append(fetch_item)
# data, uid
if type(fetch_item) is tuple:
raw_uid_data = fetch_item[0]
raw_message_data = fetch_item[1]
return raw_message_data, raw_uid_data, raw_flag_data
def retrieve_messages(self,
message_ids: t.List[int],
folder: t.Optional[str] = None,
headers_only: bool = False) -> t.List[Message]:
"""For each message ID request message flags and contents and parse it to Message."""
requested_parts = ['FLAGS']
# The BODY.PEEK[] is a functional equivalent of obsolete RFC822.PEEK,
# see https://www.ietf.org/rfc/rfc2062 for details.
requested_parts.append(
'BODY.PEEK[HEADER]' if headers_only else 'BODY.PEEK[]')
# flags_data = self.retrieve_messages_parts(message_ids, ['FLAGS'], folder)
# print(flags_data[0])
# all_data = self.retrieve_messages_parts(message_ids, ['FLAGS', 'BODY.PEEK[]'], folder)
# print(all_data[0])
messages_data = self.retrieve_messages_parts(message_ids,
requested_parts, folder)
messages = []
for message_id, (metadata, message) in zip(message_ids, messages_data):
email_message = email.message_from_bytes(message)
if headers_only:
email_message.defects = [
defect for defect in email_message.defects
if not isinstance(defect, HEADER_ONLY_IGNORED_DEFECTS)
]
if email_message.defects:
_LOG.error('%s: message #%i in "%s" has defects: %s', self,
message_id, self._folder, email_message.defects)
message = Message(email_message, self, self._folder, message_id)
flags = imaplib.ParseFlags(metadata)
for raw_flag in flags:
flag = raw_flag.decode()
if flag.startswith('\\'):
flag = flag[1:]
else:
_LOG.warning('atypical flag "%s" detected in "%s"', flag,
flags)
message.flags.add(flag)
messages.append(message)
return messages
def Mail(session, mailbox, message_uid, expunge=False):
# select mailbox
ok, response = session.select(mailbox)
if ok != "OK":
raise Exception(response)
# fetch message and split response
ok, response = session.uid('fetch', str(message_uid), "(FLAGS INTERNALDATE RFC822)")
if ok != "OK":
raise Exception(response)
metadata = response[0][0]
rfcbody = response[0][1]
# parse flags, date and message body
flags = " ".join([f.decode() for f in imaplib.ParseFlags(metadata)])
date = imaplib.Internaldate2tuple(metadata)
mail = email.message_from_bytes(rfcbody)
# debug
with color(COLOR.BLUE):
print("Message UID:", mailbox, f"({message_uid})")
print("From :", mail["From"])
print("Date :", mail["Date"])
print("Subject :", mail["Subject"])
# yield mail for editing
state = {"mail": mail, "dirty": False}
yield state
# if message was edited, append to mailbox and
# set deletion flag on old message
if state["dirty"] == True:
with color(COLOR.RED):
print("Message modified. Reuploading ..")
session.append(mailbox, flags, date, mail.as_bytes())
session.uid('store', message_uid, "+FLAGS", "\\DELETED")
# optionally directly expunge
if expunge:
session.expunge()
def stats(self):
"""List all flags"""
res, data = self.connection.search(None, 'ALL')
if res != "OK":
logger.error("Unable to fetch mails")
return []
lst = []
stats = {'tot': 0, 'flags': {}}
for mail_id in data[0].split():
stats['tot'] += 1
res, flags_data = self.connection.fetch(mail_id, '(FLAGS)')
if res != "OK":
logger.error(
"Unable to fetch flags for mail {0}".format(mail_id))
continue
for flag in imaplib.ParseFlags(flags_data[0]):
flag = flag.decode()
if flag not in stats['flags']:
stats['flags'][flag] = 0
stats['flags'][flag] += 1
return stats
