def test_Internaldate2tuple(self):
t0 = calendar.timegm((2000, 1, 1, 0, 0, 0, -1, -1, -1))
tt = imaplib.Internaldate2tuple(
b'25 (INTERNALDATE "01-Jan-2000 00:00:00 +0000")')
self.assertEqual(time.mktime(tt), t0)
tt = imaplib.Internaldate2tuple(
b'25 (INTERNALDATE "01-Jan-2000 11:30:00 +1130")')
self.assertEqual(time.mktime(tt), t0)
tt = imaplib.Internaldate2tuple(
b'25 (INTERNALDATE "31-Dec-1999 12:30:00 -1130")')
self.assertEqual(time.mktime(tt), t0)
def fetch_message(self, uid):
with self._imap as c:
# Select the IMAP mailbox.
code, count = c.select("[Gmail]/All Mail", readonly=True)
if code != u"OK":
raise IMAPSyncError(
u"Couldn't SELECT 'All Mail' ({1})".format(code))
code, data = c.uid(
u"fetch", unicode(uid), u"(BODY.PEEK[] "
u"X-GM-MSGID X-GM-THRID X-GM-LABELS "
u"FLAGS INTERNALDATE)")
if code != "OK":
raise IMAPSyncError(u"Couldn't FETCH body.")
doc = dict(
zip([u"msgid", u"thrid", u"labels", u"flags", u"uid"],
self._do_header_parse(data[0][0])))
for k, val in self.parse_header.findall(data[0][0]):
doc[k.strip().lower()] = val.strip()
doc["time"] = time.mktime(imaplib.Internaldate2tuple(data[0][0]))
doc["message"] = email.message_from_string(data[0][1])
for k in ["from", "to", "subject"]:
doc[k] = ",".join(doc["message"].get_all(k))
return doc
def getDatetime(s):
'Returns a datetime object for an INTERNALDATE response.'
timetuple = imaplib.Internaldate2tuple(s)
if timetuple is not None:
return datetime.fromtimestamp(time.mktime(timetuple))
def PopulateField(self, name, value):
if name == "UID":
self.__uid = value
elif name == "RFC822.SIZE":
self.size = int(value)
elif name == "FLAGS":
self.__flags = value
elif name == "INTERNALDATE":
self.__date_string = value
self.__hasDate = True
self.__date_tuple = imaplib.Internaldate2tuple(
'INTERNALDATE "%s"' % value)
self.__date_sec = time.mktime(self.__date_tuple)
if self.__date_sec > MessageInfo.__newestMessageSec:
MessageInfo.__newestMessageSec = self.__date_sec
if self.__date_sec < MessageInfo.__oldestMessageSec:
MessageInfo.__oldestMessageSec = self.__date_sec
elif name == "RFC822.HEADER":
try:
unicode(value, errors='strict')
except:
value = unicode(value, errors='ignore')
self.headers = email.message_from_string(value)
else:
raise AssertionError("unknown field: %s" % name)
def export_and_remove(mid, exp, flag):
parts = '(UID INTERNALDATE RFC822.HEADER BODY.PEEK[TEXT])'
rc, res = self.imap.fetch(mid, parts)
if not rc == 'OK': raise IOError('unkown fetch response(%s)' % res)
((res1, rawhead), (res2, rawtext), ep) = res
received = imaplib.Internaldate2tuple(res1)
mu = UID_PATTERN.search(res1)
if not mu: raise IOError('no uid response(%s)' % res1)
uid = int(mu.group(1))
#print rc, res1, res2, ep
#print 'uid', uid, 'received', received
exp.write_message('%s%s' % (rawhead, rawtext), received, uid)
if rc != 'OK': return False
self.lastUID = uid
if flag:
rc, rec = self.imap.store(mid, '+FLAGS.SILENT', DELETED)
if cli.verbose: log.debug('store-delete: %s: %s', rc, res[0])
if rc != 'OK': return False
return True
def moveMsg(self,boxname):
# Move messages from defined boxname
selectedbox = self.From.select(boxname)
print("Fetching emails from %s"%boxname)
# Clear up the flag emails (incase it has stalled and you've started again)
self.expunge()
typ, data = self.From.search(None, 'ALL')
print(">>>>" + typ)
print("Found %d Messages"%len(data[0].split()))
regex = r"Message-ID:[\s|\S]?<(.*?)>\s"
for muid in data[0].split():
# start messageUID process
status, msg = self.From.fetch(muid,"(FLAGS INTERNALDATE BODY.PEEK[])") #Get email
Email = msg[0][1]
flags = imaplib.ParseFlags(msg[0][0]) # get Flags
if "\\Recent" in flags:
flag_str = "\\Seen"
else:
flag_str = " ".join(flags)
print(">>>>>" + flag_str)
date = imaplib.Time2Internaldate(imaplib.Internaldate2tuple(msg[0][0])) #get date
copy_result = self.To.append(boxname, flag_str, date, Email) # Copy to Archive
if copy_result[0] == 'OK':
del_msg = self.From.store(muid, '+FLAGS', '\\Deleted') # Mark for deletion (this is deleted via expunge)
self.expunge()
def compute_email_days(login: str, password: str) -> float:
with imaplib.IMAP4_SSL(IMAP_SERVER, IMAP_PORT) as M:
M.login(login, password)
M.select('INBOX', readonly=True)
typ, msgnums = M.search(None, 'UNSEEN')
if typ != 'OK':
print(f'Something went wrong: received typ={typ}')
return
inbox_len = len(msgnums[0].split())
print(f'Unread messages: {inbox_len}')
current_time = time.time()
DAY = 60 * 60 * 24
email_days = 0
# M.fetch takes limited number of messages, so email number list is split in chunks
pack_start = 0
PACK_SIZE = 1000
while pack_start < inbox_len:
typ, data = M.fetch(
b','.join(msgnums[0].split()[pack_start:pack_start +
PACK_SIZE]),
'INTERNALDATE',
)
for i, datestring in enumerate(data):
email_days += (current_time - time.mktime(
imaplib.Internaldate2tuple(datestring))) / DAY
pack_start += PACK_SIZE
return email_days
def process_email(id, conn_server, folder, uidc, key):
""" Sanitize, if necessary, an email.
id - String containing the id of the email to fetch;
conn_server - imaplib connection to the server;
folder - Current folder of the client;
uidc - True if command contains UID flag
key - key used to verify integrity of email
If the email is not sanitized yet, make a sanitized copy in the same folder
and an unsanitized copy if the Quarantine folder. The original email is deleted.
"""
conn_server.select(folder)
if has_CIRCL_signature(id, conn_server, uidc):
return
print('Email not sanitized')
# -- No CIRCL signature or incorrect value --
bmail = fetch_entire_email(id, conn_server, uidc)
if not bmail:
return
mail = email.message_from_bytes(bmail)
# Get the DATE of the email
date_str = mail.get('Date')
date = imaplib.Internaldate2tuple(
date_str.encode()) if date_str else imaplib.Time2Internaldate(
time.time())
# Get the payload and hash
digest_original = hash_payload(get_payload(mail), key)
# Sanitize the email
content = sanitize_email(bmail)
if not content:
return
# Copy of the sanitized email
smail = email.message_from_bytes(content.getvalue())
digest_sanitized = hash_payload(get_payload(smail), key)
append_email(conn_server, smail, digest_sanitized, VALUE_SANITIZED, date,
folder)
# Copy of the original email in the Quarantine folder
append_email(conn_server, mail, digest_original, VALUE_ORIGINAL, date,
QUARANTINE_FOLDER)
# Delete original
conn_server.uid('STORE', id, '+FLAGS',
'(\Deleted)') if uidc else conn_server.store(
id, '+FLAGS', '(\Deleted)')
conn_server.expunge()
def fetch_header(self, mid):
"ヘッダ情報を取得する"
if not self.imap: raise IOError(self.not_connected)
parts = '(UID FLAGS INTERNALDATE RFC822.HEADER)'
rc, res = self.imap.fetch(mid, parts)
if not rc == 'OK':
raise IOError('unkown fetch response(%s)' % res)
msgs = []
rlen = len(res)
#print mid, 'rlen:', rlen
ct = 0
ect = 0
mid = 0
while ct + 2 <= rlen:
try:
#print res[ct:ct+2]
((res1, rawhead), ep) = res[ct:ct + 2]
emsg = EMessage()
mid = res1.split()[0]
emsg.id = int(mid)
emsg.received = imaplib.Internaldate2tuple(res1)
flags = imaplib.ParseFlags(res1)
emsg.flags = flag_text(flags)
mr = UID_PATTERN.search(res1)
if not mr: raise IOError('no uid response(%s)' % res1)
emsg.uid = int(mr.group(1))
mr = SIZE_PATTERN.search(res1)
emsg.size = int(mr.group(1)) if mr else None
#print res1, ep
#print 'uid', uid, 'received', received
emsg.parse(rawhead, headersonly=True)
msgs.append(emsg)
except Exception as e:
ect += 1
if log:
if ect == 1:
log.exception('parse: %s: %s', mid, e)
else:
log.warn('parse: %s: %s', mid, e)
else:
log.debug('WARN: parse: %s: %s', mid, e)
finally:
ct += 2
return msgs
def fetch_timestamp(mail_id: str) -> int:
# Alternative, more flexible but slower implementation using <DATE> rather than
# <INTERNALDATE> - maybe we should make this selectable
# msg = self._mailbox.fetch(mail_id, "(RFC822)")[1]
# mail = email.message_from_string(msg[0][1].decode())
# parsed = email.utils.parsedate_tz(mail["DATE"])
# return int(time.time()) if parsed is None else email.utils.mktime_tz(parsed)
return int(
time.mktime(
imaplib.Internaldate2tuple(
verified_result(self._connection.fetch(mail_id, "INTERNALDATE"))[0])))
def do_INTERNALDATE(self, arg):
'''Process an INTERNALDATE response
@param arg: A quoted IMAP INTERNALDATE string
(eg. " 9-Feb-2007 17:08:08 +0000")
@return: datetime.datetime instance for the given time (in UTC)
'''
t = imaplib.Internaldate2tuple('INTERNALDATE "%s"' % arg)
if t is None:
return None
else:
return datetime.datetime(*t[:6])
def fetch_timestamp(mail_id: str) -> int:
# Alternative, more flexible but slower implementation using <DATE> rather than
# <INTERNALDATE> - maybe we should make this selectable
# msg = self._mailbox.fetch(mail_id, "(RFC822)")[1]
# mail = email.message_from_string(msg[0][1].decode())
# parsed = email.utils.parsedate_tz(mail["DATE"])
# return int(time.time()) if parsed is None else email.utils.mktime_tz(parsed)
raw_number = verified_result(
self._connection.fetch(mail_id, "INTERNALDATE"))[0]
assert isinstance(raw_number, bytes)
# typeshed bug: https://github.com/python/typeshed/issues/7781
return int(time.mktime(imaplib.Internaldate2tuple(
raw_number))) # type: ignore[arg-type]
def get_internaldate(self, uid):
""" Return a time tuple representing the internal date for the
message with UID
Raise exception if there if there is no message with that UID.
"""
try:
(code, data) = self._server.uid('fetch', uid, '(INTERNALDATE)')
dateresult = data[0]
if code != 'OK':
raise ImapNotOkError("%s in get_internaldate(%s)" % (code, uid))
if dateresult is None:
raise NoSuchUIDError("No message %s in get_internaldate" % uid)
return imaplib.Internaldate2tuple(dateresult)
except (TypeError, ValueError):
raise ValueError("Unexpected results while fetching flags " \
+ "from server for message %s" % uid)
def _fetch_from_date_imap(self, imap_server, count, failed):
MailThread = self.env['mail.thread']
messages = []
date_uids = {}
last_date = False
# last_internal_date = datetime.strptime(self.last_internal_date,"%Y-%m-%d %H:%M:%S")
last_internal_date = self.last_internal_date
search_status, uids = imap_server.search(
None, 'SINCE', '%s' % last_internal_date.strftime('%d-%b-%Y'))
new_uids = uids[0].split()
for new_uid in new_uids:
fetch_status, date = imap_server.fetch(new_uid, 'INTERNALDATE')
internaldate = imaplib.Internaldate2tuple(date[0])
internaldate_msg = datetime.fromtimestamp(
time.mktime(internaldate))
if internaldate_msg > last_internal_date:
messages.append(new_uid)
date_uids[new_uid] = internaldate_msg
result_unseen, data_unseen = imap_server.search(None, '(UNSEEN)')
for num in messages:
# SEARCH command *always* returns at least the most
# recent message, even if it has already been synced
res_id = None
result, data = imap_server.fetch(num, '(RFC822)')
if data and data[0]:
try:
res_id = MailThread.message_process(
self.object_id.model,
data[0][1],
save_original=self.original,
strip_attachments=(not self.attach))
except Exception:
_logger.exception(
'Failed to process mail \
from %s server %s.', self.type, self.name)
failed += 1
if num.decode('utf-8') in data_unseen[0].decode('utf-8'):
imap_server.store(num, '-FLAGS', '\\Seen')
self._cr.commit()
count += 1
last_date = not failed and date_uids[num] or False
return count, failed, last_date
def Mail(session, mailbox, message_uid, expunge=False):
# select mailbox
ok, response = session.select(mailbox)
if ok != "OK":
raise Exception(response)
# fetch message and split response
ok, response = session.uid('fetch', str(message_uid), "(FLAGS INTERNALDATE RFC822)")
if ok != "OK":
raise Exception(response)
metadata = response[0][0]
rfcbody = response[0][1]
# parse flags, date and message body
flags = " ".join([f.decode() for f in imaplib.ParseFlags(metadata)])
date = imaplib.Internaldate2tuple(metadata)
mail = email.message_from_bytes(rfcbody)
# debug
with color(COLOR.BLUE):
print("Message UID:", mailbox, f"({message_uid})")
print("From :", mail["From"])
print("Date :", mail["Date"])
print("Subject :", mail["Subject"])
# yield mail for editing
state = {"mail": mail, "dirty": False}
yield state
# if message was edited, append to mailbox and
# set deletion flag on old message
if state["dirty"] == True:
with color(COLOR.RED):
print("Message modified. Reuploading ..")
session.append(mailbox, flags, date, mail.as_bytes())
session.uid('store', message_uid, "+FLAGS", "\\DELETED")
# optionally directly expunge
if expunge:
session.expunge()
def simple_list(self, q=None):
with self._imap as c:
if q is None:
q = u"in:inbox"
code, count = c.select("[Gmail]/All Mail", readonly=True)
if code != u"OK":
raise IMAPSyncError(
u"Couldn't SELECT 'All Mail' ({1})".format(code))
code, uids = c.uid(u"search", None, u"X-GM-RAW",
u"\"{0}\"".format(q))
if code != u"OK":
raise IMAPSyncError(u"Couldn't run query '{0}' ({1})".format(
q, code))
uids = u",".join(uids[0].split())
code, data = c.uid(
u"fetch", uids, u"(BODY.PEEK[HEADER.FIELDS "
u"(Subject From To Date)] "
u"X-GM-MSGID X-GM-THRID X-GM-LABELS "
u"FLAGS INTERNALDATE)")
if code != "OK":
raise IMAPSyncError(u"Couldn't FETCH flags.")
results = []
for d in data:
if d != u")":
doc = dict(
zip([u"msgid", u"thrid", u"labels", u"flags", u"uid"],
self._do_header_parse(d[0])))
for k, val in self.parse_header.findall(d[1]):
doc[k.strip().lower()] = val.strip()
doc["time"] = time.mktime(imaplib.Internaldate2tuple(d[0]))
results.append(doc)
return sorted(results, reverse=True, key=lambda d: d["time"])
def PopulateField(self, name, value):
if name == "UID": self.__uid = value
elif name == "RFC822.SIZE": self.size = int(value)
elif name == "FLAGS": self.__flags = value
elif name == "INTERNALDATE":
self.__date_string = value
if MessageInfo.__parseDates:
self.__date_tuple = \
imaplib.Internaldate2tuple('INTERNALDATE "%s"' % value)
self.__date_sec = time.mktime(self.__date_tuple)
if self.__date_sec > MessageInfo.__newestMessageSec:
MessageInfo.__newestMessageSec = self.__date_sec
if self.__date_sec < MessageInfo.__oldestMessageSec:
MessageInfo.__oldestMessageSec = self.__date_sec
elif name == "RFC822.HEADER":
self.headers = email.message_from_string(value)
else: raise AssertionError("unknown field: %s" % name)
def fetch_source(self, mid):
if not self.imap: raise IOError(self.not_connected)
parts = '(UID INTERNALDATE RFC822.HEADER BODY.PEEK[TEXT])'
rc, res = self.imap.fetch(mid, parts)
if not rc == 'OK':
raise IOError('unkown fetch response(%s)' % res)
#print rc, res
if False:
xres = list(res)
for xr in res:
t = type(xr)
if t == tuple:
trace(t, '(', xr[0], '.. %d' % len(xr), ')')
else:
trace(t, xr)
rlen = len(res)
msgs = []
ct = 0
mid = 0
while ct + 3 <= rlen:
((res1, rawhead), (res2, rawtext), ep) = res[ct:ct + 3]
mid = int(res1.split()[0])
received = imaplib.Internaldate2tuple(res1)
mr = UID_PATTERN.search(res1)
if not mr: raise IOError('no uid response(%s)' % res1)
uid = int(mr.group(1))
#print rc, res1, res2, ep
#print 'uid', uid, 'received', received
msgs.append((rawhead, rawtext, received, uid, mid))
ct += 3
return msgs
def fetch_message(self, mid):
"""メッセージを取得する"""
if not self.imap: raise IOError(self.not_connected)
parts = '(UID INTERNALDATE RFC822.HEADER BODY.PEEK[TEXT])'
rc, res = self.imap.fetch(mid, parts)
if not rc == 'OK':
raise IOError('unkown fetch response(%s)' % res)
msgs = []
rlen = len(res)
#print mid, 'rlen:', rlen
ct = 0
mid = 0
while ct + 3 <= rlen:
((res1, rawhead), (res2, rawtext), ep) = res[ct:ct + 3]
emsg = EMessage()
mid = res1[0:res1.find(' ')]
emsg.id = int(mid)
emsg.received = imaplib.Internaldate2tuple(res1)
mr = UID_PATTERN.search(res1)
if not mr:
raise IOError('no uid response(%s)' % res1)
emsg.uid = int(mr.group(1))
mr = SIZE_PATTERN.search(res1)
emsg.size = int(mr.group(1)) if mr else None
#print res1, ep
#print 'uid', uid, 'received', received
m = emsg.parse('%s%s' % (rawhead, rawtext))
msgs.append(emsg)
ct += 3
return msgs
def add_message(self,
message_parts: t.Tuple[bytes, bytes],
folder: t.Optional[str] = None) -> None:
"""Add a message to a folder using APPEND command.
:param message_parts: tuple (envelope: bytes, body: bytes), with both elements properly set,
which is exactly the same type as received via:
parts = retrieve_message_parts(uid, parts=['FLAGS', 'INTERNALDATE', 'BODY.PEEK[]'])
"""
assert isinstance(message_parts, tuple), type(message_parts)
assert len(message_parts) == 2, len(message_parts)
envelope, body = message_parts
assert isinstance(envelope, bytes), type(envelope)
assert isinstance(body, bytes), type(body)
if folder is None:
folder = self._folder
self.open_folder(folder)
flags = f'({" ".join(_.decode() for _ in imaplib.ParseFlags(envelope))})'
assert isinstance(flags, str), flags
date = imaplib.Time2Internaldate(imaplib.Internaldate2tuple(envelope))
assert date is not None
status = None
try:
status, response = self._link.append(folder, flags, date, body)
_LOG.info(
'%s%s%s: append("%s", %s, "%s", ... (%i bytes)) status: %s, response: %s',
colorama.Style.DIM, self, colorama.Style.RESET_ALL, folder,
flags, date, len(body), status, [r for r in response])
except imaplib.IMAP4.error as err:
_LOG.exception('%s: append("%s", %s, "%s", ... (%i bytes)) failed',
self, folder, flags, date, len(body))
raise RuntimeError('add_message() failed') from err
if status != 'OK':
raise RuntimeError('add_message() failed')
def process_email(id, conn_server, conn_client, folder, uidc, key):
""" Decrypt email and verify sign, if possible.
id - String containing the id of the email to fetch;
conn_server - imaplib connection to the server;
folder - Current folder of the client;
uidc - True if command contains UID flag
key - key used to verify integrity of email
"""
conn_server.select(folder)
# -- No signature or incorrect value --
bmail, decrypt_value, verify_sign_value = fetch_entire_email(
id, conn_server, uidc, key)
#print(bmail)
if not bmail:
print('return ++++++++++++++++ 3 ')
return
mail = email.message_from_string(bmail)
# Get the DATE of the email
date_str = mail.get('Date')
date = imaplib.Internaldate2tuple(
date_str.encode()) if date_str else imaplib.Time2Internaldate(
time.time())
# Append decrytped message
if decrypt_value == 'Decryption successful':
mail = append_email(conn_server, mail, decrypt_value,
verify_sign_value, date, folder, conn_client)
#conn_server.uid('STORE', id, '+FLAGS', '(\Deleted)') if uidc else conn_server.store(id, '+FLAGS', '(\Deleted)')
return mail
else:
return
def get_date(id):
import datetime
_, response = imap_server.fetch(id, '(UID INTERNALDATE)')
return imaplib.Internaldate2tuple(response[0])
def test_Internaldate2tuple_issue10941(self):
self.assertNotEqual(
imaplib.Internaldate2tuple(
b'25 (INTERNALDATE "02-Apr-2000 02:30:00 +0000")'),
imaplib.Internaldate2tuple(
b'25 (INTERNALDATE "02-Apr-2000 03:30:00 +0000")'))
def internaldate_from_string(self, internaldatestring):
""" Set the internaldate from a string as it is returned
by self.internaldatestring()
"""
self.internaldate = imaplib.Internaldate2tuple(internaldatestring)
def process_email(username, id, conn_server, folder, uidc,
logger): # todo decode!!!
print('-' * 40)
print(folder)
conn_server.select(folder)
bmail = fetch_entire_email(id, conn_server, uidc)
if has_SCM_decrypt(id, conn_server, uidc):
return False
if not bmail:
return False
mail = email.message_from_bytes(bmail)
stringm = str(mail)
headers = stringm.split('\n\n')[0]
body = ''.join(stringm.split('\n\n')[1].split('\n.')[0].split('\r\n'))
body = body.replace('\n', '')
body = body.replace('\n', '')
data = mail.get_payload()
userFrom = ''.join(re.findall(r'(?<=<).*(?=>)', mail['From']))
uid = mail['UID']
if not uid:
return False
response = requests.post(os.getenv('URL_TICKETER') + "/getKey",
data={
'userTo': username,
'userFrom': userFrom,
'uid': uid
})
key = response.text
if key.find('HTML') != -1:
return False
fernet = Fernet(key)
try:
if isinstance(data, list):
for part in data:
payload = part.get_payload()
decryptt = str(fernet.decrypt(payload.encode()), 'utf-8')
subject = re.findall(r'(?<=Subject: ).*(?=\n)', decryptt)
if subject:
headers = re.sub(r'(?<=Subject: ).*(?=\n)', subject,
headers)
decryptt = re.sub(r'Subject: .*\n', "", decryptt)
part.set_payload(decryptt)
mail.set_payload(data)
else:
decryptt = str(fernet.decrypt(body.encode()), 'utf-8')
#subject = re.findall(r'(?<=Subject: ).*(?=\n)', headers)
#body = re.sub(r'Subject: .*', "", body)
subject = re.findall(r'(?<=Subject: ).*(?=\n)', decryptt)
if subject:
headers = re.sub(r'(?<=Subject: ).*(?=\n)', subject[0],
headers)
decryptt = re.sub(r'Subject: .*\n', "", decryptt)
mail = email.message_from_string(headers + '\r\n\r\n' + decryptt)
except:
logger.error("Error of decrypting, uid - " + uid)
print("ERROR")
return False
if mail['SCM']:
mail.replace_header('SCM', 'decrypt')
else:
return False
# Get the DATE of the email
date_str = mail.get('Date')
date = imaplib.Internaldate2tuple(
date_str.encode()) if date_str else imaplib.Time2Internaldate(
time.time())
content = mail
if not content:
return False
conn_server.append(folder, '', date, str(mail).encode())
# Delete original
conn_server.uid('STORE', id, '+FLAGS',
'(\Deleted)') if uidc else conn_server.store(
id, '+FLAGS', '(\Deleted)')
conn_server.expunge()
return True
def process_emails(imap, folder):
"""Process all emails from Netto in a specific folder
Parameters
----------
imap : imaplib connection
Connection to your IMAP server. From the `connect_to_imap` function.
folder : str
Folder where you store your Netto emails (e.g. "INBOX")
Returns
-------
out : Pandas Dataframe with all your purchases. The dataframe contains the following
columns,
- time: The time of purchase
- product: Name of the product
- amount: Number of product purchases
- price: The price paid
"""
# Get emails
status, messages = imap.select(folder, readonly=True)
status, messages = imap.search("utf-8", 'FROM', '"*****@*****.**"')
dfs = []
for msg in messages[0].split():
# Get the email body (i.e. content) and when it was sent
status, response = imap.fetch(msg, '(BODY[TEXT] INTERNALDATE)')
msg_info, msg_body = response[0]
# Get message datetime - E.g. 23-Jul-2020 12:02:04 -0400 -> local time
msg_time = imaplib.Internaldate2tuple(msg_info)
msg_time = datetime(msg_time.tm_year, msg_time.tm_mon,
msg_time.tm_mday, msg_time.tm_hour,
msg_time.tm_min, msg_time.tm_sec)
# Convert the email body encoded in quoted-printable to text
utf8_msg = quopri.decodestring(msg_body)
soup = BeautifulSoup(utf8_msg, features="lxml")
# Find the first HTML table in the message body
table = soup.find("table")
try:
rows = list()
# Iterate all HTML rows in the table and process every row of class `items`
for row in table.findAll("tr"):
if "items" in row.attrs["class"]:
# The row (tr) has three data elemets (td)
# Produce name - amount - price
product, amount, price = row.findAll("td")
clean_price = price.text.strip()
# Handle bottle pant
if "pant" in clean_price:
# A product with pant is listed as: 4.50\n+ pant 3.00
clean_price, pant_item = clean_price.split('\n+')
pant_price = convert_to_us_decimal(
pant_item.split('pant')[1].strip())
# Add pant as seperate item
rows.append({
"time": msg_time,
"product": "Pant",
"amount": 1,
"price": float(pant_price)
})
# Remove newline charaters and make amount a number
rows.append({
"time":
msg_time,
"product":
product.text.strip(),
"amount":
int(amount.text.replace("stk", "").strip()),
"price":
float(convert_to_us_decimal(clean_price))
})
except Exception as e:
raise RuntimeError(
f"Error: unable to process item {row} from {msg_time}") from e
df = pd.DataFrame(rows)
dfs.append(df)
# Logout of email
imap.logout()
if len(dfs) == 0:
raise RuntimeError(
f"Error: could not find any emails from Netto in {EMAIL_FOLDER}")
# Combine into a single dataframe
ddf = pd.concat(dfs)
return ddf
for box in from_server['box_names']:
box_select = From.select(
box, readonly=False) #open box which will have its contents copied
print 'Fetching messages from \'%s\'...' % box
resp, items = From.search(None, 'ALL') #get all messages in the box
msg_nums = items[0].split()
print '%s messages to archive' % len(msg_nums)
for msg_num in msg_nums:
resp, data = From.fetch(
msg_num, "(FLAGS INTERNALDATE BODY.PEEK[])") # get email
message = data[0][1]
flags = imaplib.ParseFlags(data[0][0]) # get flags
flag_str = " ".join(flags)
date = imaplib.Time2Internaldate(
imaplib.Internaldate2tuple(data[0][0])) #get date
copy_result = To.append(to_server['box_name'], flag_str, date,
message) # copy to archive
if copy_result[0] == 'OK':
del_msg = From.store(msg_num, '+FLAGS',
'\\Deleted') # mark for deletion
ex = From.expunge() # delete marked
print 'expunge status: %s' % ex[0]
if not ex[1][
0]: # result can be ['OK', [None]] if no messages need to be deleted
print 'expunge count: 0'
else:
print 'expunge count: %s' % len(ex[1])
def update_event(self, inp=-1):
self.set_output_val(0, imaplib.Internaldate2tuple(self.input(0)))
#status, data = connection.search(None, 'FROM', '"{}"'.format(mail))
# ---------------------------
status, data = connection.search(None, '(FROM %s SINCE %s)' %(mail,date2))
datalist = data[0].decode().split()
#print(datalist)
count = 0
lenDataList = str(len(datalist))
print('Número de Email captados: ' + lenDataList)
for n in datalist:
status, raw_data = connection.fetch(n, '(BODY[HEADER.FIELDS (MESSAGE-ID)])')
status, raw_date = connection.fetch(n, '(INTERNALDATE)')
# ----- change date format -----
timestruct = imaplib.Internaldate2tuple(raw_date[0])
str_datetime = raw_date[0].decode().split('"')[1]
timezone_aware = email.utils.parsedate_to_datetime(str_datetime)
date = timezone_aware.strftime("%d/%m/%Y")
str_data = str(raw_data[0][1])[15:][:-10]
# ----- show all data (un-comment to show all msg-id and dates from all emails)-----
#print('Message ID: ' + str_data)
#print('Date: ' + date)
# ----- checking regex -----
check = re.fullmatch(regex, str_data)
if not check:
count = count + 1
part = '(UID INTERNALDATE RFC822.SIZE FLAGS )'
msgset = '34:36'
#if 'MESSAGES' in status: msgset = status['MESSAGES']
tt = mbox._fetch(msgset, part)
puts(len(tt))
for tn in tt:
puts(len(tn), type(tn))
if type(tn) == tuple:
mr = SIZE_PATTERN.search(tn[0])
if mr: puts('size:', int(mr.group(1)))
for uu in tn:
if len(uu) > 30:
cli.printf('%s ..', uu[0:30])
else:
puts(uu)
else:
mr = SIZE_PATTERN.search(tn)
if mr: puts('size:', int(mr.group(1)))
received = imaplib.Internaldate2tuple(tn)
if received:
puts('received',
time.strftime('%Y-%m-%d %H:%M:%S', received))
puts(tn)
finally:
mbox.disconnect()
puts('done.')
